From 129a3f3f3e24163a0d24a0f21ee03eeff4187985 Mon Sep 17 00:00:00 2001 From: Aidan Daly Date: Mon, 14 Sep 2026 19:52:41 +0000 Subject: [PATCH 1/8] feat: scaffold and read harness YAML configuration --- README.md | 88 +++++++ bun.lock | 1 + package.json | 1 + src/assets/cdk/bin/cdk.ts | 11 +- src/assets/cdk/io/harnessConfig.ts | 101 +++++++ src/assets/cdk/package.json | 3 +- src/assets/cdk/test/harness.test.ts | 59 +++++ src/assets/cdk/tsconfig.json | 2 +- .../__snapshots__/manager.test.ts.snap | 10 + src/core/project/manager.export.test.ts | 49 ++++ src/core/project/manager.tsx | 22 +- src/core/project/templates/export.ts | 4 +- src/core/project/templates/harness.test.ts | 149 +++++++++++ src/core/project/templates/harness.ts | 19 +- src/core/project/templates/harnessYaml.ts | 158 +++++++++++ .../project/add/harness/index.test.ts | 21 +- .../project/create/create.screen.test.tsx | 5 +- src/handlers/project/export/harness.test.ts | 2 + .../project/export/serviceHarness.test.ts | 2 +- src/handlers/project/project.test.ts | 11 +- src/handlers/project/status/screen.tsx | 2 +- src/io/harnessConfig.test.ts | 246 ++++++++++++++++++ src/io/harnessConfig.ts | 104 ++++++++ src/projectSchemas/harness.test.ts | 2 +- 24 files changed, 1032 insertions(+), 40 deletions(-) create mode 100644 src/assets/cdk/io/harnessConfig.ts create mode 100644 src/assets/cdk/test/harness.test.ts create mode 100644 src/core/project/templates/harness.test.ts create mode 100644 src/core/project/templates/harnessYaml.ts create mode 100644 src/io/harnessConfig.test.ts create mode 100644 src/io/harnessConfig.ts diff --git a/README.md b/README.md index 79822cc32..f6ac2b37c 100644 --- a/README.md +++ b/README.md @@ -157,6 +157,94 @@ pre-built container image or a custom Dockerfile, that is reported in `EXPORT_NOTES.md` rather than rebuilt. Path-based skills are not supported, since the exported agent has no container filesystem to read them from. +### Harness Project Files + +`project create` (without `--template`) and `project add harness` share the same +scaffolding flow. Each harness has `app//harness.yaml` and +`app//system-prompt.md`. The YAML contains the supplied settings and +commented optional examples. Tools are opt-in. Newly scaffolded harnesses +explicitly use `memory: { mode: managed }` unless another memory configuration +was supplied. Reading an existing file with no `memory` setting still means +disabled memory; reading never adds the scaffold default. + +```yaml +name: assistant +model: + provider: bedrock + modelId: global.anthropic.claude-sonnet-4-6 +systemPrompt: file://./system-prompt.md +memory: + mode: managed +``` + +Only `systemPrompt` and +`truncation.config.summarization.summarizationSystemPrompt` resolve local +`file://` references. The prefix is removed and the remaining filesystem path +is resolved relative to **the YAML file's directory**, not the shell's working +directory. `file://./prompt.md`, `file://../shared/prompt.md`, and absolute +filesystem paths are supported. Paths use native filesystem spelling (including +Windows drive paths), not URL host or percent-encoding rules; spaces, `#`, and +`%` in a filename stay literal. YAML quoting preserves special characters. + +```yaml +systemPrompt: | + You are a concise assistant. +truncation: + strategy: summarization + config: + summarization: + summarizationSystemPrompt: "file://../shared/summary #1.md" +``` + +Both build/deploy and local export resolve these files to literal text before +schema validation. Explicit prompt text or a reference takes precedence over +`system-prompt.md`. That conventional file is used only when `systemPrompt` is +omitted. A bad explicit reference never falls back silently. Referenced files +must be readable, nonempty UTF-8 text of at most **1 MiB each**; whitespace-only +files are rejected. Inline text and file contents preserve their whitespace. +File contents are not interpreted as further references. + +Skills are unchanged: skill paths refer to the **runtime/container filesystem**, +not local files to package. Other fields do not support local includes. +Malformed YAML, duplicate keys, and existing schema violations fail the read. +Build, deploy, and export do not rewrite harness YAML or remove its comments. +`agentcore.json`, deployment targets, JSON CLI flags/output, and service payloads +are unchanged. + +### Migrate Harness JSON + +On the `refactor` branch, `harness.yaml` replaces `harness.json`; the new reader +does not fall back to the old filename. Migrate both the harness file **and the +CDK app already copied into your project**: + +1. Commit or back up your project. Rename each `app//harness.json` to + `harness.yaml` and convert its object to YAML without changing its values. + JSON syntax is valid YAML, so the renamed file can retain its JSON syntax + while you reformat it. Keep string values quoted where needed, and preserve + omitted, disabled, or existing memory settings. +2. Keep an inline `systemPrompt`, or set `systemPrompt: file://./system-prompt.md` + to select the conventional prompt explicitly. If both previously existed, + choose the intended prompt: explicit text now wins consistently for local + export and CDK. Bare paths such as `./prompt.md` are not references. +3. Generate a separate reference project with the updated CLI: + `agentcore project create --name HarnessYamlReference --template empty --skip-install --skip-git`. + Compare its `agentcore/cdk/` with your project's copy. Port the YAML reader + (`io/harnessConfig.ts`) and the harness-loading changes in `bin/cdk.ts`; add + the direct `yaml` dependency from `package.json` and the `io/**/*` include + from `tsconfig.json`. Preserve your custom CDK code, especially + `lib/cdk-stack.ts`. The reference app also includes a harness synthesis test. +4. Reinstall dependencies in your project's `agentcore/cdk/`, compile it, and + run `agentcore project build` from the project root before deploying. + This does not require a newer `@aws/agentcore-cdk` release. + +Upgrading the CLI does **not** overwrite a project's copied CDK app. +`project add harness` in an older project produces YAML but likewise does not +upgrade that app; complete step 3 before building or deploying the new harness. +An old app may still report a missing `harness.json` after the rename. That +means its reader needs migrating, not that the YAML should be renamed back. +The updated reader reports an obsolete-JSON error when only `harness.json` +exists. Neither path silently renames files or enables memory in existing ones. + Global flags (declared at the root, available on every command): | Flag | Purpose | diff --git a/bun.lock b/bun.lock index 9cff5d36c..8cb03228d 100644 --- a/bun.lock +++ b/bun.lock @@ -35,6 +35,7 @@ "string-width": "^8.2.2", "winston": "^3.19.0", "winston-daily-rotate-file": "^5.0.0", + "yaml": "^2.8.1", "zod": "^4.4.3", }, "devDependencies": { diff --git a/package.json b/package.json index c2b2792cf..868b3b63d 100644 --- a/package.json +++ b/package.json @@ -92,6 +92,7 @@ "string-width": "^8.2.2", "winston": "^3.19.0", "winston-daily-rotate-file": "^5.0.0", + "yaml": "^2.8.1", "zod": "^4.4.3" }, "overrides": { diff --git a/src/assets/cdk/bin/cdk.ts b/src/assets/cdk/bin/cdk.ts index 9e308d1de..834ff126b 100644 --- a/src/assets/cdk/bin/cdk.ts +++ b/src/assets/cdk/bin/cdk.ts @@ -4,6 +4,7 @@ import { ConfigIO, HarnessSpecSchema, type AwsDeploymentTarget } from '@aws/agen import { App, type Environment } from 'aws-cdk-lib'; import * as path from 'path'; import * as fs from 'fs'; +import { HarnessConfigReader } from '../io/harnessConfig'; function toEnvironment(target: AwsDeploymentTarget): Environment { return { @@ -67,13 +68,13 @@ function resolveConnectorParametersByFile( // Synthesize a HarnessConfig for each harness entry in the spec. The full validated // spec drives the AWS::BedrockAgentCore::Harness CFN resource; the role-scoped // fields drive the IAM role + container build. -function resolveHarnessConfigs(spec: SpecWithLatestFields, projectRoot: string): HarnessConfig[] { +async function resolveHarnessConfigs(spec: SpecWithLatestFields, projectRoot: string): Promise { const harnessConfigs: HarnessConfig[] = []; for (const entry of spec.harnesses ?? []) { const harnessDir = path.resolve(projectRoot, entry.path); - const harnessPath = path.resolve(harnessDir, 'harness.json'); + const harnessPath = path.resolve(harnessDir, 'harness.yaml'); try { - const harnessSpec = HarnessSpecSchema.parse(JSON.parse(fs.readFileSync(harnessPath, 'utf-8'))); + const harnessSpec = HarnessSpecSchema.parse(await new HarnessConfigReader().read(harnessPath)); harnessConfigs.push({ name: entry.name, executionRoleArn: harnessSpec.executionRoleArn, @@ -96,7 +97,7 @@ function resolveHarnessConfigs(spec: SpecWithLatestFields, projectRoot: string): }); } catch (err) { throw new Error( - `Could not read harness.json for "${entry.name}" at ${harnessPath}: ${err instanceof Error ? err.message : err}` + `Could not read harness.yaml for "${entry.name}" at ${harnessPath}: ${err instanceof Error ? err.message : err}` ); } } @@ -122,7 +123,7 @@ async function main() { const mcpSpec = resolveMcpSpec(specAny); const connectorParametersByFile = resolveConnectorParametersByFile(specAny, projectRoot); - const harnessConfigs = resolveHarnessConfigs(specAny, projectRoot); + const harnessConfigs = await resolveHarnessConfigs(specAny, projectRoot); // Read deployed state for credential ARNs (populated by pre-deploy identity setup). // Under agentcore/.cli/ to match the released CLI's location. diff --git a/src/assets/cdk/io/harnessConfig.ts b/src/assets/cdk/io/harnessConfig.ts new file mode 100644 index 000000000..89e54dc01 --- /dev/null +++ b/src/assets/cdk/io/harnessConfig.ts @@ -0,0 +1,101 @@ +import { open, readFile, stat } from "node:fs/promises"; +import { dirname, join, resolve } from "node:path"; +import { parseDocument } from "yaml"; + +const MAX_PROMPT_FILE_SIZE = 1024 * 1024; +const PROMPT_FIELDS = [ + ["systemPrompt"], + ["truncation", "config", "summarization", "summarizationSystemPrompt"], +] as const; + +/** Project-file I/O only; callers validate the resolved data with their HarnessSpecSchema. */ +export class HarnessConfigReader { + async read(filePath: string): Promise { + const configPath = resolve(filePath); + try { + let raw: string; + try { + raw = await readFile(configPath, "utf8"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + const obsoletePath = join(dirname(configPath), "harness.json"); + if (await stat(obsoletePath).catch(() => undefined)) { + throw new Error( + `Obsolete harness.json at '${obsoletePath}'. Migrate it to harness.yaml and update ` + + "the copied agentcore/cdk app; see the README harness migration guide.", + ); + } + } + throw error; + } + const document = parseDocument(raw); + if (document.errors.length) throw document.errors[0]; + const data: unknown = document.toJS(); + if (!isRecord(data)) return data; + + // These are the only local-file fields. Skills and other runtime paths stay untouched. + promptFields: for (const keys of PROMPT_FIELDS) { + let parent = data; + for (const key of keys.slice(0, -1)) { + const child = parent[key]; + if (!isRecord(child)) continue promptFields; + parent[key] = { ...child }; + parent = parent[key] as Record; + } + const key = keys[keys.length - 1]!; + const value = parent[key]; + if (typeof value === "string" && value.startsWith("file://")) { + const source = value.slice("file://".length); + if (!source) throw new Error(`${keys.join(".")}: file:// requires a path`); + parent[key] = await this.readPrompt( + resolve(dirname(configPath), source), + keys.join("."), + ); + } + } + if (data.systemPrompt === undefined) { + const fallback = join(dirname(configPath), "system-prompt.md"); + try { + data.systemPrompt = await this.readPrompt(fallback, "systemPrompt"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + } + return data; + } catch (error) { + throw new Error( + `Could not read harness.yaml at '${configPath}': ${error instanceof Error ? error.message : error}`, + { cause: error }, + ); + } + } + + private async readPrompt(filePath: string, field: string): Promise { + const file = await open(filePath, "r").catch((error: NodeJS.ErrnoException) => { + throw Object.assign(new Error(`${field}: cannot read prompt file '${filePath}': ${error.message}`, { cause: error }), { code: error.code }); + }); + try { + const info = await file.stat(); + if (!info.isFile()) throw new Error(`${field}: '${filePath}' is not a regular prompt file`); + const bytes = Buffer.alloc(MAX_PROMPT_FILE_SIZE + 1); + let length = 0; + while (length < bytes.length) { + const { bytesRead } = await file.read(bytes, length, bytes.length - length, null); + if (!bytesRead) break; + length += bytesRead; + } + if (length > MAX_PROMPT_FILE_SIZE) { + throw new Error(`${field}: prompt file '${filePath}' exceeds the 1 MiB limit`); + } + const text = new TextDecoder("utf-8", { fatal: true }).decode(bytes.subarray(0, length)); + if (!text.trim()) throw new Error(`${field}: prompt file '${filePath}' is empty or whitespace-only`); + return text; + } finally { + await file.close(); + } + } +} + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === "object" && !Array.isArray(value); +} diff --git a/src/assets/cdk/package.json b/src/assets/cdk/package.json index 8fc72c2de..9691987a2 100644 --- a/src/assets/cdk/package.json +++ b/src/assets/cdk/package.json @@ -25,6 +25,7 @@ "dependencies": { "@aws/agentcore-cdk": "0.1.0-alpha.52", "aws-cdk-lib": "~2.266.0", - "constructs": "~10.7.0" + "constructs": "~10.7.0", + "yaml": "^2.8.1" } } diff --git a/src/assets/cdk/test/harness.test.ts b/src/assets/cdk/test/harness.test.ts new file mode 100644 index 000000000..e0e41a551 --- /dev/null +++ b/src/assets/cdk/test/harness.test.ts @@ -0,0 +1,59 @@ +import { execFileSync } from 'node:child_process'; +import { mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { stringify } from 'yaml'; + +const entrypoint = resolve(__dirname, '..', 'dist/bin/cdk.js'); +const roots: string[] = []; + +afterEach(() => { + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); +}); + +test.each(['literal', 'file', 'fallback'])('generated app synthesizes the %s prompt and summary as literal text', (source) => { + const root = mkdtempSync(join(tmpdir(), 'harness-yaml-synth-')); + roots.push(root); + const configRoot = join(root, 'agentcore'); + const cdkRoot = join(configRoot, 'cdk'); + const harnessDir = join(root, 'app', 'assistant'); + mkdirSync(cdkRoot, { recursive: true }); + mkdirSync(harnessDir, { recursive: true }); + writeFileSync(join(configRoot, 'agentcore.json'), JSON.stringify({ + name: 'YamlProject', + version: 1, + managedBy: 'CDK', + harnesses: [{ name: 'assistant', path: 'app/assistant' }], + })); + writeFileSync(join(configRoot, 'aws-targets.json'), '[]'); + const prompt = ' Selected prompt: # 100%\n'; + const summary = 'Keep decisions and open questions.\n'; + writeFileSync(join(harnessDir, 'chosen #100%.md'), prompt); + writeFileSync(join(harnessDir, 'system-prompt.md'), source === 'fallback' ? prompt : 'Conventional prompt loses.'); + writeFileSync(join(harnessDir, 'summary.md'), summary); + const yaml = '# Customer comment stays intact.\n' + stringify({ + name: 'assistant', + model: { provider: 'bedrock', modelId: 'global.anthropic.claude-sonnet-4-6' }, + systemPrompt: source === 'fallback' ? undefined : source === 'literal' ? prompt : 'file://./chosen #100%.md', + memory: { mode: 'disabled' }, + truncation: { + strategy: 'summarization', + config: { summarization: { summaryRatio: 0.3, preserveRecentMessages: 0, summarizationSystemPrompt: 'file://./summary.md' } }, + }, + }); + writeFileSync(join(harnessDir, 'harness.yaml'), yaml); + const outdir = join(root, 'cdk.out'); + execFileSync(process.execPath, [entrypoint], { + cwd: cdkRoot, + env: { ...process.env, INIT_CWD: root, CDK_OUTDIR: outdir }, + stdio: 'pipe', + }); + const templateFile = readdirSync(outdir).find((name) => name.endsWith('.template.json'))!; + const template = JSON.parse(readFileSync(join(outdir, templateFile), 'utf8')); + const harness = Object.values(template.Resources).find((resource: any) => resource.Type === 'AWS::BedrockAgentCore::Harness') as any; + expect(harness.Properties.SystemPrompt).toEqual([{ Text: prompt }]); + expect(harness.Properties.Memory).toEqual({ Disabled: {} }); + expect(JSON.stringify(harness.Properties)).toContain(JSON.stringify(summary).slice(1, -1)); + expect(JSON.stringify(harness.Properties)).not.toContain('file://'); + expect(readFileSync(join(harnessDir, 'harness.yaml'), 'utf8')).toBe(yaml); +}); diff --git a/src/assets/cdk/tsconfig.json b/src/assets/cdk/tsconfig.json index c70b0d444..55e13ec7d 100644 --- a/src/assets/cdk/tsconfig.json +++ b/src/assets/cdk/tsconfig.json @@ -23,6 +23,6 @@ "rootDir": ".", "outDir": "dist" }, - "include": ["bin/**/*", "lib/**/*", "test/**/*"], + "include": ["bin/**/*", "lib/**/*", "io/**/*", "test/**/*"], "exclude": ["node_modules", "cdk.out", "dist"] } diff --git a/src/core/project/__snapshots__/manager.test.ts.snap b/src/core/project/__snapshots__/manager.test.ts.snap index a079f027d..697a0de78 100644 --- a/src/core/project/__snapshots__/manager.test.ts.snap +++ b/src/core/project/__snapshots__/manager.test.ts.snap @@ -12,10 +12,12 @@ exports[`FsProjectManager.create scaffolds the expected file tree into a fresh d "agentcore/cdk/README.md", "agentcore/cdk/bin/cdk.ts", "agentcore/cdk/cdk.json", + "agentcore/cdk/io/harnessConfig.ts", "agentcore/cdk/jest.config.js", "agentcore/cdk/lib/cdk-stack.ts", "agentcore/cdk/package.json", "agentcore/cdk/test/cdk.test.ts", + "agentcore/cdk/test/harness.test.ts", "agentcore/cdk/tsconfig.json", "app/agent_python_minimal/README.md", "app/agent_python_minimal/main.py", @@ -36,10 +38,12 @@ exports[`FsProjectManager.create snapshots the Strands project manifest and runt "agentcore/cdk/README.md", "agentcore/cdk/bin/cdk.ts", "agentcore/cdk/cdk.json", + "agentcore/cdk/io/harnessConfig.ts", "agentcore/cdk/jest.config.js", "agentcore/cdk/lib/cdk-stack.ts", "agentcore/cdk/package.json", "agentcore/cdk/test/cdk.test.ts", + "agentcore/cdk/test/harness.test.ts", "agentcore/cdk/tsconfig.json", "app/agent_python_strands/.gitignore", "app/agent_python_strands/README.md", @@ -111,10 +115,12 @@ exports[`FsProjectManager.create snapshots the Strands TypeScript project manife "agentcore/cdk/README.md", "agentcore/cdk/bin/cdk.ts", "agentcore/cdk/cdk.json", + "agentcore/cdk/io/harnessConfig.ts", "agentcore/cdk/jest.config.js", "agentcore/cdk/lib/cdk-stack.ts", "agentcore/cdk/package.json", "agentcore/cdk/test/cdk.test.ts", + "agentcore/cdk/test/harness.test.ts", "agentcore/cdk/tsconfig.json", "app/agent_typescript_strands/.gitignore", "app/agent_typescript_strands/README.md", @@ -186,10 +192,12 @@ exports[`FsProjectManager.create snapshots the Strands A2A project manifest and "agentcore/cdk/README.md", "agentcore/cdk/bin/cdk.ts", "agentcore/cdk/cdk.json", + "agentcore/cdk/io/harnessConfig.ts", "agentcore/cdk/jest.config.js", "agentcore/cdk/lib/cdk-stack.ts", "agentcore/cdk/package.json", "agentcore/cdk/test/cdk.test.ts", + "agentcore/cdk/test/harness.test.ts", "agentcore/cdk/tsconfig.json", "app/a2a_python_strands/.gitignore", "app/a2a_python_strands/README.md", @@ -261,10 +269,12 @@ exports[`FsProjectManager.create snapshots the LangChain project manifest and ru "agentcore/cdk/README.md", "agentcore/cdk/bin/cdk.ts", "agentcore/cdk/cdk.json", + "agentcore/cdk/io/harnessConfig.ts", "agentcore/cdk/jest.config.js", "agentcore/cdk/lib/cdk-stack.ts", "agentcore/cdk/package.json", "agentcore/cdk/test/cdk.test.ts", + "agentcore/cdk/test/harness.test.ts", "agentcore/cdk/tsconfig.json", "app/agent_python_langchain/.gitignore", "app/agent_python_langchain/README.md", diff --git a/src/core/project/manager.export.test.ts b/src/core/project/manager.export.test.ts index 8bac9bd0b..bc9e11965 100644 --- a/src/core/project/manager.export.test.ts +++ b/src/core/project/manager.export.test.ts @@ -4,6 +4,7 @@ import { existsSync } from "node:fs"; import { join } from "node:path"; import { tmpdir } from "node:os"; import z from "zod"; +import { parse, stringify } from "yaml"; import { FsProjectManager } from "./manager"; import { FsReadWriteJson, type ReadWriteJson } from "../../io"; import { createSilentLogger, TestIdentityClient } from "../../testing"; @@ -71,6 +72,7 @@ async function projectWithHarness( name: "assistant", model: { provider: "bedrock", modelId: "us.amazon.nova-lite-v1:0" }, systemPrompt: "You are a terse assistant.", + memory: { mode: "disabled" }, ...harness, } as z.input, }), @@ -316,6 +318,53 @@ describe("FsProjectManager.exportHarness rendered tree", () => { }); describe("FsProjectManager.exportHarness side effects", () => { + test.each(["literal", "file", "fallback"] as const)( + "exports the %s prompt and resolved summary without rewriting YAML", + async (source) => { + const { manager: subject } = manager(); + const project = await projectWithHarness(subject); + const dir = join(project.rootPath, "app", "assistant"); + const configPath = join(dir, "harness.yaml"); + const config = parse(await Bun.file(configPath).text()); + const prompt = " Explicit prompt.\nKeep its whitespace.\n"; + await Bun.write( + join(dir, "system-prompt.md"), + source === "fallback" ? prompt : "Conventional prompt loses.", + ); + await Bun.write(join(dir, "chosen #1%.md"), prompt); + await Bun.write(join(dir, "summary.md"), " Keep the decisions.\n"); + if (source === "fallback") delete config.systemPrompt; + else config.systemPrompt = source === "literal" ? prompt : "file://./chosen #1%.md"; + config.truncation = { + strategy: "summarization", + config: { summarization: { summarizationSystemPrompt: "file://./summary.md" } }, + }; + const yaml = "# Keep this customer comment.\n" + stringify(config); + await Bun.write(configPath, yaml); + const result = await drain(subject.exportHarness(project, exportInput())); + const main = await Bun.file(join(result.agentPath, "main.py")).text(); + expect(main).toContain(prompt); + expect(main).toContain("Keep the decisions."); + expect(main).not.toContain("file://"); + expect(main).not.toContain("Conventional prompt loses."); + expect(await Bun.file(configPath).text()).toBe(yaml); + }, + ); + + test("reports schema errors with the YAML path before creating export output", async () => { + const { manager: subject } = manager(); + const project = await projectWithHarness(subject); + const configPath = join(project.rootPath, "app", "assistant", "harness.yaml"); + await Bun.write( + configPath, + "name: assistant\nmodel: {provider: bedrock, modelId: example}\nmaxIterations: 0\n", + ); + await expect(drain(subject.exportHarness(project, exportInput()))).rejects.toThrow( + /Invalid harness.yaml.*maxIterations/s, + ); + expect(existsSync(join(project.rootPath, "app", "assistantAgent"))).toBe(false); + }); + test("writes MCP header secrets to .env.local and registers their credentials", async () => { const { manager: subject } = manager(); const project = await projectWithHarness(subject, { diff --git a/src/core/project/manager.tsx b/src/core/project/manager.tsx index bda377b96..d902b5896 100644 --- a/src/core/project/manager.tsx +++ b/src/core/project/manager.tsx @@ -46,6 +46,7 @@ import { mapHarnessToExportPlan, } from "./templates/export"; import { HarnessSpecSchema } from "../../projectSchemas/harness"; +import { HarnessConfigReader } from "../../io/harnessConfig"; import { FsTreeNode } from "./templates/fsTree"; import { getEvaluatorTemplateResolver } from "./templates/evaluator"; import { ProjectSpecSchema, type ManagedBy } from "../../projectSchemas/project"; @@ -737,17 +738,18 @@ export class FsProjectManager implements ProjectManager { harnessDir = join(project.rootPath, entry.path); yield { type: "step", - message: `Reading harness configuration from '${join(entry.path, "harness.json")}'`, + message: `Reading harness configuration from '${join(entry.path, "harness.yaml")}'`, }; - spec = await this.json.read(join(harnessDir, "harness.json"), HarnessSpecSchema); - const promptPath = join(harnessDir, "system-prompt.md"); - const filePrompt = existsSync(promptPath) - ? (await readFile(promptPath, "utf-8")).trim() - : undefined; - systemPrompt = - filePrompt && filePrompt.length > 0 - ? filePrompt - : (spec.systemPrompt ?? DEFAULT_EXPORT_SYSTEM_PROMPT); + const harnessPath = join(harnessDir, "harness.yaml"); + const parsed = HarnessSpecSchema.safeParse(await new HarnessConfigReader().read(harnessPath)); + if (!parsed.success) { + throw new InputValidationError( + `Invalid harness.yaml at '${harnessPath}': ${z.prettifyError(parsed.error)}`, + { cause: parsed.error }, + ); + } + spec = parsed.data; + systemPrompt = spec.systemPrompt ?? DEFAULT_EXPORT_SYSTEM_PROMPT; } // Refuse to overwrite anything: the target name must be free in the spec diff --git a/src/core/project/templates/export.ts b/src/core/project/templates/export.ts index d42c29e5b..33e1ab59b 100644 --- a/src/core/project/templates/export.ts +++ b/src/core/project/templates/export.ts @@ -42,9 +42,9 @@ export interface ExportNoteLine { export interface HarnessExportInput { harnessName: string; targetAgentName: string; - /** The parsed harness spec (from app//harness.json or the service). */ + /** The parsed harness spec (from app//harness.yaml or the service). */ spec: HarnessSpec; - /** The resolved system prompt text (system-prompt.md > spec.systemPrompt > default). */ + /** The resolved system prompt text (explicit prompt > conventional file > default). */ systemPrompt: string; /** The current project spec, for memory lookups and credential dedup. */ projectSpec: ProjectSpec; diff --git a/src/core/project/templates/harness.test.ts b/src/core/project/templates/harness.test.ts new file mode 100644 index 000000000..bef972e0b --- /dev/null +++ b/src/core/project/templates/harness.test.ts @@ -0,0 +1,149 @@ +import { afterEach, expect, test } from "bun:test"; +import { mkdtemp, readFile, readdir, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { parse } from "yaml"; +import type z from "zod"; +import { HarnessSpecSchema } from "../../../projectSchemas/harness"; +import { HarnessConfigReader } from "../../../io/harnessConfig"; +import { getHarnessTemplateResolver } from "./harness"; + +const roots: string[] = []; +const model = { provider: "bedrock", modelId: "example" } as const; +afterEach(async () => { + await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); +}); + +async function scaffold(overrides: Partial> = {}) { + const root = await mkdtemp(join(tmpdir(), "harness-yaml-template-")); + roots.push(root); + const { tree } = await getHarnessTemplateResolver().resolve({ + name: "assistant", + model, + ...overrides, + }); + await tree.write(root); + const directory = join(root, "assistant"); + const path = join(directory, "harness.yaml"); + const yaml = await readFile(path, "utf8"); + return { directory, path, yaml, data: parse(yaml) }; +} + +test("actual scaffold includes semantic comments and inactive examples, without default tools", async () => { + const { directory, path, yaml, data } = await scaffold(); + expect((await readdir(directory)).sort()).toEqual(["harness.yaml", "system-prompt.md"]); + expect(data).toEqual({ + name: "assistant", + model, + systemPrompt: "file://./system-prompt.md", + memory: { mode: "managed" }, + }); + expect(yaml).not.toMatch(/^(tools|skills):/m); + expect(yaml).toContain("# Prompt file paths are relative to this YAML file."); + expect(yaml).toContain( + "# Output tokens per model call, rather than across the whole invocation.", + ); + expect(yaml).toContain("# Execution limits apply per invocation, across all model calls."); + expect(yaml).toContain( + "# Skill path sources refer to files already present in the runtime container.", + ); + expect(yaml).toContain( + "# Truncation changes the context sent to the model, not the saved memory.", + ); + expect(yaml).toContain("# maxTokens: 20000"); + expect(yaml).toContain("agentcore_code_interpreter"); + expect(yaml).toContain("remote_mcp"); + expect(yaml).toContain( + "https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/gateway.html", + ); + expect(yaml).toContain( + "https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/memory.html", + ); + expect(yaml).not.toMatch(/uncomment|replace \[\]|\bexa\b/i); + expect(HarnessSpecSchema.parse(await new HarnessConfigReader().read(path)).systemPrompt).toBe( + "You are a helpful assistant", + ); +}); + +test.each([ + { mode: "disabled" }, + { + mode: "existing", + name: "ConversationMemory", + actorId: "007", + retrievalConfig: { relevanceScore: 0 }, + }, + { + mode: "existing", + arn: "arn:aws:bedrock-agentcore:us-east-1:123456789012:memory/example-1234567890", + }, + { mode: "managed", strategies: ["EPISODIC"], eventExpiryDuration: 365 }, +] as const)("preserves explicit memory: %j", async (memory) => { + const { data } = await scaffold({ memory }); + expect(data.memory).toEqual(memory); +}); + +test("serializes supplied nested strings, arrays, maps, and zero values without example substitution", async () => { + const overrides: Partial> = { + model: { + provider: "lite_llm", + modelId: "true", + temperature: 0, + topP: 0, + maxTokens: 27, + additionalParams: { + quoted: 'a: "b" # comment\nnext', + flags: [false, 0, "007", "null"], + nested: { "a: b": "[x]" }, + }, + }, + systemPrompt: " Exact prompt.\r\nWith whitespace.\n", + tools: [ + { + name: "custom", + type: "remote_mcp", + config: { + remoteMcp: { + url: "https://example.com/a?x=#y", + headers: { Authorization: 'Bearer "secret": # not a comment' }, + }, + }, + }, + ], + allowedTools: ["@custom/search"], + skills: [{ path: "/opt/runtime-only" }], + environmentVariables: { YES: "true", NUMBER: "007", NULL: "null", OTHER: "a: b\nc" }, + tags: { team: "false" }, + truncation: { + strategy: "summarization", + config: { + summarization: { + summaryRatio: 0, + preserveRecentMessages: 0, + summarizationSystemPrompt: "inline: # summary\n", + }, + }, + }, + maxIterations: 2, + timeoutSeconds: 19, + }; + const { data, path } = await scaffold(overrides); + const expected = HarnessSpecSchema.parse({ + name: "assistant", + model, + ...overrides, + memory: { mode: "managed" }, + }); + expect(data).toEqual({ ...expected, systemPrompt: "file://./system-prompt.md" }); + expect(HarnessSpecSchema.parse(await new HarnessConfigReader().read(path))).toEqual(expected); +}); + +test("preserves a supplied explicit prompt reference", async () => { + const { data, directory } = await scaffold({ systemPrompt: "file://../shared.md" }); + expect(data.systemPrompt).toBe("file://../shared.md"); + expect(await readFile(join(directory, "system-prompt.md"), "utf8")).not.toContain("file://"); +}); + +test("defaults only absent memory and does not mask an invalid supplied setting", async () => { + await expect(scaffold({ memory: null })).rejects.toThrow(); +}); diff --git a/src/core/project/templates/harness.ts b/src/core/project/templates/harness.ts index 152a30157..f0d16d8b9 100644 --- a/src/core/project/templates/harness.ts +++ b/src/core/project/templates/harness.ts @@ -4,9 +4,9 @@ import { HarnessSpecSchema } from "../../../projectSchemas/harness"; import { FsTreeNode } from "./fsTree"; import { InputValidationError, ResourceNotFoundError } from "../../../errors/errors"; import type { TemplateResolver } from "./types"; +import { HarnessYamlRenderer } from "./harnessYaml"; const DEFAULT_SYSTEM_PROMPT = "You are a helpful assistant"; -const json = (value: unknown): string => `${JSON.stringify(value, null, 2)}\n`; /** Given a harness spec, resolve the {@link TemplateResolver} that renders its config directory **/ export function getHarnessTemplateResolver(): TemplateResolver> { @@ -14,19 +14,24 @@ export function getHarnessTemplateResolver(): TemplateResolver json(parsed)), - FsTreeNode.createFile( - "system-prompt.md", - async () => systemPrompt ?? DEFAULT_SYSTEM_PROMPT, + FsTreeNode.createFile("harness.yaml", async () => + new HarnessYamlRenderer().render({ ...parsed, systemPrompt: promptReference }), + ), + FsTreeNode.createFile("system-prompt.md", async () => + systemPrompt?.startsWith("file://") + ? DEFAULT_SYSTEM_PROMPT + : (systemPrompt ?? DEFAULT_SYSTEM_PROMPT), ), ...(spec.dockerfile ? [FsTreeNode.fromTextFile("Dockerfile", spec.dockerfile)] : []), ]); diff --git a/src/core/project/templates/harnessYaml.ts b/src/core/project/templates/harnessYaml.ts new file mode 100644 index 000000000..eaf5cf317 --- /dev/null +++ b/src/core/project/templates/harnessYaml.ts @@ -0,0 +1,158 @@ +import { stringify } from "yaml"; +import type { HarnessSpec } from "../../../projectSchemas/harness"; + +const TOOL_EXAMPLES = [ + { name: "code_interpreter", type: "agentcore_code_interpreter" }, + { name: "browser", type: "agentcore_browser" }, + { + name: "research", + type: "remote_mcp", + config: { remoteMcp: { url: "https://mcp.example.com/mcp" } }, + }, + { + name: "company_tools", + type: "agentcore_gateway", + config: { + agentCoreGateway: { + gatewayArn: "arn:aws:bedrock-agentcore:us-west-2:123456789012:gateway/example-1234567890", + outboundAuth: { awsIam: {} }, + }, + }, + }, +]; + +const OPTIONAL_SECTIONS = [ + { + comment: + "Tool patterns: @/ or @builtin.\nThis controls agent tool selection, not IAM permissions.", + values: { allowedTools: ["@builtin", "@research/search"] }, + }, + { + comment: "Skill path sources refer to files already present in the runtime container.", + values: { + skills: [ + { s3Uri: "s3://your-skills-bucket/skills/research/" }, + { gitUrl: "https://github.com/your-org/agent-skills.git", path: "skills/research" }, + { path: "/opt/skills/research" }, + ], + }, + }, + { + comment: "Execution limits apply per invocation, across all model calls.", + values: { maxIterations: 15, maxTokens: 20000, timeoutSeconds: 300 }, + }, + { + comment: "Truncation changes the context sent to the model, not the saved memory.", + values: { + truncation: { strategy: "sliding_window", config: { slidingWindow: { messagesCount: 40 } } }, + }, + }, + { + comment: + "dockerfile and containerUri are mutually exclusive. With neither set, the\nservice-provided environment is used. Dockerfile paths are relative to this directory.", + values: { + dockerfile: "Dockerfile", + containerUri: "123456789012.dkr.ecr.us-west-2.amazonaws.com/my-harness:latest", + }, + }, + { + comment: "Environment values are stored in plaintext.", + values: { environmentVariables: { LOG_LEVEL: "info" } }, + }, + { + comment: "Deployment creates a role when executionRoleArn is absent.", + values: { executionRoleArn: "arn:aws:iam::123456789012:role/MyHarnessRole" }, + }, + { comment: "Tags", values: { tags: { team: "support", environment: "development" } } }, +]; + +/** Serializes supplied values separately from inactive examples, so examples cannot become defaults. */ +export class HarnessYamlRenderer { + render(spec: HarnessSpec & { systemPrompt: string }): string { + const sections = [ + "# Optional settings are shown with example values.\n", + stringify({ name: spec.name }), + this.comment("Prompt file paths are relative to this YAML file.") + + stringify({ systemPrompt: spec.systemPrompt }), + this.comment("Model") + + stringify({ model: spec.model }) + + this.examples( + { maxTokens: 4096 }, + spec.model, + " ", + "Output tokens per model call, rather than across the whole invocation.", + ), + this.comment( + "Tools\nCode Interpreter and Browser use built-in resources when no ARN is specified.\nawsIam uses the harness execution role, which must allow Gateway invocation.\nhttps://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/gateway.html", + ) + + (spec.tools.length + ? stringify({ tools: spec.tools }) + : this.comment(stringify({ tools: TOOL_EXAMPLES }).trimEnd())), + ]; + const rendered = new Set(["name", "systemPrompt", "model", "tools", "memory"]); + for (const section of OPTIONAL_SECTIONS) { + let body = this.comment(section.comment); + for (const [key, example] of Object.entries(section.values)) { + rendered.add(key); + const value = spec[key as keyof HarnessSpec]; + body += + value === undefined || (key === "skills" && Array.isArray(value) && value.length === 0) + ? this.comment(stringify({ [key]: example }).trimEnd()) + : stringify({ [key]: value }); + } + sections.push(body); + if ("skills" in section.values) sections.push(this.memory(spec.memory)); + } + for (const [key, value] of Object.entries(spec)) { + if (!rendered.has(key) && value !== undefined) sections.push(stringify({ [key]: value })); + } + return sections.join("\n"); + } + + private memory(memory: HarnessSpec["memory"]): string { + return ( + this.comment( + "Memory\nhttps://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/memory.html\nAlternatives: existing (name or ARN), disabled.", + ) + + stringify({ memory }) + + (memory?.mode === "managed" + ? this.examples( + { strategies: ["SEMANTIC", "SUMMARIZATION"], eventExpiryDuration: 30 }, + memory, + " ", + "Managed-memory settings. Absent values use service defaults.\nEvent retention is in days.", + ) + : "") + + this.examples( + { + name: "ConversationMemory", + arn: "arn:aws:bedrock-agentcore:us-west-2:123456789012:memory/example-1234567890", + }, + memory ?? {}, + " ", + "Existing-memory reference: project resource name or external ARN.", + ) + ); + } + + private comment(text: string, indent = ""): string { + return text + .split("\n") + .map((line) => `${indent}#${line ? ` ${line}` : ""}\n`) + .join(""); + } + + private examples( + examples: Record, + actual: object, + indent: string, + semantics: string, + ): string { + const missing = Object.fromEntries( + Object.entries(examples).filter(([key]) => !(key in actual)), + ); + return Object.keys(missing).length + ? this.comment(semantics, indent) + this.comment(stringify(missing).trimEnd(), indent) + : ""; + } +} diff --git a/src/handlers/project/add/harness/index.test.ts b/src/handlers/project/add/harness/index.test.ts index df7a9cc75..0ee76e2fd 100644 --- a/src/handlers/project/add/harness/index.test.ts +++ b/src/handlers/project/add/harness/index.test.ts @@ -1,6 +1,7 @@ import { afterEach, describe, expect, test } from "bun:test"; import { existsSync } from "node:fs"; import { join } from "node:path"; +import { parse } from "yaml"; import { createRootHandler } from "../../../index"; import { createSilentLogger, @@ -415,8 +416,8 @@ describe("project add harness", () => { cleanups.push(cleanup); await run(["add", "harness", ...flags]); - const harnessJson = await Bun.file(join(projectRoot, "app", "x", "harness.json")).json(); - expect(harnessJson).toMatchObject(expected); + const harnessYaml = parse(await Bun.file(join(projectRoot, "app", "x", "harness.yaml")).text()); + expect(harnessYaml).toMatchObject(expected); const agentcoreJson = await Bun.file(join(projectRoot, "agentcore", "agentcore.json")).json(); expect(agentcoreJson.harnesses).toContainEqual({ @@ -433,8 +434,12 @@ describe("project add harness", () => { const prompt = await Bun.file(join(projectRoot, "app", "x", "system-prompt.md")).text(); expect(prompt).toBe("You are a pirate."); - const harnessJson = await Bun.file(join(projectRoot, "app", "x", "harness.json")).json(); - expect(harnessJson).not.toHaveProperty("systemPrompt"); + const harnessYaml = parse(await Bun.file(join(projectRoot, "app", "x", "harness.yaml")).text()); + expect(harnessYaml.systemPrompt).toBe("file://./system-prompt.md"); + expect(harnessYaml.memory).toEqual({ mode: "managed" }); + expect(harnessYaml.tools).toBeUndefined(); + expect(harnessYaml.skills).toBeUndefined(); + expect(existsSync(join(projectRoot, "app", "x", "harness.json"))).toBe(false); }); test("--dockerfile copies the file into the harness directory and stores the relative path", async () => { @@ -449,8 +454,8 @@ describe("project add harness", () => { const copiedContent = await Bun.file(join(projectRoot, "app", "x", "Dockerfile")).text(); expect(copiedContent).toBe("FROM python:3.12-slim\nCOPY . /app\n"); - const harnessJson = await Bun.file(join(projectRoot, "app", "x", "harness.json")).json(); - expect(harnessJson.dockerfile).toBe("Dockerfile"); + const harnessYaml = parse(await Bun.file(join(projectRoot, "app", "x", "harness.yaml")).text()); + expect(harnessYaml.dockerfile).toBe("Dockerfile"); }); test("--dockerfile with VPC mode succeeds when vpcId is in network-config", async () => { @@ -473,8 +478,8 @@ describe("project add harness", () => { '{"subnets":["subnet-0123456789abcdef0"],"securityGroups":["sg-0123456789abcdef0"],"vpcId":"vpc-0123456789abcdef0"}', ]); - const harnessJson = await Bun.file(join(projectRoot, "app", "x", "harness.json")).json(); - expect(harnessJson).toMatchObject({ + const harnessYaml = parse(await Bun.file(join(projectRoot, "app", "x", "harness.yaml")).text()); + expect(harnessYaml).toMatchObject({ dockerfile: "Dockerfile", networkMode: "VPC", networkConfig: { diff --git a/src/handlers/project/create/create.screen.test.tsx b/src/handlers/project/create/create.screen.test.tsx index a2c382631..d80595575 100644 --- a/src/handlers/project/create/create.screen.test.tsx +++ b/src/handlers/project/create/create.screen.test.tsx @@ -2,6 +2,7 @@ import { test, expect, describe, afterEach } from "bun:test"; import { existsSync } from "node:fs"; import { mkdir, readdir } from "node:fs/promises"; import { join } from "node:path"; +import { parse } from "yaml"; import { renderScreen, waitForText, @@ -203,12 +204,14 @@ describe("project create wizard", () => { const root = join(directory, "OpenAIApp"); const spec = await Bun.file(join(root, "agentcore", "agentcore.json")).json(); expect(spec.credentials).toEqual([]); - const harness = await Bun.file(join(root, "app", "OpenAIApp", "harness.json")).json(); + const harness = parse(await Bun.file(join(root, "app", "OpenAIApp", "harness.yaml")).text()); expect(harness.model).toEqual({ provider: "open_ai", modelId: "gpt-5", apiKeyArn, }); + expect(harness.memory).toEqual({ mode: "managed" }); + expect(harness.systemPrompt).toBe("file://./system-prompt.md"); r.unmount(); }, 10000); diff --git a/src/handlers/project/export/harness.test.ts b/src/handlers/project/export/harness.test.ts index 348f636da..4d5e292d2 100644 --- a/src/handlers/project/export/harness.test.ts +++ b/src/handlers/project/export/harness.test.ts @@ -59,6 +59,8 @@ async function inProjectWithHarness( JSON.stringify({ provider: "bedrock", modelId: "us.amazon.nova-lite-v1:0", maxTokens: 256 }), "--system-prompt", "You are a terse assistant.", + "--memory", + '{"mode":"disabled"}', ]); return projectRoot; } diff --git a/src/handlers/project/export/serviceHarness.test.ts b/src/handlers/project/export/serviceHarness.test.ts index 5f2e0d1f4..db7f0e30e 100644 --- a/src/handlers/project/export/serviceHarness.test.ts +++ b/src/handlers/project/export/serviceHarness.test.ts @@ -282,7 +282,7 @@ describe("mapServiceHarnessToSpec", () => { }); // The pinned CDK only maps additionalParams for lite_llm, so carrying it on another provider - // would produce a harness.json that fails at synth. The lite_llm keep-path is already asserted + // would produce a harness.yaml that fails at synth. The lite_llm keep-path is already asserted // by "maps openai and litellm model configs" above. test("notes additionalParams the CDK cannot map", () => { const { spec, notes } = mapServiceHarnessToSpec( diff --git a/src/handlers/project/project.test.ts b/src/handlers/project/project.test.ts index 61b188de4..3efc3607c 100644 --- a/src/handlers/project/project.test.ts +++ b/src/handlers/project/project.test.ts @@ -2,6 +2,7 @@ import { afterEach, test, expect, describe } from "bun:test"; import { existsSync } from "node:fs"; import { mkdir, readdir, rm } from "node:fs/promises"; import { join } from "node:path"; +import { parse } from "yaml"; import { createRootHandler } from "../index"; import { createSilentLogger, @@ -74,12 +75,18 @@ describe("project create", () => { expect(spec.harnesses).toEqual([{ name: "MyAgent", path: "app/MyAgent" }]); expect(spec.runtimes).toEqual([]); - const harness = await Bun.file(join(projectRoot, "app", "MyAgent", "harness.json")).json(); + const harness = parse( + await Bun.file(join(projectRoot, "app", "MyAgent", "harness.yaml")).text(), + ); expect(harness.model).toEqual({ provider: "bedrock", modelId: "global.anthropic.claude-sonnet-4-6", }); - expect(harness.memory).toBeUndefined(); + expect(harness.memory).toEqual({ mode: "managed" }); + expect(harness.systemPrompt).toBe("file://./system-prompt.md"); + expect(harness.tools).toBeUndefined(); + expect(harness.skills).toBeUndefined(); + expect(existsSync(join(projectRoot, "app", "MyAgent", "harness.json"))).toBe(false); expect(await Bun.file(join(projectRoot, "app", "MyAgent", "system-prompt.md")).exists()).toBe( true, ); diff --git a/src/handlers/project/status/screen.tsx b/src/handlers/project/status/screen.tsx index b40c73fb4..e352fed59 100644 --- a/src/handlers/project/status/screen.tsx +++ b/src/handlers/project/status/screen.tsx @@ -68,7 +68,7 @@ function routeFor( // env var for every declared memory into every runtime (see // src/core/project/templates/runtime.ts), so each declared memory is reachable // from each runtime agent. A harness's memory binding lives in its own -// harness.json (HarnessMemoryRefSchema), not in the project spec this report is +// harness.yaml (HarnessMemoryRefSchema), not in the project spec this report is // built from — a managed one is provisioned inside the harness and never // appears here — so harness groups list just the harness itself and memories a // harness may reference by name stay visible under the project group. diff --git a/src/io/harnessConfig.test.ts b/src/io/harnessConfig.test.ts new file mode 100644 index 000000000..c19c0bd48 --- /dev/null +++ b/src/io/harnessConfig.test.ts @@ -0,0 +1,246 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { chmod, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { stringify } from "yaml"; +import { HarnessConfigReader } from "./harnessConfig"; +import { HarnessConfigReader as CdkHarnessConfigReader } from "../assets/cdk/io/harnessConfig"; +import { HarnessSpecSchema } from "../projectSchemas/harness"; + +const roots: string[] = []; +const originalCwd = process.cwd(); +const model = { provider: "bedrock", modelId: "example" }; + +afterEach(async () => { + process.chdir(originalCwd); + await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); +}); + +async function fixture(config: unknown) { + const root = await mkdtemp(join(tmpdir(), "harness-yaml-reader-")); + roots.push(root); + const directory = join(root, "nested harness"); + await mkdir(directory); + const path = join(directory, "harness.yaml"); + await writeFile( + path, + typeof config === "string" + ? config + : stringify({ name: "assistant", model, ...(config as object) }), + ); + return { root, directory, path }; +} + +for (const [label, Reader] of [ + ["CLI", HarnessConfigReader], + ["generated CDK", CdkHarnessConfigReader], +] as const) { + describe(`${label} harness YAML reader`, () => { + test.each(["system", "summary"] as const)( + "resolves only the %s prompt field from a YAML-relative path", + async (field) => { + const reference = "file://../chosen #100%.md"; + const config = + field === "system" + ? { systemPrompt: reference } + : { + truncation: { + strategy: "summarization", + config: { summarization: { summarizationSystemPrompt: reference } }, + }, + }; + const { root, path } = await fixture({ + ...config, + skills: [{ path: "file://./runtime-only" }], + environmentVariables: { PROMPT: "file://./not-an-include" }, + model: { + provider: "lite_llm", + modelId: "test", + additionalParams: { systemPrompt: "file://./not-an-include" }, + }, + }); + const text = " Literal contents: # 100%\r\nfile://./not-a-recursive-include\n"; + await writeFile(join(root, "chosen #100%.md"), text); + process.chdir(root); + const before = await readFile(path, "utf8"); + const data = HarnessSpecSchema.parse(await new Reader().read(path)); + expect( + field === "system" + ? data.systemPrompt + : data.truncation?.config && + "summarization" in data.truncation.config && + data.truncation.config.summarization.summarizationSystemPrompt, + ).toBe(text); + expect(data.skills).toEqual([{ path: "file://./runtime-only" }]); + expect(data.environmentVariables?.PROMPT).toBe("file://./not-an-include"); + expect(data.model.additionalParams?.systemPrompt).toBe("file://./not-an-include"); + expect(await readFile(path, "utf8")).toBe(before); + }, + ); + + test("preserves inline prompts, including whitespace and YAML-special characters", async () => { + const literal = ' "hello": #yes\n[one, two] * & %\n'; + const { directory, path } = await fixture({ + systemPrompt: literal, + truncation: { + strategy: "summarization", + config: { summarization: { summarizationSystemPrompt: literal } }, + }, + }); + await mkdir(join(directory, "system-prompt.md")); + const data = HarnessSpecSchema.parse(await new Reader().read(path)); + expect(data.systemPrompt).toBe(literal); + expect(data.truncation?.config).toEqual({ + summarization: { summarizationSystemPrompt: literal }, + }); + }); + + test.each(["./chosen.md", "../chosen.md", "absolute"] as const)( + "supports the %s filesystem reference", + async (source) => { + const { directory, root, path } = await fixture({}); + const promptPath = join(source === "../chosen.md" ? root : directory, "chosen.md"); + await writeFile(promptPath, "Selected prompt."); + await writeFile(join(directory, "system-prompt.md"), "Fallback loses."); + await writeFile( + path, + stringify({ + name: "assistant", + model, + systemPrompt: `file://${source === "absolute" ? promptPath : source}`, + }), + ); + expect(HarnessSpecSchema.parse(await new Reader().read(path)).systemPrompt).toBe( + "Selected prompt.", + ); + }, + ); + + test("uses the conventional prompt only when explicit text is absent, without defaulting memory", async () => { + const { path, directory } = await fixture({}); + expect(HarnessSpecSchema.parse(await new Reader().read(path)).systemPrompt).toBeUndefined(); + await writeFile(join(directory, "system-prompt.md"), " Conventional prompt.\n"); + const data = HarnessSpecSchema.parse(await new Reader().read(path)); + expect(data.systemPrompt).toBe(" Conventional prompt.\n"); + expect(data.memory).toBeUndefined(); + }); + + test.each([ + ["malformed YAML", "name: [", /flow sequence|YAML/i], + ["duplicate keys", "name: one\nname: two", /unique/], + ["nested duplicate keys", "model:\n provider: bedrock\n provider: lite_llm\n", /unique/], + ["multiple documents", "name: one\n---\nname: two", /multiple documents/i], + ] as const)("reports %s with the source path", async (_label, yaml, message) => { + const { path } = await fixture(yaml); + await expect(new Reader().read(path)).rejects.toThrow(message); + await expect(new Reader().read(path)).rejects.toThrow(path); + }); + + test.each([ + "", + "null", + "[]", + "name: assistant\nmodel: false", + "name: assistant\nmodel: {provider: bedrock, modelId: example}\nsystemPrompt: ''", + ])("leaves invalid schema data to HarnessSpecSchema: %s", async (yaml) => { + const { path } = await fixture(yaml); + expect(HarnessSpecSchema.safeParse(await new Reader().read(path)).success).toBe(false); + }); + + test.each([ + "missing", + "empty", + "whitespace", + "directory", + "oversized", + "invalid UTF-8", + ] as const)("rejects %s referenced files in both fields", async (condition) => { + const { directory, path } = await fixture({}); + const promptPath = join(directory, "prompt.md"); + if (condition === "directory") await mkdir(promptPath); + else if (condition !== "missing") + await writeFile( + promptPath, + condition === "oversized" + ? "x".repeat(1024 * 1024 + 1) + : condition === "invalid UTF-8" + ? Buffer.from([0xff]) + : condition === "whitespace" + ? " \r\n\t" + : "", + ); + for (const config of [ + { systemPrompt: "file://./prompt.md" }, + { + truncation: { + strategy: "summarization", + config: { summarization: { summarizationSystemPrompt: "file://./prompt.md" } }, + }, + }, + ]) { + await writeFile(path, stringify({ name: "assistant", model, ...config })); + await expect(new Reader().read(path)).rejects.toThrow(); + await expect(new Reader().read(path)).rejects.toThrow(path); + } + }); + + test("accepts exactly 1 MiB and rejects an empty conventional file", async () => { + const { directory, path } = await fixture({}); + const promptPath = join(directory, "system-prompt.md"); + const text = "x".repeat(1024 * 1024); + await writeFile(promptPath, text); + expect(HarnessSpecSchema.parse(await new Reader().read(path)).systemPrompt).toBe(text); + await writeFile(promptPath, ""); + await expect(new Reader().read(path)).rejects.toThrow(/empty/); + }); + + test.skipIf(process.platform === "win32" || process.getuid?.() === 0)( + "reports unreadable referenced files", + async () => { + const { directory, path } = await fixture({ systemPrompt: "file://./locked.md" }); + const promptPath = join(directory, "locked.md"); + await writeFile(promptPath, "Private prompt."); + await chmod(promptPath, 0); + try { + await expect(new Reader().read(path)).rejects.toThrow( + /cannot read prompt file.*locked.md/, + ); + } finally { + await chmod(promptPath, 0o600); + } + }, + ); + + test("does not read misplaced prompt keys or overwrite shared YAML aliases", async () => { + const { path, directory } = await fixture( + "name: assistant\nmodel: {provider: bedrock, modelId: test}\nsummarizationSystemPrompt: file://./missing\ntruncation:\n strategy: summarization\n config:\n summarization: &summary\n summarizationSystemPrompt: file://./prompt.md\nother: *summary\n", + ); + await writeFile(join(directory, "prompt.md"), "Summary text."); + const data = (await new Reader().read(path)) as Record; + expect(data.summarizationSystemPrompt).toBe("file://./missing"); + expect(data.other).toEqual({ summarizationSystemPrompt: "file://./prompt.md" }); + }); + + test("diagnoses obsolete JSON without reading or renaming it", async () => { + const { directory, path } = await fixture({}); + await rm(path); + const json = join(directory, "harness.json"); + await writeFile(json, "{}"); + await expect(new Reader().read(path)).rejects.toThrow( + /Obsolete harness.json.*Migrate.*copied agentcore\/cdk/s, + ); + expect(await readFile(json, "utf8")).toBe("{}"); + }); + + test("does not fall back after an explicit reference fails", async () => { + const { directory, path } = await fixture({ systemPrompt: "file://" }); + await writeFile(join(directory, "system-prompt.md"), "Fallback cannot hide this error."); + await expect(new Reader().read(path)).rejects.toThrow(/requires a path/); + await writeFile( + path, + stringify({ name: "assistant", model, systemPrompt: "file://./missing.md" }), + ); + await expect(new Reader().read(path)).rejects.toThrow(/missing.md/); + }); + }); +} diff --git a/src/io/harnessConfig.ts b/src/io/harnessConfig.ts new file mode 100644 index 000000000..e4e6ad92c --- /dev/null +++ b/src/io/harnessConfig.ts @@ -0,0 +1,104 @@ +import { open, readFile, stat } from "node:fs/promises"; +import { dirname, join, resolve } from "node:path"; +import { parseDocument } from "yaml"; + +const MAX_PROMPT_FILE_SIZE = 1024 * 1024; +const PROMPT_FIELDS = [ + ["systemPrompt"], + ["truncation", "config", "summarization", "summarizationSystemPrompt"], +] as const; + +/** Project-file I/O only; callers validate the resolved data with their HarnessSpecSchema. */ +export class HarnessConfigReader { + async read(filePath: string): Promise { + const configPath = resolve(filePath); + try { + let raw: string; + try { + raw = await readFile(configPath, "utf8"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + const obsoletePath = join(dirname(configPath), "harness.json"); + if (await stat(obsoletePath).catch(() => undefined)) { + throw new Error( + `Obsolete harness.json at '${obsoletePath}'. Migrate it to harness.yaml and update ` + + "the copied agentcore/cdk app; see the README harness migration guide.", + ); + } + } + throw error; + } + const document = parseDocument(raw); + if (document.errors.length) throw document.errors[0]; + const data: unknown = document.toJS(); + if (!isRecord(data)) return data; + + // These are the only local-file fields. Skills and other runtime paths stay untouched. + promptFields: for (const keys of PROMPT_FIELDS) { + let parent = data; + for (const key of keys.slice(0, -1)) { + const child = parent[key]; + if (!isRecord(child)) continue promptFields; + parent[key] = { ...child }; + parent = parent[key] as Record; + } + const key = keys[keys.length - 1]!; + const value = parent[key]; + if (typeof value === "string" && value.startsWith("file://")) { + const source = value.slice("file://".length); + if (!source) throw new Error(`${keys.join(".")}: file:// requires a path`); + parent[key] = await this.readPrompt(resolve(dirname(configPath), source), keys.join(".")); + } + } + if (data.systemPrompt === undefined) { + const fallback = join(dirname(configPath), "system-prompt.md"); + try { + data.systemPrompt = await this.readPrompt(fallback, "systemPrompt"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + } + return data; + } catch (error) { + throw new Error( + `Could not read harness.yaml at '${configPath}': ${error instanceof Error ? error.message : error}`, + { cause: error }, + ); + } + } + + private async readPrompt(filePath: string, field: string): Promise { + const file = await open(filePath, "r").catch((error: NodeJS.ErrnoException) => { + throw Object.assign( + new Error(`${field}: cannot read prompt file '${filePath}': ${error.message}`, { + cause: error, + }), + { code: error.code }, + ); + }); + try { + const info = await file.stat(); + if (!info.isFile()) throw new Error(`${field}: '${filePath}' is not a regular prompt file`); + const bytes = Buffer.alloc(MAX_PROMPT_FILE_SIZE + 1); + let length = 0; + while (length < bytes.length) { + const { bytesRead } = await file.read(bytes, length, bytes.length - length, null); + if (!bytesRead) break; + length += bytesRead; + } + if (length > MAX_PROMPT_FILE_SIZE) { + throw new Error(`${field}: prompt file '${filePath}' exceeds the 1 MiB limit`); + } + const text = new TextDecoder("utf-8", { fatal: true }).decode(bytes.subarray(0, length)); + if (!text.trim()) + throw new Error(`${field}: prompt file '${filePath}' is empty or whitespace-only`); + return text; + } finally { + await file.close(); + } + } +} + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === "object" && !Array.isArray(value); +} diff --git a/src/projectSchemas/harness.test.ts b/src/projectSchemas/harness.test.ts index b830337aa..ab93a1ac8 100644 --- a/src/projectSchemas/harness.test.ts +++ b/src/projectSchemas/harness.test.ts @@ -42,7 +42,7 @@ describe("harness custom validation", () => { ).toBe(false); }); // The pinned @aws/agentcore-cdk rejects additionalParams on every provider but lite_llm, and - // re-parses harness.json at synth — so accepting it here would defer the failure to + // re-parses harness.yaml at synth — so accepting it here would defer the failure to // `project build` instead of surfacing it at authoring time. it("accepts additional parameters only for the lite_llm provider", () => { expect( From e161ab4545d9b60597a48998c0f9498d98ad761d Mon Sep 17 00:00:00 2001 From: Aidan Daly Date: Mon, 14 Sep 2026 20:20:37 +0000 Subject: [PATCH 2/8] fix: validate harness authoring and prompt sources Keep YAML configuration failures user-classified, validate project prompt references before literal-domain validation, and reject FIFO sources without blocking. Build the generated CDK app before tests to prevent missing or stale executable output. --- README.md | 5 ++ src/assets/cdk/README.md | 2 +- src/assets/cdk/io/harnessConfig.ts | 47 ++++++++----- src/assets/cdk/package.json | 2 +- src/core/project/templates/harness.test.ts | 7 ++ src/core/project/templates/harness.ts | 7 +- .../project/add/harness/index.test.ts | 33 +++++++++ src/handlers/project/add/harness/index.ts | 4 +- src/handlers/project/export/harness.test.ts | 52 ++++++++++++++ src/io/harnessConfig.test.ts | 51 ++++++++++++++ src/io/harnessConfig.ts | 67 +++++++++++-------- src/projectSchemas/harness-authoring.ts | 27 ++++++++ 12 files changed, 251 insertions(+), 53 deletions(-) create mode 100644 src/projectSchemas/harness-authoring.ts diff --git a/README.md b/README.md index f6ac2b37c..96b52a315 100644 --- a/README.md +++ b/README.md @@ -204,6 +204,11 @@ must be readable, nonempty UTF-8 text of at most **1 MiB each**; whitespace-only files are rejected. Inline text and file contents preserve their whitespace. File contents are not interpreted as further references. +`project add harness --system-prompt file://./selected.md` preserves that +reference in the generated YAML. Scaffolding validates reference syntax without +reading or copying the selected file; build and export resolve it from the new +harness directory. A literal `--system-prompt` is written to `system-prompt.md`. + Skills are unchanged: skill paths refer to the **runtime/container filesystem**, not local files to package. Other fields do not support local includes. Malformed YAML, duplicate keys, and existing schema violations fail the read. diff --git a/src/assets/cdk/README.md b/src/assets/cdk/README.md index be548731f..f6fd808dd 100644 --- a/src/assets/cdk/README.md +++ b/src/assets/cdk/README.md @@ -11,7 +11,7 @@ This CDK project is managed by the AgentCore CLI. It deploys your agent infrastr ## Useful commands - `npm run build` compile TypeScript to JavaScript -- `npm run test` run unit tests +- `npm run test` compile the app and run unit tests - `npx cdk synth` emit the synthesized CloudFormation template - `npx cdk deploy` deploy this stack to your default AWS account/region - `npx cdk diff` compare deployed stack with current state diff --git a/src/assets/cdk/io/harnessConfig.ts b/src/assets/cdk/io/harnessConfig.ts index 89e54dc01..efbd93e95 100644 --- a/src/assets/cdk/io/harnessConfig.ts +++ b/src/assets/cdk/io/harnessConfig.ts @@ -1,3 +1,4 @@ +import { constants } from "node:fs"; import { open, readFile, stat } from "node:fs/promises"; import { dirname, join, resolve } from "node:path"; import { parseDocument } from "yaml"; @@ -71,27 +72,37 @@ export class HarnessConfigReader { } private async readPrompt(filePath: string, field: string): Promise { - const file = await open(filePath, "r").catch((error: NodeJS.ErrnoException) => { - throw Object.assign(new Error(`${field}: cannot read prompt file '${filePath}': ${error.message}`, { cause: error }), { code: error.code }); - }); try { - const info = await file.stat(); + const info = await stat(filePath); if (!info.isFile()) throw new Error(`${field}: '${filePath}' is not a regular prompt file`); - const bytes = Buffer.alloc(MAX_PROMPT_FILE_SIZE + 1); - let length = 0; - while (length < bytes.length) { - const { bytesRead } = await file.read(bytes, length, bytes.length - length, null); - if (!bytesRead) break; - length += bytesRead; - } - if (length > MAX_PROMPT_FILE_SIZE) { - throw new Error(`${field}: prompt file '${filePath}' exceeds the 1 MiB limit`); + // A path can change after stat; nonblocking POSIX opens avoid waiting on a replacement FIFO. + const flags = constants.O_RDONLY | (process.platform === "win32" ? 0 : constants.O_NONBLOCK); + const file = await open(filePath, flags); + try { + if (!(await file.stat()).isFile()) throw new Error(`${field}: '${filePath}' is not a regular prompt file`); + const bytes = Buffer.alloc(MAX_PROMPT_FILE_SIZE + 1); + let length = 0; + while (length < bytes.length) { + const { bytesRead } = await file.read(bytes, length, bytes.length - length, null); + if (!bytesRead) break; + length += bytesRead; + } + if (length > MAX_PROMPT_FILE_SIZE) { + throw new Error(`${field}: prompt file '${filePath}' exceeds the 1 MiB limit`); + } + const text = new TextDecoder("utf-8", { fatal: true }).decode(bytes.subarray(0, length)); + if (!text.trim()) throw new Error(`${field}: prompt file '${filePath}' is empty or whitespace-only`); + return text; + } finally { + await file.close(); } - const text = new TextDecoder("utf-8", { fatal: true }).decode(bytes.subarray(0, length)); - if (!text.trim()) throw new Error(`${field}: prompt file '${filePath}' is empty or whitespace-only`); - return text; - } finally { - await file.close(); + } catch (error) { + throw Object.assign( + new Error(`${field}: cannot read prompt file '${filePath}': ${error instanceof Error ? error.message : error}`, { + cause: error, + }), + { code: (error as NodeJS.ErrnoException).code }, + ); } } } diff --git a/src/assets/cdk/package.json b/src/assets/cdk/package.json index 9691987a2..9da1463ed 100644 --- a/src/assets/cdk/package.json +++ b/src/assets/cdk/package.json @@ -7,7 +7,7 @@ "scripts": { "build": "tsc", "watch": "tsc -w", - "test": "jest", + "test": "npm run build && jest", "cdk": "npm run build && cdk", "clean": "rm -rf dist", "format": "prettier --write .", diff --git a/src/core/project/templates/harness.test.ts b/src/core/project/templates/harness.test.ts index bef972e0b..959ffc428 100644 --- a/src/core/project/templates/harness.test.ts +++ b/src/core/project/templates/harness.test.ts @@ -147,3 +147,10 @@ test("preserves a supplied explicit prompt reference", async () => { test("defaults only absent memory and does not mask an invalid supplied setting", async () => { await expect(scaffold({ memory: null })).rejects.toThrow(); }); + +test.each(["file://", "", " \n", "./legacy.md"])( + "shared project scaffolding rejects invalid authoring prompt %j", + async (systemPrompt) => { + await expect(scaffold({ systemPrompt })).rejects.toThrow(); + }, +); diff --git a/src/core/project/templates/harness.ts b/src/core/project/templates/harness.ts index f0d16d8b9..9aae6f8f3 100644 --- a/src/core/project/templates/harness.ts +++ b/src/core/project/templates/harness.ts @@ -1,6 +1,7 @@ import { existsSync } from "node:fs"; import { ZodError, z } from "zod"; import { HarnessSpecSchema } from "../../../projectSchemas/harness"; +import { HarnessAuthoringSchema } from "../../../projectSchemas/harness-authoring"; import { FsTreeNode } from "./fsTree"; import { InputValidationError, ResourceNotFoundError } from "../../../errors/errors"; import type { TemplateResolver } from "./types"; @@ -14,12 +15,12 @@ export function getHarnessTemplateResolver(): TemplateResolver) { try { - return HarnessSpecSchema.parse(spec); + return HarnessAuthoringSchema.parse(spec); } catch (err) { if (err instanceof ZodError) throw new InputValidationError(z.prettifyError(err)); throw err; diff --git a/src/handlers/project/add/harness/index.test.ts b/src/handlers/project/add/harness/index.test.ts index 0ee76e2fd..16945ad37 100644 --- a/src/handlers/project/add/harness/index.test.ts +++ b/src/handlers/project/add/harness/index.test.ts @@ -2,6 +2,8 @@ import { afterEach, describe, expect, test } from "bun:test"; import { existsSync } from "node:fs"; import { join } from "node:path"; import { parse } from "yaml"; +import { HarnessConfigReader } from "../../../../io/harnessConfig"; +import { HarnessSpecSchema } from "../../../../projectSchemas/harness"; import { createRootHandler } from "../../../index"; import { createSilentLogger, @@ -442,6 +444,37 @@ describe("project add harness", () => { expect(existsSync(join(projectRoot, "app", "x", "harness.json"))).toBe(false); }); + test.each(["selected.md", "selected.txt", "selected", "chosen #100%.md"])( + "preserves the %s authoring reference relative to the resulting YAML directory", + async (filename) => { + const { projectRoot, cleanup } = await initProject(); + cleanups.push(cleanup); + const reference = `file://./${filename}`; + await Bun.write(join(projectRoot, filename), "Caller CWD decoy."); + await run(["add", "harness", "--name", "Reference", "--system-prompt", reference]); + const directory = join(projectRoot, "app", "Reference"); + const path = join(directory, "harness.yaml"); + expect(parse(await Bun.file(path).text()).systemPrompt).toBe(reference); + expect(await Bun.file(join(directory, "system-prompt.md")).text()).toBe( + "You are a helpful assistant", + ); + await expect(new HarnessConfigReader().read(path)).rejects.toThrow(filename); + await Bun.write(join(directory, filename), " YAML-relative contents.\r\n"); + expect(HarnessSpecSchema.parse(await new HarnessConfigReader().read(path)).systemPrompt).toBe( + " YAML-relative contents.\r\n", + ); + }, + ); + + test("rejects an empty project authoring reference before writing files", async () => { + const { projectRoot, cleanup } = await initProject(); + cleanups.push(cleanup); + await expect( + run(["add", "harness", "--name", "EmptyReference", "--system-prompt", "file://"]), + ).rejects.toThrow(/systemPrompt.*file:\/\/ requires a path/s); + expect(existsSync(join(projectRoot, "app", "EmptyReference"))).toBe(false); + }); + test("--dockerfile copies the file into the harness directory and stores the relative path", async () => { const { projectRoot, cleanup } = await initProject(); cleanups.push(cleanup); diff --git a/src/handlers/project/add/harness/index.ts b/src/handlers/project/add/harness/index.ts index 90b4b8b7b..44ccc7374 100644 --- a/src/handlers/project/add/harness/index.ts +++ b/src/handlers/project/add/harness/index.ts @@ -4,7 +4,7 @@ import type { AddProjectResourceConfig } from "../types"; import { addProjectResource } from "../shared"; import { parseJsonFlag, parseTags } from "../../../utils"; import { InputValidationError } from "../../../../errors"; -import { HarnessSpecSchema } from "../../../../projectSchemas/harness"; +import { HarnessAuthoringSchema } from "../../../../projectSchemas/harness-authoring"; /** The model a harness runs on when none is configured; `project create`'s * harness path shares it so the two entry points cannot drift. */ @@ -105,7 +105,7 @@ export const createAddHarnessHandler = (config: AddProjectResourceConfig) => dockerfile: flags["dockerfile"], }; - const result = HarnessSpecSchema.safeParse(harnessInput); + const result = HarnessAuthoringSchema.safeParse(harnessInput); if (!result.success) throw new InputValidationError(z.prettifyError(result.error), { cause: result.error }); diff --git a/src/handlers/project/export/harness.test.ts b/src/handlers/project/export/harness.test.ts index 4d5e292d2..66fa2541d 100644 --- a/src/handlers/project/export/harness.test.ts +++ b/src/handlers/project/export/harness.test.ts @@ -1,6 +1,9 @@ import { afterEach, describe, expect, test } from "bun:test"; import { existsSync } from "node:fs"; +import { mkdir, writeFile } from "node:fs/promises"; import { join } from "node:path"; +import { parse, stringify } from "yaml"; +import { AgentCoreCLIError, DeserializationError } from "../../../errors"; import { createRootHandler } from "../../index"; import { createSilentLogger, @@ -66,6 +69,55 @@ async function inProjectWithHarness( } describe("project export harness handler", () => { + test.each([ + "malformed YAML", + "missing main", + "missing summary", + "empty main", + "directory", + "invalid UTF-8", + ] as const)( + "classifies %s as customer configuration through the CLI boundary", + async (failure) => { + const subject = testExportCommand(); + const projectRoot = await inProjectWithHarness(subject); + const directory = join(projectRoot, "app", "exportme"); + const path = join(directory, "harness.yaml"); + const config = parse(await Bun.file(path).text()); + let field = "systemPrompt"; + if (failure === "missing summary") { + field = "truncation.config.summarization.summarizationSystemPrompt"; + config.truncation = { + strategy: "summarization", + config: { summarization: { summarizationSystemPrompt: "file://./missing.md" } }, + }; + } else { + config.systemPrompt = "file://./selected.md"; + } + if (failure === "empty main") await writeFile(join(directory, "selected.md"), " \n"); + if (failure === "invalid UTF-8") + await writeFile(join(directory, "selected.md"), Buffer.from([0xff])); + if (failure === "directory") await mkdir(join(directory, "selected.md")); + await writeFile(path, failure === "malformed YAML" ? "name: [" : stringify(config)); + const specPath = join(projectRoot, "agentcore", "agentcore.json"); + const before = await Bun.file(specPath).text(); + const error = await subject + .run(["--name", "exportme", "--json"]) + .catch(AgentCoreCLIError.fromError); + expect(error).toBeInstanceOf(DeserializationError); + expect(error).toMatchObject({ + source: "user", + exitCode: 1, + name: "DeserializationError", + cause: expect.any(Error), + }); + expect((error as Error).message).toContain(path); + if (failure !== "malformed YAML") expect((error as Error).message).toContain(field); + expect(existsSync(join(projectRoot, "app", "exportmeAgent"))).toBe(false); + expect(await Bun.file(specPath).text()).toBe(before); + }, + ); + test("requires exactly one of --name and --arn", async () => { const subject = testExportCommand(); await inProjectWithHarness(subject); diff --git a/src/io/harnessConfig.test.ts b/src/io/harnessConfig.test.ts index c19c0bd48..38b066ac6 100644 --- a/src/io/harnessConfig.test.ts +++ b/src/io/harnessConfig.test.ts @@ -1,7 +1,9 @@ import { afterEach, describe, expect, test } from "bun:test"; +import { spawnSync } from "node:child_process"; import { chmod, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { fileURLToPath } from "node:url"; import { stringify } from "yaml"; import { HarnessConfigReader } from "./harnessConfig"; import { HarnessConfigReader as CdkHarnessConfigReader } from "../assets/cdk/io/harnessConfig"; @@ -36,6 +38,55 @@ for (const [label, Reader] of [ ["generated CDK", CdkHarnessConfigReader], ] as const) { describe(`${label} harness YAML reader`, () => { + test.skipIf(process.platform === "win32").each(["system", "summary", "fallback"] as const)( + "rejects a FIFO %s prompt in a bounded subprocess", + async (field) => { + const { directory, path } = await fixture( + field === "system" + ? { systemPrompt: "file://./pipe" } + : field === "summary" + ? { + truncation: { + strategy: "summarization", + config: { summarization: { summarizationSystemPrompt: "file://./pipe" } }, + }, + } + : {}, + ); + const fifo = join(directory, field === "fallback" ? "system-prompt.md" : "pipe"); + const setup = spawnSync("mkfifo", [fifo], { timeout: 2000, encoding: "utf8" }); + expect(setup.error).toBeUndefined(); + expect(setup.status).toBe(0); + const readerPath = fileURLToPath( + new URL( + label === "CLI" ? "./harnessConfig.ts" : "../assets/cdk/io/harnessConfig.ts", + import.meta.url, + ), + ); + const result = spawnSync( + process.execPath, + [ + "--eval", + ` + import { HarnessConfigReader } from ${JSON.stringify(readerPath)}; + try { + await new HarnessConfigReader().read(${JSON.stringify(path)}); + process.exitCode = 2; + } catch (error) { + console.error(error.message); + process.exitCode = 1; + } + `, + ], + { timeout: 2000, killSignal: "SIGKILL", encoding: "utf8" }, + ); + expect(result.error).toBeUndefined(); + expect(result.status).toBe(1); + expect(result.stderr).toContain("not a regular prompt file"); + expect(result.stderr).toContain(fifo); + }, + ); + test.each(["system", "summary"] as const)( "resolves only the %s prompt field from a YAML-relative path", async (field) => { diff --git a/src/io/harnessConfig.ts b/src/io/harnessConfig.ts index e4e6ad92c..68808f37e 100644 --- a/src/io/harnessConfig.ts +++ b/src/io/harnessConfig.ts @@ -1,6 +1,8 @@ +import { constants } from "node:fs"; import { open, readFile, stat } from "node:fs/promises"; import { dirname, join, resolve } from "node:path"; import { parseDocument } from "yaml"; +import { DeserializationError } from "../errors"; const MAX_PROMPT_FILE_SIZE = 1024 * 1024; const PROMPT_FIELDS = [ @@ -60,41 +62,50 @@ export class HarnessConfigReader { } return data; } catch (error) { - throw new Error( - `Could not read harness.yaml at '${configPath}': ${error instanceof Error ? error.message : error}`, - { cause: error }, - ); + throw new DeserializationError(configPath, { + cause: error, + details: error instanceof Error ? error.message : String(error), + }); } } private async readPrompt(filePath: string, field: string): Promise { - const file = await open(filePath, "r").catch((error: NodeJS.ErrnoException) => { - throw Object.assign( - new Error(`${field}: cannot read prompt file '${filePath}': ${error.message}`, { - cause: error, - }), - { code: error.code }, - ); - }); try { - const info = await file.stat(); + const info = await stat(filePath); if (!info.isFile()) throw new Error(`${field}: '${filePath}' is not a regular prompt file`); - const bytes = Buffer.alloc(MAX_PROMPT_FILE_SIZE + 1); - let length = 0; - while (length < bytes.length) { - const { bytesRead } = await file.read(bytes, length, bytes.length - length, null); - if (!bytesRead) break; - length += bytesRead; - } - if (length > MAX_PROMPT_FILE_SIZE) { - throw new Error(`${field}: prompt file '${filePath}' exceeds the 1 MiB limit`); + // A path can change after stat; nonblocking POSIX opens avoid waiting on a replacement FIFO. + const flags = constants.O_RDONLY | (process.platform === "win32" ? 0 : constants.O_NONBLOCK); + const file = await open(filePath, flags); + try { + if (!(await file.stat()).isFile()) + throw new Error(`${field}: '${filePath}' is not a regular prompt file`); + const bytes = Buffer.alloc(MAX_PROMPT_FILE_SIZE + 1); + let length = 0; + while (length < bytes.length) { + const { bytesRead } = await file.read(bytes, length, bytes.length - length, null); + if (!bytesRead) break; + length += bytesRead; + } + if (length > MAX_PROMPT_FILE_SIZE) { + throw new Error(`${field}: prompt file '${filePath}' exceeds the 1 MiB limit`); + } + const text = new TextDecoder("utf-8", { fatal: true }).decode(bytes.subarray(0, length)); + if (!text.trim()) + throw new Error(`${field}: prompt file '${filePath}' is empty or whitespace-only`); + return text; + } finally { + await file.close(); } - const text = new TextDecoder("utf-8", { fatal: true }).decode(bytes.subarray(0, length)); - if (!text.trim()) - throw new Error(`${field}: prompt file '${filePath}' is empty or whitespace-only`); - return text; - } finally { - await file.close(); + } catch (error) { + throw Object.assign( + new Error( + `${field}: cannot read prompt file '${filePath}': ${error instanceof Error ? error.message : error}`, + { + cause: error, + }, + ), + { code: (error as NodeJS.ErrnoException).code }, + ); } } } diff --git a/src/projectSchemas/harness-authoring.ts b/src/projectSchemas/harness-authoring.ts new file mode 100644 index 000000000..5ef56fefb --- /dev/null +++ b/src/projectSchemas/harness-authoring.ts @@ -0,0 +1,27 @@ +import { z } from "zod"; +import { HarnessSpecSchema } from "./harness"; + +/** Scaffold references name future YAML-relative files; only literal prompts enter the domain schema. */ +export const HarnessAuthoringSchema = z + .object({ systemPrompt: z.string().optional() }) + .passthrough() + .transform(({ systemPrompt, ...rest }, ctx) => { + const reference = systemPrompt?.startsWith("file://") ? systemPrompt : undefined; + if (reference === "file://") { + ctx.addIssue({ + code: "custom", + path: ["systemPrompt"], + message: "systemPrompt: file:// requires a path", + }); + return z.NEVER; + } + const parsed = HarnessSpecSchema.safeParse({ + ...rest, + systemPrompt: reference === undefined ? systemPrompt : undefined, + }); + if (!parsed.success) { + for (const issue of parsed.error.issues) ctx.addIssue({ ...issue }); + return z.NEVER; + } + return reference === undefined ? parsed.data : { ...parsed.data, systemPrompt: reference }; + }); From 3bdd0a7a51501dec73f7569132bcf979cf551601 Mon Sep 17 00:00:00 2001 From: Aidan Daly Date: Mon, 14 Sep 2026 23:02:09 +0000 Subject: [PATCH 3/8] fix: reject unknown harness root and model fields --- README.md | 8 +- src/assets/cdk/bin/cdk.ts | 3 +- src/assets/cdk/lib/harness-schema.ts | 6 ++ src/assets/cdk/test/harness.test.ts | 67 ++++++++++++++- .../__snapshots__/manager.test.ts.snap | 5 ++ src/core/project/templates/export.test.ts | 1 - src/core/project/templates/harness.test.ts | 18 +++++ .../project/add/harness/index.test.ts | 24 ++++++ src/handlers/project/export/harness.test.ts | 31 ++++++- src/projectSchemas/harness.test.ts | 81 +++++++++++++++++++ src/projectSchemas/harness.ts | 2 + 11 files changed, 238 insertions(+), 8 deletions(-) create mode 100644 src/assets/cdk/lib/harness-schema.ts diff --git a/README.md b/README.md index 96b52a315..546cc6d92 100644 --- a/README.md +++ b/README.md @@ -212,6 +212,11 @@ harness directory. A literal `--system-prompt` is written to `system-prompt.md`. Skills are unchanged: skill paths refer to the **runtime/container filesystem**, not local files to package. Other fields do not support local includes. Malformed YAML, duplicate keys, and existing schema violations fail the read. +Unknown fields at the harness root and directly inside `model` are rejected, +not silently removed or corrected. Nested configurations keep their existing +validation contracts; this is not a recursive unknown-field check. Free-form +maps such as headers, tags, environment variables, `additionalParams`, and +`inputSchema` still accept arbitrary keys. Build, deploy, and export do not rewrite harness YAML or remove its comments. `agentcore.json`, deployment targets, JSON CLI flags/output, and service payloads are unchanged. @@ -234,7 +239,8 @@ CDK app already copied into your project**: 3. Generate a separate reference project with the updated CLI: `agentcore project create --name HarnessYamlReference --template empty --skip-install --skip-git`. Compare its `agentcore/cdk/` with your project's copy. Port the YAML reader - (`io/harnessConfig.ts`) and the harness-loading changes in `bin/cdk.ts`; add + (`io/harnessConfig.ts`), schema composition (`lib/harness-schema.ts`), and + harness-loading changes in `bin/cdk.ts`; add the direct `yaml` dependency from `package.json` and the `io/**/*` include from `tsconfig.json`. Preserve your custom CDK code, especially `lib/cdk-stack.ts`. The reference app also includes a harness synthesis test. diff --git a/src/assets/cdk/bin/cdk.ts b/src/assets/cdk/bin/cdk.ts index 834ff126b..49a2973f9 100644 --- a/src/assets/cdk/bin/cdk.ts +++ b/src/assets/cdk/bin/cdk.ts @@ -1,10 +1,11 @@ #!/usr/bin/env node import { AgentCoreStack, type HarnessConfig } from '../lib/cdk-stack'; -import { ConfigIO, HarnessSpecSchema, type AwsDeploymentTarget } from '@aws/agentcore-cdk'; +import { ConfigIO, type AwsDeploymentTarget } from '@aws/agentcore-cdk'; import { App, type Environment } from 'aws-cdk-lib'; import * as path from 'path'; import * as fs from 'fs'; import { HarnessConfigReader } from '../io/harnessConfig'; +import { HarnessSpecSchema } from '../lib/harness-schema'; function toEnvironment(target: AwsDeploymentTarget): Environment { return { diff --git a/src/assets/cdk/lib/harness-schema.ts b/src/assets/cdk/lib/harness-schema.ts new file mode 100644 index 000000000..0590f3483 --- /dev/null +++ b/src/assets/cdk/lib/harness-schema.ts @@ -0,0 +1,6 @@ +import { HarnessSpecSchema as PublishedHarnessSpecSchema } from '@aws/agentcore-cdk'; + +// Match CLI root/model strictness without dropping the published schema's refinements. +export const HarnessSpecSchema: typeof PublishedHarnessSpecSchema = PublishedHarnessSpecSchema.strict().safeExtend({ + model: PublishedHarnessSpecSchema.shape.model.strict(), +}); diff --git a/src/assets/cdk/test/harness.test.ts b/src/assets/cdk/test/harness.test.ts index e0e41a551..6417ae0af 100644 --- a/src/assets/cdk/test/harness.test.ts +++ b/src/assets/cdk/test/harness.test.ts @@ -1,8 +1,9 @@ -import { execFileSync } from 'node:child_process'; +import { spawnSync } from 'node:child_process'; import { mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; import { stringify } from 'yaml'; +import { HarnessSpecSchema } from '../lib/harness-schema'; const entrypoint = resolve(__dirname, '..', 'dist/bin/cdk.js'); const roots: string[] = []; @@ -11,7 +12,49 @@ afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); }); -test.each(['literal', 'file', 'fallback'])('generated app synthesizes the %s prompt and summary as literal text', (source) => { +test.each([ + { provider: 'bedrock', modelId: 'example', apiFormat: 'converse_stream', temperature: 0, topP: 0, maxTokens: 512 }, + { provider: 'open_ai', modelId: 'example', apiKeyArn: 'arn:key', apiFormat: 'responses' }, + { provider: 'gemini', modelId: 'example', apiKeyArn: 'arn:key', topK: 40 }, + { provider: 'lite_llm', modelId: 'example', apiBase: 'https://example.com', additionalParams: { maxToken: 512, model: { maxIteration: 3 } } }, +])('preserves provider fields and free-form maps: %j', (model) => { + const map = { maxToken: '512', maxIteration: '3', model: 'custom' }; + const input = { + name: 'assistant', + model, + maxIterations: 3, + maxTokens: 1024, + timeoutSeconds: 60, + tags: map, + environmentVariables: map, + tools: [ + { type: 'remote_mcp', name: 'mcp', config: { remoteMcp: { url: 'https://example.com', headers: map } } }, + { type: 'inline_function', name: 'fn', config: { inlineFunction: { description: 'Custom schema', inputSchema: { maxToken: 512, properties: { maxIteration: { type: 'number' } } } } } }, + { type: 'agentcore_gateway', name: 'gateway', config: { agentCoreGateway: { gatewayArn: 'arn:gateway', maxIteration: 3 } } }, + ], + }; + expect(HarnessSpecSchema.parse(input)).toEqual({ ...input, skills: [] }); +}); + +test.each([ + { model: { provider: 'bedrock', modelId: 'example', topK: 40 } }, + { model: { provider: 'open_ai', modelId: 'example' } }, + { model: { provider: 'open_ai', modelId: 'example', apiKeyArn: 'arn:key', apiFormat: 'converse_stream' } }, + { model: { provider: 'gemini', modelId: 'example', apiKeyArn: 'arn:key', apiBase: 'https://example.com' } }, + { model: { provider: 'bedrock', modelId: 'example', additionalParams: { maxToken: 512 } } }, + { networkMode: 'VPC' }, + { authorizerType: 'CUSTOM_JWT' }, + { containerUri: '123456789012.dkr.ecr.us-east-1.amazonaws.com/repo:tag', dockerfile: 'Dockerfile' }, + { tools: [{ type: 'agentcore_browser', name: 'same' }, { type: 'agentcore_browser', name: 'same' }] }, +])('retains published refinements for %j', (overrides) => { + expect(HarnessSpecSchema.safeParse({ + name: 'assistant', + model: { provider: 'bedrock', modelId: 'example' }, + ...overrides, + }).success).toBe(false); +}); + +test.each(['literal', 'file', 'fallback', 'maxIteration', 'model.maxToken'])('generated app validates %s configuration without rewriting it', (source) => { const root = mkdtempSync(join(tmpdir(), 'harness-yaml-synth-')); roots.push(root); const configRoot = join(root, 'agentcore'); @@ -33,7 +76,12 @@ test.each(['literal', 'file', 'fallback'])('generated app synthesizes the %s pro writeFileSync(join(harnessDir, 'summary.md'), summary); const yaml = '# Customer comment stays intact.\n' + stringify({ name: 'assistant', - model: { provider: 'bedrock', modelId: 'global.anthropic.claude-sonnet-4-6' }, + model: { + provider: 'bedrock', + modelId: 'global.anthropic.claude-sonnet-4-6', + ...(source === 'model.maxToken' ? { maxToken: 512 } : {}), + }, + ...(source === 'maxIteration' ? { maxIteration: 3 } : {}), systemPrompt: source === 'fallback' ? undefined : source === 'literal' ? prompt : 'file://./chosen #100%.md', memory: { mode: 'disabled' }, truncation: { @@ -43,11 +91,22 @@ test.each(['literal', 'file', 'fallback'])('generated app synthesizes the %s pro }); writeFileSync(join(harnessDir, 'harness.yaml'), yaml); const outdir = join(root, 'cdk.out'); - execFileSync(process.execPath, [entrypoint], { + const result = spawnSync(process.execPath, [entrypoint], { cwd: cdkRoot, env: { ...process.env, INIT_CWD: root, CDK_OUTDIR: outdir }, stdio: 'pipe', + encoding: 'utf8', + timeout: 30000, }); + expect(result.error).toBeUndefined(); + expect(readFileSync(join(harnessDir, 'harness.yaml'), 'utf8')).toBe(yaml); + if (source === 'maxIteration' || source === 'model.maxToken') { + expect(result.status).toBe(1); + expect(result.stderr).toContain(join(harnessDir, 'harness.yaml')); + for (const key of source.split('.')) expect(result.stderr).toContain(key); + return; + } + expect(result.status).toBe(0); const templateFile = readdirSync(outdir).find((name) => name.endsWith('.template.json'))!; const template = JSON.parse(readFileSync(join(outdir, templateFile), 'utf8')); const harness = Object.values(template.Resources).find((resource: any) => resource.Type === 'AWS::BedrockAgentCore::Harness') as any; diff --git a/src/core/project/__snapshots__/manager.test.ts.snap b/src/core/project/__snapshots__/manager.test.ts.snap index 697a0de78..c646be3f6 100644 --- a/src/core/project/__snapshots__/manager.test.ts.snap +++ b/src/core/project/__snapshots__/manager.test.ts.snap @@ -15,6 +15,7 @@ exports[`FsProjectManager.create scaffolds the expected file tree into a fresh d "agentcore/cdk/io/harnessConfig.ts", "agentcore/cdk/jest.config.js", "agentcore/cdk/lib/cdk-stack.ts", + "agentcore/cdk/lib/harness-schema.ts", "agentcore/cdk/package.json", "agentcore/cdk/test/cdk.test.ts", "agentcore/cdk/test/harness.test.ts", @@ -41,6 +42,7 @@ exports[`FsProjectManager.create snapshots the Strands project manifest and runt "agentcore/cdk/io/harnessConfig.ts", "agentcore/cdk/jest.config.js", "agentcore/cdk/lib/cdk-stack.ts", + "agentcore/cdk/lib/harness-schema.ts", "agentcore/cdk/package.json", "agentcore/cdk/test/cdk.test.ts", "agentcore/cdk/test/harness.test.ts", @@ -118,6 +120,7 @@ exports[`FsProjectManager.create snapshots the Strands TypeScript project manife "agentcore/cdk/io/harnessConfig.ts", "agentcore/cdk/jest.config.js", "agentcore/cdk/lib/cdk-stack.ts", + "agentcore/cdk/lib/harness-schema.ts", "agentcore/cdk/package.json", "agentcore/cdk/test/cdk.test.ts", "agentcore/cdk/test/harness.test.ts", @@ -195,6 +198,7 @@ exports[`FsProjectManager.create snapshots the Strands A2A project manifest and "agentcore/cdk/io/harnessConfig.ts", "agentcore/cdk/jest.config.js", "agentcore/cdk/lib/cdk-stack.ts", + "agentcore/cdk/lib/harness-schema.ts", "agentcore/cdk/package.json", "agentcore/cdk/test/cdk.test.ts", "agentcore/cdk/test/harness.test.ts", @@ -272,6 +276,7 @@ exports[`FsProjectManager.create snapshots the LangChain project manifest and ru "agentcore/cdk/io/harnessConfig.ts", "agentcore/cdk/jest.config.js", "agentcore/cdk/lib/cdk-stack.ts", + "agentcore/cdk/lib/harness-schema.ts", "agentcore/cdk/package.json", "agentcore/cdk/test/cdk.test.ts", "agentcore/cdk/test/harness.test.ts", diff --git a/src/core/project/templates/export.test.ts b/src/core/project/templates/export.test.ts index a431dde6b..c57f9e9cd 100644 --- a/src/core/project/templates/export.test.ts +++ b/src/core/project/templates/export.test.ts @@ -473,7 +473,6 @@ describe("mapHarnessToExportPlan skills", () => { test("maps s3 and git skills and generates the S3 read policy", () => { const result = plan({ spec: harness({ - build: undefined, skills: [ { s3Uri: "s3://skills-bucket/team/" }, { gitUrl: "https://github.com/example/skills.git", path: "subdir" }, diff --git a/src/core/project/templates/harness.test.ts b/src/core/project/templates/harness.test.ts index 959ffc428..8f93488f5 100644 --- a/src/core/project/templates/harness.test.ts +++ b/src/core/project/templates/harness.test.ts @@ -4,6 +4,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { parse } from "yaml"; import type z from "zod"; +import { InputValidationError } from "../../../errors"; import { HarnessSpecSchema } from "../../../projectSchemas/harness"; import { HarnessConfigReader } from "../../../io/harnessConfig"; import { getHarnessTemplateResolver } from "./harness"; @@ -154,3 +155,20 @@ test.each(["file://", "", " \n", "./legacy.md"])( await expect(scaffold({ systemPrompt })).rejects.toThrow(); }, ); + +test.each([{ maxIteration: 3 }, { model: { ...model, maxToken: 512 } }])( + "shared scaffolding rejects unknown fixed fields: %j", + async (overrides) => { + const error = await getHarnessTemplateResolver() + .resolve({ + name: "assistant", + model, + systemPrompt: "file://./selected.md", + ...overrides, + }) + .catch((error: unknown) => error); + expect(error).toBeInstanceOf(InputValidationError); + expect(error).toMatchObject({ source: "user" }); + expect((error as Error).message).toContain("Unrecognized key"); + }, +); diff --git a/src/handlers/project/add/harness/index.test.ts b/src/handlers/project/add/harness/index.test.ts index 16945ad37..a4feb1db6 100644 --- a/src/handlers/project/add/harness/index.test.ts +++ b/src/handlers/project/add/harness/index.test.ts @@ -33,6 +33,30 @@ async function run(args: string[], opts?: { core?: TestCoreClient }) { describe("project add harness", () => { const defaultModel = { provider: "bedrock", modelId: "global.anthropic.claude-sonnet-4-6" }; + test("rejects a model typo as user input without scaffolding or registering a harness", async () => { + const { projectRoot, cleanup } = await initProject(); + cleanups.push(cleanup); + const specPath = join(projectRoot, "agentcore", "agentcore.json"); + const before = await Bun.file(specPath).text(); + const error = await run([ + "add", + "harness", + "--name", + "typo", + "--model", + JSON.stringify({ ...defaultModel, maxToken: 512 }), + "--system-prompt", + "file://./selected.md", + "--json", + ]).catch((error: unknown) => error); + expect(error).toBeInstanceOf(InputValidationError); + expect(error).toMatchObject({ source: "user", exitCode: 1 }); + expect((error as Error).message).toContain("model"); + expect((error as Error).message).toContain("maxToken"); + expect(existsSync(join(projectRoot, "app", "typo"))).toBe(false); + expect(await Bun.file(specPath).text()).toBe(before); + }); + test.each<[string, string[], Record]>([ ["minimal — name only", ["--name", "x"], { model: defaultModel }], [ diff --git a/src/handlers/project/export/harness.test.ts b/src/handlers/project/export/harness.test.ts index 66fa2541d..b19cad331 100644 --- a/src/handlers/project/export/harness.test.ts +++ b/src/handlers/project/export/harness.test.ts @@ -3,7 +3,7 @@ import { existsSync } from "node:fs"; import { mkdir, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { parse, stringify } from "yaml"; -import { AgentCoreCLIError, DeserializationError } from "../../../errors"; +import { AgentCoreCLIError, DeserializationError, InputValidationError } from "../../../errors"; import { createRootHandler } from "../../index"; import { createSilentLogger, @@ -69,6 +69,35 @@ async function inProjectWithHarness( } describe("project export harness handler", () => { + test.each(["maxIteration", "model.maxToken"])( + "rejects %s with the source file and field, without export side effects", + async (field) => { + const subject = testExportCommand(); + const projectRoot = await inProjectWithHarness(subject); + const path = join(projectRoot, "app", "exportme", "harness.yaml"); + const config = parse(await Bun.file(path).text()); + if (field === "maxIteration") config.maxIteration = 3; + else config.model.maxToken = 512; + const yaml = "# Customer comment\n" + stringify(config); + await writeFile(path, yaml); + const specPath = join(projectRoot, "agentcore", "agentcore.json"); + const before = await Bun.file(specPath).text(); + const error = await subject + .run(["--name", "exportme", "--json"]) + .catch(AgentCoreCLIError.fromError); + expect(error).toBeInstanceOf(InputValidationError); + expect(error).toMatchObject({ source: "user", exitCode: 1, cause: expect.any(Error) }); + expect((error as Error).message).toContain(path); + for (const key of field.split(".")) expect((error as Error).message).toContain(key); + expect(existsSync(join(projectRoot, "app", "exportmeAgent"))).toBe(false); + expect(await Bun.file(specPath).text()).toBe(before); + expect(await Bun.file(path).text()).toBe(yaml); + expect(subject.core.projectCommands).not.toContainEqual( + expect.objectContaining({ command: ["uv", "sync"] }), + ); + }, + ); + test.each([ "malformed YAML", "missing main", diff --git a/src/projectSchemas/harness.test.ts b/src/projectSchemas/harness.test.ts index ab93a1ac8..75542e9ca 100644 --- a/src/projectSchemas/harness.test.ts +++ b/src/projectSchemas/harness.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it } from "bun:test"; +import { HarnessAuthoringSchema } from "./harness-authoring"; import { HarnessMemoryRefSchema, HarnessModelSchema, @@ -8,6 +9,7 @@ import { HarnessMemoryRetrievalConfigSchema, looksLikeLegacyPromptPath, validateApiFormat, + type HarnessSpec, } from "./harness"; const minimalHarness = { name: "harness", @@ -17,6 +19,85 @@ const networkConfig = { subnets: ["subnet-0123456789abcdef0"], securityGroups: ["sg-0123456789abcdef0"], }; +describe.each([ + ["normalized", HarnessSpecSchema], + ["authoring", HarnessAuthoringSchema], +] as const)("%s harness unknown fields", (_name, schema) => { + it("rejects root and model typos together instead of discarding them", () => { + const result = schema.safeParse({ + ...minimalHarness, + maxIteration: 3, + model: { ...minimalHarness.model, maxToken: 512 }, + }); + expect(result.success).toBe(false); + if (!result.success) { + expect(result.error.issues).toEqual( + expect.arrayContaining([ + expect.objectContaining({ code: "unrecognized_keys", path: [], keys: ["maxIteration"] }), + expect.objectContaining({ + code: "unrecognized_keys", + path: ["model"], + keys: ["maxToken"], + }), + ]), + ); + } + }); + + it.each([ + { + provider: "bedrock", + modelId: "example", + apiFormat: "converse_stream", + temperature: 0, + topP: 0, + maxTokens: 512, + }, + { provider: "open_ai", modelId: "example", apiKeyArn: "arn:key", apiFormat: "responses" }, + { provider: "gemini", modelId: "example", apiKeyArn: "arn:key", topK: 40 }, + { + provider: "lite_llm", + modelId: "example", + apiBase: "https://example.com", + additionalParams: { maxToken: 512, model: { maxIteration: 3 } }, + }, + ])("preserves valid provider fields and arbitrary map keys: %j", (model) => { + const map = { maxToken: "512", maxIteration: "3", model: "custom" }; + const input = { + ...minimalHarness, + model, + maxIterations: 3, + maxTokens: 1024, + timeoutSeconds: 60, + tags: map, + environmentVariables: map, + tools: [ + { + type: "remote_mcp", + name: "mcp", + config: { remoteMcp: { url: "https://example.com", headers: map } }, + }, + { + type: "inline_function", + name: "fn", + config: { + inlineFunction: { + description: "Custom schema", + inputSchema: { maxToken: 512, properties: { maxIteration: { type: "number" } } }, + }, + }, + }, + { + type: "agentcore_gateway", + name: "gateway", + config: { agentCoreGateway: { gatewayArn: "arn:gateway", maxIteration: 3 } }, + }, + ], + } satisfies Omit; + expect(schema.parse(input)).toEqual({ ...input, skills: [] }); + }); +}); + describe("harness custom validation", () => { it("binds model-only fields to their providers", () => { expect(HarnessModelSchema.safeParse({ provider: "open_ai", modelId: "gpt" }).success).toBe( diff --git a/src/projectSchemas/harness.ts b/src/projectSchemas/harness.ts index fd058fed3..17d07d9f6 100644 --- a/src/projectSchemas/harness.ts +++ b/src/projectSchemas/harness.ts @@ -51,6 +51,7 @@ export const HarnessModelSchema = z apiBase: z.string().min(1).max(MAX_LITE_LLM_API_BASE_LENGTH).optional(), additionalParams: z.record(z.string(), z.unknown()).optional(), }) + .strict() .superRefine((model, ctx) => { if (model.topK !== undefined && model.provider !== "gemini") { ctx.addIssue({ @@ -501,6 +502,7 @@ export const HarnessSpecSchema = z connections: z.array(ConnectionSchema).optional(), tags: TagsSchema.optional(), }) + .strict() .superRefine((data, ctx) => { if (data.containerUri !== undefined && data.dockerfile !== undefined) { ctx.addIssue({ From a8f81e8249827769640a005e2d4fac85713b62ec Mon Sep 17 00:00:00 2001 From: Aidan Daly Date: Tue, 15 Sep 2026 02:30:24 +0000 Subject: [PATCH 4/8] fix: remove legacy harness configuration checks --- README.md | 40 +++------------------ src/assets/cdk/io/harnessConfig.ts | 16 +-------- src/assets/cdk/lib/harness-schema.ts | 6 +++- src/assets/cdk/package.json | 3 +- src/assets/cdk/test/harness.test.ts | 18 ++++++++-- src/core/project/templates/harness.test.ts | 2 +- src/handlers/project/export/harness.test.ts | 20 +++++++++++ src/io/harnessConfig.test.ts | 13 +++---- src/io/harnessConfig.ts | 16 +-------- src/projectSchemas/harness-authoring.ts | 30 +++------------- src/projectSchemas/harness.test.ts | 22 ++++++------ src/projectSchemas/harness.ts | 9 ----- 12 files changed, 73 insertions(+), 122 deletions(-) diff --git a/README.md b/README.md index 546cc6d92..4f4271d7f 100644 --- a/README.md +++ b/README.md @@ -161,7 +161,8 @@ since the exported agent has no container filesystem to read them from. `project create` (without `--template`) and `project add harness` share the same scaffolding flow. Each harness has `app//harness.yaml` and -`app//system-prompt.md`. The YAML contains the supplied settings and +`app//system-prompt.md`. YAML is the harness configuration format. +The YAML contains the supplied settings and commented optional examples. Tools are opt-in. Newly scaffolded harnesses explicitly use `memory: { mode: managed }` unless another memory configuration was supplied. Reading an existing file with no `memory` setting still means @@ -203,6 +204,8 @@ omitted. A bad explicit reference never falls back silently. Referenced files must be readable, nonempty UTF-8 text of at most **1 MiB each**; whitespace-only files are rejected. Inline text and file contents preserve their whitespace. File contents are not interpreted as further references. +Plain strings such as `README.md`, `./instructions.md`, and HTTPS URLs are +literal prompt text, not file references. `project add harness --system-prompt file://./selected.md` preserves that reference in the generated YAML. Scaffolding validates reference syntax without @@ -221,41 +224,6 @@ Build, deploy, and export do not rewrite harness YAML or remove its comments. `agentcore.json`, deployment targets, JSON CLI flags/output, and service payloads are unchanged. -### Migrate Harness JSON - -On the `refactor` branch, `harness.yaml` replaces `harness.json`; the new reader -does not fall back to the old filename. Migrate both the harness file **and the -CDK app already copied into your project**: - -1. Commit or back up your project. Rename each `app//harness.json` to - `harness.yaml` and convert its object to YAML without changing its values. - JSON syntax is valid YAML, so the renamed file can retain its JSON syntax - while you reformat it. Keep string values quoted where needed, and preserve - omitted, disabled, or existing memory settings. -2. Keep an inline `systemPrompt`, or set `systemPrompt: file://./system-prompt.md` - to select the conventional prompt explicitly. If both previously existed, - choose the intended prompt: explicit text now wins consistently for local - export and CDK. Bare paths such as `./prompt.md` are not references. -3. Generate a separate reference project with the updated CLI: - `agentcore project create --name HarnessYamlReference --template empty --skip-install --skip-git`. - Compare its `agentcore/cdk/` with your project's copy. Port the YAML reader - (`io/harnessConfig.ts`), schema composition (`lib/harness-schema.ts`), and - harness-loading changes in `bin/cdk.ts`; add - the direct `yaml` dependency from `package.json` and the `io/**/*` include - from `tsconfig.json`. Preserve your custom CDK code, especially - `lib/cdk-stack.ts`. The reference app also includes a harness synthesis test. -4. Reinstall dependencies in your project's `agentcore/cdk/`, compile it, and - run `agentcore project build` from the project root before deploying. - This does not require a newer `@aws/agentcore-cdk` release. - -Upgrading the CLI does **not** overwrite a project's copied CDK app. -`project add harness` in an older project produces YAML but likewise does not -upgrade that app; complete step 3 before building or deploying the new harness. -An old app may still report a missing `harness.json` after the rename. That -means its reader needs migrating, not that the YAML should be renamed back. -The updated reader reports an obsolete-JSON error when only `harness.json` -exists. Neither path silently renames files or enables memory in existing ones. - Global flags (declared at the root, available on every command): | Flag | Purpose | diff --git a/src/assets/cdk/io/harnessConfig.ts b/src/assets/cdk/io/harnessConfig.ts index efbd93e95..954f3f9b8 100644 --- a/src/assets/cdk/io/harnessConfig.ts +++ b/src/assets/cdk/io/harnessConfig.ts @@ -14,21 +14,7 @@ export class HarnessConfigReader { async read(filePath: string): Promise { const configPath = resolve(filePath); try { - let raw: string; - try { - raw = await readFile(configPath, "utf8"); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ENOENT") { - const obsoletePath = join(dirname(configPath), "harness.json"); - if (await stat(obsoletePath).catch(() => undefined)) { - throw new Error( - `Obsolete harness.json at '${obsoletePath}'. Migrate it to harness.yaml and update ` + - "the copied agentcore/cdk app; see the README harness migration guide.", - ); - } - } - throw error; - } + const raw = await readFile(configPath, "utf8"); const document = parseDocument(raw); if (document.errors.length) throw document.errors[0]; const data: unknown = document.toJS(); diff --git a/src/assets/cdk/lib/harness-schema.ts b/src/assets/cdk/lib/harness-schema.ts index 0590f3483..dc1ba8520 100644 --- a/src/assets/cdk/lib/harness-schema.ts +++ b/src/assets/cdk/lib/harness-schema.ts @@ -1,6 +1,10 @@ import { HarnessSpecSchema as PublishedHarnessSpecSchema } from '@aws/agentcore-cdk'; +import { z } from 'zod'; -// Match CLI root/model strictness without dropping the published schema's refinements. +// Match CLI strictness and literal prompt policy while retaining the published object refinements. export const HarnessSpecSchema: typeof PublishedHarnessSpecSchema = PublishedHarnessSpecSchema.strict().safeExtend({ model: PublishedHarnessSpecSchema.shape.model.strict(), + systemPrompt: z.string() + .refine(val => val.trim().length > 0, { message: 'systemPrompt must not be empty or whitespace-only' }) + .optional(), }); diff --git a/src/assets/cdk/package.json b/src/assets/cdk/package.json index 9da1463ed..e9406f15b 100644 --- a/src/assets/cdk/package.json +++ b/src/assets/cdk/package.json @@ -26,6 +26,7 @@ "@aws/agentcore-cdk": "0.1.0-alpha.52", "aws-cdk-lib": "~2.266.0", "constructs": "~10.7.0", - "yaml": "^2.8.1" + "yaml": "^2.8.1", + "zod": "^4.4.3" } } diff --git a/src/assets/cdk/test/harness.test.ts b/src/assets/cdk/test/harness.test.ts index 6417ae0af..7f1e36e27 100644 --- a/src/assets/cdk/test/harness.test.ts +++ b/src/assets/cdk/test/harness.test.ts @@ -46,6 +46,8 @@ test.each([ { authorizerType: 'CUSTOM_JWT' }, { containerUri: '123456789012.dkr.ecr.us-east-1.amazonaws.com/repo:tag', dockerfile: 'Dockerfile' }, { tools: [{ type: 'agentcore_browser', name: 'same' }, { type: 'agentcore_browser', name: 'same' }] }, + { systemPrompt: '' }, + { systemPrompt: ' \r\n\t' }, ])('retains published refinements for %j', (overrides) => { expect(HarnessSpecSchema.safeParse({ name: 'assistant', @@ -54,7 +56,16 @@ test.each([ }).success).toBe(false); }); -test.each(['literal', 'file', 'fallback', 'maxIteration', 'model.maxToken'])('generated app validates %s configuration without rewriting it', (source) => { +test.each([ + ...['literal', 'file', 'fallback', 'maxIteration', 'model.maxToken'].map(source => ({ + source, prompt: ' Selected prompt: # 100%\n', + })), + ...['README.md\n', './instructions.md', 'https://example.com/prompt.md\n', 'file://./not-a-recursive-include.md'].map(prompt => ({ + source: 'file', prompt, + })), + { source: 'literal', prompt: './instructions.md' }, + { source: 'fallback', prompt: 'README.md\n' }, +])('generated app validates $source prompt $prompt without rewriting it', ({ source, prompt }) => { const root = mkdtempSync(join(tmpdir(), 'harness-yaml-synth-')); roots.push(root); const configRoot = join(root, 'agentcore'); @@ -69,7 +80,6 @@ test.each(['literal', 'file', 'fallback', 'maxIteration', 'model.maxToken'])('ge harnesses: [{ name: 'assistant', path: 'app/assistant' }], })); writeFileSync(join(configRoot, 'aws-targets.json'), '[]'); - const prompt = ' Selected prompt: # 100%\n'; const summary = 'Keep decisions and open questions.\n'; writeFileSync(join(harnessDir, 'chosen #100%.md'), prompt); writeFileSync(join(harnessDir, 'system-prompt.md'), source === 'fallback' ? prompt : 'Conventional prompt loses.'); @@ -113,6 +123,8 @@ test.each(['literal', 'file', 'fallback', 'maxIteration', 'model.maxToken'])('ge expect(harness.Properties.SystemPrompt).toEqual([{ Text: prompt }]); expect(harness.Properties.Memory).toEqual({ Disabled: {} }); expect(JSON.stringify(harness.Properties)).toContain(JSON.stringify(summary).slice(1, -1)); - expect(JSON.stringify(harness.Properties)).not.toContain('file://'); + expect(JSON.stringify(harness.Properties)).not.toContain('file://./chosen #100%.md'); + expect(JSON.stringify(harness.Properties)).not.toContain('file://./summary.md'); + expect(readFileSync(join(harnessDir, 'chosen #100%.md'), 'utf8')).toBe(prompt); expect(readFileSync(join(harnessDir, 'harness.yaml'), 'utf8')).toBe(yaml); }); diff --git a/src/core/project/templates/harness.test.ts b/src/core/project/templates/harness.test.ts index 8f93488f5..5305ab026 100644 --- a/src/core/project/templates/harness.test.ts +++ b/src/core/project/templates/harness.test.ts @@ -149,7 +149,7 @@ test("defaults only absent memory and does not mask an invalid supplied setting" await expect(scaffold({ memory: null })).rejects.toThrow(); }); -test.each(["file://", "", " \n", "./legacy.md"])( +test.each(["file://", "", " \n"])( "shared project scaffolding rejects invalid authoring prompt %j", async (systemPrompt) => { await expect(scaffold({ systemPrompt })).rejects.toThrow(); diff --git a/src/handlers/project/export/harness.test.ts b/src/handlers/project/export/harness.test.ts index b19cad331..edeec6394 100644 --- a/src/handlers/project/export/harness.test.ts +++ b/src/handlers/project/export/harness.test.ts @@ -69,6 +69,26 @@ async function inProjectWithHarness( } describe("project export harness handler", () => { + test.each([ + "README.md\n", + "./instructions.md", + "https://example.com/prompt.md\n", + "file://./not-a-recursive-include.md", + ])("exports prompt file contents %j as exact literal text", async (prompt) => { + const subject = testExportCommand(); + const projectRoot = await inProjectWithHarness(subject); + const directory = join(projectRoot, "app", "exportme"); + const path = join(directory, "harness.yaml"); + const yaml = await Bun.file(path).text(); + await writeFile(join(directory, "system-prompt.md"), prompt); + await subject.run(["--name", "exportme"]); + expect(await Bun.file(join(projectRoot, "app", "exportmeAgent", "main.py")).text()).toContain( + `DEFAULT_SYSTEM_PROMPT = """${prompt}"""`, + ); + expect(await Bun.file(path).text()).toBe(yaml); + expect(await Bun.file(join(directory, "system-prompt.md")).text()).toBe(prompt); + }); + test.each(["maxIteration", "model.maxToken"])( "rejects %s with the source file and field, without export side effects", async (field) => { diff --git a/src/io/harnessConfig.test.ts b/src/io/harnessConfig.test.ts index 38b066ac6..ba446e90c 100644 --- a/src/io/harnessConfig.test.ts +++ b/src/io/harnessConfig.test.ts @@ -272,15 +272,16 @@ for (const [label, Reader] of [ expect(data.other).toEqual({ summarizationSystemPrompt: "file://./prompt.md" }); }); - test("diagnoses obsolete JSON without reading or renaming it", async () => { + test.each([false, true])("reports missing YAML with nearby JSON=%s", async (nearbyJson) => { const { directory, path } = await fixture({}); await rm(path); const json = join(directory, "harness.json"); - await writeFile(json, "{}"); - await expect(new Reader().read(path)).rejects.toThrow( - /Obsolete harness.json.*Migrate.*copied agentcore\/cdk/s, - ); - expect(await readFile(json, "utf8")).toBe("{}"); + if (nearbyJson) await writeFile(json, "not valid JSON"); + const error = await new Reader().read(path).catch((error: Error) => error); + expect(error).toMatchObject({ cause: expect.objectContaining({ code: "ENOENT", path }) }); + expect((error as Error).message).toContain("harness.yaml"); + expect((error as Error).message).not.toContain("harness.json"); + if (nearbyJson) expect(await readFile(json, "utf8")).toBe("not valid JSON"); }); test("does not fall back after an explicit reference fails", async () => { diff --git a/src/io/harnessConfig.ts b/src/io/harnessConfig.ts index 68808f37e..35c04da82 100644 --- a/src/io/harnessConfig.ts +++ b/src/io/harnessConfig.ts @@ -15,21 +15,7 @@ export class HarnessConfigReader { async read(filePath: string): Promise { const configPath = resolve(filePath); try { - let raw: string; - try { - raw = await readFile(configPath, "utf8"); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ENOENT") { - const obsoletePath = join(dirname(configPath), "harness.json"); - if (await stat(obsoletePath).catch(() => undefined)) { - throw new Error( - `Obsolete harness.json at '${obsoletePath}'. Migrate it to harness.yaml and update ` + - "the copied agentcore/cdk app; see the README harness migration guide.", - ); - } - } - throw error; - } + const raw = await readFile(configPath, "utf8"); const document = parseDocument(raw); if (document.errors.length) throw document.errors[0]; const data: unknown = document.toJS(); diff --git a/src/projectSchemas/harness-authoring.ts b/src/projectSchemas/harness-authoring.ts index 5ef56fefb..c934ba1d2 100644 --- a/src/projectSchemas/harness-authoring.ts +++ b/src/projectSchemas/harness-authoring.ts @@ -1,27 +1,7 @@ -import { z } from "zod"; import { HarnessSpecSchema } from "./harness"; -/** Scaffold references name future YAML-relative files; only literal prompts enter the domain schema. */ -export const HarnessAuthoringSchema = z - .object({ systemPrompt: z.string().optional() }) - .passthrough() - .transform(({ systemPrompt, ...rest }, ctx) => { - const reference = systemPrompt?.startsWith("file://") ? systemPrompt : undefined; - if (reference === "file://") { - ctx.addIssue({ - code: "custom", - path: ["systemPrompt"], - message: "systemPrompt: file:// requires a path", - }); - return z.NEVER; - } - const parsed = HarnessSpecSchema.safeParse({ - ...rest, - systemPrompt: reference === undefined ? systemPrompt : undefined, - }); - if (!parsed.success) { - for (const issue of parsed.error.issues) ctx.addIssue({ ...issue }); - return z.NEVER; - } - return reference === undefined ? parsed.data : { ...parsed.data, systemPrompt: reference }; - }); +/** Scaffold references name future YAML-relative files; validate syntax without reading them. */ +export const HarnessAuthoringSchema = HarnessSpecSchema.refine( + ({ systemPrompt }) => systemPrompt !== "file://", + { path: ["systemPrompt"], message: "systemPrompt: file:// requires a path" }, +); diff --git a/src/projectSchemas/harness.test.ts b/src/projectSchemas/harness.test.ts index 75542e9ca..e71d89653 100644 --- a/src/projectSchemas/harness.test.ts +++ b/src/projectSchemas/harness.test.ts @@ -7,7 +7,6 @@ import { HarnessToolSchema, HarnessTruncationConfigSchema, HarnessMemoryRetrievalConfigSchema, - looksLikeLegacyPromptPath, validateApiFormat, type HarnessSpec, } from "./harness"; @@ -236,15 +235,18 @@ describe("harness custom validation", () => { }).success, ).toBe(false); }); - it("rejects legacy path-shaped and blank system prompts", () => { - expect(looksLikeLegacyPromptPath("./prompt.md")).toBe(true); - expect(looksLikeLegacyPromptPath("Use prompt.md when needed")).toBe(false); - expect( - HarnessSpecSchema.safeParse({ ...minimalHarness, systemPrompt: "./prompt.md" }).success, - ).toBe(false); - expect(HarnessSpecSchema.safeParse({ ...minimalHarness, systemPrompt: " " }).success).toBe( - false, - ); + it.each(["README.md\n", "./instructions.md", "../prompt.txt", "https://example.com/prompt.md"])( + "preserves %j as literal prompt text in normalized and authoring schemas", + (systemPrompt) => { + for (const schema of [HarnessSpecSchema, HarnessAuthoringSchema]) { + expect(schema.parse({ ...minimalHarness, systemPrompt }).systemPrompt).toBe(systemPrompt); + } + }, + ); + it.each(["", " \r\n\t"])("rejects blank system prompts: %j", (systemPrompt) => { + for (const schema of [HarnessSpecSchema, HarnessAuthoringSchema]) { + expect(schema.safeParse({ ...minimalHarness, systemPrompt }).success).toBe(false); + } }); it("rejects duplicate tools and excessive environment variables", () => { expect( diff --git a/src/projectSchemas/harness.ts b/src/projectSchemas/harness.ts index 17d07d9f6..2bc5071f6 100644 --- a/src/projectSchemas/harness.ts +++ b/src/projectSchemas/harness.ts @@ -436,11 +436,6 @@ export const AllowedToolSchema = z .min(1) .max(64) .regex(/^(\*|@?[^/]+(\/[^/]+)?)$/, 'Must be "*" or a tool name pattern (max 64 chars)'); -export function looksLikeLegacyPromptPath(value: string): boolean { - const v = value.trim(); - if (!/^\S+$/.test(v)) return false; - return /^\.\.?\//.test(v) || /\.(md|txt)$/i.test(v); -} export const HarnessSpecSchema = z .object({ name: HarnessNameSchema, @@ -450,10 +445,6 @@ export const HarnessSpecSchema = z .refine((val) => val.trim().length > 0, { message: "systemPrompt must not be empty or whitespace-only", }) - .refine((val) => !looksLikeLegacyPromptPath(val), { - message: - "systemPrompt looks like a file path. It is now always literal text — put file-backed prompts in a `system-prompt.md` in the harness directory (auto-discovered), or inline the prompt text here.", - }) .optional(), tools: z .array(HarnessToolSchema) From 846430eb4993a030d92da892fda65a25ac056d68 Mon Sep 17 00:00:00 2001 From: Aidan Daly Date: Tue, 15 Sep 2026 03:25:00 +0000 Subject: [PATCH 5/8] fix: preserve UTF-8 BOM in harness prompt files TextDecoder removes the leading BOM by default. Set ignoreBOM alongside fatal in both readers to preserve prompt contents without weakening UTF-8 validation. Cover main, summary, and conventional prompts plus BOM-only rejection through the shared reader suite. --- src/assets/cdk/io/harnessConfig.ts | 2 +- src/io/harnessConfig.test.ts | 35 +++++++++++++++++++++++++++++- src/io/harnessConfig.ts | 4 +++- 3 files changed, 38 insertions(+), 3 deletions(-) diff --git a/src/assets/cdk/io/harnessConfig.ts b/src/assets/cdk/io/harnessConfig.ts index 954f3f9b8..f33447274 100644 --- a/src/assets/cdk/io/harnessConfig.ts +++ b/src/assets/cdk/io/harnessConfig.ts @@ -76,7 +76,7 @@ export class HarnessConfigReader { if (length > MAX_PROMPT_FILE_SIZE) { throw new Error(`${field}: prompt file '${filePath}' exceeds the 1 MiB limit`); } - const text = new TextDecoder("utf-8", { fatal: true }).decode(bytes.subarray(0, length)); + const text = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(bytes.subarray(0, length)); if (!text.trim()) throw new Error(`${field}: prompt file '${filePath}' is empty or whitespace-only`); return text; } finally { diff --git a/src/io/harnessConfig.test.ts b/src/io/harnessConfig.test.ts index ba446e90c..dbbfafbdf 100644 --- a/src/io/harnessConfig.test.ts +++ b/src/io/harnessConfig.test.ts @@ -129,6 +129,36 @@ for (const [label, Reader] of [ }, ); + test.each(["system", "summary", "fallback"] as const)( + "preserves the UTF-8 BOM and CRLF in the %s prompt", + async (field) => { + const { directory, path } = await fixture( + field === "system" + ? { systemPrompt: "file://./prompt.md" } + : field === "summary" + ? { + truncation: { + strategy: "summarization", + config: { summarization: { summarizationSystemPrompt: "file://./prompt.md" } }, + }, + } + : {}, + ); + await writeFile( + join(directory, field === "fallback" ? "system-prompt.md" : "prompt.md"), + "\uFEFFHi\r\n", + ); + const data = HarnessSpecSchema.parse(await new Reader().read(path)); + expect( + field === "summary" + ? data.truncation?.config && + "summarization" in data.truncation.config && + data.truncation.config.summarization.summarizationSystemPrompt + : data.systemPrompt, + ).toBe("\uFEFFHi\r\n"); + }, + ); + test("preserves inline prompts, including whitespace and YAML-special characters", async () => { const literal = ' "hello": #yes\n[one, two] * & %\n'; const { directory, path } = await fixture({ @@ -202,6 +232,7 @@ for (const [label, Reader] of [ "missing", "empty", "whitespace", + "BOM-only", "directory", "oversized", "invalid UTF-8", @@ -218,7 +249,9 @@ for (const [label, Reader] of [ ? Buffer.from([0xff]) : condition === "whitespace" ? " \r\n\t" - : "", + : condition === "BOM-only" + ? "\uFEFF" + : "", ); for (const config of [ { systemPrompt: "file://./prompt.md" }, diff --git a/src/io/harnessConfig.ts b/src/io/harnessConfig.ts index 35c04da82..3573aea51 100644 --- a/src/io/harnessConfig.ts +++ b/src/io/harnessConfig.ts @@ -75,7 +75,9 @@ export class HarnessConfigReader { if (length > MAX_PROMPT_FILE_SIZE) { throw new Error(`${field}: prompt file '${filePath}' exceeds the 1 MiB limit`); } - const text = new TextDecoder("utf-8", { fatal: true }).decode(bytes.subarray(0, length)); + const text = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode( + bytes.subarray(0, length), + ); if (!text.trim()) throw new Error(`${field}: prompt file '${filePath}' is empty or whitespace-only`); return text; From 11514f29f2dda113df53aa71a92564c571333f37 Mon Sep 17 00:00:00 2001 From: Aidan Daly Date: Tue, 15 Sep 2026 14:25:49 +0000 Subject: [PATCH 6/8] docs: clarify harness prompt input options --- src/core/project/templates/harness.test.ts | 2 +- src/core/project/templates/harnessYaml.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/project/templates/harness.test.ts b/src/core/project/templates/harness.test.ts index 5305ab026..9ff5d401f 100644 --- a/src/core/project/templates/harness.test.ts +++ b/src/core/project/templates/harness.test.ts @@ -40,7 +40,7 @@ test("actual scaffold includes semantic comments and inactive examples, without memory: { mode: "managed" }, }); expect(yaml).not.toMatch(/^(tools|skills):/m); - expect(yaml).toContain("# Prompt file paths are relative to this YAML file."); + expect(yaml).toContain("# Inline prompt text or a file:// path relative to this YAML file."); expect(yaml).toContain( "# Output tokens per model call, rather than across the whole invocation.", ); diff --git a/src/core/project/templates/harnessYaml.ts b/src/core/project/templates/harnessYaml.ts index eaf5cf317..d4696aa8c 100644 --- a/src/core/project/templates/harnessYaml.ts +++ b/src/core/project/templates/harnessYaml.ts @@ -72,7 +72,7 @@ export class HarnessYamlRenderer { const sections = [ "# Optional settings are shown with example values.\n", stringify({ name: spec.name }), - this.comment("Prompt file paths are relative to this YAML file.") + + this.comment("Inline prompt text or a file:// path relative to this YAML file.") + stringify({ systemPrompt: spec.systemPrompt }), this.comment("Model") + stringify({ model: spec.model }) + From 1a452b7bed5fb09e386c161942e7c227e7279dcd Mon Sep 17 00:00:00 2001 From: Aidan Daly Date: Tue, 15 Sep 2026 14:30:12 +0000 Subject: [PATCH 7/8] test: avoid locking harness comment wording --- src/core/project/templates/harness.test.ts | 24 +++------------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/src/core/project/templates/harness.test.ts b/src/core/project/templates/harness.test.ts index 9ff5d401f..2eaaa628d 100644 --- a/src/core/project/templates/harness.test.ts +++ b/src/core/project/templates/harness.test.ts @@ -30,7 +30,7 @@ async function scaffold(overrides: Partial> = return { directory, path, yaml, data: parse(yaml) }; } -test("actual scaffold includes semantic comments and inactive examples, without default tools", async () => { +test("scaffolds valid YAML with inactive examples and a resolvable prompt", async () => { const { directory, path, yaml, data } = await scaffold(); expect((await readdir(directory)).sort()).toEqual(["harness.yaml", "system-prompt.md"]); expect(data).toEqual({ @@ -39,28 +39,10 @@ test("actual scaffold includes semantic comments and inactive examples, without systemPrompt: "file://./system-prompt.md", memory: { mode: "managed" }, }); - expect(yaml).not.toMatch(/^(tools|skills):/m); - expect(yaml).toContain("# Inline prompt text or a file:// path relative to this YAML file."); - expect(yaml).toContain( - "# Output tokens per model call, rather than across the whole invocation.", - ); - expect(yaml).toContain("# Execution limits apply per invocation, across all model calls."); - expect(yaml).toContain( - "# Skill path sources refer to files already present in the runtime container.", - ); - expect(yaml).toContain( - "# Truncation changes the context sent to the model, not the saved memory.", - ); - expect(yaml).toContain("# maxTokens: 20000"); + expect(yaml).toMatch(/^# maxTokens:/m); expect(yaml).toContain("agentcore_code_interpreter"); expect(yaml).toContain("remote_mcp"); - expect(yaml).toContain( - "https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/gateway.html", - ); - expect(yaml).toContain( - "https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/memory.html", - ); - expect(yaml).not.toMatch(/uncomment|replace \[\]|\bexa\b/i); + expect(yaml).toMatch(/^# https:\/\/docs\.aws\.amazon\.com\//m); expect(HarnessSpecSchema.parse(await new HarnessConfigReader().read(path)).systemPrompt).toBe( "You are a helpful assistant", ); From c38ab9e6baaacf8d172cafe40fd77829863388e9 Mon Sep 17 00:00:00 2001 From: Aidan Daly Date: Tue, 15 Sep 2026 18:54:13 +0000 Subject: [PATCH 8/8] test: remove strict harness validation test expansion --- src/assets/cdk/test/harness.test.ts | 50 +----------- src/core/project/templates/harness.test.ts | 18 ----- .../project/add/harness/index.test.ts | 24 ------ src/handlers/project/export/harness.test.ts | 31 +------ src/projectSchemas/harness.test.ts | 80 ------------------- 5 files changed, 4 insertions(+), 199 deletions(-) diff --git a/src/assets/cdk/test/harness.test.ts b/src/assets/cdk/test/harness.test.ts index 7f1e36e27..717bac345 100644 --- a/src/assets/cdk/test/harness.test.ts +++ b/src/assets/cdk/test/harness.test.ts @@ -1,4 +1,4 @@ -import { spawnSync } from 'node:child_process'; +import { execFileSync } from 'node:child_process'; import { mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join, resolve } from 'node:path'; @@ -13,39 +13,6 @@ afterEach(() => { }); test.each([ - { provider: 'bedrock', modelId: 'example', apiFormat: 'converse_stream', temperature: 0, topP: 0, maxTokens: 512 }, - { provider: 'open_ai', modelId: 'example', apiKeyArn: 'arn:key', apiFormat: 'responses' }, - { provider: 'gemini', modelId: 'example', apiKeyArn: 'arn:key', topK: 40 }, - { provider: 'lite_llm', modelId: 'example', apiBase: 'https://example.com', additionalParams: { maxToken: 512, model: { maxIteration: 3 } } }, -])('preserves provider fields and free-form maps: %j', (model) => { - const map = { maxToken: '512', maxIteration: '3', model: 'custom' }; - const input = { - name: 'assistant', - model, - maxIterations: 3, - maxTokens: 1024, - timeoutSeconds: 60, - tags: map, - environmentVariables: map, - tools: [ - { type: 'remote_mcp', name: 'mcp', config: { remoteMcp: { url: 'https://example.com', headers: map } } }, - { type: 'inline_function', name: 'fn', config: { inlineFunction: { description: 'Custom schema', inputSchema: { maxToken: 512, properties: { maxIteration: { type: 'number' } } } } } }, - { type: 'agentcore_gateway', name: 'gateway', config: { agentCoreGateway: { gatewayArn: 'arn:gateway', maxIteration: 3 } } }, - ], - }; - expect(HarnessSpecSchema.parse(input)).toEqual({ ...input, skills: [] }); -}); - -test.each([ - { model: { provider: 'bedrock', modelId: 'example', topK: 40 } }, - { model: { provider: 'open_ai', modelId: 'example' } }, - { model: { provider: 'open_ai', modelId: 'example', apiKeyArn: 'arn:key', apiFormat: 'converse_stream' } }, - { model: { provider: 'gemini', modelId: 'example', apiKeyArn: 'arn:key', apiBase: 'https://example.com' } }, - { model: { provider: 'bedrock', modelId: 'example', additionalParams: { maxToken: 512 } } }, - { networkMode: 'VPC' }, - { authorizerType: 'CUSTOM_JWT' }, - { containerUri: '123456789012.dkr.ecr.us-east-1.amazonaws.com/repo:tag', dockerfile: 'Dockerfile' }, - { tools: [{ type: 'agentcore_browser', name: 'same' }, { type: 'agentcore_browser', name: 'same' }] }, { systemPrompt: '' }, { systemPrompt: ' \r\n\t' }, ])('retains published refinements for %j', (overrides) => { @@ -57,7 +24,7 @@ test.each([ }); test.each([ - ...['literal', 'file', 'fallback', 'maxIteration', 'model.maxToken'].map(source => ({ + ...['literal', 'file', 'fallback'].map(source => ({ source, prompt: ' Selected prompt: # 100%\n', })), ...['README.md\n', './instructions.md', 'https://example.com/prompt.md\n', 'file://./not-a-recursive-include.md'].map(prompt => ({ @@ -89,9 +56,7 @@ test.each([ model: { provider: 'bedrock', modelId: 'global.anthropic.claude-sonnet-4-6', - ...(source === 'model.maxToken' ? { maxToken: 512 } : {}), }, - ...(source === 'maxIteration' ? { maxIteration: 3 } : {}), systemPrompt: source === 'fallback' ? undefined : source === 'literal' ? prompt : 'file://./chosen #100%.md', memory: { mode: 'disabled' }, truncation: { @@ -101,22 +66,13 @@ test.each([ }); writeFileSync(join(harnessDir, 'harness.yaml'), yaml); const outdir = join(root, 'cdk.out'); - const result = spawnSync(process.execPath, [entrypoint], { + execFileSync(process.execPath, [entrypoint], { cwd: cdkRoot, env: { ...process.env, INIT_CWD: root, CDK_OUTDIR: outdir }, stdio: 'pipe', - encoding: 'utf8', timeout: 30000, }); - expect(result.error).toBeUndefined(); expect(readFileSync(join(harnessDir, 'harness.yaml'), 'utf8')).toBe(yaml); - if (source === 'maxIteration' || source === 'model.maxToken') { - expect(result.status).toBe(1); - expect(result.stderr).toContain(join(harnessDir, 'harness.yaml')); - for (const key of source.split('.')) expect(result.stderr).toContain(key); - return; - } - expect(result.status).toBe(0); const templateFile = readdirSync(outdir).find((name) => name.endsWith('.template.json'))!; const template = JSON.parse(readFileSync(join(outdir, templateFile), 'utf8')); const harness = Object.values(template.Resources).find((resource: any) => resource.Type === 'AWS::BedrockAgentCore::Harness') as any; diff --git a/src/core/project/templates/harness.test.ts b/src/core/project/templates/harness.test.ts index 2eaaa628d..341c82ba5 100644 --- a/src/core/project/templates/harness.test.ts +++ b/src/core/project/templates/harness.test.ts @@ -4,7 +4,6 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { parse } from "yaml"; import type z from "zod"; -import { InputValidationError } from "../../../errors"; import { HarnessSpecSchema } from "../../../projectSchemas/harness"; import { HarnessConfigReader } from "../../../io/harnessConfig"; import { getHarnessTemplateResolver } from "./harness"; @@ -137,20 +136,3 @@ test.each(["file://", "", " \n"])( await expect(scaffold({ systemPrompt })).rejects.toThrow(); }, ); - -test.each([{ maxIteration: 3 }, { model: { ...model, maxToken: 512 } }])( - "shared scaffolding rejects unknown fixed fields: %j", - async (overrides) => { - const error = await getHarnessTemplateResolver() - .resolve({ - name: "assistant", - model, - systemPrompt: "file://./selected.md", - ...overrides, - }) - .catch((error: unknown) => error); - expect(error).toBeInstanceOf(InputValidationError); - expect(error).toMatchObject({ source: "user" }); - expect((error as Error).message).toContain("Unrecognized key"); - }, -); diff --git a/src/handlers/project/add/harness/index.test.ts b/src/handlers/project/add/harness/index.test.ts index a4feb1db6..16945ad37 100644 --- a/src/handlers/project/add/harness/index.test.ts +++ b/src/handlers/project/add/harness/index.test.ts @@ -33,30 +33,6 @@ async function run(args: string[], opts?: { core?: TestCoreClient }) { describe("project add harness", () => { const defaultModel = { provider: "bedrock", modelId: "global.anthropic.claude-sonnet-4-6" }; - test("rejects a model typo as user input without scaffolding or registering a harness", async () => { - const { projectRoot, cleanup } = await initProject(); - cleanups.push(cleanup); - const specPath = join(projectRoot, "agentcore", "agentcore.json"); - const before = await Bun.file(specPath).text(); - const error = await run([ - "add", - "harness", - "--name", - "typo", - "--model", - JSON.stringify({ ...defaultModel, maxToken: 512 }), - "--system-prompt", - "file://./selected.md", - "--json", - ]).catch((error: unknown) => error); - expect(error).toBeInstanceOf(InputValidationError); - expect(error).toMatchObject({ source: "user", exitCode: 1 }); - expect((error as Error).message).toContain("model"); - expect((error as Error).message).toContain("maxToken"); - expect(existsSync(join(projectRoot, "app", "typo"))).toBe(false); - expect(await Bun.file(specPath).text()).toBe(before); - }); - test.each<[string, string[], Record]>([ ["minimal — name only", ["--name", "x"], { model: defaultModel }], [ diff --git a/src/handlers/project/export/harness.test.ts b/src/handlers/project/export/harness.test.ts index edeec6394..1f65a8dfc 100644 --- a/src/handlers/project/export/harness.test.ts +++ b/src/handlers/project/export/harness.test.ts @@ -3,7 +3,7 @@ import { existsSync } from "node:fs"; import { mkdir, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { parse, stringify } from "yaml"; -import { AgentCoreCLIError, DeserializationError, InputValidationError } from "../../../errors"; +import { AgentCoreCLIError, DeserializationError } from "../../../errors"; import { createRootHandler } from "../../index"; import { createSilentLogger, @@ -89,35 +89,6 @@ describe("project export harness handler", () => { expect(await Bun.file(join(directory, "system-prompt.md")).text()).toBe(prompt); }); - test.each(["maxIteration", "model.maxToken"])( - "rejects %s with the source file and field, without export side effects", - async (field) => { - const subject = testExportCommand(); - const projectRoot = await inProjectWithHarness(subject); - const path = join(projectRoot, "app", "exportme", "harness.yaml"); - const config = parse(await Bun.file(path).text()); - if (field === "maxIteration") config.maxIteration = 3; - else config.model.maxToken = 512; - const yaml = "# Customer comment\n" + stringify(config); - await writeFile(path, yaml); - const specPath = join(projectRoot, "agentcore", "agentcore.json"); - const before = await Bun.file(specPath).text(); - const error = await subject - .run(["--name", "exportme", "--json"]) - .catch(AgentCoreCLIError.fromError); - expect(error).toBeInstanceOf(InputValidationError); - expect(error).toMatchObject({ source: "user", exitCode: 1, cause: expect.any(Error) }); - expect((error as Error).message).toContain(path); - for (const key of field.split(".")) expect((error as Error).message).toContain(key); - expect(existsSync(join(projectRoot, "app", "exportmeAgent"))).toBe(false); - expect(await Bun.file(specPath).text()).toBe(before); - expect(await Bun.file(path).text()).toBe(yaml); - expect(subject.core.projectCommands).not.toContainEqual( - expect.objectContaining({ command: ["uv", "sync"] }), - ); - }, - ); - test.each([ "malformed YAML", "missing main", diff --git a/src/projectSchemas/harness.test.ts b/src/projectSchemas/harness.test.ts index e71d89653..d8610bedc 100644 --- a/src/projectSchemas/harness.test.ts +++ b/src/projectSchemas/harness.test.ts @@ -8,7 +8,6 @@ import { HarnessTruncationConfigSchema, HarnessMemoryRetrievalConfigSchema, validateApiFormat, - type HarnessSpec, } from "./harness"; const minimalHarness = { name: "harness", @@ -18,85 +17,6 @@ const networkConfig = { subnets: ["subnet-0123456789abcdef0"], securityGroups: ["sg-0123456789abcdef0"], }; -describe.each([ - ["normalized", HarnessSpecSchema], - ["authoring", HarnessAuthoringSchema], -] as const)("%s harness unknown fields", (_name, schema) => { - it("rejects root and model typos together instead of discarding them", () => { - const result = schema.safeParse({ - ...minimalHarness, - maxIteration: 3, - model: { ...minimalHarness.model, maxToken: 512 }, - }); - expect(result.success).toBe(false); - if (!result.success) { - expect(result.error.issues).toEqual( - expect.arrayContaining([ - expect.objectContaining({ code: "unrecognized_keys", path: [], keys: ["maxIteration"] }), - expect.objectContaining({ - code: "unrecognized_keys", - path: ["model"], - keys: ["maxToken"], - }), - ]), - ); - } - }); - - it.each([ - { - provider: "bedrock", - modelId: "example", - apiFormat: "converse_stream", - temperature: 0, - topP: 0, - maxTokens: 512, - }, - { provider: "open_ai", modelId: "example", apiKeyArn: "arn:key", apiFormat: "responses" }, - { provider: "gemini", modelId: "example", apiKeyArn: "arn:key", topK: 40 }, - { - provider: "lite_llm", - modelId: "example", - apiBase: "https://example.com", - additionalParams: { maxToken: 512, model: { maxIteration: 3 } }, - }, - ])("preserves valid provider fields and arbitrary map keys: %j", (model) => { - const map = { maxToken: "512", maxIteration: "3", model: "custom" }; - const input = { - ...minimalHarness, - model, - maxIterations: 3, - maxTokens: 1024, - timeoutSeconds: 60, - tags: map, - environmentVariables: map, - tools: [ - { - type: "remote_mcp", - name: "mcp", - config: { remoteMcp: { url: "https://example.com", headers: map } }, - }, - { - type: "inline_function", - name: "fn", - config: { - inlineFunction: { - description: "Custom schema", - inputSchema: { maxToken: 512, properties: { maxIteration: { type: "number" } } }, - }, - }, - }, - { - type: "agentcore_gateway", - name: "gateway", - config: { agentCoreGateway: { gatewayArn: "arn:gateway", maxIteration: 3 } }, - }, - ], - } satisfies Omit; - expect(schema.parse(input)).toEqual({ ...input, skills: [] }); - }); -}); - describe("harness custom validation", () => { it("binds model-only fields to their providers", () => { expect(HarnessModelSchema.safeParse({ provider: "open_ai", modelId: "gpt" }).success).toBe(