diff --git a/.gitignore b/.gitignore index 78e296d..2b4cb91 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ node_modules/ dist/ site/dist/ .morphogen/ +promoted/ .env .env.* !.env.example diff --git a/AGENTS.md b/AGENTS.md index 2d04ce6..7887a91 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,14 +2,18 @@ - `src/` — the contract (`contract.ts`, `graph.ts`), the scheduler (`run.ts`), the effect seam (`effects.ts`), the store (`store.ts`), verification - (`verify.ts`), bundles (`bundle.ts`), transports (`transport.ts`), - canonical values and digests, and colocated tests. + (`verify.ts`), Vercel AI Gateway execution (`gateway.ts`), typed external + tools (`tools.ts`), foundry evaluation and search (`foundry.ts`, + `search.ts`), benchmark comparison (`bench.ts`, `bench-verify.ts`), + bundles (`bundle.ts`), transports (`transport.ts`), canonical values and + digests, and colocated tests. - `cli.ts` — the Bun CLI (`run`, `check`, `verify`, `inspect`, `explain`, - `diff`, `runs`, `digest`, `store`, `manifests`, `manifest`, `slots`, - `slot`, `pack`, `unpack`, `example`, `suite`). + `diff`, `foundry`, `bench`, `runs`, `digest`, `store`, `manifests`, + `manifest`, `slots`, `slot`, `pack`, `unpack`, `example`, `suite`). - `index.ts` — the package's public surface. - `examples/` — bundled manifests and scripted responses used by `suite`. -- `spec/v1/organism.md` — the authoritative contract prose. +- `spec/v1/organism.md`, `spec/v1/foundry.md`, `spec/v1/search.md`, + `spec/v1/bench.md` — authoritative contract prose. - `site/` — the static morphogen.dev source; `build.ts` writes `site/dist`. - `README.md`, `CONTRIBUTING.md`, `SECURITY.md` — the public contract. diff --git a/README.md b/README.md index 3d72f0e..ebf6dd6 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,27 @@ Status: early. The v1 contract, scheduler, effect seam, nested organisms, and offline verification are implemented and tested. Hosted habitats, multi-owner messaging, and workflow breeding are deliberately deferred. +## Why this is a new primitive + +Morphogen is a third thing between deterministic programs and open-ended agents: +a bounded, typed, content-addressed probabilistic program. The manifest is a +value; the receipt is evidence; and model judgment is isolated behind explicit +cells with declared contracts and budgets. See [`docs/why-unique.md`](docs/why-unique.md) +for the full comparison with prompts, agent loops, DAG engines, probabilistic +programming, smart contracts, and FaaS. + +### Where this could go + +Because manifests are values and receipts are evidence, organisms can generate, +store, and propose new organisms. A shared `Store`, `ToolRegistry`, and +`FnRegistry` becomes a habitat: a population of organisms that evolve through +foundry search and host admission. The organism cannot rewrite its own runtime, +but it can *propose* children, functions, and tools; the host decides what to +admit. See [`docs/habitats.md`](docs/habitats.md) for the design sketch, +[`examples/habitat.morphogen.json`](examples/habitat.morphogen.json) for a +deterministic working steel thread, and `bun examples/habitat/promote.ts --live` +for a live model-driven reproduction loop. + ## What is this? An **organism** is a manifest (`morphogen.organism.v1`): a set of cells with @@ -21,6 +42,11 @@ Cell kinds: - `const` — a literal producer. Ports are declared values. - `fn` — a pure function from the host's registry (`echo.v1`, `tag.v1`, `coalesce.v1`, `pick.v1`, `format.v1` ship built in). +- `tool` — a typed external effect resolved only from the host's tool registry. + Read/write class, inputs, outputs, work cost, output bytes, timeout, and + idempotency key are explicit; results and failures are receipted and replayed + without repeating live IO. Agent cells may request the same admitted tools + during bounded turns alongside pure function callbacks. - `agent` — a bounded model call: a declared context view, a prompt, a typed output contract, an optional route, declared tool callbacks, and byte, turn, and wall-clock (`budget.maxEffectMs`) budgets — a hung executor becomes a @@ -94,6 +120,48 @@ and leaves to the model only what is declared inside a cell boundary. A run is then something you can replay, diff, and audit rather than a transcript you have to trust. +## Where it wins + +Morphogen wins where the work is **structured, verifiable, and cheaper to split +into many small decisions** than to pack into one long prompt. The fastest wins +are workloads where a single LLM call is missing information or has no way to +check itself: + +- **Tool-grounded investigation** — a model call cannot look up a customer + record, run a calculation, or inspect a ledger; Morphogen routes a typed + `tool` cell before the judgment, then checks the result deterministically. +- **Multi-decision classifiers over one shared context** — dozens of narrow + `classifier` cells see only the slices they need, each with a tiny prompt, + instead of one monolithic completion. +- **Escalation by disagreement** — two cheap lanes plus an `assert.v1` guard + escalate only when the cheap models disagree; frontier inference is sparse, + not the default. +- **Verification before promotion** — a generated organism must pass train, + validation, and holdout cases, and `morphogen verify` replays every receipt + bit-for-bit before the organism is promoted. + +### Case study: billing-dispute investigation + +`examples/invest/` runs six support tickets where the correct decision depends +on a charge ledger. A lone model sees only the ticket; the Morphogen organism +retrieves the ledger through a typed `tool` cell and then classifies. + +Live Vercel AI Gateway run: + +|| system | passed | effect calls | cost | input tokens | output tokens | Pareto | +|---|---|---:|---:|---:|---:|---| +|| cheap-single (qwen3.5, no evidence) | 4/6 | 6 | $0.00371 | 759 | 14,087 | yes | +|| frontier-single (claude-opus-5, no evidence) | 4/6 | 6 | $0.02625 | 4,214 | 207 | — | +|| **organism-cheap (qwen3.5 + ledger tool)** | **6/6** | **12** | **$0.00131** | **1,210** | **4,716** | **yes** | +|| organism-ensemble (qwen3.5 + qwen3.7 + tool) | 6/6 | 19 | $0.00676 | 3,692 | 6,776 | — | + +A Qwen Flash organism with a typed ledger lookup is **100% accurate on this +workload**, while a Claude Opus call without the tool is **67% accurate**. +Opus fails the same evidence-only cases as Qwen does when neither can look up +the charges. The Pareto set keeps both the organism (quality winner) and the +frontier single call (fewest round-trips), so the tradeoff is explicit and +can be chosen per deployment. + ## First value ```sh @@ -158,6 +226,111 @@ bun run cli pack examples/inbox.morphogen.json --modules examples > bundle.json bun run cli unpack bundle.json --dir /tmp/elsewhere # installs, digests verified ``` +## Foundry: select organisms by evidence + +A foundry evaluates a bounded population against explicit train and validation +cases, promotes one manifest digest, and only then runs that winner on the +holdout split. A case passes only when the organism completes and its interface +outputs canonically equal the expected record. Every candidate manifest and run +receipt is persisted. + +Candidates may be named files or manifests emitted as data by a generator +organism. The generator runs under the same executor, registry, store, and +budgets as any other organism; its digest and receipt become the population's +lineage. A generator may itself use `each`, `repeat`, `spawn`, slots, and gates, +so bounded populations, iterative search, durable journals, and approval are +composition rather than privileged foundry code. + +```sh +bun run cli foundry examples/generated-foundry.config.json \ + --responses examples/foundry-generator.responses.json \ + --dir .morphogen --out foundry-report.json +bun run cli foundry inspect foundry-report.json +bun run cli foundry verify foundry-report.json --dir .morphogen +bun run cli foundry pack foundry-report.json --dir .morphogen --out bundles +``` + +A `morphogen.foundry.config.v1` file declares the generator, cases, and optionally +additional candidate paths: + +```json +{ + "contract": "morphogen.foundry.config.v1", + "generator": { + "manifest": "generator.morphogen.json", + "args": { "task": "Return the input unchanged." }, + "output": "candidates", + "field": "candidates" + }, + "cases": [ + { "id": "train-a", "split": "train", "args": { "q": "a" }, "expect": { "answer": "a" } }, + { "id": "validation-b", "split": "validation", "args": { "q": "b" }, "expect": { "answer": "b" } }, + { "id": "holdout-c", "split": "holdout", "args": { "q": "c" }, "expect": { "answer": "c" } } + ] +} +``` + +Paths resolve relative to the config. Promotion prefers validation pass rate, +then train pass rate, then fewer agent calls and work units, with manifest digest +as the final tie-breaker. Non-promoted candidates never run against holdout +cases. A `morphogen.foundry.v1` report records expectations, outputs, work, token +usage, manifest and receipt digests, generator lineage, and the winner's holdout result. +`foundry verify` checks the report digest, scores, selection, claimed outputs, +and every run receipt by offline replay. `foundry pack` verifies that evidence +before exporting the promoted organism's content-addressed closure. + +A bounded search repeats generation and selection while keeping holdout sealed. +The previous winner survives into the next population, and the generator sees +only prior train/validation scores, work, and manifest digests: + +```sh +bun run cli foundry search examples/search.config.json \ + --responses examples/evolving-generator.responses.json \ + --dir .morphogen --out search-report.json +bun run cli foundry search-inspect search-report.json +bun run cli foundry search-verify search-report.json --dir .morphogen +bun run cli foundry search-pack search-report.json --dir .morphogen --out bundles +``` + +`morphogen.search.v1` bounds a search to eight generations. Every generation +records its generator receipt, proposals, full population evidence, and winner. +Verification replays the complete history, checks survivor continuity and that +every proposal was evaluated, and rejects any holdout evidence in generation +records. Baseline manifests may enter through the config's `candidates` list and +compete with generated organisms from generation zero onward. + +## Bench: compare systems on one workload + +A bench measures several systems — each an organism plus a host-resolved +executor list — against the same cases. "One cheap call", "one frontier call", +and "a decomposed organism whose frontier call is a guarded escalation branch" +are the same kind of contender. A case passes only when the run completes and +its declared outputs canonically equal `expect`; every case's receipt is +persisted and replayable. + +```sh +bun run cli bench examples/bench.config.json --dir .morphogen --out bench-report.json +bun run cli bench inspect bench-report.json +bun run cli bench verify bench-report.json --dir .morphogen +``` + +A `morphogen.bench.config.v1` file names case `args`/`expect` pairs and systems +whose `executors` map names to `gateway:` (Vercel AI Gateway), +`scripted:`, or `cmd:` specs; the first entry is the default and +named entries answer `route.preset`. The report records per-case results, work, +token usage, per-model effect attribution, and the non-dominated pareto set on +(quality ↑, tokens ↓, effect calls ↓). `examples/bench.config.json` runs it +deterministically; `examples/bench-live.config.json` swaps the scripted lanes +for `alibaba/qwen3.5-flash` and `anthropic/claude-opus-5` through the gateway. + +For tool-grounded baselines, pass `--tools `: a registry of named tools +with typed signatures and `scripted:` or `cmd:` executors. The +billing-dispute case in `examples/invest/bench-invest-live.config.json` uses it +to compare a Qwen organism with a charge-ledger lookup against a Claude Opus +call that can only read the ticket. Add a `prices` map to the bench config +(`examples/invest/bench-invest-priced.config.json`) to put the Pareto in +aicharts.io-denominated dollars. + `check` admits a manifest without running it: parse, graph validation, and interface resolution only. `explain` prints the compiled signature — every cell's resolved input/output ports (including ports inherited from embedded @@ -166,10 +339,12 @@ embed others resolve sub-manifests by digest from the store; `--modules ` loads a directory of `*.morphogen.json` files first. To go live, point `--executor-cmd` at any program that reads an effect request -(JSON) on stdin and prints the model's output on stdout. Morphogen does not -broker provider access; the executor seam is where provider auth lives. -`--executors ` takes a JSON map of name → command, so a cell's -`route.provider`/`route.preset` picks its model. +(JSON) on stdin and prints the model's output on stdout, or use +`--gateway-model ` for the built-in Vercel AI Gateway executor +(short-lived OIDC or a scoped gateway key from the environment — never the +manifest). Morphogen does not broker provider access; the executor seam is +where provider auth lives. `--executors ` takes a JSON map of +name → command, so a cell's `route.provider`/`route.preset` picks its model. ## How does it behave? @@ -226,6 +401,110 @@ broker provider access; the executor seam is where provider auth lives. - This is not a hosted orchestrator, a durable job queue, or a multi-agent town. Those are later layers; the contract is designed not to need them yet. +## Plug it into your agent or provider + +Morphogen is a library and a CLI; the seams are deliberately narrow so you can +use it from a larger system without giving the system ambient authority. + +### From code + +```ts +import { builtinRegistry, runOrganism, vercelGatewayExecutor } from "morphogen"; +import { FileStore } from "morphogen/store"; // or a custom Store + +const receipt = await runOrganism({ + manifest: myManifest, + args: { src: { ticket: "I was charged twice…" } }, + fns: builtinRegistry(), + store: new FileStore(".morphogen"), + executors: [vercelGatewayExecutor({ model: "alibaba/qwen3.5-flash" })], + tools: myToolRegistry, // typed external effects +}); +``` + +The `Executor` interface is one method: `execute(effect, signal?)` returns the +raw effect output. Any provider, local model, or hard-coded fixture fits by +wrapping that method. `runOrganism` does the scheduling, binding, budget +enforcement, and receipt writing. + +### From the CLI with any provider + +```sh +# scripted replay fixture +bun run cli run ticket.morphogen.json --responses ticket.responses.json + +# Vercel AI Gateway +bun run cli run ticket.morphogen.json \ + --gateway-model alibaba/qwen3.5-flash --write + +# any command that reads JSON on stdin and writes JSON on stdout +bun run cli run ticket.morphogen.json \ + --executor-cmd "python -m my_provider_agent" +``` + +### External tools + +Agent cells can request functions from the host registry (`tools: ["pick.v1"]`), +and explicit `tool` cells can call external services. For the CLI, declare the +registry in a `--tools `: + +```json +{ + "ledger.charges.v1": { + "signature": { + "inputs": { "account": "text" }, + "outputs": { "charges": "json" }, + "effect": "read", + "cost": 50, + "maxOutputBytes": 8192 + }, + "exec": "cmd:ledger-cli" + } +} +``` + +The command receives `{ inputs, requestDigest, idempotencyKey }` on stdin and +must print a JSON object of output ports. For deterministic testing, use +`"exec": "scripted:"`. + +### As an agent tool + +Pack an organism and register it as an OpenAI or Anthropic function tool: + +```sh +morphogen pack ticket.morphogen.json --out ./tools +morphogen tool-def ticket.morphogen.json > ticket-tool.json +``` + +Then call it from an agent: + +```sh +morphogen call ./tools/.bundle.json \ + --args ticket.args.json \ + --gateway-model alibaba/qwen3.5-flash +``` + +The result is compact enough for an agent to consume: + +```json +{ + "ok": true, + "outputs": { "out": "billing" }, + "receiptDigest": "sha256:...", + "manifestDigest": "sha256:..." +} +``` + +The agent receives the output and a receipt digest it can verify later. See +`docs/agent-tool.md` for a complete example. + +### Verification and transport + +Receipts are content-addressed canonical JSON; `morphogen verify` replays them +offline with the recorded effects fixed. `morphogen pack` exports a manifest +closure — sub-manifests, `const` refs, and linked bundles — so one digest fully +describes a deployable program. + ## How claims are checked `bun run check` runs the typechecker, the linter, the test suite, and the site @@ -238,6 +517,9 @@ detection. ## Deeper documentation - `spec/v1/organism.md` — the manifest, run, and receipt contract. +- `spec/v1/foundry.md` — candidate generation, evidence, promotion, and verification. +- `spec/v1/search.md` — bounded generations, feedback, survivors, and lineage. +- `spec/v1/bench.md` — workload comparison, attribution, and the pareto claim. - `docs/` — design notes as they land. ## Related work diff --git a/cli.ts b/cli.ts index 505f185..9810b51 100644 --- a/cli.ts +++ b/cli.ts @@ -16,6 +16,7 @@ import { type Executor, } from "./src/effects"; import { errorReport, MorphogenError } from "./src/errors"; +import { vercelGatewayExecutor } from "./src/gateway"; import { builtinRegistry } from "./src/registry"; import { parseRunReceipt, runOrganism, type RunReceipt } from "./src/run"; import { packOrganism, parseBundle, unpackBundle } from "./src/bundle"; @@ -27,6 +28,22 @@ import { type Transport, } from "./src/transport"; import { diffReceipts, verifyReceipt } from "./src/verify"; +import { + parseToolSignature, + TOOL_SIGNATURE_BOUNDS, + type Tool, + type ToolRegistry, +} from "./src/tools"; +import { + generateFoundryCandidates, + runFoundry, + type FoundryCase, +} from "./src/foundry"; +import { parseFoundryReport, verifyFoundryReport } from "./src/foundry-verify"; +import { runFoundrySearch } from "./src/search"; +import { parseSearchReport, verifySearchReport } from "./src/search-verify"; +import { runBenchmark, type BenchCase, type BenchPrice, type BenchSystem } from "./src/bench"; +import { parseBenchReport, verifyBenchReport } from "./src/bench-verify"; import { canonicalBytes, canonicalize, @@ -46,11 +63,14 @@ usage: --args input-cell values (JSON) --responses scripted agent outputs (JSON map) --executor-cmd live executor: request on stdin, output on stdout + --gateway-model Vercel AI Gateway structured-output executor --executors JSON map of executor name → shell command; route.provider/route.preset pick by name --modules load *.morphogen.json into the store for organism cells --transports JSON map of transport name → bundle directory; via cells resolve remote manifests through it + --tools tool registry: name → {signature, exec}; + exec is scripted: or cmd: --dir store directory (default .morphogen) --write persist manifest + receipt under --dir --cache-effects memoize effects: identical request digests @@ -66,6 +86,29 @@ usage: morphogen runs [--dir ] list receipts stored under --dir morphogen diff compare two receipts, report divergence + morphogen foundry [--responses ] [--executor-cmd ] + [--gateway-model ] + [--executors ] [--modules ] [--transports ] [--tools ] + [--cache-effects] [--dir ] [--out ] + generate/evaluate candidates and promote a winner + morphogen foundry verify [--dir ] + replay every run in a foundry report offline + morphogen foundry inspect summarize scores, lineage, and promotion + morphogen foundry pack --out [--dir ] + export the promoted organism's verified bundle + morphogen foundry search [executor/store options] [--out ] + evolve candidates over bounded generations + morphogen foundry search-verify [--dir ] + morphogen foundry search-inspect + morphogen foundry search-pack --out [--dir ] + inspect or export a verified search winner + morphogen bench [--modules ] [--tools ] [--dir ] [--out ] + measure several systems on one workload: + quality, tokens, work, per-model attribution, + and the non-dominated pareto set (with optional prices) + morphogen bench verify [--dir ] + replay every case receipt in a bench report + morphogen bench inspect summarize a pareto comparison morphogen suite run and verify all bundled examples morphogen digest print the manifest's canonical digest morphogen store put [--dir ] @@ -87,6 +130,16 @@ usage: --out also writes .bundle.json morphogen unpack [--dir ] install a bundle into the store, digests verified + morphogen call [options] + run a packed organism and print a compact result: + { ok, outputs, receiptDigest, manifestDigest }. + options mirror morphogen run: --args, --responses, + --executor-cmd, --gateway-model, --executors, + --modules, --tools, --cache-effects, --dir + morphogen tool-def [--modules ] + print an OpenAI/Anthropic tool definition for the + organism's interface: a name, description, and a + JSON Schema of the arguments it expects morphogen --version | --help `; @@ -129,6 +182,21 @@ async function readJson(path: string): Promise { } } +async function readJsonStdin(): Promise { + const text = await Bun.stdin.text(); + if (!text.trim()) { + throw new MorphogenError("INPUT_MISSING", "stdin was empty"); + } + try { + return JSON.parse(text) as JsonValue; + } catch (e) { + throw new MorphogenError( + "PARSE_FAILED", + `stdin: ${e instanceof Error ? e.message : String(e)}`, + ); + } +} + function out(v: JsonValue | JsonObject | RunReceipt): void { process.stdout.write(canonicalize(v as JsonValue) + "\n"); } @@ -176,6 +244,179 @@ async function loadTransports( return out; } +/** Load a tool registry from a JSON file: + * { "": { "signature": {...}, "exec": "scripted:" | "cmd:" } } + * scripted maps canonical(inputs) -> output ports; cmd receives + * { inputs, requestDigest, idempotencyKey } on stdin and must print a JSON + * object of output ports. Both stay behind the signature's bounds. */ +async function loadTools(file: string): Promise { + const resolved = resolve(file); + const raw = asRecord(await readJson(resolved), "tools"); + const base = dirname(resolved); + const registry: ToolRegistry = new Map(); + for (const [name, entry] of Object.entries(raw)) { + if ( + !/^[a-z0-9][a-z0-9.-]*$/.test(name) || + name.length > TOOL_SIGNATURE_BOUNDS.maxNameLen + ) { + throw new MorphogenError("PARSE_FAILED", `invalid tool name "${name}"`); + } + const e = asRecord(entry, `tools.${name}`); + const extra = Object.keys(e).filter( + (k) => k !== "signature" && k !== "exec", + ); + if (e.signature === undefined || typeof e.exec !== "string" || extra.length > 0) { + throw new MorphogenError( + "PARSE_FAILED", + `tools.${name} requires "signature" and "exec"`, + ); + } + const signature = parseToolSignature( + e.signature, + `tools.${name}.signature`, + ); + const spec = e.exec; + let tool: Tool; + if (spec.startsWith("scripted:")) { + const data = asRecord( + await readJson(resolve(base, spec.slice("scripted:".length))), + `tools.${name} data`, + ); + tool = async (inputs) => { + const key = canonicalize(inputs); + const hit = data[key]; + if (hit === null || typeof hit !== "object" || Array.isArray(hit)) { + throw new MorphogenError( + "TOOL_FAILED", + `${name}: no scripted output for inputs ${key.slice(0, 200)}`, + ); + } + return hit as Record; + }; + } else if (spec.startsWith("cmd:")) { + const command = spec.slice("cmd:".length); + tool = async (inputs, context) => { + const proc = Bun.spawn(["sh", "-c", command], { + cwd: base, + stdin: "pipe", + stdout: "pipe", + stderr: "pipe", + }); + void proc.stdin.write( + canonicalize({ + inputs, + requestDigest: context.requestDigest, + idempotencyKey: context.idempotencyKey, + } as JsonValue), + ); + void proc.stdin.end(); + const onAbort = () => proc.kill("SIGKILL"); + const timer = setTimeout(onAbort, 30_000); + context.signal?.addEventListener("abort", onAbort); + let stdout: Uint8Array; + let code: number; + try { + stdout = new Uint8Array( + await new Response(proc.stdout).arrayBuffer(), + ); + code = await proc.exited; + } finally { + clearTimeout(timer); + context.signal?.removeEventListener("abort", onAbort); + } + if (stdout.byteLength > signature.maxOutputBytes) { + throw new MorphogenError( + "TOOL_FAILED", + `${name}: output exceeds ${signature.maxOutputBytes} bytes`, + ); + } + if (code !== 0) { + const stderr = (await new Response(proc.stderr).text()).slice(0, 2000); + throw new MorphogenError( + "TOOL_FAILED", + `${name}: exited ${code}: ${stderr}`, + ); + } + let parsed: unknown; + try { + parsed = JSON.parse(new TextDecoder().decode(stdout)); + } catch { + throw new MorphogenError("TOOL_FAILED", `${name}: stdout is not JSON`); + } + if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) { + throw new MorphogenError( + "TOOL_FAILED", + `${name}: output must be a JSON object of ports`, + ); + } + return parsed as Record; + }; + } else { + throw new MorphogenError( + "PARSE_FAILED", + `tools.${name}.exec must be scripted: or cmd:`, + ); + } + registry.set(name, { signature, tool }); + } + return registry; +} + +/** Convert a Morphogen port type to a draft-07 JSON Schema fragment. */ +function portToJsonSchema(p: { + type: string; + optional?: boolean; + many?: boolean; + labels?: string[]; + schema?: JsonObject; +}): JsonValue { + let base: JsonObject; + switch (p.type) { + case "text": + base = { type: "string" }; + break; + case "int": + base = { type: "integer" }; + break; + case "number": + base = { type: "number" }; + break; + case "bool": + base = { type: "boolean" }; + break; + case "json": + base = p.schema ? { ...p.schema } : { type: "object" }; + break; + case "choice": + base = { type: "string", enum: p.labels ?? [] }; + break; + case "ref": + base = { type: "string", pattern: "^sha256:[a-f0-9]{64}$" }; + break; + default: + base = {}; + } + return p.many ? { type: "array", items: base } : (base as JsonValue); +} + +/** Derive a public interface from the manifest's input cells when the + * manifest does not declare one. Input cells export their first output port. */ +function deriveInputs(c: { + manifest: { cells: { id: string; kind: string }[] }; + ports: Map; outputs: Record }>; +}): Record { + const inputs: Record = {}; + for (const cell of c.manifest.cells) { + if (cell.kind !== "input") continue; + const sig = c.ports.get(cell.id); + if (!sig) continue; + const first = Object.keys(sig.outputs)[0]; + if (!first) continue; + inputs[cell.id] = { cell: cell.id, port: first }; + } + return inputs; +} + async function main(): Promise { const { cmd, positional, flags } = parseArgs(process.argv.slice(2)); const dir = String(flags.dir ?? ".morphogen"); @@ -301,6 +542,177 @@ async function main(): Promise { return 0; } + case "call": { + const file = positional[0]; + if (!file) usageError("morphogen call [options]"); + if (flags.modules !== undefined) { + const n = await loadModules(String(flags.modules), store); + diag(`loaded ${n} module(s) from ${flags.modules}`); + } + + const bundle = parseBundle(await readJson(resolve(file))); + await unpackBundle(bundle, store); + const manifest = await store.getManifest(bundle.root); + if (!manifest) { + throw new MorphogenError("STORE_MISS", `bundle root ${bundle.root} not in store after unpack`); + } + + const argsRaw = + flags.args === "-" + ? asRecord(await readJsonStdin(), "args") + : flags.args !== undefined + ? asRecord(await readJson(resolve(String(flags.args))), "args") + : {}; + const args: Record> = {}; + for (const [cellId, ports] of Object.entries(argsRaw)) { + args[cellId] = asRecord(ports as JsonValue, `args.${cellId}`); + } + + const executors: Executor[] = []; + if (flags.responses !== undefined) { + const map = asRecord( + await readJson(resolve(String(flags.responses))), + "responses", + ); + executors.push(scriptedExecutor(map as Record)); + } + if (flags["executor-cmd"] !== undefined) { + executors.push(commandExecutor(String(flags["executor-cmd"]))); + } + if (flags["gateway-model"] !== undefined) { + executors.push(vercelGatewayExecutor({ model: String(flags["gateway-model"]) })); + } + if (flags.executors !== undefined) { + const map = asRecord( + await readJson(resolve(String(flags.executors))), + "executors", + ); + for (const [name, cmd] of Object.entries(map)) { + if (typeof cmd !== "string" || cmd.length === 0) { + throw new MorphogenError( + "PARSE_FAILED", + `executors.${name} must be a shell command string`, + ); + } + const inner = commandExecutor(cmd); + executors.push({ id: name, execute: (r) => inner.execute(r) }); + } + diag(`loaded ${Object.keys(map).length} named executor(s)`); + } + const transports = + flags.transports !== undefined + ? await loadTransports(String(flags.transports)) + : undefined; + const tools = + flags.tools !== undefined + ? await loadTools(String(flags.tools)) + : undefined; + + const receipt = await runOrganism({ + manifest, + args, + fns, + store, + executors: + flags["cache-effects"] !== undefined + ? executors.map((e) => cachedExecutor(e, store)) + : executors, + ...(transports ? { transports } : {}), + ...(tools ? { tools } : {}), + }); + + const outputs: JsonObject = {}; + for (const [id, cell] of Object.entries(receipt.cells)) { + if (cell.outputs) { + outputs[id] = cell.outputs as JsonValue; + } + } + const rd = await store.putReceipt(receipt as unknown as JsonValue); + const compact: JsonObject = { + ok: receipt.outcome === "complete", + outputs, + receiptDigest: rd, + manifestDigest: receipt.manifestDigest, + }; + if (receipt.outcome !== "complete") { + compact.error = receipt.failure + ? { code: receipt.failure.code, message: receipt.failure.message } + : { code: "FAILED", message: receipt.outcome }; + } + out(compact); + return receipt.outcome === "complete" ? 0 : 1; + } + + case "tool-def": { + const file = positional[0]; + if (!file) usageError("morphogen tool-def [--format openai|anthropic]"); + if (flags.modules !== undefined) { + const n = await loadModules(String(flags.modules), store); + diag(`loaded ${n} module(s) from ${flags.modules}`); + } + const manifest = parseOrganismManifest(await readJson(resolve(file))); + const compiled = await compileOrganism( + manifest, + fns, + store, + 0, + flags.transports !== undefined + ? await loadTransports(String(flags.transports)) + : undefined, + ); + + const raw = manifest.interface ?? { inputs: deriveInputs(compiled), outputs: {} }; + const properties: JsonObject = {}; + const required: string[] = []; + for (const [name, end] of Object.entries(raw.inputs)) { + const sig = compiled.ports.get(end.cell); + if (!sig) { + throw new MorphogenError("PARSE_FAILED", `interface input ${name}: cell ${end.cell} not found`); + } + const p = sig.outputs[end.port]; + if (!p) { + throw new MorphogenError("PARSE_FAILED", `interface input ${name}: port ${end.port} not found on cell ${end.cell}`); + } + properties[name] = portToJsonSchema(p as never) as JsonObject; + if (!p.optional) required.push(name); + } + const parameters: JsonObject = { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + additionalProperties: false, + properties, + }; + if (required.length) parameters.required = required; + + const baseName = + (manifest.key.split(":").pop() ?? manifest.name) + .toLowerCase() + .replace(/[^a-z0-9_-]/g, "_") + .replace(/_+/g, "_") + .replace(/^_|_$/g, "") || "organism"; + const functionName = baseName.slice(0, 64) || "organism"; + const description = manifest.note ?? manifest.name; + + const fmt = String(flags.format ?? "openai"); + if (fmt === "anthropic") { + out({ + name: functionName, + description, + input_schema: parameters, + }); + } else { + out({ + type: "function", + function: { + name: functionName, + description, + parameters, + }, + }); + } + return 0; + } + case "check": { const file = positional[0]; if (!file) usageError("morphogen check [--modules ]"); @@ -416,6 +828,9 @@ async function main(): Promise { if (flags["executor-cmd"] !== undefined) { executors.push(commandExecutor(String(flags["executor-cmd"]))); } + if (flags["gateway-model"] !== undefined) { + executors.push(vercelGatewayExecutor({ model: String(flags["gateway-model"]) })); + } if (flags.executors !== undefined) { const map = asRecord( await readJson(resolve(String(flags.executors))), @@ -437,6 +852,10 @@ async function main(): Promise { flags.transports !== undefined ? await loadTransports(String(flags.transports)) : undefined; + const tools = + flags.tools !== undefined + ? await loadTools(String(flags.tools)) + : undefined; const receipt = await runOrganism({ manifest, @@ -448,6 +867,7 @@ async function main(): Promise { ? executors.map((e) => cachedExecutor(e, store)) : executors, ...(transports ? { transports } : {}), + ...(tools ? { tools } : {}), }); if (flags.write) { @@ -460,6 +880,472 @@ async function main(): Promise { return receipt.outcome === "complete" ? 0 : 1; } + case "foundry": { + const file = positional[0]; + if (!file) usageError("morphogen foundry | foundry verify|inspect|pack "); + if (file === "verify") { + const reportFile = positional[1]; + if (!reportFile) usageError("morphogen foundry verify [--dir ]"); + const verified = await verifyFoundryReport( + await readJson(resolve(reportFile)), + store, + fns, + flags.tools !== undefined ? await loadTools(String(flags.tools)) : undefined, + ); + out(verified as unknown as JsonObject); + return verified.ok ? 0 : 1; + } + if (file === "inspect") { + const reportFile = positional[1]; + if (!reportFile) usageError("morphogen foundry inspect "); + const report = parseFoundryReport(await readJson(resolve(reportFile))); + out({ + contract: report.contract, + digest: report.digest, + promoted: report.promoted, + lineage: report.lineage ?? null, + candidates: report.candidates.map((candidate) => ({ + manifestDigest: candidate.manifestDigest, + manifestKey: candidate.manifestKey, + train: candidate.train, + validation: candidate.validation, + work: candidate.work, + usage: candidate.usage, + })), + holdout: { passed: report.holdout.passed, total: report.holdout.total }, + }); + return 0; + } + if (file === "search-verify") { + const reportFile = positional[1]; + if (!reportFile) usageError("morphogen foundry search-verify [--dir ]"); + const verified = await verifySearchReport( + await readJson(resolve(reportFile)), + store, + fns, + flags.tools !== undefined ? await loadTools(String(flags.tools)) : undefined, + ); + out(verified as unknown as JsonObject); + return verified.ok ? 0 : 1; + } + if (file === "search-inspect") { + const reportFile = positional[1]; + if (!reportFile) usageError("morphogen foundry search-inspect "); + const report = parseSearchReport(await readJson(resolve(reportFile))); + out({ + contract: report.contract, + digest: report.digest, + generatorDigest: report.generatorDigest, + generations: report.generations.map((generation) => ({ + generation: generation.generation, + proposed: generation.proposed.length, + population: generation.candidates.length, + promoted: generation.promoted, + })), + promoted: report.result.promoted, + holdout: { passed: report.result.holdout.passed, total: report.result.holdout.total }, + }); + return 0; + } + if (file === "search-pack") { + const reportFile = positional[1]; + if (!reportFile || flags.out === undefined) { + usageError("morphogen foundry search-pack --out [--dir ]"); + } + const raw = await readJson(resolve(reportFile)); + const report = parseSearchReport(raw); + const verified = await verifySearchReport( + raw, + store, + fns, + flags.tools !== undefined ? await loadTools(String(flags.tools)) : undefined, + ); + if (!verified.ok) { + throw new MorphogenError("RECEIPT_MISMATCH", `search report failed verification: ${verified.mismatches.join("; ")}`); + } + const promoted = await store.getManifest(report.result.promoted); + if (!promoted) throw new MorphogenError("STORE_MISS", `promoted manifest ${report.result.promoted} missing`); + const bundle = await packOrganism(promoted, store); + const outputDir = resolve(String(flags.out)); + const { mkdir, writeFile } = await import("node:fs/promises"); + await mkdir(outputDir, { recursive: true }); + const outputFile = join(outputDir, `${report.result.promoted.slice(7)}.bundle.json`); + await writeFile(outputFile, canonicalize(bundle as unknown as JsonValue)); + out({ bundle: outputFile, root: bundle.root, search: report.digest }); + return 0; + } + if (file === "pack") { + const reportFile = positional[1]; + if (!reportFile || flags.out === undefined) { + usageError("morphogen foundry pack --out [--dir ]"); + } + const raw = await readJson(resolve(reportFile)); + const report = parseFoundryReport(raw); + const verified = await verifyFoundryReport( + raw, + store, + fns, + flags.tools !== undefined ? await loadTools(String(flags.tools)) : undefined, + ); + if (!verified.ok) { + throw new MorphogenError("RECEIPT_MISMATCH", `foundry report failed verification: ${verified.mismatches.join("; ")}`); + } + const promoted = await store.getManifest(report.promoted); + if (!promoted) throw new MorphogenError("STORE_MISS", `promoted manifest ${report.promoted} missing`); + const bundle = await packOrganism(promoted, store); + const outputDir = resolve(String(flags.out)); + const { mkdir, writeFile } = await import("node:fs/promises"); + await mkdir(outputDir, { recursive: true }); + const outputFile = join(outputDir, `${report.promoted.slice(7)}.bundle.json`); + await writeFile(outputFile, canonicalize(bundle as unknown as JsonValue)); + out({ bundle: outputFile, root: bundle.root, foundry: report.digest }); + return 0; + } + const searchMode = file === "search"; + const configPath = searchMode ? positional[1] : file; + if (!configPath) usageError("morphogen foundry search "); + const configFile = resolve(configPath); + if (flags.modules !== undefined) { + const n = await loadModules(String(flags.modules), store); + diag(`loaded ${n} module(s) from ${flags.modules}`); + } + const config = asRecord(await readJson(configFile), "foundry config"); + const unknown = Object.keys(config).filter((k) => !["contract", "candidates", "generator", "cases", "search"].includes(k)); + if (unknown.length > 0) { + throw new MorphogenError("PARSE_FAILED", `foundry config: unknown key "${unknown[0]}"`); + } + if (config.contract !== "morphogen.foundry.config.v1") { + throw new MorphogenError("PARSE_FAILED", "foundry config.contract must be morphogen.foundry.config.v1"); + } + if (!searchMode && config.search !== undefined) { + throw new MorphogenError("PARSE_FAILED", "search settings require the foundry search command"); + } + const candidateEntries = config.candidates ?? []; + if (!Array.isArray(candidateEntries)) { + throw new MorphogenError("PARSE_FAILED", "foundry config.candidates must be a list"); + } + if (candidateEntries.length === 0 && config.generator === undefined) { + throw new MorphogenError("PARSE_FAILED", "foundry config needs candidates or a generator"); + } + if (!Array.isArray(config.cases) || config.cases.length === 0) { + throw new MorphogenError("PARSE_FAILED", "foundry config.cases must be a non-empty list"); + } + const base = dirname(configFile); + const candidates = await Promise.all(candidateEntries.map(async (candidate, i) => { + if (typeof candidate !== "string") { + throw new MorphogenError("PARSE_FAILED", `foundry config.candidates[${i}] must be a path`); + } + return parseOrganismManifest(await readJson(resolve(base, candidate))); + })); + let generator: { manifest: ReturnType; args: Record; output: string; field?: string } | undefined; + if (config.generator !== undefined) { + const raw = asRecord(config.generator, "foundry config.generator"); + const extra = Object.keys(raw).filter((k) => !["manifest", "args", "output", "field"].includes(k)); + if ( + extra.length > 0 || + typeof raw.manifest !== "string" || + typeof raw.output !== "string" || + (raw.field !== undefined && typeof raw.field !== "string") + ) { + throw new MorphogenError("PARSE_FAILED", "foundry config.generator needs manifest, args, and output"); + } + if (raw.args === undefined) { + throw new MorphogenError("PARSE_FAILED", "foundry config.generator.args must be an object"); + } + generator = { + manifest: parseOrganismManifest(await readJson(resolve(base, raw.manifest))), + args: asRecord(raw.args, "foundry config.generator.args"), + output: raw.output, + ...(typeof raw.field === "string" ? { field: raw.field } : {}), + }; + } + const cases: FoundryCase[] = config.cases.map((raw, i) => { + const c = asRecord(raw, `foundry config.cases[${i}]`); + const extra = Object.keys(c).filter((k) => !["id", "split", "args", "expect"].includes(k)); + if (extra.length > 0) { + throw new MorphogenError("PARSE_FAILED", `foundry config.cases[${i}]: unknown key "${extra[0]}"`); + } + if ( + typeof c.id !== "string" || + (c.split !== "train" && c.split !== "validation" && c.split !== "holdout") + ) { + throw new MorphogenError("PARSE_FAILED", `foundry config.cases[${i}] needs string id and train|validation|holdout split`); + } + if (c.args === undefined || c.expect === undefined) { + throw new MorphogenError("PARSE_FAILED", `foundry config.cases[${i}] needs args and expect objects`); + } + return { + id: c.id, + split: c.split, + args: asRecord(c.args, `foundry config.cases[${i}].args`), + expect: asRecord(c.expect, `foundry config.cases[${i}].expect`), + }; + }); + const executors: Executor[] = []; + if (flags.responses !== undefined) { + executors.push(scriptedExecutor(asRecord( + await readJson(resolve(String(flags.responses))), + "responses", + ) as Record)); + } + if (flags["executor-cmd"] !== undefined) { + executors.push(commandExecutor(String(flags["executor-cmd"]))); + } + if (flags["gateway-model"] !== undefined) { + executors.push(vercelGatewayExecutor({ model: String(flags["gateway-model"]) })); + } + if (flags.executors !== undefined) { + const map = asRecord(await readJson(resolve(String(flags.executors))), "executors"); + for (const [name, command] of Object.entries(map)) { + if (typeof command !== "string" || command.length === 0) { + throw new MorphogenError("PARSE_FAILED", `executors.${name} must be a shell command string`); + } + const inner = commandExecutor(command); + executors.push({ id: name, execute: (request) => inner.execute(request) }); + } + } + const activeExecutors = flags["cache-effects"] !== undefined + ? executors.map((executor) => cachedExecutor(executor, store)) + : executors; + const transports = flags.transports !== undefined + ? await loadTransports(String(flags.transports)) + : undefined; + const tools = flags.tools !== undefined + ? await loadTools(String(flags.tools)) + : undefined; + if (searchMode) { + if (!generator || config.search === undefined) { + throw new MorphogenError("PARSE_FAILED", "search config needs generator and search objects"); + } + const search = asRecord(config.search, "foundry config.search"); + const extra = Object.keys(search).filter((key) => !["maxGenerations", "feedbackInput"].includes(key)); + if ( + extra.length > 0 || + !Number.isInteger(search.maxGenerations) || + typeof search.feedbackInput !== "string" + ) { + throw new MorphogenError("PARSE_FAILED", "foundry config.search needs maxGenerations and feedbackInput"); + } + const report = await runFoundrySearch({ + generator: generator.manifest, + generatorArgs: generator.args, + feedbackInput: search.feedbackInput, + output: generator.output, + ...(generator.field ? { field: generator.field } : {}), + seeds: candidates, + cases, + maxGenerations: search.maxGenerations as number, + fns, + store, + executors: activeExecutors, + ...(transports ? { transports } : {}), + ...(tools ? { tools } : {}), + }); + if (flags.out !== undefined) { + const { writeFile } = await import("node:fs/promises"); + await writeFile(resolve(String(flags.out)), canonicalize(report as unknown as JsonValue)); + } + out(report as unknown as JsonObject); + return 0; + } + const generated = generator + ? await generateFoundryCandidates({ + generator: generator.manifest, + args: generator.args, + output: generator.output, + ...(generator.field ? { field: generator.field } : {}), + fns, + store, + executors: activeExecutors, + ...(transports ? { transports } : {}), + ...(tools ? { tools } : {}), + }) + : undefined; + if (generated) candidates.push(...generated.candidates); + const report = await runFoundry({ + candidates, + cases, + fns, + store, + executors: activeExecutors, + ...(transports ? { transports } : {}), + ...(tools ? { tools } : {}), + ...(generated ? { + lineage: { + generatorDigest: generated.generatorDigest, + receiptDigest: generated.receiptDigest, + }, + } : {}), + }); + if (flags.out !== undefined) { + const { writeFile } = await import("node:fs/promises"); + await writeFile(resolve(String(flags.out)), canonicalize(report as unknown as JsonValue)); + } + out(report as unknown as JsonObject); + return 0; + } + + case "bench": { + const file = positional[0]; + if (!file) { + usageError("morphogen bench | bench verify|inspect "); + } + if (file === "verify") { + const reportFile = positional[1]; + if (!reportFile) usageError("morphogen bench verify [--dir ]"); + const verified = await verifyBenchReport( + await readJson(resolve(reportFile)), + store, + fns, + flags.tools !== undefined ? await loadTools(String(flags.tools)) : undefined, + ); + out(verified as unknown as JsonObject); + return verified.ok ? 0 : 1; + } + if (file === "inspect") { + const reportFile = positional[1]; + if (!reportFile) usageError("morphogen bench inspect "); + const report = parseBenchReport(await readJson(resolve(reportFile))); + out({ + contract: report.contract, + digest: report.digest, + workload: report.workload, + pareto: report.pareto, + systems: report.systems.map((system) => ({ + id: system.id, + manifestKey: system.manifestKey, + manifestDigest: system.manifestDigest, + passed: system.passed, + total: system.total, + effectCalls: system.effectCalls, + work: system.work, + usage: system.usage, + attribution: system.attribution, + pareto: report.pareto.includes(system.id), + })), + }); + return 0; + } + const configFile = resolve(file); + if (flags.modules !== undefined) { + const n = await loadModules(String(flags.modules), store); + diag(`loaded ${n} module(s) from ${flags.modules}`); + } + const config = asRecord(await readJson(configFile), "bench config"); + const unknown = Object.keys(config).filter((k) => !["contract", "cases", "systems", "prices"].includes(k)); + if (unknown.length > 0) { + throw new MorphogenError("PARSE_FAILED", `bench config: unknown key "${unknown[0]}"`); + } + if (config.contract !== "morphogen.bench.config.v1") { + throw new MorphogenError("PARSE_FAILED", "bench config.contract must be morphogen.bench.config.v1"); + } + if (!Array.isArray(config.cases) || config.cases.length === 0) { + throw new MorphogenError("PARSE_FAILED", "bench config.cases must be a non-empty list"); + } + if (!Array.isArray(config.systems) || config.systems.length === 0) { + throw new MorphogenError("PARSE_FAILED", "bench config.systems must be a non-empty list"); + } + const base = dirname(configFile); + const cases: BenchCase[] = config.cases.map((raw, i) => { + const c = asRecord(raw, `bench config.cases[${i}]`); + const extra = Object.keys(c).filter((k) => !["id", "args", "expect"].includes(k)); + if (extra.length > 0) { + throw new MorphogenError("PARSE_FAILED", `bench config.cases[${i}]: unknown key "${extra[0]}"`); + } + if (typeof c.id !== "string" || c.args === undefined || c.expect === undefined) { + throw new MorphogenError("PARSE_FAILED", `bench config.cases[${i}] needs id, args, and expect`); + } + return { + id: c.id, + args: asRecord(c.args, `bench config.cases[${i}].args`), + expect: asRecord(c.expect, `bench config.cases[${i}].expect`), + }; + }); + const named = (id: string, inner: Executor): Executor => ({ + id, + execute: (request, signal) => inner.execute(request, signal), + ...(inner.executeEffect + ? { executeEffect: (request: Parameters>[0], signal?: AbortSignal) => inner.executeEffect!(request, signal) } + : {}), + ...(inner.receiptFor + ? { receiptFor: (request: Parameters>[0]) => inner.receiptFor!(request) } + : {}), + }); + const resolveSpec = async (id: string, spec: string): Promise => { + if (spec.startsWith("gateway:")) { + return named(id, vercelGatewayExecutor({ model: spec.slice("gateway:".length) })); + } + if (spec.startsWith("scripted:")) { + const responses = asRecord( + await readJson(resolve(base, spec.slice("scripted:".length))), + `bench executor ${id}`, + ) as Record; + return named(id, scriptedExecutor(responses, id)); + } + if (spec.startsWith("cmd:")) { + return named(id, commandExecutor(spec.slice("cmd:".length))); + } + throw new MorphogenError( + "PARSE_FAILED", + `bench executor "${id}": unknown spec (want gateway:, scripted:, or cmd:)`, + ); + }; + const systems: BenchSystem[] = []; + for (const [i, raw] of config.systems.entries()) { + const s = asRecord(raw, `bench config.systems[${i}]`); + const extra = Object.keys(s).filter((k) => !["id", "manifest", "executors"].includes(k)); + if (extra.length > 0) { + throw new MorphogenError("PARSE_FAILED", `bench config.systems[${i}]: unknown key "${extra[0]}"`); + } + if (typeof s.id !== "string" || typeof s.manifest !== "string" || s.executors === undefined) { + throw new MorphogenError("PARSE_FAILED", `bench config.systems[${i}] needs id, manifest, and executors`); + } + const specs = asRecord(s.executors, `bench config.systems[${i}].executors`); + const executors: Executor[] = []; + for (const [name, spec] of Object.entries(specs)) { + if (typeof spec !== "string" || spec.length === 0) { + throw new MorphogenError("PARSE_FAILED", `bench executor "${name}" must be a spec string`); + } + executors.push(await resolveSpec(name, spec)); + } + systems.push({ + id: s.id, + manifest: parseOrganismManifest(await readJson(resolve(base, s.manifest))), + executors, + }); + } + const transports = + flags.transports !== undefined + ? await loadTransports(String(flags.transports)) + : undefined; + const tools = + flags.tools !== undefined + ? await loadTools(String(flags.tools)) + : undefined; + const prices = parseBenchPrices(config.prices, "bench config.prices"); + const report = await runBenchmark({ + systems: systems.map((system) => ({ + ...system, + executors: + flags["cache-effects"] !== undefined + ? system.executors.map((e) => cachedExecutor(e, store)) + : system.executors, + })), + cases, + fns, + store, + ...(transports ? { transports } : {}), + ...(tools ? { tools } : {}), + ...(prices ? { prices } : {}), + }); + if (flags.out !== undefined) { + const { writeFile } = await import("node:fs/promises"); + await writeFile(resolve(String(flags.out)), canonicalize(report as unknown as JsonValue)); + } + out(report as unknown as JsonObject); + return 0; + } + case "verify": { const [receiptFile, manifestFile] = positional; if (!receiptFile) { @@ -499,6 +1385,9 @@ async function main(): Promise { flags.transports !== undefined ? await loadTransports(String(flags.transports)) : undefined, + flags.tools !== undefined + ? await loadTools(String(flags.tools)) + : undefined, ); out(report as unknown as JsonObject); return report.ok ? 0 : 1; @@ -807,6 +1696,33 @@ function asRecord(v: JsonValue, what: string): Record { return v as Record; } +function parseBenchPrices( + raw: JsonValue | undefined, + at: string, +): Record | undefined { + if (raw === undefined) return undefined; + const map = asRecord(raw, at); + const prices: Record = {}; + for (const [key, value] of Object.entries(map)) { + if (key.length === 0 || key.length > 256) { + throw new MorphogenError("PARSE_FAILED", `${at} has an invalid price key`); + } + const p = asRecord(value, `${at}.${key}`); + const extra = Object.keys(p).filter((k) => !["input", "output"].includes(k)); + if (extra.length > 0) { + throw new MorphogenError("PARSE_FAILED", `${at}.${key}: unknown key "${extra[0]}"`); + } + if (typeof p.input !== "number" || p.input < 0 || !Number.isFinite(p.input)) { + throw new MorphogenError("PARSE_FAILED", `${at}.${key}.input must be a non-negative number`); + } + if (typeof p.output !== "number" || p.output < 0 || !Number.isFinite(p.output)) { + throw new MorphogenError("PARSE_FAILED", `${at}.${key}.output must be a non-negative number`); + } + prices[key] = { input: p.input, output: p.output }; + } + return prices; +} + function usageError(msg: string): never { throw new MorphogenError("PARSE_FAILED", `usage: ${msg}`); } diff --git a/docs/agent-loop-and-organism.md b/docs/agent-loop-and-organism.md new file mode 100644 index 0000000..29f07c9 --- /dev/null +++ b/docs/agent-loop-and-organism.md @@ -0,0 +1,95 @@ +# Organisms and agent loops + +Morphogen organisms are themselves bounded agent loops. The interesting +question is how they relate to the larger agent loop that deploys them: when +do you compile a piece of agentic behavior into an organism, and when do you +let the outer loop stay improvisational? + +## An organism is a compiled loop + +A `repeat` cell already captures the evaluator-optimizer pattern: the same +sub-manifest runs for bounded rounds, carrying outputs forward, until a guard +fires or `maxRounds` is reached. An `each` cell maps a list through a +sub-manifest. An agent cell with `tools` and `budget.maxTurns` is a bounded +turn-based tool loop inside one cell. So the organism contains the same +primitives that make an agent loop — but they are typed, budgeted, and +content-addressed. + +The difference is not power; it is **inspectability and custody**: + +| aspect | improvised agent loop | Morphogen organism | +|---|---|---| +| state | mutable, ambient | ports on a DAG, typed | +| branching | if/else in code or prompt | `guard`ed edges, routable failures | +| tool calls | ad-hoc, hard to replay | `tool` cells and `tools` log, digest-bound | +| model choice | global config or hidden heuristics | `route.preset` / `route.provider` explicit in the graph | +| cost limit | implicit timeout | `maxSteps`, `maxAgentCalls`, `maxWork`, `maxEffectMs` | +| evidence | transcript | content-addressed receipt + offline `verify` | +| shipping | prompt/config | a manifest digest and its bundle | + +## Practical layering + +The outer agent loop should stay improvisational where the task is genuinely +open-ended: "what should I investigate next?", "is this user request in scope?", +"what is the user's intent?" The inner organism should take over once the +sub-problem is well-formed: + +- extract the next intent → outer loop; +- verify a claim against a structured source → `tool` + `classifier`; +- classify and route a ticket → `classifier` + guards; +- generate a code diff and run the test suite → `agent` + `fn` + `assert.v1`; +- search an organism topology for the cheapest one that passes validation → + `foundry search`; + +The boundary is: **the outer loop decides what to do, the organism decides how +to do it** for the sub-problems that can be described as a bounded workflow. + +## A coding agent example + +A coding agent's outer loop might look like: + +``` +read user request +→ plan next step +→ if step is well-formed, call `morphogen run` on the right organism +→ receive the receipt and outputs +→ decide the next step or stop +``` + +The organism it calls could be `patch-and-test`: + +1. `agent` cell drafts the patch. +2. `tool` cell runs the test command (declared host capability). +3. `classifier` cell reads the test result and decides `pass`, `fail:fix`, or + `fail:escalate`. +4. `on:fail` edge routes a `fail:fix` back to the agent with the test output in + its `view.cells`. +5. `repeat` cell bounds the number of fix attempts. + +The agent does not watch the patch being written. It asks the organism to +solve a structured task and gets a receipt it can pass upstream. If the +organism is good, the agent can use it as a tool without inheriting its +complexity. + +## Can the agent itself be an organism? + +Yes, at the cost of stronger assumptions. You can write the outer loop as a +Morphogen organism if: + +- the conversation has a bounded state model (a list of turns, a current goal); +- the set of next actions is closed (a `choice` of intents); +- the tools it can call are already admitted; +- the termination condition is explicit. + +That is a useful form for a *task-specific* assistant, not a general chat +interface. The practical application is not to replace the agent but to give +the agent a library of verifiable sub-routines it can invoke with confidence. + +## When this is premature + +Do not wrap a whole agent in Morphogen just to have the receipt. Use a +single organism when the inputs, outputs, and budgets are clear, and the +failure modes are worth replaying. Use the outer loop for exploration, +ambiguity, and user interaction. The two layers compose: the agent loop +answers "what problem are we solving?", the organism answers "did this +solution actually satisfy the contract?" diff --git a/docs/agent-tool.md b/docs/agent-tool.md new file mode 100644 index 0000000..2f2987e --- /dev/null +++ b/docs/agent-tool.md @@ -0,0 +1,98 @@ +# Morphogen as an agent tool + +A Morphogen organism is a content-addressed, replayable subroutine. Pack it once and any agent—OpenAI, Anthropic, a coding agent, or a shell script—can call it as a typed tool and receive a compact, verifiable result. + +## Why + +- **Manifests are the contract.** Inputs, outputs, budgets, and failure paths are declared before the run starts. +- **Receipts are the evidence.** Every tool call produces a `receiptDigest` that can be replayed offline without the original provider. +- **Bundles are the transport.** `morphogen pack` collects the organism and every embedded sub-manifest into one closure. + +## Pack + +```sh +morphogen pack examples/triage.morphogen.json --out ./tools +``` + +This writes `./tools/.bundle.json`. The bundle is self-contained and digest-verified. + +## Register the tool + +Generate an OpenAI or Anthropic tool definition from the manifest's interface: + +```sh +morphogen tool-def examples/triage.morphogen.json +morphogen tool-def examples/triage.morphogen.json --format anthropic +``` + +OpenAI output: + +```json +{ + "type": "function", + "function": { + "name": "triage", + "description": "A classifier cell routes a support ticket; the structure carries the routing decision.", + "parameters": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": false, + "properties": { + "ticket": { "type": "string" } + }, + "required": ["ticket"] + } + } +} +``` + +Register that definition with the agent. When the agent decides to call the tool, it will be asked for a JSON object like `{"ticket": "..."}`. + +## Call + +The agent invokes `morphogen call` with the bundled organism and the tool arguments. Pass `--args -` to read the arguments from stdin: + +```sh +echo '{"ticket":{"text":"I cannot log in after the update"}}' | \ + morphogen call ./tools/.bundle.json \ + --args - \ + --responses examples/triage.responses.json +``` + +Or write the args to a file: + +```sh +echo '{"ticket":{"text":"I cannot log in after the update"}}' > /tmp/triage.args.json +morphogen call ./tools/.bundle.json \ + --args /tmp/triage.args.json \ + --responses examples/triage.responses.json +``` + +The output is compact, so the agent does not need to parse the full receipt: + +```json +{ + "ok": true, + "outputs": { + "route": { "out": "bug" }, + "result": { "value": "BUG: I cannot log in after the update" } + }, + "receiptDigest": "sha256:...", + "manifestDigest": "sha256:..." +} +``` + +## Wiring into a coding agent + +A coding agent can use a Morphogen tool for any stable, repeatable, inspectable subproblem: + +- `summarize-diff` — read a git diff and classify intent (refactor, fix, feature). +- `test-patch` — run the test command and return `pass`, `fail`, or `escalate`. +- `investigate-billing` — the `examples/invest/` organism with ledger lookup. +- `extract-api-changes` — parse source files and report breaking changes. + +The outer agent keeps doing open-ended planning, user interaction, and retries. The organism owns the bounded, typed step and returns evidence the agent can trust or escalate. + +## Failure handling + +If the organism fails or gets stuck, `ok` is `false` and `error` contains the code and message. The agent can retry with different arguments, escalate to a frontier model, or ask the user. The full receipt is still written to the `--dir` store (default `.morphogen/`) so the failure can be inspected offline. diff --git a/docs/executors.md b/docs/executors.md index 839c03c..c4d4c2f 100644 --- a/docs/executors.md +++ b/docs/executors.md @@ -61,6 +61,26 @@ The loop ends when the response binds to `output` or `budget.maxTurns` is exhausted. A `{tool,inputs}` naming a ref outside `tools` is ordinary output, not a call. +## Vercel AI Gateway + +`--gateway-model ` runs cells through Vercel AI Gateway's +structured Chat Completions API. Authentication comes only from +`AI_GATEWAY_API_KEY` or the short-lived `VERCEL_OIDC_TOKEN` supplied by a linked +Vercel project. The credential never enters a request, receipt, digest, or log. + +```sh +bun run cli run examples/gateway-smoke.morphogen.json \ + --args examples/gateway-smoke.args.json \ + --gateway-model alibaba/qwen3.5-flash --write +``` + +The adapter fixes the Gateway origin, rejects redirects, bounds response bytes, +requests a strict `{ "value": ... }` JSON object, includes the declared output +contract in model-visible context, and returns the bound value to Morphogen. +Provider model identity and input/output token usage are captured after the call +on the ordinary effect receipt, so foundry reports can compare real usage and +offline replay preserves it exactly. + ## Multiple executors `--executors execs.json` maps names to commands: @@ -105,3 +125,58 @@ default when no route matches. Only successes are memoized — a recorded error may be transient — and the first record for a digest wins. A hit is recorded on the new run's receipt as `cached: true`, so reuse stays auditable. + +## Tool registries + +`tool` cells and agent-requested tools both resolve against the host's +`ToolRegistry` — the same typed, receipted boundary, supplied by the host and +never declared inside a manifest. + +For the CLI, `--tools ` loads a registry: + +```json +{ + "ledger.charges.v1": { + "signature": { + "inputs": { "account": "text" }, + "outputs": { "charges": "json" }, + "effect": "read", + "cost": 50, + "maxOutputBytes": 8192 + }, + "exec": "cmd:ledger-cli" + } +} +``` + +`exec` is either `scripted:` — a canonical-inputs → outputs map — +or `cmd:` — a shell command that receives `{ inputs, requestDigest, +idempotencyKey }` on stdin and must print a JSON object of output ports on +stdout. The signature's `maxOutputBytes` is enforced on the command's stdout; +timeouts are enforced by a 30-second hard bound; nonzero exit is a structured +failure. This keeps external IO outside the manifest while still recording it +per-activation. + +Programmatically, a `ToolRegistry` is a `Map`: + +```ts +import type { Tool, ToolRegistry } from "morphogen"; + +const tools: ToolRegistry = new Map([[ + "ledger.charges.v1", + { + signature: { + inputs: { account: { type: "text" } }, + outputs: { charges: { type: "json" } }, + effect: "read", + cost: 50, + maxOutputBytes: 8192, + }, + tool: async (inputs) => ({ charges: await myLedger.lookup(inputs.account) }), + }, +]]); +``` + +The tool receives the canonical inputs and a `ToolContext` carrying the +`requestDigest`, `idempotencyKey`, and an optional `AbortSignal`. Pass it to +`runOrganism` or `runBenchmark` through the `tools` option. diff --git a/docs/habitats.md b/docs/habitats.md new file mode 100644 index 0000000..42c40fd --- /dev/null +++ b/docs/habitats.md @@ -0,0 +1,221 @@ +# Habitats, self-reproduction, and civilization + +There is a deeper shape hidden inside Morphogen: not just a workflow runner, +but a substrate for organisms that live, reproduce, and evolve in a shared +runtime. This is not yet fully built, but the v1 contract already contains the +seeds of it. This document teases the idea apart and shows what is possible +now under the existing safety invariants. + +## What a habitat is + +A habitat is a shared runtime with four parts: + +1. **`Store`** — the memory: manifests, receipts, values, slots. +2. **`FnRegistry`** — the host's deterministic functions. +3. **`ToolRegistry`** — the host's external capabilities. +4. **`Executor[]`** — the providers and models the host admits. + +An organism in a habitat is just a manifest. Its digest is its identity. +Its receipt is its life record. Slots are durable state it can read or write +across runs. Other organisms are cells it can invoke by digest. Tools and +functions are capabilities the host exposes, never things the organism can +create on its own. + +## Organisms as species + +A manifest is a value. It can be: + +- hashed, +- stored, +- transported, +- composed into another organism, +- generated by another organism, +- selected by a foundry. + +This means a "species" is just a set of manifests with a shared lineage or +interface. Evolution does not modify a running organism; it produces new +organisms and the host decides which ones to admit. + +## Self-reproduction is already possible + +A `spawn` cell receives a manifest as data and runs it. An `agent` cell can +generate a manifest as JSON output. Combined, an organism can: + +1. inspect its own inputs, +2. ask a model to propose a child organism (a new manifest), +3. pass that manifest to a `spawn` cell, +4. observe the child run under the same bounds and registries, +5. emit the child's digest and receipt as its own outputs. + +This is reproduction, but it is strictly bounded: +- the child manifest is parsed and admitted by the runtime, +- it runs under the parent's depth and work budgets, +- it cannot escape the host's registries, +- it cannot modify the parent or the habitat directly. + +## Self-evolution is propose-and-select + +An organism cannot edit its own manifest or the runtime. What it can do is +propose a new manifest. The host then decides whether to: + +- store it, +- register it as a tool or function, +- include it in the module set, +- run a bench against it, +- promote it. + +This is exactly the `foundry` pattern. A habitat with a foundry is an +ecosystem where organisms generate candidates, a bench measures them, and the +host (or a governance organism) promotes winners. The "civilization" is the +population of admitted organisms, tools, functions, and the receipts they +leave behind. + +## Proposing new functions and tools + +An organism can propose a new `fn` or `tool` by outputting a signature and an +implementation. But the host must install it. For example, an organism could +emit a JSON record like: + +```json +{ + "name": "stats.v1", + "signature": { + "inputs": { "values": { "type": "json" } }, + "outputs": { "mean": { "type": "number" }, "count": { "type": "int" } }, + "cost": 10 + }, + "implementation": "scripted:..." +} +``` + +The host reads this from the receipt and, if it wants, adds the function to its +registry. The organism itself cannot add the function. This preserves the core +invariant: **manifests are data, authority is host-owned**. + +## Applications as civilizations + +In this view, a Morphogen application is a habitat: + +- **Modules** are organisms in the store. +- **Capabilities** are functions and tools in the registries. +- **State** is in slots and values. +- **Events** are receipts. +- **Evolution** is foundry search over generated candidates. +- **Selection** is the host's promotion policy. + +A "civilization" emerges when many organisms co-exist, call each other, +propose new members, and the host keeps promoting the ones that improve some +measured objective. The receipts form a fossil record: every run, every +proposal, every failure is content-addressed and replayable. + +## Safety invariants that make it possible + +The deep, trippy part only works because the runtime is brutal about +boundaries: + +- **No code in manifests.** A manifest is data. It cannot install itself. +- **Host-owned registries.** New functions, tools, and executors are admitted + by the host, never by the organism. +- **Bounded depth, steps, agent calls, work, bytes, and turns.** A runaway + organism exhausts a declared budget and fails closed. +- **Content-addressed everything.** Manifests and receipts are values. You can + reason about lineage, diff populations, and verify history. +- **Receipts over wall-clock.** There are no ambient time fields. Replay is + bit-for-bit. + +Without these invariants, self-evolution becomes a self-modifying virus +problem. With them, it becomes a search-and-selection problem. + +## What is not here yet + +- Multi-habitat messaging and consensus. +- Durable organism lifetimes (organisms are stateless; only slots persist). +- A governance organism that can promote candidates by itself. +- Economic or tokenized selection mechanisms. +- A public name system for organism digests. + +These are later layers. The current contract already supports the core loop: +generate, admit, run, measure, select. + +## Working example: `examples/habitat.morphogen.json` + +There is a runnable steel thread in the bundled examples. The `habitat` organism: + +1. receives a `goal` input, +2. a `designer` agent outputs a child organism manifest as JSON, +3. a `spawn` cell admits and runs the child under the parent's budgets, +4. a `push.v1` cell appends the child digest to the `population` slot, +5. a `write` slot cell persists the new population. + +Run it: + +```sh +morphogen run examples/habitat.morphogen.json \ + --args examples/habitat.args.json \ + --responses examples/habitat.responses.json \ + --write +``` + +The receipt shows `child` (the spawned manifest digest) and `population` (the +updated list). The child is data, the host still owns the store and registries, + and the whole lineage is replayable. + +## Live self-reproduction: `examples/habitat/live.morphogen.json` + +There is also a non-deterministic version of the habitat steel thread. The +`habitat-live` organism calls a live model to design the child, falls back to a +safe default if the model fails to produce a JSON manifest, and then spawns +whatever is selected. A host script, `examples/habitat/promote.ts`, decides +whether to promote the result. + +Run the scripted version to see the fallback/rejection path: + +```sh +bun examples/habitat/promote.ts +``` + +Run the live version to watch an organism design and spawn a real child: + +```sh +bun examples/habitat/promote.ts --live +# optionally: GATEWAY_MODEL=anthropic/claude-opus-5 bun examples/habitat/promote.ts --live +``` + +`promote.ts`: + +1. packs the known fallback manifest and records its digest, +2. runs `habitat-live` with a live model as the `designer` agent, +3. extracts the spawned child digest from the receipt, +4. compares it to the fallback digest, +5. if the live model produced a *different* valid manifest, packs and writes it +to `promoted/.bundle.json` and prints `promoted bundle …`. + +When this was first run with `alibaba/qwen3.7-flash`, the model generated and +the organism spawned: + +``` +fallback digest sha256:0d888ed00f06bba02259b00a3f9f895ef90d705cdbb1a8cc9c70d3bd96ee7a0e +proposed child sha256:f73c4ef9f89a3c55595f773c5a86c89780cfaf202cf29284005e76a5ad7e751c +promoted bundle sha256:f73c4ef9f89a3c55595f773c5a86c89780cfaf202cf29284005e76a5ad7e751c +wrote promoted/f73c4ef9f89a3c55595f773c5a86c89780cfaf202cf29284005e76a5ad7e751c.bundle.json +``` + +That is the first real live self-reproduction in the habitat: a parent organism +proposed a child through a model call, the runtime admitted and ran it, and the +host script promoted the child to a standalone bundle. The fallback digest gives +a deterministic baseline; any non-fallback child is a live, model-proposed +candidate. + +## A sketch of the next step + +The most concrete near-term habitat would be: + +1. A long-lived `Store` shared by an agent loop. +2. A `ToolRegistry` that can accept proposals from organisms. +3. A `foundry` config that treats past receipts as a population and evolves it. +4. A host rule: a proposed tool is installed only after `morphogen bench` + shows it Pareto-dominates the incumbent. + +The agent loop becomes the habitat's weather: it decides which organisms to +run, which proposals to admit, and which lineages to keep. The organisms do +not rule the habitat; they are the things the habitat selects. diff --git a/docs/when-morphogen-wins.md b/docs/when-morphogen-wins.md new file mode 100644 index 0000000..0aa745d --- /dev/null +++ b/docs/when-morphogen-wins.md @@ -0,0 +1,111 @@ +# When Morphogen wins + +Morphogen is a programming model, not a model model. It wins when the work +has structure that a prompt alone cannot capture — routing, typed inputs, +external evidence, budget enforcement, and the need to prove what happened. + +## The short version + +Use Morphogen when you want to: + +- compare several ways to solve the same workflow on the same workload, +- keep the model from seeing data it should not see, +- make the model call tools and record what the tool returned, +- replay a run later and prove it produced the same result, +- evolve the workflow topology while keeping the holdout blind. + +Do not use it as a drop-in replacement for every chat completion. A single +unstructured question with no routing, no tool, and no audit requirement is +cheaper as a single call. + +## Where structure pays off + +### 1. Tool-grounded decisions + +If the right answer depends on data the model does not carry — a customer +record, a ledger, a calculation, a lookup — put a `tool` cell in the graph +before the judgment. The model receives only the tool's output, typed and +bounded; the receipt records the exact inputs and outputs; the run can be +replayed without repeating the live call. + +### 2. Many narrow judgments over one context + +A long prompt that asks for many things at once is expensive and fragile. A +Morphogen organism can split the work into many `classifier` cells, each with a +declared view, and route the outputs through `guard`ed edges. The graph decides +what the next cell sees, not the model. + +### 3. Disagreement as an escalation trigger + +Two cheap model lanes plus an `assert.v1` cell can detect when they disagree. +Only disagreement fires the frontier `classifier`. This is the structural fix for +"use cheap models most of the time and frontier models only when it matters." + +### 4. Verification before promotion + +A `foundry` or `foundry search` report evaluates organisms against train, +validation, and holdout cases. Every case receipt is content-addressed and can +be replayed offline. You can prove that the promoted organism came from the +measured evidence and never saw the holdout during search. + +## Case study: billing-dispute investigation + +`examples/invest/` is a six-case workload where each ticket needs a decision: +`refund`, `escalate`, or `monitor`. The correct decision depends on the +account's charge ledger — which the model cannot see unless a tool retrieves it. + +The live Vercel AI Gateway run is priced from aicharts.io / AI//COST and +OpenRouter/Alibaba rate cards (USD per 1M tokens): + +|| system | passed | effect calls | cost | input tokens | output tokens | Pareto | +|---|---|---:|---:|---:|---:|---| +|| cheap-single (qwen3.5, no evidence) | 4/6 | 6 | $0.00371 | 759 | 14,087 | yes | +|| frontier-single (claude-opus-5, no evidence) | 4/6 | 6 | $0.02625 | 4,214 | 207 | — | +|| **organism-cheap (qwen3.5 + ledger tool)** | **6/6** | **12** | **$0.00131** | **1,210** | **4,716** | **yes** | +|| organism-ensemble (qwen3.5 + qwen3.7 + tool) | 6/6 | 19 | $0.00676 | 3,692 | 6,776 | — | + +Price card: + +```json +{ + "alibaba/qwen3.5-flash": { "input": 0.065, "output": 0.26 }, + "alibaba/qwen3.7-flash": { "input": 0.03, "output": 0.13 }, + "anthropic/claude-opus-5": { "input": 5.00, "output": 25.00 } +} +``` + +The Qwen Flash organism reaches **100% accuracy** for **$0.00131** because it +retrieves the ledger. Claude Opus 5 without the ledger reaches only **67%** for +**$0.02625** — 20× more expensive and still wrong on the same evidence-only +cases. The single-Qwen baseline is also **67%**, but it costs **2.8× as much** +as the organism and gets the same cases wrong. The Pareto set keeps the +organism (best quality and best cost) and the single Qwen (fewest round-trips); +the frontier model is not on the efficient frontier at all. + +## What the numbers mean + +- **passed** is a strict canonical equality against the expected output. A + classifier that emits the right label but with the wrong capitalization is a + miss — Morphogen does not silently normalize outputs. +- **effect calls** counts every model call and tool call. The organism's extra + calls are the tool lookups and the model decisions that use them. +- **tokens in/out** are what the provider reported; scripted runs report zero, + which is why the Pareto set also considers effect-call count. +- **cost** is `tokensIn * inputPrice + tokensOut * outputPrice` per attribution + key, in USD. It is optional and comes from a `prices` map in the bench + config. Because it is derived from reported tokens, it is also checked during + `morphogen bench verify` when the price card is in the report. +- **Pareto** means no other system is at least as good on all three axes + (quality ↑, cost signal ↓, effect calls ↓) and strictly better on one. It is + a claim that survives `morphogen bench verify`. + +## What to do next + +1. Run `bun run cli suite` to see the bundled deterministic examples. +2. Run `bun run cli bench examples/invest/bench-invest.config.json \ + --tools examples/invest/bench-invest.tools.json \ + --dir .morphogen --out invest-report.json` to see the same workload + replayed deterministically. +3. Add the `prices` map to `examples/invest/bench-invest-priced.config.json` + (or use `examples/invest/bench-invest-live.config.json` for the live + gateway version) to reproduce the cost Pareto. diff --git a/docs/why-unique.md b/docs/why-unique.md new file mode 100644 index 0000000..724190e --- /dev/null +++ b/docs/why-unique.md @@ -0,0 +1,84 @@ +# Why Morphogen is a new primitive + +Most existing abstractions fall into one of two buckets: + +1. **Deterministic programs** — code, configs, workflows, smart contracts. +2. **Open-ended agents** — LLM agents, chatbots, copilots. + +Morphogen is a third thing: a **bounded, typed, content-addressed probabilistic program**. + +## What a Morphogen organism is + +An organism is not a script. It is a finite, strongly-typed graph where: + +- Every cell has declared input and output ports. +- Every edge is a typed wire with optional guards. +- Most cells are deterministic host functions. +- Some cells are bounded agent calls with a prompt, a context view, an output contract, and a budget. +- Model output is data. It is bound to a port before it re-enters the graph. +- The whole graph is a value. It hashes, transports, and verifies. +- A run emits a content-addressed receipt that can be replayed offline without the original provider. +- Failure is explicit and routable, not an exception the caller discovers. + +That combination is what makes it a new primitive, not just a nicer prompt chain. + +## How it differs from neighboring abstractions + +| Abstraction | What it does | Why Morphogen is different | +|---|---|---| +| **Prompt engineering** | Hand-tunes a string for a model | A manifest is a graph, not a prompt. The model only sees a bounded view of the graph. | +| **LLM agent loop (ReAct, etc.)** | Open-ended reasoning with tools | The organism loop is bounded, typed, and hashable. It can be *inside* an agent, not a replacement for it. | +| **Workflow / DAG engine** | Orchestrates deterministic steps | Effects are first-class, receipted, and replayable. Model calls are cells, not opaque black boxes. | +| **Probabilistic programming** | Samples and conditions | No sampler is in the language. Non-determinism is isolated to the executor. The graph itself is deterministic given receipts. | +| **Smart contract / zkVM** | Verifies computation by proof | Morphogen does not prove correctness. It proves *what was run and what was returned*, with everything content-addressed and replayable. | +| **Function as a Service** | Runs code on demand | A Morphogen organism is content-addressed, provider-agnostic, and emits a receipt. It is a verifiable function, not just a callable endpoint. | +| **Cellular automata** | Repeated local rules on a grid | Morphogen is a typed, heterogeneous, DAG-executed graph, not a grid. But it keeps the CA spirit: local, bounded, explicit state transitions. | + +## What the primitive enables + +### 1. Programs as values + +A manifest is a pure JSON document. Its canonical digest is its identity. You can: + +- Hash it. +- Ship it. +- Cache it. +- Compose it into another organism. +- Generate candidates in a foundry and select by Pareto. +- Register it as an agent tool. + +### 2. Receipts as evidence + +Every run produces a receipt that records every cell, every edge, every effect request, and every effect response. The receipt is also content-addressed. You can: + +- Replay it offline and get the same digest. +- Diff two receipts to find the exact cell where they diverge. +- Transport it to a third party who can verify it without trusting you or the original provider. + +### 3. Boundaries by construction + +Budgets and types are in the manifest, not the runtime's head. The runtime enforces: + +- maxSteps, maxAgentCalls, maxWork, maxDepth +- maxContextBytes, maxOutputBytes +- declared tool lists and turn budgets +- exact output contracts +- no ambient authority for agent cells + +This makes it safe to execute organisms you did not write. + +### 4. Generation and selection + +Because an organism is a value, a program can generate, evaluate, and select organisms: + +- `morphogen foundry` breeds candidates on a train set and promotes the Pareto winner. +- `morphogen search` evolves populations over generations with lineage tracking. +- The promoted organism is itself a value that can be packed, shipped, and called. + +## Where it is not the right tool + +Morphogen is not for open-ended conversation, exploratory research, or tasks where the structure itself is unknown. It is for subproblems where the shape of the work can be declared: classification, routing, extraction, verification, multi-step forms, code review gates, and tool-grounded investigations. + +## Why the name "organism" + +An organism is alive in a very bounded sense: it ingests inputs, performs work through typed cells, emits a waste-free receipt, and can reproduce (generate and evolve variants). But it has no open-ended autonomy, no persistent self-interest, and no ambient access to the world. It is a value that behaves, not an agent that wants. diff --git a/examples/bench-batch-cheap.responses.json b/examples/bench-batch-cheap.responses.json new file mode 100644 index 0000000..e59083c --- /dev/null +++ b/examples/bench-batch-cheap.responses.json @@ -0,0 +1,14 @@ +{ + "batch": [ + ["billing", "technical", "billing", "billing", "technical", "billing", "billing", "other"], + ["billing", "technical", "technical", "technical", "other", "technical", "billing", "other"] + ], + "route": [ + "billing", "technical", "technical", "billing", "technical", "billing", "technical", "other", + "billing", "technical", "billing", "technical", "other", "technical", "billing", "other" + ], + "cheap-a": [ + "billing", "technical", "billing", "billing", "technical", "billing", "technical", "other", + "billing", "technical", "billing", "technical", "other", "technical", "billing", "other" + ] +} diff --git a/examples/bench-batch-cheap2.responses.json b/examples/bench-batch-cheap2.responses.json new file mode 100644 index 0000000..3bf133b --- /dev/null +++ b/examples/bench-batch-cheap2.responses.json @@ -0,0 +1,6 @@ +{ + "cheap-b": [ + "billing", "technical", "technical", "billing", "technical", "billing", "technical", "other", + "billing", "technical", "technical", "technical", "other", "technical", "billing", "other" + ] +} diff --git a/examples/bench-batch-each.args.json b/examples/bench-batch-each.args.json new file mode 100644 index 0000000..f39bab2 --- /dev/null +++ b/examples/bench-batch-each.args.json @@ -0,0 +1 @@ +{ "src": { "tickets": ["charged twice for my subscription", "app crashes on export"] } } diff --git a/examples/bench-batch-each.morphogen.json b/examples/bench-batch-each.morphogen.json new file mode 100644 index 0000000..4922aa4 --- /dev/null +++ b/examples/bench-batch-each.morphogen.json @@ -0,0 +1,24 @@ +{ + "contract": "morphogen.organism.v1", + "key": "organism:bench-batch-each", + "name": "Bench batch — each", + "note": "The organism: each maps the ticket list through a focused per-item classifier organism. Many narrow judgments instead of one diluted call.", + "budgets": { "maxSteps": 24, "maxAgentCalls": 16, "maxWork": 200000 }, + "interface": { + "inputs": { "tickets": { "cell": "src", "port": "tickets" } }, + "outputs": { "out": { "cell": "map", "port": "label" } } + }, + "cells": [ + { "id": "src", "kind": "input", "outputs": { "tickets": "json" } }, + { + "id": "map", + "kind": "each", + "manifest": "sha256:96976c77cb240277c9e24f69aee2a4674c225de0f4f52edd35444322539f7e07", + "over": "ticket", + "maxItems": 16 + } + ], + "edges": [ + { "from": { "cell": "src", "port": "tickets" }, "to": { "cell": "map", "port": "ticket" } } + ] +} diff --git a/examples/bench-batch-each.responses.json b/examples/bench-batch-each.responses.json new file mode 100644 index 0000000..b9bd5d5 --- /dev/null +++ b/examples/bench-batch-each.responses.json @@ -0,0 +1 @@ +{ "route": ["billing", "technical"] } diff --git a/examples/bench-batch-ensemble.args.json b/examples/bench-batch-ensemble.args.json new file mode 100644 index 0000000..f39bab2 --- /dev/null +++ b/examples/bench-batch-ensemble.args.json @@ -0,0 +1 @@ +{ "src": { "tickets": ["charged twice for my subscription", "app crashes on export"] } } diff --git a/examples/bench-batch-ensemble.morphogen.json b/examples/bench-batch-ensemble.morphogen.json new file mode 100644 index 0000000..74e5de6 --- /dev/null +++ b/examples/bench-batch-ensemble.morphogen.json @@ -0,0 +1,24 @@ +{ + "contract": "morphogen.organism.v1", + "key": "organism:bench-batch-ensemble", + "name": "Bench batch — ensemble each", + "note": "The full organism: each maps the ticket list through the disagreement ensemble — two cheap lanes per item, frontier only where they disagree.", + "budgets": { "maxSteps": 64, "maxAgentCalls": 48, "maxWork": 400000 }, + "interface": { + "inputs": { "tickets": { "cell": "src", "port": "tickets" } }, + "outputs": { "out": { "cell": "map", "port": "out" } } + }, + "cells": [ + { "id": "src", "kind": "input", "outputs": { "tickets": "json" } }, + { + "id": "map", + "kind": "each", + "manifest": "sha256:b18e04fc01936054c820e129523132a080531d37b390ab25e532027f3106a945", + "over": "ticket", + "maxItems": 16 + } + ], + "edges": [ + { "from": { "cell": "src", "port": "tickets" }, "to": { "cell": "map", "port": "ticket" } } + ] +} diff --git a/examples/bench-batch-ensemble.responses.json b/examples/bench-batch-ensemble.responses.json new file mode 100644 index 0000000..6189884 --- /dev/null +++ b/examples/bench-batch-ensemble.responses.json @@ -0,0 +1 @@ +{ "cheap-a": ["billing", "billing"], "cheap-b": ["billing", "technical"], "escalate": ["technical"] } diff --git a/examples/bench-batch-frontier.responses.json b/examples/bench-batch-frontier.responses.json new file mode 100644 index 0000000..c85633a --- /dev/null +++ b/examples/bench-batch-frontier.responses.json @@ -0,0 +1,7 @@ +{ + "batch": [ + ["billing", "technical", "technical", "billing", "technical", "billing", "technical", "other"], + ["billing", "technical", "billing", "technical", "other", "technical", "billing", "other"] + ], + "escalate": ["technical", "billing"] +} diff --git a/examples/bench-batch-live.config.json b/examples/bench-batch-live.config.json new file mode 100644 index 0000000..85c5f1c --- /dev/null +++ b/examples/bench-batch-live.config.json @@ -0,0 +1,67 @@ +{ + "contract": "morphogen.bench.config.v1", + "cases": [ + { + "id": "b1", + "args": { + "tickets": [ + "charged twice for my subscription", + "app crashes on export", + "invoice totals look wrong after the update", + "where is my receipt for last month", + "SSO login loops back to the sign-in page", + "can I change my billing email", + "webhook deliveries stopped since Tuesday", + "do you offer nonprofit pricing" + ] + }, + "expect": { + "out": ["billing", "technical", "technical", "billing", "technical", "billing", "technical", "other"] + } + }, + { + "id": "b2", + "args": { + "tickets": [ + "refund a duplicate annual charge", + "dark mode setting does not persist", + "upgrade seats mid-cycle proration question", + "API returns 500 on batch create", + "your ad said free migration", + "password reset email never arrives", + "sales tax missing from my quote", + "is there a student discount" + ] + }, + "expect": { + "out": ["billing", "technical", "billing", "technical", "other", "technical", "billing", "other"] + } + } + ], + "systems": [ + { + "id": "cheap-single", + "manifest": "bench-batch-single.morphogen.json", + "executors": { "cheap": "gateway:alibaba/qwen3.5-flash" } + }, + { + "id": "frontier-single", + "manifest": "bench-batch-single.morphogen.json", + "executors": { "frontier": "gateway:anthropic/claude-opus-5" } + }, + { + "id": "each-cheap", + "manifest": "bench-batch-each.morphogen.json", + "executors": { "cheap": "gateway:alibaba/qwen3.5-flash" } + }, + { + "id": "each-ensemble", + "manifest": "bench-batch-ensemble.morphogen.json", + "executors": { + "cheap": "gateway:alibaba/qwen3.5-flash", + "cheap2": "gateway:alibaba/qwen3.7-flash", + "frontier": "gateway:anthropic/claude-opus-5" + } + } + ] +} diff --git a/examples/bench-batch-single.args.json b/examples/bench-batch-single.args.json new file mode 100644 index 0000000..f39bab2 --- /dev/null +++ b/examples/bench-batch-single.args.json @@ -0,0 +1 @@ +{ "src": { "tickets": ["charged twice for my subscription", "app crashes on export"] } } diff --git a/examples/bench-batch-single.morphogen.json b/examples/bench-batch-single.morphogen.json new file mode 100644 index 0000000..ad6d42b --- /dev/null +++ b/examples/bench-batch-single.morphogen.json @@ -0,0 +1,32 @@ +{ + "contract": "morphogen.organism.v1", + "key": "organism:bench-batch-single", + "name": "Bench batch — one call", + "note": "Baseline: a single model call labels the whole ticket list. One diluted judgment versus the each organism's focused per-item pass.", + "budgets": { "maxSteps": 4, "maxAgentCalls": 1, "maxWork": 100000 }, + "interface": { + "inputs": { "tickets": { "cell": "src", "port": "tickets" } }, + "outputs": { "out": { "cell": "batch", "port": "out" } } + }, + "cells": [ + { "id": "src", "kind": "input", "outputs": { "tickets": "json" } }, + { + "id": "batch", + "kind": "agent", + "inputs": { "tickets": "json" }, + "prompt": "Classify each support ticket in the list by its primary operational destination: billing, technical, or other. Return a JSON array of labels in the same order and length as the input list.", + "view": { "inputs": ["tickets"] }, + "output": { + "kind": "json", + "schema": { + "type": "array", + "items": { "type": "string", "enum": ["billing", "technical", "other"] } + } + }, + "budget": { "maxContextBytes": 16384, "maxOutputBytes": 8192, "maxEffectMs": 120000 } + } + ], + "edges": [ + { "from": { "cell": "src", "port": "tickets" }, "to": { "cell": "batch", "port": "tickets" } } + ] +} diff --git a/examples/bench-batch-single.responses.json b/examples/bench-batch-single.responses.json new file mode 100644 index 0000000..f26751b --- /dev/null +++ b/examples/bench-batch-single.responses.json @@ -0,0 +1 @@ +{ "batch": [["billing", "technical"]] } diff --git a/examples/bench-batch.config.json b/examples/bench-batch.config.json new file mode 100644 index 0000000..3f8bbc7 --- /dev/null +++ b/examples/bench-batch.config.json @@ -0,0 +1,67 @@ +{ + "contract": "morphogen.bench.config.v1", + "cases": [ + { + "id": "b1", + "args": { + "tickets": [ + "charged twice for my subscription", + "app crashes on export", + "invoice totals look wrong after the update", + "where is my receipt for last month", + "SSO login loops back to the sign-in page", + "can I change my billing email", + "webhook deliveries stopped since Tuesday", + "do you offer nonprofit pricing" + ] + }, + "expect": { + "out": ["billing", "technical", "technical", "billing", "technical", "billing", "technical", "other"] + } + }, + { + "id": "b2", + "args": { + "tickets": [ + "refund a duplicate annual charge", + "dark mode setting does not persist", + "upgrade seats mid-cycle proration question", + "API returns 500 on batch create", + "your ad said free migration", + "password reset email never arrives", + "sales tax missing from my quote", + "is there a student discount" + ] + }, + "expect": { + "out": ["billing", "technical", "billing", "technical", "other", "technical", "billing", "other"] + } + } + ], + "systems": [ + { + "id": "cheap-single", + "manifest": "bench-batch-single.morphogen.json", + "executors": { "cheap": "scripted:bench-batch-cheap.responses.json" } + }, + { + "id": "frontier-single", + "manifest": "bench-batch-single.morphogen.json", + "executors": { "frontier": "scripted:bench-batch-frontier.responses.json" } + }, + { + "id": "each-cheap", + "manifest": "bench-batch-each.morphogen.json", + "executors": { "cheap": "scripted:bench-batch-cheap.responses.json" } + }, + { + "id": "each-ensemble", + "manifest": "bench-batch-ensemble.morphogen.json", + "executors": { + "cheap": "scripted:bench-batch-cheap.responses.json", + "cheap2": "scripted:bench-batch-cheap2.responses.json", + "frontier": "scripted:bench-batch-frontier.responses.json" + } + } + ] +} diff --git a/examples/bench-cheap.responses.json b/examples/bench-cheap.responses.json new file mode 100644 index 0000000..b910f1f --- /dev/null +++ b/examples/bench-cheap.responses.json @@ -0,0 +1,6 @@ +{ + "route": ["billing", "technical", "billing", "other"], + "cheap": ["billing", "technical", "billing", "unsure"], + "cheap-a": ["billing", "technical", "billing", "billing"] +} + diff --git a/examples/bench-cheap2.responses.json b/examples/bench-cheap2.responses.json new file mode 100644 index 0000000..2b4534f --- /dev/null +++ b/examples/bench-cheap2.responses.json @@ -0,0 +1,3 @@ +{ + "cheap-b": ["billing", "technical", "billing", "technical"] +} diff --git a/examples/bench-frontier.responses.json b/examples/bench-frontier.responses.json new file mode 100644 index 0000000..5d30f6a --- /dev/null +++ b/examples/bench-frontier.responses.json @@ -0,0 +1,4 @@ +{ + "route": ["billing", "technical", "billing", "technical"], + "escalate": ["technical"] +} diff --git a/examples/bench-item.args.json b/examples/bench-item.args.json new file mode 100644 index 0000000..c6d8cf7 --- /dev/null +++ b/examples/bench-item.args.json @@ -0,0 +1 @@ +{ "src": { "ticket": "charged twice for my subscription" } } diff --git a/examples/bench-item.morphogen.json b/examples/bench-item.morphogen.json new file mode 100644 index 0000000..7473357 --- /dev/null +++ b/examples/bench-item.morphogen.json @@ -0,0 +1,26 @@ +{ + "contract": "morphogen.organism.v1", + "key": "organism:bench-item", + "name": "Bench item — one ticket", + "note": "The per-element organism for each cells in the batch bench: one ticket in, one routing label out. Focused context per item instead of one diluted call over a whole list.", + "budgets": { "maxSteps": 4, "maxAgentCalls": 1, "maxWork": 100000 }, + "interface": { + "inputs": { "ticket": { "cell": "src", "port": "ticket" } }, + "outputs": { "label": { "cell": "route", "port": "out" } } + }, + "cells": [ + { "id": "src", "kind": "input", "outputs": { "ticket": "text" } }, + { + "id": "route", + "kind": "classifier", + "inputs": { "ticket": "text" }, + "prompt": "Classify the support ticket by its primary operational destination: billing, technical, or other.", + "view": { "inputs": ["ticket"] }, + "output": { "kind": "choice", "labels": ["billing", "technical", "other"] }, + "budget": { "maxContextBytes": 4096, "maxOutputBytes": 8192, "maxEffectMs": 120000 } + } + ], + "edges": [ + { "from": { "cell": "src", "port": "ticket" }, "to": { "cell": "route", "port": "ticket" } } + ] +} diff --git a/examples/bench-item.responses.json b/examples/bench-item.responses.json new file mode 100644 index 0000000..fc92cd9 --- /dev/null +++ b/examples/bench-item.responses.json @@ -0,0 +1 @@ +{ "route": "billing" } diff --git a/examples/bench-live.config.json b/examples/bench-live.config.json new file mode 100644 index 0000000..53ffbcc --- /dev/null +++ b/examples/bench-live.config.json @@ -0,0 +1,54 @@ +{ + "contract": "morphogen.bench.config.v1", + "cases": [ + { + "id": "t1", + "args": { "ticket": "charged twice for my subscription" }, + "expect": { "out": "billing" } + }, + { + "id": "t2", + "args": { "ticket": "app crashes on export" }, + "expect": { "out": "technical" } + }, + { + "id": "t3", + "args": { "ticket": "refund for a cancelled plan" }, + "expect": { "out": "billing" } + }, + { + "id": "t4", + "args": { "ticket": "invoice totals look wrong after the update" }, + "expect": { "out": "technical" } + } + ], + "systems": [ + { + "id": "cheap-single", + "manifest": "bench-triage-single.morphogen.json", + "executors": { "cheap": "gateway:alibaba/qwen3.5-flash" } + }, + { + "id": "frontier-single", + "manifest": "bench-triage-single.morphogen.json", + "executors": { "frontier": "gateway:anthropic/claude-opus-5" } + }, + { + "id": "circuit", + "manifest": "bench-triage-circuit.morphogen.json", + "executors": { + "cheap": "gateway:alibaba/qwen3.5-flash", + "frontier": "gateway:anthropic/claude-opus-5" + } + }, + { + "id": "ensemble", + "manifest": "bench-triage-ensemble.morphogen.json", + "executors": { + "cheap": "gateway:alibaba/qwen3.5-flash", + "cheap2": "gateway:alibaba/qwen3.7-flash", + "frontier": "gateway:anthropic/claude-opus-5" + } + } + ] +} diff --git a/examples/bench-triage-circuit.args.json b/examples/bench-triage-circuit.args.json new file mode 100644 index 0000000..03704f8 --- /dev/null +++ b/examples/bench-triage-circuit.args.json @@ -0,0 +1 @@ +{ "src": { "ticket": "invoice totals look wrong after the update" } } diff --git a/examples/bench-triage-circuit.morphogen.json b/examples/bench-triage-circuit.morphogen.json new file mode 100644 index 0000000..1be8b5b --- /dev/null +++ b/examples/bench-triage-circuit.morphogen.json @@ -0,0 +1,49 @@ +{ + "contract": "morphogen.organism.v1", + "key": "organism:bench-triage-circuit", + "name": "Bench triage — cheap-first cascade", + "note": "The organism: a cheap classifier may abstain with \"unsure\"; only then does the escalate cell (routed at the frontier preset) fire, and coalesce prefers its answer. Frontier inference is a guarded branch, not the default path.", + "budgets": { "maxSteps": 8, "maxAgentCalls": 4, "maxWork": 100000 }, + "interface": { + "inputs": { "ticket": { "cell": "src", "port": "ticket" } }, + "outputs": { "out": { "cell": "merge", "port": "value" } } + }, + "cells": [ + { "id": "src", "kind": "input", "outputs": { "ticket": "text" } }, + { + "id": "cheap", + "kind": "classifier", + "inputs": { "ticket": "text" }, + "prompt": "Classify the support ticket: billing, technical, or other. When the routing is genuinely ambiguous, abstain with unsure.", + "view": { "inputs": ["ticket"] }, + "output": { "kind": "choice", "labels": ["billing", "technical", "other", "unsure"] }, + "route": { "preset": "cheap" }, + "budget": { "maxContextBytes": 4096, "maxOutputBytes": 256, "maxEffectMs": 120000 } + }, + { + "id": "escalate", + "kind": "classifier", + "inputs": { + "ticket": "text", + "trigger": { "type": "choice", "labels": ["billing", "technical", "other", "unsure"] } + }, + "prompt": "Classify the support ticket a cheaper pass declined to route: billing, technical, or other.", + "view": { "inputs": ["ticket"] }, + "output": { "kind": "choice", "labels": ["billing", "technical", "other"] }, + "route": { "preset": "frontier" }, + "budget": { "maxContextBytes": 4096, "maxOutputBytes": 256, "maxEffectMs": 120000 } + }, + { "id": "merge", "kind": "fn", "fn": "coalesce.v1" } + ], + "edges": [ + { "from": { "cell": "src", "port": "ticket" }, "to": { "cell": "cheap", "port": "ticket" } }, + { "from": { "cell": "src", "port": "ticket" }, "to": { "cell": "escalate", "port": "ticket" } }, + { + "from": { "cell": "cheap", "port": "out" }, + "to": { "cell": "escalate", "port": "trigger" }, + "guard": { "equals": "unsure" } + }, + { "from": { "cell": "escalate", "port": "out" }, "to": { "cell": "merge", "port": "a" } }, + { "from": { "cell": "cheap", "port": "out" }, "to": { "cell": "merge", "port": "b" } } + ] +} diff --git a/examples/bench-triage-circuit.responses.json b/examples/bench-triage-circuit.responses.json new file mode 100644 index 0000000..63b45ef --- /dev/null +++ b/examples/bench-triage-circuit.responses.json @@ -0,0 +1 @@ +{ "cheap": "unsure", "escalate": "technical" } diff --git a/examples/bench-triage-ensemble.args.json b/examples/bench-triage-ensemble.args.json new file mode 100644 index 0000000..03704f8 --- /dev/null +++ b/examples/bench-triage-ensemble.args.json @@ -0,0 +1 @@ +{ "src": { "ticket": "invoice totals look wrong after the update" } } diff --git a/examples/bench-triage-ensemble.morphogen.json b/examples/bench-triage-ensemble.morphogen.json new file mode 100644 index 0000000..69ec29b --- /dev/null +++ b/examples/bench-triage-ensemble.morphogen.json @@ -0,0 +1,56 @@ +{ + "contract": "morphogen.organism.v1", + "key": "organism:bench-triage-ensemble", + "name": "Bench triage — ensemble cascade", + "note": "Uncertainty as structure: two decorrelated cheap classifiers vote; a deterministic assert compares them; only disagreement fires the frontier cell through an on:fail edge. Escalation needs no self-reported confidence — disagreement is the signal.", + "budgets": { "maxSteps": 10, "maxAgentCalls": 6, "maxWork": 100000 }, + "interface": { + "inputs": { "ticket": { "cell": "src", "port": "ticket" } }, + "outputs": { "out": { "cell": "merge", "port": "value" } } + }, + "cells": [ + { "id": "src", "kind": "input", "outputs": { "ticket": "text" } }, + { + "id": "cheap-a", + "kind": "classifier", + "inputs": { "ticket": "text" }, + "prompt": "Classify the support ticket by its primary operational destination: billing, technical, or other.", + "view": { "inputs": ["ticket"] }, + "output": { "kind": "choice", "labels": ["billing", "technical", "other"] }, + "route": { "preset": "cheap" }, + "budget": { "maxContextBytes": 4096, "maxOutputBytes": 256, "maxEffectMs": 120000 } + }, + { + "id": "cheap-b", + "kind": "classifier", + "inputs": { "ticket": "text" }, + "prompt": "Decide which team should own this support ticket: billing, technical, or other.", + "view": { "inputs": ["ticket"] }, + "output": { "kind": "choice", "labels": ["billing", "technical", "other"] }, + "route": { "preset": "cheap2" }, + "budget": { "maxContextBytes": 4096, "maxOutputBytes": 256, "maxEffectMs": 120000 } + }, + { "id": "agree", "kind": "fn", "fn": "assert.v1" }, + { + "id": "escalate", + "kind": "classifier", + "inputs": { "ticket": "text", "trigger": "json" }, + "prompt": "Two cheap classifiers disagreed on this ticket. Route it definitively: billing, technical, or other.", + "view": { "inputs": ["ticket"] }, + "output": { "kind": "choice", "labels": ["billing", "technical", "other"] }, + "route": { "preset": "frontier" }, + "budget": { "maxContextBytes": 4096, "maxOutputBytes": 256, "maxEffectMs": 120000 } + }, + { "id": "merge", "kind": "fn", "fn": "coalesce.v1" } + ], + "edges": [ + { "from": { "cell": "src", "port": "ticket" }, "to": { "cell": "cheap-a", "port": "ticket" } }, + { "from": { "cell": "src", "port": "ticket" }, "to": { "cell": "cheap-b", "port": "ticket" } }, + { "from": { "cell": "src", "port": "ticket" }, "to": { "cell": "escalate", "port": "ticket" } }, + { "from": { "cell": "cheap-a", "port": "out" }, "to": { "cell": "agree", "port": "value" } }, + { "from": { "cell": "cheap-b", "port": "out" }, "to": { "cell": "agree", "port": "expect" } }, + { "from": { "cell": "agree", "port": "value" }, "to": { "cell": "escalate", "port": "trigger" }, "on": "fail" }, + { "from": { "cell": "escalate", "port": "out" }, "to": { "cell": "merge", "port": "a" } }, + { "from": { "cell": "cheap-a", "port": "out" }, "to": { "cell": "merge", "port": "b" } } + ] +} diff --git a/examples/bench-triage-ensemble.responses.json b/examples/bench-triage-ensemble.responses.json new file mode 100644 index 0000000..207d679 --- /dev/null +++ b/examples/bench-triage-ensemble.responses.json @@ -0,0 +1 @@ +{ "cheap-a": "billing", "cheap-b": "technical", "escalate": "technical" } diff --git a/examples/bench-triage-single.args.json b/examples/bench-triage-single.args.json new file mode 100644 index 0000000..c6d8cf7 --- /dev/null +++ b/examples/bench-triage-single.args.json @@ -0,0 +1 @@ +{ "src": { "ticket": "charged twice for my subscription" } } diff --git a/examples/bench-triage-single.morphogen.json b/examples/bench-triage-single.morphogen.json new file mode 100644 index 0000000..b3fa2d9 --- /dev/null +++ b/examples/bench-triage-single.morphogen.json @@ -0,0 +1,26 @@ +{ + "contract": "morphogen.organism.v1", + "key": "organism:bench-triage-single", + "name": "Bench triage — single call", + "note": "The baseline: one classifier call routes the ticket. Used by bench.config.json to compare a lone cheap model, a lone frontier model, and the circuit.", + "budgets": { "maxSteps": 4, "maxAgentCalls": 1, "maxWork": 100000 }, + "interface": { + "inputs": { "ticket": { "cell": "src", "port": "ticket" } }, + "outputs": { "out": { "cell": "route", "port": "out" } } + }, + "cells": [ + { "id": "src", "kind": "input", "outputs": { "ticket": "text" } }, + { + "id": "route", + "kind": "classifier", + "inputs": { "ticket": "text" }, + "prompt": "Classify the support ticket by its primary operational destination: billing, technical, or other.", + "view": { "inputs": ["ticket"] }, + "output": { "kind": "choice", "labels": ["billing", "technical", "other"] }, + "budget": { "maxContextBytes": 4096, "maxOutputBytes": 256, "maxEffectMs": 120000 } + } + ], + "edges": [ + { "from": { "cell": "src", "port": "ticket" }, "to": { "cell": "route", "port": "ticket" } } + ] +} diff --git a/examples/bench-triage-single.responses.json b/examples/bench-triage-single.responses.json new file mode 100644 index 0000000..fc92cd9 --- /dev/null +++ b/examples/bench-triage-single.responses.json @@ -0,0 +1 @@ +{ "route": "billing" } diff --git a/examples/bench.config.json b/examples/bench.config.json new file mode 100644 index 0000000..71ace01 --- /dev/null +++ b/examples/bench.config.json @@ -0,0 +1,54 @@ +{ + "contract": "morphogen.bench.config.v1", + "cases": [ + { + "id": "t1", + "args": { "ticket": "charged twice for my subscription" }, + "expect": { "out": "billing" } + }, + { + "id": "t2", + "args": { "ticket": "app crashes on export" }, + "expect": { "out": "technical" } + }, + { + "id": "t3", + "args": { "ticket": "refund for a cancelled plan" }, + "expect": { "out": "billing" } + }, + { + "id": "t4", + "args": { "ticket": "invoice totals look wrong after the update" }, + "expect": { "out": "technical" } + } + ], + "systems": [ + { + "id": "cheap-single", + "manifest": "bench-triage-single.morphogen.json", + "executors": { "cheap": "scripted:bench-cheap.responses.json" } + }, + { + "id": "frontier-single", + "manifest": "bench-triage-single.morphogen.json", + "executors": { "frontier": "scripted:bench-frontier.responses.json" } + }, + { + "id": "circuit", + "manifest": "bench-triage-circuit.morphogen.json", + "executors": { + "cheap": "scripted:bench-cheap.responses.json", + "frontier": "scripted:bench-frontier.responses.json" + } + }, + { + "id": "ensemble", + "manifest": "bench-triage-ensemble.morphogen.json", + "executors": { + "cheap": "scripted:bench-cheap.responses.json", + "cheap2": "scripted:bench-cheap2.responses.json", + "frontier": "scripted:bench-frontier.responses.json" + } + } + ] +} diff --git a/examples/evolving-generator.morphogen.json b/examples/evolving-generator.morphogen.json new file mode 100644 index 0000000..ce54924 --- /dev/null +++ b/examples/evolving-generator.morphogen.json @@ -0,0 +1,36 @@ +{ + "contract": "morphogen.organism.v1", + "key": "organism:evolving-generator", + "name": "Validation-guided candidate generator", + "budgets": { "maxSteps": 8, "maxAgentCalls": 1, "maxWork": 100000 }, + "interface": { + "inputs": { + "task": { "cell": "src", "port": "task" }, + "feedback": { "cell": "src", "port": "feedback" } + }, + "outputs": { "candidates": { "cell": "writer", "port": "out" } } + }, + "cells": [ + { "id": "src", "kind": "input", "outputs": { "task": "text", "feedback": "json" } }, + { + "id": "writer", + "kind": "agent", + "inputs": { "task": "text", "feedback": "json" }, + "prompt": "Propose a bounded candidate manifest list. Use prior validation evidence to improve the population.", + "view": { "inputs": ["task", "feedback"] }, + "output": { + "kind": "json", + "schema": { + "type": "object", + "required": ["candidates"], + "properties": { "candidates": { "type": "array" } } + } + }, + "budget": { "maxContextBytes": 16384, "maxOutputBytes": 32768 } + } + ], + "edges": [ + { "from": { "cell": "src", "port": "task" }, "to": { "cell": "writer", "port": "task" } }, + { "from": { "cell": "src", "port": "feedback" }, "to": { "cell": "writer", "port": "feedback" } } + ] +} diff --git a/examples/evolving-generator.responses.json b/examples/evolving-generator.responses.json new file mode 100644 index 0000000..278466a --- /dev/null +++ b/examples/evolving-generator.responses.json @@ -0,0 +1,38 @@ +{ + "writer": [ + { + "candidates": [{ + "contract": "morphogen.organism.v1", + "key": "organism:evolved-constant", + "name": "Generation zero constant", + "interface": { + "inputs": { "q": { "cell": "src", "port": "value" } }, + "outputs": { "answer": { "cell": "out", "port": "value" } } + }, + "cells": [ + { "id": "src", "kind": "input", "outputs": { "value": "json" } }, + { "id": "out", "kind": "const", "outputs": { "value": { "type": "json", "value": "alpha" } } } + ], + "edges": [] + }] + }, + { + "candidates": [{ + "contract": "morphogen.organism.v1", + "key": "organism:evolved-echo", + "name": "Generation one echo", + "interface": { + "inputs": { "q": { "cell": "src", "port": "value" } }, + "outputs": { "answer": { "cell": "echo", "port": "value" } } + }, + "cells": [ + { "id": "src", "kind": "input", "outputs": { "value": "json" } }, + { "id": "echo", "kind": "fn", "fn": "echo.v1" } + ], + "edges": [ + { "from": { "cell": "src", "port": "value" }, "to": { "cell": "echo", "port": "value" } } + ] + }] + } + ] +} diff --git a/examples/foundry-constant.morphogen.json b/examples/foundry-constant.morphogen.json new file mode 100644 index 0000000..eb5a091 --- /dev/null +++ b/examples/foundry-constant.morphogen.json @@ -0,0 +1,14 @@ +{ + "contract": "morphogen.organism.v1", + "key": "organism:foundry-constant", + "name": "Foundry constant candidate", + "interface": { + "inputs": { "q": { "cell": "src", "port": "value" } }, + "outputs": { "answer": { "cell": "out", "port": "value" } } + }, + "cells": [ + { "id": "src", "kind": "input", "outputs": { "value": "json" } }, + { "id": "out", "kind": "const", "outputs": { "value": { "type": "json", "value": "alpha" } } } + ], + "edges": [] +} diff --git a/examples/foundry-echo.morphogen.json b/examples/foundry-echo.morphogen.json new file mode 100644 index 0000000..edfb985 --- /dev/null +++ b/examples/foundry-echo.morphogen.json @@ -0,0 +1,16 @@ +{ + "contract": "morphogen.organism.v1", + "key": "organism:foundry-echo", + "name": "Foundry echo candidate", + "interface": { + "inputs": { "q": { "cell": "src", "port": "value" } }, + "outputs": { "answer": { "cell": "echo", "port": "value" } } + }, + "cells": [ + { "id": "src", "kind": "input", "outputs": { "value": "json" } }, + { "id": "echo", "kind": "fn", "fn": "echo.v1" } + ], + "edges": [ + { "from": { "cell": "src", "port": "value" }, "to": { "cell": "echo", "port": "value" } } + ] +} diff --git a/examples/foundry-generator.morphogen.json b/examples/foundry-generator.morphogen.json new file mode 100644 index 0000000..a4803d6 --- /dev/null +++ b/examples/foundry-generator.morphogen.json @@ -0,0 +1,32 @@ +{ + "contract": "morphogen.organism.v1", + "key": "organism:foundry-generator", + "name": "Foundry candidate generator", + "budgets": { "maxSteps": 8, "maxAgentCalls": 1, "maxWork": 100000 }, + "interface": { + "inputs": { "task": { "cell": "src", "port": "task" } }, + "outputs": { "candidates": { "cell": "writer", "port": "out" } } + }, + "cells": [ + { "id": "src", "kind": "input", "outputs": { "task": "text" } }, + { + "id": "writer", + "kind": "agent", + "inputs": { "task": "text" }, + "prompt": "Emit a bounded list of morphogen.organism.v1 candidate manifests for the task.", + "view": { "inputs": ["task"] }, + "output": { + "kind": "json", + "schema": { + "type": "object", + "required": ["candidates"], + "properties": { "candidates": { "type": "array" } } + } + }, + "budget": { "maxContextBytes": 4096, "maxOutputBytes": 32768 } + } + ], + "edges": [ + { "from": { "cell": "src", "port": "task" }, "to": { "cell": "writer", "port": "task" } } + ] +} diff --git a/examples/foundry-generator.responses.json b/examples/foundry-generator.responses.json new file mode 100644 index 0000000..769db93 --- /dev/null +++ b/examples/foundry-generator.responses.json @@ -0,0 +1,36 @@ +{ + "writer": { + "candidates": [ + { + "contract": "morphogen.organism.v1", + "key": "organism:generated-constant", + "name": "Generated constant candidate", + "interface": { + "inputs": { "q": { "cell": "src", "port": "value" } }, + "outputs": { "answer": { "cell": "out", "port": "value" } } + }, + "cells": [ + { "id": "src", "kind": "input", "outputs": { "value": "json" } }, + { "id": "out", "kind": "const", "outputs": { "value": { "type": "json", "value": "alpha" } } } + ], + "edges": [] + }, + { + "contract": "morphogen.organism.v1", + "key": "organism:generated-echo", + "name": "Generated echo candidate", + "interface": { + "inputs": { "q": { "cell": "src", "port": "value" } }, + "outputs": { "answer": { "cell": "echo", "port": "value" } } + }, + "cells": [ + { "id": "src", "kind": "input", "outputs": { "value": "json" } }, + { "id": "echo", "kind": "fn", "fn": "echo.v1" } + ], + "edges": [ + { "from": { "cell": "src", "port": "value" }, "to": { "cell": "echo", "port": "value" } } + ] + } + ] + } +} diff --git a/examples/foundry.config.json b/examples/foundry.config.json new file mode 100644 index 0000000..04d3a12 --- /dev/null +++ b/examples/foundry.config.json @@ -0,0 +1,33 @@ +{ + "contract": "morphogen.foundry.config.v1", + "candidates": [ + "foundry-constant.morphogen.json", + "foundry-echo.morphogen.json" + ], + "cases": [ + { + "id": "train-alpha", + "split": "train", + "args": { "q": "alpha" }, + "expect": { "answer": "alpha" } + }, + { + "id": "train-beta", + "split": "train", + "args": { "q": "beta" }, + "expect": { "answer": "beta" } + }, + { + "id": "validation-gamma", + "split": "validation", + "args": { "q": "gamma" }, + "expect": { "answer": "gamma" } + }, + { + "id": "holdout-delta", + "split": "holdout", + "args": { "q": "delta" }, + "expect": { "answer": "delta" } + } + ] +} diff --git a/examples/gateway-smoke.args.json b/examples/gateway-smoke.args.json new file mode 100644 index 0000000..77b3fb4 --- /dev/null +++ b/examples/gateway-smoke.args.json @@ -0,0 +1,5 @@ +{ + "src": { + "ticket": "I was charged twice for the same subscription renewal and need one charge refunded." + } +} diff --git a/examples/gateway-smoke.morphogen.json b/examples/gateway-smoke.morphogen.json new file mode 100644 index 0000000..a41c450 --- /dev/null +++ b/examples/gateway-smoke.morphogen.json @@ -0,0 +1,25 @@ +{ + "contract": "morphogen.organism.v1", + "key": "organism:gateway-smoke", + "name": "Gateway smoke", + "budgets": { "maxSteps": 4, "maxAgentCalls": 1, "maxWork": 100000 }, + "interface": { + "inputs": { "ticket": { "cell": "src", "port": "ticket" } }, + "outputs": { "route": { "cell": "route", "port": "out" } } + }, + "cells": [ + { "id": "src", "kind": "input", "outputs": { "ticket": "text" } }, + { + "id": "route", + "kind": "classifier", + "inputs": { "ticket": "text" }, + "prompt": "Classify the support ticket by its primary operational destination.", + "view": { "inputs": ["ticket"] }, + "output": { "kind": "choice", "labels": ["billing", "technical", "other"] }, + "budget": { "maxContextBytes": 4096, "maxOutputBytes": 256, "maxEffectMs": 120000 } + } + ], + "edges": [ + { "from": { "cell": "src", "port": "ticket" }, "to": { "cell": "route", "port": "ticket" } } + ] +} diff --git a/examples/gateway-smoke.responses.json b/examples/gateway-smoke.responses.json new file mode 100644 index 0000000..fc92cd9 --- /dev/null +++ b/examples/gateway-smoke.responses.json @@ -0,0 +1 @@ +{ "route": "billing" } diff --git a/examples/generated-foundry.config.json b/examples/generated-foundry.config.json new file mode 100644 index 0000000..cf08e88 --- /dev/null +++ b/examples/generated-foundry.config.json @@ -0,0 +1,29 @@ +{ + "contract": "morphogen.foundry.config.v1", + "generator": { + "manifest": "foundry-generator.morphogen.json", + "args": { "task": "Return the input value unchanged." }, + "output": "candidates", + "field": "candidates" + }, + "cases": [ + { + "id": "train-alpha", + "split": "train", + "args": { "q": "alpha" }, + "expect": { "answer": "alpha" } + }, + { + "id": "validation-beta", + "split": "validation", + "args": { "q": "beta" }, + "expect": { "answer": "beta" } + }, + { + "id": "holdout-gamma", + "split": "holdout", + "args": { "q": "gamma" }, + "expect": { "answer": "gamma" } + } + ] +} diff --git a/examples/habitat.args.json b/examples/habitat.args.json new file mode 100644 index 0000000..6a53041 --- /dev/null +++ b/examples/habitat.args.json @@ -0,0 +1,3 @@ +{ + "goal": { "text": "hello" } +} diff --git a/examples/habitat.morphogen.json b/examples/habitat.morphogen.json new file mode 100644 index 0000000..b099f88 --- /dev/null +++ b/examples/habitat.morphogen.json @@ -0,0 +1,54 @@ +{ + "contract": "morphogen.organism.v1", + "key": "organism:habitat-propose", + "name": "HabitatPropose", + "note": "A parent organism proposes a child manifest, spawns it, and records the child digest in a durable population slot. The child is data, not code; the host still decides whether to admit or promote it.", + "budgets": { + "maxSteps": 32, + "maxAgentCalls": 4, + "maxWork": 10000 + }, + "interface": { + "inputs": { "goal": { "cell": "goal", "port": "text" } }, + "outputs": { + "child": { "cell": "run", "port": "digest" }, + "population": { "cell": "record", "port": "data" } + } + }, + "cells": [ + { "id": "goal", "kind": "input", "outputs": { "text": "text" } }, + { + "id": "history", + "kind": "slot", + "name": "population", + "mode": "read", + "default": [] + }, + { + "id": "designer", + "kind": "agent", + "inputs": { "goal": "text" }, + "prompt": "Design a tiny child organism that exposes an 'out' output containing the goal string.", + "output": { "kind": "json", "schema": { "type": "object" } } + }, + { "id": "run", "kind": "spawn" }, + { + "id": "append", + "kind": "fn", + "fn": "push.v1" + }, + { + "id": "record", + "kind": "slot", + "name": "population", + "mode": "write" + } + ], + "edges": [ + { "from": { "cell": "goal", "port": "text" }, "to": { "cell": "designer", "port": "goal" } }, + { "from": { "cell": "designer", "port": "out" }, "to": { "cell": "run", "port": "manifest" } }, + { "from": { "cell": "run", "port": "digest" }, "to": { "cell": "append", "port": "item" } }, + { "from": { "cell": "history", "port": "data" }, "to": { "cell": "append", "port": "list" } }, + { "from": { "cell": "append", "port": "value" }, "to": { "cell": "record", "port": "data" } } + ] +} diff --git a/examples/habitat.responses.json b/examples/habitat.responses.json new file mode 100644 index 0000000..728b639 --- /dev/null +++ b/examples/habitat.responses.json @@ -0,0 +1,19 @@ +{ + "designer": { + "contract": "morphogen.organism.v1", + "key": "organism:echo-hello", + "name": "EchoHello", + "budgets": { + "maxSteps": 8, + "maxAgentCalls": 2, + "maxWork": 1000 + }, + "interface": { + "outputs": { "out": { "cell": "msg", "port": "v" } } + }, + "cells": [ + { "id": "msg", "kind": "const", "outputs": { "v": { "type": "text", "value": "hello" } } } + ], + "edges": [] + } +} diff --git a/examples/habitat/fallback.morphogen.json b/examples/habitat/fallback.morphogen.json new file mode 100644 index 0000000..38226da --- /dev/null +++ b/examples/habitat/fallback.morphogen.json @@ -0,0 +1,17 @@ +{ + "contract": "morphogen.organism.v1", + "key": "organism:echo-hello", + "name": "EchoHello", + "budgets": { + "maxSteps": 8, + "maxAgentCalls": 1, + "maxWork": 100 + }, + "interface": { + "outputs": { "out": { "cell": "msg", "port": "v" } } + }, + "cells": [ + { "id": "msg", "kind": "const", "outputs": { "v": { "type": "text", "value": "hello" } } } + ], + "edges": [] +} diff --git a/examples/habitat/live.args.json b/examples/habitat/live.args.json new file mode 100644 index 0000000..6a53041 --- /dev/null +++ b/examples/habitat/live.args.json @@ -0,0 +1,3 @@ +{ + "goal": { "text": "hello" } +} diff --git a/examples/habitat/live.morphogen.json b/examples/habitat/live.morphogen.json new file mode 100644 index 0000000..db25ce8 --- /dev/null +++ b/examples/habitat/live.morphogen.json @@ -0,0 +1,76 @@ +{ + "contract": "morphogen.organism.v1", + "key": "organism:habitat-live", + "name": "HabitatLive", + "note": "A parent organism proposes a child manifest live through a model call, falls back to a known safe child if the proposal fails, and records the spawned child digest. The host can compare the child digest to the known fallback digest to decide whether to promote the live proposal.", + "budgets": { + "maxSteps": 32, + "maxAgentCalls": 4, + "maxWork": 10000 + }, + "interface": { + "inputs": { "goal": { "cell": "goal", "port": "text" } }, + "outputs": { + "child": { "cell": "run", "port": "digest" }, + "population": { "cell": "coalesce-pop", "port": "value" } + } + }, + "cells": [ + { "id": "goal", "kind": "input", "outputs": { "text": "text" } }, + { + "id": "history", + "kind": "slot", + "name": "population", + "mode": "read", + "default": [] + }, + { + "id": "designer", + "kind": "agent", + "inputs": { "goal": "text" }, + "prompt": "You are a code generator. Emit ONLY a JSON object. Do not wrap in markdown. Do not include comments. Replace both instances of with the provided user goal. The cells entry must use the field id, not key.\n\nTemplate:\n{\n \"contract\": \"morphogen.organism.v1\",\n \"key\": \"organism:echo-\",\n \"name\": \"Echo\",\n \"budgets\": {\"maxSteps\":8,\"maxAgentCalls\":1,\"maxWork\":100},\n \"interface\": {\"outputs\":{\"out\":{\"cell\":\"msg\",\"port\":\"v\"}}},\n \"cells\": [{\"id\":\"msg\",\"kind\":\"const\",\"outputs\":{\"v\":{\"type\":\"text\",\"value\":\"\"}}}],\n \"edges\": [],\n \"note\": \"Generated manifest for goal: \"\n}", + "output": { "kind": "json", "schema": { "type": "object" } } + }, + { + "id": "fallback", + "kind": "agent", + "inputs": { "err": "json" }, + "prompt": "Ignore the error. Output the default fallback organism manifest for goal 'hello' with key 'organism:echo-hello'.", + "output": { "kind": "json", "schema": { "type": "object" } } + }, + { + "id": "choose", + "kind": "fn", + "fn": "coalesce.v1" + }, + { "id": "run", "kind": "spawn" }, + { + "id": "append", + "kind": "fn", + "fn": "push.v1" + }, + { + "id": "record", + "kind": "slot", + "name": "population", + "mode": "write" + }, + { + "id": "coalesce-pop", + "kind": "fn", + "fn": "coalesce.v1" + } + ], + "edges": [ + { "from": { "cell": "goal", "port": "text" }, "to": { "cell": "designer", "port": "goal" } }, + { "from": { "cell": "designer", "port": "out" }, "to": { "cell": "choose", "port": "a" } }, + { "from": { "cell": "designer", "port": "out" }, "to": { "cell": "fallback", "port": "err" }, "on": "fail" }, + { "from": { "cell": "fallback", "port": "out" }, "to": { "cell": "choose", "port": "b" } }, + { "from": { "cell": "choose", "port": "value" }, "to": { "cell": "run", "port": "manifest" } }, + { "from": { "cell": "run", "port": "digest" }, "to": { "cell": "append", "port": "item" } }, + { "from": { "cell": "history", "port": "data" }, "to": { "cell": "append", "port": "list" } }, + { "from": { "cell": "append", "port": "value" }, "to": { "cell": "record", "port": "data" } }, + { "from": { "cell": "record", "port": "data" }, "to": { "cell": "coalesce-pop", "port": "a" } }, + { "from": { "cell": "history", "port": "data" }, "to": { "cell": "coalesce-pop", "port": "b" } } + ] +} diff --git a/examples/habitat/live.responses.json b/examples/habitat/live.responses.json new file mode 100644 index 0000000..707cb36 --- /dev/null +++ b/examples/habitat/live.responses.json @@ -0,0 +1,19 @@ +{ + "fallback": { + "contract": "morphogen.organism.v1", + "key": "organism:echo-hello", + "name": "EchoHello", + "budgets": { + "maxSteps": 8, + "maxAgentCalls": 1, + "maxWork": 100 + }, + "interface": { + "outputs": { "out": { "cell": "msg", "port": "v" } } + }, + "cells": [ + { "id": "msg", "kind": "const", "outputs": { "v": { "type": "text", "value": "hello" } } } + ], + "edges": [] + } +} diff --git a/examples/habitat/promote.ts b/examples/habitat/promote.ts new file mode 100644 index 0000000..c61c390 --- /dev/null +++ b/examples/habitat/promote.ts @@ -0,0 +1,121 @@ +import { mkdir, rm, writeFile } from "node:fs/promises"; +import { resolve } from "node:path"; + +const repo = resolve(import.meta.dir, "../.."); +const manifestPath = "examples/habitat/live.morphogen.json"; +const argsPath = "examples/habitat/live.args.json"; +const responsesPath = "examples/habitat/live.responses.json"; +const fallbackPath = "examples/habitat/fallback.morphogen.json"; +const dir = ".morphogen/habitat-live"; +const promotedDir = "promoted"; + +const live = process.argv.slice(2).includes("--live"); +const model = process.env.GATEWAY_MODEL ?? "alibaba/qwen3.7-flash"; + +async function runCmd( + cmd: string[], + opts?: { cwd?: string }, +): Promise<{ stdout: string; stderr: string; exitCode: number }> { + const proc = Bun.spawn(cmd, { + stdout: "pipe", + stderr: "pipe", + cwd: opts?.cwd ?? repo, + }); + const [stdout, stderr, code] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + proc.exited, + ]); + return { stdout, stderr, exitCode: code ?? 0 }; +} + +async function pack(file: string, storeDir: string): Promise { + const res = await runCmd(["bun", "cli.ts", "pack", file, "--dir", storeDir]); + if (res.exitCode !== 0) { + throw new Error(`pack failed: ${res.stderr || res.stdout}`); + } + const bundle = JSON.parse(res.stdout); + return bundle.root as string; +} + +const dirFull = resolve(repo, dir); +await rm(dirFull, { recursive: true, force: true }); +await mkdir(dirFull, { recursive: true }); + +const fallbackDigest = await pack(fallbackPath, dir); +console.log(`fallback digest ${fallbackDigest}`); + +const runCmdArgs = [ + "bun", + "cli.ts", + "run", + manifestPath, + "--args", + argsPath, + "--dir", + dir, + "--write", +]; + +if (live) { + runCmdArgs.push("--gateway-model", model); +} else { + runCmdArgs.push("--responses", responsesPath); +} + +const runRes = await runCmd(runCmdArgs); + +if (runRes.exitCode !== 0) { + console.error("habitat run failed"); + console.error(runRes.stderr); + process.exit(1); +} + +const receipt = JSON.parse(runRes.stdout); +const childDigest = receipt.cells.run.outputs.digest as string; +console.log(`proposed child ${childDigest}`); + +if (childDigest === fallbackDigest) { + console.log("rejected: proposal fell back to the default child"); + process.exit(0); +} + +const manifestRes = await runCmd([ + "bun", + "cli.ts", + "manifest", + childDigest, + "--dir", + dir, +]); +if (manifestRes.exitCode !== 0) { + console.error("could not retrieve child manifest"); + console.error(manifestRes.stderr); + process.exit(1); +} + +const childPath = resolve(dirFull, "child.json"); +await writeFile(childPath, manifestRes.stdout); + +const promotedFull = resolve(repo, promotedDir); +await mkdir(promotedFull, { recursive: true }); + +const packRes = await runCmd([ + "bun", + "cli.ts", + "pack", + childPath, + "--out", + promotedDir, + "--dir", + dir, +]); +if (packRes.exitCode !== 0) { + console.error("pack child failed"); + console.error(packRes.stderr); + process.exit(1); +} + +const bundle = JSON.parse(packRes.stdout); +console.log(`promoted bundle ${bundle.root}`); +console.log(`wrote ${promotedDir}/${bundle.root.slice("sha256:".length)}.bundle.json`); diff --git a/examples/invest/bench-invest-cheap.responses.json b/examples/invest/bench-invest-cheap.responses.json new file mode 100644 index 0000000..f439a51 --- /dev/null +++ b/examples/invest/bench-invest-cheap.responses.json @@ -0,0 +1,5 @@ +{ + "solo": ["refund", "escalate", "refund", "refund", "refund", "refund"], + "decide": ["refund", "escalate", "monitor", "escalate", "monitor", "refund"], + "decide-a": ["refund", "escalate", "monitor", "escalate", "monitor", "refund"] +} diff --git a/examples/invest/bench-invest-cheap2.responses.json b/examples/invest/bench-invest-cheap2.responses.json new file mode 100644 index 0000000..0aef673 --- /dev/null +++ b/examples/invest/bench-invest-cheap2.responses.json @@ -0,0 +1,3 @@ +{ + "decide-b": ["refund", "escalate", "monitor", "escalate", "refund", "refund"] +} diff --git a/examples/invest/bench-invest-ensemble.morphogen.json b/examples/invest/bench-invest-ensemble.morphogen.json new file mode 100644 index 0000000..51f5ff9 --- /dev/null +++ b/examples/invest/bench-invest-ensemble.morphogen.json @@ -0,0 +1,69 @@ +{ + "contract": "morphogen.organism.v1", + "key": "organism:bench-invest-ensemble", + "name": "Bench invest — ensemble with escalation", + "note": "Two decorrelated cheap lanes judge the same ticket and ledger. assert.v1 compares their answers; only disagreement spends a frontier call. One ledger lookup serves both lanes.", + "budgets": { "maxSteps": 12, "maxAgentCalls": 4, "maxWork": 150000 }, + "interface": { + "inputs": { + "ticket": { "cell": "src", "port": "ticket" }, + "account": { "cell": "src", "port": "account" } + }, + "outputs": { "out": { "cell": "merge", "port": "value" } } + }, + "cells": [ + { "id": "src", "kind": "input", "outputs": { "ticket": "text", "account": "text" } }, + { + "id": "lookup", + "kind": "tool", + "tool": "ledger.charges.v1", + "budget": { "maxEffectMs": 30000 } + }, + { + "id": "decide-a", + "kind": "classifier", + "inputs": { "ticket": "text", "charges": "json" }, + "prompt": "Decide the dispute outcome from the ticket and the account's charge ledger. Output refund when the ledger shows a genuine duplicate: two or more identical posted charges. Output monitor when the charge in question is pending or already reversed. Output escalate for anything else: unfamiliar vendors, mismatched amounts, or ambiguous evidence.", + "view": { "inputs": ["ticket", "charges"] }, + "output": { "kind": "choice", "labels": ["refund", "escalate", "monitor"] }, + "route": { "preset": "cheap" }, + "budget": { "maxContextBytes": 8192, "maxOutputBytes": 256, "maxEffectMs": 120000 } + }, + { + "id": "decide-b", + "kind": "classifier", + "inputs": { "ticket": "text", "charges": "json" }, + "prompt": "Decide the dispute outcome from the ticket and the account's charge ledger. Output refund when the ledger shows a genuine duplicate: two or more identical posted charges. Output monitor when the charge in question is pending or already reversed. Output escalate for anything else: unfamiliar vendors, mismatched amounts, or ambiguous evidence.", + "view": { "inputs": ["ticket", "charges"] }, + "output": { "kind": "choice", "labels": ["refund", "escalate", "monitor"] }, + "route": { "preset": "cheap2" }, + "budget": { "maxContextBytes": 8192, "maxOutputBytes": 256, "maxEffectMs": 120000 } + }, + { "id": "agree", "kind": "fn", "fn": "assert.v1" }, + { + "id": "escalate", + "kind": "classifier", + "inputs": { "ticket": "text", "charges": "json", "trigger": "json" }, + "prompt": "Two cheaper judges disagreed on this dispute. Decide the outcome from the ticket and the account's charge ledger. Output refund when the ledger shows a genuine duplicate: two or more identical posted charges. Output monitor when the charge in question is pending or already reversed. Output escalate for anything else: unfamiliar vendors, mismatched amounts, or ambiguous evidence.", + "view": { "inputs": ["ticket", "charges"] }, + "output": { "kind": "choice", "labels": ["refund", "escalate", "monitor"] }, + "route": { "preset": "frontier" }, + "budget": { "maxContextBytes": 8192, "maxOutputBytes": 256, "maxEffectMs": 120000 } + }, + { "id": "merge", "kind": "fn", "fn": "coalesce.v1" } + ], + "edges": [ + { "from": { "cell": "src", "port": "account" }, "to": { "cell": "lookup", "port": "account" } }, + { "from": { "cell": "src", "port": "ticket" }, "to": { "cell": "decide-a", "port": "ticket" } }, + { "from": { "cell": "lookup", "port": "charges" }, "to": { "cell": "decide-a", "port": "charges" } }, + { "from": { "cell": "src", "port": "ticket" }, "to": { "cell": "decide-b", "port": "ticket" } }, + { "from": { "cell": "lookup", "port": "charges" }, "to": { "cell": "decide-b", "port": "charges" } }, + { "from": { "cell": "decide-a", "port": "out" }, "to": { "cell": "agree", "port": "value" } }, + { "from": { "cell": "decide-b", "port": "out" }, "to": { "cell": "agree", "port": "expect" } }, + { "from": { "cell": "src", "port": "ticket" }, "to": { "cell": "escalate", "port": "ticket" } }, + { "from": { "cell": "lookup", "port": "charges" }, "to": { "cell": "escalate", "port": "charges" } }, + { "from": { "cell": "agree", "port": "value" }, "to": { "cell": "escalate", "port": "trigger" }, "on": "fail" }, + { "from": { "cell": "escalate", "port": "out" }, "to": { "cell": "merge", "port": "a" } }, + { "from": { "cell": "decide-a", "port": "out" }, "to": { "cell": "merge", "port": "b" } } + ] +} diff --git a/examples/invest/bench-invest-frontier.responses.json b/examples/invest/bench-invest-frontier.responses.json new file mode 100644 index 0000000..4efbef5 --- /dev/null +++ b/examples/invest/bench-invest-frontier.responses.json @@ -0,0 +1,4 @@ +{ + "solo": ["refund", "escalate", "monitor", "escalate", "refund", "refund"], + "escalate": ["monitor"] +} diff --git a/examples/invest/bench-invest-live.config.json b/examples/invest/bench-invest-live.config.json new file mode 100644 index 0000000..ca55ea2 --- /dev/null +++ b/examples/invest/bench-invest-live.config.json @@ -0,0 +1,97 @@ +{ + "contract": "morphogen.bench.config.v1", + "cases": [ + { + "id": "i1", + "args": { + "ticket": "I was charged twice for my subscription this month.", + "account": "acct-1" + }, + "expect": { + "out": "refund" + } + }, + { + "id": "i2", + "args": { + "ticket": "There is a charge on my card I do not recognize.", + "account": "acct-2" + }, + "expect": { + "out": "escalate" + } + }, + { + "id": "i3", + "args": { + "ticket": "You charged me but the payment is still showing as pending.", + "account": "acct-3" + }, + "expect": { + "out": "monitor" + } + }, + { + "id": "i4", + "args": { + "ticket": "Double billed again — refund it.", + "account": "acct-4" + }, + "expect": { + "out": "escalate" + } + }, + { + "id": "i5", + "args": { + "ticket": "A charge appeared after I cancelled — I want my money back.", + "account": "acct-5" + }, + "expect": { + "out": "monitor" + } + }, + { + "id": "i6", + "args": { + "ticket": "My card was charged three times this cycle.", + "account": "acct-6" + }, + "expect": { + "out": "refund" + } + } + ], + "systems": [ + { + "id": "cheap-single", + "manifest": "bench-invest-single.morphogen.json", + "executors": { + "cheap": "gateway:alibaba/qwen3.5-flash" + } + }, + { + "id": "frontier-single", + "manifest": "bench-invest-single.morphogen.json", + "executors": { + "frontier": "gateway:anthropic/claude-opus-5" + } + }, + { + "id": "organism-cheap", + "manifest": "bench-invest-org.morphogen.json", + "executors": { + "cheap": "gateway:alibaba/qwen3.5-flash" + } + }, + { + "id": "organism-ensemble", + "manifest": "bench-invest-ensemble.morphogen.json", + "executors": { + "cheap": "gateway:alibaba/qwen3.5-flash", + "cheap2": "gateway:alibaba/qwen3.7-flash", + "frontier": "gateway:anthropic/claude-opus-5" + } + } + ] +} diff --git a/examples/invest/bench-invest-org.morphogen.json b/examples/invest/bench-invest-org.morphogen.json new file mode 100644 index 0000000..ac499ac --- /dev/null +++ b/examples/invest/bench-invest-org.morphogen.json @@ -0,0 +1,38 @@ +{ + "contract": "morphogen.organism.v1", + "key": "organism:bench-invest-org", + "name": "Bench invest — tool-grounded organism", + "note": "A cheap classifier decides the dispute, but only after a tool cell retrieves the account's charge ledger. Every lookup is typed, bounded, and receipted. The deciding evidence is data, not a guess.", + "budgets": { "maxSteps": 8, "maxAgentCalls": 2, "maxWork": 100000 }, + "interface": { + "inputs": { + "ticket": { "cell": "src", "port": "ticket" }, + "account": { "cell": "src", "port": "account" } + }, + "outputs": { "out": { "cell": "decide", "port": "out" } } + }, + "cells": [ + { "id": "src", "kind": "input", "outputs": { "ticket": "text", "account": "text" } }, + { + "id": "lookup", + "kind": "tool", + "tool": "ledger.charges.v1", + "budget": { "maxEffectMs": 30000 } + }, + { + "id": "decide", + "kind": "classifier", + "inputs": { "ticket": "text", "charges": "json" }, + "prompt": "Decide the dispute outcome from the ticket and the account's charge ledger. Output refund when the ledger shows a genuine duplicate: two or more identical posted charges. Output monitor when the charge in question is pending or already reversed. Output escalate for anything else: unfamiliar vendors, mismatched amounts, or ambiguous evidence.", + "view": { "inputs": ["ticket", "charges"] }, + "output": { "kind": "choice", "labels": ["refund", "escalate", "monitor"] }, + "route": { "preset": "cheap" }, + "budget": { "maxContextBytes": 8192, "maxOutputBytes": 256, "maxEffectMs": 120000 } + } + ], + "edges": [ + { "from": { "cell": "src", "port": "account" }, "to": { "cell": "lookup", "port": "account" } }, + { "from": { "cell": "src", "port": "ticket" }, "to": { "cell": "decide", "port": "ticket" } }, + { "from": { "cell": "lookup", "port": "charges" }, "to": { "cell": "decide", "port": "charges" } } + ] +} diff --git a/examples/invest/bench-invest-priced.config.json b/examples/invest/bench-invest-priced.config.json new file mode 100644 index 0000000..c63a2c4 --- /dev/null +++ b/examples/invest/bench-invest-priced.config.json @@ -0,0 +1,84 @@ +{ + "contract": "morphogen.bench.config.v1", + "prices": { + "alibaba/qwen3.5-flash": { "input": 0.065, "output": 0.26 }, + "alibaba/qwen3.7-flash": { "input": 0.03, "output": 0.13 }, + "anthropic/claude-opus-5": { "input": 5, "output": 25 } + }, + "cases": [ + { + "id": "i1", + "args": { + "ticket": "I was charged twice for my subscription this month.", + "account": "acct-1" + }, + "expect": { "out": "refund" } + }, + { + "id": "i2", + "args": { + "ticket": "There is a charge on my card I do not recognize.", + "account": "acct-2" + }, + "expect": { "out": "escalate" } + }, + { + "id": "i3", + "args": { + "ticket": "You charged me but the payment is still showing as pending.", + "account": "acct-3" + }, + "expect": { "out": "monitor" } + }, + { + "id": "i4", + "args": { + "ticket": "Double billed again — refund it.", + "account": "acct-4" + }, + "expect": { "out": "escalate" } + }, + { + "id": "i5", + "args": { + "ticket": "A charge appeared after I cancelled — I want my money back.", + "account": "acct-5" + }, + "expect": { "out": "monitor" } + }, + { + "id": "i6", + "args": { + "ticket": "My card was charged three times this cycle.", + "account": "acct-6" + }, + "expect": { "out": "refund" } + } + ], + "systems": [ + { + "id": "cheap-single", + "manifest": "bench-invest-single.morphogen.json", + "executors": { "cheap": "gateway:alibaba/qwen3.5-flash" } + }, + { + "id": "frontier-single", + "manifest": "bench-invest-single.morphogen.json", + "executors": { "frontier": "gateway:anthropic/claude-opus-5" } + }, + { + "id": "organism-cheap", + "manifest": "bench-invest-org.morphogen.json", + "executors": { "cheap": "gateway:alibaba/qwen3.5-flash" } + }, + { + "id": "organism-ensemble", + "manifest": "bench-invest-ensemble.morphogen.json", + "executors": { + "cheap": "gateway:alibaba/qwen3.5-flash", + "cheap2": "gateway:alibaba/qwen3.7-flash", + "frontier": "gateway:anthropic/claude-opus-5" + } + } + ] +} diff --git a/examples/invest/bench-invest-single.morphogen.json b/examples/invest/bench-invest-single.morphogen.json new file mode 100644 index 0000000..100aa28 --- /dev/null +++ b/examples/invest/bench-invest-single.morphogen.json @@ -0,0 +1,30 @@ +{ + "contract": "morphogen.organism.v1", + "key": "organism:bench-invest-single", + "name": "Bench invest — single call", + "note": "Baseline: one classifier decides the dispute from the ticket and account id alone. It can see the account id but has no way to reach the charge ledger, so every decision is a guess. Compared against the tool-grounded organisms in bench-invest.config.json.", + "budgets": { "maxSteps": 4, "maxAgentCalls": 1, "maxWork": 100000 }, + "interface": { + "inputs": { + "ticket": { "cell": "src", "port": "ticket" }, + "account": { "cell": "src", "port": "account" } + }, + "outputs": { "out": { "cell": "solo", "port": "out" } } + }, + "cells": [ + { "id": "src", "kind": "input", "outputs": { "ticket": "text", "account": "text" } }, + { + "id": "solo", + "kind": "classifier", + "inputs": { "ticket": "text", "account": "text" }, + "prompt": "Decide the dispute outcome for the support ticket. Output refund when the customer reports a genuine duplicate charge. Output monitor when the issue will likely resolve itself. Output escalate when the situation needs human review.", + "view": { "inputs": ["ticket", "account"] }, + "output": { "kind": "choice", "labels": ["refund", "escalate", "monitor"] }, + "budget": { "maxContextBytes": 4096, "maxOutputBytes": 256, "maxEffectMs": 120000 } + } + ], + "edges": [ + { "from": { "cell": "src", "port": "ticket" }, "to": { "cell": "solo", "port": "ticket" } }, + { "from": { "cell": "src", "port": "account" }, "to": { "cell": "solo", "port": "account" } } + ] +} diff --git a/examples/invest/bench-invest.config.json b/examples/invest/bench-invest.config.json new file mode 100644 index 0000000..8b98649 --- /dev/null +++ b/examples/invest/bench-invest.config.json @@ -0,0 +1,97 @@ +{ + "contract": "morphogen.bench.config.v1", + "cases": [ + { + "id": "i1", + "args": { + "ticket": "I was charged twice for my subscription this month.", + "account": "acct-1" + }, + "expect": { + "out": "refund" + } + }, + { + "id": "i2", + "args": { + "ticket": "There is a charge on my card I do not recognize.", + "account": "acct-2" + }, + "expect": { + "out": "escalate" + } + }, + { + "id": "i3", + "args": { + "ticket": "You charged me but the payment is still showing as pending.", + "account": "acct-3" + }, + "expect": { + "out": "monitor" + } + }, + { + "id": "i4", + "args": { + "ticket": "Double billed again — refund it.", + "account": "acct-4" + }, + "expect": { + "out": "escalate" + } + }, + { + "id": "i5", + "args": { + "ticket": "A charge appeared after I cancelled — I want my money back.", + "account": "acct-5" + }, + "expect": { + "out": "monitor" + } + }, + { + "id": "i6", + "args": { + "ticket": "My card was charged three times this cycle.", + "account": "acct-6" + }, + "expect": { + "out": "refund" + } + } + ], + "systems": [ + { + "id": "cheap-single", + "manifest": "bench-invest-single.morphogen.json", + "executors": { + "cheap": "scripted:bench-invest-cheap.responses.json" + } + }, + { + "id": "frontier-single", + "manifest": "bench-invest-single.morphogen.json", + "executors": { + "frontier": "scripted:bench-invest-frontier.responses.json" + } + }, + { + "id": "organism-cheap", + "manifest": "bench-invest-org.morphogen.json", + "executors": { + "cheap": "scripted:bench-invest-cheap.responses.json" + } + }, + { + "id": "organism-ensemble", + "manifest": "bench-invest-ensemble.morphogen.json", + "executors": { + "cheap": "scripted:bench-invest-cheap.responses.json", + "cheap2": "scripted:bench-invest-cheap2.responses.json", + "frontier": "scripted:bench-invest-frontier.responses.json" + } + } + ] +} diff --git a/examples/invest/bench-invest.tools.json b/examples/invest/bench-invest.tools.json new file mode 100644 index 0000000..0991a22 --- /dev/null +++ b/examples/invest/bench-invest.tools.json @@ -0,0 +1,12 @@ +{ + "ledger.charges.v1": { + "signature": { + "inputs": { "account": "text" }, + "outputs": { "charges": "json" }, + "effect": "read", + "cost": 50, + "maxOutputBytes": 8192 + }, + "exec": "scripted:bench-ledger.data.json" + } +} diff --git a/examples/invest/bench-ledger.data.json b/examples/invest/bench-ledger.data.json new file mode 100644 index 0000000..0383340 --- /dev/null +++ b/examples/invest/bench-ledger.data.json @@ -0,0 +1,36 @@ +{ + "{\"account\":\"acct-1\"}": { + "charges": [ + { "id": "ch_101", "amount": 4900, "currency": "usd", "status": "posted", "memo": "SUBSCRIPTION RENEWAL" }, + { "id": "ch_102", "amount": 4900, "currency": "usd", "status": "posted", "memo": "SUBSCRIPTION RENEWAL" } + ] + }, + "{\"account\":\"acct-2\"}": { + "charges": [ + { "id": "ch_201", "amount": 18400, "currency": "usd", "status": "posted", "memo": "ZZ*FORN VENDOR 8821" } + ] + }, + "{\"account\":\"acct-3\"}": { + "charges": [ + { "id": "ch_301", "amount": 4900, "currency": "usd", "status": "pending", "memo": "SUBSCRIPTION RENEWAL" } + ] + }, + "{\"account\":\"acct-4\"}": { + "charges": [ + { "id": "ch_401", "amount": 4900, "currency": "usd", "status": "posted", "memo": "SUBSCRIPTION RENEWAL" }, + { "id": "ch_402", "amount": 12900, "currency": "usd", "status": "posted", "memo": "ADD-ON SEATS" } + ] + }, + "{\"account\":\"acct-5\"}": { + "charges": [ + { "id": "ch_501", "amount": 4900, "currency": "usd", "status": "reversed", "memo": "SUBSCRIPTION RENEWAL" } + ] + }, + "{\"account\":\"acct-6\"}": { + "charges": [ + { "id": "ch_601", "amount": 4900, "currency": "usd", "status": "posted", "memo": "SUBSCRIPTION RENEWAL" }, + { "id": "ch_602", "amount": 4900, "currency": "usd", "status": "posted", "memo": "SUBSCRIPTION RENEWAL" }, + { "id": "ch_603", "amount": 4900, "currency": "usd", "status": "posted", "memo": "SUBSCRIPTION RENEWAL" } + ] + } +} diff --git a/examples/search.config.json b/examples/search.config.json new file mode 100644 index 0000000..ce75409 --- /dev/null +++ b/examples/search.config.json @@ -0,0 +1,33 @@ +{ + "contract": "morphogen.foundry.config.v1", + "generator": { + "manifest": "evolving-generator.morphogen.json", + "args": { "task": "Return the input value unchanged." }, + "output": "candidates", + "field": "candidates" + }, + "search": { + "maxGenerations": 2, + "feedbackInput": "feedback" + }, + "cases": [ + { + "id": "train-alpha", + "split": "train", + "args": { "q": "alpha" }, + "expect": { "answer": "alpha" } + }, + { + "id": "validation-beta", + "split": "validation", + "args": { "q": "beta" }, + "expect": { "answer": "beta" } + }, + { + "id": "holdout-gamma", + "split": "holdout", + "args": { "q": "gamma" }, + "expect": { "answer": "gamma" } + } + ] +} diff --git a/index.ts b/index.ts index 73c2c4a..7690895 100644 --- a/index.ts +++ b/index.ts @@ -42,7 +42,29 @@ export { replayExecutor, scriptedExecutor, } from "./src/effects"; -export type { EffectReceipt, EffectRequest, Executor } from "./src/effects"; +export type { + EffectReceipt, + EffectRequest, + Executor, + ExecutorMetadata, + ExecutorResult, +} from "./src/effects"; + +export { VERCEL_AI_GATEWAY_BASE_URL, vercelGatewayExecutor } from "./src/gateway"; +export type { GatewayExecutorOptions, GatewayFetch } from "./src/gateway"; + +export { + emptyToolRegistry, + parseToolSignature, + TOOL_SIGNATURE_BOUNDS, +} from "./src/tools"; +export type { + Tool, + ToolContext, + ToolEffect, + ToolRegistry, + ToolSignature, +} from "./src/tools"; export { builtinRegistry } from "./src/registry"; export type { Fn, FnRegistry, FnSignature } from "./src/registry"; @@ -68,6 +90,48 @@ export type { Transport } from "./src/transport"; export { verifyReceipt } from "./src/verify"; export type { VerifyReport } from "./src/verify"; +export { + FOUNDRY_BOUNDS, + FOUNDRY_CONTRACT, + generateFoundryCandidates, + runFoundry, + selectFoundryCandidate, +} from "./src/foundry"; +export type { + FoundryCandidateResult, + FoundryCase, + FoundryCaseResult, + FoundryLineage, + FoundryOptions, + FoundryReport, + GenerateCandidatesOptions, + GeneratedCandidates, +} from "./src/foundry"; +export { parseFoundryReport, verifyFoundryReport } from "./src/foundry-verify"; +export type { FoundryVerifyReport } from "./src/foundry-verify"; + +export { SEARCH_BOUNDS, SEARCH_CONTRACT, runFoundrySearch } from "./src/search"; +export type { + SearchGeneration, + SearchOptions, + SearchReport, +} from "./src/search"; +export { parseSearchReport, verifySearchReport } from "./src/search-verify"; +export type { SearchVerifyReport } from "./src/search-verify"; + +export { BENCH_BOUNDS, BENCH_CONTRACT, benchPareto, runBenchmark } from "./src/bench"; +export type { + BenchAttribution, + BenchCase, + BenchCaseResult, + BenchOptions, + BenchReport, + BenchSystem, + BenchSystemResult, +} from "./src/bench"; +export { parseBenchReport, verifyBenchReport } from "./src/bench-verify"; +export type { BenchVerifyReport } from "./src/bench-verify"; + export { digestCanonical, digestText } from "./src/digest"; export type { Digest } from "./src/digest"; diff --git a/site/index.html b/site/index.html index 20613a9..7d07c6a 100644 --- a/site/index.html +++ b/site/index.html @@ -70,6 +70,18 @@

Structure that does the deciding.

+
+

A new primitive

+

+ Morphogen is not a prompt framework or a DAG engine. It is a + bounded, typed, content-addressed probabilistic program: a + manifest as a value, a run as a verifiable receipt, and model judgment + isolated behind declared cells. That makes an organism a new kind of + reusable object: hashable, transportable, evolvable, and safe to execute + even when you did not write it. +

+
+

The shape

@@ -117,6 +129,64 @@

Boundaries

+
+

Case study: cheap + tool beats frontier alone

+

+ A billing-dispute organism first retrieves an account's charge ledger + through a typed tool cell, then decides whether to + refund, escalate, or monitor. +

+

+ Live Vercel AI Gateway result on six cases, priced from aicharts.io / AI//COST: +

+
    +
  • Qwen 3.5 Flash + ledger tool: 6/6 correct for $0.00131
    • +
  • Claude Opus 5 without the ledger: 4/6 correct for $0.02625
    • +
+

+ Both models fail the same evidence-only cases when they cannot look up + the ledger. The structure — a typed tool call feeding a bounded + classifier — is what wins, not the model brand. +

+

+ See the invest example + · Read the intuition doc +

+
+ +
+

Use it as an agent tool

+

+ Pack an organism into a digest-verified bundle, generate an OpenAI or + Anthropic tool definition, and let a larger agent call it by running + morphogen call. +

+ 
$ morphogen pack ticket.morphogen.json --out ./tools
+$ morphogen tool-def ticket.morphogen.json
+$ morphogen call ./tools/<bundle>.bundle.json --args ticket.args.json
+{ "ok": true, "outputs": { "out": "billing" }, "receiptDigest": "sha256:..." }
+

+ The agent receives a compact result plus a receipt it can verify offline. +

+
+ +
+

Habitats and self-evolution

+

+ Because a manifest is a value, an organism can generate and propose new + organisms, functions, and tools. A shared store and registries become a + habitat: a population that evolves through foundry search and host + admission. The organism cannot rewrite its own runtime; it can only + propose. The host remains the admission gate. +

+

+ A live steel thread already works: bun examples/habitat/promote.ts --live + lets a parent organism call a model to design a child, spawn it, and the + host promotes the child to a standalone bundle if it is valid and different + from the safe fallback. +

+
+

Status

diff --git a/site/llms.txt b/site/llms.txt index a3c04dd..0a3c600 100644 --- a/site/llms.txt +++ b/site/llms.txt @@ -11,3 +11,54 @@ - [Organism contract (spec/v1/organism.md)](https://github.com/hraness/morphogen/blob/main/spec/v1/organism.md) - [Design notes](https://github.com/hraness/morphogen/tree/main/docs) - [Bundled example organisms](https://github.com/hraness/morphogen/tree/main/examples) + +## Where it wins + +Morphogen wins when a workflow has structure that a single LLM call cannot +capture: tool-grounded evidence, many narrow classifiers, guarded routing, +budgets, and the need to prove what happened. Example: a billing-dispute +organism that first looks up a charge ledger with a typed `tool` cell, then +classifies. In a live Vercel AI Gateway run, a Qwen 3.5 Flash organism with the +ledger lookup scored 6/6 for $0.00131, while a Claude Opus 5 call without the +tool scored 4/6 for $0.02625. Prices were taken from aicharts.io / AI//COST +and OpenRouter/Alibaba rate cards. + +## Why it is a new primitive + +Morphogen sits between deterministic programs and open-ended agents. A manifest +is a finite, strongly-typed graph; a run is a content-addressed, replayable +receipt; and model judgment is isolated behind bounded cells with declared +contracts. This makes an organism a new kind of value: hashable, transportable, +verifiable, and evolvable. See `docs/why-unique.md` in the repository for the +full comparison. + +## Habitats and self-evolution + +Because a manifest is a value, an organism can generate and propose new +organisms, functions, and tools. A shared `Store` and registries become a +habitat: a population that evolves through foundry search and host admission. +The organism cannot rewrite its own runtime; it can only propose. The host +remains the admission gate. A live steel thread already works: +`bun examples/habitat/promote.ts --live` lets a parent organism call a model to +design a child, spawn it, and the host promotes the child to a bundle if it is +valid and different from the safe fallback. + +## Quick start + +```sh +bun install +bun run cli suite +bun run cli bench examples/bench.config.json --dir .morphogen --out bench-report.json +``` + +## Integration + +- `Executor` is one method: `execute(effect, signal?)` → raw output. Wrap any + provider, local model, or scripted fixture. +- `ToolRegistry` is a host-supplied `Map` of typed signatures and async callables. +- CLI: `--gateway-model ` for Vercel AI Gateway; + `--tools ` for `scripted:` or `cmd:` tool executors; + add `prices` to a bench config to get a dollar-denominated Pareto. +- Agent tools: `morphogen pack` a manifest closure, `morphogen tool-def` to + emit an OpenAI/Anthropic function definition, and `morphogen call` to run it + and get `{ ok, outputs, receiptDigest, manifestDigest }`. diff --git a/spec/v1/bench.md b/spec/v1/bench.md new file mode 100644 index 0000000..fea9d5b --- /dev/null +++ b/spec/v1/bench.md @@ -0,0 +1,17 @@ +# morphogen.bench.v1 + +A bench report records one workload measured by several systems. A system is an admitted organism manifest plus a host-resolved executor list — so "one cheap call", "one frontier call", and "a decomposed organism with a guarded escalation branch" are all the same kind of contender. Bench adds no manifest primitive and no authority; it is a measurement layer over ordinary runs. + +## Systems and cases + +A bench runs 1–8 systems against 1–256 cases. Every system manifest declares an interface; case `args` cover the interface's named inputs and `expect` covers every named output — so all systems see identical inputs and are scored on identical outputs. A case passes when the run completes and its declared outputs canonically equal `expect`; there is no judge and no leniency. + +Each case result records outcome, declared outputs, expectations, run receipt digest, effect-call count, work, token usage, and per-model attribution (calls and tokens grouped by the effect's recorded model, or by executor id when none was reported — tools and scripted executors attribute to themselves). + +## Evidence and pareto + +The report embeds the complete case list (so verification needs no external config), a workload digest over it, every system's aggregate passed/total, effect calls, work, usage, and attribution, and the non-dominated system set on (passed ↑, total tokens ↓): a system is dominated when another is at least as good on both axes and strictly better on one. Ties break deterministically. + +Verification parses strictly, recomputes the report and workload digests, rechecks every pass claim and aggregate, recomputes the pareto set, confirms each case's recorded receipt ran the claimed manifest with the claimed args, and replays every receipt offline. Tampering fails even when the report digest is recomputed, because claims must match receipted runs. + +A verified bench report proves the recorded comparison is internally consistent. It does not prove the workload is representative, that token counts imply dollars (prices are host inputs, not evidence), or that future live effects will match recorded effects. diff --git a/spec/v1/foundry.md b/spec/v1/foundry.md new file mode 100644 index 0000000..146c1f5 --- /dev/null +++ b/spec/v1/foundry.md @@ -0,0 +1,37 @@ +# morphogen.foundry.v1 + +A foundry report is content-addressed evidence for selecting one bounded organism population. It records no wall-clock values and contains no executable code. + +## Selection + +A foundry run admits 1–32 distinct candidate manifests and 1–256 uniquely named cases. Every candidate declares an interface compatible with every case. Cases have three splits: + +- `train` — visible examples used by the generator or search strategy. +- `validation` — evidence used to select a candidate. +- `holdout` — run exactly once against the promoted candidate; never run against the rest of the population. + +A case passes only when its run completes and the candidate's canonical interface output record equals `expect`. Promotion orders candidates by validation pass rate, train pass rate, ascending agent calls, ascending work units, then manifest digest. + +## Candidate generation + +A host may supply candidate files or run a generator organism. The generator must declare an interface output containing a non-empty list of `morphogen.organism.v1` values, directly or under a configured object field. Every value passes through the ordinary manifest parser. Invalid, duplicate, or over-bound populations fail before evaluation. + +The generator is an ordinary organism. It may compose `repeat`, `each`, `spawn`, slots, gates, and nested organisms to implement bounded generations, populations, lineage journals, or approval. The foundry grants it no additional functions, executors, capabilities, or budgets. + +## Report + +A report contains: + +- `candidates` — manifest identity, train and validation scores, aggregate work, and case evidence; +- `promoted` — the deterministic winner's manifest digest; +- `holdout` — case evidence for the promoted manifest only; +- `lineage` — optional generator manifest and run-receipt digests; +- `digest` — the canonical digest of every preceding report field. + +Each case records its split, expected and actual interface outputs, outcome, pass claim, work, aggregate input/output token usage, and stored run-receipt digest. Candidate records aggregate work and usage across selection cases. Candidate manifests, generator manifests, and all referenced run receipts live in the host store. + +## Verification + +Verification rejects unknown fields and malformed bounds, recomputes the report digest, scores, pass claims, and deterministic promotion, resolves every referenced manifest and receipt, compares recorded outputs, outcome, work, and token usage with each receipt, and replays every run offline. Bundle export is permitted only after successful verification and packs the promoted organism's content-addressed closure. + +A verified report proves that the recorded evidence and selection are internally consistent. It does not prove that cases represent deployment, expectations are correct, the model was truthful, or the promoted organism will receive the same effects on a future live run. diff --git a/spec/v1/search.md b/spec/v1/search.md new file mode 100644 index 0000000..f5bea56 --- /dev/null +++ b/spec/v1/search.md @@ -0,0 +1,23 @@ +# morphogen.search.v1 + +A search report records bounded, validation-guided evolution over organism manifests. Search is a host layer over ordinary generator organisms and `morphogen.foundry.v1` evidence; it adds no executable manifest primitive or authority. + +## Generations + +A search runs 1–8 generations. At each generation: + +1. The generator organism receives fixed task arguments and, when configured, the prior generation's candidate digests, train and validation scores, work, token usage, and promoted digest. +2. Its declared output is parsed as 1–32 ordinary organism manifests. +3. The previous winner survives and competes with the new proposals. Duplicate manifest digests collapse. +4. Every candidate runs against train and validation cases under the host registry, executors, transports, store, and root manifest budgets. +5. Deterministic foundry ordering promotes one survivor. + +Holdout expectations, outputs, scores, and receipts never enter generation evidence or generator feedback. After the final generation, the last winner is evaluated once against holdout cases through `morphogen.foundry.v1`. + +## Evidence + +Each generation records its index, generator manifest and receipt digests, proposed manifest digests, complete candidate evidence, and promoted digest. The search report also records the generator digest, final foundry report, and a canonical digest over the complete history. + +Verification checks bounds and unknown fields, recomputes the report digest, verifies deterministic promotion and survivor continuity, confirms every proposal was evaluated, rejects holdout evidence in generations, resolves every referenced manifest and receipt, and replays generator and candidate runs offline. The final foundry report is independently verified. + +A verified search report proves the recorded evolutionary history and selection are internally consistent. It does not establish that the fitness cases are representative, prevent a generator from overfitting visible train or validation evidence, or prove future live effects will match recorded effects. diff --git a/src/bench-verify.ts b/src/bench-verify.ts new file mode 100644 index 0000000..bfc5df1 --- /dev/null +++ b/src/bench-verify.ts @@ -0,0 +1,401 @@ +import { manifestToJson } from "./contract"; +import { digestCanonical, type Digest } from "./digest"; +import { MorphogenError } from "./errors"; +import { + BENCH_BOUNDS, + BENCH_CONTRACT, + benchPareto, + type BenchAttribution, + type BenchCase, + type BenchCaseResult, + type BenchPrice, + type BenchReport, + type BenchSystemResult, +} from "./bench"; +import type { FnRegistry } from "./registry"; +import { parseRunReceipt } from "./run"; +import type { Store } from "./store"; +import type { ToolRegistry } from "./tools"; +import { verifyReceipt } from "./verify"; +import { canonicalize, type JsonObject, type JsonValue } from "./values"; + +function object(value: unknown, at: string): JsonObject { + if (value === null || typeof value !== "object" || Array.isArray(value)) { + throw new MorphogenError("PARSE_FAILED", `${at} must be an object`); + } + return value as JsonObject; +} + +function keys(value: JsonObject, allowed: string[], at: string): void { + const extra = Object.keys(value).find((key) => !allowed.includes(key)); + if (extra) throw new MorphogenError("PARSE_FAILED", `${at}: unknown key "${extra}"`); +} + +function text(value: JsonValue | undefined, at: string): string { + if (typeof value !== "string") throw new MorphogenError("PARSE_FAILED", `${at} must be text`); + return value; +} + +function id(value: JsonValue | undefined, at: string): string { + const parsed = text(value, at); + if (!/^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(parsed) || parsed.length > BENCH_BOUNDS.maxIdLen) { + throw new MorphogenError("PARSE_FAILED", `${at} is not a valid id`); + } + return parsed; +} + +function digest(value: JsonValue | undefined, at: string): Digest { + const parsed = text(value, at); + if (!/^sha256:[0-9a-f]{64}$/.test(parsed)) { + throw new MorphogenError("PARSE_FAILED", `${at} must be a sha256 digest`); + } + return parsed as Digest; +} + +function count(value: JsonValue | undefined, at: string): number { + if (!Number.isInteger(value) || (value as number) < 0) { + throw new MorphogenError("PARSE_FAILED", `${at} must be a non-negative integer`); + } + return value as number; +} + +function number_(value: JsonValue | undefined, at: string): number { + if (typeof value !== "number" || !Number.isFinite(value) || value < 0) { + throw new MorphogenError("PARSE_FAILED", `${at} must be a non-negative number`); + } + return value; +} + +function parseWork(value: JsonValue | undefined, at: string) { + const work = object(value, at); + keys(work, ["steps", "agentCalls", "units"], at); + return { + steps: count(work.steps, `${at}.steps`), + agentCalls: count(work.agentCalls, `${at}.agentCalls`), + units: count(work.units, `${at}.units`), + }; +} + +function parseUsage(value: JsonValue | undefined, at: string) { + const usage = object(value, at); + keys(usage, ["tokensIn", "tokensOut", "cost"], at); + return { + tokensIn: count(usage.tokensIn, `${at}.tokensIn`), + tokensOut: count(usage.tokensOut, `${at}.tokensOut`), + cost: number_(usage.cost, `${at}.cost`), + }; +} + +function parseAttribution(value: JsonValue | undefined, at: string): Record { + const map = object(value, at); + const out: Record = {}; + for (const [key, raw] of Object.entries(map)) { + if (key.length === 0 || key.length > 256) { + throw new MorphogenError("PARSE_FAILED", `${at} has an invalid attribution key`); + } + const entry = object(raw, `${at}.${key}`); + keys(entry, ["calls", "tokensIn", "tokensOut", "cost"], `${at}.${key}`); + out[key] = { + calls: count(entry.calls, `${at}.${key}.calls`), + tokensIn: count(entry.tokensIn, `${at}.${key}.tokensIn`), + tokensOut: count(entry.tokensOut, `${at}.${key}.tokensOut`), + cost: number_(entry.cost, `${at}.${key}.cost`), + }; + } + return out; +} + +function parseBenchCase(value: JsonValue, at: string): BenchCase { + const c = object(value, at); + keys(c, ["id", "args", "expect"], at); + return { + id: id(c.id, `${at}.id`), + args: object(c.args, `${at}.args`), + expect: object(c.expect, `${at}.expect`), + }; +} + +function parseCaseResult(value: JsonValue, at: string): BenchCaseResult { + const c = object(value, at); + keys(c, ["id", "passed", "outcome", "outputs", "expect", "receiptDigest", "effectCalls", "work", "usage", "attribution"], at); + const outcome = text(c.outcome, `${at}.outcome`); + if (outcome !== "complete" && outcome !== "failed" && outcome !== "stuck") { + throw new MorphogenError("PARSE_FAILED", `${at}.outcome is invalid`); + } + if (typeof c.passed !== "boolean") { + throw new MorphogenError("PARSE_FAILED", `${at}.passed must be boolean`); + } + return { + id: id(c.id, `${at}.id`), + passed: c.passed, + outcome, + outputs: object(c.outputs, `${at}.outputs`), + expect: object(c.expect, `${at}.expect`), + receiptDigest: digest(c.receiptDigest, `${at}.receiptDigest`), + effectCalls: count(c.effectCalls, `${at}.effectCalls`), + work: parseWork(c.work, `${at}.work`), + usage: parseUsage(c.usage, `${at}.usage`), + attribution: parseAttribution(c.attribution, `${at}.attribution`), + }; +} + +function parseSystem(value: JsonValue, i: number): BenchSystemResult { + const at = `bench.systems[${i}]`; + const s = object(value, at); + keys(s, ["id", "manifestDigest", "manifestKey", "passed", "total", "effectCalls", "work", "usage", "attribution", "cases"], at); + const total = count(s.total, `${at}.total`); + const passed = count(s.passed, `${at}.passed`); + if (total === 0 || passed > total) { + throw new MorphogenError("PARSE_FAILED", `${at} is not a valid score`); + } + if (!Array.isArray(s.cases) || s.cases.length !== total || s.cases.length > BENCH_BOUNDS.maxCases) { + throw new MorphogenError("PARSE_FAILED", `${at}.cases must match its total`); + } + return { + id: id(s.id, `${at}.id`), + manifestDigest: digest(s.manifestDigest, `${at}.manifestDigest`), + manifestKey: text(s.manifestKey, `${at}.manifestKey`), + passed, + total, + effectCalls: count(s.effectCalls, `${at}.effectCalls`), + work: parseWork(s.work, `${at}.work`), + usage: parseUsage(s.usage, `${at}.usage`), + attribution: parseAttribution(s.attribution, `${at}.attribution`), + cases: s.cases.map((entry, j) => parseCaseResult(entry, `${at}.cases[${j}]`)), + }; +} + +function parseBenchPrice(value: JsonValue | undefined, at: string): Record | undefined { + if (value === undefined) return undefined; + const map = object(value, at); + const out: Record = {}; + for (const [key, raw] of Object.entries(map)) { + if (key.length === 0 || key.length > 256) { + throw new MorphogenError("PARSE_FAILED", `${at} has an invalid price key`); + } + const p = object(raw, `${at}.${key}`); + keys(p, ["input", "output"], `${at}.${key}`); + out[key] = { + input: number_(p.input, `${at}.${key}.input`), + output: number_(p.output, `${at}.${key}.output`), + }; + } + return out; +} + +export function parseBenchReport(value: unknown): BenchReport { + const report = object(value, "bench"); + keys(report, ["contract", "workload", "cases", "prices", "systems", "pareto", "digest"], "bench"); + if (report.contract !== BENCH_CONTRACT) { + throw new MorphogenError("PARSE_FAILED", `bench.contract must be ${BENCH_CONTRACT}`); + } + if (!Array.isArray(report.cases) || report.cases.length === 0 || report.cases.length > BENCH_BOUNDS.maxCases) { + throw new MorphogenError("PARSE_FAILED", "bench.cases must be a bounded non-empty list"); + } + if (!Array.isArray(report.systems) || report.systems.length === 0 || report.systems.length > BENCH_BOUNDS.maxSystems) { + throw new MorphogenError("PARSE_FAILED", "bench.systems must be a bounded non-empty list"); + } + if (!Array.isArray(report.pareto) || report.pareto.length > BENCH_BOUNDS.maxSystems) { + throw new MorphogenError("PARSE_FAILED", "bench.pareto must be a bounded list"); + } + const systems = report.systems.map(parseSystem); + const ids = new Set(systems.map((s) => s.id)); + const pareto = report.pareto.map((entry, i) => { + const parsed = id(entry as JsonValue, `bench.pareto[${i}]`); + if (!ids.has(parsed)) { + throw new MorphogenError("PARSE_FAILED", `bench.pareto[${i}] names an unknown system`); + } + return parsed; + }); + if (new Set(pareto).size !== pareto.length) { + throw new MorphogenError("PARSE_FAILED", "bench.pareto contains duplicates"); + } + return { + contract: BENCH_CONTRACT, + workload: digest(report.workload, "bench.workload"), + cases: report.cases.map((entry, i) => parseBenchCase(entry, `bench.cases[${i}]`)), + prices: parseBenchPrice(report.prices, "bench.prices"), + systems, + pareto, + digest: digest(report.digest, "bench.digest"), + }; +} + +export type BenchVerifyReport = { + ok: boolean; + digest: Digest; + checkedReceipts: number; + mismatches: string[]; +}; + +export async function verifyBenchReport( + value: unknown, + store: Store, + fns: FnRegistry, + tools?: ToolRegistry, +): Promise { + const report = parseBenchReport(value); + const mismatches: string[] = []; + const { digest: claimed, ...base } = report; + const actual = digestCanonical(base as unknown as JsonValue); + if (actual !== claimed) mismatches.push(`digest: claimed ${claimed}, computed ${actual}`); + const workload = digestCanonical(report.cases as unknown as JsonValue); + if (workload !== report.workload) { + mismatches.push(`workload: claimed ${report.workload}, computed ${workload}`); + } + if (canonicalize(report.pareto as unknown as JsonValue) !== canonicalize(benchPareto(report.systems, report.prices !== undefined) as unknown as JsonValue)) { + mismatches.push("pareto does not match the system totals"); + } + const caseIds = new Set(); + for (const c of report.cases) { + if (caseIds.has(c.id)) mismatches.push(`duplicate case id "${c.id}"`); + caseIds.add(c.id); + } + const casesById = new Map(report.cases.map((c) => [c.id, c])); + const systemIds = new Set(); + let checkedReceipts = 0; + for (const system of report.systems) { + if (systemIds.has(system.id)) mismatches.push(`duplicate system id "${system.id}"`); + systemIds.add(system.id); + const manifest = await store.getManifest(system.manifestDigest); + if (!manifest) { + mismatches.push(`${system.id}: manifest ${system.manifestDigest} missing`); + continue; + } + if (!manifest.interface) { + mismatches.push(`${system.id}: manifest has no interface`); + } + const passed = system.cases.filter((c) => c.passed).length; + if (passed !== system.passed) mismatches.push(`${system.id}: passed does not match its cases`); + if (system.cases.length !== report.cases.length) { + mismatches.push(`${system.id}: case count differs from the workload`); + } + const work = { steps: 0, agentCalls: 0, units: 0 }; + const usage = { tokensIn: 0, tokensOut: 0, cost: 0 }; + const attribution: Record = {}; + let effectCalls = 0; + for (const c of system.cases) { + const benchCase = casesById.get(c.id); + if (!benchCase) { + mismatches.push(`${system.id}: case "${c.id}" is not in the workload`); + continue; + } + if (canonicalize(c.expect) !== canonicalize(benchCase.expect)) { + mismatches.push(`${system.id} case ${c.id}: expect differs from the workload`); + } + const expectedPass = c.outcome === "complete" && canonicalize(c.outputs) === canonicalize(c.expect); + if (c.passed !== expectedPass) { + mismatches.push(`${system.id} case ${c.id}: invalid pass claim`); + } + work.steps += c.work.steps; + work.agentCalls += c.work.agentCalls; + work.units += c.work.units; + usage.tokensIn += c.usage.tokensIn; + usage.tokensOut += c.usage.tokensOut; + usage.cost += c.usage.cost; + effectCalls += c.effectCalls; + for (const [key, value] of Object.entries(c.attribution)) { + const entry = (attribution[key] ??= { calls: 0, tokensIn: 0, tokensOut: 0, cost: 0 }); + entry.calls += value.calls; + entry.tokensIn += value.tokensIn; + entry.tokensOut += value.tokensOut; + entry.cost += value.cost; + } + const stored = await store.getReceipt(c.receiptDigest); + if (!stored) { + mismatches.push(`${system.id} case ${c.id}: receipt ${c.receiptDigest} missing`); + continue; + } + const receipt = parseRunReceipt(stored); + if (receipt.manifestDigest !== system.manifestDigest) { + mismatches.push(`${system.id} case ${c.id}: receipt ran ${receipt.manifestDigest}`); + continue; + } + if (receipt.outcome !== c.outcome) { + mismatches.push(`${system.id} case ${c.id}: outcome differs from receipt`); + } + if (manifest.interface) { + const outputs: Record = {}; + for (const [name, source] of Object.entries(manifest.interface.outputs)) { + const value = receipt.cells[source.cell]?.outputs?.[source.port]; + if (value !== undefined) outputs[name] = value; + } + if (canonicalize(outputs) !== canonicalize(c.outputs)) { + mismatches.push(`${system.id} case ${c.id}: outputs differ from receipt`); + } + const expectedArgs: Record> = {}; + for (const [name, value] of Object.entries(benchCase.args)) { + const target = manifest.interface.inputs[name]; + if (!target) { + mismatches.push(`${system.id} case ${c.id}: unknown workload input "${name}"`); + continue; + } + (expectedArgs[target.cell] ??= {})[target.port] = value; + } + if (canonicalize(receipt.args as unknown as JsonValue) !== canonicalize(expectedArgs as unknown as JsonValue)) { + mismatches.push(`${system.id} case ${c.id}: receipt args differ from the workload`); + } + } + if (canonicalize(receipt.work as unknown as JsonValue) !== canonicalize(c.work as unknown as JsonValue)) { + mismatches.push(`${system.id} case ${c.id}: work differs from receipt`); + } + if (receipt.effects.length !== c.effectCalls) { + mismatches.push(`${system.id} case ${c.id}: effectCalls differs from receipt`); + } + const receiptUsage = { tokensIn: 0, tokensOut: 0, cost: 0 }; + const receiptAttribution: Record = {}; + for (const effect of receipt.effects) { + const key = effect.usage?.model ?? effect.executor; + const entry = (receiptAttribution[key] ??= { calls: 0, tokensIn: 0, tokensOut: 0, cost: 0 }); + entry.calls += 1; + const tokensIn = effect.usage?.tokensIn ?? 0; + const tokensOut = effect.usage?.tokensOut ?? 0; + entry.tokensIn += tokensIn; + entry.tokensOut += tokensOut; + const price = report.prices?.[key]; + const extraCost = price + ? (tokensIn * price.input + tokensOut * price.output) / 1_000_000 + : 0; + entry.cost += extraCost; + receiptUsage.tokensIn += tokensIn; + receiptUsage.tokensOut += tokensOut; + receiptUsage.cost += extraCost; + } + const reportAttribution: Record = {}; + for (const [key, value] of Object.entries(c.attribution)) { + reportAttribution[key] = { ...value }; + } + if (canonicalize(receiptUsage as unknown as JsonValue) !== canonicalize(c.usage as unknown as JsonValue)) { + mismatches.push(`${system.id} case ${c.id}: usage differs from receipt`); + } + if (canonicalize(receiptAttribution as unknown as JsonValue) !== canonicalize(reportAttribution as unknown as JsonValue)) { + mismatches.push(`${system.id} case ${c.id}: attribution differs from receipt`); + } + const verified = await verifyReceipt( + receipt, + manifestToJson(manifest), + store, + fns, + undefined, + tools, + ); + checkedReceipts++; + if (!verified.ok) { + mismatches.push(`${system.id} case ${c.id}: ${verified.mismatches.join("; ")}`); + } + } + if (canonicalize(work as unknown as JsonValue) !== canonicalize(system.work as unknown as JsonValue)) { + mismatches.push(`${system.id}: work does not match its cases`); + } + if (canonicalize(usage as unknown as JsonValue) !== canonicalize(system.usage as unknown as JsonValue)) { + mismatches.push(`${system.id}: usage does not match its cases`); + } + if (canonicalize(attribution as unknown as JsonValue) !== canonicalize(system.attribution as unknown as JsonValue)) { + mismatches.push(`${system.id}: attribution does not match its cases`); + } + if (effectCalls !== system.effectCalls) { + mismatches.push(`${system.id}: effectCalls does not match its cases`); + } + } + return { ok: mismatches.length === 0, digest: claimed, checkedReceipts, mismatches }; +} diff --git a/src/bench.test.ts b/src/bench.test.ts new file mode 100644 index 0000000..8c206c2 --- /dev/null +++ b/src/bench.test.ts @@ -0,0 +1,235 @@ +import { describe, expect, test } from "bun:test"; +import { parseOrganismManifest } from "./contract"; +import { scriptedExecutor, type Executor } from "./effects"; +import { builtinRegistry } from "./registry"; +import { MemoryStore } from "./store"; +import { runBenchmark } from "./bench"; +import { parseBenchReport, verifyBenchReport } from "./bench-verify"; +import { canonicalize, type JsonValue } from "./values"; + +const single = parseOrganismManifest({ + contract: "morphogen.organism.v1", + key: "organism:bench-single", + name: "Single call", + cells: [ + { id: "src", kind: "input", outputs: { ticket: "text" } }, + { + id: "route", + kind: "classifier", + inputs: { ticket: "text" }, + prompt: "Route the ticket.", + view: { inputs: ["ticket"] }, + output: { kind: "choice", labels: ["billing", "technical", "other"] }, + }, + ], + edges: [ + { from: { cell: "src", port: "ticket" }, to: { cell: "route", port: "ticket" } }, + ], + interface: { + inputs: { ticket: { cell: "src", port: "ticket" } }, + outputs: { out: { cell: "route", port: "out" } }, + }, +}); + +const circuit = parseOrganismManifest({ + contract: "morphogen.organism.v1", + key: "organism:bench-circuit", + name: "Cheap-first cascade", + budgets: { maxSteps: 8, maxAgentCalls: 4, maxWork: 100000 }, + cells: [ + { id: "src", kind: "input", outputs: { ticket: "text" } }, + { + id: "cheap", + kind: "classifier", + inputs: { ticket: "text" }, + prompt: "Route the ticket, or abstain.", + view: { inputs: ["ticket"] }, + output: { kind: "choice", labels: ["billing", "technical", "other", "unsure"] }, + route: { preset: "cheap" }, + }, + { + id: "escalate", + kind: "classifier", + inputs: { + ticket: "text", + trigger: { type: "choice", labels: ["billing", "technical", "other", "unsure"] }, + }, + prompt: "Route the ticket the cheap pass could not.", + view: { inputs: ["ticket"] }, + output: { kind: "choice", labels: ["billing", "technical", "other"] }, + route: { preset: "frontier" }, + }, + { id: "merge", kind: "fn", fn: "coalesce.v1" }, + ], + edges: [ + { from: { cell: "src", port: "ticket" }, to: { cell: "cheap", port: "ticket" } }, + { from: { cell: "src", port: "ticket" }, to: { cell: "escalate", port: "ticket" } }, + { + from: { cell: "cheap", port: "out" }, + to: { cell: "escalate", port: "trigger" }, + guard: { equals: "unsure" }, + }, + { from: { cell: "escalate", port: "out" }, to: { cell: "merge", port: "a" } }, + { from: { cell: "cheap", port: "out" }, to: { cell: "merge", port: "b" } }, + ], + interface: { + inputs: { ticket: { cell: "src", port: "ticket" } }, + outputs: { out: { cell: "merge", port: "value" } }, + }, +}); + +const cases = [ + { id: "t1", args: { ticket: "charged twice for my subscription" }, expect: { out: "billing" } }, + { id: "t2", args: { ticket: "app crashes on export" }, expect: { out: "technical" } }, + { id: "t3", args: { ticket: "refund for a cancelled plan" }, expect: { out: "billing" } }, + { id: "t4", args: { ticket: "invoice totals look wrong after the update" }, expect: { out: "technical" } }, +]; + +/** Scripted executor that reports model usage so attribution exercises the + * per-model path — same shape the gateway executor produces. */ +function metered( + id: string, + model: string, + responses: Record, + usage: { tokensIn: number; tokensOut: number }, +): Executor { + const inner = scriptedExecutor(responses, id); + return { + id, + execute: (request, signal) => inner.execute(request, signal), + receiptFor: () => ({ usage: { model, ...usage } }), + }; +} + +const CHEAP = { tokensIn: 100, tokensOut: 10 }; +const FRONTIER = { tokensIn: 900, tokensOut: 60 }; + +async function bench() { + const store = new MemoryStore(); + const report = await runBenchmark({ + fns: builtinRegistry(), + store, + cases, + systems: [ + { + id: "cheap-single", + manifest: single, + executors: [ + metered("cheap", "qwen-flash", { route: ["billing", "technical", "billing", "other"] }, CHEAP), + ], + }, + { + id: "frontier-single", + manifest: single, + executors: [ + metered("frontier", "claude-opus", { route: ["billing", "technical", "billing", "technical"] }, FRONTIER), + ], + }, + { + id: "circuit", + manifest: circuit, + executors: [ + metered("cheap", "qwen-flash", { cheap: ["billing", "technical", "billing", "unsure"] }, CHEAP), + metered("frontier", "claude-opus", { escalate: ["technical"] }, FRONTIER), + ], + }, + ], + }); + return { store, report }; +} + +describe("bench", () => { + test("a cheap-first circuit matches frontier quality at frontier-call rate < 1", async () => { + const { report } = await bench(); + const byId = new Map(report.systems.map((s) => [s.id, s])); + expect(byId.get("cheap-single")!.passed).toBe(3); + expect(byId.get("frontier-single")!.passed).toBe(4); + expect(byId.get("circuit")!.passed).toBe(4); + // the circuit escalated once: quality of the frontier system at a + // fraction of its tokens + const circuit = byId.get("circuit")!; + expect(circuit.attribution["claude-opus"]!.calls).toBe(1); + expect(circuit.attribution["qwen-flash"]!.calls).toBe(4); + expect(circuit.usage.tokensIn).toBe(4 * CHEAP.tokensIn + FRONTIER.tokensIn); + expect(byId.get("frontier-single")!.usage.tokensIn).toBe(4 * FRONTIER.tokensIn); + // three-axis pareto: circuit dominates on tokens, frontier-single + // stays non-dominated on effect calls (4 vs the circuit's 5), + // cheap-single trades quality for cost + expect(report.pareto).toEqual(["circuit", "frontier-single", "cheap-single"]); + expect(report.workload).toMatch(/^sha256:[0-9a-f]{64}$/); + // the report embeds the workload so verification is self-contained + const reparsed = parseBenchReport(JSON.parse(canonicalize(report as unknown as JsonValue))); + expect(reparsed.digest).toBe(report.digest); + expect(reparsed.cases).toEqual(cases); + }); + + test("bench reports verify offline and detect tampering", async () => { + const { store, report } = await bench(); + const verified = await verifyBenchReport(report, store, builtinRegistry()); + expect(verified.ok).toBe(true); + expect(verified.checkedReceipts).toBe(12); + expect(verified.mismatches).toEqual([]); + + // recomputed-digest tampering still fails: the pass claim no longer + // matches the recorded outcome and outputs + const tampered = JSON.parse(canonicalize(report as unknown as JsonValue)) as { + systems: { cases: { passed: boolean }[] }[]; + digest?: string; + }; + tampered.systems[0]!.cases[3]!.passed = true; + const { digestCanonical } = await import("./digest"); + const { digest: _d, ...base } = tampered as Record & { digest: string }; + tampered.digest = digestCanonical(base as JsonValue); + const again = await verifyBenchReport(tampered, store, builtinRegistry()); + expect(again.ok).toBe(false); + expect(again.mismatches.some((m) => m.includes("invalid pass claim") || m.includes("passed does not match"))).toBe(true); + }); + + test("validation: bounds, unique ids, and interface coverage", async () => { + const store = new MemoryStore(); + const fns = builtinRegistry(); + const cheap = metered("cheap", "qwen-flash", { route: "billing" }, CHEAP); + await expect( + runBenchmark({ fns, store, cases: [], systems: [{ id: "a", manifest: single, executors: [cheap] }] }), + ).rejects.toThrow(/at least one case/); + await expect( + runBenchmark({ + fns, + store, + cases, + systems: Array.from({ length: 9 }, (_, i) => ({ + id: `s${i}`, + manifest: single, + executors: [cheap], + })), + }), + ).rejects.toThrow(/exceed 8/); + await expect( + runBenchmark({ + fns, + store, + cases, + systems: [ + { id: "dup", manifest: single, executors: [cheap] }, + { id: "dup", manifest: single, executors: [cheap] }, + ], + }), + ).rejects.toThrow(/duplicate bench system id/); + await expect( + runBenchmark({ + fns, + store, + cases: [{ id: "x1", args: { bogus: "hi" }, expect: { out: "billing" } }], + systems: [{ id: "a", manifest: single, executors: [cheap] }], + }), + ).rejects.toThrow(/unknown system input/); + await expect( + runBenchmark({ + fns, + store, + cases: [{ id: "x1", args: { ticket: "hi" }, expect: { wrong: "billing" } }], + systems: [{ id: "a", manifest: single, executors: [cheap] }], + }), + ).rejects.toThrow(/unknown expected output|missing expected output/); + }); +}); diff --git a/src/bench.ts b/src/bench.ts new file mode 100644 index 0000000..9359a43 --- /dev/null +++ b/src/bench.ts @@ -0,0 +1,340 @@ +// Benchmark evaluation: one workload, several systems, quality/cost evidence. +// A system is an admitted organism plus a host-resolved executor list — a +// cheap single call, a frontier single call, and a decomposed circuit are +// all just systems. Every case result is a replayable run receipt, so the +// report is a content-addressed claim about a Pareto comparison, not a +// leaderboard screenshot. There is no promotion here and no split: a bench +// measures, the foundry selects. + +import type { OrganismManifest } from "./contract"; +import { digestCanonical, type Digest } from "./digest"; +import type { EffectReceipt, Executor } from "./effects"; +import { MorphogenError } from "./errors"; +import type { FnRegistry } from "./registry"; +import { runOrganism } from "./run"; +import type { Store } from "./store"; +import type { Transport } from "./transport"; +import type { ToolRegistry } from "./tools"; +import { canonicalize, type JsonValue } from "./values"; + +export const BENCH_CONTRACT = "morphogen.bench.v1" as const; + +export const BENCH_BOUNDS = { + maxSystems: 8, + maxCases: 256, + maxIdLen: 64, +} as const; + +export type BenchCase = { + id: string; + args: Record; + expect: Record; +}; + +export type BenchSystem = { + id: string; + manifest: OrganismManifest; + /** Host-resolved executors: the first is the default; named ids are + * reached through route.provider / route.preset. A cheap-first cascade + * is a system whose manifest routes escalation cells at a second entry. */ + executors: Executor[]; +}; + +export type BenchPrice = { + input: number; + output: number; +}; + +export type BenchOptions = { + systems: BenchSystem[]; + cases: BenchCase[]; + fns: FnRegistry; + store: Store; + transports?: Record; + tools?: ToolRegistry; + /** Optional per-attribution price card, in USD per 1M tokens. + * Attribution keys are `usage.model` (e.g. "alibaba/qwen3.5-flash") + * or `effect.executor` for tool/scripted runs. */ + prices?: Record; +}; + +/** Effect attribution: calls, tokens, and optional cost grouped by the + * recorded model, or by executor id when no model is reported (tools, + * scripted runs). */ +export type BenchAttribution = { + calls: number; + tokensIn: number; + tokensOut: number; + cost: number; +}; + +export type BenchCaseResult = { + id: string; + passed: boolean; + outcome: "complete" | "failed" | "stuck"; + outputs: Record; + expect: Record; + receiptDigest: Digest; + effectCalls: number; + work: { steps: number; agentCalls: number; units: number }; + usage: { tokensIn: number; tokensOut: number; cost: number }; + attribution: Record; +}; + +export type BenchSystemResult = { + id: string; + manifestDigest: Digest; + manifestKey: string; + passed: number; + total: number; + effectCalls: number; + work: { steps: number; agentCalls: number; units: number }; + usage: { tokensIn: number; tokensOut: number; cost: number }; + attribution: Record; + cases: BenchCaseResult[]; +}; + +export type BenchReport = { + contract: typeof BENCH_CONTRACT; + /** Digest of the canonical case list — what every system was measured on. */ + workload: Digest; + /** The full case list, so a verifier needs no config to check provenance. */ + cases: BenchCase[]; + /** Optional USD-per-1M-token price card used to compute `cost`. */ + prices?: Record | undefined; + systems: BenchSystemResult[]; + /** Non-dominated system ids (passed ↑, cost signal ↓, calls ↓). */ + pareto: string[]; + digest: Digest; +}; + +function fail(message: string): never { + throw new MorphogenError("PARSE_FAILED", message); +} + +function validate(opts: BenchOptions): void { + if (opts.systems.length === 0) fail("bench requires at least one system"); + if (opts.systems.length > BENCH_BOUNDS.maxSystems) { + fail(`bench systems exceed ${BENCH_BOUNDS.maxSystems}`); + } + if (opts.cases.length === 0) fail("bench requires at least one case"); + if (opts.cases.length > BENCH_BOUNDS.maxCases) { + fail(`bench cases exceed ${BENCH_BOUNDS.maxCases}`); + } + const caseIds = new Set(); + for (const c of opts.cases) { + if (!/^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(c.id) || c.id.length > BENCH_BOUNDS.maxIdLen) { + fail(`invalid bench case id "${c.id}"`); + } + if (caseIds.has(c.id)) fail(`duplicate bench case id "${c.id}"`); + caseIds.add(c.id); + } + const systemIds = new Set(); + for (const system of opts.systems) { + if (!/^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(system.id) || system.id.length > BENCH_BOUNDS.maxIdLen) { + fail(`invalid bench system id "${system.id}"`); + } + if (systemIds.has(system.id)) fail(`duplicate bench system id "${system.id}"`); + systemIds.add(system.id); + if (system.executors.length === 0) { + fail(`bench system "${system.id}" requires at least one executor`); + } + if (!system.manifest.interface) { + fail(`bench system "${system.id}" manifest must declare an interface`); + } + const inputs = new Set(Object.keys(system.manifest.interface.inputs)); + const outputs = new Set(Object.keys(system.manifest.interface.outputs)); + for (const c of opts.cases) { + for (const name of Object.keys(c.args)) { + if (!inputs.has(name)) fail(`case ${c.id}: unknown system input "${name}"`); + } + for (const name of outputs) { + if (!(name in c.expect)) fail(`case ${c.id}: missing expected output "${name}"`); + } + for (const name of Object.keys(c.expect)) { + if (!outputs.has(name)) fail(`case ${c.id}: unknown expected output "${name}"`); + } + } + } +} + +function caseArgs( + manifest: OrganismManifest, + c: BenchCase, +): Record> { + const args: Record> = Object.create(null) as Record>; + for (const [name, value] of Object.entries(c.args)) { + const target = manifest.interface!.inputs[name]!; + (args[target.cell] ??= Object.create(null) as Record)[target.port] = value; + } + return args; +} + +function costFor( + price: BenchPrice | undefined, + tokensIn: number, + tokensOut: number, +): number { + if (!price) return 0; + return (tokensIn * price.input + tokensOut * price.output) / 1_000_000; +} + +function attribute( + effects: EffectReceipt[], + prices?: Record, +): { + usage: { tokensIn: number; tokensOut: number; cost: number }; + attribution: Record; +} { + const attribution: Record = Object.create(null) as Record; + const usage = { tokensIn: 0, tokensOut: 0, cost: 0 }; + for (const effect of effects) { + const key = effect.usage?.model ?? effect.executor; + const entry = (attribution[key] ??= { + calls: 0, + tokensIn: 0, + tokensOut: 0, + cost: 0, + }); + entry.calls += 1; + const tokensIn = effect.usage?.tokensIn ?? 0; + const tokensOut = effect.usage?.tokensOut ?? 0; + entry.tokensIn += tokensIn; + entry.tokensOut += tokensOut; + const extraCost = costFor(prices?.[key], tokensIn, tokensOut); + entry.cost += extraCost; + usage.tokensIn += tokensIn; + usage.tokensOut += tokensOut; + usage.cost += extraCost; + } + return { usage, attribution }; +} + +function mergeAttribution( + into: Record, + from: Record, +): void { + for (const [key, value] of Object.entries(from)) { + const entry = (into[key] ??= { calls: 0, tokensIn: 0, tokensOut: 0, cost: 0 }); + entry.calls += value.calls; + entry.tokensIn += value.tokensIn; + entry.tokensOut += value.tokensOut; + entry.cost += value.cost; + } +} + +async function evaluateCase( + system: BenchSystem, + c: BenchCase, + opts: BenchOptions, +): Promise { + const receipt = await runOrganism({ + manifest: system.manifest, + args: caseArgs(system.manifest, c), + fns: opts.fns, + store: opts.store, + executors: system.executors, + ...(opts.transports ? { transports: opts.transports } : {}), + ...(opts.tools ? { tools: opts.tools } : {}), + }); + const receiptDigest = await opts.store.putReceipt(receipt as unknown as JsonValue); + const outputs: Record = Object.create(null) as Record; + for (const [name, source] of Object.entries(system.manifest.interface!.outputs)) { + const value = receipt.cells[source.cell]?.outputs?.[source.port]; + if (value !== undefined) outputs[name] = value; + } + const { usage, attribution } = attribute(receipt.effects, opts.prices); + return { + id: c.id, + passed: receipt.outcome === "complete" && canonicalize(outputs) === canonicalize(c.expect), + outcome: receipt.outcome, + outputs, + expect: c.expect, + receiptDigest, + effectCalls: receipt.effects.length, + work: receipt.work, + usage, + attribution, + }; +} + +/** Non-dominated systems on (passed ↑, cost signal ↓, effect calls ↓). + * The cost signal is the dollar `cost` when prices were supplied, + * otherwise total token count. A system is dominated when another is at + * least as good on all three axes and strictly better on one — + * deterministic, ties broken by id. */ +export function benchPareto( + systems: BenchSystemResult[], + hasPrices = false, +): string[] { + const tokens = (s: BenchSystemResult) => s.usage.tokensIn + s.usage.tokensOut; + const cost = (s: BenchSystemResult) => (hasPrices ? s.usage.cost : tokens(s)); + const calls = (s: BenchSystemResult) => s.effectCalls; + const kept = systems.filter( + (s) => + !systems.some( + (o) => + o.id !== s.id && + o.passed >= s.passed && + cost(o) <= cost(s) && + calls(o) <= calls(s) && + (o.passed > s.passed || cost(o) < cost(s) || calls(o) < calls(s)), + ), + ); + return kept + .sort( + (a, b) => + b.passed - a.passed || + cost(a) - cost(b) || + calls(a) - calls(b) || + a.id.localeCompare(b.id), + ) + .map((s) => s.id); +} + +export async function runBenchmark(opts: BenchOptions): Promise { + validate(opts); + const systems: BenchSystemResult[] = []; + for (const system of opts.systems) { + const manifestDigest = await opts.store.putManifest(system.manifest); + const cases: BenchCaseResult[] = []; + for (const c of opts.cases) cases.push(await evaluateCase(system, c, opts)); + const attribution: Record = Object.create(null) as Record; + for (const c of cases) mergeAttribution(attribution, c.attribution); + systems.push({ + id: system.id, + manifestDigest, + manifestKey: system.manifest.key, + passed: cases.filter((c) => c.passed).length, + total: cases.length, + effectCalls: cases.reduce((t, c) => t + c.effectCalls, 0), + work: cases.reduce( + (t, c) => ({ + steps: t.steps + c.work.steps, + agentCalls: t.agentCalls + c.work.agentCalls, + units: t.units + c.work.units, + }), + { steps: 0, agentCalls: 0, units: 0 }, + ), + usage: cases.reduce( + (t, c) => ({ + tokensIn: t.tokensIn + c.usage.tokensIn, + tokensOut: t.tokensOut + c.usage.tokensOut, + cost: t.cost + c.usage.cost, + }), + { tokensIn: 0, tokensOut: 0, cost: 0 }, + ), + attribution, + cases, + }); + } + const base: Omit = { + contract: BENCH_CONTRACT, + workload: digestCanonical(opts.cases as unknown as JsonValue), + cases: opts.cases, + systems, + pareto: benchPareto(systems, opts.prices !== undefined), + }; + if (opts.prices !== undefined) base.prices = opts.prices; + return { ...base, digest: digestCanonical(base as unknown as JsonValue) }; +} diff --git a/src/contract.test.ts b/src/contract.test.ts index 81e74e2..dbf56da 100644 --- a/src/contract.test.ts +++ b/src/contract.test.ts @@ -615,4 +615,29 @@ describe("slot cells", () => { }), ).toThrow("unknown key"); }); + + test("tool cells parse, bound time, and round-trip", () => { + const raw = { + contract: "morphogen.organism.v1", + key: "organism:tool", + name: "Tool", + cells: [{ + id: "lookup", + kind: "tool", + tool: "records.lookup.v1", + budget: { maxEffectMs: 1000 }, + }], + edges: [], + }; + const parsed = parseOrganismManifest(raw); + expect(manifestToJson(parseOrganismManifest(manifestToJson(parsed)))).toEqual(manifestToJson(parsed)); + expect(() => parseOrganismManifest({ + ...raw, + cells: [{ id: "lookup", kind: "tool", tool: "records.lookup.v1", budget: { maxEffectMs: 0 } }], + })).toThrow("maxEffectMs"); + expect(() => parseOrganismManifest({ + ...raw, + cells: [{ id: "lookup", kind: "tool", tool: "records.lookup.v1", shell: "curl" }], + })).toThrow("unknown key"); + }); }); diff --git a/src/contract.ts b/src/contract.ts index 69608da..f3bf893 100644 --- a/src/contract.ts +++ b/src/contract.ts @@ -131,6 +131,7 @@ export type Cell = | { id: string; kind: "input"; outputs: PortMap } | { id: string; kind: "const"; outputs: Record } | { id: string; kind: "fn"; fn: string } + | { id: string; kind: "tool"; tool: string; budget?: { maxEffectMs?: number } } | { id: string; kind: "agent"; @@ -333,7 +334,7 @@ function fail(msg: string): never { throw new MorphogenError("PARSE_FAILED", msg); } -function parsePortMap( +export function parsePortMap( u: unknown, what: string, role: "consumer" | "producer" = "consumer", @@ -570,6 +571,30 @@ function parseCell(u: unknown, what: string): Cell { fn: asString(reqField(obj, "fn", what), `${what}.fn`, BOUNDS.maxRefLen), }; } + case "tool": { + noUnknownKeys(obj, ["id", "kind", "tool", "budget"], what); + const cell: Cell = { + id, + kind, + tool: asString(reqField(obj, "tool", what), `${what}.tool`, BOUNDS.maxRefLen), + }; + if (obj.budget !== undefined) { + const budget = asObject(obj.budget, `${what}.budget`); + noUnknownKeys(budget, ["maxEffectMs"], `${what}.budget`); + const maxEffectMs = optField(budget, "maxEffectMs"); + cell.budget = maxEffectMs === undefined + ? {} + : { + maxEffectMs: asInt( + maxEffectMs, + `${what}.budget.maxEffectMs`, + 1, + BOUNDS.maxEffectMs, + ), + }; + } + return cell; + } case "store": case "load": case "spawn": { @@ -1090,6 +1115,13 @@ export function manifestToJson(m: OrganismManifest): JsonObject { } case "fn": return { id: c.id, kind: c.kind, fn: c.fn }; + case "tool": + return { + id: c.id, + kind: c.kind, + tool: c.tool, + ...(c.budget ? { budget: c.budget as unknown as JsonValue } : {}), + }; case "store": case "load": case "spawn": diff --git a/src/effects.ts b/src/effects.ts index 4f3a99a..0c336cb 100644 --- a/src/effects.ts +++ b/src/effects.ts @@ -46,12 +46,24 @@ export type EffectReceipt = { cached?: boolean; }; +export type ExecutorMetadata = { + executor?: string; + usage?: EffectReceipt["usage"]; + cached?: boolean; +}; + +export type ExecutorResult = { + output: JsonValue; + metadata?: ExecutorMetadata; +}; + export type Executor = { id: string; /** `signal` aborts when the cell's `budget.maxEffectMs` fires — an * executor should treat abort as cancellation (commandExecutor kills its * process). Advisory: the runner already raced the call to a timeout. */ execute(request: EffectRequest, signal?: AbortSignal): Promise; + executeEffect?(request: EffectRequest, signal?: AbortSignal): Promise; /** Receipt metadata recorded for this request. Executors that replay a * prior run implement this so the rerun reproduces the original receipt's * executor id and usage — making verification bit-for-bit. Called before @@ -189,18 +201,38 @@ export function cachedExecutor(inner: Executor, store: Store): Executor { return inner.receiptFor?.(request) ?? {}; }, async execute(request, signal) { + return (await this.executeEffect!(request, signal)).output; + }, + async executeEffect(request, signal) { const hit = await lookup(request); - if (hit?.output !== undefined) return hit.output; - const out = await inner.execute(request, signal); - const meta = await inner.receiptFor?.(request); + if (hit?.output !== undefined) { + return { + output: hit.output, + metadata: { + executor: hit.executor, + ...(hit.usage ? { usage: hit.usage } : {}), + cached: true, + }, + }; + } + let result: ExecutorResult; + if (inner.executeEffect) { + result = await inner.executeEffect(request, signal); + } else { + const metadata = await inner.receiptFor?.(request); + result = { + output: await inner.execute(request, signal), + ...(metadata ? { metadata } : {}), + }; + } const entry: EffectReceipt = { requestDigest: effectRequestDigest(request), - executor: meta?.executor ?? inner.id, - output: out, + executor: result.metadata?.executor ?? inner.id, + output: result.output, }; - if (meta?.usage) entry.usage = meta.usage; + if (result.metadata?.usage) entry.usage = result.metadata.usage; await store.putEffect(entry); - return out; + return result; }, }; } diff --git a/src/errors.ts b/src/errors.ts index b7cb4f5..caad25a 100644 --- a/src/errors.ts +++ b/src/errors.ts @@ -12,6 +12,8 @@ export const ERROR_CODES = [ "INPUT_MISSING", "FN_UNKNOWN", "FN_FAILED", + "TOOL_UNKNOWN", + "TOOL_FAILED", "EFFECT_FAILED", "EFFECT_UNPARSEABLE", "EFFECT_UNBOUND", diff --git a/src/foundry-verify.ts b/src/foundry-verify.ts new file mode 100644 index 0000000..2be7473 --- /dev/null +++ b/src/foundry-verify.ts @@ -0,0 +1,312 @@ +import { manifestToJson } from "./contract"; +import { digestCanonical, type Digest } from "./digest"; +import { MorphogenError } from "./errors"; +import { + FOUNDRY_BOUNDS, + FOUNDRY_CONTRACT, + selectFoundryCandidate, + type FoundryCandidateResult, + type FoundryCaseResult, + type FoundryReport, +} from "./foundry"; +import type { FnRegistry } from "./registry"; +import { parseRunReceipt } from "./run"; +import type { Store } from "./store"; +import type { ToolRegistry } from "./tools"; +import { verifyReceipt } from "./verify"; +import { canonicalize, type JsonObject, type JsonValue } from "./values"; + +function object(value: unknown, at: string): JsonObject { + if (value === null || typeof value !== "object" || Array.isArray(value)) { + throw new MorphogenError("PARSE_FAILED", `${at} must be an object`); + } + return value as JsonObject; +} + +function keys(value: JsonObject, allowed: string[], at: string): void { + const extra = Object.keys(value).find((key) => !allowed.includes(key)); + if (extra) throw new MorphogenError("PARSE_FAILED", `${at}: unknown key "${extra}"`); +} + +function text(value: JsonValue | undefined, at: string): string { + if (typeof value !== "string") throw new MorphogenError("PARSE_FAILED", `${at} must be text`); + return value; +} + +function digest(value: JsonValue | undefined, at: string): Digest { + const parsed = text(value, at); + if (!/^sha256:[0-9a-f]{64}$/.test(parsed)) { + throw new MorphogenError("PARSE_FAILED", `${at} must be a sha256 digest`); + } + return parsed as Digest; +} + +function count(value: JsonValue | undefined, at: string): number { + if (!Number.isInteger(value) || (value as number) < 0) { + throw new MorphogenError("PARSE_FAILED", `${at} must be a non-negative integer`); + } + return value as number; +} + +function parseWork(value: JsonValue | undefined, at: string) { + const work = object(value, at); + keys(work, ["steps", "agentCalls", "units"], at); + return { + steps: count(work.steps, `${at}.steps`), + agentCalls: count(work.agentCalls, `${at}.agentCalls`), + units: count(work.units, `${at}.units`), + }; +} + +function parseUsage(value: JsonValue | undefined, at: string) { + const usage = object(value, at); + keys(usage, ["tokensIn", "tokensOut"], at); + return { + tokensIn: count(usage.tokensIn, `${at}.tokensIn`), + tokensOut: count(usage.tokensOut, `${at}.tokensOut`), + }; +} + +function parseScore(value: JsonValue | undefined, at: string) { + const score = object(value, at); + keys(score, ["passed", "total"], at); + const parsed = { + passed: count(score.passed, `${at}.passed`), + total: count(score.total, `${at}.total`), + }; + if (parsed.total === 0 || parsed.passed > parsed.total) { + throw new MorphogenError("PARSE_FAILED", `${at} is not a valid score`); + } + return parsed; +} + +function parseCase(value: JsonValue, at: string): FoundryCaseResult { + const c = object(value, at); + keys(c, ["id", "split", "passed", "outcome", "outputs", "expect", "receiptDigest", "work", "usage"], at); + const split = text(c.split, `${at}.split`); + const outcome = text(c.outcome, `${at}.outcome`); + if (split !== "train" && split !== "validation" && split !== "holdout") { + throw new MorphogenError("PARSE_FAILED", `${at}.split is invalid`); + } + if (outcome !== "complete" && outcome !== "failed" && outcome !== "stuck") { + throw new MorphogenError("PARSE_FAILED", `${at}.outcome is invalid`); + } + if (typeof c.passed !== "boolean") { + throw new MorphogenError("PARSE_FAILED", `${at}.passed must be boolean`); + } + return { + id: text(c.id, `${at}.id`), + split, + passed: c.passed, + outcome, + outputs: object(c.outputs, `${at}.outputs`), + expect: object(c.expect, `${at}.expect`), + receiptDigest: digest(c.receiptDigest, `${at}.receiptDigest`), + work: parseWork(c.work, `${at}.work`), + usage: parseUsage(c.usage, `${at}.usage`), + }; +} + +function parseCandidate(value: JsonValue, i: number): FoundryCandidateResult { + const at = `foundry.candidates[${i}]`; + const c = object(value, at); + keys(c, ["manifestDigest", "manifestKey", "train", "validation", "work", "usage", "cases"], at); + if (!Array.isArray(c.cases) || c.cases.length === 0 || c.cases.length > FOUNDRY_BOUNDS.maxCases) { + throw new MorphogenError("PARSE_FAILED", `${at}.cases must be a bounded non-empty list`); + } + return { + manifestDigest: digest(c.manifestDigest, `${at}.manifestDigest`), + manifestKey: text(c.manifestKey, `${at}.manifestKey`), + train: parseScore(c.train, `${at}.train`), + validation: parseScore(c.validation, `${at}.validation`), + work: parseWork(c.work, `${at}.work`), + usage: parseUsage(c.usage, `${at}.usage`), + cases: c.cases.map((entry, j) => parseCase(entry, `${at}.cases[${j}]`)), + }; +} + +export function parseFoundryReport(value: unknown): FoundryReport { + const report = object(value, "foundry"); + keys(report, ["contract", "candidates", "promoted", "holdout", "lineage", "digest"], "foundry"); + if (report.contract !== FOUNDRY_CONTRACT) { + throw new MorphogenError("PARSE_FAILED", `foundry.contract must be ${FOUNDRY_CONTRACT}`); + } + if (!Array.isArray(report.candidates) || report.candidates.length === 0 || report.candidates.length > FOUNDRY_BOUNDS.maxCandidates) { + throw new MorphogenError("PARSE_FAILED", "foundry.candidates must be a bounded non-empty list"); + } + const holdout = object(report.holdout, "foundry.holdout"); + keys(holdout, ["passed", "total", "cases"], "foundry.holdout"); + const holdoutScore = { + passed: count(holdout.passed, "foundry.holdout.passed"), + total: count(holdout.total, "foundry.holdout.total"), + }; + if (holdoutScore.total === 0 || holdoutScore.passed > holdoutScore.total) { + throw new MorphogenError("PARSE_FAILED", "foundry.holdout is not a valid score"); + } + if (!Array.isArray(holdout.cases) || holdout.cases.length !== holdoutScore.total) { + throw new MorphogenError("PARSE_FAILED", "foundry.holdout.cases must match its total"); + } + const lineage = report.lineage === undefined ? undefined : object(report.lineage, "foundry.lineage"); + if (lineage) keys(lineage, ["generatorDigest", "receiptDigest"], "foundry.lineage"); + return { + contract: FOUNDRY_CONTRACT, + candidates: report.candidates.map(parseCandidate), + promoted: digest(report.promoted, "foundry.promoted"), + holdout: { + ...holdoutScore, + cases: holdout.cases.map((entry, i) => parseCase(entry, `foundry.holdout.cases[${i}]`)), + }, + ...(lineage ? { + lineage: { + generatorDigest: digest(lineage.generatorDigest, "foundry.lineage.generatorDigest"), + receiptDigest: digest(lineage.receiptDigest, "foundry.lineage.receiptDigest"), + }, + } : {}), + digest: digest(report.digest, "foundry.digest"), + }; +} + +export type FoundryVerifyReport = { + ok: boolean; + digest: Digest; + checkedReceipts: number; + mismatches: string[]; +}; + +export async function verifyFoundryReport( + value: unknown, + store: Store, + fns: FnRegistry, + tools?: ToolRegistry, +): Promise { + const report = parseFoundryReport(value); + const mismatches: string[] = []; + const { digest: claimed, ...base } = report; + const actual = digestCanonical(base as unknown as JsonValue); + if (actual !== claimed) mismatches.push(`digest: claimed ${claimed}, computed ${actual}`); + if (selectFoundryCandidate(report.candidates) !== report.promoted) { + mismatches.push("promoted digest is not the deterministic winner"); + } + const checkScore = ( + label: string, + cases: FoundryCaseResult[], + split: FoundryCaseResult["split"], + claimedScore: { passed: number; total: number }, + ) => { + const selected = cases.filter((c) => c.split === split); + const passed = selected.filter((c) => c.passed).length; + if (selected.length !== claimedScore.total || passed !== claimedScore.passed) { + mismatches.push(`${label} score does not match its cases`); + } + for (const c of selected) { + const expectedPass = c.outcome === "complete" && canonicalize(c.outputs) === canonicalize(c.expect); + if (c.passed !== expectedPass) mismatches.push(`${label} case ${c.id} has an invalid pass claim`); + } + }; + for (const candidate of report.candidates) { + checkScore(candidate.manifestKey, candidate.cases, "train", candidate.train); + checkScore(candidate.manifestKey, candidate.cases, "validation", candidate.validation); + if (candidate.cases.some((c) => c.split === "holdout")) { + mismatches.push(`${candidate.manifestKey} exposes holdout results before promotion`); + } + const work = candidate.cases.reduce( + (total, c) => ({ + steps: total.steps + c.work.steps, + agentCalls: total.agentCalls + c.work.agentCalls, + units: total.units + c.work.units, + }), + { steps: 0, agentCalls: 0, units: 0 }, + ); + if (canonicalize(work as unknown as JsonValue) !== canonicalize(candidate.work as unknown as JsonValue)) { + mismatches.push(`${candidate.manifestKey} work does not match its cases`); + } + const usage = candidate.cases.reduce( + (total, c) => ({ + tokensIn: total.tokensIn + c.usage.tokensIn, + tokensOut: total.tokensOut + c.usage.tokensOut, + }), + { tokensIn: 0, tokensOut: 0 }, + ); + if (canonicalize(usage as unknown as JsonValue) !== canonicalize(candidate.usage as unknown as JsonValue)) { + mismatches.push(`${candidate.manifestKey} usage does not match its cases`); + } + } + checkScore("holdout", report.holdout.cases, "holdout", report.holdout); + if (report.holdout.cases.some((c) => c.split !== "holdout")) { + mismatches.push("holdout contains a non-holdout case"); + } + let checkedReceipts = 0; + const verifyCases = async ( + manifestDigest: Digest, + cases: FoundryCaseResult[], + verifyClaims = true, + ) => { + const manifest = await store.getManifest(manifestDigest); + if (!manifest) { + mismatches.push(`manifest ${manifestDigest} missing`); + return; + } + for (const c of cases) { + const stored = await store.getReceipt(c.receiptDigest); + if (!stored) { + mismatches.push(`receipt ${c.receiptDigest} missing`); + continue; + } + const receipt = parseRunReceipt(stored); + if (receipt.manifestDigest !== manifestDigest) { + mismatches.push(`receipt ${c.receiptDigest} ran ${receipt.manifestDigest}, expected ${manifestDigest}`); + continue; + } + if (verifyClaims) { + const outputs: Record = {}; + for (const [name, source] of Object.entries(manifest.interface?.outputs ?? {})) { + const output = receipt.cells[source.cell]?.outputs?.[source.port]; + if (output !== undefined) outputs[name] = output; + } + if (receipt.outcome !== c.outcome) mismatches.push(`case ${c.id}: outcome differs from receipt`); + if (canonicalize(outputs) !== canonicalize(c.outputs)) { + mismatches.push(`case ${c.id}: outputs differ from receipt`); + } + if (canonicalize(receipt.work as unknown as JsonValue) !== canonicalize(c.work as unknown as JsonValue)) { + mismatches.push(`case ${c.id}: work differs from receipt`); + } + const usage = receipt.effects.reduce( + (total, effect) => ({ + tokensIn: total.tokensIn + (effect.usage?.tokensIn ?? 0), + tokensOut: total.tokensOut + (effect.usage?.tokensOut ?? 0), + }), + { tokensIn: 0, tokensOut: 0 }, + ); + if (canonicalize(usage as unknown as JsonValue) !== canonicalize(c.usage as unknown as JsonValue)) { + mismatches.push(`case ${c.id}: usage differs from receipt`); + } + } + const verified = await verifyReceipt( + receipt, + manifestToJson(manifest), + store, + fns, + undefined, + tools, + ); + checkedReceipts++; + if (!verified.ok) mismatches.push(`receipt ${c.receiptDigest}: ${verified.mismatches.join("; ")}`); + } + }; + for (const candidate of report.candidates) { + await verifyCases(candidate.manifestDigest, candidate.cases); + } + await verifyCases(report.promoted, report.holdout.cases); + if (report.lineage) await verifyCases(report.lineage.generatorDigest, [{ + id: "generator", + split: "train", + passed: true, + outcome: "complete", + outputs: {}, + expect: {}, + receiptDigest: report.lineage.receiptDigest, + work: { steps: 0, agentCalls: 0, units: 0 }, + usage: { tokensIn: 0, tokensOut: 0 }, + }], false); + return { ok: mismatches.length === 0, digest: claimed, checkedReceipts, mismatches }; +} diff --git a/src/foundry.test.ts b/src/foundry.test.ts new file mode 100644 index 0000000..8df0d70 --- /dev/null +++ b/src/foundry.test.ts @@ -0,0 +1,254 @@ +import { describe, expect, test } from "bun:test"; +import { manifestToJson, parseOrganismManifest } from "./contract"; +import { digestCanonical } from "./digest"; +import { generateFoundryCandidates, runFoundry } from "./foundry"; +import { verifyFoundryReport } from "./foundry-verify"; +import { builtinRegistry } from "./registry"; +import { runFoundrySearch } from "./search"; +import { verifySearchReport } from "./search-verify"; +import { MemoryStore } from "./store"; + +const echo = parseOrganismManifest({ + contract: "morphogen.organism.v1", + key: "organism:echo-candidate", + name: "Echo candidate", + interface: { + inputs: { q: { cell: "src", port: "value" } }, + outputs: { answer: { cell: "echo", port: "value" } }, + }, + cells: [ + { id: "src", kind: "input", outputs: { value: "json" } }, + { id: "echo", kind: "fn", fn: "echo.v1" }, + ], + edges: [ + { from: { cell: "src", port: "value" }, to: { cell: "echo", port: "value" } }, + ], +}); + +const constant = parseOrganismManifest({ + contract: "morphogen.organism.v1", + key: "organism:constant-candidate", + name: "Constant candidate", + interface: { + inputs: { q: { cell: "src", port: "value" } }, + outputs: { answer: { cell: "out", port: "value" } }, + }, + cells: [ + { id: "src", kind: "input", outputs: { value: "json" } }, + { id: "out", kind: "const", outputs: { value: { type: "json", value: "a" } } }, + ], + edges: [], +}); + +describe("foundry", () => { + test("search carries validation evidence across bounded generations without exposing holdout", async () => { + const generator = parseOrganismManifest({ + contract: "morphogen.organism.v1", + key: "organism:evolving-generator", + name: "Evolving generator", + interface: { + inputs: { feedback: { cell: "src", port: "feedback" } }, + outputs: { candidates: { cell: "writer", port: "out" } }, + }, + cells: [ + { id: "src", kind: "input", outputs: { feedback: "json" } }, + { + id: "writer", + kind: "agent", + inputs: { feedback: "json" }, + prompt: "Improve the candidate population from validation evidence.", + view: { inputs: ["feedback"] }, + output: { + kind: "json", + schema: { + type: "object", + required: ["candidates"], + properties: { candidates: { type: "array" } }, + }, + }, + }, + ], + edges: [ + { from: { cell: "src", port: "feedback" }, to: { cell: "writer", port: "feedback" } }, + ], + }); + let calls = 0; + const store = new MemoryStore(); + const result = await runFoundrySearch({ + generator, + generatorArgs: {}, + feedbackInput: "feedback", + output: "candidates", + field: "candidates", + cases: [ + { id: "train-a", split: "train", args: { q: "a" }, expect: { answer: "a" } }, + { id: "validation-b", split: "validation", args: { q: "b" }, expect: { answer: "b" } }, + { id: "holdout-c", split: "holdout", args: { q: "c" }, expect: { answer: "c" } }, + ], + maxGenerations: 2, + fns: builtinRegistry(), + store, + executors: [{ + id: "evolver", + async execute() { + return { candidates: [manifestToJson(calls++ === 0 ? constant : echo)] }; + }, + }], + }); + + expect(result.generations).toHaveLength(2); + expect(result.generations[0]?.candidates[0]?.validation.passed).toBe(0); + expect(result.generations[1]?.candidates).toHaveLength(2); + expect(result.generations.every((generation) => + generation.candidates.every((candidate) => candidate.cases.every((c) => c.split !== "holdout")), + )).toBe(true); + expect(result.result.holdout.passed).toBe(1); + expect(result.result.promoted).toBe(digestCanonical(manifestToJson(echo))); + const verified = await verifySearchReport(result, store, builtinRegistry()); + expect(verified.ok).toBe(true); + expect(verified.checkedReceipts).toBeGreaterThan(0); + }); + + test("runs an organism that emits candidate manifests and records its lineage", async () => { + const generator = parseOrganismManifest({ + contract: "morphogen.organism.v1", + key: "organism:test-generator", + name: "Test generator", + interface: { inputs: {}, outputs: { candidates: { cell: "batch", port: "value" } } }, + cells: [{ + id: "batch", + kind: "const", + outputs: { + value: { + type: "json", + value: [manifestToJson(constant), manifestToJson(echo)], + }, + }, + }], + edges: [], + }); + const store = new MemoryStore(); + const generated = await generateFoundryCandidates({ + generator, + args: {}, + output: "candidates", + fns: builtinRegistry(), + store, + executors: [], + }); + const result = await runFoundry({ + candidates: generated.candidates, + cases: [ + { id: "train-a", split: "train", args: { q: "a" }, expect: { answer: "a" } }, + { id: "validation-b", split: "validation", args: { q: "b" }, expect: { answer: "b" } }, + { id: "holdout-c", split: "holdout", args: { q: "c" }, expect: { answer: "c" } }, + ], + fns: builtinRegistry(), + store, + executors: [], + lineage: { + generatorDigest: generated.generatorDigest, + receiptDigest: generated.receiptDigest, + }, + }); + + expect(generated.candidates).toHaveLength(2); + expect(generated.receiptDigest).toMatch(/^sha256:/); + expect(result.lineage?.generatorDigest).toBe(generated.generatorDigest); + expect(result.promoted).toBe(result.candidates[1]!.manifestDigest); + const verified = await verifyFoundryReport(result, store, builtinRegistry()); + expect(verified.ok).toBe(true); + expect(verified.checkedReceipts).toBe(6); + + const tampered = structuredClone(result); + tampered.holdout.cases[0]!.expect.answer = "wrong"; + const { digest: _digest, ...tamperedBase } = tampered; + tampered.digest = digestCanonical(tamperedBase as never); + const rejected = await verifyFoundryReport(tampered, store, builtinRegistry()); + expect(rejected.ok).toBe(false); + expect(rejected.mismatches).toContain("holdout case holdout-c has an invalid pass claim"); + }); + + test("evaluates train and validation cases and promotes the best candidate", async () => { + const store = new MemoryStore(); + const result = await runFoundry({ + candidates: [constant, echo], + cases: [ + { id: "train-a", split: "train", args: { q: "a" }, expect: { answer: "a" } }, + { id: "validation-b", split: "validation", args: { q: "b" }, expect: { answer: "b" } }, + { id: "holdout-c", split: "holdout", args: { q: "c" }, expect: { answer: "c" } }, + ], + fns: builtinRegistry(), + store, + executors: [], + }); + + expect(result.contract).toBe("morphogen.foundry.v1"); + expect(result.candidates).toHaveLength(2); + expect(result.candidates[0]?.train.passed).toBe(1); + expect(result.candidates[0]?.validation.passed).toBe(0); + expect(result.candidates[1]?.validation.passed).toBe(1); + expect(result.promoted).toBe(result.candidates[1]!.manifestDigest); + expect(result.holdout.passed).toBe(1); + expect(result.holdout.cases[0]?.outputs).toEqual({ answer: "c" }); + expect(result.candidates[1]?.cases[1]?.receiptDigest).toMatch(/^sha256:/); + expect(await store.getReceipt(result.candidates[1]!.cases[1]!.receiptDigest!)).toBeDefined(); + }); + + test("rejects duplicate case ids and candidates without interfaces", async () => { + const noInterface = parseOrganismManifest({ + contract: "morphogen.organism.v1", + key: "organism:no-interface", + name: "No interface", + cells: [{ id: "x", kind: "const", outputs: { value: { type: "text", value: "x" } } }], + edges: [], + }); + const base = { + fns: builtinRegistry(), + store: new MemoryStore(), + executors: [], + }; + + await expect(runFoundry({ + ...base, + candidates: [echo], + cases: [ + { id: "same", split: "train" as const, args: { q: "a" }, expect: { answer: "a" } }, + { id: "same", split: "validation" as const, args: { q: "b" }, expect: { answer: "b" } }, + { id: "holdout", split: "holdout" as const, args: { q: "c" }, expect: { answer: "c" } }, + ], + })).rejects.toThrow("duplicate foundry case id"); + await expect(runFoundry({ + ...base, + candidates: [noInterface], + cases: [ + { id: "train", split: "train", args: {}, expect: {} }, + { id: "validation", split: "validation", args: {}, expect: {} }, + { id: "holdout", split: "holdout", args: {}, expect: {} }, + ], + })).rejects.toThrow("must declare an interface"); + }); + + test("bounds candidate populations", async () => { + await expect(runFoundry({ + candidates: Array.from({ length: 33 }, () => echo), + cases: [ + { id: "train", split: "train", args: { q: "a" }, expect: { answer: "a" } }, + { id: "validation", split: "validation", args: { q: "b" }, expect: { answer: "b" } }, + ], + fns: builtinRegistry(), + store: new MemoryStore(), + executors: [], + })).rejects.toThrow("candidates exceed 32"); + }); + + test("requires both train and validation cases", async () => { + await expect(runFoundry({ + candidates: [echo], + cases: [{ id: "one", split: "train", args: { q: "a" }, expect: { answer: "a" } }], + fns: builtinRegistry(), + store: new MemoryStore(), + executors: [], + })).rejects.toThrow("at least one validation case"); + }); +}); diff --git a/src/foundry.ts b/src/foundry.ts new file mode 100644 index 0000000..b15f7c3 --- /dev/null +++ b/src/foundry.ts @@ -0,0 +1,337 @@ +import type { OrganismManifest } from "./contract"; +import { manifestToJson, parseOrganismManifest } from "./contract"; +import { digestCanonical, type Digest } from "./digest"; +import type { Executor } from "./effects"; +import { MorphogenError } from "./errors"; +import type { FnRegistry } from "./registry"; +import { runOrganism } from "./run"; +import type { Store } from "./store"; +import type { Transport } from "./transport"; +import type { ToolRegistry } from "./tools"; +import { canonicalize, type JsonValue } from "./values"; + +export const FOUNDRY_CONTRACT = "morphogen.foundry.v1" as const; + +export const FOUNDRY_BOUNDS = { + maxCandidates: 32, + maxCases: 256, + maxCaseIdLen: 64, +} as const; + +export type FoundryCase = { + id: string; + split: "train" | "validation" | "holdout"; + args: Record; + expect: Record; +}; + +export type FoundryCaseResult = { + id: string; + split: "train" | "validation" | "holdout"; + passed: boolean; + outcome: "complete" | "failed" | "stuck"; + outputs: Record; + expect: Record; + receiptDigest: Digest; + work: { steps: number; agentCalls: number; units: number }; + usage: { tokensIn: number; tokensOut: number }; +}; + +export type FoundryCandidateResult = { + manifestDigest: Digest; + manifestKey: string; + train: { passed: number; total: number }; + validation: { passed: number; total: number }; + work: { steps: number; agentCalls: number; units: number }; + usage: { tokensIn: number; tokensOut: number }; + cases: FoundryCaseResult[]; +}; + +export type FoundryReport = { + contract: typeof FOUNDRY_CONTRACT; + candidates: FoundryCandidateResult[]; + promoted: Digest; + holdout: { passed: number; total: number; cases: FoundryCaseResult[] }; + lineage?: FoundryLineage; + digest: Digest; +}; + +export type FoundryLineage = { + generatorDigest: Digest; + receiptDigest: Digest; +}; + +export type FoundryOptions = { + candidates: OrganismManifest[]; + cases: FoundryCase[]; + fns: FnRegistry; + store: Store; + executors: Executor[]; + transports?: Record; + tools?: ToolRegistry; + lineage?: FoundryLineage; +}; + +export type GenerateCandidatesOptions = { + generator: OrganismManifest; + args: Record; + output: string; + field?: string; + fns: FnRegistry; + store: Store; + executors: Executor[]; + transports?: Record; + tools?: ToolRegistry; +}; + +export type GeneratedCandidates = FoundryLineage & { + candidates: OrganismManifest[]; +}; + +function fail(message: string): never { + throw new MorphogenError("PARSE_FAILED", message); +} + +function validate(opts: FoundryOptions): void { + if (opts.candidates.length === 0) fail("foundry requires at least one candidate"); + if (opts.candidates.length > FOUNDRY_BOUNDS.maxCandidates) { + fail(`foundry candidates exceed ${FOUNDRY_BOUNDS.maxCandidates}`); + } + if (opts.cases.length === 0) fail("foundry requires at least one case"); + if (opts.cases.length > FOUNDRY_BOUNDS.maxCases) { + fail(`foundry cases exceed ${FOUNDRY_BOUNDS.maxCases}`); + } + if (!opts.cases.some((c) => c.split === "train")) { + fail("foundry requires at least one train case"); + } + if (!opts.cases.some((c) => c.split === "validation")) { + fail("foundry requires at least one validation case"); + } + if (!opts.cases.some((c) => c.split === "holdout")) { + fail("foundry requires at least one holdout case"); + } + const ids = new Set(); + for (const c of opts.cases) { + if (!/^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(c.id) || c.id.length > FOUNDRY_BOUNDS.maxCaseIdLen) { + fail(`invalid foundry case id "${c.id}"`); + } + if (ids.has(c.id)) fail(`duplicate foundry case id "${c.id}"`); + ids.add(c.id); + } + const digests = new Set(); + for (const candidate of opts.candidates) { + if (!candidate.interface) fail(`candidate ${candidate.key} must declare an interface`); + const digest = digestCanonical(manifestToJson(candidate)); + if (digests.has(digest)) fail(`duplicate foundry candidate ${digest}`); + digests.add(digest); + const inputs = new Set(Object.keys(candidate.interface.inputs)); + const outputs = new Set(Object.keys(candidate.interface.outputs)); + for (const c of opts.cases) { + for (const name of Object.keys(c.args)) { + if (!inputs.has(name)) fail(`case ${c.id}: unknown candidate input "${name}"`); + } + for (const name of outputs) { + if (!(name in c.expect)) fail(`case ${c.id}: missing expected output "${name}"`); + } + for (const name of Object.keys(c.expect)) { + if (!outputs.has(name)) fail(`case ${c.id}: unknown candidate output "${name}"`); + } + } + } +} + +function caseArgs(candidate: OrganismManifest, c: FoundryCase): Record> { + const args: Record> = Object.create(null) as Record>; + for (const [name, value] of Object.entries(c.args)) { + const target = candidate.interface!.inputs[name]!; + (args[target.cell] ??= Object.create(null) as Record)[target.port] = value; + } + return args; +} + +function caseOutputs(candidate: OrganismManifest, cells: Awaited>["cells"]): Record { + const outputs: Record = Object.create(null) as Record; + for (const [name, source] of Object.entries(candidate.interface!.outputs)) { + const value = cells[source.cell]?.outputs?.[source.port]; + if (value !== undefined) outputs[name] = value; + } + return outputs; +} + +function score(cases: FoundryCaseResult[], split: FoundryCase["split"]) { + const selected = cases.filter((c) => c.split === split); + return { passed: selected.filter((c) => c.passed).length, total: selected.length }; +} + +function better(a: FoundryCandidateResult, b: FoundryCandidateResult): number { + const ah = a.validation.passed / a.validation.total; + const bh = b.validation.passed / b.validation.total; + if (ah !== bh) return bh - ah; + const at = a.train.passed / a.train.total; + const bt = b.train.passed / b.train.total; + if (at !== bt) return bt - at; + if (a.work.agentCalls !== b.work.agentCalls) return a.work.agentCalls - b.work.agentCalls; + if (a.work.units !== b.work.units) return a.work.units - b.work.units; + return a.manifestDigest.localeCompare(b.manifestDigest); +} + +export function selectFoundryCandidate(candidates: FoundryCandidateResult[]): Digest { + if (candidates.length === 0) fail("foundry requires at least one candidate result"); + return [...candidates].sort(better)[0]!.manifestDigest; +} + +async function evaluateCase( + candidate: OrganismManifest, + c: FoundryCase, + opts: FoundryOptions, +): Promise { + const receipt = await runOrganism({ + manifest: candidate, + args: caseArgs(candidate, c), + fns: opts.fns, + store: opts.store, + executors: opts.executors, + ...(opts.transports ? { transports: opts.transports } : {}), + ...(opts.tools ? { tools: opts.tools } : {}), + }); + const receiptDigest = await opts.store.putReceipt(receipt as unknown as JsonValue); + const outputs = caseOutputs(candidate, receipt.cells); + const usage = receipt.effects.reduce( + (total, effect) => ({ + tokensIn: total.tokensIn + (effect.usage?.tokensIn ?? 0), + tokensOut: total.tokensOut + (effect.usage?.tokensOut ?? 0), + }), + { tokensIn: 0, tokensOut: 0 }, + ); + return { + id: c.id, + split: c.split, + passed: receipt.outcome === "complete" && canonicalize(outputs) === canonicalize(c.expect), + outcome: receipt.outcome, + outputs, + expect: c.expect, + receiptDigest, + work: receipt.work, + usage, + }; +} + +export async function generateFoundryCandidates( + opts: GenerateCandidatesOptions, +): Promise { + const iface = opts.generator.interface; + if (!iface) fail(`generator ${opts.generator.key} must declare an interface`); + const source = iface.outputs[opts.output]; + if (!source) fail(`generator ${opts.generator.key}: unknown interface output "${opts.output}"`); + const args: Record> = Object.create(null) as Record>; + for (const [name, value] of Object.entries(opts.args)) { + const target = iface.inputs[name]; + if (!target) fail(`generator ${opts.generator.key}: unknown interface input "${name}"`); + (args[target.cell] ??= Object.create(null) as Record)[target.port] = value; + } + const generatorDigest = await opts.store.putManifest(opts.generator); + const receipt = await runOrganism({ + manifest: opts.generator, + args, + fns: opts.fns, + store: opts.store, + executors: opts.executors, + ...(opts.transports ? { transports: opts.transports } : {}), + ...(opts.tools ? { tools: opts.tools } : {}), + }); + const receiptDigest = await opts.store.putReceipt(receipt as unknown as JsonValue); + if (receipt.outcome !== "complete") { + fail(`generator ${opts.generator.key} ended ${receipt.outcome}`); + } + const output = receipt.cells[source.cell]?.outputs?.[source.port]; + const value = opts.field !== undefined && output !== null && typeof output === "object" && !Array.isArray(output) + ? output[opts.field] + : output; + if (!Array.isArray(value) || value.length === 0) { + fail(`generator ${opts.generator.key}.${opts.output} must emit a non-empty manifest list`); + } + if (value.length > FOUNDRY_BOUNDS.maxCandidates) { + fail(`generated candidates exceed ${FOUNDRY_BOUNDS.maxCandidates}`); + } + const candidates = value.map((candidate, i) => { + try { + return parseOrganismManifest(candidate); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + fail(`generated candidate ${i}: ${message}`); + } + }); + return { generatorDigest, receiptDigest, candidates }; +} + +async function evaluateCases( + candidate: OrganismManifest, + cases: FoundryCase[], + opts: FoundryOptions, +): Promise { + const results: FoundryCaseResult[] = []; + for (const c of cases) results.push(await evaluateCase(candidate, c, opts)); + return results; +} + +export type FoundrySelection = { + candidates: FoundryCandidateResult[]; + promoted: Digest; +}; + +export async function evaluateFoundryPopulation( + opts: FoundryOptions, +): Promise { + validate(opts); + const candidates: FoundryCandidateResult[] = []; + const selectionCases = opts.cases.filter((c) => c.split !== "holdout"); + for (const candidate of opts.candidates) { + const manifestDigest = await opts.store.putManifest(candidate); + const cases = await evaluateCases(candidate, selectionCases, opts); + candidates.push({ + manifestDigest, + manifestKey: candidate.key, + train: score(cases, "train"), + validation: score(cases, "validation"), + work: cases.reduce( + (total, c) => ({ + steps: total.steps + c.work.steps, + agentCalls: total.agentCalls + c.work.agentCalls, + units: total.units + c.work.units, + }), + { steps: 0, agentCalls: 0, units: 0 }, + ), + usage: cases.reduce( + (total, c) => ({ + tokensIn: total.tokensIn + c.usage.tokensIn, + tokensOut: total.tokensOut + c.usage.tokensOut, + }), + { tokensIn: 0, tokensOut: 0 }, + ), + cases, + }); + } + const promoted = selectFoundryCandidate(candidates); + return { candidates, promoted }; +} + +export async function runFoundry(opts: FoundryOptions): Promise { + const { candidates, promoted } = await evaluateFoundryPopulation(opts); + const promotedManifest = opts.candidates.find( + (candidate) => digestCanonical(manifestToJson(candidate)) === promoted, + )!; + const holdoutCases = await evaluateCases( + promotedManifest, + opts.cases.filter((c) => c.split === "holdout"), + opts, + ); + const holdoutScore = score(holdoutCases, "holdout"); + const base = { + contract: FOUNDRY_CONTRACT, + candidates, + promoted, + holdout: { ...holdoutScore, cases: holdoutCases }, + ...(opts.lineage ? { lineage: opts.lineage } : {}), + }; + return { ...base, digest: digestCanonical(base as unknown as JsonValue) }; +} diff --git a/src/gateway.test.ts b/src/gateway.test.ts new file mode 100644 index 0000000..b6f900d --- /dev/null +++ b/src/gateway.test.ts @@ -0,0 +1,64 @@ +import { expect, test } from "bun:test"; +import { vercelGatewayExecutor } from "./gateway"; +import type { EffectRequest } from "./effects"; + +const request: EffectRequest = { + contract: "morphogen.effect.v1", + cellId: "route", + kind: "classifier", + prompt: "Classify the ticket.", + context: { inputs: { ticket: "refund please" }, turn: 0 }, + output: { kind: "choice", labels: ["billing", "other"] }, + budget: { maxContextBytes: 4096, maxOutputBytes: 256 }, +}; + +test("Vercel Gateway executor binds structured output and post-call usage", async () => { + let seen: RequestInit | undefined; + const executor = vercelGatewayExecutor({ + model: "alibaba/qwen3.5-flash", + credential: "test-credential-value", + async fetch(_input, init) { + seen = init; + return Response.json({ + model: "alibaba/qwen3.5-flash", + choices: [{ message: { content: JSON.stringify({ value: "billing" }) } }], + usage: { prompt_tokens: 123, completion_tokens: 4 }, + }); + }, + }); + + const result = await executor.executeEffect!(request); + expect(result.output).toBe("billing"); + expect(result.metadata?.usage).toEqual({ + model: "alibaba/qwen3.5-flash", + tokensIn: 123, + tokensOut: 4, + }); + expect(seen?.redirect).toBe("error"); + expect(seen?.headers).toEqual({ + authorization: "Bearer test-credential-value", + "content-type": "application/json", + }); + const body = JSON.parse(String(seen?.body)); + expect(body.messages[0].content).toContain("JSON"); + expect(body.response_format.json_schema.schema.properties.value.enum).toEqual([ + "billing", + "other", + ]); +}); + +test("Vercel Gateway executor rejects redirects and malformed output", async () => { + const redirected = vercelGatewayExecutor({ + model: "alibaba/qwen3.5-flash", + credential: "test-credential-value", + fetch: async () => new Response(null, { status: 302 }), + }); + await expect(redirected.executeEffect!(request)).rejects.toThrow("redirects are forbidden"); + + const malformed = vercelGatewayExecutor({ + model: "alibaba/qwen3.5-flash", + credential: "test-credential-value", + fetch: async () => Response.json({ choices: [{ message: { content: "{}" } }] }), + }); + await expect(malformed.executeEffect!(request)).rejects.toThrow("has no value"); +}); diff --git a/src/gateway.ts b/src/gateway.ts new file mode 100644 index 0000000..4ed6bfc --- /dev/null +++ b/src/gateway.ts @@ -0,0 +1,174 @@ +import type { AgentOutput } from "./contract"; +import type { + EffectRequest, + Executor, + ExecutorResult, +} from "./effects"; +import { MorphogenError } from "./errors"; +import { canonicalize, type JsonObject, type JsonValue } from "./values"; + +export const VERCEL_AI_GATEWAY_BASE_URL = "https://ai-gateway.vercel.sh/v1" as const; + +export type GatewayFetch = ( + input: Request | string | URL, + init?: RequestInit, +) => Promise; + +export type GatewayExecutorOptions = { + model: string; + credential?: string; + fetch?: GatewayFetch; + maxResponseBytes?: number; +}; + +function record(value: unknown, at: string): Record { + if (value === null || typeof value !== "object" || Array.isArray(value)) { + throw new MorphogenError("EFFECT_UNPARSEABLE", `${at} must be an object`); + } + return value as Record; +} + +function integer(value: unknown): number | undefined { + return Number.isSafeInteger(value) && (value as number) >= 0 + ? value as number + : undefined; +} + +function outputSchema(output: AgentOutput): JsonObject { + if (output.kind === "text") return { type: "string" }; + if (output.kind === "choice") { + return { type: "string", enum: output.labels } as unknown as JsonObject; + } + return output.schema; +} + +async function boundedJson(response: Response, maxBytes: number): Promise { + if (response.status >= 300 && response.status < 400) { + throw new MorphogenError("EFFECT_FAILED", "AI Gateway redirects are forbidden"); + } + const declared = response.headers.get("content-length"); + if (declared !== null && Number(declared) > maxBytes) { + throw new MorphogenError("EFFECT_FAILED", `AI Gateway response exceeds ${maxBytes} bytes`); + } + const bytes = new Uint8Array(await response.arrayBuffer()); + if (bytes.byteLength > maxBytes) { + throw new MorphogenError("EFFECT_FAILED", `AI Gateway response exceeds ${maxBytes} bytes`); + } + const text = new TextDecoder().decode(bytes); + if (!response.ok) { + throw new MorphogenError( + "EFFECT_FAILED", + `AI Gateway returned ${response.status}: ${text.slice(0, 500)}`, + ); + } + try { + return JSON.parse(text); + } catch { + throw new MorphogenError("EFFECT_UNPARSEABLE", "AI Gateway returned invalid JSON"); + } +} + +function credential(options: GatewayExecutorOptions): string { + const value = options.credential + ?? process.env.AI_GATEWAY_API_KEY + ?? process.env.VERCEL_OIDC_TOKEN; + if (typeof value !== "string" || value.length < 16 || value.length > 8192) { + throw new MorphogenError( + "EFFECT_FAILED", + "AI Gateway credential is not configured", + ); + } + return value; +} + +export function vercelGatewayExecutor(options: GatewayExecutorOptions): Executor { + if (!/^[a-z0-9][a-z0-9._-]*\/[a-z0-9][a-z0-9._-]*$/i.test(options.model)) { + throw new MorphogenError("PARSE_FAILED", `invalid AI Gateway model "${options.model}"`); + } + const fetcher = options.fetch ?? globalThis.fetch; + const maxResponseBytes = options.maxResponseBytes ?? 2_097_152; + const run = async (request: EffectRequest, signal?: AbortSignal): Promise => { + const token = credential(options); + const schema = { + type: "object", + additionalProperties: false, + required: ["value"], + properties: { value: outputSchema(request.output) }, + } as unknown as JsonObject; + const body = { + model: options.model, + messages: [ + { + role: "system", + content: "Execute the declared bounded cell. Return only one JSON object with exactly one key named value and no extra fields.", + }, + { + role: "user", + content: canonicalize({ + prompt: request.prompt, + context: request.context, + output: request.output as unknown as JsonValue, + }), + }, + ], + response_format: { + type: "json_schema", + json_schema: { + name: "morphogen_cell_output", + strict: true, + schema, + }, + }, + max_tokens: Math.max(1, Math.min(16_384, Math.ceil(request.budget.maxOutputBytes / 4))), + temperature: 0, + }; + const response = await fetcher(`${VERCEL_AI_GATEWAY_BASE_URL}/chat/completions`, { + method: "POST", + redirect: "error", + ...(signal ? { signal } : {}), + headers: { + authorization: `Bearer ${token}`, + "content-type": "application/json", + }, + body: canonicalize(body as unknown as JsonValue), + }); + const raw = record(await boundedJson(response, maxResponseBytes), "AI Gateway response"); + if (!Array.isArray(raw.choices) || raw.choices.length !== 1) { + throw new MorphogenError("EFFECT_UNPARSEABLE", "AI Gateway response must contain one choice"); + } + const choice = record(raw.choices[0], "AI Gateway choice"); + const message = record(choice.message, "AI Gateway message"); + if (typeof message.content !== "string") { + throw new MorphogenError("EFFECT_UNPARSEABLE", "AI Gateway message content must be text"); + } + let parsed: unknown; + try { + parsed = JSON.parse(message.content); + } catch { + throw new MorphogenError("EFFECT_UNPARSEABLE", "AI Gateway structured output is invalid JSON"); + } + const structured = record(parsed, "AI Gateway structured output"); + if (!("value" in structured)) { + throw new MorphogenError("EFFECT_UNPARSEABLE", "AI Gateway structured output has no value"); + } + const usage = record(raw.usage ?? {}, "AI Gateway usage"); + const tokensIn = integer(usage.prompt_tokens ?? usage.input_tokens); + const tokensOut = integer(usage.completion_tokens ?? usage.output_tokens); + return { + output: structured.value as JsonValue, + metadata: { + executor: `vercel:${options.model}`, + usage: { + model: typeof raw.model === "string" ? raw.model : options.model, + ...(tokensIn !== undefined ? { tokensIn } : {}), + ...(tokensOut !== undefined ? { tokensOut } : {}), + }, + }, + }; + }; + return { + id: `vercel:${options.model}`, + execute: async (request, signal) => (await run(request, signal)).output, + executeEffect: run, + }; +} diff --git a/src/graph.ts b/src/graph.ts index 03c2a4b..af0b9ae 100644 --- a/src/graph.ts +++ b/src/graph.ts @@ -16,6 +16,7 @@ import { asDigest } from "./digest"; import { parseOrganismManifest } from "./contract"; import { unpackBundle } from "./bundle"; import type { Transport } from "./transport"; +import type { ToolRegistry } from "./tools"; import type { JsonValue } from "./values"; export type CellPorts = { inputs: PortMap; outputs: PortMap }; @@ -52,6 +53,7 @@ export function cellSignature( cell: Cell, fns: FnRegistry, children: Map, + tools?: ToolRegistry, ): CellPorts { switch (cell.kind) { case "input": @@ -77,6 +79,19 @@ export function cellSignature( outputs: { ...sig.signature.outputs }, }; } + case "tool": { + const entry = tools?.get(cell.tool); + if (!entry) { + throw new MorphogenError( + "TOOL_UNKNOWN", + `cell "${cell.id}" references unknown tool "${cell.tool}"`, + ); + } + return { + inputs: { ...entry.signature.inputs }, + outputs: { ...entry.signature.outputs }, + }; + } case "agent": case "classifier": case "gate": @@ -310,6 +325,7 @@ export async function compileOrganism( store: Store, depth = 0, transports?: Record, + tools?: ToolRegistry, ): Promise { if (depth > MAX_COMPILE_DEPTH) { throw new MorphogenError( @@ -365,14 +381,14 @@ export async function compileOrganism( } children.set( cell.id, - await compileOrganism(sub, fns, store, depth + 1, transports), + await compileOrganism(sub, fns, store, depth + 1, transports, tools), ); } // signatures const ports = new Map(); for (const cell of manifest.cells) { - ports.set(cell.id, cellSignature(cell, fns, children)); + ports.set(cell.id, cellSignature(cell, fns, children, tools)); } // interface integrity (top-level manifest interface) @@ -561,10 +577,10 @@ export async function compileOrganism( } } for (const ref of (cell.kind === "gate" ? [] : cell.tools) ?? []) { - if (!fns.has(ref)) { + if (!fns.has(ref) && !tools?.has(ref)) { throw new MorphogenError( - "FN_UNKNOWN", - `cell "${cell.id}" declares unknown tool fn "${ref}"`, + "TOOL_UNKNOWN", + `cell "${cell.id}" declares unknown tool "${ref}"`, ); } } diff --git a/src/run.test.ts b/src/run.test.ts index f781ddc..f217f39 100644 --- a/src/run.test.ts +++ b/src/run.test.ts @@ -475,7 +475,7 @@ describe("scheduler", () => { store: new MemoryStore(), executors: [], }), - ).rejects.toThrowError(/unknown tool fn/); + ).rejects.toThrowError(/unknown tool/); }); test("shadow classifier records the decision but takes the declared label", async () => { @@ -3097,4 +3097,56 @@ describe("spawn cells", () => { expect(r.cells["recover"]?.outputs?.value).toBe("FN_UNKNOWN"); }); + test("executor results record usage produced by the completed call", async () => { + const m = manifest({ + contract: "morphogen.organism.v1", + key: "organism:post-call-usage", + name: "Post-call usage", + cells: [{ + id: "answer", + kind: "agent", + inputs: {}, + prompt: "Answer.", + view: { inputs: [] }, + output: { kind: "text" }, + }], + edges: [], + }); + const receipt = await runOrganism({ + manifest: m, + fns: builtinRegistry(), + store: new MemoryStore(), + executors: [{ + id: "metered", + async execute() { + throw new Error("legacy execute must not run"); + }, + async executeEffect() { + return { + output: "ok", + metadata: { + executor: "gateway:qwen-flash", + usage: { model: "alibaba/qwen3.5-flash", tokensIn: 12, tokensOut: 3 }, + }, + }; + }, + }], + }); + + expect(receipt.outcome).toBe("complete"); + expect(receipt.effects[0]?.executor).toBe("gateway:qwen-flash"); + expect(receipt.effects[0]?.usage).toEqual({ + model: "alibaba/qwen3.5-flash", + tokensIn: 12, + tokensOut: 3, + }); + const verified = await verifyReceipt( + receipt as unknown as JsonValue, + manifestToJson(m), + new MemoryStore(), + builtinRegistry(), + ); + expect(verified.ok).toBe(true); + }); + }); diff --git a/src/run.ts b/src/run.ts index 9282b35..4d9ae0e 100644 --- a/src/run.ts +++ b/src/run.ts @@ -29,10 +29,12 @@ import { type EffectReceipt, type EffectRequest, type Executor, + type ExecutorMetadata, } from "./effects"; import type { FnRegistry } from "./registry"; import type { Store } from "./store"; import type { Transport } from "./transport"; +import type { ToolRegistry } from "./tools"; import { digestCanonical, type Digest } from "./digest"; import { canonicalBytes, @@ -104,6 +106,7 @@ export type RunOptions = { fns: FnRegistry; store: Store; executors: Executor[]; + tools?: ToolRegistry; /** Named transports for `via` cells — remote manifest resolution. */ transports?: Record; /** Provenance replay: cell path → transport name recorded by the run @@ -116,6 +119,7 @@ export type RunOptions = { * default — the failure replays too). A live slot may have been * overwritten since; the record is authoritative. */ replaySlots?: Record; + replayToolEffects?: EffectReceipt[]; }; type EdgeState = "pending" | "delivered" | "dead"; @@ -130,6 +134,7 @@ type RunContext = { work: { steps: number; agentCalls: number; units: number }; failure?: { code: ErrorCode; message: string; path?: string }; seq: number; + toolReplay: Map; }; export async function runOrganism(opts: RunOptions): Promise { @@ -142,7 +147,13 @@ export async function runOrganism(opts: RunOptions): Promise { events: [], work: { steps: 0, agentCalls: 0, units: 0 }, seq: 0, + toolReplay: new Map(), }; + for (const effect of opts.replayToolEffects ?? []) { + const queue = ctx.toolReplay.get(effect.requestDigest) ?? []; + queue.push(effect); + ctx.toolReplay.set(effect.requestDigest, queue); + } emit(ctx, { kind: "run.start", digest: manifestDigest }); const compiled = await compileOrganism( opts.manifest, @@ -150,6 +161,7 @@ export async function runOrganism(opts: RunOptions): Promise { opts.store, 0, opts.transports, + opts.tools, ); const outcome = await runInto(compiled, opts.args ?? {}, "", ctx, 0); emit(ctx, { kind: "run.end", outcome }); @@ -543,6 +555,7 @@ async function activate( ctx.opts.store, depth + 1, ctx.opts.transports, + ctx.opts.tools, ); const rawArgs = inputs.args ?? {}; if ( @@ -581,6 +594,75 @@ async function activate( } return { outputs: entry.fn(inputs) }; } + case "tool": { + const entry = ctx.opts.tools?.get(cell.tool); + if (!entry) { + throw new MorphogenError("TOOL_UNKNOWN", `tool "${cell.tool}" is not configured`); + } + const requestDigest = digestCanonical({ + contract: "morphogen.tool-effect.v1", + path, + tool: cell.tool, + effect: entry.signature.effect, + inputs, + } as unknown as JsonValue); + emit(ctx, { kind: "effect", path, digest: requestDigest }); + const replay = ctx.toolReplay.get(requestDigest)?.shift(); + if (replay) { + ctx.effects.push(replay); + if (replay.error) throw new MorphogenError(replay.error.code, replay.error.message); + ctx.work.units += entry.signature.cost + canonicalBytes(replay.output!); + return { outputs: replay.output as Record, effectDigest: requestDigest }; + } + const controller = new AbortController(); + const timeout = cell.budget?.maxEffectMs; + const timer = timeout === undefined ? undefined : setTimeout(() => controller.abort(), timeout); + try { + const execute = entry.tool(inputs, { + requestDigest, + idempotencyKey: requestDigest, + ...(timeout !== undefined ? { signal: controller.signal } : {}), + }); + const outputs = timeout === undefined + ? await execute + : await Promise.race([ + execute, + new Promise((_, reject) => controller.signal.addEventListener( + "abort", + () => reject(new MorphogenError( + "BUDGET_EXHAUSTED", + `tool cell "${cell.id}" exceeded maxEffectMs ${timeout}`, + )), + { once: true }, + )), + ]); + const bytes = canonicalBytes(outputs as unknown as JsonValue); + if (bytes > entry.signature.maxOutputBytes) { + throw new MorphogenError( + "BUDGET_EXHAUSTED", + `tool cell "${cell.id}" output ${bytes}B exceeds ${entry.signature.maxOutputBytes}B`, + ); + } + ctx.work.units += entry.signature.cost + bytes; + ctx.effects.push({ + requestDigest, + output: outputs as unknown as JsonValue, + executor: `tool:${cell.tool}`, + }); + return { outputs, effectDigest: requestDigest }; + } catch (error) { + const report = errorReport(error); + ctx.effects.push({ + requestDigest, + error: { code: report.code === "INTERNAL" ? "TOOL_FAILED" : report.code, message: report.message }, + executor: `tool:${cell.tool}`, + }); + if (error instanceof MorphogenError) throw error; + throw new MorphogenError("TOOL_FAILED", report.message); + } finally { + if (timer !== undefined) clearTimeout(timer); + } + } case "agent": case "classifier": case "gate": { @@ -711,20 +793,29 @@ async function activate( ctx.work.units += WORK.effectBase + contextBytes * WORK.perContextByte; emit(ctx, { kind: "effect", path, digest: requestDigest }); - const meta = await executor.receiptFor?.(request); + let meta: ExecutorMetadata | undefined; + const invoke = async (signal?: AbortSignal): Promise => { + if (executor.executeEffect) { + const result = await executor.executeEffect(request, signal); + meta = result.metadata; + return result.output; + } + meta = await executor.receiptFor?.(request); + return executor.execute(request, signal); + }; let raw: JsonValue; // budget.maxEffectMs bounds each call wall-clock; the timeout is // recorded as an effect error so retry and replay both see it const effectMs = cell.budget?.maxEffectMs; try { if (effectMs === undefined) { - raw = await executor.execute(request); + raw = await invoke(); } else { const ac = new AbortController(); const timer = setTimeout(() => ac.abort(), effectMs); try { raw = await Promise.race([ - executor.execute(request, ac.signal), + invoke(ac.signal), new Promise((_, reject) => ac.signal.addEventListener( "abort", @@ -822,9 +913,11 @@ async function activate( // bounded callback into the automaton: run the declared fn, log the // result, re-request with the updated tool log const call = settled; - const entry = ctx.opts.fns.get(call.fn)!; - ctx.work.units += entry.signature.cost; - for (const [p, decl] of Object.entries(entry.signature.inputs)) { + const fn = ctx.opts.fns.get(call.fn); + const external = ctx.opts.tools?.get(call.fn); + const signature = fn?.signature ?? external?.signature; + if (!signature) throw new MorphogenError("TOOL_UNKNOWN", `tool "${call.fn}" is not configured`); + for (const [p, decl] of Object.entries(signature.inputs)) { const v = (call.inputs as Record)[p]; if (v === undefined) { if (!decl.optional) { @@ -837,7 +930,73 @@ async function activate( } checkValue(v, decl, `${cell.id}.tool.${p}`); } - const toolOut = entry.fn(call.inputs as Record); + let toolOut: Record; + if (fn) { + ctx.work.units += fn.signature.cost; + toolOut = fn.fn(call.inputs as Record); + } else { + const tool = external!; + const toolDigest = digestCanonical({ + contract: "morphogen.tool-effect.v1", + path: `${path}/t${turn}`, + tool: call.fn, + effect: tool.signature.effect, + inputs: call.inputs, + } as unknown as JsonValue); + emit(ctx, { kind: "effect", path, digest: toolDigest }); + const replay = ctx.toolReplay.get(toolDigest)?.shift(); + if (replay) { + ctx.effects.push(replay); + if (replay.error) throw new MorphogenError(replay.error.code, replay.error.message); + toolOut = replay.output as Record; + } else { + const controller = new AbortController(); + const timeout = cell.budget?.maxEffectMs; + const timer = timeout === undefined ? undefined : setTimeout(() => controller.abort(), timeout); + try { + const execute = tool.tool(call.inputs as Record, { + requestDigest: toolDigest, + idempotencyKey: toolDigest, + ...(timeout !== undefined ? { signal: controller.signal } : {}), + }); + toolOut = timeout === undefined + ? await execute + : await Promise.race([ + execute, + new Promise((_, reject) => controller.signal.addEventListener( + "abort", + () => reject(new MorphogenError( + "BUDGET_EXHAUSTED", + `agent tool ${call.fn} exceeded maxEffectMs ${timeout}`, + )), + { once: true }, + )), + ]); + ctx.effects.push({ + requestDigest: toolDigest, + output: toolOut as unknown as JsonValue, + executor: `tool:${call.fn}`, + }); + } catch (error) { + const report = errorReport(error); + const code = report.code === "INTERNAL" ? "TOOL_FAILED" : report.code; + ctx.effects.push({ + requestDigest: toolDigest, + error: { code, message: report.message }, + executor: `tool:${call.fn}`, + }); + throw new MorphogenError(code, report.message); + } finally { + if (timer !== undefined) clearTimeout(timer); + } + } + const bytes = canonicalBytes(toolOut as unknown as JsonValue); + if (bytes > tool.signature.maxOutputBytes) { + throw new MorphogenError("BUDGET_EXHAUSTED", `tool ${call.fn} output exceeds its byte bound`); + } + ctx.work.units += tool.signature.cost + bytes; + } + checkOutputs(cell, signature.outputs, toolOut); toolLog.push({ fn: call.fn, inputs: call.inputs, output: toolOut as JsonValue }); } } diff --git a/src/search-verify.ts b/src/search-verify.ts new file mode 100644 index 0000000..68bd1e4 --- /dev/null +++ b/src/search-verify.ts @@ -0,0 +1,182 @@ +import { manifestToJson } from "./contract"; +import { digestCanonical, type Digest } from "./digest"; +import { MorphogenError } from "./errors"; +import { + FOUNDRY_CONTRACT, + selectFoundryCandidate, + type FoundryReport, +} from "./foundry"; +import { parseFoundryReport, verifyFoundryReport } from "./foundry-verify"; +import type { FnRegistry } from "./registry"; +import { parseRunReceipt } from "./run"; +import { + SEARCH_BOUNDS, + SEARCH_CONTRACT, + type SearchGeneration, + type SearchReport, +} from "./search"; +import type { Store } from "./store"; +import type { ToolRegistry } from "./tools"; +import { verifyReceipt } from "./verify"; +import type { JsonObject, JsonValue } from "./values"; + +function object(value: unknown, at: string): JsonObject { + if (value === null || typeof value !== "object" || Array.isArray(value)) { + throw new MorphogenError("PARSE_FAILED", `${at} must be an object`); + } + return value as JsonObject; +} + +function exactKeys(value: JsonObject, allowed: string[], at: string): void { + const extra = Object.keys(value).find((key) => !allowed.includes(key)); + if (extra) throw new MorphogenError("PARSE_FAILED", `${at}: unknown key "${extra}"`); +} + +function digest(value: JsonValue | undefined, at: string): Digest { + if (typeof value !== "string" || !/^sha256:[0-9a-f]{64}$/.test(value)) { + throw new MorphogenError("PARSE_FAILED", `${at} must be a sha256 digest`); + } + return value as Digest; +} + +export function parseSearchReport(value: unknown): SearchReport { + const raw = object(value, "search"); + exactKeys(raw, ["contract", "generatorDigest", "generations", "result", "digest"], "search"); + if (raw.contract !== SEARCH_CONTRACT) { + throw new MorphogenError("PARSE_FAILED", `search.contract must be ${SEARCH_CONTRACT}`); + } + if (!Array.isArray(raw.generations) || raw.generations.length === 0 || raw.generations.length > SEARCH_BOUNDS.maxGenerations) { + throw new MorphogenError("PARSE_FAILED", "search.generations must be a bounded non-empty list"); + } + const result = parseFoundryReport(raw.result); + const generations: SearchGeneration[] = raw.generations.map((entry, index) => { + const generation = object(entry, `search.generations[${index}]`); + exactKeys( + generation, + ["generation", "generatorDigest", "receiptDigest", "proposed", "candidates", "promoted"], + `search.generations[${index}]`, + ); + if (generation.generation !== index) { + throw new MorphogenError("PARSE_FAILED", `search.generations[${index}].generation must be ${index}`); + } + if (!Array.isArray(generation.proposed) || generation.proposed.length === 0) { + throw new MorphogenError("PARSE_FAILED", `search.generations[${index}].proposed must be non-empty`); + } + const parsed = parseFoundryReport({ + contract: FOUNDRY_CONTRACT, + candidates: generation.candidates, + promoted: generation.promoted, + holdout: result.holdout, + digest: result.digest, + }); + return { + generation: index, + generatorDigest: digest(generation.generatorDigest, `search.generations[${index}].generatorDigest`), + receiptDigest: digest(generation.receiptDigest, `search.generations[${index}].receiptDigest`), + proposed: generation.proposed.map((item, i) => digest(item, `search.generations[${index}].proposed[${i}]`)), + candidates: parsed.candidates, + promoted: parsed.promoted, + }; + }); + return { + contract: SEARCH_CONTRACT, + generatorDigest: digest(raw.generatorDigest, "search.generatorDigest"), + generations, + result, + digest: digest(raw.digest, "search.digest"), + }; +} + +export type SearchVerifyReport = { + ok: boolean; + digest: Digest; + checkedReceipts: number; + mismatches: string[]; +}; + +export async function verifySearchReport( + value: unknown, + store: Store, + fns: FnRegistry, + tools?: ToolRegistry, +): Promise { + const report = parseSearchReport(value); + const mismatches: string[] = []; + const { digest: claimed, ...base } = report; + const computed = digestCanonical(base as unknown as JsonValue); + if (claimed !== computed) mismatches.push(`digest: claimed ${claimed}, computed ${computed}`); + const final = await verifyFoundryReport(report.result, store, fns, tools); + if (!final.ok) mismatches.push(...final.mismatches.map((mismatch) => `result: ${mismatch}`)); + let checkedReceipts = final.checkedReceipts; + let previousWinner: Digest | undefined; + for (const generation of report.generations) { + if (generation.generatorDigest !== report.generatorDigest) { + mismatches.push(`generation ${generation.generation}: generator digest changed`); + } + if (selectFoundryCandidate(generation.candidates) !== generation.promoted) { + mismatches.push(`generation ${generation.generation}: promoted candidate is not the winner`); + } + if (generation.candidates.some((candidate) => candidate.cases.some((c) => c.split === "holdout"))) { + mismatches.push(`generation ${generation.generation}: holdout evidence leaked into selection`); + } + if (previousWinner && !generation.candidates.some((candidate) => candidate.manifestDigest === previousWinner)) { + mismatches.push(`generation ${generation.generation}: previous winner did not survive`); + } + for (const proposed of generation.proposed) { + if (!generation.candidates.some((candidate) => candidate.manifestDigest === proposed)) { + mismatches.push(`generation ${generation.generation}: proposed ${proposed} was not evaluated`); + } + } + const promoted = generation.candidates.find((candidate) => candidate.manifestDigest === generation.promoted)!; + const evidence = promoted.cases[0]; + if (!evidence) { + mismatches.push(`generation ${generation.generation}: winner has no evidence`); + } else { + const synthetic: Omit = { + contract: FOUNDRY_CONTRACT, + candidates: generation.candidates, + promoted: generation.promoted, + holdout: { + passed: evidence.passed ? 1 : 0, + total: 1, + cases: [{ ...evidence, split: "holdout" }], + }, + }; + const foundry = { + ...synthetic, + digest: digestCanonical(synthetic as unknown as JsonValue), + }; + const verified = await verifyFoundryReport(foundry, store, fns, tools); + checkedReceipts += verified.checkedReceipts; + if (!verified.ok) { + mismatches.push(...verified.mismatches.map((mismatch) => `generation ${generation.generation}: ${mismatch}`)); + } + } + const generatorManifest = await store.getManifest(generation.generatorDigest); + const generatorReceiptValue = await store.getReceipt(generation.receiptDigest); + if (!generatorManifest) { + mismatches.push(`generation ${generation.generation}: generator manifest missing`); + } else if (!generatorReceiptValue) { + mismatches.push(`generation ${generation.generation}: generator receipt missing`); + } else { + const generatorReceipt = parseRunReceipt(generatorReceiptValue); + const verified = await verifyReceipt( + generatorReceipt, + manifestToJson(generatorManifest), + store, + fns, + undefined, + tools, + ); + checkedReceipts++; + if (!verified.ok) { + mismatches.push(`generation ${generation.generation}: generator receipt: ${verified.mismatches.join("; ")}`); + } + } + previousWinner = generation.promoted; + } + if (previousWinner !== report.result.promoted) { + mismatches.push("final result did not preserve the last generation winner"); + } + return { ok: mismatches.length === 0, digest: claimed, checkedReceipts, mismatches }; +} diff --git a/src/search.ts b/src/search.ts new file mode 100644 index 0000000..5fb86ad --- /dev/null +++ b/src/search.ts @@ -0,0 +1,154 @@ +import { manifestToJson, type OrganismManifest } from "./contract"; +import { digestCanonical, type Digest } from "./digest"; +import type { Executor } from "./effects"; +import { MorphogenError } from "./errors"; +import { + FOUNDRY_BOUNDS, + evaluateFoundryPopulation, + generateFoundryCandidates, + runFoundry, + type FoundryCase, + type FoundryLineage, + type FoundryReport, + type FoundrySelection, +} from "./foundry"; +import type { FnRegistry } from "./registry"; +import type { Store } from "./store"; +import type { Transport } from "./transport"; +import type { ToolRegistry } from "./tools"; +import type { JsonValue } from "./values"; + +export const SEARCH_CONTRACT = "morphogen.search.v1" as const; + +export const SEARCH_BOUNDS = { + maxGenerations: 8, +} as const; + +export type SearchGeneration = FoundryLineage & FoundrySelection & { + generation: number; + proposed: Digest[]; +}; + +export type SearchReport = { + contract: typeof SEARCH_CONTRACT; + generatorDigest: Digest; + generations: SearchGeneration[]; + result: FoundryReport; + digest: Digest; +}; + +export type SearchOptions = { + generator: OrganismManifest; + generatorArgs: Record; + feedbackInput?: string; + output: string; + field?: string; + seeds?: OrganismManifest[]; + cases: FoundryCase[]; + maxGenerations: number; + fns: FnRegistry; + store: Store; + executors: Executor[]; + transports?: Record; + tools?: ToolRegistry; +}; + +function feedback(generation: number, selection?: FoundrySelection): JsonValue { + if (!selection) return null; + return { + generation, + promoted: selection.promoted, + candidates: selection.candidates.map((candidate) => ({ + manifestDigest: candidate.manifestDigest, + manifestKey: candidate.manifestKey, + train: candidate.train, + validation: candidate.validation, + work: candidate.work, + usage: candidate.usage, + })), + }; +} + +function dedupe(candidates: OrganismManifest[]): OrganismManifest[] { + const seen = new Set(); + return candidates.filter((candidate) => { + const digest = digestCanonical(manifestToJson(candidate)); + if (seen.has(digest)) return false; + seen.add(digest); + return true; + }); +} + +export async function runFoundrySearch(opts: SearchOptions): Promise { + if (!Number.isInteger(opts.maxGenerations) || opts.maxGenerations < 1 || opts.maxGenerations > SEARCH_BOUNDS.maxGenerations) { + throw new MorphogenError("PARSE_FAILED", `search maxGenerations must be 1..${SEARCH_BOUNDS.maxGenerations}`); + } + let survivors = dedupe(opts.seeds ?? []); + let prior: FoundrySelection | undefined; + const generations: SearchGeneration[] = []; + for (let generation = 0; generation < opts.maxGenerations; generation++) { + const args = { + ...opts.generatorArgs, + ...(opts.feedbackInput ? { [opts.feedbackInput]: feedback(generation, prior) } : {}), + }; + const generated = await generateFoundryCandidates({ + generator: opts.generator, + args, + output: opts.output, + ...(opts.field ? { field: opts.field } : {}), + fns: opts.fns, + store: opts.store, + executors: opts.executors, + ...(opts.transports ? { transports: opts.transports } : {}), + ...(opts.tools ? { tools: opts.tools } : {}), + }); + const population = dedupe([...survivors, ...generated.candidates]); + if (population.length > FOUNDRY_BOUNDS.maxCandidates) { + throw new MorphogenError("BUDGET_EXHAUSTED", `search population exceeds ${FOUNDRY_BOUNDS.maxCandidates}`); + } + prior = await evaluateFoundryPopulation({ + candidates: population, + cases: opts.cases, + fns: opts.fns, + store: opts.store, + executors: opts.executors, + ...(opts.transports ? { transports: opts.transports } : {}), + ...(opts.tools ? { tools: opts.tools } : {}), + }); + generations.push({ + generation, + generatorDigest: generated.generatorDigest, + receiptDigest: generated.receiptDigest, + proposed: generated.candidates.map((candidate) => digestCanonical(manifestToJson(candidate))), + ...prior, + }); + survivors = population.filter( + (candidate) => digestCanonical(manifestToJson(candidate)) === prior!.promoted, + ); + } + const last = generations.at(-1)!; + const finalPopulation = dedupe([ + ...survivors, + ...(await Promise.all(last.proposed.map(async (digest) => opts.store.getManifest(digest)))) + .filter((candidate): candidate is OrganismManifest => candidate !== undefined), + ]); + const result = await runFoundry({ + candidates: finalPopulation, + cases: opts.cases, + fns: opts.fns, + store: opts.store, + executors: opts.executors, + ...(opts.transports ? { transports: opts.transports } : {}), + lineage: { + generatorDigest: last.generatorDigest, + receiptDigest: last.receiptDigest, + }, + }); + const base = { + contract: SEARCH_CONTRACT, + generatorDigest: digestCanonical(manifestToJson(opts.generator)), + generations, + result, + }; + return { ...base, digest: digestCanonical(base as unknown as JsonValue) }; +} diff --git a/src/tools.test.ts b/src/tools.test.ts new file mode 100644 index 0000000..76281a4 --- /dev/null +++ b/src/tools.test.ts @@ -0,0 +1,174 @@ +import { expect, test } from "bun:test"; +import { manifestToJson, parseOrganismManifest } from "./contract"; +import { scriptedExecutor } from "./effects"; +import { builtinRegistry } from "./registry"; +import { runOrganism } from "./run"; +import { MemoryStore } from "./store"; +import type { ToolRegistry } from "./tools"; +import { verifyReceipt } from "./verify"; +import type { JsonValue } from "./values"; + +function lookupRegistry(calls: { value: number }): ToolRegistry { + return new Map([[ + "records.lookup.v1", + { + signature: { + inputs: { id: { type: "text" } }, + outputs: { record: { type: "json" } }, + effect: "read" as const, + cost: 25, + maxOutputBytes: 4096, + }, + async tool(inputs, context) { + calls.value++; + expect(context.idempotencyKey).toBe(context.requestDigest); + return { record: { id: inputs.id ?? null, status: "active" } }; + }, + }, + ]]); +} + +const lookup = parseOrganismManifest({ + contract: "morphogen.organism.v1", + key: "organism:tool-lookup", + name: "Tool lookup", + interface: { + inputs: { id: { cell: "src", port: "id" } }, + outputs: { record: { cell: "lookup", port: "record" } }, + }, + cells: [ + { id: "src", kind: "input", outputs: { id: "text" } }, + { id: "lookup", kind: "tool", tool: "records.lookup.v1", budget: { maxEffectMs: 1000 } }, + ], + edges: [ + { from: { cell: "src", port: "id" }, to: { cell: "lookup", port: "id" } }, + ], +}); + +test("explicit tool cells are typed, receipted, and replay without live I/O", async () => { + const calls = { value: 0 }; + const tools = lookupRegistry(calls); + const store = new MemoryStore(); + const receipt = await runOrganism({ + manifest: lookup, + args: { src: { id: "customer-7" } }, + fns: builtinRegistry(), + store, + executors: [], + tools, + }); + + expect(receipt.outcome).toBe("complete"); + expect(receipt.cells.lookup?.outputs?.record).toEqual({ id: "customer-7", status: "active" }); + expect(receipt.effects[0]?.executor).toBe("tool:records.lookup.v1"); + expect(calls.value).toBe(1); + + tools.get("records.lookup.v1")!.tool = async () => { + throw new Error("live tool must not run during replay"); + }; + const verified = await verifyReceipt( + receipt as unknown as JsonValue, + manifestToJson(lookup), + store, + builtinRegistry(), + undefined, + tools, + ); + expect(verified.mismatches).toEqual([]); + expect(calls.value).toBe(1); +}); + +test("agents call declared external tools and replay the nested effect", async () => { + const manifest = parseOrganismManifest({ + contract: "morphogen.organism.v1", + key: "organism:agent-tool", + name: "Agent external tool", + cells: [{ + id: "agent", + kind: "agent", + inputs: {}, + prompt: "Look up the customer then answer.", + view: { inputs: [] }, + tools: ["records.lookup.v1"], + output: { kind: "text" }, + budget: { maxTurns: 2 }, + }], + edges: [], + }); + const calls = { value: 0 }; + const tools = lookupRegistry(calls); + const store = new MemoryStore(); + const receipt = await runOrganism({ + manifest, + fns: builtinRegistry(), + store, + tools, + executors: [scriptedExecutor({ + agent: [ + { tool: "records.lookup.v1", inputs: { id: "customer-7" } }, + "active", + ], + })], + }); + + expect(receipt.outcome).toBe("complete"); + expect(receipt.cells.agent?.outputs?.out).toBe("active"); + expect(receipt.cells.agent?.toolCalls).toEqual([{ + fn: "records.lookup.v1", + inputs: { id: "customer-7" }, + output: { record: { id: "customer-7", status: "active" } }, + }]); + expect(receipt.effects.map((effect) => effect.executor)).toEqual([ + "scripted", + "tool:records.lookup.v1", + "scripted", + ]); + tools.get("records.lookup.v1")!.tool = async () => { + throw new Error("live tool must not run during replay"); + }; + const verified = await verifyReceipt( + receipt as unknown as JsonValue, + manifestToJson(manifest), + store, + builtinRegistry(), + undefined, + tools, + ); + expect(verified.mismatches).toEqual([]); + expect(calls.value).toBe(1); +}); + +test("tool failures route through ordinary fail edges", async () => { + const manifest = parseOrganismManifest({ + contract: "morphogen.organism.v1", + key: "organism:tool-failure", + name: "Tool failure", + cells: [ + { id: "src", kind: "const", outputs: { id: { type: "text", value: "missing" } } }, + { id: "lookup", kind: "tool", tool: "records.lookup.v1" }, + { id: "field", kind: "const", outputs: { value: { type: "text", value: "code" } } }, + { id: "recover", kind: "fn", fn: "pick.v1" }, + ], + edges: [ + { from: { cell: "src", port: "id" }, to: { cell: "lookup", port: "id" } }, + { from: { cell: "lookup", port: "record" }, to: { cell: "recover", port: "record" }, on: "fail" }, + { from: { cell: "field", port: "value" }, to: { cell: "recover", port: "field" } }, + ], + }); + const tools = lookupRegistry({ value: 0 }); + tools.get("records.lookup.v1")!.tool = async () => { + throw new Error("record unavailable"); + }; + const receipt = await runOrganism({ + manifest, + fns: builtinRegistry(), + store: new MemoryStore(), + executors: [], + tools, + }); + + expect(receipt.outcome).toBe("complete"); + expect(receipt.cells.lookup?.failure?.code).toBe("TOOL_FAILED"); + expect(receipt.cells.recover?.outputs?.value).toBe("TOOL_FAILED"); + expect(receipt.effects[0]?.error?.code).toBe("TOOL_FAILED"); +}); diff --git a/src/tools.ts b/src/tools.ts new file mode 100644 index 0000000..199c214 --- /dev/null +++ b/src/tools.ts @@ -0,0 +1,75 @@ +import { BOUNDS, parsePortMap, type PortMap } from "./contract"; +import { MorphogenError } from "./errors"; +import { + asObject, + asString, + noUnknownKeys, + reqField, + type JsonValue, +} from "./values"; + +export type ToolEffect = "read" | "write"; + +export type ToolSignature = { + inputs: PortMap; + outputs: PortMap; + effect: ToolEffect; + cost: number; + maxOutputBytes: number; +}; + +export type ToolContext = { + requestDigest: `sha256:${string}`; + idempotencyKey: `sha256:${string}`; + signal?: AbortSignal; +}; + +export type Tool = ( + inputs: Record, + context: ToolContext, +) => Promise>; + +export type ToolRegistry = Map; + +export function emptyToolRegistry(): ToolRegistry { + return new Map(); +} + +export const TOOL_SIGNATURE_BOUNDS = { + maxNameLen: 128, + maxCost: 1_000_000, +} as const; + +/** Parse a tool signature from foreign data — the same PortMap grammar cells + * use, plus effect class, modeled cost, and the output bound the runtime + * enforces on every call. */ +export function parseToolSignature(u: unknown, what = "tool signature"): ToolSignature { + const obj = asObject(u, what); + noUnknownKeys(obj, ["inputs", "outputs", "effect", "cost", "maxOutputBytes"], what); + const effect = asString(reqField(obj, "effect", what), `${what}.effect`, 8); + if (effect !== "read" && effect !== "write") { + throw new MorphogenError("PARSE_FAILED", `${what}.effect must be read|write`); + } + const cost = reqField(obj, "cost", what); + if (!Number.isInteger(cost) || (cost as number) < 0 || (cost as number) > TOOL_SIGNATURE_BOUNDS.maxCost) { + throw new MorphogenError("PARSE_FAILED", `${what}.cost must be an integer 0..${TOOL_SIGNATURE_BOUNDS.maxCost}`); + } + const maxOutputBytes = reqField(obj, "maxOutputBytes", what); + if ( + !Number.isInteger(maxOutputBytes) || + (maxOutputBytes as number) <= 0 || + (maxOutputBytes as number) > BOUNDS.maxValueBytes + ) { + throw new MorphogenError( + "PARSE_FAILED", + `${what}.maxOutputBytes must be an integer 1..${BOUNDS.maxValueBytes}`, + ); + } + return { + inputs: parsePortMap(reqField(obj, "inputs", what), `${what}.inputs`), + outputs: parsePortMap(reqField(obj, "outputs", what), `${what}.outputs`, "producer"), + effect, + cost: cost as number, + maxOutputBytes: maxOutputBytes as number, + }; +} diff --git a/src/verify.ts b/src/verify.ts index 2e5502c..c8a175f 100644 --- a/src/verify.ts +++ b/src/verify.ts @@ -14,6 +14,7 @@ import { import { manifestToJson, parseOrganismManifest } from "./contract"; import type { Store } from "./store"; import type { Transport } from "./transport"; +import type { ToolRegistry } from "./tools"; import { MorphogenError } from "./errors"; import { canonicalize, type JsonValue } from "./values"; @@ -30,6 +31,7 @@ export async function verifyReceipt( store: Store, fns: FnRegistry = builtinRegistry(), transports?: Record, + tools?: ToolRegistry, ): Promise { const original = parseRunReceipt(receiptJson); const manifest = parseOrganismManifest(manifestJson); @@ -65,7 +67,9 @@ export async function verifyReceipt( executors: [replayExecutor(original.effects)], replayVia, replaySlots, + replayToolEffects: original.effects.filter((effect) => effect.executor.startsWith("tool:")), ...(transports ? { transports } : {}), + ...(tools ? { tools } : {}), }); const mismatches = diffReceipts(original, rerun);