From 299cbc390aa0805765f6830c85d292abdffa6105 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Fri, 4 Sep 2026 16:02:07 -0700 Subject: [PATCH] fix(supervisor-run): reject invalid Runtime tree ownership --- CHANGELOG.md | 9 ++ clients/python/pyproject.toml | 2 +- clients/python/src/agent_eval_rpc/__init__.py | 2 +- clients/python/uv.lock | 2 +- package.json | 2 +- src/analyst/benchmark-implementation.ts | 2 +- src/supervisor-run/runtime-reader.test.ts | 86 +++++++++++++++++++ src/supervisor-run/runtime-reader.ts | 52 +++++++++-- 8 files changed, 144 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8bfdbdfd..bd8ed026 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,15 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval- --- +## [0.173.3] — 2026-09-04 + +### Fixed + +- The Runtime supervisor reader now treats `spawned.ownedTreeRoot` as the exclusive nested tree when present. + It rejects an additional child-id tree and any descendant spawn outside its parent's owned tree. + +--- + ## [0.173.2] — 2026-09-04 ### Fixed diff --git a/clients/python/pyproject.toml b/clients/python/pyproject.toml index 327fcbdb..29b300c1 100644 --- a/clients/python/pyproject.toml +++ b/clients/python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "agent-eval-rpc" -version = "0.173.2" +version = "0.173.3" description = "Python RPC client, official optimizer bridge, and DSPy metric adapter for @tangle-network/agent-eval." readme = "README.md" requires-python = ">=3.10" diff --git a/clients/python/src/agent_eval_rpc/__init__.py b/clients/python/src/agent_eval_rpc/__init__.py index 3124055b..d3ce4f93 100644 --- a/clients/python/src/agent_eval_rpc/__init__.py +++ b/clients/python/src/agent_eval_rpc/__init__.py @@ -53,7 +53,7 @@ try: __version__ = version("agent-eval-rpc") except PackageNotFoundError: - __version__ = "0.173.2" + __version__ = "0.173.3" __all__ = [ "Client", diff --git a/clients/python/uv.lock b/clients/python/uv.lock index 705ae923..c52a1db2 100644 --- a/clients/python/uv.lock +++ b/clients/python/uv.lock @@ -34,7 +34,7 @@ conflicts = [[ [[package]] name = "agent-eval-rpc" -version = "0.173.2" +version = "0.173.3" source = { editable = "." } dependencies = [ { name = "filelock" }, diff --git a/package.json b/package.json index 4234c1f4..d69a4cf3 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-eval", - "version": "0.173.2", + "version": "0.173.3", "description": "Evaluate and improve AI agents from runs, traces, judges, and feedback. Compare candidates, cluster failures, measure lift, and gate releases.", "homepage": "https://github.com/tangle-network/agent-eval#readme", "repository": { diff --git a/src/analyst/benchmark-implementation.ts b/src/analyst/benchmark-implementation.ts index 2754d181..add0b764 100644 --- a/src/analyst/benchmark-implementation.ts +++ b/src/analyst/benchmark-implementation.ts @@ -10,7 +10,7 @@ export const ANALYST_BENCHMARK_DEPENDENCY_LOCK_FILES = Object.freeze([ ]) export const ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256 = - '7747ce4ff600eb66042957aeb46d80c9af0067fe54218e101283ca63f158aec5' + 'ff7e12bb2457745febc8d525bb099365473eaae8b9bedde5dd5fbfbe6d86e470' /** The published benchmark evidence was produced at this package version, by * the retired one-shot direct runner, before trace analysts moved to the diff --git a/src/supervisor-run/runtime-reader.test.ts b/src/supervisor-run/runtime-reader.test.ts index a54efb12..3aada9d4 100644 --- a/src/supervisor-run/runtime-reader.test.ts +++ b/src/supervisor-run/runtime-reader.test.ts @@ -570,6 +570,92 @@ describe('Runtime FileRunContext supervisor reader', () => { ]) }) + it('refuses a legacy child-id tree when the spawn names a different owned tree', async () => { + const parent = await mkdtemp(join(tmpdir(), 'runtime-supervisor-run-')) + const runDir = join(parent, 'ambiguous-owned-tree-root') + const childId = 'root:s0' + const ownedTreeRoot = `root/${childId}` + await writeJournal(runDir, [ + begin('root', 0), + event('root', { + kind: 'spawned', + id: 'root', + parent: null, + label: 'root', + identity: { profileDigest: ROOT_PROFILE }, + budget: {}, + seq: 0, + at: at(0), + }), + event('root', { + kind: 'spawned', + id: childId, + parent: 'root', + ownedTreeRoot, + label: 'nested-researcher', + identity: { profileDigest: CHILD_PROFILE }, + runtime: 'driver', + budget: {}, + seq: 0, + at: at(1), + }), + begin(childId, 1), + begin(ownedTreeRoot, 1), + ]) + + await expect(readRuntimeSupervisorRun(runDir, { strict: true })).rejects.toThrow( + /expected one top-level tree, found 2/, + ) + }) + + it('refuses a descendant spawn outside the tree owned by its parent', async () => { + const parent = await mkdtemp(join(tmpdir(), 'runtime-supervisor-run-')) + const runDir = join(parent, 'misplaced-descendant') + const childId = 'root:s0' + const ownedTreeRoot = `root/${childId}` + await writeJournal(runDir, [ + begin('root', 0), + event('root', { + kind: 'spawned', + id: 'root', + parent: null, + label: 'root', + identity: { profileDigest: ROOT_PROFILE }, + budget: {}, + seq: 0, + at: at(0), + }), + event('root', { + kind: 'spawned', + id: childId, + parent: 'root', + ownedTreeRoot, + label: 'nested-researcher', + identity: { profileDigest: CHILD_PROFILE }, + runtime: 'driver', + budget: {}, + seq: 0, + at: at(1), + }), + begin(ownedTreeRoot, 1), + event('root', { + kind: 'spawned', + id: `${childId}:s0`, + parent: childId, + label: 'misplaced-leaf', + identity: { profileDigest: LEAF_PROFILE }, + runtime: 'cli', + budget: {}, + seq: 0, + at: at(2), + }), + ]) + + await expect(readRuntimeSupervisorRun(runDir, { strict: true })).rejects.toThrow( + /parent "root:s0" owns tree "root\/root:s0"/, + ) + }) + it('retains a current Runtime tree and classifies transport rows as unavailable only', async () => { const parent = await mkdtemp(join(tmpdir(), 'runtime-supervisor-run-')) const runDir = join(parent, 'arena-full1-ramsey-bundle-a') diff --git a/src/supervisor-run/runtime-reader.ts b/src/supervisor-run/runtime-reader.ts index 60a50d7f..620bb5cf 100644 --- a/src/supervisor-run/runtime-reader.ts +++ b/src/supervisor-run/runtime-reader.ts @@ -5,10 +5,11 @@ * Each line is an envelope whose `root` identifies the local tree. A journal * can connect a nested tree with `spawned.ownedTreeRoot`. It can also use the * spawned child id as the nested root and repeat the spawn as a parentless - * marker. This reader accepts both forms, removes a duplicate marker, and - * preserves the other envelopes for the supervisor-run analyzer. Runtime - * stores profile identity below `identity` and does not emit Eval's role field. - * This boundary projects those fields without changing Runtime's dialect. + * marker when no owned tree is recorded. Descendant spawns must occur in the + * tree their parent owns. This reader removes a duplicate marker and preserves + * the other envelopes for the supervisor-run analyzer. Runtime stores profile + * identity below `identity` and does not emit Eval's role field. This boundary + * projects those fields without changing Runtime's dialect. * * The run's terminal status is Runtime's own `result.json` `kind` — `winner`, * `no-winner`, or whatever a later arm is called — read verbatim. The reader @@ -146,15 +147,15 @@ function parseEnvelopeJournal(text: string, path: string): NormalizedRuntimeJour const id = nonEmptyString(entry.event.id) if (id === null) continue if (nonEmptyString(entry.event.parent) !== null) { - const matches = parentSpawnsById.get(id) ?? [] - matches.push(entry) - parentSpawnsById.set(id, matches) - const ownedTreeRoot = nonEmptyString(entry.event.ownedTreeRoot) if (ownedTreeRoot !== null) { const owners = parentSpawnsByOwnedTreeRoot.get(ownedTreeRoot) ?? [] owners.push(entry) parentSpawnsByOwnedTreeRoot.set(ownedTreeRoot, owners) + } else { + const matches = parentSpawnsById.get(id) ?? [] + matches.push(entry) + parentSpawnsById.set(id, matches) } } if (entry.root === id && (entry.event.parent === undefined || entry.event.parent === null)) { @@ -227,6 +228,41 @@ function parseEnvelopeJournal(text: string, path: string): NormalizedRuntimeJour } } + const ownedTreeBySupervisorId = new Map([[top.root, top.root]]) + for (const [nestedRoot, parentSpawn] of nestedParentSpawns) { + const supervisorId = nonEmptyString(parentSpawn.event.id) + if (supervisorId === null) continue + const priorTree = ownedTreeBySupervisorId.get(supervisorId) + if (priorTree !== undefined && priorTree !== nestedRoot) { + throw formatError( + path, + parentSpawn.line, + `spawn ${JSON.stringify(supervisorId)} owns both ${JSON.stringify(priorTree)} and ${JSON.stringify(nestedRoot)}`, + ) + } + ownedTreeBySupervisorId.set(supervisorId, nestedRoot) + } + for (const entry of events) { + if (entry.event.kind !== 'spawned') continue + const parentId = nonEmptyString(entry.event.parent) + if (parentId === null) continue + const parentTree = ownedTreeBySupervisorId.get(parentId) + if (parentTree === undefined) { + throw formatError( + path, + entry.line, + `spawn ${JSON.stringify(entry.event.id)} names parent ${JSON.stringify(parentId)}, which owns no journal tree`, + ) + } + if (entry.root !== parentTree) { + throw formatError( + path, + entry.line, + `spawn ${JSON.stringify(entry.event.id)} is in tree ${JSON.stringify(entry.root)}, but parent ${JSON.stringify(parentId)} owns tree ${JSON.stringify(parentTree)}`, + ) + } + } + // Runtime's recursive atom has no supervisor/worker role field. A tree root // is a supervisor; a child without its own tree is a worker. const supervisorIds = new Set([