Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,15 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval-

---

## [0.173.3] — 2026-09-04

### Fixed

- The Runtime supervisor reader now treats `spawned.ownedTreeRoot` as the exclusive nested tree when present.
It rejects an additional child-id tree and any descendant spawn outside its parent's owned tree.

---

## [0.173.2] — 2026-09-04

### Fixed
Expand Down
2 changes: 1 addition & 1 deletion clients/python/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ build-backend = "hatchling.build"

[project]
name = "agent-eval-rpc"
version = "0.173.2"
version = "0.173.3"
description = "Python RPC client, official optimizer bridge, and DSPy metric adapter for @tangle-network/agent-eval."
readme = "README.md"
requires-python = ">=3.10"
Expand Down
2 changes: 1 addition & 1 deletion clients/python/src/agent_eval_rpc/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@
try:
__version__ = version("agent-eval-rpc")
except PackageNotFoundError:
__version__ = "0.173.2"
__version__ = "0.173.3"

__all__ = [
"Client",
Expand Down
2 changes: 1 addition & 1 deletion clients/python/uv.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@tangle-network/agent-eval",
"version": "0.173.2",
"version": "0.173.3",
"description": "Evaluate and improve AI agents from runs, traces, judges, and feedback. Compare candidates, cluster failures, measure lift, and gate releases.",
"homepage": "https://github.com/tangle-network/agent-eval#readme",
"repository": {
Expand Down
2 changes: 1 addition & 1 deletion src/analyst/benchmark-implementation.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ export const ANALYST_BENCHMARK_DEPENDENCY_LOCK_FILES = Object.freeze([
])

export const ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256 =
'7747ce4ff600eb66042957aeb46d80c9af0067fe54218e101283ca63f158aec5'
'ff7e12bb2457745febc8d525bb099365473eaae8b9bedde5dd5fbfbe6d86e470'

/** The published benchmark evidence was produced at this package version, by
* the retired one-shot direct runner, before trace analysts moved to the
Expand Down
86 changes: 86 additions & 0 deletions src/supervisor-run/runtime-reader.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -570,6 +570,92 @@ describe('Runtime FileRunContext supervisor reader', () => {
])
})

it('refuses a legacy child-id tree when the spawn names a different owned tree', async () => {
const parent = await mkdtemp(join(tmpdir(), 'runtime-supervisor-run-'))
const runDir = join(parent, 'ambiguous-owned-tree-root')
const childId = 'root:s0'
const ownedTreeRoot = `root/${childId}`
await writeJournal(runDir, [
begin('root', 0),
event('root', {
kind: 'spawned',
id: 'root',
parent: null,
label: 'root',
identity: { profileDigest: ROOT_PROFILE },
budget: {},
seq: 0,
at: at(0),
}),
event('root', {
kind: 'spawned',
id: childId,
parent: 'root',
ownedTreeRoot,
label: 'nested-researcher',
identity: { profileDigest: CHILD_PROFILE },
runtime: 'driver',
budget: {},
seq: 0,
at: at(1),
}),
begin(childId, 1),
begin(ownedTreeRoot, 1),
])

await expect(readRuntimeSupervisorRun(runDir, { strict: true })).rejects.toThrow(
/expected one top-level tree, found 2/,
)
})

it('refuses a descendant spawn outside the tree owned by its parent', async () => {
const parent = await mkdtemp(join(tmpdir(), 'runtime-supervisor-run-'))
const runDir = join(parent, 'misplaced-descendant')
const childId = 'root:s0'
const ownedTreeRoot = `root/${childId}`
await writeJournal(runDir, [
begin('root', 0),
event('root', {
kind: 'spawned',
id: 'root',
parent: null,
label: 'root',
identity: { profileDigest: ROOT_PROFILE },
budget: {},
seq: 0,
at: at(0),
}),
event('root', {
kind: 'spawned',
id: childId,
parent: 'root',
ownedTreeRoot,
label: 'nested-researcher',
identity: { profileDigest: CHILD_PROFILE },
runtime: 'driver',
budget: {},
seq: 0,
at: at(1),
}),
begin(ownedTreeRoot, 1),
event('root', {
kind: 'spawned',
id: `${childId}:s0`,
parent: childId,
label: 'misplaced-leaf',
identity: { profileDigest: LEAF_PROFILE },
runtime: 'cli',
budget: {},
seq: 0,
at: at(2),
}),
])

await expect(readRuntimeSupervisorRun(runDir, { strict: true })).rejects.toThrow(
/parent "root:s0" owns tree "root\/root:s0"/,
)
})

it('retains a current Runtime tree and classifies transport rows as unavailable only', async () => {
const parent = await mkdtemp(join(tmpdir(), 'runtime-supervisor-run-'))
const runDir = join(parent, 'arena-full1-ramsey-bundle-a')
Expand Down
52 changes: 44 additions & 8 deletions src/supervisor-run/runtime-reader.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,11 @@
* Each line is an envelope whose `root` identifies the local tree. A journal
* can connect a nested tree with `spawned.ownedTreeRoot`. It can also use the
* spawned child id as the nested root and repeat the spawn as a parentless
* marker. This reader accepts both forms, removes a duplicate marker, and
* preserves the other envelopes for the supervisor-run analyzer. Runtime
* stores profile identity below `identity` and does not emit Eval's role field.
* This boundary projects those fields without changing Runtime's dialect.
* marker when no owned tree is recorded. Descendant spawns must occur in the
* tree their parent owns. This reader removes a duplicate marker and preserves
* the other envelopes for the supervisor-run analyzer. Runtime stores profile
* identity below `identity` and does not emit Eval's role field. This boundary
* projects those fields without changing Runtime's dialect.
*
* The run's terminal status is Runtime's own `result.json` `kind` — `winner`,
* `no-winner`, or whatever a later arm is called — read verbatim. The reader
Expand Down Expand Up @@ -146,15 +147,15 @@ function parseEnvelopeJournal(text: string, path: string): NormalizedRuntimeJour
const id = nonEmptyString(entry.event.id)
if (id === null) continue
if (nonEmptyString(entry.event.parent) !== null) {
const matches = parentSpawnsById.get(id) ?? []
matches.push(entry)
parentSpawnsById.set(id, matches)

const ownedTreeRoot = nonEmptyString(entry.event.ownedTreeRoot)
if (ownedTreeRoot !== null) {
const owners = parentSpawnsByOwnedTreeRoot.get(ownedTreeRoot) ?? []
owners.push(entry)
parentSpawnsByOwnedTreeRoot.set(ownedTreeRoot, owners)
} else {
const matches = parentSpawnsById.get(id) ?? []
matches.push(entry)
parentSpawnsById.set(id, matches)
}
}
if (entry.root === id && (entry.event.parent === undefined || entry.event.parent === null)) {
Expand Down Expand Up @@ -227,6 +228,41 @@ function parseEnvelopeJournal(text: string, path: string): NormalizedRuntimeJour
}
}

const ownedTreeBySupervisorId = new Map<string, string>([[top.root, top.root]])
for (const [nestedRoot, parentSpawn] of nestedParentSpawns) {
const supervisorId = nonEmptyString(parentSpawn.event.id)
if (supervisorId === null) continue
const priorTree = ownedTreeBySupervisorId.get(supervisorId)
if (priorTree !== undefined && priorTree !== nestedRoot) {
throw formatError(
path,
parentSpawn.line,
`spawn ${JSON.stringify(supervisorId)} owns both ${JSON.stringify(priorTree)} and ${JSON.stringify(nestedRoot)}`,
)
}
ownedTreeBySupervisorId.set(supervisorId, nestedRoot)
}
for (const entry of events) {
if (entry.event.kind !== 'spawned') continue
const parentId = nonEmptyString(entry.event.parent)
if (parentId === null) continue
const parentTree = ownedTreeBySupervisorId.get(parentId)
if (parentTree === undefined) {
throw formatError(
path,
entry.line,
`spawn ${JSON.stringify(entry.event.id)} names parent ${JSON.stringify(parentId)}, which owns no journal tree`,
)
}
if (entry.root !== parentTree) {
throw formatError(
path,
entry.line,
`spawn ${JSON.stringify(entry.event.id)} is in tree ${JSON.stringify(entry.root)}, but parent ${JSON.stringify(parentId)} owns tree ${JSON.stringify(parentTree)}`,
)
}
}

// Runtime's recursive atom has no supervisor/worker role field. A tree root
// is a supervisor; a child without its own tree is a worker.
const supervisorIds = new Set([
Expand Down
Loading