From b0870f3b43a84dbef1bcac09d344f69c3304af8d Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Fri, 4 Sep 2026 19:23:39 -0700 Subject: [PATCH] fix(supervise): refuse in-doubt replacement dispatch --- api-surface.json | 4 +- docs/api/runtime.md | 52 ++++++++-------- docs/api/testing.md | 16 ++--- src/mcp/tools/coordination.ts | 23 +++++--- src/runtime/supervise/coordination-driver.ts | 10 ++-- src/runtime/supervise/scope.ts | 19 +++--- src/runtime/supervise/supervise.ts | 8 +-- src/runtime/supervise/types.ts | 32 +++++----- tests/helpers/resume-driver-child.ts | 15 +++-- tests/runtime/resume-aware-driver.test.ts | 62 ++++++++++++-------- tests/runtime/spawn-keys.test.ts | 49 ++++++++++++++++ 11 files changed, 179 insertions(+), 111 deletions(-) diff --git a/api-surface.json b/api-surface.json index 02be52362..f341f1e30 100644 --- a/api-surface.json +++ b/api-surface.json @@ -1542,10 +1542,10 @@ "SpawnOpts": "type 903c3a01c5cb", "SpawnPreflight": "type 8e8b479089d3", "SpawnPreflightContext": "type 21972a19ec24", - "SpawnPrior": "type f5b562b41ed4", + "SpawnPrior": "type af72d7e58e96", "SpawnRefusal": "type e0fea783ffe7", "SpawnRefusalCause": "type a9399e96f43a", - "SpawnRejection": "type dfb5b2bcb409", + "SpawnRejection": "type 0f740e25e598", "Spend": "type 6e558e6744c8", "SpendChannel": "type 978ffae337cd", "SpendGap": "type d24924855c35", diff --git a/docs/api/runtime.md b/docs/api/runtime.md index e6067ba86..b0052ca18 100644 --- a/docs/api/runtime.md +++ b/docs/api/runtime.md @@ -13659,10 +13659,10 @@ Instruction receipts are evidence and are never delivered automatically to a rep worker. The final result spans both processes' work. Unset = in-memory, fresh every call. The boundary that remains: work that was IN FLIGHT when the process died is not recovered — -the built-in executors cannot re-attach to a dead process's executions. Each such assignment -resumes as explicitly lost/in-doubt, its full declared reservation is charged conservatively, -and its token/dollar telemetry remains unknown. A retry is admitted only from safely remaining -capacity, so restart cannot mint a fresh budget or slide the original absolute deadline. +the built-in executors cannot re-attach to a dead process's executions. Each such keyed +assignment resumes as `in-doubt`, its full declared reservation stays charged, and its +token/dollar telemetry remains unknown. Runtime refuses a replacement under that key until the +exact prior execution is recovered, so restart cannot duplicate work or slide the deadline. `runId` matters here: it defaults to the constant `'supervise'`, which is fine for a single resumable run per directory but collides across concurrent runs sharing one `runDir`. @@ -17971,10 +17971,10 @@ Instruction receipts are evidence and are never delivered automatically to a rep worker. The final result spans both processes' work. Unset = in-memory, fresh every call. The boundary that remains: work that was IN FLIGHT when the process died is not recovered — -the built-in executors cannot re-attach to a dead process's executions. Each such assignment -resumes as explicitly lost/in-doubt, its full declared reservation is charged conservatively, -and its token/dollar telemetry remains unknown. A retry is admitted only from safely remaining -capacity, so restart cannot mint a fresh budget or slide the original absolute deadline. +the built-in executors cannot re-attach to a dead process's executions. Each such keyed +assignment resumes as `in-doubt`, its full declared reservation stays charged, and its +token/dollar telemetry remains unknown. Runtime refuses a replacement under that key until the +exact prior execution is recovered, so restart cannot duplicate work or slide the deadline. `runId` matters here: it defaults to the constant `'supervise'`, which is fine for a single resumable run per directory but collides across concurrent runs sharing one `runDir`. @@ -19772,10 +19772,10 @@ Semantic identity of this assignment ACROSS process lifetimes. A keyed spawn is idempotent per key: once a child spawned under a key settles `done` — in this process or in a journaled prior one — spawning the same key returns that committed result (`prior.state: 'completed'`) instead of paying for the work again. A key whose prior attempt settled `down` -or was journaled as started-but-never-settled spawns FRESH but says so explicitly -(`prior.state: 'retried' | 'lost'`), and a key that is currently LIVE is refused -(`'duplicate-key'`) — the same assignment can never run twice concurrently. Unkeyed spawns -(the default) are position-identified and always run. +spawns fresh and says so explicitly (`prior.state: 'retried'`). A key whose prior attempt was +journaled as started but never settled is refused (`'in-doubt'`): the remote execution may +still exist and must be recovered before replacement. A key that is currently LIVE is refused +(`'duplicate-key'`). Unkeyed spawns (the default) are position-identified and always run. *** @@ -19917,9 +19917,9 @@ resumed run that was not waiting. Keyed assignments from the prior journal: `SpawnOpts.key` → what the journal proves about it. `completed`/`down` carry the rehydrated settlement; `in-doubt` means the spawn was journaled -but no settlement ever landed — the process died with it in flight. `Scope.spawn` consults -this so a keyed re-spawn resolves instead of duplicating (see `SpawnOpts.key`). Empty when no -prior spawn carried a key. +but no settlement ever landed — the process died with it in flight. `Scope.spawn` refuses a +keyed replacement in that state, rather than duplicate a possibly live remote execution. Empty +when no prior spawn carried a key. ##### priorSpend @@ -19968,7 +19968,7 @@ Identity recorded when this key was first admitted. Every reuse must match it ex ##### state -> `readonly` **state**: `"completed"` \| `"down"` \| `"in-doubt"` +> `readonly` **state**: `"completed"` \| `"in-doubt"` \| `"down"` ##### settled? @@ -25139,11 +25139,11 @@ Deterministic node id — `${parent}:s${seq}` from the cursor order, never wall- ### SpawnRejection -> **SpawnRejection** = `"budget-exhausted"` \| `"usd-unbudgeted"` \| `"depth-exceeded"` \| `"duplicate-key"` \| `"invalid-identity"` \| `"key-conflict"` \| `"max-live-workers"` \| `"scope-aborted"` +> **SpawnRejection** = `"budget-exhausted"` \| `"usd-unbudgeted"` \| `"depth-exceeded"` \| `"duplicate-key"` \| `"in-doubt"` \| `"invalid-identity"` \| `"key-conflict"` \| `"max-live-workers"` \| `"scope-aborted"` Fail-closed spawn rejections: an exhausted pool, a dollar request against a root that budgets - no dollars, an exceeded recursion ceiling, a full tree-wide worker allocation, or a `key` that - is still LIVE in this scope (the same assignment may not run twice concurrently). + no dollars, an exceeded recursion ceiling, a full tree-wide worker allocation, a `key` that is + still LIVE in this scope, or a key whose prior remote execution has no terminal receipt. `usd-unbudgeted` is separate from `budget-exhausted` because the two call for opposite responses: an exhausted pool may admit a smaller request, while an unbudgeted dollar channel @@ -25153,18 +25153,14 @@ refuses every amount until the ROOT budget names a `maxUsd`. ### SpawnPrior -> **SpawnPrior**\<`Out`\> = \{ `state`: `"completed"`; `settled`: [`Settled`](index.md#settled)\<`Out`\> & `object`; \} \| \{ `state`: `"retried"`; `priorId`: [`NodeId`](#nodeid-6); `reason`: `string`; \} \| \{ `state`: `"lost"`; `priorId`: [`NodeId`](#nodeid-6); \} +> **SpawnPrior**\<`Out`\> = \{ `state`: `"completed"`; `settled`: [`Settled`](index.md#settled)\<`Out`\> & `object`; \} \| \{ `state`: `"retried"`; `priorId`: [`NodeId`](#nodeid-6); `reason`: `string`; \} What a KEYED spawn resolved to when the key had a prior attempt. Absent on a fresh key (and on every unkeyed spawn). `'completed'` is the exactly-once path: NOTHING was spawned — the handle -references the prior settled node and `settled` is the committed result. `'retried'` / -`'lost'` DID spawn fresh: the prior attempt settled `down` (retried) or was journaled as -started but never settled — the process died with it in flight and the built-in executors -cannot re-attach to a dead process's work, so the result is explicitly in doubt (lost), never -silently duplicated. On restart, an in-doubt attempt's full declared reservation is charged and -its telemetry remains unknown; a fresh retry is admitted only from safely remaining capacity. -An executor that CAN re-attach to a still-running external execution extends this union with an -adoption state; none of the built-ins can today. +references the prior settled node and `settled` is the committed result. `'retried'` DID spawn +fresh because the prior attempt settled `down`. A start with no terminal receipt is not a prior +result: `spawn` refuses it as `'in-doubt'`, retains its charged reservation, and requires exact +recovery before a replacement can run. #### Type Parameters diff --git a/docs/api/testing.md b/docs/api/testing.md index a9abade91..e5341a674 100644 --- a/docs/api/testing.md +++ b/docs/api/testing.md @@ -1055,10 +1055,10 @@ Instruction receipts are evidence and are never delivered automatically to a rep worker. The final result spans both processes' work. Unset = in-memory, fresh every call. The boundary that remains: work that was IN FLIGHT when the process died is not recovered — -the built-in executors cannot re-attach to a dead process's executions. Each such assignment -resumes as explicitly lost/in-doubt, its full declared reservation is charged conservatively, -and its token/dollar telemetry remains unknown. A retry is admitted only from safely remaining -capacity, so restart cannot mint a fresh budget or slide the original absolute deadline. +the built-in executors cannot re-attach to a dead process's executions. Each such keyed +assignment resumes as `in-doubt`, its full declared reservation stays charged, and its +token/dollar telemetry remains unknown. Runtime refuses a replacement under that key until the +exact prior execution is recovered, so restart cannot duplicate work or slide the deadline. `runId` matters here: it defaults to the constant `'supervise'`, which is fine for a single resumable run per directory but collides across concurrent runs sharing one `runDir`. @@ -1986,10 +1986,10 @@ Instruction receipts are evidence and are never delivered automatically to a rep worker. The final result spans both processes' work. Unset = in-memory, fresh every call. The boundary that remains: work that was IN FLIGHT when the process died is not recovered — -the built-in executors cannot re-attach to a dead process's executions. Each such assignment -resumes as explicitly lost/in-doubt, its full declared reservation is charged conservatively, -and its token/dollar telemetry remains unknown. A retry is admitted only from safely remaining -capacity, so restart cannot mint a fresh budget or slide the original absolute deadline. +the built-in executors cannot re-attach to a dead process's executions. Each such keyed +assignment resumes as `in-doubt`, its full declared reservation stays charged, and its +token/dollar telemetry remains unknown. Runtime refuses a replacement under that key until the +exact prior execution is recovered, so restart cannot duplicate work or slide the deadline. `runId` matters here: it defaults to the constant `'supervise'`, which is fine for a single resumable run per directory but collides across concurrent runs sharing one `runDir`. diff --git a/src/mcp/tools/coordination.ts b/src/mcp/tools/coordination.ts index 48b36b640..e97cc5821 100644 --- a/src/mcp/tools/coordination.ts +++ b/src/mcp/tools/coordination.ts @@ -2749,9 +2749,10 @@ export function createCoordinationTools(opts: CoordinationToolsOptions): Coordin 'many workers are still in flight — settle or steer one before spawning another. ' + 'Pass a `key` naming the assignment to make it run-once ACROSS restarts: a key that ' + 'already completed returns the finished result (`resumed: "completed"` — no work re-runs, ' + - 'nothing is spent), a key whose prior attempt failed or was lost with a dead process ' + - 'spawns fresh and says so (`resumed: "retried" | "lost"`), and a key still running is ' + - 'refused (`error: "duplicate-key"`). ' + + 'nothing is spent), a key whose prior attempt failed (`down`) spawns fresh and says so ' + + '(`resumed: "retried"`), and a key with no terminal receipt is refused ' + + '(`error: "in-doubt"`) until its exact prior execution is recovered. A key still running ' + + 'is refused (`error: "duplicate-key"`). ' + 'Returns `freeSlots`: how many MORE workers you can start right now (`null` = uncapped). ' + 'While `freeSlots > 0` there is idle capacity — call this again to fill it rather than ' + 'waiting; parallel workers finish the run sooner than one at a time.', @@ -2932,7 +2933,7 @@ export function createCoordinationTools(opts: CoordinationToolsOptions): Coordin } } // A `completed` key returned above, so any prior still attached here is a real re-run: - // `retried` (the prior attempt failed) or `lost` (it died in flight with its process). + // `retried` means the prior attempt reached a terminal down receipt. const priorHistory = res.ok && res.prior !== undefined && res.prior.state !== 'completed' ? { @@ -2944,8 +2945,8 @@ export function createCoordinationTools(opts: CoordinationToolsOptions): Coordin // Report the REMAINING capacity alongside the spawn, so one tool call tells the driver // both "it started" and "you can still open N more" — the feedback that lets it fill // slots instead of opening one worker per turn. `null` = uncapped. A fresh spawn under a - // key with a failed/lost prior attempt carries that history (`resumed`/`priorWorkerId`), - // so a re-run is always explicit, never a silent duplicate. + // key with a failed prior attempt carries that history (`resumed`/`priorWorkerId`), so a + // re-run is always explicit. An in-doubt prior is refused above and never duplicated. return Promise.resolve( res.ok ? { @@ -2974,7 +2975,9 @@ export function createCoordinationTools(opts: CoordinationToolsOptions): Coordin reason: res.reason === 'usd-unbudgeted' ? "this run's root budget declares no maxUsd, so a child budget naming maxUsd can never be admitted at any amount — spawn with a budget that omits maxUsd" - : `the conserved pool refused this spawn (${String(res.reason)}); the run has no allocation left to give this worker`, + : res.reason === 'in-doubt' + ? 'this key has a prior worker recorded as started without a terminal receipt; no replacement was started because that remote worker may still be running — inspect or recover the exact prior execution before retrying' + : `the conserved pool refused this spawn (${String(res.reason)}); the run has no allocation left to give this worker`, ...(res.reason === 'usd-unbudgeted' ? { hint: @@ -2983,7 +2986,11 @@ export function createCoordinationTools(opts: CoordinationToolsOptions): Coordin 'fail identically. Spawn with a budget that omits maxUsd, or ask the caller ' + 'to give the run a root maxUsd.', } - : {}), + : res.reason === 'in-doubt' + ? { + hint: 'Do not retry this key. Use the recorded prior worker identity to inspect or recover that exact execution. A terminal receipt or explicit recovery is required before replacement work can start.', + } + : {}), live: liveWorkerCount(), freeSlots: freeWorkerSlots(), }, diff --git a/src/runtime/supervise/coordination-driver.ts b/src/runtime/supervise/coordination-driver.ts index 13042bc99..1b4f41073 100644 --- a/src/runtime/supervise/coordination-driver.ts +++ b/src/runtime/supervise/coordination-driver.ts @@ -1243,7 +1243,7 @@ export function driverAgent(opts: DriverAgentOptions): Agent { /** * The factual context a resumed driver starts from — everything the durable stores prove about - * the prior process(es): committed settlements, per-key states (completed / lost / failed), + * the prior process(es): committed settlements, per-key states (completed / in-doubt / failed), * re-armed waits, carried-over questions/findings/continuation receipts, and spend already paid. * Injected as the brain's first user-context on a resumed run so it continues from unresolved work; * old continuation receipts are evidence and are never auto-delivered. @@ -1269,7 +1269,7 @@ function resumeBrief(resume: ResumedWork, prior?: PriorCoordination): s const byState = (state: 'completed' | 'in-doubt' | 'down') => [...resume.keys].filter(([, v]) => v.state === state) const completed = byState('completed') - const lost = byState('in-doubt') + const inDoubt = byState('in-doubt') const failed = byState('down') if (completed.length > 0) { lines.push( @@ -1278,11 +1278,11 @@ function resumeBrief(resume: ResumedWork, prior?: PriorCoordination): s ...completed.map(([k, v]) => `- ${k} → ${v.id} (${v.label})`), ) } - if (lost.length > 0) { + if (inDoubt.length > 0) { lines.push( '', - 'Keys LOST in flight with the prior process — this is the unresolved work; spawn_worker with the same key starts a fresh attempt:', - ...lost.map(([k, v]) => `- ${k} (prior attempt ${v.id}, ${v.label})`), + 'Keys IN DOUBT — a prior process recorded them as started but never recorded a terminal receipt. Do NOT spawn a replacement under these keys; inspect or recover each exact prior execution first:', + ...inDoubt.map(([k, v]) => `- ${k} (prior attempt ${v.id}, ${v.label})`), ) } if (failed.length > 0) { diff --git a/src/runtime/supervise/scope.ts b/src/runtime/supervise/scope.ts index 40a6a137f..6f263efb3 100644 --- a/src/runtime/supervise/scope.ts +++ b/src/runtime/supervise/scope.ts @@ -476,8 +476,8 @@ export function createScope(args: ScopeArgs): Scope { // The semantic-key registry (`SpawnOpts.key`): every keyed assignment's current state, seeded // from the prior journal on resume and updated live as keyed children spawn and settle. This is // what makes a keyed spawn idempotent per key across process lifetimes: `done` returns the - // committed result, `live` refuses a concurrent duplicate, `down`/`in-doubt` spawn fresh but - // say so explicitly. + // committed result, `live` refuses a concurrent duplicate, `down` retries after a terminal + // receipt, and `in-doubt` refuses until the exact prior execution is recovered. type KeyState = | { readonly state: 'live'; readonly id: NodeId; readonly identity: NodeExecutionIdentity } | { @@ -540,10 +540,10 @@ export function createScope(args: ScopeArgs): Scope { const task = detachedSnapshot(rawTask, 'scope.spawn task') const opts = detachedSnapshot(rawOpts, 'scope.spawn options') - // A key is an identity claim, not merely a cache label. On every reuse, prepare the requested - // agent far enough to derive the authorized profile/task identity, then compare it with the - // journal before returning an old result or retrying old work. No executor is resolved, - // constructed, reserved, or run on the completed path. + // A key is an identity claim, not merely a cache label. An in-doubt start has no terminal + // receipt, so it refuses before touching a lazy factory. Every other reuse prepares the + // requested agent far enough to compare its authorized profile/task identity with the journal. + // No executor is resolved, constructed, reserved, or run on the completed path. let prior: SpawnPrior | undefined let prepared: | { @@ -565,6 +565,11 @@ export function createScope(args: ScopeArgs): Scope { } if (opts.key !== undefined) { const existing = keyed.get(opts.key) + if (existing?.state === 'in-doubt') { + // A durable start with no terminal receipt does not prove the remote execution stopped. + // Do not invoke a lazy factory, reserve, or construct a replacement beside it. + return { ok: false, reason: 'in-doubt' } + } if (existing !== undefined) { prepared = prepare() if (!isCompleteIdentity(prepared.identity)) { @@ -584,8 +589,6 @@ export function createScope(args: ScopeArgs): Scope { } if (existing?.state === 'down') { prior = { state: 'retried', priorId: existing.id, reason: existing.reason } - } else if (existing?.state === 'in-doubt') { - prior = { state: 'lost', priorId: existing.id } } } diff --git a/src/runtime/supervise/supervise.ts b/src/runtime/supervise/supervise.ts index 0638032df..74c36ade8 100644 --- a/src/runtime/supervise/supervise.ts +++ b/src/runtime/supervise/supervise.ts @@ -1410,10 +1410,10 @@ export interface SuperviseOptions { * worker. The final result spans both processes' work. Unset = in-memory, fresh every call. * * The boundary that remains: work that was IN FLIGHT when the process died is not recovered — - * the built-in executors cannot re-attach to a dead process's executions. Each such assignment - * resumes as explicitly lost/in-doubt, its full declared reservation is charged conservatively, - * and its token/dollar telemetry remains unknown. A retry is admitted only from safely remaining - * capacity, so restart cannot mint a fresh budget or slide the original absolute deadline. + * the built-in executors cannot re-attach to a dead process's executions. Each such keyed + * assignment resumes as `in-doubt`, its full declared reservation stays charged, and its + * token/dollar telemetry remains unknown. Runtime refuses a replacement under that key until the + * exact prior execution is recovered, so restart cannot duplicate work or slide the deadline. * * `runId` matters here: it defaults to the constant `'supervise'`, which is fine for a single * resumable run per directory but collides across concurrent runs sharing one `runDir`. diff --git a/src/runtime/supervise/types.ts b/src/runtime/supervise/types.ts index 6335c9285..1049c2efc 100644 --- a/src/runtime/supervise/types.ts +++ b/src/runtime/supervise/types.ts @@ -800,17 +800,17 @@ export interface SpawnOpts { * idempotent per key: once a child spawned under a key settles `done` — in this process or in a * journaled prior one — spawning the same key returns that committed result (`prior.state: * 'completed'`) instead of paying for the work again. A key whose prior attempt settled `down` - * or was journaled as started-but-never-settled spawns FRESH but says so explicitly - * (`prior.state: 'retried' | 'lost'`), and a key that is currently LIVE is refused - * (`'duplicate-key'`) — the same assignment can never run twice concurrently. Unkeyed spawns - * (the default) are position-identified and always run. + * spawns fresh and says so explicitly (`prior.state: 'retried'`). A key whose prior attempt was + * journaled as started but never settled is refused (`'in-doubt'`): the remote execution may + * still exist and must be recovered before replacement. A key that is currently LIVE is refused + * (`'duplicate-key'`). Unkeyed spawns (the default) are position-identified and always run. */ readonly key?: string } /** Fail-closed spawn rejections: an exhausted pool, a dollar request against a root that budgets - * no dollars, an exceeded recursion ceiling, a full tree-wide worker allocation, or a `key` that - * is still LIVE in this scope (the same assignment may not run twice concurrently). + * no dollars, an exceeded recursion ceiling, a full tree-wide worker allocation, a `key` that is + * still LIVE in this scope, or a key whose prior remote execution has no terminal receipt. * * `usd-unbudgeted` is separate from `budget-exhausted` because the two call for opposite * responses: an exhausted pool may admit a smaller request, while an unbudgeted dollar channel @@ -820,6 +820,7 @@ export type SpawnRejection = | 'usd-unbudgeted' | 'depth-exceeded' | 'duplicate-key' + | 'in-doubt' | 'invalid-identity' | 'key-conflict' | 'max-live-workers' @@ -828,19 +829,14 @@ export type SpawnRejection = /** * What a KEYED spawn resolved to when the key had a prior attempt. Absent on a fresh key (and on * every unkeyed spawn). `'completed'` is the exactly-once path: NOTHING was spawned — the handle - * references the prior settled node and `settled` is the committed result. `'retried'` / - * `'lost'` DID spawn fresh: the prior attempt settled `down` (retried) or was journaled as - * started but never settled — the process died with it in flight and the built-in executors - * cannot re-attach to a dead process's work, so the result is explicitly in doubt (lost), never - * silently duplicated. On restart, an in-doubt attempt's full declared reservation is charged and - * its telemetry remains unknown; a fresh retry is admitted only from safely remaining capacity. - * An executor that CAN re-attach to a still-running external execution extends this union with an - * adoption state; none of the built-ins can today. + * references the prior settled node and `settled` is the committed result. `'retried'` DID spawn + * fresh because the prior attempt settled `down`. A start with no terminal receipt is not a prior + * result: `spawn` refuses it as `'in-doubt'`, retains its charged reservation, and requires exact + * recovery before a replacement can run. */ export type SpawnPrior = | { readonly state: 'completed'; readonly settled: Settled & { kind: 'done' } } | { readonly state: 'retried'; readonly priorId: NodeId; readonly reason: string } - | { readonly state: 'lost'; readonly priorId: NodeId } /** * A live child handle. `abort()` is defined over the ACQUIRE lifecycle: it chains into @@ -1092,9 +1088,9 @@ export interface ResumedWork { /** * Keyed assignments from the prior journal: `SpawnOpts.key` → what the journal proves about it. * `completed`/`down` carry the rehydrated settlement; `in-doubt` means the spawn was journaled - * but no settlement ever landed — the process died with it in flight. `Scope.spawn` consults - * this so a keyed re-spawn resolves instead of duplicating (see `SpawnOpts.key`). Empty when no - * prior spawn carried a key. + * but no settlement ever landed — the process died with it in flight. `Scope.spawn` refuses a + * keyed replacement in that state, rather than duplicate a possibly live remote execution. Empty + * when no prior spawn carried a key. */ readonly keys: ReadonlyMap> /** diff --git a/tests/helpers/resume-driver-child.ts b/tests/helpers/resume-driver-child.ts index d68258860..b2166afd7 100644 --- a/tests/helpers/resume-driver-child.ts +++ b/tests/helpers/resume-driver-child.ts @@ -6,13 +6,11 @@ * * Phase `1` SIGKILLs itself from inside the brain the instant it has pulled three settlements — no * unwinding, no flush. Phase `2` is a brand-new process pointed at the SAME dir + runId. Phase - * `control` runs the identical script start-to-finish in a fresh directory: the uninterrupted - * baseline the crash-then-resume pair must match. + * `control` runs the identical script start-to-finish in a fresh directory. * * Every phase runs the SAME brain script — five keyed workers, then pull events until idle. The - * brain is deliberately NOT resume-aware: it re-issues the identical plan every time. Skipping the - * committed work is therefore the RUNTIME's job (`spawn_worker`'s `key` resolving against the - * journal), which is exactly the property under test. + * brain is deliberately NOT resume-aware: it re-issues the identical plan every time. The runtime + * must return committed work and refuse a potentially live prior execution, never duplicate it. */ import { appendFileSync } from 'node:fs' @@ -112,8 +110,14 @@ function leafAgent( const brainUsage = { input: 100, output: 20 } let spawnedPlan = false +let observedToolResultCount = 0 +const inDoubtRefusals: string[] = [] const brain: ToolLoopChat = async (messages) => { const toolResults = messages.filter((m) => m.role === 'tool').map((m) => String(m.content ?? '')) + for (const result of toolResults.slice(observedToolResultCount)) { + if (result.includes('"error":"in-doubt"')) inDoubtRefusals.push(result) + } + observedToolResultCount = toolResults.length const settlementsPulled = toolResults.filter((t) => t.includes('"type":"settled"')).length // PHASE 1 CRASH: three assignments are committed (their blob + `settled` record are fsynced @@ -175,5 +179,6 @@ process.stdout.write( .filter((n) => n.status === 'done') .map((n) => n.label) .sort(), + inDoubtRefusals, })}\n`, ) diff --git a/tests/runtime/resume-aware-driver.test.ts b/tests/runtime/resume-aware-driver.test.ts index 7c1298d3c..2f595761b 100644 --- a/tests/runtime/resume-aware-driver.test.ts +++ b/tests/runtime/resume-aware-driver.test.ts @@ -6,9 +6,8 @@ * resumes on the same `runId` + `runDir` and re-issues the IDENTICAL plan — the brain is not * resume-aware, so every skip has to come from the runtime. * - * What must hold: exactly two units of work remain, no already-completed worker executes a second - * time, and the resumed run's final output and worker spend equal an uninterrupted control run's. - * The spend equality is the real bar — it is what "did not pay twice" means in a number. + * What must hold: no replacement starts for a worker whose prior remote execution has no terminal + * receipt. The resumed run keeps that work visible as unknown and charges its declared ceiling. */ import { spawn } from 'node:child_process' @@ -37,6 +36,7 @@ interface PhaseReport { readonly spentTotal: Spend readonly spentBreakdown?: { driverInference: Spend; childWork: Spend } readonly settledNodes: string[] + readonly inDoubtRefusals: string[] } async function runPhase(dir: string, runId: string, phase: Phase): Promise { @@ -71,7 +71,7 @@ async function execLog(dir: string, phase: Phase): Promise { } } -describe('resume-aware built-in driver — a killed coordinator resumes without re-paying', () => { +describe('resume-aware built-in driver — a killed coordinator never duplicates unknown work', () => { let dir: string let controlDir: string beforeEach(async () => { @@ -83,7 +83,7 @@ describe('resume-aware built-in driver — a killed coordinator resumes without await rm(controlDir, { recursive: true, force: true }) }) - it('five workers, three settled, coordinator killed: exactly two re-run and the output + worker spend match an uninterrupted control run', { + it('five workers, three settled, coordinator killed: the two unknown assignments are refused without new execution', { timeout: 180_000, }, async () => { const runId = 'five-assignments' @@ -123,37 +123,49 @@ describe('resume-aware built-in driver — a killed coordinator resumes without expect(second.code, `phase 2 stderr: ${second.stderr}`).toBe(0) const resumed = JSON.parse(second.stdout.trim()) as PhaseReport - // THE CLAIM: exactly two units of work remained, and not one already-completed worker ran - // its model call again. - const rerun = await execLog(dir, '2') - expect(rerun).toHaveLength(2) - expect(rerun).toEqual(['w4', 'w5']) - for (const done of ['w1', 'w2', 'w3']) expect(rerun).not.toContain(done) + // The safety boundary: phase 2 starts no replacement. The two keyed requests return an + // action-oriented refusal instead of treating an absent receipt as proof of death. + expect(await execLog(dir, '2')).toEqual([]) + expect(resumed.inDoubtRefusals).toHaveLength(2) + for (const refusal of resumed.inDoubtRefusals) { + expect(refusal).toContain('"error":"in-doubt"') + expect(refusal).toContain('no replacement was started') + } + + // Phase 2 emits no `spawned` records. The original five semantic assignments remain the + // whole tree, with only the three terminal records the journal can prove. + const afterResume = (await journal.loadTree(runId)) ?? [] + const resumedSpawnedKeys = afterResume + .filter((e): e is Extract => e.kind === 'spawned') + .map((e) => e.key) + .filter((k): k is string => k !== undefined) + .sort() + expect(resumedSpawnedKeys).toEqual(['w1', 'w2', 'w3', 'w4', 'w5']) - // The finalize spans both processes: all five assignments are in the tree, and the winner - // is `w5` — reachable only by combining phase 1's committed work with phase 2's. + // The default finalizer may select the best independently verified settled artifact. It does + // not invent terminal records for the two unknown assignments or claim their work completed. expect(resumed.kind).toBe('winner') - expect(resumed.settledNodes).toEqual(['w1', 'w2', 'w3', 'w4', 'w5']) + expect(resumed.out).toBe('W3') + expect(resumed.out).not.toBe(controlReport.out) + expect(resumed.settledNodes).toEqual(['w1', 'w2', 'w3']) - // The output matches control. Spend does not pretend the two killed executions were free: - // their missing receipts are charged at both declared ceilings and marked unknown. - expect(resumed.out).toBe(controlReport.out) - expect(resumed.spentBreakdown?.childWork.iterations).toBe(15) + // Spend does not pretend the two killed executions were free: their missing receipts remain + // charged at both declared ceilings and their telemetry remains unknown. + expect(resumed.spentBreakdown?.childWork.iterations).toBe(13) expect(resumed.spentBreakdown?.childWork.tokens).toEqual({ - input: 20_050, - output: 50, + input: 20_030, + output: 30, cacheBreakdownKnown: false, }) expect(resumed.spentBreakdown?.childWork.tokensKnown).toBe(false) expect(resumed.spentBreakdown?.childWork.usdKnown).toBe(false) - expect(resumed.spentBreakdown?.childWork.usd).toBeCloseTo(0.05, 10) + expect(resumed.spentBreakdown?.childWork.usd).toBeCloseTo(0.03, 10) - // The ONLY channel that differs is the coordinator's own inference: a restarted coordinator - // re-plans, and those turns are real cost the journal keeps rather than hides. Asserted as a - // strict inequality so the test would fail if that cost were ever silently dropped. + // The resumed coordinator still has a metered inference record. Refusing unknown work may use + // fewer turns than an uninterrupted control, but neither path may hide the coordinator cost. const resumedBrain = resumed.spentBreakdown?.driverInference.tokens.input ?? 0 const controlBrain = controlReport.spentBreakdown?.driverInference.tokens.input ?? 0 expect(controlBrain).toBeGreaterThan(0) - expect(resumedBrain).toBeGreaterThan(controlBrain) + expect(resumedBrain).toBeGreaterThan(0) }) }) diff --git a/tests/runtime/spawn-keys.test.ts b/tests/runtime/spawn-keys.test.ts index 4ce2dd9a4..83fcf2681 100644 --- a/tests/runtime/spawn-keys.test.ts +++ b/tests/runtime/spawn-keys.test.ts @@ -9,7 +9,9 @@ import { describe, expect, it } from 'vitest' import { InMemoryResultBlobStore, InMemorySpawnJournal } from '../../src/durable/spawn-journal' +import { createBudgetPool } from '../../src/runtime/supervise/budget' import { createExecutorRegistry } from '../../src/runtime/supervise/runtime' +import { createScope, deriveNodeExecutionIdentity } from '../../src/runtime/supervise/scope' import { createSupervisor } from '../../src/runtime/supervise/supervisor' import type { Agent, @@ -19,6 +21,7 @@ import type { Scope, UsageEvent, } from '../../src/runtime/supervise/types' +import { zeroSpend } from '../../src/runtime/util' import { testAgentProfile } from '../kernel/test-agent-profile' /** Counts its own executions so "did this key run again?" is a number, not an inference. `gate`, @@ -234,4 +237,50 @@ describe('semantic spawn keys', () => { return 'done' }) }) + + it('an in-doubt key refuses before a lazy replacement factory can run', () => { + const original = countingLeaf('unknown', 'U', { n: 0 }) + const identity = deriveNodeExecutionIdentity(original.executorSpec, 'task') + if (identity === undefined) + throw new Error('test fixture requires a complete execution identity') + + const scope = createScope({ + parentId: 'resumed', + root: 'resumed', + pool: createBudgetPool({ maxIterations: 50, maxTokens: 100_000 }, 0), + journal: new InMemorySpawnJournal(), + blobs: new InMemoryResultBlobStore(), + executors: createExecutorRegistry(), + seams: {}, + depth: 0, + signal: new AbortController().signal, + now: () => 0, + resumeFrom: { + settled: [], + view: { root: 'resumed', nodes: [], inFlight: 0, waiting: 0 }, + maxSpawnOrdinal: 0, + maxCursorSeq: 0, + maxWaitOrdinal: 0, + waits: [], + keys: new Map([ + ['assignment-1', { id: 'resumed:s0', label: 'unknown', identity, state: 'in-doubt' }], + ]), + priorSpend: { childWork: zeroSpend(), driverInference: zeroSpend() }, + }, + }) + let factoryCalls = 0 + + const retry = scope.spawn( + () => { + factoryCalls += 1 + return countingLeaf('replacement', 'R', { n: 0 }) + }, + 'changed task', + { budget: childBudget, label: 'replacement', key: 'assignment-1' }, + ) + + expect(retry).toEqual({ ok: false, reason: 'in-doubt' }) + expect(factoryCalls).toBe(0) + expect(scope.budget.tokensLeft).toBe(100_000) + }) })