fix(supervise): refuse in-doubt replacement dispatch

api-surface.json

@@ -1542,10 +1542,10 @@
       "SpawnOpts": "type 903c3a01c5cb",
       "SpawnPreflight": "type 8e8b479089d3",
       "SpawnPreflightContext": "type 21972a19ec24",
-      "SpawnPrior": "type f5b562b41ed4",
+      "SpawnPrior": "type af72d7e58e96",
       "SpawnRefusal": "type e0fea783ffe7",
       "SpawnRefusalCause": "type a9399e96f43a",
-      "SpawnRejection": "type dfb5b2bcb409",
+      "SpawnRejection": "type 0f740e25e598",
       "Spend": "type 6e558e6744c8",
       "SpendChannel": "type 978ffae337cd",
       "SpendGap": "type d24924855c35",

docs/api/runtime.md

@@ -13659,10 +13659,10 @@ Instruction receipts are evidence and are never delivered automatically to a rep
 worker. The final result spans both processes' work. Unset = in-memory, fresh every call.
 
 The boundary that remains: work that was IN FLIGHT when the process died is not recovered —
-the built-in executors cannot re-attach to a dead process's executions. Each such assignment
-resumes as explicitly lost/in-doubt, its full declared reservation is charged conservatively,
-and its token/dollar telemetry remains unknown. A retry is admitted only from safely remaining
-capacity, so restart cannot mint a fresh budget or slide the original absolute deadline.
+the built-in executors cannot re-attach to a dead process's executions. Each such keyed
+assignment resumes as `in-doubt`, its full declared reservation stays charged, and its
+token/dollar telemetry remains unknown. Runtime refuses a replacement under that key until the
+exact prior execution is recovered, so restart cannot duplicate work or slide the deadline.
 
 `runId` matters here: it defaults to the constant `'supervise'`, which is fine for a single
 resumable run per directory but collides across concurrent runs sharing one `runDir`.
@@ -17971,10 +17971,10 @@ Instruction receipts are evidence and are never delivered automatically to a rep
 worker. The final result spans both processes' work. Unset = in-memory, fresh every call.
 
 The boundary that remains: work that was IN FLIGHT when the process died is not recovered —
-the built-in executors cannot re-attach to a dead process's executions. Each such assignment
-resumes as explicitly lost/in-doubt, its full declared reservation is charged conservatively,
-and its token/dollar telemetry remains unknown. A retry is admitted only from safely remaining
-capacity, so restart cannot mint a fresh budget or slide the original absolute deadline.
+the built-in executors cannot re-attach to a dead process's executions. Each such keyed
+assignment resumes as `in-doubt`, its full declared reservation stays charged, and its
+token/dollar telemetry remains unknown. Runtime refuses a replacement under that key until the
+exact prior execution is recovered, so restart cannot duplicate work or slide the deadline.
 
 `runId` matters here: it defaults to the constant `'supervise'`, which is fine for a single
 resumable run per directory but collides across concurrent runs sharing one `runDir`.
@@ -19772,10 +19772,10 @@ Semantic identity of this assignment ACROSS process lifetimes. A keyed spawn is
 idempotent per key: once a child spawned under a key settles `done` — in this process or in a
 journaled prior one — spawning the same key returns that committed result (`prior.state:
 'completed'`) instead of paying for the work again. A key whose prior attempt settled `down`
-or was journaled as started-but-never-settled spawns FRESH but says so explicitly
-(`prior.state: 'retried' | 'lost'`), and a key that is currently LIVE is refused
-(`'duplicate-key'`) — the same assignment can never run twice concurrently. Unkeyed spawns
-(the default) are position-identified and always run.
+spawns fresh and says so explicitly (`prior.state: 'retried'`). A key whose prior attempt was
+journaled as started but never settled is refused (`'in-doubt'`): the remote execution may
+still exist and must be recovered before replacement. A key that is currently LIVE is refused
+(`'duplicate-key'`). Unkeyed spawns (the default) are position-identified and always run.
 
 ***
 
@@ -19917,9 +19917,9 @@ resumed run that was not waiting.
 
 Keyed assignments from the prior journal: `SpawnOpts.key` → what the journal proves about it.
 `completed`/`down` carry the rehydrated settlement; `in-doubt` means the spawn was journaled
-but no settlement ever landed — the process died with it in flight. `Scope.spawn` consults
-this so a keyed re-spawn resolves instead of duplicating (see `SpawnOpts.key`). Empty when no
-prior spawn carried a key.
+but no settlement ever landed — the process died with it in flight. `Scope.spawn` refuses a
+keyed replacement in that state, rather than duplicate a possibly live remote execution. Empty
+when no prior spawn carried a key.
 
 ##### priorSpend
 
@@ -19968,7 +19968,7 @@ Identity recorded when this key was first admitted. Every reuse must match it ex
 
 ##### state
 
-> `readonly` **state**: `"completed"` \| `"down"` \| `"in-doubt"`
+> `readonly` **state**: `"completed"` \| `"in-doubt"` \| `"down"`
 
 ##### settled?
 
@@ -25139,11 +25139,11 @@ Deterministic node id — `${parent}:s${seq}` from the cursor order, never wall-
 
 ### SpawnRejection
 
-> **SpawnRejection** = `"budget-exhausted"` \| `"usd-unbudgeted"` \| `"depth-exceeded"` \| `"duplicate-key"` \| `"invalid-identity"` \| `"key-conflict"` \| `"max-live-workers"` \| `"scope-aborted"`
+> **SpawnRejection** = `"budget-exhausted"` \| `"usd-unbudgeted"` \| `"depth-exceeded"` \| `"duplicate-key"` \| `"in-doubt"` \| `"invalid-identity"` \| `"key-conflict"` \| `"max-live-workers"` \| `"scope-aborted"`
 
 Fail-closed spawn rejections: an exhausted pool, a dollar request against a root that budgets
- no dollars, an exceeded recursion ceiling, a full tree-wide worker allocation, or a `key` that
- is still LIVE in this scope (the same assignment may not run twice concurrently).
+ no dollars, an exceeded recursion ceiling, a full tree-wide worker allocation, a `key` that is
+ still LIVE in this scope, or a key whose prior remote execution has no terminal receipt.
 
 `usd-unbudgeted` is separate from `budget-exhausted` because the two call for opposite
  responses: an exhausted pool may admit a smaller request, while an unbudgeted dollar channel
@@ -25153,18 +25153,14 @@ refuses every amount until the ROOT budget names a `maxUsd`.
 
 ### SpawnPrior
 
-> **SpawnPrior**\<`Out`\> = \{ `state`: `"completed"`; `settled`: [`Settled`](index.md#settled)\<`Out`\> & `object`; \} \| \{ `state`: `"retried"`; `priorId`: [`NodeId`](#nodeid-6); `reason`: `string`; \} \| \{ `state`: `"lost"`; `priorId`: [`NodeId`](#nodeid-6); \}
+> **SpawnPrior**\<`Out`\> = \{ `state`: `"completed"`; `settled`: [`Settled`](index.md#settled)\<`Out`\> & `object`; \} \| \{ `state`: `"retried"`; `priorId`: [`NodeId`](#nodeid-6); `reason`: `string`; \}
 
 What a KEYED spawn resolved to when the key had a prior attempt. Absent on a fresh key (and on
 every unkeyed spawn). `'completed'` is the exactly-once path: NOTHING was spawned — the handle
-references the prior settled node and `settled` is the committed result. `'retried'` /
-`'lost'` DID spawn fresh: the prior attempt settled `down` (retried) or was journaled as
-started but never settled — the process died with it in flight and the built-in executors
-cannot re-attach to a dead process's work, so the result is explicitly in doubt (lost), never
-silently duplicated. On restart, an in-doubt attempt's full declared reservation is charged and
-its telemetry remains unknown; a fresh retry is admitted only from safely remaining capacity.
-An executor that CAN re-attach to a still-running external execution extends this union with an
-adoption state; none of the built-ins can today.
+references the prior settled node and `settled` is the committed result. `'retried'` DID spawn
+fresh because the prior attempt settled `down`. A start with no terminal receipt is not a prior
+result: `spawn` refuses it as `'in-doubt'`, retains its charged reservation, and requires exact
+recovery before a replacement can run.
 
 #### Type Parameters
 

docs/api/testing.md

@@ -1055,10 +1055,10 @@ Instruction receipts are evidence and are never delivered automatically to a rep
 worker. The final result spans both processes' work. Unset = in-memory, fresh every call.
 
 The boundary that remains: work that was IN FLIGHT when the process died is not recovered —
-the built-in executors cannot re-attach to a dead process's executions. Each such assignment
-resumes as explicitly lost/in-doubt, its full declared reservation is charged conservatively,
-and its token/dollar telemetry remains unknown. A retry is admitted only from safely remaining
-capacity, so restart cannot mint a fresh budget or slide the original absolute deadline.
+the built-in executors cannot re-attach to a dead process's executions. Each such keyed
+assignment resumes as `in-doubt`, its full declared reservation stays charged, and its
+token/dollar telemetry remains unknown. Runtime refuses a replacement under that key until the
+exact prior execution is recovered, so restart cannot duplicate work or slide the deadline.
 
 `runId` matters here: it defaults to the constant `'supervise'`, which is fine for a single
 resumable run per directory but collides across concurrent runs sharing one `runDir`.
@@ -1986,10 +1986,10 @@ Instruction receipts are evidence and are never delivered automatically to a rep
 worker. The final result spans both processes' work. Unset = in-memory, fresh every call.
 
 The boundary that remains: work that was IN FLIGHT when the process died is not recovered —
-the built-in executors cannot re-attach to a dead process's executions. Each such assignment
-resumes as explicitly lost/in-doubt, its full declared reservation is charged conservatively,
-and its token/dollar telemetry remains unknown. A retry is admitted only from safely remaining
-capacity, so restart cannot mint a fresh budget or slide the original absolute deadline.
+the built-in executors cannot re-attach to a dead process's executions. Each such keyed
+assignment resumes as `in-doubt`, its full declared reservation stays charged, and its
+token/dollar telemetry remains unknown. Runtime refuses a replacement under that key until the
+exact prior execution is recovered, so restart cannot duplicate work or slide the deadline.
 
 `runId` matters here: it defaults to the constant `'supervise'`, which is fine for a single
 resumable run per directory but collides across concurrent runs sharing one `runDir`.

src/mcp/tools/coordination.ts

@@ -2749,9 +2749,10 @@ export function createCoordinationTools(opts: CoordinationToolsOptions): Coordin
         'many workers are still in flight — settle or steer one before spawning another. ' +
         'Pass a `key` naming the assignment to make it run-once ACROSS restarts: a key that ' +
         'already completed returns the finished result (`resumed: "completed"` — no work re-runs, ' +
-        'nothing is spent), a key whose prior attempt failed or was lost with a dead process ' +
-        'spawns fresh and says so (`resumed: "retried" | "lost"`), and a key still running is ' +
-        'refused (`error: "duplicate-key"`). ' +
+        'nothing is spent), a key whose prior attempt failed (`down`) spawns fresh and says so ' +
+        '(`resumed: "retried"`), and a key with no terminal receipt is refused ' +
+        '(`error: "in-doubt"`) until its exact prior execution is recovered. A key still running ' +
+        'is refused (`error: "duplicate-key"`). ' +
         'Returns `freeSlots`: how many MORE workers you can start right now (`null` = uncapped). ' +
         'While `freeSlots > 0` there is idle capacity — call this again to fill it rather than ' +
         'waiting; parallel workers finish the run sooner than one at a time.',
@@ -2932,7 +2933,7 @@ export function createCoordinationTools(opts: CoordinationToolsOptions): Coordin
           }
         }
         // A `completed` key returned above, so any prior still attached here is a real re-run:
-        // `retried` (the prior attempt failed) or `lost` (it died in flight with its process).
+        // `retried` means the prior attempt reached a terminal down receipt.
         const priorHistory =
           res.ok && res.prior !== undefined && res.prior.state !== 'completed'
             ? {
@@ -2944,8 +2945,8 @@ export function createCoordinationTools(opts: CoordinationToolsOptions): Coordin
         // Report the REMAINING capacity alongside the spawn, so one tool call tells the driver
         // both "it started" and "you can still open N more" — the feedback that lets it fill
         // slots instead of opening one worker per turn. `null` = uncapped. A fresh spawn under a
-        // key with a failed/lost prior attempt carries that history (`resumed`/`priorWorkerId`),
-        // so a re-run is always explicit, never a silent duplicate.
+        // key with a failed prior attempt carries that history (`resumed`/`priorWorkerId`), so a
+        // re-run is always explicit. An in-doubt prior is refused above and never duplicated.
         return Promise.resolve(
           res.ok
             ? {
@@ -2974,7 +2975,9 @@ export function createCoordinationTools(opts: CoordinationToolsOptions): Coordin
                 reason:
                   res.reason === 'usd-unbudgeted'
                     ? "this run's root budget declares no maxUsd, so a child budget naming maxUsd can never be admitted at any amount — spawn with a budget that omits maxUsd"
-                    : `the conserved pool refused this spawn (${String(res.reason)}); the run has no allocation left to give this worker`,
+                    : res.reason === 'in-doubt'
+                      ? 'this key has a prior worker recorded as started without a terminal receipt; no replacement was started because that remote worker may still be running — inspect or recover the exact prior execution before retrying'
+                      : `the conserved pool refused this spawn (${String(res.reason)}); the run has no allocation left to give this worker`,
                 ...(res.reason === 'usd-unbudgeted'
                   ? {
                       hint:
@@ -2983,7 +2986,11 @@ export function createCoordinationTools(opts: CoordinationToolsOptions): Coordin
                         'fail identically. Spawn with a budget that omits maxUsd, or ask the caller ' +
                         'to give the run a root maxUsd.',
                     }
-                  : {}),
+                  : res.reason === 'in-doubt'
+                    ? {
+                        hint: 'Do not retry this key. Use the recorded prior worker identity to inspect or recover that exact execution. A terminal receipt or explicit recovery is required before replacement work can start.',
+                      }
+                    : {}),
                 live: liveWorkerCount(),
                 freeSlots: freeWorkerSlots(),
               },

src/runtime/supervise/coordination-driver.ts

@@ -1243,7 +1243,7 @@ export function driverAgent(opts: DriverAgentOptions): Agent<unknown, unknown> {
 
 /**
  * The factual context a resumed driver starts from — everything the durable stores prove about
- * the prior process(es): committed settlements, per-key states (completed / lost / failed),
+ * the prior process(es): committed settlements, per-key states (completed / in-doubt / failed),
  * re-armed waits, carried-over questions/findings/continuation receipts, and spend already paid.
  * Injected as the brain's first user-context on a resumed run so it continues from unresolved work;
  * old continuation receipts are evidence and are never auto-delivered.
@@ -1269,7 +1269,7 @@ function resumeBrief(resume: ResumedWork<unknown>, prior?: PriorCoordination): s
   const byState = (state: 'completed' | 'in-doubt' | 'down') =>
     [...resume.keys].filter(([, v]) => v.state === state)
   const completed = byState('completed')
-  const lost = byState('in-doubt')
+  const inDoubt = byState('in-doubt')
   const failed = byState('down')
   if (completed.length > 0) {
     lines.push(
@@ -1278,11 +1278,11 @@ function resumeBrief(resume: ResumedWork<unknown>, prior?: PriorCoordination): s
       ...completed.map(([k, v]) => `- ${k} → ${v.id} (${v.label})`),
     )
   }
-  if (lost.length > 0) {
+  if (inDoubt.length > 0) {
     lines.push(
       '',
-      'Keys LOST in flight with the prior process — this is the unresolved work; spawn_worker with the same key starts a fresh attempt:',
-      ...lost.map(([k, v]) => `- ${k} (prior attempt ${v.id}, ${v.label})`),
+      'Keys IN DOUBT — a prior process recorded them as started but never recorded a terminal receipt. Do NOT spawn a replacement under these keys; inspect or recover each exact prior execution first:',
+      ...inDoubt.map(([k, v]) => `- ${k} (prior attempt ${v.id}, ${v.label})`),
     )
   }
   if (failed.length > 0) {

src/runtime/supervise/scope.ts

@@ -476,8 +476,8 @@ export function createScope<Out>(args: ScopeArgs): Scope<Out> {
   // The semantic-key registry (`SpawnOpts.key`): every keyed assignment's current state, seeded
   // from the prior journal on resume and updated live as keyed children spawn and settle. This is
   // what makes a keyed spawn idempotent per key across process lifetimes: `done` returns the
-  // committed result, `live` refuses a concurrent duplicate, `down`/`in-doubt` spawn fresh but
-  // say so explicitly.
+  // committed result, `live` refuses a concurrent duplicate, `down` retries after a terminal
+  // receipt, and `in-doubt` refuses until the exact prior execution is recovered.
   type KeyState =
     | { readonly state: 'live'; readonly id: NodeId; readonly identity: NodeExecutionIdentity }
     | {
@@ -540,10 +540,10 @@ export function createScope<Out>(args: ScopeArgs): Scope<Out> {
     const task = detachedSnapshot(rawTask, 'scope.spawn task')
     const opts = detachedSnapshot(rawOpts, 'scope.spawn options')
 
-    // A key is an identity claim, not merely a cache label. On every reuse, prepare the requested
-    // agent far enough to derive the authorized profile/task identity, then compare it with the
-    // journal before returning an old result or retrying old work. No executor is resolved,
-    // constructed, reserved, or run on the completed path.
+    // A key is an identity claim, not merely a cache label. An in-doubt start has no terminal
+    // receipt, so it refuses before touching a lazy factory. Every other reuse prepares the
+    // requested agent far enough to compare its authorized profile/task identity with the journal.
+    // No executor is resolved, constructed, reserved, or run on the completed path.
     let prior: SpawnPrior<C> | undefined
     let prepared:
       | {
@@ -565,6 +565,11 @@ export function createScope<Out>(args: ScopeArgs): Scope<Out> {
     }
     if (opts.key !== undefined) {
       const existing = keyed.get(opts.key)
+      if (existing?.state === 'in-doubt') {
+        // A durable start with no terminal receipt does not prove the remote execution stopped.
+        // Do not invoke a lazy factory, reserve, or construct a replacement beside it.
+        return { ok: false, reason: 'in-doubt' }
+      }
       if (existing !== undefined) {
         prepared = prepare()
         if (!isCompleteIdentity(prepared.identity)) {
@@ -584,8 +589,6 @@ export function createScope<Out>(args: ScopeArgs): Scope<Out> {
       }
       if (existing?.state === 'down') {
         prior = { state: 'retried', priorId: existing.id, reason: existing.reason }
-      } else if (existing?.state === 'in-doubt') {
-        prior = { state: 'lost', priorId: existing.id }
       }
     }