diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ea21f06..b90f8e17 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,20 @@ # Changelog +## 0.193.1 + +### Bridge roots refuse unavailable work before allocation + +Bridge roots now check model routing and their bulk admission lane before Runtime allocates a remote run identity. +Temporary route failures remain retryable, while an unrouted model remains terminal. +Pre-dispatch refusals record trusted zero spend and never send a cancellation for work that did not start. +Credential reads and interrupted preflights also complete before Runtime allocates the remote identity. +Caller cancellation remains an abort through route and admission reads. + +### Provisioned supervisors no longer expire by default + +`ProvisionSupervisorRequest.timeoutMs` now creates a lifecycle deadline only when the caller supplies it. +Omitting it lets the worker continue until cleanup or cancellation. + ## 0.193.0 ### Omitted code-mode deadlines no longer stop work diff --git a/docs/api/primitive-catalog.md b/docs/api/primitive-catalog.md index 41f4f548..5d419801 100644 --- a/docs/api/primitive-catalog.md +++ b/docs/api/primitive-catalog.md @@ -7,7 +7,7 @@ # Primitive catalog — the never-stale anti-reinvention inventory -> **GENERATED** from `@tangle-network/agent-runtime@0.193.0` and `@tangle-network/agent-eval@0.173.0` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. +> **GENERATED** from `@tangle-network/agent-runtime@0.193.1` and `@tangle-network/agent-eval@0.173.0` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. ## 1. agent-runtime — own public surface diff --git a/docs/api/runtime.md b/docs/api/runtime.md index ff82ca51..e6067ba8 100644 --- a/docs/api/runtime.md +++ b/docs/api/runtime.md @@ -15299,7 +15299,7 @@ Root directory for Runtime-owned `.agent/supervisor` state. > `readonly` `optional` **timeoutMs?**: `number` -Maximum wall-clock time for the complete supervisor lifecycle, including cleanup. +Maximum wall-clock time for the complete supervisor lifecycle, including cleanup. Omit for no lifecycle deadline. ##### pollMs? @@ -30437,9 +30437,9 @@ Pre-journal profile resolution for `preflightSpawn`; see (`tools`) => `void` -Called with this server's coordination tool descriptors once they exist and BEFORE the - listener opens — the seam a caller uses to give an already-bound node tool a way to call the - same verbs in code (`SupervisorToolInvocationContext.verbs`). +Called with this server's exact MCP tool descriptors once they exist and BEFORE the listener + opens — the seam a caller uses to give an already-bound node tool a way to call the same + verbs in code (`SupervisorToolInvocationContext.verbs`). #### Returns diff --git a/docs/canonical-api.md b/docs/canonical-api.md index dc802ea8..775c64b4 100644 --- a/docs/canonical-api.md +++ b/docs/canonical-api.md @@ -4,7 +4,7 @@ Generated signatures and the complete export list live in docs/api/. Run pnpm docs:freshness after editing this file. --> -> **Version 0.193.0.** +> **Version 0.193.1.** > [`docs/api/primitive-catalog.md`](./api/primitive-catalog.md) lists every export and import path. > `agent-eval` must satisfy `>=0.173.0 <0.174.0`. > `sandbox` must satisfy `>=0.36.4 <0.38.0`. diff --git a/package.json b/package.json index 603b2d9a..5c5c0ac2 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-runtime", - "version": "0.193.0", + "version": "0.193.1", "description": "Shared task-lifecycle skeleton for agents: a recursive loop kernel for chat turns, one-shot tasks, and multi-attempt loops, with trace capture and eval-gated self-improvement. Domain behavior lives in adapters; scoring and ship-gates in @tangle-network/agent-eval.", "homepage": "https://github.com/tangle-network/agent-runtime#readme", "repository": { diff --git a/src/improvement/improve.test.ts b/src/improvement/improve.test.ts index 1cbaa591..97fd3efa 100644 --- a/src/improvement/improve.test.ts +++ b/src/improvement/improve.test.ts @@ -1025,7 +1025,7 @@ describe('improve code execution', () => { } finally { repo.cleanup() } - }) + }, 120_000) it('retains and disposes the incumbent for a baseline-only code run', async () => { const repo = createRepo('improve-code-baseline-') @@ -1089,7 +1089,7 @@ describe('improve code execution', () => { } finally { repo.cleanup() } - }) + }, 120_000) it('cleans the incumbent when baseline finalization fails', async () => { const repo = createRepo('improve-code-finalize-') diff --git a/src/runtime/supervise/bridge-executor.test.ts b/src/runtime/supervise/bridge-executor.test.ts index c243fef4..89d58659 100644 --- a/src/runtime/supervise/bridge-executor.test.ts +++ b/src/runtime/supervise/bridge-executor.test.ts @@ -16,7 +16,7 @@ import { materializeTreeView, replaySpawnTree, } from '../../durable/spawn-journal' -import { BackendTransportError } from '../../errors' +import { BackendTransportError, ValidationError } from '../../errors' import { spendFromUsageEvents } from './budget' import { classifyDriverFailure } from './driver-retry' import { @@ -26,6 +26,7 @@ import { import { type BridgeModelCredential, bridgeExecutor, + bridgeStopSignalKey, captureReusableExecutorConfig, createExecutor, createExecutorRegistry, @@ -37,17 +38,30 @@ const TEST_RUN_DIGEST = `sha256:${'b'.repeat(64)}` const TEST_WORKSPACE_DIGEST = `sha256:${'a'.repeat(64)}` function respondBridgeCapabilities(req: IncomingMessage, res: ServerResponse): boolean { - if (req.method !== 'GET' || req.url !== '/') return false - res.writeHead(200, { 'content-type': 'application/json' }) - res.end( - JSON.stringify({ - capabilities: { - profileMaterialization: 'cli-bridge.profile-materialization.v2', - usageCostProvenance: 'cli-bridge.usage-cost.v1', - }, - }), - ) - return true + if (req.method !== 'GET') return false + if (req.url === '/') { + res.writeHead(200, { 'content-type': 'application/json' }) + res.end( + JSON.stringify({ + capabilities: { + profileMaterialization: 'cli-bridge.profile-materialization.v2', + usageCostProvenance: 'cli-bridge.usage-cost.v1', + }, + }), + ) + return true + } + if (req.url?.startsWith('/v1/capabilities?model=')) { + res.writeHead(200, { 'content-type': 'application/json' }) + res.end(JSON.stringify({ available: true })) + return true + } + if (req.url === '/health') { + res.writeHead(200, { 'content-type': 'application/json' }) + res.end(JSON.stringify({ admission: { active: 0, maxActive: 4 } })) + return true + } + return false } /** @@ -427,6 +441,425 @@ describe('bridgeExecutor upstream-error propagation', () => { expect(posts).toBe(0) }) + it('refuses an unroutable manager profile before any model POST', async () => { + const requests: Array<{ method: string | undefined; url: string | undefined }> = [] + server = createServer((req, res) => { + requests.push({ method: req.method, url: req.url }) + if (req.method === 'GET' && req.url === '/') { + respondBridgeCapabilities(req, res) + return + } + if (req.method === 'GET' && req.url?.startsWith('/v1/capabilities?model=')) { + res.writeHead(404, { 'content-type': 'application/json' }) + res.end(JSON.stringify({ error: 'no backend matches model' })) + return + } + res.writeHead(500) + res.end() + }) + await new Promise((resolve) => server?.listen(0, '127.0.0.1', resolve)) + const { port } = server.address() as AddressInfo + const executor = makeExecutor(`http://127.0.0.1:${port}`) + + const failure = await drain( + executor.execute( + 'must not dispatch', + new AbortController().signal, + ) as AsyncIterable, + ).then( + () => undefined, + (error: unknown) => error, + ) + expect(failure).toMatchObject({ + message: expect.stringMatching(/routes no backend for model "pi\/tangle-router\/glm-5\.2"/u), + }) + expect(failure).toBeInstanceOf(ValidationError) + expect(classifyDriverFailure(failure)).toBe('terminal') + await expect(executor.teardown('brutalKill')).resolves.toEqual({ destroyed: true }) + expect(requests).toEqual([ + { method: 'GET', url: '/' }, + { + method: 'GET', + url: '/v1/capabilities?model=pi%2Ftangle-router%2Fglm-5.2', + }, + ]) + expect(runtimeOwnedExecutorProviderEvidence(executor)).toEqual({ + status: 'unknown', + attempts: [{ observations: [], providerDispatch: 'not_started' }], + models: [], + reason: 'provider-model-missing', + }) + }) + + it('classifies a temporarily unavailable manager route as transient before any model POST', async () => { + const requests: Array<{ method: string | undefined; url: string | undefined }> = [] + server = createServer((req, res) => { + requests.push({ method: req.method, url: req.url }) + if (req.method === 'GET' && req.url === '/') { + respondBridgeCapabilities(req, res) + return + } + if (req.method === 'GET' && req.url?.startsWith('/v1/capabilities?model=')) { + res.writeHead(503, { 'content-type': 'application/json' }) + res.end(JSON.stringify({ error: 'backend is not ready' })) + return + } + res.writeHead(500) + res.end() + }) + await new Promise((resolve) => server?.listen(0, '127.0.0.1', resolve)) + const { port } = server.address() as AddressInfo + const executor = makeExecutor(`http://127.0.0.1:${port}`) + + const failure = await drain( + executor.execute( + 'must not dispatch', + new AbortController().signal, + ) as AsyncIterable, + ).then( + () => undefined, + (error: unknown) => error, + ) + expect(failure).toMatchObject({ + message: expect.stringMatching(/answered 503 for model "pi\/tangle-router\/glm-5\.2"/u), + }) + expect(failure).toBeInstanceOf(BackendTransportError) + expect((failure as BackendTransportError).status).toBe(503) + expect(classifyDriverFailure(failure)).toBe('transient') + await expect(executor.teardown('brutalKill')).resolves.toEqual({ destroyed: true }) + expect(requests).toEqual([ + { method: 'GET', url: '/' }, + { + method: 'GET', + url: '/v1/capabilities?model=pi%2Ftangle-router%2Fglm-5.2', + }, + ]) + expect(runtimeOwnedExecutorProviderEvidence(executor)).toEqual({ + status: 'unknown', + attempts: [{ observations: [], providerDispatch: 'not_started' }], + models: [], + reason: 'provider-model-missing', + }) + }) + + it('resumes a forceful steer that interrupts preflight without cancelling an uncreated run', async () => { + let routeReads = 0 + let markFirstRouteRead!: () => void + const firstRouteRead = new Promise((resolve) => { + markFirstRouteRead = resolve + }) + const requestBodies: Array> = [] + const cancelledRuns: string[] = [] + server = createServer(async (req, res) => { + if (req.method === 'GET' && req.url === '/') { + respondBridgeCapabilities(req, res) + return + } + if (req.method === 'GET' && req.url?.startsWith('/v1/capabilities?model=')) { + routeReads += 1 + if (routeReads === 1) { + markFirstRouteRead() + return + } + res.writeHead(200, { 'content-type': 'application/json' }) + res.end(JSON.stringify({ available: true })) + return + } + if (req.method === 'GET' && req.url === '/health') { + res.writeHead(200, { 'content-type': 'application/json' }) + res.end(JSON.stringify({ admission: { active: 0, maxActive: 4 } })) + return + } + const cancelledId = cancelledRunId(req.url) + if (cancelledId !== undefined) { + cancelledRuns.push(cancelledId) + res.writeHead(200, { 'content-type': 'application/json' }) + res.end(terminalCancelBody(cancelledId)) + return + } + const chunks: Buffer[] = [] + for await (const chunk of req) chunks.push(Buffer.from(chunk)) + const body = JSON.parse(Buffer.concat(chunks).toString('utf8')) as Record + requestBodies.push(body) + const runId = String(body.run_id) + res.writeHead(200, { + 'content-type': 'text/event-stream', + ...durableRunHeaders(runId), + }) + res.end( + numberSseDataFrames( + bridgeProtocolSse( + `data: ${JSON.stringify({ choices: [{ delta: { content: 'resumed answer' } }], usage: { prompt_tokens: 3, completion_tokens: 1, cost: 0.01 } })}\n\ndata: [DONE]\n\n`, + body, + ), + ), + ) + }) + await new Promise((resolve) => server?.listen(0, '127.0.0.1', resolve)) + const { port } = server.address() as AddressInfo + const executor = makeExecutor(`http://127.0.0.1:${port}`) + const draining = drain( + executor.execute( + 'discard this plan', + new AbortController().signal, + ) as AsyncIterable, + ) + + await firstRouteRead + expect(executor.deliver?.({ steer: 'use the corrected plan', interrupt: true })).toBe(true) + await draining + + expect(routeReads).toBe(2) + expect(requestBodies).toHaveLength(1) + expect(requestBodies[0]?.messages).toEqual([ + { role: 'user', content: expect.stringContaining('use the corrected plan') }, + ]) + expect(cancelledRuns).toEqual([]) + expect(executor.resultArtifact().out).toMatchObject({ content: 'resumed answer' }) + }) + + it('settles a stopped hanging preflight without recording an attempt or sending a POST', async () => { + const stop = new AbortController() + const executeAbort = new AbortController() + const requests: string[] = [] + let markCapabilityRead!: () => void + const capabilityRead = new Promise((resolve) => { + markCapabilityRead = resolve + }) + server = createServer((req, res) => { + requests.push(`${req.method} ${req.url}`) + if (req.method === 'GET' && req.url === '/') { + markCapabilityRead() + req.once('aborted', () => res.destroy()) + return + } + res.writeHead(500) + res.end('unexpected request') + }) + await new Promise((resolve) => server?.listen(0, '127.0.0.1', resolve)) + const { port } = server.address() as AddressInfo + const executor = bridgeExecutor( + { + profile: { + name: 'stopped-preflight', + harness: 'pi', + model: { provider: 'tangle-router', default: 'glm-5.2' }, + }, + harness: null, + }, + { + signal: new AbortController().signal, + seams: { + bridge: { + bridgeUrl: `http://127.0.0.1:${port}`, + bridgeBearer: 'test-bearer', + }, + [bridgeStopSignalKey]: stop.signal, + }, + }, + ) + const draining = drain( + executor.execute('stop before dispatch', executeAbort.signal) as AsyncIterable, + ) + + await capabilityRead + stop.abort('completion requested') + const outcome = await Promise.race([ + draining.then( + () => 'settled' as const, + () => 'rejected' as const, + ), + new Promise<'timed-out'>((resolve) => setTimeout(() => resolve('timed-out'), 250)), + ]) + if (outcome !== 'settled') { + executeAbort.abort('test cleanup') + await draining.catch(() => undefined) + } + + expect(outcome).toBe('settled') + expect(requests).toEqual(['GET /']) + expect(runtimeOwnedExecutorProviderEvidence(executor)).toBeUndefined() + await expect(executor.teardown('brutalKill')).resolves.toEqual({ destroyed: true }) + }) + + it('preserves a caller abort during route preflight without sending a model POST', async () => { + const executeAbort = new AbortController() + const requests: string[] = [] + let markRouteRead!: () => void + const routeRead = new Promise((resolve) => { + markRouteRead = resolve + }) + server = createServer((req, res) => { + requests.push(`${req.method} ${req.url}`) + if (req.method === 'GET' && req.url === '/') { + res.writeHead(200, { 'content-type': 'application/json' }) + res.end( + JSON.stringify({ + capabilities: { + profileMaterialization: 'cli-bridge.profile-materialization.v2', + usageCostProvenance: 'cli-bridge.usage-cost.v1', + }, + }), + ) + return + } + if (req.method === 'GET' && req.url?.startsWith('/v1/capabilities?model=')) { + markRouteRead() + req.once('aborted', () => res.destroy()) + return + } + res.writeHead(500) + res.end('unexpected request') + }) + await new Promise((resolve) => server?.listen(0, '127.0.0.1', resolve)) + const { port } = server.address() as AddressInfo + const executor = makeExecutor(`http://127.0.0.1:${port}`) + const draining = drain( + executor.execute( + 'stop during route preflight', + executeAbort.signal, + ) as AsyncIterable, + ).then( + () => undefined, + (error: unknown) => error, + ) + + await routeRead + executeAbort.abort('caller stopped the route preflight') + + await expect(draining).resolves.toMatchObject({ name: 'AbortError' }) + expect(requests).toEqual(['GET /', 'GET /v1/capabilities?model=pi%2Ftangle-router%2Fglm-5.2']) + await expect(executor.teardown('brutalKill')).resolves.toEqual({ destroyed: true }) + }) + + it('settles a stopped credential preflight without allocating a remote run', async () => { + const stop = new AbortController() + const requests: string[] = [] + let releaseCredential!: () => void + const credentialBlocked = new Promise((resolve) => { + releaseCredential = resolve + }) + let markCredentialRead!: () => void + const credentialRead = new Promise((resolve) => { + markCredentialRead = resolve + }) + server = createServer((req, res) => { + requests.push(`${req.method} ${req.url}`) + if (respondBridgeCapabilities(req, res)) return + res.writeHead(500) + res.end('unexpected request') + }) + await new Promise((resolve) => server?.listen(0, '127.0.0.1', resolve)) + const { port } = server.address() as AddressInfo + const executor = bridgeExecutor( + { + profile: { + name: 'stopped-credential-preflight', + harness: 'pi', + model: { provider: 'tangle-router', default: 'glm-5.2' }, + }, + harness: null, + }, + { + signal: new AbortController().signal, + seams: { + bridge: { + bridgeUrl: `http://127.0.0.1:${port}`, + bridgeBearer: 'test-bearer', + modelCredential: { + key: 'MODEL_GATEWAY_TOKEN', + baseUrlKey: 'MODEL_GATEWAY_BASE_URL', + provider: { + get: async (key: string) => { + markCredentialRead() + await credentialBlocked + return key === 'MODEL_GATEWAY_TOKEN' + ? 'model-token' + : 'https://router.tangle.tools/v1' + }, + }, + }, + }, + [bridgeStopSignalKey]: stop.signal, + }, + }, + ) + const draining = drain( + executor.execute( + 'stop before credential dispatch', + new AbortController().signal, + ) as AsyncIterable, + ) + + await credentialRead + stop.abort('completion requested') + const outcome = await Promise.race([ + draining.then( + () => 'settled' as const, + () => 'rejected' as const, + ), + new Promise<'timed-out'>((resolve) => setTimeout(() => resolve('timed-out'), 250)), + ]) + releaseCredential() + await draining.catch(() => undefined) + + expect(outcome).toBe('settled') + expect(requests).toEqual([ + 'GET /', + 'GET /v1/capabilities?model=pi%2Ftangle-router%2Fglm-5.2', + 'GET /health', + ]) + expect(runtimeOwnedExecutorProviderEvidence(executor)).toBeUndefined() + await expect(executor.teardown('brutalKill')).resolves.toEqual({ destroyed: true }) + }) + + it('refuses a full bridge before any model POST', async () => { + const requests: Array<{ method: string | undefined; url: string | undefined }> = [] + server = createServer((req, res) => { + requests.push({ method: req.method, url: req.url }) + if (req.method === 'GET' && req.url === '/health') { + res.writeHead(200, { 'content-type': 'application/json' }) + res.end(JSON.stringify({ admission: { active: 2, maxActive: 2 } })) + return + } + if (respondBridgeCapabilities(req, res)) return + res.writeHead(500) + res.end() + }) + await new Promise((resolve) => server?.listen(0, '127.0.0.1', resolve)) + const { port } = server.address() as AddressInfo + const executor = makeExecutor(`http://127.0.0.1:${port}`) + + const failure = await drain( + executor.execute( + 'must not dispatch', + new AbortController().signal, + ) as AsyncIterable, + ).then( + () => undefined, + (error: unknown) => error, + ) + expect(failure).toMatchObject({ + message: expect.stringMatching(/admission is full: active 2 of maxActive 2/u), + }) + expect(classifyDriverFailure(failure)).toBe('transient') + await expect(executor.teardown('brutalKill')).resolves.toEqual({ destroyed: true }) + expect(requests).toEqual([ + { method: 'GET', url: '/' }, + { + method: 'GET', + url: '/v1/capabilities?model=pi%2Ftangle-router%2Fglm-5.2', + }, + { method: 'GET', url: '/health' }, + ]) + expect(runtimeOwnedExecutorProviderEvidence(executor)).toEqual({ + status: 'unknown', + attempts: [{ observations: [], providerDispatch: 'not_started' }], + models: [], + reason: 'provider-model-missing', + }) + }) + it('rejects a v2 bridge that completes without its terminal profile acknowledgement', async () => { const stub = await startBridgeStub( `data: ${JSON.stringify({ choices: [{ delta: { content: 'untrusted' } }] })}\n\ndata: [DONE]\n\n`, @@ -892,6 +1325,7 @@ describe('bridgeExecutor upstream-error propagation', () => { executor.execute('must refuse', new AbortController().signal) as AsyncIterable, ), ).rejects.toThrow(/no usable value for 'MODEL_GATEWAY_TOKEN'/u) + await expect(executor.teardown('brutalKill')).resolves.toEqual({ destroyed: true }) expect(posts).toBe(0) }) @@ -1451,6 +1885,101 @@ describe('bridgeExecutor upstream-error propagation', () => { expect(liveRuns.size).toBe(0) }) + it('cancels a live bridge run when an interrupted reconnect credential lookup hangs', async () => { + const requestBodies: Array> = [] + const cancelledRuns: string[] = [] + let markReconnectCredentialRead!: () => void + const reconnectCredentialRead = new Promise((resolve) => { + markReconnectCredentialRead = resolve + }) + let releaseReconnectCredential!: () => void + const reconnectCredentialBlocked = new Promise((resolve) => { + releaseReconnectCredential = resolve + }) + let markCancelSeen!: () => void + const cancelSeen = new Promise((resolve) => { + markCancelSeen = resolve + }) + server = createServer(async (req, res) => { + if (respondBridgeCapabilities(req, res)) return + const cancelledId = cancelledRunId(req.url) + if (cancelledId !== undefined) { + cancelledRuns.push(cancelledId) + markCancelSeen() + res.writeHead(200, { + 'content-type': 'application/json', + ...durableRunHeaders(cancelledId), + }) + res.end(terminalCancelBody(cancelledId)) + return + } + const chunks: Buffer[] = [] + for await (const chunk of req) chunks.push(Buffer.from(chunk)) + const body = JSON.parse(Buffer.concat(chunks).toString('utf8')) as Record + requestBodies.push(body) + const runId = String(body.run_id) + res.writeHead(200, { + 'content-type': 'text/event-stream', + ...durableRunHeaders(runId), + }) + if (requestBodies.length === 1) { + res.end( + `id: 1\n${bridgeProtocolSse(`data: ${JSON.stringify({ usage: { prompt_tokens: 2, completion_tokens: 1 } })}\n\n`, body)}`, + ) + return + } + res.end( + numberSseDataFrames( + bridgeProtocolSse( + `data: ${JSON.stringify({ choices: [{ delta: { content: 'resumed after cancel' } }], usage: { prompt_tokens: 3, completion_tokens: 1 } })}\n\ndata: [DONE]\n\n`, + body, + ), + ), + ) + }) + await new Promise((resolve) => server?.listen(0, '127.0.0.1', resolve)) + const { port } = server.address() as AddressInfo + let tokenReads = 0 + const executor = makeExecutor(`http://127.0.0.1:${port}`, { + key: 'MODEL_GATEWAY_TOKEN', + baseUrlKey: 'MODEL_GATEWAY_BASE_URL', + provider: { + get: async (key) => { + if (key === 'MODEL_GATEWAY_TOKEN') { + tokenReads += 1 + if (tokenReads === 2) { + markReconnectCredentialRead() + await reconnectCredentialBlocked + } + return 'reconnect-secret' + } + return 'https://router.tangle.tools/v1' + }, + }, + }) + const draining = drain( + executor.execute('do the task', new AbortController().signal) as AsyncIterable, + ) + + await reconnectCredentialRead + expect( + executor.deliver?.({ steer: 'resume after cancelling the lost reader', interrupt: true }), + ).toBe(true) + const cancellation = await Promise.race([ + cancelSeen.then(() => 'seen' as const), + new Promise<'timed-out'>((resolve) => setTimeout(() => resolve('timed-out'), 250)), + ]) + releaseReconnectCredential() + + expect(cancellation).toBe('seen') + await draining + expect(cancelledRuns).toEqual([requestBodies[0]?.run_id]) + expect(requestBodies).toHaveLength(2) + expect(requestBodies[1]?.messages).toEqual([ + { role: 'user', content: expect.stringContaining('resume after cancelling the lost reader') }, + ]) + }) + it('interrupts an active response body, accounts its partial usage, and resumes with the steer', async () => { const requestBodies: Array> = [] const liveRuns = new Set() diff --git a/src/runtime/supervise/coordination-mcp.ts b/src/runtime/supervise/coordination-mcp.ts index 0467e5ef..ff12923d 100644 --- a/src/runtime/supervise/coordination-mcp.ts +++ b/src/runtime/supervise/coordination-mcp.ts @@ -169,9 +169,9 @@ export async function serveCoordinationMcp(opts: { /** Pre-journal profile resolution for `preflightSpawn`; see * `CoordinationToolsOptions.resolveSpawnProfile`. */ resolveSpawnProfile?: (profile: AgentProfile) => AgentProfile - /** Called with this server's coordination tool descriptors once they exist and BEFORE the - * listener opens — the seam a caller uses to give an already-bound node tool a way to call the - * same verbs in code (`SupervisorToolInvocationContext.verbs`). */ + /** Called with this server's exact MCP tool descriptors once they exist and BEFORE the listener + * opens — the seam a caller uses to give an already-bound node tool a way to call the same + * verbs in code (`SupervisorToolInvocationContext.verbs`). */ onCoordinationTools?: (tools: ReadonlyArray) => void }): Promise { const host = opts.host ?? '127.0.0.1' @@ -228,13 +228,15 @@ export async function serveCoordinationMcp(opts: { : {}), }) await coord.ready() - // Before the listener opens: a node tool invoked on the first request must already be able to - // call these verbs. - opts.onCoordinationTools?.(coord.tools) + const servedTools = [...coord.tools, ...(opts.nodeTools ?? [])] const mcp = createMcpServer({ - extraTools: [...coord.tools, ...(opts.nodeTools ?? [])], + extraTools: servedTools, serverName: 'coordination', }) + // Before the listener opens: a node tool invoked on the first request must already be able to + // call these verbs. Read back the server's ordered set so every consumer records the exact MCP + // surface, including the shared tools that `createMcpServer` mounts before coordination tools. + opts.onCoordinationTools?.([...mcp.tools.values()]) const server: Server = createServer((req, res) => { if (req.method !== 'POST') { diff --git a/src/runtime/supervise/interactive-worker.test.ts b/src/runtime/supervise/interactive-worker.test.ts index 34f7dda2..b28b5488 100644 --- a/src/runtime/supervise/interactive-worker.test.ts +++ b/src/runtime/supervise/interactive-worker.test.ts @@ -294,7 +294,7 @@ describe('workerFromInteractiveProvider', () => { }) describe('provisionSupervisor', () => { - it('provisions a real root and worker, exposes controls, attaches, and cleans up exactly once', async () => { + it('provisions a real root and worker without a lifecycle deadline', async () => { const fixture = interactiveProviderFixture() const root = mkdtempSync(join(tmpdir(), 'agent-runtime-provision-')) try { @@ -306,7 +306,6 @@ describe('provisionSupervisor', () => { metadata: { purpose: 'provision-test' }, }, workspaceDir: root, - timeoutMs: 5_000, pollMs: 2, profile, connection: { provider: fixture.provider }, @@ -471,7 +470,7 @@ describe('provisionSupervisor', () => { } }) - it('uses timeoutMs for the live lifecycle after worker admission', async () => { + it('uses caller-supplied timeoutMs for the live lifecycle after worker admission', async () => { const fixture = interactiveProviderFixture() const root = mkdtempSync(join(tmpdir(), 'agent-runtime-provision-deadline-')) try { diff --git a/src/runtime/supervise/provision-supervisor.ts b/src/runtime/supervise/provision-supervisor.ts index 92d3d27e..02d63807 100644 --- a/src/runtime/supervise/provision-supervisor.ts +++ b/src/runtime/supervise/provision-supervisor.ts @@ -39,7 +39,6 @@ import { createRootHandle, createSupervisor } from './supervisor' import type { Agent, Budget, Scope, SpawnEvent, SupervisedResult } from './types' import { readWorkerInteractiveBinding } from './worker-interactive' -const DEFAULT_TIMEOUT_MS = 60_000 const DEFAULT_POLL_MS = 25 const ROOT_MAX_ITERATIONS = 100 const ROOT_MAX_TOKENS = 100_000 @@ -73,7 +72,7 @@ export interface ProvisionSupervisorRequest { readonly workerEnvironment?: InteractiveWorkerEnvironment /** Root directory for Runtime-owned `.agent/supervisor` state. */ readonly workspaceDir?: string - /** Maximum wall-clock time for the complete supervisor lifecycle, including cleanup. */ + /** Maximum wall-clock time for the complete supervisor lifecycle, including cleanup. Omit for no lifecycle deadline. */ readonly timeoutMs?: number /** Poll cadence for lifecycle/control readiness. */ readonly pollMs?: number @@ -399,7 +398,7 @@ export async function provisionSupervisor( function normalizeRequest(request: ProvisionSupervisorRequest): ProvisionSupervisorRequest & { readonly invocationId: string readonly task: string - readonly timeoutMs: number + readonly timeoutMs: number | undefined readonly pollMs: number readonly profile: AgentProfile readonly connection: ProvisionSupervisorConnection @@ -412,7 +411,8 @@ function normalizeRequest(request: ProvisionSupervisorRequest): ProvisionSupervi if (request.connection === undefined) { throw new SupervisorProvisionUnavailableError('provider connection is required') } - const timeoutMs = positiveNumber(request.timeoutMs ?? DEFAULT_TIMEOUT_MS, 'timeoutMs') + const timeoutMs = + request.timeoutMs === undefined ? undefined : positiveNumber(request.timeoutMs, 'timeoutMs') const pollMs = positiveNumber(request.pollMs ?? DEFAULT_POLL_MS, 'pollMs') return { ...request, invocationId, task, timeoutMs, pollMs, profile } } @@ -433,12 +433,12 @@ function supervisorIdFor(invocationId: string): string { return `runtime-supervisor-${digest.slice('sha256:'.length)}` } -/** The root deadline covers the complete supervisor lifecycle from run start through cleanup. */ -function rootBudget(timeoutMs: number): Budget { +/** A caller-supplied deadline covers the complete supervisor lifecycle from run start through cleanup. */ +function rootBudget(timeoutMs: number | undefined): Budget { return { maxIterations: ROOT_MAX_ITERATIONS, maxTokens: ROOT_MAX_TOKENS, - deadlineMs: timeoutMs, + ...(timeoutMs === undefined ? {} : { deadlineMs: timeoutMs }), } } @@ -568,7 +568,7 @@ function isObject(value: unknown): value is Record { async function waitForWorkerSpawn( worker: Promise, run: Promise>, - timeoutMs: number, + timeoutMs: number | undefined, ): Promise { return await withTimeout( Promise.race([ @@ -585,7 +585,7 @@ async function waitForWorkerSpawn( async function waitForWorkerRunning( running: Promise, run: Promise>, - timeoutMs: number, + timeoutMs: number | undefined, ): Promise { await withTimeout( Promise.race([ @@ -603,7 +603,7 @@ async function waitForInteractiveBinding( eventDir: string, workerId: string, run: Promise>, - timeoutMs: number, + timeoutMs: number | undefined, pollMs: number, ): Promise { await withTimeout( @@ -640,7 +640,12 @@ async function pollUntil( } } -async function withTimeout(promise: Promise, timeoutMs: number, label: string): Promise { +async function withTimeout( + promise: Promise, + timeoutMs: number | undefined, + label: string, +): Promise { + if (timeoutMs === undefined) return await promise let timer: ReturnType | undefined const timeout = new Promise((_resolve, reject) => { timer = setTimeout( @@ -694,16 +699,16 @@ async function waitForWorkerTerminal( journal: import('./types').SpawnJournal, root: string, workerId: string, - timeoutMs: number, + timeoutMs: number | undefined, pollMs: number, ): Promise { - const deadline = Date.now() + timeoutMs + const deadline = timeoutMs === undefined ? undefined : Date.now() + timeoutMs for (;;) { const events = await journal.loadTree(root) if (events !== undefined && workerStatusFromEvents(events, workerId) !== 'running') { return events } - if (Date.now() >= deadline) { + if (deadline !== undefined && Date.now() >= deadline) { throw unavailable(`Runtime supervisor timed out waiting for worker '${workerId}' to settle`) } // Cleanup is an explicit lifecycle operation. Keep this bounded poll referenced so a caller diff --git a/src/runtime/supervise/runtime.ts b/src/runtime/supervise/runtime.ts index de59acbf..c9842d24 100644 --- a/src/runtime/supervise/runtime.ts +++ b/src/runtime/supervise/runtime.ts @@ -121,7 +121,7 @@ import { unmeteredSpend, zeroTokenUsage, } from '../util' -import { linkAbort } from './abortable' +import { linkAbort, runAbortable } from './abortable' import { priceUnreceiptedWork } from './cost-estimate' import { executableAgentProfileSnapshot, executableAgentSpecSnapshot } from './executable-spec' import { createInPlaceCliExecutor } from './in-place-cli-executor' @@ -2366,10 +2366,15 @@ function bridgeGet( seam: BridgeSeam, path: string, timeoutMs: number, + signal?: AbortSignal, ): Promise<{ status: number; body: string }> { const target = new URL(`${seam.bridgeUrl.replace(/\/$/, '')}${path}`) const requestFn = target.protocol === 'https:' ? httpsRequest : httpRequest return new Promise<{ status: number; body: string }>((resolve, reject) => { + if (signal?.aborted) { + reject(new DOMException(`bridge GET ${path} aborted before request`, 'AbortError')) + return + } const req = requestFn( target, { @@ -2385,24 +2390,37 @@ function bridgeGet( })().catch(reject) }, ) + const abort = () => req.destroy(new DOMException(`bridge GET ${path} aborted`, 'AbortError')) + signal?.addEventListener('abort', abort, { once: true }) req.on('timeout', () => req.destroy(new Error(`bridge GET ${path} timed out`))) - req.on('error', reject) + req.on('error', (error) => { + signal?.removeEventListener('abort', abort) + reject(error) + }) + req.on('close', () => signal?.removeEventListener('abort', abort)) req.end() }) } +export interface BridgeModelRouteRefusal { + readonly detail: string + readonly retryable: boolean + readonly status?: number +} + /** * `GET /v1/capabilities?model=` — does this bridge route this model AT ALL. * * The bridge answers exactly this question and 404s `no backend matches model "…"`, which is the * error a child would otherwise discover after it was spawned and metered. Returns `undefined` - * when the route resolves; otherwise the operator-facing reason it did not. FAIL CLOSED: a - * transport error and an unexpected status are both reasons, never a silent pass. + * when the route resolves; otherwise it returns the reason and whether the failure can recover. + * A transport error and an unexpected status never become a silent pass. */ export async function bridgeModelRouteRefusal( seam: BridgeSeam, wireModel: string, -): Promise { + signal?: AbortSignal, +): Promise { const base = seam.bridgeUrl.replace(/\/$/, '') let answer: { status: number; body: string } try { @@ -2410,28 +2428,44 @@ export async function bridgeModelRouteRefusal( seam, `/v1/capabilities?model=${encodeURIComponent(wireModel)}`, BRIDGE_RUN_STATE_TIMEOUT_MS, + signal, ) } catch (error) { - return `bridge ${base} did not answer for model ${JSON.stringify(wireModel)}: ${error instanceof Error ? error.message : String(error)}` + if (signal?.aborted) throw error + return { + detail: `bridge ${base} did not answer for model ${JSON.stringify(wireModel)}: ${error instanceof Error ? error.message : String(error)}`, + retryable: true, + } } if (answer.status === 200) return undefined - return answer.status === 404 - ? `bridge ${base} routes no backend for model ${JSON.stringify(wireModel)}` - : `bridge ${base} answered ${answer.status} for model ${JSON.stringify(wireModel)}` + if (answer.status === 404) { + return { + detail: `bridge ${base} routes no backend for model ${JSON.stringify(wireModel)}`, + retryable: false, + status: answer.status, + } + } + return { + detail: `bridge ${base} answered ${answer.status} for model ${JSON.stringify(wireModel)}`, + retryable: answer.status === 408 || answer.status === 429 || answer.status >= 500, + status: answer.status, + } } /** - * `GET /health` → `admission.{active,maxActive}`, or `undefined` when the bridge does not report - * them. ADVISORY by nature — admission fills and drains — so an unanswered or admission-less - * `/health` is not evidence about capacity and must not be read as one. + * Read the default bulk lane from `GET /health`. Runtime sends no reserved-client header, so + * overall capacity can look free while the lane this request will use is already full. Older + * bridges expose only the overall counters; those retain the prior fallback. */ -export async function bridgeAdmissionRead( +export async function bridgeAdmissionRefusal( seam: BridgeSeam, -): Promise<{ active: number; maxActive: number } | undefined> { + signal?: AbortSignal, +): Promise { let answer: { status: number; body: string } try { - answer = await bridgeGet(seam, '/health', BRIDGE_RUN_STATE_TIMEOUT_MS) - } catch { + answer = await bridgeGet(seam, '/health', BRIDGE_RUN_STATE_TIMEOUT_MS, signal) + } catch (error) { + if (signal?.aborted) throw error return undefined } let parsed: unknown @@ -2440,14 +2474,47 @@ export async function bridgeAdmissionRead( } catch { return undefined } - const admission = (parsed as { admission?: { active?: unknown; maxActive?: unknown } } | null) - ?.admission + const admission = ( + parsed as { + admission?: { + active?: unknown + maxActive?: unknown + bulkMaxActive?: unknown + activeByClass?: { bulk?: unknown } + } + } | null + )?.admission const active = admission?.active const maxActive = admission?.maxActive - if (typeof active !== 'number' || typeof maxActive !== 'number' || maxActive <= 0) { + const bulkActive = admission?.activeByClass?.bulk + const bulkMaxActive = admission?.bulkMaxActive + if ( + typeof bulkActive === 'number' && + Number.isFinite(bulkActive) && + bulkActive >= 0 && + typeof bulkMaxActive === 'number' && + Number.isFinite(bulkMaxActive) && + bulkMaxActive >= 0 + ) { + if (bulkActive < bulkMaxActive) return undefined + const total = + typeof active === 'number' && typeof maxActive === 'number' + ? ` (total active ${active} of maxActive ${maxActive})` + : '' + return `bridge ${seam.bridgeUrl.replace(/\/$/, '')} bulk admission is full: active ${bulkActive} of bulkMaxActive ${bulkMaxActive}${total}` + } + if ( + typeof active !== 'number' || + !Number.isFinite(active) || + active < 0 || + typeof maxActive !== 'number' || + !Number.isFinite(maxActive) || + maxActive <= 0 || + active < maxActive + ) { return undefined } - return { active, maxActive } + return `bridge ${seam.bridgeUrl.replace(/\/$/, '')} admission is full: active ${active} of maxActive ${maxActive}` } /** `GET /v1/runs/:id` — the bridge's durable-run registry read. Resolves `undefined` for any @@ -2626,7 +2693,7 @@ async function* streamBridgeSession(args: StreamBridgeArgs): AsyncIterable resolveBridgeModelCredential(seam.modelCredential, 'bridgeExecutor'), + preflightSignal, + 'bridgeExecutor: credential preflight aborted', + ) + if (preflightSignal.aborted) { + throw new DOMException('bridgeExecutor: aborted before bridge dispatch', 'AbortError') + } + } catch (error) { + if (args.stopSignal?.aborted) { + cleanup() + observation.note = 'settled after completion request' + break + } + const interruptedBeforeDispatch = + interruptSig.aborted && !args.signal.aborted && !args.controller.signal.aborted + try { + args.onProviderAttemptStart() + args.onProviderDispatchNotStarted() + } finally { + cleanup() + } + if (interruptedBeforeDispatch) { + observation.note = `turn ${turns + 1} interrupted before dispatch, resuming` + continue + } + throw error + } + if (args.stopSignal?.aborted) { + observation.note = 'settled after completion request' + cleanup() + break + } + args.onProviderAttemptStart() + const activeRun: ActiveBridgeRun = { id: `bridge-run-${randomUUID()}`, transportAttempts: 0, @@ -2728,20 +2847,12 @@ async function* streamBridgeSession(args: StreamBridgeArgs): AsyncIterable { - await assertBridgeExecutionCapabilities(args.seam, args.signal, args.requiresRuntimeAttachments) let reconnects = 0 let pendingUpstreamError: Error | undefined for (;;) { let res: BridgeResponse try { + const modelCredential = + args.run.transportAttempts === 0 + ? args.initialModelCredential + : await runAbortable( + () => resolveBridgeModelCredential(args.seam.modelCredential, 'bridgeExecutor'), + args.signal, + 'bridgeExecutor: reconnect credential lookup aborted', + ) args.run.transportAttempts += 1 res = await bridgeStreamPost(args.seam.bridgeUrl, { bearer: args.seam.bridgeBearer, - modelCredential: args.seam.modelCredential, + modelCredential, sessionId: args.sessionId, runId: args.run.id, afterEventId: args.run.lastEventId, @@ -3375,7 +3493,7 @@ async function* streamDurableBridgeRun( * is not evidence about the process that will receive the next POST. The terminal receipt remains * mandatory because the bridge can still restart between this GET and the run request. */ async function assertBridgeExecutionCapabilities( - seam: BridgeSeam, + seam: ResolvedBridgeSeam, signal: AbortSignal, requiresRuntimeAttachments: boolean, ): Promise { @@ -3446,6 +3564,24 @@ async function assertBridgeExecutionCapabilities( ) } } + + const routeRefusal = await bridgeModelRouteRefusal(seam, seam.model, signal) + if (routeRefusal !== undefined) { + if (routeRefusal.retryable) { + throw new BackendTransportError('bridge', `bridgeExecutor: ${routeRefusal.detail}`, { + ...(routeRefusal.status === undefined ? {} : { status: routeRefusal.status }), + providerDispatch: 'not_started', + }) + } + throw new ValidationError(`bridgeExecutor: ${routeRefusal.detail}`) + } + + const admissionRefusal = await bridgeAdmissionRefusal(seam, signal) + if (admissionRefusal !== undefined) { + throw new BackendTransportError('bridge', `bridgeExecutor: ${admissionRefusal}`, { + providerDispatch: 'not_started', + }) + } } /** The subset of `Response` `streamBridgeSession` consumes: status gate, an error @@ -3460,7 +3596,7 @@ interface BridgeResponse { interface BridgeStreamPostArgs { bearer: string - modelCredential?: BridgeModelCredential + modelCredential?: ResolvedBridgeModelCredential sessionId: string runId: string afterEventId: number @@ -3483,7 +3619,6 @@ interface BridgeStreamPostArgs { * shared `parseSseChatStream` consumes it unchanged. */ async function bridgeStreamPost(url: string, args: BridgeStreamPostArgs): Promise { - const modelCredential = await resolveBridgeModelCredential(args.modelCredential, 'bridgeExecutor') const target = new URL(`${url.replace(/\/$/, '')}/v1/chat/completions`) const payload = JSON.stringify(args.body) const requestFn = target.protocol === 'https:' ? httpsRequest : httpRequest @@ -3502,11 +3637,11 @@ async function bridgeStreamPost(url: string, args: BridgeStreamPostArgs): Promis ...args.traceHeaders, 'content-type': 'application/json', authorization: `Bearer ${args.bearer}`, - ...(modelCredential === undefined + ...(args.modelCredential === undefined ? {} : { - [bridgeModelCredentialHeader]: modelCredential.token, - [bridgeModelBaseUrlHeader]: modelCredential.baseUrl, + [bridgeModelCredentialHeader]: args.modelCredential.token, + [bridgeModelBaseUrlHeader]: args.modelCredential.baseUrl, }), 'x-session-id': args.sessionId, 'x-run-id': args.runId, diff --git a/src/runtime/supervise/supervise.ts b/src/runtime/supervise/supervise.ts index 205ca13d..0638032d 100644 --- a/src/runtime/supervise/supervise.ts +++ b/src/runtime/supervise/supervise.ts @@ -60,7 +60,7 @@ import { composeRuntimeHooks, type RuntimeHooks } from '../../runtime-hooks' import { agentHarness, harnessRunsAgent } from '../harness-role' import type { RouterTransportConfig } from '../router-client' import type { ToolLoopChat, ToolLoopCompactionOptions } from '../tool-loop' -import { unmeteredSpend } from '../util' +import { unmeteredSpend, zeroSpend } from '../util' import { assertValidBudget, spendFromUsageEvents } from './budget' import { type DeliverableSpec, gateOnDeliverable } from './completion-gate' import { DEFAULT_SUCCESSFUL_SHUTDOWN_MS, teardownExecutor } from './deadline' @@ -96,7 +96,7 @@ import { readRunCancellation, readRunCancelRequest, writeRunCancellation } from import { type BridgeSeam, bindReusableExecutorExecutionId, - bridgeAdmissionRead, + bridgeAdmissionRefusal, bridgeModelRouteRefusal, bridgeRuntimeAttachmentsKey, bridgeStopSignalKey, @@ -584,10 +584,9 @@ function unmountedCoordinationTools(profile: AgentProfile): readonly string[] { * question and 404s `no backend matches model "…"`. FAIL CLOSED: any answer that is not a route * refuses, including a transport error or an unexpected status, because a pre-flight that skips * itself on an error is the silent admission it exists to remove. - * - `bridge-full` — `GET /health` → `admission.active >= admission.maxActive`. ADVISORY by - * nature (admission can fill or drain a moment later), so only a POSITIVE reading of fullness - * refuses: a `/health` that does not answer, or answers without an admission snapshot, is not - * evidence that the bridge is full and admits the spawn. + * - `bridge-full` — `GET /health` checks the bulk lane that an unreserved Runtime request uses. + * Older bridges fall back to the overall admission counters. Capacity is advisory, so only a + * positive reading of fullness refuses the spawn. */ function bridgeSpawnPreflight(seam: BridgeSeam): SpawnPreflight { return async (profile) => { @@ -606,14 +605,9 @@ function bridgeSpawnPreflight(seam: BridgeSeam): SpawnPreflight { } } const routeRefusal = await bridgeModelRouteRefusal(seam, wireModel) - if (routeRefusal !== undefined) return { cause: 'model-route', detail: routeRefusal } - const admission = await bridgeAdmissionRead(seam) - if (admission && admission.active >= admission.maxActive) { - return { - cause: 'bridge-full', - detail: `bridge ${seam.bridgeUrl.replace(/\/$/, '')} admission is full: active ${admission.active} of maxActive ${admission.maxActive}`, - } - } + if (routeRefusal !== undefined) return { cause: 'model-route', detail: routeRefusal.detail } + const admissionRefusal = await bridgeAdmissionRefusal(seam) + if (admissionRefusal !== undefined) return { cause: 'bridge-full', detail: admissionRefusal } return undefined } } @@ -773,6 +767,19 @@ function driveHarnessFromBackend( } const meterPending = async (forceUnknown = false) => { if (pendingUsage.length === 0) return + // A forceful steer can finish one preflight attempt before the resumed turn produces usage. + // Consume those explicit no-dispatch attempts first, so the paid batch keeps its own served + // identity instead of inheriting the oldest positional evidence. + for (;;) { + const evidence = runtimeOwnedExecutorProviderEvidence(executor) + const next = evidence?.attempts[meteredProviderAttempts] + if (next?.providerDispatch !== 'not_started') break + await meterRuntimeOwnedProviderAttempt(scope, zeroSpend(), providerEvidenceForNextMeter(), { + role: 'driver', + runtime: executor.runtime, + telemetry: 'known-zero-before-dispatch', + }) + } const batch = pendingUsage pendingUsage = [] const measured = spendFromUsageEvents(batch) @@ -985,13 +992,22 @@ function driveHarnessFromBackend( // Persist one unknown-cost marker for every unmetered attempt; dropping a later model // observation would let an earlier model appear homogeneous by accident. for (;;) { - const attempts = runtimeOwnedExecutorProviderEvidence(executor)?.attempts.length ?? 0 + const evidence = runtimeOwnedExecutorProviderEvidence(executor) + const attempts = evidence?.attempts.length ?? 0 if (attempts <= meteredProviderAttempts) break + const providerDispatchDidNotStart = + evidence?.attempts[meteredProviderAttempts]?.providerDispatch === 'not_started' await meterRuntimeOwnedProviderAttempt( scope, - unmeteredSpend(0), + providerDispatchDidNotStart ? zeroSpend() : unmeteredSpend(0), providerEvidenceForNextMeter(), - { role: 'driver', runtime: executor.runtime, telemetry: 'unknown-after-failure' }, + { + role: 'driver', + runtime: executor.runtime, + telemetry: providerDispatchDidNotStart + ? 'known-zero-before-dispatch' + : 'unknown-after-failure', + }, ) } if (meteredProviderAttempts === 0) { diff --git a/src/runtime/supervise/supervisor-agent.ts b/src/runtime/supervise/supervisor-agent.ts index 9e218623..7de26bf5 100644 --- a/src/runtime/supervise/supervisor-agent.ts +++ b/src/runtime/supervise/supervisor-agent.ts @@ -290,7 +290,7 @@ export interface CoordinationToolFace { */ interface VerbSlot { readonly verbs: CoordinationVerbs - readonly descriptors: () => ReadonlyArray + readonly descriptors: () => ReadonlyArray> bind(tools: ReadonlyArray): void } @@ -324,7 +324,7 @@ function createVerbSlot(): VerbSlot { readJournal: verb('read_journal'), defineAnalyst: verb('define_analyst'), }), - descriptors(): ReadonlyArray { + descriptors(): ReadonlyArray> { if (bound === undefined) { throw new ValidationError( "supervisorAgent: coordinationTools() was called before this manager's coordination tools were bound", @@ -332,11 +332,7 @@ function createVerbSlot(): VerbSlot { } return Object.freeze( bound.map(({ name, description, inputSchema }) => - Object.freeze({ - name, - ...(description === undefined ? {} : { description }), - ...(inputSchema === undefined ? {} : { inputSchema }), - }), + Object.freeze({ name, description, inputSchema }), ), ) }, @@ -902,6 +898,7 @@ function buildSupervisorAgent( onCoordinationTools: (tools) => slot.bind(tools), }) ledger = mcp + const coordinationTools = slot.descriptors() try { // The retry's progress mark. `tokensLeft` only falls, so the difference from the first // reading is everything this run has spent from the shared pool — the driver's own turns @@ -944,11 +941,7 @@ function buildSupervisorAgent( scope, coordinationMcpUrl: mcp.url, stopSignal: stopController.signal, - coordinationTools: (nodeTools ?? []).map(({ name, description, inputSchema }) => ({ - name, - description, - inputSchema, - })), + coordinationTools, }) } catch (error) { // Once the injected check has accepted a result, a later backend shutdown/timeout diff --git a/src/testing/fixtures/agent-improvement-proposal.json b/src/testing/fixtures/agent-improvement-proposal.json index 617262f4..2de6d2aa 100644 --- a/src/testing/fixtures/agent-improvement-proposal.json +++ b/src/testing/fixtures/agent-improvement-proposal.json @@ -1,6 +1,6 @@ { "changedSurfaces": ["prompt"], - "digest": "sha256:433f998c383e7b0a04044a656edda018a995622d84e4bbfc4e408d7117e60330", + "digest": "sha256:6d74288ce43d6b1e2a73377160d62b7ba4e6b294c8bfc5fdc2eac8ce8aba175e", "evaluation": { "decision": { "contributingChecks": [ @@ -4882,7 +4882,7 @@ ], "metadata": { "fixture": "agent-improvement-proposal", - "runtimeVersion": "0.193.0" + "runtimeVersion": "0.193.1" }, "objectives": [ { @@ -4993,8 +4993,8 @@ "baselineContentHash": "sha256:5c21ee53e513fc604cb09754e21c392b24a424da0ef37dbf8f1ee4a8a0b08f09", "candidateContentHash": "sha256:60fcbb1c728194bd51d7d19cb732d1c3f1881dce7e0a6266b41c8b98cfd65693", "kind": "agent-eval-loop", - "recordDigest": "sha256:a71d4d9d25b9621bec6ef0179e96046f0b0eed2f2715a27c3451fdd2364cec87", - "runId": "agent-runtime-0.193.0-proposal-fixture", + "recordDigest": "sha256:565f134e0589f18ce455cfd7fef2d76cbf5af335bc8e38befc4b587e640768bf", + "runId": "agent-runtime-0.193.1-proposal-fixture", "schema": "agent-candidate-experiment" } }, @@ -5021,5 +5021,5 @@ ], "kind": "agent-improvement-proposal", "proposedAt": "2026-07-10T01:00:00.000Z", - "runId": "agent-runtime-0.193.0-proposal-fixture" + "runId": "agent-runtime-0.193.1-proposal-fixture" } diff --git a/src/testing/fixtures/agent-profile-improvement-proposal.json b/src/testing/fixtures/agent-profile-improvement-proposal.json index 834f40b7..49ede633 100644 --- a/src/testing/fixtures/agent-profile-improvement-proposal.json +++ b/src/testing/fixtures/agent-profile-improvement-proposal.json @@ -1,6 +1,6 @@ { "changedSurfaces": ["prompt", "skills"], - "digest": "sha256:1f35e72238574b46a51a31d680bbb75017df5eb568cdff02e0ccf72f61c894de", + "digest": "sha256:4f1f9175b8ae98f66fa860a54f215006e0a8c011db73d26623b2c7d2902ba9fb", "evaluation": { "decision": { "contributingChecks": [ @@ -1715,7 +1715,7 @@ ], "metadata": { "fixture": "agent-profile-improvement-proposal", - "runtimeVersion": "0.193.0" + "runtimeVersion": "0.193.1" }, "objectives": [ { @@ -1826,7 +1826,7 @@ "baselineContentHash": "sha256:21c495a37c418c10bde64fbaa188beddeed31f1f051ea60a6a6582a9ee0db704", "candidateContentHash": "sha256:103f77bc8481601eef1ad5fe6ba84a40dffabc3a44f421f8c8559121edab84e9", "kind": "agent-eval-loop", - "recordDigest": "sha256:2dadb5ec86bc3360b6b8800b0b0bcf04892c70ec973f548e6f45d04571a56991", + "recordDigest": "sha256:d897dd5e49008c6352decb6befa70952e578dff0069351194185b4b202411085", "runId": "profile-improvement-1", "schema": "agent-profile-improvement-experiment" } diff --git a/tests/candidate-bundle-builder.test.ts b/tests/candidate-bundle-builder.test.ts index 1e81e82d..1c4f9ce2 100644 --- a/tests/candidate-bundle-builder.test.ts +++ b/tests/candidate-bundle-builder.test.ts @@ -146,7 +146,7 @@ describe('public agent candidate bundle builder', () => { } finally { await adapter.discard(worktree) } - }) + }, 120_000) it('rejects CodeSurface drift before sealing and exports the low-level sealer', async () => { const fixture = createCandidateExecutionFixture(true) @@ -180,7 +180,7 @@ describe('public agent candidate bundle builder', () => { } finally { await adapter.discard(worktree) } - }) + }, 120_000) it('fails closed when a generic profile would lose behavior or byte identity', () => { const fixture = createCandidateExecutionFixture(false) diff --git a/tests/improvement-driver.test.ts b/tests/improvement-driver.test.ts index ed7f63d9..1926b48a 100644 --- a/tests/improvement-driver.test.ts +++ b/tests/improvement-driver.test.ts @@ -192,7 +192,7 @@ describe('improvementDriver — reflective generator', () => { 'edited from raw traces\n', ) } - }) + }, 120_000) it('wraps labeled generator results into ProposedCandidate so the loop keeps attribution', async () => { // Gen-3 proposer fan-out contract: a generator that names its proposer @@ -231,7 +231,7 @@ describe('improvementDriver — reflective generator', () => { expect(surface.kind).toBe('code') verifyCodeSurface(surface) }) - }) + }, 120_000) it('forks isolated generation-two candidates from the promoted generation-one surface', async () => { const worktree = gitWorktreeAdapter({ repoRoot }) @@ -326,7 +326,7 @@ describe('improvementDriver — reflective generator', () => { expect(existsSync(winner.worktreeRef)).toBe(true) expect(verifyCodeSurface(winner).contentHash).toMatch(/^sha256:/) expect(git(['show', 'main:prompt.md'], repoRoot)).toBe('lax rubric') - }) + }, 120_000) it('rethrows and leaves NO orphaned worktree when the generator throws', async () => { // A generator whose generate() throws mid-candidate must not leak the diff --git a/tests/kernel/loop-dispatch.test.ts b/tests/kernel/loop-dispatch.test.ts index ff3c0379..10792eda 100644 --- a/tests/kernel/loop-dispatch.test.ts +++ b/tests/kernel/loop-dispatch.test.ts @@ -183,6 +183,16 @@ async function startPiBridge( ) return } + if (req.method === 'GET' && req.url?.startsWith('/v1/capabilities?model=')) { + res.writeHead(200, { 'content-type': 'application/json' }) + res.end(JSON.stringify({ available: true })) + return + } + if (req.method === 'GET' && req.url === '/health') { + res.writeHead(200, { 'content-type': 'application/json' }) + res.end(JSON.stringify({ admission: { active: 0, maxActive: 1 } })) + return + } if (req.method !== 'POST' || req.url !== '/v1/chat/completions') { res.writeHead(404) res.end() diff --git a/tests/kernel/supervise-full-profile-bridge.test.ts b/tests/kernel/supervise-full-profile-bridge.test.ts index a69aa03a..5216cea9 100644 --- a/tests/kernel/supervise-full-profile-bridge.test.ts +++ b/tests/kernel/supervise-full-profile-bridge.test.ts @@ -77,7 +77,14 @@ interface FakeBridgeRoutes { /** Wire ids this bridge routes. Any other id 404s. Omit = route everything. */ routedModels?: ReadonlySet /** Admission counters `/health` reports. Omit = report no admission snapshot at all. */ - admission?: { active: number; maxActive: number } + admission?: { + active: number + maxActive: number + bulkMaxActive?: number + activeByClass?: { bulk: number; reserved: number } + } + /** Every request at the bridge boundary, before the fake handles it. */ + onRequest?: (request: { method: string | undefined; url: string | undefined }) => void } function createBridgeServer( @@ -86,6 +93,7 @@ function createBridgeServer( ): Server { const sessionBindings = new Map() return createServer((req, res) => { + routes.onRequest?.({ method: req.method, url: req.url }) if (req.method === 'GET' && req.url?.startsWith('/v1/capabilities')) { const model = new URL(req.url, 'http://bridge.test').searchParams.get('model') ?? '' if (routes.routedModels && !routes.routedModels.has(model)) { @@ -419,6 +427,241 @@ describe('supervise — complete profiles over recursive cli-bridge managers', ( expect(() => handle.deliver({ steer: 'after completion' })).toThrow() }) + it('meters an interrupted preflight as zero spend before the resumed paid attempt', async () => { + const runId = 'bridge-preflight-meter-order' + const events: SpawnEvent[] = [] + const journal = recordingJournal(events) + const handle = createRootHandle() + let routeReads = 0 + let markFirstRouteRead!: () => void + const firstRouteRead = new Promise((resolve) => { + markFirstRouteRead = resolve + }) + const requests: BridgeRequest[] = [] + server = createServer(async (req, res) => { + if (req.method === 'GET' && req.url === '/') { + res.writeHead(200, { 'content-type': 'application/json' }) + res.end( + JSON.stringify({ + capabilities: { + profileMaterialization: 'cli-bridge.profile-materialization.v2', + usageCostProvenance: 'cli-bridge.usage-cost.v1', + runtimeAttachments: { mcp: true }, + }, + }), + ) + return + } + if (req.method === 'GET' && req.url?.startsWith('/v1/capabilities?model=')) { + routeReads += 1 + if (routeReads === 1) { + markFirstRouteRead() + return + } + res.writeHead(200, { 'content-type': 'application/json' }) + res.end(JSON.stringify({ available: true })) + return + } + if (req.method === 'GET' && req.url === '/health') { + res.writeHead(200, { 'content-type': 'application/json' }) + res.end(JSON.stringify({ admission: { active: 0, maxActive: 4 } })) + return + } + const body = await readJson(req) + requests.push(body) + const servedModel = 'openai/test@fp_preflight_resume' + const stream = [ + `data: ${JSON.stringify({ model: servedModel, choices: [{ delta: { content: 'served' } }] })}`, + `data: ${JSON.stringify({ + model: servedModel, + usage: { prompt_tokens: 3, completion_tokens: 1, cost: 0.01 }, + })}`, + 'data: [DONE]', + '', + ].join('\n\n') + respondWithBridgeStream(res, body, stream) + }) + await new Promise((resolve) => server?.listen(0, '127.0.0.1', resolve)) + const { port } = server.address() as AddressInfo + const running = supervise(codexTestProfile('bridge-root', 'Lead.'), 'Choose.', { + rootHandle: handle, + backend: { + backend: 'bridge', + bridgeUrl: `http://127.0.0.1:${port}`, + bridgeBearer: 'test-token', + }, + budget: { maxIterations: 4, maxTokens: 10_000 }, + driverRetry: { enabled: false }, + journal, + runId, + }) + + await firstRouteRead + expect(handle.deliver({ steer: 'use the corrected plan', interrupt: true })).toBe(true) + await running + + expect(routeReads).toBe(2) + expect(requests).toHaveLength(1) + const metered = events.filter( + (event): event is Extract => + event.kind === 'metered' && event.id === runId, + ) + expect(metered).toHaveLength(2) + expect(metered[0]).toMatchObject({ + spend: { iterations: 0, tokens: { input: 0, output: 0 }, usd: 0 }, + providerModel: { + attempts: [{ observations: [], providerDispatch: 'not_started' }], + }, + }) + expect(metered[1]).toMatchObject({ + spend: { tokens: { input: 3, output: 1 }, usd: 0.01 }, + providerModel: { + attempts: [{ observations: ['openai/test@fp_preflight_resume'] }], + }, + }) + }) + + it('retries a transient root route preflight before allocating a remote run', async () => { + const requests: Array<{ method: string | undefined; url: string | undefined }> = [] + let routeReads = 0 + server = createServer(async (req, res) => { + requests.push({ method: req.method, url: req.url }) + if (req.method === 'GET' && req.url === '/') { + res.writeHead(200, { 'content-type': 'application/json' }) + res.end( + JSON.stringify({ + capabilities: { + profileMaterialization: 'cli-bridge.profile-materialization.v2', + usageCostProvenance: 'cli-bridge.usage-cost.v1', + runtimeAttachments: { mcp: true }, + }, + }), + ) + return + } + if (req.method === 'GET' && req.url?.startsWith('/v1/capabilities?model=')) { + routeReads += 1 + res.writeHead(routeReads === 1 ? 503 : 200, { 'content-type': 'application/json' }) + res.end( + JSON.stringify(routeReads === 1 ? { error: 'backend is warming' } : { available: true }), + ) + return + } + if (req.method === 'GET' && req.url === '/health') { + res.writeHead(200, { 'content-type': 'application/json' }) + res.end(JSON.stringify({ admission: { active: 0, maxActive: 4 } })) + return + } + const body = await readJson(req) + respondWithBridgeStream(res, body, successStream('managed after route recovery')) + }) + await new Promise((resolve) => server?.listen(0, '127.0.0.1', resolve)) + const { port } = server.address() as AddressInfo + const attempts: DriverAttemptRecord[] = [] + + const result = await supervise(codexTestProfile('bridge-root', 'Lead.'), 'Choose.', { + backend: { + backend: 'bridge', + bridgeUrl: `http://127.0.0.1:${port}`, + bridgeBearer: 'test-token', + }, + budget: { maxIterations: 4, maxTokens: 10_000 }, + driverRetry: { initialBackoffMs: 1 }, + onDriverAttempt: (attempt) => void attempts.push(attempt), + }) + + expect(attempts.map((attempt) => attempt.stop ?? attempt.classification)).toEqual([ + 'transient', + 'completed', + ]) + expect(result.kind === 'no-winner' ? result.reason : 'winner').not.toBe('driver-failed') + expect(requests).toEqual([ + { method: 'GET', url: '/' }, + { method: 'GET', url: '/v1/capabilities?model=codex%2Fopenai%2Ftest' }, + { method: 'GET', url: '/' }, + { method: 'GET', url: '/v1/capabilities?model=codex%2Fopenai%2Ftest' }, + { method: 'GET', url: '/health' }, + { method: 'POST', url: '/v1/chat/completions' }, + ]) + }) + + it('records the exact served root MCP tools in bridge materialization evidence', async () => { + const servedToolSets: Array< + Array<{ name: string; description?: string; inputSchema?: unknown }> + > = [] + const journal = new InMemorySpawnJournal() + server = createBridgeServer(async (req, res) => { + const body = await readJson(req) + const coordination = body.runtime_attachments?.mcp['agent-runtime-coordination'] + if (coordination?.url === undefined) { + throw new Error('root bridge request did not mount the coordination MCP') + } + const response = await fetch(coordination.url, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ + jsonrpc: '2.0', + id: 'listed-tools', + method: 'tools/list', + params: {}, + }), + }) + const payload = (await response.json()) as { + result?: { tools?: Array<{ name: string; description?: string; inputSchema?: unknown }> } + } + if (!response.ok || payload.result?.tools === undefined) { + throw new Error('coordination MCP did not return tools/list') + } + servedToolSets.push(payload.result.tools) + respondWithBridgeStream(res, body, successStream('managed')) + }) + await new Promise((resolve) => server?.listen(0, '127.0.0.1', resolve)) + const { port } = server.address() as AddressInfo + const rootProfile = codexTestProfile('tool-evidence-root', 'Lead with exact tool evidence.') + + await supervise(rootProfile, 'Choose.', { + backend: { + backend: 'bridge', + bridgeUrl: `http://127.0.0.1:${port}`, + bridgeBearer: 'test-token', + }, + budget: { maxIterations: 4, maxTokens: 10_000 }, + journal, + runId: 'bridge-tool-evidence', + resolveSupervisorTools: async () => [ + { + name: 'read_root_evidence', + description: 'Read one root-scoped evidence record', + inputSchema: { type: 'object', properties: { id: { type: 'string' } } }, + handler: async () => ({ value: 'not-called' }), + }, + ], + }) + + expect(servedToolSets).toHaveLength(1) + const servedNames = servedToolSets[0]?.map((tool) => tool.name) ?? [] + expect(servedNames).toContain('spawn_worker') + expect(servedNames).toContain('read_root_evidence') + expect(new Set(servedNames).size).toBe(servedNames.length) + const events = await journal.loadTree('bridge-tool-evidence') + const materialized = events?.find( + (event) => event.kind === 'materialized' && event.id === 'bridge-tool-evidence', + ) + expect( + materialized?.kind === 'materialized' + ? materialized.receipt.platformAttachmentsDigest + : undefined, + ).toBe( + canonicalCandidateDigest({ + 'agent-runtime-coordination': { + kind: 'coordination-mcp', + transport: 'http', + tools: servedToolSets[0], + }, + }), + ) + }) + it('maxTurns caps the bridge manager turns; the same run without it keeps resuming', async () => { // The harness owns its own loop, so `maxTurns` can only bound it at a turn boundary. Each // steer would produce one more bridge request; the cap stops the loop after N of them. @@ -912,6 +1155,94 @@ describe('supervise — complete profiles over recursive cli-bridge managers', ( expect(refusal).toContain('pi/tangle-router/deepseek-v4-flash') }) + it('refuses a bulk-full root bridge in a split topology with known zero spend and no remote run', async () => { + const bridgeRequests: Array<{ method: string | undefined; url: string | undefined }> = [] + server = createBridgeServer( + (_req, res) => { + res.writeHead(500) + res.end('unexpected request') + }, + { + admission: { + active: 3, + maxActive: 4, + bulkMaxActive: 3, + activeByClass: { bulk: 3, reserved: 0 }, + }, + onRequest: (request) => bridgeRequests.push(request), + }, + ) + await new Promise((resolve) => server?.listen(0, '127.0.0.1', resolve)) + const { port } = server.address() as AddressInfo + const events: SpawnEvent[] = [] + const attempts: DriverAttemptRecord[] = [] + const result = await supervise(codexTestProfile('root', 'Lead.'), 'Choose.', { + backend: { + backend: 'sandbox', + sandboxClient: { + create: async () => { + throw new Error('split topology must not create a worker box') + }, + }, + }, + driverBackend: { + backend: 'bridge', + bridgeUrl: `http://127.0.0.1:${port}`, + bridgeBearer: 'test-token', + }, + budget: { maxIterations: 4, maxTokens: 10_000 }, + driverRetry: { enabled: false }, + journal: recordingJournal(events), + runId: 'split-root-full-bridge', + onDriverAttempt: (attempt) => void attempts.push(attempt), + }) + + expect(result.kind).toBe('no-winner') + if (result.kind !== 'no-winner') return + expect(result.reason).toBe('driver-failed') + expect(attempts).toMatchObject([ + { classification: 'transient', stop: 'retry-disabled', madeProgress: false }, + ]) + expect(bridgeRequests).toEqual([ + { method: 'GET', url: '/' }, + { + method: 'GET', + url: '/v1/capabilities?model=codex%2Fopenai%2Ftest', + }, + { method: 'GET', url: '/health' }, + ]) + expect(result.rootProviderModel).toEqual({ + status: 'unknown', + attempts: [{ observations: [], providerDispatch: 'not_started' }], + models: [], + reason: 'provider-model-missing', + }) + expect(result.providerModel).toEqual(result.rootProviderModel) + expect(result.spentTotal).toMatchObject({ + iterations: 0, + tokens: { input: 0, output: 0 }, + tokensKnown: true, + usd: 0, + usdKnown: true, + }) + expect( + events.filter( + (event): event is Extract => event.kind === 'metered', + ), + ).toMatchObject([ + { + spend: { + iterations: 0, + tokens: { input: 0, output: 0 }, + usd: 0, + }, + providerModel: { + attempts: [{ observations: [], providerDispatch: 'not_started' }], + }, + }, + ]) + }) + it('refuses into a full bridge and reports the refusal count', async () => { const requests: BridgeRequest[] = [] server = createBridgeServer( diff --git a/tests/runtime/worktree-cli-executor.test.ts b/tests/runtime/worktree-cli-executor.test.ts index 1f56682f..3d927d89 100644 --- a/tests/runtime/worktree-cli-executor.test.ts +++ b/tests/runtime/worktree-cli-executor.test.ts @@ -44,6 +44,31 @@ vi.mock('node:http', async () => { cb(response) return } + if ( + (opts as { method?: string }).method === 'GET' && + url.pathname === '/v1/capabilities' + ) { + const response = new PassThrough() as Readable & { + statusCode?: number + headers?: Record + } + response.statusCode = 200 + response.headers = { 'content-type': 'application/json' } + response.end(JSON.stringify({ available: true })) + cb(response) + return + } + if ((opts as { method?: string }).method === 'GET' && url.pathname === '/health') { + const response = new PassThrough() as Readable & { + statusCode?: number + headers?: Record + } + response.statusCode = 200 + response.headers = { 'content-type': 'application/json' } + response.end(JSON.stringify({ admission: { active: 0, maxActive: 1 } })) + cb(response) + return + } const payload = JSON.parse(body || '{}') as Record if (!bridgeHttpHandler) throw new Error('bridgeHttpHandler not set') const res = bridgeHttpHandler(payload) as Readable & {