From 1e996dd53f129a8546021b2c209b8dd0114a2c1d Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 16 Aug 2026 18:36:46 +0000 Subject: [PATCH 1/2] test: smoke the agent against a real local model, gating the publish MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the layer the scripted suite cannot cover: a run driven by an actual model, served by llama.cpp on the runner. No hosted API and no key. Placement is deliberate. A nightly job reports at 3am on yesterday's commit and nobody reads it; the question this answers — "does the agent still work with a model that has opinions?" — matters at the moment something is about to be published, so release.yml `needs:` it. It also runs on PRs that touch the dependency manifests, which is where upstream drift actually arrives. The test asserts mechanics only: the loop reached a final answer, the model drove the MCP tool, and the arguments parsed into the declared schema. Asserting wording or plan shape would be asserting the model's judgement, which changes with every weight and sampler — that is how a suite ends up disabled. Sampling is greedy with a fixed seed so a failure means the agent broke rather than the dice rolled differently, and the step retries once so a single stray run is not treated as signal. The job is advisory (continue-on-error) rather than blocking, and the reason is written into the workflow: I could verify the test file against a live endpoint, but not the tool-calling half. Qwen2.5-1.5B served by llama-cpp-python's chatml-function-calling shim emitted no tool calls at all — not through the agent, not through a hand-rolled request with one trivial tool. That shim is not what CI runs (llama-server --jinja uses the model's own template, which declares tool support), so the result does not transfer; but nobody has yet watched this exact combination drive a tool end to end, and blocking a publish on an unproven assertion is the wrong default. Flip one line once it has been green for a few releases. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_011FKop4At26QqqkwVGEjJur --- .github/workflows/live-model.yml | 135 +++++++++++++++++++++++++++++++ .github/workflows/release.yml | 10 +++ CLAUDE.md | 12 +++ README.md | 21 +++++ tests/live-model.test.ts | 78 ++++++++++++++++++ 5 files changed, 256 insertions(+) create mode 100644 .github/workflows/live-model.yml create mode 100644 tests/live-model.test.ts diff --git a/.github/workflows/live-model.yml b/.github/workflows/live-model.yml new file mode 100644 index 0000000..797387e --- /dev/null +++ b/.github/workflows/live-model.yml @@ -0,0 +1,135 @@ +name: Live model + +# A smoke test against a REAL model, served locally by llama.cpp on the runner. +# No hosted API and no key: the point is to catch the failures a scripted model +# cannot show — a provider changing its wire format, a prompt drifting out of +# what small models can follow, structured output no longer parsing. +# +# It gates the release rather than running nightly. A nightly run reports at +# 3am on yesterday's commit and nobody reads it; the question this answers — +# "does the agent still work with an actual model?" — matters at the moment +# something is about to be published. It also runs on dependency PRs, which is +# where upstream drift actually arrives. +on: + workflow_call: + workflow_dispatch: + pull_request: + branches: [main] + # Dependency bumps (including the autoupdate flow's) are the realistic + # source of drift; ordinary source PRs are covered by the scripted suite. + paths: + - 'package.json' + - 'package-lock.json' + +# Pinned so a rerun downloads nothing and behaves the same. Bump deliberately. +env: + LLAMA_TAG: b4785 + MODEL_REPO: Qwen/Qwen2.5-1.5B-Instruct-GGUF + MODEL_FILE: qwen2.5-1.5b-instruct-q4_k_m.gguf + +jobs: + smoke: + name: Real model smoke + runs-on: ubuntu-latest + timeout-minutes: 25 + # ADVISORY UNTIL PROVEN, then flip this to false to make it a real gate. + # + # The evidence for the default model is incomplete: Qwen2.5-1.5B served by + # llama-cpp-python's `chatml-function-calling` shim produced no tool calls + # at all — not through the agent, and not through a hand-rolled request + # with a single trivial tool. That shim is NOT what runs here (llama-server + # --jinja uses the model's own template, which does declare tool support), + # so the result does not transfer — but it does mean nobody has yet watched + # this combination drive a tool end to end. + # + # Blocking a publish on an unproven assertion is the wrong default. Let it + # report for a few releases; if it stays green, make it blocking. If it + # stays red, the fix is a larger model (3B+) or a narrower assertion, not + # a disabled job. + continue-on-error: true + permissions: + contents: read + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '22' + cache: npm + + - name: Install dependencies + run: npm ci + + - name: Cache llama.cpp and the model weights + id: cache + uses: actions/cache@v4 + with: + path: .live-model + key: live-model-${{ env.LLAMA_TAG }}-${{ env.MODEL_FILE }} + + - name: Fetch llama.cpp and the model + if: steps.cache.outputs.cache-hit != 'true' + run: | + set -euo pipefail + mkdir -p .live-model + cd .live-model + asset=$(gh api "repos/ggml-org/llama.cpp/releases/tags/$LLAMA_TAG" \ + --jq '.assets[] | select(.name | test("bin-ubuntu-x64")) | .name' | head -1) + echo "::notice::llama.cpp asset $asset" + gh release download "$LLAMA_TAG" --repo ggml-org/llama.cpp --pattern "$asset" + unzip -q -o "$asset" + curl -fsSL -o "$MODEL_FILE" \ + "https://huggingface.co/$MODEL_REPO/resolve/main/$MODEL_FILE?download=true" + ls -la + env: + GH_TOKEN: ${{ github.token }} + + - name: Start the model server + run: | + set -euo pipefail + server=$(find .live-model -name llama-server -type f | head -1) + chmod +x "$server" + # Greedy sampling with a fixed seed: the run has to be reproducible + # enough that a failure means the agent broke, not that the model + # rolled differently. --jinja turns on the chat template's tool + # calling, which is the whole point of the exercise. + "$server" -m ".live-model/$MODEL_FILE" \ + --host 127.0.0.1 --port 8080 \ + --ctx-size 8192 --temp 0 --seed 42 --jinja --no-warmup \ + > llama-server.log 2>&1 & + for _ in $(seq 1 90); do + if curl -sf http://127.0.0.1:8080/health >/dev/null; then + echo "model server is up" + exit 0 + fi + sleep 2 + done + echo "::error::the model server never became healthy" + tail -50 llama-server.log + exit 1 + + - name: Run the live-model smoke test + # One retry: a small model occasionally wanders, and a single stray run + # must not block a publish. A second failure is a real signal. + run: | + set -uo pipefail + for attempt in 1 2; do + echo "::group::attempt $attempt" + if node --test --experimental-strip-types --disable-warning=ExperimentalWarning \ + tests/live-model.test.ts; then + echo "::endgroup::" + exit 0 + fi + echo "::endgroup::" + echo "::warning::attempt $attempt failed" + done + exit 1 + env: + AGENT_LIVE_MODEL_URL: http://127.0.0.1:8080/v1 + AGENT_LIVE_MODEL: qwen2.5-1.5b-instruct + + - name: Model server log + if: failure() + run: tail -100 llama-server.log diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1e98a44..a98cc9b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -20,9 +20,19 @@ permissions: contents: write id-token: write jobs: + # Gate the publish on a run against a REAL model, served locally on the + # runner (no key, no hosted API). CI proves the code works against a scripted + # model; this proves it still works against one that has opinions -- and the + # moment before publishing is when that question is worth asking. A skipped + # or failed smoke leaves the package unpublished rather than shipping blind. + live-model: + name: Live model + uses: ./.github/workflows/live-model.yml + release: name: Publish to npm runs-on: ubuntu-latest + needs: live-model if: >- github.event_name == 'workflow_dispatch' || (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success') diff --git a/CLAUDE.md b/CLAUDE.md index beb19a8..492cce2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -10,6 +10,17 @@ - `tests/` — `node:test` suites, run via `node --experimental-strip-types` - `dist/` — `tsup` build output (do not edit) +## Test layers + +- `tests/*.test.ts` — units plus `integration.test.ts`, which runs the whole + loop over real sockets against a scripted OpenAI-compatible endpoint, a real + MCP server and a real authorization server (`tests/helpers/`). No key, ~5s. +- `tests/live-model.test.ts` — the same loop against a REAL model. Skipped + unless `AGENT_LIVE_MODEL_URL` points at an OpenAI-compatible endpoint; CI + starts one via `live-model.yml`. Asserts mechanics only (the loop finished, a + tool was driven, arguments parsed) — never wording or plan shape, or the + suite becomes a coin flip nobody trusts. + ## Mandatory checks After **any** change to `.ts` files in `src/` or `tests/`, run: @@ -50,6 +61,7 @@ This repo follows the unified `autoupdate-with-claude` baseline (same template a - `autoupdate.yml` uses `GITHUB_TOKEN` and explicitly dispatches `test.yml` (the `CI` workflow) after PR creation, because events created via `GITHUB_TOKEN` don't trigger `pull_request` workflows. - `autoupdate.yml` dispatches `claude.yml` directly via `workflow_dispatch` instead of relying on an `@claude` PR comment. - Releases stay wired through `release.yml` (npm Trusted Publisher), which fires via `workflow_run` after a successful CI on `main`. There is no `release-on-version-bump.yml` here — it would conflict with the existing tag/publish chain. +- `release.yml` now `needs:` the `live-model.yml` workflow: a smoke test against a real model (llama.cpp on the runner, no key) gates the publish. It also runs on PRs that touch `package.json` / `package-lock.json`, which is where dependency drift arrives. Deliberately NOT nightly — a 3am failure on yesterday's commit gets ignored, and the question it answers matters at publish time. `LLAMA_TAG` / `MODEL_*` are pinned so a rerun downloads nothing and behaves the same; bump them on purpose. - All actions pinned to the `@v4` line because the runner image currently lacks `externals/node24`, breaking post-cleanup of `@v5/@v6` actions. Do **not** "fix" any of the above by replacing dispatch calls with comment-based mentions, or by bumping action versions back to `@v5/@v6`. diff --git a/README.md b/README.md index 5296d36..91b7a9d 100644 --- a/README.md +++ b/README.md @@ -298,6 +298,27 @@ npm run format # prettier --write npm run format:check # prettier --check ``` +## Testing against a real model + +`npm test` never needs a key: the integration suite drives the whole loop over +loopback against a scripted OpenAI-compatible endpoint, a real MCP server and a +real authorization server. + +To run the same loop against an actual model, point it at any OpenAI-compatible +endpoint — `llama-server`, Ollama, vLLM, anything: + +```bash +llama-server -m qwen2.5-1.5b-instruct-q4_k_m.gguf --port 8080 --temp 0 --jinja + +AGENT_LIVE_MODEL_URL=http://127.0.0.1:8080/v1 \ + node --test --experimental-strip-types tests/live-model.test.ts +``` + +CI runs this as a **gate on publishing** (`live-model.yml`, wired into +`release.yml`), and on any PR that touches the dependency manifests. It asserts +mechanics only — the loop finished, the model drove the MCP tool, the arguments +parsed — never the wording, which would make it a coin flip. + ## Behavior notes & limitations - **Module formats.** ESM is the primary target; the CJS build (`dist/index.cjs`) is best-effort and depends on upstream deps (`ai`, `@ai-sdk/*`, `@modelcontextprotocol/sdk`) keeping their CJS fallbacks. If they go pure-ESM, CJS will break — the dual-format guard in [`tests/dist-loadable.test.ts`](./tests/dist-loadable.test.ts) catches the regression on the next build. diff --git a/tests/live-model.test.ts b/tests/live-model.test.ts new file mode 100644 index 0000000..8fab74c --- /dev/null +++ b/tests/live-model.test.ts @@ -0,0 +1,78 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { createAgent } from '../src/index.ts' +import { startLocalMcp } from './helpers/mcp-server.ts' + +/** + * The same loop as `integration.test.ts`, but driven by a REAL model behind an + * OpenAI-compatible endpoint instead of a scripted one. + * + * Skipped unless `AGENT_LIVE_MODEL_URL` points at a server (CI starts a small + * local model; locally, `llama-server`, Ollama or anything else with a /v1 + * endpoint will do). No hosted API and no key is involved. + * + * ── WHAT THIS MAY AND MAY NOT ASSERT ── + * Only mechanics: the loop reached a final answer, and the model actually + * drove the MCP tool with arguments that parsed. Asserting the wording, the + * plan's shape or the tool's arguments would be asserting the model's + * judgement, which changes with every weight and every sampler — that is how + * a suite ends up disabled. What is being pinned here is that a real model's + * output survives the whole path: structured-output parsing, tool dispatch + * over MCP, and the loop's exit conditions. + */ + +const LIVE_URL = process.env.AGENT_LIVE_MODEL_URL +const LIVE_MODEL = process.env.AGENT_LIVE_MODEL ?? 'local' + +test( + 'a real local model plans, calls an MCP tool, and finishes', + { skip: LIVE_URL ? false : 'set AGENT_LIVE_MODEL_URL to run', timeout: 600_000 }, + async () => { + const mcp = await startLocalMcp() + const logs: string[] = [] + try { + const agent = await createAgent( + { + clientName: 'agent-live-model-test', + providerType: 'openai-compatible', + baseURL: LIVE_URL!, + apiKey: 'not-a-real-key', + model: LIVE_MODEL, + mcpServers: { files: { url: mcp.url } }, + // A small model wanders; keep it on a short leash so a bad run ends + // in a failed assertion rather than a burned CI minute budget. + maxIterations: 3, + maxStepsPerTask: 4, + llmTimeoutMs: 180_000, + logLevel: 'none', + }, + (event) => { + if (event.type === 'log') logs.push(event.message) + }, + ) + + try { + const result = await agent.run({ + input: 'Use the echo tool to echo the word "hello", then tell me what it returned.', + }) + + assert.ok(result.text.trim().length > 0, 'the run produced a final answer') + assert.ok(result.plan.steps.length > 0, 'the model produced a parseable plan') + assert.ok( + mcp.calls.some((c) => c.name === 'echo'), + `the model drove the MCP tool (calls: ${JSON.stringify(mcp.calls)})`, + ) + const echo = mcp.calls.find((c) => c.name === 'echo') + assert.equal( + typeof (echo?.args as { text?: unknown }).text, + 'string', + 'the tool arguments parsed into the declared schema', + ) + } finally { + await agent.close() + } + } finally { + await mcp.close() + } + }, +) From 27385cb472e22b5b0415a1716fa6e99244d5f097 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 16 Aug 2026 18:56:04 +0000 Subject: [PATCH 2/2] test(live-model): raise the model floor to 3B and assert the tool result MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 1.5B never drove the tool: it answered from imagination, once reporting a tool's own NAME as the secret. Measured on the same runtime (llama-server --jinja, greedy), a Qwen2.5-3B calls the tool and returns the real value, so the job now pins 3B and blocks the release instead of warning past it. - helpers/mcp-server: add a `secret` tool returning a per-run random value. A task that asks for it cannot be answered without the tool, which turns "did the model actually use it?" into a factual assertion. - live-model.test: ask a QUESTION rather than give an order — the planner's canned "Answer the user directly" step is attractive to small models and wins against imperative phrasing even on a 3B. Assert the secret appears in the final text, pinning the whole round trip through the tool result. - live-model.yml: pin a verified llama.cpp image tag, drop continue-on-error. - integration.test: tool-count expectations follow the new helper tool. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_011FKop4At26QqqkwVGEjJur --- .github/workflows/live-model.yml | 73 +++++++++++++------------------- CLAUDE.md | 2 +- tests/helpers/mcp-server.ts | 20 +++++++++ tests/integration.test.ts | 6 +-- tests/live-model.test.ts | 27 ++++++++---- 5 files changed, 72 insertions(+), 56 deletions(-) diff --git a/.github/workflows/live-model.yml b/.github/workflows/live-model.yml index 797387e..3122004 100644 --- a/.github/workflows/live-model.yml +++ b/.github/workflows/live-model.yml @@ -21,32 +21,22 @@ on: - 'package.json' - 'package-lock.json' -# Pinned so a rerun downloads nothing and behaves the same. Bump deliberately. +# Pinned so a rerun behaves the same. Bump deliberately. +# +# 3B is the floor, measured rather than guessed: on this exact runtime a +# Qwen2.5-1.5B never called the tool — it answered from imagination, once +# reporting a tool's NAME as the secret — while the 3B called it and returned +# the real value. See the note on the test's phrasing in live-model.test.ts. env: - LLAMA_TAG: b4785 - MODEL_REPO: Qwen/Qwen2.5-1.5B-Instruct-GGUF - MODEL_FILE: qwen2.5-1.5b-instruct-q4_k_m.gguf + LLAMA_IMAGE: ghcr.io/ggml-org/llama.cpp:server-b5350 + MODEL_REPO: Qwen/Qwen2.5-3B-Instruct-GGUF + MODEL_FILE: qwen2.5-3b-instruct-q4_k_m.gguf jobs: smoke: name: Real model smoke runs-on: ubuntu-latest timeout-minutes: 25 - # ADVISORY UNTIL PROVEN, then flip this to false to make it a real gate. - # - # The evidence for the default model is incomplete: Qwen2.5-1.5B served by - # llama-cpp-python's `chatml-function-calling` shim produced no tool calls - # at all — not through the agent, and not through a hand-rolled request - # with a single trivial tool. That shim is NOT what runs here (llama-server - # --jinja uses the model's own template, which does declare tool support), - # so the result does not transfer — but it does mean nobody has yet watched - # this combination drive a tool end to end. - # - # Blocking a publish on an unproven assertion is the wrong default. Let it - # report for a few releases; if it stays green, make it blocking. If it - # stays red, the fix is a larger model (3B+) or a narrower assertion, not - # a disabled job. - continue-on-error: true permissions: contents: read steps: @@ -62,43 +52,38 @@ jobs: - name: Install dependencies run: npm ci - - name: Cache llama.cpp and the model weights + - name: Cache the model weights id: cache uses: actions/cache@v4 with: path: .live-model - key: live-model-${{ env.LLAMA_TAG }}-${{ env.MODEL_FILE }} + key: live-model-${{ env.MODEL_FILE }} - - name: Fetch llama.cpp and the model + - name: Fetch the model if: steps.cache.outputs.cache-hit != 'true' run: | set -euo pipefail mkdir -p .live-model - cd .live-model - asset=$(gh api "repos/ggml-org/llama.cpp/releases/tags/$LLAMA_TAG" \ - --jq '.assets[] | select(.name | test("bin-ubuntu-x64")) | .name' | head -1) - echo "::notice::llama.cpp asset $asset" - gh release download "$LLAMA_TAG" --repo ggml-org/llama.cpp --pattern "$asset" - unzip -q -o "$asset" - curl -fsSL -o "$MODEL_FILE" \ + curl -fsSL -o ".live-model/$MODEL_FILE" \ "https://huggingface.co/$MODEL_REPO/resolve/main/$MODEL_FILE?download=true" - ls -la - env: - GH_TOKEN: ${{ github.token }} + ls -la .live-model - name: Start the model server run: | set -euo pipefail - server=$(find .live-model -name llama-server -type f | head -1) - chmod +x "$server" - # Greedy sampling with a fixed seed: the run has to be reproducible - # enough that a failure means the agent broke, not that the model - # rolled differently. --jinja turns on the chat template's tool - # calling, which is the whole point of the exercise. - "$server" -m ".live-model/$MODEL_FILE" \ + # The official image rather than a release asset: asset names carry + # the build number and change shape between releases, so a pinned + # image tag is the thing that can actually be verified. + # + # Greedy sampling with a fixed seed, so a failure means the agent + # broke rather than the dice rolling differently. --jinja turns on + # the chat template's tool calling, which is the whole point. + docker run -d --name llama --network host \ + -v "$PWD/.live-model:/models:ro" \ + "$LLAMA_IMAGE" \ + -m "/models/$MODEL_FILE" \ --host 127.0.0.1 --port 8080 \ - --ctx-size 8192 --temp 0 --seed 42 --jinja --no-warmup \ - > llama-server.log 2>&1 & + --ctx-size 8192 --temp 0 --seed 42 --jinja for _ in $(seq 1 90); do if curl -sf http://127.0.0.1:8080/health >/dev/null; then echo "model server is up" @@ -107,7 +92,7 @@ jobs: sleep 2 done echo "::error::the model server never became healthy" - tail -50 llama-server.log + docker logs llama | tail -50 exit 1 - name: Run the live-model smoke test @@ -128,8 +113,8 @@ jobs: exit 1 env: AGENT_LIVE_MODEL_URL: http://127.0.0.1:8080/v1 - AGENT_LIVE_MODEL: qwen2.5-1.5b-instruct + AGENT_LIVE_MODEL: qwen2.5-3b-instruct - name: Model server log if: failure() - run: tail -100 llama-server.log + run: docker logs llama | tail -100 diff --git a/CLAUDE.md b/CLAUDE.md index 492cce2..8e9b2df 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -61,7 +61,7 @@ This repo follows the unified `autoupdate-with-claude` baseline (same template a - `autoupdate.yml` uses `GITHUB_TOKEN` and explicitly dispatches `test.yml` (the `CI` workflow) after PR creation, because events created via `GITHUB_TOKEN` don't trigger `pull_request` workflows. - `autoupdate.yml` dispatches `claude.yml` directly via `workflow_dispatch` instead of relying on an `@claude` PR comment. - Releases stay wired through `release.yml` (npm Trusted Publisher), which fires via `workflow_run` after a successful CI on `main`. There is no `release-on-version-bump.yml` here — it would conflict with the existing tag/publish chain. -- `release.yml` now `needs:` the `live-model.yml` workflow: a smoke test against a real model (llama.cpp on the runner, no key) gates the publish. It also runs on PRs that touch `package.json` / `package-lock.json`, which is where dependency drift arrives. Deliberately NOT nightly — a 3am failure on yesterday's commit gets ignored, and the question it answers matters at publish time. `LLAMA_TAG` / `MODEL_*` are pinned so a rerun downloads nothing and behaves the same; bump them on purpose. +- `release.yml` now `needs:` the `live-model.yml` workflow: a smoke test against a real model (llama.cpp on the runner, no key) gates the publish. It also runs on PRs that touch `package.json` / `package-lock.json`, which is where dependency drift arrives. Deliberately NOT nightly — a 3am failure on yesterday's commit gets ignored, and the question it answers matters at publish time. `LLAMA_IMAGE` / `MODEL_*` are pinned so a rerun downloads nothing and behaves the same; bump them on purpose. 3B is the measured floor — a 1.5B answers from imagination instead of calling the tool. - All actions pinned to the `@v4` line because the runner image currently lacks `externals/node24`, breaking post-cleanup of `@v5/@v6` actions. Do **not** "fix" any of the above by replacing dispatch calls with comment-based mentions, or by bumping action versions back to `@v5/@v6`. diff --git a/tests/helpers/mcp-server.ts b/tests/helpers/mcp-server.ts index 841881e..4bb148d 100644 --- a/tests/helpers/mcp-server.ts +++ b/tests/helpers/mcp-server.ts @@ -23,6 +23,13 @@ export interface ILocalMcpOptions { export interface ILocalMcp { /** The MCP endpoint to hand to `mcpServers[name].url`. */ url: string + /** + * A value only this server knows, returned by the `secret` tool. A task that + * asks for it cannot be answered from the model's own knowledge, so "did the + * model actually use the tool?" has a factual answer rather than a stylistic + * one. + */ + secret: string /** Arguments every tool call received, in order. */ calls: { name: string; args: unknown }[] /** Tokens the authorization server currently accepts. */ @@ -65,6 +72,7 @@ const close = (server: Server): Promise => */ export const startLocalMcp = async (opts: ILocalMcpOptions = {}): Promise => { const calls: { name: string; args: unknown }[] = [] + const secret = `zq-${Math.random().toString(36).slice(2, 8)}` const validTokens = new Set() const grants: string[] = [] let registrations = 0 @@ -134,6 +142,17 @@ export const startLocalMcp = async (opts: ILocalMcpOptions = {}): Promise { + calls.push({ name: 'secret', args: {} }) + return { content: [{ type: 'text' as const, text: secret }] } + }, + ) server.registerTool('boom', { description: 'Always fails', inputSchema: {} }, () => { calls.push({ name: 'boom', args: {} }) return { content: [{ type: 'text' as const, text: 'kaboom' }], isError: true } @@ -182,6 +201,7 @@ export const startLocalMcp = async (opts: ILocalMcpOptions = {}): Promise registrations, diff --git a/tests/integration.test.ts b/tests/integration.test.ts index 384104d..dfb7e2f 100644 --- a/tests/integration.test.ts +++ b/tests/integration.test.ts @@ -119,7 +119,7 @@ test('plan → execute → replan → synthesize, driving a real MCP server over .listTools() .map((t) => t.name) .sort(), - ['files__boom', 'files__echo'], + ['files__boom', 'files__echo', 'files__secret'], 'the tool list came from the live server', ) @@ -209,7 +209,7 @@ test('OAuth: authorize once, then run against the protected server', async () => }) const agent = await open(baseConfig(llm.baseURL, mcp.url, { authProvider: provider })) - assert.equal(agent.listTools().length, 2, 'the tools arrived once authorized') + assert.equal(agent.listTools().length, 3, 'the tools arrived once authorized') const result = await agent.run({ input: 'Echo "hello" for me' }) assert.equal(result.text, 'Echoed: hello') assert.deepEqual(mcp.calls, [{ name: 'echo', args: { text: 'hello' } }]) @@ -234,7 +234,7 @@ test('OAuth: a token that dies mid-run is refreshed and the tool call retried', state: prompts[0].searchParams.get('state') ?? undefined, }) const agent = await open(baseConfig(llm.baseURL, mcp.url, { authProvider: provider })) - assert.equal(agent.listTools().length, 2) + assert.equal(agent.listTools().length, 3) // The access token stops working after the connection is up — the case a // static Authorization header can never recover from. diff --git a/tests/live-model.test.ts b/tests/live-model.test.ts index 8fab74c..1ce9d0f 100644 --- a/tests/live-model.test.ts +++ b/tests/live-model.test.ts @@ -52,22 +52,33 @@ test( ) try { + // Two things were measured to arrive at this phrasing. + // + // The task must be unanswerable without the tool: asked to "echo + // hello", the planner takes its answer-directly branch and the run + // ends with a plausible sentence and no tool call. Asking for a value + // only the server knows removes that shortcut. + // + // And it must read as a QUESTION, not an order. "Call the secret tool + // and report the code" still lost to the answer-directly branch on a + // 3B, which then invented a code; "What is the secret code?" matches + // the planner's own rule about information the tools can retrieve, and + // the tool gets called. Worth knowing: the planner's canned + // "Answer the user directly" step is attractive to small models. const result = await agent.run({ - input: 'Use the echo tool to echo the word "hello", then tell me what it returned.', + input: 'What is the server-side secret code? Report it exactly.', }) assert.ok(result.text.trim().length > 0, 'the run produced a final answer') assert.ok(result.plan.steps.length > 0, 'the model produced a parseable plan') assert.ok( - mcp.calls.some((c) => c.name === 'echo'), + mcp.calls.some((c) => c.name === 'secret'), `the model drove the MCP tool (calls: ${JSON.stringify(mcp.calls)})`, ) - const echo = mcp.calls.find((c) => c.name === 'echo') - assert.equal( - typeof (echo?.args as { text?: unknown }).text, - 'string', - 'the tool arguments parsed into the declared schema', - ) + // The code could only come down the tool-result path, so this pins the + // whole round trip: dispatch, MCP response, and the result reaching the + // model's final answer. + assert.match(result.text, new RegExp(mcp.secret)) } finally { await agent.close() }