test: smoke the agent against a real local model, gating the publish

.github/workflows/live-model.yml

@@ -0,0 +1,120 @@
+name: Live model
+
+# A smoke test against a REAL model, served locally by llama.cpp on the runner.
+# No hosted API and no key: the point is to catch the failures a scripted model
+# cannot show — a provider changing its wire format, a prompt drifting out of
+# what small models can follow, structured output no longer parsing.
+#
+# It gates the release rather than running nightly. A nightly run reports at
+# 3am on yesterday's commit and nobody reads it; the question this answers —
+# "does the agent still work with an actual model?" — matters at the moment
+# something is about to be published. It also runs on dependency PRs, which is
+# where upstream drift actually arrives.
+on:
+  workflow_call:
+  workflow_dispatch:
+  pull_request:
+    branches: [main]
+    # Dependency bumps (including the autoupdate flow's) are the realistic
+    # source of drift; ordinary source PRs are covered by the scripted suite.
+    paths:
+      - 'package.json'
+      - 'package-lock.json'
+
+# Pinned so a rerun behaves the same. Bump deliberately.
+#
+# 3B is the floor, measured rather than guessed: on this exact runtime a
+# Qwen2.5-1.5B never called the tool — it answered from imagination, once
+# reporting a tool's NAME as the secret — while the 3B called it and returned
+# the real value. See the note on the test's phrasing in live-model.test.ts.
+env:
+  LLAMA_IMAGE: ghcr.io/ggml-org/llama.cpp:server-b5350
+  MODEL_REPO: Qwen/Qwen2.5-3B-Instruct-GGUF
+  MODEL_FILE: qwen2.5-3b-instruct-q4_k_m.gguf
+
+jobs:
+  smoke:
+    name: Real model smoke
+    runs-on: ubuntu-latest
+    timeout-minutes: 25
+    permissions:
+      contents: read
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '22'
+          cache: npm
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Cache the model weights
+        id: cache
+        uses: actions/cache@v4
+        with:
+          path: .live-model
+          key: live-model-${{ env.MODEL_FILE }}
+
+      - name: Fetch the model
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: |
+          set -euo pipefail
+          mkdir -p .live-model
+          curl -fsSL -o ".live-model/$MODEL_FILE" \
+            "https://huggingface.co/$MODEL_REPO/resolve/main/$MODEL_FILE?download=true"
+          ls -la .live-model
+
+      - name: Start the model server
+        run: |
+          set -euo pipefail
+          # The official image rather than a release asset: asset names carry
+          # the build number and change shape between releases, so a pinned
+          # image tag is the thing that can actually be verified.
+          #
+          # Greedy sampling with a fixed seed, so a failure means the agent
+          # broke rather than the dice rolling differently. --jinja turns on
+          # the chat template's tool calling, which is the whole point.
+          docker run -d --name llama --network host \
+            -v "$PWD/.live-model:/models:ro" \
+            "$LLAMA_IMAGE" \
+            -m "/models/$MODEL_FILE" \
+            --host 127.0.0.1 --port 8080 \
+            --ctx-size 8192 --temp 0 --seed 42 --jinja
+          for _ in $(seq 1 90); do
+            if curl -sf http://127.0.0.1:8080/health >/dev/null; then
+              echo "model server is up"
+              exit 0
+            fi
+            sleep 2
+          done
+          echo "::error::the model server never became healthy"
+          docker logs llama | tail -50
+          exit 1
+
+      - name: Run the live-model smoke test
+        # One retry: a small model occasionally wanders, and a single stray run
+        # must not block a publish. A second failure is a real signal.
+        run: |
+          set -uo pipefail
+          for attempt in 1 2; do
+            echo "::group::attempt $attempt"
+            if node --test --experimental-strip-types --disable-warning=ExperimentalWarning \
+                 tests/live-model.test.ts; then
+              echo "::endgroup::"
+              exit 0
+            fi
+            echo "::endgroup::"
+            echo "::warning::attempt $attempt failed"
+          done
+          exit 1
+        env:
+          AGENT_LIVE_MODEL_URL: http://127.0.0.1:8080/v1
+          AGENT_LIVE_MODEL: qwen2.5-3b-instruct
+
+      - name: Model server log
+        if: failure()
+        run: docker logs llama | tail -100

.github/workflows/release.yml

@@ -20,9 +20,19 @@ permissions:
   contents: write
   id-token: write
 jobs:
+  # Gate the publish on a run against a REAL model, served locally on the
+  # runner (no key, no hosted API). CI proves the code works against a scripted
+  # model; this proves it still works against one that has opinions -- and the
+  # moment before publishing is when that question is worth asking. A skipped
+  # or failed smoke leaves the package unpublished rather than shipping blind.
+  live-model:
+    name: Live model
+    uses: ./.github/workflows/live-model.yml
+
   release:
     name: Publish to npm
     runs-on: ubuntu-latest
+    needs: live-model
     if: >-
       github.event_name == 'workflow_dispatch' ||
       (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success')

CLAUDE.md

@@ -10,6 +10,17 @@
 - `tests/` — `node:test` suites, run via `node --experimental-strip-types`
 - `dist/` — `tsup` build output (do not edit)
 
+## Test layers
+
+- `tests/*.test.ts` — units plus `integration.test.ts`, which runs the whole
+  loop over real sockets against a scripted OpenAI-compatible endpoint, a real
+  MCP server and a real authorization server (`tests/helpers/`). No key, ~5s.
+- `tests/live-model.test.ts` — the same loop against a REAL model. Skipped
+  unless `AGENT_LIVE_MODEL_URL` points at an OpenAI-compatible endpoint; CI
+  starts one via `live-model.yml`. Asserts mechanics only (the loop finished, a
+  tool was driven, arguments parsed) — never wording or plan shape, or the
+  suite becomes a coin flip nobody trusts.
+
 ## Mandatory checks
 
 After **any** change to `.ts` files in `src/` or `tests/`, run:
@@ -50,6 +61,7 @@ This repo follows the unified `autoupdate-with-claude` baseline (same template a
 - `autoupdate.yml` uses `GITHUB_TOKEN` and explicitly dispatches `test.yml` (the `CI` workflow) after PR creation, because events created via `GITHUB_TOKEN` don't trigger `pull_request` workflows.
 - `autoupdate.yml` dispatches `claude.yml` directly via `workflow_dispatch` instead of relying on an `@claude` PR comment.
 - Releases stay wired through `release.yml` (npm Trusted Publisher), which fires via `workflow_run` after a successful CI on `main`. There is no `release-on-version-bump.yml` here — it would conflict with the existing tag/publish chain.
+- `release.yml` now `needs:` the `live-model.yml` workflow: a smoke test against a real model (llama.cpp on the runner, no key) gates the publish. It also runs on PRs that touch `package.json` / `package-lock.json`, which is where dependency drift arrives. Deliberately NOT nightly — a 3am failure on yesterday's commit gets ignored, and the question it answers matters at publish time. `LLAMA_IMAGE` / `MODEL_*` are pinned so a rerun downloads nothing and behaves the same; bump them on purpose. 3B is the measured floor — a 1.5B answers from imagination instead of calling the tool.
 - All actions pinned to the `@v4` line because the runner image currently lacks `externals/node24`, breaking post-cleanup of `@v5/@v6` actions.
 
 Do **not** "fix" any of the above by replacing dispatch calls with comment-based mentions, or by bumping action versions back to `@v5/@v6`.

README.md

@@ -298,6 +298,27 @@ npm run format        # prettier --write
 npm run format:check  # prettier --check
 ```
 
+## Testing against a real model
+
+`npm test` never needs a key: the integration suite drives the whole loop over
+loopback against a scripted OpenAI-compatible endpoint, a real MCP server and a
+real authorization server.
+
+To run the same loop against an actual model, point it at any OpenAI-compatible
+endpoint — `llama-server`, Ollama, vLLM, anything:
+
+```bash
+llama-server -m qwen2.5-1.5b-instruct-q4_k_m.gguf --port 8080 --temp 0 --jinja
+
+AGENT_LIVE_MODEL_URL=http://127.0.0.1:8080/v1 \
+  node --test --experimental-strip-types tests/live-model.test.ts
+```
+
+CI runs this as a **gate on publishing** (`live-model.yml`, wired into
+`release.yml`), and on any PR that touches the dependency manifests. It asserts
+mechanics only — the loop finished, the model drove the MCP tool, the arguments
+parsed — never the wording, which would make it a coin flip.
+
 ## Behavior notes & limitations
 
 - **Module formats.** ESM is the primary target; the CJS build (`dist/index.cjs`) is best-effort and depends on upstream deps (`ai`, `@ai-sdk/*`, `@modelcontextprotocol/sdk`) keeping their CJS fallbacks. If they go pure-ESM, CJS will break — the dual-format guard in [`tests/dist-loadable.test.ts`](./tests/dist-loadable.test.ts) catches the regression on the next build.

tests/helpers/mcp-server.ts

@@ -23,6 +23,13 @@ export interface ILocalMcpOptions {
 export interface ILocalMcp {
   /** The MCP endpoint to hand to `mcpServers[name].url`. */
   url: string
+  /**
+   * A value only this server knows, returned by the `secret` tool. A task that
+   * asks for it cannot be answered from the model's own knowledge, so "did the
+   * model actually use the tool?" has a factual answer rather than a stylistic
+   * one.
+   */
+  secret: string
   /** Arguments every tool call received, in order. */
   calls: { name: string; args: unknown }[]
   /** Tokens the authorization server currently accepts. */
@@ -65,6 +72,7 @@ const close = (server: Server): Promise<void> =>
  */
 export const startLocalMcp = async (opts: ILocalMcpOptions = {}): Promise<ILocalMcp> => {
   const calls: { name: string; args: unknown }[] = []
+  const secret = `zq-${Math.random().toString(36).slice(2, 8)}`
   const validTokens = new Set<string>()
   const grants: string[] = []
   let registrations = 0
@@ -134,6 +142,17 @@ export const startLocalMcp = async (opts: ILocalMcpOptions = {}): Promise<ILocal
         return { content: [{ type: 'text' as const, text: `echo:${text}` }] }
       },
     )
+    server.registerTool(
+      'secret',
+      {
+        description: 'Return the server-side secret code. There is no other way to learn it.',
+        inputSchema: {},
+      },
+      () => {
+        calls.push({ name: 'secret', args: {} })
+        return { content: [{ type: 'text' as const, text: secret }] }
+      },
+    )
     server.registerTool('boom', { description: 'Always fails', inputSchema: {} }, () => {
       calls.push({ name: 'boom', args: {} })
       return { content: [{ type: 'text' as const, text: 'kaboom' }], isError: true }
@@ -182,6 +201,7 @@ export const startLocalMcp = async (opts: ILocalMcpOptions = {}): Promise<ILocal
 
   return {
     url: mcpUrl,
+    secret,
     calls,
     validTokens,
     registrations: () => registrations,

tests/integration.test.ts

@@ -119,7 +119,7 @@ test('plan → execute → replan → synthesize, driving a real MCP server over
         .listTools()
         .map((t) => t.name)
         .sort(),
-      ['files__boom', 'files__echo'],
+      ['files__boom', 'files__echo', 'files__secret'],
       'the tool list came from the live server',
     )
 
@@ -209,7 +209,7 @@ test('OAuth: authorize once, then run against the protected server', async () =>
     })
 
     const agent = await open(baseConfig(llm.baseURL, mcp.url, { authProvider: provider }))
-    assert.equal(agent.listTools().length, 2, 'the tools arrived once authorized')
+    assert.equal(agent.listTools().length, 3, 'the tools arrived once authorized')
     const result = await agent.run({ input: 'Echo "hello" for me' })
     assert.equal(result.text, 'Echoed: hello')
     assert.deepEqual(mcp.calls, [{ name: 'echo', args: { text: 'hello' } }])
@@ -234,7 +234,7 @@ test('OAuth: a token that dies mid-run is refreshed and the tool call retried',
       state: prompts[0].searchParams.get('state') ?? undefined,
     })
     const agent = await open(baseConfig(llm.baseURL, mcp.url, { authProvider: provider }))
-    assert.equal(agent.listTools().length, 2)
+    assert.equal(agent.listTools().length, 3)
 
     // The access token stops working after the connection is up — the case a
     // static Authorization header can never recover from.

tests/live-model.test.ts

@@ -0,0 +1,89 @@
+import assert from 'node:assert/strict'
+import test from 'node:test'
+import { createAgent } from '../src/index.ts'
+import { startLocalMcp } from './helpers/mcp-server.ts'
+
+/**
+ * The same loop as `integration.test.ts`, but driven by a REAL model behind an
+ * OpenAI-compatible endpoint instead of a scripted one.
+ *
+ * Skipped unless `AGENT_LIVE_MODEL_URL` points at a server (CI starts a small
+ * local model; locally, `llama-server`, Ollama or anything else with a /v1
+ * endpoint will do). No hosted API and no key is involved.
+ *
+ * ── WHAT THIS MAY AND MAY NOT ASSERT ──
+ * Only mechanics: the loop reached a final answer, and the model actually
+ * drove the MCP tool with arguments that parsed. Asserting the wording, the
+ * plan's shape or the tool's arguments would be asserting the model's
+ * judgement, which changes with every weight and every sampler — that is how
+ * a suite ends up disabled. What is being pinned here is that a real model's
+ * output survives the whole path: structured-output parsing, tool dispatch
+ * over MCP, and the loop's exit conditions.
+ */
+
+const LIVE_URL = process.env.AGENT_LIVE_MODEL_URL
+const LIVE_MODEL = process.env.AGENT_LIVE_MODEL ?? 'local'
+
+test(
+  'a real local model plans, calls an MCP tool, and finishes',
+  { skip: LIVE_URL ? false : 'set AGENT_LIVE_MODEL_URL to run', timeout: 600_000 },
+  async () => {
+    const mcp = await startLocalMcp()
+    const logs: string[] = []
+    try {
+      const agent = await createAgent(
+        {
+          clientName: 'agent-live-model-test',
+          providerType: 'openai-compatible',
+          baseURL: LIVE_URL!,
+          apiKey: 'not-a-real-key',
+          model: LIVE_MODEL,
+          mcpServers: { files: { url: mcp.url } },
+          // A small model wanders; keep it on a short leash so a bad run ends
+          // in a failed assertion rather than a burned CI minute budget.
+          maxIterations: 3,
+          maxStepsPerTask: 4,
+          llmTimeoutMs: 180_000,
+          logLevel: 'none',
+        },
+        (event) => {
+          if (event.type === 'log') logs.push(event.message)
+        },
+      )
+
+      try {
+        // Two things were measured to arrive at this phrasing.
+        //
+        // The task must be unanswerable without the tool: asked to "echo
+        // hello", the planner takes its answer-directly branch and the run
+        // ends with a plausible sentence and no tool call. Asking for a value
+        // only the server knows removes that shortcut.
+        //
+        // And it must read as a QUESTION, not an order. "Call the secret tool
+        // and report the code" still lost to the answer-directly branch on a
+        // 3B, which then invented a code; "What is the secret code?" matches
+        // the planner's own rule about information the tools can retrieve, and
+        // the tool gets called. Worth knowing: the planner's canned
+        // "Answer the user directly" step is attractive to small models.
+        const result = await agent.run({
+          input: 'What is the server-side secret code? Report it exactly.',
+        })
+
+        assert.ok(result.text.trim().length > 0, 'the run produced a final answer')
+        assert.ok(result.plan.steps.length > 0, 'the model produced a parseable plan')
+        assert.ok(
+          mcp.calls.some((c) => c.name === 'secret'),
+          `the model drove the MCP tool (calls: ${JSON.stringify(mcp.calls)})`,
+        )
+        // The code could only come down the tool-result path, so this pins the
+        // whole round trip: dispatch, MCP response, and the result reaching the
+        // model's final answer.
+        assert.match(result.text, new RegExp(mcp.secret))
+      } finally {
+        await agent.close()
+      }
+    } finally {
+      await mcp.close()
+    }
+  },
+)