diff --git a/.github/workflows/live-model.yml b/.github/workflows/live-model.yml new file mode 100644 index 0000000..3122004 --- /dev/null +++ b/.github/workflows/live-model.yml @@ -0,0 +1,120 @@ +name: Live model + +# A smoke test against a REAL model, served locally by llama.cpp on the runner. +# No hosted API and no key: the point is to catch the failures a scripted model +# cannot show — a provider changing its wire format, a prompt drifting out of +# what small models can follow, structured output no longer parsing. +# +# It gates the release rather than running nightly. A nightly run reports at +# 3am on yesterday's commit and nobody reads it; the question this answers — +# "does the agent still work with an actual model?" — matters at the moment +# something is about to be published. It also runs on dependency PRs, which is +# where upstream drift actually arrives. +on: + workflow_call: + workflow_dispatch: + pull_request: + branches: [main] + # Dependency bumps (including the autoupdate flow's) are the realistic + # source of drift; ordinary source PRs are covered by the scripted suite. + paths: + - 'package.json' + - 'package-lock.json' + +# Pinned so a rerun behaves the same. Bump deliberately. +# +# 3B is the floor, measured rather than guessed: on this exact runtime a +# Qwen2.5-1.5B never called the tool — it answered from imagination, once +# reporting a tool's NAME as the secret — while the 3B called it and returned +# the real value. See the note on the test's phrasing in live-model.test.ts. +env: + LLAMA_IMAGE: ghcr.io/ggml-org/llama.cpp:server-b5350 + MODEL_REPO: Qwen/Qwen2.5-3B-Instruct-GGUF + MODEL_FILE: qwen2.5-3b-instruct-q4_k_m.gguf + +jobs: + smoke: + name: Real model smoke + runs-on: ubuntu-latest + timeout-minutes: 25 + permissions: + contents: read + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '22' + cache: npm + + - name: Install dependencies + run: npm ci + + - name: Cache the model weights + id: cache + uses: actions/cache@v4 + with: + path: .live-model + key: live-model-${{ env.MODEL_FILE }} + + - name: Fetch the model + if: steps.cache.outputs.cache-hit != 'true' + run: | + set -euo pipefail + mkdir -p .live-model + curl -fsSL -o ".live-model/$MODEL_FILE" \ + "https://huggingface.co/$MODEL_REPO/resolve/main/$MODEL_FILE?download=true" + ls -la .live-model + + - name: Start the model server + run: | + set -euo pipefail + # The official image rather than a release asset: asset names carry + # the build number and change shape between releases, so a pinned + # image tag is the thing that can actually be verified. + # + # Greedy sampling with a fixed seed, so a failure means the agent + # broke rather than the dice rolling differently. --jinja turns on + # the chat template's tool calling, which is the whole point. + docker run -d --name llama --network host \ + -v "$PWD/.live-model:/models:ro" \ + "$LLAMA_IMAGE" \ + -m "/models/$MODEL_FILE" \ + --host 127.0.0.1 --port 8080 \ + --ctx-size 8192 --temp 0 --seed 42 --jinja + for _ in $(seq 1 90); do + if curl -sf http://127.0.0.1:8080/health >/dev/null; then + echo "model server is up" + exit 0 + fi + sleep 2 + done + echo "::error::the model server never became healthy" + docker logs llama | tail -50 + exit 1 + + - name: Run the live-model smoke test + # One retry: a small model occasionally wanders, and a single stray run + # must not block a publish. A second failure is a real signal. + run: | + set -uo pipefail + for attempt in 1 2; do + echo "::group::attempt $attempt" + if node --test --experimental-strip-types --disable-warning=ExperimentalWarning \ + tests/live-model.test.ts; then + echo "::endgroup::" + exit 0 + fi + echo "::endgroup::" + echo "::warning::attempt $attempt failed" + done + exit 1 + env: + AGENT_LIVE_MODEL_URL: http://127.0.0.1:8080/v1 + AGENT_LIVE_MODEL: qwen2.5-3b-instruct + + - name: Model server log + if: failure() + run: docker logs llama | tail -100 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1e98a44..a98cc9b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -20,9 +20,19 @@ permissions: contents: write id-token: write jobs: + # Gate the publish on a run against a REAL model, served locally on the + # runner (no key, no hosted API). CI proves the code works against a scripted + # model; this proves it still works against one that has opinions -- and the + # moment before publishing is when that question is worth asking. A skipped + # or failed smoke leaves the package unpublished rather than shipping blind. + live-model: + name: Live model + uses: ./.github/workflows/live-model.yml + release: name: Publish to npm runs-on: ubuntu-latest + needs: live-model if: >- github.event_name == 'workflow_dispatch' || (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success') diff --git a/CLAUDE.md b/CLAUDE.md index beb19a8..8e9b2df 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -10,6 +10,17 @@ - `tests/` — `node:test` suites, run via `node --experimental-strip-types` - `dist/` — `tsup` build output (do not edit) +## Test layers + +- `tests/*.test.ts` — units plus `integration.test.ts`, which runs the whole + loop over real sockets against a scripted OpenAI-compatible endpoint, a real + MCP server and a real authorization server (`tests/helpers/`). No key, ~5s. +- `tests/live-model.test.ts` — the same loop against a REAL model. Skipped + unless `AGENT_LIVE_MODEL_URL` points at an OpenAI-compatible endpoint; CI + starts one via `live-model.yml`. Asserts mechanics only (the loop finished, a + tool was driven, arguments parsed) — never wording or plan shape, or the + suite becomes a coin flip nobody trusts. + ## Mandatory checks After **any** change to `.ts` files in `src/` or `tests/`, run: @@ -50,6 +61,7 @@ This repo follows the unified `autoupdate-with-claude` baseline (same template a - `autoupdate.yml` uses `GITHUB_TOKEN` and explicitly dispatches `test.yml` (the `CI` workflow) after PR creation, because events created via `GITHUB_TOKEN` don't trigger `pull_request` workflows. - `autoupdate.yml` dispatches `claude.yml` directly via `workflow_dispatch` instead of relying on an `@claude` PR comment. - Releases stay wired through `release.yml` (npm Trusted Publisher), which fires via `workflow_run` after a successful CI on `main`. There is no `release-on-version-bump.yml` here — it would conflict with the existing tag/publish chain. +- `release.yml` now `needs:` the `live-model.yml` workflow: a smoke test against a real model (llama.cpp on the runner, no key) gates the publish. It also runs on PRs that touch `package.json` / `package-lock.json`, which is where dependency drift arrives. Deliberately NOT nightly — a 3am failure on yesterday's commit gets ignored, and the question it answers matters at publish time. `LLAMA_IMAGE` / `MODEL_*` are pinned so a rerun downloads nothing and behaves the same; bump them on purpose. 3B is the measured floor — a 1.5B answers from imagination instead of calling the tool. - All actions pinned to the `@v4` line because the runner image currently lacks `externals/node24`, breaking post-cleanup of `@v5/@v6` actions. Do **not** "fix" any of the above by replacing dispatch calls with comment-based mentions, or by bumping action versions back to `@v5/@v6`. diff --git a/README.md b/README.md index 5296d36..91b7a9d 100644 --- a/README.md +++ b/README.md @@ -298,6 +298,27 @@ npm run format # prettier --write npm run format:check # prettier --check ``` +## Testing against a real model + +`npm test` never needs a key: the integration suite drives the whole loop over +loopback against a scripted OpenAI-compatible endpoint, a real MCP server and a +real authorization server. + +To run the same loop against an actual model, point it at any OpenAI-compatible +endpoint — `llama-server`, Ollama, vLLM, anything: + +```bash +llama-server -m qwen2.5-1.5b-instruct-q4_k_m.gguf --port 8080 --temp 0 --jinja + +AGENT_LIVE_MODEL_URL=http://127.0.0.1:8080/v1 \ + node --test --experimental-strip-types tests/live-model.test.ts +``` + +CI runs this as a **gate on publishing** (`live-model.yml`, wired into +`release.yml`), and on any PR that touches the dependency manifests. It asserts +mechanics only — the loop finished, the model drove the MCP tool, the arguments +parsed — never the wording, which would make it a coin flip. + ## Behavior notes & limitations - **Module formats.** ESM is the primary target; the CJS build (`dist/index.cjs`) is best-effort and depends on upstream deps (`ai`, `@ai-sdk/*`, `@modelcontextprotocol/sdk`) keeping their CJS fallbacks. If they go pure-ESM, CJS will break — the dual-format guard in [`tests/dist-loadable.test.ts`](./tests/dist-loadable.test.ts) catches the regression on the next build. diff --git a/tests/helpers/mcp-server.ts b/tests/helpers/mcp-server.ts index 841881e..4bb148d 100644 --- a/tests/helpers/mcp-server.ts +++ b/tests/helpers/mcp-server.ts @@ -23,6 +23,13 @@ export interface ILocalMcpOptions { export interface ILocalMcp { /** The MCP endpoint to hand to `mcpServers[name].url`. */ url: string + /** + * A value only this server knows, returned by the `secret` tool. A task that + * asks for it cannot be answered from the model's own knowledge, so "did the + * model actually use the tool?" has a factual answer rather than a stylistic + * one. + */ + secret: string /** Arguments every tool call received, in order. */ calls: { name: string; args: unknown }[] /** Tokens the authorization server currently accepts. */ @@ -65,6 +72,7 @@ const close = (server: Server): Promise => */ export const startLocalMcp = async (opts: ILocalMcpOptions = {}): Promise => { const calls: { name: string; args: unknown }[] = [] + const secret = `zq-${Math.random().toString(36).slice(2, 8)}` const validTokens = new Set() const grants: string[] = [] let registrations = 0 @@ -134,6 +142,17 @@ export const startLocalMcp = async (opts: ILocalMcpOptions = {}): Promise { + calls.push({ name: 'secret', args: {} }) + return { content: [{ type: 'text' as const, text: secret }] } + }, + ) server.registerTool('boom', { description: 'Always fails', inputSchema: {} }, () => { calls.push({ name: 'boom', args: {} }) return { content: [{ type: 'text' as const, text: 'kaboom' }], isError: true } @@ -182,6 +201,7 @@ export const startLocalMcp = async (opts: ILocalMcpOptions = {}): Promise registrations, diff --git a/tests/integration.test.ts b/tests/integration.test.ts index 384104d..dfb7e2f 100644 --- a/tests/integration.test.ts +++ b/tests/integration.test.ts @@ -119,7 +119,7 @@ test('plan → execute → replan → synthesize, driving a real MCP server over .listTools() .map((t) => t.name) .sort(), - ['files__boom', 'files__echo'], + ['files__boom', 'files__echo', 'files__secret'], 'the tool list came from the live server', ) @@ -209,7 +209,7 @@ test('OAuth: authorize once, then run against the protected server', async () => }) const agent = await open(baseConfig(llm.baseURL, mcp.url, { authProvider: provider })) - assert.equal(agent.listTools().length, 2, 'the tools arrived once authorized') + assert.equal(agent.listTools().length, 3, 'the tools arrived once authorized') const result = await agent.run({ input: 'Echo "hello" for me' }) assert.equal(result.text, 'Echoed: hello') assert.deepEqual(mcp.calls, [{ name: 'echo', args: { text: 'hello' } }]) @@ -234,7 +234,7 @@ test('OAuth: a token that dies mid-run is refreshed and the tool call retried', state: prompts[0].searchParams.get('state') ?? undefined, }) const agent = await open(baseConfig(llm.baseURL, mcp.url, { authProvider: provider })) - assert.equal(agent.listTools().length, 2) + assert.equal(agent.listTools().length, 3) // The access token stops working after the connection is up — the case a // static Authorization header can never recover from. diff --git a/tests/live-model.test.ts b/tests/live-model.test.ts new file mode 100644 index 0000000..1ce9d0f --- /dev/null +++ b/tests/live-model.test.ts @@ -0,0 +1,89 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { createAgent } from '../src/index.ts' +import { startLocalMcp } from './helpers/mcp-server.ts' + +/** + * The same loop as `integration.test.ts`, but driven by a REAL model behind an + * OpenAI-compatible endpoint instead of a scripted one. + * + * Skipped unless `AGENT_LIVE_MODEL_URL` points at a server (CI starts a small + * local model; locally, `llama-server`, Ollama or anything else with a /v1 + * endpoint will do). No hosted API and no key is involved. + * + * ── WHAT THIS MAY AND MAY NOT ASSERT ── + * Only mechanics: the loop reached a final answer, and the model actually + * drove the MCP tool with arguments that parsed. Asserting the wording, the + * plan's shape or the tool's arguments would be asserting the model's + * judgement, which changes with every weight and every sampler — that is how + * a suite ends up disabled. What is being pinned here is that a real model's + * output survives the whole path: structured-output parsing, tool dispatch + * over MCP, and the loop's exit conditions. + */ + +const LIVE_URL = process.env.AGENT_LIVE_MODEL_URL +const LIVE_MODEL = process.env.AGENT_LIVE_MODEL ?? 'local' + +test( + 'a real local model plans, calls an MCP tool, and finishes', + { skip: LIVE_URL ? false : 'set AGENT_LIVE_MODEL_URL to run', timeout: 600_000 }, + async () => { + const mcp = await startLocalMcp() + const logs: string[] = [] + try { + const agent = await createAgent( + { + clientName: 'agent-live-model-test', + providerType: 'openai-compatible', + baseURL: LIVE_URL!, + apiKey: 'not-a-real-key', + model: LIVE_MODEL, + mcpServers: { files: { url: mcp.url } }, + // A small model wanders; keep it on a short leash so a bad run ends + // in a failed assertion rather than a burned CI minute budget. + maxIterations: 3, + maxStepsPerTask: 4, + llmTimeoutMs: 180_000, + logLevel: 'none', + }, + (event) => { + if (event.type === 'log') logs.push(event.message) + }, + ) + + try { + // Two things were measured to arrive at this phrasing. + // + // The task must be unanswerable without the tool: asked to "echo + // hello", the planner takes its answer-directly branch and the run + // ends with a plausible sentence and no tool call. Asking for a value + // only the server knows removes that shortcut. + // + // And it must read as a QUESTION, not an order. "Call the secret tool + // and report the code" still lost to the answer-directly branch on a + // 3B, which then invented a code; "What is the secret code?" matches + // the planner's own rule about information the tools can retrieve, and + // the tool gets called. Worth knowing: the planner's canned + // "Answer the user directly" step is attractive to small models. + const result = await agent.run({ + input: 'What is the server-side secret code? Report it exactly.', + }) + + assert.ok(result.text.trim().length > 0, 'the run produced a final answer') + assert.ok(result.plan.steps.length > 0, 'the model produced a parseable plan') + assert.ok( + mcp.calls.some((c) => c.name === 'secret'), + `the model drove the MCP tool (calls: ${JSON.stringify(mcp.calls)})`, + ) + // The code could only come down the tool-result path, so this pins the + // whole round trip: dispatch, MCP response, and the result reaching the + // model's final answer. + assert.match(result.text, new RegExp(mcp.secret)) + } finally { + await agent.close() + } + } finally { + await mcp.close() + } + }, +)