diff --git a/CHANGELOG.md b/CHANGELOG.md index 024492d..5295589 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,35 @@ +## [0.16.0] - 2026-08-20 + +Minor release — backend v3.66.2 wire-validation alignment. **Behaviour change** for callers that invoke `track_llm()` / `track({"type": "llm_call", ...})` outside a paired `/check` scope. Wire-format unchanged. SDK_MIN_VERSION unchanged. + +### Changed + +- **`_route_track` no-smid branch drops llm_call events instead of falling back to /track/batch** — backend v3.66.2 closed the v1/v2 no-reservation consume path with per-event type-aware wire validation: any `llm_call` event in a batch WITHOUT `reservation_id` is rejected with 503 `BUDGET_RECHECK_FAILED` (whole-batch fail-CLOSED). The 0.12.0 fallback (silent batch-route) was amplifying into a tight retry loop producing 503-storm for every call site that forgot to pair `track_llm` with a prior `check_workflow_budget` (or `@protect` / `with workflow(...)`). Post-0.16.0 the no-smid branch: + 1. increments `metrics.runtime.dropped_llm_call_no_reservation` (new counter, exposed via `metrics.to_dict()["runtime"]["dropped_llm_call_no_reservation"]` for `/health` + operator dashboards), + 2. emits a WARNING log (not DEBUG — mirrors the 0.15.2 fail-OPEN observability fix) naming the `event_type` + `workflow_id` so operators can locate the offending call site, + 3. drops the event (no batch POST, no retry; the fix is upstream at the call site). +- **Source-pin regression tests updated to pin the corrected drop behaviour** — `tests/test_v3_wire_contract.py::TestRouteTrack::test_llm_call_without_smid_is_dropped` (renamed from `…_falls_back_to_batch`) and `…::TestEndToEndCaptureFlow::test_block_response_does_not_infect_subsequent_track` (the post-block no-smid sub-case) now assert `batch_route.call_count == 0` + drop-counter increment. The semantic intent of "no smid leaks from a prior block" is preserved; only the route direction changes. + +### Migration + +Operations hitting the new `dropped_llm_call_no_reservation` counter on `/health` are calling `track_llm()` (or `track({"type": "llm_call", ...})`) outside a paired `/check` scope. The fix is always at the call site — wrap the tracking call in one of: + +- `@protect(...)` decorator (wraps in `with workflow(...)` + `check_workflow_budget()` automatically), +- `check_workflow_budget()` before `track_llm()` (explicit two-step), +- `with workflow("wf-id"):` context manager + `check_workflow_budget()` inside. + +Bare `track_llm()` calls (no surrounding gate) silently drop the event post-0.16.0 — the call still returns its usual `{"allowed": True, ...}` dict, but no `cost_events` row is written. Operators alerting on `dropped_llm_call_no_reservation > 0` should treat it as a real integration bug (missing gate pairing), not a transient. + +### Why this is needed + +Backend v3.66.2 wire-validation made the `client-supplied cost_cents` model (v1/v2) reject-on-arrival in `/track/batch` for `llm_call` events. The 0.12.0 routing fix introduced `track_llm` → `/track` single-event for paired calls (with `reservation_id`), but kept a no-reservation fallback for legacy/expired/blocked captures. Three years of v1/v2 SDK versions shipped that no-reservation path; v3.66.2 closed it. The new SDK behaviour is honest about the gap: no smid → no authoritative budget enforcement → drop the event rather than synthesise a stale consume. + +### Compatibility + +**No SDK_MIN_VERSION bump.** Backend v3.66.2 ships since 2026-08-18 (commit `e262f1c3`). Wire-format unchanged. No public API change. Drop-in replacement for 0.15.2 for callers that always pair `track_llm` with a prior gate — those observe zero behaviour change. Callers that relied on bare `track_llm()` hitting `/track/batch` will see `dropped_llm_call_no_reservation` increment on the metrics endpoint and WARNING logs at the call site; the migration is the wrapping fix above. + +_Tests: 2 source-pin regression tests updated; both pin the new drop behaviour. No regressions in the other 1611 tests expected (the only test paths that hit the no-smid branch are the two updated above)._ + ## [0.15.2] - 2026-08-14 Patch release — observability closure + UI-UX-AUDIT 2026-08-14 fixes (F-19, F-28, F-29) + flaky-test removal. No public API change, no wire-format change, no SDK_MIN_VERSION bump. Drop-in replacement for 0.15.1. diff --git a/pyproject.toml b/pyproject.toml index 51e385a..34b94d1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "hatchling.build" name = "nullrun" # Full release history lives in CHANGELOG.md; only the current version # is pinned here. -version = "0.15.2" +version = "0.16.0" # Kept under the 200-char preview threshold so the full line is visible # without an "expand" click. The headline is the canonical §1 statement # from positioning.md — "runtime decision layer for tool-using AI agents" diff --git a/src/nullrun/observability/__init__.py b/src/nullrun/observability/__init__.py index 4257ad8..d2128bf 100644 --- a/src/nullrun/observability/__init__.py +++ b/src/nullrun/observability/__init__.py @@ -100,6 +100,15 @@ class RuntimeMetrics: # counter, the failure mode was invisible at INFO log level on # the FALLBACK path. gate_fail_open_total: int = 0 + # 2026-08-20 (v0.16.0, backend v3.66.2 alignment): v1/v2 path + # `/track/batch` with `llm_call` events missing `reservation_id` + # is now fail-CLOSED at the backend (whole-batch 503 BUDGET_RECHECK_FAILED). + # Pre-0.16.0 the SDK fell back to that path for calls without a + # paired /check — amplification into 503-storm. Now the SDK + # explicitly drops those events and increments this counter. + # Operators alert on sustained rate to detect integration bugs + # (missing gate pairing around `track_llm`). + dropped_llm_call_no_reservation: int = 0 class MetricsRegistry: diff --git a/src/nullrun/runtime.py b/src/nullrun/runtime.py index c02c07a..dff9743 100644 --- a/src/nullrun/runtime.py +++ b/src/nullrun/runtime.py @@ -3035,14 +3035,45 @@ def _route_track(self, wire_event: dict[str, Any]) -> None: smid = get_server_minted_execution_id() if not smid: - # Either no /check landed in this scope (legacy v1/v2 - # path) or the capture expired past the 295s safety - # window. Don't make up an id — fall back to batch - # which uses the no-reservation v1/v2 consume path. - self._transport.track(wire_event) - logger.debug( - "_route_track: llm_call without server-minted " - "execution_id in scope — routing via /track/batch" + # v0.16.0 (2026-08-20, backend v3.66.2 alignment): the + # 0.12.0 routing here used to fall back to /track/batch + # (the legacy v1/v2 no-reservation consume path). Backend + # v3.66.2 closed that path with per-event type-aware wire + # validation: any ``llm_call`` event in a batch WITHOUT + # ``reservation_id`` is rejected with 503 + # BUDGET_RECHECK_FAILED (whole-batch fail-CLOSED). Falling + # back here would amplify into a tight retry loop + # producing 503-storm for every call site that forgot to + # pair ``track_llm`` with a prior ``check_workflow_budget`` + # (or ``@protect`` / ``with workflow(...)``). + # + # Server-authoritative model (CLAUDE.md §22): an llm_call + # event without a paired /check reservation has no + # authoritative budget authority. Don't make up an id — + # drop the event explicitly so the operator sees the gap + # (WARNING + counter) instead of a silent batch loop. + # + # Trigger conditions: + # * no /check landed in this scope (legacy v1/v2 path) + # * capture expired past the 295s safety window + # * /check returned ``decision: "block"`` (no + # reservation_id minted on a hard block — see + # ``_capture_server_minted_execution_id``) + metrics.inc_runtime("dropped_llm_call_no_reservation") + # WARNING not DEBUG: matches the 0.15.2 fix that moved + # ``check_workflow_budget`` synthetic FALLBACK from DEBUG + # to WARNING (CHANGELOG 0.15.2). Operators should see this + # at INFO+ — a missing reservation pairing is a real + # integration bug, not a debug curiosity. + logger.warning( + "_route_track: dropping llm_call event — no " + "server-minted reservation_id in scope (no /check " + "paired, capture expired >295s, or /check returned " + "block). Wrap track_llm in @protect / " + "check_workflow_budget() / with workflow(...). " + "event_type=%s workflow_id=%s", + wire_event.get("type"), + wire_event.get("workflow_id"), ) return diff --git a/tests/contract/test_llm_call_model_wire.py b/tests/contract/test_llm_call_model_wire.py index f3e6482..4db84b1 100644 --- a/tests/contract/test_llm_call_model_wire.py +++ b/tests/contract/test_llm_call_model_wire.py @@ -160,11 +160,23 @@ def test_empty_string_in_llm_output_falls_through(): # can reject with HTTP 422. -def test_track_promotes_missing_model_to_error_and_tags_event(make_runtime, caplog): +def test_track_promotes_missing_model_to_error_and_tags_event( + make_runtime, caplog, monkeypatch +): """Regression: an ``llm_call`` event with ``model=None`` reaches ``track `` and (a) is logged at ERROR, (b) gets the ``__missing_model: True`` flag, (c) is still sent on the wire so the backend can reject with HTTP 422 (not silently free).""" + # v0.16.0 (backend v3.66.2 alignment): force the legacy batch + # route so the ``__missing_model`` flag actually reaches the + # captured buffer. Without the env var, the no-smid branch in + # ``_route_track`` would drop the event before the wire + # flag-and-log assertions can observe it. The flag/log logic + # in ``track`` is unchanged; the env var just opts out of the + # v3 routing's drop-on-no-smid so this test stays focused on + # the missing-model fail-loud surface. + monkeypatch.setenv("NULLRUN_V3_TRACK_DISABLE", "1") + rt = make_runtime() captured = [] @@ -195,9 +207,15 @@ def _capture_track(event): ) -def test_track_does_not_tag_when_model_is_set(make_runtime): +def test_track_does_not_tag_when_model_is_set(make_runtime, monkeypatch): """The happy path: ``llm_call`` event with a model passes through unchanged (no ERROR, no __missing_model flag).""" + # v0.16.0 (backend v3.66.2 alignment): same as the missing-model + # test above — opt out of the v3 routing's drop-on-no-smid so + # the captured buffer observes the wire event. The + # happy-path model field pass-through is the focus of this test. + monkeypatch.setenv("NULLRUN_V3_TRACK_DISABLE", "1") + rt = make_runtime() captured = [] rt._transport.track = lambda e: captured.append(e) diff --git a/tests/test_runtime.py b/tests/test_runtime.py index e8603a4..326019a 100644 --- a/tests/test_runtime.py +++ b/tests/test_runtime.py @@ -90,9 +90,17 @@ def test_wire_payload_strips_sensitive_fields(self, make_runtime): captured: list[dict] = [] rt._transport.track = lambda event: captured.append(dict(event)) + # v0.16.0 (backend v3.66.2 alignment): use a non-llm_call + # event type so the strip assertion exercises the batch path + # (the v3 single-event path would drop the event on no-smid + # before transport.track is observable — the strip is the + # same logic for both paths, but the batch path is the + # only one that reaches the captured buffer in this test). + # ``tool_call`` events always go through batch regardless of + # /check scope (no reservation to release). rt.track( { - "type": "llm_call", + "type": "tool_call", "provider": "openai", "model": "gpt-4o", "tokens": 15, @@ -101,6 +109,7 @@ def test_wire_payload_strips_sensitive_fields(self, make_runtime): "cache_read_tokens": 7, "finish_reason": "stop", "tool_names": ["search"], + "tool_name": "search", "has_usage": True, # These three MUST be stripped before the transport # buffer sees the event. @@ -124,7 +133,7 @@ def test_wire_payload_strips_sensitive_fields(self, make_runtime): assert "secret_routing_info" not in sent # Normalised fields pass through unchanged - assert sent["type"] == "llm_call" + assert sent["type"] == "tool_call" assert sent["input_tokens"] == 10 assert sent["cache_read_tokens"] == 7 assert sent["finish_reason"] == "stop" diff --git a/tests/test_v3_wire_contract.py b/tests/test_v3_wire_contract.py index 0fa55e0..e312da0 100644 --- a/tests/test_v3_wire_contract.py +++ b/tests/test_v3_wire_contract.py @@ -1583,7 +1583,12 @@ def test_minimal_event_only_required_fields(self): } def test_missing_workflow_id_returns_none(self): - # Caller falls back to /track/batch. + # v0.16.0 (backend v3.66.2 alignment): caller now DROPS + # llm_call events whose mapper refused (missing required + # field), not batch — backend v3.66.2 wire-validation would + # fail-CLOSE the batch anyway. Mapper returning None is the + # signal for the caller to drop instead of fabricating a + # malformed v3 payload. out = _build_v3_track_payload( {"type": "llm_call", "tokens": 1}, SERVER_MINTED_V1, @@ -1687,8 +1692,20 @@ def test_tool_call_routes_to_batch(self, make_runtime): assert batch_route.call_count == 1 @respx.mock - def test_llm_call_without_smid_falls_back_to_batch(self, make_runtime): - # No /check in scope → no smid → legacy path. + def test_llm_call_without_smid_is_dropped(self, make_runtime): + # v0.16.0 (backend v3.66.2 alignment): no /check in scope → + # no smid → DROP. Previously (0.12.0–0.15.2) this fell back + # to /track/batch (the legacy v1/v2 no-reservation consume + # path). Backend v3.66.2 closed that path with per-event + # type-aware wire validation: any ``llm_call`` event in a + # batch WITHOUT ``reservation_id`` is rejected with 503 + # BUDGET_RECHECK_FAILED (whole-batch fail-CLOSED). Falling + # back here would amplify into a tight retry loop producing + # 503-storm for every call site that forgot to pair + # ``track_llm`` with a prior ``check_workflow_budget`` (or + # ``@protect`` / ``with workflow(...)``). + from nullrun.observability import metrics + rt = make_runtime() single_route = respx.post(f"{BASE_URL}/api/v1/track").mock( @@ -1698,18 +1715,24 @@ def test_llm_call_without_smid_falls_back_to_batch(self, make_runtime): return_value=Response(200, json={"ok": True, "accepted": 1}) ) - # No capture call here — contextvar stays empty. + # Snapshot the drop counter BEFORE the call so concurrent + # tests in the same suite can't make the assertion noisy. + before = metrics.runtime.dropped_llm_call_no_reservation + # No capture call here — contextvar stays empty. rt.track_llm( input_tokens=10, output_tokens=5, model="claude-sonnet-4-6", ) - # Buffer + flush. + # Buffer + flush — neither endpoint should fire. rt._transport.flush_now() assert single_route.call_count == 0 - assert batch_route.call_count == 1 + assert batch_route.call_count == 0 + assert ( + metrics.runtime.dropped_llm_call_no_reservation == before + 1 + ), "drop counter must increment by 1 on a no-smid llm_call" @respx.mock def test_v3_track_disable_env_forces_legacy(self, make_runtime, monkeypatch): @@ -1820,6 +1843,13 @@ def test_block_response_does_not_infect_subsequent_track( from nullrun.breaker.exceptions import WorkflowKilledInterrupt from nullrun.context import workflow + from nullrun.observability import metrics + + # Snapshot the drop counter BEFORE the block+track so the + # assertion is isolated from concurrent tests in the same + # suite. + before = metrics.runtime.dropped_llm_call_no_reservation + with workflow("wf-1"): # Block path raises — WorkflowKilledInterrupt is a # BaseException (carries the kill signal @@ -1837,9 +1867,18 @@ def test_block_response_does_not_infect_subsequent_track( ) rt._transport.flush_now() - # No reservation_id was minted → falls back to batch. + # v0.16.0 (backend v3.66.2 alignment): no reservation_id + # was minted (block response carries no reservation_id; the + # capture helper clears the contextvar) → event is DROPPED, + # not batched. Pre-0.16.0 this fell back to /track/batch, + # which v3.66.2 rejects with 503 BUDGET_RECHECK_FAILED. + # The semantic intent of the test is preserved: no smid + # leaks from the block into the subsequent track. assert single_route.call_count == 0 - assert batch_route.call_count == 1 + assert batch_route.call_count == 0 + assert ( + metrics.runtime.dropped_llm_call_no_reservation == before + 1 + ), "drop counter must increment by 1 on a post-block llm_call" # ─── v3.38 wire-drift fixes ───────────────────────────────── diff --git a/uv.lock b/uv.lock index bb54ffc..9a32006 100644 --- a/uv.lock +++ b/uv.lock @@ -2870,7 +2870,7 @@ wheels = [ [[package]] name = "nullrun" -version = "0.14.6" +version = "0.16.0" source = { editable = "." } dependencies = [ { name = "httpx" },