From 6b7f950da1e258e2f9d415925347aeddd6f1d752 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 24 Aug 2026 18:35:12 +0000 Subject: [PATCH 1/2] Document the LLM Judge Scorer surface 0.6.36 consolidated grading configs into a single entity - a judge rubric plus an offline profile and an online profile, at client.monitor.judge_scorers - and soft-deprecated the two half-views that used to reach it. None of that reached the docs: EVALUATIONS.md, TRACING.md, README.md and CICD_EVAL.md had no mention of a judge scorer between them, so the only way to find the surface was to read monitor/judge_scorers.py. This repo runs no docs-vs-code CI, so nothing caught the omission either. EVALUATIONS.md gains the section, replacing the one that documented the legacy settings builder as if it were the primary surface: the three sections and what each holds, the CRUD calls and what sparse update() does with online, giving a scorer an online profile with live=True, and the calibration/tuning/history calls that resolve the online profile from the scorer id. Two things it says that are not in any docstring, because both cost real time to establish: - Which grader kwarg to write. scorer_id is the current name and evaluation_settings_id is the pre-consolidation spelling of the same id, but they are not interchangeable across versions: only evaluation_settings_id works on clients older than 0.6.36. So new code should say scorer_id, and anything that may run under a pinned older client - a CI gate, a committed evaluation harness re-run for a controlled comparison - should keep the older spelling. Stated as a table rather than left to be discovered. - Engine compatibility. client.monitor.judge_scorers calls /agent-monitoring/judge-scorers, and a self-host engine that does not serve that route returns 404 while the rest of /agent-monitoring/ answers normally - verified against a running engine, where online-evaluators and patterns return 200 and judge-scorers does not. The section says so, shows the check, and points at the legacy views as the portable path, so nobody builds on the surface and discovers this at runtime. TRACING.md marks online_evaluators as the online half-view and links across; README.md gets a short pointer next to the online-evaluator example so the front door names the unified entity. Both legacy sections stay - they document surfaces that still work, and on engines without the route they are the only ones that do. Verified rather than eyeballed: every fenced python block in the three files was extracted and resolved against 0.6.36 - 149 call sites, keywords and claims, including all eleven judge_scorers methods, every builder keyword, both grader spellings on run(), that two different ids raise, and that the cross-document anchors match real headings. Docs only; no version bump. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01WYi6akpDa8qfiojRed2cAE --- EVALUATIONS.md | 116 ++++++++++++++++++++++++++++++++++++++++++++----- README.md | 15 +++++++ TRACING.md | 4 +- 3 files changed, 123 insertions(+), 12 deletions(-) diff --git a/EVALUATIONS.md b/EVALUATIONS.md index e9605f6..aebd6d2 100644 --- a/EVALUATIONS.md +++ b/EVALUATIONS.md @@ -197,13 +197,21 @@ client.evaluations.datasets.builder(name="Support Agent v2").add_case( --- -### Evaluation Settings builder - reusable grading configs +### LLM Judge Scorers - reusable grading configs -By default, a dataset runs against the grading config it was created with (`number_of_requests`, `acceptance_criteria`, similarity metrics, etc. - see above). If you want to grade the **same dataset** against **different configs** (e.g. a strict config vs. a lenient one, or reuse one config across many datasets), create a standalone `EvaluationSettings` and pass its id to `.run()`: +By default, a dataset runs against the grading config it was created with (`number_of_requests`, `acceptance_criteria`, similarity metrics, etc. - see above). If you want to grade the **same dataset** against **different configs** (e.g. a strict config vs. a lenient one, or reuse one config across many datasets), create a standalone **LLM Judge Scorer** and pass its id to `.run()`. + +One scorer is one entity: a judge rubric plus two setting profiles. + +| Section | What it holds | +|---|---| +| `judge` | The rubric every surface grades with - acceptance / rejection / evaluation criteria, judge prompt, judge model | +| `offline` | How dataset runs grade with it - repetitions, similarity metrics, code scorers, default flag | +| `online` | Whether it *also* scores live production traffic - enabled, sample rate, scope, alert threshold. `None` means offline-only | ```python -strict_settings = ( - client.evaluations.settings +strict = ( + client.monitor.judge_scorers .builder( name="Strict grading", number_of_requests=5, @@ -215,27 +223,113 @@ strict_settings = ( report = ( client.evaluations - .run(dataset_id=dataset.id, subject={...}, evaluation_settings_id=strict_settings.id) + .run(dataset_id=dataset.id, subject={...}, scorer_id=strict.id) .execute(my_agent) .finalize() .analyze() ) ``` -Omit `evaluation_settings_id` to keep using the dataset's own config, exactly as before - this is fully additive, no existing code needs to change. The builder accepts the same config kwargs as `datasets.builder(...)` (`number_of_requests`, the three criteria fields, `vector_similarity`/`jaccard_similarity`/`bleu_score`/`rouge_score`, `sovereignty_models`, `judge_prompt`/`judge_model` below) but no `questions` - it's config-only and reusable. +**The scorer's id is the `scorer_id` a run takes** - there is no second id to keep track of. Omit `scorer_id` to keep using the dataset's own config, exactly as before - this is fully additive, no existing code needs to change. The builder accepts the same config kwargs as `datasets.builder(...)` (`number_of_requests`, the three criteria fields, `vector_similarity`/`jaccard_similarity`/`bleu_score`/`rouge_score`, `sovereignty_models`, `judge_prompt`/`judge_model` below) but no `questions` - it's config-only and reusable, plus `thresholds`, `tool_context`, `code_scorers` and the online profile below. + +```python +client.monitor.judge_scorers.get(strict.id) # -> JudgeScorer +client.monitor.judge_scorers.list() # -> list[JudgeScorer] +client.monitor.judge_scorers.update(strict.id, judge={"acceptanceCriteria": "..."}) +client.monitor.judge_scorers.delete(strict.id) # rubric, version history and online profile together +``` + +`update()` is sparse - only the sections you pass change. `online={...}` upserts the online profile (this is how an offline-only scorer goes live), and `online=None` detaches it. Section dicts use the wire's camelCase keys; `create(name, judge=..., offline=..., online=...)` takes the same three sections directly if you would rather not go through the builder. Deleting, and detaching the online profile, are both refused for the built-in Session Baseline Judge. + +A `JudgeScorer` is a `dict` subclass, so unknown fields round-trip untouched, with `.id`, `.name`, `.judge`, `.offline`, `.online` and `.online_profile_id` as conveniences. + +#### Naming: `scorer_id` and `evaluation_settings_id` + +`scorer_id` is the current name for the kwarg naming a run's grader. `evaluation_settings_id` is the pre-consolidation spelling of **the same id** - the wire still calls the field `evaluationSettingsId` - and it keeps working on `client.evaluations.run(...)` and `init_run(...)`. Passing both with *different* values raises `ValueError`. + +Which to write depends on the SDK versions your code has to run under: + +| Kwarg | `agentx-python` < 0.6.36 | >= 0.6.36 | +|---|---|---| +| `evaluation_settings_id` | works | works | +| `scorer_id` | `TypeError` | works | + +So prefer `scorer_id` in new code, and keep `evaluation_settings_id` where a script may be executed against a pinned older client - CI gates and committed evaluation harnesses that get re-run for a controlled before-and-after comparison are the usual cases. + +#### Scoring live traffic with the same scorer + +Pass `live=True` to give the scorer an online profile at creation, and the same rubric that grades your dataset runs also scores a sample of production traffic. See [`client.monitor.online_evaluators`](TRACING.md#clientmonitoronline_evaluators-self-host-only) for what the online profile does and what its fields mean. + +```python +scorer = client.monitor.judge_scorers.builder( + name="Support quality", + acceptance_criteria="Concrete, correct, cites the policy.", + live=True, # every check is a real judge call on your own provider key + sample_rate=0.2, + alert_threshold=6, +).publish() +``` + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `live` | `bool` | `False` | Create the online profile and start scoring live traffic | +| `sample_rate` | `float` | `0.1` | Fraction of traffic actually scored | +| `scope` | `str` | `"trace"` | `"trace"` scores individual traces at ingest; `"session"` scores whole conversations | +| `alert_threshold` | `float \| None` | `5` | A score below this raises a signal. `None` scores without ever raising one | +| `severity` | `str` | `"medium"` | `"low"`, `"medium"`, `"high"` or `"critical"`, applied to signals it raises | +| `agent_ids` | `list[str]` | `None` | Restrict scoring to specific agents instead of the whole workspace | +| `idle_seconds` | `int` | `120` | For `scope="session"`: how long a session must be quiet before it is judged | + +Calibration, tuning and the live-scoring history hang off the same scorer id - the SDK resolves the online profile for you: + +```python +client.monitor.judge_scorers.calibration(scorer.id, window="7d") # verdicts vs. recorded ground truth +proposal = client.monitor.judge_scorers.tune(scorer.id) # LLM call, slow +client.monitor.judge_scorers.validate_tuning(scorer.id, proposal) # re-judge with candidate criteria +client.monitor.judge_scorers.publish_tuning(scorer.id, proposal) # write it onto the rubric +client.monitor.judge_scorers.ratings(scorer.id, window="7d") # -> list[OnlineEvaluatorRatingPoint] +client.monitor.judge_scorers.events(scorer.id, window="7d") # -> list[OnlineEvaluatorEvent] +``` + +Those six cover live-traffic scoring, so calling them on an offline-only scorer raises `AgentXJudgeScorersError` naming the fix (`update(scorer_id, online={"enabled": True})`). `publish_tuning` writes to the shared rubric, so it applies everywhere the scorer is used: online scoring, offline dataset runs and the playground alike. + +#### Engine compatibility (self-host) + +`client.monitor.judge_scorers` calls `/agent-monitoring/judge-scorers`, which needs a self-host engine build that serves it. **Older engines return 404 on that route** while the rest of `/agent-monitoring/` works normally, and the SDK surfaces that as `AgentXJudgeScorersError: ... (404)`. Check before building on it: + +```python +try: + client.monitor.judge_scorers.list() +except Exception as exc: + print("unified surface unavailable on this engine:", exc) + # the legacy views below work on every engine +``` + +The legacy views are not affected - they call the long-standing `/custom-agent-evaluations/evaluation-settings` and `/agent-monitoring/online-evaluators` routes - so they remain the portable choice for code that must run against engines you do not control. This does not affect the `scorer_id` kwarg on `.run()`, which is client-side naming over a field the wire has always had. + +#### Legacy views + +Before the consolidation the same entity was reached through two half-views, and both keep working: + +| Legacy | Covers | Successor | +|---|---|---| +| `client.evaluations.settings` | the offline profile | `client.monitor.judge_scorers` | +| `client.monitor.online_evaluators` | the online profile | `client.monitor.judge_scorers` | + +Both emit a `DeprecationWarning` on first use (hidden by default; visible under `-W` or pytest) pointing at `judge_scorers`. Nothing breaks, and by design they address the same records under the same ids - an evaluation-settings id, an online-evaluator's `evaluation_settings_id` and a `scorer_id` are all the one id. They keep their pre-consolidation kwarg names deliberately: renaming compatibility surfaces would defeat their purpose. ```python -client.evaluations.settings.get(strict_settings.id) # fetch one -client.evaluations.settings.list() # list all +settings = client.evaluations.settings.builder(name="Strict grading", ...).publish() +client.evaluations.run(dataset_id=dataset.id, subject={...}, scorer_id=settings.id) ``` #### Configuring the judge -Both `datasets.builder(...)` and `settings.builder(...)` accept `judge_prompt`/`judge_model` to override how the LLM-as-judge grades responses, applying to every scoring path (native dashboard runs and SDK/custom-agent runs alike): +`datasets.builder(...)`, `judge_scorers.builder(...)` and the legacy `settings.builder(...)` all accept `judge_prompt`/`judge_model` to override how the LLM-as-judge grades responses, applying to every scoring path (native dashboard runs and SDK/custom-agent runs alike): ```python -settings = ( - client.evaluations.settings +scorer = ( + client.monitor.judge_scorers .builder( name="Strict grading", judge_model="claude-opus-4-8", # any id from list_models() diff --git a/README.md b/README.md index c0813cc..abb38f0 100644 --- a/README.md +++ b/README.md @@ -251,6 +251,21 @@ evaluator = client.monitor.online_evaluators.builder( client.monitor.online_evaluators.ratings(evaluator.id, window="7d") ``` +An online evaluator and a dataset run's grading config are two profiles of one **LLM Judge Scorer**, and `client.monitor.judge_scorers` manages both in a single call - one rubric, one id, used offline and online: + +```python +scorer = client.monitor.judge_scorers.builder( + name="Helpfulness", + acceptance_criteria="Concrete, correct, cites the policy.", + live=True, sample_rate=0.1, alert_threshold=5, # the online profile +).publish() + +client.evaluations.run(dataset_id=dataset.id, subject={...}, scorer_id=scorer.id) +client.monitor.judge_scorers.ratings(scorer.id, window="7d") +``` + +The two clients above are that entity's per-profile legacy views; they keep working and share the same ids. See [LLM Judge Scorers](EVALUATIONS.md#llm-judge-scorers---reusable-grading-configs) for the full surface, including which self-host engine builds serve it. + An online evaluator can also judge **whole conversations** instead of single traces: pass `scope="session"` and the engine scores each multi-turn session once it's been idle for `idle_seconds`, re-scoring if the conversation resumes. And to close the loop with reality, two ground-truth streams feed the dashboard's Judge Calibration view (which measures how often AgentX's automated verdicts agree with what actually happened): `client.outcomes.report(...)` for after-the-fact system results (a reopened ticket, a human confirmation), and `client.feedback.report(...)` for end-user votes forwarded from your own app's UI - a "down" raises a "Negative user feedback" signal directly, no sampling or judge call involved. All self-host features. ```python diff --git a/TRACING.md b/TRACING.md index 837de7b..acc04a6 100644 --- a/TRACING.md +++ b/TRACING.md @@ -471,7 +471,9 @@ client.monitor.profile.update("agent_123", threshold_overrides={"latencyMs": 150 ### `client.monitor.online_evaluators` (self-host only) -A real LLM judge scoring a sample of live production traffic continuously, distinct from a pattern's rule-matching: the same judge-scoring logic Evaluate's offline runs use, just pointed at production instead of a golden dataset. References an `evaluation_settings_id` (an Evaluator config: criteria, judge prompt, judge model) rather than storing its own copy, the same config datasets/Evaluate runs use. +> **Legacy view.** An online evaluator is the *online profile* of an **LLM Judge Scorer**, and this client is the half-view of it that predates the consolidation. It keeps working unchanged and its ids are the same ids, but it emits a `DeprecationWarning` on first use. Prefer [`client.monitor.judge_scorers`](EVALUATIONS.md#llm-judge-scorers---reusable-grading-configs), which manages the judge rubric, the offline (dataset-run) profile and this online profile as one entity - and note that route needs a self-host engine build that serves it, where this one works on every engine. + +A real LLM judge scoring a sample of live production traffic continuously, distinct from a pattern's rule-matching: the same judge-scoring logic Evaluate's offline runs use, just pointed at production instead of a golden dataset. References an `evaluation_settings_id` (an Evaluator config: criteria, judge prompt, judge model) rather than storing its own copy, the same config datasets/Evaluate runs use - post-consolidation that id is a judge scorer's id, and the two names address the same record. ```python evaluator = client.monitor.online_evaluators.builder( From 2d7bf1b9cd048f6498f31da42dbd3929ba53758b Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 24 Aug 2026 18:47:34 +0000 Subject: [PATCH 2/2] Add the CI that keeps the docs true The previous commit documented a surface that had shipped undocumented for a reason: nothing in this repo checks the docs. release.yml only publishes, so 0.6.36 could consolidate grading configs into an LLM Judge Scorer, rename the kwarg naming a run's grader, and soft-deprecate two clients, without a single job noticing that four documentation files still described the world before it. tests/test_docs_match_sdk.py closes that. It extracts the fenced python from EVALUATIONS.md, TRACING.md, README.md and CICD_EVAL.md and resolves what they show against the installed package: every client.monitor.judge_scorers method, every builder keyword, every keyword on evaluations.run, that the legacy views the docs promise still work are still there, that both grader spellings resolve to one id and two different ids raise, and that the cross-document links point at headings that exist. Prose is out of scope on purpose - only fenced blocks are read, because that is what a reader copies. Each sweep asserts it still found roughly what it found when written. A regex that quietly stops matching is the failure mode that matters in a checker like this: it does not go red, it goes green while checking nothing. Proven to fire rather than assumed, against a copy of the tree: a documented method the SDK dropped, a builder keyword that no longer exists, a run keyword that no longer exists, and a heading renamed out from under the links that point at it. Each fails exactly one test; the restored tree passes. docs.yml runs it on pull requests and pushes to main, on 3.9 (what release.yml builds the wheel with) and 3.12. Scope is deliberate and commented in the file: it runs this plus the judge-scorer unit tests, not the whole suite, because test_integrations.py and test_span_tree.py need optional extras (google-adk) and fail on ImportError without them - four failures on a clean checkout today. A badge that is red for reasons unrelated to the docs would say nothing about the docs. Widening it wants those extras installed first, and is its own change. Verified with the exact CI command in a fresh virtualenv, the same thing the job builds: 21 passed. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01WYi6akpDa8qfiojRed2cAE --- .github/workflows/docs.yml | 35 +++++++ tests/test_docs_match_sdk.py | 176 +++++++++++++++++++++++++++++++++++ 2 files changed, 211 insertions(+) create mode 100644 .github/workflows/docs.yml create mode 100644 tests/test_docs_match_sdk.py diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..a3fc3cc --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,35 @@ +# The docs are the only description of the SDK most people read, and nothing checked them: +# 0.6.36 consolidated grading configs into an LLM Judge Scorer and shipped with no mention of +# it anywhere in EVALUATIONS.md, TRACING.md, README.md or CICD_EVAL.md. release.yml only +# publishes, so there was no job that could have noticed. +# +# Scope is deliberate. This runs the checks that hold the documentation against the package, +# plus the unit tests for the surface it documents - both green on the base install. It is +# not the whole suite: tests/test_integrations.py and tests/test_span_tree.py need optional +# extras (google-adk) and fail on ImportError without them, so running everything here would +# ship a red badge that says nothing about the docs. Widening this to the full suite wants +# the extras installed first, and is its own change. +name: docs + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +jobs: + docs: + runs-on: ubuntu-latest + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + # The version release.yml builds the published wheel with, and a current one. + python: ["3.9", "3.12"] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + - run: pip install -e ".[langchain]" pytest + - run: python -m pytest tests/test_docs_match_sdk.py tests/test_judge_scorers.py -q diff --git a/tests/test_docs_match_sdk.py b/tests/test_docs_match_sdk.py new file mode 100644 index 0000000..221ae93 --- /dev/null +++ b/tests/test_docs_match_sdk.py @@ -0,0 +1,176 @@ +"""Hold the documentation against the SDK it documents. + +0.6.36 consolidated grading configs into an LLM Judge Scorer and shipped with no doc +coverage at all: EVALUATIONS.md, TRACING.md, README.md and CICD_EVAL.md had no mention of +a judge scorer between them. Nothing caught that, because nothing checks the docs. + +This does. It extracts the fenced python from the docs and resolves what they show +against the installed package - the methods, the keyword arguments, and the +cross-document links. A renamed kwarg or a dropped method is a failing test here rather +than a copy-pasted snippet that raises TypeError in someone's project. + +Prose is deliberately out of scope; only fenced ```python blocks are read, which is what +a reader actually copies. +""" + +from __future__ import annotations + +import inspect +import re +from pathlib import Path + +import pytest + +from agentx.evaluations.client import EvaluationsClient, _resolve_scorer_id +from agentx.evaluations.runner import EvaluationsRunner +from agentx.monitor.judge_scorers import JudgeScorerBuilder, JudgeScorersClient + +ROOT = Path(__file__).resolve().parent.parent +DOCS = ("EVALUATIONS.md", "TRACING.md", "README.md", "CICD_EVAL.md") + +BUILDER_PARAMS = set(inspect.signature(JudgeScorersClient.builder).parameters) - {"self"} +RUN_PARAMS = set(inspect.signature(EvaluationsRunner.run).parameters) - {"self"} + +# A regex that quietly stops matching turns this file into a green light wired to nothing, +# so each sweep asserts it still found roughly what it found when written. +MIN_JUDGE_SCORER_CALLS = 10 +MIN_BUILDER_KEYWORDS = 10 +MIN_RUN_KEYWORDS = 3 + + +def _fenced_python(text: str) -> str: + return "\n".join(re.findall(r"```python\n(.*?)```", text, re.S)) + + +def _calls(src: str, prefix: str) -> list[tuple[str, str]]: + """(method, argument-text) for each ``.(...)``, parens balanced. + + The docs are not importable - they carry placeholders like ``subject={...}`` - so this + reads them the way a reader does, by eye, rather than by parsing them as Python. + """ + found = [] + for match in re.finditer(re.escape(prefix) + r"\s*\.\s*(\w+)\(", src): + open_paren = match.end() - 1 + depth, index = 0, open_paren + while index < len(src): + if src[index] == "(": + depth += 1 + elif src[index] == ")": + depth -= 1 + if depth == 0: + break + index += 1 + found.append((match.group(1), src[open_paren + 1 : index])) + return found + + +def _keywords(argument_text: str) -> list[str]: + """Keyword names in a call's argument text, ignoring keys inside a value's own dict.""" + return re.findall(r"(?:^|[(,]\s*|\n\s*)(\w+)\s*=", argument_text) + + +def _documented() -> dict[str, str]: + return {name: _fenced_python((ROOT / name).read_text()) for name in DOCS} + + +def _judge_scorer_calls() -> list[tuple[str, str, str]]: + return [ + (doc, method, args) + for doc, src in _documented().items() + for method, args in _calls(src, "client.monitor.judge_scorers") + ] + + +def test_documented_judge_scorer_methods_exist(): + calls = _judge_scorer_calls() + assert len(calls) >= MIN_JUDGE_SCORER_CALLS, ( + f"only found {len(calls)} judge_scorers calls in the docs - the sweep is no longer " + "finding what it should, or the surface stopped being documented" + ) + missing = [ + f"{doc}: client.monitor.judge_scorers.{method}()" + for doc, method, _ in calls + if not hasattr(JudgeScorersClient, method) + ] + assert not missing, "documented but not on JudgeScorersClient: " + ", ".join(missing) + + +def test_documented_builder_keywords_are_real_parameters(): + keywords = [ + (doc, keyword) + for doc, method, args in _judge_scorer_calls() + if method == "builder" + for keyword in _keywords(args) + ] + assert len(keywords) >= MIN_BUILDER_KEYWORDS, ( + f"only found {len(keywords)} builder keywords in the docs - the sweep is no longer " + "finding what it should" + ) + unknown = [f"{doc}: builder({kw}=...)" for doc, kw in keywords if kw not in BUILDER_PARAMS] + assert not unknown, "documented but not a builder parameter: " + ", ".join(unknown) + + +def test_documented_run_keywords_are_real_parameters(): + keywords = [ + (doc, keyword) + for doc, src in _documented().items() + for method, args in _calls(src, "client.evaluations") + if method == "run" + for keyword in _keywords(args) + ] + assert len(keywords) >= MIN_RUN_KEYWORDS, ( + f"only found {len(keywords)} evaluations.run keywords in the docs - the sweep is no " + "longer finding what it should" + ) + unknown = [f"{doc}: run({kw}=...)" for doc, kw in keywords if kw not in RUN_PARAMS] + assert not unknown, "documented but not a run() parameter: " + ", ".join(unknown) + + +def test_builder_publish_is_documented_and_real(): + """Every builder example ends in .publish(); it has to be there.""" + assert any(".publish()" in src for src in _documented().values()) + assert hasattr(JudgeScorerBuilder, "publish") + + +def test_legacy_views_still_exist(): + """The docs tell readers the pre-consolidation clients keep working. They must.""" + assert hasattr(EvaluationsClient, "settings") + from agentx.monitor.online_evaluators import MonitorOnlineEvaluatorClient + + assert hasattr(MonitorOnlineEvaluatorClient, "builder") + + +def test_both_grader_spellings_resolve_to_one_id(): + """EVALUATIONS.md states scorer_id and evaluation_settings_id are the same id, and + that passing two different ids raises. Both halves are load-bearing for readers + choosing which to write.""" + assert "scorer_id" in RUN_PARAMS and "evaluation_settings_id" in RUN_PARAMS + assert _resolve_scorer_id("abc", None) == "abc" + assert _resolve_scorer_id(None, "abc") == "abc" + assert _resolve_scorer_id("abc", "abc") == "abc" + with pytest.raises(ValueError): + _resolve_scorer_id("abc", "def") + + +@pytest.mark.parametrize( + "link, target, heading", + [ + ( + "EVALUATIONS.md#llm-judge-scorers---reusable-grading-configs", + "EVALUATIONS.md", + "### LLM Judge Scorers - reusable grading configs", + ), + ( + "TRACING.md#clientmonitoronline_evaluators-self-host-only", + "TRACING.md", + "### `client.monitor.online_evaluators` (self-host only)", + ), + ], +) +def test_cross_document_links_resolve(link, target, heading): + """A renamed heading silently breaks every link pointing at it.""" + linking = [name for name in DOCS if link in (ROOT / name).read_text()] + assert linking, f"nothing links to {link} any more - drop this case or fix the link" + assert heading in (ROOT / target).read_text(), ( + f"{linking} link to {link}, but {target} has no heading rendering to that anchor" + )