diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..a3fc3cc --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,35 @@ +# The docs are the only description of the SDK most people read, and nothing checked them: +# 0.6.36 consolidated grading configs into an LLM Judge Scorer and shipped with no mention of +# it anywhere in EVALUATIONS.md, TRACING.md, README.md or CICD_EVAL.md. release.yml only +# publishes, so there was no job that could have noticed. +# +# Scope is deliberate. This runs the checks that hold the documentation against the package, +# plus the unit tests for the surface it documents - both green on the base install. It is +# not the whole suite: tests/test_integrations.py and tests/test_span_tree.py need optional +# extras (google-adk) and fail on ImportError without them, so running everything here would +# ship a red badge that says nothing about the docs. Widening this to the full suite wants +# the extras installed first, and is its own change. +name: docs + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +jobs: + docs: + runs-on: ubuntu-latest + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + # The version release.yml builds the published wheel with, and a current one. + python: ["3.9", "3.12"] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + - run: pip install -e ".[langchain]" pytest + - run: python -m pytest tests/test_docs_match_sdk.py tests/test_judge_scorers.py -q diff --git a/EVALUATIONS.md b/EVALUATIONS.md index e9605f6..aebd6d2 100644 --- a/EVALUATIONS.md +++ b/EVALUATIONS.md @@ -197,13 +197,21 @@ client.evaluations.datasets.builder(name="Support Agent v2").add_case( --- -### Evaluation Settings builder - reusable grading configs +### LLM Judge Scorers - reusable grading configs -By default, a dataset runs against the grading config it was created with (`number_of_requests`, `acceptance_criteria`, similarity metrics, etc. - see above). If you want to grade the **same dataset** against **different configs** (e.g. a strict config vs. a lenient one, or reuse one config across many datasets), create a standalone `EvaluationSettings` and pass its id to `.run()`: +By default, a dataset runs against the grading config it was created with (`number_of_requests`, `acceptance_criteria`, similarity metrics, etc. - see above). If you want to grade the **same dataset** against **different configs** (e.g. a strict config vs. a lenient one, or reuse one config across many datasets), create a standalone **LLM Judge Scorer** and pass its id to `.run()`. + +One scorer is one entity: a judge rubric plus two setting profiles. + +| Section | What it holds | +|---|---| +| `judge` | The rubric every surface grades with - acceptance / rejection / evaluation criteria, judge prompt, judge model | +| `offline` | How dataset runs grade with it - repetitions, similarity metrics, code scorers, default flag | +| `online` | Whether it *also* scores live production traffic - enabled, sample rate, scope, alert threshold. `None` means offline-only | ```python -strict_settings = ( - client.evaluations.settings +strict = ( + client.monitor.judge_scorers .builder( name="Strict grading", number_of_requests=5, @@ -215,27 +223,113 @@ strict_settings = ( report = ( client.evaluations - .run(dataset_id=dataset.id, subject={...}, evaluation_settings_id=strict_settings.id) + .run(dataset_id=dataset.id, subject={...}, scorer_id=strict.id) .execute(my_agent) .finalize() .analyze() ) ``` -Omit `evaluation_settings_id` to keep using the dataset's own config, exactly as before - this is fully additive, no existing code needs to change. The builder accepts the same config kwargs as `datasets.builder(...)` (`number_of_requests`, the three criteria fields, `vector_similarity`/`jaccard_similarity`/`bleu_score`/`rouge_score`, `sovereignty_models`, `judge_prompt`/`judge_model` below) but no `questions` - it's config-only and reusable. +**The scorer's id is the `scorer_id` a run takes** - there is no second id to keep track of. Omit `scorer_id` to keep using the dataset's own config, exactly as before - this is fully additive, no existing code needs to change. The builder accepts the same config kwargs as `datasets.builder(...)` (`number_of_requests`, the three criteria fields, `vector_similarity`/`jaccard_similarity`/`bleu_score`/`rouge_score`, `sovereignty_models`, `judge_prompt`/`judge_model` below) but no `questions` - it's config-only and reusable, plus `thresholds`, `tool_context`, `code_scorers` and the online profile below. + +```python +client.monitor.judge_scorers.get(strict.id) # -> JudgeScorer +client.monitor.judge_scorers.list() # -> list[JudgeScorer] +client.monitor.judge_scorers.update(strict.id, judge={"acceptanceCriteria": "..."}) +client.monitor.judge_scorers.delete(strict.id) # rubric, version history and online profile together +``` + +`update()` is sparse - only the sections you pass change. `online={...}` upserts the online profile (this is how an offline-only scorer goes live), and `online=None` detaches it. Section dicts use the wire's camelCase keys; `create(name, judge=..., offline=..., online=...)` takes the same three sections directly if you would rather not go through the builder. Deleting, and detaching the online profile, are both refused for the built-in Session Baseline Judge. + +A `JudgeScorer` is a `dict` subclass, so unknown fields round-trip untouched, with `.id`, `.name`, `.judge`, `.offline`, `.online` and `.online_profile_id` as conveniences. + +#### Naming: `scorer_id` and `evaluation_settings_id` + +`scorer_id` is the current name for the kwarg naming a run's grader. `evaluation_settings_id` is the pre-consolidation spelling of **the same id** - the wire still calls the field `evaluationSettingsId` - and it keeps working on `client.evaluations.run(...)` and `init_run(...)`. Passing both with *different* values raises `ValueError`. + +Which to write depends on the SDK versions your code has to run under: + +| Kwarg | `agentx-python` < 0.6.36 | >= 0.6.36 | +|---|---|---| +| `evaluation_settings_id` | works | works | +| `scorer_id` | `TypeError` | works | + +So prefer `scorer_id` in new code, and keep `evaluation_settings_id` where a script may be executed against a pinned older client - CI gates and committed evaluation harnesses that get re-run for a controlled before-and-after comparison are the usual cases. + +#### Scoring live traffic with the same scorer + +Pass `live=True` to give the scorer an online profile at creation, and the same rubric that grades your dataset runs also scores a sample of production traffic. See [`client.monitor.online_evaluators`](TRACING.md#clientmonitoronline_evaluators-self-host-only) for what the online profile does and what its fields mean. + +```python +scorer = client.monitor.judge_scorers.builder( + name="Support quality", + acceptance_criteria="Concrete, correct, cites the policy.", + live=True, # every check is a real judge call on your own provider key + sample_rate=0.2, + alert_threshold=6, +).publish() +``` + +| Parameter | Type | Default | Description | +|---|---|---|---| +| `live` | `bool` | `False` | Create the online profile and start scoring live traffic | +| `sample_rate` | `float` | `0.1` | Fraction of traffic actually scored | +| `scope` | `str` | `"trace"` | `"trace"` scores individual traces at ingest; `"session"` scores whole conversations | +| `alert_threshold` | `float \| None` | `5` | A score below this raises a signal. `None` scores without ever raising one | +| `severity` | `str` | `"medium"` | `"low"`, `"medium"`, `"high"` or `"critical"`, applied to signals it raises | +| `agent_ids` | `list[str]` | `None` | Restrict scoring to specific agents instead of the whole workspace | +| `idle_seconds` | `int` | `120` | For `scope="session"`: how long a session must be quiet before it is judged | + +Calibration, tuning and the live-scoring history hang off the same scorer id - the SDK resolves the online profile for you: + +```python +client.monitor.judge_scorers.calibration(scorer.id, window="7d") # verdicts vs. recorded ground truth +proposal = client.monitor.judge_scorers.tune(scorer.id) # LLM call, slow +client.monitor.judge_scorers.validate_tuning(scorer.id, proposal) # re-judge with candidate criteria +client.monitor.judge_scorers.publish_tuning(scorer.id, proposal) # write it onto the rubric +client.monitor.judge_scorers.ratings(scorer.id, window="7d") # -> list[OnlineEvaluatorRatingPoint] +client.monitor.judge_scorers.events(scorer.id, window="7d") # -> list[OnlineEvaluatorEvent] +``` + +Those six cover live-traffic scoring, so calling them on an offline-only scorer raises `AgentXJudgeScorersError` naming the fix (`update(scorer_id, online={"enabled": True})`). `publish_tuning` writes to the shared rubric, so it applies everywhere the scorer is used: online scoring, offline dataset runs and the playground alike. + +#### Engine compatibility (self-host) + +`client.monitor.judge_scorers` calls `/agent-monitoring/judge-scorers`, which needs a self-host engine build that serves it. **Older engines return 404 on that route** while the rest of `/agent-monitoring/` works normally, and the SDK surfaces that as `AgentXJudgeScorersError: ... (404)`. Check before building on it: + +```python +try: + client.monitor.judge_scorers.list() +except Exception as exc: + print("unified surface unavailable on this engine:", exc) + # the legacy views below work on every engine +``` + +The legacy views are not affected - they call the long-standing `/custom-agent-evaluations/evaluation-settings` and `/agent-monitoring/online-evaluators` routes - so they remain the portable choice for code that must run against engines you do not control. This does not affect the `scorer_id` kwarg on `.run()`, which is client-side naming over a field the wire has always had. + +#### Legacy views + +Before the consolidation the same entity was reached through two half-views, and both keep working: + +| Legacy | Covers | Successor | +|---|---|---| +| `client.evaluations.settings` | the offline profile | `client.monitor.judge_scorers` | +| `client.monitor.online_evaluators` | the online profile | `client.monitor.judge_scorers` | + +Both emit a `DeprecationWarning` on first use (hidden by default; visible under `-W` or pytest) pointing at `judge_scorers`. Nothing breaks, and by design they address the same records under the same ids - an evaluation-settings id, an online-evaluator's `evaluation_settings_id` and a `scorer_id` are all the one id. They keep their pre-consolidation kwarg names deliberately: renaming compatibility surfaces would defeat their purpose. ```python -client.evaluations.settings.get(strict_settings.id) # fetch one -client.evaluations.settings.list() # list all +settings = client.evaluations.settings.builder(name="Strict grading", ...).publish() +client.evaluations.run(dataset_id=dataset.id, subject={...}, scorer_id=settings.id) ``` #### Configuring the judge -Both `datasets.builder(...)` and `settings.builder(...)` accept `judge_prompt`/`judge_model` to override how the LLM-as-judge grades responses, applying to every scoring path (native dashboard runs and SDK/custom-agent runs alike): +`datasets.builder(...)`, `judge_scorers.builder(...)` and the legacy `settings.builder(...)` all accept `judge_prompt`/`judge_model` to override how the LLM-as-judge grades responses, applying to every scoring path (native dashboard runs and SDK/custom-agent runs alike): ```python -settings = ( - client.evaluations.settings +scorer = ( + client.monitor.judge_scorers .builder( name="Strict grading", judge_model="claude-opus-4-8", # any id from list_models() diff --git a/README.md b/README.md index c0813cc..abb38f0 100644 --- a/README.md +++ b/README.md @@ -251,6 +251,21 @@ evaluator = client.monitor.online_evaluators.builder( client.monitor.online_evaluators.ratings(evaluator.id, window="7d") ``` +An online evaluator and a dataset run's grading config are two profiles of one **LLM Judge Scorer**, and `client.monitor.judge_scorers` manages both in a single call - one rubric, one id, used offline and online: + +```python +scorer = client.monitor.judge_scorers.builder( + name="Helpfulness", + acceptance_criteria="Concrete, correct, cites the policy.", + live=True, sample_rate=0.1, alert_threshold=5, # the online profile +).publish() + +client.evaluations.run(dataset_id=dataset.id, subject={...}, scorer_id=scorer.id) +client.monitor.judge_scorers.ratings(scorer.id, window="7d") +``` + +The two clients above are that entity's per-profile legacy views; they keep working and share the same ids. See [LLM Judge Scorers](EVALUATIONS.md#llm-judge-scorers---reusable-grading-configs) for the full surface, including which self-host engine builds serve it. + An online evaluator can also judge **whole conversations** instead of single traces: pass `scope="session"` and the engine scores each multi-turn session once it's been idle for `idle_seconds`, re-scoring if the conversation resumes. And to close the loop with reality, two ground-truth streams feed the dashboard's Judge Calibration view (which measures how often AgentX's automated verdicts agree with what actually happened): `client.outcomes.report(...)` for after-the-fact system results (a reopened ticket, a human confirmation), and `client.feedback.report(...)` for end-user votes forwarded from your own app's UI - a "down" raises a "Negative user feedback" signal directly, no sampling or judge call involved. All self-host features. ```python diff --git a/TRACING.md b/TRACING.md index 837de7b..acc04a6 100644 --- a/TRACING.md +++ b/TRACING.md @@ -471,7 +471,9 @@ client.monitor.profile.update("agent_123", threshold_overrides={"latencyMs": 150 ### `client.monitor.online_evaluators` (self-host only) -A real LLM judge scoring a sample of live production traffic continuously, distinct from a pattern's rule-matching: the same judge-scoring logic Evaluate's offline runs use, just pointed at production instead of a golden dataset. References an `evaluation_settings_id` (an Evaluator config: criteria, judge prompt, judge model) rather than storing its own copy, the same config datasets/Evaluate runs use. +> **Legacy view.** An online evaluator is the *online profile* of an **LLM Judge Scorer**, and this client is the half-view of it that predates the consolidation. It keeps working unchanged and its ids are the same ids, but it emits a `DeprecationWarning` on first use. Prefer [`client.monitor.judge_scorers`](EVALUATIONS.md#llm-judge-scorers---reusable-grading-configs), which manages the judge rubric, the offline (dataset-run) profile and this online profile as one entity - and note that route needs a self-host engine build that serves it, where this one works on every engine. + +A real LLM judge scoring a sample of live production traffic continuously, distinct from a pattern's rule-matching: the same judge-scoring logic Evaluate's offline runs use, just pointed at production instead of a golden dataset. References an `evaluation_settings_id` (an Evaluator config: criteria, judge prompt, judge model) rather than storing its own copy, the same config datasets/Evaluate runs use - post-consolidation that id is a judge scorer's id, and the two names address the same record. ```python evaluator = client.monitor.online_evaluators.builder( diff --git a/tests/test_docs_match_sdk.py b/tests/test_docs_match_sdk.py new file mode 100644 index 0000000..221ae93 --- /dev/null +++ b/tests/test_docs_match_sdk.py @@ -0,0 +1,176 @@ +"""Hold the documentation against the SDK it documents. + +0.6.36 consolidated grading configs into an LLM Judge Scorer and shipped with no doc +coverage at all: EVALUATIONS.md, TRACING.md, README.md and CICD_EVAL.md had no mention of +a judge scorer between them. Nothing caught that, because nothing checks the docs. + +This does. It extracts the fenced python from the docs and resolves what they show +against the installed package - the methods, the keyword arguments, and the +cross-document links. A renamed kwarg or a dropped method is a failing test here rather +than a copy-pasted snippet that raises TypeError in someone's project. + +Prose is deliberately out of scope; only fenced ```python blocks are read, which is what +a reader actually copies. +""" + +from __future__ import annotations + +import inspect +import re +from pathlib import Path + +import pytest + +from agentx.evaluations.client import EvaluationsClient, _resolve_scorer_id +from agentx.evaluations.runner import EvaluationsRunner +from agentx.monitor.judge_scorers import JudgeScorerBuilder, JudgeScorersClient + +ROOT = Path(__file__).resolve().parent.parent +DOCS = ("EVALUATIONS.md", "TRACING.md", "README.md", "CICD_EVAL.md") + +BUILDER_PARAMS = set(inspect.signature(JudgeScorersClient.builder).parameters) - {"self"} +RUN_PARAMS = set(inspect.signature(EvaluationsRunner.run).parameters) - {"self"} + +# A regex that quietly stops matching turns this file into a green light wired to nothing, +# so each sweep asserts it still found roughly what it found when written. +MIN_JUDGE_SCORER_CALLS = 10 +MIN_BUILDER_KEYWORDS = 10 +MIN_RUN_KEYWORDS = 3 + + +def _fenced_python(text: str) -> str: + return "\n".join(re.findall(r"```python\n(.*?)```", text, re.S)) + + +def _calls(src: str, prefix: str) -> list[tuple[str, str]]: + """(method, argument-text) for each ``.(...)``, parens balanced. + + The docs are not importable - they carry placeholders like ``subject={...}`` - so this + reads them the way a reader does, by eye, rather than by parsing them as Python. + """ + found = [] + for match in re.finditer(re.escape(prefix) + r"\s*\.\s*(\w+)\(", src): + open_paren = match.end() - 1 + depth, index = 0, open_paren + while index < len(src): + if src[index] == "(": + depth += 1 + elif src[index] == ")": + depth -= 1 + if depth == 0: + break + index += 1 + found.append((match.group(1), src[open_paren + 1 : index])) + return found + + +def _keywords(argument_text: str) -> list[str]: + """Keyword names in a call's argument text, ignoring keys inside a value's own dict.""" + return re.findall(r"(?:^|[(,]\s*|\n\s*)(\w+)\s*=", argument_text) + + +def _documented() -> dict[str, str]: + return {name: _fenced_python((ROOT / name).read_text()) for name in DOCS} + + +def _judge_scorer_calls() -> list[tuple[str, str, str]]: + return [ + (doc, method, args) + for doc, src in _documented().items() + for method, args in _calls(src, "client.monitor.judge_scorers") + ] + + +def test_documented_judge_scorer_methods_exist(): + calls = _judge_scorer_calls() + assert len(calls) >= MIN_JUDGE_SCORER_CALLS, ( + f"only found {len(calls)} judge_scorers calls in the docs - the sweep is no longer " + "finding what it should, or the surface stopped being documented" + ) + missing = [ + f"{doc}: client.monitor.judge_scorers.{method}()" + for doc, method, _ in calls + if not hasattr(JudgeScorersClient, method) + ] + assert not missing, "documented but not on JudgeScorersClient: " + ", ".join(missing) + + +def test_documented_builder_keywords_are_real_parameters(): + keywords = [ + (doc, keyword) + for doc, method, args in _judge_scorer_calls() + if method == "builder" + for keyword in _keywords(args) + ] + assert len(keywords) >= MIN_BUILDER_KEYWORDS, ( + f"only found {len(keywords)} builder keywords in the docs - the sweep is no longer " + "finding what it should" + ) + unknown = [f"{doc}: builder({kw}=...)" for doc, kw in keywords if kw not in BUILDER_PARAMS] + assert not unknown, "documented but not a builder parameter: " + ", ".join(unknown) + + +def test_documented_run_keywords_are_real_parameters(): + keywords = [ + (doc, keyword) + for doc, src in _documented().items() + for method, args in _calls(src, "client.evaluations") + if method == "run" + for keyword in _keywords(args) + ] + assert len(keywords) >= MIN_RUN_KEYWORDS, ( + f"only found {len(keywords)} evaluations.run keywords in the docs - the sweep is no " + "longer finding what it should" + ) + unknown = [f"{doc}: run({kw}=...)" for doc, kw in keywords if kw not in RUN_PARAMS] + assert not unknown, "documented but not a run() parameter: " + ", ".join(unknown) + + +def test_builder_publish_is_documented_and_real(): + """Every builder example ends in .publish(); it has to be there.""" + assert any(".publish()" in src for src in _documented().values()) + assert hasattr(JudgeScorerBuilder, "publish") + + +def test_legacy_views_still_exist(): + """The docs tell readers the pre-consolidation clients keep working. They must.""" + assert hasattr(EvaluationsClient, "settings") + from agentx.monitor.online_evaluators import MonitorOnlineEvaluatorClient + + assert hasattr(MonitorOnlineEvaluatorClient, "builder") + + +def test_both_grader_spellings_resolve_to_one_id(): + """EVALUATIONS.md states scorer_id and evaluation_settings_id are the same id, and + that passing two different ids raises. Both halves are load-bearing for readers + choosing which to write.""" + assert "scorer_id" in RUN_PARAMS and "evaluation_settings_id" in RUN_PARAMS + assert _resolve_scorer_id("abc", None) == "abc" + assert _resolve_scorer_id(None, "abc") == "abc" + assert _resolve_scorer_id("abc", "abc") == "abc" + with pytest.raises(ValueError): + _resolve_scorer_id("abc", "def") + + +@pytest.mark.parametrize( + "link, target, heading", + [ + ( + "EVALUATIONS.md#llm-judge-scorers---reusable-grading-configs", + "EVALUATIONS.md", + "### LLM Judge Scorers - reusable grading configs", + ), + ( + "TRACING.md#clientmonitoronline_evaluators-self-host-only", + "TRACING.md", + "### `client.monitor.online_evaluators` (self-host only)", + ), + ], +) +def test_cross_document_links_resolve(link, target, heading): + """A renamed heading silently breaks every link pointing at it.""" + linking = [name for name in DOCS if link in (ROOT / name).read_text()] + assert linking, f"nothing links to {link} any more - drop this case or fix the link" + assert heading in (ROOT / target).read_text(), ( + f"{linking} link to {link}, but {target} has no heading rendering to that anchor" + )