From d94a19bd6afbc6e173a6d2cc3741ae08ca7fea44 Mon Sep 17 00:00:00 2001 From: Robin Date: Tue, 25 Aug 2026 00:12:21 -0700 Subject: [PATCH 1/2] feat: agentx.testing.assert_evaluation - pytest-native quality gates The DeepEval assert_test ergonomic on AgentX primitives: run an evaluation however you like, then assert_evaluation(report, min_rating=7, no_regression=True) fails the test with a per-check verdict when quality drops. Plain AssertionError subclass - works in any runner, no plugin registration - and rides the engine CI gate with caller="pytest", so a red test and the dashboard's gate-history row are the same recorded event. Co-Authored-By: Claude Fable 5 --- agentx/testing.py | 90 +++++++++++++++++++++++++++++++++++++++++++ tests/test_testing.py | 49 +++++++++++++++++++++++ 2 files changed, 139 insertions(+) create mode 100644 agentx/testing.py create mode 100644 tests/test_testing.py diff --git a/agentx/testing.py b/agentx/testing.py new file mode 100644 index 0000000..9913dfe --- /dev/null +++ b/agentx/testing.py @@ -0,0 +1,90 @@ +"""pytest-friendly assertions over evaluation runs. + +The DeepEval-style dev loop (``assert_test`` inside a pytest suite) on top of AgentX's existing +run + CI-gate primitives: run the evaluation however you like, then make the test fail with a +readable verdict when quality drops. No plugin registration needed - it's a plain function that +raises ``AssertionError``, so it works in any test runner and any CI. + +Usage:: + + from agentx import AgentX + from agentx.testing import assert_evaluation + + def test_support_agent_quality(): + client = AgentX.from_env() + report = ( + client.evaluations + .run(dataset_id=DATASET_ID, scorer_id=SCORER_ID, subject=SUBJECT) + .execute(my_agent) + .finalize() + ) + assert_evaluation(report, min_rating=7.0, no_regression=True) + +The check rides the engine's CI gate, so every pytest verdict is also recorded in the +dashboard's gate history (CI Gates tab) with ``caller="pytest"`` - a red test and the +dashboard's gate row are the same event, not two systems drifting apart. +""" + +from typing import Any, List, Optional + + +class EvaluationAssertionError(AssertionError): + """Raised when an evaluation run fails its quality checks. + + Subclasses ``AssertionError`` so pytest renders it as a plain test failure; carries the + ``gate`` result for programmatic inspection in test hooks. + """ + + def __init__(self, message: str, gate: Any = None): + super().__init__(message) + self.gate = gate + + +def _format_failures(gate: Any) -> str: + lines: List[str] = [] + checks = getattr(gate, "checks", None) or [] + for check in checks: + get = check.get if isinstance(check, dict) else lambda k, d=None: getattr(check, k, d) + status = "PASS" if get("passed") else "FAIL" + lines.append(f" [{status}] {get('name', 'check')}: {get('detail', '')}") + average = getattr(gate, "average_rating", None) + if average is not None: + lines.append(f" average rating: {average}") + return "\n".join(lines) if lines else f" gate: {gate!r}" + + +def assert_evaluation( + report: Any, + *, + min_rating: Optional[float] = None, + no_regression: bool = False, + tolerance: Optional[float] = None, + caller: str = "pytest", +) -> Any: + """Assert a finalized evaluation run meets its quality floor. + + ``report`` is the finalized :class:`~agentx.evaluations.runner.EvaluationRunContext` + returned by ``.execute(...).finalize()`` (or any object exposing the same ``.gate()``). + + - ``min_rating`` - fail when the run's average judge rating is below this floor (0-10). + - ``no_regression`` - fail when the average dropped more than ``tolerance`` (default 0.5; + judge scores are noisy) below the dataset's previous completed run. + + At least one check is required. Returns the ``GateResult`` on success; raises + :class:`EvaluationAssertionError` with a per-check verdict on failure. + """ + if min_rating is None and not no_regression: + raise ValueError("assert_evaluation needs at least one check: min_rating and/or no_regression=True") + gate = report.gate( + fail_under=min_rating, + no_regression=no_regression, + tolerance=tolerance, + caller=caller, + ) + if getattr(gate, "passed", False): + return gate + run_id = getattr(report, "run_id", None) or getattr(getattr(report, "_run", None), "run_id", "?") + raise EvaluationAssertionError( + f"Evaluation run {run_id} failed its quality gate:\n{_format_failures(gate)}", + gate=gate, + ) diff --git a/tests/test_testing.py b/tests/test_testing.py new file mode 100644 index 0000000..e5c1bfa --- /dev/null +++ b/tests/test_testing.py @@ -0,0 +1,49 @@ +"""agentx.testing.assert_evaluation - the pytest-native quality gate wrapper.""" + +import pytest + +from agentx.evaluations.runner import GateResult +from agentx.testing import EvaluationAssertionError, assert_evaluation + + +class FakeReport: + def __init__(self, gate_payload): + self.run_id = "run-123" + self.gate_kwargs = None + self._payload = gate_payload + + def gate(self, **kwargs): + self.gate_kwargs = kwargs + return GateResult(self._payload) + + +def test_passing_gate_returns_result_and_forwards_checks(): + report = FakeReport({"passed": True, "averageRating": 8.2, "checks": [{"name": "floor", "passed": True}]}) + gate = assert_evaluation(report, min_rating=7.0, no_regression=True, tolerance=0.3) + assert gate.passed is True + assert report.gate_kwargs == {"fail_under": 7.0, "no_regression": True, "tolerance": 0.3, "caller": "pytest"} + + +def test_failing_gate_raises_assertion_error_with_verdict(): + report = FakeReport({ + "passed": False, + "averageRating": 5.1, + "checks": [ + {"name": "floor", "passed": False, "detail": "average 5.1 below fail_under 7"}, + {"name": "regression", "passed": True, "detail": "no baseline"}, + ], + }) + with pytest.raises(EvaluationAssertionError) as excinfo: + assert_evaluation(report, min_rating=7.0) + message = str(excinfo.value) + assert "run-123" in message + assert "[FAIL] floor" in message + assert "average 5.1 below fail_under 7" in message + # AssertionError subclass, so pytest treats it as a normal test failure. + assert isinstance(excinfo.value, AssertionError) + assert excinfo.value.gate.average_rating == 5.1 + + +def test_requires_at_least_one_check(): + with pytest.raises(ValueError): + assert_evaluation(FakeReport({"passed": True})) From 6085fba10b387efa0b5d7d30e7c7d006a582e703 Mon Sep 17 00:00:00 2001 From: Robin Date: Tue, 25 Aug 2026 00:26:26 -0700 Subject: [PATCH 2/2] bump version --- agentx/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agentx/version.py b/agentx/version.py index fe00d35..1cf6267 100644 --- a/agentx/version.py +++ b/agentx/version.py @@ -1 +1 @@ -VERSION = "0.6.39" +VERSION = "0.1.0"