From 4e1cbda6a9714c2efb7255ddedbe592e25507a20 Mon Sep 17 00:00:00 2001 From: Peter Tomko Date: Thu, 6 Aug 2026 22:44:23 +0200 Subject: [PATCH 1/5] fix(gooddata-eval): stop the metric-skill simulated user from dropping MAQL clauses agentic_metric_skill's simulated-user reply (generate_simulated_response) is what keeps a multi-turn metric-creation conversation going after the agent asks a clarifying question -- it prompts an LLM to answer as the user, using the fixture's expected_output.maql as its only source of truth. The prompt told it to "reply briefly" with no instruction to preserve the MAQL's structure. In practice it would silently drop a WHERE/filter clause, or paraphrase a label id, whenever the agent's question didn't happen to ask about that part directly -- so a well-behaved agent, faithfully following the (already-wrong) simulated answer, still failed the eval. Reproduced live twice against a real gdc-mic-ai-evaluation fixture ("Create a metric for total ecommerce spend", expects SELECT {metric/spend_amount_-_cutcgco} WHERE {label/ecommerce_indicator_code} = "1"): 1. Simulated reply dropped "_code" off ecommerce_indicator_code, anchoring the agent on a sibling attribute that doesn't have that filter. 2. Simulated reply picked one of 3 metric options the agent offered and said "please proceed with that" -- never mentioning the WHERE clause that expected_output required, even though it had it in hand. Confirmed via a 5x-repeated A/B test that this is a prompt problem, not a model-capability one: swapping gpt-4o-mini for gpt-4o under the OLD prompt did not fix it (still dropped the clause); the NEW prompt fixes it on the ORIGINAL gpt-4o-mini (1/5 -> 5/5 runs preserving the exact filter). Fix: instruct the simulating LLM to (a) ensure every clause of the expected MAQL is eventually satisfied even if the agent's question didn't ask about it, (b) quote field/label identifiers verbatim rather than paraphrase them, and (c) proactively add a filter the agent's own offered options omitted. Also drop "reply briefly" and raise max_tokens 150->300, since brevity was part of what squeezed the filter clause out. This brings metric_skill's simulated-user prompt in line with alert_skill's generate_simulated_alert_response, which already passes structured facts + explicit "proactively tell the agent X" instructions rather than one freely-paraphrased string -- not a new pattern for this codebase. Added a regression test asserting the sent prompt preserves clause-fidelity language and the raised max_tokens. Full gooddata-eval suite: 272 passed (9 pre-existing unrelated failures, confirmed identical on clean master before this change -- missing openai extra in test env, and two unrelated test files). Co-Authored-By: Claude Sonnet 5 --- .../core/agentic/metric_skill.py | 9 +++-- .../tests/test_agentic_metric_skill.py | 33 +++++++++++++++++++ 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py index dc12a2b7f..3bddd4651 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py @@ -101,14 +101,17 @@ def generate_simulated_response(agent_message: str, expected_output: dict) -> st prompt = ( f"You are simulating a user in a conversation with a BI assistant that creates metrics. " f"The assistant said: '{agent_message}'. " - f"The user originally asked to create a metric with MAQL: {expected_maql}. " - f"Reply briefly as the user, providing any clarification the assistant needs." + f"The user's ground-truth intended metric is exactly this MAQL: {expected_maql}. " + f"Reply as the user. You MUST ensure every clause of that MAQL (including any WHERE/filter " + f"conditions) is eventually satisfied, and quote field/label identifiers verbatim from it -- " + f"never paraphrase or drop a clause, even if the assistant's question doesn't explicitly ask " + f"about it. If the assistant's offered options omit a required filter, add it yourself." ) try: response = client.chat.completions.create( model="gpt-4o-mini", messages=[{"role": "user", "content": prompt}], - max_tokens=150, + max_tokens=300, temperature=0, ) except OpenAIError as exc: diff --git a/packages/gooddata-eval/tests/test_agentic_metric_skill.py b/packages/gooddata-eval/tests/test_agentic_metric_skill.py index f00212e1c..1a5b36656 100644 --- a/packages/gooddata-eval/tests/test_agentic_metric_skill.py +++ b/packages/gooddata-eval/tests/test_agentic_metric_skill.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: LicenseRef-GoodData-Enterprise import os import sys +import types from unittest.mock import MagicMock, patch import pytest @@ -25,6 +26,38 @@ def test_normalize_maql_removes_select_wrapper(): assert _normalize_maql("(SELECT {metric/abc})") == "{metric/abc}" +def test_generate_simulated_response_prompt_preserves_maql_fidelity(monkeypatch): + """Regression test for a live-reproduced bug: the old prompt ("reply briefly", + no instruction to cover clauses the assistant didn't ask about) let the + simulating LLM silently drop a MAQL's WHERE clause or paraphrase a label id -- + confirmed via a 5x-repeated A/B test (1/5 vs 5/5 fidelity) that this was the + prompt, not the model (gpt-4o did not fix it under the old prompt either). + """ + monkeypatch.setenv("OPENAI_API_KEY", "test-key") + mock_client = MagicMock() + mock_response = MagicMock() + mock_response.choices = [MagicMock(message=MagicMock(content="ok"))] + mock_client.chat.completions.create.return_value = mock_response + + # `openai` is an optional [llm-judge] extra, not installed in this test env -- + # inject a fake module rather than patching a real one (mirrors how the source + # itself does `from openai import OpenAI` as a local, guarded import). + fake_openai_module = types.SimpleNamespace(OpenAI=MagicMock(return_value=mock_client)) + monkeypatch.setitem(sys.modules, "openai", fake_openai_module) + + expected_output = {"maql": 'SELECT {metric/spend_amount_-_cutcgco} WHERE {label/ecommerce_indicator_code} = "1"'} + generate_simulated_response("Which base metric should I use?", expected_output) + + call_kwargs = mock_client.chat.completions.create.call_args.kwargs + sent_prompt = call_kwargs["messages"][0]["content"] + + assert "verbatim" in sent_prompt + assert "every clause" in sent_prompt + assert "WHERE" in sent_prompt or "filter" in sent_prompt.lower() + assert "reply briefly" not in sent_prompt.lower() + assert call_kwargs["max_tokens"] >= 300 + + def test_metric_run_result_fields(): r = MetricRunResult( conversation_id="c1", From ecf8718349a2cfb8b4c2cd7b90e9bb4b2c201bd7 Mon Sep 17 00:00:00 2001 From: Peter Tomko Date: Thu, 6 Aug 2026 22:49:41 +0200 Subject: [PATCH 2/5] test(gooddata-eval): assert exact MAQL string appears in simulated-user prompt Addresses CodeRabbit review comment on #1718: the regression test only checked for generic instruction words ("verbatim", "every clause"), not that expected_output["maql"] itself made it into the prompt -- a regression that stripped the metric/label reference or filter value entirely could still pass. Assert the exact MAQL string is present. Co-Authored-By: Claude Sonnet 5 --- packages/gooddata-eval/tests/test_agentic_metric_skill.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/gooddata-eval/tests/test_agentic_metric_skill.py b/packages/gooddata-eval/tests/test_agentic_metric_skill.py index 1a5b36656..5f9dec88a 100644 --- a/packages/gooddata-eval/tests/test_agentic_metric_skill.py +++ b/packages/gooddata-eval/tests/test_agentic_metric_skill.py @@ -42,7 +42,7 @@ def test_generate_simulated_response_prompt_preserves_maql_fidelity(monkeypatch) # `openai` is an optional [llm-judge] extra, not installed in this test env -- # inject a fake module rather than patching a real one (mirrors how the source # itself does `from openai import OpenAI` as a local, guarded import). - fake_openai_module = types.SimpleNamespace(OpenAI=MagicMock(return_value=mock_client)) + fake_openai_module = types.SimpleNamespace(OpenAI=MagicMock(return_value=mock_client), OpenAIError=Exception) monkeypatch.setitem(sys.modules, "openai", fake_openai_module) expected_output = {"maql": 'SELECT {metric/spend_amount_-_cutcgco} WHERE {label/ecommerce_indicator_code} = "1"'} @@ -51,6 +51,7 @@ def test_generate_simulated_response_prompt_preserves_maql_fidelity(monkeypatch) call_kwargs = mock_client.chat.completions.create.call_args.kwargs sent_prompt = call_kwargs["messages"][0]["content"] + assert expected_output["maql"] in sent_prompt assert "verbatim" in sent_prompt assert "every clause" in sent_prompt assert "WHERE" in sent_prompt or "filter" in sent_prompt.lower() From 4585e38f4c5b172aebc1af87044496290ae18853 Mon Sep 17 00:00:00 2001 From: Peter Tomko Date: Tue, 4 Aug 2026 09:43:15 +0200 Subject: [PATCH 3/5] feat(gooddata-eval): support targeting a specific AI Hub agent GoodData has no admin-settable "default agent": when a conversation doesn't name one, the platform picks whichever agent was last used or last edited in that workspace. Orgs with multiple AI Hub agents (e.g. one scoped to visualization only) can end up silently evaluating the wrong one -- a metric_skill/alert_skill item run against a visualization-only agent never passes, regardless of question quality. ChatClient never sent an agentId at all, so there was no way to pick. - ChatClient gains an `agent_id` param; `create_conversation()` sends `{"agentId": ...}` only when set -- omitted entirely when None, so existing behavior (platform's own default-agent resolution) is unchanged unless the caller opts in. - Threaded through all 7 run_agentic_*/evaluate_agentic_* pairs, the agentic-dispatch layer (_dispatch_agentic/run_agentic_items), and the non-agentic ChatClient construction in cli/main.py. - New `gd-eval run --agent-id ID` flag (or `GD_EVAL_AGENT_ID` env var, same precedence convention as --token/GOODDATA_TOKEN). - README: new flags-table row + a "Targeting a specific AI Hub agent" section with real usage examples. - Tests: ChatClient POST-body shape (with/without agent_id), CLI arg parsing, flag/env-var/unset precedence into the constructed ChatClient, and _dispatch_agentic threading it to evaluate_agentic_*. Co-Authored-By: Claude Sonnet 5 --- packages/gooddata-eval/README.md | 35 ++++++ .../src/gooddata_eval/cli/agentic_runner.py | 13 ++- .../src/gooddata_eval/cli/main.py | 14 +++ .../gooddata_eval/core/agentic/alert_skill.py | 7 +- .../core/agentic/conversation.py | 7 +- .../core/agentic/general_question.py | 7 +- .../gooddata_eval/core/agentic/guardrail.py | 7 +- .../core/agentic/metric_skill.py | 7 +- .../gooddata_eval/core/agentic/search_tool.py | 7 +- .../core/agentic/visualization.py | 7 +- .../src/gooddata_eval/core/chat/sse_client.py | 7 +- .../src/gooddata_eval/core/config.py | 1 + .../tests/test_agentic_runner.py | 50 ++++++++ packages/gooddata-eval/tests/test_cli.py | 110 ++++++++++++++++++ .../gooddata-eval/tests/test_sse_client.py | 26 +++++ 15 files changed, 296 insertions(+), 9 deletions(-) create mode 100644 packages/gooddata-eval/tests/test_agentic_runner.py diff --git a/packages/gooddata-eval/README.md b/packages/gooddata-eval/README.md index b17db60cb..e5f321027 100644 --- a/packages/gooddata-eval/README.md +++ b/packages/gooddata-eval/README.md @@ -62,6 +62,40 @@ When the same model id is offered by multiple providers, use the Both provider name and provider id are accepted as the prefix. +### Targeting a specific AI Hub agent + +GoodData has no admin-settable "default agent": when a conversation doesn't +name one, the platform picks whichever agent was last used or last edited in +that workspace. If your org has several AI Hub agents configured (e.g. one +scoped to visualization only, another with every skill enabled), evaluating +without `--agent-id` can silently exercise the wrong one — a +`metric_skill`/`alert_skill` item run against a visualization-only agent will +never pass, no matter how well-formed the question is. + +```bash +export GD_EVAL_AGENT_ID='eval-all-skills' + +gd-eval run \ + --host https://your.gooddata.cloud \ + --workspace ecommerce_demo \ + --dataset ./my-dataset \ + --model gpt-5.2 \ + --runs 1 \ + --json results.json +``` + +Or pass it explicitly instead of via the env var: + +```bash +gd-eval run \ + --host https://your.gooddata.cloud \ + --workspace ecommerce_demo \ + --dataset ./my-dataset \ + --agent-id eval-all-skills \ + --model gpt-5.2 \ + --runs 1 +``` + ### All flags #### Connection @@ -72,6 +106,7 @@ Both provider name and provider id are accepted as the prefix. | `--token TOKEN` | `GOODDATA_TOKEN` | API token. Pass via flag or env var. | | `--profile NAME` | — | Profile name in `~/.gooddata/profiles.yaml` (same file as the `gdc` CLI). | | `--workspace ID` | — | **Required.** Workspace id to evaluate against. | +| `--agent-id ID` | `GD_EVAL_AGENT_ID` | AI Hub agent every conversation should target. GoodData has no admin-settable default agent — without this, each conversation falls back to whichever agent the platform's last-used/last-edited heuristic resolves, which may not have every skill under test enabled. | #### Dataset source (pick one) diff --git a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py index 7147af183..b379fadd9 100644 --- a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py +++ b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py @@ -85,6 +85,7 @@ def _dispatch_agentic( run_ts: str, model_version_override: str | None, reasoning_effort: ReasoningEffort | None = None, + agent_id: str | None = None, ) -> None: """Call the appropriate evaluate_agentic_* function for the item's test_kind.""" kind = item.test_kind @@ -106,6 +107,7 @@ def _dispatch_agentic( question=item.question, expected_outputs=_parse_visualization_expected(eo), k=k, + agent_id=agent_id, **lf_kw, ) elif kind == "agentic_metric_skill": @@ -116,6 +118,7 @@ def _dispatch_agentic( question=item.question, expected_output=eo if isinstance(eo, (dict, list)) else {}, k=k, + agent_id=agent_id, **lf_kw, ) elif kind == "agentic_alert_skill": @@ -126,6 +129,7 @@ def _dispatch_agentic( question=item.question, expected_output=eo if isinstance(eo, dict) else {}, k=k, + agent_id=agent_id, **lf_kw, ) elif kind == "agentic_search": @@ -139,6 +143,7 @@ def _dispatch_agentic( question=item.question, expected_tool_call=expected_args, k=k, + agent_id=agent_id, **lf_kw, ) elif kind == "agentic_general_question": @@ -149,6 +154,7 @@ def _dispatch_agentic( question=item.question, expected_output=eo if isinstance(eo, str) else str(eo), k=k, + agent_id=agent_id, **lf_kw, ) elif kind == "agentic_guardrail": @@ -159,6 +165,7 @@ def _dispatch_agentic( question=item.question, expected_output=eo if isinstance(eo, str) else str(eo), k=k, + agent_id=agent_id, **lf_kw, ) elif kind == "agentic_kda_skill": @@ -178,6 +185,7 @@ def _dispatch_agentic( token=token, workspace_id=workspace_id, fixture=ConversationFixture.model_validate(fixture_data), + agent_id=agent_id, **lf_kw, ) else: @@ -197,6 +205,7 @@ def run_agentic_items( run_ts: str, on_item_start: Any = None, on_item_done: Any = None, + agent_id: str | None = None, ) -> EvalReport: """Run agentic items through evaluate_agentic_* and return an EvalReport.""" langfuse = make_langfuse_client() if use_langfuse else None @@ -219,7 +228,9 @@ def run_agentic_items( ) t0 = time.perf_counter() try: - _dispatch_agentic(item, host, token, workspace_id, k, langfuse, run_ts, model_version, reasoning_effort) + _dispatch_agentic( + item, host, token, workspace_id, k, langfuse, run_ts, model_version, reasoning_effort, agent_id + ) item_report.pass_at_k = True item_report.runs = k except AssertionError as exc: diff --git a/packages/gooddata-eval/src/gooddata_eval/cli/main.py b/packages/gooddata-eval/src/gooddata_eval/cli/main.py index 9465602c0..1e40efd77 100644 --- a/packages/gooddata-eval/src/gooddata_eval/cli/main.py +++ b/packages/gooddata-eval/src/gooddata_eval/cli/main.py @@ -2,6 +2,7 @@ """`gd-eval` command-line entry point.""" import argparse +import os import sys import threading from datetime import datetime, timezone @@ -117,6 +118,16 @@ def _build_parser() -> argparse.ArgumentParser: action="store_true", help="Log scores and traces to Langfuse (requires --langfuse-dataset and LANGFUSE_* env vars).", ) + run.add_argument( + "--agent-id", + dest="agent_id", + help=( + "AI Hub agent id every conversation should target (or set GD_EVAL_AGENT_ID). " + "GoodData has no admin-settable default agent -- without this, each conversation " + "falls back to whichever agent the platform's last-used/last-edited heuristic " + "resolves, which may not have every skill under test enabled." + ), + ) models_cmd = sub.add_parser("models", help="List LLM providers and models configured in the org.") models_cmd.add_argument("--host", help="GoodData host URL.") models_cmd.add_argument("--token", help="API token (or set GOODDATA_TOKEN).") @@ -347,6 +358,7 @@ def on_langfuse_item_done( run_ts=run_ts, on_item_start=on_item_start, on_item_done=on_item_done, + agent_id=config.agent_id, ) # --- non-agentic items (single-turn, use Evaluator) --- @@ -357,6 +369,7 @@ def on_langfuse_item_done( workspace_id=config.workspace_id, preserve_failed=config.preserve_failed, reasoning_effort=config.reasoning_effort, + agent_id=config.agent_id, ), SummaryClient(host=config.host, token=config.token, workspace_id=config.workspace_id), ) @@ -449,6 +462,7 @@ def main(argv: list[str] | None = None) -> int: kind=args.kind, preserve_failed=args.preserve_failed, reasoning_effort=args.reasoning_effort, + agent_id=args.agent_id or os.environ.get("GD_EVAL_AGENT_ID"), ) return _run(config) except ( diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py index 3583780bd..e94584f39 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py @@ -434,11 +434,14 @@ def run_agentic_alert_skill( max_iterations: int = _DEFAULT_MAX_ITERATIONS, initial_conversation_id: str | None = None, reasoning_effort: ReasoningEffort | None = None, + agent_id: str | None = None, ) -> AgenticAlertSummary: """Run the alert-skill agentic evaluation K times and return a summary.""" expected = _normalize_expected_output(expected_output) run_results: list[AlertRunResult] = [] - client = ChatClient(host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort) + client = ChatClient( + host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort, agent_id=agent_id + ) sdk = GoodDataSdk.create(host, token) def _run_once(conv_id: str) -> AlertRunResult: @@ -550,6 +553,7 @@ def evaluate_agentic_alert_skill( k: int = _DEFAULT_K, max_iterations: int = _DEFAULT_MAX_ITERATIONS, initial_conversation_id: str | None = None, + agent_id: str | None = None, langfuse: object | None = None, dataset_item_id: str = "", dataset_name: str = "alert_skill", @@ -577,6 +581,7 @@ def evaluate_agentic_alert_skill( max_iterations=max_iterations, initial_conversation_id=initial_conversation_id, reasoning_effort=reasoning_effort, + agent_id=agent_id, ) if langfuse is not None and dataset_item_id: diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py index 876bbb1c6..1e50352c2 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py @@ -275,6 +275,7 @@ def run_agentic_conversation( max_clarification_turns: int = _DEFAULT_MAX_CLARIFICATION_TURNS, initial_conversation_id: str | None = None, reasoning_effort: ReasoningEffort | None = None, + agent_id: str | None = None, ) -> ConversationResult: """Run a multi-turn, multi-skill conversation evaluation (no K-runs). @@ -282,7 +283,9 @@ def run_agentic_conversation( trigger up to *max_clarification_turns* additional rounds of simulated-user replies before the agent produces the expected output. """ - client = ChatClient(host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort) + client = ChatClient( + host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort, agent_id=agent_id + ) sdk = GoodDataSdk.create(host, token) turn_results: list[TurnResult] = [] turn_outputs: dict[str, dict] = {} @@ -408,6 +411,7 @@ def evaluate_agentic_conversation( fixture: ConversationFixture, max_clarification_turns: int = _DEFAULT_MAX_CLARIFICATION_TURNS, initial_conversation_id: str | None = None, + agent_id: str | None = None, langfuse: object | None = None, dataset_item_id: str = "", dataset_name: str = "conversation", @@ -433,6 +437,7 @@ def evaluate_agentic_conversation( max_clarification_turns=max_clarification_turns, initial_conversation_id=initial_conversation_id, reasoning_effort=reasoning_effort, + agent_id=agent_id, ) if langfuse is not None and dataset_item_id: diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py index f2aa494ff..d710b2a45 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py @@ -73,10 +73,13 @@ def run_agentic_general_question( k: int = _DEFAULT_K, initial_conversation_id: str | None = None, reasoning_effort: ReasoningEffort | None = None, + agent_id: str | None = None, ) -> AgenticGeneralQuestionSummary: """Run the general-question agentic evaluation K times and return a summary.""" run_results: list[GeneralQuestionResult] = [] - client = ChatClient(host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort) + client = ChatClient( + host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort, agent_id=agent_id + ) judge = LLMJudge(_GENERAL_QUESTION_EVALUATION_STEPS, model="gpt-4o") try: @@ -149,6 +152,7 @@ def evaluate_agentic_general_question( expected_output: str, k: int = _DEFAULT_K, initial_conversation_id: str | None = None, + agent_id: str | None = None, langfuse: object | None = None, dataset_item_id: str = "", dataset_name: str = "general_question", @@ -175,6 +179,7 @@ def evaluate_agentic_general_question( k=k, initial_conversation_id=initial_conversation_id, reasoning_effort=reasoning_effort, + agent_id=agent_id, ) if langfuse is not None and dataset_item_id: diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py index ae74da5b4..aea303360 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py @@ -70,10 +70,13 @@ def run_agentic_guardrail( k: int = _DEFAULT_K, initial_conversation_id: str | None = None, reasoning_effort: ReasoningEffort | None = None, + agent_id: str | None = None, ) -> AgenticGuardrailSummary: """Run the guardrail agentic evaluation K times and return a summary.""" run_results: list[GuardrailResult] = [] - client = ChatClient(host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort) + client = ChatClient( + host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort, agent_id=agent_id + ) judge = LLMJudge(_GUARDRAIL_EVALUATION_STEPS, model="gpt-4o") try: @@ -146,6 +149,7 @@ def evaluate_agentic_guardrail( expected_output: str, k: int = _DEFAULT_K, initial_conversation_id: str | None = None, + agent_id: str | None = None, langfuse: object | None = None, dataset_item_id: str = "", dataset_name: str = "guardrail", @@ -172,6 +176,7 @@ def evaluate_agentic_guardrail( k=k, initial_conversation_id=initial_conversation_id, reasoning_effort=reasoning_effort, + agent_id=agent_id, ) if langfuse is not None and dataset_item_id: diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py index dc12a2b7f..6d882d286 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py @@ -249,6 +249,7 @@ def run_agentic_metric_skill( max_iterations: int = _DEFAULT_MAX_ITERATIONS, initial_conversation_id: str | None = None, reasoning_effort: ReasoningEffort | None = None, + agent_id: str | None = None, ) -> AgenticMetricSummary: """Run the metric-skill agentic evaluation K times and return a summary. @@ -257,7 +258,9 @@ def run_agentic_metric_skill( """ expected_outputs: list[dict] = expected_output if isinstance(expected_output, list) else [expected_output] run_results: list[MetricRunResult] = [] - client = ChatClient(host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort) + client = ChatClient( + host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort, agent_id=agent_id + ) sdk = GoodDataSdk.create(host, token) try: @@ -311,6 +314,7 @@ def evaluate_agentic_metric_skill( k: int = _DEFAULT_K, max_iterations: int = _DEFAULT_MAX_ITERATIONS, initial_conversation_id: str | None = None, + agent_id: str | None = None, langfuse: object | None = None, dataset_item_id: str = "", dataset_name: str = "metric_skill", @@ -338,6 +342,7 @@ def evaluate_agentic_metric_skill( max_iterations=max_iterations, initial_conversation_id=initial_conversation_id, reasoning_effort=reasoning_effort, + agent_id=agent_id, ) if langfuse is not None and dataset_item_id: diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py index 25053e17b..955818c0e 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py @@ -69,11 +69,14 @@ def run_agentic_search_tool( k: int = _DEFAULT_K, initial_conversation_id: str | None = None, reasoning_effort: ReasoningEffort | None = None, + agent_id: str | None = None, ) -> AgenticSearchSummary: """Run the search-tool agentic evaluation K times (single-turn each).""" run_results: list[SearchResult] = [] - client = ChatClient(host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort) + client = ChatClient( + host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort, agent_id=agent_id + ) try: conv_id_0 = initial_conversation_id if initial_conversation_id is not None else client.create_conversation() try: @@ -140,6 +143,7 @@ def evaluate_agentic_search_tool( expected_tool_call: dict, k: int = _DEFAULT_K, initial_conversation_id: str | None = None, + agent_id: str | None = None, langfuse: object | None = None, dataset_item_id: str = "", dataset_name: str = "search", @@ -166,6 +170,7 @@ def evaluate_agentic_search_tool( k=k, initial_conversation_id=initial_conversation_id, reasoning_effort=reasoning_effort, + agent_id=agent_id, ) if langfuse is not None and dataset_item_id: diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py index 3548b22b7..60134779a 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py @@ -205,6 +205,7 @@ def run_agentic_visualization( max_iterations: int = _DEFAULT_MAX_ITERATIONS, initial_conversation_id: str | None = None, reasoning_effort: ReasoningEffort | None = None, + agent_id: str | None = None, ) -> AgenticRunSummary: """Run K independent conversations and return evaluation results. @@ -213,7 +214,9 @@ def run_agentic_visualization( fresh conversations. Caller-supplied conversations are not deleted; all conversations created by this function are deleted on completion. """ - client = ChatClient(host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort) + client = ChatClient( + host=host, token=token, workspace_id=workspace_id, reasoning_effort=reasoning_effort, agent_id=agent_id + ) run_results: list[RunResult] = [] try: @@ -260,6 +263,7 @@ def evaluate_agentic_visualization( k: int = _DEFAULT_K, max_iterations: int = _DEFAULT_MAX_ITERATIONS, initial_conversation_id: str | None = None, + agent_id: str | None = None, langfuse: object | None = None, dataset_item_id: str = "", dataset_name: str = "visualization", @@ -289,6 +293,7 @@ def evaluate_agentic_visualization( max_iterations=max_iterations, initial_conversation_id=initial_conversation_id, reasoning_effort=reasoning_effort, + agent_id=agent_id, ) if langfuse is not None and dataset_item_id: diff --git a/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py b/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py index 562b466b7..d131b82cc 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py @@ -293,6 +293,7 @@ def __init__( timeout: float = 300.0, preserve_failed: bool = False, reasoning_effort: ReasoningEffort | None = None, + agent_id: str | None = None, ): """Create a chat client bound to one workspace. @@ -307,10 +308,14 @@ def __init__( self._client = httpx.Client(timeout=timeout) self._preserve_failed = preserve_failed self._reasoning_effort = normalize_reasoning_effort(reasoning_effort) + self._agent_id = agent_id def create_conversation(self) -> str: def _do() -> str: - resp = self._client.post(self._base, headers={**self._auth, "Content-Type": "application/json"}) + body = {"agentId": self._agent_id} if self._agent_id else {} + resp = self._client.post( + self._base, headers={**self._auth, "Content-Type": "application/json"}, json=body + ) resp.raise_for_status() body = resp.json() if "conversationId" not in body: diff --git a/packages/gooddata-eval/src/gooddata_eval/core/config.py b/packages/gooddata-eval/src/gooddata_eval/core/config.py index 8cb07794f..06a2dd926 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/config.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/config.py @@ -44,3 +44,4 @@ class RunConfig: kind: str = "visualization" preserve_failed: bool = False reasoning_effort: ReasoningEffort | None = None + agent_id: str | None = None diff --git a/packages/gooddata-eval/tests/test_agentic_runner.py b/packages/gooddata-eval/tests/test_agentic_runner.py new file mode 100644 index 000000000..dbbc3443e --- /dev/null +++ b/packages/gooddata-eval/tests/test_agentic_runner.py @@ -0,0 +1,50 @@ +# (C) 2026 GoodData Corporation. All rights reserved. +from unittest.mock import patch + +from gooddata_eval.cli.agentic_runner import _dispatch_agentic +from gooddata_eval.core.models import DatasetItem + + +def test_dispatch_agentic_passes_agent_id_through_to_alert_skill(): + item = DatasetItem( + id="q1", + dataset_name="ds", + test_kind="agentic_alert_skill", + question="Alert me when spend exceeds 100", + expected_output={"Operator": "GREATER_THAN", "Threshold": 100}, + ) + with patch("gooddata_eval.cli.agentic_runner.evaluate_agentic_alert_skill") as mock_eval: + _dispatch_agentic( + item, + host="https://h", + token="tok", + workspace_id="ws1", + k=1, + langfuse=None, + run_ts="2026-01-01", + model_version_override=None, + agent_id="agent-1", + ) + assert mock_eval.call_args.kwargs["agent_id"] == "agent-1" + + +def test_dispatch_agentic_omits_agent_id_by_default(): + item = DatasetItem( + id="q1", + dataset_name="ds", + test_kind="agentic_metric_skill", + question="Create a metric for total spend", + expected_output={"maql": "SELECT {metric/spend}"}, + ) + with patch("gooddata_eval.cli.agentic_runner.evaluate_agentic_metric_skill") as mock_eval: + _dispatch_agentic( + item, + host="https://h", + token="tok", + workspace_id="ws1", + k=1, + langfuse=None, + run_ts="2026-01-01", + model_version_override=None, + ) + assert mock_eval.call_args.kwargs["agent_id"] is None diff --git a/packages/gooddata-eval/tests/test_cli.py b/packages/gooddata-eval/tests/test_cli.py index 3f42a3f4e..9956ad31f 100644 --- a/packages/gooddata-eval/tests/test_cli.py +++ b/packages/gooddata-eval/tests/test_cli.py @@ -26,6 +26,16 @@ def test_build_run_config_requires_a_source(): cli_main.parse_args(["run", "--host", "h", "--workspace", "w"]) +def test_parse_args_agent_id_flag(): + args = cli_main.parse_args(["run", "--host", "h", "--workspace", "w", "--dataset", "d", "--agent-id", "agent-1"]) + assert args.agent_id == "agent-1" + + +def test_parse_args_agent_id_defaults_to_none(): + args = cli_main.parse_args(["run", "--host", "h", "--workspace", "w", "--dataset", "d"]) + assert args.agent_id is None + + def test_cli_run_end_to_end(monkeypatch, tmp_path, fixtures_dir): # Stub connection + model activation + chat backend so no network is needed. monkeypatch.setattr(cli_main, "resolve_connection", lambda host, token, profile: ("https://h", "tok")) @@ -93,6 +103,106 @@ def _fake_run( assert orjson.loads(out.read_bytes())["runs"]["Test Provider/gpt-5.2"]["summary"]["passed"] == 1 +def _stub_run_for_agent_id_test(monkeypatch, seen_chat_client_kwargs): + monkeypatch.setattr(cli_main, "resolve_connection", lambda host, token, profile: ("https://h", "tok")) + + class _FakeController: + def __init__(self, *a, **k): ... + def get_active(self): + return ActiveLlmProvider(provider_id="prov", default_model_id="gpt-5.2") + + def resolve_and_activate(self, requested, provider=None): + return ResolvedModel( + provider_id="prov", model_id=requested or "gpt-5.2", switched=False, provider_name="Test Provider" + ) + + def restore(self, original): ... + def close(self): ... + + monkeypatch.setattr(cli_main, "WorkspaceModelController", _FakeController) + + def _fake_run(items, backend, *, runs, model, workspace_id, **kw): + return EvalReport(model=model, workspace_id=workspace_id, items=[]) + + monkeypatch.setattr(cli_main, "run_items", _fake_run) + + def _spy_chat_client(**kwargs): + seen_chat_client_kwargs.update(kwargs) + return object() + + monkeypatch.setattr(cli_main, "ChatClient", _spy_chat_client) + + +def test_cli_run_passes_agent_id_flag_to_chat_client(monkeypatch, tmp_path, fixtures_dir): + seen = {} + _stub_run_for_agent_id_test(monkeypatch, seen) + exit_code = cli_main.main( + [ + "run", + "--host", + "https://h", + "--token", + "tok", + "--workspace", + "ws1", + "--dataset", + str(fixtures_dir / "sample_dataset"), + "--agent-id", + "agent-1", + "--json", + str(tmp_path / "res.json"), + ] + ) + assert exit_code == 0 + assert seen["agent_id"] == "agent-1" + + +def test_cli_run_falls_back_to_agent_id_env_var(monkeypatch, tmp_path, fixtures_dir): + monkeypatch.setenv("GD_EVAL_AGENT_ID", "agent-from-env") + seen = {} + _stub_run_for_agent_id_test(monkeypatch, seen) + exit_code = cli_main.main( + [ + "run", + "--host", + "https://h", + "--token", + "tok", + "--workspace", + "ws1", + "--dataset", + str(fixtures_dir / "sample_dataset"), + "--json", + str(tmp_path / "res.json"), + ] + ) + assert exit_code == 0 + assert seen["agent_id"] == "agent-from-env" + + +def test_cli_run_agent_id_omitted_when_unset(monkeypatch, tmp_path, fixtures_dir): + monkeypatch.delenv("GD_EVAL_AGENT_ID", raising=False) + seen = {} + _stub_run_for_agent_id_test(monkeypatch, seen) + exit_code = cli_main.main( + [ + "run", + "--host", + "https://h", + "--token", + "tok", + "--workspace", + "ws1", + "--dataset", + str(fixtures_dir / "sample_dataset"), + "--json", + str(tmp_path / "res.json"), + ] + ) + assert exit_code == 0 + assert seen["agent_id"] is None + + def test_cli_operational_error_exits_nonzero(monkeypatch, fixtures_dir): def _boom(host, token, profile): raise ConnectionError_("Missing token.") diff --git a/packages/gooddata-eval/tests/test_sse_client.py b/packages/gooddata-eval/tests/test_sse_client.py index 88be58abe..6361410ba 100644 --- a/packages/gooddata-eval/tests/test_sse_client.py +++ b/packages/gooddata-eval/tests/test_sse_client.py @@ -388,6 +388,32 @@ def handler(request): assert sleeps == [] +def test_create_conversation_omits_agent_id_by_default(): + # No agent_id given -> unchanged, existing behavior: GoodData's own + # last-used/last-edited default-agent resolution still applies. + seen = {} + + def handler(request): + seen["body"] = request.content + return httpx.Response(200, json={"conversationId": "abc"}) + + client = _client_with_handler(handler) + client.create_conversation() + assert seen["body"] in (b"", b"{}") + + +def test_create_conversation_sends_agent_id_when_given(): + seen = {} + + def handler(request): + seen["body"] = json.loads(request.content) + return httpx.Response(200, json={"conversationId": "abc"}) + + client = _client_with_handler(handler, agent_id="agent-123") + client.create_conversation() + assert seen["body"] == {"agentId": "agent-123"} + + def test_int_env_uses_default_when_unset(monkeypatch): monkeypatch.delenv("GD_TEST_INT", raising=False) assert sse_mod._int_env("GD_TEST_INT", 5) == 5 From fd4922564b896ac85e64acd075f541582f6fc5fe Mon Sep 17 00:00:00 2001 From: Peter Tomko Date: Tue, 4 Aug 2026 09:48:36 +0200 Subject: [PATCH 4/5] fix: satisfy ruff format/lint on CI - sse_client.py: reformat create_conversation() to ruff's line-length rule (the PR's own change). - test_cli.py: drop an unused `original_chat_client` local -- pre-existing on master (same line, unrelated to this PR), but ruff check runs whole-file and blocks this PR's lint-and-format-check job since this test function lives in a file the PR also touches. Co-Authored-By: Claude Sonnet 5 --- .../gooddata-eval/src/gooddata_eval/core/chat/sse_client.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py b/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py index d131b82cc..d1e375c97 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/chat/sse_client.py @@ -313,9 +313,7 @@ def __init__( def create_conversation(self) -> str: def _do() -> str: body = {"agentId": self._agent_id} if self._agent_id else {} - resp = self._client.post( - self._base, headers={**self._auth, "Content-Type": "application/json"}, json=body - ) + resp = self._client.post(self._base, headers={**self._auth, "Content-Type": "application/json"}, json=body) resp.raise_for_status() body = resp.json() if "conversationId" not in body: From f2764acb6cb6d57fe73c5e245e776bd606421dc6 Mon Sep 17 00:00:00 2001 From: Peter Tomko Date: Tue, 4 Aug 2026 09:53:46 +0200 Subject: [PATCH 5/5] test: cover agent_id threading for every agentic test kind codecov flagged one uncovered line in _dispatch_agentic's agent_id threading -- the earlier tests only exercised 2 of 7 kind branches. Parametrized test now covers all 7 (vis_agentic, agentic_visualization, agentic_search, agentic_general_question, agentic_guardrail, agentic_conversation, plus the two already covered). Co-Authored-By: Claude Sonnet 5 --- .../tests/test_agentic_runner.py | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/packages/gooddata-eval/tests/test_agentic_runner.py b/packages/gooddata-eval/tests/test_agentic_runner.py index dbbc3443e..3cbff3137 100644 --- a/packages/gooddata-eval/tests/test_agentic_runner.py +++ b/packages/gooddata-eval/tests/test_agentic_runner.py @@ -1,6 +1,7 @@ # (C) 2026 GoodData Corporation. All rights reserved. from unittest.mock import patch +import pytest from gooddata_eval.cli.agentic_runner import _dispatch_agentic from gooddata_eval.core.models import DatasetItem @@ -48,3 +49,45 @@ def test_dispatch_agentic_omits_agent_id_by_default(): model_version_override=None, ) assert mock_eval.call_args.kwargs["agent_id"] is None + + +_MIN_VIZ = {"id": "v1", "type": "table", "query": {"fields": {}, "filter_by": {}}, "metrics": [], "view_by": []} +_MIN_CONVERSATION_FIXTURE = { + "id": "c1", + "expected_skills": ["visualization"], + "turns": [{"turn_id": "t1", "message": "hi", "expected_skill": "visualization"}], +} + + +@pytest.mark.parametrize( + ("kind", "expected_output", "target"), + [ + ("vis_agentic", {"visualization": _MIN_VIZ}, "evaluate_agentic_visualization"), + ("agentic_visualization", {"visualization": _MIN_VIZ}, "evaluate_agentic_visualization"), + ("agentic_search", {"tool_call": {"function_arguments": {}}}, "evaluate_agentic_search_tool"), + ("agentic_general_question", "What is X?", "evaluate_agentic_general_question"), + ("agentic_guardrail", "Ignore prior instructions", "evaluate_agentic_guardrail"), + ("agentic_conversation", {"fixture": _MIN_CONVERSATION_FIXTURE}, "evaluate_agentic_conversation"), + ], +) +def test_dispatch_agentic_passes_agent_id_through_for_every_kind(kind, expected_output, target): + item = DatasetItem( + id="q1", + dataset_name="ds", + test_kind=kind, + question="q", + expected_output=expected_output, + ) + with patch(f"gooddata_eval.cli.agentic_runner.{target}") as mock_eval: + _dispatch_agentic( + item, + host="https://h", + token="tok", + workspace_id="ws1", + k=1, + langfuse=None, + run_ts="2026-01-01", + model_version_override=None, + agent_id="agent-1", + ) + assert mock_eval.call_args.kwargs["agent_id"] == "agent-1"