diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index af03368..371994d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,13 +1,18 @@ # Until this workflow existed, nothing in the repo ran a test: release.yml only publishes. # Two things had rotted quietly as a result - the docs never mentioned the LLM Judge Scorer -# surface 0.6.36 shipped, and tests/test_span_tree.py had been failing since 40c6f6e changed -# trace_tool_call()'s dual-write behaviour four days after the test was written. +# surface 0.6.36 shipped, and four tests had been asserting on a performance_summary payload +# that 9d45dd1 replaced with real child spans, some of them for over two weeks. # -# It runs the whole suite. Tests for optional extras (crewai, litellm, llamaindex, autogen, -# google-adk) skip themselves via pytest.importorskip when the extra is absent, so the base -# install is green - a skip is honest about not being covered here, where a failure would -# have drowned out real ones. Installing those extras to actually run them is a fair next -# step; it is a slower job, not a different one. +# Two jobs, because they answer different questions at different costs: +# +# tests - the base install on both supported Pythons. Fast feedback on the core, and the +# configuration most contributors actually have. Integration tests for optional +# extras skip themselves here via pytest.importorskip. +# extras - every framework integration installed, so those tests actually run instead of +# skipping. Slower (chromadb, onnxruntime and friends come along), so it runs on +# one Python rather than the matrix. autogen-ext is installed explicitly: its +# ReplayChatCompletionClient is a test double, so it belongs in CI rather than in +# the package's "autogen" extra. name: tests on: @@ -34,3 +39,14 @@ jobs: # tests/test_integration.py talks to the hosted API and skips itself without a key; # no secret is passed here on purpose, so pull requests from forks behave the same. - run: python -m pytest tests/ -q + + extras: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - run: pip install -e ".[all]" autogen-ext pytest + - run: python -m pytest tests/ -q diff --git a/tests/test_integrations.py b/tests/test_integrations.py index fe11dc9..ae2cc26 100644 --- a/tests/test_integrations.py +++ b/tests/test_integrations.py @@ -640,22 +640,32 @@ def kickoff(self, inputs=None): return FakeCrewOutput(raw="final output", tasks_output=[output1, output2]) - tracer = make_tracer() + # Boundary-mocked rather than make_tracer(): each task is a real child span now (9d45dd1 + # replaced the synthetic performance_summary step list), and mocking tracer._send would + # bypass the very _send/_dispatch/child_span chain that builds them. Same idiom as + # tests/test_span_tree.py. + tracer = Tracer(ingest_client=MagicMock()) observer = AgentXCrewObserver(tracer, name="my-crew") result = observer.kickoff(FakeCrew(), inputs={"topic": "AI"}) assert result.raw == "final output" - tracer._send.assert_called_once() - _, kwargs = tracer._send.call_args - steps = kwargs["performance_summary"]["execution_steps"] - assert len(steps) == 2 - assert steps[0]["name"] == "Research topic" - assert steps[1]["name"] == "Write summary" - # Real timing, not an even split — task 1 slept ~3x longer than task 2. - assert steps[0]["duration_ms"] > steps[1]["duration_ms"] * 1.5 - assert steps[0]["output"] == "research done" - assert steps[1]["output"] == "summary done" + wires = [call.args[0] for call in tracer._client.enqueue.call_args_list] + assert len(wires) == 3 + research, summary, root = wires + assert root["name"] == "my-crew" + assert root["output"] == "final output" + + # Each task is its own child span under the crew's root, carrying its own real duration. + assert research["name"] == "Research topic" + assert summary["name"] == "Write summary" + assert research["parent_span_id"] == root["span_id"] + assert summary["parent_span_id"] == root["span_id"] + assert research["output"] == "research done" + assert summary["output"] == "summary done" + # The point of the test: real timing, not an even split — task 1 slept ~3x longer than + # task 2, and the old "divide latency evenly" approximation reported them identical. + assert research["latency_ms"] > summary["latency_ms"] * 1.5 def test_crewai_falls_back_to_even_split_without_event_bus(): @@ -930,6 +940,10 @@ def stream_complete(self, prompt: str, **kwargs: Any): def test_autogen_agent_run_traces_text_reply(): """Drives a real AssistantAgent.run() via AutoGen's own ReplayChatCompletionClient (no network/API keys).""" pytest.importorskip("autogen_agentchat") + # autogen-ext is a separate distribution and is NOT part of the "autogen" extra - + # ReplayChatCompletionClient below is a test double that lives there, so guard it too + # or this fails with ModuleNotFoundError instead of skipping. + pytest.importorskip("autogen_ext") from autogen_agentchat.agents import AssistantAgent from autogen_ext.models.replay import ReplayChatCompletionClient @@ -938,25 +952,33 @@ def test_autogen_agent_run_traces_text_reply(): model_client = ReplayChatCompletionClient(["Hello from AutoGen!"]) agent = AssistantAgent("assistant", model_client=model_client) - tracer = make_tracer() + # Boundary-mocked, not make_tracer(): the agent's turn is a real child span since + # 9d45dd1, and mocking tracer._send would bypass the chain that builds it. + tracer = Tracer(ingest_client=MagicMock()) observer = AgentXAutoGenObserver(tracer, name="my-agent") result = asyncio.run(observer.run(agent, task="Say hello")) assert result.messages[-1].content == "Hello from AutoGen!" - tracer._send.assert_called_once() - _, kwargs = tracer._send.call_args - assert kwargs["input"] == "Say hello" - assert kwargs["output"] == "Hello from AutoGen!" - assert kwargs["input_tokens"] == 22 - assert kwargs["output_tokens"] == 3 - steps = kwargs["performance_summary"]["execution_steps"] - assert len(steps) == 1 - assert steps[0]["output"] == "Hello from AutoGen!" + wires = [call.args[0] for call in tracer._client.enqueue.call_args_list] + assert len(wires) == 2 + step, root = wires + assert root["input"] == "Say hello" + assert root["output"] == "Hello from AutoGen!" + assert root["input_tokens"] == 22 + assert root["output_tokens"] == 3 + # The turn itself, as its own child span rather than a performance_summary step. + assert step["name"] == "assistant" + assert step["output"] == "Hello from AutoGen!" + assert step["parent_span_id"] == root["span_id"] def test_autogen_agent_run_traces_tool_call(): pytest.importorskip("autogen_agentchat") + # autogen-ext is a separate distribution and is NOT part of the "autogen" extra - + # ReplayChatCompletionClient below is a test double that lives there, so guard it too + # or this fails with ModuleNotFoundError instead of skipping. + pytest.importorskip("autogen_ext") import json from autogen_agentchat.agents import AssistantAgent @@ -986,16 +1008,24 @@ def get_weather(city: str) -> str: ) agent = AssistantAgent("assistant", model_client=model_client, tools=[tool]) - tracer = make_tracer() + tracer = Tracer(ingest_client=MagicMock()) observer = AgentXAutoGenObserver(tracer, name="my-agent") asyncio.run(observer.run(agent, task="What is the weather in NYC?")) - tracer._send.assert_called_once() - _, kwargs = tracer._send.call_args - perf = kwargs["performance_summary"] - assert len(perf["tool_calls"]) == 1 - tool_call = perf["tool_calls"][0] + wires = [call.args[0] for call in tracer._client.enqueue.call_args_list] + assert len(wires) == 2 + child, root = wires + # The tool call is a real child span... + assert child["name"] == "get_weather" + assert "NYC" in child["input"] + assert child["output"] == "sunny in NYC" + assert child["parent_span_id"] == root["span_id"] + # ...and is mirrored onto the ROOT's flat tool_calls, which is what the engine's built-in + # "Tool failure" check and the dashboard's Tool quality column read. Same deliberate + # dual-write as trace_tool_call() - see test_span_tree.py. + assert len(root["tool_calls"]) == 1 + tool_call = root["tool_calls"][0] assert tool_call["name"] == "get_weather" assert "NYC" in tool_call["input"] assert tool_call["output"] == "sunny in NYC"