diff --git a/src/anthropic/lib/_parse/_response.py b/src/anthropic/lib/_parse/_response.py index ee8326e5f..06f2af1d9 100644 --- a/src/anthropic/lib/_parse/_response.py +++ b/src/anthropic/lib/_parse/_response.py @@ -20,6 +20,19 @@ def parse_text(text: str, output_format: ResponseFormatT | NotGiven) -> Response return None +def _parse_text_for_stop_reason( + text: str, + output_format: ResponseFormatT | NotGiven, + stop_reason: str | None, +) -> ResponseFormatT | None: + # Structured output is not guaranteed to match the requested schema when + # generation is refused or truncated. Preserve the response and its stop + # reason instead of masking it with a JSON/schema validation exception. + if stop_reason == "refusal" or stop_reason == "max_tokens": + return None + return parse_text(text, output_format) + + def parse_beta_response( *, output_format: ResponseFormatT | NotGiven, @@ -31,7 +44,14 @@ def parse_beta_response( content_list.append( construct_type_unchecked( type_=ParsedBetaTextBlock[ResponseFormatT], - value={**content.to_dict(), "parsed_output": parse_text(content.text, output_format)}, + value={ + **content.to_dict(), + "parsed_output": _parse_text_for_stop_reason( + content.text, + output_format, + response.stop_reason, + ), + }, ) ) else: @@ -57,7 +77,14 @@ def parse_response( content_list.append( construct_type_unchecked( type_=ParsedTextBlock[ResponseFormatT], - value={**content.to_dict(), "parsed_output": parse_text(content.text, output_format)}, + value={ + **content.to_dict(), + "parsed_output": _parse_text_for_stop_reason( + content.text, + output_format, + response.stop_reason, + ), + }, ) ) else: diff --git a/tests/lib/_parse/test_stop_reason_parsing.py b/tests/lib/_parse/test_stop_reason_parsing.py new file mode 100644 index 000000000..858c8fbdf --- /dev/null +++ b/tests/lib/_parse/test_stop_reason_parsing.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +from typing import Literal + +import pytest +from pydantic import BaseModel, ValidationError + +from anthropic import _compat +from anthropic.lib._parse._response import parse_beta_response, parse_response +from anthropic.types import Message, TextBlock, Usage +from anthropic.types.beta import BetaMessage, BetaTextBlock, BetaUsage + + +pytestmark = pytest.mark.skipif(_compat.PYDANTIC_V1, reason="structured outputs not supported with pydantic v1") + + +class ParsedValue(BaseModel): + value: int + + +def _message(stop_reason: Literal["refusal", "max_tokens", "end_turn"], text: str) -> Message: + return Message.construct( + id="msg_test", + type="message", + role="assistant", + content=[TextBlock.construct(type="text", text=text, citations=None)], + model="claude-test", + stop_reason=stop_reason, + stop_sequence=None, + usage=Usage.construct(input_tokens=1, output_tokens=1), + ) + + +def _beta_message(stop_reason: Literal["refusal", "max_tokens", "end_turn"], text: str) -> BetaMessage: + return BetaMessage.construct( + id="msg_test", + type="message", + role="assistant", + content=[BetaTextBlock.construct(type="text", text=text, citations=None)], + model="claude-test", + stop_reason=stop_reason, + stop_sequence=None, + usage=BetaUsage.construct(input_tokens=1, output_tokens=1), + ) + + +@pytest.mark.parametrize( + ("stop_reason", "text"), + [ + ("refusal", "I cannot provide that."), + ("max_tokens", '{"value":'), + ], +) +def test_parse_response_preserves_non_schema_terminal_output( + stop_reason: Literal["refusal", "max_tokens"], + text: str, +) -> None: + response = parse_response(output_format=ParsedValue, response=_message(stop_reason, text)) + + assert response.stop_reason == stop_reason + assert response.parsed_output is None + + +@pytest.mark.parametrize( + ("stop_reason", "text"), + [ + ("refusal", "I cannot provide that."), + ("max_tokens", '{"value":'), + ], +) +def test_parse_beta_response_preserves_non_schema_terminal_output( + stop_reason: Literal["refusal", "max_tokens"], + text: str, +) -> None: + response = parse_beta_response(output_format=ParsedValue, response=_beta_message(stop_reason, text)) + + assert response.stop_reason == stop_reason + assert response.parsed_output is None + + +def test_completed_response_still_validates_structured_output() -> None: + with pytest.raises(ValidationError): + parse_response(output_format=ParsedValue, response=_message("end_turn", "not json")) + + with pytest.raises(ValidationError): + parse_beta_response(output_format=ParsedValue, response=_beta_message("end_turn", "not json"))