From 0b57ca5d81d7ff6a254232371f27af5d00bcb257 Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Wed, 16 Sep 2026 15:13:34 -0400 Subject: [PATCH 1/4] feat(integrations): add discovery engine v1 instrumentation Trace Discovery Engine answer generation, conversations, ranking, and grounding checks through Braintrust's integrations API. ## Automatic instrumentation Install google-cloud-discoveryengine alongside braintrust, configure Google Cloud credentials, then enable automatic instrumentation before making calls: ```python import braintrust from google.cloud import discoveryengine_v1 as discoveryengine braintrust.init_logger(project="my-braintrust-project") braintrust.auto_instrument() client = discoveryengine.ConversationalSearchServiceClient() response = client.answer_query(request={ "serving_config": serving_config, "query": {"text": "What was last year's revenue?"}, }) ``` Discovery Engine is enabled by default. Importing the provider before or after setup works. Repeated setup is idempotent. To opt out while enabling other integrations, use braintrust.auto_instrument(discoveryengine=False). For explicit package-level setup or one client instance, use these alternatives: ```python from braintrust.integrations.discoveryengine import ( setup_discoveryengine, wrap_discoveryengine, ) setup_discoveryengine() # Instrument supported Discovery Engine v1 clients. # Alternatively, wrap only this instance: client = wrap_discoveryengine(discoveryengine.RankServiceClient()) ``` ## Supported versions and APIs The integration requires google-cloud-discoveryengine >= 0.20.3. The provider matrix currently tests 0.20.3; later releases pass the minimum-version gate but are not covered by this commit's recordings. Targets are the v1 clients only: | Client (and corresponding AsyncClient) | Methods | | --- | --- | | ConversationalSearchServiceClient | answer_query, stream_answer_query, converse_conversation | | RankServiceClient | rank | | GroundedGenerationServiceClient | check_grounding | Patch exactly these ten public sync/async methods. Search, Assistant, CRUD, v1alpha/v1beta, and answer_query(asynchronous_mode=True) are excluded. Defer generate_grounded_content and stream_generate_grounded_content because live v1 endpoints returned method-not-found errors. ## Span shape and Google integration coexistence Each supported call emits one llm span with provider="google" and origin "discoveryengine-auto". Log a model only when requested and omit unavailable token metrics. Streaming aggregates into one output and records time to first text; Google's final complete answer replaces accumulated deltas. Explicit closure/cancellation finalizes partial output without leaving the span current between reads. Retain structured citations and cap logged ranking output at 100 records without changing the provider's returned result. With a user-created task span around separate calls: ```text answer workflow [task; created by the application] |-- discoveryengine.rank [llm] |-- discoveryengine.stream_answer_query [llm; one span for the entire stream] `-- generate_content [llm; separate google.genai call, if made] ``` Discovery Engine targets google.cloud.discoveryengine_v1; the existing Google GenAI integration targets google.genai. Discovery Engine calls its own service transport rather than invoking the local GenAI client, so enabling both does not create a duplicate GenAI span for the same Discovery Engine request. Server-side retrieval/model work is not exposed as local child spans. Other Google integrations retain their own patch scopes. This integration does not patch shared Google authentication, HTTP, or gRPC code. Calls made inside an instrumented agent/tool follow its active parent span; simply enabling all integrations does not create an agent parent. The independent discoveryengine and google_genai flags allow either integration to be disabled at setup. ## Validation and recording Use real REST VCR and async gRPC recordings. The test-only gRPC helper replaces one callable on one transport instance and restores it after each test. Replay matches requests and reconstructs responses/errors; it does not validate network behavior, retry timing, or real cancellation. The final suite has 26 passing offline tests, including automatic import order, manual/setup idempotence, span shape, stream lifecycle, and error propagation. The broader core suite passed with 853 tests; pylint and pre-commit checks pass. Earlier red/green checks exposed duplicate final streaming snapshots and repeated conversation summaries before their fixes. Document recording prerequisites and commands in the SDK VCR workflow skill. Live recording requires BRAINTRUST_DISCOVERYENGINE_PROJECT and, for generation resources, BRAINTRUST_DISCOVERYENGINE_APP and BRAINTRUST_DISCOVERYENGINE_DATASTORE, plus gcloud ADC. Playback derives resource paths from cassettes and needs no access to the recorded project. No private-resource defaults remain in test code. Refs #773 --- .agents/skills/sdk-vcr-workflows/SKILL.md | 76 + py/noxfile.py | 11 + py/pyproject.toml | 6 + py/src/braintrust/auto.py | 5 + py/src/braintrust/integrations/__init__.py | 2 + .../test_auto_discoveryengine.py | 63 + .../integrations/discoveryengine/__init__.py | 12 + .../discoveryengine/_test_grpc.py | 144 ++ .../latest/test_answer_query[False].yaml | 423 ++++ .../latest/test_answer_query[True].yaml | 864 ++++++++ .../test_answer_requested_model[False].yaml | 165 ++ .../test_answer_requested_model[True].yaml | 58 + .../latest/test_async_grpc[answer_query].json | 501 +++++ .../test_async_grpc[check_grounding].json | 44 + ...est_async_grpc[converse_conversation].json | 497 +++++ .../latest/test_async_grpc[rank].json | 33 + .../test_async_grpc[stream_answer_query].json | 1755 +++++++++++++++++ .../latest/test_async_provider_error.json | 26 + .../test_async_stream_provider_error.json | 18 + .../latest/test_check_grounding.yaml | 60 + .../latest/test_converse_conversation.yaml | 514 +++++ .../cassettes/latest/test_provider_error.yaml | 57 + .../cassettes/latest/test_rank.yaml | 60 + .../latest/test_rank_output_limit.yaml | 296 +++ .../latest/test_stream_provider_error.yaml | 56 + .../discoveryengine/integration.py | 13 + .../integrations/discoveryengine/patchers.py | 92 + .../discoveryengine/test_discoveryengine.py | 630 ++++++ .../integrations/discoveryengine/tracing.py | 319 +++ py/uv.lock | 136 +- 30 files changed, 6901 insertions(+), 35 deletions(-) create mode 100644 py/src/braintrust/integrations/auto_test_scripts/test_auto_discoveryengine.py create mode 100644 py/src/braintrust/integrations/discoveryengine/__init__.py create mode 100644 py/src/braintrust/integrations/discoveryengine/_test_grpc.py create mode 100644 py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_answer_query[False].yaml create mode 100644 py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_answer_query[True].yaml create mode 100644 py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_answer_requested_model[False].yaml create mode 100644 py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_answer_requested_model[True].yaml create mode 100644 py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_async_grpc[answer_query].json create mode 100644 py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_async_grpc[check_grounding].json create mode 100644 py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_async_grpc[converse_conversation].json create mode 100644 py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_async_grpc[rank].json create mode 100644 py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_async_grpc[stream_answer_query].json create mode 100644 py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_async_provider_error.json create mode 100644 py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_async_stream_provider_error.json create mode 100644 py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_check_grounding.yaml create mode 100644 py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_converse_conversation.yaml create mode 100644 py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_provider_error.yaml create mode 100644 py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_rank.yaml create mode 100644 py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_rank_output_limit.yaml create mode 100644 py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_stream_provider_error.yaml create mode 100644 py/src/braintrust/integrations/discoveryengine/integration.py create mode 100644 py/src/braintrust/integrations/discoveryengine/patchers.py create mode 100644 py/src/braintrust/integrations/discoveryengine/test_discoveryengine.py create mode 100644 py/src/braintrust/integrations/discoveryengine/tracing.py diff --git a/.agents/skills/sdk-vcr-workflows/SKILL.md b/.agents/skills/sdk-vcr-workflows/SKILL.md index c35ae5a1e..cbbb33e1d 100644 --- a/.agents/skills/sdk-vcr-workflows/SKILL.md +++ b/.agents/skills/sdk-vcr-workflows/SKILL.md @@ -266,6 +266,82 @@ Important differences: Do not try to force ordinary HTTP VCR patterns onto Claude Agent SDK subprocess tests. +## Discovery Engine Recording + +Discovery Engine tests use HTTP VCR for sync REST calls and the test-only +`integrations/discoveryengine/_test_grpc.py` helper for async gRPC calls. Both +recording formats live under `py/src/braintrust/integrations/discoveryengine/cassettes//`. + +### Prerequisites + +- Install `gcloud` and use a Google Cloud project with billing and the + `discoveryengine.googleapis.com` API enabled. +- Authenticate with Application Default Credentials (ADC). The account needs + permission to invoke Discovery Engine in the target project. A `GEMINI_API_KEY` + alone does not authenticate these tests. +- For answer and conversation tests, create a search app with generative responses + enabled and attach a populated datastore. Wait for indexing to finish before + recording. Ranking and grounding-check tests do not require indexed documents. + +```sh +gcloud config set project YOUR_PROJECT_ID +gcloud auth application-default login +gcloud auth application-default set-quota-project YOUR_PROJECT_ID +``` + +The recording fixture obtains an access token using +`gcloud auth application-default print-access-token`, outside the recorded call. +Playback uses anonymous credentials and needs no Google account. + +Live recording requires explicit resource IDs through environment variables. +There are no private-project defaults in the test code. Tests use the `global` +location. Ranking and grounding checks require the project; answer and +conversation coverage also requires the app and datastore. + +A datastore can be populated from Google's public Alphabet earnings-report +sample PDFs at `gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs`. +Wait for indexing to finish before recording. + +```sh +export BRAINTRUST_DISCOVERYENGINE_PROJECT="your-project" +export BRAINTRUST_DISCOVERYENGINE_APP="your-app-id" +export BRAINTRUST_DISCOVERYENGINE_DATASTORE="your-datastore-id" +``` + +### Record and replay + +From `py/`, select a focused scenario. `--vcr-record=all` enables recording for +both REST and gRPC; the gRPC helper otherwise requires an existing cassette. + +```sh +# REST ranking, including manual/setup entry-point coverage. +mise exec -- nox -s 'test_discoveryengine(latest)' -- --vcr-record=all -k 'test_rank and not test_rank_output_limit' + +# Async gRPC ranking. +mise exec -- nox -s 'test_discoveryengine(latest)' -- --vcr-record=all -k 'test_async_grpc and rank' + +# Verify all recordings without network access to Google. +mise exec -- nox -R -s 'test_discoveryengine(latest)' -- --vcr-record=none +``` + +Playback derives resource paths from the checked-in REST cassettes and ignores +resource environment overrides. No access to the recorded project's resources is +needed. The auto-instrument subprocess test also reads its ranking resource from +the cassette. Keep the project's recordings consistent when recording against a +different project. + +The gRPC helper temporarily replaces one callable on one client transport, +records real protobuf requests/responses and errors as JSON, then restores the +callable. Playback reconstructs provider response objects and checks requests; +it does not exercise networking, retries, or real cancellation behavior. +HTTP authorization headers are filtered by shared VCR configuration; gRPC +recordings omit credentials and transport metadata. Inspect new recordings before +checking them in. + +`generate_grounded_content` and `stream_generate_grounded_content` are deferred: +the v1 endpoints returned method-not-found errors during initial recording. +Do not fabricate success recordings for those methods. + ## Relationship To Other Skills - Use `sdk-integrations` when the main task is integration implementation, patchers, tracing, or provider package structure. diff --git a/py/noxfile.py b/py/noxfile.py index 351d9383b..8240e1017 100644 --- a/py/noxfile.py +++ b/py/noxfile.py @@ -569,6 +569,17 @@ def test_google_genai(session, version): _run_tests(session, f"{INTEGRATION_DIR}/google_genai/test_google_genai.py", version=version) +DISCOVERYENGINE_VERSIONS = _get_matrix_versions("google-cloud-discoveryengine") + + +@nox.session() +@nox.parametrize("version", DISCOVERYENGINE_VERSIONS, ids=DISCOVERYENGINE_VERSIONS) +def test_discoveryengine(session, version): + _install_test_deps(session) + _install_matrix_dep(session, "google-cloud-discoveryengine", version) + _run_tests(session, f"{INTEGRATION_DIR}/discoveryengine", version=version) + + DSPY_VERSIONS = _get_matrix_versions("dspy") diff --git a/py/pyproject.toml b/py/pyproject.toml index 5b3109600..5c2d11705 100644 --- a/py/pyproject.toml +++ b/py/pyproject.toml @@ -267,6 +267,7 @@ lint = [ "dspy", "google-adk", "google-genai", + "google-cloud-discoveryengine", "instructor", "litellm>=1.83.10", "livekit-agents", @@ -462,6 +463,9 @@ latest = "google-genai==2.23.0" "1.75.0" = "google-genai==1.75.0" "1.30.0" = "google-genai==1.30.0" +[tool.braintrust.matrix.google-cloud-discoveryengine] +latest = "google-cloud-discoveryengine==0.20.3" + [tool.braintrust.matrix.dspy] latest = "dspy==3.3.1" "2.6.0" = "dspy==2.6.0" @@ -571,6 +575,7 @@ cursor_sdk = ["cursor-sdk"] crewai = ["crewai"] dspy = ["dspy"] google_genai = ["google-genai"] +discoveryengine = ["google-cloud-discoveryengine"] huggingface_hub = ["huggingface-hub"] harbor = ["harbor"] instructor = ["instructor"] @@ -604,6 +609,7 @@ cursor-sdk = "cursor_sdk" dspy = "dspy" google-adk = "google.adk" google-genai = "google.genai" +google-cloud-discoveryengine = "google.cloud.discoveryengine_v1" litellm = "litellm" livekit-agents = "livekit.agents" mistralai = "mistralai" diff --git a/py/src/braintrust/auto.py b/py/src/braintrust/auto.py index 40b8f0060..ce628c294 100644 --- a/py/src/braintrust/auto.py +++ b/py/src/braintrust/auto.py @@ -19,6 +19,7 @@ CohereIntegration, CrewAIIntegration, CursorSDKIntegration, + DiscoveryEngineIntegration, DSPyIntegration, GoogleGenAIIntegration, HuggingFaceHubIntegration, @@ -64,6 +65,7 @@ def auto_instrument( ai_sdk: bool = True, pydantic_ai: bool = True, google_genai: bool = True, + discoveryengine: bool = True, instructor: bool = True, openrouter: bool = True, mistral: bool = True, @@ -102,6 +104,7 @@ def auto_instrument( litellm: Enable LiteLLM instrumentation (default: True) ai_sdk: Enable Vercel AI SDK for Python instrumentation (default: True) pydantic_ai: Enable Pydantic AI instrumentation (default: True) + discoveryengine: Enable Google Discovery Engine v1 instrumentation (default: True) google_genai: Enable Google GenAI instrumentation (default: True) instructor: Enable Instructor (structured-output) instrumentation (default: True) openrouter: Enable OpenRouter instrumentation (default: True) @@ -184,6 +187,8 @@ def auto_instrument( results["pydantic_ai"] = _instrument_integration(PydanticAIIntegration) if google_genai: results["google_genai"] = _instrument_integration(GoogleGenAIIntegration) + if discoveryengine: + results["discoveryengine"] = _instrument_integration(DiscoveryEngineIntegration) if instructor: results["instructor"] = _instrument_integration(InstructorIntegration) if openrouter: diff --git a/py/src/braintrust/integrations/__init__.py b/py/src/braintrust/integrations/__init__.py index 024428190..5995cbcb0 100644 --- a/py/src/braintrust/integrations/__init__.py +++ b/py/src/braintrust/integrations/__init__.py @@ -9,6 +9,7 @@ from .cohere import CohereIntegration from .crewai import CrewAIIntegration from .cursor_sdk import CursorSDKIntegration +from .discoveryengine import DiscoveryEngineIntegration from .dspy import DSPyIntegration from .google_genai import GoogleGenAIIntegration from .huggingface_hub import HuggingFaceHubIntegration @@ -41,6 +42,7 @@ "CrewAIIntegration", "CursorSDKIntegration", "DSPyIntegration", + "DiscoveryEngineIntegration", "GoogleGenAIIntegration", "HuggingFaceHubIntegration", "InstructorIntegration", diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_discoveryengine.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_discoveryengine.py new file mode 100644 index 000000000..b9ed32569 --- /dev/null +++ b/py/src/braintrust/integrations/auto_test_scripts/test_auto_discoveryengine.py @@ -0,0 +1,63 @@ +"""Both import orders and opt-out, using the real REST ranking cassette.""" + +import inspect +import subprocess +import sys +from pathlib import Path +from urllib.parse import urlsplit + +import yaml +from braintrust.auto import auto_instrument +from braintrust.integrations.conftest import _versioned_cassette_dir +from braintrust.integrations.test_utils import autoinstrument_test_context + + +if len(sys.argv) == 1: + for order in ("before", "after"): + subprocess.run([sys.executable, __file__, order], check=True) + print("SUCCESS") + sys.exit(0) + +options = {name: False for name in inspect.signature(auto_instrument).parameters} +assert auto_instrument(**options) == {} +RankServiceClient = None +if sys.argv[1] == "before": + from google.cloud.discoveryengine_v1 import RankServiceClient + +options["discoveryengine"] = True +assert auto_instrument(**options) == {"discoveryengine": True} +assert auto_instrument(**options) == {"discoveryengine": True} +from google.auth.credentials import AnonymousCredentials + + +if sys.argv[1] == "after": + from google.cloud.discoveryengine_v1 import RankServiceClient + + +cassette_dir = Path(_versioned_cassette_dir(str(Path(__file__).parent.parent / "discoveryengine" / "cassettes"))) +cassette = yaml.safe_load((cassette_dir / "test_rank.yaml").read_text()) +ranking_config = urlsplit(cassette["interactions"][0]["request"]["uri"]).path.removeprefix("/v1/").split(":rank")[0] + +assert RankServiceClient is not None +with autoinstrument_test_context( + "test_rank", integration="discoveryengine", vcr_config={"record_mode": "none"} +) as memory_logger: + client = RankServiceClient(transport="rest", credentials=AnonymousCredentials()) + result = client.rank( + request={ + "ranking_config": ranking_config, + "model": "semantic-ranker-512@latest", + "query": "What is Braintrust?", + "records": [ + {"id": "1", "content": "Braintrust is a platform for evaluating and monitoring AI applications."}, + {"id": "2", "content": "The moon orbits the Earth."}, + ], + "top_n": 1, + }, + retry=None, + ) + assert result.records[0].id == "1" + spans = memory_logger.pop() + assert len(spans) == 1 + assert spans[0]["metadata"]["provider"] == "google" + assert spans[0]["context"]["span_origin"]["instrumentation"]["name"] == "discoveryengine-auto" diff --git a/py/src/braintrust/integrations/discoveryengine/__init__.py b/py/src/braintrust/integrations/discoveryengine/__init__.py new file mode 100644 index 000000000..e6b7b2d20 --- /dev/null +++ b/py/src/braintrust/integrations/discoveryengine/__init__.py @@ -0,0 +1,12 @@ +"""Braintrust integration for google-cloud-discoveryengine v1.""" + +from .integration import DiscoveryEngineIntegration +from .patchers import wrap_discoveryengine + + +__all__ = ["DiscoveryEngineIntegration", "setup_discoveryengine", "wrap_discoveryengine"] + + +def setup_discoveryengine() -> bool: + """Instrument supported v1 clients in this process.""" + return DiscoveryEngineIntegration.setup() diff --git a/py/src/braintrust/integrations/discoveryengine/_test_grpc.py b/py/src/braintrust/integrations/discoveryengine/_test_grpc.py new file mode 100644 index 000000000..685f1267b --- /dev/null +++ b/py/src/braintrust/integrations/discoveryengine/_test_grpc.py @@ -0,0 +1,144 @@ +"""Test-only recording at a single generated client's gRPC callable boundary. + +The public GAPIC method still coerces/validates requests and dispatches normally. +Only the selected entry in this transport instance's ``_wrapped_methods`` is +replaced. No global gRPC patches, credentials, or auth metadata are recorded. +""" + +import json +from contextlib import contextmanager + +from google.api_core import exceptions +from grpc.aio import EOF +from wrapt import ObjectProxy + + +def _encode(message): + return type(message).to_dict(message, preserving_proto_field_name=True) + + +def _raise_recorded_error(exchange): + error = exchange.get("error") + if error: + error_type = { + "InvalidArgument": exceptions.InvalidArgument, + "InternalServerError": exceptions.InternalServerError, + }[error["type"]] + raise error_type(error["message"]) + + +def _record_error(exchange, error): + exchange["error"] = {"type": type(error).__name__, "message": error.message} + + +class _ReplayStream: + def __init__(self, exchange, response_type): + self.exchange = exchange + self.responses = iter(exchange["responses"]) + self.response_type = response_type + self._cancelled = False + + def __aiter__(self): + return self + + async def __anext__(self): + if self._cancelled: + raise StopAsyncIteration + try: + response = next(self.responses) + except StopIteration: + _raise_recorded_error(self.exchange) + raise StopAsyncIteration from None + return self.response_type(response) + + async def read(self): + try: + return await self.__anext__() + except StopAsyncIteration: + return EOF + + def cancel(self): + self._cancelled = True + return True + + def cancelled(self): + return self._cancelled + + +class _RecordingStream(ObjectProxy): + def __init__(self, stream, exchange): + super().__init__(stream) + self._self_exchange = exchange + self._self_iterator = None + + def __aiter__(self): + return self + + async def __anext__(self): + try: + if self._self_iterator is None: + self._self_iterator = self.__wrapped__.__aiter__() + response = await self._self_iterator.__anext__() + except exceptions.GoogleAPICallError as error: + _record_error(self._self_exchange, error) + raise + self._self_exchange["responses"].append(_encode(response)) + return response + + async def read(self): + try: + response = await self.__wrapped__.read() + except exceptions.GoogleAPICallError as error: + _record_error(self._self_exchange, error) + raise + if response is not EOF: + self._self_exchange["responses"].append(_encode(response)) + return response + + +@contextmanager +def grpc_cassette(client, method, response_type, path, *, record=False, streaming=False): + transport = client.transport + rpc = getattr(transport, method) + original = transport._wrapped_methods[rpc] + saved = None if record else json.loads(path.read_text()) + exchange = {"method": method, "requests": [], "responses": []} + + def observe_request(request): + exchange["requests"].append(_encode(request)) + if saved is not None: + index = len(exchange["requests"]) - 1 + assert exchange["requests"][index] == saved["requests"][index] + return request + + async def invoke(request, **kwargs): + observe_request(request) + if saved is not None: + assert saved["method"] == method + if saved.get("error_at") == "call": + _raise_recorded_error(saved) + if streaming: + return _ReplayStream(saved, response_type) + _raise_recorded_error(saved) + return response_type(saved["responses"][0]) + try: + result = await original(request, **kwargs) + except exceptions.GoogleAPICallError as error: + _record_error(exchange, error) + exchange["error_at"] = "call" + raise + if streaming: + return _RecordingStream(result, exchange) + exchange["responses"].append(_encode(result)) + return result + + transport._wrapped_methods[rpc] = invoke + try: + yield + finally: + transport._wrapped_methods[rpc] = original + if record and (exchange["responses"] or "error" in exchange): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(exchange, indent=2) + "\n") + elif saved is not None: + assert len(exchange["requests"]) == len(saved["requests"]) diff --git a/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_answer_query[False].yaml b/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_answer_query[False].yaml new file mode 100644 index 000000000..9651910b8 --- /dev/null +++ b/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_answer_query[False].yaml @@ -0,0 +1,423 @@ +interactions: +- request: + body: "{\n \"query\": {\n \"text\": \"What was Alphabet's revenue in 2022?\"\n + \ },\n \"answerGenerationSpec\": {\n \"includeCitations\": true\n }\n}" + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '133' + Content-Type: + - application/json + User-Agent: + - python-requests/2.34.2 + x-goog-api-client: + - gl-python/3.14.7 grpc/1.84.0 gax/2.37.0 gapic/0.20.3 pb/7.36.1 cred-type/u + x-goog-request-params: + - serving_config=projects/sdk-dev-508013/locations/global/collections/default_collection/engines/braintrust-sdk-test_1789580029521/servingConfigs/default_search + method: POST + uri: https://discoveryengine.googleapis.com/v1/projects/sdk-dev-508013/locations/global/collections/default_collection/engines/braintrust-sdk-test_1789580029521/servingConfigs/default_search:answer?%24alt=json%3Benum-encoding%3Dint + response: + body: + string: "{\n \"answer\": {\n \"state\": 3,\n \"answerText\": \"Alphabet's + total revenue in 2022 was $282,836 million. This represents a 10% increase + year-over-year, or 14% in constant currency.\\n\\nHere's a breakdown of Alphabet's + 2022 revenues by segment and type:\\n\\n**By Segment:**\\n* Google Services: + $253,528 million\\n* Google Cloud: $26,280 million\\n* Other Bets: $1,068 + million\\n* Hedging gains (losses): $1,960 million\\n\\n**By Type (within + Google Services):**\\n* Google Search & other: $162,450 million\\n* YouTube + ads: $29,243 million\\n* Google Network: $32,780 million\\n* Google advertising + (total): $224,473 million\\n* Google other: $29,055 million\\n\\n**By Geography:**\\n* + \ United States: $134,814 million (48%)\\n* EMEA (Europe, Middle East, + and Africa): $82,062 million (29%)\\n* APAC (Asia-Pacific): $47,024 million + (16%)\\n* Other Americas: $16,976 million (6%)\\n* Hedging gains (losses): + $1,960 million (1%)\\n\\nAlphabet's CFO, Ruth Porat, stated that Q4 2022 consolidated + revenues were $76 billion, up 1% year over year, or up 7% in constant currency, + and the full year 2022 revenue was $283 billion, up 10%, or up 14% in constant + currency.\\n\\nFor the first quarter of 2022, Alphabet reported revenues of + $68,011 million, a 23% year-over-year growth. In the third quarter of 2022, + revenues were $69.1 billion, up 6% versus the prior year or up 11% on a constant + currency basis.\",\n \"citations\": [\n {\n \"endIndex\": \"54\",\n + \ \"sources\": [\n {\n \"referenceId\": \"0\"\n + \ },\n {\n \"referenceId\": \"4\"\n },\n + \ {\n \"referenceId\": \"8\"\n },\n {\n + \ \"referenceId\": \"9\"\n }\n ]\n },\n {\n + \ \"startIndex\": \"55\",\n \"endIndex\": \"130\",\n \"sources\": + [\n {\n \"referenceId\": \"2\"\n }\n ]\n + \ },\n {\n \"startIndex\": \"217\",\n \"endIndex\": + \"254\",\n \"sources\": [\n {\n \"referenceId\": + \"8\"\n },\n {\n \"referenceId\": \"9\"\n }\n + \ ]\n },\n {\n \"startIndex\": \"255\",\n \"endIndex\": + \"288\",\n \"sources\": [\n {\n \"referenceId\": + \"8\"\n },\n {\n \"referenceId\": \"9\"\n }\n + \ ]\n },\n {\n \"startIndex\": \"289\",\n \"endIndex\": + \"319\",\n \"sources\": [\n {\n \"referenceId\": + \"8\"\n },\n {\n \"referenceId\": \"9\"\n }\n + \ ]\n },\n {\n \"startIndex\": \"320\",\n \"endIndex\": + \"362\",\n \"sources\": [\n {\n \"referenceId\": + \"8\"\n },\n {\n \"referenceId\": \"9\"\n }\n + \ ]\n },\n {\n \"startIndex\": \"402\",\n \"endIndex\": + \"445\",\n \"sources\": [\n {\n \"referenceId\": + \"9\"\n }\n ]\n },\n {\n \"startIndex\": + \"446\",\n \"endIndex\": \"478\",\n \"sources\": [\n {\n + \ \"referenceId\": \"9\"\n }\n ]\n },\n {\n + \ \"startIndex\": \"479\",\n \"endIndex\": \"514\",\n \"sources\": + [\n {\n \"referenceId\": \"9\"\n }\n ]\n + \ },\n {\n \"startIndex\": \"515\",\n \"endIndex\": + \"563\",\n \"sources\": [\n {\n \"referenceId\": + \"9\"\n }\n ]\n },\n {\n \"startIndex\": + \"564\",\n \"endIndex\": \"597\",\n \"sources\": [\n {\n + \ \"referenceId\": \"9\"\n }\n ]\n },\n {\n + \ \"startIndex\": \"617\",\n \"endIndex\": \"658\",\n \"sources\": + [\n {\n \"referenceId\": \"9\"\n }\n ]\n + \ },\n {\n \"startIndex\": \"659\",\n \"endIndex\": + \"724\",\n \"sources\": [\n {\n \"referenceId\": + \"9\"\n }\n ]\n },\n {\n \"startIndex\": + \"725\",\n \"endIndex\": \"771\",\n \"sources\": [\n {\n + \ \"referenceId\": \"9\"\n }\n ]\n },\n {\n + \ \"startIndex\": \"772\",\n \"endIndex\": \"812\",\n \"sources\": + [\n {\n \"referenceId\": \"9\"\n }\n ]\n + \ },\n {\n \"startIndex\": \"813\",\n \"endIndex\": + \"860\",\n \"sources\": [\n {\n \"referenceId\": + \"9\"\n }\n ]\n },\n {\n \"startIndex\": + \"862\",\n \"endIndex\": \"1091\",\n \"sources\": [\n {\n + \ \"referenceId\": \"2\"\n }\n ]\n },\n {\n + \ \"startIndex\": \"1093\",\n \"endIndex\": \"1199\",\n \"sources\": + [\n {\n \"referenceId\": \"1\"\n }\n ]\n + \ },\n {\n \"startIndex\": \"1200\",\n \"endIndex\": + \"1326\",\n \"sources\": [\n {\n \"referenceId\": + \"5\"\n }\n ]\n }\n ],\n \"references\": [\n {\n + \ \"chunkInfo\": {\n \"content\": \"ALPHABET INC. ANNUAL REPORT + 45 PART II\\nITEM\_8\_\_FINANCIAL STATEMENTS AND SUPPLEMENTARY DATA Alphabet + Inc. CONSOLIDATED STATEMENTS OF INCOME (In millions, except per share amounts) + Year Ended December 31,\\n2020 2021 2022 Revenues $ 182,527 $ 257,637 $ 282,836 + Costs and expenses:\\nCost of revenues 84,732 110,939 126,203 Research and + development 27,573 31,562 39,500 Sales and marketing 17,946 22,912 26,567 + General and administrative 11,052 13,510 15,724 Total costs and expenses 141,303 + 178,923 207,994 Income from operations 41,224 78,714 74,842 Other income (expense), + net 6,858 12,020 (3,514) Income before income taxes 48,082 90,734 71,328 Provision + for income taxes 7,813 14,701 11,356 Net income $ 40,269 $ 76,033 $ 59,972 + Basic net income per share of Class A, Class B, and Class C stock $ 2.96 $ + 5.69 $ 4.59 Diluted net income per share of Class A, Class B, and Class C + stock $ 2.93 $ 5.61 $ 4.56 See accompanying notes. \",\n \"relevanceScore\": + 0.9,\n \"documentMetadata\": {\n \"document\": \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/276cee4c4086600303bc561483691f9a\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022_alphabet_annual_report.pdf\",\n + \ \"title\": \"2022_alphabet_annual_report\",\n \"pageIdentifier\": + \"83\"\n }\n }\n },\n {\n \"chunkInfo\": + {\n \"content\": \"Alphabet Announces First Quarter 2022 Results\\nMOUNTAIN + VIEW, Calif. \u2013 April 26, 2022 \u2013 Alphabet Inc. (NASDAQ: GOOG, GOOGL) + today announced financial\\nresults for the quarter ended March 31, 2022. + Sundar Pichai, CEO of Alphabet and Google, said: \u201CQ1 saw strong growth + in Search and Cloud, in particular, which\\nare both helping people and businesses + as the digital transformation continues. We\u2019ll keep investing in great\\nproducts + and services, and creating opportunities for partners and local communities + around the world.\u201D Ruth Porat, CFO of Alphabet and Google, said: \u201CWe + are pleased with Q1 revenue growth of 23% year over year. We\\ncontinue to + make considered investments in Capex, R&D and talent to support long-term + value creation for all\\nstakeholders.\u201D Q1 2022 financial highlights\\nThe + following table summarizes our consolidated financial results for the quarters + ended March 31, 2021 and 2022\\n(in millions, except for per share information + and percentages; unaudited). Quarter Ended March 31,\\n2021 2022 Revenues + $ 55,314 $ 68,011 Change in revenues year over year 34 % 23 % Change in constant + currency revenues year over year(1) 32 % 26 % Operating income $ 16,437 $ + 20,094 Operating margin 30 % 30 % Other income (expense), net $ 4,846 $ (1,160) + Net income $ 17,930 $ 16,436 \",\n \"relevanceScore\": 0.9,\n \"documentMetadata\": + {\n \"document\": \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/c201efed2be20c6a4116611ea16be9cd\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q1_alphabet_earnings_release.pdf\",\n + \ \"title\": \"GOOG Exhibit 99.1 Q1 2022\",\n \"pageIdentifier\": + \"1\"\n }\n }\n },\n {\n \"chunkInfo\": {\n + \ \"content\": \"Alphabet Announces Fourth Quarter and Fiscal Year + 2022 Results\\nMOUNTAIN VIEW, Calif. \u2013 February 2, 2023 \u2013 Alphabet + Inc. (NASDAQ: GOOG, GOOGL) today announced financial\\nresults for the quarter + and fiscal year ended December 31, 2022. Sundar Pichai, CEO of Alphabet and + Google, said: \u201COur long-term investments in deep computer science make + us\\nextremely well-positioned as AI reaches an inflection point, and I\u2019m + excited by the AI-driven leaps we\u2019re about to unveil\\nin Search and + beyond. There\u2019s also great momentum in Cloud, YouTube subscriptions, + and our Pixel devices. We\u2019re\\non an important journey to re-engineer + our cost structure in a durable way and to build financially sustainable, + vibrant,\\ngrowing businesses across Alphabet.\u201D Ruth Porat, CFO of Alphabet + and Google, said: \u201COur Q4 consolidated revenues were $76 billion, up + 1% year over year,\\nor up 7% in constant currency, and $283 billion for the + full year 2022, up 10%, or up 14% in constant currency. We\\nhave significant + work underway to improve all aspects of our cost structure, in support of + our investments in our\\nhighest growth priorities to deliver long-term, profitable + growth.\u201D \",\n \"relevanceScore\": 0.8,\n \"documentMetadata\": + {\n \"document\": \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/513da7d1b9f7739087606aadc626f2dc\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q4_alphabet_earnings_release.pdf\",\n + \ \"title\": \"GOOG Exhibit 99.1 Q4 2022\",\n \"pageIdentifier\": + \"1\"\n }\n }\n },\n {\n \"chunkInfo\": {\n + \ \"content\": \"Alphabet Announces Third Quarter 2023 Results\\nMOUNTAIN + VIEW, Calif. \u2013 October 24, 2023 \u2013 Alphabet Inc. (NASDAQ: GOOG, GOOGL) + today announced\\nfinancial results for the quarter ended September 30, 2023. + Sundar Pichai, CEO, said: \u201CI\u2019m pleased with our financial results + and our product momentum this quarter, with AI\\ndriven innovations across + Search, YouTube, Cloud, our Pixel devices and more. We\u2019re continuing + to focus on\\nmaking AI more helpful for everyone; there\u2019s exciting progress + and lots more to come.\u201D Ruth Porat, President and Chief Investment Officer; + CFO said: \u201CThe fundamental strength of our business was\\napparent again + in Q3, with $77 billion in revenue, up 11% year over year, driven by meaningful + growth in Search\\nand YouTube, and momentum in Cloud. We continue to focus + on judicious capital allocation to deliver sustainable\\nfinancial value.\u201D + Q3 2023 Financial Highlights (unaudited)\\nThe following table summarizes + our consolidated financial results for the quarters ended September 30, 2022 + and\\n2023 (in millions, except for per share information and percentages). + Quarter Ended September 30,\\n2022 2023 Revenues $ 69,092 $ 76,693 Change + in revenues year over year 6 % 11 % Change in constant currency revenues year + over year(1) 11 % 11 % Operating income $ 17,135 $ 21,343 Operating margin + 25 % \",\n \"relevanceScore\": 0.8,\n \"documentMetadata\": + {\n \"document\": \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/4c7389ffaf06b73e6e380eac7a39ea17\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2023q3-alphabet-earnings-release.pdf\",\n + \ \"title\": \"GOOG Exhibit 99.1 Q3 2023\",\n \"pageIdentifier\": + \"1\"\n }\n }\n },\n {\n \"chunkInfo\": {\n + \ \"content\": \"Alphabet Inc. Consolidated Statements of Income Year + Ended December 31, (in millions, except per share amounts) 2021 2022 2023 + Revenues $ 257,637 $ 282,836 $ 307,394 Costs and expenses:\\nCost of revenues + 110,939 126,203 133,332 Research and development 31,562 39,500 45,427 Sales + and marketing 22,912 26,567 27,917 General and administrative 13,510 15,724 + 16,425 Total costs and expenses 178,923 207,994 223,101 Income from operations + 78,714 74,842 84,293 Other income (expense), net 12,020 (3,514) 1,424 Income + before income taxes 90,734 71,328 85,717 Provision for income taxes 14,701 + 11,356 11,922 Net income $ 76,033 $ 59,972 $ 73,795 Basic net income per share + of Class A, Class B, and Class C stock $ 5.69 $ 4.59 $ 5.84 Diluted net income + per share of Class A, Class B, and Class C stock $ 5.61 $ 4.56 $ 5.80 See + accompanying notes. 50 Alphabet 2023 Annual Report Part I Part II Part III + Part IV \",\n \"relevanceScore\": 0.8,\n \"documentMetadata\": + {\n \"document\": \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/3c617f6f4b023305d82916f0093ca13d\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/goog023-alphabet-2023-annual-report-web-1.pdf\",\n + \ \"title\": \"goog023-alphabet-2023-annual-report-web-1\",\n \"pageIdentifier\": + \"60\"\n }\n }\n },\n {\n \"chunkInfo\": + {\n \"content\": \"Alphabet Announces Third Quarter 2022 Results\\nMOUNTAIN + VIEW, Calif. \u2013 October 25, 2022 \u2013 Alphabet Inc. (NASDAQ: GOOG, GOOGL) + today announced\\nfinancial results for the quarter ended September 30, 2022. + Sundar Pichai, CEO of Alphabet and Google, said: \u201CWe\u2019re sharpening + our focus on a clear set of product and\\nbusiness priorities. Product announcements + we\u2019ve made in just the past month alone have shown that very clearly,\\nincluding + significant improvements to both Search and Cloud, powered by AI, and new + ways to monetize YouTube\\nShorts. We are focused on both investing responsibly + for the long term and being responsive to the economic\\nenvironment.\u201D + Ruth Porat, CFO of Alphabet and Google, said: \u201COur third quarter revenues + were $69.1 billion, up 6% versus last\\nyear or up 11% on a constant currency + basis. Financial results for the third quarter reflect healthy fundamental\\ngrowth + in Search and momentum in Cloud, while affected by foreign exchange. We\u2019re + working to realign resources\\nto fuel our highest growth priorities.\u201D + Q3 2022 financial highlights\\nThe following table summarizes our consolidated + financial results for the quarters ended September 30, 2021 and\\n2022 (in + millions, except for per share information and percentages; unaudited). Quarter + Ended September 30,\\n2021 2022 Revenues $ 65,118 $ 69,092 Change in revenues + year over year 41 % \",\n \"relevanceScore\": 0.8,\n \"documentMetadata\": + {\n \"document\": \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/2eab9446e934f432e3df7808af857300\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q3_alphabet_earnings_release.pdf\",\n + \ \"title\": \"GOOG Exhibit 99.1 Q3 2022\",\n \"pageIdentifier\": + \"1\"\n }\n }\n },\n {\n \"chunkInfo\": {\n + \ \"content\": \"Alphabet Inc. CONSOLIDATED STATEMENTS OF CASH FLOWS\\n(In + millions, unaudited)\\nQuarter Ended September 30, Year to Date September + 30, 2021 2022 2021 2022 Operating activities\\nNet income $ 18,936 $ 13,910 + $ 55,391 $ 46,348 Adjustments:\\nDepreciation and impairment of property and\\nequipment + 3,085 3,933 8,340 11,222 Amortization and impairment of intangible assets + 219 113 662 505 Stock-based compensation expense 3,874 4,976 11,422 14,262 + Deferred income taxes (1,287) (1,920) 192 (6,157) (Gain) loss on debt and + equity securities, net (2,158) 1,378 (9,792) 3,856 Other 64 167 (199) 369 + Changes in assets and liabilities, net of effects of\\nacquisitions:\\nAccounts + receivable (2,409) (97) (3,276) 2,298 Income taxes, net 3,041 (609) 2,744 + (862) Other assets (1,255) (2,647) (1,447) (4,268) Accounts payable 238 1,907 + (874) 735 Accrued expenses and other liabilities 2,562 2,210 2,763 491 Accrued + revenue share 357 (80) 386 (1,022) Deferred revenue 272 112 406 104 Net cash + provided by operating activities 25,539 23,353 66,718 67,881 Investing activities\\nPurchases + of property and equipment (6,819) (7,276) (18,257) (23,890) Purchases of marketable + securities (34,497) (17,054) (95,106) (67,253) Maturities and sales of marketable + securities 31,459 28,713 92,126 84,087 Purchases of non-marketable securities + \",\n \"relevanceScore\": 0.8,\n \"documentMetadata\": {\n + \ \"document\": \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/2eab9446e934f432e3df7808af857300\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q3_alphabet_earnings_release.pdf\",\n + \ \"title\": \"GOOG Exhibit 99.1 Q3 2022\",\n \"pageIdentifier\": + \"6\"\n }\n }\n },\n {\n \"chunkInfo\": {\n + \ \"content\": \"Alphabet Inc. CONSOLIDATED STATEMENTS OF CASH FLOWS\\n(In + millions, unaudited)\\nQuarter Ended June 30, Year To Date June 30,\\n2021 + 2022 2021 2022 Operating activities\\nNet income $ 18,525 $ 16,002 $ 36,455 + $ 32,438 Adjustments:\\nDepreciation and impairment of property and equipment + 2,730 3,698 5,255 7,289 Amortization and impairment of intangible assets 215 + 201 443 392 Stock-based compensation expense 3,803 4,782 7,548 9,286 Deferred + income taxes 379 (2,147) 1,479 (4,237) (Gain) loss on debt and equity securities, + net (2,883) 1,041 (7,634) 2,478 Other (8) 62 (263) 202 Changes in assets and + liabilities, net of effects of acquisitions:\\nAccounts receivable (3,661) + (1,969) (867) 2,395 Income taxes, net (1,082) (4,073) (297) (253) Other assets + (199) (845) (192) (1,621) Accounts payable (130) 1,201 (1,112) (1,172) Accrued + expenses and other liabilities 3,731 1,497 201 (1,719) Accrued revenue share + 473 (114) 29 (942) Deferred revenue (3) 86 134 (8) Net cash provided by operating + activities 21,890 19,422 41,179 44,528 Investing activities\\nPurchases of + property and equipment (5,496) (6,828) (11,438) (16,614) Purchases of marketable + securities (24,183) (21,737) (60,609) (50,199) \",\n \"relevanceScore\": + 0.8,\n \"documentMetadata\": {\n \"document\": \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/376dab8cf7b3807296f2bf1c6228e078\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q2_alphabet_earnings_release.pdf\",\n + \ \"title\": \"GOOG Exhibit 99.1 Q2 2022\",\n \"pageIdentifier\": + \"6\"\n }\n }\n },\n {\n \"chunkInfo\": {\n + \ \"content\": \"The associated costs,\\nincluding depreciation and + impairment, are allocated to operating segments as a service cost generally + based on usage or headcount. Unallocated corporate costs primarily include + corporate initiatives, corporate shared costs, such as finance and legal, + including certain\\nfines and settlements, as well as costs associated with + certain shared R&D activities. Additionally, hedging gains (losses) related + to\\nrevenue are included in corporate costs. As AI is critical to delivering + our mission of bringing our breakthrough innovations into the real world, + beginning in January 2023, we\\nwill update our segment reporting relating + to certain of Alphabet\u2019s AI activities. DeepMind, previously reported + within Other Bets, will\\nbe reported as part of Alphabet\u2019s corporate + costs, reflecting its increasing collaboration with Google Services, Google + Cloud, and\\nOther Bets. Prior periods will be recast to conform to the revised + presentation. Our operating segments are not evaluated using asset information. + The following table presents information about our segments (in millions): + Year Ended December 31,\\n2020 2021 2022 Revenues:\\nGoogle Services $ 168,635 + $ 237,529 $ 253,528 Google Cloud 13,059 19,206 26,280 Other Bets 657 753 1,068 + Hedging gains (losses) 176 149 1,960 Total revenues $ 182,527 $ 257,637 $ + 282,836 Operating income (loss):\\nGoogle Services $ 54,606 $ 91,855 $ 86,572 + Google Cloud (5,607) (3,099) (2,968) Other Bets \",\n \"relevanceScore\": + 0.8,\n \"documentMetadata\": {\n \"document\": \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/276cee4c4086600303bc561483691f9a\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022_alphabet_annual_report.pdf\",\n + \ \"title\": \"2022_alphabet_annual_report\",\n \"pageIdentifier\": + \"119\"\n }\n }\n },\n {\n \"chunkInfo\": + {\n \"content\": \"56 ALPHABET INC. ANNUAL REPORT PART II\\nITEM\_8\_\_FINANCIAL + STATEMENTS AND SUPPLEMENTARY DATA Note 2. Revenues\\nDisaggregated Revenues\\nThe + following table presents revenues disaggregated by type (in millions): Year + Ended December 31,\\n2020 2021 2022 Google Search & other $ 104,062 $ 148,951 + $ 162,450 YouTube ads 19,772 28,845 29,243 Google Network 23,090 31,701 32,780 + Google advertising 146,924 209,497 224,473 Google other 21,711 28,032 29,055 + Google Services total 168,635 237,529 253,528 Google Cloud 13,059 19,206 26,280 + Other Bets 657 753 1,068 Hedging gains (losses) 176 149 1,960 Total revenues + $ 182,527 $ 257,637 $ 282,836\\nNo individual customer or groups of affiliated + customers represented more than 10% of our revenues in 2020, 2021, or 2022. + The following table presents revenues disaggregated by geography, based on + the addresses of our customers (in millions):\\nYear Ended December 31, 2020 + 2021 2022 United States $ 85,014 47 % $ 117,854 46 % $ 134,814 48% EMEA(1) + 55,370 30 79,107 31 82,062 29 APAC(1) 32,550 18 46,123 18 47,024 16 Other + Americas(1) 9,417 5 14,404 5 16,976 6 Hedging gains (losses) 176 0 149 0 1,960 + 1 Total revenues $ 182,527 100% $ 257,637 100 % $ 282,836 100% \",\n \"relevanceScore\": + 0.8,\n \"documentMetadata\": {\n \"document\": \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/276cee4c4086600303bc561483691f9a\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022_alphabet_annual_report.pdf\",\n + \ \"title\": \"2022_alphabet_annual_report\",\n \"pageIdentifier\": + \"94\"\n }\n }\n }\n ],\n \"steps\": [\n {\n + \ \"state\": 3,\n \"description\": \"Rephrase the query and search.\",\n + \ \"actions\": [\n {\n \"searchAction\": {\n \"query\": + \"What was Alphabet's revenue in 2022?\"\n },\n \"observation\": + {\n \"searchResults\": [\n {\n \"document\": + \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/513da7d1b9f7739087606aadc626f2dc\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q4_alphabet_earnings_release.pdf\",\n + \ \"title\": \"GOOG Exhibit 99.1 Q4 2022\",\n \"snippetInfo\": + [\n {\n \"snippet\": \"... \\u003cb\\u003eAlphabet\\u003c/b\\u003e.\u201D + Ruth Porat, CFO of \\u003cb\\u003eAlphabet\\u003c/b\\u003e and Google, said: + \u201COur Q4 consolidated \\u003cb\\u003erevenues\\u003c/b\\u003e were $76 + billion, up 1% year over year, or up 7% in constant currency ...\",\n + \ \"snippetStatus\": \"SUCCESS\"\n }\n + \ ]\n },\n {\n \"document\": + \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/c201efed2be20c6a4116611ea16be9cd\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q1_alphabet_earnings_release.pdf\",\n + \ \"title\": \"GOOG Exhibit 99.1 Q1 2022\",\n \"snippetInfo\": + [\n {\n \"snippet\": \"\u2013 April + 26, \\u003cb\\u003e2022\\u003c/b\\u003e \u2013 \\u003cb\\u003eAlphabet\\u003c/b\\u003e + Inc. (NASDAQ: GOOG ... \\u003cb\\u003erevenue\\u003c/b\\u003e growth of 23% + year over year. ... Quarter Ended March 31, 2021 \\u003cb\\u003e2022 Revenues\\u003c/b\\u003e + $ 55,314 $ 68,011 Change in ...\",\n \"snippetStatus\": + \"SUCCESS\"\n }\n ]\n },\n + \ {\n \"document\": \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/376dab8cf7b3807296f2bf1c6228e078\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q2_alphabet_earnings_release.pdf\",\n + \ \"title\": \"GOOG Exhibit 99.1 Q2 2022\",\n \"snippetInfo\": + [\n {\n \"snippet\": \"\u2013 July + 26, \\u003cb\\u003e2022\\u003c/b\\u003e \u2013 \\u003cb\\u003eAlphabet\\u003c/b\\u003e + Inc. (NASDAQ: GOOG ... Quarter Ended June 30, 2021 \\u003cb\\u003e2022 Revenues\\u003c/b\\u003e + $ 61,880 $ 69,685 Change in \\u003cb\\u003erevenues\\u003c/b\\u003e ... \\u003cb\\u003erevenue\\u003c/b\\u003e + growth rates by 3.7%.\",\n \"snippetStatus\": \"SUCCESS\"\n + \ }\n ]\n },\n {\n + \ \"document\": \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/4a4eabb0cefc954375df0bd9eec85c96\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2023q4-alphabet-earnings-release.pdf\",\n + \ \"title\": \"2023q4-alphabet-earnings-release\",\n \"snippetInfo\": + [\n {\n \"snippet\": \"... \\u003cb\\u003eRevenues\\u003c/b\\u003e + As Reported Less Hedging Effect Less FX Effect Constant Currency \\u003cb\\u003e2022\\u003c/b\\u003e + 2023 \\u003cb\\u003eRevenues\\u003c/b\\u003e United States $ 134,814 $ 146,286 + $ 0 $ 146,286 9 % 0 % 9 ...\",\n \"snippetStatus\": + \"SUCCESS\"\n }\n ]\n },\n + \ {\n \"document\": \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/2eab9446e934f432e3df7808af857300\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q3_alphabet_earnings_release.pdf\",\n + \ \"title\": \"GOOG Exhibit 99.1 Q3 2022\",\n \"snippetInfo\": + [\n {\n \"snippet\": \"\\u003cb\\u003eAlphabet\\u003c/b\\u003e + Announces Third Quarter \\u003cb\\u003e2022\\u003c/b\\u003e Results MOUNTAIN + VIEW, Calif. \u2013 October 25, \\u003cb\\u003e2022\\u003c/b\\u003e \u2013 + \\u003cb\\u003eAlphabet\\u003c/b\\u003e Inc. ... Quarter Ended September 30, + 2021 \\u003cb\\u003e2022 Revenues\\u003c/b\\u003e $ 65,118 ...\",\n \"snippetStatus\": + \"SUCCESS\"\n }\n ]\n },\n + \ {\n \"document\": \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/276cee4c4086600303bc561483691f9a\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022_alphabet_annual_report.pdf\",\n + \ \"title\": \"2022_alphabet_annual_report\",\n \"snippetInfo\": + [\n {\n \"snippet\": \"... \\u003cb\\u003eAlphabet's\\u003c/b\\u003e + AI activities. DeepMind, previously reported within Other Bets, will be reported + as part of \\u003cb\\u003eAlphabet's\\u003c/b\\u003e corporate costs, + reflecting its increasing ...\",\n \"snippetStatus\": + \"SUCCESS\"\n }\n ]\n },\n + \ {\n \"document\": \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/3c617f6f4b023305d82916f0093ca13d\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/goog023-alphabet-2023-annual-report-web-1.pdf\",\n + \ \"title\": \"goog023-alphabet-2023-annual-report-web-1\",\n + \ \"snippetInfo\": [\n {\n \"snippet\": + \"... \\u003cb\\u003e2022\\u003c/b\\u003e 2023 \\u003cb\\u003eRevenues\\u003c/b\\u003e + $ 257,637 $ 282,836 $ 307,394 Costs and expenses: Cost of \\u003cb\\u003erevenues\\u003c/b\\u003e + 110,939 126,203 133,332 Research and development 31,562 39,500 45,427 ...\",\n + \ \"snippetStatus\": \"SUCCESS\"\n }\n + \ ]\n },\n {\n \"document\": + \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/404cb139d857b704a13ee3d8e0c6c5fb\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2021Q4_alphabet_earnings_release.pdf\",\n + \ \"title\": \"GOOG Exhibit 99.1 Q4 2021\",\n \"snippetInfo\": + [\n {\n \"snippet\": \"\\u003cb\\u003eAlphabet\\u003c/b\\u003e + Announces Fourth Quarter and Fiscal Year 2021 Results MOUNTAIN VIEW, Calif. + \u2013 February 1, \\u003cb\\u003e2022\\u003c/b\\u003e \u2013 \\u003cb\\u003eAlphabet\\u003c/b\\u003e + Inc. ... \\u003cb\\u003erevenue\\u003c/b\\u003e growth from Google Cloud.\",\n + \ \"snippetStatus\": \"SUCCESS\"\n }\n + \ ]\n },\n {\n \"document\": + \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/e0ac436b3f82b7df8fc94580f3a84b0a\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2023q1-alphabet-earnings-release.pdf\",\n + \ \"title\": \"GOOG Exhibit 99.1 Q1 2023\",\n \"snippetInfo\": + [\n {\n \"snippet\": \"Ruth Porat, + CFO of \\u003cb\\u003eAlphabet\\u003c/b\\u003e and Google, said: \u201CResilience + in Search and momentum in Cloud resulted in Q1 consolidated \\u003cb\\u003erevenues\\u003c/b\\u003e + of $69.8 billion, up 3% year over ...\",\n \"snippetStatus\": + \"SUCCESS\"\n }\n ]\n },\n + \ {\n \"document\": \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/4c7389ffaf06b73e6e380eac7a39ea17\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2023q3-alphabet-earnings-release.pdf\",\n + \ \"title\": \"GOOG Exhibit 99.1 Q3 2023\",\n \"snippetInfo\": + [\n {\n \"snippet\": \"\\u003cb\\u003eAlphabet\\u003c/b\\u003e + Announces Third Quarter 2023 Results MOUNTAIN VIEW, Calif. ... \\u003cb\\u003erevenue\\u003c/b\\u003e, + up 11% year over year, driven by ... \\u003cb\\u003e2022\\u003c/b\\u003e and + 2023 (in millions, except for per ...\",\n \"snippetStatus\": + \"SUCCESS\"\n }\n ]\n }\n + \ ]\n }\n }\n ]\n }\n ]\n },\n + \ \"answerQueryToken\": \"NMwKDAjDravVBhCSk66vARIkNmFiNDUxYWMtMDAwMC0yOGQ5LWJhYjctMDg5ZTA4MjUyYzgw\"\n}\n" + headers: + Alt-Svc: + - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 + Content-Type: + - application/json; charset=UTF-8 + Date: + - Wed, 16 Sep 2026 17:49:55 GMT + Server: + - ESF + Server-Timing: + - gfet4t7; dur=7587 + Transfer-Encoding: + - chunked + Vary: + - Origin + - X-Origin + - Referer + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-XSS-Protection: + - '0' + content-length: + - '32045' + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_answer_query[True].yaml b/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_answer_query[True].yaml new file mode 100644 index 000000000..5df7772db --- /dev/null +++ b/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_answer_query[True].yaml @@ -0,0 +1,864 @@ +interactions: +- request: + body: "{\n \"query\": {\n \"text\": \"What was Alphabet's revenue in 2022?\"\n + \ },\n \"answerGenerationSpec\": {\n \"includeCitations\": true\n }\n}" + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '133' + Content-Type: + - application/json + User-Agent: + - python-requests/2.34.2 + x-goog-api-client: + - gl-python/3.14.7 grpc/1.84.0 gax/2.37.0 gapic/0.20.3 pb/7.36.1 cred-type/u + x-goog-request-params: + - serving_config=projects/sdk-dev-508013/locations/global/collections/default_collection/engines/braintrust-sdk-test_1789580029521/servingConfigs/default_search + method: POST + uri: https://discoveryengine.googleapis.com/v1/projects/sdk-dev-508013/locations/global/collections/default_collection/engines/braintrust-sdk-test_1789580029521/servingConfigs/default_search:streamAnswer?%24alt=json%3Benum-encoding%3Dint + response: + body: + string: "[{\n \"answer\": {\n \"state\": 4,\n \"steps\": [\n {\n + \ \"state\": 3,\n \"description\": \"Rephrase the query and search.\",\n + \ \"actions\": [\n {\n \"searchAction\": {\n \"query\": + \"What was Alphabet's revenue in 2022?\"\n },\n \"observation\": + {\n \"searchResults\": [\n {\n \"document\": + \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/513da7d1b9f7739087606aadc626f2dc\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q4_alphabet_earnings_release.pdf\",\n + \ \"title\": \"GOOG Exhibit 99.1 Q4 2022\",\n \"snippetInfo\": + [\n {\n \"snippet\": \"... \\u003cb\\u003eAlphabet\\u003c/b\\u003e.\u201D + Ruth Porat, CFO of \\u003cb\\u003eAlphabet\\u003c/b\\u003e and Google, said: + \u201COur Q4 consolidated \\u003cb\\u003erevenues\\u003c/b\\u003e were $76 + billion, up 1% year over year, or up 7% in constant currency ...\",\n + \ \"snippetStatus\": \"SUCCESS\"\n }\n + \ ]\n },\n {\n \"document\": + \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/c201efed2be20c6a4116611ea16be9cd\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q1_alphabet_earnings_release.pdf\",\n + \ \"title\": \"GOOG Exhibit 99.1 Q1 2022\",\n \"snippetInfo\": + [\n {\n \"snippet\": \"\u2013 April + 26, \\u003cb\\u003e2022\\u003c/b\\u003e \u2013 \\u003cb\\u003eAlphabet\\u003c/b\\u003e + Inc. (NASDAQ: GOOG ... \\u003cb\\u003erevenue\\u003c/b\\u003e growth of 23% + year over year. ... Quarter Ended March 31, 2021 \\u003cb\\u003e2022 Revenues\\u003c/b\\u003e + $ 55,314 $ 68,011 Change in ...\",\n \"snippetStatus\": + \"SUCCESS\"\n }\n ]\n },\n + \ {\n \"document\": \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/376dab8cf7b3807296f2bf1c6228e078\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q2_alphabet_earnings_release.pdf\",\n + \ \"title\": \"GOOG Exhibit 99.1 Q2 2022\",\n \"snippetInfo\": + [\n {\n \"snippet\": \"\u2013 July + 26, \\u003cb\\u003e2022\\u003c/b\\u003e \u2013 \\u003cb\\u003eAlphabet\\u003c/b\\u003e + Inc. (NASDAQ: GOOG ... Quarter Ended June 30, 2021 \\u003cb\\u003e2022 Revenues\\u003c/b\\u003e + $ 61,880 $ 69,685 Change in \\u003cb\\u003erevenues\\u003c/b\\u003e ... \\u003cb\\u003erevenue\\u003c/b\\u003e + growth rates by 3.7%.\",\n \"snippetStatus\": \"SUCCESS\"\n + \ }\n ]\n },\n {\n + \ \"document\": \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/4a4eabb0cefc954375df0bd9eec85c96\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2023q4-alphabet-earnings-release.pdf\",\n + \ \"title\": \"2023q4-alphabet-earnings-release\",\n \"snippetInfo\": + [\n {\n \"snippet\": \"... \\u003cb\\u003eRevenues\\u003c/b\\u003e + As Reported Less Hedging Effect Less FX Effect Constant Currency \\u003cb\\u003e2022\\u003c/b\\u003e + 2023 \\u003cb\\u003eRevenues\\u003c/b\\u003e United States $ 134,814 $ 146,286 + $ 0 $ 146,286 9 % 0 % 9 ...\",\n \"snippetStatus\": + \"SUCCESS\"\n }\n ]\n },\n + \ {\n \"document\": \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/2eab9446e934f432e3df7808af857300\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q3_alphabet_earnings_release.pdf\",\n + \ \"title\": \"GOOG Exhibit 99.1 Q3 2022\",\n \"snippetInfo\": + [\n {\n \"snippet\": \"\\u003cb\\u003eAlphabet\\u003c/b\\u003e + Announces Third Quarter \\u003cb\\u003e2022\\u003c/b\\u003e Results MOUNTAIN + VIEW, Calif. \u2013 October 25, \\u003cb\\u003e2022\\u003c/b\\u003e \u2013 + \\u003cb\\u003eAlphabet\\u003c/b\\u003e Inc. ... Quarter Ended September 30, + 2021 \\u003cb\\u003e2022 Revenues\\u003c/b\\u003e $ 65,118 ...\",\n \"snippetStatus\": + \"SUCCESS\"\n }\n ]\n },\n + \ {\n \"document\": \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/276cee4c4086600303bc561483691f9a\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022_alphabet_annual_report.pdf\",\n + \ \"title\": \"2022_alphabet_annual_report\",\n \"snippetInfo\": + [\n {\n \"snippet\": \"... \\u003cb\\u003eAlphabet's\\u003c/b\\u003e + AI activities. DeepMind, previously reported within Other Bets, will be reported + as part of \\u003cb\\u003eAlphabet's\\u003c/b\\u003e corporate costs, + reflecting its increasing ...\",\n \"snippetStatus\": + \"SUCCESS\"\n }\n ]\n },\n + \ {\n \"document\": \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/3c617f6f4b023305d82916f0093ca13d\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/goog023-alphabet-2023-annual-report-web-1.pdf\",\n + \ \"title\": \"goog023-alphabet-2023-annual-report-web-1\",\n + \ \"snippetInfo\": [\n {\n \"snippet\": + \"... \\u003cb\\u003e2022\\u003c/b\\u003e 2023 \\u003cb\\u003eRevenues\\u003c/b\\u003e + $ 257,637 $ 282,836 $ 307,394 Costs and expenses: Cost of \\u003cb\\u003erevenues\\u003c/b\\u003e + 110,939 126,203 133,332 Research and development 31,562 39,500 45,427 ...\",\n + \ \"snippetStatus\": \"SUCCESS\"\n }\n + \ ]\n },\n {\n \"document\": + \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/404cb139d857b704a13ee3d8e0c6c5fb\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2021Q4_alphabet_earnings_release.pdf\",\n + \ \"title\": \"GOOG Exhibit 99.1 Q4 2021\",\n \"snippetInfo\": + [\n {\n \"snippet\": \"\\u003cb\\u003eAlphabet\\u003c/b\\u003e + Announces Fourth Quarter and Fiscal Year 2021 Results MOUNTAIN VIEW, Calif. + \u2013 February 1, \\u003cb\\u003e2022\\u003c/b\\u003e \u2013 \\u003cb\\u003eAlphabet\\u003c/b\\u003e + Inc. ... \\u003cb\\u003erevenue\\u003c/b\\u003e growth from Google Cloud.\",\n + \ \"snippetStatus\": \"SUCCESS\"\n }\n + \ ]\n },\n {\n \"document\": + \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/e0ac436b3f82b7df8fc94580f3a84b0a\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2023q1-alphabet-earnings-release.pdf\",\n + \ \"title\": \"GOOG Exhibit 99.1 Q1 2023\",\n \"snippetInfo\": + [\n {\n \"snippet\": \"Ruth Porat, + CFO of \\u003cb\\u003eAlphabet\\u003c/b\\u003e and Google, said: \u201CResilience + in Search and momentum in Cloud resulted in Q1 consolidated \\u003cb\\u003erevenues\\u003c/b\\u003e + of $69.8 billion, up 3% year over ...\",\n \"snippetStatus\": + \"SUCCESS\"\n }\n ]\n },\n + \ {\n \"document\": \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/4c7389ffaf06b73e6e380eac7a39ea17\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2023q3-alphabet-earnings-release.pdf\",\n + \ \"title\": \"GOOG Exhibit 99.1 Q3 2023\",\n \"snippetInfo\": + [\n {\n \"snippet\": \"\\u003cb\\u003eAlphabet\\u003c/b\\u003e + Announces Third Quarter 2023 Results MOUNTAIN VIEW, Calif. ... \\u003cb\\u003erevenue\\u003c/b\\u003e, + up 11% year over year, driven by ... \\u003cb\\u003e2022\\u003c/b\\u003e and + 2023 (in millions, except for per ...\",\n \"snippetStatus\": + \"SUCCESS\"\n }\n ]\n }\n + \ ]\n }\n }\n ]\n }\n ]\n },\n + \ \"answerQueryToken\": \"NMwKDAjEravVBhCr-_ORAxIkNmFjNGMzMTUtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi\"\n}\n,\r\n{\n + \ \"answer\": {\n \"state\": 4,\n \"references\": [\n {\n \"chunkInfo\": + {\n \"content\": \"ALPHABET INC. ANNUAL REPORT 45 PART II\\nITEM\_8\_\_FINANCIAL + STATEMENTS AND SUPPLEMENTARY DATA Alphabet Inc. CONSOLIDATED STATEMENTS OF + INCOME (In millions, except per share amounts) Year Ended December 31,\\n2020 + 2021 2022 Revenues $ 182,527 $ 257,637 $ 282,836 Costs and expenses:\\nCost + of revenues 84,732 110,939 126,203 Research and development 27,573 31,562 + 39,500 Sales and marketing 17,946 22,912 26,567 General and administrative + 11,052 13,510 15,724 Total costs and expenses 141,303 178,923 207,994 Income + from operations 41,224 78,714 74,842 Other income (expense), net 6,858 12,020 + (3,514) Income before income taxes 48,082 90,734 71,328 Provision for income + taxes 7,813 14,701 11,356 Net income $ 40,269 $ 76,033 $ 59,972 Basic net + income per share of Class A, Class B, and Class C stock $ 2.96 $ 5.69 $ 4.59 + Diluted net income per share of Class A, Class B, and Class C stock $ 2.93 + $ 5.61 $ 4.56 See accompanying notes. \",\n \"documentMetadata\": + {\n \"document\": \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/276cee4c4086600303bc561483691f9a\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022_alphabet_annual_report.pdf\",\n + \ \"title\": \"2022_alphabet_annual_report\"\n }\n }\n + \ },\n {\n \"chunkInfo\": {\n \"content\": \"Alphabet + Announces First Quarter 2022 Results\\nMOUNTAIN VIEW, Calif. \u2013 April + 26, 2022 \u2013 Alphabet Inc. (NASDAQ: GOOG, GOOGL) today announced financial\\nresults + for the quarter ended March 31, 2022. Sundar Pichai, CEO of Alphabet and Google, + said: \u201CQ1 saw strong growth in Search and Cloud, in particular, which\\nare + both helping people and businesses as the digital transformation continues. + We\u2019ll keep investing in great\\nproducts and services, and creating opportunities + for partners and local communities around the world.\u201D Ruth Porat, CFO + of Alphabet and Google, said: \u201CWe are pleased with Q1 revenue growth + of 23% year over year. We\\ncontinue to make considered investments in Capex, + R&D and talent to support long-term value creation for all\\nstakeholders.\u201D + Q1 2022 financial highlights\\nThe following table summarizes our consolidated + financial results for the quarters ended March 31, 2021 and 2022\\n(in millions, + except for per share information and percentages; unaudited). Quarter Ended + March 31,\\n2021 2022 Revenues $ 55,314 $ 68,011 Change in revenues year over + year 34 % 23 % Change in constant currency revenues year over year(1) 32 % + 26 % Operating income $ 16,437 $ 20,094 Operating margin 30 % 30 % Other income + (expense), net $ 4,846 $ (1,160) Net income $ 17,930 $ 16,436 \",\n \"documentMetadata\": + {\n \"document\": \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/c201efed2be20c6a4116611ea16be9cd\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q1_alphabet_earnings_release.pdf\",\n + \ \"title\": \"GOOG Exhibit 99.1 Q1 2022\"\n }\n }\n + \ },\n {\n \"chunkInfo\": {\n \"content\": \"Alphabet + Announces Fourth Quarter and Fiscal Year 2022 Results\\nMOUNTAIN VIEW, Calif. + \u2013 February 2, 2023 \u2013 Alphabet Inc. (NASDAQ: GOOG, GOOGL) today announced + financial\\nresults for the quarter and fiscal year ended December 31, 2022. + Sundar Pichai, CEO of Alphabet and Google, said: \u201COur long-term investments + in deep computer science make us\\nextremely well-positioned as AI reaches + an inflection point, and I\u2019m excited by the AI-driven leaps we\u2019re + about to unveil\\nin Search and beyond. There\u2019s also great momentum in + Cloud, YouTube subscriptions, and our Pixel devices. We\u2019re\\non an important + journey to re-engineer our cost structure in a durable way and to build financially + sustainable, vibrant,\\ngrowing businesses across Alphabet.\u201D Ruth Porat, + CFO of Alphabet and Google, said: \u201COur Q4 consolidated revenues were + $76 billion, up 1% year over year,\\nor up 7% in constant currency, and $283 + billion for the full year 2022, up 10%, or up 14% in constant currency. We\\nhave + significant work underway to improve all aspects of our cost structure, in + support of our investments in our\\nhighest growth priorities to deliver long-term, + profitable growth.\u201D \",\n \"documentMetadata\": {\n \"document\": + \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/513da7d1b9f7739087606aadc626f2dc\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q4_alphabet_earnings_release.pdf\",\n + \ \"title\": \"GOOG Exhibit 99.1 Q4 2022\"\n }\n }\n + \ },\n {\n \"chunkInfo\": {\n \"content\": \"Alphabet + Announces Third Quarter 2023 Results\\nMOUNTAIN VIEW, Calif. \u2013 October + 24, 2023 \u2013 Alphabet Inc. (NASDAQ: GOOG, GOOGL) today announced\\nfinancial + results for the quarter ended September 30, 2023. Sundar Pichai, CEO, said: + \u201CI\u2019m pleased with our financial results and our product momentum + this quarter, with AI\\ndriven innovations across Search, YouTube, Cloud, + our Pixel devices and more. We\u2019re continuing to focus on\\nmaking AI + more helpful for everyone; there\u2019s exciting progress and lots more to + come.\u201D Ruth Porat, President and Chief Investment Officer; CFO said: + \u201CThe fundamental strength of our business was\\napparent again in Q3, + with $77 billion in revenue, up 11% year over year, driven by meaningful growth + in Search\\nand YouTube, and momentum in Cloud. We continue to focus on judicious + capital allocation to deliver sustainable\\nfinancial value.\u201D Q3 2023 + Financial Highlights (unaudited)\\nThe following table summarizes our consolidated + financial results for the quarters ended September 30, 2022 and\\n2023 (in + millions, except for per share information and percentages). Quarter Ended + September 30,\\n2022 2023 Revenues $ 69,092 $ 76,693 Change in revenues year + over year 6 % 11 % Change in constant currency revenues year over year(1) + 11 % 11 % Operating income $ 17,135 $ 21,343 Operating margin 25 % \",\n \"documentMetadata\": + {\n \"document\": \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/4c7389ffaf06b73e6e380eac7a39ea17\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2023q3-alphabet-earnings-release.pdf\",\n + \ \"title\": \"GOOG Exhibit 99.1 Q3 2023\"\n }\n }\n + \ },\n {\n \"chunkInfo\": {\n \"content\": \"Alphabet + Inc. Consolidated Statements of Income Year Ended December 31, (in millions, + except per share amounts) 2021 2022 2023 Revenues $ 257,637 $ 282,836 $ 307,394 + Costs and expenses:\\nCost of revenues 110,939 126,203 133,332 Research and + development 31,562 39,500 45,427 Sales and marketing 22,912 26,567 27,917 + General and administrative 13,510 15,724 16,425 Total costs and expenses 178,923 + 207,994 223,101 Income from operations 78,714 74,842 84,293 Other income (expense), + net 12,020 (3,514) 1,424 Income before income taxes 90,734 71,328 85,717 Provision + for income taxes 14,701 11,356 11,922 Net income $ 76,033 $ 59,972 $ 73,795 + Basic net income per share of Class A, Class B, and Class C stock $ 5.69 $ + 4.59 $ 5.84 Diluted net income per share of Class A, Class B, and Class C + stock $ 5.61 $ 4.56 $ 5.80 See accompanying notes. 50 Alphabet 2023 Annual + Report Part I Part II Part III Part IV \",\n \"documentMetadata\": + {\n \"document\": \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/3c617f6f4b023305d82916f0093ca13d\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/goog023-alphabet-2023-annual-report-web-1.pdf\",\n + \ \"title\": \"goog023-alphabet-2023-annual-report-web-1\"\n }\n + \ }\n },\n {\n \"chunkInfo\": {\n \"content\": + \"Alphabet Announces Third Quarter 2022 Results\\nMOUNTAIN VIEW, Calif. \u2013 + October 25, 2022 \u2013 Alphabet Inc. (NASDAQ: GOOG, GOOGL) today announced\\nfinancial + results for the quarter ended September 30, 2022. Sundar Pichai, CEO of Alphabet + and Google, said: \u201CWe\u2019re sharpening our focus on a clear set of + product and\\nbusiness priorities. Product announcements we\u2019ve made in + just the past month alone have shown that very clearly,\\nincluding significant + improvements to both Search and Cloud, powered by AI, and new ways to monetize + YouTube\\nShorts. We are focused on both investing responsibly for the long + term and being responsive to the economic\\nenvironment.\u201D Ruth Porat, + CFO of Alphabet and Google, said: \u201COur third quarter revenues were $69.1 + billion, up 6% versus last\\nyear or up 11% on a constant currency basis. + Financial results for the third quarter reflect healthy fundamental\\ngrowth + in Search and momentum in Cloud, while affected by foreign exchange. We\u2019re + working to realign resources\\nto fuel our highest growth priorities.\u201D + Q3 2022 financial highlights\\nThe following table summarizes our consolidated + financial results for the quarters ended September 30, 2021 and\\n2022 (in + millions, except for per share information and percentages; unaudited). Quarter + Ended September 30,\\n2021 2022 Revenues $ 65,118 $ 69,092 Change in revenues + year over year 41 % \",\n \"documentMetadata\": {\n \"document\": + \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/2eab9446e934f432e3df7808af857300\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q3_alphabet_earnings_release.pdf\",\n + \ \"title\": \"GOOG Exhibit 99.1 Q3 2022\"\n }\n }\n + \ },\n {\n \"chunkInfo\": {\n \"content\": \"Alphabet + Inc. CONSOLIDATED STATEMENTS OF CASH FLOWS\\n(In millions, unaudited)\\nQuarter + Ended September 30, Year to Date September 30, 2021 2022 2021 2022 Operating + activities\\nNet income $ 18,936 $ 13,910 $ 55,391 $ 46,348 Adjustments:\\nDepreciation + and impairment of property and\\nequipment 3,085 3,933 8,340 11,222 Amortization + and impairment of intangible assets 219 113 662 505 Stock-based compensation + expense 3,874 4,976 11,422 14,262 Deferred income taxes (1,287) (1,920) 192 + (6,157) (Gain) loss on debt and equity securities, net (2,158) 1,378 (9,792) + 3,856 Other 64 167 (199) 369 Changes in assets and liabilities, net of effects + of\\nacquisitions:\\nAccounts receivable (2,409) (97) (3,276) 2,298 Income + taxes, net 3,041 (609) 2,744 (862) Other assets (1,255) (2,647) (1,447) (4,268) + Accounts payable 238 1,907 (874) 735 Accrued expenses and other liabilities + 2,562 2,210 2,763 491 Accrued revenue share 357 (80) 386 (1,022) Deferred + revenue 272 112 406 104 Net cash provided by operating activities 25,539 23,353 + 66,718 67,881 Investing activities\\nPurchases of property and equipment (6,819) + (7,276) (18,257) (23,890) Purchases of marketable securities (34,497) (17,054) + (95,106) (67,253) Maturities and sales of marketable securities 31,459 28,713 + 92,126 84,087 Purchases of non-marketable securities \",\n \"documentMetadata\": + {\n \"document\": \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/2eab9446e934f432e3df7808af857300\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q3_alphabet_earnings_release.pdf\",\n + \ \"title\": \"GOOG Exhibit 99.1 Q3 2022\"\n }\n }\n + \ },\n {\n \"chunkInfo\": {\n \"content\": \"Alphabet + Inc. CONSOLIDATED STATEMENTS OF CASH FLOWS\\n(In millions, unaudited)\\nQuarter + Ended June 30, Year To Date June 30,\\n2021 2022 2021 2022 Operating activities\\nNet + income $ 18,525 $ 16,002 $ 36,455 $ 32,438 Adjustments:\\nDepreciation and + impairment of property and equipment 2,730 3,698 5,255 7,289 Amortization + and impairment of intangible assets 215 201 443 392 Stock-based compensation + expense 3,803 4,782 7,548 9,286 Deferred income taxes 379 (2,147) 1,479 (4,237) + (Gain) loss on debt and equity securities, net (2,883) 1,041 (7,634) 2,478 + Other (8) 62 (263) 202 Changes in assets and liabilities, net of effects of + acquisitions:\\nAccounts receivable (3,661) (1,969) (867) 2,395 Income taxes, + net (1,082) (4,073) (297) (253) Other assets (199) (845) (192) (1,621) Accounts + payable (130) 1,201 (1,112) (1,172) Accrued expenses and other liabilities + 3,731 1,497 201 (1,719) Accrued revenue share 473 (114) 29 (942) Deferred + revenue (3) 86 134 (8) Net cash provided by operating activities 21,890 19,422 + 41,179 44,528 Investing activities\\nPurchases of property and equipment (5,496) + (6,828) (11,438) (16,614) Purchases of marketable securities (24,183) (21,737) + (60,609) (50,199) \",\n \"documentMetadata\": {\n \"document\": + \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/376dab8cf7b3807296f2bf1c6228e078\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q2_alphabet_earnings_release.pdf\",\n + \ \"title\": \"GOOG Exhibit 99.1 Q2 2022\"\n }\n }\n + \ },\n {\n \"chunkInfo\": {\n \"content\": \"The + associated costs,\\nincluding depreciation and impairment, are allocated to + operating segments as a service cost generally based on usage or headcount. + Unallocated corporate costs primarily include corporate initiatives, corporate + shared costs, such as finance and legal, including certain\\nfines and settlements, + as well as costs associated with certain shared R&D activities. Additionally, + hedging gains (losses) related to\\nrevenue are included in corporate costs. + As AI is critical to delivering our mission of bringing our breakthrough innovations + into the real world, beginning in January 2023, we\\nwill update our segment + reporting relating to certain of Alphabet\u2019s AI activities. DeepMind, + previously reported within Other Bets, will\\nbe reported as part of Alphabet\u2019s + corporate costs, reflecting its increasing collaboration with Google Services, + Google Cloud, and\\nOther Bets. Prior periods will be recast to conform to + the revised presentation. Our operating segments are not evaluated using asset + information. The following table presents information about our segments (in + millions): Year Ended December 31,\\n2020 2021 2022 Revenues:\\nGoogle Services + $ 168,635 $ 237,529 $ 253,528 Google Cloud 13,059 19,206 26,280 Other Bets + 657 753 1,068 Hedging gains (losses) 176 149 1,960 Total revenues $ 182,527 + $ 257,637 $ 282,836 Operating income (loss):\\nGoogle Services $ 54,606 $ + 91,855 $ 86,572 Google Cloud (5,607) (3,099) (2,968) Other Bets \",\n \"documentMetadata\": + {\n \"document\": \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/276cee4c4086600303bc561483691f9a\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022_alphabet_annual_report.pdf\",\n + \ \"title\": \"2022_alphabet_annual_report\"\n }\n }\n + \ },\n {\n \"chunkInfo\": {\n \"content\": \"56 ALPHABET + INC. ANNUAL REPORT PART II\\nITEM\_8\_\_FINANCIAL STATEMENTS AND SUPPLEMENTARY + DATA Note 2. Revenues\\nDisaggregated Revenues\\nThe following table presents + revenues disaggregated by type (in millions): Year Ended December 31,\\n2020 + 2021 2022 Google Search & other $ 104,062 $ 148,951 $ 162,450 YouTube ads + 19,772 28,845 29,243 Google Network 23,090 31,701 32,780 Google advertising + 146,924 209,497 224,473 Google other 21,711 28,032 29,055 Google Services + total 168,635 237,529 253,528 Google Cloud 13,059 19,206 26,280 Other Bets + 657 753 1,068 Hedging gains (losses) 176 149 1,960 Total revenues $ 182,527 + $ 257,637 $ 282,836\\nNo individual customer or groups of affiliated customers + represented more than 10% of our revenues in 2020, 2021, or 2022. The following + table presents revenues disaggregated by geography, based on the addresses + of our customers (in millions):\\nYear Ended December 31, 2020 2021 2022 United + States $ 85,014 47 % $ 117,854 46 % $ 134,814 48% EMEA(1) 55,370 30 79,107 + 31 82,062 29 APAC(1) 32,550 18 46,123 18 47,024 16 Other Americas(1) 9,417 + 5 14,404 5 16,976 6 Hedging gains (losses) 176 0 149 0 1,960 1 Total revenues + $ 182,527 100% $ 257,637 100 % $ 282,836 100% \",\n \"documentMetadata\": + {\n \"document\": \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/276cee4c4086600303bc561483691f9a\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022_alphabet_annual_report.pdf\",\n + \ \"title\": \"2022_alphabet_annual_report\"\n }\n }\n + \ }\n ]\n },\n \"answerQueryToken\": \"NMwKDAjEravVBhCr-_ORAxIkNmFjNGMzMTUtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi\"\n}\n,\r\n{\n + \ \"answer\": {\n \"state\": 4,\n \"answerText\": \"Alphabet\"\n },\n + \ \"answerQueryToken\": \"NMwKDAjEravVBhCr-_ORAxIkNmFjNGMzMTUtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi\"\n}\n,\r\n{\n + \ \"answer\": {\n \"state\": 4,\n \"answerText\": \"'s total revenue\"\n + \ },\n \"answerQueryToken\": \"NMwKDAjEravVBhCr-_ORAxIkNmFjNGMzMTUtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi\"\n}\n,\r\n{\n + \ \"answer\": {\n \"state\": 4,\n \"answerText\": \" in 2022 was $282,836 + million. \"\n },\n \"answerQueryToken\": \"NMwKDAjEravVBhCr-_ORAxIkNmFjNGMzMTUtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi\"\n}\n,\r\n{\n + \ \"answer\": {\n \"state\": 4,\n \"citations\": [\n {\n \"endIndex\": + \"54\",\n \"sources\": [\n {\n \"referenceId\": + \"0\"\n },\n {\n \"referenceId\": \"4\"\n },\n + \ {\n \"referenceId\": \"8\"\n },\n {\n + \ \"referenceId\": \"9\"\n }\n ]\n }\n ]\n + \ },\n \"answerQueryToken\": \"NMwKDAjEravVBhCr-_ORAxIkNmFjNGMzMTUtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi\"\n}\n,\r\n{\n + \ \"answer\": {\n \"state\": 4,\n \"answerText\": \"This represents + a 10% increase\"\n },\n \"answerQueryToken\": \"NMwKDAjEravVBhCr-_ORAxIkNmFjNGMzMTUtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi\"\n}\n,\r\n{\n + \ \"answer\": {\n \"state\": 4,\n \"answerText\": \" year-over-year, + or 14% in constant currency.\\n\\n\"\n },\n \"answerQueryToken\": \"NMwKDAjEravVBhCr-_ORAxIkNmFjNGMzMTUtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi\"\n}\n,\r\n{\n + \ \"answer\": {\n \"state\": 4,\n \"citations\": [\n {\n \"startIndex\": + \"55\",\n \"endIndex\": \"130\",\n \"sources\": [\n {\n + \ \"referenceId\": \"2\"\n }\n ]\n }\n ]\n + \ },\n \"answerQueryToken\": \"NMwKDAjEravVBhCr-_ORAxIkNmFjNGMzMTUtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi\"\n}\n,\r\n{\n + \ \"answer\": {\n \"state\": 4,\n \"answerText\": \"Here's a breakdown + of Alphabet's 2022 revenues by\"\n },\n \"answerQueryToken\": \"NMwKDAjEravVBhCr-_ORAxIkNmFjNGMzMTUtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi\"\n}\n,\r\n{\n + \ \"answer\": {\n \"state\": 4,\n \"answerText\": \" segment and type:\\n\\n\"\n + \ },\n \"answerQueryToken\": \"NMwKDAjEravVBhCr-_ORAxIkNmFjNGMzMTUtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi\"\n}\n,\r\n{\n + \ \"answer\": {\n \"state\": 4,\n \"answerText\": \"**By Segment:**\\n* + \ Google Services: $253,528 million\\n\"\n },\n \"answerQueryToken\": \"NMwKDAjEravVBhCr-_ORAxIkNmFjNGMzMTUtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi\"\n}\n,\r\n{\n + \ \"answer\": {\n \"state\": 4,\n \"citations\": [\n {\n \"startIndex\": + \"217\",\n \"endIndex\": \"254\",\n \"sources\": [\n {\n + \ \"referenceId\": \"8\"\n },\n {\n \"referenceId\": + \"9\"\n }\n ]\n }\n ]\n },\n \"answerQueryToken\": + \"NMwKDAjEravVBhCr-_ORAxIkNmFjNGMzMTUtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi\"\n}\n,\r\n{\n + \ \"answer\": {\n \"state\": 4,\n \"answerText\": \"* Google Cloud: + $26,280 million\\n\"\n },\n \"answerQueryToken\": \"NMwKDAjEravVBhCr-_ORAxIkNmFjNGMzMTUtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi\"\n}\n,\r\n{\n + \ \"answer\": {\n \"state\": 4,\n \"citations\": [\n {\n \"startIndex\": + \"255\",\n \"endIndex\": \"288\",\n \"sources\": [\n {\n + \ \"referenceId\": \"8\"\n },\n {\n \"referenceId\": + \"9\"\n }\n ]\n }\n ]\n },\n \"answerQueryToken\": + \"NMwKDAjEravVBhCr-_ORAxIkNmFjNGMzMTUtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi\"\n}\n,\r\n{\n + \ \"answer\": {\n \"state\": 4,\n \"answerText\": \"* Other Bets: + $1,068 million\\n\"\n },\n \"answerQueryToken\": \"NMwKDAjEravVBhCr-_ORAxIkNmFjNGMzMTUtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi\"\n}\n,\r\n{\n + \ \"answer\": {\n \"state\": 4,\n \"citations\": [\n {\n \"startIndex\": + \"289\",\n \"endIndex\": \"319\",\n \"sources\": [\n {\n + \ \"referenceId\": \"8\"\n },\n {\n \"referenceId\": + \"9\"\n }\n ]\n }\n ]\n },\n \"answerQueryToken\": + \"NMwKDAjEravVBhCr-_ORAxIkNmFjNGMzMTUtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi\"\n}\n,\r\n{\n + \ \"answer\": {\n \"state\": 4,\n \"answerText\": \"* Hedging gains + (losses\"\n },\n \"answerQueryToken\": \"NMwKDAjEravVBhCr-_ORAxIkNmFjNGMzMTUtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi\"\n}\n,\r\n{\n + \ \"answer\": {\n \"state\": 4,\n \"answerText\": \"): $1,960 million\\n\\n\"\n + \ },\n \"answerQueryToken\": \"NMwKDAjEravVBhCr-_ORAxIkNmFjNGMzMTUtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi\"\n}\n,\r\n{\n + \ \"answer\": {\n \"state\": 4,\n \"citations\": [\n {\n \"startIndex\": + \"320\",\n \"endIndex\": \"362\",\n \"sources\": [\n {\n + \ \"referenceId\": \"8\"\n },\n {\n \"referenceId\": + \"9\"\n }\n ]\n }\n ]\n },\n \"answerQueryToken\": + \"NMwKDAjEravVBhCr-_ORAxIkNmFjNGMzMTUtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi\"\n}\n,\r\n{\n + \ \"answer\": {\n \"state\": 4,\n \"answerText\": \"**By Type (within + Google Services):**\\n\"\n },\n \"answerQueryToken\": \"NMwKDAjEravVBhCr-_ORAxIkNmFjNGMzMTUtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi\"\n}\n,\r\n{\n + \ \"answer\": {\n \"state\": 4,\n \"answerText\": \"* Google Search + & other: $162,450 million\\n\"\n },\n \"answerQueryToken\": \"NMwKDAjEravVBhCr-_ORAxIkNmFjNGMzMTUtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi\"\n}\n,\r\n{\n + \ \"answer\": {\n \"state\": 4,\n \"citations\": [\n {\n \"startIndex\": + \"402\",\n \"endIndex\": \"445\",\n \"sources\": [\n {\n + \ \"referenceId\": \"9\"\n }\n ]\n }\n ]\n + \ },\n \"answerQueryToken\": \"NMwKDAjEravVBhCr-_ORAxIkNmFjNGMzMTUtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi\"\n}\n,\r\n{\n + \ \"answer\": {\n \"state\": 4,\n \"answerText\": \"* YouTube\"\n + \ },\n \"answerQueryToken\": \"NMwKDAjEravVBhCr-_ORAxIkNmFjNGMzMTUtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi\"\n}\n,\r\n{\n + \ \"answer\": {\n \"state\": 4,\n \"answerText\": \" ads: $29,243 million\\n\"\n + \ },\n \"answerQueryToken\": \"NMwKDAjEravVBhCr-_ORAxIkNmFjNGMzMTUtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi\"\n}\n,\r\n{\n + \ \"answer\": {\n \"state\": 4,\n \"citations\": [\n {\n \"startIndex\": + \"446\",\n \"endIndex\": \"478\",\n \"sources\": [\n {\n + \ \"referenceId\": \"9\"\n }\n ]\n }\n ]\n + \ },\n \"answerQueryToken\": \"NMwKDAjEravVBhCr-_ORAxIkNmFjNGMzMTUtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi\"\n}\n,\r\n{\n + \ \"answer\": {\n \"state\": 4,\n \"answerText\": \"* Google Network: + $32,780 million\\n\"\n },\n \"answerQueryToken\": \"NMwKDAjEravVBhCr-_ORAxIkNmFjNGMzMTUtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi\"\n}\n,\r\n{\n + \ \"answer\": {\n \"state\": 4,\n \"citations\": [\n {\n \"startIndex\": + \"479\",\n \"endIndex\": \"514\",\n \"sources\": [\n {\n + \ \"referenceId\": \"9\"\n }\n ]\n }\n ]\n + \ },\n \"answerQueryToken\": \"NMwKDAjEravVBhCr-_ORAxIkNmFjNGMzMTUtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi\"\n}\n,\r\n{\n + \ \"answer\": {\n \"state\": 4,\n \"answerText\": \"* Google advertising + (total): $224,473 million\\n\"\n },\n \"answerQueryToken\": \"NMwKDAjEravVBhCr-_ORAxIkNmFjNGMzMTUtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi\"\n}\n,\r\n{\n + \ \"answer\": {\n \"state\": 4,\n \"citations\": [\n {\n \"startIndex\": + \"515\",\n \"endIndex\": \"563\",\n \"sources\": [\n {\n + \ \"referenceId\": \"9\"\n }\n ]\n }\n ]\n + \ },\n \"answerQueryToken\": \"NMwKDAjEravVBhCr-_ORAxIkNmFjNGMzMTUtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi\"\n}\n,\r\n{\n + \ \"answer\": {\n \"state\": 4,\n \"answerText\": \"* Google other: + $29,055 million\\n\\n\"\n },\n \"answerQueryToken\": \"NMwKDAjEravVBhCr-_ORAxIkNmFjNGMzMTUtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi\"\n}\n,\r\n{\n + \ \"answer\": {\n \"state\": 4,\n \"citations\": [\n {\n \"startIndex\": + \"564\",\n \"endIndex\": \"597\",\n \"sources\": [\n {\n + \ \"referenceId\": \"9\"\n }\n ]\n }\n ]\n + \ },\n \"answerQueryToken\": \"NMwKDAjEravVBhCr-_ORAxIkNmFjNGMzMTUtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi\"\n}\n,\r\n{\n + \ \"answer\": {\n \"state\": 4,\n \"answerText\": \"**By Geography:**\\n\"\n + \ },\n \"answerQueryToken\": \"NMwKDAjEravVBhCr-_ORAxIkNmFjNGMzMTUtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi\"\n}\n,\r\n{\n + \ \"answer\": {\n \"state\": 4,\n \"answerText\": \"* United States: + $134,814 million (48%)\\n\"\n },\n \"answerQueryToken\": \"NMwKDAjEravVBhCr-_ORAxIkNmFjNGMzMTUtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi\"\n}\n,\r\n{\n + \ \"answer\": {\n \"state\": 4,\n \"citations\": [\n {\n \"startIndex\": + \"617\",\n \"endIndex\": \"658\",\n \"sources\": [\n {\n + \ \"referenceId\": \"9\"\n }\n ]\n }\n ]\n + \ },\n \"answerQueryToken\": \"NMwKDAjEravVBhCr-_ORAxIkNmFjNGMzMTUtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi\"\n}\n,\r\n{\n + \ \"answer\": {\n \"state\": 4,\n \"answerText\": \"* EMEA (Europe, + Middle East, and Africa): $82,062 million (29%)\\n\"\n },\n \"answerQueryToken\": + \"NMwKDAjEravVBhCr-_ORAxIkNmFjNGMzMTUtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi\"\n}\n,\r\n{\n + \ \"answer\": {\n \"state\": 4,\n \"citations\": [\n {\n \"startIndex\": + \"659\",\n \"endIndex\": \"724\",\n \"sources\": [\n {\n + \ \"referenceId\": \"9\"\n }\n ]\n }\n ]\n + \ },\n \"answerQueryToken\": \"NMwKDAjEravVBhCr-_ORAxIkNmFjNGMzMTUtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi\"\n}\n,\r\n{\n + \ \"answer\": {\n \"state\": 4,\n \"answerText\": \"* APAC (\"\n },\n + \ \"answerQueryToken\": \"NMwKDAjEravVBhCr-_ORAxIkNmFjNGMzMTUtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi\"\n}\n,\r\n{\n + \ \"answer\": {\n \"state\": 4,\n \"answerText\": \"Asia-Pacific): $47,024 + million (16%)\\n\"\n },\n \"answerQueryToken\": \"NMwKDAjEravVBhCr-_ORAxIkNmFjNGMzMTUtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi\"\n}\n,\r\n{\n + \ \"answer\": {\n \"state\": 4,\n \"citations\": [\n {\n \"startIndex\": + \"725\",\n \"endIndex\": \"771\",\n \"sources\": [\n {\n + \ \"referenceId\": \"9\"\n }\n ]\n }\n ]\n + \ },\n \"answerQueryToken\": \"NMwKDAjEravVBhCr-_ORAxIkNmFjNGMzMTUtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi\"\n}\n,\r\n{\n + \ \"answer\": {\n \"state\": 4,\n \"answerText\": \"* Other Americas: + $16,976 million (6%)\\n\"\n },\n \"answerQueryToken\": \"NMwKDAjEravVBhCr-_ORAxIkNmFjNGMzMTUtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi\"\n}\n,\r\n{\n + \ \"answer\": {\n \"state\": 4,\n \"citations\": [\n {\n \"startIndex\": + \"772\",\n \"endIndex\": \"812\",\n \"sources\": [\n {\n + \ \"referenceId\": \"9\"\n }\n ]\n }\n ]\n + \ },\n \"answerQueryToken\": \"NMwKDAjEravVBhCr-_ORAxIkNmFjNGMzMTUtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi\"\n}\n,\r\n{\n + \ \"answer\": {\n \"state\": 4,\n \"answerText\": \"* Hedging gains + (losses): $1,960 million (1%)\\n\\n\"\n },\n \"answerQueryToken\": \"NMwKDAjEravVBhCr-_ORAxIkNmFjNGMzMTUtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi\"\n}\n,\r\n{\n + \ \"answer\": {\n \"state\": 4,\n \"citations\": [\n {\n \"startIndex\": + \"813\",\n \"endIndex\": \"860\",\n \"sources\": [\n {\n + \ \"referenceId\": \"9\"\n }\n ]\n }\n ]\n + \ },\n \"answerQueryToken\": \"NMwKDAjEravVBhCr-_ORAxIkNmFjNGMzMTUtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi\"\n}\n,\r\n{\n + \ \"answer\": {\n \"state\": 4,\n \"answerText\": \"Alphabet\"\n },\n + \ \"answerQueryToken\": \"NMwKDAjEravVBhCr-_ORAxIkNmFjNGMzMTUtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi\"\n}\n,\r\n{\n + \ \"answer\": {\n \"state\": 4,\n \"answerText\": \"'s CFO, Ruth Porat, + stated that Q4 2022 consolidated revenues were $76 billion, up 1% year over + year, or up 7% in constant currency, and the full year 2022 revenue was $283 + billion, up 10%,\"\n },\n \"answerQueryToken\": \"NMwKDAjEravVBhCr-_ORAxIkNmFjNGMzMTUtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi\"\n}\n,\r\n{\n + \ \"answer\": {\n \"state\": 4,\n \"answerText\": \" or up 14% in constant + currency.\\n\\n\"\n },\n \"answerQueryToken\": \"NMwKDAjEravVBhCr-_ORAxIkNmFjNGMzMTUtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi\"\n}\n,\r\n{\n + \ \"answer\": {\n \"state\": 4,\n \"citations\": [\n {\n \"startIndex\": + \"862\",\n \"endIndex\": \"1091\",\n \"sources\": [\n {\n + \ \"referenceId\": \"2\"\n }\n ]\n }\n ]\n + \ },\n \"answerQueryToken\": \"NMwKDAjEravVBhCr-_ORAxIkNmFjNGMzMTUtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi\"\n}\n,\r\n{\n + \ \"answer\": {\n \"state\": 4,\n \"answerText\": \"For the first quarter + of 2022, Alphabet reported revenues of $68,011 million, a 23% year-over-year + growth. \"\n },\n \"answerQueryToken\": \"NMwKDAjEravVBhCr-_ORAxIkNmFjNGMzMTUtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi\"\n}\n,\r\n{\n + \ \"answer\": {\n \"state\": 4,\n \"citations\": [\n {\n \"startIndex\": + \"1093\",\n \"endIndex\": \"1199\",\n \"sources\": [\n {\n + \ \"referenceId\": \"1\"\n }\n ]\n }\n ]\n + \ },\n \"answerQueryToken\": \"NMwKDAjEravVBhCr-_ORAxIkNmFjNGMzMTUtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi\"\n}\n,\r\n{\n + \ \"answer\": {\n \"state\": 4,\n \"answerText\": \"In the third quarter + of 2022, revenues were\"\n },\n \"answerQueryToken\": \"NMwKDAjEravVBhCr-_ORAxIkNmFjNGMzMTUtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi\"\n}\n,\r\n{\n + \ \"answer\": {\n \"state\": 4,\n \"answerText\": \" $69.1 billion, + up 6% versus the prior year or up 11% on a constant currency basis.\"\n },\n + \ \"answerQueryToken\": \"NMwKDAjEravVBhCr-_ORAxIkNmFjNGMzMTUtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi\"\n}\n,\r\n{\n + \ \"answer\": {\n \"state\": 4,\n \"citations\": [\n {\n \"startIndex\": + \"1200\",\n \"endIndex\": \"1326\",\n \"sources\": [\n {\n + \ \"referenceId\": \"5\"\n }\n ]\n }\n ]\n + \ },\n \"answerQueryToken\": \"NMwKDAjEravVBhCr-_ORAxIkNmFjNGMzMTUtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi\"\n}\n,\r\n{\n + \ \"answer\": {\n \"state\": 3,\n \"answerText\": \"Alphabet's total + revenue in 2022 was $282,836 million. This represents a 10% increase year-over-year, + or 14% in constant currency.\\n\\nHere's a breakdown of Alphabet's 2022 revenues + by segment and type:\\n\\n**By Segment:**\\n* Google Services: $253,528 + million\\n* Google Cloud: $26,280 million\\n* Other Bets: $1,068 million\\n* + \ Hedging gains (losses): $1,960 million\\n\\n**By Type (within Google Services):**\\n* + \ Google Search & other: $162,450 million\\n* YouTube ads: $29,243 million\\n* + \ Google Network: $32,780 million\\n* Google advertising (total): $224,473 + million\\n* Google other: $29,055 million\\n\\n**By Geography:**\\n* United + States: $134,814 million (48%)\\n* EMEA (Europe, Middle East, and Africa): + $82,062 million (29%)\\n* APAC (Asia-Pacific): $47,024 million (16%)\\n* + \ Other Americas: $16,976 million (6%)\\n* Hedging gains (losses): $1,960 + million (1%)\\n\\nAlphabet's CFO, Ruth Porat, stated that Q4 2022 consolidated + revenues were $76 billion, up 1% year over year, or up 7% in constant currency, + and the full year 2022 revenue was $283 billion, up 10%, or up 14% in constant + currency.\\n\\nFor the first quarter of 2022, Alphabet reported revenues of + $68,011 million, a 23% year-over-year growth. In the third quarter of 2022, + revenues were $69.1 billion, up 6% versus the prior year or up 11% on a constant + currency basis.\",\n \"citations\": [\n {\n \"endIndex\": \"54\",\n + \ \"sources\": [\n {\n \"referenceId\": \"0\"\n + \ },\n {\n \"referenceId\": \"4\"\n },\n + \ {\n \"referenceId\": \"8\"\n },\n {\n + \ \"referenceId\": \"9\"\n }\n ]\n },\n {\n + \ \"startIndex\": \"55\",\n \"endIndex\": \"130\",\n \"sources\": + [\n {\n \"referenceId\": \"2\"\n }\n ]\n + \ },\n {\n \"startIndex\": \"217\",\n \"endIndex\": + \"254\",\n \"sources\": [\n {\n \"referenceId\": + \"8\"\n },\n {\n \"referenceId\": \"9\"\n }\n + \ ]\n },\n {\n \"startIndex\": \"255\",\n \"endIndex\": + \"288\",\n \"sources\": [\n {\n \"referenceId\": + \"8\"\n },\n {\n \"referenceId\": \"9\"\n }\n + \ ]\n },\n {\n \"startIndex\": \"289\",\n \"endIndex\": + \"319\",\n \"sources\": [\n {\n \"referenceId\": + \"8\"\n },\n {\n \"referenceId\": \"9\"\n }\n + \ ]\n },\n {\n \"startIndex\": \"320\",\n \"endIndex\": + \"362\",\n \"sources\": [\n {\n \"referenceId\": + \"8\"\n },\n {\n \"referenceId\": \"9\"\n }\n + \ ]\n },\n {\n \"startIndex\": \"402\",\n \"endIndex\": + \"445\",\n \"sources\": [\n {\n \"referenceId\": + \"9\"\n }\n ]\n },\n {\n \"startIndex\": + \"446\",\n \"endIndex\": \"478\",\n \"sources\": [\n {\n + \ \"referenceId\": \"9\"\n }\n ]\n },\n {\n + \ \"startIndex\": \"479\",\n \"endIndex\": \"514\",\n \"sources\": + [\n {\n \"referenceId\": \"9\"\n }\n ]\n + \ },\n {\n \"startIndex\": \"515\",\n \"endIndex\": + \"563\",\n \"sources\": [\n {\n \"referenceId\": + \"9\"\n }\n ]\n },\n {\n \"startIndex\": + \"564\",\n \"endIndex\": \"597\",\n \"sources\": [\n {\n + \ \"referenceId\": \"9\"\n }\n ]\n },\n {\n + \ \"startIndex\": \"617\",\n \"endIndex\": \"658\",\n \"sources\": + [\n {\n \"referenceId\": \"9\"\n }\n ]\n + \ },\n {\n \"startIndex\": \"659\",\n \"endIndex\": + \"724\",\n \"sources\": [\n {\n \"referenceId\": + \"9\"\n }\n ]\n },\n {\n \"startIndex\": + \"725\",\n \"endIndex\": \"771\",\n \"sources\": [\n {\n + \ \"referenceId\": \"9\"\n }\n ]\n },\n {\n + \ \"startIndex\": \"772\",\n \"endIndex\": \"812\",\n \"sources\": + [\n {\n \"referenceId\": \"9\"\n }\n ]\n + \ },\n {\n \"startIndex\": \"813\",\n \"endIndex\": + \"860\",\n \"sources\": [\n {\n \"referenceId\": + \"9\"\n }\n ]\n },\n {\n \"startIndex\": + \"862\",\n \"endIndex\": \"1091\",\n \"sources\": [\n {\n + \ \"referenceId\": \"2\"\n }\n ]\n },\n {\n + \ \"startIndex\": \"1093\",\n \"endIndex\": \"1199\",\n \"sources\": + [\n {\n \"referenceId\": \"1\"\n }\n ]\n + \ },\n {\n \"startIndex\": \"1200\",\n \"endIndex\": + \"1326\",\n \"sources\": [\n {\n \"referenceId\": + \"5\"\n }\n ]\n }\n ],\n \"references\": [\n {\n + \ \"chunkInfo\": {\n \"content\": \"ALPHABET INC. ANNUAL REPORT + 45 PART II\\nITEM\_8\_\_FINANCIAL STATEMENTS AND SUPPLEMENTARY DATA Alphabet + Inc. CONSOLIDATED STATEMENTS OF INCOME (In millions, except per share amounts) + Year Ended December 31,\\n2020 2021 2022 Revenues $ 182,527 $ 257,637 $ 282,836 + Costs and expenses:\\nCost of revenues 84,732 110,939 126,203 Research and + development 27,573 31,562 39,500 Sales and marketing 17,946 22,912 26,567 + General and administrative 11,052 13,510 15,724 Total costs and expenses 141,303 + 178,923 207,994 Income from operations 41,224 78,714 74,842 Other income (expense), + net 6,858 12,020 (3,514) Income before income taxes 48,082 90,734 71,328 Provision + for income taxes 7,813 14,701 11,356 Net income $ 40,269 $ 76,033 $ 59,972 + Basic net income per share of Class A, Class B, and Class C stock $ 2.96 $ + 5.69 $ 4.59 Diluted net income per share of Class A, Class B, and Class C + stock $ 2.93 $ 5.61 $ 4.56 See accompanying notes. \",\n \"documentMetadata\": + {\n \"document\": \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/276cee4c4086600303bc561483691f9a\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022_alphabet_annual_report.pdf\",\n + \ \"title\": \"2022_alphabet_annual_report\"\n }\n }\n + \ },\n {\n \"chunkInfo\": {\n \"content\": \"Alphabet + Announces First Quarter 2022 Results\\nMOUNTAIN VIEW, Calif. \u2013 April + 26, 2022 \u2013 Alphabet Inc. (NASDAQ: GOOG, GOOGL) today announced financial\\nresults + for the quarter ended March 31, 2022. Sundar Pichai, CEO of Alphabet and Google, + said: \u201CQ1 saw strong growth in Search and Cloud, in particular, which\\nare + both helping people and businesses as the digital transformation continues. + We\u2019ll keep investing in great\\nproducts and services, and creating opportunities + for partners and local communities around the world.\u201D Ruth Porat, CFO + of Alphabet and Google, said: \u201CWe are pleased with Q1 revenue growth + of 23% year over year. We\\ncontinue to make considered investments in Capex, + R&D and talent to support long-term value creation for all\\nstakeholders.\u201D + Q1 2022 financial highlights\\nThe following table summarizes our consolidated + financial results for the quarters ended March 31, 2021 and 2022\\n(in millions, + except for per share information and percentages; unaudited). Quarter Ended + March 31,\\n2021 2022 Revenues $ 55,314 $ 68,011 Change in revenues year over + year 34 % 23 % Change in constant currency revenues year over year(1) 32 % + 26 % Operating income $ 16,437 $ 20,094 Operating margin 30 % 30 % Other income + (expense), net $ 4,846 $ (1,160) Net income $ 17,930 $ 16,436 \",\n \"documentMetadata\": + {\n \"document\": \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/c201efed2be20c6a4116611ea16be9cd\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q1_alphabet_earnings_release.pdf\",\n + \ \"title\": \"GOOG Exhibit 99.1 Q1 2022\"\n }\n }\n + \ },\n {\n \"chunkInfo\": {\n \"content\": \"Alphabet + Announces Fourth Quarter and Fiscal Year 2022 Results\\nMOUNTAIN VIEW, Calif. + \u2013 February 2, 2023 \u2013 Alphabet Inc. (NASDAQ: GOOG, GOOGL) today announced + financial\\nresults for the quarter and fiscal year ended December 31, 2022. + Sundar Pichai, CEO of Alphabet and Google, said: \u201COur long-term investments + in deep computer science make us\\nextremely well-positioned as AI reaches + an inflection point, and I\u2019m excited by the AI-driven leaps we\u2019re + about to unveil\\nin Search and beyond. There\u2019s also great momentum in + Cloud, YouTube subscriptions, and our Pixel devices. We\u2019re\\non an important + journey to re-engineer our cost structure in a durable way and to build financially + sustainable, vibrant,\\ngrowing businesses across Alphabet.\u201D Ruth Porat, + CFO of Alphabet and Google, said: \u201COur Q4 consolidated revenues were + $76 billion, up 1% year over year,\\nor up 7% in constant currency, and $283 + billion for the full year 2022, up 10%, or up 14% in constant currency. We\\nhave + significant work underway to improve all aspects of our cost structure, in + support of our investments in our\\nhighest growth priorities to deliver long-term, + profitable growth.\u201D \",\n \"documentMetadata\": {\n \"document\": + \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/513da7d1b9f7739087606aadc626f2dc\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q4_alphabet_earnings_release.pdf\",\n + \ \"title\": \"GOOG Exhibit 99.1 Q4 2022\"\n }\n }\n + \ },\n {\n \"chunkInfo\": {\n \"content\": \"Alphabet + Announces Third Quarter 2023 Results\\nMOUNTAIN VIEW, Calif. \u2013 October + 24, 2023 \u2013 Alphabet Inc. (NASDAQ: GOOG, GOOGL) today announced\\nfinancial + results for the quarter ended September 30, 2023. Sundar Pichai, CEO, said: + \u201CI\u2019m pleased with our financial results and our product momentum + this quarter, with AI\\ndriven innovations across Search, YouTube, Cloud, + our Pixel devices and more. We\u2019re continuing to focus on\\nmaking AI + more helpful for everyone; there\u2019s exciting progress and lots more to + come.\u201D Ruth Porat, President and Chief Investment Officer; CFO said: + \u201CThe fundamental strength of our business was\\napparent again in Q3, + with $77 billion in revenue, up 11% year over year, driven by meaningful growth + in Search\\nand YouTube, and momentum in Cloud. We continue to focus on judicious + capital allocation to deliver sustainable\\nfinancial value.\u201D Q3 2023 + Financial Highlights (unaudited)\\nThe following table summarizes our consolidated + financial results for the quarters ended September 30, 2022 and\\n2023 (in + millions, except for per share information and percentages). Quarter Ended + September 30,\\n2022 2023 Revenues $ 69,092 $ 76,693 Change in revenues year + over year 6 % 11 % Change in constant currency revenues year over year(1) + 11 % 11 % Operating income $ 17,135 $ 21,343 Operating margin 25 % \",\n \"documentMetadata\": + {\n \"document\": \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/4c7389ffaf06b73e6e380eac7a39ea17\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2023q3-alphabet-earnings-release.pdf\",\n + \ \"title\": \"GOOG Exhibit 99.1 Q3 2023\"\n }\n }\n + \ },\n {\n \"chunkInfo\": {\n \"content\": \"Alphabet + Inc. Consolidated Statements of Income Year Ended December 31, (in millions, + except per share amounts) 2021 2022 2023 Revenues $ 257,637 $ 282,836 $ 307,394 + Costs and expenses:\\nCost of revenues 110,939 126,203 133,332 Research and + development 31,562 39,500 45,427 Sales and marketing 22,912 26,567 27,917 + General and administrative 13,510 15,724 16,425 Total costs and expenses 178,923 + 207,994 223,101 Income from operations 78,714 74,842 84,293 Other income (expense), + net 12,020 (3,514) 1,424 Income before income taxes 90,734 71,328 85,717 Provision + for income taxes 14,701 11,356 11,922 Net income $ 76,033 $ 59,972 $ 73,795 + Basic net income per share of Class A, Class B, and Class C stock $ 5.69 $ + 4.59 $ 5.84 Diluted net income per share of Class A, Class B, and Class C + stock $ 5.61 $ 4.56 $ 5.80 See accompanying notes. 50 Alphabet 2023 Annual + Report Part I Part II Part III Part IV \",\n \"documentMetadata\": + {\n \"document\": \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/3c617f6f4b023305d82916f0093ca13d\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/goog023-alphabet-2023-annual-report-web-1.pdf\",\n + \ \"title\": \"goog023-alphabet-2023-annual-report-web-1\"\n }\n + \ }\n },\n {\n \"chunkInfo\": {\n \"content\": + \"Alphabet Announces Third Quarter 2022 Results\\nMOUNTAIN VIEW, Calif. \u2013 + October 25, 2022 \u2013 Alphabet Inc. (NASDAQ: GOOG, GOOGL) today announced\\nfinancial + results for the quarter ended September 30, 2022. Sundar Pichai, CEO of Alphabet + and Google, said: \u201CWe\u2019re sharpening our focus on a clear set of + product and\\nbusiness priorities. Product announcements we\u2019ve made in + just the past month alone have shown that very clearly,\\nincluding significant + improvements to both Search and Cloud, powered by AI, and new ways to monetize + YouTube\\nShorts. We are focused on both investing responsibly for the long + term and being responsive to the economic\\nenvironment.\u201D Ruth Porat, + CFO of Alphabet and Google, said: \u201COur third quarter revenues were $69.1 + billion, up 6% versus last\\nyear or up 11% on a constant currency basis. + Financial results for the third quarter reflect healthy fundamental\\ngrowth + in Search and momentum in Cloud, while affected by foreign exchange. We\u2019re + working to realign resources\\nto fuel our highest growth priorities.\u201D + Q3 2022 financial highlights\\nThe following table summarizes our consolidated + financial results for the quarters ended September 30, 2021 and\\n2022 (in + millions, except for per share information and percentages; unaudited). Quarter + Ended September 30,\\n2021 2022 Revenues $ 65,118 $ 69,092 Change in revenues + year over year 41 % \",\n \"documentMetadata\": {\n \"document\": + \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/2eab9446e934f432e3df7808af857300\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q3_alphabet_earnings_release.pdf\",\n + \ \"title\": \"GOOG Exhibit 99.1 Q3 2022\"\n }\n }\n + \ },\n {\n \"chunkInfo\": {\n \"content\": \"Alphabet + Inc. CONSOLIDATED STATEMENTS OF CASH FLOWS\\n(In millions, unaudited)\\nQuarter + Ended September 30, Year to Date September 30, 2021 2022 2021 2022 Operating + activities\\nNet income $ 18,936 $ 13,910 $ 55,391 $ 46,348 Adjustments:\\nDepreciation + and impairment of property and\\nequipment 3,085 3,933 8,340 11,222 Amortization + and impairment of intangible assets 219 113 662 505 Stock-based compensation + expense 3,874 4,976 11,422 14,262 Deferred income taxes (1,287) (1,920) 192 + (6,157) (Gain) loss on debt and equity securities, net (2,158) 1,378 (9,792) + 3,856 Other 64 167 (199) 369 Changes in assets and liabilities, net of effects + of\\nacquisitions:\\nAccounts receivable (2,409) (97) (3,276) 2,298 Income + taxes, net 3,041 (609) 2,744 (862) Other assets (1,255) (2,647) (1,447) (4,268) + Accounts payable 238 1,907 (874) 735 Accrued expenses and other liabilities + 2,562 2,210 2,763 491 Accrued revenue share 357 (80) 386 (1,022) Deferred + revenue 272 112 406 104 Net cash provided by operating activities 25,539 23,353 + 66,718 67,881 Investing activities\\nPurchases of property and equipment (6,819) + (7,276) (18,257) (23,890) Purchases of marketable securities (34,497) (17,054) + (95,106) (67,253) Maturities and sales of marketable securities 31,459 28,713 + 92,126 84,087 Purchases of non-marketable securities \",\n \"documentMetadata\": + {\n \"document\": \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/2eab9446e934f432e3df7808af857300\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q3_alphabet_earnings_release.pdf\",\n + \ \"title\": \"GOOG Exhibit 99.1 Q3 2022\"\n }\n }\n + \ },\n {\n \"chunkInfo\": {\n \"content\": \"Alphabet + Inc. CONSOLIDATED STATEMENTS OF CASH FLOWS\\n(In millions, unaudited)\\nQuarter + Ended June 30, Year To Date June 30,\\n2021 2022 2021 2022 Operating activities\\nNet + income $ 18,525 $ 16,002 $ 36,455 $ 32,438 Adjustments:\\nDepreciation and + impairment of property and equipment 2,730 3,698 5,255 7,289 Amortization + and impairment of intangible assets 215 201 443 392 Stock-based compensation + expense 3,803 4,782 7,548 9,286 Deferred income taxes 379 (2,147) 1,479 (4,237) + (Gain) loss on debt and equity securities, net (2,883) 1,041 (7,634) 2,478 + Other (8) 62 (263) 202 Changes in assets and liabilities, net of effects of + acquisitions:\\nAccounts receivable (3,661) (1,969) (867) 2,395 Income taxes, + net (1,082) (4,073) (297) (253) Other assets (199) (845) (192) (1,621) Accounts + payable (130) 1,201 (1,112) (1,172) Accrued expenses and other liabilities + 3,731 1,497 201 (1,719) Accrued revenue share 473 (114) 29 (942) Deferred + revenue (3) 86 134 (8) Net cash provided by operating activities 21,890 19,422 + 41,179 44,528 Investing activities\\nPurchases of property and equipment (5,496) + (6,828) (11,438) (16,614) Purchases of marketable securities (24,183) (21,737) + (60,609) (50,199) \",\n \"documentMetadata\": {\n \"document\": + \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/376dab8cf7b3807296f2bf1c6228e078\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q2_alphabet_earnings_release.pdf\",\n + \ \"title\": \"GOOG Exhibit 99.1 Q2 2022\"\n }\n }\n + \ },\n {\n \"chunkInfo\": {\n \"content\": \"The + associated costs,\\nincluding depreciation and impairment, are allocated to + operating segments as a service cost generally based on usage or headcount. + Unallocated corporate costs primarily include corporate initiatives, corporate + shared costs, such as finance and legal, including certain\\nfines and settlements, + as well as costs associated with certain shared R&D activities. Additionally, + hedging gains (losses) related to\\nrevenue are included in corporate costs. + As AI is critical to delivering our mission of bringing our breakthrough innovations + into the real world, beginning in January 2023, we\\nwill update our segment + reporting relating to certain of Alphabet\u2019s AI activities. DeepMind, + previously reported within Other Bets, will\\nbe reported as part of Alphabet\u2019s + corporate costs, reflecting its increasing collaboration with Google Services, + Google Cloud, and\\nOther Bets. Prior periods will be recast to conform to + the revised presentation. Our operating segments are not evaluated using asset + information. The following table presents information about our segments (in + millions): Year Ended December 31,\\n2020 2021 2022 Revenues:\\nGoogle Services + $ 168,635 $ 237,529 $ 253,528 Google Cloud 13,059 19,206 26,280 Other Bets + 657 753 1,068 Hedging gains (losses) 176 149 1,960 Total revenues $ 182,527 + $ 257,637 $ 282,836 Operating income (loss):\\nGoogle Services $ 54,606 $ + 91,855 $ 86,572 Google Cloud (5,607) (3,099) (2,968) Other Bets \",\n \"documentMetadata\": + {\n \"document\": \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/276cee4c4086600303bc561483691f9a\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022_alphabet_annual_report.pdf\",\n + \ \"title\": \"2022_alphabet_annual_report\"\n }\n }\n + \ },\n {\n \"chunkInfo\": {\n \"content\": \"56 ALPHABET + INC. ANNUAL REPORT PART II\\nITEM\_8\_\_FINANCIAL STATEMENTS AND SUPPLEMENTARY + DATA Note 2. Revenues\\nDisaggregated Revenues\\nThe following table presents + revenues disaggregated by type (in millions): Year Ended December 31,\\n2020 + 2021 2022 Google Search & other $ 104,062 $ 148,951 $ 162,450 YouTube ads + 19,772 28,845 29,243 Google Network 23,090 31,701 32,780 Google advertising + 146,924 209,497 224,473 Google other 21,711 28,032 29,055 Google Services + total 168,635 237,529 253,528 Google Cloud 13,059 19,206 26,280 Other Bets + 657 753 1,068 Hedging gains (losses) 176 149 1,960 Total revenues $ 182,527 + $ 257,637 $ 282,836\\nNo individual customer or groups of affiliated customers + represented more than 10% of our revenues in 2020, 2021, or 2022. The following + table presents revenues disaggregated by geography, based on the addresses + of our customers (in millions):\\nYear Ended December 31, 2020 2021 2022 United + States $ 85,014 47 % $ 117,854 46 % $ 134,814 48% EMEA(1) 55,370 30 79,107 + 31 82,062 29 APAC(1) 32,550 18 46,123 18 47,024 16 Other Americas(1) 9,417 + 5 14,404 5 16,976 6 Hedging gains (losses) 176 0 149 0 1,960 1 Total revenues + $ 182,527 100% $ 257,637 100 % $ 282,836 100% \",\n \"documentMetadata\": + {\n \"document\": \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/276cee4c4086600303bc561483691f9a\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022_alphabet_annual_report.pdf\",\n + \ \"title\": \"2022_alphabet_annual_report\"\n }\n }\n + \ }\n ],\n \"steps\": [\n {\n \"state\": 3,\n \"description\": + \"Rephrase the query and search.\",\n \"actions\": [\n {\n + \ \"searchAction\": {\n \"query\": \"What was Alphabet's + revenue in 2022?\"\n },\n \"observation\": {\n \"searchResults\": + [\n {\n \"document\": \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/513da7d1b9f7739087606aadc626f2dc\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q4_alphabet_earnings_release.pdf\",\n + \ \"title\": \"GOOG Exhibit 99.1 Q4 2022\",\n \"snippetInfo\": + [\n {\n \"snippet\": \"... \\u003cb\\u003eAlphabet\\u003c/b\\u003e.\u201D + Ruth Porat, CFO of \\u003cb\\u003eAlphabet\\u003c/b\\u003e and Google, said: + \u201COur Q4 consolidated \\u003cb\\u003erevenues\\u003c/b\\u003e were $76 + billion, up 1% year over year, or up 7% in constant currency ...\",\n + \ \"snippetStatus\": \"SUCCESS\"\n }\n + \ ]\n },\n {\n \"document\": + \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/c201efed2be20c6a4116611ea16be9cd\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q1_alphabet_earnings_release.pdf\",\n + \ \"title\": \"GOOG Exhibit 99.1 Q1 2022\",\n \"snippetInfo\": + [\n {\n \"snippet\": \"\u2013 April + 26, \\u003cb\\u003e2022\\u003c/b\\u003e \u2013 \\u003cb\\u003eAlphabet\\u003c/b\\u003e + Inc. (NASDAQ: GOOG ... \\u003cb\\u003erevenue\\u003c/b\\u003e growth of 23% + year over year. ... Quarter Ended March 31, 2021 \\u003cb\\u003e2022 Revenues\\u003c/b\\u003e + $ 55,314 $ 68,011 Change in ...\",\n \"snippetStatus\": + \"SUCCESS\"\n }\n ]\n },\n + \ {\n \"document\": \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/376dab8cf7b3807296f2bf1c6228e078\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q2_alphabet_earnings_release.pdf\",\n + \ \"title\": \"GOOG Exhibit 99.1 Q2 2022\",\n \"snippetInfo\": + [\n {\n \"snippet\": \"\u2013 July + 26, \\u003cb\\u003e2022\\u003c/b\\u003e \u2013 \\u003cb\\u003eAlphabet\\u003c/b\\u003e + Inc. (NASDAQ: GOOG ... Quarter Ended June 30, 2021 \\u003cb\\u003e2022 Revenues\\u003c/b\\u003e + $ 61,880 $ 69,685 Change in \\u003cb\\u003erevenues\\u003c/b\\u003e ... \\u003cb\\u003erevenue\\u003c/b\\u003e + growth rates by 3.7%.\",\n \"snippetStatus\": \"SUCCESS\"\n + \ }\n ]\n },\n {\n + \ \"document\": \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/4a4eabb0cefc954375df0bd9eec85c96\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2023q4-alphabet-earnings-release.pdf\",\n + \ \"title\": \"2023q4-alphabet-earnings-release\",\n \"snippetInfo\": + [\n {\n \"snippet\": \"... \\u003cb\\u003eRevenues\\u003c/b\\u003e + As Reported Less Hedging Effect Less FX Effect Constant Currency \\u003cb\\u003e2022\\u003c/b\\u003e + 2023 \\u003cb\\u003eRevenues\\u003c/b\\u003e United States $ 134,814 $ 146,286 + $ 0 $ 146,286 9 % 0 % 9 ...\",\n \"snippetStatus\": + \"SUCCESS\"\n }\n ]\n },\n + \ {\n \"document\": \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/2eab9446e934f432e3df7808af857300\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q3_alphabet_earnings_release.pdf\",\n + \ \"title\": \"GOOG Exhibit 99.1 Q3 2022\",\n \"snippetInfo\": + [\n {\n \"snippet\": \"\\u003cb\\u003eAlphabet\\u003c/b\\u003e + Announces Third Quarter \\u003cb\\u003e2022\\u003c/b\\u003e Results MOUNTAIN + VIEW, Calif. \u2013 October 25, \\u003cb\\u003e2022\\u003c/b\\u003e \u2013 + \\u003cb\\u003eAlphabet\\u003c/b\\u003e Inc. ... Quarter Ended September 30, + 2021 \\u003cb\\u003e2022 Revenues\\u003c/b\\u003e $ 65,118 ...\",\n \"snippetStatus\": + \"SUCCESS\"\n }\n ]\n },\n + \ {\n \"document\": \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/276cee4c4086600303bc561483691f9a\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022_alphabet_annual_report.pdf\",\n + \ \"title\": \"2022_alphabet_annual_report\",\n \"snippetInfo\": + [\n {\n \"snippet\": \"... \\u003cb\\u003eAlphabet's\\u003c/b\\u003e + AI activities. DeepMind, previously reported within Other Bets, will be reported + as part of \\u003cb\\u003eAlphabet's\\u003c/b\\u003e corporate costs, + reflecting its increasing ...\",\n \"snippetStatus\": + \"SUCCESS\"\n }\n ]\n },\n + \ {\n \"document\": \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/3c617f6f4b023305d82916f0093ca13d\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/goog023-alphabet-2023-annual-report-web-1.pdf\",\n + \ \"title\": \"goog023-alphabet-2023-annual-report-web-1\",\n + \ \"snippetInfo\": [\n {\n \"snippet\": + \"... \\u003cb\\u003e2022\\u003c/b\\u003e 2023 \\u003cb\\u003eRevenues\\u003c/b\\u003e + $ 257,637 $ 282,836 $ 307,394 Costs and expenses: Cost of \\u003cb\\u003erevenues\\u003c/b\\u003e + 110,939 126,203 133,332 Research and development 31,562 39,500 45,427 ...\",\n + \ \"snippetStatus\": \"SUCCESS\"\n }\n + \ ]\n },\n {\n \"document\": + \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/404cb139d857b704a13ee3d8e0c6c5fb\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2021Q4_alphabet_earnings_release.pdf\",\n + \ \"title\": \"GOOG Exhibit 99.1 Q4 2021\",\n \"snippetInfo\": + [\n {\n \"snippet\": \"\\u003cb\\u003eAlphabet\\u003c/b\\u003e + Announces Fourth Quarter and Fiscal Year 2021 Results MOUNTAIN VIEW, Calif. + \u2013 February 1, \\u003cb\\u003e2022\\u003c/b\\u003e \u2013 \\u003cb\\u003eAlphabet\\u003c/b\\u003e + Inc. ... \\u003cb\\u003erevenue\\u003c/b\\u003e growth from Google Cloud.\",\n + \ \"snippetStatus\": \"SUCCESS\"\n }\n + \ ]\n },\n {\n \"document\": + \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/e0ac436b3f82b7df8fc94580f3a84b0a\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2023q1-alphabet-earnings-release.pdf\",\n + \ \"title\": \"GOOG Exhibit 99.1 Q1 2023\",\n \"snippetInfo\": + [\n {\n \"snippet\": \"Ruth Porat, + CFO of \\u003cb\\u003eAlphabet\\u003c/b\\u003e and Google, said: \u201CResilience + in Search and momentum in Cloud resulted in Q1 consolidated \\u003cb\\u003erevenues\\u003c/b\\u003e + of $69.8 billion, up 3% year over ...\",\n \"snippetStatus\": + \"SUCCESS\"\n }\n ]\n },\n + \ {\n \"document\": \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/4c7389ffaf06b73e6e380eac7a39ea17\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2023q3-alphabet-earnings-release.pdf\",\n + \ \"title\": \"GOOG Exhibit 99.1 Q3 2023\",\n \"snippetInfo\": + [\n {\n \"snippet\": \"\\u003cb\\u003eAlphabet\\u003c/b\\u003e + Announces Third Quarter 2023 Results MOUNTAIN VIEW, Calif. ... \\u003cb\\u003erevenue\\u003c/b\\u003e, + up 11% year over year, driven by ... \\u003cb\\u003e2022\\u003c/b\\u003e and + 2023 (in millions, except for per ...\",\n \"snippetStatus\": + \"SUCCESS\"\n }\n ]\n }\n + \ ]\n }\n }\n ]\n }\n ]\n },\n + \ \"answerQueryToken\": \"NMwKDAjEravVBhCr-_ORAxIkNmFjNGMzMTUtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi\"\n}\n]" + headers: + Alt-Svc: + - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 + Content-Type: + - application/json; charset=UTF-8 + Date: + - Wed, 16 Sep 2026 17:49:56 GMT + Server: + - ESF + Server-Timing: + - gfet4t7; dur=1375 + Transfer-Encoding: + - chunked + Vary: + - Origin + - X-Origin + - Referer + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-XSS-Protection: + - '0' + content-length: + - '71033' + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_answer_requested_model[False].yaml b/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_answer_requested_model[False].yaml new file mode 100644 index 000000000..53b30d1ef --- /dev/null +++ b/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_answer_requested_model[False].yaml @@ -0,0 +1,165 @@ +interactions: +- request: + body: "{\n \"query\": {\n \"text\": \"What was Alphabet's revenue in 2022?\"\n + \ },\n \"answerGenerationSpec\": {\n \"modelSpec\": {\n \"modelVersion\": + \"stable\"\n }\n }\n}" + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '160' + Content-Type: + - application/json + User-Agent: + - python-requests/2.34.2 + x-goog-api-client: + - gl-python/3.14.7 grpc/1.84.0 gax/2.37.0 gapic/0.20.3 pb/7.36.1 cred-type/u + x-goog-request-params: + - serving_config=projects/sdk-dev-508013/locations/global/collections/default_collection/engines/braintrust-sdk-test_1789580029521/servingConfigs/default_search + method: POST + uri: https://discoveryengine.googleapis.com/v1/projects/sdk-dev-508013/locations/global/collections/default_collection/engines/braintrust-sdk-test_1789580029521/servingConfigs/default_search:answer?%24alt=json%3Benum-encoding%3Dint + response: + body: + string: "{\n \"answer\": {\n \"state\": 3,\n \"answerText\": \"Alphabet's + total revenue in 2022 was $282,836 million. This represents a 10% increase + year-over-year, or 14% in constant currency.\\n\\nHere's a breakdown of Alphabet's + 2022 revenues by segment and type:\\n\\n**By Segment:**\\n* Google Services: + $253,528 million\\n* Google Cloud: $26,280 million\\n* Other Bets: $1,068 + million\\n* Hedging gains (losses): $1,960 million\\n\\n**By Type (within + Google Services):**\\n* Google Search & other: $162,450 million\\n* YouTube + ads: $29,243 million\\n* Google Network: $32,780 million\\n* Google advertising + (total): $224,473 million\\n* Google other: $29,055 million\\n\\n**By Geography:**\\n* + \ United States: $134,814 million (48%)\\n* EMEA (Europe, Middle East, + and Africa): $82,062 million (29%)\\n* APAC (Asia-Pacific): $47,024 million + (16%)\\n* Other Americas: $16,976 million (6%)\\n* Hedging gains (losses): + $1,960 million (1%)\\n\\nAlphabet's CFO, Ruth Porat, stated that Q4 2022 consolidated + revenues were $76 billion, up 1% year over year, or up 7% in constant currency, + and the full year 2022 revenue was $283 billion, up 10%, or up 14% in constant + currency.\\n\\nFor the first quarter of 2022, Alphabet reported revenues of + $68,011 million, a 23% year-over-year growth. In the third quarter of 2022, + revenues were $69.1 billion, up 6% versus the prior year or up 11% on a constant + currency basis.\",\n \"steps\": [\n {\n \"state\": 3,\n \"description\": + \"Rephrase the query and search.\",\n \"actions\": [\n {\n + \ \"searchAction\": {\n \"query\": \"What was Alphabet's + revenue in 2022?\"\n },\n \"observation\": {\n \"searchResults\": + [\n {\n \"document\": \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/513da7d1b9f7739087606aadc626f2dc\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q4_alphabet_earnings_release.pdf\",\n + \ \"title\": \"GOOG Exhibit 99.1 Q4 2022\",\n \"snippetInfo\": + [\n {\n \"snippet\": \"... \\u003cb\\u003eAlphabet\\u003c/b\\u003e.\u201D + Ruth Porat, CFO of \\u003cb\\u003eAlphabet\\u003c/b\\u003e and Google, said: + \u201COur Q4 consolidated \\u003cb\\u003erevenues\\u003c/b\\u003e were $76 + billion, up 1% year over year, or up 7% in constant currency ...\",\n + \ \"snippetStatus\": \"SUCCESS\"\n }\n + \ ]\n },\n {\n \"document\": + \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/c201efed2be20c6a4116611ea16be9cd\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q1_alphabet_earnings_release.pdf\",\n + \ \"title\": \"GOOG Exhibit 99.1 Q1 2022\",\n \"snippetInfo\": + [\n {\n \"snippet\": \"\u2013 April + 26, \\u003cb\\u003e2022\\u003c/b\\u003e \u2013 \\u003cb\\u003eAlphabet\\u003c/b\\u003e + Inc. (NASDAQ: GOOG ... \\u003cb\\u003erevenue\\u003c/b\\u003e growth of 23% + year over year. ... Quarter Ended March 31, 2021 \\u003cb\\u003e2022 Revenues\\u003c/b\\u003e + $ 55,314 $ 68,011 Change in ...\",\n \"snippetStatus\": + \"SUCCESS\"\n }\n ]\n },\n + \ {\n \"document\": \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/376dab8cf7b3807296f2bf1c6228e078\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q2_alphabet_earnings_release.pdf\",\n + \ \"title\": \"GOOG Exhibit 99.1 Q2 2022\",\n \"snippetInfo\": + [\n {\n \"snippet\": \"\u2013 July + 26, \\u003cb\\u003e2022\\u003c/b\\u003e \u2013 \\u003cb\\u003eAlphabet\\u003c/b\\u003e + Inc. (NASDAQ: GOOG ... Quarter Ended June 30, 2021 \\u003cb\\u003e2022 Revenues\\u003c/b\\u003e + $ 61,880 $ 69,685 Change in \\u003cb\\u003erevenues\\u003c/b\\u003e ... \\u003cb\\u003erevenue\\u003c/b\\u003e + growth rates by 3.7%.\",\n \"snippetStatus\": \"SUCCESS\"\n + \ }\n ]\n },\n {\n + \ \"document\": \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/4a4eabb0cefc954375df0bd9eec85c96\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2023q4-alphabet-earnings-release.pdf\",\n + \ \"title\": \"2023q4-alphabet-earnings-release\",\n \"snippetInfo\": + [\n {\n \"snippet\": \"... \\u003cb\\u003eRevenues\\u003c/b\\u003e + As Reported Less Hedging Effect Less FX Effect Constant Currency \\u003cb\\u003e2022\\u003c/b\\u003e + 2023 \\u003cb\\u003eRevenues\\u003c/b\\u003e United States $ 134,814 $ 146,286 + $ 0 $ 146,286 9 % 0 % 9 ...\",\n \"snippetStatus\": + \"SUCCESS\"\n }\n ]\n },\n + \ {\n \"document\": \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/2eab9446e934f432e3df7808af857300\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q3_alphabet_earnings_release.pdf\",\n + \ \"title\": \"GOOG Exhibit 99.1 Q3 2022\",\n \"snippetInfo\": + [\n {\n \"snippet\": \"\\u003cb\\u003eAlphabet\\u003c/b\\u003e + Announces Third Quarter \\u003cb\\u003e2022\\u003c/b\\u003e Results MOUNTAIN + VIEW, Calif. \u2013 October 25, \\u003cb\\u003e2022\\u003c/b\\u003e \u2013 + \\u003cb\\u003eAlphabet\\u003c/b\\u003e Inc. ... Quarter Ended September 30, + 2021 \\u003cb\\u003e2022 Revenues\\u003c/b\\u003e $ 65,118 ...\",\n \"snippetStatus\": + \"SUCCESS\"\n }\n ]\n },\n + \ {\n \"document\": \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/276cee4c4086600303bc561483691f9a\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022_alphabet_annual_report.pdf\",\n + \ \"title\": \"2022_alphabet_annual_report\",\n \"snippetInfo\": + [\n {\n \"snippet\": \"... \\u003cb\\u003eAlphabet's\\u003c/b\\u003e + AI activities. DeepMind, previously reported within Other Bets, will be reported + as part of \\u003cb\\u003eAlphabet's\\u003c/b\\u003e corporate costs, + reflecting its increasing ...\",\n \"snippetStatus\": + \"SUCCESS\"\n }\n ]\n },\n + \ {\n \"document\": \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/3c617f6f4b023305d82916f0093ca13d\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/goog023-alphabet-2023-annual-report-web-1.pdf\",\n + \ \"title\": \"goog023-alphabet-2023-annual-report-web-1\",\n + \ \"snippetInfo\": [\n {\n \"snippet\": + \"... \\u003cb\\u003e2022\\u003c/b\\u003e 2023 \\u003cb\\u003eRevenues\\u003c/b\\u003e + $ 257,637 $ 282,836 $ 307,394 Costs and expenses: Cost of \\u003cb\\u003erevenues\\u003c/b\\u003e + 110,939 126,203 133,332 Research and development 31,562 39,500 45,427 ...\",\n + \ \"snippetStatus\": \"SUCCESS\"\n }\n + \ ]\n },\n {\n \"document\": + \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/404cb139d857b704a13ee3d8e0c6c5fb\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2021Q4_alphabet_earnings_release.pdf\",\n + \ \"title\": \"GOOG Exhibit 99.1 Q4 2021\",\n \"snippetInfo\": + [\n {\n \"snippet\": \"\\u003cb\\u003eAlphabet\\u003c/b\\u003e + Announces Fourth Quarter and Fiscal Year 2021 Results MOUNTAIN VIEW, Calif. + \u2013 February 1, \\u003cb\\u003e2022\\u003c/b\\u003e \u2013 \\u003cb\\u003eAlphabet\\u003c/b\\u003e + Inc. ... \\u003cb\\u003erevenue\\u003c/b\\u003e growth from Google Cloud.\",\n + \ \"snippetStatus\": \"SUCCESS\"\n }\n + \ ]\n },\n {\n \"document\": + \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/e0ac436b3f82b7df8fc94580f3a84b0a\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2023q1-alphabet-earnings-release.pdf\",\n + \ \"title\": \"GOOG Exhibit 99.1 Q1 2023\",\n \"snippetInfo\": + [\n {\n \"snippet\": \"Ruth Porat, + CFO of \\u003cb\\u003eAlphabet\\u003c/b\\u003e and Google, said: \u201CResilience + in Search and momentum in Cloud resulted in Q1 consolidated \\u003cb\\u003erevenues\\u003c/b\\u003e + of $69.8 billion, up 3% year over ...\",\n \"snippetStatus\": + \"SUCCESS\"\n }\n ]\n },\n + \ {\n \"document\": \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/4c7389ffaf06b73e6e380eac7a39ea17\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2023q3-alphabet-earnings-release.pdf\",\n + \ \"title\": \"GOOG Exhibit 99.1 Q3 2023\",\n \"snippetInfo\": + [\n {\n \"snippet\": \"\\u003cb\\u003eAlphabet\\u003c/b\\u003e + Announces Third Quarter 2023 Results MOUNTAIN VIEW, Calif. ... \\u003cb\\u003erevenue\\u003c/b\\u003e, + up 11% year over year, driven by ... \\u003cb\\u003e2022\\u003c/b\\u003e and + 2023 (in millions, except for per ...\",\n \"snippetStatus\": + \"SUCCESS\"\n }\n ]\n }\n + \ ]\n }\n }\n ]\n }\n ]\n },\n + \ \"answerQueryToken\": \"NMwKDAiarKvVBhDA2pCkAxIkNmFjNGJmYWMtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi\"\n}\n" + headers: + Alt-Svc: + - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 + Content-Type: + - application/json; charset=UTF-8 + Date: + - Wed, 16 Sep 2026 17:47:06 GMT + Server: + - ESF + Server-Timing: + - gfet4t7; dur=13418 + Transfer-Encoding: + - chunked + Vary: + - Origin + - X-Origin + - Referer + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-XSS-Protection: + - '0' + content-length: + - '10723' + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_answer_requested_model[True].yaml b/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_answer_requested_model[True].yaml new file mode 100644 index 000000000..4ab82421b --- /dev/null +++ b/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_answer_requested_model[True].yaml @@ -0,0 +1,58 @@ +interactions: +- request: + body: "{\n \"query\": {\n \"text\": \"What was Alphabet's revenue in 2022?\"\n + \ },\n \"answerGenerationSpec\": {\n \"modelSpec\": {\n \"modelVersion\": + \"stable\"\n }\n },\n \"asynchronousMode\": true\n}" + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '188' + Content-Type: + - application/json + User-Agent: + - python-requests/2.34.2 + x-goog-api-client: + - gl-python/3.14.7 grpc/1.84.0 gax/2.37.0 gapic/0.20.3 pb/7.36.1 cred-type/u + x-goog-request-params: + - serving_config=projects/sdk-dev-508013/locations/global/collections/default_collection/engines/braintrust-sdk-test_1789580029521/servingConfigs/default_search + method: POST + uri: https://discoveryengine.googleapis.com/v1/projects/sdk-dev-508013/locations/global/collections/default_collection/engines/braintrust-sdk-test_1789580029521/servingConfigs/default_search:answer?%24alt=json%3Benum-encoding%3Dint + response: + body: + string: "{\n \"error\": {\n \"code\": 400,\n \"message\": \"Request contains + an invalid argument: asynchronous mode is deprecated.\",\n \"status\": + \"INVALID_ARGUMENT\"\n }\n}\n" + headers: + Alt-Svc: + - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 + Content-Type: + - application/json; charset=UTF-8 + Date: + - Wed, 16 Sep 2026 17:46:53 GMT + Server: + - ESF + Server-Timing: + - gfet4t7; dur=104 + Transfer-Encoding: + - chunked + Vary: + - Origin + - X-Origin + - Referer + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-XSS-Protection: + - '0' + content-length: + - '160' + status: + code: 400 + message: Bad Request +version: 1 diff --git a/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_async_grpc[answer_query].json b/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_async_grpc[answer_query].json new file mode 100644 index 000000000..ff6a91f0d --- /dev/null +++ b/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_async_grpc[answer_query].json @@ -0,0 +1,501 @@ +{ + "method": "answer_query", + "requests": [ + { + "serving_config": "projects/sdk-dev-508013/locations/global/collections/default_collection/engines/braintrust-sdk-test_1789580029521/servingConfigs/default_search", + "query": { + "text": "What was Alphabet's revenue in 2022?", + "query_id": "" + }, + "answer_generation_spec": { + "include_citations": true, + "answer_language_code": "", + "ignore_adversarial_query": false, + "ignore_non_answer_seeking_query": false, + "ignore_jail_breaking_query": false + }, + "session": "", + "asynchronous_mode": false, + "user_pseudo_id": "", + "user_labels": {} + } + ], + "responses": [ + { + "answer": { + "state": 3, + "answer_text": "Alphabet's total revenue in 2022 was $282,836 million. This represents a 10% increase year-over-year, or 14% in constant currency.\n\nHere's a breakdown of Alphabet's 2022 revenues by segment and type:\n\n**By Segment:**\n* Google Services: $253,528 million\n* Google Cloud: $26,280 million\n* Other Bets: $1,068 million\n* Hedging gains (losses): $1,960 million\n\n**By Type (within Google Services):**\n* Google Search & other: $162,450 million\n* YouTube ads: $29,243 million\n* Google Network: $32,780 million\n* Google advertising (total): $224,473 million\n* Google other: $29,055 million\n\n**By Geography:**\n* United States: $134,814 million (48%)\n* EMEA (Europe, Middle East, and Africa): $82,062 million (29%)\n* APAC (Asia-Pacific): $47,024 million (16%)\n* Other Americas: $16,976 million (6%)\n* Hedging gains (losses): $1,960 million (1%)\n\nAlphabet's CFO, Ruth Porat, stated that Q4 2022 consolidated revenues were $76 billion, up 1% year over year, or up 7% in constant currency, and the full year 2022 revenue was $283 billion, up 10%, or up 14% in constant currency.\n\nFor the first quarter of 2022, Alphabet reported revenues of $68,011 million, a 23% year-over-year growth. In the third quarter of 2022, revenues were $69.1 billion, up 6% versus the prior year or up 11% on a constant currency basis.", + "citations": [ + { + "end_index": "54", + "sources": [ + { + "reference_id": "0" + }, + { + "reference_id": "4" + }, + { + "reference_id": "8" + }, + { + "reference_id": "9" + } + ], + "start_index": "0" + }, + { + "start_index": "55", + "end_index": "130", + "sources": [ + { + "reference_id": "2" + } + ] + }, + { + "start_index": "217", + "end_index": "254", + "sources": [ + { + "reference_id": "8" + }, + { + "reference_id": "9" + } + ] + }, + { + "start_index": "255", + "end_index": "288", + "sources": [ + { + "reference_id": "8" + }, + { + "reference_id": "9" + } + ] + }, + { + "start_index": "289", + "end_index": "319", + "sources": [ + { + "reference_id": "8" + }, + { + "reference_id": "9" + } + ] + }, + { + "start_index": "320", + "end_index": "362", + "sources": [ + { + "reference_id": "8" + }, + { + "reference_id": "9" + } + ] + }, + { + "start_index": "402", + "end_index": "445", + "sources": [ + { + "reference_id": "9" + } + ] + }, + { + "start_index": "446", + "end_index": "478", + "sources": [ + { + "reference_id": "9" + } + ] + }, + { + "start_index": "479", + "end_index": "514", + "sources": [ + { + "reference_id": "9" + } + ] + }, + { + "start_index": "515", + "end_index": "563", + "sources": [ + { + "reference_id": "9" + } + ] + }, + { + "start_index": "564", + "end_index": "597", + "sources": [ + { + "reference_id": "9" + } + ] + }, + { + "start_index": "617", + "end_index": "658", + "sources": [ + { + "reference_id": "9" + } + ] + }, + { + "start_index": "659", + "end_index": "724", + "sources": [ + { + "reference_id": "9" + } + ] + }, + { + "start_index": "725", + "end_index": "771", + "sources": [ + { + "reference_id": "9" + } + ] + }, + { + "start_index": "772", + "end_index": "812", + "sources": [ + { + "reference_id": "9" + } + ] + }, + { + "start_index": "813", + "end_index": "860", + "sources": [ + { + "reference_id": "9" + } + ] + }, + { + "start_index": "862", + "end_index": "1091", + "sources": [ + { + "reference_id": "2" + } + ] + }, + { + "start_index": "1093", + "end_index": "1199", + "sources": [ + { + "reference_id": "1" + } + ] + }, + { + "start_index": "1200", + "end_index": "1326", + "sources": [ + { + "reference_id": "5" + } + ] + } + ], + "references": [ + { + "chunk_info": { + "content": "ALPHABET INC. ANNUAL REPORT 45 PART II\nITEM\u00a08\u00a0\u00a0FINANCIAL STATEMENTS AND SUPPLEMENTARY DATA Alphabet Inc. CONSOLIDATED STATEMENTS OF INCOME (In millions, except per share amounts) Year Ended December 31,\n2020 2021 2022 Revenues $ 182,527 $ 257,637 $ 282,836 Costs and expenses:\nCost of revenues 84,732 110,939 126,203 Research and development 27,573 31,562 39,500 Sales and marketing 17,946 22,912 26,567 General and administrative 11,052 13,510 15,724 Total costs and expenses 141,303 178,923 207,994 Income from operations 41,224 78,714 74,842 Other income (expense), net 6,858 12,020 (3,514) Income before income taxes 48,082 90,734 71,328 Provision for income taxes 7,813 14,701 11,356 Net income $ 40,269 $ 76,033 $ 59,972 Basic net income per share of Class A, Class B, and Class C stock $ 2.96 $ 5.69 $ 4.59 Diluted net income per share of Class A, Class B, and Class C stock $ 2.93 $ 5.61 $ 4.56 See accompanying notes. ", + "relevance_score": 0.9, + "document_metadata": { + "document": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/276cee4c4086600303bc561483691f9a", + "uri": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022_alphabet_annual_report.pdf", + "title": "2022_alphabet_annual_report", + "page_identifier": "83" + }, + "chunk": "" + } + }, + { + "chunk_info": { + "content": "Alphabet Announces First Quarter 2022 Results\nMOUNTAIN VIEW, Calif. \u2013 April 26, 2022 \u2013 Alphabet Inc. (NASDAQ: GOOG, GOOGL) today announced financial\nresults for the quarter ended March 31, 2022. Sundar Pichai, CEO of Alphabet and Google, said: \u201cQ1 saw strong growth in Search and Cloud, in particular, which\nare both helping people and businesses as the digital transformation continues. We\u2019ll keep investing in great\nproducts and services, and creating opportunities for partners and local communities around the world.\u201d Ruth Porat, CFO of Alphabet and Google, said: \u201cWe are pleased with Q1 revenue growth of 23% year over year. We\ncontinue to make considered investments in Capex, R&D and talent to support long-term value creation for all\nstakeholders.\u201d Q1 2022 financial highlights\nThe following table summarizes our consolidated financial results for the quarters ended March 31, 2021 and 2022\n(in millions, except for per share information and percentages; unaudited). Quarter Ended March 31,\n2021 2022 Revenues $ 55,314 $ 68,011 Change in revenues year over year 34 % 23 % Change in constant currency revenues year over year(1) 32 % 26 % Operating income $ 16,437 $ 20,094 Operating margin 30 % 30 % Other income (expense), net $ 4,846 $ (1,160) Net income $ 17,930 $ 16,436 ", + "relevance_score": 0.9, + "document_metadata": { + "document": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/c201efed2be20c6a4116611ea16be9cd", + "uri": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q1_alphabet_earnings_release.pdf", + "title": "GOOG Exhibit 99.1 Q1 2022", + "page_identifier": "1" + }, + "chunk": "" + } + }, + { + "chunk_info": { + "content": "Alphabet Announces Fourth Quarter and Fiscal Year 2022 Results\nMOUNTAIN VIEW, Calif. \u2013 February 2, 2023 \u2013 Alphabet Inc. (NASDAQ: GOOG, GOOGL) today announced financial\nresults for the quarter and fiscal year ended December 31, 2022. Sundar Pichai, CEO of Alphabet and Google, said: \u201cOur long-term investments in deep computer science make us\nextremely well-positioned as AI reaches an inflection point, and I\u2019m excited by the AI-driven leaps we\u2019re about to unveil\nin Search and beyond. There\u2019s also great momentum in Cloud, YouTube subscriptions, and our Pixel devices. We\u2019re\non an important journey to re-engineer our cost structure in a durable way and to build financially sustainable, vibrant,\ngrowing businesses across Alphabet.\u201d Ruth Porat, CFO of Alphabet and Google, said: \u201cOur Q4 consolidated revenues were $76 billion, up 1% year over year,\nor up 7% in constant currency, and $283 billion for the full year 2022, up 10%, or up 14% in constant currency. We\nhave significant work underway to improve all aspects of our cost structure, in support of our investments in our\nhighest growth priorities to deliver long-term, profitable growth.\u201d ", + "relevance_score": 0.8, + "document_metadata": { + "document": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/513da7d1b9f7739087606aadc626f2dc", + "uri": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q4_alphabet_earnings_release.pdf", + "title": "GOOG Exhibit 99.1 Q4 2022", + "page_identifier": "1" + }, + "chunk": "" + } + }, + { + "chunk_info": { + "content": "Alphabet Announces Third Quarter 2023 Results\nMOUNTAIN VIEW, Calif. \u2013 October 24, 2023 \u2013 Alphabet Inc. (NASDAQ: GOOG, GOOGL) today announced\nfinancial results for the quarter ended September 30, 2023. Sundar Pichai, CEO, said: \u201cI\u2019m pleased with our financial results and our product momentum this quarter, with AI\ndriven innovations across Search, YouTube, Cloud, our Pixel devices and more. We\u2019re continuing to focus on\nmaking AI more helpful for everyone; there\u2019s exciting progress and lots more to come.\u201d Ruth Porat, President and Chief Investment Officer; CFO said: \u201cThe fundamental strength of our business was\napparent again in Q3, with $77 billion in revenue, up 11% year over year, driven by meaningful growth in Search\nand YouTube, and momentum in Cloud. We continue to focus on judicious capital allocation to deliver sustainable\nfinancial value.\u201d Q3 2023 Financial Highlights (unaudited)\nThe following table summarizes our consolidated financial results for the quarters ended September 30, 2022 and\n2023 (in millions, except for per share information and percentages). Quarter Ended September 30,\n2022 2023 Revenues $ 69,092 $ 76,693 Change in revenues year over year 6 % 11 % Change in constant currency revenues year over year(1) 11 % 11 % Operating income $ 17,135 $ 21,343 Operating margin 25 % ", + "relevance_score": 0.8, + "document_metadata": { + "document": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/4c7389ffaf06b73e6e380eac7a39ea17", + "uri": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2023q3-alphabet-earnings-release.pdf", + "title": "GOOG Exhibit 99.1 Q3 2023", + "page_identifier": "1" + }, + "chunk": "" + } + }, + { + "chunk_info": { + "content": "Alphabet Inc. Consolidated Statements of Income Year Ended December 31, (in millions, except per share amounts) 2021 2022 2023 Revenues $ 257,637 $ 282,836 $ 307,394 Costs and expenses:\nCost of revenues 110,939 126,203 133,332 Research and development 31,562 39,500 45,427 Sales and marketing 22,912 26,567 27,917 General and administrative 13,510 15,724 16,425 Total costs and expenses 178,923 207,994 223,101 Income from operations 78,714 74,842 84,293 Other income (expense), net 12,020 (3,514) 1,424 Income before income taxes 90,734 71,328 85,717 Provision for income taxes 14,701 11,356 11,922 Net income $ 76,033 $ 59,972 $ 73,795 Basic net income per share of Class A, Class B, and Class C stock $ 5.69 $ 4.59 $ 5.84 Diluted net income per share of Class A, Class B, and Class C stock $ 5.61 $ 4.56 $ 5.80 See accompanying notes. 50 Alphabet 2023 Annual Report Part I Part II Part III Part IV ", + "relevance_score": 0.8, + "document_metadata": { + "document": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/3c617f6f4b023305d82916f0093ca13d", + "uri": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/goog023-alphabet-2023-annual-report-web-1.pdf", + "title": "goog023-alphabet-2023-annual-report-web-1", + "page_identifier": "60" + }, + "chunk": "" + } + }, + { + "chunk_info": { + "content": "Alphabet Announces Third Quarter 2022 Results\nMOUNTAIN VIEW, Calif. \u2013 October 25, 2022 \u2013 Alphabet Inc. (NASDAQ: GOOG, GOOGL) today announced\nfinancial results for the quarter ended September 30, 2022. Sundar Pichai, CEO of Alphabet and Google, said: \u201cWe\u2019re sharpening our focus on a clear set of product and\nbusiness priorities. Product announcements we\u2019ve made in just the past month alone have shown that very clearly,\nincluding significant improvements to both Search and Cloud, powered by AI, and new ways to monetize YouTube\nShorts. We are focused on both investing responsibly for the long term and being responsive to the economic\nenvironment.\u201d Ruth Porat, CFO of Alphabet and Google, said: \u201cOur third quarter revenues were $69.1 billion, up 6% versus last\nyear or up 11% on a constant currency basis. Financial results for the third quarter reflect healthy fundamental\ngrowth in Search and momentum in Cloud, while affected by foreign exchange. We\u2019re working to realign resources\nto fuel our highest growth priorities.\u201d Q3 2022 financial highlights\nThe following table summarizes our consolidated financial results for the quarters ended September 30, 2021 and\n2022 (in millions, except for per share information and percentages; unaudited). Quarter Ended September 30,\n2021 2022 Revenues $ 65,118 $ 69,092 Change in revenues year over year 41 % ", + "relevance_score": 0.8, + "document_metadata": { + "document": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/2eab9446e934f432e3df7808af857300", + "uri": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q3_alphabet_earnings_release.pdf", + "title": "GOOG Exhibit 99.1 Q3 2022", + "page_identifier": "1" + }, + "chunk": "" + } + }, + { + "chunk_info": { + "content": "Alphabet Inc. CONSOLIDATED STATEMENTS OF CASH FLOWS\n(In millions, unaudited)\nQuarter Ended September 30, Year to Date September 30, 2021 2022 2021 2022 Operating activities\nNet income $ 18,936 $ 13,910 $ 55,391 $ 46,348 Adjustments:\nDepreciation and impairment of property and\nequipment 3,085 3,933 8,340 11,222 Amortization and impairment of intangible assets 219 113 662 505 Stock-based compensation expense 3,874 4,976 11,422 14,262 Deferred income taxes (1,287) (1,920) 192 (6,157) (Gain) loss on debt and equity securities, net (2,158) 1,378 (9,792) 3,856 Other 64 167 (199) 369 Changes in assets and liabilities, net of effects of\nacquisitions:\nAccounts receivable (2,409) (97) (3,276) 2,298 Income taxes, net 3,041 (609) 2,744 (862) Other assets (1,255) (2,647) (1,447) (4,268) Accounts payable 238 1,907 (874) 735 Accrued expenses and other liabilities 2,562 2,210 2,763 491 Accrued revenue share 357 (80) 386 (1,022) Deferred revenue 272 112 406 104 Net cash provided by operating activities 25,539 23,353 66,718 67,881 Investing activities\nPurchases of property and equipment (6,819) (7,276) (18,257) (23,890) Purchases of marketable securities (34,497) (17,054) (95,106) (67,253) Maturities and sales of marketable securities 31,459 28,713 92,126 84,087 Purchases of non-marketable securities ", + "relevance_score": 0.8, + "document_metadata": { + "document": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/2eab9446e934f432e3df7808af857300", + "uri": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q3_alphabet_earnings_release.pdf", + "title": "GOOG Exhibit 99.1 Q3 2022", + "page_identifier": "6" + }, + "chunk": "" + } + }, + { + "chunk_info": { + "content": "Alphabet Inc. CONSOLIDATED STATEMENTS OF CASH FLOWS\n(In millions, unaudited)\nQuarter Ended June 30, Year To Date June 30,\n2021 2022 2021 2022 Operating activities\nNet income $ 18,525 $ 16,002 $ 36,455 $ 32,438 Adjustments:\nDepreciation and impairment of property and equipment 2,730 3,698 5,255 7,289 Amortization and impairment of intangible assets 215 201 443 392 Stock-based compensation expense 3,803 4,782 7,548 9,286 Deferred income taxes 379 (2,147) 1,479 (4,237) (Gain) loss on debt and equity securities, net (2,883) 1,041 (7,634) 2,478 Other (8) 62 (263) 202 Changes in assets and liabilities, net of effects of acquisitions:\nAccounts receivable (3,661) (1,969) (867) 2,395 Income taxes, net (1,082) (4,073) (297) (253) Other assets (199) (845) (192) (1,621) Accounts payable (130) 1,201 (1,112) (1,172) Accrued expenses and other liabilities 3,731 1,497 201 (1,719) Accrued revenue share 473 (114) 29 (942) Deferred revenue (3) 86 134 (8) Net cash provided by operating activities 21,890 19,422 41,179 44,528 Investing activities\nPurchases of property and equipment (5,496) (6,828) (11,438) (16,614) Purchases of marketable securities (24,183) (21,737) (60,609) (50,199) ", + "relevance_score": 0.8, + "document_metadata": { + "document": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/376dab8cf7b3807296f2bf1c6228e078", + "uri": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q2_alphabet_earnings_release.pdf", + "title": "GOOG Exhibit 99.1 Q2 2022", + "page_identifier": "6" + }, + "chunk": "" + } + }, + { + "chunk_info": { + "content": "The associated costs,\nincluding depreciation and impairment, are allocated to operating segments as a service cost generally based on usage or headcount. Unallocated corporate costs primarily include corporate initiatives, corporate shared costs, such as finance and legal, including certain\nfines and settlements, as well as costs associated with certain shared R&D activities. Additionally, hedging gains (losses) related to\nrevenue are included in corporate costs. As AI is critical to delivering our mission of bringing our breakthrough innovations into the real world, beginning in January 2023, we\nwill update our segment reporting relating to certain of Alphabet\u2019s AI activities. DeepMind, previously reported within Other Bets, will\nbe reported as part of Alphabet\u2019s corporate costs, reflecting its increasing collaboration with Google Services, Google Cloud, and\nOther Bets. Prior periods will be recast to conform to the revised presentation. Our operating segments are not evaluated using asset information. The following table presents information about our segments (in millions): Year Ended December 31,\n2020 2021 2022 Revenues:\nGoogle Services $ 168,635 $ 237,529 $ 253,528 Google Cloud 13,059 19,206 26,280 Other Bets 657 753 1,068 Hedging gains (losses) 176 149 1,960 Total revenues $ 182,527 $ 257,637 $ 282,836 Operating income (loss):\nGoogle Services $ 54,606 $ 91,855 $ 86,572 Google Cloud (5,607) (3,099) (2,968) Other Bets ", + "relevance_score": 0.8, + "document_metadata": { + "document": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/276cee4c4086600303bc561483691f9a", + "uri": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022_alphabet_annual_report.pdf", + "title": "2022_alphabet_annual_report", + "page_identifier": "119" + }, + "chunk": "" + } + }, + { + "chunk_info": { + "content": "56 ALPHABET INC. ANNUAL REPORT PART II\nITEM\u00a08\u00a0\u00a0FINANCIAL STATEMENTS AND SUPPLEMENTARY DATA Note 2. Revenues\nDisaggregated Revenues\nThe following table presents revenues disaggregated by type (in millions): Year Ended December 31,\n2020 2021 2022 Google Search & other $ 104,062 $ 148,951 $ 162,450 YouTube ads 19,772 28,845 29,243 Google Network 23,090 31,701 32,780 Google advertising 146,924 209,497 224,473 Google other 21,711 28,032 29,055 Google Services total 168,635 237,529 253,528 Google Cloud 13,059 19,206 26,280 Other Bets 657 753 1,068 Hedging gains (losses) 176 149 1,960 Total revenues $ 182,527 $ 257,637 $ 282,836\nNo individual customer or groups of affiliated customers represented more than 10% of our revenues in 2020, 2021, or 2022. The following table presents revenues disaggregated by geography, based on the addresses of our customers (in millions):\nYear Ended December 31, 2020 2021 2022 United States $ 85,014 47 % $ 117,854 46 % $ 134,814 48% EMEA(1) 55,370 30 79,107 31 82,062 29 APAC(1) 32,550 18 46,123 18 47,024 16 Other Americas(1) 9,417 5 14,404 5 16,976 6 Hedging gains (losses) 176 0 149 0 1,960 1 Total revenues $ 182,527 100% $ 257,637 100 % $ 282,836 100% ", + "relevance_score": 0.8, + "document_metadata": { + "document": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/276cee4c4086600303bc561483691f9a", + "uri": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022_alphabet_annual_report.pdf", + "title": "2022_alphabet_annual_report", + "page_identifier": "94" + }, + "chunk": "" + } + } + ], + "steps": [ + { + "state": 3, + "description": "Rephrase the query and search.", + "actions": [ + { + "search_action": { + "query": "What was Alphabet's revenue in 2022?" + }, + "observation": { + "search_results": [ + { + "document": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/513da7d1b9f7739087606aadc626f2dc", + "uri": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q4_alphabet_earnings_release.pdf", + "title": "GOOG Exhibit 99.1 Q4 2022", + "snippet_info": [ + { + "snippet": "... Alphabet.\u201d Ruth Porat, CFO of Alphabet and Google, said: \u201cOur Q4 consolidated revenues were $76 billion, up 1% year over year, or up 7% in constant currency ...", + "snippet_status": "SUCCESS" + } + ], + "chunk_info": [] + }, + { + "document": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/c201efed2be20c6a4116611ea16be9cd", + "uri": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q1_alphabet_earnings_release.pdf", + "title": "GOOG Exhibit 99.1 Q1 2022", + "snippet_info": [ + { + "snippet": "\u2013 April 26, 2022 \u2013 Alphabet Inc. (NASDAQ: GOOG ... revenue growth of 23% year over year. ... Quarter Ended March 31, 2021 2022 Revenues $ 55,314 $ 68,011 Change in ...", + "snippet_status": "SUCCESS" + } + ], + "chunk_info": [] + }, + { + "document": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/376dab8cf7b3807296f2bf1c6228e078", + "uri": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q2_alphabet_earnings_release.pdf", + "title": "GOOG Exhibit 99.1 Q2 2022", + "snippet_info": [ + { + "snippet": "\u2013 July 26, 2022 \u2013 Alphabet Inc. (NASDAQ: GOOG ... Quarter Ended June 30, 2021 2022 Revenues $ 61,880 $ 69,685 Change in revenues ... revenue growth rates by 3.7%.", + "snippet_status": "SUCCESS" + } + ], + "chunk_info": [] + }, + { + "document": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/4a4eabb0cefc954375df0bd9eec85c96", + "uri": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2023q4-alphabet-earnings-release.pdf", + "title": "2023q4-alphabet-earnings-release", + "snippet_info": [ + { + "snippet": "... Revenues As Reported Less Hedging Effect Less FX Effect Constant Currency 2022 2023 Revenues United States $ 134,814 $ 146,286 $ 0 $ 146,286 9 % 0 % 9 ...", + "snippet_status": "SUCCESS" + } + ], + "chunk_info": [] + }, + { + "document": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/2eab9446e934f432e3df7808af857300", + "uri": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q3_alphabet_earnings_release.pdf", + "title": "GOOG Exhibit 99.1 Q3 2022", + "snippet_info": [ + { + "snippet": "Alphabet Announces Third Quarter 2022 Results MOUNTAIN VIEW, Calif. \u2013 October 25, 2022 \u2013 Alphabet Inc. ... Quarter Ended September 30, 2021 2022 Revenues $ 65,118 ...", + "snippet_status": "SUCCESS" + } + ], + "chunk_info": [] + }, + { + "document": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/276cee4c4086600303bc561483691f9a", + "uri": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022_alphabet_annual_report.pdf", + "title": "2022_alphabet_annual_report", + "snippet_info": [ + { + "snippet": "... Alphabet's AI activities. DeepMind, previously reported within Other Bets, will be reported as part of Alphabet's corporate costs, reflecting its increasing ...", + "snippet_status": "SUCCESS" + } + ], + "chunk_info": [] + }, + { + "document": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/3c617f6f4b023305d82916f0093ca13d", + "uri": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/goog023-alphabet-2023-annual-report-web-1.pdf", + "title": "goog023-alphabet-2023-annual-report-web-1", + "snippet_info": [ + { + "snippet": "... 2022 2023 Revenues $ 257,637 $ 282,836 $ 307,394 Costs and expenses: Cost of revenues 110,939 126,203 133,332 Research and development 31,562 39,500 45,427 ...", + "snippet_status": "SUCCESS" + } + ], + "chunk_info": [] + }, + { + "document": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/404cb139d857b704a13ee3d8e0c6c5fb", + "uri": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2021Q4_alphabet_earnings_release.pdf", + "title": "GOOG Exhibit 99.1 Q4 2021", + "snippet_info": [ + { + "snippet": "Alphabet Announces Fourth Quarter and Fiscal Year 2021 Results MOUNTAIN VIEW, Calif. \u2013 February 1, 2022 \u2013 Alphabet Inc. ... revenue growth from Google Cloud.", + "snippet_status": "SUCCESS" + } + ], + "chunk_info": [] + }, + { + "document": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/e0ac436b3f82b7df8fc94580f3a84b0a", + "uri": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2023q1-alphabet-earnings-release.pdf", + "title": "GOOG Exhibit 99.1 Q1 2023", + "snippet_info": [ + { + "snippet": "Ruth Porat, CFO of Alphabet and Google, said: \u201cResilience in Search and momentum in Cloud resulted in Q1 consolidated revenues of $69.8 billion, up 3% year over ...", + "snippet_status": "SUCCESS" + } + ], + "chunk_info": [] + }, + { + "document": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/4c7389ffaf06b73e6e380eac7a39ea17", + "uri": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2023q3-alphabet-earnings-release.pdf", + "title": "GOOG Exhibit 99.1 Q3 2023", + "snippet_info": [ + { + "snippet": "Alphabet Announces Third Quarter 2023 Results MOUNTAIN VIEW, Calif. ... revenue, up 11% year over year, driven by ... 2022 and 2023 (in millions, except for per ...", + "snippet_status": "SUCCESS" + } + ], + "chunk_info": [] + } + ] + } + } + ], + "thought": "" + } + ], + "name": "", + "grounding_supports": [], + "related_questions": [], + "answer_skipped_reasons": [], + "safety_ratings": [] + }, + "answer_query_token": "NMwKDAjOravVBhDu95WBAxIkNmFiNDUxZDQtMDAwMC0yOGQ5LWJhYjctMDg5ZTA4MjUyYzgw" + } + ] +} diff --git a/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_async_grpc[check_grounding].json b/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_async_grpc[check_grounding].json new file mode 100644 index 000000000..57eaa6693 --- /dev/null +++ b/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_async_grpc[check_grounding].json @@ -0,0 +1,44 @@ +{ + "method": "check_grounding", + "requests": [ + { + "grounding_config": "projects/sdk-dev-508013/locations/global/groundingConfigs/default_grounding_config", + "answer_candidate": "Braintrust evaluates AI applications.", + "facts": [ + { + "fact_text": "Braintrust is a platform for evaluating AI applications.", + "attributes": {} + } + ], + "user_labels": {} + } + ], + "responses": [ + { + "support_score": 0.99110764, + "cited_chunks": [ + { + "chunk_text": "Braintrust is a platform for evaluating AI applications.", + "source": "0", + "index": 0, + "source_metadata": {}, + "uri": "", + "title": "", + "domain": "" + } + ], + "claims": [ + { + "start_pos": 0, + "end_pos": 37, + "claim_text": "Braintrust evaluates AI applications.", + "citation_indices": [ + 0 + ], + "grounding_check_required": true + } + ], + "cited_facts": [] + } + ] +} diff --git a/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_async_grpc[converse_conversation].json b/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_async_grpc[converse_conversation].json new file mode 100644 index 000000000..8b0784b5f --- /dev/null +++ b/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_async_grpc[converse_conversation].json @@ -0,0 +1,497 @@ +{ + "method": "converse_conversation", + "requests": [ + { + "name": "projects/sdk-dev-508013/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/conversations/-", + "query": { + "input": "What was Alphabet's revenue in 2022?" + }, + "serving_config": "projects/sdk-dev-508013/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/servingConfigs/default_search", + "summary_spec": { + "summary_result_count": 3, + "include_citations": false, + "ignore_adversarial_query": false, + "ignore_non_summary_seeking_query": false, + "ignore_low_relevant_content": false, + "ignore_jail_breaking_query": false, + "language_code": "", + "use_semantic_chunks": false + }, + "safe_search": false, + "user_labels": {}, + "filter": "" + } + ], + "responses": [ + { + "reply": { + "summary": { + "summary_text": "Alphabet's consolidated revenues for the year ended December 31, 2022, were $282,836 million.\n\nLooking at the quarterly revenues for 2022:\n* Q1 2022 revenues were $68,011 million. This was a 23% change in revenues year over year, or 26% in constant currency.\n* Q2 2022 revenues were $69,685 million. This represented a 13% change in revenues year over year, or 16% in constant currency.\n* Q4 2022 consolidated revenues were $76,048 million, which was up 1% year over year, or up 7% in constant currency.\n\nFor the year-to-date period ending June 30, 2022, Alphabet's revenues were $137,696 million.\n\nSpecific revenue breakdowns for Q4 2022 include:\n* Google Search & other: $42,604 million.\n* YouTube ads: $7,963 million.\n* Google Network: $8,475 million.\n* Google advertising: $59,042 million.", + "summary_skipped_reasons": [] + } + }, + "conversation": { + "name": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/conversations/6828753367195968599", + "state": 1, + "user_pseudo_id": "6828753367195968599", + "messages": [ + { + "user_input": { + "input": "What was Alphabet's revenue in 2022?" + } + }, + { + "reply": { + "summary": { + "summary_text": "Alphabet's consolidated revenues for the year ended December 31, 2022, were $282,836 million.\n\nLooking at the quarterly revenues for 2022:\n* Q1 2022 revenues were $68,011 million. This was a 23% change in revenues year over year, or 26% in constant currency.\n* Q2 2022 revenues were $69,685 million. This represented a 13% change in revenues year over year, or 16% in constant currency.\n* Q4 2022 consolidated revenues were $76,048 million, which was up 1% year over year, or up 7% in constant currency.\n\nFor the year-to-date period ending June 30, 2022, Alphabet's revenues were $137,696 million.\n\nSpecific revenue breakdowns for Q4 2022 include:\n* Google Search & other: $42,604 million.\n* YouTube ads: $7,963 million.\n* Google Network: $8,475 million.\n* Google advertising: $59,042 million.", + "summary_skipped_reasons": [] + } + } + } + ] + }, + "search_results": [ + { + "id": "513da7d1b9f7739087606aadc626f2dc", + "document": { + "name": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/513da7d1b9f7739087606aadc626f2dc", + "id": "513da7d1b9f7739087606aadc626f2dc", + "derived_struct_data": { + "link": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q4_alphabet_earnings_release.pdf", + "extractive_answers": [ + { + "pageNumber": "1", + "content": "We're on an important journey to re-engineer our cost structure in a durable way and to build financially sustainable, vibrant, growing businesses across Alphabet.\u201d Ruth Porat, CFO of Alphabet and Google, said: \u201cOur Q4 consolidated revenues were $76 billion, up 1% year over year, or up 7% in constant currency, and $283 ..." + }, + { + "pageNumber": "5", + "content": "Alphabet Inc. CONSOLIDATED STATEMENTS OF INCOME (In millions, except per share amounts) Quarter Ended December 31, Year Ended December 31, 2021 2022 2021 2022 (unaudited) (unaudited) Revenues $ 75325 $ 76048 $ 257637 $ 282836 Costs and expenses: Cost of revenues 32988 35342 110939 126203 Research and development 8708 ..." + }, + { + "pageNumber": "2", + "content": "Q4 2022 supplemental information (in millions, except for number of employees; unaudited) Revenues, Traffic Acquisition Costs (TAC) and number of employees Quarter Ended December 31, 2021 2022 Google Search & other $ 43301 $ 42604 YouTube ads 8633 7963 Google Network 9305 8475 Google advertising 61239 59042 Google ..." + } + ], + "snippets": [ + { + "snippet_status": "SUCCESS", + "snippet": "... Alphabet.\u201d Ruth Porat, CFO of Alphabet and Google, said: \u201cOur Q4 consolidated revenues were $76 billion, up 1% year over year, or up 7% in constant currency ..." + } + ], + "title": "GOOG Exhibit 99.1 Q4 2022", + "can_fetch_raw_content": "true" + }, + "schema_id": "", + "parent_document_id": "" + }, + "rank_signals": { + "keyword_similarity_score": 2.4196758, + "semantic_similarity_score": 0.78510624, + "topicality_rank": 4.0, + "document_age": 497105.84, + "boosting_factor": 0.0, + "default_rank": 1.0, + "custom_signals": [] + }, + "model_scores": {} + }, + { + "id": "c201efed2be20c6a4116611ea16be9cd", + "document": { + "name": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/c201efed2be20c6a4116611ea16be9cd", + "id": "c201efed2be20c6a4116611ea16be9cd", + "derived_struct_data": { + "link": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q1_alphabet_earnings_release.pdf", + "extractive_answers": [ + { + "pageNumber": "1", + "content": "Quarter Ended March 31, 2021 2022 Revenues $ 55314 $ 68011 Change in revenues year over year 34 % 23 % Change in constant currency revenues year over year(1) 32 % 26 % Operating income $ 16437 $ 20094 Operating margin 30 % 30 % Other income (expense), net $ 4846 $ (1160) Net income $ 17930 $ 16436 Diluted EPS $ 26.29 $ ..." + }, + { + "pageNumber": "5", + "content": "Alphabet Inc. CONSOLIDATED STATEMENTS OF INCOME (In millions, except share amounts which are reflected in thousands and per share amounts) Quarter Ended March 31, 2021 2022 (unaudited) Revenues $ 55314 $ 68011 Costs and expenses: Cost of revenues 24103 29599 Research and development 7485 9119 Sales and marketing 4516 ..." + }, + { + "pageNumber": "8", + "content": "Comparison from the Quarter Ended March 31, 2021 to the Quarter Ended March 31, 2022 Quarter Ended March 31, 2021 March 31, 2022 % Change from Prior Year EMEA revenues $ 17031 $ 20317 19 % EMEA constant currency revenues 21628 27 % APAC revenues 10455 11841 13 % APAC constant currency revenues 12440 19 % Other Americas ..." + } + ], + "snippets": [ + { + "snippet_status": "SUCCESS", + "snippet": "\u2013 April 26, 2022 \u2013 Alphabet Inc. (NASDAQ: GOOG ... revenue growth of 23% year over year. ... Quarter Ended March 31, 2021 2022 Revenues $ 55,314 $ 68,011 Change in ..." + } + ], + "title": "GOOG Exhibit 99.1 Q1 2022", + "can_fetch_raw_content": "true" + }, + "schema_id": "", + "parent_document_id": "" + }, + "rank_signals": { + "keyword_similarity_score": 2.303273, + "semantic_similarity_score": 0.7624767, + "topicality_rank": 2.0, + "document_age": 497105.84, + "boosting_factor": 0.0, + "default_rank": 2.0, + "custom_signals": [] + }, + "model_scores": {} + }, + { + "id": "376dab8cf7b3807296f2bf1c6228e078", + "document": { + "name": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/376dab8cf7b3807296f2bf1c6228e078", + "id": "376dab8cf7b3807296f2bf1c6228e078", + "derived_struct_data": { + "link": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q2_alphabet_earnings_release.pdf", + "extractive_answers": [ + { + "pageNumber": "1", + "content": "Quarter Ended June 30, 2021 2022 Revenues $ 61880 $ 69685 Change in revenues year over year (1) 62 % 13 % Change in constant currency revenues year over year(1) (2) 57 % 16 % Operating income $ 19361 $ 19453 Operating margin 31 % 28 % Other income (expense), net $ 2624 $ (439) Net income $ 18525 $ 16002 Diluted EPS $ ..." + }, + { + "pageNumber": "5", + "content": "Alphabet Inc. CONSOLIDATED STATEMENTS OF INCOME (In millions, except per share amounts, unaudited) Quarter Ended June 30, Year To Date June 30, 2021 2022 2021 2022 Revenues $ 61880 $ 69685 $ 117194 $ 137696 Costs and expenses: Cost of revenues 26227 30104 50330 59703 Research and development 7675 9841 15160 18960 Sales ..." + }, + { + "pageNumber": "6", + "content": "Alphabet Inc. CONSOLIDATED STATEMENTS OF CASH FLOWS (In millions, unaudited) Quarter Ended June 30, Year To Date June 30, 2021 2022 2021 2022 Operating activities Net income $ 18525 $ 16002 $ 36455 $ 32438 Adjustments: Depreciation and impairment of property and equipment 2730 3698 5255 7289 Amortization and impairment ..." + } + ], + "snippets": [ + { + "snippet_status": "SUCCESS", + "snippet": "\u2013 July 26, 2022 \u2013 Alphabet Inc. (NASDAQ: GOOG ... Quarter Ended June 30, 2021 2022 Revenues $ 61,880 $ 69,685 Change in revenues ... revenue growth rates by 3.7%." + } + ], + "title": "GOOG Exhibit 99.1 Q2 2022", + "can_fetch_raw_content": "true" + }, + "schema_id": "", + "parent_document_id": "" + }, + "rank_signals": { + "keyword_similarity_score": 2.5845962, + "semantic_similarity_score": 0.7609745, + "topicality_rank": 5.0, + "document_age": 497105.84, + "boosting_factor": 0.0, + "default_rank": 3.0, + "custom_signals": [] + }, + "model_scores": {} + }, + { + "id": "4a4eabb0cefc954375df0bd9eec85c96", + "document": { + "name": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/4a4eabb0cefc954375df0bd9eec85c96", + "id": "4a4eabb0cefc954375df0bd9eec85c96", + "derived_struct_data": { + "link": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2023q4-alphabet-earnings-release.pdf", + "extractive_answers": [ + { + "pageNumber": "6", + "content": "Alphabet Inc. CONSOLIDATED STATEMENTS OF INCOME (In millions, except per share amounts) Quarter Ended December 31, Year Ended December 31, 2022 2023 2022 2023 (unaudited) (unaudited) Revenues $ 76048 $ 86310 $ 282836 $ 307394 Costs and expenses: Cost of revenues 35342 37575 126203 133332 Research and development 10267 ..." + }, + { + "pageNumber": "11", + "content": "Total Revenues \u2014 Prior Year Comparative Periods Comparison from the Quarter Ended December 31, 2021 to the Quarter Ended December 31, 2022 Quarter Ended December 31, 2022 Quarter Ended December 31, % Change from Prior Period Less FX Effect Constant Currency Revenues As Reported Less Hedging Effect Less FX Effect ..." + }, + { + "pageNumber": "2", + "content": "Q4 2023 Supplemental Information (in millions, except for number of employees; unaudited) Revenues, Traffic Acquisition Costs (TAC), and Number of Employees Quarter Ended December 31, 2022 2023 Google Search & other $ 42604 $ 48020 YouTube ads 7963 9200 Google Network 8475 8297 Google advertising 59042 65517 Google ..." + } + ], + "snippets": [ + { + "snippet_status": "SUCCESS", + "snippet": "... 2022 2023 2022 2023 (unaudited) (unaudited) Revenues $ 76,048 $ 86,310 $ 282,836 $ 307,394 Costs and expenses: Cost of revenues 35,342 37,575 126,203 ..." + } + ], + "title": "2023q4-alphabet-earnings-release", + "can_fetch_raw_content": "true" + }, + "schema_id": "", + "parent_document_id": "" + }, + "rank_signals": { + "keyword_similarity_score": 2.0151296, + "semantic_similarity_score": 0.771026, + "topicality_rank": 7.0, + "document_age": 497105.84, + "boosting_factor": 0.0, + "default_rank": 4.0, + "custom_signals": [] + }, + "model_scores": {} + }, + { + "id": "2eab9446e934f432e3df7808af857300", + "document": { + "name": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/2eab9446e934f432e3df7808af857300", + "id": "2eab9446e934f432e3df7808af857300", + "derived_struct_data": { + "link": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q3_alphabet_earnings_release.pdf", + "extractive_answers": [ + { + "pageNumber": "1", + "content": "Quarter Ended September 30, 2021 2022 Revenues $ 65118 $ 69092 Change in revenues year over year 41 % 6 % Change in constant currency revenues year over year(1) 39 % 11 % Operating income $ 21031 $ 17135 Operating margin 32 % 25 % Other income (expense), net $ 2033 $ (902) Net income $ 18936 $ 13910 Diluted EPS $ 1.40 ..." + }, + { + "pageNumber": "5", + "content": "Alphabet Inc. CONSOLIDATED STATEMENTS OF INCOME (In millions, except per share amounts, unaudited) Quarter Ended September 30, Year to Date September 30, 2021 2022 2021 2022 Revenues $ 65118 $ 69092 $ 182312 $ 206788 Costs and expenses: Cost of revenues 27621 31158 77951 90861 Research and development 7694 10273 22854 ..." + }, + { + "pageNumber": "6", + "content": "Alphabet Inc. CONSOLIDATED STATEMENTS OF CASH FLOWS (In millions, unaudited) Quarter Ended September 30, Year to Date September 30, 2021 2022 2021 2022 Operating activities Net income $ 18936 $ 13910 $ 55391 $ 46348 Adjustments: Depreciation and impairment of property and equipment 3085 3933 8340 11222 Amortization and ..." + } + ], + "snippets": [ + { + "snippet_status": "SUCCESS", + "snippet": "Alphabet Announces Third Quarter 2022 Results MOUNTAIN VIEW, Calif. \u2013 October 25, 2022 \u2013 Alphabet Inc. ... Quarter Ended September 30, 2021 2022 Revenues $ 65,118 ..." + } + ], + "title": "GOOG Exhibit 99.1 Q3 2022", + "can_fetch_raw_content": "true" + }, + "schema_id": "", + "parent_document_id": "" + }, + "rank_signals": { + "keyword_similarity_score": 2.1627069, + "semantic_similarity_score": 0.76224506, + "topicality_rank": 8.0, + "document_age": 497105.84, + "boosting_factor": 0.0, + "default_rank": 5.0, + "custom_signals": [] + }, + "model_scores": {} + }, + { + "id": "3c617f6f4b023305d82916f0093ca13d", + "document": { + "name": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/3c617f6f4b023305d82916f0093ca13d", + "id": "3c617f6f4b023305d82916f0093ca13d", + "derived_struct_data": { + "link": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/goog023-alphabet-2023-annual-report-web-1.pdf", + "extractive_answers": [ + { + "pageNumber": "60", + "content": "Alphabet Inc. Consolidated Statements of Income Year Ended December 31, (in millions, except per share amounts) 2021 2022 2023 Revenues $ 257637 $ 282836 $ 307394 Costs and expenses: Cost of revenues 110939 126203 133332 Research and development 31562 39500 45427 Sales and marketing 22912 26567 27917 General and ..." + }, + { + "pageNumber": "46", + "content": "Year Ended December 31, 2022 2023 Operating income (loss): Google Services $ 82699 $ 95858 Google Cloud (1922) 1716 Other Bets (4636) (4095) Alphabet-level activities(1) (1299) (9186) Total income from operations $ 74842 $ 84293 (1) In addition to the costs included in Alphabet-level activities, hedging gains (losses) ..." + }, + { + "pageNumber": "97", + "content": "The following table presents information about our segments (in millions): Year Ended December 31, 2021 2022 2023 Revenues: Google Services $ 237529 $ 253528 $ 272543 Google Cloud 19206 26280 33088 Other Bets 753 1068 1527 Hedging gains (losses) 149 1960 236 Total revenues $ 257637 $ 282836 $ 307394 Operating income ( ..." + } + ], + "snippets": [ + { + "snippet_status": "SUCCESS", + "snippet": "... 2022 2023 Revenues $ 257,637 $ 282,836 $ 307,394 Costs and expenses: Cost of revenues 110,939 126,203 133,332 Research and development 31,562 39,500 45,427 ..." + } + ], + "title": "goog023-alphabet-2023-annual-report-web-1", + "can_fetch_raw_content": "true" + }, + "schema_id": "", + "parent_document_id": "" + }, + "rank_signals": { + "keyword_similarity_score": 1.782047, + "semantic_similarity_score": 0.772955, + "topicality_rank": 11.0, + "document_age": 497105.84, + "boosting_factor": 0.0, + "default_rank": 7.0, + "custom_signals": [] + }, + "model_scores": {} + }, + { + "id": "276cee4c4086600303bc561483691f9a", + "document": { + "name": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/276cee4c4086600303bc561483691f9a", + "id": "276cee4c4086600303bc561483691f9a", + "derived_struct_data": { + "link": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022_alphabet_annual_report.pdf", + "extractive_answers": [ + { + "pageNumber": "119", + "content": "Prior periods will be recast to conform to the revised presentation. Our operating segments are not evaluated using asset information. The following table presents information about our segments (in millions): Year Ended December 31, 2020 2021 2022 Revenues: Google Services $ 168635 $ 237529 $ 253528 Google Cloud 13059 ..." + }, + { + "pageNumber": "94", + "content": "Deferred revenues primarily relate to Google Cloud and Google other. Total deferred revenue as of December 31, 2021 was $3.8 billion, of which $2.5 billion was recognized as revenues for the year ending December 31, 2022." + }, + { + "pageNumber": "83", + "content": "ALPHABET INC. \u25cf ANNUAL REPORT 45 PART II ITEM 8 FINANCIAL STATEMENTS AND SUPPLEMENTARY DATA Alphabet Inc. CONSOLIDATED STATEMENTS OF INCOME (In millions, except per share amounts) Year Ended December 31, 2020 2021 2022 Revenues $ 182527 $ 257637 $ 282836 Costs and expenses: Cost of revenues 84732 110939 126203 Research ..." + } + ], + "snippets": [ + { + "snippet_status": "SUCCESS", + "snippet": "... Alphabet's AI activities. DeepMind, previously reported within Other Bets, will be reported as part of Alphabet's corporate costs, reflecting its increasing ..." + } + ], + "title": "2022_alphabet_annual_report", + "can_fetch_raw_content": "true" + }, + "schema_id": "", + "parent_document_id": "" + }, + "rank_signals": { + "keyword_similarity_score": 1.9559898, + "semantic_similarity_score": 0.7700404, + "topicality_rank": 9.0, + "document_age": 497105.84, + "boosting_factor": 0.0, + "default_rank": 8.0, + "custom_signals": [] + }, + "model_scores": {} + }, + { + "id": "404cb139d857b704a13ee3d8e0c6c5fb", + "document": { + "name": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/404cb139d857b704a13ee3d8e0c6c5fb", + "id": "404cb139d857b704a13ee3d8e0c6c5fb", + "derived_struct_data": { + "link": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2021Q4_alphabet_earnings_release.pdf", + "extractive_answers": [ + { + "pageNumber": "1", + "content": "Sundar Pichai, CEO of Alphabet and Google, said: \u201cOur deep investment in AI technologies continues to drive extraordinary and helpful experiences for people and businesses, across our most important products. Q4 saw ongoing strong growth in our advertising business, which helped millions of businesses thrive and find ..." + }, + { + "pageNumber": "5", + "content": "Alphabet Inc. CONSOLIDATED STATEMENTS OF INCOME (In millions, except share amounts which are reflected in thousands and per share amounts) Quarter Ended December 31, Year Ended December 31, 2020 2021 2020 2021 (unaudited) (unaudited) Revenues $ 56898 $ 75325 $ 182527 $ 257637 Costs and expenses: Cost of revenues 26080 ..." + }, + { + "pageNumber": "8", + "content": "Non-GAAP constant currency revenues are calculated by translating current quarter revenues using prior period exchange rates and excluding any hedging effect recognized in the current quarter." + } + ], + "snippets": [ + { + "snippet_status": "SUCCESS", + "snippet": "Alphabet Announces Fourth Quarter and Fiscal Year 2021 Results MOUNTAIN VIEW, Calif. \u2013 February 1, 2022 \u2013 Alphabet Inc. ... revenue growth from Google Cloud." + } + ], + "title": "GOOG Exhibit 99.1 Q4 2021", + "can_fetch_raw_content": "true" + }, + "schema_id": "", + "parent_document_id": "" + }, + "rank_signals": { + "keyword_similarity_score": 1.4731116, + "semantic_similarity_score": 0.76674277, + "topicality_rank": 23.0, + "document_age": 497105.84, + "boosting_factor": 0.0, + "default_rank": 18.0, + "custom_signals": [] + }, + "model_scores": {} + }, + { + "id": "e0ac436b3f82b7df8fc94580f3a84b0a", + "document": { + "name": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/e0ac436b3f82b7df8fc94580f3a84b0a", + "id": "e0ac436b3f82b7df8fc94580f3a84b0a", + "derived_struct_data": { + "link": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2023q1-alphabet-earnings-release.pdf", + "extractive_answers": [ + { + "pageNumber": "1", + "content": "Quarter Ended March 31, 2022 2023 Revenues $ 68011 $ 69787 Change in revenues year over year 23 % 3 % Change in constant currency revenues year over year(1) 26 % 6 % Operating income $ 20094 $ 17415 Operating margin 30 % 25 % Other income (expense), net $ (1160) $ 790 Net income $ 16436 $ 15051 Diluted EPS $ 1.23 $ ..." + }, + { + "pageNumber": "6", + "content": "Alphabet Inc. CONSOLIDATED STATEMENTS OF INCOME (In millions, except per share amounts, unaudited) Quarter Ended March 31, 2022 2023 Revenues $ 68011 $ 69787 Costs and expenses: Cost of revenues 29599 30612 Research and development 9119 11468 Sales and marketing 5825 6533 General and administrative 3374 3759 Total ..." + }, + { + "pageNumber": "2", + "content": "Quarter Ended March 31, 2022 2023 (recast) Operating income (loss): Google Services $ 21973 $ 21737 Google Cloud (706) 191 Other Bets (835) (1225) Corporate costs, unallocated(1) (338) (3288) Total income from operations $ 20094 $ 17415 (1) Hedging gains (losses) related to revenue included in unallocated corporate ..." + } + ], + "snippets": [ + { + "snippet_status": "SUCCESS", + "snippet": "Ruth Porat, CFO of Alphabet and Google, said: \u201cResilience in Search and momentum in Cloud resulted in Q1 consolidated revenues of $69.8 billion, up 3% year over ..." + } + ], + "title": "GOOG Exhibit 99.1 Q1 2023", + "can_fetch_raw_content": "true" + }, + "schema_id": "", + "parent_document_id": "" + }, + "rank_signals": { + "keyword_similarity_score": 1.8275325, + "semantic_similarity_score": 0.7408051, + "topicality_rank": 20.0, + "document_age": 497105.84, + "boosting_factor": 0.0, + "default_rank": 19.0, + "custom_signals": [] + }, + "model_scores": {} + }, + { + "id": "4c7389ffaf06b73e6e380eac7a39ea17", + "document": { + "name": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/4c7389ffaf06b73e6e380eac7a39ea17", + "id": "4c7389ffaf06b73e6e380eac7a39ea17", + "derived_struct_data": { + "link": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2023q3-alphabet-earnings-release.pdf", + "extractive_answers": [ + { + "pageNumber": "1", + "content": "We're continuing to focus on making AI more helpful for everyone; there's exciting progress and lots more to come.\u201d Ruth Porat, President and Chief Investment Officer; CFO said: \u201cThe fundamental strength of our business was apparent again in Q3, with $77 billion in revenue, up 11% year over year, driven by meaningful ..." + }, + { + "pageNumber": "6", + "content": "Alphabet Inc. CONSOLIDATED STATEMENTS OF INCOME (In millions, except per share amounts, unaudited) Quarter Ended September 30, Year to Date September 30, 2022 2023 2022 2023 Revenues $ 69092 $ 76693 $ 206788 $ 221084 Costs and expenses: Cost of revenues 31158 33229 90861 95757 Research and development 10273 11258 29233 ..." + }, + { + "pageNumber": "2", + "content": "Quarter Ended September 30, 2022 2023 (recast) Operating income (loss): Google Services $ 18883 $ 23937 Google Cloud (440) 266 Other Bets (1225) (1194) Corporate costs, unallocated(1) (83) (1666) Total income from operations $ 17135 $ 21343 (1) In addition to the Alphabet-level costs included in unallocated corporate ..." + } + ], + "snippets": [ + { + "snippet_status": "SUCCESS", + "snippet": "Alphabet Announces Third Quarter 2023 Results MOUNTAIN VIEW, Calif. ... revenue, up 11% year over year, driven by ... 2022 and 2023 (in millions, except for per ..." + } + ], + "title": "GOOG Exhibit 99.1 Q3 2023", + "can_fetch_raw_content": "true" + }, + "schema_id": "", + "parent_document_id": "" + }, + "rank_signals": { + "keyword_similarity_score": 1.7931039, + "semantic_similarity_score": 0.73606783, + "topicality_rank": 30.0, + "document_age": 497105.84, + "boosting_factor": 0.0, + "default_rank": 20.0, + "custom_signals": [] + }, + "model_scores": {} + } + ] + } + ] +} diff --git a/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_async_grpc[rank].json b/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_async_grpc[rank].json new file mode 100644 index 000000000..26f5be92c --- /dev/null +++ b/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_async_grpc[rank].json @@ -0,0 +1,33 @@ +{ + "method": "rank", + "requests": [ + { + "ranking_config": "projects/sdk-dev-508013/locations/global/rankingConfigs/default_ranking_config", + "query": "What is Braintrust?", + "records": [ + { + "id": "1", + "content": "Braintrust evaluates AI applications.", + "title": "", + "score": 0.0 + } + ], + "model": "", + "top_n": 0, + "ignore_record_details_in_response": false, + "user_labels": {} + } + ], + "responses": [ + { + "records": [ + { + "id": "1", + "content": "Braintrust evaluates AI applications.", + "score": 0.6252, + "title": "" + } + ] + } + ] +} diff --git a/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_async_grpc[stream_answer_query].json b/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_async_grpc[stream_answer_query].json new file mode 100644 index 000000000..eff807c8b --- /dev/null +++ b/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_async_grpc[stream_answer_query].json @@ -0,0 +1,1755 @@ +{ + "method": "stream_answer_query", + "requests": [ + { + "serving_config": "projects/sdk-dev-508013/locations/global/collections/default_collection/engines/braintrust-sdk-test_1789580029521/servingConfigs/default_search", + "query": { + "text": "What was Alphabet's revenue in 2022?", + "query_id": "" + }, + "answer_generation_spec": { + "include_citations": true, + "answer_language_code": "", + "ignore_adversarial_query": false, + "ignore_non_answer_seeking_query": false, + "ignore_jail_breaking_query": false + }, + "session": "", + "asynchronous_mode": false, + "user_pseudo_id": "", + "user_labels": {} + } + ], + "responses": [ + { + "answer": { + "state": 4, + "steps": [ + { + "state": 3, + "description": "Rephrase the query and search.", + "actions": [ + { + "search_action": { + "query": "What was Alphabet's revenue in 2022?" + }, + "observation": { + "search_results": [ + { + "document": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/513da7d1b9f7739087606aadc626f2dc", + "uri": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q4_alphabet_earnings_release.pdf", + "title": "GOOG Exhibit 99.1 Q4 2022", + "snippet_info": [ + { + "snippet": "... Alphabet.\u201d Ruth Porat, CFO of Alphabet and Google, said: \u201cOur Q4 consolidated revenues were $76 billion, up 1% year over year, or up 7% in constant currency ...", + "snippet_status": "SUCCESS" + } + ], + "chunk_info": [] + }, + { + "document": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/c201efed2be20c6a4116611ea16be9cd", + "uri": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q1_alphabet_earnings_release.pdf", + "title": "GOOG Exhibit 99.1 Q1 2022", + "snippet_info": [ + { + "snippet": "\u2013 April 26, 2022 \u2013 Alphabet Inc. (NASDAQ: GOOG ... revenue growth of 23% year over year. ... Quarter Ended March 31, 2021 2022 Revenues $ 55,314 $ 68,011 Change in ...", + "snippet_status": "SUCCESS" + } + ], + "chunk_info": [] + }, + { + "document": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/376dab8cf7b3807296f2bf1c6228e078", + "uri": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q2_alphabet_earnings_release.pdf", + "title": "GOOG Exhibit 99.1 Q2 2022", + "snippet_info": [ + { + "snippet": "\u2013 July 26, 2022 \u2013 Alphabet Inc. (NASDAQ: GOOG ... Quarter Ended June 30, 2021 2022 Revenues $ 61,880 $ 69,685 Change in revenues ... revenue growth rates by 3.7%.", + "snippet_status": "SUCCESS" + } + ], + "chunk_info": [] + }, + { + "document": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/4a4eabb0cefc954375df0bd9eec85c96", + "uri": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2023q4-alphabet-earnings-release.pdf", + "title": "2023q4-alphabet-earnings-release", + "snippet_info": [ + { + "snippet": "... Revenues As Reported Less Hedging Effect Less FX Effect Constant Currency 2022 2023 Revenues United States $ 134,814 $ 146,286 $ 0 $ 146,286 9 % 0 % 9 ...", + "snippet_status": "SUCCESS" + } + ], + "chunk_info": [] + }, + { + "document": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/2eab9446e934f432e3df7808af857300", + "uri": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q3_alphabet_earnings_release.pdf", + "title": "GOOG Exhibit 99.1 Q3 2022", + "snippet_info": [ + { + "snippet": "Alphabet Announces Third Quarter 2022 Results MOUNTAIN VIEW, Calif. \u2013 October 25, 2022 \u2013 Alphabet Inc. ... Quarter Ended September 30, 2021 2022 Revenues $ 65,118 ...", + "snippet_status": "SUCCESS" + } + ], + "chunk_info": [] + }, + { + "document": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/276cee4c4086600303bc561483691f9a", + "uri": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022_alphabet_annual_report.pdf", + "title": "2022_alphabet_annual_report", + "snippet_info": [ + { + "snippet": "... Alphabet's AI activities. DeepMind, previously reported within Other Bets, will be reported as part of Alphabet's corporate costs, reflecting its increasing ...", + "snippet_status": "SUCCESS" + } + ], + "chunk_info": [] + }, + { + "document": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/3c617f6f4b023305d82916f0093ca13d", + "uri": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/goog023-alphabet-2023-annual-report-web-1.pdf", + "title": "goog023-alphabet-2023-annual-report-web-1", + "snippet_info": [ + { + "snippet": "... 2022 2023 Revenues $ 257,637 $ 282,836 $ 307,394 Costs and expenses: Cost of revenues 110,939 126,203 133,332 Research and development 31,562 39,500 45,427 ...", + "snippet_status": "SUCCESS" + } + ], + "chunk_info": [] + }, + { + "document": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/404cb139d857b704a13ee3d8e0c6c5fb", + "uri": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2021Q4_alphabet_earnings_release.pdf", + "title": "GOOG Exhibit 99.1 Q4 2021", + "snippet_info": [ + { + "snippet": "Alphabet Announces Fourth Quarter and Fiscal Year 2021 Results MOUNTAIN VIEW, Calif. \u2013 February 1, 2022 \u2013 Alphabet Inc. ... revenue growth from Google Cloud.", + "snippet_status": "SUCCESS" + } + ], + "chunk_info": [] + }, + { + "document": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/e0ac436b3f82b7df8fc94580f3a84b0a", + "uri": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2023q1-alphabet-earnings-release.pdf", + "title": "GOOG Exhibit 99.1 Q1 2023", + "snippet_info": [ + { + "snippet": "Ruth Porat, CFO of Alphabet and Google, said: \u201cResilience in Search and momentum in Cloud resulted in Q1 consolidated revenues of $69.8 billion, up 3% year over ...", + "snippet_status": "SUCCESS" + } + ], + "chunk_info": [] + }, + { + "document": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/4c7389ffaf06b73e6e380eac7a39ea17", + "uri": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2023q3-alphabet-earnings-release.pdf", + "title": "GOOG Exhibit 99.1 Q3 2023", + "snippet_info": [ + { + "snippet": "Alphabet Announces Third Quarter 2023 Results MOUNTAIN VIEW, Calif. ... revenue, up 11% year over year, driven by ... 2022 and 2023 (in millions, except for per ...", + "snippet_status": "SUCCESS" + } + ], + "chunk_info": [] + } + ] + } + } + ], + "thought": "" + } + ], + "name": "", + "answer_text": "", + "citations": [], + "grounding_supports": [], + "references": [], + "related_questions": [], + "answer_skipped_reasons": [], + "safety_ratings": [] + }, + "answer_query_token": "NMwKDAjPravVBhD9nMWHAxIkNmFjNGMzNWYtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi" + }, + { + "answer": { + "state": 4, + "references": [ + { + "chunk_info": { + "content": "ALPHABET INC. ANNUAL REPORT 45 PART II\nITEM\u00a08\u00a0\u00a0FINANCIAL STATEMENTS AND SUPPLEMENTARY DATA Alphabet Inc. CONSOLIDATED STATEMENTS OF INCOME (In millions, except per share amounts) Year Ended December 31,\n2020 2021 2022 Revenues $ 182,527 $ 257,637 $ 282,836 Costs and expenses:\nCost of revenues 84,732 110,939 126,203 Research and development 27,573 31,562 39,500 Sales and marketing 17,946 22,912 26,567 General and administrative 11,052 13,510 15,724 Total costs and expenses 141,303 178,923 207,994 Income from operations 41,224 78,714 74,842 Other income (expense), net 6,858 12,020 (3,514) Income before income taxes 48,082 90,734 71,328 Provision for income taxes 7,813 14,701 11,356 Net income $ 40,269 $ 76,033 $ 59,972 Basic net income per share of Class A, Class B, and Class C stock $ 2.96 $ 5.69 $ 4.59 Diluted net income per share of Class A, Class B, and Class C stock $ 2.93 $ 5.61 $ 4.56 See accompanying notes. ", + "document_metadata": { + "document": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/276cee4c4086600303bc561483691f9a", + "uri": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022_alphabet_annual_report.pdf", + "title": "2022_alphabet_annual_report", + "page_identifier": "" + }, + "chunk": "" + } + }, + { + "chunk_info": { + "content": "Alphabet Announces First Quarter 2022 Results\nMOUNTAIN VIEW, Calif. \u2013 April 26, 2022 \u2013 Alphabet Inc. (NASDAQ: GOOG, GOOGL) today announced financial\nresults for the quarter ended March 31, 2022. Sundar Pichai, CEO of Alphabet and Google, said: \u201cQ1 saw strong growth in Search and Cloud, in particular, which\nare both helping people and businesses as the digital transformation continues. We\u2019ll keep investing in great\nproducts and services, and creating opportunities for partners and local communities around the world.\u201d Ruth Porat, CFO of Alphabet and Google, said: \u201cWe are pleased with Q1 revenue growth of 23% year over year. We\ncontinue to make considered investments in Capex, R&D and talent to support long-term value creation for all\nstakeholders.\u201d Q1 2022 financial highlights\nThe following table summarizes our consolidated financial results for the quarters ended March 31, 2021 and 2022\n(in millions, except for per share information and percentages; unaudited). Quarter Ended March 31,\n2021 2022 Revenues $ 55,314 $ 68,011 Change in revenues year over year 34 % 23 % Change in constant currency revenues year over year(1) 32 % 26 % Operating income $ 16,437 $ 20,094 Operating margin 30 % 30 % Other income (expense), net $ 4,846 $ (1,160) Net income $ 17,930 $ 16,436 ", + "document_metadata": { + "document": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/c201efed2be20c6a4116611ea16be9cd", + "uri": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q1_alphabet_earnings_release.pdf", + "title": "GOOG Exhibit 99.1 Q1 2022", + "page_identifier": "" + }, + "chunk": "" + } + }, + { + "chunk_info": { + "content": "Alphabet Announces Fourth Quarter and Fiscal Year 2022 Results\nMOUNTAIN VIEW, Calif. \u2013 February 2, 2023 \u2013 Alphabet Inc. (NASDAQ: GOOG, GOOGL) today announced financial\nresults for the quarter and fiscal year ended December 31, 2022. Sundar Pichai, CEO of Alphabet and Google, said: \u201cOur long-term investments in deep computer science make us\nextremely well-positioned as AI reaches an inflection point, and I\u2019m excited by the AI-driven leaps we\u2019re about to unveil\nin Search and beyond. There\u2019s also great momentum in Cloud, YouTube subscriptions, and our Pixel devices. We\u2019re\non an important journey to re-engineer our cost structure in a durable way and to build financially sustainable, vibrant,\ngrowing businesses across Alphabet.\u201d Ruth Porat, CFO of Alphabet and Google, said: \u201cOur Q4 consolidated revenues were $76 billion, up 1% year over year,\nor up 7% in constant currency, and $283 billion for the full year 2022, up 10%, or up 14% in constant currency. We\nhave significant work underway to improve all aspects of our cost structure, in support of our investments in our\nhighest growth priorities to deliver long-term, profitable growth.\u201d ", + "document_metadata": { + "document": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/513da7d1b9f7739087606aadc626f2dc", + "uri": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q4_alphabet_earnings_release.pdf", + "title": "GOOG Exhibit 99.1 Q4 2022", + "page_identifier": "" + }, + "chunk": "" + } + }, + { + "chunk_info": { + "content": "Alphabet Announces Third Quarter 2023 Results\nMOUNTAIN VIEW, Calif. \u2013 October 24, 2023 \u2013 Alphabet Inc. (NASDAQ: GOOG, GOOGL) today announced\nfinancial results for the quarter ended September 30, 2023. Sundar Pichai, CEO, said: \u201cI\u2019m pleased with our financial results and our product momentum this quarter, with AI\ndriven innovations across Search, YouTube, Cloud, our Pixel devices and more. We\u2019re continuing to focus on\nmaking AI more helpful for everyone; there\u2019s exciting progress and lots more to come.\u201d Ruth Porat, President and Chief Investment Officer; CFO said: \u201cThe fundamental strength of our business was\napparent again in Q3, with $77 billion in revenue, up 11% year over year, driven by meaningful growth in Search\nand YouTube, and momentum in Cloud. We continue to focus on judicious capital allocation to deliver sustainable\nfinancial value.\u201d Q3 2023 Financial Highlights (unaudited)\nThe following table summarizes our consolidated financial results for the quarters ended September 30, 2022 and\n2023 (in millions, except for per share information and percentages). Quarter Ended September 30,\n2022 2023 Revenues $ 69,092 $ 76,693 Change in revenues year over year 6 % 11 % Change in constant currency revenues year over year(1) 11 % 11 % Operating income $ 17,135 $ 21,343 Operating margin 25 % ", + "document_metadata": { + "document": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/4c7389ffaf06b73e6e380eac7a39ea17", + "uri": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2023q3-alphabet-earnings-release.pdf", + "title": "GOOG Exhibit 99.1 Q3 2023", + "page_identifier": "" + }, + "chunk": "" + } + }, + { + "chunk_info": { + "content": "Alphabet Inc. Consolidated Statements of Income Year Ended December 31, (in millions, except per share amounts) 2021 2022 2023 Revenues $ 257,637 $ 282,836 $ 307,394 Costs and expenses:\nCost of revenues 110,939 126,203 133,332 Research and development 31,562 39,500 45,427 Sales and marketing 22,912 26,567 27,917 General and administrative 13,510 15,724 16,425 Total costs and expenses 178,923 207,994 223,101 Income from operations 78,714 74,842 84,293 Other income (expense), net 12,020 (3,514) 1,424 Income before income taxes 90,734 71,328 85,717 Provision for income taxes 14,701 11,356 11,922 Net income $ 76,033 $ 59,972 $ 73,795 Basic net income per share of Class A, Class B, and Class C stock $ 5.69 $ 4.59 $ 5.84 Diluted net income per share of Class A, Class B, and Class C stock $ 5.61 $ 4.56 $ 5.80 See accompanying notes. 50 Alphabet 2023 Annual Report Part I Part II Part III Part IV ", + "document_metadata": { + "document": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/3c617f6f4b023305d82916f0093ca13d", + "uri": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/goog023-alphabet-2023-annual-report-web-1.pdf", + "title": "goog023-alphabet-2023-annual-report-web-1", + "page_identifier": "" + }, + "chunk": "" + } + }, + { + "chunk_info": { + "content": "Alphabet Announces Third Quarter 2022 Results\nMOUNTAIN VIEW, Calif. \u2013 October 25, 2022 \u2013 Alphabet Inc. (NASDAQ: GOOG, GOOGL) today announced\nfinancial results for the quarter ended September 30, 2022. Sundar Pichai, CEO of Alphabet and Google, said: \u201cWe\u2019re sharpening our focus on a clear set of product and\nbusiness priorities. Product announcements we\u2019ve made in just the past month alone have shown that very clearly,\nincluding significant improvements to both Search and Cloud, powered by AI, and new ways to monetize YouTube\nShorts. We are focused on both investing responsibly for the long term and being responsive to the economic\nenvironment.\u201d Ruth Porat, CFO of Alphabet and Google, said: \u201cOur third quarter revenues were $69.1 billion, up 6% versus last\nyear or up 11% on a constant currency basis. Financial results for the third quarter reflect healthy fundamental\ngrowth in Search and momentum in Cloud, while affected by foreign exchange. We\u2019re working to realign resources\nto fuel our highest growth priorities.\u201d Q3 2022 financial highlights\nThe following table summarizes our consolidated financial results for the quarters ended September 30, 2021 and\n2022 (in millions, except for per share information and percentages; unaudited). Quarter Ended September 30,\n2021 2022 Revenues $ 65,118 $ 69,092 Change in revenues year over year 41 % ", + "document_metadata": { + "document": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/2eab9446e934f432e3df7808af857300", + "uri": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q3_alphabet_earnings_release.pdf", + "title": "GOOG Exhibit 99.1 Q3 2022", + "page_identifier": "" + }, + "chunk": "" + } + }, + { + "chunk_info": { + "content": "Alphabet Inc. CONSOLIDATED STATEMENTS OF CASH FLOWS\n(In millions, unaudited)\nQuarter Ended September 30, Year to Date September 30, 2021 2022 2021 2022 Operating activities\nNet income $ 18,936 $ 13,910 $ 55,391 $ 46,348 Adjustments:\nDepreciation and impairment of property and\nequipment 3,085 3,933 8,340 11,222 Amortization and impairment of intangible assets 219 113 662 505 Stock-based compensation expense 3,874 4,976 11,422 14,262 Deferred income taxes (1,287) (1,920) 192 (6,157) (Gain) loss on debt and equity securities, net (2,158) 1,378 (9,792) 3,856 Other 64 167 (199) 369 Changes in assets and liabilities, net of effects of\nacquisitions:\nAccounts receivable (2,409) (97) (3,276) 2,298 Income taxes, net 3,041 (609) 2,744 (862) Other assets (1,255) (2,647) (1,447) (4,268) Accounts payable 238 1,907 (874) 735 Accrued expenses and other liabilities 2,562 2,210 2,763 491 Accrued revenue share 357 (80) 386 (1,022) Deferred revenue 272 112 406 104 Net cash provided by operating activities 25,539 23,353 66,718 67,881 Investing activities\nPurchases of property and equipment (6,819) (7,276) (18,257) (23,890) Purchases of marketable securities (34,497) (17,054) (95,106) (67,253) Maturities and sales of marketable securities 31,459 28,713 92,126 84,087 Purchases of non-marketable securities ", + "document_metadata": { + "document": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/2eab9446e934f432e3df7808af857300", + "uri": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q3_alphabet_earnings_release.pdf", + "title": "GOOG Exhibit 99.1 Q3 2022", + "page_identifier": "" + }, + "chunk": "" + } + }, + { + "chunk_info": { + "content": "Alphabet Inc. CONSOLIDATED STATEMENTS OF CASH FLOWS\n(In millions, unaudited)\nQuarter Ended June 30, Year To Date June 30,\n2021 2022 2021 2022 Operating activities\nNet income $ 18,525 $ 16,002 $ 36,455 $ 32,438 Adjustments:\nDepreciation and impairment of property and equipment 2,730 3,698 5,255 7,289 Amortization and impairment of intangible assets 215 201 443 392 Stock-based compensation expense 3,803 4,782 7,548 9,286 Deferred income taxes 379 (2,147) 1,479 (4,237) (Gain) loss on debt and equity securities, net (2,883) 1,041 (7,634) 2,478 Other (8) 62 (263) 202 Changes in assets and liabilities, net of effects of acquisitions:\nAccounts receivable (3,661) (1,969) (867) 2,395 Income taxes, net (1,082) (4,073) (297) (253) Other assets (199) (845) (192) (1,621) Accounts payable (130) 1,201 (1,112) (1,172) Accrued expenses and other liabilities 3,731 1,497 201 (1,719) Accrued revenue share 473 (114) 29 (942) Deferred revenue (3) 86 134 (8) Net cash provided by operating activities 21,890 19,422 41,179 44,528 Investing activities\nPurchases of property and equipment (5,496) (6,828) (11,438) (16,614) Purchases of marketable securities (24,183) (21,737) (60,609) (50,199) ", + "document_metadata": { + "document": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/376dab8cf7b3807296f2bf1c6228e078", + "uri": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q2_alphabet_earnings_release.pdf", + "title": "GOOG Exhibit 99.1 Q2 2022", + "page_identifier": "" + }, + "chunk": "" + } + }, + { + "chunk_info": { + "content": "The associated costs,\nincluding depreciation and impairment, are allocated to operating segments as a service cost generally based on usage or headcount. Unallocated corporate costs primarily include corporate initiatives, corporate shared costs, such as finance and legal, including certain\nfines and settlements, as well as costs associated with certain shared R&D activities. Additionally, hedging gains (losses) related to\nrevenue are included in corporate costs. As AI is critical to delivering our mission of bringing our breakthrough innovations into the real world, beginning in January 2023, we\nwill update our segment reporting relating to certain of Alphabet\u2019s AI activities. DeepMind, previously reported within Other Bets, will\nbe reported as part of Alphabet\u2019s corporate costs, reflecting its increasing collaboration with Google Services, Google Cloud, and\nOther Bets. Prior periods will be recast to conform to the revised presentation. Our operating segments are not evaluated using asset information. The following table presents information about our segments (in millions): Year Ended December 31,\n2020 2021 2022 Revenues:\nGoogle Services $ 168,635 $ 237,529 $ 253,528 Google Cloud 13,059 19,206 26,280 Other Bets 657 753 1,068 Hedging gains (losses) 176 149 1,960 Total revenues $ 182,527 $ 257,637 $ 282,836 Operating income (loss):\nGoogle Services $ 54,606 $ 91,855 $ 86,572 Google Cloud (5,607) (3,099) (2,968) Other Bets ", + "document_metadata": { + "document": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/276cee4c4086600303bc561483691f9a", + "uri": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022_alphabet_annual_report.pdf", + "title": "2022_alphabet_annual_report", + "page_identifier": "" + }, + "chunk": "" + } + }, + { + "chunk_info": { + "content": "56 ALPHABET INC. ANNUAL REPORT PART II\nITEM\u00a08\u00a0\u00a0FINANCIAL STATEMENTS AND SUPPLEMENTARY DATA Note 2. Revenues\nDisaggregated Revenues\nThe following table presents revenues disaggregated by type (in millions): Year Ended December 31,\n2020 2021 2022 Google Search & other $ 104,062 $ 148,951 $ 162,450 YouTube ads 19,772 28,845 29,243 Google Network 23,090 31,701 32,780 Google advertising 146,924 209,497 224,473 Google other 21,711 28,032 29,055 Google Services total 168,635 237,529 253,528 Google Cloud 13,059 19,206 26,280 Other Bets 657 753 1,068 Hedging gains (losses) 176 149 1,960 Total revenues $ 182,527 $ 257,637 $ 282,836\nNo individual customer or groups of affiliated customers represented more than 10% of our revenues in 2020, 2021, or 2022. The following table presents revenues disaggregated by geography, based on the addresses of our customers (in millions):\nYear Ended December 31, 2020 2021 2022 United States $ 85,014 47 % $ 117,854 46 % $ 134,814 48% EMEA(1) 55,370 30 79,107 31 82,062 29 APAC(1) 32,550 18 46,123 18 47,024 16 Other Americas(1) 9,417 5 14,404 5 16,976 6 Hedging gains (losses) 176 0 149 0 1,960 1 Total revenues $ 182,527 100% $ 257,637 100 % $ 282,836 100% ", + "document_metadata": { + "document": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/276cee4c4086600303bc561483691f9a", + "uri": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022_alphabet_annual_report.pdf", + "title": "2022_alphabet_annual_report", + "page_identifier": "" + }, + "chunk": "" + } + } + ], + "name": "", + "answer_text": "", + "citations": [], + "grounding_supports": [], + "related_questions": [], + "steps": [], + "answer_skipped_reasons": [], + "safety_ratings": [] + }, + "answer_query_token": "NMwKDAjPravVBhD9nMWHAxIkNmFjNGMzNWYtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi" + }, + { + "answer": { + "state": 4, + "answer_text": "Alphabet's total revenue", + "name": "", + "citations": [], + "grounding_supports": [], + "references": [], + "related_questions": [], + "steps": [], + "answer_skipped_reasons": [], + "safety_ratings": [] + }, + "answer_query_token": "NMwKDAjPravVBhD9nMWHAxIkNmFjNGMzNWYtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi" + }, + { + "answer": { + "state": 4, + "answer_text": " in 2022 was $282,836 million. ", + "name": "", + "citations": [], + "grounding_supports": [], + "references": [], + "related_questions": [], + "steps": [], + "answer_skipped_reasons": [], + "safety_ratings": [] + }, + "answer_query_token": "NMwKDAjPravVBhD9nMWHAxIkNmFjNGMzNWYtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi" + }, + { + "answer": { + "state": 4, + "citations": [ + { + "end_index": "54", + "sources": [ + { + "reference_id": "0" + }, + { + "reference_id": "4" + }, + { + "reference_id": "8" + }, + { + "reference_id": "9" + } + ], + "start_index": "0" + } + ], + "name": "", + "answer_text": "", + "grounding_supports": [], + "references": [], + "related_questions": [], + "steps": [], + "answer_skipped_reasons": [], + "safety_ratings": [] + }, + "answer_query_token": "NMwKDAjPravVBhD9nMWHAxIkNmFjNGMzNWYtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi" + }, + { + "answer": { + "state": 4, + "answer_text": "This represents a 10% increase", + "name": "", + "citations": [], + "grounding_supports": [], + "references": [], + "related_questions": [], + "steps": [], + "answer_skipped_reasons": [], + "safety_ratings": [] + }, + "answer_query_token": "NMwKDAjPravVBhD9nMWHAxIkNmFjNGMzNWYtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi" + }, + { + "answer": { + "state": 4, + "answer_text": " year-over-year, or 14% in constant currency.\n\n", + "name": "", + "citations": [], + "grounding_supports": [], + "references": [], + "related_questions": [], + "steps": [], + "answer_skipped_reasons": [], + "safety_ratings": [] + }, + "answer_query_token": "NMwKDAjPravVBhD9nMWHAxIkNmFjNGMzNWYtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi" + }, + { + "answer": { + "state": 4, + "citations": [ + { + "start_index": "55", + "end_index": "130", + "sources": [ + { + "reference_id": "2" + } + ] + } + ], + "name": "", + "answer_text": "", + "grounding_supports": [], + "references": [], + "related_questions": [], + "steps": [], + "answer_skipped_reasons": [], + "safety_ratings": [] + }, + "answer_query_token": "NMwKDAjPravVBhD9nMWHAxIkNmFjNGMzNWYtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi" + }, + { + "answer": { + "state": 4, + "answer_text": "Here's a breakdown of Alphabet's 2022 revenues by", + "name": "", + "citations": [], + "grounding_supports": [], + "references": [], + "related_questions": [], + "steps": [], + "answer_skipped_reasons": [], + "safety_ratings": [] + }, + "answer_query_token": "NMwKDAjPravVBhD9nMWHAxIkNmFjNGMzNWYtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi" + }, + { + "answer": { + "state": 4, + "answer_text": " segment and type:\n\n", + "name": "", + "citations": [], + "grounding_supports": [], + "references": [], + "related_questions": [], + "steps": [], + "answer_skipped_reasons": [], + "safety_ratings": [] + }, + "answer_query_token": "NMwKDAjPravVBhD9nMWHAxIkNmFjNGMzNWYtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi" + }, + { + "answer": { + "state": 4, + "answer_text": "**By Segment:**\n* Google Services: $253,528 million\n", + "name": "", + "citations": [], + "grounding_supports": [], + "references": [], + "related_questions": [], + "steps": [], + "answer_skipped_reasons": [], + "safety_ratings": [] + }, + "answer_query_token": "NMwKDAjPravVBhD9nMWHAxIkNmFjNGMzNWYtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi" + }, + { + "answer": { + "state": 4, + "citations": [ + { + "start_index": "217", + "end_index": "254", + "sources": [ + { + "reference_id": "8" + }, + { + "reference_id": "9" + } + ] + } + ], + "name": "", + "answer_text": "", + "grounding_supports": [], + "references": [], + "related_questions": [], + "steps": [], + "answer_skipped_reasons": [], + "safety_ratings": [] + }, + "answer_query_token": "NMwKDAjPravVBhD9nMWHAxIkNmFjNGMzNWYtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi" + }, + { + "answer": { + "state": 4, + "answer_text": "* Google Cloud: $26,280 million\n", + "name": "", + "citations": [], + "grounding_supports": [], + "references": [], + "related_questions": [], + "steps": [], + "answer_skipped_reasons": [], + "safety_ratings": [] + }, + "answer_query_token": "NMwKDAjPravVBhD9nMWHAxIkNmFjNGMzNWYtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi" + }, + { + "answer": { + "state": 4, + "citations": [ + { + "start_index": "255", + "end_index": "288", + "sources": [ + { + "reference_id": "8" + }, + { + "reference_id": "9" + } + ] + } + ], + "name": "", + "answer_text": "", + "grounding_supports": [], + "references": [], + "related_questions": [], + "steps": [], + "answer_skipped_reasons": [], + "safety_ratings": [] + }, + "answer_query_token": "NMwKDAjPravVBhD9nMWHAxIkNmFjNGMzNWYtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi" + }, + { + "answer": { + "state": 4, + "answer_text": "* Other Bets: $1,068 million\n", + "name": "", + "citations": [], + "grounding_supports": [], + "references": [], + "related_questions": [], + "steps": [], + "answer_skipped_reasons": [], + "safety_ratings": [] + }, + "answer_query_token": "NMwKDAjPravVBhD9nMWHAxIkNmFjNGMzNWYtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi" + }, + { + "answer": { + "state": 4, + "citations": [ + { + "start_index": "289", + "end_index": "319", + "sources": [ + { + "reference_id": "8" + }, + { + "reference_id": "9" + } + ] + } + ], + "name": "", + "answer_text": "", + "grounding_supports": [], + "references": [], + "related_questions": [], + "steps": [], + "answer_skipped_reasons": [], + "safety_ratings": [] + }, + "answer_query_token": "NMwKDAjPravVBhD9nMWHAxIkNmFjNGMzNWYtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi" + }, + { + "answer": { + "state": 4, + "answer_text": "* Hedging gains (losses", + "name": "", + "citations": [], + "grounding_supports": [], + "references": [], + "related_questions": [], + "steps": [], + "answer_skipped_reasons": [], + "safety_ratings": [] + }, + "answer_query_token": "NMwKDAjPravVBhD9nMWHAxIkNmFjNGMzNWYtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi" + }, + { + "answer": { + "state": 4, + "answer_text": "): $1,960 million\n\n", + "name": "", + "citations": [], + "grounding_supports": [], + "references": [], + "related_questions": [], + "steps": [], + "answer_skipped_reasons": [], + "safety_ratings": [] + }, + "answer_query_token": "NMwKDAjPravVBhD9nMWHAxIkNmFjNGMzNWYtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi" + }, + { + "answer": { + "state": 4, + "citations": [ + { + "start_index": "320", + "end_index": "362", + "sources": [ + { + "reference_id": "8" + }, + { + "reference_id": "9" + } + ] + } + ], + "name": "", + "answer_text": "", + "grounding_supports": [], + "references": [], + "related_questions": [], + "steps": [], + "answer_skipped_reasons": [], + "safety_ratings": [] + }, + "answer_query_token": "NMwKDAjPravVBhD9nMWHAxIkNmFjNGMzNWYtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi" + }, + { + "answer": { + "state": 4, + "answer_text": "**By Type (within Google Services):**\n", + "name": "", + "citations": [], + "grounding_supports": [], + "references": [], + "related_questions": [], + "steps": [], + "answer_skipped_reasons": [], + "safety_ratings": [] + }, + "answer_query_token": "NMwKDAjPravVBhD9nMWHAxIkNmFjNGMzNWYtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi" + }, + { + "answer": { + "state": 4, + "answer_text": "* Google Search & other: $162,450 million\n", + "name": "", + "citations": [], + "grounding_supports": [], + "references": [], + "related_questions": [], + "steps": [], + "answer_skipped_reasons": [], + "safety_ratings": [] + }, + "answer_query_token": "NMwKDAjPravVBhD9nMWHAxIkNmFjNGMzNWYtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi" + }, + { + "answer": { + "state": 4, + "citations": [ + { + "start_index": "402", + "end_index": "445", + "sources": [ + { + "reference_id": "9" + } + ] + } + ], + "name": "", + "answer_text": "", + "grounding_supports": [], + "references": [], + "related_questions": [], + "steps": [], + "answer_skipped_reasons": [], + "safety_ratings": [] + }, + "answer_query_token": "NMwKDAjPravVBhD9nMWHAxIkNmFjNGMzNWYtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi" + }, + { + "answer": { + "state": 4, + "answer_text": "* YouTube", + "name": "", + "citations": [], + "grounding_supports": [], + "references": [], + "related_questions": [], + "steps": [], + "answer_skipped_reasons": [], + "safety_ratings": [] + }, + "answer_query_token": "NMwKDAjPravVBhD9nMWHAxIkNmFjNGMzNWYtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi" + }, + { + "answer": { + "state": 4, + "answer_text": " ads: $29,243 million\n", + "name": "", + "citations": [], + "grounding_supports": [], + "references": [], + "related_questions": [], + "steps": [], + "answer_skipped_reasons": [], + "safety_ratings": [] + }, + "answer_query_token": "NMwKDAjPravVBhD9nMWHAxIkNmFjNGMzNWYtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi" + }, + { + "answer": { + "state": 4, + "citations": [ + { + "start_index": "446", + "end_index": "478", + "sources": [ + { + "reference_id": "9" + } + ] + } + ], + "name": "", + "answer_text": "", + "grounding_supports": [], + "references": [], + "related_questions": [], + "steps": [], + "answer_skipped_reasons": [], + "safety_ratings": [] + }, + "answer_query_token": "NMwKDAjPravVBhD9nMWHAxIkNmFjNGMzNWYtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi" + }, + { + "answer": { + "state": 4, + "answer_text": "* Google Network: $32,780 million\n", + "name": "", + "citations": [], + "grounding_supports": [], + "references": [], + "related_questions": [], + "steps": [], + "answer_skipped_reasons": [], + "safety_ratings": [] + }, + "answer_query_token": "NMwKDAjPravVBhD9nMWHAxIkNmFjNGMzNWYtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi" + }, + { + "answer": { + "state": 4, + "citations": [ + { + "start_index": "479", + "end_index": "514", + "sources": [ + { + "reference_id": "9" + } + ] + } + ], + "name": "", + "answer_text": "", + "grounding_supports": [], + "references": [], + "related_questions": [], + "steps": [], + "answer_skipped_reasons": [], + "safety_ratings": [] + }, + "answer_query_token": "NMwKDAjPravVBhD9nMWHAxIkNmFjNGMzNWYtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi" + }, + { + "answer": { + "state": 4, + "answer_text": "* Google advertising (total): $224,473 million\n", + "name": "", + "citations": [], + "grounding_supports": [], + "references": [], + "related_questions": [], + "steps": [], + "answer_skipped_reasons": [], + "safety_ratings": [] + }, + "answer_query_token": "NMwKDAjPravVBhD9nMWHAxIkNmFjNGMzNWYtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi" + }, + { + "answer": { + "state": 4, + "citations": [ + { + "start_index": "515", + "end_index": "563", + "sources": [ + { + "reference_id": "9" + } + ] + } + ], + "name": "", + "answer_text": "", + "grounding_supports": [], + "references": [], + "related_questions": [], + "steps": [], + "answer_skipped_reasons": [], + "safety_ratings": [] + }, + "answer_query_token": "NMwKDAjPravVBhD9nMWHAxIkNmFjNGMzNWYtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi" + }, + { + "answer": { + "state": 4, + "answer_text": "* Google other: $29,055 million", + "name": "", + "citations": [], + "grounding_supports": [], + "references": [], + "related_questions": [], + "steps": [], + "answer_skipped_reasons": [], + "safety_ratings": [] + }, + "answer_query_token": "NMwKDAjPravVBhD9nMWHAxIkNmFjNGMzNWYtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi" + }, + { + "answer": { + "state": 4, + "answer_text": "\n\n", + "name": "", + "citations": [], + "grounding_supports": [], + "references": [], + "related_questions": [], + "steps": [], + "answer_skipped_reasons": [], + "safety_ratings": [] + }, + "answer_query_token": "NMwKDAjPravVBhD9nMWHAxIkNmFjNGMzNWYtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi" + }, + { + "answer": { + "state": 4, + "citations": [ + { + "start_index": "564", + "end_index": "597", + "sources": [ + { + "reference_id": "9" + } + ] + } + ], + "name": "", + "answer_text": "", + "grounding_supports": [], + "references": [], + "related_questions": [], + "steps": [], + "answer_skipped_reasons": [], + "safety_ratings": [] + }, + "answer_query_token": "NMwKDAjPravVBhD9nMWHAxIkNmFjNGMzNWYtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi" + }, + { + "answer": { + "state": 4, + "answer_text": "**By Geography:**\n", + "name": "", + "citations": [], + "grounding_supports": [], + "references": [], + "related_questions": [], + "steps": [], + "answer_skipped_reasons": [], + "safety_ratings": [] + }, + "answer_query_token": "NMwKDAjPravVBhD9nMWHAxIkNmFjNGMzNWYtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi" + }, + { + "answer": { + "state": 4, + "answer_text": "* United States: $134,814 million (48%)\n", + "name": "", + "citations": [], + "grounding_supports": [], + "references": [], + "related_questions": [], + "steps": [], + "answer_skipped_reasons": [], + "safety_ratings": [] + }, + "answer_query_token": "NMwKDAjPravVBhD9nMWHAxIkNmFjNGMzNWYtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi" + }, + { + "answer": { + "state": 4, + "citations": [ + { + "start_index": "617", + "end_index": "658", + "sources": [ + { + "reference_id": "9" + } + ] + } + ], + "name": "", + "answer_text": "", + "grounding_supports": [], + "references": [], + "related_questions": [], + "steps": [], + "answer_skipped_reasons": [], + "safety_ratings": [] + }, + "answer_query_token": "NMwKDAjPravVBhD9nMWHAxIkNmFjNGMzNWYtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi" + }, + { + "answer": { + "state": 4, + "answer_text": "* EMEA (Europe, Middle East, and Africa): $82,062 million (29%)\n", + "name": "", + "citations": [], + "grounding_supports": [], + "references": [], + "related_questions": [], + "steps": [], + "answer_skipped_reasons": [], + "safety_ratings": [] + }, + "answer_query_token": "NMwKDAjPravVBhD9nMWHAxIkNmFjNGMzNWYtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi" + }, + { + "answer": { + "state": 4, + "citations": [ + { + "start_index": "659", + "end_index": "724", + "sources": [ + { + "reference_id": "9" + } + ] + } + ], + "name": "", + "answer_text": "", + "grounding_supports": [], + "references": [], + "related_questions": [], + "steps": [], + "answer_skipped_reasons": [], + "safety_ratings": [] + }, + "answer_query_token": "NMwKDAjPravVBhD9nMWHAxIkNmFjNGMzNWYtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi" + }, + { + "answer": { + "state": 4, + "answer_text": "* APAC (", + "name": "", + "citations": [], + "grounding_supports": [], + "references": [], + "related_questions": [], + "steps": [], + "answer_skipped_reasons": [], + "safety_ratings": [] + }, + "answer_query_token": "NMwKDAjPravVBhD9nMWHAxIkNmFjNGMzNWYtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi" + }, + { + "answer": { + "state": 4, + "answer_text": "Asia-Pacific): $47,024 million (16%)\n", + "name": "", + "citations": [], + "grounding_supports": [], + "references": [], + "related_questions": [], + "steps": [], + "answer_skipped_reasons": [], + "safety_ratings": [] + }, + "answer_query_token": "NMwKDAjPravVBhD9nMWHAxIkNmFjNGMzNWYtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi" + }, + { + "answer": { + "state": 4, + "citations": [ + { + "start_index": "725", + "end_index": "771", + "sources": [ + { + "reference_id": "9" + } + ] + } + ], + "name": "", + "answer_text": "", + "grounding_supports": [], + "references": [], + "related_questions": [], + "steps": [], + "answer_skipped_reasons": [], + "safety_ratings": [] + }, + "answer_query_token": "NMwKDAjPravVBhD9nMWHAxIkNmFjNGMzNWYtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi" + }, + { + "answer": { + "state": 4, + "answer_text": "* Other Americas: $16,976 million (6%)\n", + "name": "", + "citations": [], + "grounding_supports": [], + "references": [], + "related_questions": [], + "steps": [], + "answer_skipped_reasons": [], + "safety_ratings": [] + }, + "answer_query_token": "NMwKDAjPravVBhD9nMWHAxIkNmFjNGMzNWYtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi" + }, + { + "answer": { + "state": 4, + "citations": [ + { + "start_index": "772", + "end_index": "812", + "sources": [ + { + "reference_id": "9" + } + ] + } + ], + "name": "", + "answer_text": "", + "grounding_supports": [], + "references": [], + "related_questions": [], + "steps": [], + "answer_skipped_reasons": [], + "safety_ratings": [] + }, + "answer_query_token": "NMwKDAjPravVBhD9nMWHAxIkNmFjNGMzNWYtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi" + }, + { + "answer": { + "state": 4, + "answer_text": "* Hedging gains (losses): $1,960 million (1%)\n\n", + "name": "", + "citations": [], + "grounding_supports": [], + "references": [], + "related_questions": [], + "steps": [], + "answer_skipped_reasons": [], + "safety_ratings": [] + }, + "answer_query_token": "NMwKDAjPravVBhD9nMWHAxIkNmFjNGMzNWYtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi" + }, + { + "answer": { + "state": 4, + "citations": [ + { + "start_index": "813", + "end_index": "860", + "sources": [ + { + "reference_id": "9" + } + ] + } + ], + "name": "", + "answer_text": "", + "grounding_supports": [], + "references": [], + "related_questions": [], + "steps": [], + "answer_skipped_reasons": [], + "safety_ratings": [] + }, + "answer_query_token": "NMwKDAjPravVBhD9nMWHAxIkNmFjNGMzNWYtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi" + }, + { + "answer": { + "state": 4, + "answer_text": "Alphabet", + "name": "", + "citations": [], + "grounding_supports": [], + "references": [], + "related_questions": [], + "steps": [], + "answer_skipped_reasons": [], + "safety_ratings": [] + }, + "answer_query_token": "NMwKDAjPravVBhD9nMWHAxIkNmFjNGMzNWYtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi" + }, + { + "answer": { + "state": 4, + "answer_text": "'s CFO, Ruth Porat, stated that Q4 2022 consolidated revenues were $76 billion, up 1% year over year, or up 7% in constant currency, and the full year 2022 revenue was $283 billion, up 10%,", + "name": "", + "citations": [], + "grounding_supports": [], + "references": [], + "related_questions": [], + "steps": [], + "answer_skipped_reasons": [], + "safety_ratings": [] + }, + "answer_query_token": "NMwKDAjPravVBhD9nMWHAxIkNmFjNGMzNWYtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi" + }, + { + "answer": { + "state": 4, + "answer_text": " or up 14% in constant currency.\n\n", + "name": "", + "citations": [], + "grounding_supports": [], + "references": [], + "related_questions": [], + "steps": [], + "answer_skipped_reasons": [], + "safety_ratings": [] + }, + "answer_query_token": "NMwKDAjPravVBhD9nMWHAxIkNmFjNGMzNWYtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi" + }, + { + "answer": { + "state": 4, + "citations": [ + { + "start_index": "862", + "end_index": "1091", + "sources": [ + { + "reference_id": "2" + } + ] + } + ], + "name": "", + "answer_text": "", + "grounding_supports": [], + "references": [], + "related_questions": [], + "steps": [], + "answer_skipped_reasons": [], + "safety_ratings": [] + }, + "answer_query_token": "NMwKDAjPravVBhD9nMWHAxIkNmFjNGMzNWYtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi" + }, + { + "answer": { + "state": 4, + "answer_text": "For the first quarter of 2022, Alphabet reported revenues of $68,011 million, a 23% year-over-year growth. ", + "name": "", + "citations": [], + "grounding_supports": [], + "references": [], + "related_questions": [], + "steps": [], + "answer_skipped_reasons": [], + "safety_ratings": [] + }, + "answer_query_token": "NMwKDAjPravVBhD9nMWHAxIkNmFjNGMzNWYtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi" + }, + { + "answer": { + "state": 4, + "citations": [ + { + "start_index": "1093", + "end_index": "1199", + "sources": [ + { + "reference_id": "1" + } + ] + } + ], + "name": "", + "answer_text": "", + "grounding_supports": [], + "references": [], + "related_questions": [], + "steps": [], + "answer_skipped_reasons": [], + "safety_ratings": [] + }, + "answer_query_token": "NMwKDAjPravVBhD9nMWHAxIkNmFjNGMzNWYtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi" + }, + { + "answer": { + "state": 4, + "answer_text": "In the third quarter of 2022, revenues were", + "name": "", + "citations": [], + "grounding_supports": [], + "references": [], + "related_questions": [], + "steps": [], + "answer_skipped_reasons": [], + "safety_ratings": [] + }, + "answer_query_token": "NMwKDAjPravVBhD9nMWHAxIkNmFjNGMzNWYtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi" + }, + { + "answer": { + "state": 4, + "answer_text": " $69.1 billion, up 6% versus the prior year or up 11% on a constant currency basis.", + "name": "", + "citations": [], + "grounding_supports": [], + "references": [], + "related_questions": [], + "steps": [], + "answer_skipped_reasons": [], + "safety_ratings": [] + }, + "answer_query_token": "NMwKDAjPravVBhD9nMWHAxIkNmFjNGMzNWYtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi" + }, + { + "answer": { + "state": 4, + "citations": [ + { + "start_index": "1200", + "end_index": "1326", + "sources": [ + { + "reference_id": "5" + } + ] + } + ], + "name": "", + "answer_text": "", + "grounding_supports": [], + "references": [], + "related_questions": [], + "steps": [], + "answer_skipped_reasons": [], + "safety_ratings": [] + }, + "answer_query_token": "NMwKDAjPravVBhD9nMWHAxIkNmFjNGMzNWYtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi" + }, + { + "answer": { + "state": 3, + "answer_text": "Alphabet's total revenue in 2022 was $282,836 million. This represents a 10% increase year-over-year, or 14% in constant currency.\n\nHere's a breakdown of Alphabet's 2022 revenues by segment and type:\n\n**By Segment:**\n* Google Services: $253,528 million\n* Google Cloud: $26,280 million\n* Other Bets: $1,068 million\n* Hedging gains (losses): $1,960 million\n\n**By Type (within Google Services):**\n* Google Search & other: $162,450 million\n* YouTube ads: $29,243 million\n* Google Network: $32,780 million\n* Google advertising (total): $224,473 million\n* Google other: $29,055 million\n\n**By Geography:**\n* United States: $134,814 million (48%)\n* EMEA (Europe, Middle East, and Africa): $82,062 million (29%)\n* APAC (Asia-Pacific): $47,024 million (16%)\n* Other Americas: $16,976 million (6%)\n* Hedging gains (losses): $1,960 million (1%)\n\nAlphabet's CFO, Ruth Porat, stated that Q4 2022 consolidated revenues were $76 billion, up 1% year over year, or up 7% in constant currency, and the full year 2022 revenue was $283 billion, up 10%, or up 14% in constant currency.\n\nFor the first quarter of 2022, Alphabet reported revenues of $68,011 million, a 23% year-over-year growth. In the third quarter of 2022, revenues were $69.1 billion, up 6% versus the prior year or up 11% on a constant currency basis.", + "citations": [ + { + "end_index": "54", + "sources": [ + { + "reference_id": "0" + }, + { + "reference_id": "4" + }, + { + "reference_id": "8" + }, + { + "reference_id": "9" + } + ], + "start_index": "0" + }, + { + "start_index": "55", + "end_index": "130", + "sources": [ + { + "reference_id": "2" + } + ] + }, + { + "start_index": "217", + "end_index": "254", + "sources": [ + { + "reference_id": "8" + }, + { + "reference_id": "9" + } + ] + }, + { + "start_index": "255", + "end_index": "288", + "sources": [ + { + "reference_id": "8" + }, + { + "reference_id": "9" + } + ] + }, + { + "start_index": "289", + "end_index": "319", + "sources": [ + { + "reference_id": "8" + }, + { + "reference_id": "9" + } + ] + }, + { + "start_index": "320", + "end_index": "362", + "sources": [ + { + "reference_id": "8" + }, + { + "reference_id": "9" + } + ] + }, + { + "start_index": "402", + "end_index": "445", + "sources": [ + { + "reference_id": "9" + } + ] + }, + { + "start_index": "446", + "end_index": "478", + "sources": [ + { + "reference_id": "9" + } + ] + }, + { + "start_index": "479", + "end_index": "514", + "sources": [ + { + "reference_id": "9" + } + ] + }, + { + "start_index": "515", + "end_index": "563", + "sources": [ + { + "reference_id": "9" + } + ] + }, + { + "start_index": "564", + "end_index": "597", + "sources": [ + { + "reference_id": "9" + } + ] + }, + { + "start_index": "617", + "end_index": "658", + "sources": [ + { + "reference_id": "9" + } + ] + }, + { + "start_index": "659", + "end_index": "724", + "sources": [ + { + "reference_id": "9" + } + ] + }, + { + "start_index": "725", + "end_index": "771", + "sources": [ + { + "reference_id": "9" + } + ] + }, + { + "start_index": "772", + "end_index": "812", + "sources": [ + { + "reference_id": "9" + } + ] + }, + { + "start_index": "813", + "end_index": "860", + "sources": [ + { + "reference_id": "9" + } + ] + }, + { + "start_index": "862", + "end_index": "1091", + "sources": [ + { + "reference_id": "2" + } + ] + }, + { + "start_index": "1093", + "end_index": "1199", + "sources": [ + { + "reference_id": "1" + } + ] + }, + { + "start_index": "1200", + "end_index": "1326", + "sources": [ + { + "reference_id": "5" + } + ] + } + ], + "references": [ + { + "chunk_info": { + "content": "ALPHABET INC. ANNUAL REPORT 45 PART II\nITEM\u00a08\u00a0\u00a0FINANCIAL STATEMENTS AND SUPPLEMENTARY DATA Alphabet Inc. CONSOLIDATED STATEMENTS OF INCOME (In millions, except per share amounts) Year Ended December 31,\n2020 2021 2022 Revenues $ 182,527 $ 257,637 $ 282,836 Costs and expenses:\nCost of revenues 84,732 110,939 126,203 Research and development 27,573 31,562 39,500 Sales and marketing 17,946 22,912 26,567 General and administrative 11,052 13,510 15,724 Total costs and expenses 141,303 178,923 207,994 Income from operations 41,224 78,714 74,842 Other income (expense), net 6,858 12,020 (3,514) Income before income taxes 48,082 90,734 71,328 Provision for income taxes 7,813 14,701 11,356 Net income $ 40,269 $ 76,033 $ 59,972 Basic net income per share of Class A, Class B, and Class C stock $ 2.96 $ 5.69 $ 4.59 Diluted net income per share of Class A, Class B, and Class C stock $ 2.93 $ 5.61 $ 4.56 See accompanying notes. ", + "document_metadata": { + "document": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/276cee4c4086600303bc561483691f9a", + "uri": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022_alphabet_annual_report.pdf", + "title": "2022_alphabet_annual_report", + "page_identifier": "" + }, + "chunk": "" + } + }, + { + "chunk_info": { + "content": "Alphabet Announces First Quarter 2022 Results\nMOUNTAIN VIEW, Calif. \u2013 April 26, 2022 \u2013 Alphabet Inc. (NASDAQ: GOOG, GOOGL) today announced financial\nresults for the quarter ended March 31, 2022. Sundar Pichai, CEO of Alphabet and Google, said: \u201cQ1 saw strong growth in Search and Cloud, in particular, which\nare both helping people and businesses as the digital transformation continues. We\u2019ll keep investing in great\nproducts and services, and creating opportunities for partners and local communities around the world.\u201d Ruth Porat, CFO of Alphabet and Google, said: \u201cWe are pleased with Q1 revenue growth of 23% year over year. We\ncontinue to make considered investments in Capex, R&D and talent to support long-term value creation for all\nstakeholders.\u201d Q1 2022 financial highlights\nThe following table summarizes our consolidated financial results for the quarters ended March 31, 2021 and 2022\n(in millions, except for per share information and percentages; unaudited). Quarter Ended March 31,\n2021 2022 Revenues $ 55,314 $ 68,011 Change in revenues year over year 34 % 23 % Change in constant currency revenues year over year(1) 32 % 26 % Operating income $ 16,437 $ 20,094 Operating margin 30 % 30 % Other income (expense), net $ 4,846 $ (1,160) Net income $ 17,930 $ 16,436 ", + "document_metadata": { + "document": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/c201efed2be20c6a4116611ea16be9cd", + "uri": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q1_alphabet_earnings_release.pdf", + "title": "GOOG Exhibit 99.1 Q1 2022", + "page_identifier": "" + }, + "chunk": "" + } + }, + { + "chunk_info": { + "content": "Alphabet Announces Fourth Quarter and Fiscal Year 2022 Results\nMOUNTAIN VIEW, Calif. \u2013 February 2, 2023 \u2013 Alphabet Inc. (NASDAQ: GOOG, GOOGL) today announced financial\nresults for the quarter and fiscal year ended December 31, 2022. Sundar Pichai, CEO of Alphabet and Google, said: \u201cOur long-term investments in deep computer science make us\nextremely well-positioned as AI reaches an inflection point, and I\u2019m excited by the AI-driven leaps we\u2019re about to unveil\nin Search and beyond. There\u2019s also great momentum in Cloud, YouTube subscriptions, and our Pixel devices. We\u2019re\non an important journey to re-engineer our cost structure in a durable way and to build financially sustainable, vibrant,\ngrowing businesses across Alphabet.\u201d Ruth Porat, CFO of Alphabet and Google, said: \u201cOur Q4 consolidated revenues were $76 billion, up 1% year over year,\nor up 7% in constant currency, and $283 billion for the full year 2022, up 10%, or up 14% in constant currency. We\nhave significant work underway to improve all aspects of our cost structure, in support of our investments in our\nhighest growth priorities to deliver long-term, profitable growth.\u201d ", + "document_metadata": { + "document": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/513da7d1b9f7739087606aadc626f2dc", + "uri": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q4_alphabet_earnings_release.pdf", + "title": "GOOG Exhibit 99.1 Q4 2022", + "page_identifier": "" + }, + "chunk": "" + } + }, + { + "chunk_info": { + "content": "Alphabet Announces Third Quarter 2023 Results\nMOUNTAIN VIEW, Calif. \u2013 October 24, 2023 \u2013 Alphabet Inc. (NASDAQ: GOOG, GOOGL) today announced\nfinancial results for the quarter ended September 30, 2023. Sundar Pichai, CEO, said: \u201cI\u2019m pleased with our financial results and our product momentum this quarter, with AI\ndriven innovations across Search, YouTube, Cloud, our Pixel devices and more. We\u2019re continuing to focus on\nmaking AI more helpful for everyone; there\u2019s exciting progress and lots more to come.\u201d Ruth Porat, President and Chief Investment Officer; CFO said: \u201cThe fundamental strength of our business was\napparent again in Q3, with $77 billion in revenue, up 11% year over year, driven by meaningful growth in Search\nand YouTube, and momentum in Cloud. We continue to focus on judicious capital allocation to deliver sustainable\nfinancial value.\u201d Q3 2023 Financial Highlights (unaudited)\nThe following table summarizes our consolidated financial results for the quarters ended September 30, 2022 and\n2023 (in millions, except for per share information and percentages). Quarter Ended September 30,\n2022 2023 Revenues $ 69,092 $ 76,693 Change in revenues year over year 6 % 11 % Change in constant currency revenues year over year(1) 11 % 11 % Operating income $ 17,135 $ 21,343 Operating margin 25 % ", + "document_metadata": { + "document": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/4c7389ffaf06b73e6e380eac7a39ea17", + "uri": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2023q3-alphabet-earnings-release.pdf", + "title": "GOOG Exhibit 99.1 Q3 2023", + "page_identifier": "" + }, + "chunk": "" + } + }, + { + "chunk_info": { + "content": "Alphabet Inc. Consolidated Statements of Income Year Ended December 31, (in millions, except per share amounts) 2021 2022 2023 Revenues $ 257,637 $ 282,836 $ 307,394 Costs and expenses:\nCost of revenues 110,939 126,203 133,332 Research and development 31,562 39,500 45,427 Sales and marketing 22,912 26,567 27,917 General and administrative 13,510 15,724 16,425 Total costs and expenses 178,923 207,994 223,101 Income from operations 78,714 74,842 84,293 Other income (expense), net 12,020 (3,514) 1,424 Income before income taxes 90,734 71,328 85,717 Provision for income taxes 14,701 11,356 11,922 Net income $ 76,033 $ 59,972 $ 73,795 Basic net income per share of Class A, Class B, and Class C stock $ 5.69 $ 4.59 $ 5.84 Diluted net income per share of Class A, Class B, and Class C stock $ 5.61 $ 4.56 $ 5.80 See accompanying notes. 50 Alphabet 2023 Annual Report Part I Part II Part III Part IV ", + "document_metadata": { + "document": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/3c617f6f4b023305d82916f0093ca13d", + "uri": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/goog023-alphabet-2023-annual-report-web-1.pdf", + "title": "goog023-alphabet-2023-annual-report-web-1", + "page_identifier": "" + }, + "chunk": "" + } + }, + { + "chunk_info": { + "content": "Alphabet Announces Third Quarter 2022 Results\nMOUNTAIN VIEW, Calif. \u2013 October 25, 2022 \u2013 Alphabet Inc. (NASDAQ: GOOG, GOOGL) today announced\nfinancial results for the quarter ended September 30, 2022. Sundar Pichai, CEO of Alphabet and Google, said: \u201cWe\u2019re sharpening our focus on a clear set of product and\nbusiness priorities. Product announcements we\u2019ve made in just the past month alone have shown that very clearly,\nincluding significant improvements to both Search and Cloud, powered by AI, and new ways to monetize YouTube\nShorts. We are focused on both investing responsibly for the long term and being responsive to the economic\nenvironment.\u201d Ruth Porat, CFO of Alphabet and Google, said: \u201cOur third quarter revenues were $69.1 billion, up 6% versus last\nyear or up 11% on a constant currency basis. Financial results for the third quarter reflect healthy fundamental\ngrowth in Search and momentum in Cloud, while affected by foreign exchange. We\u2019re working to realign resources\nto fuel our highest growth priorities.\u201d Q3 2022 financial highlights\nThe following table summarizes our consolidated financial results for the quarters ended September 30, 2021 and\n2022 (in millions, except for per share information and percentages; unaudited). Quarter Ended September 30,\n2021 2022 Revenues $ 65,118 $ 69,092 Change in revenues year over year 41 % ", + "document_metadata": { + "document": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/2eab9446e934f432e3df7808af857300", + "uri": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q3_alphabet_earnings_release.pdf", + "title": "GOOG Exhibit 99.1 Q3 2022", + "page_identifier": "" + }, + "chunk": "" + } + }, + { + "chunk_info": { + "content": "Alphabet Inc. CONSOLIDATED STATEMENTS OF CASH FLOWS\n(In millions, unaudited)\nQuarter Ended September 30, Year to Date September 30, 2021 2022 2021 2022 Operating activities\nNet income $ 18,936 $ 13,910 $ 55,391 $ 46,348 Adjustments:\nDepreciation and impairment of property and\nequipment 3,085 3,933 8,340 11,222 Amortization and impairment of intangible assets 219 113 662 505 Stock-based compensation expense 3,874 4,976 11,422 14,262 Deferred income taxes (1,287) (1,920) 192 (6,157) (Gain) loss on debt and equity securities, net (2,158) 1,378 (9,792) 3,856 Other 64 167 (199) 369 Changes in assets and liabilities, net of effects of\nacquisitions:\nAccounts receivable (2,409) (97) (3,276) 2,298 Income taxes, net 3,041 (609) 2,744 (862) Other assets (1,255) (2,647) (1,447) (4,268) Accounts payable 238 1,907 (874) 735 Accrued expenses and other liabilities 2,562 2,210 2,763 491 Accrued revenue share 357 (80) 386 (1,022) Deferred revenue 272 112 406 104 Net cash provided by operating activities 25,539 23,353 66,718 67,881 Investing activities\nPurchases of property and equipment (6,819) (7,276) (18,257) (23,890) Purchases of marketable securities (34,497) (17,054) (95,106) (67,253) Maturities and sales of marketable securities 31,459 28,713 92,126 84,087 Purchases of non-marketable securities ", + "document_metadata": { + "document": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/2eab9446e934f432e3df7808af857300", + "uri": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q3_alphabet_earnings_release.pdf", + "title": "GOOG Exhibit 99.1 Q3 2022", + "page_identifier": "" + }, + "chunk": "" + } + }, + { + "chunk_info": { + "content": "Alphabet Inc. CONSOLIDATED STATEMENTS OF CASH FLOWS\n(In millions, unaudited)\nQuarter Ended June 30, Year To Date June 30,\n2021 2022 2021 2022 Operating activities\nNet income $ 18,525 $ 16,002 $ 36,455 $ 32,438 Adjustments:\nDepreciation and impairment of property and equipment 2,730 3,698 5,255 7,289 Amortization and impairment of intangible assets 215 201 443 392 Stock-based compensation expense 3,803 4,782 7,548 9,286 Deferred income taxes 379 (2,147) 1,479 (4,237) (Gain) loss on debt and equity securities, net (2,883) 1,041 (7,634) 2,478 Other (8) 62 (263) 202 Changes in assets and liabilities, net of effects of acquisitions:\nAccounts receivable (3,661) (1,969) (867) 2,395 Income taxes, net (1,082) (4,073) (297) (253) Other assets (199) (845) (192) (1,621) Accounts payable (130) 1,201 (1,112) (1,172) Accrued expenses and other liabilities 3,731 1,497 201 (1,719) Accrued revenue share 473 (114) 29 (942) Deferred revenue (3) 86 134 (8) Net cash provided by operating activities 21,890 19,422 41,179 44,528 Investing activities\nPurchases of property and equipment (5,496) (6,828) (11,438) (16,614) Purchases of marketable securities (24,183) (21,737) (60,609) (50,199) ", + "document_metadata": { + "document": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/376dab8cf7b3807296f2bf1c6228e078", + "uri": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q2_alphabet_earnings_release.pdf", + "title": "GOOG Exhibit 99.1 Q2 2022", + "page_identifier": "" + }, + "chunk": "" + } + }, + { + "chunk_info": { + "content": "The associated costs,\nincluding depreciation and impairment, are allocated to operating segments as a service cost generally based on usage or headcount. Unallocated corporate costs primarily include corporate initiatives, corporate shared costs, such as finance and legal, including certain\nfines and settlements, as well as costs associated with certain shared R&D activities. Additionally, hedging gains (losses) related to\nrevenue are included in corporate costs. As AI is critical to delivering our mission of bringing our breakthrough innovations into the real world, beginning in January 2023, we\nwill update our segment reporting relating to certain of Alphabet\u2019s AI activities. DeepMind, previously reported within Other Bets, will\nbe reported as part of Alphabet\u2019s corporate costs, reflecting its increasing collaboration with Google Services, Google Cloud, and\nOther Bets. Prior periods will be recast to conform to the revised presentation. Our operating segments are not evaluated using asset information. The following table presents information about our segments (in millions): Year Ended December 31,\n2020 2021 2022 Revenues:\nGoogle Services $ 168,635 $ 237,529 $ 253,528 Google Cloud 13,059 19,206 26,280 Other Bets 657 753 1,068 Hedging gains (losses) 176 149 1,960 Total revenues $ 182,527 $ 257,637 $ 282,836 Operating income (loss):\nGoogle Services $ 54,606 $ 91,855 $ 86,572 Google Cloud (5,607) (3,099) (2,968) Other Bets ", + "document_metadata": { + "document": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/276cee4c4086600303bc561483691f9a", + "uri": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022_alphabet_annual_report.pdf", + "title": "2022_alphabet_annual_report", + "page_identifier": "" + }, + "chunk": "" + } + }, + { + "chunk_info": { + "content": "56 ALPHABET INC. ANNUAL REPORT PART II\nITEM\u00a08\u00a0\u00a0FINANCIAL STATEMENTS AND SUPPLEMENTARY DATA Note 2. Revenues\nDisaggregated Revenues\nThe following table presents revenues disaggregated by type (in millions): Year Ended December 31,\n2020 2021 2022 Google Search & other $ 104,062 $ 148,951 $ 162,450 YouTube ads 19,772 28,845 29,243 Google Network 23,090 31,701 32,780 Google advertising 146,924 209,497 224,473 Google other 21,711 28,032 29,055 Google Services total 168,635 237,529 253,528 Google Cloud 13,059 19,206 26,280 Other Bets 657 753 1,068 Hedging gains (losses) 176 149 1,960 Total revenues $ 182,527 $ 257,637 $ 282,836\nNo individual customer or groups of affiliated customers represented more than 10% of our revenues in 2020, 2021, or 2022. The following table presents revenues disaggregated by geography, based on the addresses of our customers (in millions):\nYear Ended December 31, 2020 2021 2022 United States $ 85,014 47 % $ 117,854 46 % $ 134,814 48% EMEA(1) 55,370 30 79,107 31 82,062 29 APAC(1) 32,550 18 46,123 18 47,024 16 Other Americas(1) 9,417 5 14,404 5 16,976 6 Hedging gains (losses) 176 0 149 0 1,960 1 Total revenues $ 182,527 100% $ 257,637 100 % $ 282,836 100% ", + "document_metadata": { + "document": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/276cee4c4086600303bc561483691f9a", + "uri": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022_alphabet_annual_report.pdf", + "title": "2022_alphabet_annual_report", + "page_identifier": "" + }, + "chunk": "" + } + } + ], + "steps": [ + { + "state": 3, + "description": "Rephrase the query and search.", + "actions": [ + { + "search_action": { + "query": "What was Alphabet's revenue in 2022?" + }, + "observation": { + "search_results": [ + { + "document": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/513da7d1b9f7739087606aadc626f2dc", + "uri": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q4_alphabet_earnings_release.pdf", + "title": "GOOG Exhibit 99.1 Q4 2022", + "snippet_info": [ + { + "snippet": "... Alphabet.\u201d Ruth Porat, CFO of Alphabet and Google, said: \u201cOur Q4 consolidated revenues were $76 billion, up 1% year over year, or up 7% in constant currency ...", + "snippet_status": "SUCCESS" + } + ], + "chunk_info": [] + }, + { + "document": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/c201efed2be20c6a4116611ea16be9cd", + "uri": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q1_alphabet_earnings_release.pdf", + "title": "GOOG Exhibit 99.1 Q1 2022", + "snippet_info": [ + { + "snippet": "\u2013 April 26, 2022 \u2013 Alphabet Inc. (NASDAQ: GOOG ... revenue growth of 23% year over year. ... Quarter Ended March 31, 2021 2022 Revenues $ 55,314 $ 68,011 Change in ...", + "snippet_status": "SUCCESS" + } + ], + "chunk_info": [] + }, + { + "document": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/376dab8cf7b3807296f2bf1c6228e078", + "uri": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q2_alphabet_earnings_release.pdf", + "title": "GOOG Exhibit 99.1 Q2 2022", + "snippet_info": [ + { + "snippet": "\u2013 July 26, 2022 \u2013 Alphabet Inc. (NASDAQ: GOOG ... Quarter Ended June 30, 2021 2022 Revenues $ 61,880 $ 69,685 Change in revenues ... revenue growth rates by 3.7%.", + "snippet_status": "SUCCESS" + } + ], + "chunk_info": [] + }, + { + "document": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/4a4eabb0cefc954375df0bd9eec85c96", + "uri": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2023q4-alphabet-earnings-release.pdf", + "title": "2023q4-alphabet-earnings-release", + "snippet_info": [ + { + "snippet": "... Revenues As Reported Less Hedging Effect Less FX Effect Constant Currency 2022 2023 Revenues United States $ 134,814 $ 146,286 $ 0 $ 146,286 9 % 0 % 9 ...", + "snippet_status": "SUCCESS" + } + ], + "chunk_info": [] + }, + { + "document": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/2eab9446e934f432e3df7808af857300", + "uri": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q3_alphabet_earnings_release.pdf", + "title": "GOOG Exhibit 99.1 Q3 2022", + "snippet_info": [ + { + "snippet": "Alphabet Announces Third Quarter 2022 Results MOUNTAIN VIEW, Calif. \u2013 October 25, 2022 \u2013 Alphabet Inc. ... Quarter Ended September 30, 2021 2022 Revenues $ 65,118 ...", + "snippet_status": "SUCCESS" + } + ], + "chunk_info": [] + }, + { + "document": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/276cee4c4086600303bc561483691f9a", + "uri": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022_alphabet_annual_report.pdf", + "title": "2022_alphabet_annual_report", + "snippet_info": [ + { + "snippet": "... Alphabet's AI activities. DeepMind, previously reported within Other Bets, will be reported as part of Alphabet's corporate costs, reflecting its increasing ...", + "snippet_status": "SUCCESS" + } + ], + "chunk_info": [] + }, + { + "document": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/3c617f6f4b023305d82916f0093ca13d", + "uri": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/goog023-alphabet-2023-annual-report-web-1.pdf", + "title": "goog023-alphabet-2023-annual-report-web-1", + "snippet_info": [ + { + "snippet": "... 2022 2023 Revenues $ 257,637 $ 282,836 $ 307,394 Costs and expenses: Cost of revenues 110,939 126,203 133,332 Research and development 31,562 39,500 45,427 ...", + "snippet_status": "SUCCESS" + } + ], + "chunk_info": [] + }, + { + "document": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/404cb139d857b704a13ee3d8e0c6c5fb", + "uri": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2021Q4_alphabet_earnings_release.pdf", + "title": "GOOG Exhibit 99.1 Q4 2021", + "snippet_info": [ + { + "snippet": "Alphabet Announces Fourth Quarter and Fiscal Year 2021 Results MOUNTAIN VIEW, Calif. \u2013 February 1, 2022 \u2013 Alphabet Inc. ... revenue growth from Google Cloud.", + "snippet_status": "SUCCESS" + } + ], + "chunk_info": [] + }, + { + "document": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/e0ac436b3f82b7df8fc94580f3a84b0a", + "uri": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2023q1-alphabet-earnings-release.pdf", + "title": "GOOG Exhibit 99.1 Q1 2023", + "snippet_info": [ + { + "snippet": "Ruth Porat, CFO of Alphabet and Google, said: \u201cResilience in Search and momentum in Cloud resulted in Q1 consolidated revenues of $69.8 billion, up 3% year over ...", + "snippet_status": "SUCCESS" + } + ], + "chunk_info": [] + }, + { + "document": "projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/4c7389ffaf06b73e6e380eac7a39ea17", + "uri": "gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2023q3-alphabet-earnings-release.pdf", + "title": "GOOG Exhibit 99.1 Q3 2023", + "snippet_info": [ + { + "snippet": "Alphabet Announces Third Quarter 2023 Results MOUNTAIN VIEW, Calif. ... revenue, up 11% year over year, driven by ... 2022 and 2023 (in millions, except for per ...", + "snippet_status": "SUCCESS" + } + ], + "chunk_info": [] + } + ] + } + } + ], + "thought": "" + } + ], + "name": "", + "grounding_supports": [], + "related_questions": [], + "answer_skipped_reasons": [], + "safety_ratings": [] + }, + "answer_query_token": "NMwKDAjPravVBhD9nMWHAxIkNmFjNGMzNWYtMDAwMC0yOWQ0LTg1Y2EtMzQzOTE2MTY3MmZi" + } + ] +} diff --git a/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_async_provider_error.json b/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_async_provider_error.json new file mode 100644 index 000000000..75731c8b6 --- /dev/null +++ b/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_async_provider_error.json @@ -0,0 +1,26 @@ +{ + "method": "rank", + "requests": [ + { + "ranking_config": "projects/sdk-dev-508013/locations/global/rankingConfigs/default_ranking_config", + "query": "test", + "records": [ + { + "id": "empty", + "title": "", + "content": "", + "score": 0.0 + } + ], + "model": "", + "top_n": 0, + "ignore_record_details_in_response": false, + "user_labels": {} + } + ], + "responses": [], + "error": { + "type": "InvalidArgument", + "message": "RankRequest.records(0).title() and RankRequest.records(0).content() are empty at the same time." + } +} diff --git a/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_async_stream_provider_error.json b/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_async_stream_provider_error.json new file mode 100644 index 000000000..cdf1e4aed --- /dev/null +++ b/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_async_stream_provider_error.json @@ -0,0 +1,18 @@ +{ + "method": "stream_answer_query", + "requests": [ + { + "serving_config": "projects/sdk-dev-508013/locations/global/collections/default_collection/engines/braintrust-sdk-test_1789580029521/servingConfigs/default_search", + "session": "", + "asynchronous_mode": false, + "user_pseudo_id": "", + "user_labels": {} + } + ], + "responses": [], + "error": { + "type": "InternalServerError", + "message": "Internal error encountered. Please try again. If the issue persists, please contact our support team." + }, + "error_at": "call" +} diff --git a/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_check_grounding.yaml b/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_check_grounding.yaml new file mode 100644 index 000000000..5cafd2ec9 --- /dev/null +++ b/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_check_grounding.yaml @@ -0,0 +1,60 @@ +interactions: +- request: + body: "{\n \"answerCandidate\": \"Braintrust evaluates AI applications.\",\n + \ \"facts\": [\n {\n \"factText\": \"Braintrust is a platform for evaluating + AI applications.\"\n }\n ]\n}" + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '171' + Content-Type: + - application/json + User-Agent: + - python-requests/2.34.2 + x-goog-api-client: + - gl-python/3.14.7 grpc/1.84.0 gax/2.37.0 gapic/0.20.3 pb/7.36.1 cred-type/u + x-goog-request-params: + - grounding_config=projects/sdk-dev-508013/locations/global/groundingConfigs/default_grounding_config + method: POST + uri: https://discoveryengine.googleapis.com/v1/projects/sdk-dev-508013/locations/global/groundingConfigs/default_grounding_config:check?%24alt=json%3Benum-encoding%3Dint + response: + body: + string: "{\n \"supportScore\": 0.99110764,\n \"citedChunks\": [\n {\n \"chunkText\": + \"Braintrust is a platform for evaluating AI applications.\",\n \"source\": + \"0\"\n }\n ],\n \"claims\": [\n {\n \"startPos\": 0,\n \"endPos\": + 37,\n \"claimText\": \"Braintrust evaluates AI applications.\",\n \"citationIndices\": + [\n 0\n ],\n \"groundingCheckRequired\": true\n }\n ]\n}\n" + headers: + Alt-Svc: + - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 + Content-Type: + - application/json; charset=UTF-8 + Date: + - Wed, 16 Sep 2026 17:38:47 GMT + Server: + - ESF + Server-Timing: + - gfet4t7; dur=189 + Transfer-Encoding: + - chunked + Vary: + - Origin + - X-Origin + - Referer + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-XSS-Protection: + - '0' + content-length: + - '383' + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_converse_conversation.yaml b/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_converse_conversation.yaml new file mode 100644 index 000000000..b99b5c197 --- /dev/null +++ b/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_converse_conversation.yaml @@ -0,0 +1,514 @@ +interactions: +- request: + body: "{\n \"query\": {\n \"input\": \"What was Alphabet's revenue in 2022?\"\n + \ },\n \"servingConfig\": \"projects/sdk-dev-508013/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/servingConfigs/default_search\",\n + \ \"summarySpec\": {\n \"summaryResultCount\": 3,\n \"includeCitations\": + true\n }\n}" + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '328' + Content-Type: + - application/json + User-Agent: + - python-requests/2.34.2 + x-goog-api-client: + - gl-python/3.14.7 grpc/1.84.0 gax/2.37.0 gapic/0.20.3 pb/7.36.1 cred-type/u + x-goog-request-params: + - name=projects/sdk-dev-508013/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/conversations/- + method: POST + uri: https://discoveryengine.googleapis.com/v1/projects/sdk-dev-508013/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/conversations/-:converse?%24alt=json%3Benum-encoding%3Dint + response: + body: + string: "{\n \"reply\": {\n \"summary\": {\n \"summaryText\": \"Alphabet's + consolidated revenues for the year ended December 31, 2022, were $282,836 + million [1].\\n\\nLooking at the quarterly revenues for 2022:\\n* Q1 2022 + revenues were $68,011 million [2]. This was a 23% change in revenues year + over year, or 26% in constant currency [2].\\n* Q2 2022 revenues were $69,685 + million [3]. This represented a 13% change in revenues year over year, or + 16% in constant currency [3].\\n* Q4 2022 consolidated revenues were $76,048 + million, which was up 1% year over year, or up 7% in constant currency [1].\\n\\nFor + the year-to-date period ending June 30, 2022, Alphabet's revenues were $137,696 + million [3].\\n\\nSpecific revenue breakdowns for Q4 2022 include:\\n* Google + Search & other: $42,604 million [1].\\n* YouTube ads: $7,963 million [1].\\n* + \ Google Network: $8,475 million [1].\\n* Google advertising: $59,042 million + [1].\",\n \"summaryWithMetadata\": {\n \"summary\": \"Alphabet's + consolidated revenues for the year ended December 31, 2022, were $282,836 + million.\\n\\nLooking at the quarterly revenues for 2022:\\n* Q1 2022 revenues + were $68,011 million. This was a 23% change in revenues year over year, or + 26% in constant currency.\\n* Q2 2022 revenues were $69,685 million. This + represented a 13% change in revenues year over year, or 16% in constant currency.\\n* + \ Q4 2022 consolidated revenues were $76,048 million, which was up 1% year + over year, or up 7% in constant currency.\\n\\nFor the year-to-date period + ending June 30, 2022, Alphabet's revenues were $137,696 million.\\n\\nSpecific + revenue breakdowns for Q4 2022 include:\\n* Google Search & other: $42,604 + million.\\n* YouTube ads: $7,963 million.\\n* Google Network: $8,475 million.\\n* + \ Google advertising: $59,042 million.\",\n \"citationMetadata\": + {\n \"citations\": [\n {\n \"endIndex\": + \"93\",\n \"sources\": [\n {}\n ]\n + \ },\n {\n \"startIndex\": \"139\",\n \"endIndex\": + \"181\",\n \"sources\": [\n {\n \"referenceIndex\": + \"1\"\n }\n ]\n },\n {\n + \ \"startIndex\": \"182\",\n \"endIndex\": \"260\",\n + \ \"sources\": [\n {\n \"referenceIndex\": + \"1\"\n }\n ]\n },\n {\n + \ \"startIndex\": \"261\",\n \"endIndex\": \"303\",\n + \ \"sources\": [\n {\n \"referenceIndex\": + \"2\"\n }\n ]\n },\n {\n + \ \"startIndex\": \"304\",\n \"endIndex\": \"390\",\n + \ \"sources\": [\n {\n \"referenceIndex\": + \"2\"\n }\n ]\n },\n {\n + \ \"startIndex\": \"391\",\n \"endIndex\": \"509\",\n + \ \"sources\": [\n {}\n ]\n },\n + \ {\n \"startIndex\": \"511\",\n \"endIndex\": + \"603\",\n \"sources\": [\n {\n \"referenceIndex\": + \"2\"\n }\n ]\n },\n {\n + \ \"startIndex\": \"654\",\n \"endIndex\": \"697\",\n + \ \"sources\": [\n {}\n ]\n },\n + \ {\n \"startIndex\": \"698\",\n \"endIndex\": + \"730\",\n \"sources\": [\n {}\n ]\n + \ },\n {\n \"startIndex\": \"731\",\n \"endIndex\": + \"766\",\n \"sources\": [\n {}\n ]\n + \ },\n {\n \"startIndex\": \"767\",\n \"endIndex\": + \"807\",\n \"sources\": [\n {}\n ]\n + \ }\n ]\n },\n \"references\": [\n {\n + \ \"title\": \"GOOG Exhibit 99.1 Q4 2022\",\n \"document\": + \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/513da7d1b9f7739087606aadc626f2dc\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q4_alphabet_earnings_release.pdf\"\n + \ },\n {\n \"title\": \"GOOG Exhibit 99.1 Q1 2022\",\n + \ \"document\": \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/c201efed2be20c6a4116611ea16be9cd\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q1_alphabet_earnings_release.pdf\"\n + \ },\n {\n \"title\": \"GOOG Exhibit 99.1 Q2 2022\",\n + \ \"document\": \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/376dab8cf7b3807296f2bf1c6228e078\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q2_alphabet_earnings_release.pdf\"\n + \ }\n ]\n }\n }\n },\n \"conversation\": {\n \"name\": + \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/conversations/16515746392651966416\",\n + \ \"state\": 1,\n \"userPseudoId\": \"16515746392651966416\",\n \"messages\": + [\n {\n \"userInput\": {\n \"input\": \"What was Alphabet's + revenue in 2022?\"\n }\n },\n {\n \"reply\": {\n \"summary\": + {\n \"summaryText\": \"Alphabet's consolidated revenues for the + year ended December 31, 2022, were $282,836 million [1].\\n\\nLooking at the + quarterly revenues for 2022:\\n* Q1 2022 revenues were $68,011 million [2]. + This was a 23% change in revenues year over year, or 26% in constant currency + [2].\\n* Q2 2022 revenues were $69,685 million [3]. This represented a 13% + change in revenues year over year, or 16% in constant currency [3].\\n* Q4 + 2022 consolidated revenues were $76,048 million, which was up 1% year over + year, or up 7% in constant currency [1].\\n\\nFor the year-to-date period + ending June 30, 2022, Alphabet's revenues were $137,696 million [3].\\n\\nSpecific + revenue breakdowns for Q4 2022 include:\\n* Google Search & other: $42,604 + million [1].\\n* YouTube ads: $7,963 million [1].\\n* Google Network: + $8,475 million [1].\\n* Google advertising: $59,042 million [1].\",\n \"summaryWithMetadata\": + {\n \"summary\": \"Alphabet's consolidated revenues for the year + ended December 31, 2022, were $282,836 million.\\n\\nLooking at the quarterly + revenues for 2022:\\n* Q1 2022 revenues were $68,011 million. This was a + 23% change in revenues year over year, or 26% in constant currency.\\n* Q2 + 2022 revenues were $69,685 million. This represented a 13% change in revenues + year over year, or 16% in constant currency.\\n* Q4 2022 consolidated revenues + were $76,048 million, which was up 1% year over year, or up 7% in constant + currency.\\n\\nFor the year-to-date period ending June 30, 2022, Alphabet's + revenues were $137,696 million.\\n\\nSpecific revenue breakdowns for Q4 2022 + include:\\n* Google Search & other: $42,604 million.\\n* YouTube ads: + $7,963 million.\\n* Google Network: $8,475 million.\\n* Google advertising: + $59,042 million.\",\n \"citationMetadata\": {\n \"citations\": + [\n {\n \"endIndex\": \"93\",\n \"sources\": + [\n {}\n ]\n },\n + \ {\n \"startIndex\": \"139\",\n \"endIndex\": + \"181\",\n \"sources\": [\n {\n \"referenceIndex\": + \"1\"\n }\n ]\n },\n + \ {\n \"startIndex\": \"182\",\n \"endIndex\": + \"260\",\n \"sources\": [\n {\n \"referenceIndex\": + \"1\"\n }\n ]\n },\n + \ {\n \"startIndex\": \"261\",\n \"endIndex\": + \"303\",\n \"sources\": [\n {\n \"referenceIndex\": + \"2\"\n }\n ]\n },\n + \ {\n \"startIndex\": \"304\",\n \"endIndex\": + \"390\",\n \"sources\": [\n {\n \"referenceIndex\": + \"2\"\n }\n ]\n },\n + \ {\n \"startIndex\": \"391\",\n \"endIndex\": + \"509\",\n \"sources\": [\n {}\n ]\n + \ },\n {\n \"startIndex\": + \"511\",\n \"endIndex\": \"603\",\n \"sources\": + [\n {\n \"referenceIndex\": \"2\"\n + \ }\n ]\n },\n {\n + \ \"startIndex\": \"654\",\n \"endIndex\": + \"697\",\n \"sources\": [\n {}\n ]\n + \ },\n {\n \"startIndex\": + \"698\",\n \"endIndex\": \"730\",\n \"sources\": + [\n {}\n ]\n },\n + \ {\n \"startIndex\": \"731\",\n \"endIndex\": + \"766\",\n \"sources\": [\n {}\n ]\n + \ },\n {\n \"startIndex\": + \"767\",\n \"endIndex\": \"807\",\n \"sources\": + [\n {}\n ]\n }\n + \ ]\n },\n \"references\": [\n {\n + \ \"title\": \"GOOG Exhibit 99.1 Q4 2022\",\n \"document\": + \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/513da7d1b9f7739087606aadc626f2dc\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q4_alphabet_earnings_release.pdf\"\n + \ },\n {\n \"title\": \"GOOG + Exhibit 99.1 Q1 2022\",\n \"document\": \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/c201efed2be20c6a4116611ea16be9cd\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q1_alphabet_earnings_release.pdf\"\n + \ },\n {\n \"title\": \"GOOG + Exhibit 99.1 Q2 2022\",\n \"document\": \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/376dab8cf7b3807296f2bf1c6228e078\",\n + \ \"uri\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q2_alphabet_earnings_release.pdf\"\n + \ }\n ]\n }\n }\n }\n + \ }\n ]\n },\n \"searchResults\": [\n {\n \"id\": \"513da7d1b9f7739087606aadc626f2dc\",\n + \ \"document\": {\n \"name\": \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/513da7d1b9f7739087606aadc626f2dc\",\n + \ \"id\": \"513da7d1b9f7739087606aadc626f2dc\",\n \"derivedStructData\": + {\n \"can_fetch_raw_content\": \"true\",\n \"title\": \"GOOG + Exhibit 99.1 Q4 2022\",\n \"extractive_answers\": [\n {\n + \ \"content\": \"We're on an important journey to re-engineer + our cost structure in a durable way and to build financially sustainable, + vibrant, growing businesses across Alphabet.\u201D Ruth Porat, CFO of Alphabet + and Google, said: \u201COur Q4 consolidated revenues were $76 billion, up + 1% year over year, or up 7% in constant currency, and $283 ...\",\n \"pageNumber\": + \"1\"\n },\n {\n \"pageNumber\": \"5\",\n + \ \"content\": \"Alphabet Inc. CONSOLIDATED STATEMENTS OF INCOME + (In millions, except per share amounts) Quarter Ended December 31, Year Ended + December 31, 2021 2022 2021 2022 (unaudited) (unaudited) Revenues $ 75325 + $ 76048 $ 257637 $ 282836 Costs and expenses: Cost of revenues 32988 35342 + 110939 126203 Research and development 8708 ...\"\n },\n {\n + \ \"pageNumber\": \"2\",\n \"content\": \"Q4 2022 + supplemental information (in millions, except for number of employees; unaudited) + Revenues, Traffic Acquisition Costs (TAC) and number of employees Quarter + Ended December 31, 2021 2022 Google Search & other $ 43301 $ 42604 YouTube + ads 8633 7963 Google Network 9305 8475 Google advertising 61239 59042 Google + ...\"\n }\n ],\n \"snippets\": [\n {\n + \ \"snippet_status\": \"SUCCESS\",\n \"snippet\": + \"... \\u003cb\\u003eAlphabet\\u003c/b\\u003e.\u201D Ruth Porat, CFO of \\u003cb\\u003eAlphabet\\u003c/b\\u003e + and Google, said: \u201COur Q4 consolidated \\u003cb\\u003erevenues\\u003c/b\\u003e + were $76 billion, up 1% year over year, or up 7% in constant currency ...\"\n + \ }\n ],\n \"link\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q4_alphabet_earnings_release.pdf\"\n + \ }\n },\n \"rankSignals\": {\n \"keywordSimilarityScore\": + 2.4196758,\n \"semanticSimilarityScore\": 0.78510624,\n \"topicalityRank\": + 4,\n \"documentAge\": 497105.84,\n \"boostingFactor\": 0,\n + \ \"defaultRank\": 1\n },\n \"retrievalSignals\": {\n \"retrievalSources\": + [\n 1,\n 2,\n 1,\n 2,\n 1,\n + \ 2\n ],\n \"semanticRelevanceScore\": 0.7206375\n }\n + \ },\n {\n \"id\": \"c201efed2be20c6a4116611ea16be9cd\",\n \"document\": + {\n \"name\": \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/c201efed2be20c6a4116611ea16be9cd\",\n + \ \"id\": \"c201efed2be20c6a4116611ea16be9cd\",\n \"derivedStructData\": + {\n \"snippets\": [\n {\n \"snippet_status\": + \"SUCCESS\",\n \"snippet\": \"\u2013 April 26, \\u003cb\\u003e2022\\u003c/b\\u003e + \u2013 \\u003cb\\u003eAlphabet\\u003c/b\\u003e Inc. (NASDAQ: GOOG ... \\u003cb\\u003erevenue\\u003c/b\\u003e + growth of 23% year over year. ... Quarter Ended March 31, 2021 \\u003cb\\u003e2022 + Revenues\\u003c/b\\u003e $ 55,314 $ 68,011 Change in ...\"\n }\n + \ ],\n \"link\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q1_alphabet_earnings_release.pdf\",\n + \ \"title\": \"GOOG Exhibit 99.1 Q1 2022\",\n \"can_fetch_raw_content\": + \"true\",\n \"extractive_answers\": [\n {\n \"content\": + \"Quarter Ended March 31, 2021 2022 Revenues $ 55314 $ 68011 Change in revenues + year over year 34 % 23 % Change in constant currency revenues year over year(1) + 32 % 26 % Operating income $ 16437 $ 20094 Operating margin 30 % 30 % Other + income (expense), net $ 4846 $ (1160) Net income $ 17930 $ 16436 Diluted EPS + $ 26.29 $ ...\",\n \"pageNumber\": \"1\"\n },\n {\n + \ \"pageNumber\": \"5\",\n \"content\": \"Alphabet + Inc. CONSOLIDATED STATEMENTS OF INCOME (In millions, except share amounts + which are reflected in thousands and per share amounts) Quarter Ended March + 31, 2021 2022 (unaudited) Revenues \\u003cb\\u003e$ 55314\\u003c/b\\u003e + $ 68011 Costs and expenses: Cost of revenues 24103 29599 Research and development + 7485 9119 Sales and marketing 4516 ...\"\n },\n {\n + \ \"content\": \"Comparison from the Quarter Ended March 31, 2021 + to the Quarter Ended March 31, 2022 Quarter Ended March 31, 2021 March 31, + 2022 % Change from Prior Year EMEA revenues $ 17031 $ 20317 19 % EMEA constant + currency revenues 21628 27 % APAC revenues 10455 11841 13 % APAC constant + currency revenues 12440 19 % Other Americas ...\",\n \"pageNumber\": + \"8\"\n }\n ]\n }\n },\n \"rankSignals\": + {\n \"keywordSimilarityScore\": 2.303273,\n \"semanticSimilarityScore\": + 0.7624767,\n \"topicalityRank\": 2,\n \"documentAge\": 497105.84,\n + \ \"boostingFactor\": 0,\n \"defaultRank\": 2\n },\n \"retrievalSignals\": + {\n \"retrievalSources\": [\n 1,\n 2,\n 1,\n + \ 2,\n 2\n ],\n \"semanticRelevanceScore\": + 0.6153616\n }\n },\n {\n \"id\": \"376dab8cf7b3807296f2bf1c6228e078\",\n + \ \"document\": {\n \"name\": \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/376dab8cf7b3807296f2bf1c6228e078\",\n + \ \"id\": \"376dab8cf7b3807296f2bf1c6228e078\",\n \"derivedStructData\": + {\n \"can_fetch_raw_content\": \"true\",\n \"snippets\": + [\n {\n \"snippet_status\": \"SUCCESS\",\n \"snippet\": + \"\u2013 July 26, \\u003cb\\u003e2022\\u003c/b\\u003e \u2013 \\u003cb\\u003eAlphabet\\u003c/b\\u003e + Inc. (NASDAQ: GOOG ... Quarter Ended June 30, 2021 \\u003cb\\u003e2022 Revenues\\u003c/b\\u003e + $ 61,880 $ 69,685 Change in \\u003cb\\u003erevenues\\u003c/b\\u003e ... \\u003cb\\u003erevenue\\u003c/b\\u003e + growth rates by 3.7%.\"\n }\n ],\n \"title\": + \"GOOG Exhibit 99.1 Q2 2022\",\n \"extractive_answers\": [\n {\n + \ \"content\": \"Quarter Ended June 30, 2021 2022 Revenues $ 61880 + $ 69685 Change in revenues year over year (1) 62 % 13 % Change in constant + currency revenues year over year(1) (2) 57 % 16 % Operating income $ 19361 + $ 19453 Operating margin 31 % 28 % Other income (expense), net $ 2624 $ (439) + Net income $ 18525 $ 16002 Diluted EPS $ ...\",\n \"pageNumber\": + \"1\"\n },\n {\n \"content\": \"Alphabet + Inc. CONSOLIDATED STATEMENTS OF INCOME (In millions, except per share amounts, + unaudited) Quarter Ended June 30, Year To Date June 30, 2021 2022 2021 2022 + Revenues $ 61880 $ 69685 $ 117194 $ 137696 Costs and expenses: Cost of revenues + 26227 30104 50330 59703 Research and development 7675 9841 15160 18960 Sales + ...\",\n \"pageNumber\": \"5\"\n },\n {\n + \ \"pageNumber\": \"6\",\n \"content\": \"Alphabet + Inc. CONSOLIDATED STATEMENTS OF CASH FLOWS (In millions, unaudited) Quarter + Ended June 30, Year To Date June 30, 2021 2022 2021 2022 Operating activities + Net income $ 18525 $ 16002 $ 36455 $ 32438 Adjustments: Depreciation and impairment + of property and equipment 2730 3698 5255 7289 Amortization and impairment + ...\"\n }\n ],\n \"link\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q2_alphabet_earnings_release.pdf\"\n + \ }\n },\n \"rankSignals\": {\n \"keywordSimilarityScore\": + 2.5845962,\n \"semanticSimilarityScore\": 0.7609745,\n \"topicalityRank\": + 5,\n \"documentAge\": 497105.84,\n \"boostingFactor\": 0,\n + \ \"defaultRank\": 3\n },\n \"retrievalSignals\": {\n \"retrievalSources\": + [\n 1,\n 2,\n 1,\n 2,\n 1,\n + \ 2\n ],\n \"semanticRelevanceScore\": 0.6050133\n }\n + \ },\n {\n \"id\": \"4a4eabb0cefc954375df0bd9eec85c96\",\n \"document\": + {\n \"name\": \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/4a4eabb0cefc954375df0bd9eec85c96\",\n + \ \"id\": \"4a4eabb0cefc954375df0bd9eec85c96\",\n \"derivedStructData\": + {\n \"snippets\": [\n {\n \"snippet\": \"... + \\u003cb\\u003e2022\\u003c/b\\u003e 2023 \\u003cb\\u003e2022\\u003c/b\\u003e + 2023 (unaudited) (unaudited) \\u003cb\\u003eRevenues\\u003c/b\\u003e $ 76,048 + $ 86,310 $ 282,836 $ 307,394 Costs and expenses: Cost of \\u003cb\\u003erevenues\\u003c/b\\u003e + 35,342 37,575 126,203 ...\",\n \"snippet_status\": \"SUCCESS\"\n + \ }\n ],\n \"can_fetch_raw_content\": \"true\",\n + \ \"extractive_answers\": [\n {\n \"pageNumber\": + \"6\",\n \"content\": \"Alphabet Inc. CONSOLIDATED STATEMENTS + OF INCOME (In millions, except per share amounts) Quarter Ended December 31, + Year Ended December 31, 2022 2023 2022 2023 (unaudited) (unaudited) Revenues + $ 76048 $ 86310 $ 282836 $ 307394 Costs and expenses: Cost of revenues 35342 + 37575 126203 133332 Research and development 10267 ...\"\n },\n + \ {\n \"content\": \"Total Revenues \u2014 Prior Year + Comparative Periods Comparison from the Quarter Ended December 31, 2021 to + the Quarter Ended December 31, 2022 Quarter Ended December 31, 2022 Quarter + Ended December 31, % Change from Prior Period Less FX Effect Constant Currency + Revenues As Reported Less Hedging Effect Less FX Effect ...\",\n \"pageNumber\": + \"11\"\n },\n {\n \"content\": \"Q4 2023 + Supplemental Information (in millions, except for number of employees; unaudited) + Revenues, Traffic Acquisition Costs (TAC), and Number of Employees Quarter + Ended December 31, 2022 2023 Google Search & other $ 42604 $ 48020 YouTube + ads 7963 9200 Google Network 8475 8297 Google advertising 59042 65517 Google + ...\",\n \"pageNumber\": \"2\"\n }\n ],\n + \ \"title\": \"2023q4-alphabet-earnings-release\",\n \"link\": + \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2023q4-alphabet-earnings-release.pdf\"\n + \ }\n },\n \"rankSignals\": {\n \"keywordSimilarityScore\": + 2.0151296,\n \"semanticSimilarityScore\": 0.771026,\n \"topicalityRank\": + 7,\n \"documentAge\": 497105.84,\n \"boostingFactor\": 0,\n + \ \"defaultRank\": 4\n },\n \"retrievalSignals\": {\n \"retrievalSources\": + [\n 1,\n 2,\n 1,\n 2,\n 1,\n + \ 2\n ],\n \"semanticRelevanceScore\": 0.6663862\n }\n + \ },\n {\n \"id\": \"2eab9446e934f432e3df7808af857300\",\n \"document\": + {\n \"name\": \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/2eab9446e934f432e3df7808af857300\",\n + \ \"id\": \"2eab9446e934f432e3df7808af857300\",\n \"derivedStructData\": + {\n \"title\": \"GOOG Exhibit 99.1 Q3 2022\",\n \"extractive_answers\": + [\n {\n \"content\": \"Quarter Ended September 30, + 2021 2022 Revenues $ 65118 $ 69092 Change in revenues year over year 41 % + 6 % Change in constant currency revenues year over year(1) 39 % 11 % Operating + income $ 21031 $ 17135 Operating margin 32 % 25 % Other income (expense), + net $ 2033 $ (902) Net income $ 18936 $ 13910 Diluted EPS $ 1.40 ...\",\n + \ \"pageNumber\": \"1\"\n },\n {\n \"pageNumber\": + \"5\",\n \"content\": \"Alphabet Inc. CONSOLIDATED STATEMENTS + OF INCOME (In millions, except per share amounts, unaudited) Quarter Ended + September 30, Year to Date September 30, 2021 2022 2021 2022 Revenues $ 65118 + $ 69092 $ 182312 $ 206788 Costs and expenses: Cost of revenues 27621 31158 + 77951 90861 Research and development 7694 10273 22854 ...\"\n },\n + \ {\n \"pageNumber\": \"6\",\n \"content\": + \"Alphabet Inc. CONSOLIDATED STATEMENTS OF CASH FLOWS (In millions, unaudited) + Quarter Ended September 30, Year to Date September 30, 2021 2022 2021 2022 + Operating activities Net income $ 18936 $ 13910 $ 55391 $ 46348 Adjustments: + Depreciation and impairment of property and equipment 3085 3933 8340 11222 + Amortization and ...\"\n }\n ],\n \"link\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022Q3_alphabet_earnings_release.pdf\",\n + \ \"can_fetch_raw_content\": \"true\",\n \"snippets\": [\n + \ {\n \"snippet_status\": \"SUCCESS\",\n \"snippet\": + \"\\u003cb\\u003eAlphabet\\u003c/b\\u003e Announces Third Quarter \\u003cb\\u003e2022\\u003c/b\\u003e + Results MOUNTAIN VIEW, Calif. \u2013 October 25, \\u003cb\\u003e2022\\u003c/b\\u003e + \u2013 \\u003cb\\u003eAlphabet\\u003c/b\\u003e Inc. ... Quarter Ended September + 30, 2021 \\u003cb\\u003e2022 Revenues\\u003c/b\\u003e $ 65,118 ...\"\n + \ }\n ]\n }\n },\n \"rankSignals\": {\n + \ \"keywordSimilarityScore\": 2.1627069,\n \"semanticSimilarityScore\": + 0.76224506,\n \"topicalityRank\": 8,\n \"documentAge\": 497105.84,\n + \ \"boostingFactor\": 0,\n \"defaultRank\": 5\n },\n \"retrievalSignals\": + {\n \"retrievalSources\": [\n 1,\n 2,\n 1,\n + \ 2,\n 1,\n 2\n ],\n \"semanticRelevanceScore\": + 0.61376595\n }\n },\n {\n \"id\": \"3c617f6f4b023305d82916f0093ca13d\",\n + \ \"document\": {\n \"name\": \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/3c617f6f4b023305d82916f0093ca13d\",\n + \ \"id\": \"3c617f6f4b023305d82916f0093ca13d\",\n \"derivedStructData\": + {\n \"snippets\": [\n {\n \"snippet\": \"... + \\u003cb\\u003e2022\\u003c/b\\u003e 2023 \\u003cb\\u003eRevenues\\u003c/b\\u003e + $ 257,637 $ 282,836 $ 307,394 Costs and expenses: Cost of \\u003cb\\u003erevenues\\u003c/b\\u003e + 110,939 126,203 133,332 Research and development 31,562 39,500 45,427 ...\",\n + \ \"snippet_status\": \"SUCCESS\"\n }\n ],\n + \ \"link\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/goog023-alphabet-2023-annual-report-web-1.pdf\",\n + \ \"title\": \"goog023-alphabet-2023-annual-report-web-1\",\n \"extractive_answers\": + [\n {\n \"pageNumber\": \"60\",\n \"content\": + \"Alphabet Inc. Consolidated Statements of Income Year Ended December 31, + (in millions, except per share amounts) 2021 2022 2023 Revenues $ 257637 $ + 282836 $ 307394 Costs and expenses: Cost of revenues 110939 126203 133332 + Research and development 31562 39500 45427 Sales and marketing 22912 26567 + 27917 General and ...\"\n },\n {\n \"content\": + \"Year Ended December 31, 2022 2023 Operating income (loss): Google Services + $ 82699 $ 95858 Google Cloud (1922) 1716 Other Bets (4636) (4095) Alphabet-level + activities(1) (1299) (9186) Total income from operations $ 74842 $ 84293 (1) + In addition to the costs included in Alphabet-level activities, hedging gains + (losses) ...\",\n \"pageNumber\": \"46\"\n },\n {\n + \ \"content\": \"The following table presents information about + our segments (in millions): Year Ended December 31, 2021 2022 2023 Revenues: + Google Services $ 237529 $ 253528 $ 272543 Google Cloud 19206 26280 33088 + Other Bets 753 1068 1527 Hedging gains (losses) 149 1960 236 Total revenues + $ 257637 $ 282836 $ 307394 Operating income ( ...\",\n \"pageNumber\": + \"97\"\n }\n ],\n \"can_fetch_raw_content\": + \"true\"\n }\n },\n \"rankSignals\": {\n \"keywordSimilarityScore\": + 1.782047,\n \"semanticSimilarityScore\": 0.772955,\n \"topicalityRank\": + 11,\n \"documentAge\": 497105.84,\n \"boostingFactor\": 0,\n + \ \"defaultRank\": 7\n },\n \"retrievalSignals\": {\n \"retrievalSources\": + [\n 1,\n 2,\n 1,\n 2,\n 1,\n + \ 2\n ],\n \"semanticRelevanceScore\": 0.67124826\n + \ }\n },\n {\n \"id\": \"276cee4c4086600303bc561483691f9a\",\n + \ \"document\": {\n \"name\": \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/276cee4c4086600303bc561483691f9a\",\n + \ \"id\": \"276cee4c4086600303bc561483691f9a\",\n \"derivedStructData\": + {\n \"extractive_answers\": [\n {\n \"content\": + \"Prior periods will be recast to conform to the revised presentation. Our + operating segments are not evaluated using asset information. The following + table presents information about our segments (in millions): Year Ended December + 31, 2020 2021 2022 Revenues: Google Services $ 168635 $ 237529 $ 253528 Google + Cloud 13059 ...\",\n \"pageNumber\": \"119\"\n },\n + \ {\n \"content\": \"Deferred revenues primarily relate + to Google Cloud and Google other. Total deferred revenue as of December 31, + 2021 was $3.8 billion, of which $2.5 billion was recognized as revenues for + the year ending December 31, 2022.\",\n \"pageNumber\": \"94\"\n + \ },\n {\n \"pageNumber\": \"83\",\n \"content\": + \"ALPHABET INC. \u25CF ANNUAL REPORT 45 PART II ITEM 8 FINANCIAL STATEMENTS + AND SUPPLEMENTARY DATA Alphabet Inc. CONSOLIDATED STATEMENTS OF INCOME (In + millions, except per share amounts) Year Ended December 31, 2020 2021 2022 + Revenues $ 182527 $ 257637 $ 282836 Costs and expenses: Cost of revenues 84732 + 110939 126203 Research ...\"\n }\n ],\n \"link\": + \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2022_alphabet_annual_report.pdf\",\n + \ \"snippets\": [\n {\n \"snippet_status\": + \"SUCCESS\",\n \"snippet\": \"... \\u003cb\\u003eAlphabet's\\u003c/b\\u003e + AI activities. DeepMind, previously reported within Other Bets, will be reported + as part of \\u003cb\\u003eAlphabet's\\u003c/b\\u003e corporate costs, + reflecting its increasing ...\"\n }\n ],\n \"title\": + \"2022_alphabet_annual_report\",\n \"can_fetch_raw_content\": \"true\"\n + \ }\n },\n \"rankSignals\": {\n \"keywordSimilarityScore\": + 1.9559898,\n \"semanticSimilarityScore\": 0.7700404,\n \"topicalityRank\": + 9,\n \"documentAge\": 497105.84,\n \"boostingFactor\": 0,\n + \ \"defaultRank\": 8\n },\n \"retrievalSignals\": {\n \"retrievalSources\": + [\n 1,\n 2,\n 1,\n 2,\n 1,\n + \ 2\n ],\n \"semanticRelevanceScore\": 0.7249754\n }\n + \ },\n {\n \"id\": \"404cb139d857b704a13ee3d8e0c6c5fb\",\n \"document\": + {\n \"name\": \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/404cb139d857b704a13ee3d8e0c6c5fb\",\n + \ \"id\": \"404cb139d857b704a13ee3d8e0c6c5fb\",\n \"derivedStructData\": + {\n \"snippets\": [\n {\n \"snippet_status\": + \"SUCCESS\",\n \"snippet\": \"\\u003cb\\u003eAlphabet\\u003c/b\\u003e + Announces Fourth Quarter and Fiscal Year 2021 Results MOUNTAIN VIEW, Calif. + \u2013 February 1, \\u003cb\\u003e2022\\u003c/b\\u003e \u2013 \\u003cb\\u003eAlphabet\\u003c/b\\u003e + Inc. ... \\u003cb\\u003erevenue\\u003c/b\\u003e growth from Google Cloud.\"\n + \ }\n ],\n \"can_fetch_raw_content\": \"true\",\n + \ \"link\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2021Q4_alphabet_earnings_release.pdf\",\n + \ \"extractive_answers\": [\n {\n \"content\": + \"Sundar Pichai, CEO of Alphabet and Google, said: \u201COur deep investment + in AI technologies continues to drive extraordinary and helpful experiences + for people and businesses, across our most important products. Q4 saw ongoing + strong growth in our advertising business, which helped millions of businesses + thrive and find ...\",\n \"pageNumber\": \"1\"\n },\n + \ {\n \"content\": \"Alphabet Inc. CONSOLIDATED STATEMENTS + OF INCOME (In millions, except share amounts which are reflected in thousands + and per share amounts) Quarter Ended December 31, Year Ended December 31, + 2020 2021 2020 2021 (unaudited) (unaudited) Revenues $ 56898 $ 75325 $ 182527 + $ 257637 Costs and expenses: Cost of revenues 26080 ...\",\n \"pageNumber\": + \"5\"\n },\n {\n \"content\": \"Non-GAAP + constant currency revenues are calculated by translating current quarter revenues + using prior period exchange rates and excluding any hedging effect recognized + in the current quarter.\",\n \"pageNumber\": \"8\"\n }\n + \ ],\n \"title\": \"GOOG Exhibit 99.1 Q4 2021\"\n }\n + \ },\n \"rankSignals\": {\n \"keywordSimilarityScore\": 1.4731116,\n + \ \"semanticSimilarityScore\": 0.76674277,\n \"topicalityRank\": + 23,\n \"documentAge\": 497105.84,\n \"boostingFactor\": 0,\n + \ \"defaultRank\": 18\n },\n \"retrievalSignals\": {\n \"retrievalSources\": + [\n 1,\n 2,\n 2,\n 2\n ],\n \"semanticRelevanceScore\": + 0.64287335\n }\n },\n {\n \"id\": \"e0ac436b3f82b7df8fc94580f3a84b0a\",\n + \ \"document\": {\n \"name\": \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/e0ac436b3f82b7df8fc94580f3a84b0a\",\n + \ \"id\": \"e0ac436b3f82b7df8fc94580f3a84b0a\",\n \"derivedStructData\": + {\n \"title\": \"GOOG Exhibit 99.1 Q1 2023\",\n \"snippets\": + [\n {\n \"snippet\": \"Ruth Porat, CFO of \\u003cb\\u003eAlphabet\\u003c/b\\u003e + and Google, said: \u201CResilience in Search and momentum in Cloud resulted + in Q1 consolidated \\u003cb\\u003erevenues\\u003c/b\\u003e of $69.8 billion, + up 3% year over ...\",\n \"snippet_status\": \"SUCCESS\"\n + \ }\n ],\n \"can_fetch_raw_content\": \"true\",\n + \ \"extractive_answers\": [\n {\n \"pageNumber\": + \"1\",\n \"content\": \"Quarter Ended March 31, 2022 2023 Revenues + $ 68011 $ 69787 Change in revenues year over year 23 % 3 % Change in constant + currency revenues year over year(1) 26 % 6 % Operating income $ 20094 $ 17415 + Operating margin 30 % 25 % Other income (expense), net $ (1160) $ 790 Net + income $ 16436 $ 15051 Diluted EPS $ 1.23 $ ...\"\n },\n {\n + \ \"content\": \"Alphabet Inc. CONSOLIDATED STATEMENTS OF INCOME + (In millions, except per share amounts, unaudited) Quarter Ended March 31, + 2022 2023 Revenues $ 68011 $ 69787 Costs and expenses: Cost of revenues 29599 + 30612 Research and development 9119 11468 Sales and marketing 5825 6533 General + and administrative 3374 3759 Total ...\",\n \"pageNumber\": \"6\"\n + \ },\n {\n \"content\": \"Quarter Ended + March 31, 2022 2023 (recast) Operating income (loss): Google Services $ 21973 + $ 21737 Google Cloud (706) 191 Other Bets (835) (1225) Corporate costs, unallocated(1) + (338) (3288) Total income from operations $ 20094 $ 17415 (1) Hedging gains + (losses) related to revenue included in unallocated corporate ...\",\n \"pageNumber\": + \"2\"\n }\n ],\n \"link\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2023q1-alphabet-earnings-release.pdf\"\n + \ }\n },\n \"rankSignals\": {\n \"keywordSimilarityScore\": + 1.8275325,\n \"semanticSimilarityScore\": 0.7408051,\n \"topicalityRank\": + 20,\n \"documentAge\": 497105.84,\n \"boostingFactor\": 0,\n + \ \"defaultRank\": 19\n },\n \"retrievalSignals\": {\n \"retrievalSources\": + [\n 1,\n 2,\n 1,\n 2,\n 1,\n + \ 2\n ],\n \"semanticRelevanceScore\": 0.48590127\n + \ }\n },\n {\n \"id\": \"4c7389ffaf06b73e6e380eac7a39ea17\",\n + \ \"document\": {\n \"name\": \"projects/479238727071/locations/global/collections/default_collection/dataStores/braintrust-sdk-test-docs_1789580119475/branches/0/documents/4c7389ffaf06b73e6e380eac7a39ea17\",\n + \ \"id\": \"4c7389ffaf06b73e6e380eac7a39ea17\",\n \"derivedStructData\": + {\n \"extractive_answers\": [\n {\n \"content\": + \"We're continuing to focus on making AI more helpful for everyone; there's + exciting progress and lots more to come.\u201D Ruth Porat, President and Chief + Investment Officer; CFO said: \u201CThe fundamental strength of our business + was apparent again in Q3, with $77 billion in revenue, up 11% year over year, + driven by meaningful ...\",\n \"pageNumber\": \"1\"\n },\n + \ {\n \"content\": \"Alphabet Inc. CONSOLIDATED STATEMENTS + OF INCOME (In millions, except per share amounts, unaudited) Quarter Ended + September 30, Year to Date September 30, 2022 2023 2022 2023 Revenues $ 69092 + $ 76693 $ 206788 $ 221084 Costs and expenses: Cost of revenues 31158 33229 + 90861 95757 Research and development 10273 11258 29233 ...\",\n \"pageNumber\": + \"6\"\n },\n {\n \"content\": \"Quarter + Ended September 30, 2022 2023 (recast) Operating income (loss): Google Services + $ 18883 $ 23937 Google Cloud (440) 266 Other Bets (1225) (1194) Corporate + costs, unallocated(1) (83) (1666) Total income from operations $ 17135 $ 21343 + (1) In addition to the Alphabet-level costs included in unallocated corporate + ...\",\n \"pageNumber\": \"2\"\n }\n ],\n + \ \"snippets\": [\n {\n \"snippet\": \"\\u003cb\\u003eAlphabet\\u003c/b\\u003e + Announces Third Quarter 2023 Results MOUNTAIN VIEW, Calif. ... \\u003cb\\u003erevenue\\u003c/b\\u003e, + up 11% year over year, driven by ... \\u003cb\\u003e2022\\u003c/b\\u003e and + 2023 (in millions, except for per ...\",\n \"snippet_status\": + \"SUCCESS\"\n }\n ],\n \"link\": \"gs://cloud-samples-data/gen-app-builder/search/alphabet-investor-pdfs/2023q3-alphabet-earnings-release.pdf\",\n + \ \"can_fetch_raw_content\": \"true\",\n \"title\": \"GOOG + Exhibit 99.1 Q3 2023\"\n }\n },\n \"rankSignals\": {\n \"keywordSimilarityScore\": + 1.7931039,\n \"semanticSimilarityScore\": 0.73606783,\n \"topicalityRank\": + 30,\n \"documentAge\": 497105.84,\n \"boostingFactor\": 0,\n + \ \"defaultRank\": 20\n },\n \"retrievalSignals\": {\n \"retrievalSources\": + [\n 1,\n 2,\n 1,\n 2,\n 1,\n + \ 2\n ],\n \"semanticRelevanceScore\": 0.46072376\n + \ }\n }\n ]\n}\n" + headers: + Alt-Svc: + - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 + Content-Type: + - application/json; charset=UTF-8 + Date: + - Wed, 16 Sep 2026 17:50:02 GMT + Server: + - ESF + Server-Timing: + - gfet4t7; dur=2503 + Transfer-Encoding: + - chunked + Vary: + - Origin + - X-Origin + - Referer + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-XSS-Protection: + - '0' + content-length: + - '38349' + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_provider_error.yaml b/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_provider_error.yaml new file mode 100644 index 000000000..161390870 --- /dev/null +++ b/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_provider_error.yaml @@ -0,0 +1,57 @@ +interactions: +- request: + body: "{\n \"query\": \"test\",\n \"records\": [\n {\n \"id\": \"empty\"\n + \ }\n ]\n}" + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '73' + Content-Type: + - application/json + User-Agent: + - python-requests/2.34.2 + x-goog-api-client: + - gl-python/3.14.7 grpc/1.84.0 gax/2.37.0 gapic/0.20.3 pb/7.36.1 cred-type/u + x-goog-request-params: + - ranking_config=projects/sdk-dev-508013/locations/global/rankingConfigs/default_ranking_config + method: POST + uri: https://discoveryengine.googleapis.com/v1/projects/sdk-dev-508013/locations/global/rankingConfigs/default_ranking_config:rank?%24alt=json%3Benum-encoding%3Dint + response: + body: + string: "{\n \"error\": {\n \"code\": 400,\n \"message\": \"RankRequest.records(0).title() + and RankRequest.records(0).content() are empty at the same time.\",\n \"status\": + \"INVALID_ARGUMENT\"\n }\n}\n" + headers: + Alt-Svc: + - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 + Content-Type: + - application/json; charset=UTF-8 + Date: + - Wed, 16 Sep 2026 17:45:01 GMT + Server: + - ESF + Server-Timing: + - gfet4t7; dur=71 + Transfer-Encoding: + - chunked + Vary: + - Origin + - X-Origin + - Referer + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-XSS-Protection: + - '0' + content-length: + - '185' + status: + code: 400 + message: Bad Request +version: 1 diff --git a/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_rank.yaml b/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_rank.yaml new file mode 100644 index 000000000..1ceaa8483 --- /dev/null +++ b/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_rank.yaml @@ -0,0 +1,60 @@ +interactions: +- request: + body: "{\n \"model\": \"semantic-ranker-512@latest\",\n \"topN\": 1,\n \"query\": + \"What is Braintrust?\",\n \"records\": [\n {\n \"id\": \"1\",\n \"content\": + \"Braintrust is a platform for evaluating and monitoring AI applications.\"\n + \ },\n {\n \"id\": \"2\",\n \"content\": \"The moon orbits the + Earth.\"\n }\n ]\n}" + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate, zstd + Connection: + - keep-alive + Content-Length: + - '306' + Content-Type: + - application/json + User-Agent: + - python-requests/2.34.2 + x-goog-api-client: + - gl-python/3.14.7 grpc/1.84.0 gax/2.37.0 gapic/0.20.3 pb/7.36.1 cred-type/u + x-goog-request-params: + - ranking_config=projects/sdk-dev-508013/locations/global/rankingConfigs/default_ranking_config + method: POST + uri: https://discoveryengine.googleapis.com/v1/projects/sdk-dev-508013/locations/global/rankingConfigs/default_ranking_config:rank?%24alt=json%3Benum-encoding%3Dint + response: + body: + string: "{\n \"records\": [\n {\n \"id\": \"1\",\n \"content\": + \"Braintrust is a platform for evaluating and monitoring AI applications.\",\n + \ \"score\": 0.8597\n }\n ]\n}\n" + headers: + Alt-Svc: + - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 + Content-Type: + - application/json; charset=UTF-8 + Date: + - Wed, 16 Sep 2026 17:37:35 GMT + Server: + - ESF + Server-Timing: + - gfet4t7; dur=105 + Transfer-Encoding: + - chunked + Vary: + - Origin + - X-Origin + - Referer + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-XSS-Protection: + - '0' + content-length: + - '166' + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_rank_output_limit.yaml b/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_rank_output_limit.yaml new file mode 100644 index 000000000..91bfe97de --- /dev/null +++ b/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_rank_output_limit.yaml @@ -0,0 +1,296 @@ +interactions: +- request: + body: "{\n \"query\": \"AI evaluation\",\n \"records\": [\n {\n \"id\": + \"0\",\n \"content\": \"Evaluation example 0.\"\n },\n {\n \"id\": + \"1\",\n \"content\": \"Evaluation example 1.\"\n },\n {\n \"id\": + \"2\",\n \"content\": \"Evaluation example 2.\"\n },\n {\n \"id\": + \"3\",\n \"content\": \"Evaluation example 3.\"\n },\n {\n \"id\": + \"4\",\n \"content\": \"Evaluation example 4.\"\n },\n {\n \"id\": + \"5\",\n \"content\": \"Evaluation example 5.\"\n },\n {\n \"id\": + \"6\",\n \"content\": \"Evaluation example 6.\"\n },\n {\n \"id\": + \"7\",\n \"content\": \"Evaluation example 7.\"\n },\n {\n \"id\": + \"8\",\n \"content\": \"Evaluation example 8.\"\n },\n {\n \"id\": + \"9\",\n \"content\": \"Evaluation example 9.\"\n },\n {\n \"id\": + \"10\",\n \"content\": \"Evaluation example 10.\"\n },\n {\n \"id\": + \"11\",\n \"content\": \"Evaluation example 11.\"\n },\n {\n \"id\": + \"12\",\n \"content\": \"Evaluation example 12.\"\n },\n {\n \"id\": + \"13\",\n \"content\": \"Evaluation example 13.\"\n },\n {\n \"id\": + \"14\",\n \"content\": \"Evaluation example 14.\"\n },\n {\n \"id\": + \"15\",\n \"content\": \"Evaluation example 15.\"\n },\n {\n \"id\": + \"16\",\n \"content\": \"Evaluation example 16.\"\n },\n {\n \"id\": + \"17\",\n \"content\": \"Evaluation example 17.\"\n },\n {\n \"id\": + \"18\",\n \"content\": \"Evaluation example 18.\"\n },\n {\n \"id\": + \"19\",\n \"content\": \"Evaluation example 19.\"\n },\n {\n \"id\": + \"20\",\n \"content\": \"Evaluation example 20.\"\n },\n {\n \"id\": + \"21\",\n \"content\": \"Evaluation example 21.\"\n },\n {\n \"id\": + \"22\",\n \"content\": \"Evaluation example 22.\"\n },\n {\n \"id\": + \"23\",\n \"content\": \"Evaluation example 23.\"\n },\n {\n \"id\": + \"24\",\n \"content\": \"Evaluation example 24.\"\n },\n {\n \"id\": + \"25\",\n \"content\": \"Evaluation example 25.\"\n },\n {\n \"id\": + \"26\",\n \"content\": \"Evaluation example 26.\"\n },\n {\n \"id\": + \"27\",\n \"content\": \"Evaluation example 27.\"\n },\n {\n \"id\": + \"28\",\n \"content\": \"Evaluation example 28.\"\n },\n {\n \"id\": + \"29\",\n \"content\": \"Evaluation example 29.\"\n },\n {\n \"id\": + \"30\",\n \"content\": \"Evaluation example 30.\"\n },\n {\n \"id\": + \"31\",\n \"content\": \"Evaluation example 31.\"\n },\n {\n \"id\": + \"32\",\n \"content\": \"Evaluation example 32.\"\n },\n {\n \"id\": + \"33\",\n \"content\": \"Evaluation example 33.\"\n },\n {\n \"id\": + \"34\",\n \"content\": \"Evaluation example 34.\"\n },\n {\n \"id\": + \"35\",\n \"content\": \"Evaluation example 35.\"\n },\n {\n \"id\": + \"36\",\n \"content\": \"Evaluation example 36.\"\n },\n {\n \"id\": + \"37\",\n \"content\": \"Evaluation example 37.\"\n },\n {\n \"id\": + \"38\",\n \"content\": \"Evaluation example 38.\"\n },\n {\n \"id\": + \"39\",\n \"content\": \"Evaluation example 39.\"\n },\n {\n \"id\": + \"40\",\n \"content\": \"Evaluation example 40.\"\n },\n {\n \"id\": + \"41\",\n \"content\": \"Evaluation example 41.\"\n },\n {\n \"id\": + \"42\",\n \"content\": \"Evaluation example 42.\"\n },\n {\n \"id\": + \"43\",\n \"content\": \"Evaluation example 43.\"\n },\n {\n \"id\": + \"44\",\n \"content\": \"Evaluation example 44.\"\n },\n {\n \"id\": + \"45\",\n \"content\": \"Evaluation example 45.\"\n },\n {\n \"id\": + \"46\",\n \"content\": \"Evaluation example 46.\"\n },\n {\n \"id\": + \"47\",\n \"content\": \"Evaluation example 47.\"\n },\n {\n \"id\": + \"48\",\n \"content\": \"Evaluation example 48.\"\n },\n {\n \"id\": + \"49\",\n \"content\": \"Evaluation example 49.\"\n },\n {\n \"id\": + \"50\",\n \"content\": \"Evaluation example 50.\"\n },\n {\n \"id\": + \"51\",\n \"content\": \"Evaluation example 51.\"\n },\n {\n \"id\": + \"52\",\n \"content\": \"Evaluation example 52.\"\n },\n {\n \"id\": + \"53\",\n \"content\": \"Evaluation example 53.\"\n },\n {\n \"id\": + \"54\",\n \"content\": \"Evaluation example 54.\"\n },\n {\n \"id\": + \"55\",\n \"content\": \"Evaluation example 55.\"\n },\n {\n \"id\": + \"56\",\n \"content\": \"Evaluation example 56.\"\n },\n {\n \"id\": + \"57\",\n \"content\": \"Evaluation example 57.\"\n },\n {\n \"id\": + \"58\",\n \"content\": \"Evaluation example 58.\"\n },\n {\n \"id\": + \"59\",\n \"content\": \"Evaluation example 59.\"\n },\n {\n \"id\": + \"60\",\n \"content\": \"Evaluation example 60.\"\n },\n {\n \"id\": + \"61\",\n \"content\": \"Evaluation example 61.\"\n },\n {\n \"id\": + \"62\",\n \"content\": \"Evaluation example 62.\"\n },\n {\n \"id\": + \"63\",\n \"content\": \"Evaluation example 63.\"\n },\n {\n \"id\": + \"64\",\n \"content\": \"Evaluation example 64.\"\n },\n {\n \"id\": + \"65\",\n \"content\": \"Evaluation example 65.\"\n },\n {\n \"id\": + \"66\",\n \"content\": \"Evaluation example 66.\"\n },\n {\n \"id\": + \"67\",\n \"content\": \"Evaluation example 67.\"\n },\n {\n \"id\": + \"68\",\n \"content\": \"Evaluation example 68.\"\n },\n {\n \"id\": + \"69\",\n \"content\": \"Evaluation example 69.\"\n },\n {\n \"id\": + \"70\",\n \"content\": \"Evaluation example 70.\"\n },\n {\n \"id\": + \"71\",\n \"content\": \"Evaluation example 71.\"\n },\n {\n \"id\": + \"72\",\n \"content\": \"Evaluation example 72.\"\n },\n {\n \"id\": + \"73\",\n \"content\": \"Evaluation example 73.\"\n },\n {\n \"id\": + \"74\",\n \"content\": \"Evaluation example 74.\"\n },\n {\n \"id\": + \"75\",\n \"content\": \"Evaluation example 75.\"\n },\n {\n \"id\": + \"76\",\n \"content\": \"Evaluation example 76.\"\n },\n {\n \"id\": + \"77\",\n \"content\": \"Evaluation example 77.\"\n },\n {\n \"id\": + \"78\",\n \"content\": \"Evaluation example 78.\"\n },\n {\n \"id\": + \"79\",\n \"content\": \"Evaluation example 79.\"\n },\n {\n \"id\": + \"80\",\n \"content\": \"Evaluation example 80.\"\n },\n {\n \"id\": + \"81\",\n \"content\": \"Evaluation example 81.\"\n },\n {\n \"id\": + \"82\",\n \"content\": \"Evaluation example 82.\"\n },\n {\n \"id\": + \"83\",\n \"content\": \"Evaluation example 83.\"\n },\n {\n \"id\": + \"84\",\n \"content\": \"Evaluation example 84.\"\n },\n {\n \"id\": + \"85\",\n \"content\": \"Evaluation example 85.\"\n },\n {\n \"id\": + \"86\",\n \"content\": \"Evaluation example 86.\"\n },\n {\n \"id\": + \"87\",\n \"content\": \"Evaluation example 87.\"\n },\n {\n \"id\": + \"88\",\n \"content\": \"Evaluation example 88.\"\n },\n {\n \"id\": + \"89\",\n \"content\": \"Evaluation example 89.\"\n },\n {\n \"id\": + \"90\",\n \"content\": \"Evaluation example 90.\"\n },\n {\n \"id\": + \"91\",\n \"content\": \"Evaluation example 91.\"\n },\n {\n \"id\": + \"92\",\n \"content\": \"Evaluation example 92.\"\n },\n {\n \"id\": + \"93\",\n \"content\": \"Evaluation example 93.\"\n },\n {\n \"id\": + \"94\",\n \"content\": \"Evaluation example 94.\"\n },\n {\n \"id\": + \"95\",\n \"content\": \"Evaluation example 95.\"\n },\n {\n \"id\": + \"96\",\n \"content\": \"Evaluation example 96.\"\n },\n {\n \"id\": + \"97\",\n \"content\": \"Evaluation example 97.\"\n },\n {\n \"id\": + \"98\",\n \"content\": \"Evaluation example 98.\"\n },\n {\n \"id\": + \"99\",\n \"content\": \"Evaluation example 99.\"\n },\n {\n \"id\": + \"100\",\n \"content\": \"Evaluation example 100.\"\n }\n ]\n}" + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '7404' + Content-Type: + - application/json + User-Agent: + - python-requests/2.34.2 + x-goog-api-client: + - gl-python/3.11.15 grpc/1.84.0 gax/2.37.0 gapic/0.20.3 pb/7.36.1 cred-type/u + x-goog-request-params: + - ranking_config=projects/sdk-dev-508013/locations/global/rankingConfigs/default_ranking_config + method: POST + uri: https://discoveryengine.googleapis.com/v1/projects/sdk-dev-508013/locations/global/rankingConfigs/default_ranking_config:rank?%24alt=json%3Benum-encoding%3Dint + response: + body: + string: "{\n \"records\": [\n {\n \"id\": \"1\",\n \"content\": + \"Evaluation example 1.\",\n \"score\": 0.2568\n },\n {\n \"id\": + \"7\",\n \"content\": \"Evaluation example 7.\",\n \"score\": 0.2355\n + \ },\n {\n \"id\": \"8\",\n \"content\": \"Evaluation example + 8.\",\n \"score\": 0.2339\n },\n {\n \"id\": \"5\",\n \"content\": + \"Evaluation example 5.\",\n \"score\": 0.2322\n },\n {\n \"id\": + \"9\",\n \"content\": \"Evaluation example 9.\",\n \"score\": 0.232\n + \ },\n {\n \"id\": \"4\",\n \"content\": \"Evaluation example + 4.\",\n \"score\": 0.2305\n },\n {\n \"id\": \"6\",\n \"content\": + \"Evaluation example 6.\",\n \"score\": 0.23\n },\n {\n \"id\": + \"3\",\n \"content\": \"Evaluation example 3.\",\n \"score\": 0.2286\n + \ },\n {\n \"id\": \"0\",\n \"content\": \"Evaluation example + 0.\",\n \"score\": 0.2246\n },\n {\n \"id\": \"2\",\n \"content\": + \"Evaluation example 2.\",\n \"score\": 0.2178\n },\n {\n \"id\": + \"39\",\n \"content\": \"Evaluation example 39.\",\n \"score\": + 0.2166\n },\n {\n \"id\": \"47\",\n \"content\": \"Evaluation + example 47.\",\n \"score\": 0.2157\n },\n {\n \"id\": \"38\",\n + \ \"content\": \"Evaluation example 38.\",\n \"score\": 0.2154\n + \ },\n {\n \"id\": \"41\",\n \"content\": \"Evaluation example + 41.\",\n \"score\": 0.2149\n },\n {\n \"id\": \"34\",\n \"content\": + \"Evaluation example 34.\",\n \"score\": 0.214\n },\n {\n \"id\": + \"68\",\n \"content\": \"Evaluation example 68.\",\n \"score\": + 0.2139\n },\n {\n \"id\": \"54\",\n \"content\": \"Evaluation + example 54.\",\n \"score\": 0.2129\n },\n {\n \"id\": \"67\",\n + \ \"content\": \"Evaluation example 67.\",\n \"score\": 0.2128\n + \ },\n {\n \"id\": \"49\",\n \"content\": \"Evaluation example + 49.\",\n \"score\": 0.2123\n },\n {\n \"id\": \"81\",\n \"content\": + \"Evaluation example 81.\",\n \"score\": 0.2123\n },\n {\n \"id\": + \"58\",\n \"content\": \"Evaluation example 58.\",\n \"score\": + 0.2122\n },\n {\n \"id\": \"37\",\n \"content\": \"Evaluation + example 37.\",\n \"score\": 0.2111\n },\n {\n \"id\": \"61\",\n + \ \"content\": \"Evaluation example 61.\",\n \"score\": 0.2108\n + \ },\n {\n \"id\": \"11\",\n \"content\": \"Evaluation example + 11.\",\n \"score\": 0.2107\n },\n {\n \"id\": \"59\",\n \"content\": + \"Evaluation example 59.\",\n \"score\": 0.2104\n },\n {\n \"id\": + \"42\",\n \"content\": \"Evaluation example 42.\",\n \"score\": + 0.2097\n },\n {\n \"id\": \"43\",\n \"content\": \"Evaluation + example 43.\",\n \"score\": 0.2093\n },\n {\n \"id\": \"31\",\n + \ \"content\": \"Evaluation example 31.\",\n \"score\": 0.2093\n + \ },\n {\n \"id\": \"44\",\n \"content\": \"Evaluation example + 44.\",\n \"score\": 0.2093\n },\n {\n \"id\": \"48\",\n \"content\": + \"Evaluation example 48.\",\n \"score\": 0.209\n },\n {\n \"id\": + \"69\",\n \"content\": \"Evaluation example 69.\",\n \"score\": + 0.2087\n },\n {\n \"id\": \"100\",\n \"content\": \"Evaluation + example 100.\",\n \"score\": 0.2085\n },\n {\n \"id\": \"17\",\n + \ \"content\": \"Evaluation example 17.\",\n \"score\": 0.2085\n + \ },\n {\n \"id\": \"32\",\n \"content\": \"Evaluation example + 32.\",\n \"score\": 0.208\n },\n {\n \"id\": \"10\",\n \"content\": + \"Evaluation example 10.\",\n \"score\": 0.2078\n },\n {\n \"id\": + \"33\",\n \"content\": \"Evaluation example 33.\",\n \"score\": + 0.2077\n },\n {\n \"id\": \"45\",\n \"content\": \"Evaluation + example 45.\",\n \"score\": 0.2077\n },\n {\n \"id\": \"36\",\n + \ \"content\": \"Evaluation example 36.\",\n \"score\": 0.2076\n + \ },\n {\n \"id\": \"21\",\n \"content\": \"Evaluation example + 21.\",\n \"score\": 0.2076\n },\n {\n \"id\": \"19\",\n \"content\": + \"Evaluation example 19.\",\n \"score\": 0.2075\n },\n {\n \"id\": + \"86\",\n \"content\": \"Evaluation example 86.\",\n \"score\": + 0.2074\n },\n {\n \"id\": \"53\",\n \"content\": \"Evaluation + example 53.\",\n \"score\": 0.2072\n },\n {\n \"id\": \"76\",\n + \ \"content\": \"Evaluation example 76.\",\n \"score\": 0.2071\n + \ },\n {\n \"id\": \"89\",\n \"content\": \"Evaluation example + 89.\",\n \"score\": 0.207\n },\n {\n \"id\": \"91\",\n \"content\": + \"Evaluation example 91.\",\n \"score\": 0.2069\n },\n {\n \"id\": + \"51\",\n \"content\": \"Evaluation example 51.\",\n \"score\": + 0.2068\n },\n {\n \"id\": \"57\",\n \"content\": \"Evaluation + example 57.\",\n \"score\": 0.2067\n },\n {\n \"id\": \"40\",\n + \ \"content\": \"Evaluation example 40.\",\n \"score\": 0.2067\n + \ },\n {\n \"id\": \"63\",\n \"content\": \"Evaluation example + 63.\",\n \"score\": 0.2064\n },\n {\n \"id\": \"90\",\n \"content\": + \"Evaluation example 90.\",\n \"score\": 0.2063\n },\n {\n \"id\": + \"15\",\n \"content\": \"Evaluation example 15.\",\n \"score\": + 0.2061\n },\n {\n \"id\": \"55\",\n \"content\": \"Evaluation + example 55.\",\n \"score\": 0.2061\n },\n {\n \"id\": \"16\",\n + \ \"content\": \"Evaluation example 16.\",\n \"score\": 0.206\n },\n + \ {\n \"id\": \"84\",\n \"content\": \"Evaluation example 84.\",\n + \ \"score\": 0.2059\n },\n {\n \"id\": \"20\",\n \"content\": + \"Evaluation example 20.\",\n \"score\": 0.2059\n },\n {\n \"id\": + \"14\",\n \"content\": \"Evaluation example 14.\",\n \"score\": + 0.2058\n },\n {\n \"id\": \"71\",\n \"content\": \"Evaluation + example 71.\",\n \"score\": 0.2058\n },\n {\n \"id\": \"64\",\n + \ \"content\": \"Evaluation example 64.\",\n \"score\": 0.2056\n + \ },\n {\n \"id\": \"73\",\n \"content\": \"Evaluation example + 73.\",\n \"score\": 0.2054\n },\n {\n \"id\": \"74\",\n \"content\": + \"Evaluation example 74.\",\n \"score\": 0.2053\n },\n {\n \"id\": + \"83\",\n \"content\": \"Evaluation example 83.\",\n \"score\": + 0.2052\n },\n {\n \"id\": \"82\",\n \"content\": \"Evaluation + example 82.\",\n \"score\": 0.2052\n },\n {\n \"id\": \"60\",\n + \ \"content\": \"Evaluation example 60.\",\n \"score\": 0.205\n },\n + \ {\n \"id\": \"79\",\n \"content\": \"Evaluation example 79.\",\n + \ \"score\": 0.2049\n },\n {\n \"id\": \"88\",\n \"content\": + \"Evaluation example 88.\",\n \"score\": 0.2049\n },\n {\n \"id\": + \"46\",\n \"content\": \"Evaluation example 46.\",\n \"score\": + 0.2048\n },\n {\n \"id\": \"87\",\n \"content\": \"Evaluation + example 87.\",\n \"score\": 0.2046\n },\n {\n \"id\": \"18\",\n + \ \"content\": \"Evaluation example 18.\",\n \"score\": 0.2042\n + \ },\n {\n \"id\": \"80\",\n \"content\": \"Evaluation example + 80.\",\n \"score\": 0.2042\n },\n {\n \"id\": \"52\",\n \"content\": + \"Evaluation example 52.\",\n \"score\": 0.204\n },\n {\n \"id\": + \"62\",\n \"content\": \"Evaluation example 62.\",\n \"score\": + 0.2037\n },\n {\n \"id\": \"56\",\n \"content\": \"Evaluation + example 56.\",\n \"score\": 0.2037\n },\n {\n \"id\": \"78\",\n + \ \"content\": \"Evaluation example 78.\",\n \"score\": 0.2036\n + \ },\n {\n \"id\": \"13\",\n \"content\": \"Evaluation example + 13.\",\n \"score\": 0.2035\n },\n {\n \"id\": \"12\",\n \"content\": + \"Evaluation example 12.\",\n \"score\": 0.2034\n },\n {\n \"id\": + \"85\",\n \"content\": \"Evaluation example 85.\",\n \"score\": + 0.2034\n },\n {\n \"id\": \"35\",\n \"content\": \"Evaluation + example 35.\",\n \"score\": 0.2033\n },\n {\n \"id\": \"92\",\n + \ \"content\": \"Evaluation example 92.\",\n \"score\": 0.2031\n + \ },\n {\n \"id\": \"95\",\n \"content\": \"Evaluation example + 95.\",\n \"score\": 0.2031\n },\n {\n \"id\": \"66\",\n \"content\": + \"Evaluation example 66.\",\n \"score\": 0.203\n },\n {\n \"id\": + \"29\",\n \"content\": \"Evaluation example 29.\",\n \"score\": + 0.2028\n },\n {\n \"id\": \"28\",\n \"content\": \"Evaluation + example 28.\",\n \"score\": 0.2027\n },\n {\n \"id\": \"65\",\n + \ \"content\": \"Evaluation example 65.\",\n \"score\": 0.2027\n + \ },\n {\n \"id\": \"24\",\n \"content\": \"Evaluation example + 24.\",\n \"score\": 0.2027\n },\n {\n \"id\": \"99\",\n \"content\": + \"Evaluation example 99.\",\n \"score\": 0.2019\n },\n {\n \"id\": + \"72\",\n \"content\": \"Evaluation example 72.\",\n \"score\": + 0.2015\n },\n {\n \"id\": \"77\",\n \"content\": \"Evaluation + example 77.\",\n \"score\": 0.2015\n },\n {\n \"id\": \"96\",\n + \ \"content\": \"Evaluation example 96.\",\n \"score\": 0.2012\n + \ },\n {\n \"id\": \"50\",\n \"content\": \"Evaluation example + 50.\",\n \"score\": 0.2011\n },\n {\n \"id\": \"94\",\n \"content\": + \"Evaluation example 94.\",\n \"score\": 0.2008\n },\n {\n \"id\": + \"75\",\n \"content\": \"Evaluation example 75.\",\n \"score\": + 0.2008\n },\n {\n \"id\": \"26\",\n \"content\": \"Evaluation + example 26.\",\n \"score\": 0.2006\n },\n {\n \"id\": \"70\",\n + \ \"content\": \"Evaluation example 70.\",\n \"score\": 0.2006\n + \ },\n {\n \"id\": \"23\",\n \"content\": \"Evaluation example + 23.\",\n \"score\": 0.2005\n },\n {\n \"id\": \"27\",\n \"content\": + \"Evaluation example 27.\",\n \"score\": 0.1989\n },\n {\n \"id\": + \"25\",\n \"content\": \"Evaluation example 25.\",\n \"score\": + 0.1986\n },\n {\n \"id\": \"22\",\n \"content\": \"Evaluation + example 22.\",\n \"score\": 0.1985\n },\n {\n \"id\": \"98\",\n + \ \"content\": \"Evaluation example 98.\",\n \"score\": 0.1981\n + \ },\n {\n \"id\": \"30\",\n \"content\": \"Evaluation example + 30.\",\n \"score\": 0.1979\n },\n {\n \"id\": \"93\",\n \"content\": + \"Evaluation example 93.\",\n \"score\": 0.1955\n },\n {\n \"id\": + \"97\",\n \"content\": \"Evaluation example 97.\",\n \"score\": + 0.1953\n }\n ]\n}\n" + headers: + Alt-Svc: + - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 + Content-Type: + - application/json; charset=UTF-8 + Date: + - Wed, 16 Sep 2026 17:55:15 GMT + Server: + - ESF + Server-Timing: + - gfet4t7; dur=151 + Transfer-Encoding: + - chunked + Vary: + - Origin + - X-Origin + - Referer + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-XSS-Protection: + - '0' + content-length: + - '9689' + status: + code: 200 + message: OK +version: 1 diff --git a/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_stream_provider_error.yaml b/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_stream_provider_error.yaml new file mode 100644 index 000000000..355c7c4a1 --- /dev/null +++ b/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_stream_provider_error.yaml @@ -0,0 +1,56 @@ +interactions: +- request: + body: '{}' + headers: + Accept: + - '*/*' + Accept-Encoding: + - gzip, deflate + Connection: + - keep-alive + Content-Length: + - '2' + Content-Type: + - application/json + User-Agent: + - python-requests/2.34.2 + x-goog-api-client: + - gl-python/3.10.19 grpc/1.84.0 gax/2.37.0 gapic/0.20.3 pb/7.36.1 cred-type/u + x-goog-request-params: + - serving_config=projects/sdk-dev-508013/locations/global/collections/default_collection/engines/braintrust-sdk-test_1789580029521/servingConfigs/default_search + method: POST + uri: https://discoveryengine.googleapis.com/v1/projects/sdk-dev-508013/locations/global/collections/default_collection/engines/braintrust-sdk-test_1789580029521/servingConfigs/default_search:streamAnswer?%24alt=json%3Benum-encoding%3Dint + response: + body: + string: "[{\n \"error\": {\n \"code\": 500,\n \"message\": \"Internal + error encountered. Please try again. If the issue persists, please contact + our support team.\",\n \"status\": \"INTERNAL\"\n }\n}\n]" + headers: + Alt-Svc: + - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 + Content-Type: + - application/json; charset=UTF-8 + Date: + - Wed, 16 Sep 2026 17:56:16 GMT + Server: + - ESF + Server-Timing: + - gfet4t7; dur=83 + Transfer-Encoding: + - chunked + Vary: + - Origin + - X-Origin + - Referer + X-Content-Type-Options: + - nosniff + X-Frame-Options: + - SAMEORIGIN + X-XSS-Protection: + - '0' + content-length: + - '185' + status: + code: 500 + message: Internal Server Error +version: 1 diff --git a/py/src/braintrust/integrations/discoveryengine/integration.py b/py/src/braintrust/integrations/discoveryengine/integration.py new file mode 100644 index 000000000..179c18461 --- /dev/null +++ b/py/src/braintrust/integrations/discoveryengine/integration.py @@ -0,0 +1,13 @@ +"""Discovery Engine integration orchestration.""" + +from braintrust.integrations.base import BaseIntegration + +from .patchers import PATCHERS + + +class DiscoveryEngineIntegration(BaseIntegration): + name = "discoveryengine" + import_names = ("google.cloud.discoveryengine_v1",) + distribution_names = ("google-cloud-discoveryengine",) + min_version = "0.20.3" + patchers = PATCHERS diff --git a/py/src/braintrust/integrations/discoveryengine/patchers.py b/py/src/braintrust/integrations/discoveryengine/patchers.py new file mode 100644 index 000000000..02de3ccdd --- /dev/null +++ b/py/src/braintrust/integrations/discoveryengine/patchers.py @@ -0,0 +1,92 @@ +"""Exact v1 GAPIC targets. No transport, constructor, or v1alpha/v1beta patches.""" + +from functools import partial + +from braintrust.integrations.base import FunctionWrapperPatcher + +from .tracing import _async_call, _call + + +class AnswerQueryPatcher(FunctionWrapperPatcher): + name = "discoveryengine.answer_query" + target_path = "ConversationalSearchServiceClient.answer_query" + wrapper = partial(_call, "answer_query") + + +class AsyncAnswerQueryPatcher(FunctionWrapperPatcher): + name = "discoveryengine.async.answer_query" + target_path = "ConversationalSearchServiceAsyncClient.answer_query" + wrapper = partial(_async_call, "answer_query") + + +class StreamAnswerQueryPatcher(FunctionWrapperPatcher): + name = "discoveryengine.stream_answer_query" + target_path = "ConversationalSearchServiceClient.stream_answer_query" + wrapper = partial(_call, "stream_answer_query") + + +class AsyncStreamAnswerQueryPatcher(FunctionWrapperPatcher): + name = "discoveryengine.async.stream_answer_query" + target_path = "ConversationalSearchServiceAsyncClient.stream_answer_query" + wrapper = partial(_async_call, "stream_answer_query") + + +class ConverseConversationPatcher(FunctionWrapperPatcher): + name = "discoveryengine.converse_conversation" + target_path = "ConversationalSearchServiceClient.converse_conversation" + wrapper = partial(_call, "converse_conversation") + + +class AsyncConverseConversationPatcher(FunctionWrapperPatcher): + name = "discoveryengine.async.converse_conversation" + target_path = "ConversationalSearchServiceAsyncClient.converse_conversation" + wrapper = partial(_async_call, "converse_conversation") + + +class CheckGroundingPatcher(FunctionWrapperPatcher): + name = "discoveryengine.check_grounding" + target_path = "GroundedGenerationServiceClient.check_grounding" + wrapper = partial(_call, "check_grounding") + + +class AsyncCheckGroundingPatcher(FunctionWrapperPatcher): + name = "discoveryengine.async.check_grounding" + target_path = "GroundedGenerationServiceAsyncClient.check_grounding" + wrapper = partial(_async_call, "check_grounding") + + +class RankPatcher(FunctionWrapperPatcher): + name = "discoveryengine.rank" + target_path = "RankServiceClient.rank" + wrapper = partial(_call, "rank") + + +class AsyncRankPatcher(FunctionWrapperPatcher): + name = "discoveryengine.async.rank" + target_path = "RankServiceAsyncClient.rank" + wrapper = partial(_async_call, "rank") + + +PATCHERS = ( + AnswerQueryPatcher, + AsyncAnswerQueryPatcher, + StreamAnswerQueryPatcher, + AsyncStreamAnswerQueryPatcher, + ConverseConversationPatcher, + AsyncConverseConversationPatcher, + CheckGroundingPatcher, + AsyncCheckGroundingPatcher, + RankPatcher, + AsyncRankPatcher, +) + + +def wrap_discoveryengine(client): + """Instrument one v1 client instance, returning the same client.""" + from google.cloud import discoveryengine_v1 + + for patcher in PATCHERS: + client_type = getattr(discoveryengine_v1, patcher.target_path.split(".")[0]) + if isinstance(client, client_type) and not patcher.is_patched(discoveryengine_v1, None): + patcher.wrap_target(client) + return client diff --git a/py/src/braintrust/integrations/discoveryengine/test_discoveryengine.py b/py/src/braintrust/integrations/discoveryengine/test_discoveryengine.py new file mode 100644 index 000000000..f8cd2240c --- /dev/null +++ b/py/src/braintrust/integrations/discoveryengine/test_discoveryengine.py @@ -0,0 +1,630 @@ +"""Real Discovery Engine responses, recorded over REST and gRPC.""" + +import json +import os +import subprocess +from contextlib import nullcontext +from pathlib import Path +from urllib.parse import urlsplit + +import pytest +import yaml +from braintrust import auto_instrument, logger +from braintrust.conftest import get_vcr_config +from braintrust.test_helpers import init_test_logger + + +pytest.importorskip("google.cloud.discoveryengine_v1") + +from google.auth.credentials import AnonymousCredentials +from google.cloud import discoveryengine_v1 as discoveryengine +from google.oauth2.credentials import Credentials + + +def _resource(request, cassette_dir, env_name, cassette_name, separator): + if request.config.getoption("--vcr-record") == "all": + value = os.getenv(f"BRAINTRUST_DISCOVERYENGINE_{env_name}") + if not value: + pytest.fail(f"Set BRAINTRUST_DISCOVERYENGINE_{env_name} to record Discovery Engine tests") + return value + cassette = yaml.safe_load((Path(cassette_dir) / cassette_name).read_text()) + resource = urlsplit(cassette["interactions"][0]["request"]["uri"]).path.removeprefix("/v1/") + return resource.split(separator)[0] + + +@pytest.fixture +def LOCATION(request, vcr_cassette_dir): + project = _resource(request, vcr_cassette_dir, "PROJECT", "test_rank.yaml", "/locations/") + return ( + f"projects/{project}/locations/global" + if request.config.getoption("--vcr-record") == "all" + else f"{project}/locations/global" + ) + + +@pytest.fixture +def SERVING_CONFIG(request, vcr_cassette_dir, LOCATION): + app = _resource(request, vcr_cassette_dir, "APP", "test_answer_query[False].yaml", ":answer") + return ( + f"{LOCATION}/collections/default_collection/engines/{app}/servingConfigs/default_search" + if request.config.getoption("--vcr-record") == "all" + else app + ) + + +@pytest.fixture +def DATASTORE_CONFIG(request, vcr_cassette_dir, LOCATION): + datastore = _resource(request, vcr_cassette_dir, "DATASTORE", "test_converse_conversation.yaml", "/conversations/") + return ( + f"{LOCATION}/collections/default_collection/dataStores/{datastore}/servingConfigs/default_search" + if request.config.getoption("--vcr-record") == "all" + else f"{datastore}/servingConfigs/default_search" + ) + + +@pytest.fixture +def DATASTORE(DATASTORE_CONFIG): + return DATASTORE_CONFIG.split("/dataStores/")[1].split("/")[0] + + +@pytest.fixture +def vcr_cassette_name(request): + marker = request.node.get_closest_marker("vcr") + return marker.args[0] if marker and marker.args else request.node.name + + +@pytest.fixture(scope="module") +def vcr_config(): + return {**get_vcr_config(), "match_on": ["method", "scheme", "host", "port", "path", "query", "body"]} + + +@pytest.fixture(scope="session") +def credentials(request): + if request.config.getoption("--vcr-record") == "all": + if not os.getenv("BRAINTRUST_DISCOVERYENGINE_PROJECT"): + pytest.fail("Set BRAINTRUST_DISCOVERYENGINE_PROJECT to record Discovery Engine tests") + # Refresh outside the recorded HTTP call; credentials never enter cassettes. + token = subprocess.check_output( + ["gcloud", "auth", "application-default", "print-access-token"], text=True + ).strip() + return Credentials(token=token) + return AnonymousCredentials() + + +@pytest.fixture +def memory_logger(): + init_test_logger("test-discoveryengine") + with logger._internal_with_memory_background_logger() as bgl: + yield bgl + + +@pytest.fixture +def rank_request(LOCATION): + return { + "ranking_config": f"{LOCATION}/rankingConfigs/default_ranking_config", + "model": "semantic-ranker-512@latest", + "query": "What is Braintrust?", + "records": [ + {"id": "1", "content": "Braintrust is a platform for evaluating and monitoring AI applications."}, + {"id": "2", "content": "The moon orbits the Earth."}, + ], + "top_n": 1, + } + + +@pytest.mark.vcr("test_rank.yaml") +@pytest.mark.parametrize("mode", ["manual", "manual_then_setup", "setup_then_manual"]) +def test_rank(memory_logger, credentials, rank_request, mode): + from braintrust.integrations.discoveryengine import setup_discoveryengine, wrap_discoveryengine + + client = discoveryengine.RankServiceClient(transport="rest", credentials=credentials) + untouched = discoveryengine.RankServiceClient(transport="rest", credentials=credentials) + original = untouched.rank + if mode == "setup_then_manual": + assert setup_discoveryengine() + assert setup_discoveryengine() + assert wrap_discoveryengine(client) is client + assert wrap_discoveryengine(client) is client + if mode == "manual_then_setup": + assert setup_discoveryengine() + if mode == "manual": + assert untouched.rank == original + assert not hasattr(untouched.rank, "__wrapped__") + result = client.rank(request=rank_request, retry=None) + assert result.records[0].id == "1" + spans = memory_logger.pop() + assert len(spans) == 1 + span = spans[0] + assert span["span_attributes"]["name"] == "discoveryengine.rank" + assert span["span_attributes"]["type"] == "llm" + assert span["metadata"]["provider"] == "google" + assert span["metadata"]["model"] == "semantic-ranker-512@latest" + assert span["input"]["query"] == "What is Braintrust?" + assert span["output"][0]["id"] == "1" + assert span["output"][0]["score"] == result.records[0].score + assert not {"tokens", "prompt_tokens", "completion_tokens"} & span["metrics"].keys() + assert span["context"]["span_origin"]["instrumentation"]["name"] == "discoveryengine-auto" + + +QUERY = "What was Alphabet's revenue in 2022?" + + +def _assert_generation_span(memory_logger, method, text): + spans = memory_logger.pop() + assert len(spans) == 1 + span = spans[0] + assert span["span_attributes"]["name"] == f"discoveryengine.{method}" + assert span["span_attributes"]["type"] == "llm" + assert span["metadata"]["provider"] == "google" + assert "model" not in span["metadata"] + assert span["output"][0]["message"]["content"] == text + assert not {"tokens", "prompt_tokens", "completion_tokens"} & span["metrics"].keys() + assert span["context"]["span_origin"]["instrumentation"]["name"] == "discoveryengine-auto" + json.dumps({key: span[key] for key in ("input", "output", "metadata")}) + return span + + +@pytest.mark.vcr +@pytest.mark.parametrize("stream", [False, True]) +def test_answer_query(memory_logger, credentials, stream, SERVING_CONFIG): + auto_instrument() + client = discoveryengine.ConversationalSearchServiceClient(transport="rest", credentials=credentials) + request = discoveryengine.AnswerQueryRequest( + serving_config=SERVING_CONFIG, + query={"text": QUERY}, + answer_generation_spec={"include_citations": True}, + ) + if stream: + chunks = list(client.stream_answer_query(request, retry=None, timeout=90)) + assert chunks[-1].answer.state == discoveryengine.Answer.State.SUCCEEDED + text = chunks[-1].answer.answer_text + method = "stream_answer_query" + else: + result = client.answer_query(request, retry=None, timeout=90) + text = result.answer.answer_text + method = "answer_query" + assert text and "could not be generated" not in text + span = _assert_generation_span(memory_logger, method, text) + assert span["input"] == [{"role": "user", "content": QUERY}] + assert span["output"][0]["citations"] + assert span["output"][0]["references"] + final_answer = chunks[-1].answer if stream else result.answer + assert len(span["output"][0]["references"]) == len(final_answer.references) + assert len(span["output"][0]["citations"]) == len(final_answer.citations) + if stream: + assert len([chunk for chunk in chunks if chunk.answer.answer_text]) > 1 + assert span["metrics"]["time_to_first_token"] >= 0 + + +@pytest.mark.vcr +def test_converse_conversation(memory_logger, credentials, DATASTORE, DATASTORE_CONFIG, LOCATION): + auto_instrument() + client = discoveryengine.ConversationalSearchServiceClient(transport="rest", credentials=credentials) + result = client.converse_conversation( + request={ + "name": f"{LOCATION}/collections/default_collection/dataStores/{DATASTORE}/conversations/-", + "query": {"input": QUERY}, + "serving_config": DATASTORE_CONFIG, + "summary_spec": {"summary_result_count": 3, "include_citations": True}, + }, + retry=None, + timeout=90, + ) + assert result.reply.summary.summary_text + span = _assert_generation_span(memory_logger, "converse_conversation", result.reply.summary.summary_text) + choice = span["output"][0] + assert "summary_with_metadata" not in choice + assert len(choice["references"]) == len(result.reply.summary.summary_with_metadata.references) + assert choice["citation_metadata"] + + +@pytest.mark.vcr +def test_check_grounding(memory_logger, credentials, LOCATION): + auto_instrument() + client = discoveryengine.GroundedGenerationServiceClient(transport="rest", credentials=credentials) + result = client.check_grounding( + request={ + "grounding_config": f"{LOCATION}/groundingConfigs/default_grounding_config", + "answer_candidate": "Braintrust evaluates AI applications.", + "facts": [{"fact_text": "Braintrust is a platform for evaluating AI applications."}], + }, + retry=None, + timeout=90, + ) + assert result.support_score > 0 + spans = memory_logger.pop() + assert len(spans) == 1 + assert spans[0]["span_attributes"]["name"] == "discoveryengine.check_grounding" + assert spans[0]["output"]["support_score"] == result.support_score + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "method", + [ + "answer_query", + "stream_answer_query", + "converse_conversation", + "check_grounding", + "rank", + ], +) +async def test_async_grpc( + memory_logger, + credentials, + request, + vcr_cassette_dir, + method, + DATASTORE, + DATASTORE_CONFIG, + LOCATION, + SERVING_CONFIG, +): + from braintrust.integrations.discoveryengine._test_grpc import grpc_cassette + + auto_instrument() + if method in ("answer_query", "stream_answer_query", "converse_conversation"): + client = discoveryengine.ConversationalSearchServiceAsyncClient(credentials=credentials) + response_type = discoveryengine.AnswerQueryResponse + payload = discoveryengine.AnswerQueryRequest( + serving_config=SERVING_CONFIG, query={"text": QUERY}, answer_generation_spec={"include_citations": True} + ) + if method == "converse_conversation": + response_type = discoveryengine.ConverseConversationResponse + payload = discoveryengine.ConverseConversationRequest( + name=f"{LOCATION}/collections/default_collection/dataStores/{DATASTORE}/conversations/-", + serving_config=DATASTORE_CONFIG, + query={"input": QUERY}, + summary_spec={"summary_result_count": 3}, + ) + elif method == "rank": + client = discoveryengine.RankServiceAsyncClient(credentials=credentials) + response_type = discoveryengine.RankResponse + payload = discoveryengine.RankRequest( + ranking_config=f"{LOCATION}/rankingConfigs/default_ranking_config", + query="What is Braintrust?", + records=[{"id": "1", "content": "Braintrust evaluates AI applications."}], + ) + else: + client = discoveryengine.GroundedGenerationServiceAsyncClient(credentials=credentials) + response_type = discoveryengine.CheckGroundingResponse + payload = discoveryengine.CheckGroundingRequest( + grounding_config=f"{LOCATION}/groundingConfigs/default_grounding_config", + answer_candidate="Braintrust evaluates AI applications.", + facts=[{"fact_text": "Braintrust is a platform for evaluating AI applications."}], + ) + streaming = method.startswith("stream_") + path = Path(vcr_cassette_dir) / f"test_async_grpc[{method}].json" + + try: + with grpc_cassette( + client, + method, + response_type, + path, + record=request.config.getoption("--vcr-record") == "all", + streaming=streaming, + ): + result = await getattr(client, method)(payload, retry=None, timeout=90) + if streaming: + chunks = [chunk async for chunk in result] + assert chunks + else: + assert isinstance(result, response_type) + spans = memory_logger.pop() + assert len(spans) == 1 + span = spans[0] + assert span["span_attributes"]["name"] == f"discoveryengine.{method}" + assert span["span_attributes"]["type"] == "llm" + assert span["metadata"]["provider"] == "google" + assert "model" not in span["metadata"] + assert span["context"]["span_origin"]["instrumentation"]["name"] == "discoveryengine-auto" + assert span["metrics"]["end"] >= span["metrics"]["start"] + assert not {"tokens", "prompt_tokens", "completion_tokens"} & span["metrics"].keys() + if method in ("answer_query", "stream_answer_query", "converse_conversation"): + text = ( + chunks[-1].answer.answer_text + if streaming + else result.reply.summary.summary_text + if method == "converse_conversation" + else result.answer.answer_text + ) + assert text + assert span["input"] == [{"role": "user", "content": QUERY}] + assert span["output"][0]["message"]["content"] == text + if streaming: + assert span["metrics"]["time_to_first_token"] >= 0 + elif method == "rank": + assert span["output"][0]["id"] == result.records[0].id + assert span["output"][0]["score"] == result.records[0].score + elif method == "check_grounding": + assert span["output"]["support_score"] == result.support_score + assert span["input"]["answer_candidate"] == payload.answer_candidate + json.dumps({key: span[key] for key in ("input", "output", "metadata")}) + finally: + await client.transport.close() + + +@pytest.fixture(autouse=True) +def restore_methods(): + from braintrust.integrations.discoveryengine.patchers import PATCHERS + + originals = [] + for patcher in PATCHERS: + class_name, method = patcher.target_path.split(".") + cls = getattr(discoveryengine, class_name) + original = getattr(cls, method) + originals.append((cls, method, original, patcher.patch_marker_attr())) + yield + for cls, method, original, marker in originals: + setattr(cls, method, original) + if hasattr(original, marker): + delattr(original, marker) + + +def test_patch_scope(): + import inspect + + from braintrust.integrations.discoveryengine import setup_discoveryengine + from braintrust.integrations.discoveryengine.patchers import PATCHERS + from google.cloud import discoveryengine_v1alpha, discoveryengine_v1beta + + untouched = [ + (discoveryengine.GroundedGenerationServiceClient, "generate_grounded_content"), + (discoveryengine.GroundedGenerationServiceClient, "stream_generate_grounded_content"), + (discoveryengine.GroundedGenerationServiceAsyncClient, "generate_grounded_content"), + (discoveryengine.GroundedGenerationServiceAsyncClient, "stream_generate_grounded_content"), + (discoveryengine.SearchServiceClient, "search"), + (discoveryengine.SearchServiceClient, "search_lite"), + (discoveryengine.AssistantServiceClient, "stream_assist"), + (discoveryengine.ConversationalSearchServiceClient, "get_answer"), + (discoveryengine.ConversationalSearchServiceClient, "create_conversation"), + (discoveryengine_v1alpha.RankServiceClient, "rank"), + (discoveryengine_v1beta.RankServiceClient, "rank"), + ] + originals = [inspect.getattr_static(cls, name) for cls, name in untouched] + assert setup_discoveryengine() + for (cls, name), original in zip(untouched, originals): + assert inspect.getattr_static(cls, name) is original + for patcher in PATCHERS: + assert patcher.is_patched(discoveryengine, "0.20.3") + target = patcher.resolve_target(discoveryengine, "0.20.3") + assert hasattr(target, "__wrapped__") + assert not hasattr(target.__wrapped__, "__wrapped__") + + +@pytest.mark.vcr("test_answer_query[True].yaml") +def test_stream_close_preserves_parent(memory_logger, credentials, SERVING_CONFIG): + from braintrust import current_span, start_span + from braintrust.integrations.discoveryengine import wrap_discoveryengine + + client = wrap_discoveryengine( + discoveryengine.ConversationalSearchServiceClient(transport="rest", credentials=credentials) + ) + with start_span(name="caller") as parent: + stream = client.stream_answer_query( + discoveryengine.AnswerQueryRequest( + serving_config=SERVING_CONFIG, + query={"text": QUERY}, + answer_generation_spec={"include_citations": True}, + ), + retry=None, + ) + assert current_span() is parent + next(stream) + assert current_span() is parent + stream.close() + stream.close() + assert current_span() is parent + spans = memory_logger.pop() + assert len(spans) == 2 + child = next(span for span in spans if span["span_attributes"]["name"] == "discoveryengine.stream_answer_query") + parent_row = next(span for span in spans if span["span_attributes"]["name"] == "caller") + assert child["span_parents"] == [parent_row["span_id"]] + assert "end" in child["metrics"] + + +def test_auto_instrument_subprocess(): + from braintrust.integrations.test_utils import verify_autoinstrument_script + + verify_autoinstrument_script("test_auto_discoveryengine.py") + + +@pytest.mark.vcr +@pytest.mark.parametrize("asynchronous_mode", [True, False]) +def test_answer_requested_model(memory_logger, credentials, asynchronous_mode, SERVING_CONFIG): + from braintrust.integrations.discoveryengine import setup_discoveryengine + + setup_discoveryengine() + client = discoveryengine.ConversationalSearchServiceClient(transport="rest", credentials=credentials) + from google.api_core.exceptions import BadRequest + + expected = ( + pytest.raises(BadRequest, match="asynchronous mode is deprecated") if asynchronous_mode else nullcontext() + ) + with expected: + result = client.answer_query( + request={ + "serving_config": SERVING_CONFIG, + "query": {"text": QUERY}, + "answer_generation_spec": {"model_spec": {"model_version": "stable"}}, + "asynchronous_mode": asynchronous_mode, + }, + retry=None, + timeout=90, + ) + spans = memory_logger.pop() + if asynchronous_mode: + assert spans == [] + else: + assert isinstance(result, discoveryengine.AnswerQueryResponse) + assert len(spans) == 1 + assert spans[0]["metadata"]["model"] == "stable" + + +@pytest.mark.vcr("test_rank.yaml") +def test_normalization_failure_does_not_change_result(memory_logger, credentials, monkeypatch, rank_request): + from braintrust.integrations.discoveryengine import setup_discoveryengine, tracing + + def broken(*args): + raise ValueError("injected extraction failure") + + monkeypatch.setattr(tracing, "_prepare", broken) + monkeypatch.setattr(tracing, "_output", broken) + setup_discoveryengine() + client = discoveryengine.RankServiceClient(transport="rest", credentials=credentials) + result = client.rank(request=rank_request, retry=None) + assert result.records[0].id == "1" + span = memory_logger.pop()[0] + assert "error" not in span + + +@pytest.mark.asyncio +@pytest.mark.parametrize("consume", ["read", "cancel", "aclose"]) +async def test_async_stream_lifecycle(memory_logger, credentials, vcr_cassette_dir, consume, SERVING_CONFIG): + from braintrust import current_span, start_span + from braintrust.integrations.discoveryengine import wrap_discoveryengine + from braintrust.integrations.discoveryengine._test_grpc import grpc_cassette + from grpc.aio import EOF + + client = wrap_discoveryengine(discoveryengine.ConversationalSearchServiceAsyncClient(credentials=credentials)) + payload = discoveryengine.AnswerQueryRequest( + serving_config=SERVING_CONFIG, + query={"text": QUERY}, + answer_generation_spec={"include_citations": True}, + ) + path = Path(vcr_cassette_dir) / "test_async_grpc[stream_answer_query].json" + try: + with ( + start_span(name="caller") as parent, + grpc_cassette( + client, + "stream_answer_query", + discoveryengine.AnswerQueryResponse, + path, + streaming=True, + ), + ): + stream = await client.stream_answer_query(payload, retry=None) + assert current_span() is parent + chunks = [] + if consume == "read": + while (chunk := await stream.read()) is not EOF: + chunks.append(chunk) + assert current_span() is parent + assert await stream.read() is EOF + else: + chunks.append(await stream.__anext__()) + if consume == "cancel": + assert stream.cancel() + else: + await stream.aclose() + assert stream.cancelled() + assert current_span() is parent + spans = memory_logger.pop() + assert len(spans) == 2 + child = next( + span for span in spans if span["span_attributes"]["name"] == "discoveryengine.stream_answer_query" + ) + parent_row = next(span for span in spans if span["span_attributes"]["name"] == "caller") + assert child["span_parents"] == [parent_row["span_id"]] + expected_text = ( + chunks[-1].answer.answer_text + if consume == "read" + else "".join(chunk.answer.answer_text for chunk in chunks) + ) + assert child["output"][0]["message"]["content"] == expected_text + assert "end" in child["metrics"] + finally: + await client.transport.close() + + +@pytest.mark.vcr +def test_rank_output_limit(memory_logger, credentials, LOCATION): + from braintrust.integrations.discoveryengine import setup_discoveryengine + + setup_discoveryengine() + client = discoveryengine.RankServiceClient(transport="rest", credentials=credentials) + result = client.rank( + request={ + "ranking_config": f"{LOCATION}/rankingConfigs/default_ranking_config", + "query": "AI evaluation", + "records": [{"id": str(i), "content": f"Evaluation example {i}."} for i in range(101)], + }, + retry=None, + ) + assert len(result.records) == 101 + span = memory_logger.pop()[0] + assert len(span["input"]["records"]) == 101 + assert len(span["output"]) == 100 + assert [record["id"] for record in span["output"]] == [result.records[i].id for i in range(100)] + + +@pytest.fixture +def error_request(stream, LOCATION, SERVING_CONFIG): + if stream: + return {"serving_config": SERVING_CONFIG} + return { + "ranking_config": f"{LOCATION}/rankingConfigs/default_ranking_config", + "query": "test", + "records": [{"id": "empty"}], + } + + +@pytest.mark.vcr +@pytest.mark.parametrize( + "stream,vcr_cassette_name", + [(False, "test_provider_error"), (True, "test_stream_provider_error")], +) +def test_provider_error(memory_logger, credentials, stream, error_request, vcr_cassette_name): + from braintrust.integrations.discoveryengine import setup_discoveryengine + from google.api_core.exceptions import BadRequest, InternalServerError + + setup_discoveryengine() + client_type = discoveryengine.ConversationalSearchServiceClient if stream else discoveryengine.RankServiceClient + client = client_type(transport="rest", credentials=credentials) + method = client.stream_answer_query if stream else client.rank + with pytest.raises(InternalServerError if stream else BadRequest): + result = method(request=error_request, retry=None) + if stream: + list(result) + spans = memory_logger.pop() + assert len(spans) == 1 + assert spans[0]["error"] + assert spans[0]["metrics"]["end"] >= spans[0]["metrics"]["start"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stream", [False, True]) +async def test_async_provider_error(memory_logger, credentials, request, vcr_cassette_dir, stream, error_request): + from braintrust.integrations.discoveryengine import setup_discoveryengine + from braintrust.integrations.discoveryengine._test_grpc import grpc_cassette + from google.api_core.exceptions import InternalServerError, InvalidArgument + + setup_discoveryengine() + client_type = ( + discoveryengine.ConversationalSearchServiceAsyncClient if stream else discoveryengine.RankServiceAsyncClient + ) + client = client_type(credentials=credentials) + method = "stream_answer_query" if stream else "rank" + cassette = "test_async_stream_provider_error.json" if stream else "test_async_provider_error.json" + try: + with grpc_cassette( + client, + method, + discoveryengine.AnswerQueryResponse if stream else discoveryengine.RankResponse, + Path(vcr_cassette_dir) / cassette, + record=request.config.getoption("--vcr-record") == "all", + streaming=stream, + ): + with pytest.raises(InternalServerError if stream else InvalidArgument): + result = await getattr(client, method)(request=error_request, retry=None) + if stream: + async for _ in result: + pass + spans = memory_logger.pop() + assert len(spans) == 1 + assert spans[0]["error"] + assert spans[0]["metrics"]["end"] >= spans[0]["metrics"]["start"] + finally: + await client.transport.close() diff --git a/py/src/braintrust/integrations/discoveryengine/tracing.py b/py/src/braintrust/integrations/discoveryengine/tracing.py new file mode 100644 index 000000000..867403403 --- /dev/null +++ b/py/src/braintrust/integrations/discoveryengine/tracing.py @@ -0,0 +1,319 @@ +"""Tracing for Discovery Engine's direct v1 generation and ranking calls. + +Requests are read without constructing or serializing a second GAPIC request. +Only selected protobuf submessages (citations, references, configuration) need +conversion; neither complete responses nor streams are serialized to JSON here. +""" + +import logging +import time +from collections.abc import Mapping +from itertools import islice + +from braintrust.logger import start_span +from wrapt import ObjectProxy + + +_LOG = logging.getLogger(__name__) +_INSTRUMENTATION = "discoveryengine-auto" +_MAX_RANK_RESULTS = 100 +_ANSWER_DETAILS = ("citations", "references", "grounding_supports", "related_questions", "answer_skipped_reasons") + + +def _get(value, name, default=None): + return value.get(name, default) if isinstance(value, Mapping) else getattr(value, name, default) + + +def _message_dict(value): + # Braintrust's serializer falls back to text for proto-plus messages. + # Convert only selected protobuf fields, leaving ordinary values alone. + if isinstance(value, Mapping): + return value + return type(value).to_dict(value, preserving_proto_field_name=True, always_print_fields_with_no_presence=False) + + +def _details(value, names): + return {name: field for name in names if (field := _get(value, name))} + + +def _request(args, kwargs): + request = args[0] if args else kwargs.get("request") + return request if request is not None else kwargs + + +def _prepare(method, request): + metadata = {"provider": "google"} + if method == "rank": + model = _get(request, "model") + metadata.update(_details(request, ("ranking_config", "top_n", "ignore_record_details_in_response"))) + span_input = { + "query": _get(request, "query", ""), + "records": [_details(record, ("id", "title", "content")) for record in _get(request, "records", ())], + } + elif method == "check_grounding": + model = None + metadata.update(_details(request, ("grounding_config",))) + grounding_spec = _get(request, "grounding_spec") + if grounding_spec: + metadata["grounding_spec"] = _message_dict(grounding_spec) + span_input = { + "answer_candidate": _get(request, "answer_candidate", ""), + "facts": [_message_dict(fact) for fact in _get(request, "facts", ())], + } + else: + converse = method == "converse_conversation" + spec = _get(request, "summary_spec" if converse else "answer_generation_spec") + model = _get(_get(spec, "model_spec"), "version" if converse else "model_version") + query = _get(request, "query") + span_input = [{"role": "user", "content": _get(query, "input" if converse else "text", "")}] + preamble = _get(_get(spec, "model_prompt_spec" if converse else "prompt_spec"), "preamble") + if preamble: + span_input.insert(0, {"role": "system", "content": preamble}) + metadata.update(_details(request, ("serving_config", "session", "name"))) + metadata.update(_details(spec, ("include_citations", "answer_language_code", "summary_result_count"))) + if model: + metadata["model"] = model + return span_input, metadata + + +def _choice(text, **details): + return {"index": 0, "message": {"role": "assistant", "content": text}, **details} + + +def _answer_details(answer): + details = {} + for name in _ANSWER_DETAILS: + values = _get(answer, name) + if values: + details[name] = ( + [_message_dict(value) for value in values] + if name in ("citations", "references", "grounding_supports") + else list(values) + ) + state = _get(answer, "state") + if state: + details["state"] = state.name if hasattr(state, "name") else state + # An optional zero score is meaningful; don't drop it by testing truthiness. + if answer is not None and "grounding_score" in answer: + details["grounding_score"] = _get(answer, "grounding_score") + return details + + +def _output(method, response): + if method == "rank": + return [ + {**_details(record, ("id", "title", "content")), "score": record.score} + for record in islice(response.records, _MAX_RANK_RESULTS) + ] + if method == "check_grounding": + return { + "support_score": response.support_score, + **{ + name: [_message_dict(item) for item in values] + for name in ("cited_chunks", "cited_facts", "claims") + if (values := getattr(response, name)) + }, + } + if method == "converse_conversation": + summary = response.reply.summary + summary_metadata = summary.summary_with_metadata + details = {} + if summary_metadata.citation_metadata: + details["citation_metadata"] = _message_dict(summary_metadata.citation_metadata) + if summary_metadata.references: + details["references"] = [_message_dict(reference) for reference in summary_metadata.references] + if summary.summary_skipped_reasons: + details["summary_skipped_reasons"] = list(summary.summary_skipped_reasons) + return [_choice(summary.summary_text, **details)] + return [_choice(response.answer.answer_text, **_answer_details(response.answer))] + + +def _safe_extract(fn, *args, default=None): + try: + return fn(*args) + except Exception: + _LOG.warning("Could not extract Discovery Engine trace data", exc_info=True) + return default + + +def _start(method, request): + span_input, metadata = _safe_extract(_prepare, method, request, default=(None, {"provider": "google"})) + return start_span( + name=f"discoveryengine.{method}", + type="llm", + input=span_input, + metadata=metadata, + internal={"instrumentation": _INSTRUMENTATION}, + set_current=method != "stream_answer_query", + ) + + +class _AnswerStreamState: + def __init__(self, span): + self.span = span + self.started = time.monotonic() + self.first_token = None + self.text = [] + self.details = {} + self.ended = False + + def add(self, response): + answer = response.answer + text = answer.answer_text + from google.cloud.discoveryengine_v1 import Answer + + # SUCCEEDED is a complete snapshot, following the text/citation deltas. + # Replace accumulated fields before consuming it to avoid duplication. + if answer.state == Answer.State.SUCCEEDED: + self.text.clear() + self.details.clear() + if text: + if self.first_token is None: + self.first_token = time.monotonic() - self.started + self.text.append(text) + # Keep protobuf leaves until final logging; no per-chunk serialization. + for name in _ANSWER_DETAILS: + values = getattr(answer, name) + if values: + self.details.setdefault(name, []).extend(values) + for name in ("state", "grounding_score"): + if name in answer: + self.details[name] = getattr(answer, name) + + def finish(self, error=None): + if self.ended: + return + self.ended = True + details = _safe_extract(_answer_details, self.details, default={}) + output = [_choice("".join(self.text), **details)] + metrics = {"time_to_first_token": self.first_token} if self.first_token is not None else {} + self.span.log(output=output, metrics=metrics, **({"error": error} if error is not None else {})) + self.span.end() + + +class _AnswerStream(ObjectProxy): + def __init__(self, stream, state): + super().__init__(stream) + self._self_state = state + self._self_iterator = iter(stream) + + def __iter__(self): + return self + + def __next__(self): + try: + response = next(self._self_iterator) + except StopIteration: + self._self_state.finish() + raise + except BaseException as error: + self._self_state.finish(error) + raise + _safe_extract(self._self_state.add, response) + return response + + def close(self): + try: + close = getattr(self.__wrapped__, "close", None) + if close is not None: + return close() + return self.__wrapped__.cancel() + finally: + self._self_state.finish() + + def cancel(self): + try: + return self.__wrapped__.cancel() + finally: + self._self_state.finish() + + +class _AsyncAnswerStream(ObjectProxy): + def __init__(self, stream, state): + super().__init__(stream) + self._self_state = state + self._self_iterator = None + + def __aiter__(self): + return self + + async def __anext__(self): + try: + if self._self_iterator is None: + self._self_iterator = self.__wrapped__.__aiter__() + response = await self._self_iterator.__anext__() + except StopAsyncIteration: + self._self_state.finish() + raise + except BaseException as error: + self._self_state.finish(error) + raise + _safe_extract(self._self_state.add, response) + return response + + async def read(self): + from grpc.aio import EOF + + try: + response = await self.__wrapped__.read() + except BaseException as error: + self._self_state.finish(error) + raise + if response is EOF: + self._self_state.finish() + else: + _safe_extract(self._self_state.add, response) + return response + + async def aclose(self): + try: + close = getattr(self.__wrapped__, "aclose", None) + if close is not None: + return await close() + self.__wrapped__.cancel() + finally: + self._self_state.finish() + + def cancel(self): + try: + return self.__wrapped__.cancel() + finally: + self._self_state.finish() + + +def _call(method, wrapped, instance, args, kwargs): + request = _request(args, kwargs) + if method == "answer_query" and _get(request, "asynchronous_mode", False): + return wrapped(*args, **kwargs) + span = _start(method, request) + if method == "stream_answer_query": + state = _AnswerStreamState(span) + try: + return _AnswerStream(wrapped(*args, **kwargs), state) + except BaseException as error: + state.finish(error) + raise + with span: + result = wrapped(*args, **kwargs) + output = _safe_extract(_output, method, result) + span.log(output=output) + return result + + +async def _async_call(method, wrapped, instance, args, kwargs): + request = _request(args, kwargs) + if method == "answer_query" and _get(request, "asynchronous_mode", False): + return await wrapped(*args, **kwargs) + span = _start(method, request) + if method == "stream_answer_query": + state = _AnswerStreamState(span) + try: + return _AsyncAnswerStream(await wrapped(*args, **kwargs), state) + except BaseException as error: + state.finish(error) + raise + with span: + result = await wrapped(*args, **kwargs) + output = _safe_extract(_output, method, result) + span.log(output=output) + return result diff --git a/py/uv.lock b/py/uv.lock index 289000b16..84e4db239 100644 --- a/py/uv.lock +++ b/py/uv.lock @@ -835,6 +835,7 @@ lint = [ { name = "cohere" }, { name = "dspy" }, { name = "google-adk" }, + { name = "google-cloud-discoveryengine" }, { name = "google-genai" }, { name = "huggingface-hub" }, { name = "instructor" }, @@ -1066,6 +1067,7 @@ lint = [ { name = "cohere" }, { name = "dspy" }, { name = "google-adk" }, + { name = "google-cloud-discoveryengine" }, { name = "google-genai" }, { name = "huggingface-hub" }, { name = "instructor" }, @@ -1735,7 +1737,7 @@ name = "coloredlogs" version = "15.0.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "humanfriendly", marker = "(python_full_version < '3.12' and extra == 'group-10-braintrust-lint') or extra == 'group-10-braintrust-test-livekit-agents' or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-openai-agents' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-openai-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-openai-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-openai-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-openai-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-openai-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-crewai' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-crewai' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-deepagents' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-deepagents' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-langchain' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-langchain' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands')" }, + { name = "humanfriendly", marker = "(python_full_version < '3.11' and extra == 'group-10-braintrust-lint') or extra == 'group-10-braintrust-test-livekit-agents' or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-openai-agents' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-openai-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-openai-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-openai-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-openai-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-openai-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-crewai' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-crewai' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-deepagents' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-deepagents' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-langchain' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-langchain' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/cc/c7/eed8f27100517e8c0e6b923d5f0845d0cb99763da6fdee00478f91db7325/coloredlogs-15.0.1.tar.gz", hash = "sha256:7c991aa71a4577af2f82600d8f8f3a89f936baeaf9b50a9c197da014e5bf16b0", size = 278520, upload-time = "2021-06-11T10:22:45.202Z" } wheels = [ @@ -2423,8 +2425,8 @@ name = "genai-prices" version = "0.0.73" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "httpx2" }, - { name = "pydantic" }, + { name = "httpx2", marker = "extra == 'group-10-braintrust-lint' or extra == 'group-10-braintrust-test-agentscope' or extra == 'group-10-braintrust-test-agno' or extra == 'group-10-braintrust-test-crewai' or extra == 'group-10-braintrust-test-deepagents' or extra == 'group-10-braintrust-test-langchain' or extra == 'group-10-braintrust-test-litellm' or extra != 'group-10-braintrust-test-livekit-agents' or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-otel-events') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands')" }, + { name = "pydantic", marker = "extra == 'group-10-braintrust-lint' or extra == 'group-10-braintrust-test-agentscope' or extra == 'group-10-braintrust-test-agno' or extra == 'group-10-braintrust-test-crewai' or extra == 'group-10-braintrust-test-deepagents' or extra == 'group-10-braintrust-test-langchain' or extra == 'group-10-braintrust-test-litellm' or extra != 'group-10-braintrust-test-livekit-agents' or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-otel-events') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/84/0b/4b430d9c6ff0c76e42b91fc4cedff84dfbefa1a35b885fad45a22d29303f/genai_prices-0.0.73.tar.gz", hash = "sha256:ddad5b23dadd7aac8a58ab5af7506addb062114037e96427324703e50bc35077", size = 88406, upload-time = "2026-07-29T12:48:58.215Z" } wheels = [ @@ -2483,6 +2485,28 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/61/39/24cd361daa2deba1f5ed35e401eff3cdc72b0371efe55a1145df7315c4c2/google_adk-2.9.0-py3-none-any.whl", hash = "sha256:3b15b91fdaf8e7193006d0ee4e745f45b4f39c62bdc1a6a8e60a4a8b1972bd3b", size = 4630844, upload-time = "2026-09-10T22:11:11.443Z" }, ] +[[package]] +name = "google-api-core" +version = "2.25.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-auth" }, + { name = "googleapis-common-protos", version = "1.75.0", source = { registry = "https://pypi.org/simple" } }, + { name = "proto-plus" }, + { name = "protobuf", version = "5.29.6", source = { registry = "https://pypi.org/simple" } }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/09/cd/63f1557235c2440fe0577acdbc32577c5c002684c58c7f4d770a92366a24/google_api_core-2.25.2.tar.gz", hash = "sha256:1c63aa6af0d0d5e37966f157a77f9396d820fba59f9e43e9415bc3dc5baff300", size = 166266, upload-time = "2025-10-03T00:07:34.778Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c8/d8/894716a5423933f5c8d2d5f04b16f052a515f78e815dab0c2c6f1fd105dc/google_api_core-2.25.2-py3-none-any.whl", hash = "sha256:e9a8f62d363dc8424a8497f4c2a47d6bcda6c16514c935629c257ab5d10210e7", size = 162489, upload-time = "2025-10-03T00:07:32.924Z" }, +] + +[package.optional-dependencies] +grpc = [ + { name = "grpcio" }, + { name = "grpcio-status" }, +] + [[package]] name = "google-auth" version = "2.58.0" @@ -2504,6 +2528,22 @@ requests = [ { name = "requests" }, ] +[[package]] +name = "google-cloud-discoveryengine" +version = "0.20.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-api-core", extra = ["grpc"], marker = "extra == 'group-10-braintrust-lint' or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-otel-events') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-openai-agents' and extra == 'group-10-braintrust-test-strands')" }, + { name = "google-auth" }, + { name = "grpcio" }, + { name = "proto-plus" }, + { name = "protobuf", version = "5.29.6", source = { registry = "https://pypi.org/simple" } }, +] +sdist = { url = "https://files.pythonhosted.org/packages/69/42/01c7f142b4c41abe9a6b6112bab357c3acf1cc5caf91bacfecc994762a34/google_cloud_discoveryengine-0.20.0.tar.gz", hash = "sha256:4268d32f4b72f6d19748b2040b21882100c5a4790985bcdb6fcf161a1cebcba9", size = 3720499, upload-time = "2026-06-03T15:28:14.173Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/3b/6a02a11e7d9b2e5a5ef3021558fffeae73a9f2a2a4574913d7a96414cdb5/google_cloud_discoveryengine-0.20.0-py3-none-any.whl", hash = "sha256:75e1a6501ad4882eba91f68f995bb593b158599b250a4686491096e210cc9b72", size = 3411962, upload-time = "2026-06-03T15:27:27.25Z" }, +] + [[package]] name = "google-genai" version = "2.23.0" @@ -2750,6 +2790,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/65/22/fc9a622d885a7a37ff972a12faaef443d74e47407181da70d0ab62ab41f0/grpcio-1.83.1-cp314-cp314-win_amd64.whl", hash = "sha256:47e6934ad38779271e2e7cc5f78a63a407cf3d98114c65c1fdbcd3f5a716f29b", size = 5302032, upload-time = "2026-08-28T07:09:09.285Z" }, ] +[[package]] +name = "grpcio-status" +version = "1.71.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "googleapis-common-protos", version = "1.75.0", source = { registry = "https://pypi.org/simple" } }, + { name = "grpcio" }, + { name = "protobuf", version = "5.29.6", source = { registry = "https://pypi.org/simple" } }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fd/d1/b6e9877fedae3add1afdeae1f89d1927d296da9cf977eca0eb08fb8a460e/grpcio_status-1.71.2.tar.gz", hash = "sha256:c7a97e176df71cdc2c179cd1847d7fc86cca5832ad12e9798d7fed6b7a1aab50", size = 13677, upload-time = "2025-06-28T04:24:05.426Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/67/58/317b0134129b556a93a3b0afe00ee675b5657f0155509e22fcb853bafe2d/grpcio_status-1.71.2-py3-none-any.whl", hash = "sha256:803c98cb6a8b7dc6dbb785b1111aed739f241ab5e9da0bba96888aa74704cfd3", size = 14424, upload-time = "2025-06-28T04:23:42.136Z" }, +] + [[package]] name = "h11" version = "0.16.0" @@ -2823,8 +2877,8 @@ name = "httpcore2" version = "2.12.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "h11" }, - { name = "truststore" }, + { name = "h11", marker = "extra == 'group-10-braintrust-lint' or extra == 'group-10-braintrust-test-agentscope' or extra == 'group-10-braintrust-test-agno' or extra == 'group-10-braintrust-test-crewai' or extra == 'group-10-braintrust-test-deepagents' or extra == 'group-10-braintrust-test-langchain' or extra == 'group-10-braintrust-test-litellm' or extra != 'group-10-braintrust-test-livekit-agents' or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-otel-events') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands')" }, + { name = "truststore", marker = "extra == 'group-10-braintrust-lint' or extra == 'group-10-braintrust-test-agentscope' or extra == 'group-10-braintrust-test-agno' or extra == 'group-10-braintrust-test-crewai' or extra == 'group-10-braintrust-test-deepagents' or extra == 'group-10-braintrust-test-langchain' or extra == 'group-10-braintrust-test-litellm' or extra != 'group-10-braintrust-test-livekit-agents' or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-otel-events') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/be/ad/f4f0e57345f1870f3e8cb624e058d7eca6e5a27d33bcc3311d9b618734cd/httpcore2-2.12.0.tar.gz", hash = "sha256:9293522bba0aa7c4c8e9e3f040c16575bd8868e155a77fa30c7a9085a5eae648", size = 67548, upload-time = "2026-08-18T13:22:08.211Z" } wheels = [ @@ -2868,7 +2922,7 @@ dependencies = [ { name = "anyio", marker = "(sys_platform != 'emscripten' and extra == 'group-10-braintrust-lint') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-test-agentscope') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-test-agno') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-test-crewai') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-test-deepagents') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-test-langchain') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-test-litellm') or (sys_platform != 'emscripten' and extra != 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-otel-events') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-openai-agents' and extra == 'group-10-braintrust-test-strands')" }, { name = "httpcore2", marker = "(sys_platform != 'emscripten' and extra == 'group-10-braintrust-lint') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-test-agentscope') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-test-agno') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-test-crewai') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-test-deepagents') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-test-langchain') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-test-litellm') or (sys_platform != 'emscripten' and extra != 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-otel-events') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-openai-agents' and extra == 'group-10-braintrust-test-strands')" }, { name = "httpx2-jsfetch", marker = "(python_full_version >= '3.12' and sys_platform == 'emscripten' and extra == 'group-10-braintrust-lint') or (python_full_version >= '3.12' and sys_platform == 'emscripten' and extra == 'group-10-braintrust-test-agentscope') or (python_full_version >= '3.12' and sys_platform == 'emscripten' and extra == 'group-10-braintrust-test-agno') or (python_full_version >= '3.12' and sys_platform == 'emscripten' and extra == 'group-10-braintrust-test-crewai') or (python_full_version >= '3.12' and sys_platform == 'emscripten' and extra == 'group-10-braintrust-test-deepagents') or (python_full_version >= '3.12' and sys_platform == 'emscripten' and extra == 'group-10-braintrust-test-langchain') or (python_full_version >= '3.12' and sys_platform == 'emscripten' and extra == 'group-10-braintrust-test-litellm') or (python_full_version >= '3.12' and sys_platform == 'emscripten' and extra != 'group-10-braintrust-test-livekit-agents') or (python_full_version < '3.12' and extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope') or (python_full_version < '3.12' and extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno') or (python_full_version < '3.12' and extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai') or (python_full_version < '3.12' and extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-deepagents') or (python_full_version < '3.12' and extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-langchain') or (python_full_version < '3.12' and extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-litellm') or (python_full_version < '3.12' and extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-livekit-agents') or (python_full_version < '3.12' and extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-openai-agents') or (python_full_version < '3.12' and extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (python_full_version < '3.12' and extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-strands') or (python_full_version < '3.12' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-agno') or (python_full_version < '3.12' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-crewai') or (python_full_version < '3.12' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-deepagents') or (python_full_version < '3.12' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-langchain') or (python_full_version < '3.12' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-litellm') or (python_full_version < '3.12' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-livekit-agents') or (python_full_version < '3.12' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-openai-agents') or (python_full_version < '3.12' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-strands') or (python_full_version < '3.12' and extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-crewai') or (python_full_version < '3.12' and extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-deepagents') or (python_full_version < '3.12' and extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-langchain') or (python_full_version < '3.12' and extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-litellm') or (python_full_version < '3.12' and extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-livekit-agents') or (python_full_version < '3.12' and extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-openai-agents') or (python_full_version < '3.12' and extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-strands') or (python_full_version < '3.12' and extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-deepagents') or (python_full_version < '3.12' and extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-langchain') or (python_full_version < '3.12' and extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-litellm') or (python_full_version < '3.12' and extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-livekit-agents') or (python_full_version < '3.12' and extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-openai-agents') or (python_full_version < '3.12' and extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-strands') or (python_full_version < '3.12' and extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-langchain') or (python_full_version < '3.12' and extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-litellm') or (python_full_version < '3.12' and extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-livekit-agents') or (python_full_version < '3.12' and extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-openai-agents') or (python_full_version < '3.12' and extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-strands') or (python_full_version < '3.12' and extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-litellm') or (python_full_version < '3.12' and extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-livekit-agents') or (python_full_version < '3.12' and extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-openai-agents') or (python_full_version < '3.12' and extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-strands') or (python_full_version < '3.12' and extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-livekit-agents') or (python_full_version < '3.12' and extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-openai-agents') or (python_full_version < '3.12' and extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-strands') or (python_full_version < '3.12' and extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (python_full_version < '3.12' and extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (python_full_version < '3.12' and extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-otel-events') or (python_full_version < '3.12' and extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (python_full_version < '3.12' and extra == 'group-10-braintrust-test-openai-agents' and extra == 'group-10-braintrust-test-strands') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-deepagents') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-langchain') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-litellm') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-livekit-agents') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-openai-agents') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-strands') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-agno') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-crewai') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-deepagents') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-langchain') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-litellm') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-livekit-agents') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-openai-agents') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-strands') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-crewai') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-deepagents') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-langchain') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-litellm') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-livekit-agents') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-openai-agents') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-strands') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-deepagents') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-langchain') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-litellm') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-livekit-agents') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-openai-agents') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-strands') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-langchain') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-litellm') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-livekit-agents') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-openai-agents') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-strands') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-litellm') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-livekit-agents') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-openai-agents') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-strands') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-livekit-agents') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-openai-agents') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-strands') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-otel-events') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-test-openai-agents' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-crewai' and extra != 'group-10-braintrust-test-deepagents' and extra != 'group-10-braintrust-test-langchain' and extra != 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra != 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-crewai' and extra != 'group-10-braintrust-test-deepagents' and extra != 'group-10-braintrust-test-langchain' and extra != 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra != 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-crewai' and extra != 'group-10-braintrust-test-deepagents' and extra != 'group-10-braintrust-test-langchain' and extra != 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-otel-events') or (extra != 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-crewai' and extra != 'group-10-braintrust-test-deepagents' and extra != 'group-10-braintrust-test-langchain' and extra != 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands')" }, - { name = "idna" }, + { name = "idna", marker = "extra == 'group-10-braintrust-lint' or extra == 'group-10-braintrust-test-agentscope' or extra == 'group-10-braintrust-test-agno' or extra == 'group-10-braintrust-test-crewai' or extra == 'group-10-braintrust-test-deepagents' or extra == 'group-10-braintrust-test-langchain' or extra == 'group-10-braintrust-test-litellm' or extra != 'group-10-braintrust-test-livekit-agents' or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-otel-events') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands')" }, { name = "truststore", marker = "(sys_platform != 'emscripten' and extra == 'group-10-braintrust-lint') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-test-agentscope') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-test-agno') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-test-crewai') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-test-deepagents') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-test-langchain') or (sys_platform != 'emscripten' and extra == 'group-10-braintrust-test-litellm') or (sys_platform != 'emscripten' and extra != 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-otel-events') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-openai-agents' and extra == 'group-10-braintrust-test-strands')" }, { name = "typing-extensions", marker = "(python_full_version < '3.13' and extra == 'group-10-braintrust-lint') or (python_full_version < '3.13' and extra == 'group-10-braintrust-test-agentscope') or (python_full_version < '3.13' and extra == 'group-10-braintrust-test-agno') or (python_full_version < '3.13' and extra == 'group-10-braintrust-test-crewai') or (python_full_version < '3.13' and extra == 'group-10-braintrust-test-deepagents') or (python_full_version < '3.13' and extra == 'group-10-braintrust-test-langchain') or (python_full_version < '3.13' and extra == 'group-10-braintrust-test-litellm') or (python_full_version < '3.13' and extra != 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-otel-events') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-openai-agents' and extra == 'group-10-braintrust-test-strands')" }, ] @@ -2911,7 +2965,7 @@ name = "humanfriendly" version = "10.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pyreadline3", marker = "(python_full_version < '3.12' and sys_platform == 'win32' and extra == 'group-10-braintrust-lint') or (python_full_version >= '3.12' and extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope') or (python_full_version >= '3.12' and extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno') or (python_full_version >= '3.12' and extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai') or (python_full_version >= '3.12' and extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-deepagents') or (python_full_version >= '3.12' and extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-langchain') or (python_full_version >= '3.12' and extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-litellm') or (python_full_version >= '3.12' and extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-livekit-agents') or (python_full_version >= '3.12' and extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-openai-agents') or (python_full_version >= '3.12' and extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (python_full_version >= '3.12' and extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-strands') or (sys_platform != 'win32' and extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope') or (sys_platform != 'win32' and extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno') or (sys_platform != 'win32' and extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai') or (sys_platform != 'win32' and extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-deepagents') or (sys_platform != 'win32' and extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-langchain') or (sys_platform != 'win32' and extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-litellm') or (sys_platform != 'win32' and extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-livekit-agents') or (sys_platform != 'win32' and extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-openai-agents') or (sys_platform != 'win32' and extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (sys_platform != 'win32' and extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-strands') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-agno') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-crewai') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-deepagents') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-langchain') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-litellm') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-livekit-agents') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-openai-agents') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-strands') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-crewai') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-deepagents') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-langchain') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-litellm') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-livekit-agents') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-openai-agents') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-strands') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-deepagents') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-langchain') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-litellm') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-livekit-agents') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-openai-agents') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-strands') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-langchain') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-litellm') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-livekit-agents') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-openai-agents') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-strands') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-litellm') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-livekit-agents') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-openai-agents') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-strands') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-livekit-agents') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-openai-agents') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-strands') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-otel-events') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-openai-agents' and extra == 'group-10-braintrust-test-strands') or (sys_platform == 'win32' and extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-livekit-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-agno') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-crewai') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-deepagents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-langchain') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-litellm') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-livekit-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-openai-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-crewai') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-deepagents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-langchain') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-litellm') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-livekit-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-openai-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-deepagents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-langchain') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-litellm') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-livekit-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-openai-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-langchain') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-litellm') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-livekit-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-openai-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-litellm') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-livekit-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-openai-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-livekit-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-openai-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents' and extra == 'group-10-braintrust-test-strands')" }, + { name = "pyreadline3", marker = "(python_full_version < '3.11' and sys_platform == 'win32' and extra == 'group-10-braintrust-lint') or (python_full_version >= '3.11' and extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope') or (python_full_version >= '3.11' and extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno') or (python_full_version >= '3.11' and extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai') or (python_full_version >= '3.11' and extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-deepagents') or (python_full_version >= '3.11' and extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-langchain') or (python_full_version >= '3.11' and extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-litellm') or (python_full_version >= '3.11' and extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-livekit-agents') or (python_full_version >= '3.11' and extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-openai-agents') or (python_full_version >= '3.11' and extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (python_full_version >= '3.11' and extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-strands') or (sys_platform != 'win32' and extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope') or (sys_platform != 'win32' and extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno') or (sys_platform != 'win32' and extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai') or (sys_platform != 'win32' and extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-deepagents') or (sys_platform != 'win32' and extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-langchain') or (sys_platform != 'win32' and extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-litellm') or (sys_platform != 'win32' and extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-livekit-agents') or (sys_platform != 'win32' and extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-openai-agents') or (sys_platform != 'win32' and extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (sys_platform != 'win32' and extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-strands') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-agno') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-crewai') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-deepagents') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-langchain') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-litellm') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-livekit-agents') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-openai-agents') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-strands') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-crewai') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-deepagents') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-langchain') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-litellm') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-livekit-agents') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-openai-agents') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-strands') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-deepagents') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-langchain') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-litellm') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-livekit-agents') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-openai-agents') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-strands') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-langchain') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-litellm') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-livekit-agents') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-openai-agents') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-strands') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-litellm') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-livekit-agents') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-openai-agents') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-strands') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-livekit-agents') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-openai-agents') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-strands') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-otel-events') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (sys_platform != 'win32' and extra == 'group-10-braintrust-test-openai-agents' and extra == 'group-10-braintrust-test-strands') or (sys_platform == 'win32' and extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-livekit-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-agno') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-crewai') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-deepagents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-langchain') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-litellm') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-livekit-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-openai-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-crewai') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-deepagents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-langchain') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-litellm') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-livekit-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-openai-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-deepagents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-langchain') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-litellm') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-livekit-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-openai-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-langchain') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-litellm') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-livekit-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-openai-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-litellm') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-livekit-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-openai-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-livekit-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-openai-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents' and extra == 'group-10-braintrust-test-strands')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/cc/3f/2c29224acb2e2df4d2046e4c73ee2662023c58ff5b113c4c1adac0886c43/humanfriendly-10.0.tar.gz", hash = "sha256:6b0b831ce8f15f7300721aa49829fc4e83921a9a301cc7f606be6686a2288ddc", size = 360702, upload-time = "2021-09-17T21:40:43.31Z" } wheels = [ @@ -4085,16 +4139,16 @@ name = "logfire" version = "4.32.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "executing" }, - { name = "opentelemetry-exporter-otlp-proto-http", version = "1.39.1", source = { registry = "https://pypi.org/simple" } }, - { name = "opentelemetry-instrumentation" }, - { name = "opentelemetry-sdk", version = "1.39.1", source = { registry = "https://pypi.org/simple" } }, + { name = "executing", marker = "extra == 'group-10-braintrust-lint' or extra == 'group-10-braintrust-test-agentscope' or extra == 'group-10-braintrust-test-agno' or extra == 'group-10-braintrust-test-crewai' or extra == 'group-10-braintrust-test-deepagents' or extra == 'group-10-braintrust-test-langchain' or extra == 'group-10-braintrust-test-litellm' or extra != 'group-10-braintrust-test-livekit-agents' or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-otel-events') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands')" }, + { name = "opentelemetry-exporter-otlp-proto-http", version = "1.39.1", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'group-10-braintrust-lint' or extra == 'group-10-braintrust-test-agentscope' or extra == 'group-10-braintrust-test-agno' or extra == 'group-10-braintrust-test-crewai' or extra == 'group-10-braintrust-test-deepagents' or extra == 'group-10-braintrust-test-langchain' or extra == 'group-10-braintrust-test-litellm' or extra != 'group-10-braintrust-test-livekit-agents' or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-otel-events') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands')" }, + { name = "opentelemetry-instrumentation", marker = "extra == 'group-10-braintrust-lint' or extra == 'group-10-braintrust-test-agentscope' or extra == 'group-10-braintrust-test-agno' or extra == 'group-10-braintrust-test-crewai' or extra == 'group-10-braintrust-test-deepagents' or extra == 'group-10-braintrust-test-langchain' or extra == 'group-10-braintrust-test-litellm' or extra != 'group-10-braintrust-test-livekit-agents' or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-otel-events') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands')" }, + { name = "opentelemetry-sdk", version = "1.39.1", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'group-10-braintrust-lint' or extra == 'group-10-braintrust-test-agentscope' or extra == 'group-10-braintrust-test-agno' or extra == 'group-10-braintrust-test-crewai' or extra == 'group-10-braintrust-test-deepagents' or extra == 'group-10-braintrust-test-langchain' or extra == 'group-10-braintrust-test-litellm' or extra != 'group-10-braintrust-test-livekit-agents' or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-otel-events') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands')" }, { name = "protobuf", version = "5.29.6", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'group-10-braintrust-lint' or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-otel-events') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-openai-agents' and extra == 'group-10-braintrust-test-strands')" }, { name = "protobuf", version = "6.33.6", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'group-10-braintrust-test-agentscope' or extra == 'group-10-braintrust-test-agno' or extra == 'group-10-braintrust-test-crewai' or extra == 'group-10-braintrust-test-deepagents' or extra == 'group-10-braintrust-test-langchain' or extra == 'group-10-braintrust-test-litellm' or extra == 'group-10-braintrust-test-openai-agents' or extra == 'group-10-braintrust-test-pydantic-ai-logfire' or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-livekit-agents') or (extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-otel-events') or (extra != 'group-10-braintrust-test-pydantic-ai-otel-events' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-livekit-agents')" }, { name = "rich", version = "14.3.4", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'group-10-braintrust-lint' or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-otel-events') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-openai-agents' and extra == 'group-10-braintrust-test-strands')" }, { name = "rich", version = "15.0.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'group-10-braintrust-test-agentscope' or extra == 'group-10-braintrust-test-agno' or extra == 'group-10-braintrust-test-crewai' or extra == 'group-10-braintrust-test-deepagents' or extra == 'group-10-braintrust-test-langchain' or extra == 'group-10-braintrust-test-litellm' or extra == 'group-10-braintrust-test-openai-agents' or extra == 'group-10-braintrust-test-pydantic-ai-logfire' or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-livekit-agents') or (extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-otel-events') or (extra != 'group-10-braintrust-test-pydantic-ai-otel-events' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-livekit-agents')" }, { name = "tomli", marker = "(python_full_version < '3.11' and extra == 'group-10-braintrust-lint') or (python_full_version < '3.11' and extra == 'group-10-braintrust-test-agentscope') or (python_full_version < '3.11' and extra == 'group-10-braintrust-test-agno') or (python_full_version < '3.11' and extra == 'group-10-braintrust-test-crewai') or (python_full_version < '3.11' and extra == 'group-10-braintrust-test-deepagents') or (python_full_version < '3.11' and extra == 'group-10-braintrust-test-langchain') or (python_full_version < '3.11' and extra == 'group-10-braintrust-test-litellm') or (python_full_version < '3.11' and extra != 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-otel-events') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-openai-agents' and extra == 'group-10-braintrust-test-strands')" }, - { name = "typing-extensions" }, + { name = "typing-extensions", marker = "extra == 'group-10-braintrust-lint' or extra == 'group-10-braintrust-test-agentscope' or extra == 'group-10-braintrust-test-agno' or extra == 'group-10-braintrust-test-crewai' or extra == 'group-10-braintrust-test-deepagents' or extra == 'group-10-braintrust-test-langchain' or extra == 'group-10-braintrust-test-litellm' or extra != 'group-10-braintrust-test-livekit-agents' or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-otel-events') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b5/d7/70c6def7f3f459b2d57aa7fb37863d31b8d877e391547f200ee8c31d2e30/logfire-4.32.1.tar.gz", hash = "sha256:8e7ff418b5f2629c8a8e9426283ff82c760a30f24516c4c389d6cbb1d9768c58", size = 1089612, upload-time = "2026-04-15T14:11:57.518Z" } wheels = [ @@ -4893,15 +4947,15 @@ name = "onnxruntime" version = "1.23.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "coloredlogs", marker = "(python_full_version < '3.12' and extra == 'group-10-braintrust-lint') or extra == 'group-10-braintrust-test-livekit-agents' or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-openai-agents' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-openai-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-openai-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-openai-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-openai-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-openai-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-crewai' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-crewai' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-deepagents' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-deepagents' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-langchain' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-langchain' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands')" }, - { name = "flatbuffers", marker = "(python_full_version < '3.12' and extra == 'group-10-braintrust-lint') or extra == 'group-10-braintrust-test-livekit-agents' or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-openai-agents' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-openai-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-openai-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-openai-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-openai-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-openai-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-crewai' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-crewai' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-deepagents' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-deepagents' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-langchain' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-langchain' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands')" }, + { name = "coloredlogs", marker = "(python_full_version < '3.11' and extra == 'group-10-braintrust-lint') or extra == 'group-10-braintrust-test-livekit-agents' or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-openai-agents' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-openai-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-openai-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-openai-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-openai-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-openai-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-crewai' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-crewai' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-deepagents' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-deepagents' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-langchain' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-langchain' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands')" }, + { name = "flatbuffers", marker = "(python_full_version < '3.11' and extra == 'group-10-braintrust-lint') or extra == 'group-10-braintrust-test-livekit-agents' or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-openai-agents' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-openai-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-openai-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-openai-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-openai-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-openai-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-crewai' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-crewai' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-deepagents' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-deepagents' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-langchain' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-langchain' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands')" }, { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and extra == 'group-10-braintrust-lint') or (python_full_version < '3.11' and extra != 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-crewai' and extra != 'group-10-braintrust-test-deepagents' and extra != 'group-10-braintrust-test-langchain' and extra != 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-otel-events') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-openai-agents' and extra == 'group-10-braintrust-test-strands')" }, - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version == '3.11.*' and extra == 'group-10-braintrust-lint') or (python_full_version == '3.11.*' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-otel-events') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-openai-agents' and extra == 'group-10-braintrust-test-strands')" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version == '3.11.*' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-otel-events') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-openai-agents' and extra == 'group-10-braintrust-test-strands')" }, { name = "numpy", version = "2.5.3", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version >= '3.12' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-otel-events') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-openai-agents' and extra == 'group-10-braintrust-test-strands')" }, - { name = "packaging", marker = "(python_full_version < '3.12' and extra == 'group-10-braintrust-lint') or extra == 'group-10-braintrust-test-livekit-agents' or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-openai-agents' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-openai-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-openai-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-openai-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-openai-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-openai-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-crewai' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-crewai' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-deepagents' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-deepagents' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-langchain' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-langchain' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands')" }, - { name = "protobuf", version = "5.29.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.12' and extra == 'group-10-braintrust-lint') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-otel-events') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-openai-agents' and extra == 'group-10-braintrust-test-strands')" }, + { name = "packaging", marker = "(python_full_version < '3.11' and extra == 'group-10-braintrust-lint') or extra == 'group-10-braintrust-test-livekit-agents' or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-openai-agents' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-openai-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-openai-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-openai-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-openai-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-openai-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-crewai' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-crewai' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-deepagents' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-deepagents' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-langchain' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-langchain' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands')" }, + { name = "protobuf", version = "5.29.6", source = { registry = "https://pypi.org/simple" }, marker = "(python_full_version < '3.11' and extra == 'group-10-braintrust-lint') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-otel-events') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-openai-agents' and extra == 'group-10-braintrust-test-strands')" }, { name = "protobuf", version = "6.33.6", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'group-10-braintrust-test-livekit-agents' or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-openai-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-crewai' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-crewai' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-deepagents' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-deepagents' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-langchain' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-langchain' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands')" }, - { name = "sympy", marker = "(python_full_version < '3.12' and extra == 'group-10-braintrust-lint') or extra == 'group-10-braintrust-test-livekit-agents' or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-openai-agents' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-openai-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-openai-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-openai-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-openai-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-openai-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-crewai' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-crewai' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-deepagents' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-deepagents' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-langchain' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-langchain' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands')" }, + { name = "sympy", marker = "(python_full_version < '3.11' and extra == 'group-10-braintrust-lint') or extra == 'group-10-braintrust-test-livekit-agents' or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-openai-agents' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-openai-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-openai-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-openai-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-openai-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-openai-agents') or (extra != 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-crewai' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-crewai' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-deepagents' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-deepagents' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-langchain' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-langchain' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/b3/84/42b8a11c9ebfb042071aaab73d17829fc094126e30caf65b18a94c3a5116/onnxruntime-1.23.1-cp310-cp310-macosx_13_0_arm64.whl", hash = "sha256:6b5257157d319abc87aa17294a9acf17119c6ecfdf9531017239b9022334f9b7", size = 17192895, upload-time = "2025-10-08T04:25:21.961Z" }, @@ -5007,15 +5061,15 @@ resolution-markers = [ "python_full_version < '3.11' and sys_platform == 'darwin'", ] dependencies = [ - { name = "anyio" }, - { name = "distro" }, - { name = "httpx" }, + { name = "anyio", marker = "extra == 'group-10-braintrust-lint' or extra == 'group-10-braintrust-test-crewai' or extra == 'group-10-braintrust-test-deepagents' or extra == 'group-10-braintrust-test-langchain' or extra == 'group-10-braintrust-test-livekit-agents' or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-openai-agents' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-litellm' and extra != 'group-10-braintrust-test-openai-agents')" }, + { name = "distro", marker = "extra == 'group-10-braintrust-lint' or extra == 'group-10-braintrust-test-crewai' or extra == 'group-10-braintrust-test-deepagents' or extra == 'group-10-braintrust-test-langchain' or extra == 'group-10-braintrust-test-livekit-agents' or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-openai-agents' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-litellm' and extra != 'group-10-braintrust-test-openai-agents')" }, + { name = "httpx", marker = "extra == 'group-10-braintrust-lint' or extra == 'group-10-braintrust-test-crewai' or extra == 'group-10-braintrust-test-deepagents' or extra == 'group-10-braintrust-test-langchain' or extra == 'group-10-braintrust-test-livekit-agents' or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-openai-agents' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-litellm' and extra != 'group-10-braintrust-test-openai-agents')" }, { name = "jiter", version = "0.14.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'group-10-braintrust-lint' or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-otel-events') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-openai-agents' and extra == 'group-10-braintrust-test-strands')" }, { name = "jiter", version = "0.17.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'group-10-braintrust-test-crewai' or extra == 'group-10-braintrust-test-deepagents' or extra == 'group-10-braintrust-test-langchain' or extra == 'group-10-braintrust-test-livekit-agents' or extra == 'group-10-braintrust-test-strands' or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agentscope') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-crewai' and extra != 'group-10-braintrust-test-deepagents' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-litellm') or (extra != 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-litellm' and extra != 'group-10-braintrust-test-openai-agents' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-crewai' and extra != 'group-10-braintrust-test-deepagents' and extra != 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-litellm') or (extra != 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-litellm' and extra != 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-crewai' and extra != 'group-10-braintrust-test-deepagents' and extra != 'group-10-braintrust-test-langchain' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-crewai' and extra != 'group-10-braintrust-test-deepagents' and extra != 'group-10-braintrust-test-langchain' and extra != 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents')" }, - { name = "pydantic" }, - { name = "sniffio" }, - { name = "tqdm" }, - { name = "typing-extensions" }, + { name = "pydantic", marker = "extra == 'group-10-braintrust-lint' or extra == 'group-10-braintrust-test-crewai' or extra == 'group-10-braintrust-test-deepagents' or extra == 'group-10-braintrust-test-langchain' or extra == 'group-10-braintrust-test-livekit-agents' or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-openai-agents' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-litellm' and extra != 'group-10-braintrust-test-openai-agents')" }, + { name = "sniffio", marker = "extra == 'group-10-braintrust-lint' or extra == 'group-10-braintrust-test-crewai' or extra == 'group-10-braintrust-test-deepagents' or extra == 'group-10-braintrust-test-langchain' or extra == 'group-10-braintrust-test-livekit-agents' or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-openai-agents' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-litellm' and extra != 'group-10-braintrust-test-openai-agents')" }, + { name = "tqdm", marker = "extra == 'group-10-braintrust-lint' or extra == 'group-10-braintrust-test-crewai' or extra == 'group-10-braintrust-test-deepagents' or extra == 'group-10-braintrust-test-langchain' or extra == 'group-10-braintrust-test-livekit-agents' or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-openai-agents' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-litellm' and extra != 'group-10-braintrust-test-openai-agents')" }, + { name = "typing-extensions", marker = "extra == 'group-10-braintrust-lint' or extra == 'group-10-braintrust-test-crewai' or extra == 'group-10-braintrust-test-deepagents' or extra == 'group-10-braintrust-test-langchain' or extra == 'group-10-braintrust-test-livekit-agents' or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-openai-agents' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-litellm' and extra != 'group-10-braintrust-test-openai-agents')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ed/59/bdcc6b759b8c42dd73afaf5bf8f902c04b37987a5514dbc1c64dba390fef/openai-2.32.0.tar.gz", hash = "sha256:c54b27a9e4cb8d51f0dd94972ffd1a04437efeb259a9e60d8922b8bd26fe55e0", size = 693286, upload-time = "2026-04-15T22:28:19.434Z" } wheels = [ @@ -5074,15 +5128,15 @@ resolution-markers = [ "python_full_version < '3.11' and sys_platform == 'darwin'", ] dependencies = [ - { name = "griffelib" }, + { name = "griffelib", marker = "extra == 'group-10-braintrust-lint' or extra == 'group-10-braintrust-test-agentscope' or extra == 'group-10-braintrust-test-agno' or extra == 'group-10-braintrust-test-crewai' or extra == 'group-10-braintrust-test-deepagents' or extra == 'group-10-braintrust-test-langchain' or extra != 'group-10-braintrust-test-litellm' or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-strands')" }, { name = "mcp", marker = "extra == 'group-10-braintrust-lint' or extra == 'group-10-braintrust-test-agentscope' or extra == 'group-10-braintrust-test-agno' or extra == 'group-10-braintrust-test-crewai' or extra == 'group-10-braintrust-test-deepagents' or extra == 'group-10-braintrust-test-langchain' or extra != 'group-10-braintrust-test-litellm' or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-strands')" }, { name = "openai", version = "2.31.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'group-10-braintrust-test-agentscope' or extra == 'group-10-braintrust-test-agno' or extra == 'group-10-braintrust-test-openai-agents' or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-crewai') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-lint' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-deepagents') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-crewai' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-langchain') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-deepagents' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-langchain' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-otel-events') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-openai-agents' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-lint' and extra != 'group-10-braintrust-test-openai-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-crewai' and extra != 'group-10-braintrust-test-openai-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-deepagents' and extra != 'group-10-braintrust-test-openai-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-langchain' and extra != 'group-10-braintrust-test-openai-agents' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-litellm' and extra != 'group-10-braintrust-test-openai-agents' and extra == 'group-10-braintrust-test-strands')" }, { name = "openai", version = "2.32.0", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'group-10-braintrust-lint' or extra == 'group-10-braintrust-test-crewai' or extra == 'group-10-braintrust-test-deepagents' or extra == 'group-10-braintrust-test-langchain' or extra == 'group-10-braintrust-test-livekit-agents' or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-agno') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agentscope' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-litellm') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-agno' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-strands') or (extra == 'group-10-braintrust-test-openai-agents' and extra == 'group-10-braintrust-test-strands') or (extra != 'group-10-braintrust-test-agentscope' and extra != 'group-10-braintrust-test-agno' and extra != 'group-10-braintrust-test-litellm' and extra != 'group-10-braintrust-test-openai-agents')" }, - { name = "pydantic" }, - { name = "requests" }, - { name = "types-requests" }, - { name = "typing-extensions" }, - { name = "websockets" }, + { name = "pydantic", marker = "extra == 'group-10-braintrust-lint' or extra == 'group-10-braintrust-test-agentscope' or extra == 'group-10-braintrust-test-agno' or extra == 'group-10-braintrust-test-crewai' or extra == 'group-10-braintrust-test-deepagents' or extra == 'group-10-braintrust-test-langchain' or extra != 'group-10-braintrust-test-litellm' or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-strands')" }, + { name = "requests", marker = "extra == 'group-10-braintrust-lint' or extra == 'group-10-braintrust-test-agentscope' or extra == 'group-10-braintrust-test-agno' or extra == 'group-10-braintrust-test-crewai' or extra == 'group-10-braintrust-test-deepagents' or extra == 'group-10-braintrust-test-langchain' or extra != 'group-10-braintrust-test-litellm' or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-strands')" }, + { name = "types-requests", marker = "extra == 'group-10-braintrust-lint' or extra == 'group-10-braintrust-test-agentscope' or extra == 'group-10-braintrust-test-agno' or extra == 'group-10-braintrust-test-crewai' or extra == 'group-10-braintrust-test-deepagents' or extra == 'group-10-braintrust-test-langchain' or extra != 'group-10-braintrust-test-litellm' or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-strands')" }, + { name = "typing-extensions", marker = "extra == 'group-10-braintrust-lint' or extra == 'group-10-braintrust-test-agentscope' or extra == 'group-10-braintrust-test-agno' or extra == 'group-10-braintrust-test-crewai' or extra == 'group-10-braintrust-test-deepagents' or extra == 'group-10-braintrust-test-langchain' or extra != 'group-10-braintrust-test-litellm' or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-strands')" }, + { name = "websockets", marker = "extra == 'group-10-braintrust-lint' or extra == 'group-10-braintrust-test-agentscope' or extra == 'group-10-braintrust-test-agno' or extra == 'group-10-braintrust-test-crewai' or extra == 'group-10-braintrust-test-deepagents' or extra == 'group-10-braintrust-test-langchain' or extra != 'group-10-braintrust-test-litellm' or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-livekit-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-litellm' and extra == 'group-10-braintrust-test-strands')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/fd/16/b79c1849125eb6d19cae98c21ff35caa2e55b5ec8d7a02b354b711917ef7/openai_agents-0.17.3.tar.gz", hash = "sha256:63b6dda6bd4fb51169e2a2cbd5d187a4e5ce823bbd15f965c8ed1d3b89072eec", size = 5406135, upload-time = "2026-05-19T01:28:15.971Z" } wheels = [ @@ -5378,10 +5432,10 @@ name = "opentelemetry-instrumentation" version = "0.60b1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "opentelemetry-api", version = "1.39.1", source = { registry = "https://pypi.org/simple" } }, - { name = "opentelemetry-semantic-conventions", version = "0.60b1", source = { registry = "https://pypi.org/simple" } }, - { name = "packaging" }, - { name = "wrapt" }, + { name = "opentelemetry-api", version = "1.39.1", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'group-10-braintrust-lint' or extra == 'group-10-braintrust-test-agentscope' or extra == 'group-10-braintrust-test-agno' or extra == 'group-10-braintrust-test-crewai' or extra == 'group-10-braintrust-test-deepagents' or extra == 'group-10-braintrust-test-langchain' or extra == 'group-10-braintrust-test-litellm' or extra != 'group-10-braintrust-test-livekit-agents' or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-otel-events') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands')" }, + { name = "opentelemetry-semantic-conventions", version = "0.60b1", source = { registry = "https://pypi.org/simple" }, marker = "extra == 'group-10-braintrust-lint' or extra == 'group-10-braintrust-test-agentscope' or extra == 'group-10-braintrust-test-agno' or extra == 'group-10-braintrust-test-crewai' or extra == 'group-10-braintrust-test-deepagents' or extra == 'group-10-braintrust-test-langchain' or extra == 'group-10-braintrust-test-litellm' or extra != 'group-10-braintrust-test-livekit-agents' or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-otel-events') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands')" }, + { name = "packaging", marker = "extra == 'group-10-braintrust-lint' or extra == 'group-10-braintrust-test-agentscope' or extra == 'group-10-braintrust-test-agno' or extra == 'group-10-braintrust-test-crewai' or extra == 'group-10-braintrust-test-deepagents' or extra == 'group-10-braintrust-test-langchain' or extra == 'group-10-braintrust-test-litellm' or extra != 'group-10-braintrust-test-livekit-agents' or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-otel-events') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands')" }, + { name = "wrapt", marker = "extra == 'group-10-braintrust-lint' or extra == 'group-10-braintrust-test-agentscope' or extra == 'group-10-braintrust-test-agno' or extra == 'group-10-braintrust-test-crewai' or extra == 'group-10-braintrust-test-deepagents' or extra == 'group-10-braintrust-test-langchain' or extra == 'group-10-braintrust-test-litellm' or extra != 'group-10-braintrust-test-livekit-agents' or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-openai-agents') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-logfire') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-pydantic-ai-otel-events') or (extra == 'group-10-braintrust-test-livekit-agents' and extra == 'group-10-braintrust-test-strands')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/41/0f/7e6b713ac117c1f5e4e3300748af699b9902a2e5e34c9cf443dde25a01fa/opentelemetry_instrumentation-0.60b1.tar.gz", hash = "sha256:57ddc7974c6eb35865af0426d1a17132b88b2ed8586897fee187fd5b8944bd6a", size = 31706, upload-time = "2025-12-11T13:36:42.515Z" } wheels = [ @@ -6196,6 +6250,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3a/ed/1cdcab6ba3d6ab7feca11fc14f0eeea80755bb53ef4e892079f31b10a25f/propcache-0.5.2-py3-none-any.whl", hash = "sha256:be1ddfcbb376e3de5d2e2db1d58d6d67463e6b4f9f040c000de8e300295465fe", size = 14036, upload-time = "2026-05-08T21:02:10.673Z" }, ] +[[package]] +name = "proto-plus" +version = "1.28.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf", version = "5.29.6", source = { registry = "https://pypi.org/simple" } }, +] +sdist = { url = "https://files.pythonhosted.org/packages/73/3e/29e0d6a2c5adde6ab5772253fd16ab346324026b89a66e354689c86d0584/proto_plus-1.28.2.tar.gz", hash = "sha256:26d843eb99c1e32fdf1d20ff0faae56607f7748fe774acf9ecd5cfe6c6472501", size = 58063, upload-time = "2026-07-22T16:28:29.119Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9d/84/4e9a53a062d4073c74897a6bd20fff74d55307341b3e85c081002462b3ef/proto_plus-1.28.2-py3-none-any.whl", hash = "sha256:b874236fcac2358f601e4330bcb76cb8b89c851303ccf4078408b3d4774d1c52", size = 50693, upload-time = "2026-07-22T16:28:24.059Z" }, +] + [[package]] name = "protobuf" version = "5.29.6" From 528ee7c6113ded93a8efdfdca2329b61a24aa279 Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Wed, 16 Sep 2026 15:28:40 -0400 Subject: [PATCH 2/4] rename --- .agents/skills/sdk-vcr-workflows/SKILL.md | 16 ++-- py/noxfile.py | 8 +- py/pyproject.toml | 2 +- py/src/braintrust/auto.py | 10 +-- py/src/braintrust/integrations/__init__.py | 4 +- ...py => test_auto_google_discoveryengine.py} | 14 +-- .../integrations/discoveryengine/__init__.py | 12 --- .../google_discoveryengine/__init__.py | 12 +++ .../_test_grpc.py | 0 .../latest/test_answer_query[False].yaml | 0 .../latest/test_answer_query[True].yaml | 0 .../test_answer_requested_model[False].yaml | 0 .../test_answer_requested_model[True].yaml | 0 .../latest/test_async_grpc[answer_query].json | 0 .../test_async_grpc[check_grounding].json | 0 ...est_async_grpc[converse_conversation].json | 0 .../latest/test_async_grpc[rank].json | 0 .../test_async_grpc[stream_answer_query].json | 0 .../latest/test_async_provider_error.json | 0 .../test_async_stream_provider_error.json | 0 .../latest/test_check_grounding.yaml | 0 .../latest/test_converse_conversation.yaml | 0 .../cassettes/latest/test_provider_error.yaml | 0 .../cassettes/latest/test_rank.yaml | 0 .../latest/test_rank_output_limit.yaml | 0 .../latest/test_stream_provider_error.yaml | 0 .../integration.py | 4 +- .../patchers.py | 22 ++--- .../test_google_discoveryengine.py} | 89 ++++++++++--------- .../tracing.py | 4 +- 30 files changed, 103 insertions(+), 94 deletions(-) rename py/src/braintrust/integrations/auto_test_scripts/{test_auto_discoveryengine.py => test_auto_google_discoveryengine.py} (81%) delete mode 100644 py/src/braintrust/integrations/discoveryengine/__init__.py create mode 100644 py/src/braintrust/integrations/google_discoveryengine/__init__.py rename py/src/braintrust/integrations/{discoveryengine => google_discoveryengine}/_test_grpc.py (100%) rename py/src/braintrust/integrations/{discoveryengine => google_discoveryengine}/cassettes/latest/test_answer_query[False].yaml (100%) rename py/src/braintrust/integrations/{discoveryengine => google_discoveryengine}/cassettes/latest/test_answer_query[True].yaml (100%) rename py/src/braintrust/integrations/{discoveryengine => google_discoveryengine}/cassettes/latest/test_answer_requested_model[False].yaml (100%) rename py/src/braintrust/integrations/{discoveryengine => google_discoveryengine}/cassettes/latest/test_answer_requested_model[True].yaml (100%) rename py/src/braintrust/integrations/{discoveryengine => google_discoveryengine}/cassettes/latest/test_async_grpc[answer_query].json (100%) rename py/src/braintrust/integrations/{discoveryengine => google_discoveryengine}/cassettes/latest/test_async_grpc[check_grounding].json (100%) rename py/src/braintrust/integrations/{discoveryengine => google_discoveryengine}/cassettes/latest/test_async_grpc[converse_conversation].json (100%) rename py/src/braintrust/integrations/{discoveryengine => google_discoveryengine}/cassettes/latest/test_async_grpc[rank].json (100%) rename py/src/braintrust/integrations/{discoveryengine => google_discoveryengine}/cassettes/latest/test_async_grpc[stream_answer_query].json (100%) rename py/src/braintrust/integrations/{discoveryengine => google_discoveryengine}/cassettes/latest/test_async_provider_error.json (100%) rename py/src/braintrust/integrations/{discoveryengine => google_discoveryengine}/cassettes/latest/test_async_stream_provider_error.json (100%) rename py/src/braintrust/integrations/{discoveryengine => google_discoveryengine}/cassettes/latest/test_check_grounding.yaml (100%) rename py/src/braintrust/integrations/{discoveryengine => google_discoveryengine}/cassettes/latest/test_converse_conversation.yaml (100%) rename py/src/braintrust/integrations/{discoveryengine => google_discoveryengine}/cassettes/latest/test_provider_error.yaml (100%) rename py/src/braintrust/integrations/{discoveryengine => google_discoveryengine}/cassettes/latest/test_rank.yaml (100%) rename py/src/braintrust/integrations/{discoveryengine => google_discoveryengine}/cassettes/latest/test_rank_output_limit.yaml (100%) rename py/src/braintrust/integrations/{discoveryengine => google_discoveryengine}/cassettes/latest/test_stream_provider_error.yaml (100%) rename py/src/braintrust/integrations/{discoveryengine => google_discoveryengine}/integration.py (76%) rename py/src/braintrust/integrations/{discoveryengine => google_discoveryengine}/patchers.py (82%) rename py/src/braintrust/integrations/{discoveryengine/test_discoveryengine.py => google_discoveryengine/test_google_discoveryengine.py} (88%) rename py/src/braintrust/integrations/{discoveryengine => google_discoveryengine}/tracing.py (99%) diff --git a/.agents/skills/sdk-vcr-workflows/SKILL.md b/.agents/skills/sdk-vcr-workflows/SKILL.md index cbbb33e1d..daaa2a23f 100644 --- a/.agents/skills/sdk-vcr-workflows/SKILL.md +++ b/.agents/skills/sdk-vcr-workflows/SKILL.md @@ -269,8 +269,8 @@ Do not try to force ordinary HTTP VCR patterns onto Claude Agent SDK subprocess ## Discovery Engine Recording Discovery Engine tests use HTTP VCR for sync REST calls and the test-only -`integrations/discoveryengine/_test_grpc.py` helper for async gRPC calls. Both -recording formats live under `py/src/braintrust/integrations/discoveryengine/cassettes//`. +`integrations/google_discoveryengine/_test_grpc.py` helper for async gRPC calls. Both +recording formats live under `py/src/braintrust/integrations/google_discoveryengine/cassettes//`. ### Prerequisites @@ -303,9 +303,9 @@ sample PDFs at `gs://cloud-samples-data/gen-app-builder/search/alphabet-investor Wait for indexing to finish before recording. ```sh -export BRAINTRUST_DISCOVERYENGINE_PROJECT="your-project" -export BRAINTRUST_DISCOVERYENGINE_APP="your-app-id" -export BRAINTRUST_DISCOVERYENGINE_DATASTORE="your-datastore-id" +export BRAINTRUST_GOOGLE_DISCOVERYENGINE_PROJECT="your-project" +export BRAINTRUST_GOOGLE_DISCOVERYENGINE_APP="your-app-id" +export BRAINTRUST_GOOGLE_DISCOVERYENGINE_DATASTORE="your-datastore-id" ``` ### Record and replay @@ -315,13 +315,13 @@ both REST and gRPC; the gRPC helper otherwise requires an existing cassette. ```sh # REST ranking, including manual/setup entry-point coverage. -mise exec -- nox -s 'test_discoveryengine(latest)' -- --vcr-record=all -k 'test_rank and not test_rank_output_limit' +mise exec -- nox -s 'test_google_discoveryengine(latest)' -- --vcr-record=all -k 'test_rank and not test_rank_output_limit' # Async gRPC ranking. -mise exec -- nox -s 'test_discoveryengine(latest)' -- --vcr-record=all -k 'test_async_grpc and rank' +mise exec -- nox -s 'test_google_discoveryengine(latest)' -- --vcr-record=all -k 'test_async_grpc and rank' # Verify all recordings without network access to Google. -mise exec -- nox -R -s 'test_discoveryengine(latest)' -- --vcr-record=none +mise exec -- nox -R -s 'test_google_discoveryengine(latest)' -- --vcr-record=none ``` Playback derives resource paths from the checked-in REST cassettes and ignores diff --git a/py/noxfile.py b/py/noxfile.py index 8240e1017..83eb26bb1 100644 --- a/py/noxfile.py +++ b/py/noxfile.py @@ -569,15 +569,15 @@ def test_google_genai(session, version): _run_tests(session, f"{INTEGRATION_DIR}/google_genai/test_google_genai.py", version=version) -DISCOVERYENGINE_VERSIONS = _get_matrix_versions("google-cloud-discoveryengine") +GOOGLE_DISCOVERYENGINE_VERSIONS = _get_matrix_versions("google-cloud-discoveryengine") @nox.session() -@nox.parametrize("version", DISCOVERYENGINE_VERSIONS, ids=DISCOVERYENGINE_VERSIONS) -def test_discoveryengine(session, version): +@nox.parametrize("version", GOOGLE_DISCOVERYENGINE_VERSIONS, ids=GOOGLE_DISCOVERYENGINE_VERSIONS) +def test_google_discoveryengine(session, version): _install_test_deps(session) _install_matrix_dep(session, "google-cloud-discoveryengine", version) - _run_tests(session, f"{INTEGRATION_DIR}/discoveryengine", version=version) + _run_tests(session, f"{INTEGRATION_DIR}/google_discoveryengine", version=version) DSPY_VERSIONS = _get_matrix_versions("dspy") diff --git a/py/pyproject.toml b/py/pyproject.toml index 5c2d11705..bad984f68 100644 --- a/py/pyproject.toml +++ b/py/pyproject.toml @@ -575,7 +575,7 @@ cursor_sdk = ["cursor-sdk"] crewai = ["crewai"] dspy = ["dspy"] google_genai = ["google-genai"] -discoveryengine = ["google-cloud-discoveryengine"] +google_discoveryengine = ["google-cloud-discoveryengine"] huggingface_hub = ["huggingface-hub"] harbor = ["harbor"] instructor = ["instructor"] diff --git a/py/src/braintrust/auto.py b/py/src/braintrust/auto.py index ce628c294..a8c1f0319 100644 --- a/py/src/braintrust/auto.py +++ b/py/src/braintrust/auto.py @@ -19,8 +19,8 @@ CohereIntegration, CrewAIIntegration, CursorSDKIntegration, - DiscoveryEngineIntegration, DSPyIntegration, + GoogleDiscoveryEngineIntegration, GoogleGenAIIntegration, HuggingFaceHubIntegration, InstructorIntegration, @@ -65,7 +65,7 @@ def auto_instrument( ai_sdk: bool = True, pydantic_ai: bool = True, google_genai: bool = True, - discoveryengine: bool = True, + google_discoveryengine: bool = True, instructor: bool = True, openrouter: bool = True, mistral: bool = True, @@ -104,7 +104,7 @@ def auto_instrument( litellm: Enable LiteLLM instrumentation (default: True) ai_sdk: Enable Vercel AI SDK for Python instrumentation (default: True) pydantic_ai: Enable Pydantic AI instrumentation (default: True) - discoveryengine: Enable Google Discovery Engine v1 instrumentation (default: True) + google_discoveryengine: Enable Google Discovery Engine v1 instrumentation (default: True) google_genai: Enable Google GenAI instrumentation (default: True) instructor: Enable Instructor (structured-output) instrumentation (default: True) openrouter: Enable OpenRouter instrumentation (default: True) @@ -187,8 +187,8 @@ def auto_instrument( results["pydantic_ai"] = _instrument_integration(PydanticAIIntegration) if google_genai: results["google_genai"] = _instrument_integration(GoogleGenAIIntegration) - if discoveryengine: - results["discoveryengine"] = _instrument_integration(DiscoveryEngineIntegration) + if google_discoveryengine: + results["google_discoveryengine"] = _instrument_integration(GoogleDiscoveryEngineIntegration) if instructor: results["instructor"] = _instrument_integration(InstructorIntegration) if openrouter: diff --git a/py/src/braintrust/integrations/__init__.py b/py/src/braintrust/integrations/__init__.py index 5995cbcb0..3691d395e 100644 --- a/py/src/braintrust/integrations/__init__.py +++ b/py/src/braintrust/integrations/__init__.py @@ -9,8 +9,8 @@ from .cohere import CohereIntegration from .crewai import CrewAIIntegration from .cursor_sdk import CursorSDKIntegration -from .discoveryengine import DiscoveryEngineIntegration from .dspy import DSPyIntegration +from .google_discoveryengine import GoogleDiscoveryEngineIntegration from .google_genai import GoogleGenAIIntegration from .huggingface_hub import HuggingFaceHubIntegration from .instructor import InstructorIntegration @@ -42,7 +42,7 @@ "CrewAIIntegration", "CursorSDKIntegration", "DSPyIntegration", - "DiscoveryEngineIntegration", + "GoogleDiscoveryEngineIntegration", "GoogleGenAIIntegration", "HuggingFaceHubIntegration", "InstructorIntegration", diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_discoveryengine.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_google_discoveryengine.py similarity index 81% rename from py/src/braintrust/integrations/auto_test_scripts/test_auto_discoveryengine.py rename to py/src/braintrust/integrations/auto_test_scripts/test_auto_google_discoveryengine.py index b9ed32569..dd7fa19fe 100644 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_discoveryengine.py +++ b/py/src/braintrust/integrations/auto_test_scripts/test_auto_google_discoveryengine.py @@ -24,9 +24,9 @@ if sys.argv[1] == "before": from google.cloud.discoveryengine_v1 import RankServiceClient -options["discoveryengine"] = True -assert auto_instrument(**options) == {"discoveryengine": True} -assert auto_instrument(**options) == {"discoveryengine": True} +options["google_discoveryengine"] = True +assert auto_instrument(**options) == {"google_discoveryengine": True} +assert auto_instrument(**options) == {"google_discoveryengine": True} from google.auth.credentials import AnonymousCredentials @@ -34,13 +34,15 @@ from google.cloud.discoveryengine_v1 import RankServiceClient -cassette_dir = Path(_versioned_cassette_dir(str(Path(__file__).parent.parent / "discoveryengine" / "cassettes"))) +cassette_dir = Path( + _versioned_cassette_dir(str(Path(__file__).parent.parent / "google_discoveryengine" / "cassettes")) +) cassette = yaml.safe_load((cassette_dir / "test_rank.yaml").read_text()) ranking_config = urlsplit(cassette["interactions"][0]["request"]["uri"]).path.removeprefix("/v1/").split(":rank")[0] assert RankServiceClient is not None with autoinstrument_test_context( - "test_rank", integration="discoveryengine", vcr_config={"record_mode": "none"} + "test_rank", integration="google_discoveryengine", vcr_config={"record_mode": "none"} ) as memory_logger: client = RankServiceClient(transport="rest", credentials=AnonymousCredentials()) result = client.rank( @@ -60,4 +62,4 @@ spans = memory_logger.pop() assert len(spans) == 1 assert spans[0]["metadata"]["provider"] == "google" - assert spans[0]["context"]["span_origin"]["instrumentation"]["name"] == "discoveryengine-auto" + assert spans[0]["context"]["span_origin"]["instrumentation"]["name"] == "google-discoveryengine-auto" diff --git a/py/src/braintrust/integrations/discoveryengine/__init__.py b/py/src/braintrust/integrations/discoveryengine/__init__.py deleted file mode 100644 index e6b7b2d20..000000000 --- a/py/src/braintrust/integrations/discoveryengine/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -"""Braintrust integration for google-cloud-discoveryengine v1.""" - -from .integration import DiscoveryEngineIntegration -from .patchers import wrap_discoveryengine - - -__all__ = ["DiscoveryEngineIntegration", "setup_discoveryengine", "wrap_discoveryengine"] - - -def setup_discoveryengine() -> bool: - """Instrument supported v1 clients in this process.""" - return DiscoveryEngineIntegration.setup() diff --git a/py/src/braintrust/integrations/google_discoveryengine/__init__.py b/py/src/braintrust/integrations/google_discoveryengine/__init__.py new file mode 100644 index 000000000..543b57d93 --- /dev/null +++ b/py/src/braintrust/integrations/google_discoveryengine/__init__.py @@ -0,0 +1,12 @@ +"""Braintrust integration for google-cloud-discoveryengine v1.""" + +from .integration import GoogleDiscoveryEngineIntegration +from .patchers import wrap_google_discoveryengine + + +__all__ = ["GoogleDiscoveryEngineIntegration", "setup_google_discoveryengine", "wrap_google_discoveryengine"] + + +def setup_google_discoveryengine() -> bool: + """Instrument supported v1 clients in this process.""" + return GoogleDiscoveryEngineIntegration.setup() diff --git a/py/src/braintrust/integrations/discoveryengine/_test_grpc.py b/py/src/braintrust/integrations/google_discoveryengine/_test_grpc.py similarity index 100% rename from py/src/braintrust/integrations/discoveryengine/_test_grpc.py rename to py/src/braintrust/integrations/google_discoveryengine/_test_grpc.py diff --git a/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_answer_query[False].yaml b/py/src/braintrust/integrations/google_discoveryengine/cassettes/latest/test_answer_query[False].yaml similarity index 100% rename from py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_answer_query[False].yaml rename to py/src/braintrust/integrations/google_discoveryengine/cassettes/latest/test_answer_query[False].yaml diff --git a/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_answer_query[True].yaml b/py/src/braintrust/integrations/google_discoveryengine/cassettes/latest/test_answer_query[True].yaml similarity index 100% rename from py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_answer_query[True].yaml rename to py/src/braintrust/integrations/google_discoveryengine/cassettes/latest/test_answer_query[True].yaml diff --git a/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_answer_requested_model[False].yaml b/py/src/braintrust/integrations/google_discoveryengine/cassettes/latest/test_answer_requested_model[False].yaml similarity index 100% rename from py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_answer_requested_model[False].yaml rename to py/src/braintrust/integrations/google_discoveryengine/cassettes/latest/test_answer_requested_model[False].yaml diff --git a/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_answer_requested_model[True].yaml b/py/src/braintrust/integrations/google_discoveryengine/cassettes/latest/test_answer_requested_model[True].yaml similarity index 100% rename from py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_answer_requested_model[True].yaml rename to py/src/braintrust/integrations/google_discoveryengine/cassettes/latest/test_answer_requested_model[True].yaml diff --git a/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_async_grpc[answer_query].json b/py/src/braintrust/integrations/google_discoveryengine/cassettes/latest/test_async_grpc[answer_query].json similarity index 100% rename from py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_async_grpc[answer_query].json rename to py/src/braintrust/integrations/google_discoveryengine/cassettes/latest/test_async_grpc[answer_query].json diff --git a/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_async_grpc[check_grounding].json b/py/src/braintrust/integrations/google_discoveryengine/cassettes/latest/test_async_grpc[check_grounding].json similarity index 100% rename from py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_async_grpc[check_grounding].json rename to py/src/braintrust/integrations/google_discoveryengine/cassettes/latest/test_async_grpc[check_grounding].json diff --git a/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_async_grpc[converse_conversation].json b/py/src/braintrust/integrations/google_discoveryengine/cassettes/latest/test_async_grpc[converse_conversation].json similarity index 100% rename from py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_async_grpc[converse_conversation].json rename to py/src/braintrust/integrations/google_discoveryengine/cassettes/latest/test_async_grpc[converse_conversation].json diff --git a/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_async_grpc[rank].json b/py/src/braintrust/integrations/google_discoveryengine/cassettes/latest/test_async_grpc[rank].json similarity index 100% rename from py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_async_grpc[rank].json rename to py/src/braintrust/integrations/google_discoveryengine/cassettes/latest/test_async_grpc[rank].json diff --git a/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_async_grpc[stream_answer_query].json b/py/src/braintrust/integrations/google_discoveryengine/cassettes/latest/test_async_grpc[stream_answer_query].json similarity index 100% rename from py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_async_grpc[stream_answer_query].json rename to py/src/braintrust/integrations/google_discoveryengine/cassettes/latest/test_async_grpc[stream_answer_query].json diff --git a/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_async_provider_error.json b/py/src/braintrust/integrations/google_discoveryengine/cassettes/latest/test_async_provider_error.json similarity index 100% rename from py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_async_provider_error.json rename to py/src/braintrust/integrations/google_discoveryengine/cassettes/latest/test_async_provider_error.json diff --git a/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_async_stream_provider_error.json b/py/src/braintrust/integrations/google_discoveryengine/cassettes/latest/test_async_stream_provider_error.json similarity index 100% rename from py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_async_stream_provider_error.json rename to py/src/braintrust/integrations/google_discoveryengine/cassettes/latest/test_async_stream_provider_error.json diff --git a/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_check_grounding.yaml b/py/src/braintrust/integrations/google_discoveryengine/cassettes/latest/test_check_grounding.yaml similarity index 100% rename from py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_check_grounding.yaml rename to py/src/braintrust/integrations/google_discoveryengine/cassettes/latest/test_check_grounding.yaml diff --git a/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_converse_conversation.yaml b/py/src/braintrust/integrations/google_discoveryengine/cassettes/latest/test_converse_conversation.yaml similarity index 100% rename from py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_converse_conversation.yaml rename to py/src/braintrust/integrations/google_discoveryengine/cassettes/latest/test_converse_conversation.yaml diff --git a/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_provider_error.yaml b/py/src/braintrust/integrations/google_discoveryengine/cassettes/latest/test_provider_error.yaml similarity index 100% rename from py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_provider_error.yaml rename to py/src/braintrust/integrations/google_discoveryengine/cassettes/latest/test_provider_error.yaml diff --git a/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_rank.yaml b/py/src/braintrust/integrations/google_discoveryengine/cassettes/latest/test_rank.yaml similarity index 100% rename from py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_rank.yaml rename to py/src/braintrust/integrations/google_discoveryengine/cassettes/latest/test_rank.yaml diff --git a/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_rank_output_limit.yaml b/py/src/braintrust/integrations/google_discoveryengine/cassettes/latest/test_rank_output_limit.yaml similarity index 100% rename from py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_rank_output_limit.yaml rename to py/src/braintrust/integrations/google_discoveryengine/cassettes/latest/test_rank_output_limit.yaml diff --git a/py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_stream_provider_error.yaml b/py/src/braintrust/integrations/google_discoveryengine/cassettes/latest/test_stream_provider_error.yaml similarity index 100% rename from py/src/braintrust/integrations/discoveryengine/cassettes/latest/test_stream_provider_error.yaml rename to py/src/braintrust/integrations/google_discoveryengine/cassettes/latest/test_stream_provider_error.yaml diff --git a/py/src/braintrust/integrations/discoveryengine/integration.py b/py/src/braintrust/integrations/google_discoveryengine/integration.py similarity index 76% rename from py/src/braintrust/integrations/discoveryengine/integration.py rename to py/src/braintrust/integrations/google_discoveryengine/integration.py index 179c18461..2ed7f3990 100644 --- a/py/src/braintrust/integrations/discoveryengine/integration.py +++ b/py/src/braintrust/integrations/google_discoveryengine/integration.py @@ -5,8 +5,8 @@ from .patchers import PATCHERS -class DiscoveryEngineIntegration(BaseIntegration): - name = "discoveryengine" +class GoogleDiscoveryEngineIntegration(BaseIntegration): + name = "google_discoveryengine" import_names = ("google.cloud.discoveryengine_v1",) distribution_names = ("google-cloud-discoveryengine",) min_version = "0.20.3" diff --git a/py/src/braintrust/integrations/discoveryengine/patchers.py b/py/src/braintrust/integrations/google_discoveryengine/patchers.py similarity index 82% rename from py/src/braintrust/integrations/discoveryengine/patchers.py rename to py/src/braintrust/integrations/google_discoveryengine/patchers.py index 02de3ccdd..2d2632f1e 100644 --- a/py/src/braintrust/integrations/discoveryengine/patchers.py +++ b/py/src/braintrust/integrations/google_discoveryengine/patchers.py @@ -8,61 +8,61 @@ class AnswerQueryPatcher(FunctionWrapperPatcher): - name = "discoveryengine.answer_query" + name = "google_discoveryengine.answer_query" target_path = "ConversationalSearchServiceClient.answer_query" wrapper = partial(_call, "answer_query") class AsyncAnswerQueryPatcher(FunctionWrapperPatcher): - name = "discoveryengine.async.answer_query" + name = "google_discoveryengine.async.answer_query" target_path = "ConversationalSearchServiceAsyncClient.answer_query" wrapper = partial(_async_call, "answer_query") class StreamAnswerQueryPatcher(FunctionWrapperPatcher): - name = "discoveryengine.stream_answer_query" + name = "google_discoveryengine.stream_answer_query" target_path = "ConversationalSearchServiceClient.stream_answer_query" wrapper = partial(_call, "stream_answer_query") class AsyncStreamAnswerQueryPatcher(FunctionWrapperPatcher): - name = "discoveryengine.async.stream_answer_query" + name = "google_discoveryengine.async.stream_answer_query" target_path = "ConversationalSearchServiceAsyncClient.stream_answer_query" wrapper = partial(_async_call, "stream_answer_query") class ConverseConversationPatcher(FunctionWrapperPatcher): - name = "discoveryengine.converse_conversation" + name = "google_discoveryengine.converse_conversation" target_path = "ConversationalSearchServiceClient.converse_conversation" wrapper = partial(_call, "converse_conversation") class AsyncConverseConversationPatcher(FunctionWrapperPatcher): - name = "discoveryengine.async.converse_conversation" + name = "google_discoveryengine.async.converse_conversation" target_path = "ConversationalSearchServiceAsyncClient.converse_conversation" wrapper = partial(_async_call, "converse_conversation") class CheckGroundingPatcher(FunctionWrapperPatcher): - name = "discoveryengine.check_grounding" + name = "google_discoveryengine.check_grounding" target_path = "GroundedGenerationServiceClient.check_grounding" wrapper = partial(_call, "check_grounding") class AsyncCheckGroundingPatcher(FunctionWrapperPatcher): - name = "discoveryengine.async.check_grounding" + name = "google_discoveryengine.async.check_grounding" target_path = "GroundedGenerationServiceAsyncClient.check_grounding" wrapper = partial(_async_call, "check_grounding") class RankPatcher(FunctionWrapperPatcher): - name = "discoveryengine.rank" + name = "google_discoveryengine.rank" target_path = "RankServiceClient.rank" wrapper = partial(_call, "rank") class AsyncRankPatcher(FunctionWrapperPatcher): - name = "discoveryengine.async.rank" + name = "google_discoveryengine.async.rank" target_path = "RankServiceAsyncClient.rank" wrapper = partial(_async_call, "rank") @@ -81,7 +81,7 @@ class AsyncRankPatcher(FunctionWrapperPatcher): ) -def wrap_discoveryengine(client): +def wrap_google_discoveryengine(client): """Instrument one v1 client instance, returning the same client.""" from google.cloud import discoveryengine_v1 diff --git a/py/src/braintrust/integrations/discoveryengine/test_discoveryengine.py b/py/src/braintrust/integrations/google_discoveryengine/test_google_discoveryengine.py similarity index 88% rename from py/src/braintrust/integrations/discoveryengine/test_discoveryengine.py rename to py/src/braintrust/integrations/google_discoveryengine/test_google_discoveryengine.py index f8cd2240c..f43f7f786 100644 --- a/py/src/braintrust/integrations/discoveryengine/test_discoveryengine.py +++ b/py/src/braintrust/integrations/google_discoveryengine/test_google_discoveryengine.py @@ -23,9 +23,9 @@ def _resource(request, cassette_dir, env_name, cassette_name, separator): if request.config.getoption("--vcr-record") == "all": - value = os.getenv(f"BRAINTRUST_DISCOVERYENGINE_{env_name}") + value = os.getenv(f"BRAINTRUST_GOOGLE_DISCOVERYENGINE_{env_name}") if not value: - pytest.fail(f"Set BRAINTRUST_DISCOVERYENGINE_{env_name} to record Discovery Engine tests") + pytest.fail(f"Set BRAINTRUST_GOOGLE_DISCOVERYENGINE_{env_name} to record Discovery Engine tests") return value cassette = yaml.safe_load((Path(cassette_dir) / cassette_name).read_text()) resource = urlsplit(cassette["interactions"][0]["request"]["uri"]).path.removeprefix("/v1/") @@ -81,8 +81,8 @@ def vcr_config(): @pytest.fixture(scope="session") def credentials(request): if request.config.getoption("--vcr-record") == "all": - if not os.getenv("BRAINTRUST_DISCOVERYENGINE_PROJECT"): - pytest.fail("Set BRAINTRUST_DISCOVERYENGINE_PROJECT to record Discovery Engine tests") + if not os.getenv("BRAINTRUST_GOOGLE_DISCOVERYENGINE_PROJECT"): + pytest.fail("Set BRAINTRUST_GOOGLE_DISCOVERYENGINE_PROJECT to record Discovery Engine tests") # Refresh outside the recorded HTTP call; credentials never enter cassettes. token = subprocess.check_output( ["gcloud", "auth", "application-default", "print-access-token"], text=True @@ -115,18 +115,21 @@ def rank_request(LOCATION): @pytest.mark.vcr("test_rank.yaml") @pytest.mark.parametrize("mode", ["manual", "manual_then_setup", "setup_then_manual"]) def test_rank(memory_logger, credentials, rank_request, mode): - from braintrust.integrations.discoveryengine import setup_discoveryengine, wrap_discoveryengine + from braintrust.integrations.google_discoveryengine import ( + setup_google_discoveryengine, + wrap_google_discoveryengine, + ) client = discoveryengine.RankServiceClient(transport="rest", credentials=credentials) untouched = discoveryengine.RankServiceClient(transport="rest", credentials=credentials) original = untouched.rank if mode == "setup_then_manual": - assert setup_discoveryengine() - assert setup_discoveryengine() - assert wrap_discoveryengine(client) is client - assert wrap_discoveryengine(client) is client + assert setup_google_discoveryengine() + assert setup_google_discoveryengine() + assert wrap_google_discoveryengine(client) is client + assert wrap_google_discoveryengine(client) is client if mode == "manual_then_setup": - assert setup_discoveryengine() + assert setup_google_discoveryengine() if mode == "manual": assert untouched.rank == original assert not hasattr(untouched.rank, "__wrapped__") @@ -135,7 +138,7 @@ def test_rank(memory_logger, credentials, rank_request, mode): spans = memory_logger.pop() assert len(spans) == 1 span = spans[0] - assert span["span_attributes"]["name"] == "discoveryengine.rank" + assert span["span_attributes"]["name"] == "google_discoveryengine.rank" assert span["span_attributes"]["type"] == "llm" assert span["metadata"]["provider"] == "google" assert span["metadata"]["model"] == "semantic-ranker-512@latest" @@ -143,7 +146,7 @@ def test_rank(memory_logger, credentials, rank_request, mode): assert span["output"][0]["id"] == "1" assert span["output"][0]["score"] == result.records[0].score assert not {"tokens", "prompt_tokens", "completion_tokens"} & span["metrics"].keys() - assert span["context"]["span_origin"]["instrumentation"]["name"] == "discoveryengine-auto" + assert span["context"]["span_origin"]["instrumentation"]["name"] == "google-discoveryengine-auto" QUERY = "What was Alphabet's revenue in 2022?" @@ -153,13 +156,13 @@ def _assert_generation_span(memory_logger, method, text): spans = memory_logger.pop() assert len(spans) == 1 span = spans[0] - assert span["span_attributes"]["name"] == f"discoveryengine.{method}" + assert span["span_attributes"]["name"] == f"google_discoveryengine.{method}" assert span["span_attributes"]["type"] == "llm" assert span["metadata"]["provider"] == "google" assert "model" not in span["metadata"] assert span["output"][0]["message"]["content"] == text assert not {"tokens", "prompt_tokens", "completion_tokens"} & span["metrics"].keys() - assert span["context"]["span_origin"]["instrumentation"]["name"] == "discoveryengine-auto" + assert span["context"]["span_origin"]["instrumentation"]["name"] == "google-discoveryengine-auto" json.dumps({key: span[key] for key in ("input", "output", "metadata")}) return span @@ -234,7 +237,7 @@ def test_check_grounding(memory_logger, credentials, LOCATION): assert result.support_score > 0 spans = memory_logger.pop() assert len(spans) == 1 - assert spans[0]["span_attributes"]["name"] == "discoveryengine.check_grounding" + assert spans[0]["span_attributes"]["name"] == "google_discoveryengine.check_grounding" assert spans[0]["output"]["support_score"] == result.support_score @@ -260,7 +263,7 @@ async def test_async_grpc( LOCATION, SERVING_CONFIG, ): - from braintrust.integrations.discoveryengine._test_grpc import grpc_cassette + from braintrust.integrations.google_discoveryengine._test_grpc import grpc_cassette auto_instrument() if method in ("answer_query", "stream_answer_query", "converse_conversation"): @@ -314,11 +317,11 @@ async def test_async_grpc( spans = memory_logger.pop() assert len(spans) == 1 span = spans[0] - assert span["span_attributes"]["name"] == f"discoveryengine.{method}" + assert span["span_attributes"]["name"] == f"google_discoveryengine.{method}" assert span["span_attributes"]["type"] == "llm" assert span["metadata"]["provider"] == "google" assert "model" not in span["metadata"] - assert span["context"]["span_origin"]["instrumentation"]["name"] == "discoveryengine-auto" + assert span["context"]["span_origin"]["instrumentation"]["name"] == "google-discoveryengine-auto" assert span["metrics"]["end"] >= span["metrics"]["start"] assert not {"tokens", "prompt_tokens", "completion_tokens"} & span["metrics"].keys() if method in ("answer_query", "stream_answer_query", "converse_conversation"): @@ -347,7 +350,7 @@ async def test_async_grpc( @pytest.fixture(autouse=True) def restore_methods(): - from braintrust.integrations.discoveryengine.patchers import PATCHERS + from braintrust.integrations.google_discoveryengine.patchers import PATCHERS originals = [] for patcher in PATCHERS: @@ -365,8 +368,8 @@ def restore_methods(): def test_patch_scope(): import inspect - from braintrust.integrations.discoveryengine import setup_discoveryengine - from braintrust.integrations.discoveryengine.patchers import PATCHERS + from braintrust.integrations.google_discoveryengine import setup_google_discoveryengine + from braintrust.integrations.google_discoveryengine.patchers import PATCHERS from google.cloud import discoveryengine_v1alpha, discoveryengine_v1beta untouched = [ @@ -383,7 +386,7 @@ def test_patch_scope(): (discoveryengine_v1beta.RankServiceClient, "rank"), ] originals = [inspect.getattr_static(cls, name) for cls, name in untouched] - assert setup_discoveryengine() + assert setup_google_discoveryengine() for (cls, name), original in zip(untouched, originals): assert inspect.getattr_static(cls, name) is original for patcher in PATCHERS: @@ -396,9 +399,9 @@ def test_patch_scope(): @pytest.mark.vcr("test_answer_query[True].yaml") def test_stream_close_preserves_parent(memory_logger, credentials, SERVING_CONFIG): from braintrust import current_span, start_span - from braintrust.integrations.discoveryengine import wrap_discoveryengine + from braintrust.integrations.google_discoveryengine import wrap_google_discoveryengine - client = wrap_discoveryengine( + client = wrap_google_discoveryengine( discoveryengine.ConversationalSearchServiceClient(transport="rest", credentials=credentials) ) with start_span(name="caller") as parent: @@ -418,7 +421,9 @@ def test_stream_close_preserves_parent(memory_logger, credentials, SERVING_CONFI assert current_span() is parent spans = memory_logger.pop() assert len(spans) == 2 - child = next(span for span in spans if span["span_attributes"]["name"] == "discoveryengine.stream_answer_query") + child = next( + span for span in spans if span["span_attributes"]["name"] == "google_discoveryengine.stream_answer_query" + ) parent_row = next(span for span in spans if span["span_attributes"]["name"] == "caller") assert child["span_parents"] == [parent_row["span_id"]] assert "end" in child["metrics"] @@ -427,15 +432,15 @@ def test_stream_close_preserves_parent(memory_logger, credentials, SERVING_CONFI def test_auto_instrument_subprocess(): from braintrust.integrations.test_utils import verify_autoinstrument_script - verify_autoinstrument_script("test_auto_discoveryengine.py") + verify_autoinstrument_script("test_auto_google_discoveryengine.py") @pytest.mark.vcr @pytest.mark.parametrize("asynchronous_mode", [True, False]) def test_answer_requested_model(memory_logger, credentials, asynchronous_mode, SERVING_CONFIG): - from braintrust.integrations.discoveryengine import setup_discoveryengine + from braintrust.integrations.google_discoveryengine import setup_google_discoveryengine - setup_discoveryengine() + setup_google_discoveryengine() client = discoveryengine.ConversationalSearchServiceClient(transport="rest", credentials=credentials) from google.api_core.exceptions import BadRequest @@ -464,14 +469,14 @@ def test_answer_requested_model(memory_logger, credentials, asynchronous_mode, S @pytest.mark.vcr("test_rank.yaml") def test_normalization_failure_does_not_change_result(memory_logger, credentials, monkeypatch, rank_request): - from braintrust.integrations.discoveryengine import setup_discoveryengine, tracing + from braintrust.integrations.google_discoveryengine import setup_google_discoveryengine, tracing def broken(*args): raise ValueError("injected extraction failure") monkeypatch.setattr(tracing, "_prepare", broken) monkeypatch.setattr(tracing, "_output", broken) - setup_discoveryengine() + setup_google_discoveryengine() client = discoveryengine.RankServiceClient(transport="rest", credentials=credentials) result = client.rank(request=rank_request, retry=None) assert result.records[0].id == "1" @@ -483,11 +488,13 @@ def broken(*args): @pytest.mark.parametrize("consume", ["read", "cancel", "aclose"]) async def test_async_stream_lifecycle(memory_logger, credentials, vcr_cassette_dir, consume, SERVING_CONFIG): from braintrust import current_span, start_span - from braintrust.integrations.discoveryengine import wrap_discoveryengine - from braintrust.integrations.discoveryengine._test_grpc import grpc_cassette + from braintrust.integrations.google_discoveryengine import wrap_google_discoveryengine + from braintrust.integrations.google_discoveryengine._test_grpc import grpc_cassette from grpc.aio import EOF - client = wrap_discoveryengine(discoveryengine.ConversationalSearchServiceAsyncClient(credentials=credentials)) + client = wrap_google_discoveryengine( + discoveryengine.ConversationalSearchServiceAsyncClient(credentials=credentials) + ) payload = discoveryengine.AnswerQueryRequest( serving_config=SERVING_CONFIG, query={"text": QUERY}, @@ -524,7 +531,7 @@ async def test_async_stream_lifecycle(memory_logger, credentials, vcr_cassette_d spans = memory_logger.pop() assert len(spans) == 2 child = next( - span for span in spans if span["span_attributes"]["name"] == "discoveryengine.stream_answer_query" + span for span in spans if span["span_attributes"]["name"] == "google_discoveryengine.stream_answer_query" ) parent_row = next(span for span in spans if span["span_attributes"]["name"] == "caller") assert child["span_parents"] == [parent_row["span_id"]] @@ -541,9 +548,9 @@ async def test_async_stream_lifecycle(memory_logger, credentials, vcr_cassette_d @pytest.mark.vcr def test_rank_output_limit(memory_logger, credentials, LOCATION): - from braintrust.integrations.discoveryengine import setup_discoveryengine + from braintrust.integrations.google_discoveryengine import setup_google_discoveryengine - setup_discoveryengine() + setup_google_discoveryengine() client = discoveryengine.RankServiceClient(transport="rest", credentials=credentials) result = client.rank( request={ @@ -577,10 +584,10 @@ def error_request(stream, LOCATION, SERVING_CONFIG): [(False, "test_provider_error"), (True, "test_stream_provider_error")], ) def test_provider_error(memory_logger, credentials, stream, error_request, vcr_cassette_name): - from braintrust.integrations.discoveryengine import setup_discoveryengine + from braintrust.integrations.google_discoveryengine import setup_google_discoveryengine from google.api_core.exceptions import BadRequest, InternalServerError - setup_discoveryengine() + setup_google_discoveryengine() client_type = discoveryengine.ConversationalSearchServiceClient if stream else discoveryengine.RankServiceClient client = client_type(transport="rest", credentials=credentials) method = client.stream_answer_query if stream else client.rank @@ -597,11 +604,11 @@ def test_provider_error(memory_logger, credentials, stream, error_request, vcr_c @pytest.mark.asyncio @pytest.mark.parametrize("stream", [False, True]) async def test_async_provider_error(memory_logger, credentials, request, vcr_cassette_dir, stream, error_request): - from braintrust.integrations.discoveryengine import setup_discoveryengine - from braintrust.integrations.discoveryengine._test_grpc import grpc_cassette + from braintrust.integrations.google_discoveryengine import setup_google_discoveryengine + from braintrust.integrations.google_discoveryengine._test_grpc import grpc_cassette from google.api_core.exceptions import InternalServerError, InvalidArgument - setup_discoveryengine() + setup_google_discoveryengine() client_type = ( discoveryengine.ConversationalSearchServiceAsyncClient if stream else discoveryengine.RankServiceAsyncClient ) diff --git a/py/src/braintrust/integrations/discoveryengine/tracing.py b/py/src/braintrust/integrations/google_discoveryengine/tracing.py similarity index 99% rename from py/src/braintrust/integrations/discoveryengine/tracing.py rename to py/src/braintrust/integrations/google_discoveryengine/tracing.py index 867403403..79ead8bb4 100644 --- a/py/src/braintrust/integrations/discoveryengine/tracing.py +++ b/py/src/braintrust/integrations/google_discoveryengine/tracing.py @@ -15,7 +15,7 @@ _LOG = logging.getLogger(__name__) -_INSTRUMENTATION = "discoveryengine-auto" +_INSTRUMENTATION = "google-discoveryengine-auto" _MAX_RANK_RESULTS = 100 _ANSWER_DETAILS = ("citations", "references", "grounding_supports", "related_questions", "answer_skipped_reasons") @@ -139,7 +139,7 @@ def _safe_extract(fn, *args, default=None): def _start(method, request): span_input, metadata = _safe_extract(_prepare, method, request, default=(None, {"provider": "google"})) return start_span( - name=f"discoveryengine.{method}", + name=f"google_discoveryengine.{method}", type="llm", input=span_input, metadata=metadata, From 6e91a7fc4ad570a279cb589bab3d813e4edff929 Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Wed, 16 Sep 2026 15:38:05 -0400 Subject: [PATCH 3/4] fix(integrations): use task spans for discovery engine assessments Classify google_discoveryengine.rank and check_grounding as task spans: these APIs rank or assess existing content rather than generate answers. Keep answer_query, stream_answer_query, and converse_conversation as llm. Automatic setup and manual wrapping use the same classification for sync and async clients. ```text answer workflow [task] |-- google_discoveryengine.rank [task] |-- google_discoveryengine.stream_answer_query [llm] `-- google_discoveryengine.check_grounding [task] ``` Update existing REST/gRPC cassette tests: six assertions fail before the change, and all 26 integration tests pass afterward. Pylint and pre-commit pass. Reuse recordings because provider requests and responses are unchanged. --- .../google_discoveryengine/test_google_discoveryengine.py | 5 +++-- .../integrations/google_discoveryengine/tracing.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/py/src/braintrust/integrations/google_discoveryengine/test_google_discoveryengine.py b/py/src/braintrust/integrations/google_discoveryengine/test_google_discoveryengine.py index f43f7f786..fab3a9d51 100644 --- a/py/src/braintrust/integrations/google_discoveryengine/test_google_discoveryengine.py +++ b/py/src/braintrust/integrations/google_discoveryengine/test_google_discoveryengine.py @@ -139,7 +139,7 @@ def test_rank(memory_logger, credentials, rank_request, mode): assert len(spans) == 1 span = spans[0] assert span["span_attributes"]["name"] == "google_discoveryengine.rank" - assert span["span_attributes"]["type"] == "llm" + assert span["span_attributes"]["type"] == "task" assert span["metadata"]["provider"] == "google" assert span["metadata"]["model"] == "semantic-ranker-512@latest" assert span["input"]["query"] == "What is Braintrust?" @@ -238,6 +238,7 @@ def test_check_grounding(memory_logger, credentials, LOCATION): spans = memory_logger.pop() assert len(spans) == 1 assert spans[0]["span_attributes"]["name"] == "google_discoveryengine.check_grounding" + assert spans[0]["span_attributes"]["type"] == "task" assert spans[0]["output"]["support_score"] == result.support_score @@ -318,7 +319,7 @@ async def test_async_grpc( assert len(spans) == 1 span = spans[0] assert span["span_attributes"]["name"] == f"google_discoveryengine.{method}" - assert span["span_attributes"]["type"] == "llm" + assert span["span_attributes"]["type"] == ("task" if method in ("rank", "check_grounding") else "llm") assert span["metadata"]["provider"] == "google" assert "model" not in span["metadata"] assert span["context"]["span_origin"]["instrumentation"]["name"] == "google-discoveryengine-auto" diff --git a/py/src/braintrust/integrations/google_discoveryengine/tracing.py b/py/src/braintrust/integrations/google_discoveryengine/tracing.py index 79ead8bb4..d900af1ad 100644 --- a/py/src/braintrust/integrations/google_discoveryengine/tracing.py +++ b/py/src/braintrust/integrations/google_discoveryengine/tracing.py @@ -140,7 +140,7 @@ def _start(method, request): span_input, metadata = _safe_extract(_prepare, method, request, default=(None, {"provider": "google"})) return start_span( name=f"google_discoveryengine.{method}", - type="llm", + type="task" if method in ("rank", "check_grounding") else "llm", input=span_input, metadata=metadata, internal={"instrumentation": _INSTRUMENTATION}, From fd166312fc816f1ecbd1376fbd7b3c08338d090b Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Thu, 17 Sep 2026 08:45:38 -0400 Subject: [PATCH 4/4] streaming fix --- .agents/skills/sdk-integrations/SKILL.md | 22 ++++++++++ .../test_google_discoveryengine.py | 44 +++++++++++++++---- .../google_discoveryengine/tracing.py | 7 +++ 3 files changed, 64 insertions(+), 9 deletions(-) diff --git a/.agents/skills/sdk-integrations/SKILL.md b/.agents/skills/sdk-integrations/SKILL.md index 36d9cd820..fadc984c6 100644 --- a/.agents/skills/sdk-integrations/SKILL.md +++ b/.agents/skills/sdk-integrations/SKILL.md @@ -263,6 +263,28 @@ Assert on emitted spans (not just provider return values): For streaming, assert both the provider iterator/async-iterator still works AND the final span has aggregated `output` + stream-specific `metrics`. +### Streaming lifecycle review + +Review every exit path, not just full consumption. Check the real provider's +iterator, context-manager, cancellation, and garbage-collection behavior before +choosing a wrapper. A proxy can preserve transport cleanup while still losing +the span's final output and end time. + +- Cover exhaustion, provider errors, explicit close/cancel, and context-manager + exit where the provider supports it. Preserve exception and return semantics. +- Cover `break` followed by dropping the last stream reference, and dropping a + stream before consuming any chunks. `break` alone does not close a retained + iterator; do not promise immediate finalization while callers still hold it. +- Use existing recordings for partial-consumption tests. Drop the proxy, force + collection, and assert partial (or empty) output, `metrics.end`, correct + parentage, and no duplicate finalization after explicit close or exhaustion. +- A GC fallback must not retain the stream through its callback or closure. + For async streams, do not run or schedule event-loop work from a finalizer; + finalize trace state and preserve the provider's own cleanup behavior. +- Check that neither iteration nor cleanup leaves the stream span current in + the caller's context. GC is a best-effort fallback, not a substitute for + deterministic cleanup when the caller explicitly closes the stream. + Cassettes live in `integrations//cassettes//` (e.g. `cassettes/latest/`, `cassettes/0.48.0/`). Nox sets `BRAINTRUST_TEST_PACKAGE_VERSION` so cassettes land correctly. Do not add per-test `vcr_cassette_dir` / `cassette_library_dir` fixtures — `integrations/conftest.py` handles it. Re-record only when behavior intentionally changed. Sanitize binary media in both request and response bodies so checked-in cassettes do not retain large base64 payloads. Confirm the exact session name from `noxfile.py` — don't assume it matches the folder. diff --git a/py/src/braintrust/integrations/google_discoveryengine/test_google_discoveryengine.py b/py/src/braintrust/integrations/google_discoveryengine/test_google_discoveryengine.py index fab3a9d51..2d116f313 100644 --- a/py/src/braintrust/integrations/google_discoveryengine/test_google_discoveryengine.py +++ b/py/src/braintrust/integrations/google_discoveryengine/test_google_discoveryengine.py @@ -1,8 +1,10 @@ """Real Discovery Engine responses, recorded over REST and gRPC.""" +import gc import json import os import subprocess +import weakref from contextlib import nullcontext from pathlib import Path from urllib.parse import urlsplit @@ -398,7 +400,8 @@ def test_patch_scope(): @pytest.mark.vcr("test_answer_query[True].yaml") -def test_stream_close_preserves_parent(memory_logger, credentials, SERVING_CONFIG): +@pytest.mark.parametrize("consume", ["close", "abandon", "unstarted"]) +def test_stream_lifecycle(memory_logger, credentials, SERVING_CONFIG, consume): from braintrust import current_span, start_span from braintrust.integrations.google_discoveryengine import wrap_google_discoveryengine @@ -415,10 +418,20 @@ def test_stream_close_preserves_parent(memory_logger, credentials, SERVING_CONFI retry=None, ) assert current_span() is parent - next(stream) + chunks = [] + if consume != "unstarted": + for chunk in stream: + chunks.append(chunk) + if chunk.answer.answer_text: + break assert current_span() is parent - stream.close() - stream.close() + if consume == "close": + stream.close() + stream.close() + stream_ref = weakref.ref(stream) + del stream + gc.collect() + assert stream_ref() is None assert current_span() is parent spans = memory_logger.pop() assert len(spans) == 2 @@ -427,7 +440,10 @@ def test_stream_close_preserves_parent(memory_logger, credentials, SERVING_CONFI ) parent_row = next(span for span in spans if span["span_attributes"]["name"] == "caller") assert child["span_parents"] == [parent_row["span_id"]] + assert child["output"][0]["message"]["content"] == "".join(chunk.answer.answer_text for chunk in chunks) assert "end" in child["metrics"] + gc.collect() + assert memory_logger.pop() == [] def test_auto_instrument_subprocess(): @@ -486,7 +502,7 @@ def broken(*args): @pytest.mark.asyncio -@pytest.mark.parametrize("consume", ["read", "cancel", "aclose"]) +@pytest.mark.parametrize("consume", ["read", "cancel", "aclose", "abandon", "unstarted"]) async def test_async_stream_lifecycle(memory_logger, credentials, vcr_cassette_dir, consume, SERVING_CONFIG): from braintrust import current_span, start_span from braintrust.integrations.google_discoveryengine import wrap_google_discoveryengine @@ -521,13 +537,21 @@ async def test_async_stream_lifecycle(memory_logger, credentials, vcr_cassette_d chunks.append(chunk) assert current_span() is parent assert await stream.read() is EOF - else: - chunks.append(await stream.__anext__()) + elif consume != "unstarted": + async for chunk in stream: + chunks.append(chunk) + if chunk.answer.answer_text: + break if consume == "cancel": assert stream.cancel() - else: + elif consume == "aclose": await stream.aclose() - assert stream.cancelled() + if consume in ("cancel", "aclose"): + assert stream.cancelled() + stream_ref = weakref.ref(stream) + del stream + gc.collect() + assert stream_ref() is None assert current_span() is parent spans = memory_logger.pop() assert len(spans) == 2 @@ -543,6 +567,8 @@ async def test_async_stream_lifecycle(memory_logger, credentials, vcr_cassette_d ) assert child["output"][0]["message"]["content"] == expected_text assert "end" in child["metrics"] + gc.collect() + assert memory_logger.pop() == [] finally: await client.transport.close() diff --git a/py/src/braintrust/integrations/google_discoveryengine/tracing.py b/py/src/braintrust/integrations/google_discoveryengine/tracing.py index d900af1ad..2be500ad0 100644 --- a/py/src/braintrust/integrations/google_discoveryengine/tracing.py +++ b/py/src/braintrust/integrations/google_discoveryengine/tracing.py @@ -7,6 +7,7 @@ import logging import time +import weakref from collections.abc import Mapping from itertools import islice @@ -195,6 +196,9 @@ class _AnswerStream(ObjectProxy): def __init__(self, stream, state): super().__init__(stream) self._self_state = state + # Retain only trace state, not the proxy/provider stream. GC can finalize + # partial output while the provider handles its own transport cleanup. + weakref.finalize(self, state.finish) self._self_iterator = iter(stream) def __iter__(self): @@ -232,6 +236,9 @@ class _AsyncAnswerStream(ObjectProxy): def __init__(self, stream, state): super().__init__(stream) self._self_state = state + # Retain only trace state, not the proxy/provider stream. GC can finalize + # partial output while the provider handles its own transport cleanup. + weakref.finalize(self, state.finish) self._self_iterator = None def __aiter__(self):