Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions py/src/braintrust/devserver/server.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
)
)

from ..bt_json import bt_dumps
from ..framework import (
EvalAsync,
EvalHooks,
Expand Down Expand Up @@ -369,10 +370,10 @@ def scorer_fn(input, output, expected, metadata):
"mode": "auto",
"strict": True,
}
headers = {"Accept": "application/json"}
headers = {"Accept": "application/json", "Content-Type": "application/json"}
if project_id:
headers["x-bt-project-id"] = project_id
result = state.proxy_conn().post("function/invoke", json=request, headers=headers)
result = state.proxy_conn().post("function/invoke", data=bt_dumps(request).encode("utf-8"), headers=headers)
result.raise_for_status()
data = result.json()
return data
Expand Down
52 changes: 52 additions & 0 deletions py/src/braintrust/devserver/test_server_integration.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
import os
from pathlib import Path
from typing import Any
from unittest.mock import MagicMock

import pytest
from braintrust.framework import _evals
Expand Down Expand Up @@ -236,6 +237,57 @@ def test_eval_error_handling(client, api_key, org_name):
assert "not found" in error["error"].lower()


def test_remote_scorer_serializes_readonly_attachments(monkeypatch):
from braintrust.devserver.server import make_scorer
from braintrust.logger import BraintrustState, ReadonlyAttachment

attachment_reference = {
"type": "braintrust_attachment",
"filename": "document.pdf",
"content_type": "application/pdf",
"key": "attachment-key",
}
attachment = ReadonlyAttachment(attachment_reference)

response = MagicMock()
response.json.return_value = {"score": 1}
proxy_conn = MagicMock()
proxy_conn.post.return_value = response

state = BraintrustState()
monkeypatch.setattr(state, "proxy_conn", lambda: proxy_conn)

scorer = make_scorer(state, "remote-scorer", {"function_id": "scorer-id"}, project_id="project-id")
result = scorer(
input={"file": attachment},
output="extracted text",
expected="expected text",
metadata={"source": "test"},
)

assert result == {"score": 1}
response.raise_for_status.assert_called_once_with()
proxy_conn.post.assert_called_once()

args, kwargs = proxy_conn.post.call_args
assert args == ("function/invoke",)
assert set(kwargs) == {"data", "headers"}
assert kwargs["headers"] == {
"Accept": "application/json",
"Content-Type": "application/json",
"x-bt-project-id": "project-id",
}
assert isinstance(kwargs["data"], bytes)

request = json.loads(kwargs["data"].decode("utf-8"))
assert request["input"] == {
"input": {"file": attachment_reference},
"output": "extracted text",
"expected": "expected text",
"metadata": {"source": "test"},
}


@pytest.mark.skipif(not HAS_PYDANTIC, reason="pydantic not installed")
def test_eval_uses_inline_request_parameters(api_key, org_name, monkeypatch):
from braintrust import Evaluator
Expand Down