From 627834575b036f704da5485aa06d0ae34eb21f0d Mon Sep 17 00:00:00 2001 From: "Peter B. Johnson" Date: Wed, 9 Sep 2026 21:21:38 +0100 Subject: [PATCH] Fix truncation of evaluation requests larger than 4 KB The pinned lf_toolkit commit read at most `size` bytes from the socket instead of reading to the newline delimiter, so any request longer than the 4096-byte read hint arrived truncated. The worker could not decode it and never replied, and Shimmy returned a 500 after its 30 second deadline. Any submission over roughly 45 notes was affected. The platform sends `response` and `answer` as JSON, so most real pieces exceeded the limit. Upstream fixed the framing in toolkit-python#10. Update the lock to pick it up, and add regression tests in transport_test.py that push a payload past the read hint so a future pin cannot reintroduce the bug. Verified against the container: a 200-note request (18.8 KB) now returns in 0.12 s, where it previously timed out after 30 s. Co-Authored-By: Claude Opus 5 --- evaluation_function/transport_test.py | 115 ++++++++++++++++++++++++++ poetry.lock | 5 +- 2 files changed, 118 insertions(+), 2 deletions(-) create mode 100644 evaluation_function/transport_test.py diff --git a/evaluation_function/transport_test.py b/evaluation_function/transport_test.py new file mode 100644 index 0000000..a1210b9 --- /dev/null +++ b/evaluation_function/transport_test.py @@ -0,0 +1,115 @@ +""" +transport_test.py +================= +Regression tests for the transport that carries evaluation requests from +Shimmy to this function. + +The Lambda Feedback platform sends `response` and `answer` as JSON, and a +real piece of music easily runs to thousands of bytes. Requests travel over +a unix socket using lf_toolkit's newline-delimited framing, so a framing +bug that truncates long messages silently breaks every realistic +submission while short test cases keep passing. + +These tests exercise lf_toolkit's framing directly with a payload larger +than the 4096-byte read hint, so that a regression in the pinned toolkit +version is caught here rather than in production. + +Run locally with: python -m pytest evaluation_function/transport_test.py -v +""" + +import json + +import anyio + +from lf_toolkit.io.stream_io import NewlineStreamIO + +from .evaluation_test import make_midi + + +# Helpers +# ------------------------------------------------------------------------------ +class FakeStream: + """ + Minimal in-memory stand-in for a socket. + + read(size) returns at most `size` bytes from the pending buffer, exactly + as a real socket may return a short read. + """ + + def __init__(self, data=b""): + self._pending = data + self.written = b"" + + async def read(self, size): + chunk = self._pending[:size] + self._pending = self._pending[size:] + return chunk + + async def write(self, data): + self.written += data + + +def make_long_request(note_count): + """ + Build a realistic evaluation request of roughly `note_count` notes, + serialised the way the platform sends it. + """ + pitches = [60 + (i % 12) for i in range(note_count)] + starts = [round(i * 0.5, 3) for i in range(note_count)] + durations = [0.4] * note_count + + midi = make_midi(pitches, starts, durations) + return json.dumps({"response": midi, "answer": midi, "params": {}}) + + +def round_trip(payload): + """ + Write `payload` through NewlineStreamIO and read it back, the way the + IPC server frames one request. + """ + + async def run(): + writer_stream = FakeStream() + await NewlineStreamIO(writer_stream).write(payload.encode("utf-8")) + + reader = NewlineStreamIO(FakeStream(writer_stream.written)) + return await reader.read(4096) + + return anyio.run(run) + + +# Tests +# ------------------------------------------------------------------------------ +def test_short_request_round_trips(): + """A request comfortably under the read hint must survive unchanged.""" + payload = make_long_request(5) + assert len(payload) < 4096 + + assert round_trip(payload).decode("utf-8") == payload + + +def test_request_larger_than_read_hint_is_not_truncated(): + """ + A request larger than the 4096-byte read hint must survive unchanged. + + The framing is newline-delimited, so the read must continue to the + delimiter rather than stopping at the size hint. + """ + payload = make_long_request(200) + assert len(payload) > 4096, "test payload must exceed the read hint" + + assert round_trip(payload).decode("utf-8") == payload + + +def test_large_request_still_parses_as_json(): + """ + The practical symptom of truncation: the worker receives a prefix of the + request and cannot decode it, so it never replies and the caller times out. + """ + payload = make_long_request(200) + + received = round_trip(payload).decode("utf-8") + parsed = json.loads(received) + + assert len(parsed["response"]["notes"]) == 200 + assert len(parsed["answer"]["notes"]) == 200 diff --git a/poetry.lock b/poetry.lock index b065fde..5c555e9 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.1.4 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.2.0 and should not be changed by hand. [[package]] name = "absl-py" @@ -2148,6 +2148,7 @@ sympy = ">=1.12,<2.0" ujson = "5.10.0" [package.extras] +gcs = ["google-cloud-storage (>=2.18,<3.0)"] http = ["fastapi (>=0.115.0,<0.116.0)"] ipc = ["pywin32 (>=306,<307) ; sys_platform == \"win32\""] parsing = ["antlr4-python3-runtime (==4.13.2)", "lark (==1.2.2)", "latex2sympy @ git+https://github.com/purdue-tlt/latex2sympy.git@1.12.0"] @@ -2156,7 +2157,7 @@ parsing = ["antlr4-python3-runtime (==4.13.2)", "lark (==1.2.2)", "latex2sympy @ type = "git" url = "https://github.com/lambda-feedback/toolkit-python.git" reference = "main" -resolved_reference = "8a687d35284c156f045dde4da15a2e717112c9de" +resolved_reference = "ae52fa6f23c585f2595e6f32dc56ac8984c8fc50" [[package]] name = "libclang"