Skip to content

Commit abb596c

Browse files
committed
Add support for unwrapping {"code", "files"} payloads in answers and submissions
Refactors `_resolve_submission` into `_unwrap_payload` and updates answer handling to support the `{"code", "files"}` structure. Adds unit tests for evaluation behavior with structured answers.
1 parent ce273e1 commit abb596c

2 files changed

Lines changed: 64 additions & 15 deletions

File tree

evaluation_function/evaluation.py

Lines changed: 28 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -343,15 +343,15 @@ def _coerce_file_specs(raw: Any) -> list:
343343
return specs
344344

345345

346-
def _resolve_submission(response: Any, params: Params) -> tuple[str, list]:
347-
"""Split the submission into (code, file_specs).
346+
def _unwrap_payload(value: Any) -> tuple[str, list]:
347+
"""Return (code, file_specs) from a submission or answer value.
348348
349-
When file upload is enabled, the LF web client delivers the response
350-
payload as {"code": ..., "files": [...]} (sometimes as a JSON string of
351-
that object) rather than a bare code string. Files listed in the
352-
response take precedence; params["files"] is the fallback.
349+
When file upload is enabled, the LF web client delivers the value as
350+
{"code": ..., "files": [...]} (sometimes as a JSON string of that
351+
object) rather than a bare code string. A plain string is returned
352+
unchanged with no files.
353353
"""
354-
payload = response
354+
payload = value
355355
if isinstance(payload, str):
356356
try:
357357
parsed = json.loads(payload)
@@ -361,14 +361,27 @@ def _resolve_submission(response: Any, params: Params) -> tuple[str, list]:
361361
payload = parsed
362362

363363
if isinstance(payload, dict):
364-
code = payload.get("code") or ""
365-
response_files = _coerce_file_specs(payload.get("files"))
366-
else:
367-
code = payload if isinstance(payload, str) else str(payload)
368-
response_files = []
364+
return str(payload.get("code") or ""), _coerce_file_specs(payload.get("files"))
365+
if isinstance(payload, str):
366+
return payload, []
367+
return str(payload), []
368+
369369

370+
def _resolve_submission(response: Any, params: Params) -> tuple[str, list]:
371+
"""Split the submission into (code, file_specs).
372+
373+
Files listed in the response take precedence; params["files"] is the
374+
fallback.
375+
"""
376+
code, response_files = _unwrap_payload(response)
370377
file_specs = response_files or _coerce_file_specs(params.get("files"))
371-
return str(code), file_specs
378+
return code, file_specs
379+
380+
381+
def _answer_code(answer: Any) -> str:
382+
"""The code string from the answer field, unwrapping a {code, files}
383+
payload the same way the submission is unwrapped."""
384+
return _unwrap_payload(answer)[0]
372385

373386

374387
def evaluation_function(response: Any, answer: Any, params: Params) -> Result:
@@ -398,10 +411,10 @@ def evaluation_function(response: Any, answer: Any, params: Params) -> Result:
398411
if mode == "demo":
399412
result = _evaluate_demo(code, result, files_dir)
400413
elif mode == "io_test":
401-
ans = str(answer) if params.get("use_answer_as_expected_output") else ""
414+
ans = _answer_code(answer) if params.get("use_answer_as_expected_output") else ""
402415
result = _evaluate_io(code, params.get("tests", []), result, answer=ans, files_dir=files_dir)
403416
else:
404-
test_code = str(answer) if params.get("use_answer_as_test_code") else params.get("test_code", "")
417+
test_code = _answer_code(answer) if params.get("use_answer_as_test_code") else params.get("test_code", "")
405418
result = _evaluate_unit(code, test_code, result, files_dir=files_dir)
406419

407420
for warning in file_warnings:

evaluation_function/evaluation_test.py

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -479,6 +479,42 @@ def test_response_dict_without_files_no_download(self):
479479
mock_download.assert_not_called()
480480

481481

482+
class TestAnswerFieldPayload(unittest.TestCase):
483+
"""With the file-upload widget the answer field is delivered in the same
484+
{"code", "files"} shape as the submission, not as a bare string."""
485+
486+
def test_unit_test_code_from_answer_dict(self):
487+
response = {"code": "def square(n):\n return n * n\n"}
488+
answer = {"code": "def test_sq():\n assert square(4) == 16\n", "files": []}
489+
params = {"mode": "unit_test", "use_answer_as_test_code": True}
490+
result = evaluation_function(response, answer, params).to_dict()
491+
492+
self.assertTrue(result["is_correct"])
493+
self.assertIn("1/1 tests passed", result["feedback"])
494+
495+
def test_unit_test_code_from_answer_json_string(self):
496+
answer = json.dumps({"code": "def test_ok():\n assert True\n", "files": []})
497+
params = {"mode": "unit_test", "use_answer_as_test_code": True}
498+
result = evaluation_function({"code": ""}, answer, params).to_dict()
499+
500+
self.assertIn("1/1 tests passed", result["feedback"])
501+
502+
def test_io_expected_output_from_answer_dict(self):
503+
response = {"code": "print(6)"}
504+
answer = {"code": "print(2 * 3)", "files": []}
505+
params = {"mode": "io_test", "use_answer_as_expected_output": True,
506+
"tests": [{"input": ""}]}
507+
result = evaluation_function(response, answer, params).to_dict()
508+
509+
self.assertTrue(result["is_correct"])
510+
511+
def test_plain_string_answer_still_works(self):
512+
params = {"mode": "unit_test", "use_answer_as_test_code": True}
513+
result = evaluation_function("x = 1", "def test_ok():\n assert True\n", params).to_dict()
514+
515+
self.assertIn("1/1 tests passed", result["feedback"])
516+
517+
482518
class TestUnexpectedExceptionHandling(unittest.TestCase):
483519

484520
@patch("evaluation_function.evaluation._run_code")

0 commit comments

Comments
 (0)