From dafce45a2a867d699e51a46a09be13c9a6e46c0e Mon Sep 17 00:00:00 2001 From: hushuming Date: Mon, 31 Aug 2026 09:28:34 +0800 Subject: [PATCH 1/2] fix(examples): disable prefix caching in multimodal QA and document the symptom vLLM's prefix cache desyncs from the multimodal receiver cache across the trainer's sleep/wake cycles (vllm#42995), surfacing as 'AssertionError: Expected a cached item for mm_hash=...' or a silent rollout hang. Turn it off in the example config and add a troubleshooting note to the example doc. --- docs/80-example-multimodal-qa.md | 4 ++++ examples/multimodal_qa/train_multimodal_qa.py | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/docs/80-example-multimodal-qa.md b/docs/80-example-multimodal-qa.md index c61f1d095..8c1b5fd53 100644 --- a/docs/80-example-multimodal-qa.md +++ b/docs/80-example-multimodal-qa.md @@ -46,3 +46,7 @@ Multimodal training requires `agentlightning.trace_aggregator.level: transition` - Rollout replies converge towards the correct count and `training/reward` climbs from chance level (~0.2) towards 1.0. - At each training step the batch carries `multi_modal_inputs` in `non_tensor_batch` and `position_ids` with `dim() == 3` (mrope). To see the vision tensors reach the model forward, log `model_inputs` keys in verl's `prepare_model_inputs` — `pixel_values` and `image_grid_thw` should be present. - Without the multimodal data path (or with images silently dropped), training still runs but the warning `rollout traces contain images but RolloutAdapter has no processor` appears and the vision signal never reaches the training forward. + +## Troubleshooting + +- **`AssertionError: Expected a cached item for mm_hash=...`, or rollouts hanging after a sleep/wake cycle**: this is a vLLM multimodal prefix-cache desync ([vllm#42995](https://github.com/vllm-project/vllm/issues/42995)), not an Agent Lightning bug. The trainer sleeps and wakes the vLLM replicas around each update step, which is exactly what triggers it. Set `actor_rollout_ref.rollout.enable_prefix_caching: False` (already the default in this example) and restart from the latest checkpoint. diff --git a/examples/multimodal_qa/train_multimodal_qa.py b/examples/multimodal_qa/train_multimodal_qa.py index 62c6f2511..a104daa49 100644 --- a/examples/multimodal_qa/train_multimodal_qa.py +++ b/examples/multimodal_qa/train_multimodal_qa.py @@ -80,6 +80,11 @@ def verl_default_config() -> dict[str, Any]: "multi_turn": {"format": "hermes"}, "name": "vllm", "gpu_memory_utilization": 0.6, + # vLLM's prefix cache can desync from the multimodal receiver cache + # across sleep/wake cycles (`AssertionError: Expected a cached item + # for mm_hash=...`, or a silent rollout hang). Keep it off for + # multimodal training. See https://github.com/vllm-project/vllm/issues/42995 + "enable_prefix_caching": False, }, "actor": { "ppo_mini_batch_size": 8, From 3c3534c22ce5fc1baf021b87609a35fbc95ad992 Mon Sep 17 00:00:00 2001 From: hushuming Date: Mon, 31 Aug 2026 09:38:03 +0800 Subject: [PATCH 2/2] docs(examples): qualify affected vLLM versions for the prefix-cache note --- docs/80-example-multimodal-qa.md | 2 +- examples/multimodal_qa/train_multimodal_qa.py | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/80-example-multimodal-qa.md b/docs/80-example-multimodal-qa.md index 8c1b5fd53..6cd6e9e5c 100644 --- a/docs/80-example-multimodal-qa.md +++ b/docs/80-example-multimodal-qa.md @@ -49,4 +49,4 @@ Multimodal training requires `agentlightning.trace_aggregator.level: transition` ## Troubleshooting -- **`AssertionError: Expected a cached item for mm_hash=...`, or rollouts hanging after a sleep/wake cycle**: this is a vLLM multimodal prefix-cache desync ([vllm#42995](https://github.com/vllm-project/vllm/issues/42995)), not an Agent Lightning bug. The trainer sleeps and wakes the vLLM replicas around each update step, which is exactly what triggers it. Set `actor_rollout_ref.rollout.enable_prefix_caching: False` (already the default in this example) and restart from the latest checkpoint. +- **`AssertionError: Expected a cached item for mm_hash=...`, or rollouts hanging after a sleep/wake cycle**: on vLLM < 0.22.0 the prefix cache can desync from the multimodal receiver cache across the trainer's sleep/wake cycles ([vllm#42995](https://github.com/vllm-project/vllm/issues/42995), fixed upstream by [vllm#43001](https://github.com/vllm-project/vllm/pull/43001)). This is a vLLM version issue, not an Agent Lightning bug. On affected versions, keep `actor_rollout_ref.rollout.enable_prefix_caching: False` (the default in this example) and restart the run — resuming from the latest checkpoint if you have checkpointing enabled. diff --git a/examples/multimodal_qa/train_multimodal_qa.py b/examples/multimodal_qa/train_multimodal_qa.py index a104daa49..982ef4439 100644 --- a/examples/multimodal_qa/train_multimodal_qa.py +++ b/examples/multimodal_qa/train_multimodal_qa.py @@ -80,10 +80,11 @@ def verl_default_config() -> dict[str, Any]: "multi_turn": {"format": "hermes"}, "name": "vllm", "gpu_memory_utilization": 0.6, - # vLLM's prefix cache can desync from the multimodal receiver cache - # across sleep/wake cycles (`AssertionError: Expected a cached item - # for mm_hash=...`, or a silent rollout hang). Keep it off for - # multimodal training. See https://github.com/vllm-project/vllm/issues/42995 + # vLLM < 0.22.0: the prefix cache can desync from the multimodal + # receiver cache across sleep/wake cycles (`AssertionError: + # Expected a cached item for mm_hash=...`, or a silent rollout + # hang). Keep it off for multimodal training. + # See https://github.com/vllm-project/vllm/issues/42995 "enable_prefix_caching": False, }, "actor": {