From 5090bffe75a8bf0fb704b7593394fa55d60543e0 Mon Sep 17 00:00:00 2001 From: Anai Guo Date: Fri, 17 Jul 2026 03:11:21 -0700 Subject: [PATCH 1/2] fix(llama): release a vision chat handler's mtmd context on close (#2342) `Llama.close()` tore down the model and its context but left the chat handler's `_exit_stack` untouched, so the mtmd/clip context it built from that model was never freed. Handlers routinely outlive the `Llama` that initialized them -- callers construct one handler and reuse it across loads -- and `_init_mtmd_context()` returns early while `mtmd_ctx` is set. After the first `close()` the handler therefore kept a context bound to an already-freed model and handed it back on the next load, where it surfaces as a null `mtmd_ctx` on the C++ side. Register the handler's exit stack on the Llama's `_stack`. It unwinds LIFO, so the mtmd context is released ahead of the model teardown registered earlier in `__init__`, and `mtmd_free` resets `mtmd_ctx` to `None` so the next load re-initializes cleanly. Duck-typed on `_exit_stack`, which both `Llava15ChatHandler` and `MTMDChatHandler` (and every subclass) expose. --- llama_cpp/llama.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index 14e2f8500..d7d97363d 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -463,6 +463,16 @@ def free_lora_adapter(): self.chat_format = chat_format self.chat_handler = chat_handler + # A vision chat handler builds its mtmd/clip context from this model, so + # that context must be released before the model itself is freed. The + # handler object can outlive the Llama (callers commonly reuse a single + # handler across loads), and it skips re-initialization while mtmd_ctx + # is set -- leaving a context bound to an already-freed model behind. + # `_stack` unwinds LIFO, so this runs before the model teardown that was + # registered earlier in __init__. + handler_stack = getattr(chat_handler, "_exit_stack", None) + if handler_stack is not None: + self._stack.callback(handler_stack.close) self._chat_handlers: Dict[ str, llama_chat_format.LlamaChatCompletionHandler ] = {} From 11aeacda2a2e65ed4264a3ce10e13fa05d330c93 Mon Sep 17 00:00:00 2001 From: abetlen Date: Mon, 17 Aug 2026 00:23:27 -0700 Subject: [PATCH 2/2] fix: bind multimodal context cleanup to model --- CHANGELOG.md | 1 + llama_cpp/llama.py | 10 ---------- llama_cpp/llama_chat_format.py | 7 ++----- 3 files changed, 3 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 853a7ca37..cc48609da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - fix(example): retain recurrent state for server MTP rollback - feat: update llama.cpp to ggml-org/llama.cpp@adb55e514 - fix(server): show falsey defaults in CLI help by @cupkk in #2355 +- fix: release multimodal contexts when closing Llama by @Anai-Guo in #2343 ## [0.3.34] diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py index d7d97363d..14e2f8500 100644 --- a/llama_cpp/llama.py +++ b/llama_cpp/llama.py @@ -463,16 +463,6 @@ def free_lora_adapter(): self.chat_format = chat_format self.chat_handler = chat_handler - # A vision chat handler builds its mtmd/clip context from this model, so - # that context must be released before the model itself is freed. The - # handler object can outlive the Llama (callers commonly reuse a single - # handler across loads), and it skips re-initialization while mtmd_ctx - # is set -- leaving a context bound to an already-freed model behind. - # `_stack` unwinds LIFO, so this runs before the model teardown that was - # registered earlier in __init__. - handler_stack = getattr(chat_handler, "_exit_stack", None) - if handler_stack is not None: - self._stack.callback(handler_stack.close) self._chat_handlers: Dict[ str, llama_chat_format.LlamaChatCompletionHandler ] = {} diff --git a/llama_cpp/llama_chat_format.py b/llama_cpp/llama_chat_format.py index 4f41c2eb7..62e236415 100644 --- a/llama_cpp/llama_chat_format.py +++ b/llama_cpp/llama_chat_format.py @@ -9,7 +9,6 @@ import string from datetime import datetime -from contextlib import ExitStack from typing import ( Any, Dict, @@ -2779,7 +2778,6 @@ def __init__(self, clip_model_path: str, verbose: bool = True): self.clip_model_path = clip_model_path self.verbose = verbose self._mtmd_cpp = mtmd_cpp - self._exit_stack = ExitStack() self.mtmd_ctx: Optional[mtmd_cpp.mtmd_context_p] = None if not os.path.exists(clip_model_path): @@ -2825,7 +2823,7 @@ def mtmd_free(): self._mtmd_cpp.mtmd_free(self.mtmd_ctx) self.mtmd_ctx = None - self._exit_stack.callback(mtmd_free) + llama_model._stack.callback(mtmd_free) def load_image(self, image_url: str) -> bytes: return self._load_image(image_url) @@ -3278,7 +3276,6 @@ def __init__( self.verbose = verbose self.use_gpu = use_gpu self._mtmd_cpp = mtmd_cpp - self._exit_stack = ExitStack() self.mtmd_ctx: Optional[mtmd_cpp.mtmd_context_p] = None if not os.path.exists(clip_model_path): @@ -3321,7 +3318,7 @@ def mtmd_free(): self._mtmd_cpp.mtmd_free(self.mtmd_ctx) self.mtmd_ctx = None - self._exit_stack.callback(mtmd_free) + llama_model._stack.callback(mtmd_free) def load_image(self, image_url: str) -> bytes: return self._load_image(image_url)