From 3e168e39c4ae968175fcea546b057cc0df5a7799 Mon Sep 17 00:00:00 2001 From: james-333i Date: Tue, 25 Aug 2026 11:29:46 -0700 Subject: [PATCH 1/2] Fix LlamaLanguageModel build against current llama.swift The open-ended dependency range resolves llama.swift to releases wrapping current llama.cpp builds, where the Llama trait no longer compiles: llama_sampler_init_penalties regained its leading n_vocab parameter, and llama_model_params replaced use_mmap and use_mlock with a llama_load_mode enum. Pass the vocabulary size at all three penalties call sites and set load_mode to LLAMA_LOAD_MODE_MMAP, matching the previous mmap-only behavior. Verified against llama.swift 2.10549.0 with the full live test suite. --- Package.resolved | 9 +++++++++ Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift | 6 ++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/Package.resolved b/Package.resolved index b689a691..0b86bc46 100644 --- a/Package.resolved +++ b/Package.resolved @@ -19,6 +19,15 @@ "version" : "1.3.1" } }, + { + "identity" : "llama.swift", + "kind" : "remoteSourceControl", + "location" : "https://github.com/mattt/llama.swift", + "state" : { + "revision" : "716419d4d7aa542fce301e809cde7234c68ddbc6", + "version" : "2.10549.0" + } + }, { "identity" : "partialjsondecoder", "kind" : "remoteSourceControl", diff --git a/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift b/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift index 6bcd8045..56d858a9 100644 --- a/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift @@ -673,8 +673,7 @@ import Foundation params.n_gpu_layers = 0 // Try to reduce memory usage - params.use_mmap = true - params.use_mlock = false + params.load_mode = LLAMA_LOAD_MODE_MMAP return params } @@ -829,6 +828,7 @@ import Foundation llama_sampler_chain_add( samplerPtr, llama_sampler_init_penalties( + llama_vocab_n_tokens(vocab), effectiveRepeatLastN, effectiveRepeatPenalty, effectiveFrequencyPenalty, @@ -958,6 +958,7 @@ import Foundation llama_sampler_chain_add( samplerPointer, llama_sampler_init_penalties( + llama_vocab_n_tokens(vocab), options.repeatLastN, options.repeatPenalty, options.frequencyPenalty, @@ -1197,6 +1198,7 @@ import Foundation llama_sampler_chain_add( samplerPtr, llama_sampler_init_penalties( + llama_vocab_n_tokens(vocab), effectiveRepeatLastN, effectiveRepeatPenalty, effectiveFrequencyPenalty, From 360279caf612e3bf925c95bc99c4e426cbd9dc18 Mon Sep 17 00:00:00 2001 From: james-333i Date: Tue, 25 Aug 2026 12:06:33 -0700 Subject: [PATCH 2/2] Fix constrained sampling ignoring the allowed-token set LlamaTokenBackend.sample(from:) masked disallowed tokens by writing negative infinity into the buffer returned by llama_get_logits, then called llama_sampler_sample. The context can refresh that buffer when the sampler fetches logits, so the mask is dropped and sampling runs over the full vocabulary. Constrained JSON generation then emits tokens outside the allowed set and fails. Build a llama_token_data_array containing only the allowed tokens and their logits, apply the sampler chain to it with llama_sampler_apply, and return the selected candidate. --- .../Models/LlamaLanguageModel.swift | 33 +++++++++++++++---- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift b/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift index 56d858a9..03fbd609 100644 --- a/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift +++ b/Sources/AnyLanguageModel/Models/LlamaLanguageModel.swift @@ -1110,18 +1110,37 @@ import Foundation } mutating func sample(from allowedTokens: Set) async throws -> Int { - guard let logits = llama_get_logits(context) else { + // Masking llama_get_logits in place does not constrain + // llama_sampler_sample: the context can refresh that buffer when + // the sampler fetches logits, dropping the mask. Build a candidate + // array holding only the allowed tokens and run the sampler chain + // over it instead. + guard let logits = llama_get_logits_ith(context, batch.pointee.n_tokens - 1) else { return eosToken } - for tokenIndex in 0 ..< vocabSize { - if !allowedTokens.contains(tokenIndex) { - logits[tokenIndex] = -Float.infinity - } + var candidates = allowedTokens.compactMap { token -> llama_token_data? in + guard token >= 0, token < vocabSize else { return nil } + return llama_token_data(id: llama_token(token), logit: logits[token], p: 0) + } + guard !candidates.isEmpty else { + return eosToken } - let tokenIndex = batch.pointee.n_tokens - 1 - return Int(llama_sampler_sample(sampler, context, tokenIndex)) + let selectedToken = candidates.withUnsafeMutableBufferPointer { buffer -> Int in + var array = llama_token_data_array( + data: buffer.baseAddress, + size: buffer.count, + selected: -1, + sorted: false + ) + llama_sampler_apply(sampler, &array) + guard array.selected >= 0, array.selected < Int64(array.size), let data = array.data else { + return eosToken + } + return Int(data[Int(array.selected)].id) + } + return selectedToken } }