feat: granite 4.2

docs/docs/models/granite.md

@@ -0,0 +1,29 @@
+---
+layout: docs
+title: Granite
+nav_order: 14
+parent: Models
+---
+
+## 🧩 Model Card: [ibm-granite/granite-4.2-3b](https://huggingface.co/ibm-granite/granite-4.2-3b)
+
+- **Type:** Text-to-Text
+- **Think:** Yes
+- **Tool Calling Support:** Yes
+- **Base Model:** [ibm-granite/granite-4.2-3b](https://huggingface.co/ibm-granite/granite-4.2-3b)
+- **Quantization:** Q4_1
+- **Max Context Length:** 128k tokens
+- **Default Context Length:** 8k tokens ([change default](https://fastflowlm.com/docs/instructions/cli/#-change-default-context-length-max))
+- **[Set Context Length at Launch](https://fastflowlm.com/docs/instructions/cli/#-set-context-length-at-launch)**
+
+▶️ Run with FastFlowLM in PowerShell:
+
+```shell
+flm run granite:3b
+```
+
+Granite 4.2 is a reasoning model: the chat template opens a `<think>` block in
+the generation prompt, so the model emits a reasoning trace, closes it with
+`</think>`, and then answers.
+
+---

docs/docs/models/index.md

@@ -27,4 +27,5 @@ has_children: true
 - [Nanbeige](nanbeige/)
 - [Whisper](whisper/)
 - [EmbeddingGemma](embeddinggemma/)
-- [SmolVLA](smolvla/)
+- [SmolVLA](smolvla/)
+- [Granite](granite/)

src/common/AutoModel/modeling_granite.cpp

@@ -0,0 +1,86 @@
+/// \file modeling_granite.cpp
+/// \brief IBM Granite (dense) family. See modeling_granite.hpp.
+
+#include "AutoModel/modeling_granite.hpp"
+
+/************              Granite family            **************/
+Granite::Granite(flm_rt::device* npu_device_inst) : AutoModel(npu_device_inst, "Granite") {}
+
+void Granite::load_model(std::string model_path, json model_info, int default_context_length,
+                         bool enable_preemption) {
+    this->_shared_load_model(model_path, model_info, default_context_length, enable_preemption);
+
+    this->q4nx = std::make_unique<Q4NX>(this->model_path);
+    this->lm_engine = std::make_unique<granite_npu>(*this->lm_config, this->npu.get(), this->MAX_L);
+
+    this->lm_engine->load_weights(*this->q4nx);
+    this->q4nx.reset();
+
+    this->lm_engine->clear_context();
+    this->setup_tokenizer(model_path);
+    this->sampler.reset();
+
+    // granite-4.2 is a reasoning model; its own generation_config ships
+    // temperature 1.0 / top_p 0.95, and a repetition guard keeps long chains of
+    // thought from looping.
+    sampler_config config;
+    config.top_k = 40;
+    config.top_p = 0.95;
+    config.min_p = 0.0;
+    config.temperature = 1.0;
+    config.rep_penalty = 1.05;
+
+    this->set_sampler(config);
+    for (size_t i = 0; i < PROFILER_TYPE_NUM; i++) {
+        this->profiler_list[i].reset();
+    }
+}
+
+void Granite::setup_tokenizer(std::string model_path) {
+    auto tokenizer_config = this->_shared_setup_tokenizer(model_path);
+}
+
+std::string Granite::apply_chat_template(nlohmann::ordered_json& messages,
+                                         nlohmann::ordered_json tools) {
+    minja::chat_template_inputs inputs;
+    inputs.add_generation_prompt = true;
+    inputs.messages = messages;
+    inputs.extra_context = this->extra_context;
+    return this->chat_tmpl->apply(inputs);
+}
+
+bool Granite::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input,
+                     std::function<bool()> is_cancelled) {
+    this->profiler_list[TKOEN_ENCODE_TIME].start();
+    std::string templated_text;
+    if (input.messages.empty() && input.prompt.empty()) {
+        header_print("WARNING", "No messages or prompt provided");
+        return false;
+    }
+    if (!input.messages.empty()) {
+        templated_text = this->apply_chat_template(input.messages);
+    }
+    else if (!input.prompt.empty()) {
+        nlohmann::ordered_json messages;
+        messages.push_back({ {"role", "user"}, {"content", input.prompt} });
+        templated_text = this->apply_chat_template(messages);
+    }
+
+    std::vector<int> tokens = this->tokenizer->encode(templated_text);
+    this->profiler_list[TKOEN_ENCODE_TIME].stop(tokens.size());
+
+    return this->_shared_insert(meta_info, tokens, is_cancelled);
+}
+
+std::string Granite::generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os,
+                              std::function<bool()> is_cancelled) {
+    return this->_shared_generate(meta_info, length_limit, os, is_cancelled);
+}
+
+std::string Granite::generate_with_prompt(chat_meta_info_t& meta_info, lm_uniform_input_t& input,
+                                          int length_limit, std::ostream& os) {
+    if (!this->insert(meta_info, input)) {
+        return "";
+    }
+    return this->_shared_generate(meta_info, length_limit, os);
+}