From ad1878dfda9f19adc8adb140f4fbf74f2352cd9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CChi?= Date: Mon, 31 Aug 2026 20:34:01 -0700 Subject: [PATCH 001/117] feat: add optional corelib ABI loader Co-authored-by: Cursor --- .gitignore | 1 + src/CMakeLists.txt | 29 ++ src/common/corelib/corelib_api.cpp | 317 ++++++++++++++ src/common/corelib/corelib_sources.cmake | 2 + src/include/corelib/corelib_api.hpp | 101 +++++ src/include/corelib/corelib_object.hpp | 77 ++++ src/test/phi4_corelib_aie4/CMakeLists.txt | 61 +++ src/test/phi4_corelib_aie4/fake_corelib.cpp | 339 +++++++++++++++ src/test/phi4_corelib_aie4/fake_corelib.hpp | 19 + .../phi4_corelib_aie4/test_corelib_api.cpp | 405 ++++++++++++++++++ src/test/phi4_corelib_aie4/test_support.hpp | 33 ++ 11 files changed, 1384 insertions(+) create mode 100644 src/common/corelib/corelib_api.cpp create mode 100644 src/common/corelib/corelib_sources.cmake create mode 100644 src/include/corelib/corelib_api.hpp create mode 100644 src/include/corelib/corelib_object.hpp create mode 100644 src/test/phi4_corelib_aie4/CMakeLists.txt create mode 100644 src/test/phi4_corelib_aie4/fake_corelib.cpp create mode 100644 src/test/phi4_corelib_aie4/fake_corelib.hpp create mode 100644 src/test/phi4_corelib_aie4/test_corelib_api.cpp create mode 100644 src/test/phi4_corelib_aie4/test_support.hpp diff --git a/.gitignore b/.gitignore index 8bf22b3e..64998c4d 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ __pycache__ _site .jekyll-metadata *.csv +/docs/superpowers/ diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index dd4d33fe..5b79b840 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -48,6 +48,30 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_RELEASE ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) # ——————————————————————————————————————————————— option(FLM_USE_HRX "Use the HRX amdxdna NPU runtime instead of XRT (0=XRT default, 1=HRX)" OFF) option(FLM_PORTABLE_BUILD "Build portable distribution with bundled runtime libraries" OFF) +option(FLM_ENABLE_CORELIB_AIE4 + "Enable optional Phi-4 AIE4 execution through ryzenai-corelib" OFF) + +if(FLM_ENABLE_CORELIB_AIE4 AND (NOT WIN32 OR FLM_USE_HRX)) + message(FATAL_ERROR + "FLM_ENABLE_CORELIB_AIE4 requires Windows and the XRT backend") +endif() + +if(FLM_ENABLE_CORELIB_AIE4) + find_path(RYZENAI_CORELIB_INCLUDE_DIR + NAMES ryzenai/corelib.h + REQUIRED) + include("${CMAKE_SOURCE_DIR}/common/corelib/corelib_sources.cmake") + add_library(flm_corelib_aie4 STATIC + ${FLM_CORELIB_AIE4_SOURCES}) + target_include_directories(flm_corelib_aie4 PUBLIC + ${CMAKE_SOURCE_DIR}/include + ${RYZENAI_CORELIB_INCLUDE_DIR}) + target_compile_definitions(flm_corelib_aie4 PUBLIC + FLM_ENABLE_CORELIB_AIE4=1 + WIN32_LEAN_AND_MEAN + NOMINMAX) + target_compile_options(flm_corelib_aie4 PRIVATE /fp:precise) +endif() if(FLM_USE_HRX) set(FLM_RUNTIME_NAME "hrx") @@ -236,6 +260,7 @@ add_subdirectory(${CMAKE_SOURCE_DIR}/../third_party/tokenizers-cpp # Gather your sources # ——————————————————————————————————————————————— file(GLOB SOURCES "src/*.cpp" "runner/*.cpp" "common/*.cpp" "common/*/*.cpp" "server/*.cpp" "pull/*.cpp" ) +list(FILTER SOURCES EXCLUDE REGEX ".*/common/corelib/.*\\.cpp$") file(GLOB HEADERS "include/*.hpp" "runner/*.hpp" "common/*.hpp" "common/*/*.hpp" "server/*.hpp" "pull/*.hpp") # Exclude files that depend on missing libraries for Linux @@ -269,6 +294,10 @@ endif() add_executable(flm ${SOURCES} ${HEADERS}) +if(FLM_ENABLE_CORELIB_AIE4) + target_link_libraries(flm PRIVATE flm_corelib_aie4) +endif() + if(WIN32) if(VCPKG_TOOLCHAIN) # A vcpkg toolchain is active (e.g. the rocm-npu-staging dev.py build or diff --git a/src/common/corelib/corelib_api.cpp b/src/common/corelib/corelib_api.cpp new file mode 100644 index 00000000..abfcff26 --- /dev/null +++ b/src/common/corelib/corelib_api.cpp @@ -0,0 +1,317 @@ +#include + +#include + +#include +#include +#include +#include +#include +#include + +namespace flm::corelib { +namespace { + +constexpr wchar_t kCorelibPathEnvironment[] = L"RYZENAI_CORELIB_PATH"; +constexpr wchar_t kCorelibFilename[] = L"ryzenai_corelib.dll"; + +std::string FormatCorelibError( + std::string_view call, + std::string_view status_text, + std::string_view detail) { + std::string message(call); + message += " failed: "; + message += status_text; + if (!detail.empty()) { + message += ": "; + message += detail; + } + return message; +} + +std::optional ReadWideEnvironment(const wchar_t* name) { + SetLastError(ERROR_SUCCESS); + const DWORD required = GetEnvironmentVariableW(name, nullptr, 0); + if (required == 0) { + const DWORD error = GetLastError(); + if (error == ERROR_SUCCESS || error == ERROR_ENVVAR_NOT_FOUND) { + return std::nullopt; + } + throw std::system_error( + static_cast(error), + std::system_category(), + "GetEnvironmentVariableW failed"); + } + + std::wstring value(required, L'\0'); + const DWORD written = + GetEnvironmentVariableW(name, value.data(), required); + if (written == 0 || written >= required) { + throw std::system_error( + static_cast(GetLastError()), + std::system_category(), + "GetEnvironmentVariableW failed"); + } + value.resize(written); + return value; +} + +std::filesystem::path MakeAbsolute(std::filesystem::path path) { + if (!path.is_absolute()) { + path = std::filesystem::absolute(path); + } + return path.lexically_normal(); +} + +template +Function ResolveRequired( + const CorelibApi::Resolver& resolver, + std::string_view name) { + void* address = resolver(name); + if (address == nullptr) { + throw std::runtime_error( + "missing required ryzenai-corelib symbol: " + + std::string(name)); + } + return reinterpret_cast(address); +} + +CorelibFunctions ResolveFunctions(const CorelibApi::Resolver& resolver) { + return CorelibFunctions{ + ResolveRequired< + decltype(&::ryzenai_corelib_status_to_string)>( + resolver, + "ryzenai_corelib_status_to_string"), + ResolveRequired< + decltype(&::ryzenai_corelib_get_last_error_message)>( + resolver, + "ryzenai_corelib_get_last_error_message"), + ResolveRequired< + decltype(&::ryzenai_corelib_selftest_dependencies)>( + resolver, + "ryzenai_corelib_selftest_dependencies"), + ResolveRequired< + decltype(&::ryzenai_corelib_has_device_context)>( + resolver, + "ryzenai_corelib_has_device_context"), + ResolveRequired< + decltype(&::ryzenai_corelib_object_release)>( + resolver, + "ryzenai_corelib_object_release"), + ResolveRequired< + decltype(&::ryzenai_corelib_create_stream)>( + resolver, + "ryzenai_corelib_create_stream"), + ResolveRequired< + decltype(&::ryzenai_corelib_stream_synchronize)>( + resolver, + "ryzenai_corelib_stream_synchronize"), + ResolveRequired< + decltype(&::ryzenai_corelib_create_device_tensor)>( + resolver, + "ryzenai_corelib_create_device_tensor"), + ResolveRequired< + decltype(&::ryzenai_corelib_tensor_write)>( + resolver, + "ryzenai_corelib_tensor_write"), + ResolveRequired< + decltype(&::ryzenai_corelib_tensor_read)>( + resolver, + "ryzenai_corelib_tensor_read"), + ResolveRequired< + decltype(&::ryzenai_corelib_tensor_get_byte_size)>( + resolver, + "ryzenai_corelib_tensor_get_byte_size"), + ResolveRequired< + decltype(&::ryzenai_corelib_convert)>( + resolver, + "ryzenai_corelib_convert"), + ResolveRequired< + decltype(&::ryzenai_corelib_convert_strided)>( + resolver, + "ryzenai_corelib_convert_strided"), + ResolveRequired< + decltype(&::ryzenai_corelib_matmul_bf16_pad_shape)>( + resolver, + "ryzenai_corelib_matmul_bf16_pad_shape"), + ResolveRequired( + resolver, + "ryzenai_corelib_matmul_bf16_weights_create_from_onnx_components"), + ResolveRequired( + resolver, + "ryzenai_corelib_matmul_bf16_weights_get_data"), + ResolveRequired< + decltype(&::ryzenai_corelib_matmul_bf16)>( + resolver, + "ryzenai_corelib_matmul_bf16"), + ResolveRequired< + decltype(&::ryzenai_corelib_ssmlp_bf16_pad_rows)>( + resolver, + "ryzenai_corelib_ssmlp_bf16_pad_rows"), + ResolveRequired( + resolver, + "ryzenai_corelib_ssmlp_bf16_weights_create_from_onnx_components"), + ResolveRequired( + resolver, + "ryzenai_corelib_ssmlp_bf16_weights_get_data"), + ResolveRequired< + decltype(&::ryzenai_corelib_ssmlp_bf16)>( + resolver, + "ryzenai_corelib_ssmlp_bf16"), + ResolveRequired< + decltype(&::ryzenai_corelib_flat_mha_bf16_pad_rows)>( + resolver, + "ryzenai_corelib_flat_mha_bf16_pad_rows"), + ResolveRequired< + decltype(&::ryzenai_corelib_flat_mha_bf16)>( + resolver, + "ryzenai_corelib_flat_mha_bf16"), + ResolveRequired< + decltype(&::ryzenai_corelib_cleanup)>( + resolver, + "ryzenai_corelib_cleanup"), + }; +} + +std::string LoadFailureMessage( + const std::filesystem::path& path, + DWORD error) { + return "LoadLibraryExW failed for " + path.string() + + " (Win32 error " + std::to_string(error) + ")"; +} + +} // namespace + +CorelibError::CorelibError( + ryzenai_corelib_status status_value, + std::string call_value, + std::string detail_value, + std::string status_text) + : std::runtime_error(FormatCorelibError( + call_value, + status_text, + detail_value)), + status(status_value), + call(std::move(call_value)), + detail(std::move(detail_value)) {} + +std::shared_ptr CorelibApi::Load( + const std::filesystem::path& absolute_path) { + if (!absolute_path.is_absolute()) { + throw std::invalid_argument( + "CorelibApi::Load requires an absolute DLL path"); + } + + HMODULE module = LoadLibraryExW( + absolute_path.c_str(), + nullptr, + LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR | LOAD_LIBRARY_SEARCH_DEFAULT_DIRS); + if (module == nullptr) { + const DWORD error = GetLastError(); + throw std::runtime_error( + LoadFailureMessage(absolute_path, error)); + } + + try { + Resolver resolver = [module](std::string_view name) -> void* { + const std::string symbol(name); + return reinterpret_cast( + GetProcAddress(module, symbol.c_str())); + }; + auto functions = ResolveFunctions(resolver); + return std::shared_ptr( + new CorelibApi( + module, + absolute_path.lexically_normal(), + std::move(functions))); + } catch (...) { + FreeLibrary(module); + throw; + } +} + +std::shared_ptr CorelibApi::ResolveForTest( + Resolver resolver) { + auto functions = ResolveFunctions(resolver); + return std::shared_ptr( + new CorelibApi(nullptr, {}, std::move(functions))); +} + +std::filesystem::path CorelibApi::ResolveLibraryPath( + const std::filesystem::path& executable_dir) { + if (const auto configured = + ReadWideEnvironment(kCorelibPathEnvironment)) { + auto path = MakeAbsolute(std::filesystem::path(*configured)); + std::error_code error; + if (std::filesystem::is_directory(path, error)) { + path /= kCorelibFilename; + } + return path.lexically_normal(); + } + + return MakeAbsolute( + executable_dir / "aie4" / kCorelibFilename); +} + +CorelibApi::CorelibApi( + void* module, + std::filesystem::path library_path, + CorelibFunctions functions) + : module_(module), + library_path_(std::move(library_path)), + functions_(std::move(functions)) {} + +CorelibApi::~CorelibApi() { + if (module_ != nullptr) { + FreeLibrary(static_cast(module_)); + } +} + +const CorelibFunctions& CorelibApi::functions() const noexcept { + return functions_; +} + +void CorelibApi::Check( + ryzenai_corelib_status status, + std::string_view call) const { + if (status == ryzenai_corelib_status_success) { + return; + } + const char* raw = functions_.get_last_error_message(); + std::string detail = raw == nullptr ? std::string() : std::string(raw); + const char* status_text = functions_.status_to_string(status); + throw CorelibError{ + status, + std::string(call), + std::move(detail), + status_text == nullptr ? "corelib failure" : status_text}; +} + +void CorelibApi::RegisterObject() const noexcept { + live_object_count_.fetch_add(1, std::memory_order_relaxed); +} + +void CorelibApi::Release(void* value) const noexcept { + if (value == nullptr) { + return; + } + + functions_.object_release(value); + const auto previous = + live_object_count_.fetch_sub(1, std::memory_order_acq_rel); + assert(previous > 0); +} + +std::size_t CorelibApi::live_object_count() const noexcept { + return live_object_count_.load(std::memory_order_acquire); +} + +const std::filesystem::path& CorelibApi::library_path() const noexcept { + return library_path_; +} + +} // namespace flm::corelib diff --git a/src/common/corelib/corelib_sources.cmake b/src/common/corelib/corelib_sources.cmake new file mode 100644 index 00000000..6c396406 --- /dev/null +++ b/src/common/corelib/corelib_sources.cmake @@ -0,0 +1,2 @@ +set(FLM_CORELIB_AIE4_SOURCES + "${CMAKE_CURRENT_LIST_DIR}/corelib_api.cpp") diff --git a/src/include/corelib/corelib_api.hpp b/src/include/corelib/corelib_api.hpp new file mode 100644 index 00000000..9c04ca8b --- /dev/null +++ b/src/include/corelib/corelib_api.hpp @@ -0,0 +1,101 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace flm::corelib { + +struct CorelibError final : std::runtime_error { + CorelibError( + ryzenai_corelib_status status, + std::string call, + std::string detail, + std::string status_text); + + ryzenai_corelib_status status; + std::string call; + std::string detail; +}; + +struct CorelibFunctions { + decltype(&::ryzenai_corelib_status_to_string) status_to_string; + decltype(&::ryzenai_corelib_get_last_error_message) + get_last_error_message; + decltype(&::ryzenai_corelib_selftest_dependencies) + selftest_dependencies; + decltype(&::ryzenai_corelib_has_device_context) has_device_context; + decltype(&::ryzenai_corelib_object_release) object_release; + decltype(&::ryzenai_corelib_create_stream) create_stream; + decltype(&::ryzenai_corelib_stream_synchronize) stream_synchronize; + decltype(&::ryzenai_corelib_create_device_tensor) create_device_tensor; + decltype(&::ryzenai_corelib_tensor_write) tensor_write; + decltype(&::ryzenai_corelib_tensor_read) tensor_read; + decltype(&::ryzenai_corelib_tensor_get_byte_size) tensor_get_byte_size; + decltype(&::ryzenai_corelib_convert) convert; + decltype(&::ryzenai_corelib_convert_strided) convert_strided; + decltype(&::ryzenai_corelib_matmul_bf16_pad_shape) matmul_pad_shape; + decltype( + &::ryzenai_corelib_matmul_bf16_weights_create_from_onnx_components) + matmul_weights_from_onnx; + decltype(&::ryzenai_corelib_matmul_bf16_weights_get_data) + matmul_weights_get_data; + decltype(&::ryzenai_corelib_matmul_bf16) matmul; + decltype(&::ryzenai_corelib_ssmlp_bf16_pad_rows) ssmlp_pad_rows; + decltype( + &::ryzenai_corelib_ssmlp_bf16_weights_create_from_onnx_components) + ssmlp_weights_from_onnx; + decltype(&::ryzenai_corelib_ssmlp_bf16_weights_get_data) + ssmlp_weights_get_data; + decltype(&::ryzenai_corelib_ssmlp_bf16) ssmlp; + decltype(&::ryzenai_corelib_flat_mha_bf16_pad_rows) flat_mha_pad_rows; + decltype(&::ryzenai_corelib_flat_mha_bf16) flat_mha; + decltype(&::ryzenai_corelib_cleanup) cleanup; +}; + +class CorelibApi final { +public: + using Resolver = std::function; + + static std::shared_ptr Load( + const std::filesystem::path& absolute_path); + static std::shared_ptr ResolveForTest(Resolver resolver); + static std::filesystem::path ResolveLibraryPath( + const std::filesystem::path& executable_dir); + + ~CorelibApi(); + + CorelibApi(const CorelibApi&) = delete; + CorelibApi& operator=(const CorelibApi&) = delete; + CorelibApi(CorelibApi&&) = delete; + CorelibApi& operator=(CorelibApi&&) = delete; + + const CorelibFunctions& functions() const noexcept; + void Check( + ryzenai_corelib_status status, + std::string_view call) const; + void RegisterObject() const noexcept; + void Release(void* value) const noexcept; + std::size_t live_object_count() const noexcept; + const std::filesystem::path& library_path() const noexcept; + +private: + CorelibApi( + void* module, + std::filesystem::path library_path, + CorelibFunctions functions); + + void* module_ = nullptr; + std::filesystem::path library_path_; + CorelibFunctions functions_; + mutable std::atomic live_object_count_{0}; +}; + +} // namespace flm::corelib diff --git a/src/include/corelib/corelib_object.hpp b/src/include/corelib/corelib_object.hpp new file mode 100644 index 00000000..0af8fc04 --- /dev/null +++ b/src/include/corelib/corelib_object.hpp @@ -0,0 +1,77 @@ +#pragma once + +#include + +#include +#include +#include + +namespace flm::corelib { + +template +class UniqueObject { +public: + UniqueObject() noexcept = default; + + UniqueObject( + std::shared_ptr api, + void* value) noexcept + : api_(std::move(api)), + value_(value) { + if (value_ != nullptr) { + assert(api_ != nullptr); + api_->RegisterObject(); + } + } + + UniqueObject(UniqueObject&& other) noexcept + : api_(std::move(other.api_)), + value_(std::exchange(other.value_, nullptr)) {} + + UniqueObject& operator=(UniqueObject&& other) noexcept { + if (this != &other) { + reset(); + api_ = std::move(other.api_); + value_ = std::exchange(other.value_, nullptr); + } + return *this; + } + + ~UniqueObject() noexcept { + reset(); + } + + UniqueObject(const UniqueObject&) = delete; + UniqueObject& operator=(const UniqueObject&) = delete; + + void* get() const noexcept { + return value_; + } + + explicit operator bool() const noexcept { + return value_ != nullptr; + } + + void reset() noexcept { + if (value_ != nullptr) { + api_->Release(std::exchange(value_, nullptr)); + } + api_.reset(); + } + +private: + std::shared_ptr api_; + void* value_ = nullptr; +}; + +struct StreamTag; +struct TensorTag; +struct MatMulWeightsTag; +struct SsMlpWeightsTag; + +using UniqueStream = UniqueObject; +using UniqueTensor = UniqueObject; +using UniqueMatMulWeights = UniqueObject; +using UniqueSsMlpWeights = UniqueObject; + +} // namespace flm::corelib diff --git a/src/test/phi4_corelib_aie4/CMakeLists.txt b/src/test/phi4_corelib_aie4/CMakeLists.txt new file mode 100644 index 00000000..6c2222ce --- /dev/null +++ b/src/test/phi4_corelib_aie4/CMakeLists.txt @@ -0,0 +1,61 @@ +cmake_minimum_required(VERSION 3.22) +project(phi4_corelib_aie4_tests LANGUAGES CXX) + +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +find_path(RYZENAI_CORELIB_INCLUDE_DIR + NAMES ryzenai/corelib.h + REQUIRED) + +get_filename_component( + FASTFLOW_SOURCE_DIR + "${CMAKE_CURRENT_LIST_DIR}/../.." + ABSOLUTE) +set(XRT_INCLUDE_DIR + "C:/dev/XRT/src/runtime_src/core/include" + CACHE PATH "Where XRT headers live") +set(XRT_LIB_DIR + "C:/dev/xrtNPUfromDLL" + CACHE PATH "Where XRT libraries live") + +include("${FASTFLOW_SOURCE_DIR}/common/corelib/corelib_sources.cmake") +add_library(flm_corelib_aie4_testlib STATIC + ${FLM_CORELIB_AIE4_SOURCES}) +target_include_directories(flm_corelib_aie4_testlib PUBLIC + ${FASTFLOW_SOURCE_DIR}/include + ${RYZENAI_CORELIB_INCLUDE_DIR} + ${XRT_INCLUDE_DIR}) +target_link_directories(flm_corelib_aie4_testlib PUBLIC + ${XRT_LIB_DIR}) +target_compile_definitions(flm_corelib_aie4_testlib PUBLIC + FLM_ENABLE_CORELIB_AIE4=1 + DEV_BUILD=1 + __WINDOWS__ + USEAVX2=1 + DISABLE_ABI_CHECK=1 + _ENABLE_EXTENDED_ALIGNED_STORAGE + CMAKE_INSTALL_PREFIX=\"${FASTFLOW_SOURCE_DIR}/build/phi4_corelib_aie4-tests\" + CMAKE_XCLBIN_PREFIX=\"${FASTFLOW_SOURCE_DIR}/xclbins\" + __FLM_VERSION__=\"${FLM_VERSION}\" + __NPU_VERSION__=\"${NPU_VERSION}\" + WIN32_LEAN_AND_MEAN + NOMINMAX) +target_compile_options(flm_corelib_aie4_testlib PRIVATE /fp:precise) +target_link_libraries(flm_corelib_aie4_testlib PUBLIC + xrt_coreutil) + +function(add_corelib_host_test TEST_NAME TEST_SOURCE) + add_executable(${TEST_NAME} + ${TEST_SOURCE} + fake_corelib.cpp) + target_link_libraries(${TEST_NAME} PRIVATE + flm_corelib_aie4_testlib) + add_test(NAME ${TEST_NAME} COMMAND ${TEST_NAME}) + set_tests_properties(${TEST_NAME} PROPERTIES + ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:${XRT_LIB_DIR}") +endfunction() + +enable_testing() +add_corelib_host_test(test_corelib_api test_corelib_api.cpp) diff --git a/src/test/phi4_corelib_aie4/fake_corelib.cpp b/src/test/phi4_corelib_aie4/fake_corelib.cpp new file mode 100644 index 00000000..5143e7c9 --- /dev/null +++ b/src/test/phi4_corelib_aie4/fake_corelib.cpp @@ -0,0 +1,339 @@ +#include "fake_corelib.hpp" + +#include +#include +#include +#include + +namespace flm::test { +namespace { + +thread_local std::string g_last_error; +std::unordered_map g_release_counts; +std::size_t g_release_count = 0; +void* g_last_released_object = nullptr; + +const char* FakeStatusToString(ryzenai_corelib_status status) { + g_last_error = "overwritten by status_to_string"; + switch (status) { + case ryzenai_corelib_status_success: + return "success"; + case ryzenai_corelib_status_failure: + return "failure"; + case ryzenai_corelib_status_bad_argument: + return "bad argument"; + case ryzenai_corelib_status_unsupported: + return "unsupported"; + } + return "unknown"; +} + +const char* FakeGetLastErrorMessage() { + return g_last_error.c_str(); +} + +ryzenai_corelib_status FakeSelftestDependencies() { + return ryzenai_corelib_status_success; +} + +bool FakeHasDeviceContext() { + return true; +} + +void FakeObjectRelease(ryzenai_corelib_object_ptr object) { + ++g_release_count; + ++g_release_counts[object]; + g_last_released_object = object; +} + +ryzenai_corelib_status FakeCreateStream(ryzenai_corelib_stream_ptr* out) { + if (out != nullptr) { + *out = nullptr; + } + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status FakeStreamSynchronize( + ryzenai_corelib_stream_ptr) { + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status FakeCreateDeviceTensor( + ryzenai_corelib_data_type, + const int64_t*, + std::size_t, + ryzenai_corelib_tensor_ptr* out) { + if (out != nullptr) { + *out = nullptr; + } + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status FakeTensorWrite( + ryzenai_corelib_tensor_ptr, + const void*, + std::size_t, + std::size_t) { + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status FakeTensorRead( + ryzenai_corelib_tensor_ptr, + void*, + std::size_t, + std::size_t) { + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status FakeTensorGetByteSize( + ryzenai_corelib_tensor_ptr, + std::size_t* out) { + if (out != nullptr) { + *out = 0; + } + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status FakeConvert( + ryzenai_corelib_data_type, + const void*, + ryzenai_corelib_data_type, + void*, + std::size_t) { + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status FakeConvertStrided( + ryzenai_corelib_data_type, + const void*, + std::size_t, + ryzenai_corelib_data_type, + void*, + std::size_t, + std::size_t, + std::size_t) { + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status FakeMatmulPadShape( + int64_t*, + int64_t*, + int64_t*, + uint32_t) { + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status FakeMatmulWeightsFromOnnx( + const ryzenai_corelib_matmul_bf16_weights_desc*, + const ryzenai_corelib_matmul_bf16_onnx_weights_components*, + ryzenai_corelib_matmul_bf16_weights_ptr* out) { + if (out != nullptr) { + *out = nullptr; + } + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status FakeMatmulWeightsGetData( + ryzenai_corelib_matmul_bf16_weights_ptr, + const void** data, + std::size_t* size) { + if (data != nullptr) { + *data = nullptr; + } + if (size != nullptr) { + *size = 0; + } + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status FakeMatmul( + ryzenai_corelib_stream_ptr, + ryzenai_corelib_tensor_ptr, + int64_t, + ryzenai_corelib_matmul_bf16_weights_ptr, + ryzenai_corelib_tensor_ptr) { + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status FakeSsmlpPadRows( + int64_t*, + int64_t, + int64_t, + uint32_t) { + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status FakeSsmlpWeightsFromOnnx( + const ryzenai_corelib_ssmlp_bf16_weights_desc*, + const ryzenai_corelib_ssmlp_bf16_onnx_weights_components*, + ryzenai_corelib_ssmlp_bf16_weights_ptr* out) { + if (out != nullptr) { + *out = nullptr; + } + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status FakeSsmlpWeightsGetData( + ryzenai_corelib_ssmlp_bf16_weights_ptr, + const void** data, + std::size_t* size) { + if (data != nullptr) { + *data = nullptr; + } + if (size != nullptr) { + *size = 0; + } + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status FakeSsmlp( + ryzenai_corelib_stream_ptr, + ryzenai_corelib_tensor_ptr, + ryzenai_corelib_tensor_ptr, + int64_t, + ryzenai_corelib_ssmlp_bf16_weights_ptr, + ryzenai_corelib_tensor_ptr, + ryzenai_corelib_tensor_ptr) { + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status FakeFlatMhaPadRows( + int64_t*, + const ryzenai_corelib_flat_mha_bf16_desc*) { + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status FakeFlatMha( + ryzenai_corelib_stream_ptr, + const ryzenai_corelib_flat_mha_bf16_desc*, + ryzenai_corelib_tensor_ptr, + ryzenai_corelib_tensor_ptr, + int64_t, + int64_t, + ryzenai_corelib_tensor_ptr, + ryzenai_corelib_tensor_ptr, + ryzenai_corelib_tensor_ptr, + ryzenai_corelib_tensor_ptr, + ryzenai_corelib_tensor_ptr) { + return ryzenai_corelib_status_success; +} + +void FakeCleanup() {} + +template +void* FunctionAddress(Function function) { + return reinterpret_cast(function); +} + +#define FLM_FAKE_ENTRY(symbol, replacement) \ + { \ + #symbol, FunctionAddress( \ + static_cast(&replacement)) \ + } + +} // namespace + +std::unordered_map CompleteCorelibResolver() { + return { + FLM_FAKE_ENTRY( + ryzenai_corelib_status_to_string, + FakeStatusToString), + FLM_FAKE_ENTRY( + ryzenai_corelib_get_last_error_message, + FakeGetLastErrorMessage), + FLM_FAKE_ENTRY( + ryzenai_corelib_selftest_dependencies, + FakeSelftestDependencies), + FLM_FAKE_ENTRY( + ryzenai_corelib_has_device_context, + FakeHasDeviceContext), + FLM_FAKE_ENTRY( + ryzenai_corelib_object_release, + FakeObjectRelease), + FLM_FAKE_ENTRY( + ryzenai_corelib_create_stream, + FakeCreateStream), + FLM_FAKE_ENTRY( + ryzenai_corelib_stream_synchronize, + FakeStreamSynchronize), + FLM_FAKE_ENTRY( + ryzenai_corelib_create_device_tensor, + FakeCreateDeviceTensor), + FLM_FAKE_ENTRY( + ryzenai_corelib_tensor_write, + FakeTensorWrite), + FLM_FAKE_ENTRY( + ryzenai_corelib_tensor_read, + FakeTensorRead), + FLM_FAKE_ENTRY( + ryzenai_corelib_tensor_get_byte_size, + FakeTensorGetByteSize), + FLM_FAKE_ENTRY( + ryzenai_corelib_convert, + FakeConvert), + FLM_FAKE_ENTRY( + ryzenai_corelib_convert_strided, + FakeConvertStrided), + FLM_FAKE_ENTRY( + ryzenai_corelib_matmul_bf16_pad_shape, + FakeMatmulPadShape), + FLM_FAKE_ENTRY( + ryzenai_corelib_matmul_bf16_weights_create_from_onnx_components, + FakeMatmulWeightsFromOnnx), + FLM_FAKE_ENTRY( + ryzenai_corelib_matmul_bf16_weights_get_data, + FakeMatmulWeightsGetData), + FLM_FAKE_ENTRY( + ryzenai_corelib_matmul_bf16, + FakeMatmul), + FLM_FAKE_ENTRY( + ryzenai_corelib_ssmlp_bf16_pad_rows, + FakeSsmlpPadRows), + FLM_FAKE_ENTRY( + ryzenai_corelib_ssmlp_bf16_weights_create_from_onnx_components, + FakeSsmlpWeightsFromOnnx), + FLM_FAKE_ENTRY( + ryzenai_corelib_ssmlp_bf16_weights_get_data, + FakeSsmlpWeightsGetData), + FLM_FAKE_ENTRY( + ryzenai_corelib_ssmlp_bf16, + FakeSsmlp), + FLM_FAKE_ENTRY( + ryzenai_corelib_flat_mha_bf16_pad_rows, + FakeFlatMhaPadRows), + FLM_FAKE_ENTRY( + ryzenai_corelib_flat_mha_bf16, + FakeFlatMha), + FLM_FAKE_ENTRY( + ryzenai_corelib_cleanup, + FakeCleanup), + }; +} + +#undef FLM_FAKE_ENTRY + +void ResetFakeCorelib() { + g_last_error.clear(); + g_release_counts.clear(); + g_release_count = 0; + g_last_released_object = nullptr; +} + +void SetLastErrorMessage(std::string message) { + g_last_error = std::move(message); +} + +std::size_t ObjectReleaseCount() noexcept { + return g_release_count; +} + +std::size_t ObjectReleaseCountFor(void* value) noexcept { + const auto found = g_release_counts.find(value); + return found == g_release_counts.end() ? 0 : found->second; +} + +void* LastReleasedObject() noexcept { + return g_last_released_object; +} + +} // namespace flm::test diff --git a/src/test/phi4_corelib_aie4/fake_corelib.hpp b/src/test/phi4_corelib_aie4/fake_corelib.hpp new file mode 100644 index 00000000..14d2eef8 --- /dev/null +++ b/src/test/phi4_corelib_aie4/fake_corelib.hpp @@ -0,0 +1,19 @@ +#pragma once + +#include + +#include +#include +#include + +namespace flm::test { + +std::unordered_map CompleteCorelibResolver(); + +void ResetFakeCorelib(); +void SetLastErrorMessage(std::string message); +std::size_t ObjectReleaseCount() noexcept; +std::size_t ObjectReleaseCountFor(void* value) noexcept; +void* LastReleasedObject() noexcept; + +} // namespace flm::test diff --git a/src/test/phi4_corelib_aie4/test_corelib_api.cpp b/src/test/phi4_corelib_aie4/test_corelib_api.cpp new file mode 100644 index 00000000..7b99c516 --- /dev/null +++ b/src/test/phi4_corelib_aie4/test_corelib_api.cpp @@ -0,0 +1,405 @@ +#include "fake_corelib.hpp" +#include "test_support.hpp" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using flm::corelib::CorelibApi; + +constexpr std::array kRequiredSymbols{ + "ryzenai_corelib_status_to_string", + "ryzenai_corelib_get_last_error_message", + "ryzenai_corelib_selftest_dependencies", + "ryzenai_corelib_has_device_context", + "ryzenai_corelib_object_release", + "ryzenai_corelib_create_stream", + "ryzenai_corelib_stream_synchronize", + "ryzenai_corelib_create_device_tensor", + "ryzenai_corelib_tensor_write", + "ryzenai_corelib_tensor_read", + "ryzenai_corelib_tensor_get_byte_size", + "ryzenai_corelib_convert", + "ryzenai_corelib_convert_strided", + "ryzenai_corelib_matmul_bf16_pad_shape", + "ryzenai_corelib_matmul_bf16_weights_create_from_onnx_components", + "ryzenai_corelib_matmul_bf16_weights_get_data", + "ryzenai_corelib_matmul_bf16", + "ryzenai_corelib_ssmlp_bf16_pad_rows", + "ryzenai_corelib_ssmlp_bf16_weights_create_from_onnx_components", + "ryzenai_corelib_ssmlp_bf16_weights_get_data", + "ryzenai_corelib_ssmlp_bf16", + "ryzenai_corelib_flat_mha_bf16_pad_rows", + "ryzenai_corelib_flat_mha_bf16", + "ryzenai_corelib_cleanup", +}; + +std::shared_ptr ResolveCompleteCorelib() { + auto resolver = flm::test::CompleteCorelibResolver(); + return CorelibApi::ResolveForTest( + [resolver = std::move(resolver)](std::string_view name) mutable + -> void* { + const auto found = resolver.find(std::string(name)); + return found == resolver.end() ? nullptr : found->second; + }); +} + +std::optional ReadEnvironment(const wchar_t* name) { + std::size_t required = 0; + if (_wgetenv_s(&required, nullptr, 0, name) != 0) { + throw std::runtime_error("failed to read environment variable"); + } + if (required == 0) { + return std::nullopt; + } + + std::vector value(required); + if (_wgetenv_s(&required, value.data(), value.size(), name) != 0) { + throw std::runtime_error("failed to read environment variable"); + } + return std::wstring(value.data()); +} + +class ScopedEnvironment final { +public: + ScopedEnvironment( + std::wstring name, + std::optional value) + : name_(std::move(name)), + original_(ReadEnvironment(name_.c_str())) { + Set(value); + } + + ~ScopedEnvironment() noexcept { + _wputenv_s( + name_.c_str(), + original_.has_value() ? original_->c_str() : L""); + } + + ScopedEnvironment(const ScopedEnvironment&) = delete; + ScopedEnvironment& operator=(const ScopedEnvironment&) = delete; + +private: + void Set(const std::optional& value) { + if (_wputenv_s( + name_.c_str(), + value.has_value() ? value->c_str() : L"") != 0) { + throw std::runtime_error("failed to set environment variable"); + } + } + + std::wstring name_; + std::optional original_; +}; + +class ScopedCurrentPath final { +public: + explicit ScopedCurrentPath(const std::filesystem::path& value) + : original_(std::filesystem::current_path()) { + std::filesystem::current_path(value); + } + + ~ScopedCurrentPath() noexcept { + std::error_code error; + std::filesystem::current_path(original_, error); + } + + ScopedCurrentPath(const ScopedCurrentPath&) = delete; + ScopedCurrentPath& operator=(const ScopedCurrentPath&) = delete; + +private: + std::filesystem::path original_; +}; + +class TempDirectory final { +public: + TempDirectory() { + const auto nonce = + std::chrono::steady_clock::now().time_since_epoch().count(); + path_ = std::filesystem::temp_directory_path() / + ("fastflowlm-corelib-api-" + std::to_string(nonce)); + std::filesystem::create_directories(path_); + } + + ~TempDirectory() noexcept { + std::error_code error; + std::filesystem::remove_all(path_, error); + } + + TempDirectory(const TempDirectory&) = delete; + TempDirectory& operator=(const TempDirectory&) = delete; + + const std::filesystem::path& path() const noexcept { + return path_; + } + +private: + std::filesystem::path path_; +}; + +void Touch(const std::filesystem::path& path) { + std::ofstream file(path, std::ios::binary); + if (!file) { + throw std::runtime_error("failed to create test file"); + } +} + +void TestCompleteResolution() { + auto resolver = flm::test::CompleteCorelibResolver(); + CHECK(resolver.size() == kRequiredSymbols.size()); + for (const auto name : kRequiredSymbols) { + CHECK(resolver.contains(std::string(name))); + } + + std::vector requested; + auto api = CorelibApi::ResolveForTest( + [&resolver, &requested](std::string_view name) -> void* { + requested.emplace_back(name); + const auto found = resolver.find(std::string(name)); + return found == resolver.end() ? nullptr : found->second; + }); + + CHECK(api != nullptr); + CHECK(requested.size() == kRequiredSymbols.size()); + for (const auto name : kRequiredSymbols) { + CHECK(std::count(requested.begin(), requested.end(), name) == 1); + } + + const auto& functions = api->functions(); + CHECK(functions.status_to_string != nullptr); + CHECK(functions.get_last_error_message != nullptr); + CHECK(functions.selftest_dependencies != nullptr); + CHECK(functions.has_device_context != nullptr); + CHECK(functions.object_release != nullptr); + CHECK(functions.create_stream != nullptr); + CHECK(functions.stream_synchronize != nullptr); + CHECK(functions.create_device_tensor != nullptr); + CHECK(functions.tensor_write != nullptr); + CHECK(functions.tensor_read != nullptr); + CHECK(functions.tensor_get_byte_size != nullptr); + CHECK(functions.convert != nullptr); + CHECK(functions.convert_strided != nullptr); + CHECK(functions.matmul_pad_shape != nullptr); + CHECK(functions.matmul_weights_from_onnx != nullptr); + CHECK(functions.matmul_weights_get_data != nullptr); + CHECK(functions.matmul != nullptr); + CHECK(functions.ssmlp_pad_rows != nullptr); + CHECK(functions.ssmlp_weights_from_onnx != nullptr); + CHECK(functions.ssmlp_weights_get_data != nullptr); + CHECK(functions.ssmlp != nullptr); + CHECK(functions.flat_mha_pad_rows != nullptr); + CHECK(functions.flat_mha != nullptr); + CHECK(functions.cleanup != nullptr); +} + +void TestMissingSymbolFailsAtomically() { + auto resolver = flm::test::CompleteCorelibResolver(); + resolver.erase("ryzenai_corelib_flat_mha_bf16"); + CheckThrowsContains( + [&] { + flm::corelib::CorelibApi::ResolveForTest( + [resolver](std::string_view name) mutable -> void* { + const auto found = resolver.find(std::string(name)); + return found == resolver.end() ? nullptr : found->second; + }); + }, + "ryzenai_corelib_flat_mha_bf16"); +} + +void TestErrorDetailSurvivesStatusConversion() { + flm::test::ResetFakeCorelib(); + auto api = ResolveCompleteCorelib(); + const std::string expected_detail = + "durable detail copied before the next corelib call"; + flm::test::SetLastErrorMessage(expected_detail); + + try { + api->Check( + ryzenai_corelib_status_bad_argument, + "ryzenai_corelib_test_call"); + } catch (const flm::corelib::CorelibError& error) { + CHECK(error.status == ryzenai_corelib_status_bad_argument); + CHECK(error.call == "ryzenai_corelib_test_call"); + CHECK(error.detail == expected_detail); + CHECK(std::string_view(error.what()).find( + "ryzenai_corelib_test_call") != std::string_view::npos); + CHECK(std::string_view(error.what()).find("bad argument") != + std::string_view::npos); + CHECK(std::string_view(error.what()).find(expected_detail) != + std::string_view::npos); + return; + } + throw std::runtime_error("expected CorelibError was not thrown"); +} + +void TestSuccessfulStatusDoesNotReadErrorState() { + flm::test::ResetFakeCorelib(); + auto api = ResolveCompleteCorelib(); + flm::test::SetLastErrorMessage("must remain untouched"); + + api->Check(ryzenai_corelib_status_success, "successful_call"); + + CHECK(std::string_view( + api->functions().get_last_error_message()) == + "must remain untouched"); +} + +void TestMoveConstructionReleasesExactlyOnce() { + flm::test::ResetFakeCorelib(); + auto api = ResolveCompleteCorelib(); + int storage = 0; + void* handle = &storage; + + { + flm::corelib::UniqueStream original(api, handle); + CHECK(api->live_object_count() == 1); + + flm::corelib::UniqueStream moved(std::move(original)); + CHECK(!original); + CHECK(moved.get() == handle); + CHECK(api->live_object_count() == 1); + CHECK(flm::test::ObjectReleaseCount() == 0); + } + + CHECK(flm::test::ObjectReleaseCount() == 1); + CHECK(flm::test::ObjectReleaseCountFor(handle) == 1); + CHECK(flm::test::LastReleasedObject() == handle); + CHECK(api->live_object_count() == 0); +} + +void TestMoveAssignmentReleasesEachObjectOnce() { + flm::test::ResetFakeCorelib(); + auto api = ResolveCompleteCorelib(); + int source_storage = 0; + int target_storage = 0; + void* source_handle = &source_storage; + void* target_handle = &target_storage; + + { + flm::corelib::UniqueTensor source(api, source_handle); + flm::corelib::UniqueTensor target(api, target_handle); + CHECK(api->live_object_count() == 2); + + target = std::move(source); + CHECK(!source); + CHECK(target.get() == source_handle); + CHECK(flm::test::ObjectReleaseCountFor(target_handle) == 1); + CHECK(flm::test::ObjectReleaseCountFor(source_handle) == 0); + CHECK(api->live_object_count() == 1); + } + + CHECK(flm::test::ObjectReleaseCount() == 2); + CHECK(flm::test::ObjectReleaseCountFor(target_handle) == 1); + CHECK(flm::test::ObjectReleaseCountFor(source_handle) == 1); + CHECK(api->live_object_count() == 0); +} + +void TestNullResetDoesNotRelease() { + flm::test::ResetFakeCorelib(); + auto api = ResolveCompleteCorelib(); + + flm::corelib::UniqueMatMulWeights empty; + empty.reset(); + flm::corelib::UniqueSsMlpWeights null_value(api, nullptr); + null_value.reset(); + + CHECK(flm::test::ObjectReleaseCount() == 0); + CHECK(api->live_object_count() == 0); +} + +void TestExplicitCorelibFileWins() { + TempDirectory temp; + const auto executable_dir = temp.path() / "bin"; + const auto explicit_file = temp.path() / "chosen-corelib.dll"; + std::filesystem::create_directories(executable_dir); + Touch(explicit_file); + ScopedEnvironment override_path{ + L"RYZENAI_CORELIB_PATH", + explicit_file.wstring()}; + + CHECK(CorelibApi::ResolveLibraryPath(executable_dir) == + explicit_file.lexically_normal()); +} + +void TestExplicitCorelibDirectoryWins() { + TempDirectory temp; + const auto executable_dir = temp.path() / "bin"; + const auto explicit_directory = temp.path() / "runtime"; + std::filesystem::create_directories(executable_dir); + std::filesystem::create_directories(explicit_directory); + ScopedEnvironment override_path{ + L"RYZENAI_CORELIB_PATH", + explicit_directory.wstring()}; + + CHECK(CorelibApi::ResolveLibraryPath(executable_dir) == + (explicit_directory / "ryzenai_corelib.dll").lexically_normal()); +} + +void TestFallbackIgnoresCurrentDirectoryAndPath() { + TempDirectory temp; + const auto executable_dir = temp.path() / "application"; + const auto trap_directory = temp.path() / "trap"; + std::filesystem::create_directories(executable_dir); + std::filesystem::create_directories(trap_directory); + Touch(trap_directory / "ryzenai_corelib.dll"); + + ScopedEnvironment no_override{ + L"RYZENAI_CORELIB_PATH", + std::nullopt}; + ScopedEnvironment trap_path{ + L"PATH", + trap_directory.wstring()}; + ScopedCurrentPath trap_current_directory{trap_directory}; + + CHECK(CorelibApi::ResolveLibraryPath(executable_dir) == + (executable_dir / "aie4" / "ryzenai_corelib.dll") + .lexically_normal()); +} + +static_assert( + !std::is_copy_constructible_v); +static_assert( + !std::is_copy_assignable_v); +static_assert( + std::is_nothrow_move_constructible_v); +static_assert( + std::is_nothrow_move_assignable_v); + +} // namespace + +int main() { + try { + TestCompleteResolution(); + TestMissingSymbolFailsAtomically(); + TestErrorDetailSurvivesStatusConversion(); + TestSuccessfulStatusDoesNotReadErrorState(); + TestMoveConstructionReleasesExactlyOnce(); + TestMoveAssignmentReleasesEachObjectOnce(); + TestNullResetDoesNotRelease(); + TestExplicitCorelibFileWins(); + TestExplicitCorelibDirectoryWins(); + TestFallbackIgnoresCurrentDirectoryAndPath(); + std::cout << "test_corelib_api: PASS\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/src/test/phi4_corelib_aie4/test_support.hpp b/src/test/phi4_corelib_aie4/test_support.hpp new file mode 100644 index 00000000..cd1b5b10 --- /dev/null +++ b/src/test/phi4_corelib_aie4/test_support.hpp @@ -0,0 +1,33 @@ +#pragma once + +#include +#include +#include +#include + +inline void Check( + bool condition, + std::string_view expression, + const char* file, + int line) { + if (!condition) { + throw std::runtime_error( + std::string(file) + ":" + std::to_string(line) + + " CHECK failed: " + std::string(expression)); + } +} + +#define CHECK(expression) \ + Check(static_cast(expression), #expression, __FILE__, __LINE__) + +template +void CheckThrowsContains(Function&& function, std::string_view expected) { + try { + function(); + } catch (const std::exception& error) { + CHECK(std::string_view(error.what()).find(expected) != + std::string_view::npos); + return; + } + throw std::runtime_error("expected exception was not thrown"); +} From 9c8de1ce17db9e10bafba8daf72ad20e461afd13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CChi?= Date: Mon, 31 Aug 2026 21:09:02 -0700 Subject: [PATCH 002/117] feat: add corelib runtime failure handling Co-authored-by: Cursor --- .gitignore | 15 +- src/CMakeLists.txt | 10 + src/common/corelib/corelib_api.cpp | 8 +- src/common/corelib/corelib_fatal_record.cpp | 655 ++++++++++++++++ src/common/corelib/corelib_runtime.cpp | 204 +++++ src/common/corelib/corelib_sources.cmake | 4 +- src/include/corelib/corelib_fatal_record.hpp | 68 ++ src/include/corelib/corelib_runtime.hpp | 72 ++ src/test/phi4_corelib_aie4/CMakeLists.txt | 20 +- src/test/phi4_corelib_aie4/fake_corelib.cpp | 340 ++++++++- src/test/phi4_corelib_aie4/fake_corelib.hpp | 5 + .../phi4_corelib_aie4/test_corelib_api.cpp | 120 ++- .../test_corelib_fatal_record.cpp | 710 ++++++++++++++++++ 13 files changed, 2180 insertions(+), 51 deletions(-) create mode 100644 src/common/corelib/corelib_fatal_record.cpp create mode 100644 src/common/corelib/corelib_runtime.cpp create mode 100644 src/include/corelib/corelib_fatal_record.hpp create mode 100644 src/include/corelib/corelib_runtime.hpp create mode 100644 src/test/phi4_corelib_aie4/test_corelib_fatal_record.cpp diff --git a/.gitignore b/.gitignore index 64998c4d..1ef2e712 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,9 @@ -build -build_*/ -.vscode -__pycache__ -_site -.jekyll-metadata -*.csv +build +build_*/ +/src/build-*/ +.vscode +__pycache__ +_site +.jekyll-metadata +*.csv /docs/superpowers/ diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 5b79b840..418a3f73 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -97,6 +97,16 @@ else() if(WIN32) set(XRT_INCLUDE_DIR C:/dev/XRT/src/runtime_src/core/include CACHE PATH "Where XRT headers live") set(XRT_LIB_DIR C:/dev/xrtNPUfromDLL CACHE PATH "Where XRT libs live") + if(FLM_ENABLE_CORELIB_AIE4) + target_include_directories(flm_corelib_aie4 PUBLIC + ${XRT_INCLUDE_DIR}) + target_link_directories(flm_corelib_aie4 PUBLIC + ${XRT_LIB_DIR}) + target_link_libraries(flm_corelib_aie4 PUBLIC + xrt_coreutil + shell32 + ole32) + endif() else() find_package(PkgConfig) if(PkgConfig_FOUND) diff --git a/src/common/corelib/corelib_api.cpp b/src/common/corelib/corelib_api.cpp index abfcff26..4413c3e3 100644 --- a/src/common/corelib/corelib_api.cpp +++ b/src/common/corelib/corelib_api.cpp @@ -245,7 +245,13 @@ std::filesystem::path CorelibApi::ResolveLibraryPath( const std::filesystem::path& executable_dir) { if (const auto configured = ReadWideEnvironment(kCorelibPathEnvironment)) { - auto path = MakeAbsolute(std::filesystem::path(*configured)); + std::filesystem::path path(*configured); + if (!path.is_absolute()) { + throw std::invalid_argument( + "RYZENAI_CORELIB_PATH must be an absolute file or " + "directory path"); + } + path = path.lexically_normal(); std::error_code error; if (std::filesystem::is_directory(path, error)) { path /= kCorelibFilename; diff --git a/src/common/corelib/corelib_fatal_record.cpp b/src/common/corelib/corelib_fatal_record.cpp new file mode 100644 index 00000000..b62fbcad --- /dev/null +++ b/src/common/corelib/corelib_fatal_record.cpp @@ -0,0 +1,655 @@ +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace flm::corelib { +namespace { + +using HundredNanoseconds = + std::chrono::duration>; + +constexpr uint64_t kWindowsToUnixEpochTicks = 116'444'736'000'000'000ULL; +constexpr std::string_view kPendingPrefix = "pending-corelib-fatal-"; +constexpr std::string_view kPendingSuffix = ".tmp"; +constexpr std::string_view kFinalPrefix = "corelib-fatal-"; +constexpr std::string_view kFinalSuffix = ".json"; + +std::runtime_error FatalRecordError( + std::string_view action, + const std::filesystem::path& path, + unsigned long error) { + return std::runtime_error( + "AIE4 fatal record " + std::string(action) + " failed for " + + path.string() + " (error " + std::to_string(error) + ")"); +} + +std::string FormatUtc( + std::chrono::system_clock::time_point value) { + const auto ticks = + std::chrono::duration_cast( + value.time_since_epoch()) + .count(); + constexpr int64_t ticks_per_second = 10'000'000; + int64_t seconds = ticks / ticks_per_second; + int64_t fraction = ticks % ticks_per_second; + if (fraction < 0) { + fraction += ticks_per_second; + --seconds; + } + + const std::time_t calendar_seconds = + static_cast(seconds); + std::tm utc{}; + if (gmtime_s(&utc, &calendar_seconds) != 0) { + throw std::runtime_error( + "AIE4 fatal record timestamp conversion failed"); + } + + std::ostringstream output; + output << std::put_time(&utc, "%Y%m%dT%H%M%S") + << std::setfill('0') << std::setw(7) << fraction << 'Z'; + return output.str(); +} + +std::optional ParseUtc( + std::string_view value) { + if (value.size() != 23 || value[8] != 'T' || value[22] != 'Z') { + return std::nullopt; + } + + const auto parse_part = + [value](std::size_t offset, std::size_t length) + -> std::optional { + int result = 0; + const char* first = value.data() + offset; + const char* last = first + length; + const auto parsed = std::from_chars(first, last, result); + if (parsed.ec != std::errc{} || parsed.ptr != last) { + return std::nullopt; + } + return result; + }; + + const auto year = parse_part(0, 4); + const auto month = parse_part(4, 2); + const auto day = parse_part(6, 2); + const auto hour = parse_part(9, 2); + const auto minute = parse_part(11, 2); + const auto second = parse_part(13, 2); + const auto fraction = parse_part(15, 7); + if (!year || !month || !day || !hour || !minute || !second || + !fraction) { + return std::nullopt; + } + + std::tm utc{}; + utc.tm_year = *year - 1900; + utc.tm_mon = *month - 1; + utc.tm_mday = *day; + utc.tm_hour = *hour; + utc.tm_min = *minute; + utc.tm_sec = *second; + const __time64_t seconds = _mkgmtime64(&utc); + if (seconds == -1) { + return std::nullopt; + } + + const HundredNanoseconds duration{ + static_cast(seconds) * 10'000'000 + *fraction}; + const auto result = std::chrono::system_clock::time_point{ + std::chrono::duration_cast( + duration)}; + if (FormatUtc(result) != value) { + return std::nullopt; + } + return result; +} + +std::chrono::system_clock::time_point FileTimeToSystemClock( + const FILETIME& value) { + ULARGE_INTEGER ticks{}; + ticks.LowPart = value.dwLowDateTime; + ticks.HighPart = value.dwHighDateTime; + if (ticks.QuadPart < kWindowsToUnixEpochTicks) { + throw std::runtime_error( + "AIE4 fatal record process start time predates Unix epoch"); + } + const auto unix_ticks = ticks.QuadPart - kWindowsToUnixEpochTicks; + if (unix_ticks > + static_cast(std::numeric_limits::max())) { + throw std::runtime_error( + "AIE4 fatal record process start time is out of range"); + } + return std::chrono::system_clock::time_point{ + std::chrono::duration_cast( + HundredNanoseconds{static_cast(unix_ticks)})}; +} + +std::chrono::system_clock::time_point QueryProcessStart(HANDLE process) { + FILETIME creation{}; + FILETIME exit{}; + FILETIME kernel{}; + FILETIME user{}; + if (!GetProcessTimes( + process, + &creation, + &exit, + &kernel, + &user)) { + throw FatalRecordError( + "process-time query", + {}, + GetLastError()); + } + return FileTimeToSystemClock(creation); +} + +std::optional ProbeProcessStart( + DWORD pid) { + HANDLE process = OpenProcess( + PROCESS_QUERY_LIMITED_INFORMATION, + FALSE, + pid); + if (process == nullptr) { + if (GetLastError() == ERROR_INVALID_PARAMETER) { + return std::chrono::system_clock::time_point{}; + } + return std::nullopt; + } + + try { + const auto start = QueryProcessStart(process); + CloseHandle(process); + return start; + } catch (...) { + CloseHandle(process); + return std::nullopt; + } +} + +std::filesystem::path LocalAppDataLogRoot() { + PWSTR raw = nullptr; + const HRESULT result = SHGetKnownFolderPath( + FOLDERID_LocalAppData, + KF_FLAG_DEFAULT, + nullptr, + &raw); + if (FAILED(result)) { + throw std::runtime_error( + "AIE4 fatal record LocalAppData resolution failed (HRESULT " + + std::to_string(static_cast(result)) + ")"); + } + + std::filesystem::path root(raw); + CoTaskMemFree(raw); + return root / "FastFlowLM" / "logs"; +} + +std::string EscapeJson(std::string_view value) { + std::ostringstream output; + output << std::hex << std::uppercase; + for (const unsigned char character : value) { + switch (character) { + case '"': + output << "\\\""; + break; + case '\\': + output << "\\\\"; + break; + case '\b': + output << "\\b"; + break; + case '\f': + output << "\\f"; + break; + case '\n': + output << "\\n"; + break; + case '\r': + output << "\\r"; + break; + case '\t': + output << "\\t"; + break; + default: + if (character < 0x20) { + output << "\\u" + << std::setfill('0') << std::setw(4) + << static_cast(character); + } else { + output << static_cast(character); + } + break; + } + } + return output.str(); +} + +std::string SerializeFailure( + const FailureContext& failure, + std::string_view process_start, + std::string_view failure_utc, + DWORD pid) { + std::ostringstream output; + output << '{' + << "\"status\":" << static_cast(failure.status) << ',' + << "\"call\":\"" << EscapeJson(failure.call) << "\"," + << "\"detail\":\"" << EscapeJson(failure.detail) << "\"," + << "\"phase\":\"" << EscapeJson(failure.phase) << "\"," + << "\"layer\":"; + if (failure.layer.has_value()) { + output << *failure.layer; + } else { + output << "null"; + } + output << ',' + << "\"rows\":" << failure.rows << ',' + << "\"position\":" << failure.position << ',' + << "\"process_start_utc\":\"" << process_start << "\"," + << "\"failure_utc\":\"" << failure_utc << "\"," + << "\"pid\":" << pid + << "}\n"; + return output.str(); +} + +void WriteAll(HANDLE file, std::string_view contents) { + std::size_t offset = 0; + while (offset < contents.size()) { + const auto remaining = contents.size() - offset; + const DWORD chunk = static_cast( + (std::min)( + remaining, + static_cast( + std::numeric_limits::max()))); + DWORD written = 0; + if (!WriteFile( + file, + contents.data() + offset, + chunk, + &written, + nullptr) || + written != chunk) { + throw FatalRecordError( + "write", + {}, + GetLastError()); + } + offset += written; + } +} + +void RewindAndTruncate(HANDLE file) { + LARGE_INTEGER beginning{}; + if (!SetFilePointerEx(file, beginning, nullptr, FILE_BEGIN) || + !SetEndOfFile(file)) { + throw FatalRecordError( + "truncate", + {}, + GetLastError()); + } +} + +void EmitRecord(std::ostream& output, std::string_view record) { + output.write( + record.data(), + static_cast(record.size())); + output.flush(); + if (!output) { + throw std::runtime_error( + "AIE4 fatal record startup reporting failed"); + } +} + +bool HasPrefixAndSuffix( + std::string_view value, + std::string_view prefix, + std::string_view suffix) { + return value.starts_with(prefix) && value.ends_with(suffix) && + value.size() > prefix.size() + suffix.size(); +} + +struct PendingIdentity { + std::string timestamp; + DWORD pid; + std::chrono::system_clock::time_point start; +}; + +std::optional ParsePendingIdentity( + const std::filesystem::path& path) { + const std::string filename = path.filename().string(); + if (!HasPrefixAndSuffix( + filename, + kPendingPrefix, + kPendingSuffix)) { + return std::nullopt; + } + + const std::string_view body(filename.data() + kPendingPrefix.size(), + filename.size() - kPendingPrefix.size() - + kPendingSuffix.size()); + const auto separator = body.rfind('-'); + if (separator == std::string_view::npos) { + return std::nullopt; + } + const std::string_view timestamp = body.substr(0, separator); + const std::string_view pid_text = body.substr(separator + 1); + unsigned long pid_value = 0; + const auto parsed = std::from_chars( + pid_text.data(), + pid_text.data() + pid_text.size(), + pid_value); + if (parsed.ec != std::errc{} || + parsed.ptr != pid_text.data() + pid_text.size() || + pid_value > std::numeric_limits::max()) { + return std::nullopt; + } + const auto start = ParseUtc(timestamp); + if (!start.has_value()) { + return std::nullopt; + } + return PendingIdentity{ + std::string(timestamp), + static_cast(pid_value), + *start}; +} + +std::string ReadRecord(const std::filesystem::path& path) { + std::ifstream input(path, std::ios::binary); + if (!input) { + throw FatalRecordError("read", path, GetLastError()); + } + std::string contents{ + std::istreambuf_iterator(input), + std::istreambuf_iterator()}; + if (input.bad()) { + throw FatalRecordError("read", path, GetLastError()); + } + return contents; +} + +void RemoveReportedRecord(const std::filesystem::path& path) { + std::error_code error; + const bool removed = std::filesystem::remove(path, error); + if (error || !removed) { + throw std::runtime_error( + "AIE4 fatal record removal failed for " + path.string() + + (error ? ": " + error.message() : "")); + } +} + +} // namespace + +FatalRecordStore::FatalRecordStore( + std::filesystem::path root, + DWORD pid, + std::chrono::system_clock::time_point process_start, + ProcessProbe process_probe) + : root_(std::move(root)), + pid_(pid), + process_start_(process_start), + process_probe_(std::move(process_probe)) {} + +FatalRecordStore::FatalRecordStore( + FatalRecordStore&& other) noexcept + : root_(std::move(other.root_)), + pid_(other.pid_), + process_start_(other.process_start_), + process_probe_(std::move(other.process_probe_)), + pending_path_(std::move(other.pending_path_)), + pending_handle_( + std::exchange(other.pending_handle_, INVALID_HANDLE_VALUE)), + persisted_(other.persisted_) { + other.pending_path_.clear(); + other.persisted_ = true; +} + +FatalRecordStore& FatalRecordStore::operator=( + FatalRecordStore&& other) noexcept { + if (this != &other) { + RemoveUnusedPending(); + root_ = std::move(other.root_); + pid_ = other.pid_; + process_start_ = other.process_start_; + process_probe_ = std::move(other.process_probe_); + pending_path_ = std::move(other.pending_path_); + pending_handle_ = + std::exchange(other.pending_handle_, INVALID_HANDLE_VALUE); + persisted_ = other.persisted_; + other.pending_path_.clear(); + other.persisted_ = true; + } + return *this; +} + +FatalRecordStore::~FatalRecordStore() noexcept { + RemoveUnusedPending(); +} + +FatalRecordStore FatalRecordStore::ForCurrentProcess() { + const DWORD pid = GetCurrentProcessId(); + return FatalRecordStore( + LocalAppDataLogRoot(), + pid, + QueryProcessStart(GetCurrentProcess()), + ProbeProcessStart); +} + +void FatalRecordStore::Prepare() { + if (pending_handle_ != INVALID_HANDLE_VALUE || persisted_) { + return; + } + + std::error_code error; + std::filesystem::create_directories(root_, error); + if (error || !std::filesystem::is_directory(root_, error) || error) { + throw std::runtime_error( + "AIE4 fatal record directory preparation failed for " + + root_.string() + + (error ? ": " + error.message() : "")); + } + + pending_path_ = + root_ / + (std::string(kPendingPrefix) + FormatUtc(process_start_) + "-" + + std::to_string(pid_) + std::string(kPendingSuffix)); + pending_handle_ = CreateFileW( + pending_path_.c_str(), + GENERIC_WRITE, + FILE_SHARE_READ, + nullptr, + CREATE_NEW, + FILE_ATTRIBUTE_NORMAL | FILE_FLAG_WRITE_THROUGH, + nullptr); + if (pending_handle_ == INVALID_HANDLE_VALUE) { + throw FatalRecordError( + "initialization", + pending_path_, + GetLastError()); + } + + try { + WriteAll(pending_handle_, "P"); + if (!FlushFileBuffers(pending_handle_)) { + throw FatalRecordError( + "initial flush", + pending_path_, + GetLastError()); + } + RewindAndTruncate(pending_handle_); + if (!FlushFileBuffers(pending_handle_)) { + throw FatalRecordError( + "initial flush", + pending_path_, + GetLastError()); + } + } catch (...) { + ClosePending(); + DeleteFileW(pending_path_.c_str()); + throw; + } +} + +std::filesystem::path FatalRecordStore::Persist( + const FailureContext& failure) { + if (pending_handle_ == INVALID_HANDLE_VALUE || persisted_) { + throw std::logic_error( + "AIE4 fatal record store was not prepared"); + } + + const auto failure_time = std::chrono::system_clock::now(); + const std::string failure_utc = FormatUtc(failure_time); + const std::string record = SerializeFailure( + failure, + FormatUtc(process_start_), + failure_utc, + pid_); + RewindAndTruncate(pending_handle_); + WriteAll(pending_handle_, record); + if (!FlushFileBuffers(pending_handle_)) { + throw FatalRecordError( + "flush", + pending_path_, + GetLastError()); + } + ClosePending(); + + const auto final_path = + root_ / + (std::string(kFinalPrefix) + failure_utc + "-" + + std::to_string(pid_) + std::string(kFinalSuffix)); + if (!MoveFileExW( + pending_path_.c_str(), + final_path.c_str(), + MOVEFILE_WRITE_THROUGH)) { + throw FatalRecordError( + "atomic rename", + final_path, + GetLastError()); + } + persisted_ = true; + return final_path; +} + +std::vector FatalRecordStore::DrainPriorRecords( + const std::filesystem::path& root, + ProcessProbe process_probe, + std::ostream& output) { + std::error_code error; + if (!std::filesystem::exists(root, error)) { + if (error) { + throw std::filesystem::filesystem_error( + "failed to inspect AIE4 fatal record directory", + root, + error); + } + return {}; + } + if (!std::filesystem::is_directory(root, error) || error) { + throw std::runtime_error( + "AIE4 fatal record root is not a readable directory: " + + root.string()); + } + + std::vector final_paths; + std::vector pending_paths; + for (std::filesystem::directory_iterator iterator(root, error), end; + !error && iterator != end; + iterator.increment(error)) { + const auto filename = iterator->path().filename().string(); + if (HasPrefixAndSuffix( + filename, + kFinalPrefix, + kFinalSuffix)) { + final_paths.push_back(iterator->path()); + } else if (HasPrefixAndSuffix( + filename, + kPendingPrefix, + kPendingSuffix)) { + pending_paths.push_back(iterator->path()); + } + } + if (error) { + throw std::filesystem::filesystem_error( + "failed to enumerate AIE4 fatal records", + root, + error); + } + std::sort(final_paths.begin(), final_paths.end()); + std::sort(pending_paths.begin(), pending_paths.end()); + + std::vector records; + records.reserve(final_paths.size() + pending_paths.size()); + for (const auto& path : final_paths) { + std::string record = ReadRecord(path); + EmitRecord(output, record); + records.push_back(std::move(record)); + RemoveReportedRecord(path); + } + + for (const auto& path : pending_paths) { + const auto identity = ParsePendingIdentity(path); + if (!identity.has_value() || !process_probe) { + continue; + } + + std::optional current_start; + try { + current_start = process_probe(identity->pid); + } catch (...) { + continue; + } + if (!current_start.has_value() || + FormatUtc(*current_start) == identity->timestamp) { + continue; + } + + std::string record = + "incomplete corelib fatal record: " + + path.filename().string() + "\n"; + EmitRecord(output, record); + records.push_back(std::move(record)); + RemoveReportedRecord(path); + } + return records; +} + +void FatalRecordStore::RemoveUnusedPending() noexcept { + if (persisted_) { + return; + } + ClosePending(); + if (!pending_path_.empty()) { + DeleteFileW(pending_path_.c_str()); + } +} + +const std::filesystem::path& FatalRecordStore::pending_path() const noexcept { + return pending_path_; +} + +void FatalRecordStore::ClosePending() noexcept { + if (pending_handle_ != INVALID_HANDLE_VALUE) { + CloseHandle(pending_handle_); + pending_handle_ = INVALID_HANDLE_VALUE; + } +} + +} // namespace flm::corelib diff --git a/src/common/corelib/corelib_runtime.cpp b/src/common/corelib/corelib_runtime.cpp new file mode 100644 index 00000000..47f523a1 --- /dev/null +++ b/src/common/corelib/corelib_runtime.cpp @@ -0,0 +1,204 @@ +#include + +#include + +#include +#include +#include +#include + +namespace flm::corelib { +namespace { + +constexpr unsigned int kFatalExitCode = 0xE0040001u; + +struct ProcessRuntimeSlot { + std::mutex mutex; + std::shared_ptr runtime; +}; + +ProcessRuntimeSlot& RuntimeSlot() { + static auto* slot = new ProcessRuntimeSlot; + return *slot; +} + +[[noreturn]] void TerminateCurrentProcess(unsigned int code) { + TerminateProcess(GetCurrentProcess(), code); + std::abort(); +} + +} // namespace + +std::shared_ptr CorelibRuntime::GetOrCreate( + const std::filesystem::path& executable_dir) { + auto& slot = RuntimeSlot(); + std::lock_guard lock(slot.mutex); + if (slot.runtime) { + if (slot.runtime->state() != ProcessState::Healthy) { + throw std::runtime_error( + "corelib process runtime is not healthy"); + } + return slot.runtime; + } + + auto api = CorelibApi::Load( + CorelibApi::ResolveLibraryPath(executable_dir)); + auto runtime = Create( + std::move(api), + FatalRecordStore::ForCurrentProcess(), + TerminateCurrentProcess); + slot.runtime = runtime; + return runtime; +} + +std::shared_ptr CorelibRuntime::Create( + std::shared_ptr api, + FatalRecordStore records, + Terminator terminator) { + if (!api) { + throw std::invalid_argument( + "CorelibRuntime::Create requires a CorelibApi"); + } + if (!terminator) { + throw std::invalid_argument( + "CorelibRuntime::Create requires a terminator"); + } + + auto runtime = std::shared_ptr( + new CorelibRuntime( + std::move(api), + std::move(records), + std::move(terminator))); + try { + runtime->records_.Prepare(); + runtime->api_->Check( + runtime->api_->functions().selftest_dependencies(), + "ryzenai_corelib_selftest_dependencies"); + if (!runtime->api_->functions().has_device_context()) { + throw std::runtime_error( + "ryzenai_corelib_has_device_context reported no " + "AIE4 device context"); + } + runtime->state_.store( + ProcessState::Healthy, + std::memory_order_release); + return runtime; + } catch (...) { + runtime->api_->functions().cleanup(); + runtime->cleanup_called_ = true; + throw; + } +} + +void CorelibRuntime::ShutdownProcess() { + auto& slot = RuntimeSlot(); + std::shared_ptr runtime; + { + std::lock_guard lock(slot.mutex); + if (!slot.runtime) { + return; + } + runtime = slot.runtime; + runtime->ShutdownHealthy(); + slot.runtime.reset(); + } + runtime.reset(); +} + +CorelibRuntime::CorelibRuntime( + std::shared_ptr api, + FatalRecordStore records, + Terminator terminator) + : api_(std::move(api)), + records_(std::move(records)), + terminator_(std::move(terminator)) {} + +ExecutionLease CorelibRuntime::AcquireExecution() { + ExecutionLease lease(execution_mutex_); + if (state_.load(std::memory_order_acquire) != + ProcessState::Healthy) { + throw std::runtime_error( + "corelib process runtime is not accepting execution"); + } + return lease; +} + +bool CorelibRuntime::admission_open() const noexcept { + return state_.load(std::memory_order_acquire) == + ProcessState::Healthy; +} + +ProcessState CorelibRuntime::state() const noexcept { + return state_.load(std::memory_order_acquire); +} + +const std::shared_ptr& CorelibRuntime::api() const noexcept { + return api_; +} + +void CorelibRuntime::ShutdownHealthy() { + std::lock_guard shutdown_lock(shutdown_mutex_); + ProcessState expected = ProcessState::Healthy; + if (!state_.compare_exchange_strong( + expected, + ProcessState::Shutdown, + std::memory_order_acq_rel, + std::memory_order_acquire)) { + if (expected == ProcessState::Shutdown) { + return; + } + throw std::logic_error( + "cannot clean up a terminating corelib process runtime"); + } + + ExecutionLease lease(execution_mutex_); + if (state_.load(std::memory_order_acquire) == + ProcessState::Terminating) { + throw std::logic_error( + "cannot clean up a terminating corelib process runtime"); + } + if (api_->live_object_count() != 0) { + state_.store(ProcessState::Healthy, std::memory_order_release); + throw std::logic_error( + "cannot clean up corelib while live corelib objects remain"); + } + + if (!cleanup_called_) { + api_->functions().cleanup(); + cleanup_called_ = true; + } + records_.RemoveUnusedPending(); +} + +[[noreturn]] void CorelibRuntime::TerminateAfterFailure( + const FailureContext& failure) { + state_.store(ProcessState::Terminating, std::memory_order_release); + + std::cerr << "AIE4 terminal failure: call=" << failure.call + << " phase=" << failure.phase + << " layer="; + if (failure.layer.has_value()) { + std::cerr << *failure.layer; + } else { + std::cerr << "none"; + } + std::cerr << " rows=" << failure.rows + << " position=" << failure.position + << " detail=" << failure.detail << '\n'; + try { + const auto path = records_.Persist(failure); + std::cerr << "AIE4 fatal record: " << path.string() << '\n'; + } catch (const std::exception& error) { + std::cerr << "AIE4 fatal record persistence failed: " + << error.what() << '\n'; + } catch (...) { + std::cerr << "AIE4 fatal record persistence failed with an " + "unknown error\n"; + } + std::cerr.flush(); + + terminator_(kFatalExitCode); + std::abort(); +} + +} // namespace flm::corelib diff --git a/src/common/corelib/corelib_sources.cmake b/src/common/corelib/corelib_sources.cmake index 6c396406..60641974 100644 --- a/src/common/corelib/corelib_sources.cmake +++ b/src/common/corelib/corelib_sources.cmake @@ -1,2 +1,4 @@ set(FLM_CORELIB_AIE4_SOURCES - "${CMAKE_CURRENT_LIST_DIR}/corelib_api.cpp") + "${CMAKE_CURRENT_LIST_DIR}/corelib_api.cpp" + "${CMAKE_CURRENT_LIST_DIR}/corelib_fatal_record.cpp" + "${CMAKE_CURRENT_LIST_DIR}/corelib_runtime.cpp") diff --git a/src/include/corelib/corelib_fatal_record.hpp b/src/include/corelib/corelib_fatal_record.hpp new file mode 100644 index 00000000..72c8c0c6 --- /dev/null +++ b/src/include/corelib/corelib_fatal_record.hpp @@ -0,0 +1,68 @@ +#pragma once + +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace flm::corelib { + +struct FailureContext { + ryzenai_corelib_status status; + std::string call; + std::string detail; + std::string phase; + std::optional layer; + int64_t rows; + int64_t position; +}; + +class FatalRecordStore { +public: + using ProcessProbe = + std::function< + std::optional(DWORD)>; + + FatalRecordStore( + std::filesystem::path root, + DWORD pid, + std::chrono::system_clock::time_point process_start, + ProcessProbe process_probe); + FatalRecordStore(FatalRecordStore&& other) noexcept; + FatalRecordStore& operator=(FatalRecordStore&& other) noexcept; + ~FatalRecordStore() noexcept; + FatalRecordStore(const FatalRecordStore&) = delete; + FatalRecordStore& operator=(const FatalRecordStore&) = delete; + + static FatalRecordStore ForCurrentProcess(); + + void Prepare(); + std::filesystem::path Persist(const FailureContext& failure); + static std::vector DrainPriorRecords( + const std::filesystem::path& root, + ProcessProbe process_probe, + std::ostream& output); + void RemoveUnusedPending() noexcept; + const std::filesystem::path& pending_path() const noexcept; + +private: + void ClosePending() noexcept; + + std::filesystem::path root_; + DWORD pid_ = 0; + std::chrono::system_clock::time_point process_start_; + ProcessProbe process_probe_; + std::filesystem::path pending_path_; + HANDLE pending_handle_ = INVALID_HANDLE_VALUE; + bool persisted_ = false; +}; + +} // namespace flm::corelib diff --git a/src/include/corelib/corelib_runtime.hpp b/src/include/corelib/corelib_runtime.hpp new file mode 100644 index 00000000..a5a09654 --- /dev/null +++ b/src/include/corelib/corelib_runtime.hpp @@ -0,0 +1,72 @@ +#pragma once + +#include +#include + +#include +#include +#include +#include +#include + +namespace flm::corelib { + +enum class ProcessState { Healthy, Terminating, Shutdown }; +using ExecutionLease = std::unique_lock; + +class StepSubmissionState { +public: + void MarkSuccessfulSubmit() noexcept { + submitted_ = true; + } + + bool irrevocable() const noexcept { + return submitted_; + } + +private: + bool submitted_ = false; +}; + +class CorelibRuntime final { +public: + using Terminator = std::function; + + static std::shared_ptr GetOrCreate( + const std::filesystem::path& executable_dir); + static std::shared_ptr Create( + std::shared_ptr api, + FatalRecordStore records, + Terminator terminator); + static void ShutdownProcess(); + + ~CorelibRuntime() = default; + CorelibRuntime(const CorelibRuntime&) = delete; + CorelibRuntime& operator=(const CorelibRuntime&) = delete; + CorelibRuntime(CorelibRuntime&&) = delete; + CorelibRuntime& operator=(CorelibRuntime&&) = delete; + + ExecutionLease AcquireExecution(); + bool admission_open() const noexcept; + ProcessState state() const noexcept; + const std::shared_ptr& api() const noexcept; + void ShutdownHealthy(); + [[noreturn]] void TerminateAfterFailure( + const FailureContext& failure); + +private: + CorelibRuntime( + std::shared_ptr api, + FatalRecordStore records, + Terminator terminator); + + std::shared_ptr api_; + FatalRecordStore records_; + Terminator terminator_; + std::mutex shutdown_mutex_; + mutable std::mutex execution_mutex_; + std::atomic state_{ProcessState::Shutdown}; + bool cleanup_called_ = false; +}; + +} // namespace flm::corelib diff --git a/src/test/phi4_corelib_aie4/CMakeLists.txt b/src/test/phi4_corelib_aie4/CMakeLists.txt index 6c2222ce..70c3af13 100644 --- a/src/test/phi4_corelib_aie4/CMakeLists.txt +++ b/src/test/phi4_corelib_aie4/CMakeLists.txt @@ -43,7 +43,17 @@ target_compile_definitions(flm_corelib_aie4_testlib PUBLIC NOMINMAX) target_compile_options(flm_corelib_aie4_testlib PRIVATE /fp:precise) target_link_libraries(flm_corelib_aie4_testlib PUBLIC - xrt_coreutil) + xrt_coreutil + shell32 + ole32) + +add_library(fake_ryzenai_corelib SHARED + fake_corelib.cpp) +target_include_directories(fake_ryzenai_corelib PRIVATE + ${RYZENAI_CORELIB_INCLUDE_DIR}) +target_compile_definitions(fake_ryzenai_corelib PRIVATE + RYZENAI_CORELIB_EXPORT=1 + FLM_FAKE_CORELIB_DLL=1) function(add_corelib_host_test TEST_NAME TEST_SOURCE) add_executable(${TEST_NAME} @@ -59,3 +69,11 @@ endfunction() enable_testing() add_corelib_host_test(test_corelib_api test_corelib_api.cpp) +add_corelib_host_test( + test_corelib_fatal_record + test_corelib_fatal_record.cpp) +add_dependencies( + test_corelib_fatal_record + fake_ryzenai_corelib) +target_link_libraries(test_corelib_fatal_record PRIVATE + advapi32) diff --git a/src/test/phi4_corelib_aie4/fake_corelib.cpp b/src/test/phi4_corelib_aie4/fake_corelib.cpp index 5143e7c9..cc430b1e 100644 --- a/src/test/phi4_corelib_aie4/fake_corelib.cpp +++ b/src/test/phi4_corelib_aie4/fake_corelib.cpp @@ -1,17 +1,25 @@ #include "fake_corelib.hpp" #include +#include +#include #include #include #include +#include namespace flm::test { -namespace { +namespace detail { thread_local std::string g_last_error; std::unordered_map g_release_counts; std::size_t g_release_count = 0; void* g_last_released_object = nullptr; +ryzenai_corelib_status g_selftest_status = + ryzenai_corelib_status_success; +bool g_has_device_context = true; +std::size_t g_cleanup_count = 0; +std::vector g_events; const char* FakeStatusToString(ryzenai_corelib_status status) { g_last_error = "overwritten by status_to_string"; @@ -33,17 +41,18 @@ const char* FakeGetLastErrorMessage() { } ryzenai_corelib_status FakeSelftestDependencies() { - return ryzenai_corelib_status_success; + return g_selftest_status; } bool FakeHasDeviceContext() { - return true; + return g_has_device_context; } void FakeObjectRelease(ryzenai_corelib_object_ptr object) { ++g_release_count; ++g_release_counts[object]; g_last_released_object = object; + g_events.emplace_back("release"); } ryzenai_corelib_status FakeCreateStream(ryzenai_corelib_stream_ptr* out) { @@ -218,17 +227,294 @@ ryzenai_corelib_status FakeFlatMha( return ryzenai_corelib_status_success; } -void FakeCleanup() {} +void FakeCleanup() { + ++g_cleanup_count; + g_events.emplace_back("cleanup"); + + std::size_t required = 0; + if (_wgetenv_s( + &required, + nullptr, + 0, + L"FLM_FAKE_CORELIB_CLEANUP_MARKER") != 0 || + required == 0) { + return; + } + std::vector marker_path(required); + if (_wgetenv_s( + &required, + marker_path.data(), + marker_path.size(), + L"FLM_FAKE_CORELIB_CLEANUP_MARKER") != 0) { + return; + } + std::ofstream marker( + std::filesystem::path(marker_path.data()), + std::ios::app | std::ios::binary); + marker << "cleanup\n"; +} + +} // namespace detail +} // namespace flm::test + +#if defined(FLM_FAKE_CORELIB_DLL) + +const char* ryzenai_corelib_status_to_string( + ryzenai_corelib_status status) { + return flm::test::detail::FakeStatusToString(status); +} + +const char* ryzenai_corelib_get_last_error_message() { + return flm::test::detail::FakeGetLastErrorMessage(); +} + +ryzenai_corelib_status ryzenai_corelib_selftest_dependencies() { + return flm::test::detail::FakeSelftestDependencies(); +} + +bool ryzenai_corelib_has_device_context() { + return flm::test::detail::FakeHasDeviceContext(); +} + +void ryzenai_corelib_object_release(ryzenai_corelib_object_ptr object) { + flm::test::detail::FakeObjectRelease(object); +} + +ryzenai_corelib_status ryzenai_corelib_create_stream( + ryzenai_corelib_stream_ptr* out) { + return flm::test::detail::FakeCreateStream(out); +} + +ryzenai_corelib_status ryzenai_corelib_stream_synchronize( + ryzenai_corelib_stream_ptr stream) { + return flm::test::detail::FakeStreamSynchronize(stream); +} + +ryzenai_corelib_status ryzenai_corelib_create_device_tensor( + ryzenai_corelib_data_type data_type, + const int64_t* shape, + std::size_t shape_len, + ryzenai_corelib_tensor_ptr* out) { + return flm::test::detail::FakeCreateDeviceTensor( + data_type, + shape, + shape_len, + out); +} + +ryzenai_corelib_status ryzenai_corelib_tensor_write( + ryzenai_corelib_tensor_ptr tensor, + const void* source, + std::size_t size, + std::size_t offset) { + return flm::test::detail::FakeTensorWrite( + tensor, + source, + size, + offset); +} + +ryzenai_corelib_status ryzenai_corelib_tensor_read( + ryzenai_corelib_tensor_ptr tensor, + void* destination, + std::size_t size, + std::size_t offset) { + return flm::test::detail::FakeTensorRead( + tensor, + destination, + size, + offset); +} + +ryzenai_corelib_status ryzenai_corelib_tensor_get_byte_size( + ryzenai_corelib_tensor_ptr tensor, + std::size_t* out) { + return flm::test::detail::FakeTensorGetByteSize(tensor, out); +} + +ryzenai_corelib_status ryzenai_corelib_convert( + ryzenai_corelib_data_type source_type, + const void* source, + ryzenai_corelib_data_type destination_type, + void* destination, + std::size_t count) { + return flm::test::detail::FakeConvert( + source_type, + source, + destination_type, + destination, + count); +} + +ryzenai_corelib_status ryzenai_corelib_convert_strided( + ryzenai_corelib_data_type source_type, + const void* source, + std::size_t source_stride, + ryzenai_corelib_data_type destination_type, + void* destination, + std::size_t destination_stride, + std::size_t count, + std::size_t row) { + return flm::test::detail::FakeConvertStrided( + source_type, + source, + source_stride, + destination_type, + destination, + destination_stride, + count, + row); +} + +ryzenai_corelib_status ryzenai_corelib_matmul_bf16_pad_shape( + int64_t* m, + int64_t* k, + int64_t* n, + uint32_t group_size) { + return flm::test::detail::FakeMatmulPadShape( + m, + k, + n, + group_size); +} + +ryzenai_corelib_status +ryzenai_corelib_matmul_bf16_weights_create_from_onnx_components( + const ryzenai_corelib_matmul_bf16_weights_desc* desc, + const ryzenai_corelib_matmul_bf16_onnx_weights_components* components, + ryzenai_corelib_matmul_bf16_weights_ptr* out) { + return flm::test::detail::FakeMatmulWeightsFromOnnx( + desc, + components, + out); +} + +ryzenai_corelib_status ryzenai_corelib_matmul_bf16_weights_get_data( + ryzenai_corelib_matmul_bf16_weights_ptr weights, + const void** data, + std::size_t* size) { + return flm::test::detail::FakeMatmulWeightsGetData( + weights, + data, + size); +} + +ryzenai_corelib_status ryzenai_corelib_matmul_bf16( + ryzenai_corelib_stream_ptr stream, + ryzenai_corelib_tensor_ptr input, + int64_t rows, + ryzenai_corelib_matmul_bf16_weights_ptr weights, + ryzenai_corelib_tensor_ptr output) { + return flm::test::detail::FakeMatmul( + stream, + input, + rows, + weights, + output); +} + +ryzenai_corelib_status ryzenai_corelib_ssmlp_bf16_pad_rows( + int64_t* m, + int64_t k, + int64_t n, + uint32_t group_size) { + return flm::test::detail::FakeSsmlpPadRows( + m, + k, + n, + group_size); +} + +ryzenai_corelib_status +ryzenai_corelib_ssmlp_bf16_weights_create_from_onnx_components( + const ryzenai_corelib_ssmlp_bf16_weights_desc* desc, + const ryzenai_corelib_ssmlp_bf16_onnx_weights_components* components, + ryzenai_corelib_ssmlp_bf16_weights_ptr* out) { + return flm::test::detail::FakeSsmlpWeightsFromOnnx( + desc, + components, + out); +} + +ryzenai_corelib_status ryzenai_corelib_ssmlp_bf16_weights_get_data( + ryzenai_corelib_ssmlp_bf16_weights_ptr weights, + const void** data, + std::size_t* size) { + return flm::test::detail::FakeSsmlpWeightsGetData( + weights, + data, + size); +} + +ryzenai_corelib_status ryzenai_corelib_ssmlp_bf16( + ryzenai_corelib_stream_ptr stream, + ryzenai_corelib_tensor_ptr input, + ryzenai_corelib_tensor_ptr residual, + int64_t rows, + ryzenai_corelib_ssmlp_bf16_weights_ptr weights, + ryzenai_corelib_tensor_ptr skip_sum, + ryzenai_corelib_tensor_ptr normalized) { + return flm::test::detail::FakeSsmlp( + stream, + input, + residual, + rows, + weights, + skip_sum, + normalized); +} + +ryzenai_corelib_status ryzenai_corelib_flat_mha_bf16_pad_rows( + int64_t* m, + const ryzenai_corelib_flat_mha_bf16_desc* desc) { + return flm::test::detail::FakeFlatMhaPadRows(m, desc); +} + +ryzenai_corelib_status ryzenai_corelib_flat_mha_bf16( + ryzenai_corelib_stream_ptr stream, + const ryzenai_corelib_flat_mha_bf16_desc* desc, + ryzenai_corelib_tensor_ptr query, + ryzenai_corelib_tensor_ptr key, + int64_t rows, + int64_t position, + ryzenai_corelib_tensor_ptr cos, + ryzenai_corelib_tensor_ptr sin, + ryzenai_corelib_tensor_ptr key_cache, + ryzenai_corelib_tensor_ptr value_cache, + ryzenai_corelib_tensor_ptr output) { + return flm::test::detail::FakeFlatMha( + stream, + desc, + query, + key, + rows, + position, + cos, + sin, + key_cache, + value_cache, + output); +} + +void ryzenai_corelib_cleanup() { + flm::test::detail::FakeCleanup(); +} + +#endif + +namespace flm::test { +namespace { template void* FunctionAddress(Function function) { return reinterpret_cast(function); } -#define FLM_FAKE_ENTRY(symbol, replacement) \ - { \ - #symbol, FunctionAddress( \ - static_cast(&replacement)) \ +#define FLM_FAKE_ENTRY(symbol, replacement) \ + { \ + #symbol, FunctionAddress( \ + static_cast( \ + &detail::replacement)) \ } } // namespace @@ -313,27 +599,47 @@ std::unordered_map CompleteCorelibResolver() { #undef FLM_FAKE_ENTRY void ResetFakeCorelib() { - g_last_error.clear(); - g_release_counts.clear(); - g_release_count = 0; - g_last_released_object = nullptr; + detail::g_last_error.clear(); + detail::g_release_counts.clear(); + detail::g_release_count = 0; + detail::g_last_released_object = nullptr; + detail::g_selftest_status = ryzenai_corelib_status_success; + detail::g_has_device_context = true; + detail::g_cleanup_count = 0; + detail::g_events.clear(); } void SetLastErrorMessage(std::string message) { - g_last_error = std::move(message); + detail::g_last_error = std::move(message); +} + +void SetSelftestStatus(ryzenai_corelib_status status) noexcept { + detail::g_selftest_status = status; +} + +void SetHasDeviceContext(bool value) noexcept { + detail::g_has_device_context = value; } std::size_t ObjectReleaseCount() noexcept { - return g_release_count; + return detail::g_release_count; } std::size_t ObjectReleaseCountFor(void* value) noexcept { - const auto found = g_release_counts.find(value); - return found == g_release_counts.end() ? 0 : found->second; + const auto found = detail::g_release_counts.find(value); + return found == detail::g_release_counts.end() ? 0 : found->second; } void* LastReleasedObject() noexcept { - return g_last_released_object; + return detail::g_last_released_object; +} + +std::size_t CleanupCount() noexcept { + return detail::g_cleanup_count; +} + +std::vector FakeCorelibEvents() { + return detail::g_events; } } // namespace flm::test diff --git a/src/test/phi4_corelib_aie4/fake_corelib.hpp b/src/test/phi4_corelib_aie4/fake_corelib.hpp index 14d2eef8..58906464 100644 --- a/src/test/phi4_corelib_aie4/fake_corelib.hpp +++ b/src/test/phi4_corelib_aie4/fake_corelib.hpp @@ -5,6 +5,7 @@ #include #include #include +#include namespace flm::test { @@ -12,8 +13,12 @@ std::unordered_map CompleteCorelibResolver(); void ResetFakeCorelib(); void SetLastErrorMessage(std::string message); +void SetSelftestStatus(ryzenai_corelib_status status) noexcept; +void SetHasDeviceContext(bool value) noexcept; std::size_t ObjectReleaseCount() noexcept; std::size_t ObjectReleaseCountFor(void* value) noexcept; void* LastReleasedObject() noexcept; +std::size_t CleanupCount() noexcept; +std::vector FakeCorelibEvents(); } // namespace flm::test diff --git a/src/test/phi4_corelib_aie4/test_corelib_api.cpp b/src/test/phi4_corelib_aie4/test_corelib_api.cpp index 7b99c516..920451d2 100644 --- a/src/test/phi4_corelib_aie4/test_corelib_api.cpp +++ b/src/test/phi4_corelib_aie4/test_corelib_api.cpp @@ -183,30 +183,66 @@ void TestCompleteResolution() { } const auto& functions = api->functions(); - CHECK(functions.status_to_string != nullptr); - CHECK(functions.get_last_error_message != nullptr); - CHECK(functions.selftest_dependencies != nullptr); - CHECK(functions.has_device_context != nullptr); - CHECK(functions.object_release != nullptr); - CHECK(functions.create_stream != nullptr); - CHECK(functions.stream_synchronize != nullptr); - CHECK(functions.create_device_tensor != nullptr); - CHECK(functions.tensor_write != nullptr); - CHECK(functions.tensor_read != nullptr); - CHECK(functions.tensor_get_byte_size != nullptr); - CHECK(functions.convert != nullptr); - CHECK(functions.convert_strided != nullptr); - CHECK(functions.matmul_pad_shape != nullptr); - CHECK(functions.matmul_weights_from_onnx != nullptr); - CHECK(functions.matmul_weights_get_data != nullptr); - CHECK(functions.matmul != nullptr); - CHECK(functions.ssmlp_pad_rows != nullptr); - CHECK(functions.ssmlp_weights_from_onnx != nullptr); - CHECK(functions.ssmlp_weights_get_data != nullptr); - CHECK(functions.ssmlp != nullptr); - CHECK(functions.flat_mha_pad_rows != nullptr); - CHECK(functions.flat_mha != nullptr); - CHECK(functions.cleanup != nullptr); +#define CHECK_MEMBER_IDENTITY(member, symbol) \ + CHECK(reinterpret_cast(functions.member) == \ + resolver.at(#symbol)) + CHECK_MEMBER_IDENTITY( + status_to_string, + ryzenai_corelib_status_to_string); + CHECK_MEMBER_IDENTITY( + get_last_error_message, + ryzenai_corelib_get_last_error_message); + CHECK_MEMBER_IDENTITY( + selftest_dependencies, + ryzenai_corelib_selftest_dependencies); + CHECK_MEMBER_IDENTITY( + has_device_context, + ryzenai_corelib_has_device_context); + CHECK_MEMBER_IDENTITY( + object_release, + ryzenai_corelib_object_release); + CHECK_MEMBER_IDENTITY(create_stream, ryzenai_corelib_create_stream); + CHECK_MEMBER_IDENTITY( + stream_synchronize, + ryzenai_corelib_stream_synchronize); + CHECK_MEMBER_IDENTITY( + create_device_tensor, + ryzenai_corelib_create_device_tensor); + CHECK_MEMBER_IDENTITY(tensor_write, ryzenai_corelib_tensor_write); + CHECK_MEMBER_IDENTITY(tensor_read, ryzenai_corelib_tensor_read); + CHECK_MEMBER_IDENTITY( + tensor_get_byte_size, + ryzenai_corelib_tensor_get_byte_size); + CHECK_MEMBER_IDENTITY(convert, ryzenai_corelib_convert); + CHECK_MEMBER_IDENTITY( + convert_strided, + ryzenai_corelib_convert_strided); + CHECK_MEMBER_IDENTITY( + matmul_pad_shape, + ryzenai_corelib_matmul_bf16_pad_shape); + CHECK_MEMBER_IDENTITY( + matmul_weights_from_onnx, + ryzenai_corelib_matmul_bf16_weights_create_from_onnx_components); + CHECK_MEMBER_IDENTITY( + matmul_weights_get_data, + ryzenai_corelib_matmul_bf16_weights_get_data); + CHECK_MEMBER_IDENTITY(matmul, ryzenai_corelib_matmul_bf16); + CHECK_MEMBER_IDENTITY( + ssmlp_pad_rows, + ryzenai_corelib_ssmlp_bf16_pad_rows); + CHECK_MEMBER_IDENTITY( + ssmlp_weights_from_onnx, + ryzenai_corelib_ssmlp_bf16_weights_create_from_onnx_components); + CHECK_MEMBER_IDENTITY( + ssmlp_weights_get_data, + ryzenai_corelib_ssmlp_bf16_weights_get_data); + CHECK_MEMBER_IDENTITY(ssmlp, ryzenai_corelib_ssmlp_bf16); + CHECK_MEMBER_IDENTITY( + flat_mha_pad_rows, + ryzenai_corelib_flat_mha_bf16_pad_rows); + CHECK_MEMBER_IDENTITY(flat_mha, ryzenai_corelib_flat_mha_bf16); + CHECK_MEMBER_IDENTITY(cleanup, ryzenai_corelib_cleanup); +#undef CHECK_MEMBER_IDENTITY } void TestMissingSymbolFailsAtomically() { @@ -352,6 +388,40 @@ void TestExplicitCorelibDirectoryWins() { (explicit_directory / "ryzenai_corelib.dll").lexically_normal()); } +void TestRelativeCorelibFileOverrideIsRejected() { + TempDirectory temp; + const auto executable_dir = temp.path() / "bin"; + std::filesystem::create_directories(executable_dir); + Touch(temp.path() / "relative-corelib.dll"); + ScopedCurrentPath current_path{temp.path()}; + ScopedEnvironment override_path{ + L"RYZENAI_CORELIB_PATH", + L"relative-corelib.dll"}; + + CheckThrowsContains( + [&] { + (void)CorelibApi::ResolveLibraryPath(executable_dir); + }, + "absolute"); +} + +void TestRelativeCorelibDirectoryOverrideIsRejected() { + TempDirectory temp; + const auto executable_dir = temp.path() / "bin"; + std::filesystem::create_directories(executable_dir); + std::filesystem::create_directories(temp.path() / "relative-runtime"); + ScopedCurrentPath current_path{temp.path()}; + ScopedEnvironment override_path{ + L"RYZENAI_CORELIB_PATH", + L"relative-runtime"}; + + CheckThrowsContains( + [&] { + (void)CorelibApi::ResolveLibraryPath(executable_dir); + }, + "absolute"); +} + void TestFallbackIgnoresCurrentDirectoryAndPath() { TempDirectory temp; const auto executable_dir = temp.path() / "application"; @@ -395,6 +465,8 @@ int main() { TestNullResetDoesNotRelease(); TestExplicitCorelibFileWins(); TestExplicitCorelibDirectoryWins(); + TestRelativeCorelibFileOverrideIsRejected(); + TestRelativeCorelibDirectoryOverrideIsRejected(); TestFallbackIgnoresCurrentDirectoryAndPath(); std::cout << "test_corelib_api: PASS\n"; return 0; diff --git a/src/test/phi4_corelib_aie4/test_corelib_fatal_record.cpp b/src/test/phi4_corelib_aie4/test_corelib_fatal_record.cpp new file mode 100644 index 00000000..db2a8a4a --- /dev/null +++ b/src/test/phi4_corelib_aie4/test_corelib_fatal_record.cpp @@ -0,0 +1,710 @@ +#include "fake_corelib.hpp" +#include "test_support.hpp" + +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using flm::corelib::CorelibApi; +using flm::corelib::CorelibRuntime; +using flm::corelib::FailureContext; +using flm::corelib::FatalRecordStore; +using flm::corelib::ProcessState; + +class TempDirectory final { +public: + TempDirectory() { + const auto nonce = + std::chrono::steady_clock::now().time_since_epoch().count(); + path_ = std::filesystem::temp_directory_path() / + ("fastflowlm-corelib-runtime-" + std::to_string(nonce)); + std::filesystem::create_directories(path_); + } + + ~TempDirectory() noexcept { + std::error_code error; + std::filesystem::remove_all(path_, error); + } + + TempDirectory(const TempDirectory&) = delete; + TempDirectory& operator=(const TempDirectory&) = delete; + + const std::filesystem::path& path() const noexcept { + return path_; + } + +private: + std::filesystem::path path_; +}; + +std::optional ReadEnvironment(const wchar_t* name) { + std::size_t required = 0; + if (_wgetenv_s(&required, nullptr, 0, name) != 0) { + throw std::runtime_error("failed to read environment variable"); + } + if (required == 0) { + return std::nullopt; + } + + std::vector value(required); + if (_wgetenv_s(&required, value.data(), value.size(), name) != 0) { + throw std::runtime_error("failed to read environment variable"); + } + return std::wstring(value.data()); +} + +class ScopedEnvironment final { +public: + ScopedEnvironment( + std::wstring name, + std::optional value) + : name_(std::move(name)), + original_(ReadEnvironment(name_.c_str())) { + if (_wputenv_s( + name_.c_str(), + value.has_value() ? value->c_str() : L"") != 0) { + throw std::runtime_error("failed to set environment variable"); + } + } + + ~ScopedEnvironment() noexcept { + _wputenv_s( + name_.c_str(), + original_.has_value() ? original_->c_str() : L""); + } + + ScopedEnvironment(const ScopedEnvironment&) = delete; + ScopedEnvironment& operator=(const ScopedEnvironment&) = delete; + +private: + std::wstring name_; + std::optional original_; +}; + +std::chrono::system_clock::time_point KnownStartTime() { + using namespace std::chrono; + return sys_days{year{2026} / August / day{31}} + hours{17}; +} + +std::shared_ptr ResolveCompleteCorelib() { + auto resolver = flm::test::CompleteCorelibResolver(); + return CorelibApi::ResolveForTest( + [resolver = std::move(resolver)](std::string_view name) mutable + -> void* { + const auto found = resolver.find(std::string(name)); + return found == resolver.end() ? nullptr : found->second; + }); +} + +void WriteText( + const std::filesystem::path& path, + std::string_view contents) { + std::ofstream output(path, std::ios::binary); + if (!output) { + throw std::runtime_error("failed to create test record"); + } + output.write( + contents.data(), + static_cast(contents.size())); + if (!output) { + throw std::runtime_error("failed to write test record"); + } +} + +std::string ReadText(const std::filesystem::path& path) { + std::ifstream input(path, std::ios::binary); + if (!input) { + throw std::runtime_error("failed to open test record"); + } + return std::string( + std::istreambuf_iterator(input), + std::istreambuf_iterator()); +} + +std::vector MatchingFiles( + const std::filesystem::path& root, + std::string_view prefix) { + std::vector paths; + std::error_code error; + for (std::filesystem::directory_iterator iterator(root, error), end; + !error && iterator != end; + iterator.increment(error)) { + const auto name = iterator->path().filename().string(); + if (name.starts_with(prefix)) { + paths.push_back(iterator->path()); + } + } + if (error) { + throw std::filesystem::filesystem_error( + "failed to enumerate test records", + root, + error); + } + std::sort(paths.begin(), paths.end()); + return paths; +} + +std::filesystem::path CurrentExecutablePath() { + std::wstring buffer(32768, L'\0'); + const DWORD size = GetModuleFileNameW( + nullptr, + buffer.data(), + static_cast(buffer.size())); + if (size == 0 || size == buffer.size()) { + throw std::runtime_error("GetModuleFileNameW failed"); + } + buffer.resize(size); + return std::filesystem::path(std::move(buffer)); +} + +void CreateReadOnlyDirectory(const std::filesystem::path& path) { + PSECURITY_DESCRIPTOR descriptor = nullptr; + if (!ConvertStringSecurityDescriptorToSecurityDescriptorW( + L"D:P(D;;GW;;;WD)(A;;GRGXSD;;;WD)", + SDDL_REVISION_1, + &descriptor, + nullptr)) { + throw std::runtime_error( + "failed to construct read-only directory ACL"); + } + SECURITY_ATTRIBUTES attributes{ + sizeof(SECURITY_ATTRIBUTES), + descriptor, + FALSE}; + const BOOL created = CreateDirectoryW(path.c_str(), &attributes); + const DWORD error = created ? ERROR_SUCCESS : GetLastError(); + LocalFree(descriptor); + if (!created) { + throw std::runtime_error( + "failed to create read-only test directory (error " + + std::to_string(error) + ")"); + } +} + +void TestPendingNamesUseStartTimeAndPid() { + TempDirectory temp; + const auto start = KnownStartTime(); + auto probe = [](DWORD) + -> std::optional { + return std::nullopt; + }; + FatalRecordStore store_a(temp.path(), 1001, start, probe); + FatalRecordStore store_b(temp.path(), 1002, start, probe); + + store_a.Prepare(); + store_b.Prepare(); + + CHECK(store_a.pending_path().filename() == + "pending-corelib-fatal-20260831T1700000000000Z-1001.tmp"); + CHECK(store_b.pending_path().filename() == + "pending-corelib-fatal-20260831T1700000000000Z-1002.tmp"); + CHECK(store_a.pending_path() != store_b.pending_path()); + CHECK(std::filesystem::exists(store_a.pending_path())); + CHECK(std::filesystem::exists(store_b.pending_path())); + + const auto pending_a = store_a.pending_path(); + const auto pending_b = store_b.pending_path(); + store_a.RemoveUnusedPending(); + store_b.RemoveUnusedPending(); + CHECK(!std::filesystem::exists(pending_a)); + CHECK(!std::filesystem::exists(pending_b)); +} + +void TestPersistWritesCompleteUniqueRecords() { + TempDirectory temp; + const auto start = KnownStartTime(); + auto probe = [](DWORD) + -> std::optional { + return std::nullopt; + }; + FatalRecordStore store_a(temp.path(), 1001, start, probe); + FatalRecordStore store_b(temp.path(), 1002, start, probe); + store_a.Prepare(); + store_b.Prepare(); + + const FailureContext failure{ + ryzenai_corelib_status_bad_argument, + "matmul_q", + "quoted \"detail\"\nnext line", + "qkv", + 7, + 32, + 128}; + const auto final_a = store_a.Persist(failure); + const auto final_b = store_b.Persist(failure); + + CHECK(final_a != final_b); + CHECK(final_a.filename().string().starts_with("corelib-fatal-")); + CHECK(final_a.filename().string().ends_with("-1001.json")); + CHECK(final_b.filename().string().ends_with("-1002.json")); + CHECK(std::filesystem::exists(final_a)); + CHECK(std::filesystem::exists(final_b)); + CHECK(!std::filesystem::exists(store_a.pending_path())); + CHECK(!std::filesystem::exists(store_b.pending_path())); + + const auto record = ReadText(final_a); + CHECK(record.find("\"status\":2") != std::string::npos); + CHECK(record.find("\"call\":\"matmul_q\"") != std::string::npos); + CHECK(record.find("quoted \\\"detail\\\"\\nnext line") != + std::string::npos); + CHECK(record.find("\"phase\":\"qkv\"") != std::string::npos); + CHECK(record.find("\"layer\":7") != std::string::npos); + CHECK(record.find("\"rows\":32") != std::string::npos); + CHECK(record.find("\"position\":128") != std::string::npos); + CHECK(record.find( + "\"process_start_utc\":\"20260831T1700000000000Z\"") != + std::string::npos); + CHECK(record.find("\"failure_utc\":\"") != std::string::npos); + CHECK(record.find("\"pid\":1001") != std::string::npos); +} + +void TestDrainOrdersFinalRecordsAndRemovesAfterEmission() { + TempDirectory temp; + const auto first = + temp.path() / "corelib-fatal-20260831T1700000000000Z-1001.json"; + const auto same_time_second = + temp.path() / "corelib-fatal-20260831T1700000000000Z-1002.json"; + const auto later = + temp.path() / "corelib-fatal-20260831T1800000000000Z-1000.json"; + WriteText(later, "later\n"); + WriteText(same_time_second, "same-time-second\n"); + WriteText(first, "first\n"); + std::size_t probe_calls = 0; + std::ostringstream output; + + const auto records = FatalRecordStore::DrainPriorRecords( + temp.path(), + [&probe_calls](DWORD) + -> std::optional { + ++probe_calls; + return std::nullopt; + }, + output); + + const std::vector expected{ + "first\n", + "same-time-second\n", + "later\n"}; + CHECK(records == expected); + CHECK(output.str() == "first\nsame-time-second\nlater\n"); + CHECK(probe_calls == 0); + CHECK(!std::filesystem::exists(first)); + CHECK(!std::filesystem::exists(same_time_second)); + CHECK(!std::filesystem::exists(later)); +} + +void TestDrainPreservesLivePendingRecord() { + TempDirectory temp; + const auto pending = + temp.path() / + "pending-corelib-fatal-20260831T1700000000000Z-1001.tmp"; + WriteText(pending, "pending"); + std::ostringstream output; + + const auto records = FatalRecordStore::DrainPriorRecords( + temp.path(), + [](DWORD pid) + -> std::optional { + CHECK(pid == 1001); + return KnownStartTime(); + }, + output); + + CHECK(records.empty()); + CHECK(output.str().empty()); + CHECK(std::filesystem::exists(pending)); +} + +void TestDrainPreservesPendingWhenProbeFails() { + TempDirectory temp; + const auto pending = + temp.path() / + "pending-corelib-fatal-20260831T1700000000000Z-1001.tmp"; + WriteText(pending, "pending"); + std::ostringstream output; + + const auto records = FatalRecordStore::DrainPriorRecords( + temp.path(), + [](DWORD) + -> std::optional { + return std::nullopt; + }, + output); + + CHECK(records.empty()); + CHECK(output.str().empty()); + CHECK(std::filesystem::exists(pending)); +} + +void TestDrainReportsAndRemovesStalePendingRecord() { + TempDirectory temp; + const auto pending = + temp.path() / + "pending-corelib-fatal-20260831T1700000000000Z-1001.tmp"; + WriteText(pending, "pending"); + std::ostringstream output; + + const auto records = FatalRecordStore::DrainPriorRecords( + temp.path(), + [](DWORD) + -> std::optional { + return KnownStartTime() + std::chrono::seconds{1}; + }, + output); + + CHECK(records.size() == 1); + CHECK(records.front().find("incomplete corelib fatal record") != + std::string::npos); + CHECK(records.front().find(pending.filename().string()) != + std::string::npos); + CHECK(output.str() == records.front()); + CHECK(!std::filesystem::exists(pending)); +} + +void TestPrepareRejectsUnwritableRoot() { + TempDirectory temp; + const auto read_only = temp.path() / "read-only"; + CreateReadOnlyDirectory(read_only); + FatalRecordStore records( + read_only, + 1001, + KnownStartTime(), + [](DWORD) + -> std::optional { + return std::nullopt; + }); + + CheckThrowsContains( + [&] { + records.Prepare(); + }, + "fatal record"); +} + +void TestUnusedPendingIsRemovedByDestructor() { + TempDirectory temp; + std::filesystem::path pending; + { + FatalRecordStore records( + temp.path(), + 1001, + KnownStartTime(), + [](DWORD) + -> std::optional { + return std::nullopt; + }); + records.Prepare(); + pending = records.pending_path(); + CHECK(std::filesystem::exists(pending)); + } + CHECK(!std::filesystem::exists(pending)); +} + +FatalRecordStore MakeTestRecords( + const std::filesystem::path& root, + DWORD pid = 1001) { + return FatalRecordStore( + root, + pid, + KnownStartTime(), + [](DWORD) + -> std::optional { + return std::nullopt; + }); +} + +void TestRuntimePublishesHealthyAndShutsDownOnce() { + TempDirectory temp; + flm::test::ResetFakeCorelib(); + auto api = ResolveCompleteCorelib(); + bool terminated = false; + auto runtime = CorelibRuntime::Create( + api, + MakeTestRecords(temp.path()), + [&terminated](unsigned int) { + terminated = true; + throw std::runtime_error("unexpected termination"); + }); + + CHECK(runtime->state() == ProcessState::Healthy); + CHECK(runtime->admission_open()); + CHECK(runtime->api().get() == api.get()); + { + auto lease = runtime->AcquireExecution(); + CHECK(lease.owns_lock()); + } + + runtime->ShutdownHealthy(); + runtime->ShutdownHealthy(); + CHECK(runtime->state() == ProcessState::Shutdown); + CHECK(!runtime->admission_open()); + CHECK(flm::test::CleanupCount() == 1); + CHECK(!terminated); + CHECK(MatchingFiles( + temp.path(), + "pending-corelib-fatal-").empty()); +} + +void TestHealthyShutdownClosesAdmissionBeforeWaiting() { + TempDirectory temp; + flm::test::ResetFakeCorelib(); + auto runtime = CorelibRuntime::Create( + ResolveCompleteCorelib(), + MakeTestRecords(temp.path()), + [](unsigned int) { + throw std::runtime_error("unexpected termination"); + }); + auto active_execution = runtime->AcquireExecution(); + std::atomic shutdown_started = false; + std::exception_ptr shutdown_error; + std::thread shutdown([&] { + shutdown_started.store(true, std::memory_order_release); + try { + runtime->ShutdownHealthy(); + } catch (...) { + shutdown_error = std::current_exception(); + } + }); + while (!shutdown_started.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + + const auto deadline = + std::chrono::steady_clock::now() + std::chrono::seconds{1}; + while (runtime->admission_open() && + std::chrono::steady_clock::now() < deadline) { + std::this_thread::yield(); + } + const bool closed_before_active_execution_finished = + !runtime->admission_open(); + active_execution.unlock(); + shutdown.join(); + + if (shutdown_error) { + std::rethrow_exception(shutdown_error); + } + CHECK(closed_before_active_execution_finished); + CHECK(runtime->state() == ProcessState::Shutdown); +} + +void TestInitializationFailureCleansUpAndRemovesPending() { + TempDirectory temp; + flm::test::ResetFakeCorelib(); + flm::test::SetLastErrorMessage("dependency probe failed"); + flm::test::SetSelftestStatus(ryzenai_corelib_status_failure); + auto api = ResolveCompleteCorelib(); + + CheckThrowsContains( + [&] { + (void)CorelibRuntime::Create( + api, + MakeTestRecords(temp.path()), + [](unsigned int) { + throw std::runtime_error("unexpected termination"); + }); + }, + "ryzenai_corelib_selftest_dependencies"); + + CHECK(flm::test::CleanupCount() == 1); + CHECK(MatchingFiles( + temp.path(), + "pending-corelib-fatal-").empty()); +} + +void TestMissingDeviceContextCleansUpAndRemovesPending() { + TempDirectory temp; + flm::test::ResetFakeCorelib(); + flm::test::SetHasDeviceContext(false); + auto api = ResolveCompleteCorelib(); + + CheckThrowsContains( + [&] { + (void)CorelibRuntime::Create( + api, + MakeTestRecords(temp.path()), + [](unsigned int) { + throw std::runtime_error("unexpected termination"); + }); + }, + "device context"); + + CHECK(flm::test::CleanupCount() == 1); + CHECK(MatchingFiles( + temp.path(), + "pending-corelib-fatal-").empty()); +} + +void TestHealthyCleanupFollowsLastObjectRelease() { + TempDirectory temp; + flm::test::ResetFakeCorelib(); + auto api = ResolveCompleteCorelib(); + auto runtime = CorelibRuntime::Create( + api, + MakeTestRecords(temp.path()), + [](unsigned int) { + throw std::runtime_error("unexpected termination"); + }); + int storage = 0; + flm::corelib::UniqueTensor tensor(api, &storage); + + CheckThrowsContains( + [&] { + runtime->ShutdownHealthy(); + }, + "live corelib"); + CHECK(runtime->state() == ProcessState::Healthy); + CHECK(flm::test::CleanupCount() == 0); + + tensor.reset(); + runtime->ShutdownHealthy(); + + const std::vector expected{"release", "cleanup"}; + CHECK(flm::test::FakeCorelibEvents() == expected); + CHECK(flm::test::CleanupCount() == 1); +} + +struct TerminationIntercept final {}; + +void TestTerminationClosesAdmissionBeforeTerminator() { + TempDirectory temp; + flm::test::ResetFakeCorelib(); + auto api = ResolveCompleteCorelib(); + std::shared_ptr runtime; + bool terminator_called = false; + unsigned int termination_code = 0; + runtime = CorelibRuntime::Create( + api, + MakeTestRecords(temp.path()), + [&](unsigned int code) { + terminator_called = true; + termination_code = code; + CHECK(runtime->state() == ProcessState::Terminating); + CHECK(!runtime->admission_open()); + throw TerminationIntercept{}; + }); + const FailureContext failure{ + ryzenai_corelib_status_failure, + "ryzenai_corelib_matmul_bf16", + "dispatch failed", + "qkv", + 12, + 4, + 256}; + + try { + runtime->TerminateAfterFailure(failure); + } catch (const TerminationIntercept&) { + } + + CHECK(terminator_called); + CHECK(termination_code == 0xE0040001u); + CHECK(runtime->state() == ProcessState::Terminating); + CHECK(!runtime->admission_open()); + CHECK(flm::test::CleanupCount() == 0); + const auto records = MatchingFiles(temp.path(), "corelib-fatal-"); + CHECK(records.size() == 1); + const auto contents = ReadText(records.front()); + CHECK(contents.find("\"call\":\"ryzenai_corelib_matmul_bf16\"") != + std::string::npos); + CHECK(contents.find("\"phase\":\"qkv\"") != std::string::npos); + CHECK(contents.find("\"layer\":12") != std::string::npos); + CHECK(contents.find("\"rows\":4") != std::string::npos); + CHECK(contents.find("\"position\":256") != std::string::npos); +} + +void TestStepSubmissionStateCrossesIrrevocableBoundary() { + flm::corelib::StepSubmissionState submission; + CHECK(!submission.irrevocable()); + submission.MarkSuccessfulSubmit(); + CHECK(submission.irrevocable()); +} + +void TestGetOrCreateKeepsOneProcessRuntimeUntilExplicitShutdown() { + TempDirectory temp; + const auto fake_dll = + CurrentExecutablePath().parent_path() / "fake_ryzenai_corelib.dll"; + const auto cleanup_marker = temp.path() / "cleanup-marker.txt"; + CHECK(std::filesystem::exists(fake_dll)); + ScopedEnvironment corelib_path{ + L"RYZENAI_CORELIB_PATH", + fake_dll.wstring()}; + ScopedEnvironment marker_path{ + L"FLM_FAKE_CORELIB_CLEANUP_MARKER", + cleanup_marker.wstring()}; + + auto first = CorelibRuntime::GetOrCreate( + CurrentExecutablePath().parent_path()); + const auto first_runtime = first.get(); + const auto first_api = first->api().get(); + std::weak_ptr runtime_weak = first; + std::weak_ptr api_weak = first->api(); + first.reset(); + + CHECK(!runtime_weak.expired()); + auto second = CorelibRuntime::GetOrCreate( + CurrentExecutablePath().parent_path()); + CHECK(second.get() == first_runtime); + CHECK(second->api().get() == first_api); + CHECK(!std::filesystem::exists(cleanup_marker)); + second.reset(); + + CorelibRuntime::ShutdownProcess(); + + CHECK(runtime_weak.expired()); + CHECK(api_weak.expired()); + CHECK(ReadText(cleanup_marker) == "cleanup\n"); +} + +} // namespace + +int main() { + try { + TestPendingNamesUseStartTimeAndPid(); + TestPersistWritesCompleteUniqueRecords(); + TestDrainOrdersFinalRecordsAndRemovesAfterEmission(); + TestDrainPreservesLivePendingRecord(); + TestDrainPreservesPendingWhenProbeFails(); + TestDrainReportsAndRemovesStalePendingRecord(); + TestPrepareRejectsUnwritableRoot(); + TestUnusedPendingIsRemovedByDestructor(); + TestRuntimePublishesHealthyAndShutsDownOnce(); + TestHealthyShutdownClosesAdmissionBeforeWaiting(); + TestInitializationFailureCleansUpAndRemovesPending(); + TestMissingDeviceContextCleansUpAndRemovesPending(); + TestHealthyCleanupFollowsLastObjectRelease(); + TestTerminationClosesAdmissionBeforeTerminator(); + TestStepSubmissionStateCrossesIrrevocableBoundary(); + TestGetOrCreateKeepsOneProcessRuntimeUntilExplicitShutdown(); + std::cout << "test_corelib_fatal_record: PASS\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } catch (...) { + std::cerr << "unexpected non-standard exception\n"; + return 1; + } +} From 2344ec6490dde0e5b9a9b1f0833c8758b47c3b59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CChi?= Date: Mon, 31 Aug 2026 21:28:02 -0700 Subject: [PATCH 003/117] fix: harden corelib runtime foundation Co-authored-by: Cursor --- src/common/corelib/corelib_fatal_record.cpp | 129 ++++++++++---- src/common/corelib/corelib_runtime.cpp | 19 +- src/include/corelib/corelib_runtime.hpp | 7 + src/test/phi4_corelib_aie4/CMakeLists.txt | 1 + src/test/phi4_corelib_aie4/fake_corelib.cpp | 4 +- .../phi4_corelib_aie4/test_corelib_api.cpp | 51 ++++++ .../test_corelib_fatal_record.cpp | 168 ++++++++++++++++++ 7 files changed, 343 insertions(+), 36 deletions(-) diff --git a/src/common/corelib/corelib_fatal_record.cpp b/src/common/corelib/corelib_fatal_record.cpp index b62fbcad..767e8443 100644 --- a/src/common/corelib/corelib_fatal_record.cpp +++ b/src/common/corelib/corelib_fatal_record.cpp @@ -385,13 +385,40 @@ std::string ReadRecord(const std::filesystem::path& path) { return contents; } -void RemoveReportedRecord(const std::filesystem::path& path) { - std::error_code error; - const bool removed = std::filesystem::remove(path, error); - if (error || !removed) { - throw std::runtime_error( - "AIE4 fatal record removal failed for " + path.string() + - (error ? ": " + error.message() : "")); +void EmitDrainWarning( + std::ostream& output, + std::string_view operation, + const std::filesystem::path& path, + std::string_view detail) noexcept { + try { + output << "AIE4 fatal record warning: failed to " << operation + << ' ' << path.string(); + if (!detail.empty()) { + output << ": " << detail; + } + output << '\n'; + } catch (...) { + } +} + +void RemoveReportedRecord( + const std::filesystem::path& path, + std::ostream& output) noexcept { + try { + std::error_code error; + const bool removed = std::filesystem::remove(path, error); + if (!error && removed) { + return; + } + EmitDrainWarning( + output, + "remove", + path, + error ? error.message() : "record was not removed"); + } catch (const std::exception& exception) { + EmitDrainWarning(output, "remove", path, exception.what()); + } catch (...) { + EmitDrainWarning(output, "remove", path, "unknown error"); } } @@ -555,42 +582,61 @@ std::vector FatalRecordStore::DrainPriorRecords( std::error_code error; if (!std::filesystem::exists(root, error)) { if (error) { - throw std::filesystem::filesystem_error( - "failed to inspect AIE4 fatal record directory", + EmitDrainWarning( + output, + "inspect", root, - error); + error.message()); } return {}; } if (!std::filesystem::is_directory(root, error) || error) { - throw std::runtime_error( - "AIE4 fatal record root is not a readable directory: " + - root.string()); + EmitDrainWarning( + output, + "inspect", + root, + error ? error.message() : "not a directory"); + return {}; } std::vector final_paths; std::vector pending_paths; - for (std::filesystem::directory_iterator iterator(root, error), end; - !error && iterator != end; - iterator.increment(error)) { - const auto filename = iterator->path().filename().string(); - if (HasPrefixAndSuffix( - filename, - kFinalPrefix, - kFinalSuffix)) { - final_paths.push_back(iterator->path()); - } else if (HasPrefixAndSuffix( - filename, - kPendingPrefix, - kPendingSuffix)) { - pending_paths.push_back(iterator->path()); + try { + for (std::filesystem::directory_iterator iterator(root, error), end; + !error && iterator != end; + iterator.increment(error)) { + const auto filename = iterator->path().filename().string(); + if (HasPrefixAndSuffix( + filename, + kFinalPrefix, + kFinalSuffix)) { + final_paths.push_back(iterator->path()); + } else if (HasPrefixAndSuffix( + filename, + kPendingPrefix, + kPendingSuffix)) { + pending_paths.push_back(iterator->path()); + } } + } catch (const std::exception& exception) { + EmitDrainWarning( + output, + "enumerate", + root, + exception.what()); + } catch (...) { + EmitDrainWarning( + output, + "enumerate", + root, + "unknown error"); } if (error) { - throw std::filesystem::filesystem_error( - "failed to enumerate AIE4 fatal records", + EmitDrainWarning( + output, + "enumerate", root, - error); + error.message()); } std::sort(final_paths.begin(), final_paths.end()); std::sort(pending_paths.begin(), pending_paths.end()); @@ -598,10 +644,27 @@ std::vector FatalRecordStore::DrainPriorRecords( std::vector records; records.reserve(final_paths.size() + pending_paths.size()); for (const auto& path : final_paths) { - std::string record = ReadRecord(path); + std::string record; + try { + record = ReadRecord(path); + } catch (const std::exception& exception) { + EmitDrainWarning( + output, + "read", + path, + exception.what()); + continue; + } catch (...) { + EmitDrainWarning( + output, + "read", + path, + "unknown error"); + continue; + } EmitRecord(output, record); records.push_back(std::move(record)); - RemoveReportedRecord(path); + RemoveReportedRecord(path, output); } for (const auto& path : pending_paths) { @@ -626,7 +689,7 @@ std::vector FatalRecordStore::DrainPriorRecords( path.filename().string() + "\n"; EmitRecord(output, record); records.push_back(std::move(record)); - RemoveReportedRecord(path); + RemoveReportedRecord(path, output); } return records; } diff --git a/src/common/corelib/corelib_runtime.cpp b/src/common/corelib/corelib_runtime.cpp index 47f523a1..827de4c9 100644 --- a/src/common/corelib/corelib_runtime.cpp +++ b/src/common/corelib/corelib_runtime.cpp @@ -158,7 +158,17 @@ void CorelibRuntime::ShutdownHealthy() { "cannot clean up a terminating corelib process runtime"); } if (api_->live_object_count() != 0) { - state_.store(ProcessState::Healthy, std::memory_order_release); +#if defined(FLM_CORELIB_TESTING) + if (before_live_object_rollback_for_test_) { + before_live_object_rollback_for_test_(); + } +#endif + expected = ProcessState::Shutdown; + state_.compare_exchange_strong( + expected, + ProcessState::Healthy, + std::memory_order_acq_rel, + std::memory_order_acquire); throw std::logic_error( "cannot clean up corelib while live corelib objects remain"); } @@ -170,6 +180,13 @@ void CorelibRuntime::ShutdownHealthy() { records_.RemoveUnusedPending(); } +#if defined(FLM_CORELIB_TESTING) +void CorelibRuntime::SetBeforeLiveObjectRollbackForTest( + std::function hook) { + before_live_object_rollback_for_test_ = std::move(hook); +} +#endif + [[noreturn]] void CorelibRuntime::TerminateAfterFailure( const FailureContext& failure) { state_.store(ProcessState::Terminating, std::memory_order_release); diff --git a/src/include/corelib/corelib_runtime.hpp b/src/include/corelib/corelib_runtime.hpp index a5a09654..73dc30e2 100644 --- a/src/include/corelib/corelib_runtime.hpp +++ b/src/include/corelib/corelib_runtime.hpp @@ -53,6 +53,10 @@ class CorelibRuntime final { void ShutdownHealthy(); [[noreturn]] void TerminateAfterFailure( const FailureContext& failure); +#if defined(FLM_CORELIB_TESTING) + void SetBeforeLiveObjectRollbackForTest( + std::function hook); +#endif private: CorelibRuntime( @@ -67,6 +71,9 @@ class CorelibRuntime final { mutable std::mutex execution_mutex_; std::atomic state_{ProcessState::Shutdown}; bool cleanup_called_ = false; +#if defined(FLM_CORELIB_TESTING) + std::function before_live_object_rollback_for_test_; +#endif }; } // namespace flm::corelib diff --git a/src/test/phi4_corelib_aie4/CMakeLists.txt b/src/test/phi4_corelib_aie4/CMakeLists.txt index 70c3af13..1775eae7 100644 --- a/src/test/phi4_corelib_aie4/CMakeLists.txt +++ b/src/test/phi4_corelib_aie4/CMakeLists.txt @@ -30,6 +30,7 @@ target_link_directories(flm_corelib_aie4_testlib PUBLIC ${XRT_LIB_DIR}) target_compile_definitions(flm_corelib_aie4_testlib PUBLIC FLM_ENABLE_CORELIB_AIE4=1 + FLM_CORELIB_TESTING=1 DEV_BUILD=1 __WINDOWS__ USEAVX2=1 diff --git a/src/test/phi4_corelib_aie4/fake_corelib.cpp b/src/test/phi4_corelib_aie4/fake_corelib.cpp index cc430b1e..8db8ecec 100644 --- a/src/test/phi4_corelib_aie4/fake_corelib.cpp +++ b/src/test/phi4_corelib_aie4/fake_corelib.cpp @@ -150,7 +150,7 @@ ryzenai_corelib_status FakeMatmulWeightsGetData( *data = nullptr; } if (size != nullptr) { - *size = 0; + *size = 0x4D4D; } return ryzenai_corelib_status_success; } @@ -190,7 +190,7 @@ ryzenai_corelib_status FakeSsmlpWeightsGetData( *data = nullptr; } if (size != nullptr) { - *size = 0; + *size = 0x5353; } return ryzenai_corelib_status_success; } diff --git a/src/test/phi4_corelib_aie4/test_corelib_api.cpp b/src/test/phi4_corelib_aie4/test_corelib_api.cpp index 920451d2..e47d4353 100644 --- a/src/test/phi4_corelib_aie4/test_corelib_api.cpp +++ b/src/test/phi4_corelib_aie4/test_corelib_api.cpp @@ -161,6 +161,18 @@ void Touch(const std::filesystem::path& path) { } } +void CheckGetDataIdentities( + const CorelibApi& api, + const std::unordered_map& expected) { + const auto& functions = api.functions(); + CHECK(reinterpret_cast(functions.matmul_weights_get_data) == + expected.at( + "ryzenai_corelib_matmul_bf16_weights_get_data")); + CHECK(reinterpret_cast(functions.ssmlp_weights_get_data) == + expected.at( + "ryzenai_corelib_ssmlp_bf16_weights_get_data")); +} + void TestCompleteResolution() { auto resolver = flm::test::CompleteCorelibResolver(); CHECK(resolver.size() == kRequiredSymbols.size()); @@ -245,6 +257,44 @@ void TestCompleteResolution() { #undef CHECK_MEMBER_IDENTITY } +void TestTypeIdenticalGetDataSymbolsCannotBeSwapped() { + auto expected = flm::test::CompleteCorelibResolver(); + const auto matmul_name = + "ryzenai_corelib_matmul_bf16_weights_get_data"; + const auto ssmlp_name = + "ryzenai_corelib_ssmlp_bf16_weights_get_data"; + CHECK(expected.at(matmul_name) != expected.at(ssmlp_name)); + + auto api = CorelibApi::ResolveForTest( + [&expected](std::string_view name) -> void* { + return expected.at(std::string(name)); + }); + std::size_t matmul_size = 0; + std::size_t ssmlp_size = 0; + CHECK(api->functions().matmul_weights_get_data( + nullptr, + nullptr, + &matmul_size) == ryzenai_corelib_status_success); + CHECK(api->functions().ssmlp_weights_get_data( + nullptr, + nullptr, + &ssmlp_size) == ryzenai_corelib_status_success); + CHECK(matmul_size == 0x4D4D); + CHECK(ssmlp_size == 0x5353); + + auto swapped = expected; + std::swap(swapped.at(matmul_name), swapped.at(ssmlp_name)); + auto swapped_api = CorelibApi::ResolveForTest( + [&swapped](std::string_view name) -> void* { + return swapped.at(std::string(name)); + }); + CheckThrowsContains( + [&] { + CheckGetDataIdentities(*swapped_api, expected); + }, + "matmul_weights_get_data"); +} + void TestMissingSymbolFailsAtomically() { auto resolver = flm::test::CompleteCorelibResolver(); resolver.erase("ryzenai_corelib_flat_mha_bf16"); @@ -457,6 +507,7 @@ static_assert( int main() { try { TestCompleteResolution(); + TestTypeIdenticalGetDataSymbolsCannotBeSwapped(); TestMissingSymbolFailsAtomically(); TestErrorDetailSurvivesStatusConversion(); TestSuccessfulStatusDoesNotReadErrorState(); diff --git a/src/test/phi4_corelib_aie4/test_corelib_fatal_record.cpp b/src/test/phi4_corelib_aie4/test_corelib_fatal_record.cpp index db2a8a4a..315f4187 100644 --- a/src/test/phi4_corelib_aie4/test_corelib_fatal_record.cpp +++ b/src/test/phi4_corelib_aie4/test_corelib_fatal_record.cpp @@ -60,6 +60,37 @@ class TempDirectory final { std::filesystem::path path_; }; +class ScopedHandle final { +public: + ScopedHandle( + const std::filesystem::path& path, + DWORD share_mode) + : handle_(CreateFileW( + path.c_str(), + GENERIC_READ, + share_mode, + nullptr, + OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, + nullptr)) { + if (handle_ == INVALID_HANDLE_VALUE) { + throw std::runtime_error( + "failed to lock test record (error " + + std::to_string(GetLastError()) + ")"); + } + } + + ~ScopedHandle() noexcept { + CloseHandle(handle_); + } + + ScopedHandle(const ScopedHandle&) = delete; + ScopedHandle& operator=(const ScopedHandle&) = delete; + +private: + HANDLE handle_; +}; + std::optional ReadEnvironment(const wchar_t* name) { std::size_t required = 0; if (_wgetenv_s(&required, nullptr, 0, name) != 0) { @@ -316,6 +347,62 @@ void TestDrainOrdersFinalRecordsAndRemovesAfterEmission() { CHECK(!std::filesystem::exists(later)); } +void TestDrainContinuesPastUnreadableAndUnremovableRecords() { + TempDirectory temp; + const auto unreadable = + temp.path() / "corelib-fatal-20260831T1700000000000Z-1001.json"; + const auto unremovable = + temp.path() / "corelib-fatal-20260831T1800000000000Z-1002.json"; + const auto successful = + temp.path() / "corelib-fatal-20260831T1900000000000Z-1003.json"; + WriteText(unreadable, "unreadable\n"); + WriteText(unremovable, "unremovable\n"); + WriteText(successful, "successful\n"); + ScopedHandle deny_read(unreadable, 0); + ScopedHandle deny_delete( + unremovable, + FILE_SHARE_READ | FILE_SHARE_WRITE); + std::ostringstream output; + + const auto records = FatalRecordStore::DrainPriorRecords( + temp.path(), + {}, + output); + + const std::vector expected{ + "unremovable\n", + "successful\n"}; + CHECK(records == expected); + CHECK(output.str().find("warning") != std::string::npos); + CHECK(output.str().find("read") != std::string::npos); + CHECK(output.str().find(unreadable.filename().string()) != + std::string::npos); + CHECK(output.str().find("remove") != std::string::npos); + CHECK(output.str().find(unremovable.filename().string()) != + std::string::npos); + CHECK(output.str().find("unremovable\n") != std::string::npos); + CHECK(output.str().find("successful\n") != std::string::npos); + CHECK(std::filesystem::exists(unreadable)); + CHECK(std::filesystem::exists(unremovable)); + CHECK(!std::filesystem::exists(successful)); +} + +void TestDrainWarnsAndReturnsForInvalidRoot() { + TempDirectory temp; + const auto root_file = temp.path() / "not-a-directory"; + WriteText(root_file, "not a directory"); + std::ostringstream output; + + const auto records = FatalRecordStore::DrainPriorRecords( + root_file, + {}, + output); + + CHECK(records.empty()); + CHECK(output.str().find("warning") != std::string::npos); + CHECK(output.str().find(root_file.string()) != std::string::npos); +} + void TestDrainPreservesLivePendingRecord() { TempDirectory temp; const auto pending = @@ -588,6 +675,84 @@ void TestHealthyCleanupFollowsLastObjectRelease() { struct TerminationIntercept final {}; +void TestFailedShutdownRollbackPreservesConcurrentTermination() { + TempDirectory temp; + flm::test::ResetFakeCorelib(); + auto api = ResolveCompleteCorelib(); + auto runtime = CorelibRuntime::Create( + api, + MakeTestRecords(temp.path()), + [](unsigned int) { + throw TerminationIntercept{}; + }); + int storage = 0; + flm::corelib::UniqueTensor tensor(api, &storage); + std::atomic rollback_reached = false; + std::atomic release_rollback = false; + runtime->SetBeforeLiveObjectRollbackForTest([&] { + rollback_reached.store(true, std::memory_order_release); + while (!release_rollback.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + }); + + std::exception_ptr shutdown_error; + std::thread shutdown([&] { + try { + runtime->ShutdownHealthy(); + } catch (...) { + shutdown_error = std::current_exception(); + } + }); + const auto rollback_deadline = + std::chrono::steady_clock::now() + std::chrono::seconds{1}; + while (!rollback_reached.load(std::memory_order_acquire) && + std::chrono::steady_clock::now() < rollback_deadline) { + std::this_thread::yield(); + } + const bool reached_rollback = + rollback_reached.load(std::memory_order_acquire); + if (!reached_rollback) { + release_rollback.store(true, std::memory_order_release); + shutdown.join(); + CHECK(reached_rollback); + } + + const FailureContext failure{ + ryzenai_corelib_status_failure, + "ryzenai_corelib_matmul_bf16", + "concurrent terminal failure", + "qkv", + 12, + 4, + 256}; + std::exception_ptr termination_error; + std::thread termination([&] { + try { + runtime->TerminateAfterFailure(failure); + } catch (const TerminationIntercept&) { + } catch (...) { + termination_error = std::current_exception(); + } + }); + termination.join(); + CHECK(!termination_error); + CHECK(runtime->state() == ProcessState::Terminating); + + release_rollback.store(true, std::memory_order_release); + shutdown.join(); + + CHECK(shutdown_error); + CheckThrowsContains( + [&] { + std::rethrow_exception(shutdown_error); + }, + "live corelib"); + CHECK(runtime->state() == ProcessState::Terminating); + CHECK(!runtime->admission_open()); + tensor.reset(); +} + void TestTerminationClosesAdmissionBeforeTerminator() { TempDirectory temp; flm::test::ResetFakeCorelib(); @@ -685,6 +850,8 @@ int main() { TestPendingNamesUseStartTimeAndPid(); TestPersistWritesCompleteUniqueRecords(); TestDrainOrdersFinalRecordsAndRemovesAfterEmission(); + TestDrainContinuesPastUnreadableAndUnremovableRecords(); + TestDrainWarnsAndReturnsForInvalidRoot(); TestDrainPreservesLivePendingRecord(); TestDrainPreservesPendingWhenProbeFails(); TestDrainReportsAndRemovesStalePendingRecord(); @@ -695,6 +862,7 @@ int main() { TestInitializationFailureCleansUpAndRemovesPending(); TestMissingDeviceContextCleansUpAndRemovesPending(); TestHealthyCleanupFollowsLastObjectRelease(); + TestFailedShutdownRollbackPreservesConcurrentTermination(); TestTerminationClosesAdmissionBeforeTerminator(); TestStepSubmissionStateCrossesIrrevocableBoundary(); TestGetOrCreateKeepsOneProcessRuntimeUntilExplicitShutdown(); From 91c32bfb459a81c12fa7b6f804fe968f51c7ed12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CChi?= Date: Mon, 31 Aug 2026 21:46:21 -0700 Subject: [PATCH 004/117] feat: add Phi-4 ONNX manifest tooling Co-authored-by: Cursor --- tools/__init__.py | 1 + tools/generate_phi4_corelib_manifest.py | 627 ++++++++++++++++ tools/requirements-phi4-corelib.txt | 3 + tools/tests/__init__.py | 1 + .../test_generate_phi4_corelib_manifest.py | 672 ++++++++++++++++++ 5 files changed, 1304 insertions(+) create mode 100644 tools/__init__.py create mode 100644 tools/generate_phi4_corelib_manifest.py create mode 100644 tools/requirements-phi4-corelib.txt create mode 100644 tools/tests/__init__.py create mode 100644 tools/tests/test_generate_phi4_corelib_manifest.py diff --git a/tools/__init__.py b/tools/__init__.py new file mode 100644 index 00000000..1ddde4c7 --- /dev/null +++ b/tools/__init__.py @@ -0,0 +1 @@ +"""Offline FastFlowLM tooling.""" diff --git a/tools/generate_phi4_corelib_manifest.py b/tools/generate_phi4_corelib_manifest.py new file mode 100644 index 00000000..99664cf2 --- /dev/null +++ b/tools/generate_phi4_corelib_manifest.py @@ -0,0 +1,627 @@ +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path, PurePosixPath, PureWindowsPath + +import onnx +from onnx import TensorProto + + +SCHEMA_VERSION = 1 +MAX_U64 = (1 << 64) - 1 +EMBEDDED_INITIALIZERS_FILE = "corelib_embedded_initializers.bin" + +MODEL_IDENTITY: dict[str, object] = { + "family": "phi4", + "layers": 32, + "hidden_size": 3072, + "intermediate_size": 8192, + "num_heads": 24, + "kv_heads": 8, + "head_size": 128, + "vocab_size": 200064, + "group_size": 128, + "rope_dim": 96, + "rms_epsilon": 0.00001, +} + +_DTYPE_INFO = { + TensorProto.UINT8: ("uint8", 1), + TensorProto.FLOAT16: ("float16", 2), + TensorProto.FLOAT: ("float32", 4), + TensorProto.INT64: ("int64", 8), +} + + +def _positive_integer(value: int, field: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value <= 0: + raise ValueError(f"{field} must be a positive integer") + return value + + +def expected_matmul_components( + k: int, + n: int, + group: int, +) -> dict[str, tuple[object, list[int]]]: + """Return the exact ONNX MatMulNBits component contracts.""" + k = _positive_integer(k, "k") + n = _positive_integer(n, "n") + group = _positive_integer(group, "group") + if k % 2 != 0: + raise ValueError("k must be even for nibble-packed qweight") + if k % group != 0: + raise ValueError("k must be divisible by group") + + groups = k // group + return { + "qweight": ("uint8", [n, k // 2]), + "scales": ({"float16", "float32"}, [n, groups]), + "qzeros": ("uint8", [n, (groups + 1) // 2]), + } + + +def _component_role( + role: str, + weight_object: str, + dtype: object, + shape: list[int], +) -> dict[str, object]: + dtypes = {dtype} if isinstance(dtype, str) else set(dtype) + return { + "role": role, + "weight_object": weight_object, + "dtypes": dtypes, + "shape": list(shape), + } + + +def matmul_roles( + prefix: str, + k: int, + n: int, + group: int, +) -> dict[str, dict[str, object]]: + """Return fully qualified initializer contracts for one MatMul object.""" + if not prefix: + raise ValueError("MatMul initializer prefix must not be empty") + expected = expected_matmul_components(k, n, group) + return { + f"{prefix}.{component}": _component_role( + f"matmul.{component}", + prefix, + dtype, + shape, + ) + for component, (dtype, shape) in expected.items() + } + + +def ssmlp_roles(layer: int) -> dict[str, dict[str, object]]: + """Return exact projection and norm contracts for one fused SSMLP.""" + if isinstance(layer, bool) or not isinstance(layer, int): + raise ValueError("layer must be an integer") + if layer < 0 or layer >= 32: + raise ValueError("layer must be in [0, 31]") + + base = f"model.layers.{layer}" + weight_object = f"{base}.ssmlp" + roles: dict[str, dict[str, object]] = {} + for projection, k, n in ( + ("gate", 3072, 8192), + ("up", 3072, 8192), + ("down", 8192, 3072), + ): + prefix = f"{base}.mlp.{projection}_proj.MatMulNBits" + for component, (dtype, shape) in expected_matmul_components( + k, + n, + 128, + ).items(): + roles[f"{prefix}.{component}"] = _component_role( + f"ssmlp.{projection}.{component}", + weight_object, + dtype, + shape, + ) + + norm_dtypes = {"float16", "float32"} + roles[f"{base}.post_attention_layernorm.weight"] = { + "role": "ssmlp.norm0", + "weight_object": weight_object, + "dtypes": set(norm_dtypes), + "shape": [3072], + } + next_norm = ( + "model.layers.32.final_norm_layernorm.weight" + if layer == 31 + else f"model.layers.{layer + 1}.input_layernorm.weight" + ) + roles[next_norm] = { + "role": "ssmlp.norm1", + "weight_object": weight_object, + "dtypes": set(norm_dtypes), + "shape": [3072], + } + return roles + + +def host_role(role: str) -> dict[str, object]: + """Return the accepted source contract for one host tensor.""" + if role == "embedding": + return { + "role": role, + "dtypes": {"float16"}, + "shape": [200064, 3072], + } + if role == "input_norm": + return { + "role": role, + "dtypes": {"float16", "float32"}, + "shape": [3072], + } + if role in {"cos_cache", "sin_cache"}: + return { + "role": role, + "dtypes": {"float16", "float32"}, + "rank": 2, + "minimum_shape": [4096, 48], + } + raise ValueError(f"unknown host tensor role: {role}") + + +def _merge_roles( + destination: dict[str, dict[str, object]], + additions: dict[str, dict[str, object]], +) -> None: + duplicates = sorted(destination.keys() & additions.keys()) + if duplicates: + raise RuntimeError( + "initializer role generated more than once: " + duplicates[0] + ) + destination.update(additions) + + +def required_initializer_roles() -> dict[str, dict[str, object]]: + """Return all 743 required source initializers in the driver name map.""" + roles: dict[str, dict[str, object]] = {} + for layer in range(32): + base = f"model.layers.{layer}" + # The driver maps o_proj from Q_DIM to HIDDEN. Both are 3072 for + # Phi-4-mini, but keeping both logical dimensions explicit prevents a + # future name-only inference from changing the source contract. + for projection, k, n in ( + ("attn.q_proj", 3072, 3072), + ("attn.k_proj", 3072, 1024), + ("attn.v_proj", 3072, 1024), + ("attn.o_proj", 3072, 3072), + ): + prefix = f"{base}.{projection}.MatMulNBits" + _merge_roles(roles, matmul_roles(prefix, k, n, 128)) + _merge_roles(roles, ssmlp_roles(layer)) + + _merge_roles( + roles, + matmul_roles("lm_head.MatMulNBits", 3072, 200064, 128), + ) + _merge_roles( + roles, + { + "model.embed_tokens.weight": host_role("embedding"), + "model.layers.0.input_layernorm.weight": host_role("input_norm"), + "cos_cache": host_role("cos_cache"), + "sin_cache": host_role("sin_cache"), + }, + ) + + weight_objects = { + record["weight_object"] + for record in roles.values() + if "weight_object" in record + } + if len(roles) != 743 or len(weight_objects) != 161: + raise RuntimeError( + "internal Phi-4 role map mismatch: " + f"{len(roles)} initializers, {len(weight_objects)} weight objects" + ) + return roles + + +def _parse_u64(value: str, field: str, initializer: str) -> int: + if not value or not value.isdecimal(): + raise ValueError( + f"{initializer}: external {field} must be an unsigned decimal" + ) + parsed = int(value) + if parsed > MAX_U64: + raise ValueError(f"{initializer}: external {field} exceeds uint64") + return parsed + + +def _safe_location(location: str, initializer: str) -> str: + if not location or "\x00" in location: + raise ValueError(f"{initializer}: invalid external location") + + windows = PureWindowsPath(location) + posix = PurePosixPath(location.replace("\\", "/")) + if ( + windows.is_absolute() + or bool(windows.drive) + or bool(windows.root) + or posix.is_absolute() + or bool(posix.root) + ): + raise ValueError( + f"{initializer}: external location must be a relative path" + ) + + parts = tuple( + part + for part in posix.parts + if part not in {"", "."} + ) + if not parts or ".." in parts: + raise ValueError( + f"{initializer}: external location contains path traversal" + ) + return PurePosixPath(*parts).as_posix() + + +def _external_source( + tensor: TensorProto, + initializer: str, + model_dir: Path, +) -> tuple[str, Path, int, int]: + metadata: dict[str, str] = {} + for item in tensor.external_data: + if item.key in metadata: + raise ValueError( + f"{initializer}: duplicate external metadata key {item.key}" + ) + metadata[item.key] = item.value + + if "location" not in metadata: + raise ValueError(f"{initializer}: missing external location") + if "length" not in metadata: + raise ValueError(f"{initializer}: missing external length") + + location = _safe_location(metadata["location"], initializer) + offset = _parse_u64(metadata.get("offset", "0"), "offset", initializer) + length = _parse_u64(metadata["length"], "length", initializer) + if length == 0: + raise ValueError(f"{initializer}: external length must be positive") + if offset > MAX_U64 - length: + raise ValueError(f"{initializer}: external range overflow") + + source = model_dir.joinpath(*PurePosixPath(location).parts) + try: + resolved = source.resolve(strict=True) + except FileNotFoundError as error: + raise ValueError( + f"{initializer}: external file does not exist: {location}" + ) from error + if not resolved.is_relative_to(model_dir): + raise ValueError( + f"{initializer}: external location escapes the model directory" + ) + if not resolved.is_file(): + raise ValueError( + f"{initializer}: external location is not a file: {location}" + ) + return location, resolved, offset, length + + +def _dtype_and_item_size( + tensor: TensorProto, + initializer: str, +) -> tuple[str, int]: + try: + return _DTYPE_INFO[tensor.data_type] + except KeyError as error: + type_name = TensorProto.DataType.Name(tensor.data_type) + raise ValueError( + f"{initializer}: unsupported ONNX dtype {type_name}" + ) from error + + +def _checked_byte_count( + shape: list[int], + item_size: int, + initializer: str, +) -> int: + elements = 1 + for dimension in shape: + if dimension <= 0: + raise ValueError( + f"{initializer}: shape dimensions must be positive" + ) + if elements > MAX_U64 // dimension: + raise ValueError(f"{initializer}: shape element count overflow") + elements *= dimension + if elements > MAX_U64 // item_size: + raise ValueError(f"{initializer}: tensor byte count overflow") + return elements * item_size + + +def _validate_contract( + initializer: str, + tensor: TensorProto, + contract: dict[str, object], +) -> tuple[str, int, list[int], int]: + dtype, item_size = _dtype_and_item_size(tensor, initializer) + accepted_dtypes = contract.get("dtypes") + if not isinstance(accepted_dtypes, set) or dtype not in accepted_dtypes: + expected = ", ".join(sorted(accepted_dtypes or ())) + raise ValueError( + f"{initializer}: dtype {dtype} does not match {expected}" + ) + + shape = [int(dimension) for dimension in tensor.dims] + if "shape" in contract: + expected_shape = contract["shape"] + if shape != expected_shape: + raise ValueError( + f"{initializer}: shape {shape} does not match " + f"{expected_shape}" + ) + else: + rank = contract.get("rank") + minimum_shape = contract.get("minimum_shape") + if len(shape) != rank: + raise ValueError( + f"{initializer}: shape rank {len(shape)} does not match {rank}" + ) + if ( + not isinstance(minimum_shape, list) + or len(minimum_shape) != len(shape) + or any( + actual < minimum + for actual, minimum in zip(shape, minimum_shape) + ) + ): + raise ValueError( + f"{initializer}: shape {shape} is smaller than " + f"{minimum_shape}" + ) + + byte_count = _checked_byte_count(shape, item_size, initializer) + return dtype, item_size, shape, byte_count + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + while chunk := stream.read(1024 * 1024): + digest.update(chunk) + return digest.hexdigest() + + +def _file_record(path: Path, full_hash: bool) -> dict[str, object]: + size = path.stat().st_size + if size < 0 or size > MAX_U64: + raise ValueError(f"file size exceeds uint64: {path}") + record: dict[str, object] = {"size": size} + if full_hash: + record["sha256"] = _sha256(path) + return record + + +def _load_initializers(model_path: Path) -> dict[str, TensorProto]: + try: + model = onnx.load(str(model_path), load_external_data=False) + except Exception as error: + raise ValueError(f"failed to parse ONNX model {model_path}") from error + + initializers: dict[str, TensorProto] = {} + for tensor in model.graph.initializer: + if tensor.name in initializers: + raise ValueError(f"duplicate initializer: {tensor.name}") + initializers[tensor.name] = tensor + return initializers + + +def _initializer_record( + contract: dict[str, object], + *, + dtype: str, + shape: list[int], + file: str, + offset: int, + length: int, +) -> dict[str, object]: + record: dict[str, object] = { + "file": file, + "offset": offset, + "length": length, + "dtype": dtype, + "shape": shape, + "role": contract["role"], + } + return record + + +def _generate_manifest( + model_dir: Path, + output: Path, + full_hash: bool, + roles: dict[str, dict[str, object]], +) -> dict[str, object]: + """Generate a manifest using an explicit role map. + + The public generator always supplies ``required_initializer_roles()``. + The explicit map keeps small synthetic unit models practical without + allocating the accepted model's multi-gigabyte tensors. + """ + model_dir = Path(model_dir).resolve(strict=True) + output = Path(output) + if not model_dir.is_dir(): + raise ValueError(f"model directory is not a directory: {model_dir}") + if not isinstance(full_hash, bool): + raise ValueError("full_hash must be a boolean") + + model_path = model_dir / "model.onnx" + if not model_path.is_file(): + raise ValueError(f"model.onnx does not exist in {model_dir}") + if not output.parent.exists() or not output.parent.is_dir(): + raise ValueError(f"output parent directory does not exist: {output.parent}") + + initializers = _load_initializers(model_path) + missing = sorted(set(roles) - set(initializers)) + if missing: + preview = ", ".join(missing[:8]) + if len(missing) > 8: + preview += f", ... ({len(missing)} total)" + raise ValueError(f"missing initializer(s): {preview}") + + records: dict[str, dict[str, object]] = {} + external_files: dict[str, Path] = {} + embedded: list[tuple[str, bytes]] = [] + embedded_offset = 0 + + for name in sorted(roles): + tensor = initializers[name] + contract = roles[name] + dtype, item_size, shape, byte_count = _validate_contract( + name, + tensor, + contract, + ) + is_external = ( + tensor.data_location == TensorProto.EXTERNAL + or bool(tensor.external_data) + ) + if is_external: + location, path, offset, length = _external_source( + tensor, + name, + model_dir, + ) + if length != byte_count: + raise ValueError( + f"{name}: external byte count {length} does not match " + f"dtype/shape byte count {byte_count}" + ) + if offset % item_size != 0: + raise ValueError( + f"{name}: external offset is not dtype-aligned" + ) + size = path.stat().st_size + if size > MAX_U64: + raise ValueError(f"{name}: external file size exceeds uint64") + if offset > size or length > size - offset: + raise ValueError( + f"{name}: external range exceeds file size" + ) + previous = external_files.setdefault(location, path) + if previous != path: + raise ValueError( + f"{name}: external location resolves inconsistently" + ) + records[name] = _initializer_record( + contract, + dtype=dtype, + shape=shape, + file=location, + offset=offset, + length=length, + ) + continue + + raw_data = bytes(tensor.raw_data) + if not raw_data: + raise ValueError( + f"{name}: embedded initializer must use raw_data" + ) + if len(raw_data) != byte_count: + raise ValueError( + f"{name}: embedded byte count {len(raw_data)} does not match " + f"dtype/shape byte count {byte_count}" + ) + if embedded_offset > MAX_U64 - byte_count: + raise ValueError("embedded initializer sidecar offset overflow") + records[name] = _initializer_record( + contract, + dtype=dtype, + shape=shape, + file=EMBEDDED_INITIALIZERS_FILE, + offset=embedded_offset, + length=byte_count, + ) + embedded.append((name, raw_data)) + embedded_offset += byte_count + + sidecar_path = model_dir / EMBEDDED_INITIALIZERS_FILE + if embedded and EMBEDDED_INITIALIZERS_FILE in external_files: + raise ValueError( + "embedded initializer sidecar conflicts with an external data file" + ) + + protected_paths = {model_path.resolve()} + protected_paths.update(external_files.values()) + if embedded: + protected_paths.add(sidecar_path.resolve(strict=False)) + output_resolved = output.resolve(strict=False) + if output_resolved in protected_paths: + raise ValueError("output path would overwrite ONNX initializer data") + + if embedded: + with sidecar_path.open("wb") as stream: + for _, raw_data in embedded: + stream.write(raw_data) + + files: dict[str, dict[str, object]] = { + "model.onnx": _file_record(model_path, full_hash) + } + for location in sorted(external_files): + files[location] = _file_record(external_files[location], full_hash) + if embedded: + files[EMBEDDED_INITIALIZERS_FILE] = _file_record( + sidecar_path, + full_hash, + ) + + manifest: dict[str, object] = { + "schema_version": SCHEMA_VERSION, + "execution_backend": "corelib_aie4", + "model": dict(MODEL_IDENTITY), + "backend": {"max_seq": 4096}, + "files": files, + "initializers": records, + } + serialized = json.dumps( + manifest, + ensure_ascii=False, + indent=2, + sort_keys=True, + ) + output.write_text(serialized + "\n", encoding="utf-8", newline="\n") + return manifest + + +def generate_manifest( + model_dir: Path, + output: Path, + full_hash: bool, +) -> dict[str, object]: + return _generate_manifest( + model_dir, + output, + full_hash, + required_initializer_roles(), + ) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--model-dir", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--full-hash", action="store_true") + args = parser.parse_args() + generate_manifest(args.model_dir, args.output, args.full_hash) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/requirements-phi4-corelib.txt b/tools/requirements-phi4-corelib.txt new file mode 100644 index 00000000..05e2a42b --- /dev/null +++ b/tools/requirements-phi4-corelib.txt @@ -0,0 +1,3 @@ +numpy +onnx +tokenizers diff --git a/tools/tests/__init__.py b/tools/tests/__init__.py new file mode 100644 index 00000000..73c719f2 --- /dev/null +++ b/tools/tests/__init__.py @@ -0,0 +1 @@ +"""Tests for offline FastFlowLM tooling.""" diff --git a/tools/tests/test_generate_phi4_corelib_manifest.py b/tools/tests/test_generate_phi4_corelib_manifest.py new file mode 100644 index 00000000..ac6c8cd5 --- /dev/null +++ b/tools/tests/test_generate_phi4_corelib_manifest.py @@ -0,0 +1,672 @@ +from __future__ import annotations + +import hashlib +import json +import tempfile +import unittest +from pathlib import Path + +import onnx +from onnx import TensorProto, helper + +from tools.generate_phi4_corelib_manifest import ( + MAX_U64, + _generate_manifest, + expected_matmul_components, + host_role, + matmul_roles, + required_initializer_roles, + ssmlp_roles, +) + + +class ManifestGeneratorTests(unittest.TestCase): + def _write_model( + self, + root: Path, + initializers: list[TensorProto], + ) -> Path: + graph = helper.make_graph( + nodes=[], + name="synthetic-phi4", + inputs=[], + outputs=[], + initializer=initializers, + ) + model = helper.make_model(graph) + path = root / "model.onnx" + path.write_bytes(model.SerializeToString()) + return path + + def _embedded( + self, + name: str, + data_type: int, + shape: list[int], + raw_data: bytes, + ) -> TensorProto: + tensor = TensorProto() + tensor.name = name + tensor.data_type = data_type + tensor.dims.extend(shape) + tensor.raw_data = raw_data + return tensor + + def _external( + self, + name: str, + data_type: int, + shape: list[int], + metadata: list[tuple[str, str]], + ) -> TensorProto: + tensor = TensorProto() + tensor.name = name + tensor.data_type = data_type + tensor.dims.extend(shape) + tensor.data_location = TensorProto.EXTERNAL + for key, value in metadata: + item = tensor.external_data.add() + item.key = key + item.value = value + return tensor + + def _exact_roles( + self, + name: str, + *, + dtypes: set[str] | None = None, + shape: list[int] | None = None, + ) -> dict[str, dict[str, object]]: + return { + name: { + "role": "test.tensor", + "dtypes": dtypes or {"uint8"}, + "shape": shape or [2, 2], + } + } + + def test_matmul_component_shapes(self): + expected = expected_matmul_components(k=3072, n=1024, group=128) + self.assertEqual(expected["qweight"], ("uint8", [1024, 1536])) + self.assertEqual( + expected["scales"], + ({"float16", "float32"}, [1024, 24]), + ) + self.assertEqual(expected["qzeros"], ("uint8", [1024, 12])) + + odd_group_count = expected_matmul_components( + k=384, + n=5, + group=128, + ) + self.assertEqual(odd_group_count["qzeros"], ("uint8", [5, 2])) + + def test_matmul_roles_use_exact_names_and_constraints(self): + prefix = "model.layers.7.attn.k_proj.MatMulNBits" + self.assertEqual( + matmul_roles(prefix, 3072, 1024, 128), + { + f"{prefix}.qweight": { + "role": "matmul.qweight", + "weight_object": prefix, + "dtypes": {"uint8"}, + "shape": [1024, 1536], + }, + f"{prefix}.scales": { + "role": "matmul.scales", + "weight_object": prefix, + "dtypes": {"float16", "float32"}, + "shape": [1024, 24], + }, + f"{prefix}.qzeros": { + "role": "matmul.qzeros", + "weight_object": prefix, + "dtypes": {"uint8"}, + "shape": [1024, 12], + }, + }, + ) + + def test_ssmlp_gate_up_down_and_norm_shapes(self): + roles = ssmlp_roles(0) + base = "model.layers.0" + group = f"{base}.ssmlp" + + self.assertEqual( + roles[f"{base}.mlp.gate_proj.MatMulNBits.qweight"], + { + "role": "ssmlp.gate.qweight", + "weight_object": group, + "dtypes": {"uint8"}, + "shape": [8192, 1536], + }, + ) + self.assertEqual( + roles[f"{base}.mlp.up_proj.MatMulNBits.scales"]["shape"], + [8192, 24], + ) + self.assertEqual( + roles[f"{base}.mlp.up_proj.MatMulNBits.qzeros"]["shape"], + [8192, 12], + ) + self.assertEqual( + roles[f"{base}.mlp.down_proj.MatMulNBits.qweight"]["shape"], + [3072, 4096], + ) + self.assertEqual( + roles[f"{base}.mlp.down_proj.MatMulNBits.scales"]["shape"], + [3072, 64], + ) + self.assertEqual( + roles[f"{base}.mlp.down_proj.MatMulNBits.qzeros"]["shape"], + [3072, 32], + ) + self.assertEqual( + roles[f"{base}.post_attention_layernorm.weight"], + { + "role": "ssmlp.norm0", + "weight_object": group, + "dtypes": {"float16", "float32"}, + "shape": [3072], + }, + ) + self.assertEqual( + roles["model.layers.1.input_layernorm.weight"]["role"], + "ssmlp.norm1", + ) + + def test_required_roles_cover_exact_32_layer_driver_map(self): + roles = required_initializer_roles() + self.assertEqual(len(roles), 743) + + groups = { + record["weight_object"] + for record in roles.values() + if "weight_object" in record + } + self.assertEqual(len(groups), 161) + expected_attention_groups = { + f"model.layers.{layer}.attn.{projection}.MatMulNBits" + for layer in range(32) + for projection in ("q_proj", "k_proj", "v_proj", "o_proj") + } + self.assertTrue(expected_attention_groups.issubset(groups)) + self.assertEqual( + { + group + for group in groups + if str(group).endswith(".ssmlp") + }, + {f"model.layers.{layer}.ssmlp" for layer in range(32)}, + ) + self.assertIn("lm_head.MatMulNBits", groups) + + o_prefix = "model.layers.31.attn.o_proj.MatMulNBits" + self.assertEqual( + roles[f"{o_prefix}.qweight"]["shape"], + [3072, 1536], + ) + self.assertNotIn( + "model.layers.32.attn.q_proj.MatMulNBits.qweight", + roles, + ) + + def test_layer_31_uses_phantom_layer_32_final_norm(self): + roles = ssmlp_roles(31) + final_norm = "model.layers.32.final_norm_layernorm.weight" + self.assertIn(final_norm, roles) + self.assertEqual(roles[final_norm]["role"], "ssmlp.norm1") + self.assertNotIn("model.layers.32.input_layernorm.weight", roles) + self.assertNotIn("model.norm.weight", required_initializer_roles()) + + def test_host_tensor_constraints(self): + self.assertEqual( + host_role("embedding"), + { + "role": "embedding", + "dtypes": {"float16"}, + "shape": [200064, 3072], + }, + ) + self.assertEqual( + host_role("input_norm"), + { + "role": "input_norm", + "dtypes": {"float16", "float32"}, + "shape": [3072], + }, + ) + for role in ("cos_cache", "sin_cache"): + self.assertEqual( + host_role(role), + { + "role": role, + "dtypes": {"float16", "float32"}, + "rank": 2, + "minimum_shape": [4096, 48], + }, + ) + + def test_external_initializer_metadata_and_hash(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + data_path = root / "weights" / "data.bin" + data_path.parent.mkdir() + data_path.write_bytes(b"HEADER" + bytes([1, 2, 3, 4]) + b"TAIL") + tensor = self._external( + "test.weight", + TensorProto.UINT8, + [2, 2], + [ + ("location", "weights/data.bin"), + ("offset", "6"), + ("length", "4"), + ], + ) + model_path = self._write_model(root, [tensor]) + output = root / "manifest.json" + + manifest = _generate_manifest( + root, + output, + True, + self._exact_roles("test.weight"), + ) + + self.assertEqual( + manifest["initializers"]["test.weight"], + { + "dtype": "uint8", + "file": "weights/data.bin", + "length": 4, + "offset": 6, + "role": "test.tensor", + "shape": [2, 2], + }, + ) + self.assertEqual( + set(manifest["files"]), + {"model.onnx", "weights/data.bin"}, + ) + self.assertEqual( + manifest["files"]["weights/data.bin"], + { + "size": data_path.stat().st_size, + "sha256": hashlib.sha256(data_path.read_bytes()).hexdigest(), + }, + ) + self.assertEqual( + manifest["files"]["model.onnx"]["sha256"], + hashlib.sha256(model_path.read_bytes()).hexdigest(), + ) + + def test_full_hash_flag_controls_sha256_metadata(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + tensor = self._embedded( + "test.weight", + TensorProto.UINT8, + [2, 2], + b"data", + ) + self._write_model(root, [tensor]) + + manifest = _generate_manifest( + root, + root / "manifest.json", + False, + self._exact_roles("test.weight"), + ) + + self.assertTrue(manifest["files"]) + for record in manifest["files"].values(): + self.assertNotIn("sha256", record) + + def test_initializer_schema_omits_validation_only_group_metadata(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + tensor = self._embedded( + "test.weight", + TensorProto.UINT8, + [2, 2], + b"data", + ) + self._write_model(root, [tensor]) + roles = self._exact_roles("test.weight") + roles["test.weight"]["weight_object"] = "test.matmul" + + manifest = _generate_manifest( + root, + root / "manifest.json", + False, + roles, + ) + + self.assertEqual( + manifest["initializers"]["test.weight"], + { + "dtype": "uint8", + "file": "corelib_embedded_initializers.bin", + "length": 4, + "offset": 0, + "role": "test.tensor", + "shape": [2, 2], + }, + ) + + def test_embedded_initializers_are_sorted_into_one_sidecar(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + z_tensor = self._embedded( + "z.tensor", + TensorProto.UINT8, + [3], + b"XYZ", + ) + a_tensor = self._embedded( + "a.tensor", + TensorProto.UINT8, + [2], + b"ab", + ) + self._write_model(root, [z_tensor, a_tensor]) + roles = { + "z.tensor": { + "role": "test.z", + "dtypes": {"uint8"}, + "shape": [3], + }, + "a.tensor": { + "role": "test.a", + "dtypes": {"uint8"}, + "shape": [2], + }, + } + + first = root / "first.json" + second = root / "second.json" + manifest = _generate_manifest(root, first, True, roles) + _generate_manifest(root, second, True, roles) + + sidecar = root / "corelib_embedded_initializers.bin" + self.assertEqual(sidecar.read_bytes(), b"abXYZ") + self.assertEqual( + manifest["initializers"]["a.tensor"]["offset"], + 0, + ) + self.assertEqual( + manifest["initializers"]["z.tensor"]["offset"], + 2, + ) + self.assertEqual( + manifest["files"]["corelib_embedded_initializers.bin"], + { + "size": 5, + "sha256": hashlib.sha256(b"abXYZ").hexdigest(), + }, + ) + self.assertEqual(first.read_bytes(), second.read_bytes()) + decoded = json.loads(first.read_text(encoding="utf-8")) + self.assertEqual( + list(decoded["initializers"]), + ["a.tensor", "z.tensor"], + ) + + def test_wider_and_longer_rope_source_is_recorded(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + rows = 4097 + columns = 64 + tensor = self._embedded( + "cos_cache", + TensorProto.FLOAT16, + [rows, columns], + bytes(rows * columns * 2), + ) + self._write_model(root, [tensor]) + + manifest = _generate_manifest( + root, + root / "manifest.json", + False, + {"cos_cache": host_role("cos_cache")}, + ) + + self.assertEqual( + manifest["initializers"]["cos_cache"]["shape"], + [4097, 64], + ) + self.assertEqual( + manifest["initializers"]["cos_cache"]["dtype"], + "float16", + ) + + def test_path_traversal_and_absolute_locations_are_rejected(self): + bad_locations = ( + "../escape.bin", + "safe/../../escape.bin", + r"safe\..\escape.bin", + "/absolute.bin", + "C:/absolute.bin", + r"\\server\share\absolute.bin", + ) + for location in bad_locations: + with self.subTest(location=location): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + tensor = self._external( + "test.weight", + TensorProto.UINT8, + [2, 2], + [ + ("location", location), + ("offset", "0"), + ("length", "4"), + ], + ) + self._write_model(root, [tensor]) + + with self.assertRaisesRegex(ValueError, "path|location"): + _generate_manifest( + root, + root / "manifest.json", + False, + self._exact_roles("test.weight"), + ) + + def test_missing_length_and_range_overflow_are_rejected(self): + cases = ( + ( + [ + ("location", "weights.bin"), + ("offset", "0"), + ], + "length", + ), + ( + [ + ("location", "weights.bin"), + ("offset", str(MAX_U64)), + ("length", "2"), + ], + "overflow", + ), + ) + for metadata, message in cases: + with self.subTest(message=message): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "weights.bin").write_bytes(b"data") + tensor = self._external( + "test.weight", + TensorProto.UINT8, + [2, 2], + metadata, + ) + self._write_model(root, [tensor]) + + with self.assertRaisesRegex(ValueError, message): + _generate_manifest( + root, + root / "manifest.json", + False, + self._exact_roles("test.weight"), + ) + + def test_wrong_dtype_rank_and_shape_are_rejected(self): + cases = ( + ( + TensorProto.FLOAT16, + [2, 2], + bytes(8), + "dtype", + ), + ( + TensorProto.UINT8, + [4], + bytes(4), + "shape", + ), + ( + TensorProto.UINT8, + [2, 3], + bytes(6), + "shape", + ), + ) + for data_type, shape, raw_data, message in cases: + with self.subTest(data_type=data_type, shape=shape): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + tensor = self._embedded( + "test.weight", + data_type, + shape, + raw_data, + ) + self._write_model(root, [tensor]) + + with self.assertRaisesRegex(ValueError, message): + _generate_manifest( + root, + root / "manifest.json", + False, + self._exact_roles("test.weight"), + ) + + def test_byte_count_mismatches_are_rejected(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + embedded = self._embedded( + "test.weight", + TensorProto.UINT8, + [2, 2], + b"abc", + ) + self._write_model(root, [embedded]) + with self.assertRaisesRegex(ValueError, "byte count"): + _generate_manifest( + root, + root / "embedded.json", + False, + self._exact_roles("test.weight"), + ) + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "weights.bin").write_bytes(b"abc") + external = self._external( + "test.weight", + TensorProto.UINT8, + [2, 2], + [ + ("location", "weights.bin"), + ("offset", "0"), + ("length", "3"), + ], + ) + self._write_model(root, [external]) + with self.assertRaisesRegex(ValueError, "byte count"): + _generate_manifest( + root, + root / "external.json", + False, + self._exact_roles("test.weight"), + ) + + def test_missing_and_duplicate_required_initializers_are_rejected(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self._write_model(root, []) + with self.assertRaisesRegex(ValueError, "missing initializer"): + _generate_manifest( + root, + root / "missing.json", + False, + self._exact_roles("test.weight"), + ) + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + first = self._embedded( + "test.weight", + TensorProto.UINT8, + [2, 2], + b"abcd", + ) + second = self._embedded( + "test.weight", + TensorProto.UINT8, + [2, 2], + b"efgh", + ) + self._write_model(root, [first, second]) + with self.assertRaisesRegex(ValueError, "duplicate initializer"): + _generate_manifest( + root, + root / "duplicate.json", + False, + self._exact_roles("test.weight"), + ) + + def test_manifest_has_locked_schema_and_model_identity(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + tensor = self._embedded( + "test.weight", + TensorProto.UINT8, + [2, 2], + b"abcd", + ) + self._write_model(root, [tensor]) + + manifest = _generate_manifest( + root, + root / "manifest.json", + False, + self._exact_roles("test.weight"), + ) + + self.assertEqual(manifest["schema_version"], 1) + self.assertEqual(manifest["execution_backend"], "corelib_aie4") + self.assertEqual( + manifest["model"], + { + "family": "phi4", + "group_size": 128, + "head_size": 128, + "hidden_size": 3072, + "intermediate_size": 8192, + "kv_heads": 8, + "layers": 32, + "num_heads": 24, + "rms_epsilon": 0.00001, + "rope_dim": 96, + "vocab_size": 200064, + }, + ) + self.assertEqual(manifest["backend"], {"max_seq": 4096}) + + +if __name__ == "__main__": + unittest.main() From aefe2a9c2ffce6d190eecb12e71919f6a35d5cae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CChi?= Date: Mon, 31 Aug 2026 22:00:35 -0700 Subject: [PATCH 005/117] fix: complete Phi-4 manifest verification Co-authored-by: Cursor --- tools/generate_phi4_corelib_manifest.py | 179 ++++++- .../test_generate_phi4_corelib_manifest.py | 486 ++++++++++++++++++ 2 files changed, 658 insertions(+), 7 deletions(-) diff --git a/tools/generate_phi4_corelib_manifest.py b/tools/generate_phi4_corelib_manifest.py index 99664cf2..f2431d18 100644 --- a/tools/generate_phi4_corelib_manifest.py +++ b/tools/generate_phi4_corelib_manifest.py @@ -27,6 +27,13 @@ "rms_epsilon": 0.00001, } +_ATTENTION_PROJECTIONS = ( + ("q_proj", 3072, 3072), + ("k_proj", 3072, 1024), + ("v_proj", 3072, 1024), + ("o_proj", 3072, 3072), +) + _DTYPE_INFO = { TensorProto.UINT8: ("uint8", 1), TensorProto.FLOAT16: ("float16", 2), @@ -192,13 +199,8 @@ def required_initializer_roles() -> dict[str, dict[str, object]]: # The driver maps o_proj from Q_DIM to HIDDEN. Both are 3072 for # Phi-4-mini, but keeping both logical dimensions explicit prevents a # future name-only inference from changing the source contract. - for projection, k, n in ( - ("attn.q_proj", 3072, 3072), - ("attn.k_proj", 3072, 1024), - ("attn.v_proj", 3072, 1024), - ("attn.o_proj", 3072, 3072), - ): - prefix = f"{base}.{projection}.MatMulNBits" + for projection, k, n in _ATTENTION_PROJECTIONS: + prefix = f"{base}.attn.{projection}.MatMulNBits" _merge_roles(roles, matmul_roles(prefix, k, n, 128)) _merge_roles(roles, ssmlp_roles(layer)) @@ -229,6 +231,95 @@ def required_initializer_roles() -> dict[str, dict[str, object]]: return roles +def _matmul_weight_object( + prefix: str, + k: int, + n: int, + group_size: int, +) -> dict[str, object]: + return { + "name": prefix, + "kind": "matmul", + "descriptor": { + "k": k, + "n": n, + "group_size": group_size, + "has_bias": False, + }, + "roles": { + component: f"{prefix}.{component}" + for component in ("qweight", "scales", "qzeros") + }, + } + + +def _ssmlp_weight_object(layer: int) -> dict[str, object]: + base = f"model.layers.{layer}" + next_norm = ( + "model.layers.32.final_norm_layernorm.weight" + if layer == 31 + else f"model.layers.{layer + 1}.input_layernorm.weight" + ) + roles = { + "norm0": f"{base}.post_attention_layernorm.weight", + "norm1": next_norm, + } + for projection in ("gate", "up", "down"): + prefix = f"{base}.mlp.{projection}_proj.MatMulNBits" + for component in ("qweight", "scales", "qzeros"): + roles[f"{projection}_{component}"] = f"{prefix}.{component}" + return { + "name": f"{base}.ssmlp", + "kind": "ssmlp", + "descriptor": { + "k": 3072, + "n": 8192, + "group_size": 128, + }, + "roles": roles, + } + + +def required_weight_objects() -> list[dict[str, object]]: + """Return the deterministic 161-object corelib construction plan.""" + objects: list[dict[str, object]] = [] + for layer in range(32): + base = f"model.layers.{layer}.attn" + for projection, k, n in _ATTENTION_PROJECTIONS: + objects.append( + _matmul_weight_object( + f"{base}.{projection}.MatMulNBits", + k, + n, + 128, + ) + ) + objects.append(_ssmlp_weight_object(layer)) + objects.append( + _matmul_weight_object( + "lm_head.MatMulNBits", + 3072, + 200064, + 128, + ) + ) + + names = [record["name"] for record in objects] + initializer_names = set(required_initializer_roles()) + references = { + initializer_name + for record in objects + for initializer_name in record["roles"].values() + } + if ( + len(objects) != 161 + or len(names) != len(set(names)) + or not references.issubset(initializer_names) + ): + raise RuntimeError("internal Phi-4 weight-object map mismatch") + return objects + + def _parse_u64(value: str, field: str, initializer: str) -> int: if not value or not value.isdecimal(): raise ValueError( @@ -442,11 +533,79 @@ def _initializer_record( return record +def _validate_weight_objects( + weight_objects: list[dict[str, object]], + initializer_names: set[str], +) -> None: + names: set[str] = set() + expected_roles = { + "matmul": {"qweight", "scales", "qzeros"}, + "ssmlp": { + "norm0", + "norm1", + "gate_qweight", + "gate_scales", + "gate_qzeros", + "up_qweight", + "up_scales", + "up_qzeros", + "down_qweight", + "down_scales", + "down_qzeros", + }, + } + expected_descriptor_keys = { + "matmul": {"k", "n", "group_size", "has_bias"}, + "ssmlp": {"k", "n", "group_size"}, + } + + for record in weight_objects: + name = record.get("name") + kind = record.get("kind") + descriptor = record.get("descriptor") + roles = record.get("roles") + if not isinstance(name, str) or not name: + raise ValueError("weight object has an invalid name") + if name in names: + raise ValueError(f"duplicate weight object: {name}") + names.add(name) + if kind not in expected_roles: + raise ValueError(f"{name}: invalid weight object kind") + if ( + not isinstance(descriptor, dict) + or set(descriptor) != expected_descriptor_keys[kind] + ): + raise ValueError(f"{name}: invalid weight object descriptor") + for field in ("k", "n", "group_size"): + value = descriptor[field] + if ( + isinstance(value, bool) + or not isinstance(value, int) + or value <= 0 + ): + raise ValueError(f"{name}: invalid descriptor {field}") + if kind == "matmul" and descriptor["has_bias"] is not False: + raise ValueError(f"{name}: MatMul has_bias must be false") + if not isinstance(roles, dict) or set(roles) != expected_roles[kind]: + raise ValueError(f"{name}: invalid weight object role map") + if len(set(roles.values())) != len(roles): + raise ValueError(f"{name}: duplicate initializer role reference") + for initializer_name in roles.values(): + if ( + not isinstance(initializer_name, str) + or initializer_name not in initializer_names + ): + raise ValueError( + f"{name}: unresolved initializer {initializer_name}" + ) + + def _generate_manifest( model_dir: Path, output: Path, full_hash: bool, roles: dict[str, dict[str, object]], + weight_objects: list[dict[str, object]] | None = None, ) -> dict[str, object]: """Generate a manifest using an explicit role map. @@ -582,6 +741,10 @@ def _generate_manifest( full_hash, ) + emitted_weight_objects = ( + [] if weight_objects is None else weight_objects + ) + _validate_weight_objects(emitted_weight_objects, set(records)) manifest: dict[str, object] = { "schema_version": SCHEMA_VERSION, "execution_backend": "corelib_aie4", @@ -589,6 +752,7 @@ def _generate_manifest( "backend": {"max_seq": 4096}, "files": files, "initializers": records, + "weight_objects": emitted_weight_objects, } serialized = json.dumps( manifest, @@ -610,6 +774,7 @@ def generate_manifest( output, full_hash, required_initializer_roles(), + required_weight_objects(), ) diff --git a/tools/tests/test_generate_phi4_corelib_manifest.py b/tools/tests/test_generate_phi4_corelib_manifest.py index ac6c8cd5..da8eb9dc 100644 --- a/tools/tests/test_generate_phi4_corelib_manifest.py +++ b/tools/tests/test_generate_phi4_corelib_manifest.py @@ -1,14 +1,19 @@ from __future__ import annotations +from collections import Counter import hashlib import json +import os +import sys import tempfile import unittest from pathlib import Path +from unittest.mock import patch import onnx from onnx import TensorProto, helper +from tools import generate_phi4_corelib_manifest as manifest_tool from tools.generate_phi4_corelib_manifest import ( MAX_U64, _generate_manifest, @@ -474,6 +479,196 @@ def test_path_traversal_and_absolute_locations_are_rejected(self): self._exact_roles("test.weight"), ) + def test_symlink_escape_is_rejected(self): + with tempfile.TemporaryDirectory() as directory: + parent = Path(directory) + root = parent / "model" + root.mkdir() + outside = parent / "outside.bin" + outside.write_bytes(b"data") + link = root / "weights.bin" + try: + link.symlink_to(outside) + except (NotImplementedError, OSError) as error: + if os.name == "nt": + self.skipTest( + f"Windows cannot create the required symlink: {error}" + ) + raise + + tensor = self._external( + "test.weight", + TensorProto.UINT8, + [2, 2], + [ + ("location", "weights.bin"), + ("offset", "0"), + ("length", "4"), + ], + ) + self._write_model(root, [tensor]) + + with self.assertRaisesRegex(ValueError, "escapes"): + _generate_manifest( + root, + root / "manifest.json", + False, + self._exact_roles("test.weight"), + ) + + def test_out_of_file_range_reaches_range_guard(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "weights.bin").write_bytes(b"ab") + tensor = self._external( + "test.weight", + TensorProto.UINT8, + [2], + [ + ("location", "weights.bin"), + ("offset", "1"), + ("length", "2"), + ], + ) + self._write_model(root, [tensor]) + + with self.assertRaisesRegex(ValueError, "range exceeds file size"): + _generate_manifest( + root, + root / "manifest.json", + False, + self._exact_roles("test.weight", shape=[2]), + ) + + def test_multibyte_dtype_rejects_odd_external_offset(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "weights.bin").write_bytes(b"x" + bytes(4)) + tensor = self._external( + "test.scale", + TensorProto.FLOAT16, + [2], + [ + ("location", "weights.bin"), + ("offset", "1"), + ("length", "4"), + ], + ) + self._write_model(root, [tensor]) + + with self.assertRaisesRegex(ValueError, "dtype-aligned"): + _generate_manifest( + root, + root / "manifest.json", + False, + self._exact_roles( + "test.scale", + dtypes={"float16"}, + shape=[2], + ), + ) + + def test_duplicate_external_metadata_key_is_rejected(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "weights.bin").write_bytes(b"data") + tensor = self._external( + "test.weight", + TensorProto.UINT8, + [2, 2], + [ + ("location", "weights.bin"), + ("location", "weights.bin"), + ("offset", "0"), + ("length", "4"), + ], + ) + self._write_model(root, [tensor]) + + with self.assertRaisesRegex( + ValueError, + "duplicate external metadata key location", + ): + _generate_manifest( + root, + root / "manifest.json", + False, + self._exact_roles("test.weight"), + ) + + def test_generated_sidecar_cannot_replace_required_external_data(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + sidecar = root / "corelib_embedded_initializers.bin" + sidecar.write_bytes(b"source") + external = self._external( + "external.weight", + TensorProto.UINT8, + [6], + [ + ("location", "corelib_embedded_initializers.bin"), + ("offset", "0"), + ("length", "6"), + ], + ) + embedded = self._embedded( + "embedded.weight", + TensorProto.UINT8, + [4], + b"data", + ) + self._write_model(root, [external, embedded]) + roles = { + "external.weight": { + "role": "test.external", + "dtypes": {"uint8"}, + "shape": [6], + }, + "embedded.weight": { + "role": "test.embedded", + "dtypes": {"uint8"}, + "shape": [4], + }, + } + + with self.assertRaisesRegex(ValueError, "sidecar conflicts"): + _generate_manifest( + root, + root / "manifest.json", + False, + roles, + ) + self.assertEqual(sidecar.read_bytes(), b"source") + + def test_output_cannot_overwrite_required_external_data(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + data_path = root / "weights.bin" + data_path.write_bytes(b"data") + tensor = self._external( + "test.weight", + TensorProto.UINT8, + [2, 2], + [ + ("location", "weights.bin"), + ("offset", "0"), + ("length", "4"), + ], + ) + self._write_model(root, [tensor]) + + with self.assertRaisesRegex( + ValueError, + "output path would overwrite", + ): + _generate_manifest( + root, + data_path, + False, + self._exact_roles("test.weight"), + ) + self.assertEqual(data_path.read_bytes(), b"data") + def test_missing_length_and_range_overflow_are_rejected(self): cases = ( ( @@ -667,6 +862,297 @@ def test_manifest_has_locked_schema_and_model_identity(self): ) self.assertEqual(manifest["backend"], {"max_seq": 4096}) + def test_required_weight_objects_have_exact_descriptors_and_roles(self): + objects = manifest_tool.required_weight_objects() + self.assertEqual(len(objects), 161) + self.assertEqual(objects, manifest_tool.required_weight_objects()) + + names = [record["name"] for record in objects] + self.assertEqual(len(names), len(set(names))) + self.assertEqual( + Counter(record["kind"] for record in objects), + {"matmul": 129, "ssmlp": 32}, + ) + self.assertEqual( + sum( + record["kind"] == "matmul" + and ".attn." in record["name"] + for record in objects + ), + 128, + ) + expected_names = [ + name + for layer in range(32) + for name in ( + f"model.layers.{layer}.attn.q_proj.MatMulNBits", + f"model.layers.{layer}.attn.k_proj.MatMulNBits", + f"model.layers.{layer}.attn.v_proj.MatMulNBits", + f"model.layers.{layer}.attn.o_proj.MatMulNBits", + f"model.layers.{layer}.ssmlp", + ) + ] + expected_names.append("lm_head.MatMulNBits") + self.assertEqual(names, expected_names) + self.assertEqual( + names[:5], + [ + "model.layers.0.attn.q_proj.MatMulNBits", + "model.layers.0.attn.k_proj.MatMulNBits", + "model.layers.0.attn.v_proj.MatMulNBits", + "model.layers.0.attn.o_proj.MatMulNBits", + "model.layers.0.ssmlp", + ], + ) + self.assertEqual(names[-1], "lm_head.MatMulNBits") + + by_name = {record["name"]: record for record in objects} + projection_descriptors = ( + ("q_proj", 3072, 3072), + ("k_proj", 3072, 1024), + ("v_proj", 3072, 1024), + ("o_proj", 3072, 3072), + ) + for layer in range(32): + for projection, k, n in projection_descriptors: + prefix = ( + f"model.layers.{layer}.attn.{projection}.MatMulNBits" + ) + self.assertEqual( + by_name[prefix]["descriptor"], + { + "k": k, + "n": n, + "group_size": 128, + "has_bias": False, + }, + ) + self.assertEqual( + by_name[prefix]["roles"], + { + "qweight": f"{prefix}.qweight", + "scales": f"{prefix}.scales", + "qzeros": f"{prefix}.qzeros", + }, + ) + + base = f"model.layers.{layer}" + next_norm = ( + "model.layers.32.final_norm_layernorm.weight" + if layer == 31 + else f"model.layers.{layer + 1}.input_layernorm.weight" + ) + expected_ssmlp_roles = { + "norm0": f"{base}.post_attention_layernorm.weight", + "norm1": next_norm, + } + for projection in ("gate", "up", "down"): + prefix = f"{base}.mlp.{projection}_proj.MatMulNBits" + for component in ("qweight", "scales", "qzeros"): + expected_ssmlp_roles[f"{projection}_{component}"] = ( + f"{prefix}.{component}" + ) + self.assertEqual( + by_name[f"{base}.ssmlp"], + { + "name": f"{base}.ssmlp", + "kind": "ssmlp", + "descriptor": { + "k": 3072, + "n": 8192, + "group_size": 128, + }, + "roles": expected_ssmlp_roles, + }, + ) + + self.assertEqual( + by_name["model.layers.0.attn.q_proj.MatMulNBits"], + { + "name": "model.layers.0.attn.q_proj.MatMulNBits", + "kind": "matmul", + "descriptor": { + "k": 3072, + "n": 3072, + "group_size": 128, + "has_bias": False, + }, + "roles": { + "qweight": ( + "model.layers.0.attn.q_proj." + "MatMulNBits.qweight" + ), + "scales": ( + "model.layers.0.attn.q_proj." + "MatMulNBits.scales" + ), + "qzeros": ( + "model.layers.0.attn.q_proj." + "MatMulNBits.qzeros" + ), + }, + }, + ) + self.assertEqual( + by_name["model.layers.0.attn.k_proj.MatMulNBits"]["descriptor"], + { + "k": 3072, + "n": 1024, + "group_size": 128, + "has_bias": False, + }, + ) + self.assertEqual( + by_name["model.layers.0.attn.v_proj.MatMulNBits"]["descriptor"], + { + "k": 3072, + "n": 1024, + "group_size": 128, + "has_bias": False, + }, + ) + self.assertEqual( + by_name["model.layers.0.attn.o_proj.MatMulNBits"]["descriptor"], + { + "k": 3072, + "n": 3072, + "group_size": 128, + "has_bias": False, + }, + ) + self.assertEqual( + by_name["lm_head.MatMulNBits"]["descriptor"], + { + "k": 3072, + "n": 200064, + "group_size": 128, + "has_bias": False, + }, + ) + + final_ssmlp = by_name["model.layers.31.ssmlp"] + self.assertEqual( + final_ssmlp["descriptor"], + {"k": 3072, "n": 8192, "group_size": 128}, + ) + self.assertEqual( + final_ssmlp["roles"]["norm1"], + "model.layers.32.final_norm_layernorm.weight", + ) + self.assertNotIn("epsilon", final_ssmlp["roles"]) + self.assertEqual(manifest_tool.MODEL_IDENTITY["rms_epsilon"], 0.00001) + + initializers = required_initializer_roles() + for weight_object in objects: + for initializer_name in weight_object["roles"].values(): + self.assertIn(initializer_name, initializers) + + def test_manifest_emits_top_level_weight_objects_only(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + names = ("test.qweight", "test.scales", "test.qzeros") + tensors = [ + self._embedded(name, TensorProto.UINT8, [1], bytes([index])) + for index, name in enumerate(names, start=1) + ] + self._write_model(root, tensors) + roles = { + name: { + "role": f"test.{name.rsplit('.', 1)[-1]}", + "weight_object": "test.matmul", + "dtypes": {"uint8"}, + "shape": [1], + } + for name in names + } + weight_objects = [ + { + "name": "test.matmul", + "kind": "matmul", + "descriptor": { + "k": 2, + "n": 1, + "group_size": 1, + "has_bias": False, + }, + "roles": { + "qweight": "test.qweight", + "scales": "test.scales", + "qzeros": "test.qzeros", + }, + } + ] + + manifest = _generate_manifest( + root, + root / "manifest.json", + False, + roles, + weight_objects, + ) + + self.assertEqual(manifest["weight_objects"], weight_objects) + for initializer in manifest["initializers"].values(): + self.assertNotIn("weight_object", initializer) + + def test_generate_manifest_binds_real_role_and_weight_maps(self): + model_dir = Path("model package") + output = Path("output manifest.json") + sentinel = {"result": "sentinel"} + + with patch.object( + manifest_tool, + "_generate_manifest", + return_value=sentinel, + ) as lower: + result = manifest_tool.generate_manifest( + model_dir, + output, + True, + ) + + self.assertIs(result, sentinel) + self.assertEqual(len(lower.call_args.args), 5) + self.assertEqual(lower.call_args.args[:3], (model_dir, output, True)) + self.assertEqual( + lower.call_args.args[3], + required_initializer_roles(), + ) + self.assertEqual( + lower.call_args.args[4], + manifest_tool.required_weight_objects(), + ) + + def test_main_wires_exact_paths_and_full_hash_flag(self): + cases = ( + ([], False), + (["--full-hash"], True), + ) + for extra, expected_full_hash in cases: + with self.subTest(full_hash=expected_full_hash): + argv = [ + "generate_phi4_corelib_manifest.py", + "--model-dir", + "model package", + "--output", + "output manifest.json", + *extra, + ] + with ( + patch.object(sys, "argv", argv), + patch.object( + manifest_tool, + "generate_manifest", + ) as generate, + ): + self.assertEqual(manifest_tool.main(), 0) + + generate.assert_called_once_with( + Path("model package"), + Path("output manifest.json"), + expected_full_hash, + ) + if __name__ == "__main__": unittest.main() From 398a7326690a0dd04ebb4f2b4b270f8633f94c8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CChi?= Date: Mon, 31 Aug 2026 22:26:55 -0700 Subject: [PATCH 006/117] feat: load Phi-4 AIE4 model packages Co-authored-by: Cursor --- src/common/corelib/corelib_sources.cmake | 3 +- src/common/corelib/phi4_corelib_manifest.cpp | 1312 +++++++++++++++++ .../models/phi4/phi4_corelib_manifest.hpp | 127 ++ src/test/phi4_corelib_aie4/CMakeLists.txt | 1 + .../phi4_corelib_aie4/test_phi4_manifest.cpp | 1196 +++++++++++++++ 5 files changed, 2638 insertions(+), 1 deletion(-) create mode 100644 src/common/corelib/phi4_corelib_manifest.cpp create mode 100644 src/include/models/phi4/phi4_corelib_manifest.hpp create mode 100644 src/test/phi4_corelib_aie4/test_phi4_manifest.cpp diff --git a/src/common/corelib/corelib_sources.cmake b/src/common/corelib/corelib_sources.cmake index 60641974..e94480bc 100644 --- a/src/common/corelib/corelib_sources.cmake +++ b/src/common/corelib/corelib_sources.cmake @@ -1,4 +1,5 @@ set(FLM_CORELIB_AIE4_SOURCES "${CMAKE_CURRENT_LIST_DIR}/corelib_api.cpp" "${CMAKE_CURRENT_LIST_DIR}/corelib_fatal_record.cpp" - "${CMAKE_CURRENT_LIST_DIR}/corelib_runtime.cpp") + "${CMAKE_CURRENT_LIST_DIR}/corelib_runtime.cpp" + "${CMAKE_CURRENT_LIST_DIR}/phi4_corelib_manifest.cpp") diff --git a/src/common/corelib/phi4_corelib_manifest.cpp b/src/common/corelib/phi4_corelib_manifest.cpp new file mode 100644 index 00000000..99573f26 --- /dev/null +++ b/src/common/corelib/phi4_corelib_manifest.cpp @@ -0,0 +1,1312 @@ +#include + +#include "../../pull/picosha2.h" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace flm::phi4 { +namespace { + +using nlohmann::json; + +constexpr std::string_view kManifestName = + "corelib_phi4_manifest.json"; +constexpr std::size_t kExpectedInitializers = 743; +constexpr std::size_t kExpectedWeightObjects = 161; +constexpr std::int64_t kLayers = 32; +constexpr std::int64_t kHidden = 3072; +constexpr std::int64_t kIntermediate = 8192; +constexpr std::int64_t kVocab = 200064; +constexpr std::int64_t kMaxSeq = 4096; +constexpr std::int64_t kRopeColumns = 48; +constexpr std::uint32_t kGroupSize = 128; + +[[noreturn]] void Throw( + std::string_view context, + std::string_view detail) { + throw std::runtime_error( + std::string(context) + ": " + std::string(detail)); +} + +[[noreturn]] void ThrowWin32( + std::string_view operation, + const std::filesystem::path& path, + DWORD error) { + std::ostringstream message; + message << operation << " failed for " << path.string() + << " (Win32 error " << error << ")"; + throw std::runtime_error(message.str()); +} + +void RequireExactKeys( + const json& value, + std::initializer_list expected, + std::string_view context) { + if (!value.is_object()) { + Throw(context, "must be an object"); + } + if (value.size() != expected.size()) { + Throw(context, "has an invalid field set"); + } + for (const std::string_view key : expected) { + if (!value.contains(std::string(key))) { + Throw( + context, + std::string("is missing field ") + std::string(key)); + } + } +} + +std::string ReadString( + const json& value, + std::string_view context) { + if (!value.is_string()) { + Throw(context, "must be a string"); + } + const std::string result = value.get(); + if (result.empty() || + result.find('\0') != std::string::npos) { + Throw(context, "must be a non-empty string"); + } + return result; +} + +std::uint64_t ReadU64( + const json& value, + std::string_view context) { + if (!value.is_number_unsigned()) { + Throw(context, "must be an unsigned integer"); + } + return value.get(); +} + +std::int64_t ReadPositiveI64( + const json& value, + std::string_view context) { + std::uint64_t unsigned_value = 0; + if (value.is_number_unsigned()) { + unsigned_value = value.get(); + } else if (value.is_number_integer()) { + const std::int64_t signed_value = value.get(); + if (signed_value <= 0) { + Throw(context, "must be a positive integer"); + } + return signed_value; + } else { + Throw(context, "must be a positive integer"); + } + if ( + unsigned_value == 0 || + unsigned_value > + static_cast( + std::numeric_limits::max())) { + Throw(context, "must fit a positive int64"); + } + return static_cast(unsigned_value); +} + +void RequireInteger( + const json& value, + std::int64_t expected, + std::string_view context) { + if (ReadPositiveI64(value, context) != expected) { + Throw(context, "does not match the Phi-4 model identity"); + } +} + +bool ReadBool( + const json& value, + std::string_view context) { + if (!value.is_boolean()) { + Throw(context, "must be a boolean"); + } + return value.get(); +} + +std::filesystem::path ParseRelativePath( + std::string_view raw, + std::string_view context) { + if (raw.empty() || + raw.find('\0') != std::string_view::npos) { + Throw(context, "file path must not be empty"); + } + + const auto path = std::filesystem::u8path( + raw.begin(), + raw.end()); + if ( + path.is_absolute() || path.has_root_name() || + path.has_root_directory()) { + Throw(context, "file path must be relative"); + } + + bool has_component = false; + for (const auto& component : path) { + if (component == "..") { + Throw(context, "file path contains parent traversal"); + } + if (component == "." || component.empty()) { + continue; + } + has_component = true; + } + if (!has_component) { + Throw(context, "file path must name a file"); + } + return path.lexically_normal(); +} + +bool IsWithin( + const std::filesystem::path& root, + const std::filesystem::path& candidate) { + const auto relative = candidate.lexically_relative(root); + if (relative.empty() || relative.is_absolute()) { + return false; + } + const auto first = relative.begin(); + return first != relative.end() && *first != ".."; +} + +std::filesystem::path ResolvePackageFile( + const std::filesystem::path& root, + std::string_view raw, + std::string_view context) { + const auto relative = ParseRelativePath(raw, context); + std::error_code error; + const auto resolved = + std::filesystem::canonical(root / relative, error); + if (error) { + Throw( + context, + std::string("package file does not exist: ") + + std::string(raw)); + } + if (!IsWithin(root, resolved)) { + Throw(context, "package file escapes the model directory"); + } + if (!std::filesystem::is_regular_file(resolved, error) || error) { + Throw(context, "package path is not a regular file"); + } + return resolved; +} + +std::string CalculateSha256( + const std::filesystem::path& path) { + std::ifstream stream(path, std::ios::binary); + if (!stream) { + Throw("SHA-256", "failed to open package file"); + } + std::array digest{}; + picosha2::hash256(stream, digest.begin(), digest.end()); + if (stream.bad()) { + Throw("SHA-256", "failed while reading package file"); + } + return picosha2::bytes_to_hex_string( + digest.begin(), + digest.end()); +} + +bool IsSha256(std::string_view value) { + return value.size() == picosha2::k_digest_size * 2 && + std::all_of( + value.begin(), + value.end(), + [](unsigned char character) { + return std::isxdigit(character) != 0; + }); +} + +std::string NormalizeSha256(std::string value) { + std::transform( + value.begin(), + value.end(), + value.begin(), + [](unsigned char character) { + return static_cast(std::tolower(character)); + }); + return value; +} + +SourceDType ParseDType( + std::string_view value, + std::string_view context) { + if (value == "uint8") { + return SourceDType::UInt8; + } + if (value == "float16") { + return SourceDType::Float16; + } + if (value == "float32") { + return SourceDType::Float32; + } + if (value == "int64") { + return SourceDType::Int64; + } + Throw(context, "has an unsupported dtype"); +} + +std::uint64_t ItemSize(SourceDType dtype) { + switch (dtype) { + case SourceDType::UInt8: + return 1; + case SourceDType::Float16: + return 2; + case SourceDType::Float32: + return 4; + case SourceDType::Int64: + return 8; + } + throw std::logic_error("unreachable SourceDType"); +} + +std::vector ParseShape( + const json& value, + std::string_view context) { + if (!value.is_array() || value.empty()) { + Throw(context, "shape must be a non-empty array"); + } + std::vector shape; + shape.reserve(value.size()); + for (const auto& dimension : value) { + shape.push_back(ReadPositiveI64(dimension, context)); + } + return shape; +} + +std::uint64_t CheckedByteCount( + const std::vector& shape, + SourceDType dtype, + std::string_view context) { + std::uint64_t elements = 1; + for (const std::int64_t dimension : shape) { + const auto unsigned_dimension = + static_cast(dimension); + if ( + elements > + std::numeric_limits::max() / + unsigned_dimension) { + Throw(context, "shape element count overflow"); + } + elements *= unsigned_dimension; + } + const std::uint64_t item_size = ItemSize(dtype); + if ( + elements > + std::numeric_limits::max() / item_size) { + Throw(context, "tensor byte count overflow"); + } + return elements * item_size; +} + +void RequireShape( + const InitializerView& view, + std::initializer_list expected, + std::string_view context) { + if (!std::equal( + view.shape.begin(), + view.shape.end(), + expected.begin(), + expected.end())) { + Throw(context, "has an invalid shape"); + } +} + +void RequireFloating( + const InitializerView& view, + std::string_view context) { + if ( + view.dtype != SourceDType::Float16 && + view.dtype != SourceDType::Float32) { + Throw(context, "must use a floating FP16 or FP32 source"); + } +} + +void RequireSemanticRole( + const std::map>& roles, + std::string_view initializer, + std::string_view expected) { + const auto found = roles.find(initializer); + if (found == roles.end() || found->second != expected) { + Throw( + initializer, + std::string("semantic role does not match ") + + std::string(expected)); + } +} + +struct ExpectedWeightObject { + std::string name; + WeightObjectKind kind; + std::int64_t k; + std::int64_t n; +}; + +const std::vector& ExpectedWeightObjects() { + static const std::vector expected = [] { + std::vector values; + values.reserve(kExpectedWeightObjects); + for (int layer = 0; layer < kLayers; ++layer) { + const std::string base = + "model.layers." + std::to_string(layer) + ".attn."; + values.push_back({ + base + "q_proj.MatMulNBits", + WeightObjectKind::MatMul, + 3072, + 3072}); + values.push_back({ + base + "k_proj.MatMulNBits", + WeightObjectKind::MatMul, + 3072, + 1024}); + values.push_back({ + base + "v_proj.MatMulNBits", + WeightObjectKind::MatMul, + 3072, + 1024}); + values.push_back({ + base + "o_proj.MatMulNBits", + WeightObjectKind::MatMul, + 3072, + 3072}); + values.push_back({ + "model.layers." + std::to_string(layer) + ".ssmlp", + WeightObjectKind::SsMlp, + kHidden, + kIntermediate}); + } + values.push_back({ + "lm_head.MatMulNBits", + WeightObjectKind::MatMul, + kHidden, + kVocab}); + return values; + }(); + return expected; +} + +std::set MatMulRoleNames() { + return {"qweight", "scales", "qzeros"}; +} + +std::set SsMlpRoleNames() { + return { + "norm0", + "norm1", + "gate_qweight", + "gate_scales", + "gate_qzeros", + "up_qweight", + "up_scales", + "up_qzeros", + "down_qweight", + "down_scales", + "down_qzeros"}; +} + +std::set JsonKeys(const json& value) { + std::set keys; + for (const auto& [key, _] : value.items()) { + keys.insert(key); + } + return keys; +} + +void ValidateQuantizedProjection( + const Phi4Package& package, + const std::map>& semantic_roles, + const std::map& components, + std::string_view role_prefix, + std::string_view component_prefix, + std::int64_t k, + std::int64_t n) { + if (k <= 0 || n <= 0 || k % 2 != 0 || + k % static_cast(kGroupSize) != 0) { + Throw(role_prefix, "has an invalid quantized descriptor"); + } + const std::int64_t groups = + k / static_cast(kGroupSize); + + const auto validate = [&]( + std::string_view component, + SourceDType dtype, + std::initializer_list shape, + std::string semantic) { + const std::string role_name = + std::string(component_prefix) + std::string(component); + const auto found = components.find(role_name); + if (found == components.end()) { + Throw(role_prefix, "is missing a component role"); + } + const auto& view = package.Require(found->second); + if (view.dtype != dtype) { + Throw(role_name, "has an invalid dtype"); + } + RequireShape(view, shape, role_name); + RequireSemanticRole( + semantic_roles, + found->second, + semantic); + }; + + validate( + "qweight", + SourceDType::UInt8, + {n, k / 2}, + std::string(role_prefix) + ".qweight"); + + const std::string scales_name = + std::string(component_prefix) + "scales"; + const auto scales_component = components.find(scales_name); + if (scales_component == components.end()) { + Throw(role_prefix, "is missing a scales role"); + } + const auto& scales = package.Require(scales_component->second); + RequireFloating(scales, scales_name); + RequireShape(scales, {n, groups}, scales_name); + RequireSemanticRole( + semantic_roles, + scales_component->second, + std::string(role_prefix) + ".scales"); + + validate( + "qzeros", + SourceDType::UInt8, + {n, (groups + 1) / 2}, + std::string(role_prefix) + ".qzeros"); +} + +void ValidateHostInitializers( + const Phi4Package& package, + const std::map>& semantic_roles) { + const auto& embedding = + package.Require("model.embed_tokens.weight"); + if (embedding.dtype != SourceDType::Float16) { + Throw("embedding", "must use an FP16 source"); + } + RequireShape( + embedding, + {kVocab, kHidden}, + "embedding"); + RequireSemanticRole( + semantic_roles, + "model.embed_tokens.weight", + "embedding"); + + const auto& input_norm = + package.Require("model.layers.0.input_layernorm.weight"); + RequireFloating(input_norm, "input_norm"); + RequireShape(input_norm, {kHidden}, "input_norm"); + RequireSemanticRole( + semantic_roles, + "model.layers.0.input_layernorm.weight", + "input_norm"); + + for (const std::string_view name : {"cos_cache", "sin_cache"}) { + const auto& rope = package.Require(name); + RequireFloating(rope, name); + if ( + rope.shape.size() != 2 || + rope.shape[0] < kMaxSeq || + rope.shape[1] < kRopeColumns) { + Throw(name, "must be rank 2 and at least [4096,48]"); + } + RequireSemanticRole(semantic_roles, name, name); + } +} + +ryzenai_corelib_data_type CorelibDType( + SourceDType dtype, + std::string_view context) { + switch (dtype) { + case SourceDType::Float16: + return ryzenai_corelib_data_type_fp16; + case SourceDType::Float32: + return ryzenai_corelib_data_type_fp32; + case SourceDType::UInt8: + case SourceDType::Int64: + Throw(context, "requires a floating FP16 or FP32 source"); + } + throw std::logic_error("unreachable SourceDType"); +} + +std::size_t ElementCount( + const InitializerView& view, + std::string_view context) { + const std::uint64_t count = + static_cast(view.size) / ItemSize(view.dtype); + if ( + count > + static_cast( + std::numeric_limits::max())) { + Throw(context, "element count exceeds addressable memory"); + } + return static_cast(count); +} + +void ValidateModelIdentity(const json& manifest) { + if ( + manifest.is_object() && + !manifest.contains("weight_objects")) { + Throw("weight_objects", "section is missing"); + } + RequireExactKeys( + manifest, + { + "schema_version", + "execution_backend", + "model", + "backend", + "files", + "initializers", + "weight_objects", + }, + "manifest"); + RequireInteger( + manifest.at("schema_version"), + 1, + "schema_version"); + if ( + ReadString( + manifest.at("execution_backend"), + "execution_backend") != "corelib_aie4") { + Throw( + "execution_backend", + "does not match corelib_aie4"); + } + + const auto& model = manifest.at("model"); + RequireExactKeys( + model, + { + "family", + "layers", + "hidden_size", + "intermediate_size", + "num_heads", + "kv_heads", + "head_size", + "vocab_size", + "group_size", + "rope_dim", + "rms_epsilon", + }, + "model"); + if (ReadString(model.at("family"), "model.family") != "phi4") { + Throw("model.family", "does not match phi4"); + } + RequireInteger(model.at("layers"), 32, "model.layers"); + RequireInteger( + model.at("hidden_size"), + 3072, + "model.hidden_size"); + RequireInteger( + model.at("intermediate_size"), + 8192, + "model.intermediate_size"); + RequireInteger( + model.at("num_heads"), + 24, + "model.num_heads"); + RequireInteger(model.at("kv_heads"), 8, "model.kv_heads"); + RequireInteger( + model.at("head_size"), + 128, + "model.head_size"); + RequireInteger( + model.at("vocab_size"), + 200064, + "model.vocab_size"); + RequireInteger( + model.at("group_size"), + 128, + "model.group_size"); + RequireInteger( + model.at("rope_dim"), + 96, + "model.rope_dim"); + if ( + !model.at("rms_epsilon").is_number() || + model.at("rms_epsilon").get() != 0.00001) { + Throw( + "model.rms_epsilon", + "does not match the Phi-4 model identity"); + } + + const auto& backend = manifest.at("backend"); + RequireExactKeys(backend, {"max_seq"}, "backend"); + RequireInteger( + backend.at("max_seq"), + kMaxSeq, + "backend.max_seq"); +} + +} // namespace + +MappedFile::MappedFile( + std::filesystem::path path, + void* file, + void* mapping, + const std::byte* data, + std::uint64_t size) noexcept + : path_(std::move(path)), + file_(file), + mapping_(mapping), + data_(data), + size_(size) {} + +std::shared_ptr MappedFile::OpenReadOnly( + const std::filesystem::path& path) { + // Copy before acquiring Win32 resources so an allocation failure cannot + // strand handles that have not yet been transferred to MappedFile. + std::filesystem::path owned_path = path; + HANDLE file = CreateFileW( + path.c_str(), + GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_DELETE, + nullptr, + OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, + nullptr); + if (file == INVALID_HANDLE_VALUE) { + ThrowWin32("CreateFileW", path, GetLastError()); + } + + LARGE_INTEGER raw_size{}; + if (GetFileSizeEx(file, &raw_size) == FALSE) { + const DWORD error = GetLastError(); + CloseHandle(file); + ThrowWin32("GetFileSizeEx", path, error); + } + if (raw_size.QuadPart <= 0) { + CloseHandle(file); + Throw(path.string(), "mapped file must not be empty"); + } + const auto size = static_cast(raw_size.QuadPart); + + HANDLE mapping = CreateFileMappingW( + file, + nullptr, + PAGE_READONLY, + 0, + 0, + nullptr); + if (mapping == nullptr) { + const DWORD error = GetLastError(); + CloseHandle(file); + ThrowWin32("CreateFileMappingW", path, error); + } + + const void* raw_data = + MapViewOfFile(mapping, FILE_MAP_READ, 0, 0, 0); + if (raw_data == nullptr) { + const DWORD error = GetLastError(); + CloseHandle(mapping); + CloseHandle(file); + ThrowWin32("MapViewOfFile", path, error); + } + + std::unique_ptr owner; + try { + owner.reset(new MappedFile( + std::move(owned_path), + file, + mapping, + static_cast(raw_data), + size)); + } catch (...) { + UnmapViewOfFile(raw_data); + CloseHandle(mapping); + CloseHandle(file); + throw; + } + return std::shared_ptr(std::move(owner)); +} + +MappedFile::MappedFile(MappedFile&& other) noexcept + : path_(std::move(other.path_)), + file_(std::exchange(other.file_, nullptr)), + mapping_(std::exchange(other.mapping_, nullptr)), + data_(std::exchange(other.data_, nullptr)), + size_(std::exchange(other.size_, 0)) {} + +MappedFile& MappedFile::operator=(MappedFile&& other) noexcept { + if (this != &other) { + Reset(); + path_ = std::move(other.path_); + file_ = std::exchange(other.file_, nullptr); + mapping_ = std::exchange(other.mapping_, nullptr); + data_ = std::exchange(other.data_, nullptr); + size_ = std::exchange(other.size_, 0); + } + return *this; +} + +MappedFile::~MappedFile() noexcept { + Reset(); +} + +void MappedFile::Reset() noexcept { + if (data_ != nullptr) { + UnmapViewOfFile(data_); + data_ = nullptr; + } + if (mapping_ != nullptr) { + CloseHandle(static_cast(mapping_)); + mapping_ = nullptr; + } + if (file_ != nullptr) { + CloseHandle(static_cast(file_)); + file_ = nullptr; + } + size_ = 0; +} + +const std::byte* MappedFile::data() const noexcept { + return data_; +} + +std::uint64_t MappedFile::size() const noexcept { + return size_; +} + +const std::filesystem::path& MappedFile::path() const noexcept { + return path_; +} + +Phi4Package Phi4Package::Load( + const std::filesystem::path& model_dir, + std::shared_ptr api, + bool verify_full_hash) { + if (!api) { + throw std::invalid_argument( + "Phi4Package::Load requires a CorelibApi"); + } + + std::error_code error; + const auto root = std::filesystem::canonical(model_dir, error); + if ( + error || !std::filesystem::is_directory(root, error) || + error) { + throw std::runtime_error( + "Phi4 model directory does not exist or is not a directory"); + } + + std::ifstream manifest_stream( + root / kManifestName, + std::ios::binary); + if (!manifest_stream) { + throw std::runtime_error( + "Phi-4 package is missing corelib_phi4_manifest.json"); + } + + json manifest; + try { + manifest_stream >> manifest; + } catch (const json::exception& parse_error) { + throw std::runtime_error( + std::string("failed to parse corelib_phi4_manifest.json: ") + + parse_error.what()); + } + ValidateModelIdentity(manifest); + + Phi4Package package; + package.api_ = std::move(api); + + const auto& files = manifest.at("files"); + if (!files.is_object() || files.empty()) { + Throw("files", "must be a non-empty object"); + } + if (!files.contains("model.onnx")) { + Throw("files", "is missing model.onnx"); + } + for (const auto& [name, record] : files.items()) { + const std::string context = "files." + name; + if (!record.is_object()) { + Throw(context, "must be an object"); + } + const bool has_hash = record.contains("sha256"); + RequireExactKeys( + record, + has_hash + ? std::initializer_list{ + "size", + "sha256"} + : std::initializer_list{"size"}, + context); + + const std::uint64_t expected_size = + ReadU64(record.at("size"), context + ".size"); + if (expected_size == 0) { + Throw(context, "file size must be positive"); + } + const auto resolved = + ResolvePackageFile(root, name, context); + auto mapped = MappedFile::OpenReadOnly(resolved); + if (mapped->size() != expected_size) { + Throw(context, "file size does not match the manifest"); + } + + if (has_hash) { + std::string expected_hash = ReadString( + record.at("sha256"), + context + ".sha256"); + if (!IsSha256(expected_hash)) { + Throw(context, "SHA-256 must contain 64 hexadecimal digits"); + } + expected_hash = NormalizeSha256(std::move(expected_hash)); + if ( + verify_full_hash && + CalculateSha256(resolved) != expected_hash) { + Throw(context, "SHA-256 does not match the package file"); + } + } else if (verify_full_hash) { + Throw(context, "SHA-256 is required for full verification"); + } + + package.mapped_files_.emplace(name, std::move(mapped)); + } + + const auto& initializer_records = manifest.at("initializers"); + if ( + !initializer_records.is_object() || + initializer_records.size() != kExpectedInitializers) { + Throw( + "initializers", + "must contain exactly 743 initializer records"); + } + + std::map> + semantic_roles; + for (const auto& [name, record] : initializer_records.items()) { + const std::string context = "initializers." + name; + RequireExactKeys( + record, + {"file", "offset", "length", "dtype", "shape", "role"}, + context); + const std::string file_name = + ReadString(record.at("file"), context + ".file"); + (void)ParseRelativePath(file_name, context + ".file"); + + const std::uint64_t offset = + ReadU64(record.at("offset"), context + ".offset"); + const std::uint64_t length = + ReadU64(record.at("length"), context + ".length"); + if (length == 0) { + Throw(context, "initializer length must be positive"); + } + if ( + offset > + std::numeric_limits::max() - length) { + Throw(context, "initializer range overflow"); + } + + const auto owner = + package.mapped_files_.find(file_name); + if (owner == package.mapped_files_.end()) { + Throw(context, "initializer references an unlisted file"); + } + if ( + offset > owner->second->size() || + length > owner->second->size() - offset) { + Throw(context, "initializer range exceeds file size"); + } + + const std::string dtype_name = + ReadString(record.at("dtype"), context + ".dtype"); + const SourceDType dtype = + ParseDType(dtype_name, context + ".dtype"); + if (offset % ItemSize(dtype) != 0) { + Throw(context, "initializer offset is not dtype-aligned"); + } + const auto shape = + ParseShape(record.at("shape"), context + ".shape"); + if (CheckedByteCount(shape, dtype, context) != length) { + Throw( + context, + "dtype/shape byte count does not match length"); + } + if ( + length > + static_cast( + std::numeric_limits::max()) || + offset > + static_cast( + std::numeric_limits::max())) { + Throw(context, "initializer range is not addressable"); + } + + const std::string semantic_role = + ReadString(record.at("role"), context + ".role"); + semantic_roles.emplace(name, semantic_role); + package.initializers_.emplace( + name, + InitializerView{ + dtype, + shape, + owner->second->data() + + static_cast(offset), + static_cast(length), + owner->second}); + } + + ValidateHostInitializers(package, semantic_roles); + + if (!manifest.contains("weight_objects")) { + Throw("weight_objects", "section is missing"); + } + const auto& object_records = manifest.at("weight_objects"); + if ( + !object_records.is_array() || + object_records.size() != kExpectedWeightObjects) { + Throw( + "weight_objects", + "must be a non-empty list of exactly 161 entries"); + } + + const auto& expected_objects = ExpectedWeightObjects(); + std::set object_names; + std::set referenced_initializers; + package.weight_objects_.reserve(kExpectedWeightObjects); + for (std::size_t index = 0; index < object_records.size(); ++index) { + const auto& record = object_records.at(index); + const std::string index_context = + "weight_objects[" + std::to_string(index) + "]"; + RequireExactKeys( + record, + {"name", "kind", "descriptor", "roles"}, + index_context); + + const std::string name = + ReadString(record.at("name"), index_context + ".name"); + if (!object_names.insert(name).second) { + Throw(name, "duplicate weight object name"); + } + + const std::string kind_name = + ReadString(record.at("kind"), name + ".kind"); + WeightObjectKind kind; + if (kind_name == "matmul") { + kind = WeightObjectKind::MatMul; + } else if (kind_name == "ssmlp") { + kind = WeightObjectKind::SsMlp; + } else { + Throw(name, "invalid weight object kind"); + } + + const auto& expected = expected_objects.at(index); + if (name != expected.name) { + Throw( + name, + "weight object name/order does not match the fixed " + "Phi-4 plan"); + } + if (kind != expected.kind) { + Throw( + name, + "weight object kind does not match the fixed Phi-4 plan"); + } + + const auto& descriptor = record.at("descriptor"); + RequireExactKeys( + descriptor, + kind == WeightObjectKind::MatMul + ? std::initializer_list{ + "k", + "n", + "group_size", + "has_bias"} + : std::initializer_list{ + "k", + "n", + "group_size"}, + name + ".descriptor"); + const std::int64_t k = + ReadPositiveI64(descriptor.at("k"), name + ".descriptor.k"); + const std::int64_t n = + ReadPositiveI64(descriptor.at("n"), name + ".descriptor.n"); + const std::int64_t raw_group_size = ReadPositiveI64( + descriptor.at("group_size"), + name + ".descriptor.group_size"); + if ( + raw_group_size > + static_cast( + std::numeric_limits::max())) { + Throw(name + ".descriptor", "group_size exceeds uint32"); + } + const auto group_size = + static_cast(raw_group_size); + bool has_bias = false; + if (kind == WeightObjectKind::MatMul) { + has_bias = ReadBool( + descriptor.at("has_bias"), + name + ".descriptor.has_bias"); + if (has_bias) { + Throw( + name + ".descriptor", + "MatMul has_bias must be false"); + } + } + if ( + k != expected.k || n != expected.n || + group_size != kGroupSize) { + Throw( + name + ".descriptor", + "does not match the fixed Phi-4 descriptor"); + } + + const auto& role_map = record.at("roles"); + if (!role_map.is_object()) { + Throw(name, "weight object role map must be an object"); + } + const auto expected_roles = + kind == WeightObjectKind::MatMul + ? MatMulRoleNames() + : SsMlpRoleNames(); + if (JsonKeys(role_map) != expected_roles) { + Throw(name, "invalid weight object role map"); + } + + std::map components; + std::set local_references; + for (const auto& [role, initializer] : role_map.items()) { + const std::string initializer_name = ReadString( + initializer, + name + ".roles." + role); + if (!local_references.insert(initializer_name).second) { + Throw(name, "duplicate initializer role reference"); + } + if (!package.initializers_.contains(initializer_name)) { + Throw( + name, + "unresolved initializer " + initializer_name); + } + if (!referenced_initializers.insert(initializer_name).second) { + Throw( + name, + "duplicate initializer reference across weight objects"); + } + components.emplace(role, initializer_name); + } + + if (kind == WeightObjectKind::MatMul) { + ValidateQuantizedProjection( + package, + semantic_roles, + components, + "matmul", + "", + k, + n); + } else { + const auto validate_norm = [&]( + std::string_view role, + std::string_view semantic) { + const auto component = + components.find(std::string(role)); + const auto& view = + package.Require(component->second); + RequireFloating(view, role); + RequireShape(view, {kHidden}, role); + RequireSemanticRole( + semantic_roles, + component->second, + semantic); + }; + validate_norm("norm0", "ssmlp.norm0"); + validate_norm("norm1", "ssmlp.norm1"); + ValidateQuantizedProjection( + package, + semantic_roles, + components, + "ssmlp.gate", + "gate_", + k, + n); + ValidateQuantizedProjection( + package, + semantic_roles, + components, + "ssmlp.up", + "up_", + k, + n); + ValidateQuantizedProjection( + package, + semantic_roles, + components, + "ssmlp.down", + "down_", + n, + k); + } + + package.weight_objects_.push_back({ + name, + kind, + k, + n, + group_size, + has_bias, + std::move(components)}); + } + + const std::set host_initializers{ + "model.embed_tokens.weight", + "model.layers.0.input_layernorm.weight", + "cos_cache", + "sin_cache"}; + for (const auto& [name, _] : package.initializers_) { + const bool is_host = host_initializers.contains(name); + const bool is_referenced = + referenced_initializers.contains(name); + if (is_host == is_referenced) { + Throw( + name, + is_host + ? "host initializer must not belong to a weight object" + : "initializer is not referenced by weight_objects"); + } + } + + return package; +} + +const InitializerView& Phi4Package::Require( + std::string_view name) const { + const auto found = initializers_.find(name); + if (found == initializers_.end()) { + throw std::runtime_error( + "missing initializer: " + std::string(name)); + } + return found->second; +} + +const std::vector& +Phi4Package::weight_objects() const noexcept { + return weight_objects_; +} + +std::span Phi4Package::MaterializeFp16( + std::string_view name) { + if (const auto found = fp16_buffers_.find(name); + found != fp16_buffers_.end()) { + return found->second; + } + + const auto& source = Require(name); + const auto source_type = CorelibDType(source.dtype, name); + const std::size_t count = ElementCount(source, name); + auto [buffer, inserted] = + fp16_buffers_.try_emplace(std::string(name), count); + try { + api_->Check( + api_->functions().convert( + source_type, + source.data, + ryzenai_corelib_data_type_fp16, + buffer->second.data(), + count), + "ryzenai_corelib_convert"); + } catch (...) { + if (inserted) { + fp16_buffers_.erase(buffer); + } + throw; + } + return buffer->second; +} + +std::span Phi4Package::MaterializeBf16( + std::string_view name) { + if (const auto found = bf16_buffers_.find(name); + found != bf16_buffers_.end()) { + return found->second; + } + + const auto& source = Require(name); + const auto source_type = CorelibDType(source.dtype, name); + const std::size_t count = ElementCount(source, name); + auto [buffer, inserted] = + bf16_buffers_.try_emplace(std::string(name), count); + try { + api_->Check( + api_->functions().convert( + source_type, + source.data, + ryzenai_corelib_data_type_bf16, + buffer->second.data(), + count), + "ryzenai_corelib_convert"); + } catch (...) { + if (inserted) { + bf16_buffers_.erase(buffer); + } + throw; + } + return buffer->second; +} + +std::span Phi4Package::MaterializeRopeFp32( + std::string_view name) { + if (const auto found = fp32_buffers_.find(name); + found != fp32_buffers_.end()) { + return found->second; + } + + const auto& source = Require(name); + const auto source_type = CorelibDType(source.dtype, name); + if ( + source.shape.size() != 2 || + source.shape[0] < kMaxSeq || + source.shape[1] < kRopeColumns) { + Throw( + name, + "RoPE source must be rank 2 and at least [4096,48]"); + } + const auto source_columns = + static_cast(source.shape[1]); + constexpr std::size_t count = + static_cast(kMaxSeq * kRopeColumns); + auto [buffer, inserted] = + fp32_buffers_.try_emplace(std::string(name), count); + try { + api_->Check( + api_->functions().convert_strided( + source_type, + source.data, + source_columns, + ryzenai_corelib_data_type_fp32, + buffer->second.data(), + static_cast(kRopeColumns), + count, + static_cast(kRopeColumns)), + "ryzenai_corelib_convert_strided"); + } catch (...) { + if (inserted) { + fp32_buffers_.erase(buffer); + } + throw; + } + return buffer->second; +} + +} // namespace flm::phi4 diff --git a/src/include/models/phi4/phi4_corelib_manifest.hpp b/src/include/models/phi4/phi4_corelib_manifest.hpp new file mode 100644 index 00000000..f82f560d --- /dev/null +++ b/src/include/models/phi4/phi4_corelib_manifest.hpp @@ -0,0 +1,127 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace flm::phi4 { + +class MappedFile final { +public: + static std::shared_ptr OpenReadOnly( + const std::filesystem::path& path); + + MappedFile(MappedFile&& other) noexcept; + MappedFile& operator=(MappedFile&& other) noexcept; + ~MappedFile() noexcept; + + MappedFile(const MappedFile&) = delete; + MappedFile& operator=(const MappedFile&) = delete; + + const std::byte* data() const noexcept; + std::uint64_t size() const noexcept; + const std::filesystem::path& path() const noexcept; + +private: + MappedFile( + std::filesystem::path path, + void* file, + void* mapping, + const std::byte* data, + std::uint64_t size) noexcept; + + void Reset() noexcept; + + std::filesystem::path path_; + void* file_ = nullptr; + void* mapping_ = nullptr; + const std::byte* data_ = nullptr; + std::uint64_t size_ = 0; +}; + +enum class SourceDType { + UInt8, + Float16, + Float32, + Int64 +}; + +struct InitializerView { + SourceDType dtype; + std::vector shape; + const std::byte* data; + std::size_t size; + std::shared_ptr owner; +}; + +enum class WeightObjectKind { + MatMul, + SsMlp +}; + +struct WeightObjectView { + std::string name; + WeightObjectKind kind; + std::int64_t k; + std::int64_t n; + std::uint32_t group_size; + bool has_bias; + std::map components; +}; + +class Phi4Package final { +public: + static Phi4Package Load( + const std::filesystem::path& model_dir, + std::shared_ptr api, + bool verify_full_hash); + + Phi4Package(Phi4Package&&) noexcept = default; + Phi4Package& operator=(Phi4Package&&) noexcept = default; + ~Phi4Package() = default; + + Phi4Package(const Phi4Package&) = delete; + Phi4Package& operator=(const Phi4Package&) = delete; + + const InitializerView& Require(std::string_view name) const; + const std::vector& weight_objects() const noexcept; + std::span MaterializeFp16( + std::string_view name); + std::span MaterializeBf16( + std::string_view name); + std::span MaterializeRopeFp32( + std::string_view name); + +private: + Phi4Package() = default; + + std::shared_ptr api_; + + // Declare every owner before the non-owning views. Reverse member + // destruction then releases views before their mapped/derived storage. + std::map< + std::string, + std::shared_ptr, + std::less<>> + mapped_files_; + std::map, std::less<>> + fp16_buffers_; + std::map, std::less<>> + bf16_buffers_; + std::map, std::less<>> + fp32_buffers_; + + std::map> + initializers_; + std::vector weight_objects_; +}; + +} // namespace flm::phi4 diff --git a/src/test/phi4_corelib_aie4/CMakeLists.txt b/src/test/phi4_corelib_aie4/CMakeLists.txt index 1775eae7..7e3497cd 100644 --- a/src/test/phi4_corelib_aie4/CMakeLists.txt +++ b/src/test/phi4_corelib_aie4/CMakeLists.txt @@ -70,6 +70,7 @@ endfunction() enable_testing() add_corelib_host_test(test_corelib_api test_corelib_api.cpp) +add_corelib_host_test(test_phi4_manifest test_phi4_manifest.cpp) add_corelib_host_test( test_corelib_fatal_record test_corelib_fatal_record.cpp) diff --git a/src/test/phi4_corelib_aie4/test_phi4_manifest.cpp b/src/test/phi4_corelib_aie4/test_phi4_manifest.cpp new file mode 100644 index 00000000..0f8e362b --- /dev/null +++ b/src/test/phi4_corelib_aie4/test_phi4_manifest.cpp @@ -0,0 +1,1196 @@ +#include "fake_corelib.hpp" +#include "test_support.hpp" + +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using flm::corelib::CorelibApi; +using flm::phi4::InitializerView; +using flm::phi4::MappedFile; +using flm::phi4::Phi4Package; +using flm::phi4::SourceDType; +using flm::phi4::WeightObjectKind; +using nlohmann::json; + +constexpr std::string_view kManifestName = "corelib_phi4_manifest.json"; +constexpr std::string_view kDataFile = "z-data.bin"; +constexpr std::string_view kRopeFile = "z-rope.bin"; +constexpr std::uint64_t kDataBytes = + 200064ull * 3072ull * sizeof(std::uint16_t); +constexpr std::size_t kRopeRows = 4096; +constexpr std::size_t kRopeColumns = 64; +constexpr std::uint64_t kRopeBytes = + kRopeRows * kRopeColumns * sizeof(std::uint16_t); +constexpr std::uint64_t kRopeMappedBytes = kRopeBytes + 4096; +constexpr std::uint64_t kFp32ScaleOffset = 4096; +constexpr std::uint64_t kNormOffset = 1024 * 1024; +constexpr std::uint64_t kSinOffset = 2 * 1024 * 1024; + +constexpr std::string_view kFp32Scale = + "model.layers.0.attn.q_proj.MatMulNBits.scales"; +constexpr std::string_view kFp16Scale = + "model.layers.0.attn.k_proj.MatMulNBits.scales"; +constexpr std::string_view kFp32Norm = + "model.layers.0.post_attention_layernorm.weight"; + +struct ConvertRecord { + ryzenai_corelib_data_type source_type = + ryzenai_corelib_data_type_fp32; + const void* source = nullptr; + std::size_t src_stride = 0; + ryzenai_corelib_data_type destination_type = + ryzenai_corelib_data_type_fp32; + void* destination = nullptr; + std::size_t dst_stride = 0; + std::size_t count = 0; + std::size_t row = 0; +}; + +ConvertRecord g_last_convert; +std::size_t g_convert_calls = 0; + +float HalfToFloat(std::uint16_t value) { + const bool negative = (value & 0x8000u) != 0; + const unsigned exponent = (value >> 10) & 0x1fu; + const unsigned mantissa = value & 0x03ffu; + float result = 0.0f; + if (exponent == 0) { + result = std::ldexp(static_cast(mantissa), -24); + } else if (exponent == 31) { + result = mantissa == 0 + ? std::numeric_limits::infinity() + : std::numeric_limits::quiet_NaN(); + } else { + result = std::ldexp( + 1.0f + static_cast(mantissa) / 1024.0f, + static_cast(exponent) - 15); + } + return negative ? -result : result; +} + +std::uint16_t FloatToHalf(float value) { + const std::uint32_t bits = std::bit_cast(value); + const std::uint16_t sign = + static_cast((bits >> 16) & 0x8000u); + const std::uint32_t source_exponent = (bits >> 23) & 0xffu; + const std::uint32_t source_mantissa = bits & 0x007fffffu; + + if (source_exponent == 0xffu) { + return static_cast( + sign | (source_mantissa == 0 ? 0x7c00u : 0x7e00u)); + } + + const int exponent = static_cast(source_exponent) - 127 + 15; + if (exponent >= 31) { + return static_cast(sign | 0x7c00u); + } + if (exponent <= 0) { + if (exponent < -10) { + return sign; + } + const std::uint32_t mantissa = source_mantissa | 0x00800000u; + const unsigned shift = static_cast(14 - exponent); + const std::uint32_t halfway = 1u << (shift - 1); + const std::uint32_t rounded = + (mantissa + halfway - 1u + ((mantissa >> shift) & 1u)) >> + shift; + return static_cast(sign | rounded); + } + + const std::uint32_t rounded = + source_mantissa + 0x00000fffu + + ((source_mantissa >> 13) & 1u); + if ((rounded & 0x00800000u) != 0) { + if (exponent + 1 >= 31) { + return static_cast(sign | 0x7c00u); + } + return static_cast( + sign | (static_cast(exponent + 1) << 10)); + } + return static_cast( + sign | (static_cast(exponent) << 10) | + (rounded >> 13)); +} + +std::uint16_t FloatToBf16(float value) { + std::uint32_t bits = std::bit_cast(value); + bits += 0x7fffu + ((bits >> 16) & 1u); + return static_cast(bits >> 16); +} + +float ReadElement( + ryzenai_corelib_data_type type, + const void* source, + std::size_t index) { + if (type == ryzenai_corelib_data_type_fp16) { + return HalfToFloat( + static_cast(source)[index]); + } + if (type == ryzenai_corelib_data_type_fp32) { + return static_cast(source)[index]; + } + throw std::runtime_error("test converter received unsupported source type"); +} + +void WriteElement( + ryzenai_corelib_data_type type, + void* destination, + std::size_t index, + float value) { + if (type == ryzenai_corelib_data_type_fp16) { + static_cast(destination)[index] = + FloatToHalf(value); + return; + } + if (type == ryzenai_corelib_data_type_bf16) { + static_cast(destination)[index] = + FloatToBf16(value); + return; + } + if (type == ryzenai_corelib_data_type_fp32) { + static_cast(destination)[index] = value; + return; + } + throw std::runtime_error( + "test converter received unsupported destination type"); +} + +ryzenai_corelib_status RecordingConvert( + ryzenai_corelib_data_type source_type, + const void* source, + ryzenai_corelib_data_type destination_type, + void* destination, + std::size_t count) { + g_last_convert = { + source_type, + source, + 0, + destination_type, + destination, + 0, + count, + 0}; + ++g_convert_calls; + for (std::size_t index = 0; index < count; ++index) { + WriteElement( + destination_type, + destination, + index, + ReadElement(source_type, source, index)); + } + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status RecordingConvertStrided( + ryzenai_corelib_data_type source_type, + const void* source, + std::size_t source_stride, + ryzenai_corelib_data_type destination_type, + void* destination, + std::size_t destination_stride, + std::size_t count, + std::size_t row) { + g_last_convert = { + source_type, + source, + source_stride, + destination_type, + destination, + destination_stride, + count, + row}; + ++g_convert_calls; + if ( + row == 0 || count % row != 0 || source_stride < row || + destination_stride < row) { + return ryzenai_corelib_status_bad_argument; + } + const std::size_t rows = count / row; + for (std::size_t source_row = 0; source_row < rows; ++source_row) { + for (std::size_t column = 0; column < row; ++column) { + WriteElement( + destination_type, + destination, + source_row * destination_stride + column, + ReadElement( + source_type, + source, + source_row * source_stride + column)); + } + } + return ryzenai_corelib_status_success; +} + +template +void* FunctionAddress(Function function) { + return reinterpret_cast(function); +} + +std::shared_ptr ResolveRecordingCorelib() { + auto resolver = flm::test::CompleteCorelibResolver(); + resolver["ryzenai_corelib_convert"] = FunctionAddress( + static_cast( + &RecordingConvert)); + resolver["ryzenai_corelib_convert_strided"] = FunctionAddress( + static_cast( + &RecordingConvertStrided)); + return CorelibApi::ResolveForTest( + [resolver = std::move(resolver)](std::string_view name) mutable + -> void* { + const auto found = resolver.find(std::string(name)); + return found == resolver.end() ? nullptr : found->second; + }); +} + +class TempDirectory final { +public: + TempDirectory() { + const auto nonce = + std::chrono::steady_clock::now().time_since_epoch().count(); + path_ = std::filesystem::temp_directory_path() / + ("fastflowlm-phi4-manifest-" + + std::to_string(GetCurrentProcessId()) + "-" + + std::to_string(nonce)); + std::filesystem::create_directories(path_); + } + + ~TempDirectory() noexcept { + std::error_code error; + std::filesystem::remove_all(path_, error); + } + + TempDirectory(const TempDirectory&) = delete; + TempDirectory& operator=(const TempDirectory&) = delete; + + const std::filesystem::path& path() const noexcept { + return path_; + } + +private: + std::filesystem::path path_; +}; + +void CreateSparseFile( + const std::filesystem::path& path, + std::uint64_t size) { + HANDLE file = CreateFileW( + path.c_str(), + GENERIC_READ | GENERIC_WRITE, + FILE_SHARE_READ, + nullptr, + CREATE_ALWAYS, + FILE_ATTRIBUTE_NORMAL, + nullptr); + if (file == INVALID_HANDLE_VALUE) { + throw std::runtime_error("failed to create sparse test file"); + } + + DWORD ignored = 0; + DeviceIoControl( + file, + FSCTL_SET_SPARSE, + nullptr, + 0, + nullptr, + 0, + &ignored, + nullptr); + LARGE_INTEGER end{}; + end.QuadPart = static_cast(size); + const bool success = + SetFilePointerEx(file, end, nullptr, FILE_BEGIN) != FALSE && + SetEndOfFile(file) != FALSE; + CloseHandle(file); + if (!success) { + throw std::runtime_error("failed to size sparse test file"); + } +} + +template +void WriteValues( + const std::filesystem::path& path, + std::uint64_t offset, + std::span values) { + std::fstream file( + path, + std::ios::in | std::ios::out | std::ios::binary); + if (!file) { + throw std::runtime_error("failed to open synthetic data file"); + } + file.seekp(static_cast(offset)); + file.write( + reinterpret_cast(values.data()), + static_cast(values.size_bytes())); + if (!file) { + throw std::runtime_error("failed to write synthetic data"); + } +} + +std::uint64_t ItemSize(std::string_view dtype) { + if (dtype == "uint8") { + return 1; + } + if (dtype == "float16") { + return 2; + } + if (dtype == "float32") { + return 4; + } + if (dtype == "int64") { + return 8; + } + throw std::runtime_error("unsupported synthetic dtype"); +} + +std::uint64_t ByteLength( + std::string_view dtype, + const std::vector& shape) { + std::uint64_t elements = 1; + for (const std::int64_t dimension : shape) { + elements *= static_cast(dimension); + } + return elements * ItemSize(dtype); +} + +void AddInitializer( + json& initializers, + std::string name, + std::string dtype, + std::vector shape, + std::string role, + std::string file = std::string(kDataFile), + std::uint64_t offset = 0) { + CHECK(!initializers.contains(name)); + const std::uint64_t length = ByteLength(dtype, shape); + initializers[std::move(name)] = { + {"file", std::move(file)}, + {"offset", offset}, + {"length", length}, + {"dtype", std::move(dtype)}, + {"shape", std::move(shape)}, + {"role", std::move(role)}}; +} + +void AddMatMul( + json& manifest, + const std::string& name, + std::int64_t k, + std::int64_t n) { + json roles = { + {"qweight", name + ".qweight"}, + {"scales", name + ".scales"}, + {"qzeros", name + ".qzeros"}}; + manifest["weight_objects"].push_back({ + {"name", name}, + {"kind", "matmul"}, + {"descriptor", + { + {"k", k}, + {"n", n}, + {"group_size", 128}, + {"has_bias", false}, + }}, + {"roles", roles}}); + + auto& initializers = manifest["initializers"]; + AddInitializer( + initializers, + roles["qweight"].get(), + "uint8", + {n, k / 2}, + "matmul.qweight"); + + std::string scale_dtype = "float16"; + std::uint64_t scale_offset = 0; + if (name == "model.layers.0.attn.q_proj.MatMulNBits") { + scale_dtype = "float32"; + scale_offset = kFp32ScaleOffset; + } + AddInitializer( + initializers, + roles["scales"].get(), + scale_dtype, + {n, k / 128}, + "matmul.scales", + std::string(kDataFile), + scale_offset); + AddInitializer( + initializers, + roles["qzeros"].get(), + "uint8", + {n, ((k / 128) + 1) / 2}, + "matmul.qzeros"); +} + +void AddSsMlp(json& manifest, int layer) { + const std::string base = + "model.layers." + std::to_string(layer); + const std::string object_name = base + ".ssmlp"; + const std::string norm0 = + base + ".post_attention_layernorm.weight"; + const std::string norm1 = + layer == 31 + ? "model.layers.32.final_norm_layernorm.weight" + : "model.layers." + std::to_string(layer + 1) + + ".input_layernorm.weight"; + + json roles = { + {"norm0", norm0}, + {"norm1", norm1}, + }; + auto& initializers = manifest["initializers"]; + for (const std::string projection : {"gate", "up", "down"}) { + const std::int64_t k = projection == "down" ? 8192 : 3072; + const std::int64_t n = projection == "down" ? 3072 : 8192; + const std::string prefix = + base + ".mlp." + projection + "_proj.MatMulNBits"; + for (const std::string component : + {"qweight", "scales", "qzeros"}) { + roles[projection + "_" + component] = + prefix + "." + component; + } + AddInitializer( + initializers, + prefix + ".qweight", + "uint8", + {n, k / 2}, + "ssmlp." + projection + ".qweight"); + AddInitializer( + initializers, + prefix + ".scales", + "float16", + {n, k / 128}, + "ssmlp." + projection + ".scales"); + AddInitializer( + initializers, + prefix + ".qzeros", + "uint8", + {n, ((k / 128) + 1) / 2}, + "ssmlp." + projection + ".qzeros"); + } + + AddInitializer( + initializers, + norm0, + layer == 0 ? "float32" : "float16", + {3072}, + "ssmlp.norm0", + std::string(kDataFile), + layer == 0 ? kNormOffset : 0); + AddInitializer( + initializers, + norm1, + "float16", + {3072}, + "ssmlp.norm1"); + + manifest["weight_objects"].push_back({ + {"name", object_name}, + {"kind", "ssmlp"}, + {"descriptor", + { + {"k", 3072}, + {"n", 8192}, + {"group_size", 128}, + }}, + {"roles", std::move(roles)}}); +} + +json BuildManifest(std::uint64_t model_size) { + json manifest = { + {"schema_version", 1}, + {"execution_backend", "corelib_aie4"}, + {"model", + { + {"family", "phi4"}, + {"layers", 32}, + {"hidden_size", 3072}, + {"intermediate_size", 8192}, + {"num_heads", 24}, + {"kv_heads", 8}, + {"head_size", 128}, + {"vocab_size", 200064}, + {"group_size", 128}, + {"rope_dim", 96}, + {"rms_epsilon", 0.00001}, + }}, + {"backend", {{"max_seq", 4096}}}, + {"files", + { + {"model.onnx", {{"size", model_size}}}, + {std::string(kDataFile), {{"size", kDataBytes}}}, + {std::string(kRopeFile), {{"size", kRopeMappedBytes}}}, + }}, + {"initializers", json::object()}, + {"weight_objects", json::array()}, + }; + + for (int layer = 0; layer < 32; ++layer) { + const std::string base = + "model.layers." + std::to_string(layer) + ".attn."; + AddMatMul( + manifest, + base + "q_proj.MatMulNBits", + 3072, + 3072); + AddMatMul( + manifest, + base + "k_proj.MatMulNBits", + 3072, + 1024); + AddMatMul( + manifest, + base + "v_proj.MatMulNBits", + 3072, + 1024); + AddMatMul( + manifest, + base + "o_proj.MatMulNBits", + 3072, + 3072); + AddSsMlp(manifest, layer); + } + AddMatMul(manifest, "lm_head.MatMulNBits", 3072, 200064); + + AddInitializer( + manifest["initializers"], + "model.embed_tokens.weight", + "float16", + {200064, 3072}, + "embedding"); + AddInitializer( + manifest["initializers"], + "model.layers.0.input_layernorm.weight", + "float32", + {3072}, + "input_norm", + std::string(kDataFile), + kNormOffset); + AddInitializer( + manifest["initializers"], + "cos_cache", + "float16", + {4096, 64}, + "cos_cache", + std::string(kRopeFile)); + AddInitializer( + manifest["initializers"], + "sin_cache", + "float32", + {4096, 48}, + "sin_cache", + std::string(kDataFile), + kSinOffset); + + CHECK(manifest["weight_objects"].size() == 161); + CHECK(manifest["initializers"].size() == 743); + return manifest; +} + +class SyntheticPackage final { +public: + SyntheticPackage() { + const auto model_path = temp_.path() / "model.onnx"; + { + std::ofstream model(model_path, std::ios::binary); + model << "model"; + } + CreateSparseFile(temp_.path() / kDataFile, kDataBytes); + CreateSparseFile( + temp_.path() / kRopeFile, + kRopeMappedBytes); + + const std::array half_values{ + 0x3c00u, + 0xc000u}; + const std::array float_values{1.0f, -2.0f}; + WriteValues( + temp_.path() / kDataFile, + 0, + std::span(half_values)); + WriteValues( + temp_.path() / kDataFile, + kFp32ScaleOffset, + std::span(float_values)); + WriteValues( + temp_.path() / kDataFile, + kNormOffset, + std::span(float_values)); + const std::array sin_value{4.0f}; + WriteValues( + temp_.path() / kDataFile, + kSinOffset, + std::span(sin_value)); + + const std::array one{0x3c00u}; + const std::array two{0x4000u}; + const std::array three{0x4200u}; + WriteValues( + temp_.path() / kRopeFile, + 0, + std::span(one)); + WriteValues( + temp_.path() / kRopeFile, + kRopeColumns * sizeof(std::uint16_t), + std::span(two)); + WriteValues( + temp_.path() / kRopeFile, + ((kRopeRows - 1) * kRopeColumns + 47) * + sizeof(std::uint16_t), + std::span(three)); + + manifest_ = BuildManifest( + std::filesystem::file_size(model_path)); + Write(manifest_); + } + + const std::filesystem::path& path() const noexcept { + return temp_.path(); + } + + json manifest() const { + return manifest_; + } + + void Write(const json& manifest) const { + std::ofstream stream( + temp_.path() / kManifestName, + std::ios::binary | std::ios::trunc); + if (!stream) { + throw std::runtime_error("failed to write synthetic manifest"); + } + stream << manifest.dump(2) << '\n'; + } + +private: + TempDirectory temp_; + json manifest_; +}; + +void RenameFile( + json& manifest, + std::string_view old_name, + std::string new_name) { + auto record = manifest["files"].at(std::string(old_name)); + manifest["files"].erase(std::string(old_name)); + manifest["files"][new_name] = std::move(record); + for (auto& [_, initializer] : + manifest["initializers"].items()) { + if ( + initializer["file"].get() == + std::string(old_name)) { + initializer["file"] = new_name; + } + } +} + +template +void ExpectLoadFailure( + const SyntheticPackage& fixture, + const std::shared_ptr& api, + Mutation&& mutation, + std::string_view expected, + bool verify_full_hash = false) { + json manifest = fixture.manifest(); + std::invoke( + std::forward(mutation), + manifest); + fixture.Write(manifest); + try { + (void)Phi4Package::Load( + fixture.path(), + api, + verify_full_hash); + } catch (const std::exception& error) { + if ( + std::string_view(error.what()).find(expected) == + std::string_view::npos) { + throw std::runtime_error( + "expected load failure containing '" + + std::string(expected) + "', got: " + error.what()); + } + return; + } + throw std::runtime_error( + "expected package load failure containing '" + + std::string(expected) + "'"); +} + +void TestValidMappingAndExplicitGrouping( + const SyntheticPackage& fixture, + const std::shared_ptr& api) { + json manifest = fixture.manifest(); + const std::string original = + "model.layers.0.attn.q_proj.MatMulNBits.qweight"; + const std::string opaque = "opaque-initializer-000"; + manifest["initializers"][opaque] = + manifest["initializers"].at(original); + manifest["initializers"].erase(original); + manifest["weight_objects"][0]["roles"]["qweight"] = opaque; + fixture.Write(manifest); + + auto package = Phi4Package::Load(fixture.path(), api, false); + CHECK(package.weight_objects().size() == 161); + const auto& first = package.weight_objects().front(); + CHECK(first.name == + "model.layers.0.attn.q_proj.MatMulNBits"); + CHECK(first.kind == WeightObjectKind::MatMul); + CHECK(first.k == 3072); + CHECK(first.n == 3072); + CHECK(first.group_size == 128); + CHECK(!first.has_bias); + CHECK(first.components.at("qweight") == opaque); + + const auto& last = package.weight_objects().back(); + CHECK(last.name == "lm_head.MatMulNBits"); + CHECK(last.n == 200064); + + const auto& embedding = + package.Require("model.embed_tokens.weight"); + CHECK(embedding.dtype == SourceDType::Float16); + CHECK(embedding.shape == + std::vector({200064, 3072})); + CHECK(embedding.size == kDataBytes); + CHECK(embedding.data != nullptr); + CHECK(embedding.owner != nullptr); + CHECK(embedding.owner->path().filename() == kDataFile); + CHECK(embedding.owner->size() == kDataBytes); + CheckThrowsContains( + [&] { + (void)package.Require("missing.initializer"); + }, + "missing initializer"); +} + +void TestMappedOwnerOutlivesPackage( + const SyntheticPackage& fixture, + const std::shared_ptr& api) { + fixture.Write(fixture.manifest()); + std::optional retained; + { + auto package = + Phi4Package::Load(fixture.path(), api, false); + retained = package.Require(kFp16Scale); + CHECK(retained->owner.use_count() > 1); + } + CHECK(retained->owner.use_count() == 1); + CHECK(retained->data != nullptr); + CHECK( + *reinterpret_cast(retained->data) == + 0x3c00u); + + auto direct = + MappedFile::OpenReadOnly(fixture.path() / "model.onnx"); + CHECK(direct->size() == 5); + CHECK(direct->path() == fixture.path() / "model.onnx"); + CHECK(direct->data()[0] == std::byte{'m'}); +} + +void TestPathRangeAndHashRejections( + const SyntheticPackage& fixture, + const std::shared_ptr& api) { + ExpectLoadFailure( + fixture, + api, + [](json& manifest) { + RenameFile(manifest, kDataFile, "C:/outside.bin"); + }, + "relative"); + ExpectLoadFailure( + fixture, + api, + [](json& manifest) { + RenameFile(manifest, kDataFile, "../outside.bin"); + }, + "traversal"); + ExpectLoadFailure( + fixture, + api, + [](json& manifest) { + auto& record = manifest["initializers"].at( + "model.embed_tokens.weight"); + record["offset"] = + std::numeric_limits::max() - 1; + }, + "range overflow"); + ExpectLoadFailure( + fixture, + api, + [](json& manifest) { + auto& record = + manifest["initializers"].at(std::string(kFp16Scale)); + record["length"] = + record["length"].get() - 1; + }, + "byte count"); + ExpectLoadFailure( + fixture, + api, + [](json& manifest) { + manifest["initializers"] + .at(std::string(kFp16Scale))["offset"] = 1; + }, + "dtype-aligned"); + ExpectLoadFailure( + fixture, + api, + [](json& manifest) { + manifest["files"].at(std::string(kDataFile))["size"] = + kDataBytes - 1; + }, + "file size"); + ExpectLoadFailure( + fixture, + api, + [](json& manifest) { + for (auto& [_, record] : manifest["files"].items()) { + record["sha256"] = std::string(64, '0'); + } + }, + "SHA-256", + true); +} + +void TestWeightObjectRejections( + const SyntheticPackage& fixture, + const std::shared_ptr& api) { + ExpectLoadFailure( + fixture, + api, + [](json& manifest) { + manifest.erase("weight_objects"); + }, + "weight_objects"); + ExpectLoadFailure( + fixture, + api, + [](json& manifest) { + manifest["weight_objects"] = json::array(); + }, + "weight_objects"); + ExpectLoadFailure( + fixture, + api, + [](json& manifest) { + manifest["weight_objects"].erase( + manifest["weight_objects"].end() - 1); + }, + "161"); + ExpectLoadFailure( + fixture, + api, + [](json& manifest) { + manifest["weight_objects"][1]["name"] = + manifest["weight_objects"][0]["name"]; + }, + "duplicate weight object"); + ExpectLoadFailure( + fixture, + api, + [](json& manifest) { + manifest["weight_objects"][0]["kind"] = "convolution"; + }, + "kind"); + ExpectLoadFailure( + fixture, + api, + [](json& manifest) { + manifest["weight_objects"][0]["descriptor"].erase("n"); + }, + "descriptor"); + ExpectLoadFailure( + fixture, + api, + [](json& manifest) { + manifest["weight_objects"][0]["descriptor"]["n"] = 1024; + }, + "descriptor"); + ExpectLoadFailure( + fixture, + api, + [](json& manifest) { + manifest["weight_objects"][0]["roles"]["mystery"] = + "cos_cache"; + }, + "role"); + ExpectLoadFailure( + fixture, + api, + [](json& manifest) { + const std::string target = + manifest["weight_objects"][0]["roles"]["qweight"]; + manifest["initializers"]["unreferenced-placeholder"] = + manifest["initializers"].at(target); + manifest["initializers"].erase(target); + }, + "unresolved initializer"); + ExpectLoadFailure( + fixture, + api, + [](json& manifest) { + manifest["weight_objects"][0]["roles"]["scales"] = + manifest["weight_objects"][0]["roles"]["qweight"]; + }, + "duplicate initializer"); +} + +void TestExactSourceValidation( + const SyntheticPackage& fixture, + const std::shared_ptr& api) { + ExpectLoadFailure( + fixture, + api, + [](json& manifest) { + auto& record = manifest["initializers"].at( + "model.embed_tokens.weight"); + record["dtype"] = "uint8"; + record["length"] = + 200064ull * 3072ull; + }, + "embedding"); + ExpectLoadFailure( + fixture, + api, + [](json& manifest) { + auto& record = manifest["initializers"].at( + "model.layers.0.input_layernorm.weight"); + record["dtype"] = "uint8"; + record["length"] = 3072; + }, + "input_norm"); + ExpectLoadFailure( + fixture, + api, + [](json& manifest) { + auto& record = + manifest["initializers"].at("cos_cache"); + record["shape"] = {4095, 64}; + record["length"] = + 4095ull * 64ull * sizeof(std::uint16_t); + }, + "cos_cache"); + ExpectLoadFailure( + fixture, + api, + [](json& manifest) { + const std::string name = + manifest["weight_objects"][0]["roles"]["qweight"]; + auto& record = manifest["initializers"].at(name); + record["shape"] = {3071, 1536}; + record["length"] = 3071ull * 1536ull; + }, + "qweight"); + ExpectLoadFailure( + fixture, + api, + [](json& manifest) { + const std::string name = + manifest["weight_objects"][0]["roles"]["qweight"]; + manifest["initializers"][name]["role"] = "unknown.role"; + }, + "semantic role"); +} + +void TestOwnedScaleAndNormConversions( + const SyntheticPackage& fixture, + const std::shared_ptr& api) { + fixture.Write(fixture.manifest()); + auto package = Phi4Package::Load(fixture.path(), api, false); + + g_convert_calls = 0; + const auto fp16 = package.MaterializeFp16(kFp16Scale); + CHECK(g_convert_calls == 1); + CHECK(g_last_convert.source_type == + ryzenai_corelib_data_type_fp16); + CHECK(g_last_convert.destination_type == + ryzenai_corelib_data_type_fp16); + CHECK(g_last_convert.count == 1024u * 24u); + CHECK(fp16.size() == 1024u * 24u); + CHECK(fp16[0] == 0x3c00u); + CHECK(fp16[1] == 0xc000u); + CHECK( + reinterpret_cast(fp16.data()) != + g_last_convert.source); + const auto* fp16_address = fp16.data(); + + const auto same_fp16 = package.MaterializeFp16(kFp16Scale); + CHECK(g_convert_calls == 1); + CHECK(same_fp16.data() == fp16_address); + + const auto converted = package.MaterializeFp16(kFp32Scale); + CHECK(g_convert_calls == 2); + CHECK(g_last_convert.source_type == + ryzenai_corelib_data_type_fp32); + CHECK(g_last_convert.destination_type == + ryzenai_corelib_data_type_fp16); + CHECK(g_last_convert.count == 3072u * 24u); + CHECK(converted[0] == 0x3c00u); + CHECK(converted[1] == 0xc000u); + CHECK(fp16.data() == fp16_address); + + const auto bf16 = package.MaterializeBf16(kFp32Norm); + CHECK(g_convert_calls == 3); + CHECK(g_last_convert.source_type == + ryzenai_corelib_data_type_fp32); + CHECK(g_last_convert.destination_type == + ryzenai_corelib_data_type_bf16); + CHECK(g_last_convert.count == 3072); + CHECK(bf16[0] == 0x3f80u); + CHECK(bf16[1] == 0xc000u); + + CheckThrowsContains( + [&] { + (void)package.MaterializeFp16( + "model.layers.0.attn.q_proj.MatMulNBits.qweight"); + }, + "floating"); +} + +class NoAccessGuard final { +public: + explicit NoAccessGuard(void* address) { + MEMORY_BASIC_INFORMATION region{}; + if ( + VirtualQuery(address, ®ion, sizeof(region)) != + sizeof(region) || + region.State != MEM_COMMIT) { + throw std::runtime_error("failed to query guard-page address"); + } + address_ = address; + if ( + VirtualProtect( + address, + 4096, + PAGE_NOACCESS, + &old_protection_) == FALSE) { + throw std::runtime_error( + "failed to protect no-access guard page"); + } + } + + ~NoAccessGuard() noexcept { + if (address_ != nullptr) { + DWORD ignored = 0; + VirtualProtect( + address_, + 4096, + old_protection_, + &ignored); + } + } + + NoAccessGuard(const NoAccessGuard&) = delete; + NoAccessGuard& operator=(const NoAccessGuard&) = delete; + +private: + void* address_ = nullptr; + DWORD old_protection_ = 0; +}; + +void TestRopeUsesExactStridedContractAtGuardPage( + const SyntheticPackage& fixture, + const std::shared_ptr& api) { + fixture.Write(fixture.manifest()); + auto package = Phi4Package::Load(fixture.path(), api, false); + const auto& source = package.Require("cos_cache"); + CHECK(source.size == kRopeBytes); + CHECK(source.owner->size() == kRopeMappedBytes); + CHECK(source.data == source.owner->data()); + + auto* one_past = + const_cast(source.data + source.size); + NoAccessGuard guard(one_past); + + g_convert_calls = 0; + const auto rope = package.MaterializeRopeFp32("cos_cache"); + CHECK(g_convert_calls == 1); + CHECK(g_last_convert.source_type == + ryzenai_corelib_data_type_fp16); + CHECK(g_last_convert.destination_type == + ryzenai_corelib_data_type_fp32); + CHECK(g_last_convert.row == 48); + CHECK(g_last_convert.src_stride == kRopeColumns); + CHECK(g_last_convert.dst_stride == 48); + CHECK(g_last_convert.count == 4096u * 48u); + CHECK(rope.size() == 4096u * 48u); + CHECK(rope[0] == 1.0f); + CHECK(rope[48] == 2.0f); + CHECK(rope.back() == 3.0f); + + const auto* address = rope.data(); + const auto again = package.MaterializeRopeFp32("cos_cache"); + CHECK(g_convert_calls == 1); + CHECK(again.data() == address); +} + +void TestFp32RopeSource( + const SyntheticPackage& fixture, + const std::shared_ptr& api) { + fixture.Write(fixture.manifest()); + auto package = Phi4Package::Load(fixture.path(), api, false); + + g_convert_calls = 0; + const auto rope = package.MaterializeRopeFp32("sin_cache"); + CHECK(g_convert_calls == 1); + CHECK(g_last_convert.source_type == + ryzenai_corelib_data_type_fp32); + CHECK(g_last_convert.src_stride == 48); + CHECK(g_last_convert.dst_stride == 48); + CHECK(g_last_convert.row == 48); + CHECK(g_last_convert.count == 196608); + CHECK(rope[0] == 4.0f); +} + +static_assert(!std::is_copy_constructible_v); +static_assert(!std::is_copy_assignable_v); +static_assert(std::is_nothrow_move_constructible_v); +static_assert(std::is_nothrow_move_assignable_v); + +} // namespace + +int main() { + try { + SyntheticPackage fixture; + auto api = ResolveRecordingCorelib(); + TestValidMappingAndExplicitGrouping(fixture, api); + TestMappedOwnerOutlivesPackage(fixture, api); + TestPathRangeAndHashRejections(fixture, api); + TestWeightObjectRejections(fixture, api); + TestExactSourceValidation(fixture, api); + TestOwnedScaleAndNormConversions(fixture, api); + TestRopeUsesExactStridedContractAtGuardPage(fixture, api); + TestFp32RopeSource(fixture, api); + std::cout << "test_phi4_manifest: PASS\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} From 2e6212738bfa4a3d8c906b9beb12efec88b308ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CChi?= Date: Mon, 31 Aug 2026 22:46:19 -0700 Subject: [PATCH 007/117] fix: identify invalid Phi-4 weight components Co-authored-by: Cursor --- src/common/corelib/phi4_corelib_manifest.cpp | 56 ++++++++++++-- .../phi4_corelib_aie4/test_phi4_manifest.cpp | 75 +++++++++++++++++++ 2 files changed, 123 insertions(+), 8 deletions(-) diff --git a/src/common/corelib/phi4_corelib_manifest.cpp b/src/common/corelib/phi4_corelib_manifest.cpp index 99573f26..05159086 100644 --- a/src/common/corelib/phi4_corelib_manifest.cpp +++ b/src/common/corelib/phi4_corelib_manifest.cpp @@ -430,8 +430,23 @@ std::set JsonKeys(const json& value) { return keys; } +std::string ComponentContext( + std::string_view object_name, + std::string_view role_name) { + return std::string(object_name) + "." + std::string(role_name); +} + +std::string ComponentContext( + std::string_view object_name, + std::string_view role_name, + std::string_view initializer_name) { + return ComponentContext(object_name, role_name) + " (" + + std::string(initializer_name) + ")"; +} + void ValidateQuantizedProjection( const Phi4Package& package, + std::string_view object_name, const std::map>& semantic_roles, const std::map& components, std::string_view role_prefix, @@ -454,13 +469,17 @@ void ValidateQuantizedProjection( std::string(component_prefix) + std::string(component); const auto found = components.find(role_name); if (found == components.end()) { - Throw(role_prefix, "is missing a component role"); + Throw( + ComponentContext(object_name, role_name), + "is missing a component role"); } const auto& view = package.Require(found->second); + const std::string context = + ComponentContext(object_name, role_name, found->second); if (view.dtype != dtype) { - Throw(role_name, "has an invalid dtype"); + Throw(context, "has an invalid dtype"); } - RequireShape(view, shape, role_name); + RequireShape(view, shape, context); RequireSemanticRole( semantic_roles, found->second, @@ -477,11 +496,17 @@ void ValidateQuantizedProjection( std::string(component_prefix) + "scales"; const auto scales_component = components.find(scales_name); if (scales_component == components.end()) { - Throw(role_prefix, "is missing a scales role"); + Throw( + ComponentContext(object_name, scales_name), + "is missing a scales role"); } const auto& scales = package.Require(scales_component->second); - RequireFloating(scales, scales_name); - RequireShape(scales, {n, groups}, scales_name); + const std::string scales_context = ComponentContext( + object_name, + scales_name, + scales_component->second); + RequireFloating(scales, scales_context); + RequireShape(scales, {n, groups}, scales_context); RequireSemanticRole( semantic_roles, scales_component->second, @@ -1081,6 +1106,13 @@ Phi4Package Phi4Package::Load( kind == WeightObjectKind::MatMul ? MatMulRoleNames() : SsMlpRoleNames(); + for (const auto& role : expected_roles) { + if (!role_map.contains(role)) { + Throw( + ComponentContext(name, role), + "is missing a component role"); + } + } if (JsonKeys(role_map) != expected_roles) { Throw(name, "invalid weight object role map"); } @@ -1110,6 +1142,7 @@ Phi4Package Phi4Package::Load( if (kind == WeightObjectKind::MatMul) { ValidateQuantizedProjection( package, + name, semantic_roles, components, "matmul", @@ -1124,8 +1157,12 @@ Phi4Package Phi4Package::Load( components.find(std::string(role)); const auto& view = package.Require(component->second); - RequireFloating(view, role); - RequireShape(view, {kHidden}, role); + const std::string context = ComponentContext( + name, + role, + component->second); + RequireFloating(view, context); + RequireShape(view, {kHidden}, context); RequireSemanticRole( semantic_roles, component->second, @@ -1135,6 +1172,7 @@ Phi4Package Phi4Package::Load( validate_norm("norm1", "ssmlp.norm1"); ValidateQuantizedProjection( package, + name, semantic_roles, components, "ssmlp.gate", @@ -1143,6 +1181,7 @@ Phi4Package Phi4Package::Load( n); ValidateQuantizedProjection( package, + name, semantic_roles, components, "ssmlp.up", @@ -1151,6 +1190,7 @@ Phi4Package Phi4Package::Load( n); ValidateQuantizedProjection( package, + name, semantic_roles, components, "ssmlp.down", diff --git a/src/test/phi4_corelib_aie4/test_phi4_manifest.cpp b/src/test/phi4_corelib_aie4/test_phi4_manifest.cpp index 0f8e362b..0cc7eb41 100644 --- a/src/test/phi4_corelib_aie4/test_phi4_manifest.cpp +++ b/src/test/phi4_corelib_aie4/test_phi4_manifest.cpp @@ -1018,6 +1018,80 @@ void TestExactSourceValidation( "semantic role"); } +void TestComponentDiagnosticsIdentifyWeightObjects( + const SyntheticPackage& fixture, + const std::shared_ptr& api) { + std::string failures; + const auto verify = [&]( + std::string_view scenario, + auto mutation, + std::string_view expected) { + try { + ExpectLoadFailure( + fixture, + api, + std::move(mutation), + expected); + } catch (const std::exception& error) { + failures += "\n" + std::string(scenario) + ": " + error.what(); + } + }; + + verify( + "MatMul dtype", + [](json& manifest) { + constexpr std::size_t object_index = 7u * 5u + 1u; + const std::string initializer = + manifest["weight_objects"][object_index]["roles"]["scales"]; + auto& record = manifest["initializers"].at(initializer); + record["dtype"] = "uint8"; + record["length"] = 1024u * 24u; + }, + "model.layers.7.attn.k_proj.MatMulNBits.scales " + "(model.layers.7.attn.k_proj.MatMulNBits.scales)"); + verify( + "SSMLP projection shape", + [](json& manifest) { + constexpr std::size_t object_index = 19u * 5u + 4u; + const std::string initializer = + manifest["weight_objects"][object_index]["roles"] + ["up_scales"]; + auto& record = manifest["initializers"].at(initializer); + record["shape"] = {8191, 24}; + record["length"] = + 8191u * 24u * sizeof(std::uint16_t); + }, + "model.layers.19.ssmlp.up_scales " + "(model.layers.19.mlp.up_proj.MatMulNBits.scales)"); + verify( + "missing MatMul role", + [](json& manifest) { + constexpr std::size_t object_index = 12u * 5u + 2u; + manifest["weight_objects"][object_index]["roles"].erase( + "qzeros"); + }, + "model.layers.12.attn.v_proj.MatMulNBits.qzeros"); + verify( + "SSMLP norm shape", + [](json& manifest) { + constexpr std::size_t object_index = 27u * 5u + 4u; + const std::string initializer = + manifest["weight_objects"][object_index]["roles"]["norm0"]; + auto& record = manifest["initializers"].at(initializer); + record["shape"] = {3071}; + record["length"] = + 3071u * sizeof(std::uint16_t); + }, + "model.layers.27.ssmlp.norm0 " + "(model.layers.27.post_attention_layernorm.weight)"); + + if (!failures.empty()) { + throw std::runtime_error( + "component diagnostics did not identify their objects:" + + failures); + } +} + void TestOwnedScaleAndNormConversions( const SyntheticPackage& fixture, const std::shared_ptr& api) { @@ -1184,6 +1258,7 @@ int main() { TestPathRangeAndHashRejections(fixture, api); TestWeightObjectRejections(fixture, api); TestExactSourceValidation(fixture, api); + TestComponentDiagnosticsIdentifyWeightObjects(fixture, api); TestOwnedScaleAndNormConversions(fixture, api); TestRopeUsesExactStridedContractAtGuardPage(fixture, api); TestFp32RopeSource(fixture, api); From 64f5062f562bff945ac9ecbc5a5572da7f3ad2d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CChi?= Date: Mon, 31 Aug 2026 23:03:32 -0700 Subject: [PATCH 008/117] feat: derive AIE4 tensor shapes from corelib Co-authored-by: Cursor --- src/common/corelib/corelib_sources.cmake | 3 +- .../corelib/phi4_corelib_shape_plan.cpp | 245 ++++++++ .../models/phi4/phi4_corelib_shape_plan.hpp | 59 ++ src/test/phi4_corelib_aie4/CMakeLists.txt | 1 + .../test_phi4_shape_plan.cpp | 534 ++++++++++++++++++ 5 files changed, 841 insertions(+), 1 deletion(-) create mode 100644 src/common/corelib/phi4_corelib_shape_plan.cpp create mode 100644 src/include/models/phi4/phi4_corelib_shape_plan.hpp create mode 100644 src/test/phi4_corelib_aie4/test_phi4_shape_plan.cpp diff --git a/src/common/corelib/corelib_sources.cmake b/src/common/corelib/corelib_sources.cmake index e94480bc..0d29073e 100644 --- a/src/common/corelib/corelib_sources.cmake +++ b/src/common/corelib/corelib_sources.cmake @@ -2,4 +2,5 @@ set(FLM_CORELIB_AIE4_SOURCES "${CMAKE_CURRENT_LIST_DIR}/corelib_api.cpp" "${CMAKE_CURRENT_LIST_DIR}/corelib_fatal_record.cpp" "${CMAKE_CURRENT_LIST_DIR}/corelib_runtime.cpp" - "${CMAKE_CURRENT_LIST_DIR}/phi4_corelib_manifest.cpp") + "${CMAKE_CURRENT_LIST_DIR}/phi4_corelib_manifest.cpp" + "${CMAKE_CURRENT_LIST_DIR}/phi4_corelib_shape_plan.cpp") diff --git a/src/common/corelib/phi4_corelib_shape_plan.cpp b/src/common/corelib/phi4_corelib_shape_plan.cpp new file mode 100644 index 00000000..4cf6b656 --- /dev/null +++ b/src/common/corelib/phi4_corelib_shape_plan.cpp @@ -0,0 +1,245 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace flm::phi4 { +namespace { + +constexpr std::int64_t kMaxLiveRows = 4096; +constexpr std::int64_t kHiddenSize = 3072; +constexpr std::int64_t kKvSize = 1024; +constexpr std::int64_t kMlpSize = 8192; +constexpr std::int64_t kVocabSize = 200064; +constexpr std::uint32_t kGroupSize = 128; + +constexpr ryzenai_corelib_flat_mha_bf16_desc kAttentionDesc{ + 24, + 8, + 128, + kMaxLiveRows, + 96}; + +std::size_t RowUseIndex(RowUse use) { + switch (use) { + case RowUse::QueryProjection: + return 0; + case RowUse::KvProjection: + return 1; + case RowUse::Attention: + return 2; + case RowUse::OutputProjection: + return 3; + case RowUse::SsMlp: + return 4; + case RowUse::LmHead: + return 5; + } + throw std::invalid_argument("unknown Phi-4 RowUse"); +} + +void ValidatePaddedRows( + std::string_view context, + std::int64_t live_rows, + std::int64_t padded_rows) { + if (padded_rows < live_rows) { + throw std::runtime_error( + std::string(context) + + " returned invalid padded rows: live=" + + std::to_string(live_rows) + + ", padded=" + std::to_string(padded_rows)); + } +} + +MatMulPaddedShape QueryMatMul( + const corelib::CorelibApi& api, + std::int64_t live_rows, + std::int64_t k, + std::int64_t n, + std::string_view context) { + MatMulPaddedShape shape{live_rows, k, n}; + api.Check( + api.functions().matmul_pad_shape( + &shape.m, + &shape.k, + &shape.n, + kGroupSize), + "ryzenai_corelib_matmul_bf16_pad_shape"); + if (shape.k != k || shape.n != n) { + throw std::runtime_error( + std::string(context) + + " MatMul helper changed requested K/N"); + } + ValidatePaddedRows(context, live_rows, shape.m); + return shape; +} + +std::int64_t QuerySsMlp( + const corelib::CorelibApi& api, + std::int64_t live_rows) { + std::int64_t padded_rows = live_rows; + api.Check( + api.functions().ssmlp_pad_rows( + &padded_rows, + kHiddenSize, + kMlpSize, + kGroupSize), + "ryzenai_corelib_ssmlp_bf16_pad_rows"); + ValidatePaddedRows("SSMLP", live_rows, padded_rows); + return padded_rows; +} + +std::int64_t QueryAttention( + const corelib::CorelibApi& api, + std::int64_t live_rows, + const ryzenai_corelib_flat_mha_bf16_desc& desc) { + std::int64_t padded_rows = live_rows; + api.Check( + api.functions().flat_mha_pad_rows(&padded_rows, &desc), + "ryzenai_corelib_flat_mha_bf16_pad_rows"); + ValidatePaddedRows("attention", live_rows, padded_rows); + return padded_rows; +} + +void AppendTransition( + std::vector>& transitions, + std::int64_t live_rows, + std::int64_t padded_rows) { + if (transitions.empty() || + transitions.back().second != padded_rows) { + transitions.emplace_back(live_rows, padded_rows); + } +} + +} // namespace + +Phi4ShapePlan Phi4ShapePlan::Build( + std::shared_ptr api) { + if (!api) { + throw std::invalid_argument( + "Phi4ShapePlan::Build requires a CorelibApi"); + } + + Phi4ShapePlan plan; + plan.attention_desc_ = kAttentionDesc; + + auto& query_transitions = + plan.transitions_[RowUseIndex(RowUse::QueryProjection)]; + auto& kv_transitions = + plan.transitions_[RowUseIndex(RowUse::KvProjection)]; + auto& attention_transitions = + plan.transitions_[RowUseIndex(RowUse::Attention)]; + auto& output_transitions = + plan.transitions_[RowUseIndex(RowUse::OutputProjection)]; + auto& ssmlp_transitions = + plan.transitions_[RowUseIndex(RowUse::SsMlp)]; + + for (std::int64_t live_rows = 1; + live_rows <= kMaxLiveRows; + ++live_rows) { + const auto query_shape = QueryMatMul( + *api, + live_rows, + kHiddenSize, + kHiddenSize, + "query/output projection"); + const auto kv_shape = QueryMatMul( + *api, + live_rows, + kHiddenSize, + kKvSize, + "key/value projection"); + const auto ssmlp_rows = QuerySsMlp(*api, live_rows); + const auto attention_rows = QueryAttention( + *api, + live_rows, + plan.attention_desc_); + + AppendTransition( + query_transitions, + live_rows, + query_shape.m); + AppendTransition( + output_transitions, + live_rows, + query_shape.m); + AppendTransition(kv_transitions, live_rows, kv_shape.m); + AppendTransition( + ssmlp_transitions, + live_rows, + ssmlp_rows); + AppendTransition( + attention_transitions, + live_rows, + attention_rows); + + plan.capacities_.layer_rows = std::max( + {plan.capacities_.layer_rows, + query_shape.m, + kv_shape.m, + ssmlp_rows, + attention_rows}); + } + + const auto lm_head_shape = QueryMatMul( + *api, + 1, + kHiddenSize, + kVocabSize, + "LM head"); + AppendTransition( + plan.transitions_[RowUseIndex(RowUse::LmHead)], + 1, + lm_head_shape.m); + plan.capacities_.lm_head_rows = lm_head_shape.m; + + return plan; +} + +std::int64_t Phi4ShapePlan::RowsFor( + RowUse use, + std::int64_t live_rows) const { + const auto& transitions = Transitions(use); + const std::int64_t max_live_rows = + use == RowUse::LmHead ? 1 : kMaxLiveRows; + if (live_rows < 1 || live_rows > max_live_rows) { + throw std::out_of_range( + "Phi4ShapePlan live rows are outside the cached range"); + } + + const auto next = std::upper_bound( + transitions.begin(), + transitions.end(), + live_rows, + [](std::int64_t value, const auto& transition) { + return value < transition.first; + }); + if (next == transitions.begin()) { + throw std::logic_error( + "Phi4ShapePlan has no transition for live rows"); + } + return std::prev(next)->second; +} + +const std::vector>& +Phi4ShapePlan::Transitions(RowUse use) const { + return transitions_[RowUseIndex(use)]; +} + +const Phi4Capacities& Phi4ShapePlan::capacities() const noexcept { + return capacities_; +} + +const ryzenai_corelib_flat_mha_bf16_desc& +Phi4ShapePlan::attention_desc() const noexcept { + return attention_desc_; +} + +} // namespace flm::phi4 diff --git a/src/include/models/phi4/phi4_corelib_shape_plan.hpp b/src/include/models/phi4/phi4_corelib_shape_plan.hpp new file mode 100644 index 00000000..c4aa8ec1 --- /dev/null +++ b/src/include/models/phi4/phi4_corelib_shape_plan.hpp @@ -0,0 +1,59 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include +#include + +namespace flm::phi4 { + +enum class RowUse { + QueryProjection, + KvProjection, + Attention, + OutputProjection, + SsMlp, + LmHead +}; + +struct MatMulPaddedShape { + std::int64_t m; + std::int64_t k; + std::int64_t n; +}; + +struct Phi4Capacities { + std::int64_t layer_rows; + std::int64_t lm_head_rows; +}; + +class Phi4ShapePlan final { +public: + static Phi4ShapePlan Build( + std::shared_ptr api); + + std::int64_t RowsFor(RowUse use, std::int64_t live_rows) const; + const std::vector>& + Transitions(RowUse use) const; + const Phi4Capacities& capacities() const noexcept; + const ryzenai_corelib_flat_mha_bf16_desc& attention_desc() + const noexcept; + +private: + static constexpr std::size_t kRowUseCount = 6; + + Phi4ShapePlan() = default; + + std::array< + std::vector>, + kRowUseCount> + transitions_; + Phi4Capacities capacities_{}; + ryzenai_corelib_flat_mha_bf16_desc attention_desc_{}; +}; + +} // namespace flm::phi4 diff --git a/src/test/phi4_corelib_aie4/CMakeLists.txt b/src/test/phi4_corelib_aie4/CMakeLists.txt index 7e3497cd..5aeb15ee 100644 --- a/src/test/phi4_corelib_aie4/CMakeLists.txt +++ b/src/test/phi4_corelib_aie4/CMakeLists.txt @@ -71,6 +71,7 @@ endfunction() enable_testing() add_corelib_host_test(test_corelib_api test_corelib_api.cpp) add_corelib_host_test(test_phi4_manifest test_phi4_manifest.cpp) +add_corelib_host_test(test_phi4_shape_plan test_phi4_shape_plan.cpp) add_corelib_host_test( test_corelib_fatal_record test_corelib_fatal_record.cpp) diff --git a/src/test/phi4_corelib_aie4/test_phi4_shape_plan.cpp b/src/test/phi4_corelib_aie4/test_phi4_shape_plan.cpp new file mode 100644 index 00000000..b2b36762 --- /dev/null +++ b/src/test/phi4_corelib_aie4/test_phi4_shape_plan.cpp @@ -0,0 +1,534 @@ +#include "fake_corelib.hpp" +#include "test_support.hpp" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using flm::corelib::CorelibApi; +using flm::phi4::Phi4ShapePlan; +using flm::phi4::RowUse; + +struct MatMulCall { + std::int64_t m; + std::int64_t k; + std::int64_t n; + std::uint32_t group_size; +}; + +struct SsMlpCall { + std::int64_t m; + std::int64_t k; + std::int64_t n; + std::uint32_t group_size; +}; + +struct AttentionCall { + std::int64_t m; + ryzenai_corelib_flat_mha_bf16_desc desc; +}; + +enum class HelperKind { + None, + MatMul, + SsMlp, + Attention +}; + +enum class FaultKind { + None, + Unsupported, + InvalidRows, + MutateK, + MutateN +}; + +struct Fault { + HelperKind helper = HelperKind::None; + FaultKind kind = FaultKind::None; + std::int64_t live_rows = 0; + std::int64_t matmul_n = 0; +}; + +struct HelperState { + std::vector matmul_calls; + std::vector ssmlp_calls; + std::vector attention_calls; + Fault fault; + + void Reset() { + matmul_calls.clear(); + ssmlp_calls.clear(); + attention_calls.clear(); + fault = {}; + } + + std::size_t TotalCalls() const noexcept { + return matmul_calls.size() + ssmlp_calls.size() + + attention_calls.size(); + } +}; + +HelperState g_helpers; + +bool Matches( + HelperKind helper, + std::int64_t live_rows, + std::int64_t matmul_n = 0) { + return g_helpers.fault.helper == helper && + g_helpers.fault.live_rows == live_rows && + (helper != HelperKind::MatMul || + g_helpers.fault.matmul_n == matmul_n); +} + +std::int64_t QueryProjectionRows(std::int64_t live_rows) { + if (live_rows == 1) { + return 1; + } + if (live_rows <= 32) { + return 32; + } + if (live_rows <= 512) { + return 512; + } + return 4096; +} + +std::int64_t KvProjectionRows(std::int64_t live_rows) { + if (live_rows == 1) { + return 1; + } + if (live_rows <= 64) { + return 64; + } + if (live_rows <= 1024) { + return 1024; + } + return 4160; +} + +std::int64_t SsMlpRows(std::int64_t live_rows) { + if (live_rows == 1) { + return 1; + } + if (live_rows <= 128) { + return 128; + } + if (live_rows <= 2048) { + return 2048; + } + return 4608; +} + +std::int64_t AttentionRows(std::int64_t live_rows) { + if (live_rows == 1) { + return 1; + } + if (live_rows <= 256) { + return 256; + } + if (live_rows <= 3072) { + return 3072; + } + return 4352; +} + +ryzenai_corelib_status RecordingMatMulPadShape( + std::int64_t* m, + std::int64_t* k, + std::int64_t* n, + std::uint32_t group_size) { + if (m == nullptr || k == nullptr || n == nullptr) { + return ryzenai_corelib_status_bad_argument; + } + + const MatMulCall call{*m, *k, *n, group_size}; + g_helpers.matmul_calls.push_back(call); + if (Matches(HelperKind::MatMul, call.m, call.n)) { + switch (g_helpers.fault.kind) { + case FaultKind::Unsupported: + return ryzenai_corelib_status_unsupported; + case FaultKind::InvalidRows: + *m = call.m - 1; + return ryzenai_corelib_status_success; + case FaultKind::MutateK: + ++*k; + return ryzenai_corelib_status_success; + case FaultKind::MutateN: + ++*n; + return ryzenai_corelib_status_success; + case FaultKind::None: + break; + } + } + + if (call.k != 3072 || call.group_size != 128) { + return ryzenai_corelib_status_bad_argument; + } + if (call.n == 3072) { + *m = QueryProjectionRows(call.m); + } else if (call.n == 1024) { + *m = KvProjectionRows(call.m); + } else if (call.n == 200064 && call.m == 1) { + *m = 1; + } else { + return ryzenai_corelib_status_bad_argument; + } + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status RecordingSsMlpPadRows( + std::int64_t* m, + std::int64_t k, + std::int64_t n, + std::uint32_t group_size) { + if (m == nullptr) { + return ryzenai_corelib_status_bad_argument; + } + + const SsMlpCall call{*m, k, n, group_size}; + g_helpers.ssmlp_calls.push_back(call); + if (Matches(HelperKind::SsMlp, call.m)) { + if (g_helpers.fault.kind == FaultKind::Unsupported) { + return ryzenai_corelib_status_unsupported; + } + if (g_helpers.fault.kind == FaultKind::InvalidRows) { + *m = call.m - 1; + return ryzenai_corelib_status_success; + } + } + + if (call.k != 3072 || call.n != 8192 || + call.group_size != 128) { + return ryzenai_corelib_status_bad_argument; + } + *m = SsMlpRows(call.m); + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status RecordingAttentionPadRows( + std::int64_t* m, + const ryzenai_corelib_flat_mha_bf16_desc* desc) { + if (m == nullptr || desc == nullptr) { + return ryzenai_corelib_status_bad_argument; + } + + const AttentionCall call{*m, *desc}; + g_helpers.attention_calls.push_back(call); + if (Matches(HelperKind::Attention, call.m)) { + if (g_helpers.fault.kind == FaultKind::Unsupported) { + return ryzenai_corelib_status_unsupported; + } + if (g_helpers.fault.kind == FaultKind::InvalidRows) { + *m = call.m - 1; + return ryzenai_corelib_status_success; + } + } + + if (desc->num_heads != 24 || desc->kv_num_heads != 8 || + desc->head_size != 128 || desc->max_seq != 4096 || + desc->rope_dim != 96) { + return ryzenai_corelib_status_bad_argument; + } + *m = AttentionRows(call.m); + return ryzenai_corelib_status_success; +} + +template +void* FunctionAddress(Function function) { + return reinterpret_cast(function); +} + +std::shared_ptr ResolveRecordingCorelib() { + auto resolver = flm::test::CompleteCorelibResolver(); + resolver["ryzenai_corelib_matmul_bf16_pad_shape"] = + FunctionAddress( + static_cast< + decltype(&::ryzenai_corelib_matmul_bf16_pad_shape)>( + &RecordingMatMulPadShape)); + resolver["ryzenai_corelib_ssmlp_bf16_pad_rows"] = + FunctionAddress( + static_cast< + decltype(&::ryzenai_corelib_ssmlp_bf16_pad_rows)>( + &RecordingSsMlpPadRows)); + resolver["ryzenai_corelib_flat_mha_bf16_pad_rows"] = + FunctionAddress( + static_cast< + decltype(&::ryzenai_corelib_flat_mha_bf16_pad_rows)>( + &RecordingAttentionPadRows)); + return CorelibApi::ResolveForTest( + [resolver = std::move(resolver)](std::string_view name) mutable + -> void* { + const auto found = resolver.find(std::string(name)); + return found == resolver.end() ? nullptr : found->second; + }); +} + +void CheckTransitions( + const std::vector>& actual, + std::initializer_list< + std::pair> expected) { + const std::vector> + expected_transitions(expected); + CHECK(actual == expected_transitions); +} + +void TestCompleteQueriesAndCachedTransitions( + const std::shared_ptr& api) { + g_helpers.Reset(); + const auto plan = Phi4ShapePlan::Build(api); + + CHECK(g_helpers.matmul_calls.size() == 8193); + std::array query_rows{}; + std::array kv_rows{}; + int lm_head_calls = 0; + for (const auto& call : g_helpers.matmul_calls) { + CHECK(call.k == 3072); + CHECK(call.group_size == 128); + if (call.n == 3072) { + CHECK(call.m >= 1 && call.m <= 4096); + ++query_rows[static_cast(call.m)]; + } else if (call.n == 1024) { + CHECK(call.m >= 1 && call.m <= 4096); + ++kv_rows[static_cast(call.m)]; + } else { + CHECK(call.n == 200064); + CHECK(call.m == 1); + ++lm_head_calls; + } + } + for (std::int64_t row = 1; row <= 4096; ++row) { + CHECK(query_rows[static_cast(row)] == 1); + CHECK(kv_rows[static_cast(row)] == 1); + } + CHECK(lm_head_calls == 1); + + CHECK(g_helpers.ssmlp_calls.size() == 4096); + CHECK(g_helpers.attention_calls.size() == 4096); + for (std::int64_t row = 1; row <= 4096; ++row) { + const auto index = static_cast(row - 1); + const auto& ssmlp = g_helpers.ssmlp_calls[index]; + CHECK(ssmlp.m == row); + CHECK(ssmlp.k == 3072); + CHECK(ssmlp.n == 8192); + CHECK(ssmlp.group_size == 128); + + const auto& attention = g_helpers.attention_calls[index]; + CHECK(attention.m == row); + CHECK(attention.desc.num_heads == 24); + CHECK(attention.desc.kv_num_heads == 8); + CHECK(attention.desc.head_size == 128); + CHECK(attention.desc.max_seq == 4096); + CHECK(attention.desc.rope_dim == 96); + } + + const auto& desc = plan.attention_desc(); + CHECK(desc.num_heads == 24); + CHECK(desc.kv_num_heads == 8); + CHECK(desc.head_size == 128); + CHECK(desc.max_seq == 4096); + CHECK(desc.rope_dim == 96); + + CHECK(plan.capacities().layer_rows == 4608); + CHECK(plan.capacities().lm_head_rows == 1); + + CheckTransitions( + plan.Transitions(RowUse::QueryProjection), + {{1, 1}, {2, 32}, {33, 512}, {513, 4096}}); + CheckTransitions( + plan.Transitions(RowUse::OutputProjection), + {{1, 1}, {2, 32}, {33, 512}, {513, 4096}}); + CheckTransitions( + plan.Transitions(RowUse::KvProjection), + {{1, 1}, {2, 64}, {65, 1024}, {1025, 4160}}); + CheckTransitions( + plan.Transitions(RowUse::SsMlp), + {{1, 1}, {2, 128}, {129, 2048}, {2049, 4608}}); + CheckTransitions( + plan.Transitions(RowUse::Attention), + {{1, 1}, {2, 256}, {257, 3072}, {3073, 4352}}); + CheckTransitions( + plan.Transitions(RowUse::LmHead), + {{1, 1}}); + + constexpr std::array uses{ + RowUse::QueryProjection, + RowUse::KvProjection, + RowUse::Attention, + RowUse::OutputProjection, + RowUse::SsMlp, + RowUse::LmHead}; + for (const auto use : uses) { + CHECK(plan.RowsFor(use, 1) == 1); + } + + const std::size_t calls_after_build = g_helpers.TotalCalls(); + CHECK(plan.RowsFor(RowUse::QueryProjection, 2) == 32); + CHECK(plan.RowsFor(RowUse::QueryProjection, 33) == 512); + CHECK(plan.RowsFor(RowUse::KvProjection, 4096) == 4160); + CHECK(plan.RowsFor(RowUse::Attention, 3072) == 3072); + CHECK(plan.RowsFor(RowUse::SsMlp, 2049) == 4608); + CHECK(plan.RowsFor(RowUse::LmHead, 1) == 1); + CHECK(g_helpers.TotalCalls() == calls_after_build); +} + +void TestMatMulDimensionMutationsRejectBuild( + const std::shared_ptr& api) { + struct Case { + FaultKind kind; + std::int64_t row; + std::int64_t n; + std::string_view context; + }; + constexpr std::array cases{{ + {FaultKind::MutateK, 37, 3072, "query/output projection"}, + {FaultKind::MutateN, 93, 1024, "key/value projection"}, + {FaultKind::MutateN, 1, 200064, "LM head"}, + }}; + + for (const auto& test_case : cases) { + g_helpers.Reset(); + g_helpers.fault = { + HelperKind::MatMul, + test_case.kind, + test_case.row, + test_case.n}; + CheckThrowsContains( + [&] { + (void)Phi4ShapePlan::Build(api); + }, + test_case.context); + } +} + +void TestUnsupportedRowsRejectBuild( + const std::shared_ptr& api) { + struct Case { + HelperKind helper; + std::int64_t row; + std::int64_t n; + std::string_view call; + }; + constexpr std::array cases{{ + {HelperKind::MatMul, + 41, + 3072, + "ryzenai_corelib_matmul_bf16_pad_shape"}, + {HelperKind::SsMlp, + 43, + 0, + "ryzenai_corelib_ssmlp_bf16_pad_rows"}, + {HelperKind::Attention, + 47, + 0, + "ryzenai_corelib_flat_mha_bf16_pad_rows"}, + }}; + + for (const auto& test_case : cases) { + g_helpers.Reset(); + g_helpers.fault = { + test_case.helper, + FaultKind::Unsupported, + test_case.row, + test_case.n}; + CheckThrowsContains( + [&] { + (void)Phi4ShapePlan::Build(api); + }, + test_case.call); + } +} + +void TestInvalidPaddedRowsRejectBuild( + const std::shared_ptr& api) { + struct Case { + HelperKind helper; + std::int64_t row; + std::int64_t n; + std::string_view context; + }; + constexpr std::array cases{{ + {HelperKind::MatMul, 53, 3072, "query/output projection"}, + {HelperKind::SsMlp, 59, 0, "SSMLP"}, + {HelperKind::Attention, 61, 0, "attention"}, + {HelperKind::MatMul, 1, 200064, "LM head"}, + }}; + + for (const auto& test_case : cases) { + g_helpers.Reset(); + g_helpers.fault = { + test_case.helper, + FaultKind::InvalidRows, + test_case.row, + test_case.n}; + CheckThrowsContains( + [&] { + (void)Phi4ShapePlan::Build(api); + }, + test_case.context); + } +} + +void TestInvalidInputsRejectWithoutHelperCalls( + const std::shared_ptr& api) { + g_helpers.Reset(); + CheckThrowsContains( + [] { + (void)Phi4ShapePlan::Build(nullptr); + }, + "CorelibApi"); + CHECK(g_helpers.TotalCalls() == 0); + + const auto plan = Phi4ShapePlan::Build(api); + const auto calls_after_build = g_helpers.TotalCalls(); + CheckThrowsContains( + [&] { + (void)plan.RowsFor(RowUse::QueryProjection, 0); + }, + "live rows"); + CheckThrowsContains( + [&] { + (void)plan.RowsFor(RowUse::Attention, 4097); + }, + "live rows"); + CheckThrowsContains( + [&] { + (void)plan.RowsFor(RowUse::LmHead, 2); + }, + "live rows"); + CheckThrowsContains( + [&] { + (void)plan.Transitions(static_cast(99)); + }, + "RowUse"); + CHECK(g_helpers.TotalCalls() == calls_after_build); +} + +} // namespace + +int main() { + try { + auto api = ResolveRecordingCorelib(); + TestCompleteQueriesAndCachedTransitions(api); + TestMatMulDimensionMutationsRejectBuild(api); + TestUnsupportedRowsRejectBuild(api); + TestInvalidPaddedRowsRejectBuild(api); + TestInvalidInputsRejectWithoutHelperCalls(api); + std::cout << "test_phi4_shape_plan: PASS\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} From 64c2013fd555d8c91340d26e6d2fb25994252aee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CChi?= Date: Mon, 31 Aug 2026 23:18:27 -0700 Subject: [PATCH 009/117] fix: centralize Phi-4 shape contracts Co-authored-by: Cursor --- src/common/corelib/phi4_corelib_manifest.cpp | 94 +++++++++++-------- .../corelib/phi4_corelib_shape_plan.cpp | 56 ++++++----- .../models/phi4/phi4_corelib_constants.hpp | 26 +++++ .../test_phi4_shape_plan.cpp | 35 +++++-- 4 files changed, 141 insertions(+), 70 deletions(-) create mode 100644 src/include/models/phi4/phi4_corelib_constants.hpp diff --git a/src/common/corelib/phi4_corelib_manifest.cpp b/src/common/corelib/phi4_corelib_manifest.cpp index 05159086..7dc5291e 100644 --- a/src/common/corelib/phi4_corelib_manifest.cpp +++ b/src/common/corelib/phi4_corelib_manifest.cpp @@ -1,4 +1,5 @@ #include +#include #include "../../pull/picosha2.h" @@ -32,13 +33,8 @@ constexpr std::string_view kManifestName = "corelib_phi4_manifest.json"; constexpr std::size_t kExpectedInitializers = 743; constexpr std::size_t kExpectedWeightObjects = 161; -constexpr std::int64_t kLayers = 32; -constexpr std::int64_t kHidden = 3072; -constexpr std::int64_t kIntermediate = 8192; -constexpr std::int64_t kVocab = 200064; -constexpr std::int64_t kMaxSeq = 4096; -constexpr std::int64_t kRopeColumns = 48; -constexpr std::uint32_t kGroupSize = 128; +constexpr std::int64_t kRopeColumns = + constants::kRopeDimension / 2; [[noreturn]] void Throw( std::string_view context, @@ -364,40 +360,42 @@ const std::vector& ExpectedWeightObjects() { static const std::vector expected = [] { std::vector values; values.reserve(kExpectedWeightObjects); - for (int layer = 0; layer < kLayers; ++layer) { + for (std::int64_t layer = 0; + layer < constants::kLayerCount; + ++layer) { const std::string base = "model.layers." + std::to_string(layer) + ".attn."; values.push_back({ base + "q_proj.MatMulNBits", WeightObjectKind::MatMul, - 3072, - 3072}); + constants::kHiddenSize, + constants::kQueryDimension}); values.push_back({ base + "k_proj.MatMulNBits", WeightObjectKind::MatMul, - 3072, - 1024}); + constants::kHiddenSize, + constants::kKvDimension}); values.push_back({ base + "v_proj.MatMulNBits", WeightObjectKind::MatMul, - 3072, - 1024}); + constants::kHiddenSize, + constants::kKvDimension}); values.push_back({ base + "o_proj.MatMulNBits", WeightObjectKind::MatMul, - 3072, - 3072}); + constants::kQueryDimension, + constants::kHiddenSize}); values.push_back({ "model.layers." + std::to_string(layer) + ".ssmlp", WeightObjectKind::SsMlp, - kHidden, - kIntermediate}); + constants::kHiddenSize, + constants::kIntermediateSize}); } values.push_back({ "lm_head.MatMulNBits", WeightObjectKind::MatMul, - kHidden, - kVocab}); + constants::kHiddenSize, + constants::kVocabularySize}); return values; }(); return expected; @@ -454,11 +452,13 @@ void ValidateQuantizedProjection( std::int64_t k, std::int64_t n) { if (k <= 0 || n <= 0 || k % 2 != 0 || - k % static_cast(kGroupSize) != 0) { + k % static_cast( + constants::kGroupSize) != 0) { Throw(role_prefix, "has an invalid quantized descriptor"); } const std::int64_t groups = - k / static_cast(kGroupSize); + k / static_cast( + constants::kGroupSize); const auto validate = [&]( std::string_view component, @@ -529,7 +529,7 @@ void ValidateHostInitializers( } RequireShape( embedding, - {kVocab, kHidden}, + {constants::kVocabularySize, constants::kHiddenSize}, "embedding"); RequireSemanticRole( semantic_roles, @@ -539,7 +539,10 @@ void ValidateHostInitializers( const auto& input_norm = package.Require("model.layers.0.input_layernorm.weight"); RequireFloating(input_norm, "input_norm"); - RequireShape(input_norm, {kHidden}, "input_norm"); + RequireShape( + input_norm, + {constants::kHiddenSize}, + "input_norm"); RequireSemanticRole( semantic_roles, "model.layers.0.input_layernorm.weight", @@ -550,7 +553,7 @@ void ValidateHostInitializers( RequireFloating(rope, name); if ( rope.shape.size() != 2 || - rope.shape[0] < kMaxSeq || + rope.shape[0] < constants::kMaxSequenceLength || rope.shape[1] < kRopeColumns) { Throw(name, "must be rank 2 and at least [4096,48]"); } @@ -638,39 +641,46 @@ void ValidateModelIdentity(const json& manifest) { if (ReadString(model.at("family"), "model.family") != "phi4") { Throw("model.family", "does not match phi4"); } - RequireInteger(model.at("layers"), 32, "model.layers"); + RequireInteger( + model.at("layers"), + constants::kLayerCount, + "model.layers"); RequireInteger( model.at("hidden_size"), - 3072, + constants::kHiddenSize, "model.hidden_size"); RequireInteger( model.at("intermediate_size"), - 8192, + constants::kIntermediateSize, "model.intermediate_size"); RequireInteger( model.at("num_heads"), - 24, + constants::kQueryHeadCount, "model.num_heads"); - RequireInteger(model.at("kv_heads"), 8, "model.kv_heads"); + RequireInteger( + model.at("kv_heads"), + constants::kKvHeadCount, + "model.kv_heads"); RequireInteger( model.at("head_size"), - 128, + constants::kHeadSize, "model.head_size"); RequireInteger( model.at("vocab_size"), - 200064, + constants::kVocabularySize, "model.vocab_size"); RequireInteger( model.at("group_size"), - 128, + constants::kGroupSize, "model.group_size"); RequireInteger( model.at("rope_dim"), - 96, + constants::kRopeDimension, "model.rope_dim"); if ( !model.at("rms_epsilon").is_number() || - model.at("rms_epsilon").get() != 0.00001) { + model.at("rms_epsilon").get() != + constants::kRmsEpsilon) { Throw( "model.rms_epsilon", "does not match the Phi-4 model identity"); @@ -680,7 +690,7 @@ void ValidateModelIdentity(const json& manifest) { RequireExactKeys(backend, {"max_seq"}, "backend"); RequireInteger( backend.at("max_seq"), - kMaxSeq, + constants::kMaxSequenceLength, "backend.max_seq"); } @@ -1092,7 +1102,7 @@ Phi4Package Phi4Package::Load( } if ( k != expected.k || n != expected.n || - group_size != kGroupSize) { + group_size != constants::kGroupSize) { Throw( name + ".descriptor", "does not match the fixed Phi-4 descriptor"); @@ -1162,7 +1172,10 @@ Phi4Package Phi4Package::Load( role, component->second); RequireFloating(view, context); - RequireShape(view, {kHidden}, context); + RequireShape( + view, + {constants::kHiddenSize}, + context); RequireSemanticRole( semantic_roles, component->second, @@ -1316,7 +1329,7 @@ std::span Phi4Package::MaterializeRopeFp32( const auto source_type = CorelibDType(source.dtype, name); if ( source.shape.size() != 2 || - source.shape[0] < kMaxSeq || + source.shape[0] < constants::kMaxSequenceLength || source.shape[1] < kRopeColumns) { Throw( name, @@ -1325,7 +1338,8 @@ std::span Phi4Package::MaterializeRopeFp32( const auto source_columns = static_cast(source.shape[1]); constexpr std::size_t count = - static_cast(kMaxSeq * kRopeColumns); + static_cast( + constants::kMaxSequenceLength * kRopeColumns); auto [buffer, inserted] = fp32_buffers_.try_emplace(std::string(name), count); try { diff --git a/src/common/corelib/phi4_corelib_shape_plan.cpp b/src/common/corelib/phi4_corelib_shape_plan.cpp index 4cf6b656..b199d310 100644 --- a/src/common/corelib/phi4_corelib_shape_plan.cpp +++ b/src/common/corelib/phi4_corelib_shape_plan.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -13,19 +14,19 @@ namespace flm::phi4 { namespace { -constexpr std::int64_t kMaxLiveRows = 4096; -constexpr std::int64_t kHiddenSize = 3072; -constexpr std::int64_t kKvSize = 1024; -constexpr std::int64_t kMlpSize = 8192; -constexpr std::int64_t kVocabSize = 200064; -constexpr std::uint32_t kGroupSize = 128; +constexpr std::int64_t kShapePrecomputeMaxLiveRows = 4096; +static_assert( + kShapePrecomputeMaxLiveRows == + constants::kMaxSequenceLength, + "Phi4ShapePlan v1 queries every live row through the " + "physical attention window"); constexpr ryzenai_corelib_flat_mha_bf16_desc kAttentionDesc{ - 24, - 8, - 128, - kMaxLiveRows, - 96}; + constants::kQueryHeadCount, + constants::kKvHeadCount, + constants::kHeadSize, + constants::kMaxSequenceLength, + constants::kRopeDimension}; std::size_t RowUseIndex(RowUse use) { switch (use) { @@ -70,12 +71,17 @@ MatMulPaddedShape QueryMatMul( &shape.m, &shape.k, &shape.n, - kGroupSize), + constants::kGroupSize), "ryzenai_corelib_matmul_bf16_pad_shape"); if (shape.k != k || shape.n != n) { throw std::runtime_error( std::string(context) + - " MatMul helper changed requested K/N"); + " MatMul K/N mismatch at live row " + + std::to_string(live_rows) + + ": requested K=" + std::to_string(k) + + ", N=" + std::to_string(n) + + "; returned K=" + std::to_string(shape.k) + + ", N=" + std::to_string(shape.n)); } ValidatePaddedRows(context, live_rows, shape.m); return shape; @@ -88,9 +94,9 @@ std::int64_t QuerySsMlp( api.Check( api.functions().ssmlp_pad_rows( &padded_rows, - kHiddenSize, - kMlpSize, - kGroupSize), + constants::kHiddenSize, + constants::kIntermediateSize, + constants::kGroupSize), "ryzenai_corelib_ssmlp_bf16_pad_rows"); ValidatePaddedRows("SSMLP", live_rows, padded_rows); return padded_rows; @@ -142,19 +148,19 @@ Phi4ShapePlan Phi4ShapePlan::Build( plan.transitions_[RowUseIndex(RowUse::SsMlp)]; for (std::int64_t live_rows = 1; - live_rows <= kMaxLiveRows; + live_rows <= kShapePrecomputeMaxLiveRows; ++live_rows) { const auto query_shape = QueryMatMul( *api, live_rows, - kHiddenSize, - kHiddenSize, + constants::kHiddenSize, + constants::kQueryDimension, "query/output projection"); const auto kv_shape = QueryMatMul( *api, live_rows, - kHiddenSize, - kKvSize, + constants::kHiddenSize, + constants::kKvDimension, "key/value projection"); const auto ssmlp_rows = QuerySsMlp(*api, live_rows); const auto attention_rows = QueryAttention( @@ -191,8 +197,8 @@ Phi4ShapePlan Phi4ShapePlan::Build( const auto lm_head_shape = QueryMatMul( *api, 1, - kHiddenSize, - kVocabSize, + constants::kHiddenSize, + constants::kVocabularySize, "LM head"); AppendTransition( plan.transitions_[RowUseIndex(RowUse::LmHead)], @@ -208,7 +214,9 @@ std::int64_t Phi4ShapePlan::RowsFor( std::int64_t live_rows) const { const auto& transitions = Transitions(use); const std::int64_t max_live_rows = - use == RowUse::LmHead ? 1 : kMaxLiveRows; + use == RowUse::LmHead + ? 1 + : kShapePrecomputeMaxLiveRows; if (live_rows < 1 || live_rows > max_live_rows) { throw std::out_of_range( "Phi4ShapePlan live rows are outside the cached range"); diff --git a/src/include/models/phi4/phi4_corelib_constants.hpp b/src/include/models/phi4/phi4_corelib_constants.hpp new file mode 100644 index 00000000..637bd133 --- /dev/null +++ b/src/include/models/phi4/phi4_corelib_constants.hpp @@ -0,0 +1,26 @@ +#pragma once + +#include + +namespace flm::phi4::constants { + +inline constexpr std::int64_t kLayerCount = 32; +inline constexpr std::int64_t kHiddenSize = 3072; +inline constexpr std::int64_t kIntermediateSize = 8192; +inline constexpr std::int64_t kQueryHeadCount = 24; +inline constexpr std::int64_t kKvHeadCount = 8; +inline constexpr std::int64_t kHeadSize = 128; +inline constexpr std::int64_t kQueryDimension = 3072; +inline constexpr std::int64_t kKvDimension = 1024; +inline constexpr std::int64_t kVocabularySize = 200064; +inline constexpr std::uint32_t kGroupSize = 128; +inline constexpr std::int64_t kRopeDimension = 96; +inline constexpr std::int64_t kMaxSequenceLength = 4096; +inline constexpr double kRmsEpsilon = 1.0e-5; + +static_assert( + kQueryHeadCount * kHeadSize == kQueryDimension); +static_assert(kQueryDimension == kHiddenSize); +static_assert(kKvHeadCount * kHeadSize == kKvDimension); + +} // namespace flm::phi4::constants diff --git a/src/test/phi4_corelib_aie4/test_phi4_shape_plan.cpp b/src/test/phi4_corelib_aie4/test_phi4_shape_plan.cpp index b2b36762..0dda64fc 100644 --- a/src/test/phi4_corelib_aie4/test_phi4_shape_plan.cpp +++ b/src/test/phi4_corelib_aie4/test_phi4_shape_plan.cpp @@ -284,6 +284,17 @@ void CheckTransitions( CHECK(actual == expected_transitions); } +template +void CheckThrowsEquals(Function&& function, std::string_view expected) { + try { + function(); + } catch (const std::exception& error) { + CHECK(std::string_view(error.what()) == expected); + return; + } + throw std::runtime_error("expected exception was not thrown"); +} + void TestCompleteQueriesAndCachedTransitions( const std::shared_ptr& api) { g_helpers.Reset(); @@ -389,12 +400,24 @@ void TestMatMulDimensionMutationsRejectBuild( FaultKind kind; std::int64_t row; std::int64_t n; - std::string_view context; + std::string_view message; }; constexpr std::array cases{{ - {FaultKind::MutateK, 37, 3072, "query/output projection"}, - {FaultKind::MutateN, 93, 1024, "key/value projection"}, - {FaultKind::MutateN, 1, 200064, "LM head"}, + {FaultKind::MutateK, + 37, + 3072, + "query/output projection MatMul K/N mismatch at live row 37: " + "requested K=3072, N=3072; returned K=3073, N=3072"}, + {FaultKind::MutateN, + 93, + 1024, + "key/value projection MatMul K/N mismatch at live row 93: " + "requested K=3072, N=1024; returned K=3072, N=1025"}, + {FaultKind::MutateN, + 1, + 200064, + "LM head MatMul K/N mismatch at live row 1: " + "requested K=3072, N=200064; returned K=3072, N=200065"}, }}; for (const auto& test_case : cases) { @@ -404,11 +427,11 @@ void TestMatMulDimensionMutationsRejectBuild( test_case.kind, test_case.row, test_case.n}; - CheckThrowsContains( + CheckThrowsEquals( [&] { (void)Phi4ShapePlan::Build(api); }, - test_case.context); + test_case.message); } } From eee632fdc815a587247f71e943cef51942ef7254 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CChi?= Date: Mon, 31 Aug 2026 23:37:18 -0700 Subject: [PATCH 010/117] feat: load Phi-4 weights through corelib Co-authored-by: Cursor --- src/common/corelib/corelib_sources.cmake | 1 + src/common/corelib/phi4_corelib_weights.cpp | 341 +++++ .../models/phi4/phi4_corelib_weights.hpp | 53 + src/test/phi4_corelib_aie4/CMakeLists.txt | 1 + .../phi4_corelib_aie4/test_phi4_weights.cpp | 1179 +++++++++++++++++ 5 files changed, 1575 insertions(+) create mode 100644 src/common/corelib/phi4_corelib_weights.cpp create mode 100644 src/include/models/phi4/phi4_corelib_weights.hpp create mode 100644 src/test/phi4_corelib_aie4/test_phi4_weights.cpp diff --git a/src/common/corelib/corelib_sources.cmake b/src/common/corelib/corelib_sources.cmake index 0d29073e..eac6b536 100644 --- a/src/common/corelib/corelib_sources.cmake +++ b/src/common/corelib/corelib_sources.cmake @@ -3,4 +3,5 @@ set(FLM_CORELIB_AIE4_SOURCES "${CMAKE_CURRENT_LIST_DIR}/corelib_fatal_record.cpp" "${CMAKE_CURRENT_LIST_DIR}/corelib_runtime.cpp" "${CMAKE_CURRENT_LIST_DIR}/phi4_corelib_manifest.cpp" + "${CMAKE_CURRENT_LIST_DIR}/phi4_corelib_weights.cpp" "${CMAKE_CURRENT_LIST_DIR}/phi4_corelib_shape_plan.cpp") diff --git a/src/common/corelib/phi4_corelib_weights.cpp b/src/common/corelib/phi4_corelib_weights.cpp new file mode 100644 index 00000000..e962bf43 --- /dev/null +++ b/src/common/corelib/phi4_corelib_weights.cpp @@ -0,0 +1,341 @@ +#include + +#include +#include +#include +#include +#include + +namespace flm::phi4 { +namespace { + +constexpr std::string_view kConvertCall = + "ryzenai_corelib_convert"; +constexpr std::string_view kMatMulCreateCall = + "ryzenai_corelib_matmul_bf16_weights_create_from_onnx_components"; +constexpr std::string_view kMatMulGetDataCall = + "ryzenai_corelib_matmul_bf16_weights_get_data"; +constexpr std::string_view kSsMlpCreateCall = + "ryzenai_corelib_ssmlp_bf16_weights_create_from_onnx_components"; +constexpr std::string_view kSsMlpGetDataCall = + "ryzenai_corelib_ssmlp_bf16_weights_get_data"; + +const std::string& ComponentName( + const WeightObjectView& object, + std::string_view role) { + const auto component = object.components.find(std::string(role)); + if (component == object.components.end()) { + throw std::runtime_error( + "validated component role is missing: " + + std::string(role)); + } + return component->second; +} + +void AddPackedBytes( + std::size_t packed_size, + std::size_t& total) { + if ( + packed_size > + std::numeric_limits::max() - total) { + throw std::overflow_error( + "packed weight byte total overflows size_t"); + } + total += packed_size; +} + +[[noreturn]] void ThrowObjectError( + const WeightObjectView& object, + const std::exception& error) { + throw std::runtime_error( + "failed to load Phi-4 weight object '" + object.name + + "': " + error.what()); +} + +corelib::UniqueMatMulWeights CreateMatMul( + const std::shared_ptr& api, + Phi4Package& package, + const WeightObjectView& object, + std::size_t& packed_bytes) { + try { + if (object.kind != WeightObjectKind::MatMul) { + throw std::runtime_error( + "validated weight object has the wrong kind"); + } + + const auto& qweight = + package.Require(ComponentName(object, "qweight")); + const auto scales = + package.MaterializeFp16( + ComponentName(object, "scales")); + const auto& qzeros = + package.Require(ComponentName(object, "qzeros")); + + const ryzenai_corelib_matmul_bf16_weights_desc descriptor{ + object.k, + object.n, + constants::kGroupSize, + false}; + const ryzenai_corelib_matmul_bf16_onnx_weights_components + components{ + qweight.data, + scales.data(), + qzeros.data}; + + ryzenai_corelib_matmul_bf16_weights_ptr raw = nullptr; + const auto status = + api->functions().matmul_weights_from_onnx( + &descriptor, + &components, + &raw); + corelib::UniqueMatMulWeights weights(api, raw); + api->Check(status, kMatMulCreateCall); + if (!weights) { + throw std::runtime_error( + std::string(kMatMulCreateCall) + + " succeeded with a null object"); + } + + std::size_t packed_size = 0; + api->Check( + api->functions().matmul_weights_get_data( + weights.get(), + nullptr, + &packed_size), + kMatMulGetDataCall); + AddPackedBytes(packed_size, packed_bytes); + return weights; + } catch (const std::exception& error) { + ThrowObjectError(object, error); + } +} + +corelib::UniqueSsMlpWeights CreateSsMlp( + const std::shared_ptr& api, + Phi4Package& package, + const std::uint16_t* epsilon, + const WeightObjectView& object, + std::size_t& packed_bytes) { + try { + if (object.kind != WeightObjectKind::SsMlp) { + throw std::runtime_error( + "validated weight object has the wrong kind"); + } + + const auto norm0 = + package.MaterializeBf16( + ComponentName(object, "norm0")); + const auto norm1 = + package.MaterializeBf16( + ComponentName(object, "norm1")); + + const auto& gate_qweight = + package.Require( + ComponentName(object, "gate_qweight")); + const auto gate_scales = + package.MaterializeFp16( + ComponentName(object, "gate_scales")); + const auto& gate_qzeros = + package.Require( + ComponentName(object, "gate_qzeros")); + + const auto& up_qweight = + package.Require( + ComponentName(object, "up_qweight")); + const auto up_scales = + package.MaterializeFp16( + ComponentName(object, "up_scales")); + const auto& up_qzeros = + package.Require( + ComponentName(object, "up_qzeros")); + + const auto& down_qweight = + package.Require( + ComponentName(object, "down_qweight")); + const auto down_scales = + package.MaterializeFp16( + ComponentName(object, "down_scales")); + const auto& down_qzeros = + package.Require( + ComponentName(object, "down_qzeros")); + + const ryzenai_corelib_ssmlp_bf16_weights_desc descriptor{ + object.k, + object.n, + constants::kGroupSize}; + const ryzenai_corelib_ssmlp_bf16_onnx_weights_components + components{ + epsilon, + norm0.data(), + norm1.data(), + gate_qweight.data, + gate_scales.data(), + gate_qzeros.data, + up_qweight.data, + up_scales.data(), + up_qzeros.data, + down_qweight.data, + down_scales.data(), + down_qzeros.data}; + + ryzenai_corelib_ssmlp_bf16_weights_ptr raw = nullptr; + const auto status = + api->functions().ssmlp_weights_from_onnx( + &descriptor, + &components, + &raw); + corelib::UniqueSsMlpWeights weights(api, raw); + api->Check(status, kSsMlpCreateCall); + if (!weights) { + throw std::runtime_error( + std::string(kSsMlpCreateCall) + + " succeeded with a null object"); + } + + std::size_t packed_size = 0; + api->Check( + api->functions().ssmlp_weights_get_data( + weights.get(), + nullptr, + &packed_size), + kSsMlpGetDataCall); + AddPackedBytes(packed_size, packed_bytes); + return weights; + } catch (const std::exception& error) { + ThrowObjectError(object, error); + } +} + +} // namespace + +Phi4Weights Phi4Weights::Load( + std::shared_ptr api, + std::shared_ptr package) { + if (!api) { + throw std::invalid_argument( + "Phi4Weights::Load requires a CorelibApi"); + } + if (!package) { + throw std::invalid_argument( + "Phi4Weights::Load requires a Phi4Package"); + } + + Phi4Weights result; + result.package_ = std::move(package); + + auto epsilon = std::make_shared(); + const float epsilon_fp32 = + static_cast(constants::kRmsEpsilon); + try { + api->Check( + api->functions().convert( + ryzenai_corelib_data_type_fp32, + &epsilon_fp32, + ryzenai_corelib_data_type_bf16, + epsilon.get(), + 1), + kConvertCall); + } catch (const std::exception& error) { + throw std::runtime_error( + "failed to materialize Phi-4 RMS epsilon: " + + std::string(error.what())); + } + result.epsilon_bf16_ = std::move(epsilon); + + const auto& objects = result.package_->weight_objects(); + constexpr std::size_t objects_per_layer = 5; + constexpr std::size_t expected_objects = + static_cast(constants::kLayerCount) * + objects_per_layer + + 1; + if (objects.size() != expected_objects) { + throw std::runtime_error( + "Phi4Package must contain exactly 161 validated " + "weight objects"); + } + + for (std::size_t layer = 0; + layer < + static_cast(constants::kLayerCount); + ++layer) { + const std::size_t base = layer * objects_per_layer; + auto& destination = result.layers_[layer]; + destination.q = CreateMatMul( + api, + *result.package_, + objects[base], + result.packed_bytes_); + destination.k = CreateMatMul( + api, + *result.package_, + objects[base + 1], + result.packed_bytes_); + destination.v = CreateMatMul( + api, + *result.package_, + objects[base + 2], + result.packed_bytes_); + destination.o = CreateMatMul( + api, + *result.package_, + objects[base + 3], + result.packed_bytes_); + destination.mlp = CreateSsMlp( + api, + *result.package_, + result.epsilon_bf16_.get(), + objects[base + 4], + result.packed_bytes_); + } + + result.lm_head_ = CreateMatMul( + api, + *result.package_, + objects.back(), + result.packed_bytes_); + return result; +} + +Phi4Weights& Phi4Weights::operator=(Phi4Weights&& other) noexcept { + if (this != &other) { + ResetWeightObjects(); + epsilon_bf16_.reset(); + package_.reset(); + + package_ = std::move(other.package_); + epsilon_bf16_ = std::move(other.epsilon_bf16_); + layers_ = std::move(other.layers_); + lm_head_ = std::move(other.lm_head_); + packed_bytes_ = std::exchange(other.packed_bytes_, 0); + } + return *this; +} + +const std::array& +Phi4Weights::layers() const noexcept { + return layers_; +} + +const corelib::UniqueMatMulWeights& +Phi4Weights::lm_head() const noexcept { + return lm_head_; +} + +std::size_t Phi4Weights::packed_bytes() const noexcept { + return packed_bytes_; +} + +void Phi4Weights::ResetWeightObjects() noexcept { + lm_head_.reset(); + for (auto layer = layers_.rbegin(); + layer != layers_.rend(); + ++layer) { + layer->mlp.reset(); + layer->o.reset(); + layer->v.reset(); + layer->k.reset(); + layer->q.reset(); + } +} + +} // namespace flm::phi4 diff --git a/src/include/models/phi4/phi4_corelib_weights.hpp b/src/include/models/phi4/phi4_corelib_weights.hpp new file mode 100644 index 00000000..5321b65e --- /dev/null +++ b/src/include/models/phi4/phi4_corelib_weights.hpp @@ -0,0 +1,53 @@ +#pragma once + +#include +#include +#include + +#include +#include +#include +#include + +namespace flm::phi4 { + +struct LayerWeights { + corelib::UniqueMatMulWeights q; + corelib::UniqueMatMulWeights k; + corelib::UniqueMatMulWeights v; + corelib::UniqueMatMulWeights o; + corelib::UniqueSsMlpWeights mlp; +}; + +class Phi4Weights final { +public: + static Phi4Weights Load( + std::shared_ptr api, + std::shared_ptr package); + + Phi4Weights(Phi4Weights&&) noexcept = default; + Phi4Weights& operator=(Phi4Weights&& other) noexcept; + + Phi4Weights(const Phi4Weights&) = delete; + Phi4Weights& operator=(const Phi4Weights&) = delete; + + const std::array& + layers() const noexcept; + const corelib::UniqueMatMulWeights& lm_head() const noexcept; + std::size_t packed_bytes() const noexcept; + +private: + Phi4Weights() = default; + + void ResetWeightObjects() noexcept; + + // Owners precede every weight that receives their addresses. Reverse + // destruction therefore releases corelib objects before source storage. + std::shared_ptr package_; + std::shared_ptr epsilon_bf16_; + std::array layers_; + corelib::UniqueMatMulWeights lm_head_; + std::size_t packed_bytes_ = 0; +}; + +} // namespace flm::phi4 diff --git a/src/test/phi4_corelib_aie4/CMakeLists.txt b/src/test/phi4_corelib_aie4/CMakeLists.txt index 5aeb15ee..f18182ca 100644 --- a/src/test/phi4_corelib_aie4/CMakeLists.txt +++ b/src/test/phi4_corelib_aie4/CMakeLists.txt @@ -72,6 +72,7 @@ enable_testing() add_corelib_host_test(test_corelib_api test_corelib_api.cpp) add_corelib_host_test(test_phi4_manifest test_phi4_manifest.cpp) add_corelib_host_test(test_phi4_shape_plan test_phi4_shape_plan.cpp) +add_corelib_host_test(test_phi4_weights test_phi4_weights.cpp) add_corelib_host_test( test_corelib_fatal_record test_corelib_fatal_record.cpp) diff --git a/src/test/phi4_corelib_aie4/test_phi4_weights.cpp b/src/test/phi4_corelib_aie4/test_phi4_weights.cpp new file mode 100644 index 00000000..47bc6694 --- /dev/null +++ b/src/test/phi4_corelib_aie4/test_phi4_weights.cpp @@ -0,0 +1,1179 @@ +#include "fake_corelib.hpp" +#include "test_support.hpp" + +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using flm::corelib::CorelibApi; +using flm::phi4::Phi4Package; +using flm::phi4::Phi4Weights; +using flm::phi4::WeightObjectKind; +using nlohmann::json; + +namespace constants = flm::phi4::constants; + +constexpr std::string_view kManifestName = + "corelib_phi4_manifest.json"; +constexpr std::string_view kDataFile = "weights.bin"; +constexpr std::uint64_t kDataBytes = + 200064ull * 3072ull * sizeof(std::uint16_t); +constexpr std::size_t kMatMulPackedBytes = 17; +constexpr std::size_t kSsMlpPackedBytes = 29; +constexpr std::uint16_t kBf16One = 0x3f80u; +constexpr std::uint16_t kBf16Epsilon = 0x3728u; + +enum class FailurePoint { + None, + MatMulCreate, + MatMulGetData, + SsMlpCreate, + SsMlpGetData +}; + +struct FakeWeightHandle { + std::weak_ptr package; +}; + +struct MatMulCreateRecord { + ryzenai_corelib_matmul_bf16_weights_desc desc{}; + ryzenai_corelib_matmul_bf16_onnx_weights_components components{}; + void* object = nullptr; + std::thread::id thread; +}; + +struct SsMlpCreateRecord { + ryzenai_corelib_ssmlp_bf16_weights_desc desc{}; + ryzenai_corelib_ssmlp_bf16_onnx_weights_components components{}; + void* object = nullptr; + std::thread::id thread; +}; + +struct ConvertRecord { + ryzenai_corelib_data_type source_type = + ryzenai_corelib_data_type_fp32; + ryzenai_corelib_data_type destination_type = + ryzenai_corelib_data_type_fp32; + const void* source = nullptr; + void* destination = nullptr; + std::size_t count = 0; + float first_source_value = 0.0f; + std::thread::id thread; +}; + +struct RecordingState { + std::mutex mutex; + std::weak_ptr current_package; + FailurePoint failure = FailurePoint::None; + std::size_t failure_ordinal = 1; + std::size_t matmul_create_attempts = 0; + std::size_t matmul_get_attempts = 0; + std::size_t ssmlp_create_attempts = 0; + std::size_t ssmlp_get_attempts = 0; + std::vector matmul_creates; + std::vector ssmlp_creates; + std::vector converts; + std::vector creation_order; + std::vector release_order; + std::vector package_alive_at_release; + std::vector events; + std::size_t get_data_calls = 0; + bool every_get_data_pointer_argument_was_null = true; + bool every_get_data_size_argument_was_nonnull = true; +}; + +RecordingState* g_recording = nullptr; + +RecordingState& State() { + if (g_recording == nullptr) { + throw std::runtime_error("recording corelib is not active"); + } + return *g_recording; +} + +bool ShouldFail( + FailurePoint configured, + FailurePoint current, + std::size_t ordinal) { + return configured == current && + ordinal == State().failure_ordinal; +} + +ryzenai_corelib_status RecordingConvert( + ryzenai_corelib_data_type source_type, + const void* source, + ryzenai_corelib_data_type destination_type, + void* destination, + std::size_t count) { + if (source == nullptr || destination == nullptr || count == 0) { + return ryzenai_corelib_status_bad_argument; + } + + ConvertRecord record{ + source_type, + destination_type, + source, + destination, + count, + 0.0f, + std::this_thread::get_id()}; + if (source_type == ryzenai_corelib_data_type_fp32) { + record.first_source_value = + *static_cast(source); + } + + auto& state = State(); + { + std::lock_guard lock(state.mutex); + state.converts.push_back(record); + } + + auto* output = static_cast(destination); + if ( + destination_type == ryzenai_corelib_data_type_bf16 && + source_type == ryzenai_corelib_data_type_fp32 && + count == 1 && + std::abs(record.first_source_value - 1.0e-5f) < 1.0e-10f) { + output[0] = kBf16Epsilon; + } else if ( + destination_type == ryzenai_corelib_data_type_fp16 || + destination_type == ryzenai_corelib_data_type_bf16) { + output[0] = kBf16One; + } else { + return ryzenai_corelib_status_bad_argument; + } + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status RecordingMatMulCreate( + const ryzenai_corelib_matmul_bf16_weights_desc* desc, + const ryzenai_corelib_matmul_bf16_onnx_weights_components* components, + ryzenai_corelib_matmul_bf16_weights_ptr* out) { + if (desc == nullptr || components == nullptr || out == nullptr) { + return ryzenai_corelib_status_bad_argument; + } + *out = nullptr; + + auto& state = State(); + std::lock_guard lock(state.mutex); + const std::size_t ordinal = ++state.matmul_create_attempts; + state.events.emplace_back("matmul_create"); + if (ShouldFail( + state.failure, + FailurePoint::MatMulCreate, + ordinal)) { + flm::test::SetLastErrorMessage( + "intentional Task 6 MatMul create failure"); + return ryzenai_corelib_status_unsupported; + } + + auto* handle = new FakeWeightHandle{state.current_package}; + *out = handle; + state.matmul_creates.push_back( + {*desc, *components, handle, std::this_thread::get_id()}); + state.creation_order.push_back(handle); + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status RecordingMatMulGetData( + ryzenai_corelib_matmul_bf16_weights_ptr weights, + const void** data, + std::size_t* size) { + auto& state = State(); + std::lock_guard lock(state.mutex); + const std::size_t ordinal = ++state.matmul_get_attempts; + state.events.emplace_back("matmul_get_data"); + ++state.get_data_calls; + state.every_get_data_pointer_argument_was_null = + state.every_get_data_pointer_argument_was_null && + data == nullptr; + state.every_get_data_size_argument_was_nonnull = + state.every_get_data_size_argument_was_nonnull && + size != nullptr; + if ( + weights == nullptr || data != nullptr || size == nullptr) { + return ryzenai_corelib_status_bad_argument; + } + if (ShouldFail( + state.failure, + FailurePoint::MatMulGetData, + ordinal)) { + flm::test::SetLastErrorMessage( + "intentional Task 6 MatMul get-data failure"); + return ryzenai_corelib_status_unsupported; + } + *size = kMatMulPackedBytes; + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status RecordingSsMlpCreate( + const ryzenai_corelib_ssmlp_bf16_weights_desc* desc, + const ryzenai_corelib_ssmlp_bf16_onnx_weights_components* components, + ryzenai_corelib_ssmlp_bf16_weights_ptr* out) { + if (desc == nullptr || components == nullptr || out == nullptr) { + return ryzenai_corelib_status_bad_argument; + } + *out = nullptr; + + auto& state = State(); + std::lock_guard lock(state.mutex); + const std::size_t ordinal = ++state.ssmlp_create_attempts; + state.events.emplace_back("ssmlp_create"); + if (ShouldFail( + state.failure, + FailurePoint::SsMlpCreate, + ordinal)) { + flm::test::SetLastErrorMessage( + "intentional Task 6 SSMLP create failure"); + return ryzenai_corelib_status_unsupported; + } + + auto* handle = new FakeWeightHandle{state.current_package}; + *out = handle; + state.ssmlp_creates.push_back( + {*desc, *components, handle, std::this_thread::get_id()}); + state.creation_order.push_back(handle); + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status RecordingSsMlpGetData( + ryzenai_corelib_ssmlp_bf16_weights_ptr weights, + const void** data, + std::size_t* size) { + auto& state = State(); + std::lock_guard lock(state.mutex); + const std::size_t ordinal = ++state.ssmlp_get_attempts; + state.events.emplace_back("ssmlp_get_data"); + ++state.get_data_calls; + state.every_get_data_pointer_argument_was_null = + state.every_get_data_pointer_argument_was_null && + data == nullptr; + state.every_get_data_size_argument_was_nonnull = + state.every_get_data_size_argument_was_nonnull && + size != nullptr; + if ( + weights == nullptr || data != nullptr || size == nullptr) { + return ryzenai_corelib_status_bad_argument; + } + if (ShouldFail( + state.failure, + FailurePoint::SsMlpGetData, + ordinal)) { + flm::test::SetLastErrorMessage( + "intentional Task 6 SSMLP get-data failure"); + return ryzenai_corelib_status_unsupported; + } + *size = kSsMlpPackedBytes; + return ryzenai_corelib_status_success; +} + +void RecordingRelease(ryzenai_corelib_object_ptr object) { + auto* handle = static_cast(object); + auto& state = State(); + { + std::lock_guard lock(state.mutex); + state.release_order.push_back(object); + state.package_alive_at_release.push_back( + !handle->package.expired()); + state.events.emplace_back("release"); + } + delete handle; +} + +template +void* FunctionAddress(Function function) { + return reinterpret_cast(function); +} + +std::shared_ptr ResolveRecordingCorelib( + RecordingState& state) { + g_recording = &state; + auto resolver = flm::test::CompleteCorelibResolver(); + resolver["ryzenai_corelib_object_release"] = FunctionAddress( + static_cast( + &RecordingRelease)); + resolver["ryzenai_corelib_convert"] = FunctionAddress( + static_cast( + &RecordingConvert)); + resolver + ["ryzenai_corelib_matmul_bf16_weights_create_from_onnx_components"] = + FunctionAddress( + static_cast( + &RecordingMatMulCreate)); + resolver["ryzenai_corelib_matmul_bf16_weights_get_data"] = + FunctionAddress( + static_cast( + &RecordingMatMulGetData)); + resolver + ["ryzenai_corelib_ssmlp_bf16_weights_create_from_onnx_components"] = + FunctionAddress( + static_cast( + &RecordingSsMlpCreate)); + resolver["ryzenai_corelib_ssmlp_bf16_weights_get_data"] = + FunctionAddress( + static_cast( + &RecordingSsMlpGetData)); + return CorelibApi::ResolveForTest( + [resolver = std::move(resolver)](std::string_view name) mutable + -> void* { + const auto found = resolver.find(std::string(name)); + return found == resolver.end() ? nullptr : found->second; + }); +} + +class TempDirectory final { +public: + TempDirectory() { + const auto nonce = + std::chrono::steady_clock::now().time_since_epoch().count(); + path_ = std::filesystem::temp_directory_path() / + ("fastflowlm-phi4-weights-" + + std::to_string(GetCurrentProcessId()) + "-" + + std::to_string(nonce)); + std::filesystem::create_directories(path_); + } + + ~TempDirectory() noexcept { + std::error_code error; + std::filesystem::remove_all(path_, error); + } + + TempDirectory(const TempDirectory&) = delete; + TempDirectory& operator=(const TempDirectory&) = delete; + + const std::filesystem::path& path() const noexcept { + return path_; + } + +private: + std::filesystem::path path_; +}; + +void CreateSparseFile( + const std::filesystem::path& path, + std::uint64_t size) { + HANDLE file = CreateFileW( + path.c_str(), + GENERIC_READ | GENERIC_WRITE, + FILE_SHARE_READ, + nullptr, + CREATE_ALWAYS, + FILE_ATTRIBUTE_NORMAL, + nullptr); + if (file == INVALID_HANDLE_VALUE) { + throw std::runtime_error("failed to create sparse weight file"); + } + + DWORD ignored = 0; + if ( + DeviceIoControl( + file, + FSCTL_SET_SPARSE, + nullptr, + 0, + nullptr, + 0, + &ignored, + nullptr) == FALSE) { + CloseHandle(file); + throw std::runtime_error( + "test volume does not support sparse files"); + } + + LARGE_INTEGER end{}; + end.QuadPart = static_cast(size); + const bool success = + SetFilePointerEx(file, end, nullptr, FILE_BEGIN) != FALSE && + SetEndOfFile(file) != FALSE; + CloseHandle(file); + if (!success) { + throw std::runtime_error("failed to size sparse weight file"); + } +} + +std::uint64_t ItemSize(std::string_view dtype) { + if (dtype == "uint8") { + return 1; + } + if (dtype == "float16") { + return 2; + } + if (dtype == "float32") { + return 4; + } + throw std::runtime_error("unsupported synthetic dtype"); +} + +std::uint64_t ByteLength( + std::string_view dtype, + const std::vector& shape) { + std::uint64_t elements = 1; + for (const std::int64_t dimension : shape) { + elements *= static_cast(dimension); + } + return elements * ItemSize(dtype); +} + +void AddInitializer( + json& initializers, + const std::string& name, + std::string dtype, + std::vector shape, + std::string role, + std::uint64_t offset) { + CHECK(!initializers.contains(name)); + initializers[name] = { + {"file", std::string(kDataFile)}, + {"offset", offset}, + {"length", ByteLength(dtype, shape)}, + {"dtype", std::move(dtype)}, + {"shape", std::move(shape)}, + {"role", std::move(role)}}; +} + +class ManifestBuilder final { +public: + explicit ManifestBuilder(std::uint64_t model_size) + : manifest_{ + {"schema_version", 1}, + {"execution_backend", "corelib_aie4"}, + {"model", + { + {"family", "phi4"}, + {"layers", 32}, + {"hidden_size", 3072}, + {"intermediate_size", 8192}, + {"num_heads", 24}, + {"kv_heads", 8}, + {"head_size", 128}, + {"vocab_size", 200064}, + {"group_size", 128}, + {"rope_dim", 96}, + {"rms_epsilon", 0.00001}, + }}, + {"backend", {{"max_seq", 4096}}}, + {"files", + { + {"model.onnx", {{"size", model_size}}}, + {std::string(kDataFile), {{"size", kDataBytes}}}, + }}, + {"initializers", json::object()}, + {"weight_objects", json::array()}} {} + + void AddMatMul( + const std::string& name, + std::int64_t k, + std::int64_t n, + bool opaque_qweight = false) { + const std::string qweight = + opaque_qweight ? "opaque-qweight-from-explicit-role" + : name + ".qweight"; + const std::string scales = name + ".scales"; + const std::string qzeros = name + ".qzeros"; + manifest_["weight_objects"].push_back({ + {"name", name}, + {"kind", "matmul"}, + {"descriptor", + { + {"k", k}, + {"n", n}, + {"group_size", 128}, + {"has_bias", false}, + }}, + {"roles", + { + {"qweight", qweight}, + {"scales", scales}, + {"qzeros", qzeros}, + }}}); + + AddInitializer( + manifest_["initializers"], + qweight, + "uint8", + {n, k / 2}, + "matmul.qweight", + NextOffset()); + AddInitializer( + manifest_["initializers"], + scales, + name == "model.layers.0.attn.q_proj.MatMulNBits" + ? "float32" + : "float16", + {n, k / 128}, + "matmul.scales", + NextOffset()); + AddInitializer( + manifest_["initializers"], + qzeros, + "uint8", + {n, ((k / 128) + 1) / 2}, + "matmul.qzeros", + NextOffset()); + } + + void AddSsMlp(int layer) { + const std::string base = + "model.layers." + std::to_string(layer); + const std::string object_name = base + ".ssmlp"; + const std::string norm0 = + base + ".post_attention_layernorm.weight"; + const std::string norm1 = + layer == 31 + ? "model.layers.32.final_norm_layernorm.weight" + : "model.layers." + std::to_string(layer + 1) + + ".input_layernorm.weight"; + json roles = { + {"norm0", norm0}, + {"norm1", norm1}, + }; + + for (const std::string projection : {"gate", "up", "down"}) { + const std::int64_t k = + projection == "down" ? 8192 : 3072; + const std::int64_t n = + projection == "down" ? 3072 : 8192; + const std::string prefix = + base + ".mlp." + projection + + "_proj.MatMulNBits"; + const std::string role_prefix = + "ssmlp." + projection; + for (const std::string component : + {"qweight", "scales", "qzeros"}) { + roles[projection + "_" + component] = + prefix + "." + component; + } + AddInitializer( + manifest_["initializers"], + prefix + ".qweight", + "uint8", + {n, k / 2}, + role_prefix + ".qweight", + NextOffset()); + AddInitializer( + manifest_["initializers"], + prefix + ".scales", + "float16", + {n, k / 128}, + role_prefix + ".scales", + NextOffset()); + AddInitializer( + manifest_["initializers"], + prefix + ".qzeros", + "uint8", + {n, ((k / 128) + 1) / 2}, + role_prefix + ".qzeros", + NextOffset()); + } + + AddInitializer( + manifest_["initializers"], + norm0, + layer == 0 ? "float32" : "float16", + {3072}, + "ssmlp.norm0", + NextOffset()); + AddInitializer( + manifest_["initializers"], + norm1, + "float16", + {3072}, + "ssmlp.norm1", + NextOffset()); + manifest_["weight_objects"].push_back({ + {"name", object_name}, + {"kind", "ssmlp"}, + {"descriptor", + { + {"k", 3072}, + {"n", 8192}, + {"group_size", 128}, + }}, + {"roles", std::move(roles)}}); + } + + json Finish() { + for (int layer = 0; layer < 32; ++layer) { + const std::string base = + "model.layers." + std::to_string(layer) + + ".attn."; + AddMatMul( + base + "q_proj.MatMulNBits", + 3072, + 3072, + layer == 0); + AddMatMul( + base + "k_proj.MatMulNBits", + 3072, + 1024); + AddMatMul( + base + "v_proj.MatMulNBits", + 3072, + 1024); + AddMatMul( + base + "o_proj.MatMulNBits", + 3072, + 3072); + AddSsMlp(layer); + } + AddMatMul( + "lm_head.MatMulNBits", + 3072, + 200064); + + AddInitializer( + manifest_["initializers"], + "model.embed_tokens.weight", + "float16", + {200064, 3072}, + "embedding", + 0); + AddInitializer( + manifest_["initializers"], + "model.layers.0.input_layernorm.weight", + "float16", + {3072}, + "input_norm", + NextOffset()); + AddInitializer( + manifest_["initializers"], + "cos_cache", + "float16", + {4096, 48}, + "cos_cache", + NextOffset()); + AddInitializer( + manifest_["initializers"], + "sin_cache", + "float32", + {4096, 48}, + "sin_cache", + NextOffset()); + + CHECK(manifest_["weight_objects"].size() == 161); + CHECK(manifest_["initializers"].size() == 743); + return std::move(manifest_); + } + +private: + std::uint64_t NextOffset() { + const std::uint64_t result = next_offset_; + next_offset_ += 16; + return result; + } + + json manifest_; + std::uint64_t next_offset_ = 4096; +}; + +class SyntheticPackage final { +public: + SyntheticPackage() { + const auto model_path = temp_.path() / "model.onnx"; + { + std::ofstream model(model_path, std::ios::binary); + model << "model"; + } + CreateSparseFile(temp_.path() / kDataFile, kDataBytes); + ManifestBuilder builder( + std::filesystem::file_size(model_path)); + std::ofstream manifest( + temp_.path() / kManifestName, + std::ios::binary); + manifest << builder.Finish().dump(2) << '\n'; + if (!manifest) { + throw std::runtime_error( + "failed to write synthetic weight manifest"); + } + } + + std::shared_ptr Load( + const std::shared_ptr& api) const { + return std::make_shared( + Phi4Package::Load(temp_.path(), api, false)); + } + +private: + TempDirectory temp_; +}; + +const std::string& Role( + const flm::phi4::WeightObjectView& object, + std::string_view role) { + return object.components.at(std::string(role)); +} + +void CheckMatMulComponents( + const MatMulCreateRecord& actual, + const flm::phi4::WeightObjectView& expected, + Phi4Package& package) { + CHECK(expected.kind == WeightObjectKind::MatMul); + CHECK(actual.desc.k == expected.k); + CHECK(actual.desc.n == expected.n); + CHECK(actual.desc.group_size == 128); + CHECK(actual.desc.has_bias == false); + + const auto& qweight = package.Require(Role(expected, "qweight")); + const auto scales = + package.MaterializeFp16(Role(expected, "scales")); + const auto& qzeros = package.Require(Role(expected, "qzeros")); + CHECK(actual.components.qweight == qweight.data); + CHECK(actual.components.scales == scales.data()); + CHECK(actual.components.qzeros == qzeros.data); + CHECK(actual.components.scales != + package.Require(Role(expected, "scales")).data); + CHECK( + *static_cast(actual.components.qweight) == + std::byte{0}); + CHECK( + *static_cast(actual.components.qzeros) == + std::byte{0}); + CHECK( + *static_cast(actual.components.scales) == + kBf16One); +} + +void CheckSsMlpComponents( + const SsMlpCreateRecord& actual, + const flm::phi4::WeightObjectView& expected, + Phi4Package& package) { + CHECK(expected.kind == WeightObjectKind::SsMlp); + CHECK(actual.desc.k == 3072); + CHECK(actual.desc.n == 8192); + CHECK(actual.desc.group_size == 128); + CHECK(actual.components.epsilon != nullptr); + CHECK( + *static_cast( + actual.components.epsilon) == kBf16Epsilon); + + const auto check_norm = [&]( + const void* actual_pointer, + std::string_view role) { + const std::string& name = Role(expected, role); + const auto value = package.MaterializeBf16(name); + CHECK(actual_pointer == value.data()); + CHECK(actual_pointer != package.Require(name).data); + CHECK( + *static_cast(actual_pointer) == + kBf16One); + }; + const auto check_projection = [&]( + const void* qweight, + const void* scales, + const void* qzeros, + std::string_view prefix) { + const std::string qweight_role = + std::string(prefix) + "_qweight"; + const std::string scales_role = + std::string(prefix) + "_scales"; + const std::string qzeros_role = + std::string(prefix) + "_qzeros"; + CHECK(qweight == + package.Require(Role(expected, qweight_role)).data); + CHECK(scales == + package.MaterializeFp16( + Role(expected, scales_role)) + .data()); + CHECK(qzeros == + package.Require(Role(expected, qzeros_role)).data); + CHECK(scales != + package.Require(Role(expected, scales_role)).data); + }; + + check_norm(actual.components.norm0, "norm0"); + check_norm(actual.components.norm1, "norm1"); + check_projection( + actual.components.gate_qweight, + actual.components.gate_scales, + actual.components.gate_qzeros, + "gate"); + check_projection( + actual.components.up_qweight, + actual.components.up_scales, + actual.components.up_qzeros, + "up"); + check_projection( + actual.components.down_qweight, + actual.components.down_scales, + actual.components.down_qzeros, + "down"); +} + +void CheckImmediateSizeQueries(const RecordingState& state) { + CHECK(state.events.size() == 322); + for (std::size_t index = 0; index < state.events.size(); index += 2) { + const bool matmul = + state.events[index] == "matmul_create"; + CHECK(matmul || state.events[index] == "ssmlp_create"); + CHECK( + state.events[index + 1] == + (matmul ? "matmul_get_data" : "ssmlp_get_data")); + } +} + +void CheckReleaseOrder( + const std::vector& creation_order, + const std::vector& release_order) { + CHECK(creation_order.size() == release_order.size()); + CHECK(std::equal( + release_order.begin(), + release_order.end(), + creation_order.rbegin(), + creation_order.rend())); +} + +void TestExactConstructionAndLifetime( + const SyntheticPackage& fixture) { + RecordingState state; + flm::test::ResetFakeCorelib(); + auto api = ResolveRecordingCorelib(state); + auto package = fixture.Load(api); + state.current_package = package; + const std::weak_ptr package_lifetime = package; + const std::thread::id load_thread = std::this_thread::get_id(); + + { + Phi4Weights weights = Phi4Weights::Load(api, package); + CHECK(weights.layers().size() == 32); + CHECK(state.matmul_creates.size() == 32u * 4u + 1u); + CHECK(state.ssmlp_creates.size() == 32u); + CHECK(state.get_data_calls == 161u); + CHECK(state.every_get_data_pointer_argument_was_null); + CHECK(state.every_get_data_size_argument_was_nonnull); + CHECK(weights.packed_bytes() == + 129u * kMatMulPackedBytes + + 32u * kSsMlpPackedBytes); + CHECK(api->live_object_count() == 161); + CheckImmediateSizeQueries(state); + + const auto& objects = package->weight_objects(); + CHECK(objects.size() == 161); + CHECK(Role(objects.front(), "qweight") == + "opaque-qweight-from-explicit-role"); + for (std::size_t layer = 0; layer < 32; ++layer) { + const std::size_t object_base = layer * 5; + const std::size_t matmul_base = layer * 4; + for (std::size_t projection = 0; + projection < 4; + ++projection) { + CheckMatMulComponents( + state.matmul_creates.at( + matmul_base + projection), + objects.at(object_base + projection), + *package); + } + CheckSsMlpComponents( + state.ssmlp_creates.at(layer), + objects.at(object_base + 4), + *package); + CHECK(weights.layers()[layer].q); + CHECK(weights.layers()[layer].k); + CHECK(weights.layers()[layer].v); + CHECK(weights.layers()[layer].o); + CHECK(weights.layers()[layer].mlp); + } + CheckMatMulComponents( + state.matmul_creates.back(), + objects.back(), + *package); + CHECK(weights.lm_head()); + CHECK(state.matmul_creates.front().desc.has_bias == false); + CHECK(state.matmul_creates.back().desc.n == 200064); + CHECK( + Role(objects.at(31u * 5u + 4u), "norm1") == + "model.layers.32.final_norm_layernorm.weight"); + CHECK( + state.ssmlp_creates.back().components.norm1 == + package + ->MaterializeBf16( + "model.layers.32.final_norm_layernorm.weight") + .data()); + + CHECK(state.converts.size() == 290); + CHECK(std::count_if( + state.converts.begin(), + state.converts.end(), + [](const ConvertRecord& call) { + return call.destination_type == + ryzenai_corelib_data_type_fp16; + }) == 225); + CHECK(std::count_if( + state.converts.begin(), + state.converts.end(), + [](const ConvertRecord& call) { + return call.destination_type == + ryzenai_corelib_data_type_bf16; + }) == 65); + CHECK(std::all_of( + state.converts.begin(), + state.converts.end(), + [load_thread](const ConvertRecord& call) { + return call.thread == load_thread; + })); + CHECK(std::all_of( + state.matmul_creates.begin(), + state.matmul_creates.end(), + [load_thread](const MatMulCreateRecord& call) { + return call.thread == load_thread; + })); + CHECK(std::all_of( + state.ssmlp_creates.begin(), + state.ssmlp_creates.end(), + [load_thread](const SsMlpCreateRecord& call) { + return call.thread == load_thread; + })); + + const std::vector creation_order = + state.creation_order; + Phi4Weights moved(std::move(weights)); + CHECK(state.release_order.empty()); + CHECK(api->live_object_count() == 161); + package.reset(); + CHECK(!package_lifetime.expired()); + + auto retained = package_lifetime.lock(); + CHECK(retained != nullptr); + const auto& retained_objects = retained->weight_objects(); + for (std::size_t layer = 0; layer < 32; ++layer) { + for (std::size_t projection = 0; + projection < 4; + ++projection) { + CheckMatMulComponents( + state.matmul_creates.at( + layer * 4 + projection), + retained_objects.at( + layer * 5 + projection), + *retained); + } + CheckSsMlpComponents( + state.ssmlp_creates.at(layer), + retained_objects.at(layer * 5 + 4), + *retained); + } + CheckMatMulComponents( + state.matmul_creates.back(), + retained_objects.back(), + *retained); + retained.reset(); + CHECK(!package_lifetime.expired()); + CHECK(state.converts.size() == 290); + (void)moved; + CHECK(creation_order.size() == 161); + } + + CHECK(state.release_order.size() == 161); + CHECK(std::all_of( + state.package_alive_at_release.begin(), + state.package_alive_at_release.end(), + [](bool value) { return value; })); + CheckReleaseOrder(state.creation_order, state.release_order); + CHECK(package_lifetime.expired()); + CHECK(api->live_object_count() == 0); + g_recording = nullptr; +} + +void TestMoveAssignmentReleasesBeforeOwners( + const SyntheticPackage& fixture) { + RecordingState state; + flm::test::ResetFakeCorelib(); + auto api = ResolveRecordingCorelib(state); + auto source_package = fixture.Load(api); + auto destination_package = fixture.Load(api); + const std::weak_ptr source_lifetime = source_package; + const std::weak_ptr destination_lifetime = + destination_package; + + { + state.current_package = source_package; + Phi4Weights source = + Phi4Weights::Load(api, source_package); + const std::vector source_objects = + state.creation_order; + + state.current_package = destination_package; + Phi4Weights destination = + Phi4Weights::Load(api, destination_package); + const std::vector destination_objects( + state.creation_order.begin() + 161, + state.creation_order.end()); + CHECK(source_objects.size() == 161); + CHECK(destination_objects.size() == 161); + CHECK(api->live_object_count() == 322); + + source_package.reset(); + destination_package.reset(); + destination = std::move(source); + + CHECK(state.release_order.size() == 161); + CheckReleaseOrder( + destination_objects, + state.release_order); + CHECK(destination_lifetime.expired()); + CHECK(!source_lifetime.expired()); + CHECK(api->live_object_count() == 161); + } + + CHECK(state.release_order.size() == 322); + const std::vector source_releases( + state.release_order.begin() + 161, + state.release_order.end()); + const std::vector source_objects( + state.creation_order.begin(), + state.creation_order.begin() + 161); + CheckReleaseOrder(source_objects, source_releases); + CHECK(std::all_of( + state.package_alive_at_release.begin(), + state.package_alive_at_release.end(), + [](bool value) { return value; })); + CHECK(source_lifetime.expired()); + CHECK(api->live_object_count() == 0); + g_recording = nullptr; +} + +void CheckLoadFailure( + RecordingState& state, + const std::shared_ptr& api, + const std::shared_ptr& package, + FailurePoint failure, + std::string_view object_name, + std::string_view call, + std::string_view detail) { + state.failure = failure; + state.failure_ordinal = 1; + state.matmul_create_attempts = 0; + state.matmul_get_attempts = 0; + state.ssmlp_create_attempts = 0; + state.ssmlp_get_attempts = 0; + state.matmul_creates.clear(); + state.ssmlp_creates.clear(); + state.converts.clear(); + state.creation_order.clear(); + state.release_order.clear(); + state.package_alive_at_release.clear(); + state.events.clear(); + state.get_data_calls = 0; + state.every_get_data_pointer_argument_was_null = true; + state.every_get_data_size_argument_was_nonnull = true; + state.current_package = package; + flm::test::SetLastErrorMessage({}); + + try { + (void)Phi4Weights::Load(api, package); + } catch (const std::exception& error) { + const std::string_view message(error.what()); + CHECK(message.find(object_name) != std::string_view::npos); + CHECK(message.find(call) != std::string_view::npos); + CHECK(message.find(detail) != std::string_view::npos); + CHECK(api->live_object_count() == 0); + CHECK(std::all_of( + state.package_alive_at_release.begin(), + state.package_alive_at_release.end(), + [](bool value) { return value; })); + return; + } + throw std::runtime_error( + "expected Phi4Weights::Load failure was not thrown"); +} + +void TestActionableFailures(const SyntheticPackage& fixture) { + RecordingState state; + flm::test::ResetFakeCorelib(); + auto api = ResolveRecordingCorelib(state); + auto package = fixture.Load(api); + + CheckThrowsContains( + [&] { + (void)Phi4Weights::Load(nullptr, package); + }, + "CorelibApi"); + CheckThrowsContains( + [&] { + (void)Phi4Weights::Load(api, nullptr); + }, + "Phi4Package"); + + CheckLoadFailure( + state, + api, + package, + FailurePoint::MatMulCreate, + "model.layers.0.attn.q_proj.MatMulNBits", + "ryzenai_corelib_matmul_bf16_weights_create_from_onnx_components", + "intentional Task 6 MatMul create failure"); + CheckLoadFailure( + state, + api, + package, + FailurePoint::MatMulGetData, + "model.layers.0.attn.q_proj.MatMulNBits", + "ryzenai_corelib_matmul_bf16_weights_get_data", + "intentional Task 6 MatMul get-data failure"); + CheckLoadFailure( + state, + api, + package, + FailurePoint::SsMlpCreate, + "model.layers.0.ssmlp", + "ryzenai_corelib_ssmlp_bf16_weights_create_from_onnx_components", + "intentional Task 6 SSMLP create failure"); + CheckLoadFailure( + state, + api, + package, + FailurePoint::SsMlpGetData, + "model.layers.0.ssmlp", + "ryzenai_corelib_ssmlp_bf16_weights_get_data", + "intentional Task 6 SSMLP get-data failure"); + g_recording = nullptr; +} + +static_assert(!std::is_copy_constructible_v); +static_assert(!std::is_copy_assignable_v); +static_assert(std::is_nothrow_move_constructible_v); +static_assert(std::is_nothrow_move_assignable_v); + +} // namespace + +int main() { + try { + SyntheticPackage fixture; + TestExactConstructionAndLifetime(fixture); + TestMoveAssignmentReleasesBeforeOwners(fixture); + TestActionableFailures(fixture); + std::cout << "test_phi4_weights: PASS\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} From b94ba0b6fccdc0ad39655d2c11f79399558cd009 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CChi?= Date: Mon, 31 Aug 2026 23:58:38 -0700 Subject: [PATCH 011/117] fix: preserve Phi-4 weight error contracts Co-authored-by: Cursor --- src/common/corelib/corelib_api.cpp | 17 ++- src/common/corelib/phi4_corelib_manifest.cpp | 93 ++++++++++-- src/common/corelib/phi4_corelib_weights.cpp | 11 ++ src/include/corelib/corelib_api.hpp | 5 + .../phi4_corelib_aie4/test_phi4_manifest.cpp | 56 +++++-- .../phi4_corelib_aie4/test_phi4_weights.cpp | 143 ++++++++++++++---- 6 files changed, 267 insertions(+), 58 deletions(-) diff --git a/src/common/corelib/corelib_api.cpp b/src/common/corelib/corelib_api.cpp index 4413c3e3..2123ce49 100644 --- a/src/common/corelib/corelib_api.cpp +++ b/src/common/corelib/corelib_api.cpp @@ -197,7 +197,22 @@ CorelibError::CorelibError( detail_value)), status(status_value), call(std::move(call_value)), - detail(std::move(detail_value)) {} + detail(std::move(detail_value)), + status_text_(std::move(status_text)) {} + +CorelibError CorelibError::WithContext( + std::string_view context) const { + std::string enriched_detail(context); + if (!enriched_detail.empty() && !detail.empty()) { + enriched_detail += ": "; + } + enriched_detail += detail; + return CorelibError{ + status, + call, + std::move(enriched_detail), + status_text_}; +} std::shared_ptr CorelibApi::Load( const std::filesystem::path& absolute_path) { diff --git a/src/common/corelib/phi4_corelib_manifest.cpp b/src/common/corelib/phi4_corelib_manifest.cpp index 7dc5291e..c1c2f389 100644 --- a/src/common/corelib/phi4_corelib_manifest.cpp +++ b/src/common/corelib/phi4_corelib_manifest.cpp @@ -354,48 +354,92 @@ struct ExpectedWeightObject { WeightObjectKind kind; std::int64_t k; std::int64_t n; + std::map components; }; +std::map ExpectedMatMulComponents( + const std::string& object_name) { + return { + {"qweight", object_name + ".qweight"}, + {"scales", object_name + ".scales"}, + {"qzeros", object_name + ".qzeros"}}; +} + +std::map ExpectedSsMlpComponents( + std::int64_t layer) { + const std::string base = + "model.layers." + std::to_string(layer); + std::map components{ + {"norm0", base + ".post_attention_layernorm.weight"}, + {"norm1", + layer + 1 == constants::kLayerCount + ? "model.layers.32.final_norm_layernorm.weight" + : "model.layers." + std::to_string(layer + 1) + + ".input_layernorm.weight"}}; + for (const std::string projection : {"gate", "up", "down"}) { + const std::string initializer = + base + ".mlp." + projection + "_proj.MatMulNBits."; + components.emplace( + projection + "_qweight", + initializer + "qweight"); + components.emplace( + projection + "_scales", + initializer + "scales"); + components.emplace( + projection + "_qzeros", + initializer + "qzeros"); + } + return components; +} + const std::vector& ExpectedWeightObjects() { static const std::vector expected = [] { std::vector values; values.reserve(kExpectedWeightObjects); + const auto add_matmul = [&]( + std::string name, + std::int64_t k, + std::int64_t n) { + auto components = ExpectedMatMulComponents(name); + values.push_back({ + std::move(name), + WeightObjectKind::MatMul, + k, + n, + std::move(components)}); + }; for (std::int64_t layer = 0; layer < constants::kLayerCount; ++layer) { const std::string base = "model.layers." + std::to_string(layer) + ".attn."; - values.push_back({ + add_matmul( base + "q_proj.MatMulNBits", - WeightObjectKind::MatMul, constants::kHiddenSize, - constants::kQueryDimension}); - values.push_back({ + constants::kQueryDimension); + add_matmul( base + "k_proj.MatMulNBits", - WeightObjectKind::MatMul, constants::kHiddenSize, - constants::kKvDimension}); - values.push_back({ + constants::kKvDimension); + add_matmul( base + "v_proj.MatMulNBits", - WeightObjectKind::MatMul, constants::kHiddenSize, - constants::kKvDimension}); - values.push_back({ + constants::kKvDimension); + add_matmul( base + "o_proj.MatMulNBits", - WeightObjectKind::MatMul, constants::kQueryDimension, - constants::kHiddenSize}); + constants::kHiddenSize); values.push_back({ "model.layers." + std::to_string(layer) + ".ssmlp", WeightObjectKind::SsMlp, constants::kHiddenSize, - constants::kIntermediateSize}); + constants::kIntermediateSize, + ExpectedSsMlpComponents(layer)}); } - values.push_back({ + add_matmul( "lm_head.MatMulNBits", - WeightObjectKind::MatMul, constants::kHiddenSize, - constants::kVocabularySize}); + constants::kVocabularySize); return values; }(); return expected; @@ -1148,6 +1192,23 @@ Phi4Package Phi4Package::Load( } components.emplace(role, initializer_name); } + for (const auto& [role, expected_initializer] : + expected.components) { + const auto component = components.find(role); + if ( + component == components.end() || + component->second != expected_initializer) { + Throw( + component == components.end() + ? ComponentContext(name, role) + : ComponentContext( + name, + role, + component->second), + "does not match fixed Phi-4 initializer " + + expected_initializer); + } + } if (kind == WeightObjectKind::MatMul) { ValidateQuantizedProjection( diff --git a/src/common/corelib/phi4_corelib_weights.cpp b/src/common/corelib/phi4_corelib_weights.cpp index e962bf43..9c9f0ced 100644 --- a/src/common/corelib/phi4_corelib_weights.cpp +++ b/src/common/corelib/phi4_corelib_weights.cpp @@ -44,6 +44,13 @@ void AddPackedBytes( total += packed_size; } +[[noreturn]] void ThrowObjectError( + const WeightObjectView& object, + const corelib::CorelibError& error) { + throw error.WithContext( + "Phi-4 weight object '" + object.name + "'"); +} + [[noreturn]] void ThrowObjectError( const WeightObjectView& object, const std::exception& error) { @@ -105,6 +112,8 @@ corelib::UniqueMatMulWeights CreateMatMul( kMatMulGetDataCall); AddPackedBytes(packed_size, packed_bytes); return weights; + } catch (const corelib::CorelibError& error) { + ThrowObjectError(object, error); } catch (const std::exception& error) { ThrowObjectError(object, error); } @@ -201,6 +210,8 @@ corelib::UniqueSsMlpWeights CreateSsMlp( kSsMlpGetDataCall); AddPackedBytes(packed_size, packed_bytes); return weights; + } catch (const corelib::CorelibError& error) { + ThrowObjectError(object, error); } catch (const std::exception& error) { ThrowObjectError(object, error); } diff --git a/src/include/corelib/corelib_api.hpp b/src/include/corelib/corelib_api.hpp index 9c04ca8b..960903af 100644 --- a/src/include/corelib/corelib_api.hpp +++ b/src/include/corelib/corelib_api.hpp @@ -20,9 +20,14 @@ struct CorelibError final : std::runtime_error { std::string detail, std::string status_text); + CorelibError WithContext(std::string_view context) const; + ryzenai_corelib_status status; std::string call; std::string detail; + +private: + std::string status_text_; }; struct CorelibFunctions { diff --git a/src/test/phi4_corelib_aie4/test_phi4_manifest.cpp b/src/test/phi4_corelib_aie4/test_phi4_manifest.cpp index 0cc7eb41..d9600234 100644 --- a/src/test/phi4_corelib_aie4/test_phi4_manifest.cpp +++ b/src/test/phi4_corelib_aie4/test_phi4_manifest.cpp @@ -743,18 +743,10 @@ void ExpectLoadFailure( std::string(expected) + "'"); } -void TestValidMappingAndExplicitGrouping( +void TestValidMappingAndExplicitRoles( const SyntheticPackage& fixture, const std::shared_ptr& api) { - json manifest = fixture.manifest(); - const std::string original = - "model.layers.0.attn.q_proj.MatMulNBits.qweight"; - const std::string opaque = "opaque-initializer-000"; - manifest["initializers"][opaque] = - manifest["initializers"].at(original); - manifest["initializers"].erase(original); - manifest["weight_objects"][0]["roles"]["qweight"] = opaque; - fixture.Write(manifest); + fixture.Write(fixture.manifest()); auto package = Phi4Package::Load(fixture.path(), api, false); CHECK(package.weight_objects().size() == 161); @@ -766,7 +758,9 @@ void TestValidMappingAndExplicitGrouping( CHECK(first.n == 3072); CHECK(first.group_size == 128); CHECK(!first.has_bias); - CHECK(first.components.at("qweight") == opaque); + CHECK( + first.components.at("qweight") == + "model.layers.0.attn.q_proj.MatMulNBits.qweight"); const auto& last = package.weight_objects().back(); CHECK(last.name == "lm_head.MatMulNBits"); @@ -961,6 +955,43 @@ void TestWeightObjectRejections( "duplicate initializer"); } +void TestRoleInitializerIdentityRejections( + const SyntheticPackage& fixture, + const std::shared_ptr& api) { + ExpectLoadFailure( + fixture, + api, + [](json& manifest) { + auto& q_role = + manifest["weight_objects"][0]["roles"]["qweight"]; + auto& o_role = + manifest["weight_objects"][3]["roles"]["qweight"]; + const std::string q_initializer = + q_role.get(); + q_role = o_role.get(); + o_role = q_initializer; + }, + "model.layers.0.attn.q_proj.MatMulNBits.qweight " + "(model.layers.0.attn.o_proj.MatMulNBits.qweight)"); + ExpectLoadFailure( + fixture, + api, + [](json& manifest) { + constexpr std::size_t layer5 = 5u * 5u + 4u; + constexpr std::size_t layer31 = 31u * 5u + 4u; + auto& layer5_norm = + manifest["weight_objects"][layer5]["roles"]["norm1"]; + auto& layer31_norm = + manifest["weight_objects"][layer31]["roles"]["norm1"]; + const std::string initializer = + layer5_norm.get(); + layer5_norm = layer31_norm.get(); + layer31_norm = initializer; + }, + "model.layers.5.ssmlp.norm1 " + "(model.layers.32.final_norm_layernorm.weight)"); +} + void TestExactSourceValidation( const SyntheticPackage& fixture, const std::shared_ptr& api) { @@ -1253,10 +1284,11 @@ int main() { try { SyntheticPackage fixture; auto api = ResolveRecordingCorelib(); - TestValidMappingAndExplicitGrouping(fixture, api); + TestValidMappingAndExplicitRoles(fixture, api); TestMappedOwnerOutlivesPackage(fixture, api); TestPathRangeAndHashRejections(fixture, api); TestWeightObjectRejections(fixture, api); + TestRoleInitializerIdentityRejections(fixture, api); TestExactSourceValidation(fixture, api); TestComponentDiagnosticsIdentifyWeightObjects(fixture, api); TestOwnedScaleAndNormConversions(fixture, api); diff --git a/src/test/phi4_corelib_aie4/test_phi4_weights.cpp b/src/test/phi4_corelib_aie4/test_phi4_weights.cpp index 47bc6694..2c736610 100644 --- a/src/test/phi4_corelib_aie4/test_phi4_weights.cpp +++ b/src/test/phi4_corelib_aie4/test_phi4_weights.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -32,6 +33,7 @@ namespace { using flm::corelib::CorelibApi; +using flm::corelib::CorelibError; using flm::phi4::Phi4Package; using flm::phi4::Phi4Weights; using flm::phi4::WeightObjectKind; @@ -106,6 +108,8 @@ struct RecordingState { std::size_t get_data_calls = 0; bool every_get_data_pointer_argument_was_null = true; bool every_get_data_size_argument_was_nonnull = true; + std::size_t matmul_packed_bytes = kMatMulPackedBytes; + std::size_t ssmlp_packed_bytes = kSsMlpPackedBytes; }; RecordingState* g_recording = nullptr; @@ -228,7 +232,7 @@ ryzenai_corelib_status RecordingMatMulGetData( "intentional Task 6 MatMul get-data failure"); return ryzenai_corelib_status_unsupported; } - *size = kMatMulPackedBytes; + *size = state.matmul_packed_bytes; return ryzenai_corelib_status_success; } @@ -289,7 +293,7 @@ ryzenai_corelib_status RecordingSsMlpGetData( "intentional Task 6 SSMLP get-data failure"); return ryzenai_corelib_status_unsupported; } - *size = kSsMlpPackedBytes; + *size = state.ssmlp_packed_bytes; return ryzenai_corelib_status_success; } @@ -493,11 +497,8 @@ class ManifestBuilder final { void AddMatMul( const std::string& name, std::int64_t k, - std::int64_t n, - bool opaque_qweight = false) { - const std::string qweight = - opaque_qweight ? "opaque-qweight-from-explicit-role" - : name + ".qweight"; + std::int64_t n) { + const std::string qweight = name + ".qweight"; const std::string scales = name + ".scales"; const std::string qzeros = name + ".qzeros"; manifest_["weight_objects"].push_back({ @@ -630,8 +631,7 @@ class ManifestBuilder final { AddMatMul( base + "q_proj.MatMulNBits", 3072, - 3072, - layer == 0); + 3072); AddMatMul( base + "k_proj.MatMulNBits", 3072, @@ -878,8 +878,6 @@ void TestExactConstructionAndLifetime( const auto& objects = package->weight_objects(); CHECK(objects.size() == 161); - CHECK(Role(objects.front(), "qweight") == - "opaque-qweight-from-explicit-role"); for (std::size_t layer = 0; layer < 32; ++layer) { const std::size_t object_base = layer * 5; const std::size_t matmul_base = layer * 4; @@ -1063,13 +1061,15 @@ void TestMoveAssignmentReleasesBeforeOwners( void CheckLoadFailure( RecordingState& state, const std::shared_ptr& api, - const std::shared_ptr& package, + std::shared_ptr package, FailurePoint failure, + std::size_t failure_ordinal, + std::size_t expected_created, std::string_view object_name, std::string_view call, std::string_view detail) { state.failure = failure; - state.failure_ordinal = 1; + state.failure_ordinal = failure_ordinal; state.matmul_create_attempts = 0; state.matmul_get_attempts = 0; state.ssmlp_create_attempts = 0; @@ -1088,18 +1088,35 @@ void CheckLoadFailure( flm::test::SetLastErrorMessage({}); try { - (void)Phi4Weights::Load(api, package); - } catch (const std::exception& error) { - const std::string_view message(error.what()); - CHECK(message.find(object_name) != std::string_view::npos); - CHECK(message.find(call) != std::string_view::npos); - CHECK(message.find(detail) != std::string_view::npos); + (void)Phi4Weights::Load(api, std::move(package)); + } catch (const CorelibError& error) { + const std::string expected_context = + "Phi-4 weight object '" + std::string(object_name) + "'"; + const std::string expected_detail = + expected_context + ": " + std::string(detail); + const std::string expected_what = + std::string(call) + " failed: unsupported: " + + expected_detail; + CHECK(error.status == ryzenai_corelib_status_unsupported); + CHECK(error.call == call); + CHECK(error.detail == expected_detail); + CHECK(std::string_view(error.what()) == expected_what); + CHECK(state.creation_order.size() == expected_created); + CHECK(state.release_order.size() == expected_created); + CheckReleaseOrder( + state.creation_order, + state.release_order); CHECK(api->live_object_count() == 0); CHECK(std::all_of( state.package_alive_at_release.begin(), state.package_alive_at_release.end(), [](bool value) { return value; })); + CHECK(state.current_package.expired()); return; + } catch (const std::exception& error) { + throw std::runtime_error( + "expected typed CorelibError for " + + std::string(object_name) + ", got: " + error.what()); } throw std::runtime_error( "expected Phi4Weights::Load failure was not thrown"); @@ -1111,52 +1128,119 @@ void TestActionableFailures(const SyntheticPackage& fixture) { auto api = ResolveRecordingCorelib(state); auto package = fixture.Load(api); - CheckThrowsContains( + const auto check_validation_error = []( + auto&& action, + std::string_view expected) { + try { + action(); + } catch (const CorelibError&) { + throw std::runtime_error( + "non-corelib validation became CorelibError"); + } catch (const std::invalid_argument& error) { + CHECK(std::string_view(error.what()).find(expected) != + std::string_view::npos); + return; + } + throw std::runtime_error( + "expected non-corelib validation error"); + }; + check_validation_error( [&] { (void)Phi4Weights::Load(nullptr, package); }, "CorelibApi"); - CheckThrowsContains( + check_validation_error( [&] { (void)Phi4Weights::Load(api, nullptr); }, "Phi4Package"); + package.reset(); CheckLoadFailure( state, api, - package, + fixture.Load(api), FailurePoint::MatMulCreate, - "model.layers.0.attn.q_proj.MatMulNBits", + 70, + 86, + "model.layers.17.attn.k_proj.MatMulNBits", "ryzenai_corelib_matmul_bf16_weights_create_from_onnx_components", "intentional Task 6 MatMul create failure"); CheckLoadFailure( state, api, - package, + fixture.Load(api), FailurePoint::MatMulGetData, - "model.layers.0.attn.q_proj.MatMulNBits", + 75, + 93, + "model.layers.18.attn.v_proj.MatMulNBits", "ryzenai_corelib_matmul_bf16_weights_get_data", "intentional Task 6 MatMul get-data failure"); CheckLoadFailure( state, api, - package, + fixture.Load(api), FailurePoint::SsMlpCreate, - "model.layers.0.ssmlp", + 24, + 119, + "model.layers.23.ssmlp", "ryzenai_corelib_ssmlp_bf16_weights_create_from_onnx_components", "intentional Task 6 SSMLP create failure"); CheckLoadFailure( state, api, - package, + fixture.Load(api), FailurePoint::SsMlpGetData, - "model.layers.0.ssmlp", + 29, + 145, + "model.layers.28.ssmlp", "ryzenai_corelib_ssmlp_bf16_weights_get_data", "intentional Task 6 SSMLP get-data failure"); g_recording = nullptr; } +void TestNonCorelibObjectFailureRemainsDistinct( + const SyntheticPackage& fixture) { + RecordingState state; + state.matmul_packed_bytes = + std::numeric_limits::max(); + flm::test::ResetFakeCorelib(); + auto api = ResolveRecordingCorelib(state); + auto package = fixture.Load(api); + const std::weak_ptr package_lifetime = package; + state.current_package = package; + + try { + (void)Phi4Weights::Load(api, std::move(package)); + } catch (const CorelibError&) { + throw std::runtime_error( + "packed-byte validation became CorelibError"); + } catch (const std::runtime_error& error) { + const std::string_view message(error.what()); + CHECK( + message.find( + "model.layers.0.attn.k_proj.MatMulNBits") != + std::string_view::npos); + CHECK(message.find("overflows size_t") != + std::string_view::npos); + CHECK(state.creation_order.size() == 2); + CHECK(state.release_order.size() == 2); + CheckReleaseOrder( + state.creation_order, + state.release_order); + CHECK(std::all_of( + state.package_alive_at_release.begin(), + state.package_alive_at_release.end(), + [](bool value) { return value; })); + CHECK(package_lifetime.expired()); + CHECK(api->live_object_count() == 0); + g_recording = nullptr; + return; + } + throw std::runtime_error( + "expected packed-byte validation failure"); +} + static_assert(!std::is_copy_constructible_v); static_assert(!std::is_copy_assignable_v); static_assert(std::is_nothrow_move_constructible_v); @@ -1170,6 +1254,7 @@ int main() { TestExactConstructionAndLifetime(fixture); TestMoveAssignmentReleasesBeforeOwners(fixture); TestActionableFailures(fixture); + TestNonCorelibObjectFailureRemainsDistinct(fixture); std::cout << "test_phi4_weights: PASS\n"; return 0; } catch (const std::exception& error) { From d63ba65716b016c8d72041322c64af04f9bb01a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CChi?= Date: Tue, 1 Sep 2026 00:23:08 -0700 Subject: [PATCH 012/117] feat: add Phi-4 AIE4 host operations Co-authored-by: Cursor --- .gitignore | 1 + src/common/corelib/corelib_sources.cmake | 1 + src/common/corelib/phi4_corelib_host.cpp | 289 +++++++ src/include/models/phi4/phi4_corelib_host.hpp | 55 ++ src/test/phi4_corelib_aie4/CMakeLists.txt | 7 + src/test/phi4_corelib_aie4/test_phi4_host.cpp | 711 ++++++++++++++++++ 6 files changed, 1064 insertions(+) create mode 100644 src/common/corelib/phi4_corelib_host.cpp create mode 100644 src/include/models/phi4/phi4_corelib_host.hpp create mode 100644 src/test/phi4_corelib_aie4/test_phi4_host.cpp diff --git a/.gitignore b/.gitignore index 1ef2e712..404555a3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ build build_*/ +/.build-*/ /src/build-*/ .vscode __pycache__ diff --git a/src/common/corelib/corelib_sources.cmake b/src/common/corelib/corelib_sources.cmake index eac6b536..0720fd58 100644 --- a/src/common/corelib/corelib_sources.cmake +++ b/src/common/corelib/corelib_sources.cmake @@ -2,6 +2,7 @@ set(FLM_CORELIB_AIE4_SOURCES "${CMAKE_CURRENT_LIST_DIR}/corelib_api.cpp" "${CMAKE_CURRENT_LIST_DIR}/corelib_fatal_record.cpp" "${CMAKE_CURRENT_LIST_DIR}/corelib_runtime.cpp" + "${CMAKE_CURRENT_LIST_DIR}/phi4_corelib_host.cpp" "${CMAKE_CURRENT_LIST_DIR}/phi4_corelib_manifest.cpp" "${CMAKE_CURRENT_LIST_DIR}/phi4_corelib_weights.cpp" "${CMAKE_CURRENT_LIST_DIR}/phi4_corelib_shape_plan.cpp") diff --git a/src/common/corelib/phi4_corelib_host.cpp b/src/common/corelib/phi4_corelib_host.cpp new file mode 100644 index 00000000..46ae2c00 --- /dev/null +++ b/src/common/corelib/phi4_corelib_host.cpp @@ -0,0 +1,289 @@ +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace flm::phi4 { +namespace { + +constexpr std::string_view kConvertCall = + "ryzenai_corelib_convert"; +constexpr std::string_view kTensorReadCall = + "ryzenai_corelib_tensor_read"; +constexpr std::string_view kTensorWriteCall = + "ryzenai_corelib_tensor_write"; + +std::size_t CheckedExtent( + std::int64_t rows, + std::int64_t width, + std::string_view context) { + if (rows <= 0 || width <= 0) { + throw std::invalid_argument( + std::string(context) + + " rows and width must be positive"); + } + const auto row_count = static_cast(rows); + const auto column_count = static_cast(width); + if (column_count > + std::numeric_limits::max() / row_count) { + throw std::overflow_error( + std::string(context) + " extent overflows size_t"); + } + return static_cast(row_count * column_count); +} + +} // namespace + +void GatherEmbedding( + const corelib::CorelibApi& api, + std::span embedding_fp16, + std::span token_ids, + std::span output) { + constexpr std::size_t width = + static_cast(constants::kHiddenSize); + if (embedding_fp16.empty() || + embedding_fp16.size() % width != 0) { + throw std::invalid_argument( + "Phi-4 embedding shape must be [vocabulary, 3072]"); + } + if (token_ids.size() > + std::numeric_limits::max() / width) { + throw std::overflow_error( + "Phi-4 embedding gather extent overflows size_t"); + } + const std::size_t output_count = token_ids.size() * width; + if (output.size() != output_count) { + throw std::invalid_argument( + "Phi-4 embedding output shape does not match token IDs"); + } + if (token_ids.empty()) { + return; + } + + const std::size_t vocabulary_rows = + embedding_fp16.size() / width; + thread_local std::vector staging; + staging.resize(output_count); + for (std::size_t row = 0; row < token_ids.size(); ++row) { + const int token_id = token_ids[row]; + if (token_id < 0 || + static_cast(token_id) >= vocabulary_rows) { + throw std::out_of_range( + "Phi-4 embedding token ID is outside the mapped table"); + } + const std::size_t source_offset = + static_cast(token_id) * width; + std::copy_n( + embedding_fp16.begin() + source_offset, + width, + staging.begin() + row * width); + } + + api.Check( + api.functions().convert( + ryzenai_corelib_data_type_fp16, + staging.data(), + ryzenai_corelib_data_type_fp32, + output.data(), + output_count), + kConvertCall); +} + +void RmsNorm( + std::span input, + std::span scale, + std::int64_t rows, + std::int64_t width, + float epsilon, + std::span output) { + const std::size_t element_count = + CheckedExtent(rows, width, "Phi-4 RMSNorm"); + if (input.size() != element_count || + output.size() != element_count || + scale.size() != static_cast(width)) { + throw std::invalid_argument( + "Phi-4 RMSNorm shape mismatch"); + } + if (!std::isfinite(epsilon) || epsilon < 0.0f) { + throw std::invalid_argument( + "Phi-4 RMSNorm epsilon must be finite and nonnegative"); + } + + const std::size_t row_width = static_cast(width); + for (std::size_t row = 0; + row < static_cast(rows); + ++row) { + const std::size_t base = row * row_width; + float sum_of_squares = 0.0f; + for (std::size_t column = 0; + column < row_width; + ++column) { + const float value = input[base + column]; + sum_of_squares += value * value; + } + const float mean_square = + sum_of_squares / static_cast(width); + const float denominator = + std::sqrt(mean_square + epsilon); + for (std::size_t column = 0; + column < row_width; + ++column) { + output[base + column] = + (input[base + column] / denominator) * + scale[column]; + } + } +} + +void StageBf16( + const corelib::CorelibApi& api, + std::span input, + std::int64_t live_rows, + std::int64_t padded_rows, + std::int64_t width, + std::span output) { + if (padded_rows < live_rows) { + throw std::invalid_argument( + "Phi-4 BF16 staging padded rows are smaller than live rows"); + } + const std::size_t live_count = + CheckedExtent(live_rows, width, "Phi-4 BF16 staging"); + const std::size_t padded_count = + CheckedExtent(padded_rows, width, "Phi-4 BF16 staging"); + if (input.size() != live_count || output.size() < padded_count) { + throw std::invalid_argument( + "Phi-4 BF16 staging shape mismatch"); + } + + thread_local std::vector staging; + staging.resize(padded_count); + std::copy(input.begin(), input.end(), staging.begin()); + std::fill( + staging.begin() + live_count, + staging.end(), + 0.0f); + api.Check( + api.functions().convert( + ryzenai_corelib_data_type_fp32, + staging.data(), + ryzenai_corelib_data_type_bf16, + output.data(), + padded_count), + kConvertCall); +} + +void ScatterV( + const corelib::CorelibApi& api, + ryzenai_corelib_tensor_ptr source, + ryzenai_corelib_tensor_ptr value_cache, + std::int64_t rows, + std::int64_t position, + std::vector& staging, + VScatterMetrics& metrics) { + if (source == nullptr || value_cache == nullptr) { + throw std::invalid_argument( + "Phi-4 V scatter requires source and cache tensors"); + } + if (rows <= 0 || + rows > constants::kMaxSequenceLength || + position < 0 || + position > + constants::kMaxSequenceLength - rows) { + throw std::out_of_range( + "Phi-4 V scatter live rows exceed the cache window"); + } + + constexpr std::size_t head_count = + static_cast(constants::kKvHeadCount); + constexpr std::size_t head_width = + static_cast(constants::kHeadSize); + constexpr std::size_t row_width = head_count * head_width; + const std::size_t live_rows = static_cast(rows); + const std::size_t source_count = live_rows * row_width; + const std::size_t head_staging_count = live_rows * head_width; + staging.resize(source_count + head_staging_count); + + const auto started = std::chrono::steady_clock::now(); + const std::size_t source_bytes = + source_count * sizeof(std::uint16_t); + api.Check( + api.functions().tensor_read( + source, + staging.data(), + source_bytes, + 0), + kTensorReadCall); + ++metrics.read_calls; + metrics.bytes += source_bytes; + + std::uint16_t* const head_staging = + staging.data() + source_count; + const std::size_t head_bytes = + head_staging_count * sizeof(std::uint16_t); + for (std::size_t head = 0; head < head_count; ++head) { + for (std::size_t row = 0; row < live_rows; ++row) { + const std::size_t source_offset = + (row * head_count + head) * head_width; + std::copy_n( + staging.data() + source_offset, + head_width, + head_staging + row * head_width); + } + const std::size_t cache_offset = + ((head * + static_cast( + constants::kMaxSequenceLength)) + + static_cast(position)) * + head_width * sizeof(std::uint16_t); + api.Check( + api.functions().tensor_write( + value_cache, + head_staging, + head_bytes, + cache_offset), + kTensorWriteCall); + ++metrics.write_calls; + metrics.bytes += head_bytes; + } + const auto elapsed = + std::chrono::duration_cast( + std::chrono::steady_clock::now() - started); + metrics.nanoseconds += + static_cast(elapsed.count()); +} + +int ArgmaxLowest( + std::span logits) { + if (logits.empty()) { + throw std::invalid_argument( + "Phi-4 argmax cannot consume empty logits"); + } + if (logits.size() > + static_cast( + std::numeric_limits::max())) { + throw std::overflow_error( + "Phi-4 argmax token index exceeds int"); + } + + int best_index = 0; + float best_value = static_cast(logits.front()); + for (std::size_t index = 1; index < logits.size(); ++index) { + const float value = static_cast(logits[index]); + if (value > best_value) { + best_value = value; + best_index = static_cast(index); + } + } + return best_index; +} + +} // namespace flm::phi4 diff --git a/src/include/models/phi4/phi4_corelib_host.hpp b/src/include/models/phi4/phi4_corelib_host.hpp new file mode 100644 index 00000000..953c9764 --- /dev/null +++ b/src/include/models/phi4/phi4_corelib_host.hpp @@ -0,0 +1,55 @@ +#pragma once + +#include +#include + +#include +#include +#include + +namespace flm::phi4 { + +struct VScatterMetrics { + std::uint64_t read_calls = 0; + std::uint64_t write_calls = 0; + std::uint64_t bytes = 0; + std::uint64_t nanoseconds = 0; +}; + +void GatherEmbedding( + const corelib::CorelibApi& api, + std::span embedding_fp16, + std::span token_ids, + std::span output); + +void RmsNorm( + std::span input, + std::span scale, + std::int64_t rows, + std::int64_t width, + float epsilon, + std::span output); + +// Stages only the helper-required initial hidden/residual prefix. +// Elements beyond padded_rows * width are intentionally untouched. +void StageBf16( + const corelib::CorelibApi& api, + std::span input, + std::int64_t live_rows, + std::int64_t padded_rows, + std::int64_t width, + std::span output); + +void ScatterV( + const corelib::CorelibApi& api, + ryzenai_corelib_tensor_ptr source, + ryzenai_corelib_tensor_ptr value_cache, + std::int64_t rows, + std::int64_t position, + std::vector& staging, + VScatterMetrics& metrics); + +int ArgmaxLowest( + std::span logits); + +} // namespace flm::phi4 diff --git a/src/test/phi4_corelib_aie4/CMakeLists.txt b/src/test/phi4_corelib_aie4/CMakeLists.txt index f18182ca..daa71ace 100644 --- a/src/test/phi4_corelib_aie4/CMakeLists.txt +++ b/src/test/phi4_corelib_aie4/CMakeLists.txt @@ -60,6 +60,12 @@ function(add_corelib_host_test TEST_NAME TEST_SOURCE) add_executable(${TEST_NAME} ${TEST_SOURCE} fake_corelib.cpp) + if(MSVC) + target_compile_options(${TEST_NAME} PRIVATE + /arch:AVX2 + /fp:precise + $<$:/O2>) + endif() target_link_libraries(${TEST_NAME} PRIVATE flm_corelib_aie4_testlib) add_test(NAME ${TEST_NAME} COMMAND ${TEST_NAME}) @@ -72,6 +78,7 @@ enable_testing() add_corelib_host_test(test_corelib_api test_corelib_api.cpp) add_corelib_host_test(test_phi4_manifest test_phi4_manifest.cpp) add_corelib_host_test(test_phi4_shape_plan test_phi4_shape_plan.cpp) +add_corelib_host_test(test_phi4_host test_phi4_host.cpp) add_corelib_host_test(test_phi4_weights test_phi4_weights.cpp) add_corelib_host_test( test_corelib_fatal_record diff --git a/src/test/phi4_corelib_aie4/test_phi4_host.cpp b/src/test/phi4_corelib_aie4/test_phi4_host.cpp new file mode 100644 index 00000000..78e8d6fd --- /dev/null +++ b/src/test/phi4_corelib_aie4/test_phi4_host.cpp @@ -0,0 +1,711 @@ +#include "fake_corelib.hpp" +#include "test_support.hpp" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using flm::corelib::CorelibApi; +using flm::phi4::ArgmaxLowest; +using flm::phi4::GatherEmbedding; +using flm::phi4::RmsNorm; +using flm::phi4::ScatterV; +using flm::phi4::StageBf16; +using flm::phi4::VScatterMetrics; +using bf16 = biovault::bfloat16_t; + +constexpr std::uint16_t kPoison = 0xDEADu; + +struct ConvertCall { + ryzenai_corelib_data_type source_type; + const void* source; + ryzenai_corelib_data_type destination_type; + void* destination; + std::size_t count; + std::vector fp16_source; + std::vector fp32_source_bits; +}; + +struct ReadCall { + ryzenai_corelib_tensor_ptr tensor; + void* destination; + std::size_t size; + std::size_t offset; +}; + +struct WriteCall { + ryzenai_corelib_tensor_ptr tensor; + const void* source; + std::size_t size; + std::size_t offset; + std::vector values; +}; + +struct RecordingState { + std::vector converts; + std::vector reads; + std::vector writes; + std::vector v_source; + int v_tensor_storage = 0; + int v_cache_storage = 0; + + ryzenai_corelib_tensor_ptr v_tensor() noexcept { + return &v_tensor_storage; + } + + ryzenai_corelib_tensor_ptr v_cache() noexcept { + return &v_cache_storage; + } + + void ResetCalls() { + converts.clear(); + reads.clear(); + writes.clear(); + } +}; + +RecordingState g_recording; + +bool FixtureFp16ToFp32(std::uint16_t bits, float& value) { + switch (bits) { + case 0x0000u: + value = 0.0f; + return true; + case 0x3800u: + value = 0.5f; + return true; + case 0x3C00u: + value = 1.0f; + return true; + case 0x4000u: + value = 2.0f; + return true; + case 0x4200u: + value = 3.0f; + return true; + case 0xBC00u: + value = -1.0f; + return true; + case 0xC000u: + value = -2.0f; + return true; + default: + return false; + } +} + +bool FixtureFp32ToBf16(std::uint32_t bits, std::uint16_t& value) { + switch (bits) { + case 0x00000000u: + value = 0x0000u; + return true; + case 0x3F000000u: + value = 0x3F00u; + return true; + case 0x3F800000u: + value = 0x3F80u; + return true; + case 0x3F808000u: + value = 0x3F80u; + return true; + case 0x3F818000u: + value = 0x3F82u; + return true; + case 0x40000000u: + value = 0x4000u; + return true; + case 0xBF800000u: + value = 0xBF80u; + return true; + case 0xC0000000u: + value = 0xC000u; + return true; + default: + return false; + } +} + +ryzenai_corelib_status RecordingConvert( + ryzenai_corelib_data_type source_type, + const void* source, + ryzenai_corelib_data_type destination_type, + void* destination, + std::size_t count) { + if ((source == nullptr || destination == nullptr) && count != 0) { + return ryzenai_corelib_status_bad_argument; + } + + ConvertCall call{ + source_type, + source, + destination_type, + destination, + count, + {}, + {}}; + if (source_type == ryzenai_corelib_data_type_fp16 && + destination_type == ryzenai_corelib_data_type_fp32) { + const auto* input = + static_cast(source); + auto* output = static_cast(destination); + call.fp16_source.assign(input, input + count); + for (std::size_t index = 0; index < count; ++index) { + if (!FixtureFp16ToFp32(input[index], output[index])) { + return ryzenai_corelib_status_bad_argument; + } + } + } else if ( + source_type == ryzenai_corelib_data_type_fp32 && + destination_type == ryzenai_corelib_data_type_bf16) { + const auto* input = static_cast(source); + auto* output = static_cast(destination); + call.fp32_source_bits.reserve(count); + for (std::size_t index = 0; index < count; ++index) { + const auto bits = std::bit_cast(input[index]); + call.fp32_source_bits.push_back(bits); + if (!FixtureFp32ToBf16(bits, output[index])) { + return ryzenai_corelib_status_bad_argument; + } + } + } else { + return ryzenai_corelib_status_bad_argument; + } + g_recording.converts.push_back(std::move(call)); + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status RecordingTensorRead( + ryzenai_corelib_tensor_ptr tensor, + void* destination, + std::size_t size, + std::size_t offset) { + if (tensor != g_recording.v_tensor() || destination == nullptr || + offset > g_recording.v_source.size() * sizeof(std::uint16_t) || + size > g_recording.v_source.size() * sizeof(std::uint16_t) - + offset) { + return ryzenai_corelib_status_bad_argument; + } + g_recording.reads.push_back( + ReadCall{tensor, destination, size, offset}); + std::memcpy( + destination, + reinterpret_cast( + g_recording.v_source.data()) + + offset, + size); + + const auto tick = std::chrono::steady_clock::now(); + while (std::chrono::steady_clock::now() == tick) { + } + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status RecordingTensorWrite( + ryzenai_corelib_tensor_ptr tensor, + const void* source, + std::size_t size, + std::size_t offset) { + if (tensor != g_recording.v_cache() || source == nullptr || + size % sizeof(std::uint16_t) != 0) { + return ryzenai_corelib_status_bad_argument; + } + const auto* values = static_cast(source); + g_recording.writes.push_back(WriteCall{ + tensor, + source, + size, + offset, + std::vector( + values, + values + size / sizeof(std::uint16_t))}); + return ryzenai_corelib_status_success; +} + +template +void* FunctionAddress(Function function) { + return reinterpret_cast(function); +} + +std::shared_ptr ResolveRecordingCorelib() { + auto resolver = flm::test::CompleteCorelibResolver(); + resolver["ryzenai_corelib_convert"] = + FunctionAddress( + static_cast( + &RecordingConvert)); + resolver["ryzenai_corelib_tensor_read"] = + FunctionAddress( + static_cast( + &RecordingTensorRead)); + resolver["ryzenai_corelib_tensor_write"] = + FunctionAddress( + static_cast( + &RecordingTensorWrite)); + return CorelibApi::ResolveForTest( + [resolver = std::move(resolver)](std::string_view name) mutable + -> void* { + const auto found = resolver.find(std::string(name)); + return found == resolver.end() ? nullptr : found->second; + }); +} + +void TestGatherEmbeddingUsesOneReusableContiguousConversion( + const std::shared_ptr& api) { + constexpr std::size_t width = static_cast( + flm::phi4::constants::kHiddenSize); + std::vector embedding(3u * width); + std::fill_n(embedding.begin(), width, 0x3C00u); + std::fill_n(embedding.begin() + width, width, 0xC000u); + std::fill_n(embedding.begin() + 2u * width, width, 0x4200u); + + const std::array ids{2, 0}; + std::vector output(ids.size() * width); + g_recording.ResetCalls(); + GatherEmbedding(*api, embedding, ids, output); + + CHECK(g_recording.converts.size() == 1); + const auto& first_call = g_recording.converts.front(); + CHECK( + first_call.source_type == + ryzenai_corelib_data_type_fp16); + CHECK( + first_call.destination_type == + ryzenai_corelib_data_type_fp32); + CHECK(first_call.count == ids.size() * width); + CHECK(first_call.fp16_source.size() == ids.size() * width); + CHECK(std::all_of( + first_call.fp16_source.begin(), + first_call.fp16_source.begin() + width, + [](std::uint16_t value) { return value == 0x4200u; })); + CHECK(std::all_of( + first_call.fp16_source.begin() + width, + first_call.fp16_source.end(), + [](std::uint16_t value) { return value == 0x3C00u; })); + CHECK(std::all_of( + output.begin(), + output.begin() + width, + [](float value) { return value == 3.0f; })); + CHECK(std::all_of( + output.begin() + width, + output.end(), + [](float value) { return value == 1.0f; })); + + const void* first_staging = first_call.source; + const std::array second_ids{1}; + output.resize(width); + g_recording.ResetCalls(); + GatherEmbedding(*api, embedding, second_ids, output); + CHECK(g_recording.converts.size() == 1); + CHECK(g_recording.converts.front().source == first_staging); + CHECK(std::all_of( + output.begin(), + output.end(), + [](float value) { return value == -2.0f; })); +} + +void TestRmsNormUsesFp32AccumulationAndSharedEpsilon() { + const std::array input{ + std::bit_cast(0xBE8BBBACu), + std::bit_cast(0xBCCC9DE0u), + std::bit_cast(0xBFED682Fu), + std::bit_cast(0xC2CD01EDu)}; + const std::array scale{ + 1.0f, + 1.0f, + 1.0f, + 1.0f}; + std::array output{}; + + RmsNorm( + input, + scale, + 1, + 4, + static_cast( + flm::phi4::constants::kRmsEpsilon), + output); + + constexpr std::array expected{ + 0xBBAE75DBu, + 0xB9FF781Fu, + 0xBD143451u, + 0xBFFFF509u}; + for (std::size_t index = 0; index < output.size(); ++index) { + CHECK(std::bit_cast(output[index]) == + expected[index]); + } +} + +void TestStageBf16UsesRneAndZerosOnlyInitialInputPrefixes( + const std::shared_ptr& api) { + const std::array normalized{ + std::bit_cast(0x3F800000u), + std::bit_cast(0x3F808000u), + std::bit_cast(0x3F818000u), + std::bit_cast(0xBF800000u)}; + const std::array residual{ + std::bit_cast(0x40000000u), + std::bit_cast(0xC0000000u), + std::bit_cast(0x00000000u), + std::bit_cast(0x3F000000u)}; + std::vector hidden(8, kPoison); + std::vector residual_device(8, kPoison); + + std::vector q(8, kPoison); + std::vector k(8, kPoison); + std::vector v(8, kPoison); + std::vector attention(8, kPoison); + std::vector skip_sum(8, kPoison); + std::vector next_hidden(8, kPoison); + const std::array*, 6> + kernel_outputs{ + &q, + &k, + &v, + &attention, + &skip_sum, + &next_hidden}; + for (auto* output : kernel_outputs) { + std::fill_n(output->begin(), 4, 0x1234u); + } + + g_recording.ResetCalls(); + StageBf16(*api, normalized, 2, 3, 2, hidden); + StageBf16(*api, residual, 2, 3, 2, residual_device); + + CHECK(g_recording.converts.size() == 2); + CHECK(g_recording.converts[0].count == 6); + CHECK(g_recording.converts[1].count == 6); + CHECK( + g_recording.converts[0].source_type == + ryzenai_corelib_data_type_fp32); + CHECK( + g_recording.converts[0].destination_type == + ryzenai_corelib_data_type_bf16); + CHECK( + g_recording.converts[0].source == + g_recording.converts[1].source); + + constexpr std::array expected_hidden_source{ + 0x3F800000u, + 0x3F808000u, + 0x3F818000u, + 0xBF800000u, + 0x00000000u, + 0x00000000u}; + constexpr std::array expected_residual_source{ + 0x40000000u, + 0xC0000000u, + 0x00000000u, + 0x3F000000u, + 0x00000000u, + 0x00000000u}; + CHECK( + std::equal( + g_recording.converts[0].fp32_source_bits.begin(), + g_recording.converts[0].fp32_source_bits.end(), + expected_hidden_source.begin(), + expected_hidden_source.end())); + CHECK( + std::equal( + g_recording.converts[1].fp32_source_bits.begin(), + g_recording.converts[1].fp32_source_bits.end(), + expected_residual_source.begin(), + expected_residual_source.end())); + + constexpr std::array expected_hidden{ + 0x3F80u, + 0x3F80u, + 0x3F82u, + 0xBF80u, + 0x0000u, + 0x0000u, + kPoison, + kPoison}; + constexpr std::array expected_residual{ + 0x4000u, + 0xC000u, + 0x0000u, + 0x3F00u, + 0x0000u, + 0x0000u, + kPoison, + kPoison}; + CHECK(std::equal( + hidden.begin(), + hidden.end(), + expected_hidden.begin(), + expected_hidden.end())); + CHECK(std::equal( + residual_device.begin(), + residual_device.end(), + expected_residual.begin(), + expected_residual.end())); + + for (const auto* output : kernel_outputs) { + CHECK(std::all_of( + output->begin(), + output->begin() + 4, + [](std::uint16_t value) { return value == 0x1234u; })); + CHECK(std::all_of( + output->begin() + 4, + output->end(), + [](std::uint16_t value) { return value == kPoison; })); + } +} + +std::uint16_t VValue( + std::size_t row, + std::size_t head, + std::size_t column) { + return static_cast( + row * 4096u + head * 256u + column); +} + +void PopulateVSource(std::size_t capacity_rows) { + constexpr std::size_t heads = 8; + constexpr std::size_t width = 128; + g_recording.v_source.assign( + capacity_rows * heads * width, + kPoison); + for (std::size_t row = 0; row < 3; ++row) { + for (std::size_t head = 0; head < heads; ++head) { + for (std::size_t column = 0; column < width; ++column) { + const std::size_t index = + (row * heads + head) * width + column; + g_recording.v_source[index] = + VValue(row, head, column); + } + } + } +} + +void CheckScatterWrites( + std::int64_t rows, + std::int64_t position) { + constexpr std::size_t heads = 8; + constexpr std::size_t width = 128; + CHECK(g_recording.writes.size() == heads); + for (std::size_t head = 0; head < heads; ++head) { + const auto& write = g_recording.writes[head]; + const std::size_t expected_offset = + ((head * 4096u) + + static_cast(position)) * + width * sizeof(std::uint16_t); + CHECK(write.tensor == g_recording.v_cache()); + CHECK(write.offset == expected_offset); + CHECK( + write.size == + static_cast(rows) * width * + sizeof(std::uint16_t)); + CHECK( + write.values.size() == + static_cast(rows) * width); + for (std::size_t row = 0; + row < static_cast(rows); + ++row) { + for (std::size_t column = 0; column < width; ++column) { + CHECK( + write.values[row * width + column] == + VValue(row, head, column)); + } + } + CHECK(std::none_of( + write.values.begin(), + write.values.end(), + [](std::uint16_t value) { return value == kPoison; })); + } +} + +void TestScatterVReadsOnlyLiveRowsAndReusesOneBuffer( + const std::shared_ptr& api) { + constexpr std::size_t heads = 8; + constexpr std::size_t width = 128; + PopulateVSource(5); + std::vector staging; + VScatterMetrics metrics{}; + + g_recording.ResetCalls(); + ScatterV( + *api, + g_recording.v_tensor(), + g_recording.v_cache(), + 3, + 17, + staging, + metrics); + + const std::size_t read_size = + 3u * heads * width * sizeof(std::uint16_t); + CHECK(g_recording.reads.size() == 1); + CHECK(g_recording.reads.front().tensor == + g_recording.v_tensor()); + CHECK(g_recording.reads.front().size == read_size); + CHECK(g_recording.reads.front().offset == 0); + CHECK(g_recording.reads.front().destination == staging.data()); + CheckScatterWrites(3, 17); + + const auto* staging_begin = staging.data(); + const auto* staging_end = staging.data() + staging.size(); + for (const auto& write : g_recording.writes) { + const auto* source = + static_cast(write.source); + CHECK(source >= staging_begin); + CHECK( + source + + write.size / sizeof(std::uint16_t) <= + staging_end); + } + CHECK(metrics.read_calls == 1); + CHECK(metrics.write_calls == 8); + CHECK(metrics.bytes == 2u * read_size); + CHECK(metrics.nanoseconds > 0); + CHECK(std::all_of( + g_recording.v_source.begin() + 3u * heads * width, + g_recording.v_source.end(), + [](std::uint16_t value) { return value == kPoison; })); + + const void* first_data = staging.data(); + const std::size_t first_capacity = staging.capacity(); + const std::uint64_t first_ns = metrics.nanoseconds; + g_recording.ResetCalls(); + ScatterV( + *api, + g_recording.v_tensor(), + g_recording.v_cache(), + 1, + 20, + staging, + metrics); + CHECK(staging.data() == first_data); + CHECK(staging.capacity() == first_capacity); + CHECK(g_recording.reads.size() == 1); + CHECK( + g_recording.reads.front().size == + heads * width * sizeof(std::uint16_t)); + CheckScatterWrites(1, 20); + CHECK(metrics.read_calls == 2); + CHECK(metrics.write_calls == 16); + CHECK( + metrics.bytes == + 2u * read_size + + 2u * heads * width * sizeof(std::uint16_t)); + CHECK(metrics.nanoseconds > first_ns); +} + +void TestArgmaxLowestChoosesLowestTokenOnTie() { + std::vector logits(12, bf16{0xBF80u, true}); + logits[2] = bf16{0x4000u, true}; + logits[7] = bf16{0x4040u, true}; + logits[9] = bf16{0x4040u, true}; + CHECK(ArgmaxLowest(logits) == 7); + + const std::array negative{ + bf16{0xC040u, true}, + bf16{0xC000u, true}, + bf16{0xC000u, true}}; + CHECK(ArgmaxLowest(negative) == 1); +} + +void TestInvalidHostArgumentsFailBeforeCorelib( + const std::shared_ptr& api) { + constexpr std::size_t width = static_cast( + flm::phi4::constants::kHiddenSize); + std::vector embedding(width); + const std::array invalid_id{1}; + std::vector embedding_output(width); + std::vector staged(4); + std::vector scatter_staging; + VScatterMetrics metrics{}; + + g_recording.ResetCalls(); + CheckThrowsContains( + [&] { + GatherEmbedding( + *api, + embedding, + invalid_id, + embedding_output); + }, + "token ID"); + CheckThrowsContains( + [&] { + StageBf16( + *api, + std::span{embedding_output}.first(2), + 2, + 1, + 1, + staged); + }, + "padded rows"); + CheckThrowsContains( + [&] { + ScatterV( + *api, + g_recording.v_tensor(), + g_recording.v_cache(), + 2, + 4095, + scatter_staging, + metrics); + }, + "cache"); + CheckThrowsContains( + [&] { + std::array rms_output{}; + RmsNorm( + std::span{embedding_output}.first(1), + std::span{embedding_output}.first(1), + 1, + 2, + static_cast( + flm::phi4::constants::kRmsEpsilon), + rms_output); + }, + "shape"); + CheckThrowsContains( + [] { + const std::span empty; + (void)ArgmaxLowest(empty); + }, + "empty"); + CHECK(g_recording.converts.empty()); + CHECK(g_recording.reads.empty()); + CHECK(g_recording.writes.empty()); +} + +} // namespace + +int main() { + try { + const auto api = ResolveRecordingCorelib(); + TestGatherEmbeddingUsesOneReusableContiguousConversion(api); + TestRmsNormUsesFp32AccumulationAndSharedEpsilon(); + TestStageBf16UsesRneAndZerosOnlyInitialInputPrefixes(api); + TestScatterVReadsOnlyLiveRowsAndReusesOneBuffer(api); + TestArgmaxLowestChoosesLowestTokenOnTie(); + TestInvalidHostArgumentsFailBeforeCorelib(api); + std::cout << "phi4 host operation tests passed\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} From 983d19082c59528c0429d00c12f9295c1137a4ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CChi?= Date: Tue, 1 Sep 2026 00:37:13 -0700 Subject: [PATCH 013/117] fix: clarify Phi-4 host operation contracts Co-authored-by: Cursor --- src/include/models/phi4/phi4_corelib_host.hpp | 3 ++ src/test/phi4_corelib_aie4/CMakeLists.txt | 13 +++-- src/test/phi4_corelib_aie4/test_phi4_host.cpp | 51 ++++++++----------- 3 files changed, 30 insertions(+), 37 deletions(-) diff --git a/src/include/models/phi4/phi4_corelib_host.hpp b/src/include/models/phi4/phi4_corelib_host.hpp index 953c9764..2b88333b 100644 --- a/src/include/models/phi4/phi4_corelib_host.hpp +++ b/src/include/models/phi4/phi4_corelib_host.hpp @@ -40,6 +40,9 @@ void StageBf16( std::int64_t width, std::span output); +// Precondition: the caller has successfully synchronized the Stream after +// V projection and before this host read. ScatterV deliberately owns no +// Stream and performs no synchronization. void ScatterV( const corelib::CorelibApi& api, ryzenai_corelib_tensor_ptr source, diff --git a/src/test/phi4_corelib_aie4/CMakeLists.txt b/src/test/phi4_corelib_aie4/CMakeLists.txt index daa71ace..27ef8960 100644 --- a/src/test/phi4_corelib_aie4/CMakeLists.txt +++ b/src/test/phi4_corelib_aie4/CMakeLists.txt @@ -42,7 +42,12 @@ target_compile_definitions(flm_corelib_aie4_testlib PUBLIC __NPU_VERSION__=\"${NPU_VERSION}\" WIN32_LEAN_AND_MEAN NOMINMAX) -target_compile_options(flm_corelib_aie4_testlib PRIVATE /fp:precise) +if(MSVC) + target_compile_options(flm_corelib_aie4_testlib PRIVATE + /arch:AVX2 + /fp:precise + $<$:/O2>) +endif() target_link_libraries(flm_corelib_aie4_testlib PUBLIC xrt_coreutil shell32 @@ -60,12 +65,6 @@ function(add_corelib_host_test TEST_NAME TEST_SOURCE) add_executable(${TEST_NAME} ${TEST_SOURCE} fake_corelib.cpp) - if(MSVC) - target_compile_options(${TEST_NAME} PRIVATE - /arch:AVX2 - /fp:precise - $<$:/O2>) - endif() target_link_libraries(${TEST_NAME} PRIVATE flm_corelib_aie4_testlib) add_test(NAME ${TEST_NAME} COMMAND ${TEST_NAME}) diff --git a/src/test/phi4_corelib_aie4/test_phi4_host.cpp b/src/test/phi4_corelib_aie4/test_phi4_host.cpp index 78e8d6fd..7da7c865 100644 --- a/src/test/phi4_corelib_aie4/test_phi4_host.cpp +++ b/src/test/phi4_corelib_aie4/test_phi4_host.cpp @@ -62,6 +62,7 @@ struct RecordingState { std::vector converts; std::vector reads; std::vector writes; + std::size_t synchronize_calls = 0; std::vector v_source; int v_tensor_storage = 0; int v_cache_storage = 0; @@ -78,6 +79,7 @@ struct RecordingState { converts.clear(); reads.clear(); writes.clear(); + synchronize_calls = 0; } }; @@ -238,6 +240,12 @@ ryzenai_corelib_status RecordingTensorWrite( return ryzenai_corelib_status_success; } +ryzenai_corelib_status RecordingStreamSynchronize( + ryzenai_corelib_stream_ptr) { + ++g_recording.synchronize_calls; + return ryzenai_corelib_status_success; +} + template void* FunctionAddress(Function function) { return reinterpret_cast(function); @@ -257,6 +265,11 @@ std::shared_ptr ResolveRecordingCorelib() { FunctionAddress( static_cast( &RecordingTensorWrite)); + resolver["ryzenai_corelib_stream_synchronize"] = + FunctionAddress( + static_cast< + decltype(&::ryzenai_corelib_stream_synchronize)>( + &RecordingStreamSynchronize)); return CorelibApi::ResolveForTest( [resolver = std::move(resolver)](std::string_view name) mutable -> void* { @@ -367,23 +380,10 @@ void TestStageBf16UsesRneAndZerosOnlyInitialInputPrefixes( std::vector hidden(8, kPoison); std::vector residual_device(8, kPoison); - std::vector q(8, kPoison); - std::vector k(8, kPoison); - std::vector v(8, kPoison); - std::vector attention(8, kPoison); - std::vector skip_sum(8, kPoison); - std::vector next_hidden(8, kPoison); - const std::array*, 6> - kernel_outputs{ - &q, - &k, - &v, - &attention, - &skip_sum, - &next_hidden}; - for (auto* output : kernel_outputs) { - std::fill_n(output->begin(), 4, 0x1234u); - } + // Task 7 has no production consumer for q/k/attention/skip-sum/ + // next-hidden padded tails. Task 8's dispatch test must poison those + // actual persistent buffers and prove its live/helper-authorized + // regions exclude stale values. g_recording.ResetCalls(); StageBf16(*api, normalized, 2, 3, 2, hidden); @@ -457,17 +457,6 @@ void TestStageBf16UsesRneAndZerosOnlyInitialInputPrefixes( residual_device.end(), expected_residual.begin(), expected_residual.end())); - - for (const auto* output : kernel_outputs) { - CHECK(std::all_of( - output->begin(), - output->begin() + 4, - [](std::uint16_t value) { return value == 0x1234u; })); - CHECK(std::all_of( - output->begin() + 4, - output->end(), - [](std::uint16_t value) { return value == kPoison; })); - } } std::uint16_t VValue( @@ -533,7 +522,7 @@ void CheckScatterWrites( } } -void TestScatterVReadsOnlyLiveRowsAndReusesOneBuffer( +void TestScatterVReadsOnlyLiveRowsWithoutHiddenSynchronize( const std::shared_ptr& api) { constexpr std::size_t heads = 8; constexpr std::size_t width = 128; @@ -559,6 +548,7 @@ void TestScatterVReadsOnlyLiveRowsAndReusesOneBuffer( CHECK(g_recording.reads.front().size == read_size); CHECK(g_recording.reads.front().offset == 0); CHECK(g_recording.reads.front().destination == staging.data()); + CHECK(g_recording.synchronize_calls == 0); CheckScatterWrites(3, 17); const auto* staging_begin = staging.data(); @@ -599,6 +589,7 @@ void TestScatterVReadsOnlyLiveRowsAndReusesOneBuffer( CHECK( g_recording.reads.front().size == heads * width * sizeof(std::uint16_t)); + CHECK(g_recording.synchronize_calls == 0); CheckScatterWrites(1, 20); CHECK(metrics.read_calls == 2); CHECK(metrics.write_calls == 16); @@ -699,7 +690,7 @@ int main() { TestGatherEmbeddingUsesOneReusableContiguousConversion(api); TestRmsNormUsesFp32AccumulationAndSharedEpsilon(); TestStageBf16UsesRneAndZerosOnlyInitialInputPrefixes(api); - TestScatterVReadsOnlyLiveRowsAndReusesOneBuffer(api); + TestScatterVReadsOnlyLiveRowsWithoutHiddenSynchronize(api); TestArgmaxLowestChoosesLowestTokenOnTie(); TestInvalidHostArgumentsFailBeforeCorelib(api); std::cout << "phi4 host operation tests passed\n"; From 1367d08e27a68cb2dc6ab8c0967fbf4350dc612b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CChi?= Date: Tue, 1 Sep 2026 01:07:45 -0700 Subject: [PATCH 014/117] feat: add corelib-backed Phi-4 engine Co-authored-by: Cursor --- src/common/corelib/corelib_sources.cmake | 1 + src/common/corelib/phi4_corelib_aie4.cpp | 1213 ++++++++++ src/include/models/phi4/phi4_corelib_aie4.hpp | 92 + src/test/phi4_corelib_aie4/CMakeLists.txt | 13 + .../phi4_corelib_aie4/test_phi4_engine.cpp | 2152 +++++++++++++++++ 5 files changed, 3471 insertions(+) create mode 100644 src/common/corelib/phi4_corelib_aie4.cpp create mode 100644 src/include/models/phi4/phi4_corelib_aie4.hpp create mode 100644 src/test/phi4_corelib_aie4/test_phi4_engine.cpp diff --git a/src/common/corelib/corelib_sources.cmake b/src/common/corelib/corelib_sources.cmake index 0720fd58..fe990e2a 100644 --- a/src/common/corelib/corelib_sources.cmake +++ b/src/common/corelib/corelib_sources.cmake @@ -2,6 +2,7 @@ set(FLM_CORELIB_AIE4_SOURCES "${CMAKE_CURRENT_LIST_DIR}/corelib_api.cpp" "${CMAKE_CURRENT_LIST_DIR}/corelib_fatal_record.cpp" "${CMAKE_CURRENT_LIST_DIR}/corelib_runtime.cpp" + "${CMAKE_CURRENT_LIST_DIR}/phi4_corelib_aie4.cpp" "${CMAKE_CURRENT_LIST_DIR}/phi4_corelib_host.cpp" "${CMAKE_CURRENT_LIST_DIR}/phi4_corelib_manifest.cpp" "${CMAKE_CURRENT_LIST_DIR}/phi4_corelib_weights.cpp" diff --git a/src/common/corelib/phi4_corelib_aie4.cpp b/src/common/corelib/phi4_corelib_aie4.cpp new file mode 100644 index 00000000..06a9abf8 --- /dev/null +++ b/src/common/corelib/phi4_corelib_aie4.cpp @@ -0,0 +1,1213 @@ +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace flm::phi4 { +namespace { + +constexpr std::string_view kEmbeddingName = + "model.embed_tokens.weight"; +constexpr std::string_view kInputNormName = + "model.layers.0.input_layernorm.weight"; +constexpr std::string_view kCosName = "cos_cache"; +constexpr std::string_view kSinName = "sin_cache"; +constexpr std::string_view kCreateStreamCall = + "ryzenai_corelib_create_stream"; +constexpr std::string_view kCreateTensorCall = + "ryzenai_corelib_create_device_tensor"; +constexpr std::string_view kTensorByteSizeCall = + "ryzenai_corelib_tensor_get_byte_size"; +constexpr std::string_view kTensorWriteCall = + "ryzenai_corelib_tensor_write"; +constexpr std::string_view kTensorReadCall = + "ryzenai_corelib_tensor_read"; +constexpr std::string_view kSynchronizeCall = + "ryzenai_corelib_stream_synchronize"; +constexpr std::string_view kConvertCall = + "ryzenai_corelib_convert"; + +static_assert(sizeof(bf16) == sizeof(std::uint16_t)); + +std::uint32_t ValidateMaxLength(std::uint32_t max_length) { + if ( + max_length == 0 || + max_length > + static_cast( + constants::kMaxSequenceLength)) { + throw std::out_of_range( + "Phi-4 AIE4 maximum length must be in 1..4096"); + } + return max_length; +} + +std::size_t CheckedElements( + std::int64_t rows, + std::int64_t width, + std::string_view context) { + if (rows <= 0 || width <= 0) { + throw std::invalid_argument( + std::string(context) + + " rows and width must be positive"); + } + const auto row_count = static_cast(rows); + const auto column_count = static_cast(width); + if ( + row_count > + std::numeric_limits::max() / column_count) { + throw std::overflow_error( + std::string(context) + " extent overflows size_t"); + } + return static_cast(row_count * column_count); +} + +std::size_t CheckedBytes( + std::int64_t rows, + std::int64_t width, + std::size_t element_size, + std::string_view context) { + const std::size_t elements = + CheckedElements(rows, width, context); + if ( + element_size != 0 && + elements > + std::numeric_limits::max() / element_size) { + throw std::overflow_error( + std::string(context) + " byte size overflows size_t"); + } + return elements * element_size; +} + +std::size_t TensorByteCount( + ryzenai_corelib_data_type data_type, + std::span shape) { + if (shape.empty()) { + throw std::invalid_argument( + "Phi-4 AIE4 tensor shape cannot be empty"); + } + std::size_t elements = 1; + for (const std::int64_t dimension : shape) { + if (dimension <= 0) { + throw std::invalid_argument( + "Phi-4 AIE4 tensor dimensions must be positive"); + } + const auto value = static_cast(dimension); + if ( + elements > + std::numeric_limits::max() / value) { + throw std::overflow_error( + "Phi-4 AIE4 tensor extent overflows size_t"); + } + elements *= value; + } + + std::size_t element_size = 0; + switch (data_type) { + case ryzenai_corelib_data_type_bf16: + case ryzenai_corelib_data_type_fp16: + element_size = sizeof(std::uint16_t); + break; + case ryzenai_corelib_data_type_fp32: + element_size = sizeof(float); + break; + default: + throw std::invalid_argument( + "Phi-4 AIE4 requested an unsupported tensor dtype"); + } + if ( + elements > + std::numeric_limits::max() / element_size) { + throw std::overflow_error( + "Phi-4 AIE4 tensor byte size overflows size_t"); + } + return elements * element_size; +} + +ryzenai_corelib_data_type SourceDataType( + SourceDType data_type, + std::string_view context) { + switch (data_type) { + case SourceDType::Float16: + return ryzenai_corelib_data_type_fp16; + case SourceDType::Float32: + return ryzenai_corelib_data_type_fp32; + case SourceDType::UInt8: + case SourceDType::Int64: + throw std::invalid_argument( + std::string(context) + + " requires an FP16 or FP32 source"); + } + throw std::logic_error("unreachable Phi-4 source dtype"); +} + +std::uint64_t MappedSourceBytes(const Phi4Package& package) { + std::unordered_set owners; + std::uint64_t total = 0; + const auto add = [&](const InitializerView& view) { + if (!view.owner || !owners.insert(view.owner.get()).second) { + return; + } + if ( + view.owner->size() > + std::numeric_limits::max() - total) { + throw std::overflow_error( + "Phi-4 mapped-source byte total overflows uint64_t"); + } + total += view.owner->size(); + }; + + add(package.Require(kEmbeddingName)); + add(package.Require(kInputNormName)); + add(package.Require(kCosName)); + add(package.Require(kSinName)); + for (const auto& object : package.weight_objects()) { + for (const auto& [_, initializer] : object.components) { + add(package.Require(initializer)); + } + } + return total; +} + +std::uint64_t ElapsedNanoseconds( + std::chrono::steady_clock::time_point started) { + const auto elapsed = + std::chrono::duration_cast( + std::chrono::steady_clock::now() - started); + return static_cast( + std::max(elapsed.count(), 0)); +} + +bool RecoverableBeforeSubmit( + bool synchronize_in_progress, + const corelib::StepSubmissionState& submission) noexcept { + return !synchronize_in_progress && !submission.irrevocable(); +} + +[[noreturn]] void TerminateCorelibFailure( + const std::shared_ptr& runtime, + const corelib::CorelibError& error, + std::string phase, + std::optional layer, + std::int64_t rows, + std::int64_t position) { + runtime->TerminateAfterFailure(corelib::FailureContext{ + error.status, + error.call, + error.detail, + std::move(phase), + layer, + rows, + position}); +} + +[[noreturn]] void TerminateHostFailure( + const std::shared_ptr& runtime, + const std::exception& error, + std::string phase, + std::optional layer, + std::int64_t rows, + std::int64_t position) { + runtime->TerminateAfterFailure(corelib::FailureContext{ + ryzenai_corelib_status_failure, + "host_exception", + error.what(), + std::move(phase), + layer, + rows, + position}); +} + +[[noreturn]] void TerminateUnknownFailure( + const std::shared_ptr& runtime, + std::string phase, + std::optional layer, + std::int64_t rows, + std::int64_t position) { + runtime->TerminateAfterFailure(corelib::FailureContext{ + ryzenai_corelib_status_failure, + "unknown_exception", + "non-standard exception after the irrevocable boundary", + std::move(phase), + layer, + rows, + position}); +} + +} // namespace + +struct phi4_corelib_aie4::Impl final { + Impl( + LM_Config config, + std::filesystem::path model_path, + std::shared_ptr supplied_runtime, + std::uint32_t requested_max_length) + : runtime(std::move(supplied_runtime)), + max_length(ValidateMaxLength(requested_max_length)) { + (void)config; + if (!runtime) { + throw std::invalid_argument( + "phi4_corelib_aie4 requires a CorelibRuntime"); + } + api = runtime->api(); + if (!api) { + throw std::invalid_argument( + "phi4_corelib_aie4 runtime has no CorelibApi"); + } + if (model_path.empty()) { + throw std::invalid_argument( + "phi4_corelib_aie4 requires a model path"); + } + + const auto model_load_started = + std::chrono::steady_clock::now(); + auto execution = runtime->AcquireExecution(); + + package = std::make_shared( + Phi4Package::Load(model_path, api, false)); + metrics.mapped_source_bytes = + MappedSourceBytes(*package); + shape_plan.emplace(Phi4ShapePlan::Build(api)); + + const auto& embedding_view = + package->Require(kEmbeddingName); + embedding = std::span( + reinterpret_cast( + embedding_view.data), + embedding_view.size / sizeof(std::uint16_t)); + + const auto& norm_view = package->Require(kInputNormName); + input_norm.resize( + static_cast(constants::kHiddenSize)); + api->Check( + api->functions().convert( + SourceDataType(norm_view.dtype, kInputNormName), + norm_view.data, + ryzenai_corelib_data_type_fp32, + input_norm.data(), + input_norm.size()), + kConvertCall); + + const auto cos_host = + package->MaterializeRopeFp32(kCosName); + const auto sin_host = + package->MaterializeRopeFp32(kSinName); + + const auto weight_pack_started = + std::chrono::steady_clock::now(); + weights.emplace(Phi4Weights::Load(api, package)); + metrics.weight_pack_ns = + ElapsedNanoseconds(weight_pack_started); + metrics.packed_weight_bytes = + static_cast(weights->packed_bytes()); + metrics.weight_create_count = + static_cast( + constants::kLayerCount * 5 + 1); + + const auto& capacities = shape_plan->capacities(); + const std::size_t layer_elements = CheckedElements( + capacities.layer_rows, + constants::kHiddenSize, + "Phi-4 host layer staging"); + embedding_fp32.resize(layer_elements); + normalized_fp32.resize(layer_elements); + bf16_staging.resize(layer_elements); + v_staging.reserve( + CheckedElements( + capacities.layer_rows, + constants::kKvDimension + constants::kHeadSize, + "Phi-4 V staging")); + last_hidden_staging.resize( + static_cast(constants::kHiddenSize)); + + stream = CreateStream(); + hidden_tensors[0] = CreateTensor( + ryzenai_corelib_data_type_bf16, + {capacities.layer_rows, constants::kHiddenSize}, + false); + hidden_tensors[1] = CreateTensor( + ryzenai_corelib_data_type_bf16, + {capacities.layer_rows, constants::kHiddenSize}, + false); + residual_tensor = CreateTensor( + ryzenai_corelib_data_type_bf16, + {capacities.layer_rows, constants::kHiddenSize}, + false); + skip_sum_tensor = CreateTensor( + ryzenai_corelib_data_type_bf16, + {capacities.layer_rows, constants::kHiddenSize}, + false); + query_tensor = CreateTensor( + ryzenai_corelib_data_type_bf16, + {capacities.layer_rows, constants::kQueryDimension}, + false); + key_tensor = CreateTensor( + ryzenai_corelib_data_type_bf16, + {capacities.layer_rows, constants::kKvDimension}, + false); + value_tensor = CreateTensor( + ryzenai_corelib_data_type_bf16, + {capacities.layer_rows, constants::kKvDimension}, + false); + attention_tensor = CreateTensor( + ryzenai_corelib_data_type_bf16, + {capacities.layer_rows, constants::kQueryDimension}, + false); + lm_input_tensor = CreateTensor( + ryzenai_corelib_data_type_bf16, + {capacities.lm_head_rows, constants::kHiddenSize}, + false); + lm_output_tensor = CreateTensor( + ryzenai_corelib_data_type_bf16, + {capacities.lm_head_rows, constants::kVocabularySize}, + false); + cos_tensor = CreateTensor( + ryzenai_corelib_data_type_fp32, + {constants::kMaxSequenceLength, + constants::kRopeDimension / 2}, + false); + sin_tensor = CreateTensor( + ryzenai_corelib_data_type_fp32, + {constants::kMaxSequenceLength, + constants::kRopeDimension / 2}, + false); + + for (std::size_t layer = 0; + layer < + static_cast(constants::kLayerCount); + ++layer) { + key_caches[layer] = CreateTensor( + ryzenai_corelib_data_type_bf16, + {constants::kKvHeadCount, + constants::kMaxSequenceLength, + constants::kHeadSize}, + true); + value_caches[layer] = CreateTensor( + ryzenai_corelib_data_type_bf16, + {constants::kKvHeadCount, + constants::kMaxSequenceLength, + constants::kHeadSize}, + true); + } + + const std::size_t rope_bytes = CheckedBytes( + constants::kMaxSequenceLength, + constants::kRopeDimension / 2, + sizeof(float), + "Phi-4 RoPE upload"); + api->Check( + api->functions().tensor_write( + cos_tensor.get(), + cos_host.data(), + rope_bytes, + 0), + kTensorWriteCall); + api->Check( + api->functions().tensor_write( + sin_tensor.get(), + sin_host.data(), + rope_bytes, + 0), + kTensorWriteCall); + + current_hidden = &hidden_tensors[0]; + next_hidden = &hidden_tensors[1]; + current_residual = &residual_tensor; + next_skip_sum = &skip_sum_tensor; + metrics.model_load_ns = + ElapsedNanoseconds(model_load_started); + } + + ~Impl() noexcept { +#if defined(FLM_CORELIB_TESTING) + if ( + runtime && + runtime->state() == corelib::ProcessState::Terminating) { + ReleaseResourcesWithoutSynchronization(); + return; + } +#endif + if (!stream) { + ReleaseResourcesWithoutSynchronization(); + return; + } + + try { + auto execution = runtime->AcquireExecution(); + api->Check( + api->functions().stream_synchronize(stream.get()), + kSynchronizeCall); + stream.reset(); + ReleaseTensorsWeightsAndPackage(); + } catch (const corelib::CorelibError& error) { + TerminateCorelibFailure( + runtime, + error, + "destruction", + std::nullopt, + last_live_rows, + position); + } catch (const std::exception& error) { + TerminateHostFailure( + runtime, + error, + "destruction", + std::nullopt, + last_live_rows, + position); + } catch (...) { + TerminateUnknownFailure( + runtime, + "destruction", + std::nullopt, + last_live_rows, + position); + } + } + + corelib::UniqueStream CreateStream() { + ryzenai_corelib_stream_ptr raw = nullptr; + const auto status = + api->functions().create_stream(&raw); + corelib::UniqueStream result(api, raw); + api->Check(status, kCreateStreamCall); + if (!result) { + throw std::runtime_error( + "ryzenai_corelib_create_stream succeeded with a " + "null object"); + } + return result; + } + + corelib::UniqueTensor CreateTensor( + ryzenai_corelib_data_type data_type, + std::initializer_list shape_values, + bool is_kv) { + const std::vector shape(shape_values); + ryzenai_corelib_tensor_ptr raw = nullptr; + const auto status = + api->functions().create_device_tensor( + data_type, + shape.data(), + shape.size(), + &raw); + corelib::UniqueTensor result(api, raw); + api->Check(status, kCreateTensorCall); + if (!result) { + throw std::runtime_error( + "ryzenai_corelib_create_device_tensor succeeded " + "with a null object"); + } + + std::size_t actual_bytes = 0; + api->Check( + api->functions().tensor_get_byte_size( + result.get(), + &actual_bytes), + kTensorByteSizeCall); + const std::size_t expected_bytes = + TensorByteCount(data_type, shape); + if (actual_bytes != expected_bytes) { + throw std::runtime_error( + "corelib device tensor byte size does not match " + "the requested Phi-4 shape"); + } + ++metrics.device_tensor_create_count; + if (is_kv) { + metrics.kv_bytes += actual_bytes; + } else { + metrics.scratch_bytes += actual_bytes; + } + return result; + } + + void ReleaseResourcesWithoutSynchronization() noexcept { + stream.reset(); + ReleaseTensorsWeightsAndPackage(); + } + + void ReleaseTensorsWeightsAndPackage() noexcept { + for (auto layer = value_caches.rbegin(); + layer != value_caches.rend(); + ++layer) { + layer->reset(); + } + for (auto layer = key_caches.rbegin(); + layer != key_caches.rend(); + ++layer) { + layer->reset(); + } + sin_tensor.reset(); + cos_tensor.reset(); + lm_output_tensor.reset(); + lm_input_tensor.reset(); + attention_tensor.reset(); + value_tensor.reset(); + key_tensor.reset(); + query_tensor.reset(); + skip_sum_tensor.reset(); + residual_tensor.reset(); + hidden_tensors[1].reset(); + hidden_tensors[0].reset(); + weights.reset(); + embedding = {}; + package.reset(); + } + + void ValidateTokens(std::span token_ids) const { + for (const int token_id : token_ids) { + if ( + token_id < 0 || + token_id >= constants::kVocabularySize) { + throw std::out_of_range( + "Phi-4 token ID is outside the vocabulary"); + } + } + } + + void EnsureCapacity(std::size_t token_count) const { + if (token_count == 0) { + throw std::invalid_argument( + "Phi-4 prefill requires at least one token"); + } + const auto remaining = + static_cast(max_length) - + static_cast(position); + if (token_count > remaining) { + throw std::out_of_range( + "Phi-4 AIE4 maximum context length would be " + "exceeded"); + } + } + + void StageInput(std::span token_ids) { + const auto rows = + static_cast(token_ids.size()); + const std::size_t live_elements = CheckedElements( + rows, + constants::kHiddenSize, + "Phi-4 input staging"); + if ( + live_elements > embedding_fp32.size() || + live_elements > normalized_fp32.size()) { + throw std::out_of_range( + "Phi-4 input exceeds the planned host capacity"); + } + + const auto embedding_output = + std::span(embedding_fp32).first(live_elements); + const auto normalized_output = + std::span(normalized_fp32).first(live_elements); + GatherEmbedding( + *api, + embedding, + token_ids, + embedding_output); + RmsNorm( + embedding_output, + input_norm, + rows, + constants::kHiddenSize, + static_cast(constants::kRmsEpsilon), + normalized_output); + + const std::int64_t hidden_rows = std::max( + shape_plan->RowsFor(RowUse::QueryProjection, rows), + shape_plan->RowsFor(RowUse::KvProjection, rows)); + StageBf16( + *api, + normalized_output, + rows, + hidden_rows, + constants::kHiddenSize, + bf16_staging); + const std::size_t hidden_bytes = CheckedBytes( + hidden_rows, + constants::kHiddenSize, + sizeof(std::uint16_t), + "Phi-4 hidden staging"); + api->Check( + api->functions().tensor_write( + current_hidden->get(), + bf16_staging.data(), + hidden_bytes, + 0), + kTensorWriteCall); + + const std::int64_t residual_rows = + shape_plan->RowsFor(RowUse::SsMlp, rows); + StageBf16( + *api, + embedding_output, + rows, + residual_rows, + constants::kHiddenSize, + bf16_staging); + const std::size_t residual_bytes = CheckedBytes( + residual_rows, + constants::kHiddenSize, + sizeof(std::uint16_t), + "Phi-4 residual staging"); + api->Check( + api->functions().tensor_write( + current_residual->get(), + bf16_staging.data(), + residual_bytes, + 0), + kTensorWriteCall); + } + + ryzenai_corelib_status SubmitMatMul( + const corelib::UniqueTensor& input, + const corelib::UniqueMatMulWeights& operation_weights, + corelib::UniqueTensor& output, + std::int64_t rows) { + return api->functions().matmul( + stream.get(), + input.get(), + rows, + operation_weights.get(), + output.get()); + } + + ryzenai_corelib_status SubmitMha( + std::size_t layer, + std::int64_t rows, + std::int64_t current_position) { + return api->functions().flat_mha( + stream.get(), + &shape_plan->attention_desc(), + query_tensor.get(), + key_tensor.get(), + rows, + current_position, + cos_tensor.get(), + sin_tensor.get(), + key_caches[layer].get(), + value_caches[layer].get(), + attention_tensor.get()); + } + + ryzenai_corelib_status SubmitSsMlp( + const corelib::UniqueTensor& input, + const corelib::UniqueTensor& residual, + const corelib::UniqueSsMlpWeights& operation_weights, + corelib::UniqueTensor& skip_sum, + corelib::UniqueTensor& normalized, + std::int64_t rows) { + return api->functions().ssmlp( + stream.get(), + input.get(), + residual.get(), + rows, + operation_weights.get(), + skip_sum.get(), + normalized.get()); + } + + void ScatterValue( + std::size_t layer, + std::int64_t rows, + std::int64_t current_position) { + flm::phi4::ScatterV( + *api, + value_tensor.get(), + value_caches[layer].get(), + rows, + current_position, + v_staging, + v_metrics); + metrics.v_read_calls = v_metrics.read_calls; + metrics.v_write_calls = v_metrics.write_calls; + metrics.v_bytes = v_metrics.bytes; + metrics.v_scatter_ns = v_metrics.nanoseconds; + } + + void PrepareLastHidden(std::int64_t rows) { + const std::size_t row_bytes = CheckedBytes( + 1, + constants::kHiddenSize, + sizeof(std::uint16_t), + "Phi-4 last hidden"); + const std::size_t source_offset = + static_cast(rows - 1) * row_bytes; + api->Check( + api->functions().tensor_read( + current_hidden->get(), + last_hidden_staging.data(), + row_bytes, + source_offset), + kTensorReadCall); + api->Check( + api->functions().tensor_write( + lm_input_tensor.get(), + last_hidden_staging.data(), + row_bytes, + 0), + kTensorWriteCall); + } + + void ReadLogits(buffer& output) { + constexpr std::size_t logits_bytes = + static_cast(constants::kVocabularySize) * + sizeof(std::uint16_t); + if (output.size() != + static_cast(constants::kVocabularySize)) { + throw std::logic_error( + "Phi-4 logits buffer has the wrong size"); + } + api->Check( + api->functions().tensor_read( + lm_output_tensor.get(), + output.data(), + logits_bytes, + 0), + kTensorReadCall); + } + + buffer RunRows(std::span token_ids) { + const std::int64_t rows = + static_cast(token_ids.size()); + auto execution = runtime->AcquireExecution(); + corelib::StepSubmissionState submission; + std::optional active_layer; + std::string active_phase = "stage_input"; + bool synchronize_in_progress = false; + buffer logits( + static_cast(constants::kVocabularySize)); + + try { + auto checked_submit = [&]( + ryzenai_corelib_status status, + std::string_view call) { + api->Check(status, call); + submission.MarkSuccessfulSubmit(); + ++metrics.dispatch_count; + }; + auto checked_synchronize = [&] { + synchronize_in_progress = true; + api->Check( + api->functions().stream_synchronize( + stream.get()), + kSynchronizeCall); + synchronize_in_progress = false; + ++metrics.synchronize_count; + }; + + StageInput(token_ids); + + for (int layer = 0; + layer < constants::kLayerCount; + ++layer) { + active_layer = layer; + active_phase = "qkv"; + const auto& layer_weights = + weights->layers()[static_cast(layer)]; + checked_submit( + SubmitMatMul( + *current_hidden, + layer_weights.q, + query_tensor, + rows), + "q"); + checked_submit( + SubmitMatMul( + *current_hidden, + layer_weights.k, + key_tensor, + rows), + "k"); + checked_submit( + SubmitMatMul( + *current_hidden, + layer_weights.v, + value_tensor, + rows), + "v"); + checked_synchronize(); + + active_phase = "v_scatter"; + ScatterValue( + static_cast(layer), + rows, + position); + + active_phase = "flat_mha"; + checked_submit( + SubmitMha( + static_cast(layer), + rows, + position), + "flat_mha"); + checked_synchronize(); + + active_phase = "o"; + checked_submit( + SubmitMatMul( + attention_tensor, + layer_weights.o, + *current_hidden, + rows), + "o"); + checked_synchronize(); + + active_phase = "ssmlp"; + checked_submit( + SubmitSsMlp( + *current_hidden, + *current_residual, + layer_weights.mlp, + *next_skip_sum, + *next_hidden, + rows), + "ssmlp"); + checked_synchronize(); + std::swap(current_hidden, next_hidden); + std::swap(current_residual, next_skip_sum); + } + + active_layer.reset(); + active_phase = "lm_head"; + PrepareLastHidden(rows); + checked_submit( + SubmitMatMul( + lm_input_tensor, + weights->lm_head(), + lm_output_tensor, + 1), + "lm_head"); + checked_synchronize(); + ReadLogits(logits); + } catch (const corelib::CorelibError& error) { + if (RecoverableBeforeSubmit( + synchronize_in_progress, + submission)) { + throw; + } + TerminateCorelibFailure( + runtime, + error, + active_phase, + active_layer, + rows, + position); + } catch (const std::exception& error) { + if (RecoverableBeforeSubmit( + synchronize_in_progress, + submission)) { + throw; + } + TerminateHostFailure( + runtime, + error, + active_phase, + active_layer, + rows, + position); + } catch (...) { + if (RecoverableBeforeSubmit( + synchronize_in_progress, + submission)) { + throw; + } + TerminateUnknownFailure( + runtime, + active_phase, + active_layer, + rows, + position); + } + + position += rows; + last_live_rows = rows; + return logits; + } + + std::vector ReadLiveCache( + const corelib::UniqueTensor& cache) const { + if (position == 0) { + return {}; + } + const std::size_t live_rows = + static_cast(position); + const std::size_t head_width = + static_cast(constants::kHeadSize); + std::vector result( + static_cast(constants::kKvHeadCount) * + live_rows * head_width); + const std::size_t bytes_per_head = + live_rows * head_width * sizeof(std::uint16_t); + for (std::size_t head = 0; + head < + static_cast(constants::kKvHeadCount); + ++head) { + const std::size_t source_offset = + head * + static_cast( + constants::kMaxSequenceLength) * + head_width * sizeof(std::uint16_t); + api->Check( + api->functions().tensor_read( + cache.get(), + result.data() + head * live_rows * head_width, + bytes_per_head, + source_offset), + kTensorReadCall); + } + return result; + } + +#ifdef DEV_BUILD + Phi4DebugSnapshot DebugSnapshot() const { + auto execution = runtime->AcquireExecution(); + Phi4DebugSnapshot snapshot; + snapshot.live_rows = last_live_rows; + snapshot.position = position; + snapshot.layer0_k = ReadLiveCache(key_caches.front()); + snapshot.layer0_v = ReadLiveCache(value_caches.front()); + snapshot.layer31_k = ReadLiveCache(key_caches.back()); + snapshot.layer31_v = ReadLiveCache(value_caches.back()); + snapshot.last_hidden.resize( + static_cast(constants::kHiddenSize)); + snapshot.logits.resize( + static_cast(constants::kVocabularySize)); + api->Check( + api->functions().tensor_read( + lm_input_tensor.get(), + snapshot.last_hidden.data(), + snapshot.last_hidden.size() * + sizeof(std::uint16_t), + 0), + kTensorReadCall); + api->Check( + api->functions().tensor_read( + lm_output_tensor.get(), + snapshot.logits.data(), + snapshot.logits.size() * sizeof(std::uint16_t), + 0), + kTensorReadCall); + return snapshot; + } +#endif + + std::shared_ptr runtime; + std::shared_ptr api; + std::optional shape_plan; + std::shared_ptr package; + std::optional weights; + + std::span embedding; + std::vector input_norm; + std::vector embedding_fp32; + std::vector normalized_fp32; + std::vector bf16_staging; + std::vector v_staging; + std::vector last_hidden_staging; + + std::array hidden_tensors; + corelib::UniqueTensor residual_tensor; + corelib::UniqueTensor skip_sum_tensor; + corelib::UniqueTensor query_tensor; + corelib::UniqueTensor key_tensor; + corelib::UniqueTensor value_tensor; + corelib::UniqueTensor attention_tensor; + corelib::UniqueTensor lm_input_tensor; + corelib::UniqueTensor lm_output_tensor; + corelib::UniqueTensor cos_tensor; + corelib::UniqueTensor sin_tensor; + std::array< + corelib::UniqueTensor, + static_cast(constants::kLayerCount)> + key_caches; + std::array< + corelib::UniqueTensor, + static_cast(constants::kLayerCount)> + value_caches; + corelib::UniqueTensor* current_hidden = nullptr; + corelib::UniqueTensor* next_hidden = nullptr; + corelib::UniqueTensor* current_residual = nullptr; + corelib::UniqueTensor* next_skip_sum = nullptr; + + Phi4Aie4Metrics metrics; + VScatterMetrics v_metrics; + std::uint32_t max_length; + std::int64_t position = 0; + std::int64_t last_live_rows = 0; + std::optional checkpoint_position; + + // Declared last so constructor rollback also releases the Stream before + // tensors, weights, and package storage. + corelib::UniqueStream stream; +}; + +phi4_corelib_aie4::phi4_corelib_aie4( + LM_Config config, + std::filesystem::path model_path, + std::shared_ptr runtime, + std::uint32_t max_length) + : impl_(std::make_unique( + std::move(config), + std::move(model_path), + std::move(runtime), + max_length)) {} + +phi4_corelib_aie4::~phi4_corelib_aie4() = default; + +buffer phi4_corelib_aie4::forward(int id) { + const std::array token{id}; + impl_->ValidateTokens(token); + impl_->EnsureCapacity(token.size()); + return impl_->RunRows(token); +} + +buffer phi4_corelib_aie4::prefill( + std::vector& ids, + void* payload) { + (void)payload; + const std::span token_ids(ids); + impl_->ValidateTokens(token_ids); + impl_->EnsureCapacity(token_ids.size()); + if (impl_->position == 0 || token_ids.size() == 1) { + return impl_->RunRows(token_ids); + } + + buffer logits; + for (const int token_id : token_ids) { + const std::array token{token_id}; + logits = impl_->RunRows(token); + } + return logits; +} + +void phi4_corelib_aie4::set_context_length(int length) { + if (length != impl_->position) { + throw std::invalid_argument( + "phi4_corelib_aie4 set_context_length accepts only " + "the current logical position"); + } +} + +void phi4_corelib_aie4::load_weights(Q4NX& q4nx) { + (void)q4nx; + throw std::runtime_error( + "Q4NX weight loading is unsupported by phi4_corelib_aie4"); +} + +void phi4_corelib_aie4::update_max_length( + std::uint32_t max_length) { + const std::uint32_t validated = + ValidateMaxLength(max_length); + if ( + validated < + static_cast(impl_->position)) { + throw std::out_of_range( + "Phi-4 AIE4 maximum length cannot be below the " + "current logical position"); + } + impl_->max_length = validated; +} + +void phi4_corelib_aie4::clear_context() { + impl_->position = 0; + impl_->last_live_rows = 0; + impl_->checkpoint_position.reset(); +} + +buffer phi4_corelib_aie4::get_k_cache( + int layer, + int index) { + (void)layer; + (void)index; + throw std::runtime_error( + "K-cache getters are unsupported by phi4_corelib_aie4"); +} + +buffer phi4_corelib_aie4::get_v_cache( + int layer, + int index) { + (void)layer; + (void)index; + throw std::runtime_error( + "V-cache getters are unsupported by phi4_corelib_aie4"); +} + +int phi4_corelib_aie4::get_current_context_length() { + return static_cast(impl_->position); +} + +int phi4_corelib_aie4::checkpoint() { + impl_->checkpoint_position = impl_->position; + return static_cast(impl_->position); +} + +int phi4_corelib_aie4::restore() { + if (!impl_->checkpoint_position.has_value()) { + throw std::logic_error( + "phi4_corelib_aie4 has no checkpoint to restore"); + } + impl_->position = *impl_->checkpoint_position; + impl_->last_live_rows = 0; + return static_cast(impl_->position); +} + +const Phi4Aie4Metrics& +phi4_corelib_aie4::metrics() const noexcept { + return impl_->metrics; +} + +#ifdef DEV_BUILD +Phi4DebugSnapshot phi4_corelib_aie4::debug_snapshot() const { + return impl_->DebugSnapshot(); +} +#endif + +#if defined(FLM_CORELIB_TESTING) +namespace testing { + +[[noreturn]] void ApplyCorelibFailurePolicyForTest( + const std::shared_ptr& runtime, + const corelib::CorelibError& error, + bool synchronize_in_progress, + const corelib::StepSubmissionState& submission, + std::string phase, + std::optional layer, + std::int64_t rows, + std::int64_t position) { + if (RecoverableBeforeSubmit( + synchronize_in_progress, + submission)) { + throw error; + } + TerminateCorelibFailure( + runtime, + error, + std::move(phase), + layer, + rows, + position); +} + +} // namespace testing +#endif + +} // namespace flm::phi4 diff --git a/src/include/models/phi4/phi4_corelib_aie4.hpp b/src/include/models/phi4/phi4_corelib_aie4.hpp new file mode 100644 index 00000000..ba12833d --- /dev/null +++ b/src/include/models/phi4/phi4_corelib_aie4.hpp @@ -0,0 +1,92 @@ +#pragma once + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace flm::phi4 { + +struct Phi4DebugSnapshot { + std::int64_t live_rows; + std::int64_t position; + std::vector layer0_k; + std::vector layer0_v; + std::vector layer31_k; + std::vector layer31_v; + std::vector last_hidden; + std::vector logits; +}; + +struct Phi4Aie4Metrics { + std::uint64_t model_load_ns = 0; + std::uint64_t weight_pack_ns = 0; + std::uint64_t packed_weight_bytes = 0; + std::uint64_t mapped_source_bytes = 0; + std::uint64_t kv_bytes = 0; + std::uint64_t scratch_bytes = 0; + std::uint64_t device_tensor_create_count = 0; + std::uint64_t weight_create_count = 0; + std::uint64_t dispatch_count = 0; + std::uint64_t synchronize_count = 0; + std::uint64_t v_read_calls = 0; + std::uint64_t v_write_calls = 0; + std::uint64_t v_bytes = 0; + std::uint64_t v_scatter_ns = 0; +}; + +class phi4_corelib_aie4 final : public causal_lm { +public: + phi4_corelib_aie4( + LM_Config config, + std::filesystem::path model_path, + std::shared_ptr runtime, + std::uint32_t max_length = 4096); + ~phi4_corelib_aie4() override; + + buffer forward(int id) override; + buffer prefill( + std::vector& ids, + void* payload = nullptr) override; + void set_context_length(int length) override; + void load_weights(Q4NX& q4nx) override; + void update_max_length(std::uint32_t max_length) override; + void clear_context() override; + buffer get_k_cache(int layer, int index) override; + buffer get_v_cache(int layer, int index) override; + int get_current_context_length() override; + int checkpoint() override; + int restore() override; + const Phi4Aie4Metrics& metrics() const noexcept; +#ifdef DEV_BUILD + Phi4DebugSnapshot debug_snapshot() const; +#endif + +private: + struct Impl; + std::unique_ptr impl_; +}; + +#if defined(FLM_CORELIB_TESTING) +namespace testing { + +[[noreturn]] void ApplyCorelibFailurePolicyForTest( + const std::shared_ptr& runtime, + const corelib::CorelibError& error, + bool synchronize_in_progress, + const corelib::StepSubmissionState& submission, + std::string phase, + std::optional layer, + std::int64_t rows, + std::int64_t position); + +} // namespace testing +#endif + +} // namespace flm::phi4 diff --git a/src/test/phi4_corelib_aie4/CMakeLists.txt b/src/test/phi4_corelib_aie4/CMakeLists.txt index 27ef8960..a1bcf6fa 100644 --- a/src/test/phi4_corelib_aie4/CMakeLists.txt +++ b/src/test/phi4_corelib_aie4/CMakeLists.txt @@ -7,6 +7,15 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON) find_path(RYZENAI_CORELIB_INCLUDE_DIR NAMES ryzenai/corelib.h REQUIRED) +find_path(BOOST_INCLUDE_DIR + NAMES boost/any.hpp + HINTS + "$ENV{BOOST_INCLUDEDIR}" + "$ENV{BOOST_ROOT}" + "$ENV{CONDA_PREFIX}/Library/include" + "$ENV{USERPROFILE}/anaconda3/Library/include" + "C:/dev/boost_1_88_0" + REQUIRED) get_filename_component( FASTFLOW_SOURCE_DIR @@ -25,6 +34,7 @@ add_library(flm_corelib_aie4_testlib STATIC target_include_directories(flm_corelib_aie4_testlib PUBLIC ${FASTFLOW_SOURCE_DIR}/include ${RYZENAI_CORELIB_INCLUDE_DIR} + ${BOOST_INCLUDE_DIR} ${XRT_INCLUDE_DIR}) target_link_directories(flm_corelib_aie4_testlib PUBLIC ${XRT_LIB_DIR}) @@ -79,6 +89,9 @@ add_corelib_host_test(test_phi4_manifest test_phi4_manifest.cpp) add_corelib_host_test(test_phi4_shape_plan test_phi4_shape_plan.cpp) add_corelib_host_test(test_phi4_host test_phi4_host.cpp) add_corelib_host_test(test_phi4_weights test_phi4_weights.cpp) +add_corelib_host_test(test_phi4_engine test_phi4_engine.cpp) +target_link_libraries(test_phi4_engine PRIVATE + advapi32) add_corelib_host_test( test_corelib_fatal_record test_corelib_fatal_record.cpp) diff --git a/src/test/phi4_corelib_aie4/test_phi4_engine.cpp b/src/test/phi4_corelib_aie4/test_phi4_engine.cpp new file mode 100644 index 00000000..44040bb6 --- /dev/null +++ b/src/test/phi4_corelib_aie4/test_phi4_engine.cpp @@ -0,0 +1,2152 @@ +#include "fake_corelib.hpp" +#include "test_support.hpp" + +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using flm::corelib::CorelibApi; +using flm::corelib::CorelibError; +using flm::corelib::CorelibRuntime; +using flm::corelib::FatalRecordStore; +using flm::corelib::ProcessState; +using flm::phi4::Phi4Aie4Metrics; +using flm::phi4::Phi4DebugSnapshot; +using flm::phi4::phi4_corelib_aie4; +using nlohmann::json; + +namespace constants = flm::phi4::constants; + +constexpr std::string_view kManifestName = + "corelib_phi4_manifest.json"; +constexpr std::string_view kDataFile = "weights.bin"; +constexpr std::uint64_t kDataBytes = + 200064ull * 3072ull * sizeof(std::uint16_t); +constexpr std::uint16_t kPoison = 0xDEADu; + +std::int64_t PaddedRows(std::int64_t rows) { + return rows == 1 ? 1 : ((rows + 3) / 4) * 4; +} + +class TempDirectory final { +public: + explicit TempDirectory(std::string_view stem) { + const auto nonce = + std::chrono::steady_clock::now().time_since_epoch().count(); + path_ = std::filesystem::temp_directory_path() / + (std::string(stem) + "-" + + std::to_string(GetCurrentProcessId()) + "-" + + std::to_string(nonce)); + std::filesystem::create_directories(path_); + } + + ~TempDirectory() noexcept { + std::error_code error; + std::filesystem::remove_all(path_, error); + } + + TempDirectory(const TempDirectory&) = delete; + TempDirectory& operator=(const TempDirectory&) = delete; + + const std::filesystem::path& path() const noexcept { + return path_; + } + +private: + std::filesystem::path path_; +}; + +void CreateSparseFile( + const std::filesystem::path& path, + std::uint64_t size) { + HANDLE file = CreateFileW( + path.c_str(), + GENERIC_READ | GENERIC_WRITE, + FILE_SHARE_READ, + nullptr, + CREATE_ALWAYS, + FILE_ATTRIBUTE_NORMAL, + nullptr); + if (file == INVALID_HANDLE_VALUE) { + throw std::runtime_error( + "failed to create sparse engine-test model data"); + } + + DWORD ignored = 0; + if ( + DeviceIoControl( + file, + FSCTL_SET_SPARSE, + nullptr, + 0, + nullptr, + 0, + &ignored, + nullptr) == FALSE) { + CloseHandle(file); + throw std::runtime_error( + "engine-test volume does not support sparse files"); + } + + LARGE_INTEGER end{}; + end.QuadPart = static_cast(size); + const bool success = + SetFilePointerEx(file, end, nullptr, FILE_BEGIN) != FALSE && + SetEndOfFile(file) != FALSE; + CloseHandle(file); + if (!success) { + throw std::runtime_error( + "failed to size sparse engine-test model data"); + } +} + +std::uint64_t ItemSize(std::string_view dtype) { + if (dtype == "uint8") { + return 1; + } + if (dtype == "float16") { + return 2; + } + if (dtype == "float32" || dtype == "int64") { + return dtype == "float32" ? 4 : 8; + } + throw std::runtime_error("unsupported synthetic dtype"); +} + +std::uint64_t ByteLength( + std::string_view dtype, + const std::vector& shape) { + std::uint64_t elements = 1; + for (const std::int64_t dimension : shape) { + elements *= static_cast(dimension); + } + return elements * ItemSize(dtype); +} + +void AddInitializer( + json& initializers, + const std::string& name, + std::string dtype, + std::vector shape, + std::string role, + std::uint64_t offset) { + CHECK(!initializers.contains(name)); + initializers[name] = { + {"file", std::string(kDataFile)}, + {"offset", offset}, + {"length", ByteLength(dtype, shape)}, + {"dtype", std::move(dtype)}, + {"shape", std::move(shape)}, + {"role", std::move(role)}}; +} + +class ManifestBuilder final { +public: + explicit ManifestBuilder(std::uint64_t model_size) + : manifest_{ + {"schema_version", 1}, + {"execution_backend", "corelib_aie4"}, + {"model", + { + {"family", "phi4"}, + {"layers", 32}, + {"hidden_size", 3072}, + {"intermediate_size", 8192}, + {"num_heads", 24}, + {"kv_heads", 8}, + {"head_size", 128}, + {"vocab_size", 200064}, + {"group_size", 128}, + {"rope_dim", 96}, + {"rms_epsilon", 0.00001}, + }}, + {"backend", {{"max_seq", 4096}}}, + {"files", + { + {"model.onnx", {{"size", model_size}}}, + {std::string(kDataFile), {{"size", kDataBytes}}}, + }}, + {"initializers", json::object()}, + {"weight_objects", json::array()}} {} + + void AddMatMul( + const std::string& name, + std::int64_t k, + std::int64_t n) { + const std::string qweight = name + ".qweight"; + const std::string scales = name + ".scales"; + const std::string qzeros = name + ".qzeros"; + manifest_["weight_objects"].push_back({ + {"name", name}, + {"kind", "matmul"}, + {"descriptor", + { + {"k", k}, + {"n", n}, + {"group_size", 128}, + {"has_bias", false}, + }}, + {"roles", + { + {"qweight", qweight}, + {"scales", scales}, + {"qzeros", qzeros}, + }}}); + + AddInitializer( + manifest_["initializers"], + qweight, + "uint8", + {n, k / 2}, + "matmul.qweight", + NextOffset()); + AddInitializer( + manifest_["initializers"], + scales, + "float16", + {n, k / 128}, + "matmul.scales", + NextOffset()); + AddInitializer( + manifest_["initializers"], + qzeros, + "uint8", + {n, ((k / 128) + 1) / 2}, + "matmul.qzeros", + NextOffset()); + } + + void AddSsMlp(int layer) { + const std::string base = + "model.layers." + std::to_string(layer); + const std::string object_name = base + ".ssmlp"; + const std::string norm0 = + base + ".post_attention_layernorm.weight"; + const std::string norm1 = + layer == 31 + ? "model.layers.32.final_norm_layernorm.weight" + : "model.layers." + std::to_string(layer + 1) + + ".input_layernorm.weight"; + json roles = { + {"norm0", norm0}, + {"norm1", norm1}, + }; + + for (const std::string projection : {"gate", "up", "down"}) { + const std::int64_t k = + projection == "down" ? 8192 : 3072; + const std::int64_t n = + projection == "down" ? 3072 : 8192; + const std::string prefix = + base + ".mlp." + projection + + "_proj.MatMulNBits"; + const std::string role_prefix = + "ssmlp." + projection; + for (const std::string component : + {"qweight", "scales", "qzeros"}) { + roles[projection + "_" + component] = + prefix + "." + component; + } + AddInitializer( + manifest_["initializers"], + prefix + ".qweight", + "uint8", + {n, k / 2}, + role_prefix + ".qweight", + NextOffset()); + AddInitializer( + manifest_["initializers"], + prefix + ".scales", + "float16", + {n, k / 128}, + role_prefix + ".scales", + NextOffset()); + AddInitializer( + manifest_["initializers"], + prefix + ".qzeros", + "uint8", + {n, ((k / 128) + 1) / 2}, + role_prefix + ".qzeros", + NextOffset()); + } + + AddInitializer( + manifest_["initializers"], + norm0, + "float16", + {3072}, + "ssmlp.norm0", + NextOffset()); + AddInitializer( + manifest_["initializers"], + norm1, + "float16", + {3072}, + "ssmlp.norm1", + NextOffset()); + manifest_["weight_objects"].push_back({ + {"name", object_name}, + {"kind", "ssmlp"}, + {"descriptor", + { + {"k", 3072}, + {"n", 8192}, + {"group_size", 128}, + }}, + {"roles", std::move(roles)}}); + } + + json Finish() { + for (int layer = 0; layer < 32; ++layer) { + const std::string base = + "model.layers." + std::to_string(layer) + + ".attn."; + AddMatMul( + base + "q_proj.MatMulNBits", + 3072, + 3072); + AddMatMul( + base + "k_proj.MatMulNBits", + 3072, + 1024); + AddMatMul( + base + "v_proj.MatMulNBits", + 3072, + 1024); + AddMatMul( + base + "o_proj.MatMulNBits", + 3072, + 3072); + AddSsMlp(layer); + } + AddMatMul("lm_head.MatMulNBits", 3072, 200064); + + AddInitializer( + manifest_["initializers"], + "model.embed_tokens.weight", + "float16", + {200064, 3072}, + "embedding", + 0); + AddInitializer( + manifest_["initializers"], + "model.layers.0.input_layernorm.weight", + "float16", + {3072}, + "input_norm", + NextOffset()); + AddInitializer( + manifest_["initializers"], + "cos_cache", + "float16", + {4096, 48}, + "cos_cache", + NextOffset()); + AddInitializer( + manifest_["initializers"], + "sin_cache", + "float16", + {4096, 48}, + "sin_cache", + NextOffset()); + + CHECK(manifest_["weight_objects"].size() == 161); + CHECK(manifest_["initializers"].size() == 743); + return std::move(manifest_); + } + +private: + std::uint64_t NextOffset() { + const std::uint64_t result = next_offset_; + next_offset_ += 16; + return result; + } + + json manifest_; + std::uint64_t next_offset_ = 4096; +}; + +class SyntheticPackage final { +public: + SyntheticPackage() + : temp_("fastflowlm-phi4-engine-model") { + const auto model_path = temp_.path() / "model.onnx"; + { + std::ofstream model(model_path, std::ios::binary); + model << "model"; + } + CreateSparseFile(temp_.path() / kDataFile, kDataBytes); + ManifestBuilder builder( + std::filesystem::file_size(model_path)); + std::ofstream manifest( + temp_.path() / kManifestName, + std::ios::binary); + manifest << builder.Finish().dump(2) << '\n'; + if (!manifest) { + throw std::runtime_error( + "failed to write synthetic engine manifest"); + } + } + + const std::filesystem::path& path() const noexcept { + return temp_.path(); + } + +private: + TempDirectory temp_; +}; + +float HalfToFloat(std::uint16_t value) { + const bool negative = (value & 0x8000u) != 0; + const unsigned exponent = (value >> 10) & 0x1fu; + const unsigned mantissa = value & 0x03ffu; + float result = 0.0f; + if (exponent == 0) { + result = std::ldexp(static_cast(mantissa), -24); + } else if (exponent == 31) { + result = mantissa == 0 + ? std::numeric_limits::infinity() + : std::numeric_limits::quiet_NaN(); + } else { + result = std::ldexp( + 1.0f + static_cast(mantissa) / 1024.0f, + static_cast(exponent) - 15); + } + return negative ? -result : result; +} + +std::uint16_t FloatToHalf(float value) { + const std::uint32_t bits = std::bit_cast(value); + const std::uint16_t sign = + static_cast((bits >> 16) & 0x8000u); + const std::uint32_t source_exponent = (bits >> 23) & 0xffu; + const std::uint32_t source_mantissa = bits & 0x007fffffu; + if (source_exponent == 0xffu) { + return static_cast( + sign | (source_mantissa == 0 ? 0x7c00u : 0x7e00u)); + } + const int exponent = static_cast(source_exponent) - 127 + 15; + if (exponent >= 31) { + return static_cast(sign | 0x7c00u); + } + if (exponent <= 0) { + if (exponent < -10) { + return sign; + } + const std::uint32_t mantissa = + source_mantissa | 0x00800000u; + const unsigned shift = static_cast(14 - exponent); + const std::uint32_t rounded = + (mantissa + (1u << (shift - 1)) - 1u + + ((mantissa >> shift) & 1u)) >> + shift; + return static_cast(sign | rounded); + } + const std::uint32_t rounded = + source_mantissa + 0x00000fffu + + ((source_mantissa >> 13) & 1u); + if ((rounded & 0x00800000u) != 0) { + if (exponent + 1 >= 31) { + return static_cast(sign | 0x7c00u); + } + return static_cast( + sign | (static_cast(exponent + 1) << 10)); + } + return static_cast( + sign | (static_cast(exponent) << 10) | + (rounded >> 13)); +} + +std::uint16_t FloatToBf16(float value) { + std::uint32_t bits = std::bit_cast(value); + bits += 0x7fffu + ((bits >> 16) & 1u); + return static_cast(bits >> 16); +} + +float ReadElement( + ryzenai_corelib_data_type type, + const void* source, + std::size_t index) { + switch (type) { + case ryzenai_corelib_data_type_fp16: + return HalfToFloat( + static_cast(source)[index]); + case ryzenai_corelib_data_type_bf16: { + const std::uint32_t bits = + static_cast( + static_cast( + source)[index]) + << 16; + return std::bit_cast(bits); + } + case ryzenai_corelib_data_type_fp32: + return static_cast(source)[index]; + default: + throw std::runtime_error( + "engine fake received unsupported conversion source"); + } +} + +void WriteElement( + ryzenai_corelib_data_type type, + void* destination, + std::size_t index, + float value) { + switch (type) { + case ryzenai_corelib_data_type_fp16: + static_cast(destination)[index] = + FloatToHalf(value); + return; + case ryzenai_corelib_data_type_bf16: + static_cast(destination)[index] = + FloatToBf16(value); + return; + case ryzenai_corelib_data_type_fp32: + static_cast(destination)[index] = value; + return; + default: + throw std::runtime_error( + "engine fake received unsupported conversion destination"); + } +} + +enum class ObjectKind { Stream, Tensor, MatMulWeight, SsMlpWeight }; + +struct FakeObject { + explicit FakeObject(ObjectKind kind_value, std::string label_value) + : kind(kind_value), + label(std::move(label_value)) {} + virtual ~FakeObject() = default; + + ObjectKind kind; + std::string label; + bool released = false; +}; + +struct FakeTensor final : FakeObject { + static constexpr std::size_t kPageBytes = 4096; + using Page = std::array; + + FakeTensor( + ryzenai_corelib_data_type type, + std::vector dimensions, + std::size_t bytes, + std::string label) + : FakeObject(ObjectKind::Tensor, std::move(label)), + data_type(type), + shape(std::move(dimensions)), + byte_size(bytes) {} + + std::byte DefaultByte(std::size_t offset) const noexcept { + return std::byte{ + static_cast( + offset % 2 == 0 ? 0xADu : 0xDEu)}; + } + + Page& MutablePage(std::size_t page_index) { + auto [found, inserted] = + pages.try_emplace(page_index, nullptr); + if (inserted) { + found->second = std::make_unique(); + const std::size_t base = page_index * kPageBytes; + for (std::size_t index = 0; index < kPageBytes; ++index) { + (*found->second)[index] = DefaultByte(base + index); + } + } + return *found->second; + } + + std::byte ReadByte(std::size_t offset) const { + const std::size_t page_index = offset / kPageBytes; + const auto found = pages.find(page_index); + if (found == pages.end()) { + return DefaultByte(offset); + } + return (*found->second)[offset % kPageBytes]; + } + + bool Write( + const void* source, + std::size_t size, + std::size_t offset) { + if ( + (source == nullptr && size != 0) || + offset > byte_size || + size > byte_size - offset) { + return false; + } + const auto* bytes = static_cast(source); + for (std::size_t index = 0; index < size; ++index) { + MutablePage((offset + index) / kPageBytes) + [(offset + index) % kPageBytes] = bytes[index]; + } + return true; + } + + bool Read( + void* destination, + std::size_t size, + std::size_t offset) const { + if ( + (destination == nullptr && size != 0) || + offset > byte_size || + size > byte_size - offset) { + return false; + } + auto* bytes = static_cast(destination); + for (std::size_t index = 0; index < size; ++index) { + bytes[index] = ReadByte(offset + index); + } + return true; + } + + std::uint16_t ReadWord(std::size_t word_index) const { + const std::size_t offset = word_index * sizeof(std::uint16_t); + const std::array bytes{ + ReadByte(offset), + ReadByte(offset + 1)}; + std::uint16_t value = 0; + std::memcpy(&value, bytes.data(), sizeof(value)); + return value; + } + + void WriteWord(std::size_t word_index, std::uint16_t value) { + const std::size_t offset = word_index * sizeof(value); + CHECK(Write(&value, sizeof(value), offset)); + } + + void FillWords( + std::size_t first, + std::size_t count, + std::uint16_t value) { + for (std::size_t index = 0; index < count; ++index) { + WriteWord(first + index, value); + } + } + + bool ContainsPoisonWords( + std::size_t first, + std::size_t count) const { + for (std::size_t index = 0; index < count; ++index) { + if (ReadWord(first + index) == kPoison) { + return true; + } + } + return false; + } + + ryzenai_corelib_data_type data_type; + std::vector shape; + std::size_t byte_size; + std::unordered_map> pages; +}; + +struct FakeMatMulWeight final : FakeObject { + FakeMatMulWeight( + std::string label, + ryzenai_corelib_matmul_bf16_weights_desc value) + : FakeObject(ObjectKind::MatMulWeight, std::move(label)), + desc(value) {} + + ryzenai_corelib_matmul_bf16_weights_desc desc{}; +}; + +struct FakeSsMlpWeight final : FakeObject { + FakeSsMlpWeight( + std::string label, + ryzenai_corelib_ssmlp_bf16_weights_desc value) + : FakeObject(ObjectKind::SsMlpWeight, std::move(label)), + desc(value) {} + + ryzenai_corelib_ssmlp_bf16_weights_desc desc{}; +}; + +struct MatMulCall { + std::string label; + FakeTensor* input; + FakeTensor* output; + std::int64_t rows; +}; + +struct SsMlpCall { + FakeTensor* input; + FakeTensor* residual; + FakeTensor* skip_sum; + FakeTensor* normalized; + std::int64_t rows; +}; + +struct MhaCall { + int layer; + FakeTensor* query; + FakeTensor* key; + FakeTensor* key_cache; + FakeTensor* value_cache; + FakeTensor* output; + std::int64_t rows; + std::int64_t position; +}; + +struct TensorWriteCall { + FakeTensor* tensor; + std::size_t size; + std::size_t offset; +}; + +enum class FailurePoint { + None, + FirstQ, + KAfterQ, + Synchronize, + StageBadAlloc, + ScatterBadAlloc, + ScatterUnknown +}; + +struct UnknownFailure final {}; + +struct RecordingState { + std::vector> objects; + std::unordered_map object_index; + std::vector tensors; + std::vector events; + std::vector release_labels; + std::vector matmul_calls; + std::vector ssmlp_calls; + std::vector mha_calls; + std::vector stage_writes; + std::set q_tensors; + std::set k_tensors; + std::set v_tensors; + std::set attention_tensors; + std::set skip_sum_tensors; + std::set normalized_tensors; + FakeTensor* staged_hidden = nullptr; + FakeTensor* staged_residual = nullptr; + FailurePoint failure = FailurePoint::None; + std::size_t matmul_weight_count = 0; + std::size_t ssmlp_weight_count = 0; + std::size_t synchronize_calls = 0; + int active_layer = -1; + std::int64_t active_rows = 0; + bool input_poison_observed = false; + bool host_read_poison_observed = false; + bool cache_publish_poison_observed = false; + bool terminator_called = false; + unsigned int termination_code = 0; + std::optional load_thread; + bool load_thread_consistent = true; + + template + Object* Create(Args&&... args) { + auto object = + std::make_unique(std::forward(args)...); + auto* result = object.get(); + object_index.emplace(result, result); + objects.push_back(std::move(object)); + return result; + } + + FakeObject* Object(void* value) const { + const auto found = object_index.find(value); + return found == object_index.end() ? nullptr : found->second; + } + + FakeTensor* Tensor(void* value) const { + auto* object = Object(value); + if (object == nullptr || object->kind != ObjectKind::Tensor) { + return nullptr; + } + return static_cast(object); + } + + void ObserveLoadThread() { + const auto current = std::this_thread::get_id(); + if (!load_thread.has_value()) { + load_thread = current; + } else { + load_thread_consistent = + load_thread_consistent && *load_thread == current; + } + } + + void ResetExecutionRecords() { + events.clear(); + matmul_calls.clear(); + ssmlp_calls.clear(); + mha_calls.clear(); + stage_writes.clear(); + q_tensors.clear(); + k_tensors.clear(); + v_tensors.clear(); + attention_tensors.clear(); + skip_sum_tensors.clear(); + normalized_tensors.clear(); + synchronize_calls = 0; + active_layer = -1; + active_rows = 0; + input_poison_observed = false; + host_read_poison_observed = false; + cache_publish_poison_observed = false; + } +}; + +RecordingState* g_state = nullptr; + +RecordingState& State() { + if (g_state == nullptr) { + throw std::runtime_error("engine recording fake is not active"); + } + return *g_state; +} + +std::size_t TypeSize(ryzenai_corelib_data_type type) { + switch (type) { + case ryzenai_corelib_data_type_fp16: + case ryzenai_corelib_data_type_bf16: + return 2; + case ryzenai_corelib_data_type_fp32: + return 4; + default: + throw std::runtime_error( + "engine fake cannot size this tensor dtype"); + } +} + +std::size_t TensorBytes( + ryzenai_corelib_data_type type, + std::span shape) { + std::size_t elements = 1; + for (const std::int64_t dimension : shape) { + if (dimension <= 0) { + throw std::runtime_error( + "engine fake received non-positive tensor shape"); + } + elements *= static_cast(dimension); + } + return elements * TypeSize(type); +} + +std::string MatMulLabel(std::size_t ordinal) { + if (ordinal == 128) { + return "lm_head"; + } + const std::size_t layer = ordinal / 4; + constexpr std::array projections{ + "q", + "k", + "v", + "o"}; + return std::string(projections[ordinal % 4]) + "_" + + std::to_string(layer); +} + +std::string Projection(const std::string& label) { + if (label == "lm_head") { + return label; + } + const auto separator = label.find('_'); + return separator == std::string::npos + ? label + : label.substr(0, separator); +} + +int WeightLayer(const std::string& label) { + if (label == "lm_head") { + return -1; + } + return std::stoi(label.substr(label.find('_') + 1)); +} + +ryzenai_corelib_status RecordingConvert( + ryzenai_corelib_data_type source_type, + const void* source, + ryzenai_corelib_data_type destination_type, + void* destination, + std::size_t count) { + auto& state = State(); + state.ObserveLoadThread(); + if ( + state.failure == FailurePoint::StageBadAlloc && + source_type == ryzenai_corelib_data_type_fp16 && + destination_type == ryzenai_corelib_data_type_fp32 && + count != static_cast(constants::kHiddenSize)) { + throw std::bad_alloc{}; + } + if ((source == nullptr || destination == nullptr) && count != 0) { + return ryzenai_corelib_status_bad_argument; + } + for (std::size_t index = 0; index < count; ++index) { + WriteElement( + destination_type, + destination, + index, + ReadElement(source_type, source, index)); + } + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status RecordingConvertStrided( + ryzenai_corelib_data_type source_type, + const void* source, + std::size_t source_stride, + ryzenai_corelib_data_type destination_type, + void* destination, + std::size_t destination_stride, + std::size_t count, + std::size_t row) { + State().ObserveLoadThread(); + if ( + source == nullptr || destination == nullptr || row == 0 || + count % row != 0 || source_stride < row || + destination_stride < row) { + return ryzenai_corelib_status_bad_argument; + } + const std::size_t rows = count / row; + for (std::size_t row_index = 0; row_index < rows; ++row_index) { + for (std::size_t column = 0; column < row; ++column) { + WriteElement( + destination_type, + destination, + row_index * destination_stride + column, + ReadElement( + source_type, + source, + row_index * source_stride + column)); + } + } + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status RecordingMatMulPadShape( + std::int64_t* m, + std::int64_t*, + std::int64_t*, + std::uint32_t) { + State().ObserveLoadThread(); + if (m == nullptr || *m <= 0) { + return ryzenai_corelib_status_bad_argument; + } + *m = PaddedRows(*m); + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status RecordingSsMlpPadRows( + std::int64_t* m, + std::int64_t, + std::int64_t, + std::uint32_t) { + State().ObserveLoadThread(); + if (m == nullptr || *m <= 0) { + return ryzenai_corelib_status_bad_argument; + } + *m = PaddedRows(*m); + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status RecordingMhaPadRows( + std::int64_t* m, + const ryzenai_corelib_flat_mha_bf16_desc*) { + State().ObserveLoadThread(); + if (m == nullptr || *m <= 0) { + return ryzenai_corelib_status_bad_argument; + } + *m = PaddedRows(*m); + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status RecordingCreateStream( + ryzenai_corelib_stream_ptr* out) { + auto& state = State(); + state.ObserveLoadThread(); + if (out == nullptr) { + return ryzenai_corelib_status_bad_argument; + } + *out = state.Create(ObjectKind::Stream, "stream"); + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status RecordingCreateTensor( + ryzenai_corelib_data_type data_type, + const std::int64_t* shape, + std::size_t shape_len, + ryzenai_corelib_tensor_ptr* out) { + auto& state = State(); + state.ObserveLoadThread(); + if (shape == nullptr || shape_len == 0 || out == nullptr) { + return ryzenai_corelib_status_bad_argument; + } + std::vector dimensions(shape, shape + shape_len); + const auto ordinal = state.tensors.size(); + auto* tensor = state.Create( + data_type, + dimensions, + TensorBytes(data_type, dimensions), + "tensor_" + std::to_string(ordinal)); + state.tensors.push_back(tensor); + *out = tensor; + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status RecordingTensorGetByteSize( + ryzenai_corelib_tensor_ptr tensor, + std::size_t* out) { + auto* value = State().Tensor(tensor); + if (value == nullptr || out == nullptr) { + return ryzenai_corelib_status_bad_argument; + } + *out = value->byte_size; + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status RecordingTensorWrite( + ryzenai_corelib_tensor_ptr tensor, + const void* source, + std::size_t size, + std::size_t offset) { + auto& state = State(); + auto* value = state.Tensor(tensor); + if (value == nullptr) { + return ryzenai_corelib_status_bad_argument; + } + + const bool is_cache = + value->shape == + std::vector{8, 4096, 128}; + if (is_cache) { + if ( + state.failure == FailurePoint::ScatterBadAlloc) { + // The injected allocation failure belongs to the preceding V read. + return ryzenai_corelib_status_bad_argument; + } + if (value->label.starts_with("tensor_")) { + value->label = + "v_cache_" + std::to_string(state.active_layer); + } + const auto* words = + static_cast(source); + const std::size_t word_count = size / sizeof(std::uint16_t); + state.cache_publish_poison_observed = + state.cache_publish_poison_observed || + std::any_of( + words, + words + word_count, + [](std::uint16_t word) { return word == kPoison; }); + constexpr std::size_t head_pitch_bytes = + 4096u * 128u * sizeof(std::uint16_t); + const std::size_t head = offset / head_pitch_bytes; + state.events.push_back( + "tensor_write_v_head_" + std::to_string(head)); + } else if ( + value->data_type == ryzenai_corelib_data_type_bf16 && + value->shape == + std::vector{4096, 3072}) { + state.stage_writes.push_back({value, size, offset}); + if (state.stage_writes.size() % 2 == 1) { + state.staged_hidden = value; + } else { + state.staged_residual = value; + } + } + + return value->Write(source, size, offset) + ? ryzenai_corelib_status_success + : ryzenai_corelib_status_bad_argument; +} + +ryzenai_corelib_status RecordingTensorRead( + ryzenai_corelib_tensor_ptr tensor, + void* destination, + std::size_t size, + std::size_t offset) { + auto& state = State(); + auto* value = state.Tensor(tensor); + if (value == nullptr) { + return ryzenai_corelib_status_bad_argument; + } + + if (state.v_tensors.contains(value)) { + if (state.failure == FailurePoint::ScatterBadAlloc) { + throw std::bad_alloc{}; + } + if (state.failure == FailurePoint::ScatterUnknown) { + throw UnknownFailure{}; + } + state.events.emplace_back("tensor_read_v"); + const std::size_t expected = + static_cast(state.active_rows) * + static_cast(constants::kKvDimension) * + sizeof(std::uint16_t); + if (offset != 0 || size != expected) { + state.host_read_poison_observed = true; + } + } else if ( + value->shape == + std::vector{1, 200064}) { + state.events.emplace_back("tensor_read_logits"); + } + + state.host_read_poison_observed = + state.host_read_poison_observed || + value->ContainsPoisonWords( + offset / sizeof(std::uint16_t), + size / sizeof(std::uint16_t)); + return value->Read(destination, size, offset) + ? ryzenai_corelib_status_success + : ryzenai_corelib_status_bad_argument; +} + +ryzenai_corelib_status RecordingMatMulWeightsCreate( + const ryzenai_corelib_matmul_bf16_weights_desc* desc, + const ryzenai_corelib_matmul_bf16_onnx_weights_components* components, + ryzenai_corelib_matmul_bf16_weights_ptr* out) { + auto& state = State(); + state.ObserveLoadThread(); + if (desc == nullptr || components == nullptr || out == nullptr) { + return ryzenai_corelib_status_bad_argument; + } + const auto ordinal = state.matmul_weight_count++; + *out = state.Create( + MatMulLabel(ordinal), + *desc); + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status RecordingSsMlpWeightsCreate( + const ryzenai_corelib_ssmlp_bf16_weights_desc* desc, + const ryzenai_corelib_ssmlp_bf16_onnx_weights_components* components, + ryzenai_corelib_ssmlp_bf16_weights_ptr* out) { + auto& state = State(); + state.ObserveLoadThread(); + if (desc == nullptr || components == nullptr || out == nullptr) { + return ryzenai_corelib_status_bad_argument; + } + const auto ordinal = state.ssmlp_weight_count++; + *out = state.Create( + "ssmlp_" + std::to_string(ordinal), + *desc); + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status RecordingMatMulWeightsGetData( + ryzenai_corelib_matmul_bf16_weights_ptr weights, + const void** data, + std::size_t* size) { + if ( + State().Object(weights) == nullptr || data != nullptr || + size == nullptr) { + return ryzenai_corelib_status_bad_argument; + } + *size = 17; + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status RecordingSsMlpWeightsGetData( + ryzenai_corelib_ssmlp_bf16_weights_ptr weights, + const void** data, + std::size_t* size) { + if ( + State().Object(weights) == nullptr || data != nullptr || + size == nullptr) { + return ryzenai_corelib_status_bad_argument; + } + *size = 29; + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status RecordingMatMul( + ryzenai_corelib_stream_ptr stream, + ryzenai_corelib_tensor_ptr input, + std::int64_t rows, + ryzenai_corelib_matmul_bf16_weights_ptr weights, + ryzenai_corelib_tensor_ptr output) { + auto& state = State(); + auto* stream_object = state.Object(stream); + auto* input_tensor = state.Tensor(input); + auto* output_tensor = state.Tensor(output); + auto* weight_object = state.Object(weights); + if ( + stream_object == nullptr || + stream_object->kind != ObjectKind::Stream || + input_tensor == nullptr || output_tensor == nullptr || + weight_object == nullptr || + weight_object->kind != ObjectKind::MatMulWeight || + rows <= 0) { + return ryzenai_corelib_status_bad_argument; + } + auto* weight = static_cast(weight_object); + const std::string projection = Projection(weight->label); + const int layer = WeightLayer(weight->label); + if (layer >= 0) { + state.active_layer = layer; + } + state.active_rows = rows; + state.events.push_back("matmul_" + projection); + state.matmul_calls.push_back( + {weight->label, input_tensor, output_tensor, rows}); + + if ( + state.failure == FailurePoint::FirstQ && + weight->label == "q_0") { + flm::test::SetLastErrorMessage("injected first-q failure"); + return ryzenai_corelib_status_failure; + } + if ( + state.failure == FailurePoint::KAfterQ && + weight->label == "k_0") { + flm::test::SetLastErrorMessage("injected k failure"); + return ryzenai_corelib_status_failure; + } + + const std::size_t input_words = + static_cast(PaddedRows(rows)) * + static_cast(weight->desc.k); + state.input_poison_observed = + state.input_poison_observed || + input_tensor->ContainsPoisonWords(0, input_words); + const std::size_t output_words = + static_cast(PaddedRows(rows)) * + static_cast(weight->desc.n); + output_tensor->FillWords( + 0, + output_words, + projection == "lm_head" + ? static_cast(0x3F80u) + : static_cast( + 0x1000u + static_cast( + std::max(layer, 0) * 8 + + static_cast( + state.matmul_calls.size() % 8)))); + + if (projection == "q") { + state.q_tensors.insert(output_tensor); + } else if (projection == "k") { + state.k_tensors.insert(output_tensor); + } else if (projection == "v") { + state.v_tensors.insert(output_tensor); + } + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status RecordingSsMlp( + ryzenai_corelib_stream_ptr stream, + ryzenai_corelib_tensor_ptr input, + ryzenai_corelib_tensor_ptr residual, + std::int64_t rows, + ryzenai_corelib_ssmlp_bf16_weights_ptr weights, + ryzenai_corelib_tensor_ptr skip_sum, + ryzenai_corelib_tensor_ptr normalized) { + auto& state = State(); + auto* stream_object = state.Object(stream); + auto* input_tensor = state.Tensor(input); + auto* residual_tensor = state.Tensor(residual); + auto* skip_tensor = state.Tensor(skip_sum); + auto* normalized_tensor = state.Tensor(normalized); + auto* weight_object = state.Object(weights); + if ( + stream_object == nullptr || + stream_object->kind != ObjectKind::Stream || + input_tensor == nullptr || residual_tensor == nullptr || + skip_tensor == nullptr || normalized_tensor == nullptr || + weight_object == nullptr || + weight_object->kind != ObjectKind::SsMlpWeight || + rows <= 0) { + return ryzenai_corelib_status_bad_argument; + } + state.events.emplace_back("ssmlp"); + state.ssmlp_calls.push_back( + {input_tensor, + residual_tensor, + skip_tensor, + normalized_tensor, + rows}); + + const std::size_t words = + static_cast(PaddedRows(rows)) * + static_cast(constants::kHiddenSize); + state.input_poison_observed = + state.input_poison_observed || + input_tensor->ContainsPoisonWords(0, words) || + residual_tensor->ContainsPoisonWords(0, words); + skip_tensor->FillWords(0, words, 0x2200u); + normalized_tensor->FillWords(0, words, 0x2300u); + state.skip_sum_tensors.insert(skip_tensor); + state.normalized_tensors.insert(normalized_tensor); + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status RecordingFlatMha( + ryzenai_corelib_stream_ptr stream, + const ryzenai_corelib_flat_mha_bf16_desc* desc, + ryzenai_corelib_tensor_ptr query, + ryzenai_corelib_tensor_ptr key, + std::int64_t rows, + std::int64_t position, + ryzenai_corelib_tensor_ptr cos, + ryzenai_corelib_tensor_ptr sin, + ryzenai_corelib_tensor_ptr key_cache, + ryzenai_corelib_tensor_ptr value_cache, + ryzenai_corelib_tensor_ptr output) { + auto& state = State(); + auto* stream_object = state.Object(stream); + auto* q = state.Tensor(query); + auto* k = state.Tensor(key); + auto* cos_tensor = state.Tensor(cos); + auto* sin_tensor = state.Tensor(sin); + auto* k_cache = state.Tensor(key_cache); + auto* v_cache = state.Tensor(value_cache); + auto* out = state.Tensor(output); + if ( + stream_object == nullptr || + stream_object->kind != ObjectKind::Stream || + desc == nullptr || q == nullptr || k == nullptr || + cos_tensor == nullptr || sin_tensor == nullptr || + k_cache == nullptr || v_cache == nullptr || out == nullptr || + rows <= 0 || position < 0 || + (rows > 1 && position != 0)) { + return ryzenai_corelib_status_bad_argument; + } + if (k_cache->label.starts_with("tensor_")) { + k_cache->label = + "k_cache_" + std::to_string(state.active_layer); + } + if (v_cache->label.starts_with("tensor_")) { + v_cache->label = + "v_cache_" + std::to_string(state.active_layer); + } + state.events.emplace_back("flat_mha"); + state.mha_calls.push_back( + {state.active_layer, + q, + k, + k_cache, + v_cache, + out, + rows, + position}); + + const std::size_t padded = + static_cast(PaddedRows(rows)); + state.input_poison_observed = + state.input_poison_observed || + q->ContainsPoisonWords( + 0, + padded * + static_cast( + constants::kQueryDimension)) || + k->ContainsPoisonWords( + 0, + padded * + static_cast( + constants::kKvDimension)); + for (std::size_t head = 0; head < 8; ++head) { + for (std::size_t row = 0; + row < static_cast(rows); + ++row) { + const std::size_t cache_base = + ((head * 4096u) + + static_cast(position) + row) * + 128u; + state.input_poison_observed = + state.input_poison_observed || + v_cache->ContainsPoisonWords(cache_base, 128); + k_cache->FillWords(cache_base, 128, 0x3100u); + } + } + out->FillWords( + 0, + padded * + static_cast(constants::kQueryDimension), + 0x3200u); + state.attention_tensors.insert(out); + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status RecordingSynchronize( + ryzenai_corelib_stream_ptr stream) { + auto& state = State(); + auto* object = state.Object(stream); + if (object == nullptr || object->kind != ObjectKind::Stream) { + return ryzenai_corelib_status_bad_argument; + } + ++state.synchronize_calls; + state.events.emplace_back("synchronize"); + if (state.failure == FailurePoint::Synchronize) { + flm::test::SetLastErrorMessage("injected synchronize failure"); + return ryzenai_corelib_status_failure; + } + return ryzenai_corelib_status_success; +} + +void RecordingRelease(ryzenai_corelib_object_ptr object) { + auto& state = State(); + auto* value = state.Object(object); + CHECK(value != nullptr); + CHECK(!value->released); + value->released = true; + state.release_labels.push_back(value->label); +} + +template +void* FunctionAddress(Function function) { + return reinterpret_cast(function); +} + +std::shared_ptr ResolveRecordingCorelib( + RecordingState& state) { + g_state = &state; + auto resolver = flm::test::CompleteCorelibResolver(); + resolver["ryzenai_corelib_object_release"] = FunctionAddress( + static_cast( + &RecordingRelease)); + resolver["ryzenai_corelib_create_stream"] = FunctionAddress( + static_cast( + &RecordingCreateStream)); + resolver["ryzenai_corelib_stream_synchronize"] = FunctionAddress( + static_cast( + &RecordingSynchronize)); + resolver["ryzenai_corelib_create_device_tensor"] = FunctionAddress( + static_cast( + &RecordingCreateTensor)); + resolver["ryzenai_corelib_tensor_write"] = FunctionAddress( + static_cast( + &RecordingTensorWrite)); + resolver["ryzenai_corelib_tensor_read"] = FunctionAddress( + static_cast( + &RecordingTensorRead)); + resolver["ryzenai_corelib_tensor_get_byte_size"] = FunctionAddress( + static_cast( + &RecordingTensorGetByteSize)); + resolver["ryzenai_corelib_convert"] = FunctionAddress( + static_cast( + &RecordingConvert)); + resolver["ryzenai_corelib_convert_strided"] = FunctionAddress( + static_cast( + &RecordingConvertStrided)); + resolver["ryzenai_corelib_matmul_bf16_pad_shape"] = + FunctionAddress( + static_cast< + decltype(&::ryzenai_corelib_matmul_bf16_pad_shape)>( + &RecordingMatMulPadShape)); + resolver["ryzenai_corelib_ssmlp_bf16_pad_rows"] = + FunctionAddress( + static_cast< + decltype(&::ryzenai_corelib_ssmlp_bf16_pad_rows)>( + &RecordingSsMlpPadRows)); + resolver["ryzenai_corelib_flat_mha_bf16_pad_rows"] = + FunctionAddress( + static_cast< + decltype(&::ryzenai_corelib_flat_mha_bf16_pad_rows)>( + &RecordingMhaPadRows)); + resolver + ["ryzenai_corelib_matmul_bf16_weights_create_from_onnx_components"] = + FunctionAddress( + static_cast( + &RecordingMatMulWeightsCreate)); + resolver["ryzenai_corelib_matmul_bf16_weights_get_data"] = + FunctionAddress( + static_cast( + &RecordingMatMulWeightsGetData)); + resolver + ["ryzenai_corelib_ssmlp_bf16_weights_create_from_onnx_components"] = + FunctionAddress( + static_cast( + &RecordingSsMlpWeightsCreate)); + resolver["ryzenai_corelib_ssmlp_bf16_weights_get_data"] = + FunctionAddress( + static_cast( + &RecordingSsMlpWeightsGetData)); + resolver["ryzenai_corelib_matmul_bf16"] = FunctionAddress( + static_cast( + &RecordingMatMul)); + resolver["ryzenai_corelib_ssmlp_bf16"] = FunctionAddress( + static_cast( + &RecordingSsMlp)); + resolver["ryzenai_corelib_flat_mha_bf16"] = FunctionAddress( + static_cast( + &RecordingFlatMha)); + return CorelibApi::ResolveForTest( + [resolver = std::move(resolver)](std::string_view name) mutable + -> void* { + const auto found = resolver.find(std::string(name)); + return found == resolver.end() ? nullptr : found->second; + }); +} + +std::chrono::system_clock::time_point KnownStartTime() { + using namespace std::chrono; + return sys_days{year{2026} / September / day{1}}; +} + +FatalRecordStore MakeRecords(const std::filesystem::path& root) { + return FatalRecordStore( + root, + GetCurrentProcessId(), + KnownStartTime(), + [](DWORD) + -> std::optional { + return std::nullopt; + }); +} + +struct TerminationIntercept final {}; + +class EngineFixture final { +public: + explicit EngineFixture( + const SyntheticPackage& package, + std::uint32_t max_length = 4096) + : fatal_root_("fastflowlm-phi4-engine-fatal") { + flm::test::ResetFakeCorelib(); + api_ = ResolveRecordingCorelib(state); + runtime = CorelibRuntime::Create( + api_, + MakeRecords(fatal_root_.path()), + [this](unsigned int code) { + state.terminator_called = true; + state.termination_code = code; + throw TerminationIntercept{}; + }); + LM_Config config; + engine = std::make_unique( + std::move(config), + package.path(), + runtime, + max_length); + } + + ~EngineFixture() noexcept { + try { + engine.reset(); + if (runtime && runtime->state() == ProcessState::Healthy) { + runtime->ShutdownHealthy(); + } + } catch (...) { + std::terminate(); + } + if (g_state == &state) { + g_state = nullptr; + } + } + + void DestroyHealthy() { + CHECK(runtime->state() == ProcessState::Healthy); + engine.reset(); + CHECK(api_->live_object_count() == 0); + runtime->ShutdownHealthy(); + } + + json FatalRecord() const { + for (const auto& entry : + std::filesystem::directory_iterator(fatal_root_.path())) { + const auto name = entry.path().filename().string(); + if ( + name.starts_with("corelib-fatal-") && + name.ends_with(".json")) { + std::ifstream input(entry.path(), std::ios::binary); + return json::parse(input); + } + } + throw std::runtime_error("expected corelib fatal record"); + } + + RecordingState state; + std::shared_ptr runtime; + std::unique_ptr engine; + +private: + TempDirectory fatal_root_; + std::shared_ptr api_; +}; + +void CheckLayerOrder(const std::vector& events) { + const std::array expected{ + "matmul_q", + "matmul_k", + "matmul_v", + "synchronize", + "tensor_read_v", + "tensor_write_v_head_0", + "tensor_write_v_head_1", + "tensor_write_v_head_2", + "tensor_write_v_head_3", + "tensor_write_v_head_4", + "tensor_write_v_head_5", + "tensor_write_v_head_6", + "tensor_write_v_head_7", + "flat_mha", + "synchronize", + "matmul_o", + "synchronize", + "ssmlp", + "synchronize"}; + CHECK(events.size() == 32u * expected.size() + 3u); + for (std::size_t layer = 0; layer < 32; ++layer) { + CHECK(std::equal( + expected.begin(), + expected.end(), + events.begin() + layer * expected.size())); + } + const auto final = events.end() - 3; + CHECK(final[0] == "matmul_lm_head"); + CHECK(final[1] == "synchronize"); + CHECK(final[2] == "tensor_read_logits"); +} + +void CheckDistinctAndPingPong(const RecordingState& state) { + CHECK(state.ssmlp_calls.size() == 32); + FakeTensor* first_normalized = + state.ssmlp_calls.front().normalized; + FakeTensor* second_normalized = + state.ssmlp_calls.at(1).normalized; + CHECK(first_normalized != second_normalized); + for (std::size_t layer = 0; + layer < state.ssmlp_calls.size(); + ++layer) { + const auto& call = state.ssmlp_calls[layer]; + CHECK(call.input != call.residual); + CHECK(call.input != call.skip_sum); + CHECK(call.input != call.normalized); + CHECK(call.residual != call.skip_sum); + CHECK(call.residual != call.normalized); + CHECK(call.skip_sum != call.normalized); + CHECK( + call.normalized == + (layer % 2 == 0 ? first_normalized : second_normalized)); + if (layer + 1 < state.ssmlp_calls.size()) { + CHECK( + state.ssmlp_calls[layer + 1].input == + call.normalized); + CHECK( + state.ssmlp_calls[layer + 1].residual == + call.skip_sum); + } + } +} + +void CheckActualPersistentTail( + const std::set& tensors, + std::size_t live_prefix_words) { + CHECK(!tensors.empty()); + for (const auto* tensor : tensors) { + CHECK(tensor->ReadWord(live_prefix_words) == kPoison); + } +} + +void PoisonActualPersistentTail( + const std::set& tensors, + std::size_t first_word, + std::size_t word_count) { + for (auto* tensor : tensors) { + tensor->FillWords(first_word, word_count, kPoison); + } +} + +void CheckMetrics( + const Phi4Aie4Metrics& metrics, + std::uint64_t passes) { + CHECK(metrics.model_load_ns > 0); + CHECK(metrics.weight_pack_ns > 0); + CHECK(metrics.packed_weight_bytes == + 129u * 17u + 32u * 29u); + CHECK(metrics.mapped_source_bytes == kDataBytes); + CHECK(metrics.kv_bytes == 536870912u); + CHECK(metrics.scratch_bytes > 0); + CHECK(metrics.device_tensor_create_count == 76); + CHECK(metrics.weight_create_count == 161); + CHECK(metrics.dispatch_count == passes * 193u); + CHECK(metrics.synchronize_count == passes * 129u); + CHECK(metrics.v_read_calls == passes * 32u); + CHECK(metrics.v_write_calls == passes * 32u * 8u); + CHECK(metrics.v_bytes > 0); + CHECK(metrics.v_scatter_ns > 0); +} + +void TestOrderBuffersTailsStateAndMetrics( + const SyntheticPackage& package) { + EngineFixture fixture(package); + CHECK(fixture.state.load_thread_consistent); + CHECK(fixture.state.load_thread.has_value()); + CHECK(fixture.state.matmul_weight_count == 129); + CHECK(fixture.state.ssmlp_weight_count == 32); + CHECK(fixture.state.tensors.size() == 76); + + fixture.state.ResetExecutionRecords(); + std::vector prompt{1, 2, 3}; + buffer logits = fixture.engine->prefill(prompt); + CHECK(logits.size() == 200064); + CHECK(logits.is_owner()); + CHECK(fixture.engine->get_current_context_length() == 3); + CHECK(fixture.state.synchronize_calls == 129); + CheckLayerOrder(fixture.state.events); + CheckDistinctAndPingPong(fixture.state); + CHECK(fixture.state.stage_writes.size() == 2); + const std::size_t padded_hidden_bytes = + static_cast(PaddedRows(3)) * + static_cast(constants::kHiddenSize) * + sizeof(std::uint16_t); + CHECK(fixture.state.stage_writes[0].size == + padded_hidden_bytes); + CHECK(fixture.state.stage_writes[1].size == + padded_hidden_bytes); + CHECK(!fixture.state.input_poison_observed); + CHECK(!fixture.state.host_read_poison_observed); + CHECK(!fixture.state.cache_publish_poison_observed); + + const std::size_t hidden_tail = + static_cast(PaddedRows(3)) * + static_cast(constants::kHiddenSize); + const std::size_t kv_tail = + static_cast(PaddedRows(3)) * + static_cast(constants::kKvDimension); + CheckActualPersistentTail( + fixture.state.q_tensors, + hidden_tail); + CheckActualPersistentTail( + fixture.state.k_tensors, + kv_tail); + CheckActualPersistentTail( + fixture.state.attention_tensors, + hidden_tail); + CheckActualPersistentTail( + fixture.state.skip_sum_tensors, + hidden_tail); + CheckActualPersistentTail( + fixture.state.normalized_tensors, + hidden_tail); + CHECK(fixture.state.staged_hidden->ReadWord(hidden_tail) == + kPoison); + CHECK(fixture.state.staged_residual->ReadWord(hidden_tail) == + kPoison); + +#if defined(DEV_BUILD) + fixture.state.events.clear(); + const Phi4DebugSnapshot snapshot = + fixture.engine->debug_snapshot(); + CHECK(snapshot.live_rows == 3); + CHECK(snapshot.position == 3); + CHECK(snapshot.layer0_k.size() == 8u * 3u * 128u); + CHECK(snapshot.layer0_v.size() == 8u * 3u * 128u); + CHECK(snapshot.layer31_k.size() == 8u * 3u * 128u); + CHECK(snapshot.layer31_v.size() == 8u * 3u * 128u); + CHECK(snapshot.last_hidden.size() == 3072); + CHECK(snapshot.logits.size() == 200064); + CHECK(std::none_of( + snapshot.layer0_k.begin(), + snapshot.layer0_k.end(), + [](std::uint16_t value) { return value == kPoison; })); + CHECK(std::none_of( + snapshot.layer31_v.begin(), + snapshot.layer31_v.end(), + [](std::uint16_t value) { return value == kPoison; })); +#endif + + CheckMetrics(fixture.engine->metrics(), 1); + + const std::size_t one_hidden_row = + static_cast(constants::kHiddenSize); + const std::size_t one_kv_row = + static_cast(constants::kKvDimension); + PoisonActualPersistentTail( + fixture.state.q_tensors, + one_hidden_row, + hidden_tail - one_hidden_row); + PoisonActualPersistentTail( + fixture.state.k_tensors, + one_kv_row, + kv_tail - one_kv_row); + PoisonActualPersistentTail( + fixture.state.attention_tensors, + one_hidden_row, + hidden_tail - one_hidden_row); + PoisonActualPersistentTail( + fixture.state.skip_sum_tensors, + one_hidden_row, + hidden_tail - one_hidden_row); + PoisonActualPersistentTail( + fixture.state.normalized_tensors, + one_hidden_row, + hidden_tail - one_hidden_row); + fixture.state.staged_hidden->FillWords( + one_hidden_row, + hidden_tail - one_hidden_row, + kPoison); + fixture.state.staged_residual->FillWords( + one_hidden_row, + hidden_tail - one_hidden_row, + kPoison); + + fixture.state.ResetExecutionRecords(); + std::vector suffix{4, 5, 6}; + buffer suffix_logits = fixture.engine->prefill(suffix); + CHECK(suffix_logits.is_owner()); + CHECK(fixture.engine->get_current_context_length() == 6); + CHECK(fixture.state.mha_calls.size() == 3u * 32u); + for (std::size_t pass = 0; pass < 3; ++pass) { + for (std::size_t layer = 0; layer < 32; ++layer) { + const auto& call = + fixture.state.mha_calls[pass * 32 + layer]; + CHECK(call.rows == 1); + CHECK(call.position == 3 + static_cast(pass)); + } + } + CHECK(fixture.state.synchronize_calls == 3u * 129u); + CHECK(!fixture.state.input_poison_observed); + CHECK(!fixture.state.host_read_poison_observed); + CHECK(!fixture.state.cache_publish_poison_observed); + CheckActualPersistentTail( + fixture.state.q_tensors, + one_hidden_row); + CheckActualPersistentTail( + fixture.state.k_tensors, + one_kv_row); + CheckActualPersistentTail( + fixture.state.attention_tensors, + one_hidden_row); + CheckActualPersistentTail( + fixture.state.skip_sum_tensors, + one_hidden_row); + CheckActualPersistentTail( + fixture.state.normalized_tensors, + one_hidden_row); + CHECK( + fixture.state.staged_hidden->ReadWord(one_hidden_row) == + kPoison); + CHECK( + fixture.state.staged_residual->ReadWord(one_hidden_row) == + kPoison); + + fixture.state.ResetExecutionRecords(); + buffer forward_logits = fixture.engine->forward(7); + CHECK(forward_logits.is_owner()); + CHECK(fixture.engine->get_current_context_length() == 7); + CHECK(fixture.state.mha_calls.size() == 32); + CHECK(std::all_of( + fixture.state.mha_calls.begin(), + fixture.state.mha_calls.end(), + [](const MhaCall& call) { + return call.rows == 1 && call.position == 6; + })); + + CHECK(fixture.engine->checkpoint() == 7); + (void)fixture.engine->forward(8); + CHECK(fixture.engine->get_current_context_length() == 8); + CHECK(fixture.engine->restore() == 7); + CHECK(fixture.engine->get_current_context_length() == 7); + fixture.engine->set_context_length(7); + CheckThrowsContains( + [&] { fixture.engine->set_context_length(6); }, + "current"); + + fixture.engine->update_max_length(7); + CheckThrowsContains( + [&] { (void)fixture.engine->forward(9); }, + "maximum"); + CheckThrowsContains( + [&] { fixture.engine->update_max_length(0); }, + "1..4096"); + CheckThrowsContains( + [&] { fixture.engine->update_max_length(4097); }, + "1..4096"); + CheckThrowsContains( + [&] { fixture.engine->update_max_length(6); }, + "current"); + fixture.engine->update_max_length(4096); + + CheckThrowsContains( + [&] { (void)fixture.engine->get_k_cache(0, 0); }, + "unsupported"); + CheckThrowsContains( + [&] { (void)fixture.engine->get_v_cache(0, 0); }, + "unsupported"); + alignas(Q4NX) std::array q4nx_storage{}; + auto& q4nx = + *reinterpret_cast(q4nx_storage.data()); + CheckThrowsContains( + [&] { fixture.engine->load_weights(q4nx); }, + "unsupported"); + + fixture.engine->clear_context(); + CHECK(fixture.engine->get_current_context_length() == 0); + CheckThrowsContains( + [&] { (void)fixture.engine->restore(); }, + "checkpoint"); + + fixture.state.ResetExecutionRecords(); + std::vector one_token{10}; + (void)fixture.engine->prefill(one_token); + CHECK(fixture.engine->get_current_context_length() == 1); + CHECK(std::all_of( + fixture.state.mha_calls.begin(), + fixture.state.mha_calls.end(), + [](const MhaCall& call) { + return call.rows == 1 && call.position == 0; + })); + CHECK(fixture.state.stage_writes.size() == 2); + CHECK( + fixture.state.stage_writes[0].size == + static_cast(constants::kHiddenSize) * + sizeof(std::uint16_t)); + + fixture.engine->clear_context(); + fixture.state.ResetExecutionRecords(); + std::vector too_many(4097, 0); + CheckThrowsContains( + [&] { (void)fixture.engine->prefill(too_many); }, + "maximum"); + CHECK(fixture.state.matmul_calls.empty()); + CHECK(fixture.engine->get_current_context_length() == 0); + + fixture.state.ResetExecutionRecords(); + fixture.DestroyHealthy(); + CHECK(fixture.state.synchronize_calls == 1); + CHECK(!fixture.state.release_labels.empty()); + CHECK(fixture.state.release_labels.front() == "stream"); + const auto first_weight = std::find_if( + fixture.state.release_labels.begin(), + fixture.state.release_labels.end(), + [](const std::string& label) { + return label.starts_with("q_") || + (label.starts_with("k_") && + !label.starts_with("k_cache_")) || + (label.starts_with("v_") && + !label.starts_with("v_cache_")) || + label.starts_with("o_") || + label.starts_with("ssmlp_") || + label == "lm_head"; + }); + CHECK(first_weight != fixture.state.release_labels.end()); + CHECK(std::all_of( + fixture.state.release_labels.begin() + 1, + first_weight, + [](const std::string& label) { + return label.starts_with("tensor_") || + label.starts_with("k_cache_") || + label.starts_with("v_cache_"); + })); + CHECK(std::none_of( + first_weight, + fixture.state.release_labels.end(), + [](const std::string& label) { + return label.starts_with("tensor_") || + label.starts_with("k_cache_") || + label.starts_with("v_cache_"); + })); +} + +void TestRecoverablePreSubmitFailures( + const SyntheticPackage& package) { + EngineFixture fixture(package); + std::vector prompt{1, 2}; + + fixture.state.failure = FailurePoint::StageBadAlloc; + try { + (void)fixture.engine->prefill(prompt); + } catch (const std::bad_alloc&) { + } catch (...) { + throw std::runtime_error( + "pre-submit staging bad_alloc changed exception type"); + } + CHECK(fixture.engine->get_current_context_length() == 0); + CHECK(fixture.runtime->state() == ProcessState::Healthy); + CHECK(fixture.state.matmul_calls.empty()); + + fixture.state.failure = FailurePoint::FirstQ; + try { + (void)fixture.engine->prefill(prompt); + } catch (const CorelibError& error) { + CHECK(error.call == "q"); + CHECK(error.detail == "injected first-q failure"); + } + CHECK(fixture.engine->get_current_context_length() == 0); + CHECK(fixture.runtime->state() == ProcessState::Healthy); + + fixture.state.failure = FailurePoint::None; + (void)fixture.engine->prefill(prompt); + CHECK(fixture.engine->get_current_context_length() == 2); +} + +template +void CheckFatalFailure( + const SyntheticPackage& package, + FailurePoint failure, + CheckRecord&& check_record) { + EngineFixture fixture(package); + fixture.state.failure = failure; + std::vector prompt{1, 2}; + try { + (void)fixture.engine->prefill(prompt); + } catch (const TerminationIntercept&) { + } + CHECK(fixture.state.terminator_called); + CHECK(fixture.state.termination_code == 0xE0040001u); + CHECK(fixture.runtime->state() == ProcessState::Terminating); + CHECK(fixture.engine->get_current_context_length() == 0); + check_record(fixture.FatalRecord()); +} + +void TestIrrevocableFailurePolicies( + const SyntheticPackage& package) { + CheckFatalFailure( + package, + FailurePoint::KAfterQ, + [](const json& record) { + CHECK(record.at("status") == + ryzenai_corelib_status_failure); + CHECK(record.at("call") == "k"); + CHECK(record.at("detail") == "injected k failure"); + CHECK(record.at("phase") == "qkv"); + CHECK(record.at("layer") == 0); + CHECK(record.at("rows") == 2); + CHECK(record.at("position") == 0); + }); + + CheckFatalFailure( + package, + FailurePoint::Synchronize, + [](const json& record) { + CHECK(record.at("status") == + ryzenai_corelib_status_failure); + CHECK( + record.at("call") == + "ryzenai_corelib_stream_synchronize"); + CHECK( + record.at("detail") == + "injected synchronize failure"); + CHECK(record.at("phase") == "qkv"); + CHECK(record.at("layer") == 0); + CHECK(record.at("rows") == 2); + CHECK(record.at("position") == 0); + }); + + CheckFatalFailure( + package, + FailurePoint::ScatterBadAlloc, + [](const json& record) { + CHECK(record.at("status") == + ryzenai_corelib_status_failure); + CHECK(record.at("call") == "host_exception"); + CHECK( + record.at("detail") + .get() + .find("alloc") != std::string::npos); + CHECK(record.at("phase") == "v_scatter"); + CHECK(record.at("layer") == 0); + CHECK(record.at("rows") == 2); + CHECK(record.at("position") == 0); + }); + + CheckFatalFailure( + package, + FailurePoint::ScatterUnknown, + [](const json& record) { + CHECK(record.at("status") == + ryzenai_corelib_status_failure); + CHECK(record.at("call") == "unknown_exception"); + CHECK( + record.at("detail") == + "non-standard exception after the irrevocable boundary"); + CHECK(record.at("phase") == "v_scatter"); + CHECK(record.at("layer") == 0); + }); +} + +void TestSynchronizeFailureTerminatesWithoutSubmissionFlag() { + TempDirectory fatal_root("fastflowlm-phi4-policy-fatal"); + RecordingState state; + flm::test::ResetFakeCorelib(); + auto api = ResolveRecordingCorelib(state); + auto runtime = CorelibRuntime::Create( + api, + MakeRecords(fatal_root.path()), + [&state](unsigned int code) { + state.terminator_called = true; + state.termination_code = code; + throw TerminationIntercept{}; + }); + flm::corelib::StepSubmissionState submission; + CHECK(!submission.irrevocable()); + const CorelibError error( + ryzenai_corelib_status_failure, + "ryzenai_corelib_stream_synchronize", + "injected policy synchronize failure", + "failure"); + + try { + flm::phi4::testing::ApplyCorelibFailurePolicyForTest( + runtime, + error, + true, + submission, + "qkv", + 5, + 4, + 17); + } catch (const TerminationIntercept&) { + } + CHECK(state.terminator_called); + CHECK(state.termination_code == 0xE0040001u); + CHECK(runtime->state() == ProcessState::Terminating); + g_state = nullptr; +} + +static_assert(std::is_base_of_v); +static_assert(std::has_virtual_destructor_v); + +} // namespace + +int main() { + try { + SyntheticPackage package; + TestOrderBuffersTailsStateAndMetrics(package); + TestRecoverablePreSubmitFailures(package); + TestIrrevocableFailurePolicies(package); + TestSynchronizeFailureTerminatesWithoutSubmissionFlag(); + std::cout << "test_phi4_engine: PASS\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } catch (...) { + std::cerr << "unexpected non-standard exception\n"; + return 1; + } +} From ed2a996ebbfe1ef4e7fc8bf89685b1876cb117df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CChi?= Date: Tue, 1 Sep 2026 01:39:06 -0700 Subject: [PATCH 015/117] fix: handle divergent AIE4 padding safely Co-authored-by: Cursor --- src/common/corelib/phi4_corelib_aie4.cpp | 164 ++++++- src/include/models/phi4/phi4_corelib_aie4.hpp | 5 + .../phi4_corelib_aie4/test_phi4_engine.cpp | 451 ++++++++++++++++-- 3 files changed, 569 insertions(+), 51 deletions(-) diff --git a/src/common/corelib/phi4_corelib_aie4.cpp b/src/common/corelib/phi4_corelib_aie4.cpp index 06a9abf8..c462f58a 100644 --- a/src/common/corelib/phi4_corelib_aie4.cpp +++ b/src/common/corelib/phi4_corelib_aie4.cpp @@ -332,6 +332,13 @@ struct phi4_corelib_aie4::Impl final { embedding_fp32.resize(layer_elements); normalized_fp32.resize(layer_elements); bf16_staging.resize(layer_elements); + const std::size_t lm_head_elements = CheckedElements( + capacities.lm_head_rows, + constants::kHiddenSize, + "Phi-4 LM-head padding staging"); + padding_zero_staging.resize( + std::max(layer_elements, lm_head_elements), + 0); v_staging.reserve( CheckedElements( capacities.layer_rows, @@ -439,14 +446,6 @@ struct phi4_corelib_aie4::Impl final { } ~Impl() noexcept { -#if defined(FLM_CORELIB_TESTING) - if ( - runtime && - runtime->state() == corelib::ProcessState::Terminating) { - ReleaseResourcesWithoutSynchronization(); - return; - } -#endif if (!stream) { ReleaseResourcesWithoutSynchronization(); return; @@ -600,7 +599,97 @@ struct phi4_corelib_aie4::Impl final { } } - void StageInput(std::span token_ids) { + struct RunRowExtents { + std::int64_t query_projection; + std::int64_t kv_projection; + std::int64_t attention; + std::int64_t output_projection; + std::int64_t ssmlp; + std::int64_t lm_head; + + std::int64_t ProjectionInput() const noexcept { + return std::max(query_projection, kv_projection); + } + }; + + RunRowExtents RowsForRun(std::int64_t rows) { + RunRowExtents extents{ + shape_plan->RowsFor(RowUse::QueryProjection, rows), + shape_plan->RowsFor(RowUse::KvProjection, rows), + shape_plan->RowsFor(RowUse::Attention, rows), + shape_plan->RowsFor(RowUse::OutputProjection, rows), + shape_plan->RowsFor(RowUse::SsMlp, rows), + shape_plan->RowsFor(RowUse::LmHead, 1)}; + ++metrics.attention_extent_queries; + ++metrics.output_projection_extent_queries; + ++metrics.lm_head_extent_queries; + return extents; + } + + void WriteZeroRows( + corelib::UniqueTensor& tensor, + std::int64_t first_row, + std::int64_t row_count, + std::int64_t width, + std::string_view context) { + if (row_count <= 0) { + return; + } + if (first_row < 0) { + throw std::out_of_range( + "Phi-4 padding write has a negative row offset"); + } + const std::size_t offset = + first_row == 0 + ? 0 + : CheckedBytes( + first_row, + width, + sizeof(std::uint16_t), + context); + const std::size_t bytes = CheckedBytes( + row_count, + width, + sizeof(std::uint16_t), + context); + const std::size_t words = CheckedElements( + row_count, + width, + context); + if (words > padding_zero_staging.size()) { + throw std::out_of_range( + "Phi-4 padding write exceeds host staging capacity"); + } + api->Check( + api->functions().tensor_write( + tensor.get(), + padding_zero_staging.data(), + bytes, + offset), + kTensorWriteCall); + ++metrics.padding_write_calls; + metrics.padding_bytes += bytes; + } + + void BridgePadding( + corelib::UniqueTensor& tensor, + std::int64_t producer_rows, + std::int64_t consumer_rows, + std::int64_t width, + std::string_view context) { + if (consumer_rows > producer_rows) { + WriteZeroRows( + tensor, + producer_rows, + consumer_rows - producer_rows, + width, + context); + } + } + + void StageInput( + std::span token_ids, + const RunRowExtents& extents) { const auto rows = static_cast(token_ids.size()); const std::size_t live_elements = CheckedElements( @@ -631,9 +720,8 @@ struct phi4_corelib_aie4::Impl final { static_cast(constants::kRmsEpsilon), normalized_output); - const std::int64_t hidden_rows = std::max( - shape_plan->RowsFor(RowUse::QueryProjection, rows), - shape_plan->RowsFor(RowUse::KvProjection, rows)); + const std::int64_t hidden_rows = + extents.ProjectionInput(); StageBf16( *api, normalized_output, @@ -654,8 +742,7 @@ struct phi4_corelib_aie4::Impl final { 0), kTensorWriteCall); - const std::int64_t residual_rows = - shape_plan->RowsFor(RowUse::SsMlp, rows); + const std::int64_t residual_rows = extents.ssmlp; StageBf16( *api, embedding_output, @@ -743,7 +830,9 @@ struct phi4_corelib_aie4::Impl final { metrics.v_scatter_ns = v_metrics.nanoseconds; } - void PrepareLastHidden(std::int64_t rows) { + void PrepareLastHidden( + std::int64_t rows, + std::int64_t lm_head_rows) { const std::size_t row_bytes = CheckedBytes( 1, constants::kHiddenSize, @@ -751,6 +840,12 @@ struct phi4_corelib_aie4::Impl final { "Phi-4 last hidden"); const std::size_t source_offset = static_cast(rows - 1) * row_bytes; + WriteZeroRows( + lm_input_tensor, + 0, + lm_head_rows, + constants::kHiddenSize, + "Phi-4 LM-head input initialization"); api->Check( api->functions().tensor_read( current_hidden->get(), @@ -814,7 +909,8 @@ struct phi4_corelib_aie4::Impl final { ++metrics.synchronize_count; }; - StageInput(token_ids); + const RunRowExtents extents = RowsForRun(rows); + StageInput(token_ids, extents); for (int layer = 0; layer < constants::kLayerCount; @@ -853,6 +949,18 @@ struct phi4_corelib_aie4::Impl final { position); active_phase = "flat_mha"; + BridgePadding( + query_tensor, + extents.query_projection, + extents.attention, + constants::kQueryDimension, + "Phi-4 query-to-attention padding"); + BridgePadding( + key_tensor, + extents.kv_projection, + extents.attention, + constants::kKvDimension, + "Phi-4 key-to-attention padding"); checked_submit( SubmitMha( static_cast(layer), @@ -862,6 +970,12 @@ struct phi4_corelib_aie4::Impl final { checked_synchronize(); active_phase = "o"; + BridgePadding( + attention_tensor, + extents.attention, + extents.output_projection, + constants::kQueryDimension, + "Phi-4 attention-to-output padding"); checked_submit( SubmitMatMul( attention_tensor, @@ -872,6 +986,12 @@ struct phi4_corelib_aie4::Impl final { checked_synchronize(); active_phase = "ssmlp"; + BridgePadding( + *current_hidden, + extents.output_projection, + extents.ssmlp, + constants::kHiddenSize, + "Phi-4 output-to-SSMLP padding"); checked_submit( SubmitSsMlp( *current_hidden, @@ -882,13 +1002,22 @@ struct phi4_corelib_aie4::Impl final { rows), "ssmlp"); checked_synchronize(); + if (layer + 1 < constants::kLayerCount) { + active_phase = "next_layer_padding"; + BridgePadding( + *next_hidden, + extents.ssmlp, + extents.ProjectionInput(), + constants::kHiddenSize, + "Phi-4 SSMLP-to-projection padding"); + } std::swap(current_hidden, next_hidden); std::swap(current_residual, next_skip_sum); } active_layer.reset(); active_phase = "lm_head"; - PrepareLastHidden(rows); + PrepareLastHidden(rows, extents.lm_head); checked_submit( SubmitMatMul( lm_input_tensor, @@ -1021,6 +1150,7 @@ struct phi4_corelib_aie4::Impl final { std::vector embedding_fp32; std::vector normalized_fp32; std::vector bf16_staging; + std::vector padding_zero_staging; std::vector v_staging; std::vector last_hidden_staging; diff --git a/src/include/models/phi4/phi4_corelib_aie4.hpp b/src/include/models/phi4/phi4_corelib_aie4.hpp index ba12833d..3f4953e5 100644 --- a/src/include/models/phi4/phi4_corelib_aie4.hpp +++ b/src/include/models/phi4/phi4_corelib_aie4.hpp @@ -35,6 +35,11 @@ struct Phi4Aie4Metrics { std::uint64_t weight_create_count = 0; std::uint64_t dispatch_count = 0; std::uint64_t synchronize_count = 0; + std::uint64_t padding_write_calls = 0; + std::uint64_t padding_bytes = 0; + std::uint64_t attention_extent_queries = 0; + std::uint64_t output_projection_extent_queries = 0; + std::uint64_t lm_head_extent_queries = 0; std::uint64_t v_read_calls = 0; std::uint64_t v_write_calls = 0; std::uint64_t v_bytes = 0; diff --git a/src/test/phi4_corelib_aie4/test_phi4_engine.cpp b/src/test/phi4_corelib_aie4/test_phi4_engine.cpp index 44040bb6..fa90d74a 100644 --- a/src/test/phi4_corelib_aie4/test_phi4_engine.cpp +++ b/src/test/phi4_corelib_aie4/test_phi4_engine.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -62,6 +63,17 @@ std::int64_t PaddedRows(std::int64_t rows) { return rows == 1 ? 1 : ((rows + 3) / 4) * 4; } +using PadRowsFunction = + std::function; + +struct PaddingFunctions { + PadRowsFunction query_projection = PaddedRows; + PadRowsFunction kv_projection = PaddedRows; + PadRowsFunction lm_head = PaddedRows; + PadRowsFunction ssmlp = PaddedRows; + PadRowsFunction attention = PaddedRows; +}; + class TempDirectory final { public: explicit TempDirectory(std::string_view stem) { @@ -672,6 +684,7 @@ struct FakeTensor final : FakeObject { ryzenai_corelib_data_type data_type; std::vector shape; std::size_t byte_size; + std::string last_producer; std::unordered_map> pages; }; @@ -732,6 +745,7 @@ enum class FailurePoint { FirstQ, KAfterQ, Synchronize, + DestructionSynchronize, StageBadAlloc, ScatterBadAlloc, ScatterUnknown @@ -749,12 +763,14 @@ struct RecordingState { std::vector ssmlp_calls; std::vector mha_calls; std::vector stage_writes; + PaddingFunctions padding; std::set q_tensors; std::set k_tensors; std::set v_tensors; std::set attention_tensors; std::set skip_sum_tensors; std::set normalized_tensors; + FakeTensor* lm_input = nullptr; FakeTensor* staged_hidden = nullptr; FakeTensor* staged_residual = nullptr; FailurePoint failure = FailurePoint::None; @@ -766,6 +782,8 @@ struct RecordingState { bool input_poison_observed = false; bool host_read_poison_observed = false; bool cache_publish_poison_observed = false; + bool destruction_failure_returned = false; + std::filesystem::path post_failure_marker; bool terminator_called = false; unsigned int termination_code = 0; std::optional load_thread; @@ -892,6 +910,29 @@ int WeightLayer(const std::string& label) { return std::stoi(label.substr(label.find('_') + 1)); } +std::int64_t MatMulPaddedRows( + const RecordingState& state, + const ryzenai_corelib_matmul_bf16_weights_desc& desc, + std::int64_t rows) { + if ( + desc.k == constants::kHiddenSize && + desc.n == constants::kVocabularySize) { + return state.padding.lm_head(rows); + } + if ( + desc.k == constants::kHiddenSize && + desc.n == constants::kKvDimension) { + return state.padding.kv_projection(rows); + } + if ( + desc.k == constants::kHiddenSize && + desc.n == constants::kQueryDimension) { + return state.padding.query_projection(rows); + } + throw std::runtime_error( + "engine fake received an unknown matmul shape"); +} + ryzenai_corelib_status RecordingConvert( ryzenai_corelib_data_type source_type, const void* source, @@ -954,14 +995,20 @@ ryzenai_corelib_status RecordingConvertStrided( ryzenai_corelib_status RecordingMatMulPadShape( std::int64_t* m, - std::int64_t*, - std::int64_t*, + std::int64_t* k, + std::int64_t* n, std::uint32_t) { - State().ObserveLoadThread(); - if (m == nullptr || *m <= 0) { + auto& state = State(); + state.ObserveLoadThread(); + if ( + m == nullptr || k == nullptr || n == nullptr || + *m <= 0 || *k <= 0 || *n <= 0) { return ryzenai_corelib_status_bad_argument; } - *m = PaddedRows(*m); + ryzenai_corelib_matmul_bf16_weights_desc desc{}; + desc.k = *k; + desc.n = *n; + *m = MatMulPaddedRows(state, desc, *m); return ryzenai_corelib_status_success; } @@ -970,22 +1017,24 @@ ryzenai_corelib_status RecordingSsMlpPadRows( std::int64_t, std::int64_t, std::uint32_t) { - State().ObserveLoadThread(); + auto& state = State(); + state.ObserveLoadThread(); if (m == nullptr || *m <= 0) { return ryzenai_corelib_status_bad_argument; } - *m = PaddedRows(*m); + *m = state.padding.ssmlp(*m); return ryzenai_corelib_status_success; } ryzenai_corelib_status RecordingMhaPadRows( std::int64_t* m, const ryzenai_corelib_flat_mha_bf16_desc*) { - State().ObserveLoadThread(); + auto& state = State(); + state.ObserveLoadThread(); if (m == nullptr || *m <= 0) { return ryzenai_corelib_status_bad_argument; } - *m = PaddedRows(*m); + *m = state.padding.attention(*m); return ryzenai_corelib_status_success; } @@ -1018,6 +1067,9 @@ ryzenai_corelib_status RecordingCreateTensor( TensorBytes(data_type, dimensions), "tensor_" + std::to_string(ordinal)); state.tensors.push_back(tensor); + if (ordinal == 8) { + state.lm_input = tensor; + } *out = tensor; return ryzenai_corelib_status_success; } @@ -1071,6 +1123,20 @@ ryzenai_corelib_status RecordingTensorWrite( const std::size_t head = offset / head_pitch_bytes; state.events.push_back( "tensor_write_v_head_" + std::to_string(head)); + } else if ( + state.active_layer >= 0 && + (value->last_producer == "q" || + value->last_producer == "k" || + value->last_producer == "attention" || + value->last_producer == "o" || + value->last_producer == "normalized")) { + state.events.push_back( + "padding_write_" + value->last_producer); + } else if ( + value == state.lm_input && + value->last_producer.empty()) { + state.events.emplace_back("padding_write_lm_input"); + value->last_producer = "lm_initialized"; } else if ( value->data_type == ryzenai_corelib_data_type_bf16 && value->shape == @@ -1115,9 +1181,13 @@ ryzenai_corelib_status RecordingTensorRead( state.host_read_poison_observed = true; } } else if ( - value->shape == - std::vector{1, 200064}) { + value->shape.size() == 2 && + value->shape[1] == constants::kVocabularySize) { state.events.emplace_back("tensor_read_logits"); + state.active_layer = -1; + if (state.lm_input != nullptr) { + state.lm_input->last_producer.clear(); + } } state.host_read_poison_observed = @@ -1232,14 +1302,16 @@ ryzenai_corelib_status RecordingMatMul( return ryzenai_corelib_status_failure; } + const std::int64_t padded_rows = + MatMulPaddedRows(state, weight->desc, rows); const std::size_t input_words = - static_cast(PaddedRows(rows)) * + static_cast(padded_rows) * static_cast(weight->desc.k); state.input_poison_observed = state.input_poison_observed || input_tensor->ContainsPoisonWords(0, input_words); const std::size_t output_words = - static_cast(PaddedRows(rows)) * + static_cast(padded_rows) * static_cast(weight->desc.n); output_tensor->FillWords( 0, @@ -1251,6 +1323,7 @@ ryzenai_corelib_status RecordingMatMul( std::max(layer, 0) * 8 + static_cast( state.matmul_calls.size() % 8)))); + output_tensor->last_producer = projection; if (projection == "q") { state.q_tensors.insert(output_tensor); @@ -1296,7 +1369,7 @@ ryzenai_corelib_status RecordingSsMlp( rows}); const std::size_t words = - static_cast(PaddedRows(rows)) * + static_cast(state.padding.ssmlp(rows)) * static_cast(constants::kHiddenSize); state.input_poison_observed = state.input_poison_observed || @@ -1304,6 +1377,8 @@ ryzenai_corelib_status RecordingSsMlp( residual_tensor->ContainsPoisonWords(0, words); skip_tensor->FillWords(0, words, 0x2200u); normalized_tensor->FillWords(0, words, 0x2300u); + skip_tensor->last_producer = "skip_sum"; + normalized_tensor->last_producer = "normalized"; state.skip_sum_tensors.insert(skip_tensor); state.normalized_tensors.insert(normalized_tensor); return ryzenai_corelib_status_success; @@ -1360,7 +1435,7 @@ ryzenai_corelib_status RecordingFlatMha( position}); const std::size_t padded = - static_cast(PaddedRows(rows)); + static_cast(state.padding.attention(rows)); state.input_poison_observed = state.input_poison_observed || q->ContainsPoisonWords( @@ -1392,6 +1467,7 @@ ryzenai_corelib_status RecordingFlatMha( padded * static_cast(constants::kQueryDimension), 0x3200u); + out->last_producer = "attention"; state.attention_tensors.insert(out); return ryzenai_corelib_status_success; } @@ -1405,15 +1481,33 @@ ryzenai_corelib_status RecordingSynchronize( } ++state.synchronize_calls; state.events.emplace_back("synchronize"); - if (state.failure == FailurePoint::Synchronize) { + if ( + state.failure == FailurePoint::Synchronize || + state.failure == FailurePoint::DestructionSynchronize) { + state.destruction_failure_returned = + state.failure == FailurePoint::DestructionSynchronize; flm::test::SetLastErrorMessage("injected synchronize failure"); return ryzenai_corelib_status_failure; } return ryzenai_corelib_status_success; } +void MarkPostDestructionFailure(std::string_view action) { + auto& state = State(); + if ( + !state.destruction_failure_returned || + state.post_failure_marker.empty()) { + return; + } + std::ofstream output( + state.post_failure_marker, + std::ios::binary | std::ios::app); + output << action << '\n'; +} + void RecordingRelease(ryzenai_corelib_object_ptr object) { auto& state = State(); + MarkPostDestructionFailure("release"); auto* value = state.Object(object); CHECK(value != nullptr); CHECK(!value->released); @@ -1421,6 +1515,10 @@ void RecordingRelease(ryzenai_corelib_object_ptr object) { state.release_labels.push_back(value->label); } +void RecordingCleanup() { + MarkPostDestructionFailure("cleanup"); +} + template void* FunctionAddress(Function function) { return reinterpret_cast(function); @@ -1433,6 +1531,9 @@ std::shared_ptr ResolveRecordingCorelib( resolver["ryzenai_corelib_object_release"] = FunctionAddress( static_cast( &RecordingRelease)); + resolver["ryzenai_corelib_cleanup"] = FunctionAddress( + static_cast( + &RecordingCleanup)); resolver["ryzenai_corelib_create_stream"] = FunctionAddress( static_cast( &RecordingCreateStream)); @@ -1527,15 +1628,31 @@ FatalRecordStore MakeRecords(const std::filesystem::path& root) { }); } +json ReadFatalRecord(const std::filesystem::path& root) { + for (const auto& entry : + std::filesystem::directory_iterator(root)) { + const auto name = entry.path().filename().string(); + if ( + name.starts_with("corelib-fatal-") && + name.ends_with(".json")) { + std::ifstream input(entry.path(), std::ios::binary); + return json::parse(input); + } + } + throw std::runtime_error("expected corelib fatal record"); +} + struct TerminationIntercept final {}; class EngineFixture final { public: explicit EngineFixture( const SyntheticPackage& package, - std::uint32_t max_length = 4096) + std::uint32_t max_length = 4096, + PaddingFunctions padding = {}) : fatal_root_("fastflowlm-phi4-engine-fatal") { flm::test::ResetFakeCorelib(); + state.padding = std::move(padding); api_ = ResolveRecordingCorelib(state); runtime = CorelibRuntime::Create( api_, @@ -1555,6 +1672,15 @@ class EngineFixture final { ~EngineFixture() noexcept { try { + if ( + runtime && + runtime->state() == ProcessState::Terminating) { + (void)engine.release(); + if (g_state == &state) { + g_state = nullptr; + } + return; + } engine.reset(); if (runtime && runtime->state() == ProcessState::Healthy) { runtime->ShutdownHealthy(); @@ -1575,17 +1701,7 @@ class EngineFixture final { } json FatalRecord() const { - for (const auto& entry : - std::filesystem::directory_iterator(fatal_root_.path())) { - const auto name = entry.path().filename().string(); - if ( - name.starts_with("corelib-fatal-") && - name.ends_with(".json")) { - std::ifstream input(entry.path(), std::ios::binary); - return json::parse(input); - } - } - throw std::runtime_error("expected corelib fatal record"); + return ReadFatalRecord(fatal_root_.path()); } RecordingState state; @@ -1618,17 +1734,18 @@ void CheckLayerOrder(const std::vector& events) { "synchronize", "ssmlp", "synchronize"}; - CHECK(events.size() == 32u * expected.size() + 3u); + CHECK(events.size() == 32u * expected.size() + 4u); for (std::size_t layer = 0; layer < 32; ++layer) { CHECK(std::equal( expected.begin(), expected.end(), events.begin() + layer * expected.size())); } - const auto final = events.end() - 3; - CHECK(final[0] == "matmul_lm_head"); - CHECK(final[1] == "synchronize"); - CHECK(final[2] == "tensor_read_logits"); + const auto final = events.end() - 4; + CHECK(final[0] == "padding_write_lm_input"); + CHECK(final[1] == "matmul_lm_head"); + CHECK(final[2] == "synchronize"); + CHECK(final[3] == "tensor_read_logits"); } void CheckDistinctAndPingPong(const RecordingState& state) { @@ -1694,6 +1811,15 @@ void CheckMetrics( CHECK(metrics.weight_create_count == 161); CHECK(metrics.dispatch_count == passes * 193u); CHECK(metrics.synchronize_count == passes * 129u); + CHECK(metrics.padding_write_calls == passes); + CHECK( + metrics.padding_bytes == + passes * + static_cast(constants::kHiddenSize) * + sizeof(std::uint16_t)); + CHECK(metrics.attention_extent_queries == passes); + CHECK(metrics.output_projection_extent_queries == passes); + CHECK(metrics.lm_head_extent_queries == passes); CHECK(metrics.v_read_calls == passes * 32u); CHECK(metrics.v_write_calls == passes * 32u * 8u); CHECK(metrics.v_bytes > 0); @@ -1972,6 +2098,136 @@ void TestOrderBuffersTailsStateAndMetrics( })); } +std::size_t EventCount( + const RecordingState& state, + std::string_view event) { + return static_cast(std::count( + state.events.begin(), + state.events.end(), + event)); +} + +std::int64_t AlignRows(std::int64_t rows, std::int64_t alignment) { + return ((rows + alignment - 1) / alignment) * alignment; +} + +void CheckNoStaleConsumption(const RecordingState& state) { + CHECK(!state.input_poison_observed); + CHECK(!state.host_read_poison_observed); + CHECK(!state.cache_publish_poison_observed); +} + +void TestDivergentPaddingGrids(const SyntheticPackage& package) { + { + PaddingFunctions padding; + padding.query_projection = + [](std::int64_t rows) { return AlignRows(rows, 4); }; + padding.kv_projection = padding.query_projection; + padding.attention = + [](std::int64_t rows) { return AlignRows(rows, 8); }; + padding.ssmlp = padding.attention; + padding.lm_head = + [](std::int64_t rows) { return AlignRows(rows, 8); }; + EngineFixture fixture(package, 4096, std::move(padding)); + + fixture.state.ResetExecutionRecords(); + std::vector prompt{1, 2, 3}; + (void)fixture.engine->prefill(prompt); + CheckNoStaleConsumption(fixture.state); + CHECK(EventCount(fixture.state, "padding_write_q") == 32); + CHECK(EventCount(fixture.state, "padding_write_k") == 32); + CHECK( + EventCount(fixture.state, "padding_write_attention") == + 0); + CHECK(EventCount(fixture.state, "padding_write_o") == 32); + CHECK( + EventCount(fixture.state, "padding_write_normalized") == + 0); + CHECK( + EventCount(fixture.state, "padding_write_lm_input") == + 1); + CHECK(fixture.state.stage_writes.size() == 2); + CHECK( + fixture.state.stage_writes[0].size == + 4u * static_cast(constants::kHiddenSize) * + sizeof(std::uint16_t)); + CHECK( + fixture.state.stage_writes[1].size == + 8u * static_cast(constants::kHiddenSize) * + sizeof(std::uint16_t)); + const auto& metrics = fixture.engine->metrics(); + CHECK(metrics.padding_write_calls == 97); + CHECK(metrics.attention_extent_queries == 1); + CHECK(metrics.output_projection_extent_queries == 1); + CHECK(metrics.lm_head_extent_queries == 1); + CHECK( + metrics.padding_bytes == + 32u * 4u * + static_cast( + 2 * constants::kHiddenSize + + constants::kKvDimension) * + sizeof(std::uint16_t) + + 8u * + static_cast( + constants::kHiddenSize) * + sizeof(std::uint16_t)); + } + + { + PaddingFunctions padding; + padding.query_projection = + [](std::int64_t rows) { return AlignRows(rows, 8); }; + padding.kv_projection = padding.query_projection; + padding.attention = + [](std::int64_t rows) { return AlignRows(rows, 4); }; + padding.ssmlp = padding.attention; + padding.lm_head = + [](std::int64_t rows) { return AlignRows(rows, 8); }; + EngineFixture fixture(package, 4096, std::move(padding)); + + fixture.state.ResetExecutionRecords(); + std::vector prompt{1, 2, 3}; + (void)fixture.engine->prefill(prompt); + CheckNoStaleConsumption(fixture.state); + CHECK(EventCount(fixture.state, "padding_write_q") == 0); + CHECK(EventCount(fixture.state, "padding_write_k") == 0); + CHECK( + EventCount(fixture.state, "padding_write_attention") == + 32); + CHECK(EventCount(fixture.state, "padding_write_o") == 0); + CHECK( + EventCount(fixture.state, "padding_write_normalized") == + 31); + CHECK( + EventCount(fixture.state, "padding_write_lm_input") == + 1); + CHECK(fixture.state.stage_writes.size() == 2); + CHECK( + fixture.state.stage_writes[0].size == + 8u * static_cast(constants::kHiddenSize) * + sizeof(std::uint16_t)); + CHECK( + fixture.state.stage_writes[1].size == + 4u * static_cast(constants::kHiddenSize) * + sizeof(std::uint16_t)); + const auto& metrics = fixture.engine->metrics(); + CHECK(metrics.padding_write_calls == 64); + CHECK(metrics.attention_extent_queries == 1); + CHECK(metrics.output_projection_extent_queries == 1); + CHECK(metrics.lm_head_extent_queries == 1); + CHECK( + metrics.padding_bytes == + 63u * 4u * + static_cast( + constants::kHiddenSize) * + sizeof(std::uint16_t) + + 8u * + static_cast( + constants::kHiddenSize) * + sizeof(std::uint16_t)); + } +} + void TestRecoverablePreSubmitFailures( const SyntheticPackage& package) { EngineFixture fixture(package); @@ -2128,16 +2384,143 @@ void TestSynchronizeFailureTerminatesWithoutSubmissionFlag() { g_state = nullptr; } +std::filesystem::path CurrentExecutablePath() { + std::vector buffer(32768); + const DWORD length = GetModuleFileNameW( + nullptr, + buffer.data(), + static_cast(buffer.size())); + if (length == 0 || length == buffer.size()) { + throw std::runtime_error( + "failed to resolve engine-test executable path"); + } + return std::filesystem::path( + std::wstring(buffer.data(), length)); +} + +std::wstring QuoteProcessArgument( + const std::filesystem::path& argument) { + const std::wstring value = argument.wstring(); + if (value.find(L'"') != std::wstring::npos) { + throw std::runtime_error( + "engine-test child argument contains a quote"); + } + return L"\"" + value + L"\""; +} + +int RunDestructorFailureChild( + const std::filesystem::path& model_path, + const std::filesystem::path& fatal_root, + const std::filesystem::path& post_failure_marker) { + RecordingState state; + state.post_failure_marker = post_failure_marker; + flm::test::ResetFakeCorelib(); + auto api = ResolveRecordingCorelib(state); + auto runtime = CorelibRuntime::Create( + api, + MakeRecords(fatal_root), + [](unsigned int code) { + (void)TerminateProcess(GetCurrentProcess(), code); + ExitProcess(code); + }); + LM_Config config; + { + phi4_corelib_aie4 engine( + std::move(config), + model_path, + runtime, + 4096); + state.failure = FailurePoint::DestructionSynchronize; + } + return 3; +} + +void TestDestructorSynchronizeFailureChild( + const SyntheticPackage& package) { + TempDirectory fatal_root( + "fastflowlm-phi4-destruction-fatal"); + const auto marker = + fatal_root.path() / "post-failure-cleanup.txt"; + const auto executable = CurrentExecutablePath(); + std::wstring command = + QuoteProcessArgument(executable) + + L" --destructor-failure-child " + + QuoteProcessArgument(package.path()) + L" " + + QuoteProcessArgument(fatal_root.path()) + L" " + + QuoteProcessArgument(marker); + std::vector mutable_command( + command.begin(), + command.end()); + mutable_command.push_back(L'\0'); + + STARTUPINFOW startup{}; + startup.cb = sizeof(startup); + PROCESS_INFORMATION process{}; + if (!CreateProcessW( + nullptr, + mutable_command.data(), + nullptr, + nullptr, + FALSE, + 0, + nullptr, + nullptr, + &startup, + &process)) { + throw std::runtime_error( + "failed to start destruction-failure child"); + } + CloseHandle(process.hThread); + const DWORD wait_result = + WaitForSingleObject(process.hProcess, 120000); + if (wait_result != WAIT_OBJECT_0) { + (void)TerminateProcess(process.hProcess, 2); + CloseHandle(process.hProcess); + throw std::runtime_error( + "destruction-failure child did not exit"); + } + DWORD exit_code = 0; + const BOOL queried = + GetExitCodeProcess(process.hProcess, &exit_code); + CloseHandle(process.hProcess); + CHECK(queried != FALSE); + CHECK(exit_code == 0xE0040001u); + CHECK(!std::filesystem::exists(marker)); + + const json record = ReadFatalRecord(fatal_root.path()); + CHECK( + record.at("status") == + ryzenai_corelib_status_failure); + CHECK( + record.at("call") == + "ryzenai_corelib_stream_synchronize"); + CHECK(record.at("phase") == "destruction"); + CHECK(record.at("layer").is_null()); + CHECK(record.at("rows") == 0); + CHECK(record.at("position") == 0); +} + static_assert(std::is_base_of_v); static_assert(std::has_virtual_destructor_v); } // namespace -int main() { +int wmain(int argc, wchar_t* argv[]) { try { + if ( + argc == 5 && + std::wstring_view(argv[1]) == + L"--destructor-failure-child") { + return RunDestructorFailureChild( + argv[2], + argv[3], + argv[4]); + } SyntheticPackage package; TestOrderBuffersTailsStateAndMetrics(package); + TestDivergentPaddingGrids(package); TestRecoverablePreSubmitFailures(package); + TestDestructorSynchronizeFailureChild(package); TestIrrevocableFailurePolicies(package); TestSynchronizeFailureTerminatesWithoutSubmissionFlag(); std::cout << "test_phi4_engine: PASS\n"; From d6c1760dd39ecf979a6a5f3dd3fc8422097e1873 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CChi?= Date: Tue, 1 Sep 2026 02:19:50 -0700 Subject: [PATCH 016/117] feat: route Phi-4 AIE4 models through corelib Co-authored-by: Cursor --- src/CMakeLists.txt | 2 + src/common/AutoModel/automodel.cpp | 192 +++- src/common/AutoModel/modeling_phi4.cpp | 569 ++++++++++- src/common/corelib/phi4_corelib_aie4.cpp | 113 ++- src/include/AutoModel/automodel.hpp | 45 +- src/include/AutoModel/modeling_phi4.hpp | 60 +- src/include/models/phi4/phi4_corelib_aie4.hpp | 2 + .../models/phi4/phi4_corelib_aie4_tuning.hpp | 44 + src/test/phi4_corelib_aie4/CMakeLists.txt | 86 ++ .../phi4_corelib_aie4/test_phi4_engine.cpp | 51 + .../phi4_corelib_aie4/test_phi4_frontend.cpp | 929 ++++++++++++++++++ 11 files changed, 1999 insertions(+), 94 deletions(-) create mode 100644 src/include/models/phi4/phi4_corelib_aie4_tuning.hpp create mode 100644 src/test/phi4_corelib_aie4/test_phi4_frontend.cpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 418a3f73..2d7f3275 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -305,6 +305,8 @@ endif() add_executable(flm ${SOURCES} ${HEADERS}) if(FLM_ENABLE_CORELIB_AIE4) + target_compile_definitions(flm PRIVATE + FLM_ENABLE_CORELIB_AIE4=1) target_link_libraries(flm PRIVATE flm_corelib_aie4) endif() diff --git a/src/common/AutoModel/automodel.cpp b/src/common/AutoModel/automodel.cpp index 5df2f2d2..7a084773 100644 --- a/src/common/AutoModel/automodel.cpp +++ b/src/common/AutoModel/automodel.cpp @@ -8,6 +8,22 @@ #include "AutoModel/automodel.hpp" +ModelRequestError::ModelRequestError( + int http_code, + bool session_cleared, + std::string message) + : std::runtime_error(std::move(message)), + http_code_(http_code), + session_cleared_(session_cleared) {} + +int ModelRequestError::http_code() const noexcept { + return http_code_; +} + +bool ModelRequestError::session_cleared() const noexcept { + return session_cleared_; +} + AutoModel::AutoModel(flm_rt::device* npu_device_inst, std::string current_model) { this->npu_device_inst = npu_device_inst; this->current_model = current_model; @@ -116,17 +132,21 @@ void AutoModel::_shared_load_model(std::string model_path, json model_info, int header_print("FLM", "Model already loaded: " << this->model_path); return; } + this->_shared_initialize_model_state( + std::move(model_path), + std::move(model_info), + default_context_length); + this->_shared_initialize_legacy_npu(enable_preemption); +} +void AutoModel::_shared_initialize_model_state( + std::string model_path, + json model_info, + int default_context_length) { this->model_path = model_path; header_print("FLM", "Loading model: " << this->model_path); this->lm_config = std::make_unique(); this->lm_config->from_pretrained(this->model_path); - if (this->npu_device_inst == nullptr) { - header_print("ERROR", "NPU device instance is nullptr"); - exit(1); - } - this->npu = std::make_unique(npu_device::device_npu2, this->npu_device_inst, enable_preemption); - this->enable_preemption = enable_preemption; // Set context length: use provided value if not -1, otherwise use model default if (default_context_length != -1) { this->MAX_L = default_context_length; @@ -144,62 +164,146 @@ void AutoModel::_shared_load_model(std::string model_path, json model_info, int this->total_tokens = 0; } -bool AutoModel::_shared_insert(chat_meta_info_t& meta_info, std::vector& tokens, std::function is_cancelled, void* payload, int first_len_run) { - - // print token history - // header_print("DEBUG", "Current token history: "); - // for (size_t i = 0; i < this->token_history.size(); i++) { - // std::cout << this->token_history[i] << " "; - // } - // std::cout << std::endl; - // // print tokens to insert - // header_print("DEBUG", "Tokens to insert: "); - // for (size_t i = 0; i < tokens.size(); i++) { - // std::cout << tokens[i] << " "; - // } - // std::cout << std::endl; - - // prefix check for tokens and token history to see if we can skip some tokens - const size_t idx = this->token_history.size(); - size_t skip_count = 0; - for (size_t i = 0; i < idx; i++) { - if (i < tokens.size() && tokens[i] == this->token_history[i]) { - skip_count++; - } - else { - break; - } +void AutoModel::_shared_initialize_legacy_npu( + bool enable_preemption) { + if (this->npu_device_inst == nullptr) { + header_print("ERROR", "NPU device instance is nullptr"); + exit(1); } - if (skip_count != idx) { + this->npu = std::make_unique( + npu_device::device_npu2, + this->npu_device_inst, + enable_preemption); + this->enable_preemption = enable_preemption; +} + +size_t AutoModel::_matching_prefix_length( + std::span tokens) const { + const size_t compared = + std::min(tokens.size(), this->token_history.size()); + size_t matched = 0; + while ( + matched < compared && + tokens[matched] == this->token_history[matched]) { + ++matched; + } + return matched; +} + +bool AutoModel::_shared_insert( + chat_meta_info_t& meta_info, + std::vector& tokens, + std::function is_cancelled, + void* payload, + int first_len_run) { + return this->_shared_insert( + meta_info, + tokens, + std::move(is_cancelled), + payload, + first_len_run, + PrefixHitAction::AppendSuffixBatched); +} + +bool AutoModel::_shared_insert( + chat_meta_info_t& meta_info, + std::vector& tokens, + std::function is_cancelled, + void* payload, + int first_len_run, + PrefixHitAction prefix_action) { + const auto mark_cancelled = [&] { + meta_info.stop_reason = CANCEL_DETECTED; + buffer_.clear(); + current_mode_ = StreamEventType::CONTENT; + tool_name_.clear(); + is_in_tool_block_ = false; + }; + + const size_t history_size = this->token_history.size(); + const size_t matched = this->_matching_prefix_length(tokens); + const bool prefix_hit = matched == history_size; + const bool recompute = + prefix_action == PrefixHitAction::RecomputeFull; + const bool clear_before_prefill = !prefix_hit || recompute; + const bool fresh_prefill = history_size == 0; + + if ( + (fresh_prefill || clear_before_prefill) && + is_cancelled()) { + mark_cancelled(); + return false; + } + + size_t skip_count = prefix_hit && !recompute + ? history_size + : 0; + if (clear_before_prefill) { clear_context(); - skip_count = 0; } tokens.erase(tokens.begin(), tokens.begin() + skip_count); - - if (this->total_tokens + tokens.size() >= this->MAX_L){ + if (this->total_tokens + tokens.size() >= this->MAX_L) { header_print("WARNING", "Max length reached, stopping prefilling..."); return false; } - for (int token : tokens){ - this->token_history.push_back(token); + + if (tokens.empty()) { + meta_info.prefill_duration = 0; + meta_info.prompt_tokens = 0; + return this->last_token != -1; } - buffer y; + buffer y; auto prefill_start_time = this->profiler_list[PREFILL_TIME].start(); - - y = _chunked_insert(meta_info, tokens, is_cancelled, payload, first_len_run); - auto prefill_end_time = this->profiler_list[PREFILL_TIME].stop(tokens.size()); + size_t committed_tokens = 0; + if ( + prefix_action == PrefixHitAction::AppendSuffixOneByOne && + !clear_before_prefill) { + for (const int token : tokens) { + if (is_cancelled()) { + mark_cancelled(); + break; + } + std::vector one_token{token}; + y = this->lm_engine->prefill(one_token, payload); + payload = nullptr; + this->token_history.push_back(token); + ++this->total_tokens; + ++committed_tokens; + } + } else { + for (const int token : tokens) { + this->token_history.push_back(token); + } + const bool direct_full_prefill = + clear_before_prefill && + prefix_action != PrefixHitAction::AppendSuffixBatched; + y = direct_full_prefill + ? this->lm_engine->prefill(tokens, payload) + : _chunked_insert( + meta_info, + tokens, + is_cancelled, + payload, + first_len_run); + if (meta_info.stop_reason != CANCEL_DETECTED) { + committed_tokens = tokens.size(); + this->total_tokens += tokens.size(); + } + } + + auto prefill_end_time = + this->profiler_list[PREFILL_TIME].stop(committed_tokens); meta_info.prefill_duration = (uint64_t)time_utils::duration_ns(prefill_start_time, prefill_end_time).first; - meta_info.prompt_tokens = tokens.size(); + meta_info.prompt_tokens = static_cast(committed_tokens); if (meta_info.stop_reason == CANCEL_DETECTED) { return false; } - this->total_tokens += tokens.size(); - if (this->total_tokens >= this->MAX_L){ + if (this->total_tokens >= this->MAX_L) { header_print("WARNING", "Max length reached, stopping prefilling..."); } this->profiler_list[SAMPLING_TIME].start(); diff --git a/src/common/AutoModel/modeling_phi4.cpp b/src/common/AutoModel/modeling_phi4.cpp index 06c164f5..b35843ef 100644 --- a/src/common/AutoModel/modeling_phi4.cpp +++ b/src/common/AutoModel/modeling_phi4.cpp @@ -7,74 +7,291 @@ #include "AutoModel/modeling_phi4.hpp" -/************ Phi4 family **************/ -Phi4::Phi4(flm_rt::device* npu_device_inst) : AutoModel(npu_device_inst, "Phi4") {} +#include +#include +#include +#include +#include +#include +#include -void Phi4::load_model(std::string model_path, json model_info, int default_context_length, bool enable_preemption) { - this->_shared_load_model(model_path, model_info, default_context_length, enable_preemption); - - this->q4nx = std::make_unique(this->model_path); - // model_type == phi4 - this->lm_engine = std::make_unique(*this->lm_config, this->npu.get(), this->MAX_L); - this->lm_engine->load_weights(*this->q4nx); - - //free the q4nx - this->q4nx.reset(); - - this->lm_engine->clear_context(); - this->setup_tokenizer(model_path); - this->sampler.reset(); +namespace { + +constexpr std::string_view kCorelibBackend = "corelib_aie4"; +constexpr int kPhi4Eos = 200020; +constexpr int kPhi4End = 199999; + +enum class Phi4Backend { + Legacy, + CorelibAie4 +}; + +Phi4Backend ResolveBackend(const json& model_info) { + const auto details = model_info.find("details"); + if ( + details == model_info.end() || + !details->is_object() || + !details->contains("execution_backend")) { + return Phi4Backend::Legacy; + } + + const auto& value = details->at("execution_backend"); + if (!value.is_string()) { + throw std::invalid_argument( + "Phi-4 details.execution_backend must be a string"); + } + const std::string backend = value.get(); + if (backend == kCorelibBackend) { + return Phi4Backend::CorelibAie4; + } + throw std::invalid_argument( + "Unknown Phi-4 execution backend '" + backend + "'"); +} + +std::uint32_t ResolveAie4ContextLength( + const json& model_info, + int requested_context_length) { + std::int64_t value = requested_context_length; + if (requested_context_length == -1) { + if (!model_info.contains("default_context_length")) { + throw std::invalid_argument( + "Phi-4 AIE4 model metadata has no default_context_length"); + } + value = model_info.at("default_context_length").get(); + } + if (value <= 0 || value > 4096) { + throw std::out_of_range( + "Phi-4 AIE4 maximum length must be in 1..4096"); + } + return static_cast(value); +} + +void RequireAie4FrontendFile( + const std::filesystem::path& model_path, + std::string_view filename) { + const auto path = model_path / filename; + std::ifstream input(path, std::ios::binary); + if (!input) { + throw std::runtime_error( + "Phi-4 AIE4 package is missing required frontend file: " + + path.string()); + } +} +void ConfigureDefaultSampler(Phi4& model) { sampler_config config; config.top_k = 40; config.top_p = 0.9; config.min_p = 0.1; config.temperature = 0.8; + model.set_sampler(config); +} + +} // namespace + +#if defined(FLM_CORELIB_TESTING) +Phi4::EngineFactoryForTesting Phi4::engine_factory_for_testing_; +#endif + +/************ Phi4 family **************/ +Phi4::Phi4(flm_rt::device* npu_device_inst) : AutoModel(npu_device_inst, "Phi4") {} + +void Phi4::load_model(std::string model_path, json model_info, int default_context_length, bool enable_preemption) { + const Phi4Backend backend = ResolveBackend(model_info); + if (backend == Phi4Backend::CorelibAie4) { +#if !defined(FLM_ENABLE_CORELIB_AIE4) + throw std::runtime_error( + "This binary was built without Phi-4 AIE4 corelib support"); +#else + if (enable_preemption) { + throw std::invalid_argument( + "Phi-4 AIE4 corelib execution does not support preemption"); + } + + const std::uint32_t context_length = + ResolveAie4ContextLength( + model_info, + default_context_length); + const std::filesystem::path package_path(model_path); + RequireAie4FrontendFile(package_path, "config.json"); + RequireAie4FrontendFile(package_path, "tokenizer.json"); + RequireAie4FrontendFile( + package_path, + "tokenizer_config.json"); + + this->_shared_initialize_model_state( + model_path, + model_info, + static_cast(context_length)); + this->npu.reset(); + this->enable_preemption = false; + this->setup_tokenizer(model_path, true); + + auto runtime = + flm::corelib::CorelibRuntime::GetOrCreate( + utils::get_executable_directory()); + std::unique_ptr engine; +#if defined(FLM_CORELIB_TESTING) + if (!engine_factory_for_testing_) { + throw std::logic_error( + "Phi-4 frontend test engine factory is not installed"); + } + engine = engine_factory_for_testing_( + true, + *this->lm_config, + nullptr, + package_path, + context_length); +#else + engine = std::make_unique( + *this->lm_config, + package_path, + runtime, + context_length); +#endif + engine->clear_context(); + + this->lm_engine = std::move(engine); + this->corelib_runtime_ = std::move(runtime); + this->uses_corelib_aie4_ = true; + this->last_continuation_route_.reset(); + this->last_continuation_ns_ = 0; + this->append_continuation_ns_ = 0; + this->reprefill_continuation_ns_ = 0; + this->sampler.reset(); + ConfigureDefaultSampler(*this); +#endif + } else { +#if defined(FLM_ENABLE_CORELIB_AIE4) + this->uses_corelib_aie4_ = false; + this->corelib_runtime_.reset(); + this->last_continuation_route_.reset(); +#if defined(FLM_CORELIB_TESTING) + this->metrics_for_testing_.reset(); +#endif +#endif + this->_shared_load_model( + model_path, + model_info, + default_context_length, + enable_preemption); + + std::unique_ptr engine; +#if defined(FLM_CORELIB_TESTING) + if (!engine_factory_for_testing_) { + throw std::logic_error( + "Phi-4 frontend test engine factory is not installed"); + } + engine = engine_factory_for_testing_( + false, + *this->lm_config, + this->npu.get(), + std::filesystem::path(model_path), + this->MAX_L); +#else + this->q4nx = std::make_unique(this->model_path); + engine = std::make_unique( + *this->lm_config, + this->npu.get(), + this->MAX_L); + engine->load_weights(*this->q4nx); + this->q4nx.reset(); +#endif + + engine->clear_context(); + this->setup_tokenizer(model_path, false); + this->lm_engine = std::move(engine); + this->sampler.reset(); + ConfigureDefaultSampler(*this); + } - this->set_sampler(config); for (size_t i = 0; i < PROFILER_TYPE_NUM; i++) { this->profiler_list[i].reset(); } } -void Phi4::setup_tokenizer(std::string model_path) { - // load tokenizer configurations - #ifdef _WIN32 +void Phi4::setup_tokenizer( + const std::string& model_path, + bool require_aie4_eos) { +#ifdef _WIN32 std::string tokenizer_config_path = model_path + "\\tokenizer_config.json"; - #else +#else std::string tokenizer_config_path = model_path + "/tokenizer_config.json"; - #endif +#endif std::ifstream fs_config(tokenizer_config_path, std::ios::in | std::ios::binary); if (fs_config.fail()) { - std::cerr << "Cannot open " << tokenizer_config_path << std::endl; - exit(1); + throw std::runtime_error( + "Cannot open " + tokenizer_config_path); } std::string data_config; fs_config.seekg(0, std::ios::end); - size_t size_config = static_cast(fs_config.tellg()); + const auto end = fs_config.tellg(); + if (end < 0) { + throw std::runtime_error( + "Cannot read " + tokenizer_config_path); + } + size_t size_config = static_cast(end); fs_config.seekg(0, std::ios::beg); data_config.resize(size_config); - fs_config.read(data_config.data(), size_config); + fs_config.read( + data_config.data(), + static_cast(size_config)); + if (!fs_config && size_config != 0) { + throw std::runtime_error( + "Cannot read " + tokenizer_config_path); + } fs_config.close(); auto tokenizer_config = nlohmann::json::parse(data_config); - this->has_bos_token = false; - // load chat template - this->chat_tmpl = std::make_unique( - tokenizer_config["chat_template"], + + if ( + !tokenizer_config.contains("chat_template") || + !tokenizer_config.at("chat_template").is_string()) { + throw std::invalid_argument( + "Phi-4 tokenizer_config.json requires a string chat_template"); + } + if (!tokenizer_config.contains("eos_token_id")) { + throw std::invalid_argument( + "Phi-4 tokenizer_config.json requires eos_token_id"); + } + + auto chat_template = std::make_unique( + tokenizer_config.at("chat_template").get(), "", - "" - ); + ""); + std::vector parsed_eos_ids; - if (this->has_bos_token) { - this->bos_token_id = tokenizer_config["bos_token_id"].get(); + const auto& eos_ids = tokenizer_config.at("eos_token_id"); + if (eos_ids.is_number_integer()) { + parsed_eos_ids.push_back(eos_ids.get()); + } else if (eos_ids.is_array()) { + for (const auto& token : eos_ids) { + parsed_eos_ids.push_back(token.get()); + } + } else { + throw std::invalid_argument( + "Phi-4 tokenizer_config.json eos_token_id must be " + "an integer or array"); } - else { - this->bos_token_id = -1; + + if (require_aie4_eos) { + const auto has_id = [&](int id) { + return std::find( + parsed_eos_ids.begin(), + parsed_eos_ids.end(), + id) != parsed_eos_ids.end(); + }; + if (!has_id(kPhi4Eos) || !has_id(kPhi4End)) { + throw std::invalid_argument( + "Phi-4 AIE4 tokenizer_config.json must contain " + "EOS token IDs 200020 and 199999"); + } } + + this->has_bos_token = false; + this->chat_tmpl = std::move(chat_template); + this->bos_token_id = -1; this->eos_token = ""; - for (auto& token : tokenizer_config["eos_token_id"]) { - this->eos_token_ids.push_back(token.get()); - } + this->eos_token_ids = std::move(parsed_eos_ids); this->user_system_prompt = ""; this->extra_context["user_system_prompt"] = this->user_system_prompt; } @@ -87,8 +304,55 @@ std::string Phi4::apply_chat_template(nlohmann::ordered_json& messages, nlohmann return this->chat_tmpl->apply(inputs); } +#if defined(FLM_ENABLE_CORELIB_AIE4) +void Phi4::validate_aie4_capacity( + size_t rendered_tokens, + std::optional requested_max_new_tokens) const { + if ( + requested_max_new_tokens.has_value() && + *requested_max_new_tokens < 0) { + throw ModelRequestError( + 400, + false, + "Phi-4 AIE4 requested_max_new_tokens cannot be negative"); + } + + const size_t active_cap = this->MAX_L; + const size_t requested = + requested_max_new_tokens.has_value() + ? static_cast(*requested_max_new_tokens) + : 0; + const bool prompt_has_no_generation_room = + rendered_tokens >= active_cap; + const bool explicit_request_exceeds_cap = + requested_max_new_tokens.has_value() && + (rendered_tokens > active_cap || + requested > active_cap - rendered_tokens); + if ( + prompt_has_no_generation_room || + explicit_request_exceeds_cap) { + std::stringstream message; + message + << "Phi-4 AIE4 request exceeds the active context cap " + << active_cap << ": rendered prompt has " + << rendered_tokens << " tokens"; + if (requested_max_new_tokens.has_value()) { + message << " and requested output has " + << requested << " tokens"; + } + throw ModelRequestError( + 400, + false, + message.str()); + } +} + +void Phi4::clear_after_corelib_error() { + AutoModel::clear_context(); +} +#endif + bool Phi4::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std::function is_cancelled) { - // preprocess this->profiler_list[TKOEN_ENCODE_TIME].start(); std::string templated_text; if (input.messages.empty() && input.prompt.empty()) { @@ -107,19 +371,232 @@ bool Phi4::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std::f std::vector tokens = this->tokenizer->encode(templated_text); this->profiler_list[TKOEN_ENCODE_TIME].stop(tokens.size()); - // hardware - return this->_shared_insert(meta_info, tokens, is_cancelled); +#if defined(FLM_ENABLE_CORELIB_AIE4) + if (this->uses_corelib_aie4_) { + this->validate_aie4_capacity( + tokens.size(), + input.requested_max_new_tokens); + + const size_t history_size = this->token_history.size(); + const size_t matched = this->_matching_prefix_length(tokens); + const bool prefix_hit = + history_size != 0 && matched == history_size; + const size_t suffix_tokens = + prefix_hit ? tokens.size() - matched : tokens.size(); + const flm::phi4::ContinuationRoute route = + prefix_hit + ? flm::phi4::SelectContinuationRoute( + suffix_tokens, + this->forced_continuation_route_) + : flm::phi4::ContinuationRoute::Reprefill; + const PrefixHitAction action = + route == flm::phi4::ContinuationRoute::Append + ? PrefixHitAction::AppendSuffixOneByOne + : PrefixHitAction::RecomputeFull; + const auto started = std::chrono::steady_clock::now(); + + bool inserted = false; + try { + inserted = this->_shared_insert( + meta_info, + tokens, + std::move(is_cancelled), + nullptr, + 0, + action); + } catch (const flm::corelib::CorelibError&) { + this->clear_after_corelib_error(); + throw ModelRequestError( + 500, + true, + "AIE4 inference failed before submission; the " + "current conversation was cleared."); + } + + const auto elapsed = + std::chrono::duration_cast( + std::chrono::steady_clock::now() - started) + .count(); + this->last_continuation_ns_ = + static_cast( + std::max(elapsed, 0)); + this->last_continuation_route_ = route; + if (route == flm::phi4::ContinuationRoute::Append) { + this->append_continuation_ns_ += + this->last_continuation_ns_; + } else { + this->reprefill_continuation_ns_ += + this->last_continuation_ns_; + } + return inserted; + } +#endif + + return this->_shared_insert( + meta_info, + tokens, + std::move(is_cancelled)); } std::string Phi4::generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled) { - return this->_shared_generate(meta_info, length_limit, os, is_cancelled); +#if defined(FLM_ENABLE_CORELIB_AIE4) + if (this->uses_corelib_aie4_) { + try { + return this->_shared_generate( + meta_info, + length_limit, + os, + std::move(is_cancelled)); + } catch (const flm::corelib::CorelibError&) { + this->clear_after_corelib_error(); + throw ModelRequestError( + 500, + true, + "AIE4 inference failed before submission; the " + "current conversation was cleared."); + } + } +#endif + return this->_shared_generate( + meta_info, + length_limit, + os, + std::move(is_cancelled)); } std::string Phi4::generate_with_prompt(chat_meta_info_t& meta_info, lm_uniform_input_t& input, int length_limit, std::ostream& os) { - if (!this->insert(meta_info, input)) { - return ""; + const std::optional caller_limit = + input.requested_max_new_tokens; + if ( + !input.requested_max_new_tokens.has_value() && + length_limit >= 0) { + input.requested_max_new_tokens = length_limit; + } + + try { + if (!this->insert(meta_info, input)) { + input.requested_max_new_tokens = caller_limit; + return ""; + } + } catch (...) { + input.requested_max_new_tokens = caller_limit; + throw; + } + input.requested_max_new_tokens = caller_limit; + return this->generate(meta_info, length_limit, os); +} + +void Phi4::set_max_length(unsigned int requested_max_length) { +#if defined(FLM_ENABLE_CORELIB_AIE4) + if (this->uses_corelib_aie4_) { + if ( + requested_max_length == 0 || + requested_max_length > 4096) { + throw std::out_of_range( + "Phi-4 AIE4 maximum length must be in 1..4096"); + } + const int frontend_position = + AutoModel::get_current_context_length(); + const int engine_position = + this->lm_engine->get_current_context_length(); + if (frontend_position != engine_position) { + throw std::logic_error( + "Phi-4 AIE4 frontend and engine context positions " + "are inconsistent"); + } + if ( + requested_max_length < + static_cast(frontend_position)) { + throw std::out_of_range( + "Phi-4 AIE4 maximum length cannot be below the " + "current logical position"); + } + + this->lm_engine->update_max_length(requested_max_length); + this->MAX_L = requested_max_length; + return; + } +#endif + AutoModel::set_max_length(requested_max_length); +} + +#if defined(FLM_ENABLE_CORELIB_AIE4) +const flm::phi4::Phi4Aie4Metrics& +Phi4::aie4_metrics() const { +#if defined(FLM_CORELIB_TESTING) + if (this->metrics_for_testing_.has_value()) { + return *this->metrics_for_testing_; + } +#endif + const auto* engine = + dynamic_cast( + this->lm_engine.get()); + if (engine == nullptr) { + throw std::logic_error( + "Phi-4 AIE4 profile requested for a non-corelib engine"); + } + return engine->metrics(); +} +#endif + +std::string Phi4::show_profile() { + const std::string base = AutoModel::show_profile(); +#if defined(FLM_ENABLE_CORELIB_AIE4) + if (this->uses_corelib_aie4_) { + const auto& metrics = this->aie4_metrics(); + std::stringstream profile; + profile << base; + profile << " Phi-4 AIE4:" << std::endl; + profile << " Engine: corelib_aie4" << std::endl; + profile << " Continuation route: " + << (this->last_continuation_route_.has_value() + ? flm::phi4::ContinuationRouteName( + *this->last_continuation_route_) + : "none") + << std::endl; + profile << " Append threshold: " + << flm::phi4::kContinuationAppendThreshold + << std::endl; + profile << " Corelib DLL: " + << this->corelib_runtime_->api()->library_path().string() + << std::endl; + profile << " Helper transitions: " + << metrics.helper_transition_counts[0] << "/" + << metrics.helper_transition_counts[1] << "/" + << metrics.helper_transition_counts[2] << "/" + << metrics.helper_transition_counts[3] << "/" + << metrics.helper_transition_counts[4] << "/" + << metrics.helper_transition_counts[5] + << std::endl; + profile << " Cold model load: " + << metrics.model_load_ns << " ns" << std::endl; + profile << " Cold weight pack: " + << metrics.weight_pack_ns << " ns" << std::endl; + profile << " Continuation time: " + << this->last_continuation_ns_ << " ns" << std::endl; + profile << " Warm append total: " + << this->append_continuation_ns_ << " ns" << std::endl; + profile << " Warm reprefill total: " + << this->reprefill_continuation_ns_ << " ns" + << std::endl; + profile << " Dispatches: " + << metrics.dispatch_count << std::endl; + profile << " Synchronizations: " + << metrics.synchronize_count << std::endl; + profile << " Packed weights: " + << metrics.packed_weight_bytes << " bytes" + << std::endl; + profile << " Mapped source: " + << metrics.mapped_source_bytes << " bytes" + << std::endl; + profile << " KV storage: " + << metrics.kv_bytes << " bytes" << std::endl; + profile << " Scratch storage: " + << metrics.scratch_bytes << " bytes" << std::endl; + return profile.str(); } - return this->_shared_generate(meta_info, length_limit, os); +#endif + return base; } \ No newline at end of file diff --git a/src/common/corelib/phi4_corelib_aie4.cpp b/src/common/corelib/phi4_corelib_aie4.cpp index c462f58a..b2318737 100644 --- a/src/common/corelib/phi4_corelib_aie4.cpp +++ b/src/common/corelib/phi4_corelib_aie4.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -62,6 +63,96 @@ std::uint32_t ValidateMaxLength(std::uint32_t max_length) { return max_length; } +void ValidateOptionalIntegerIdentity( + const LM_Config& config, + std::string_view key, + std::int64_t expected) { + const auto found = config._json_config.find(std::string(key)); + if ( + found == config._json_config.end() || + found->is_null()) { + return; + } + if (!found->is_number_integer()) { + throw std::invalid_argument( + "Phi-4 AIE4 LM_Config field '" + + std::string(key) + "' must be an integer"); + } + const std::int64_t actual = found->get(); + if (actual != expected) { + throw std::invalid_argument( + "Phi-4 AIE4 LM_Config field '" + + std::string(key) + + "' does not match the package identity"); + } +} + +void ValidateOptionalFloatingIdentity( + const LM_Config& config, + std::string_view key, + double expected) { + const auto found = config._json_config.find(std::string(key)); + if ( + found == config._json_config.end() || + found->is_null()) { + return; + } + if (!found->is_number()) { + throw std::invalid_argument( + "Phi-4 AIE4 LM_Config field '" + + std::string(key) + "' must be numeric"); + } + const double actual = found->get(); + if ( + !std::isfinite(actual) || + std::abs(actual - expected) > + std::numeric_limits::epsilon() * + std::max(1.0, std::abs(expected)) * 8.0) { + throw std::invalid_argument( + "Phi-4 AIE4 LM_Config field '" + + std::string(key) + + "' does not match the package identity"); + } +} + +void ValidateConfigIdentity(const LM_Config& config) { + // Legacy LM_Config files do not guarantee every fixed field. Validate + // each field they do provide; the package manifest independently remains + // the authority for absent fields. + ValidateOptionalIntegerIdentity( + config, + "num_hidden_layers", + constants::kLayerCount); + ValidateOptionalIntegerIdentity( + config, + "hidden_size", + constants::kHiddenSize); + ValidateOptionalIntegerIdentity( + config, + "intermediate_size", + constants::kIntermediateSize); + ValidateOptionalIntegerIdentity( + config, + "num_attention_heads", + constants::kQueryHeadCount); + ValidateOptionalIntegerIdentity( + config, + "num_key_value_heads", + constants::kKvHeadCount); + ValidateOptionalIntegerIdentity( + config, + "head_dim", + constants::kHeadSize); + ValidateOptionalIntegerIdentity( + config, + "vocab_size", + constants::kVocabularySize); + ValidateOptionalFloatingIdentity( + config, + "rms_norm_eps", + constants::kRmsEpsilon); +} + std::size_t CheckedElements( std::int64_t rows, std::int64_t width, @@ -264,7 +355,7 @@ struct phi4_corelib_aie4::Impl final { std::uint32_t requested_max_length) : runtime(std::move(supplied_runtime)), max_length(ValidateMaxLength(requested_max_length)) { - (void)config; + ValidateConfigIdentity(config); if (!runtime) { throw std::invalid_argument( "phi4_corelib_aie4 requires a CorelibRuntime"); @@ -288,6 +379,26 @@ struct phi4_corelib_aie4::Impl final { metrics.mapped_source_bytes = MappedSourceBytes(*package); shape_plan.emplace(Phi4ShapePlan::Build(api)); + metrics.helper_transition_counts = { + static_cast( + shape_plan->Transitions( + RowUse::QueryProjection).size()), + static_cast( + shape_plan->Transitions( + RowUse::KvProjection).size()), + static_cast( + shape_plan->Transitions( + RowUse::Attention).size()), + static_cast( + shape_plan->Transitions( + RowUse::OutputProjection).size()), + static_cast( + shape_plan->Transitions( + RowUse::SsMlp).size()), + static_cast( + shape_plan->Transitions( + RowUse::LmHead).size()), + }; const auto& embedding_view = package->Require(kEmbeddingName); diff --git a/src/include/AutoModel/automodel.hpp b/src/include/AutoModel/automodel.hpp index ed3a6726..eabaa0a5 100644 --- a/src/include/AutoModel/automodel.hpp +++ b/src/include/AutoModel/automodel.hpp @@ -7,9 +7,13 @@ #pragma once #include +#include #include -#include #include +#include +#include +#include +#include #include #include #include @@ -128,10 +132,32 @@ struct lm_uniform_input_t { std::vector audios; std::vector audio_payload_types; nlohmann::ordered_json tools; + std::optional requested_max_new_tokens; }; using json = nlohmann::ordered_json; +class ModelRequestError final : public std::runtime_error { +public: + ModelRequestError( + int http_code, + bool session_cleared, + std::string message); + + int http_code() const noexcept; + bool session_cleared() const noexcept; + +private: + int http_code_; + bool session_cleared_; +}; + +enum class PrefixHitAction { + AppendSuffixBatched, + AppendSuffixOneByOne, + RecomputeFull +}; + class AutoModel { protected: std::string model_path = ""; @@ -188,14 +214,29 @@ class AutoModel { void _shared_load_model(std::string model_path, json model_info, int default_context_length = -1, bool enable_preemption = false); + void _shared_initialize_model_state(std::string model_path, json model_info, int default_context_length); + void _shared_initialize_legacy_npu(bool enable_preemption); nlohmann::json _shared_setup_tokenizer(std::string model_path); + size_t _matching_prefix_length(std::span tokens) const; /// \brief Insert tokens into the model /// \param meta_info the meta information of the chat /// \param tokens the tokens to insert /// \param payload the payload, it shall not be used as this function is only used for chunkwised insertion, no image allowed /// \return true if the tokens were inserted successfully, false otherwise - bool _shared_insert(chat_meta_info_t& meta_info, std::vector& tokens, std::function is_cancelled = [] { return false; }, void* payload = nullptr, int first_len_run = 0); + bool _shared_insert( + chat_meta_info_t& meta_info, + std::vector& tokens, + std::function is_cancelled = [] { return false; }, + void* payload = nullptr, + int first_len_run = 0); + bool _shared_insert( + chat_meta_info_t& meta_info, + std::vector& tokens, + std::function is_cancelled, + void* payload, + int first_len_run, + PrefixHitAction prefix_action); buffer _chunked_insert(chat_meta_info_t& meta_info, std::vector& tokens, std::function is_cancelled = [] { return false; }, void* payload = nullptr, int first_len_run = 0); std::string _shared_generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled = [] { return false; }); diff --git a/src/include/AutoModel/modeling_phi4.hpp b/src/include/AutoModel/modeling_phi4.hpp index 66937d6d..dcce242d 100644 --- a/src/include/AutoModel/modeling_phi4.hpp +++ b/src/include/AutoModel/modeling_phi4.hpp @@ -7,10 +7,66 @@ #pragma once #include "AutoModel/automodel.hpp" +#if defined(FLM_ENABLE_CORELIB_AIE4) +#include "corelib/corelib_runtime.hpp" +#include "models/phi4/phi4_corelib_aie4.hpp" +#include "models/phi4/phi4_corelib_aie4_tuning.hpp" +#include +#endif + +#if defined(FLM_CORELIB_TESTING) +#include +#include +#endif + +#if defined(FLM_CORELIB_TESTING) +namespace flm::phi4::testing { +class Phi4FrontendTestAccess; +} +#endif + /************ phi4 family **************/ class Phi4 : public AutoModel { private: - void setup_tokenizer(std::string model_path); + void setup_tokenizer( + const std::string& model_path, + bool require_aie4_eos); + +#if defined(FLM_ENABLE_CORELIB_AIE4) + void validate_aie4_capacity( + size_t rendered_tokens, + std::optional requested_max_new_tokens) const; + void clear_after_corelib_error(); + const flm::phi4::Phi4Aie4Metrics& aie4_metrics() const; + + bool uses_corelib_aie4_ = false; + std::shared_ptr + corelib_runtime_; + flm::phi4::ForcedContinuationRoute + forced_continuation_route_ = + flm::phi4::ForcedContinuationRoute::Automatic; + std::optional + last_continuation_route_; + std::uint64_t last_continuation_ns_ = 0; + std::uint64_t append_continuation_ns_ = 0; + std::uint64_t reprefill_continuation_ns_ = 0; +#if defined(FLM_CORELIB_TESTING) + std::optional + metrics_for_testing_; +#endif +#endif + +#if defined(FLM_CORELIB_TESTING) + using EngineFactoryForTesting = + std::function( + bool, + const LM_Config&, + npu_xclbin_manager*, + const std::filesystem::path&, + std::uint32_t)>; + static EngineFactoryForTesting engine_factory_for_testing_; + friend class flm::phi4::testing::Phi4FrontendTestAccess; +#endif public: Phi4(flm_rt::device* npu_device_inst); @@ -21,4 +77,6 @@ class Phi4 : public AutoModel { std::string generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled = [] { return false; }) override; std::string generate_with_prompt(chat_meta_info_t& meta_info, lm_uniform_input_t& input, int length_limit, std::ostream& os = std::cout) override; std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object()) override; + void set_max_length(unsigned int MAX_L) override; + std::string show_profile() override; }; diff --git a/src/include/models/phi4/phi4_corelib_aie4.hpp b/src/include/models/phi4/phi4_corelib_aie4.hpp index 3f4953e5..66cafa78 100644 --- a/src/include/models/phi4/phi4_corelib_aie4.hpp +++ b/src/include/models/phi4/phi4_corelib_aie4.hpp @@ -4,6 +4,7 @@ #include #include +#include #include #include #include @@ -40,6 +41,7 @@ struct Phi4Aie4Metrics { std::uint64_t attention_extent_queries = 0; std::uint64_t output_projection_extent_queries = 0; std::uint64_t lm_head_extent_queries = 0; + std::array helper_transition_counts{}; std::uint64_t v_read_calls = 0; std::uint64_t v_write_calls = 0; std::uint64_t v_bytes = 0; diff --git a/src/include/models/phi4/phi4_corelib_aie4_tuning.hpp b/src/include/models/phi4/phi4_corelib_aie4_tuning.hpp new file mode 100644 index 00000000..4a4bcba9 --- /dev/null +++ b/src/include/models/phi4/phi4_corelib_aie4_tuning.hpp @@ -0,0 +1,44 @@ +#pragma once + +#include +#include + +namespace flm::phi4 { + +enum class ContinuationRoute { + Append, + Reprefill +}; + +enum class ForcedContinuationRoute { + Automatic, + Append, + Reprefill +}; + +inline constexpr std::uint32_t kContinuationAppendThreshold = 0; + +inline constexpr ContinuationRoute SelectContinuationRoute( + std::size_t suffix_tokens, + ForcedContinuationRoute forced) noexcept { + switch (forced) { + case ForcedContinuationRoute::Append: + return ContinuationRoute::Append; + case ForcedContinuationRoute::Reprefill: + return ContinuationRoute::Reprefill; + case ForcedContinuationRoute::Automatic: + return suffix_tokens <= kContinuationAppendThreshold + ? ContinuationRoute::Append + : ContinuationRoute::Reprefill; + } + return ContinuationRoute::Reprefill; +} + +inline constexpr const char* ContinuationRouteName( + ContinuationRoute route) noexcept { + return route == ContinuationRoute::Append + ? "append" + : "reprefill"; +} + +} // namespace flm::phi4 diff --git a/src/test/phi4_corelib_aie4/CMakeLists.txt b/src/test/phi4_corelib_aie4/CMakeLists.txt index a1bcf6fa..9c7da669 100644 --- a/src/test/phi4_corelib_aie4/CMakeLists.txt +++ b/src/test/phi4_corelib_aie4/CMakeLists.txt @@ -100,3 +100,89 @@ add_dependencies( fake_ryzenai_corelib) target_link_libraries(test_corelib_fatal_record PRIVATE advapi32) + +set(PHI4_FRONTEND_SOURCES + ${FASTFLOW_SOURCE_DIR}/common/AutoModel/automodel.cpp + ${FASTFLOW_SOURCE_DIR}/common/AutoModel/modeling_phi4.cpp) + +function(add_phi4_frontend_test TEST_NAME ENABLE_AIE4) + add_executable(${TEST_NAME} + test_phi4_frontend.cpp + ${PHI4_FRONTEND_SOURCES}) + target_include_directories(${TEST_NAME} PRIVATE + ${FASTFLOW_SOURCE_DIR}/include + ${FASTFLOW_SOURCE_DIR}/../third_party/tokenizers-cpp/include + ${BOOST_INCLUDE_DIR} + ${XRT_INCLUDE_DIR}) + target_compile_definitions(${TEST_NAME} PRIVATE + FLM_CORELIB_TESTING=1 + DEV_BUILD=1 + __WINDOWS__ + USEAVX2=1 + DISABLE_ABI_CHECK=1 + _ENABLE_EXTENDED_ALIGNED_STORAGE + CMAKE_INSTALL_PREFIX="${FASTFLOW_SOURCE_DIR}/build/phi4_corelib_aie4-tests" + CMAKE_XCLBIN_PREFIX="${FASTFLOW_SOURCE_DIR}/xclbins" + WIN32_LEAN_AND_MEAN + NOMINMAX) + target_compile_options(${TEST_NAME} PRIVATE + $<$:/arch:AVX2 /fp:precise>) + target_link_directories(${TEST_NAME} PRIVATE + ${XRT_LIB_DIR}) + target_link_libraries(${TEST_NAME} PRIVATE + xrt_coreutil) + + if(ENABLE_AIE4) + target_compile_definitions(${TEST_NAME} PRIVATE + FLM_ENABLE_CORELIB_AIE4=1) + target_include_directories(${TEST_NAME} PRIVATE + ${RYZENAI_CORELIB_INCLUDE_DIR}) + target_link_libraries(${TEST_NAME} PRIVATE + flm_corelib_aie4_testlib) + add_dependencies(${TEST_NAME} + fake_ryzenai_corelib) + endif() + + add_test(NAME ${TEST_NAME} COMMAND ${TEST_NAME}) + set_tests_properties(${TEST_NAME} PROPERTIES + ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:${XRT_LIB_DIR}") +endfunction() + +add_phi4_frontend_test(test_phi4_frontend_off FALSE) +add_phi4_frontend_test(test_phi4_frontend_on TRUE) + +function(add_phi4_frontend_compile_check TARGET_NAME ENABLE_AIE4) + add_library(${TARGET_NAME} OBJECT + ${PHI4_FRONTEND_SOURCES}) + target_include_directories(${TARGET_NAME} PRIVATE + ${FASTFLOW_SOURCE_DIR}/include + ${FASTFLOW_SOURCE_DIR}/../third_party/tokenizers-cpp/include + ${BOOST_INCLUDE_DIR} + ${XRT_INCLUDE_DIR}) + target_compile_definitions(${TARGET_NAME} PRIVATE + DEV_BUILD=1 + __WINDOWS__ + USEAVX2=1 + DISABLE_ABI_CHECK=1 + _ENABLE_EXTENDED_ALIGNED_STORAGE + CMAKE_INSTALL_PREFIX="${FASTFLOW_SOURCE_DIR}/build/phi4_corelib_aie4-tests" + CMAKE_XCLBIN_PREFIX="${FASTFLOW_SOURCE_DIR}/xclbins" + WIN32_LEAN_AND_MEAN + NOMINMAX) + target_compile_options(${TARGET_NAME} PRIVATE + $<$:/arch:AVX2 /fp:precise>) + if(ENABLE_AIE4) + target_compile_definitions(${TARGET_NAME} PRIVATE + FLM_ENABLE_CORELIB_AIE4=1) + target_include_directories(${TARGET_NAME} PRIVATE + ${RYZENAI_CORELIB_INCLUDE_DIR}) + endif() +endfunction() + +add_phi4_frontend_compile_check( + phi4_frontend_compile_off + FALSE) +add_phi4_frontend_compile_check( + phi4_frontend_compile_on + TRUE) diff --git a/src/test/phi4_corelib_aie4/test_phi4_engine.cpp b/src/test/phi4_corelib_aie4/test_phi4_engine.cpp index fa90d74a..874e27a4 100644 --- a/src/test/phi4_corelib_aie4/test_phi4_engine.cpp +++ b/src/test/phi4_corelib_aie4/test_phi4_engine.cpp @@ -1820,6 +1820,10 @@ void CheckMetrics( CHECK(metrics.attention_extent_queries == passes); CHECK(metrics.output_projection_extent_queries == passes); CHECK(metrics.lm_head_extent_queries == passes); + CHECK(std::all_of( + metrics.helper_transition_counts.begin(), + metrics.helper_transition_counts.end(), + [](std::uint32_t count) { return count > 0; })); CHECK(metrics.v_read_calls == passes * 32u); CHECK(metrics.v_write_calls == passes * 32u * 8u); CHECK(metrics.v_bytes > 0); @@ -2500,6 +2504,52 @@ void TestDestructorSynchronizeFailureChild( CHECK(record.at("position") == 0); } +void TestConfigIdentityMismatchIsRejected( + const SyntheticPackage& package) { + RecordingState state; + flm::test::ResetFakeCorelib(); + TempDirectory fatal_root( + "fastflowlm-phi4-config-identity"); + auto api = ResolveRecordingCorelib(state); + auto runtime = CorelibRuntime::Create( + api, + MakeRecords(fatal_root.path()), + [](unsigned int) { + throw std::runtime_error( + "config validation must not terminate"); + }); + + LM_Config config; + config._json_config = { + {"hidden_size", constants::kHiddenSize + 1}, + {"num_hidden_layers", constants::kLayerCount}, + {"vocab_size", constants::kVocabularySize}, + }; + + bool rejected = false; + try { + phi4_corelib_aie4 engine( + std::move(config), + package.path(), + runtime, + 4096); + } catch (const std::invalid_argument& error) { + CHECK( + std::string_view(error.what()).find("hidden_size") != + std::string_view::npos); + rejected = true; + } + runtime->ShutdownHealthy(); + if (g_state == &state) { + g_state = nullptr; + } + CHECK(rejected); + CHECK(state.objects.empty()); + CHECK(state.tensors.empty()); + CHECK(state.matmul_weight_count == 0); + CHECK(state.ssmlp_weight_count == 0); +} + static_assert(std::is_base_of_v); static_assert(std::has_virtual_destructor_v); @@ -2517,6 +2567,7 @@ int wmain(int argc, wchar_t* argv[]) { argv[4]); } SyntheticPackage package; + TestConfigIdentityMismatchIsRejected(package); TestOrderBuffersTailsStateAndMetrics(package); TestDivergentPaddingGrids(package); TestRecoverablePreSubmitFailures(package); diff --git a/src/test/phi4_corelib_aie4/test_phi4_frontend.cpp b/src/test/phi4_corelib_aie4/test_phi4_frontend.cpp new file mode 100644 index 00000000..ea003f80 --- /dev/null +++ b/src/test/phi4_corelib_aie4/test_phi4_frontend.cpp @@ -0,0 +1,929 @@ +#include "test_support.hpp" + +#include +#include + +#if defined(FLM_ENABLE_CORELIB_AIE4) +#include +#include +#endif + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +std::vector g_encoded_tokens; +int g_sample_token = 7; +int g_sampler_reset_count = 0; +int g_sampler_sample_count = 0; + +std::filesystem::path CurrentExecutablePath() { + std::wstring buffer(32768, L'\0'); + const DWORD written = GetModuleFileNameW( + nullptr, + buffer.data(), + static_cast(buffer.size())); + if (written == 0 || written >= buffer.size()) { + throw std::runtime_error("GetModuleFileNameW failed"); + } + buffer.resize(written); + return std::filesystem::path(buffer); +} + +class TempModelPackage final { +public: + explicit TempModelPackage( + std::vector eos_ids = {200020, 199999}, + std::optional hidden_size = 3072) { + const auto base = std::filesystem::temp_directory_path(); + for (int attempt = 0; attempt < 100; ++attempt) { + path_ = base / + ("flm-phi4-frontend-" + + std::to_string(GetCurrentProcessId()) + "-" + + std::to_string(GetTickCount64()) + "-" + + std::to_string(attempt)); + std::error_code error; + if (std::filesystem::create_directory(path_, error)) { + break; + } + if (attempt == 99) { + throw std::runtime_error( + "failed to create temporary Phi-4 package"); + } + } + + nlohmann::json config = { + {"model_type", "phi4"}, + {"num_hidden_layers", 32}, + {"hidden_size", hidden_size.value_or(3072)}, + {"intermediate_size", 8192}, + {"num_attention_heads", 24}, + {"num_key_value_heads", 8}, + {"head_dim", 128}, + {"vocab_size", 200064}, + {"rms_norm_eps", 1.0e-5}, + }; + WriteJson(path_ / "config.json", config); + + nlohmann::json tokenizer_config = { + {"chat_template", + "{% for message in messages %}" + "{{ message['content'] }}" + "{% endfor %}"}, + {"eos_token_id", std::move(eos_ids)}, + }; + WriteJson(path_ / "tokenizer_config.json", tokenizer_config); + WriteText(path_ / "tokenizer.json", "{}"); + } + + ~TempModelPackage() { + std::error_code ignored; + std::filesystem::remove_all(path_, ignored); + } + + TempModelPackage(const TempModelPackage&) = delete; + TempModelPackage& operator=(const TempModelPackage&) = delete; + + const std::filesystem::path& path() const noexcept { + return path_; + } + +private: + static void WriteJson( + const std::filesystem::path& path, + const nlohmann::json& value) { + WriteText(path, value.dump()); + } + + static void WriteText( + const std::filesystem::path& path, + std::string_view value) { + std::ofstream output(path, std::ios::binary); + if (!output) { + throw std::runtime_error( + "failed to create frontend test package file"); + } + output.write( + value.data(), + static_cast(value.size())); + } + + std::filesystem::path path_; +}; + +class FakeEngine final : public causal_lm { +public: + explicit FakeEngine(std::uint32_t max_length) + : max_length(max_length) {} + + buffer forward(int id) override { +#if defined(FLM_ENABLE_CORELIB_AIE4) + if (fail_next_forward) { + fail_next_forward = false; + throw flm::corelib::CorelibError( + ryzenai_corelib_status_failure, + "fake_forward", + "injected pre-submit forward failure", + "failure"); + } +#endif + forward_tokens.push_back(id); + ++position; + return MakeLogits(); + } + + buffer prefill( + std::vector& ids, + void*) override { +#if defined(FLM_ENABLE_CORELIB_AIE4) + if (fail_next_prefill) { + fail_next_prefill = false; + throw flm::corelib::CorelibError( + ryzenai_corelib_status_failure, + "fake_prefill", + "injected pre-submit prefill failure", + "failure"); + } +#endif + prefill_calls.push_back(ids); + position += static_cast(ids.size()); + return MakeLogits(); + } + + void set_context_length(int length) override { + position = length; + } + + void load_weights(Q4NX&) override {} + + void update_max_length(std::uint32_t requested) override { + if (requested == 0 || requested > 4096) { + throw std::out_of_range( + "fake AIE4 maximum length must be in 1..4096"); + } + if (requested < static_cast(position)) { + throw std::out_of_range( + "fake AIE4 maximum length cannot be below current"); + } + max_length = requested; + } + + void clear_context() override { + ++clear_count; + position = 0; + checkpoint_position.reset(); + } + + buffer get_k_cache(int, int) override { + return MakeLogits(); + } + + buffer get_v_cache(int, int) override { + return MakeLogits(); + } + + int get_current_context_length() override { + return position; + } + + int checkpoint() override { + checkpoint_position = position; + return position; + } + + int restore() override { + if (!checkpoint_position.has_value()) { + throw std::logic_error("fake engine has no checkpoint"); + } + position = *checkpoint_position; + return position; + } + + static buffer MakeLogits() { + return buffer(1); + } + + std::uint32_t max_length; + int position = 0; + int clear_count = 0; + std::optional checkpoint_position; + std::vector> prefill_calls; + std::vector forward_tokens; +#if defined(FLM_ENABLE_CORELIB_AIE4) + bool fail_next_prefill = false; + bool fail_next_forward = false; +#endif +}; + +struct FactoryState { + int calls = 0; + bool last_was_corelib = false; + FakeEngine* engine = nullptr; +}; + +FactoryState g_factory; + +nlohmann::ordered_json ModelInfo( + int default_context_length, + std::optional execution_backend = std::nullopt) { + nlohmann::ordered_json details = { + {"family", "phi4"}, + }; + if (execution_backend.has_value()) { + details["execution_backend"] = *execution_backend; + } + return { + {"default_context_length", default_context_length}, + {"details", std::move(details)}, + }; +} + +chat_meta_info_t Meta() { + chat_meta_info_t meta; + meta.max_prefill_len = 64; + return meta; +} + +lm_uniform_input_t Prompt( + std::optional requested = std::nullopt) { + lm_uniform_input_t input; + input.prompt = "frontend-test"; + input.requested_max_new_tokens = requested; + return input; +} + +template +void CheckRequestError( + Function&& function, + int expected_code, + bool expected_session_cleared, + std::string_view expected_message) { + try { + function(); + } catch (const ModelRequestError& error) { + CHECK(error.http_code() == expected_code); + CHECK(error.session_cleared() == expected_session_cleared); + CHECK( + std::string_view(error.what()).find(expected_message) != + std::string_view::npos); + return; + } + throw std::runtime_error("expected ModelRequestError was not thrown"); +} + +} // namespace + +Tokenizer::Tokenizer(const std::string&) { + is_doubled_encoded = false; +} + +Tokenizer::~Tokenizer() = default; + +std::vector Tokenizer::encode(const std::string&) { + return g_encoded_tokens; +} + +std::string Tokenizer::decode(const std::vector&) { + return "decoded"; +} + +std::string Tokenizer::run_time_decoder(int token) { + return "token-" + std::to_string(token); +} + +SafeTensors::~SafeTensors() = default; + +Sampler::Sampler(int features, sampler_config& config) + : in_features(features), + rep_penalty(config.rep_penalty), + freq_penalty(config.freq_penalty), + pre_penalty(config.pre_penalty), + top_k(config.top_k), + top_p(config.top_p), + min_p(config.min_p), + temperature(config.temperature), + total_tokens(0), + freq_penalty_window(config.freq_penalty_window), + rep_penalty_window(config.rep_penalty_window), + repeat_last_n(config.repeat_last_n), + use_optimized_sampling(config.use_optimized_sampling) { + logits.resize(1); + counters.resize(1); + token_positions.resize(1, -1); +} + +void Sampler::reset_penalties() { + ++g_sampler_reset_count; + std::fill(counters.begin(), counters.end(), 0); + std::fill(token_positions.begin(), token_positions.end(), -1); + token_counts_sparse.clear(); + token_history.clear(); + total_tokens = 0; +} + +int Sampler::sample(buffer&) { + ++g_sampler_sample_count; + ++total_tokens; + token_history.push_back(g_sample_token); + return g_sample_token; +} + +namespace utils { + +std::string get_executable_directory() { + return CurrentExecutablePath().parent_path().string(); +} + +} // namespace utils + +namespace flm::phi4::testing { + +class Phi4FrontendTestAccess final { +public: + using Factory = std::function( + bool, + const LM_Config&, + npu_xclbin_manager*, + const std::filesystem::path&, + std::uint32_t)>; + + static void InstallFactory() { + g_factory = {}; + Phi4::engine_factory_for_testing_ = + [](bool corelib, + const LM_Config&, + npu_xclbin_manager*, + const std::filesystem::path&, + std::uint32_t max_length) { + ++g_factory.calls; + g_factory.last_was_corelib = corelib; + auto engine = + std::make_unique(max_length); + g_factory.engine = engine.get(); + return engine; + }; + } + + static void RemoveFactory() { + Phi4::engine_factory_for_testing_ = {}; + g_factory = {}; + } + + static bool HasLegacyNpu(const Phi4& model) { + return model.npu != nullptr; + } + +#if defined(FLM_ENABLE_CORELIB_AIE4) + static bool HasRuntime(const Phi4& model) { + return model.corelib_runtime_ != nullptr; + } + + static void SetMetrics( + Phi4& model, + const Phi4Aie4Metrics& metrics) { + model.metrics_for_testing_ = metrics; + } + + static void ForceRoute( + Phi4& model, + ForcedContinuationRoute route) { + model.forced_continuation_route_ = route; + } +#endif + + static const std::vector& History(const Phi4& model) { + return model.token_history; + } +}; + +} // namespace flm::phi4::testing + +namespace { + +using flm::phi4::ContinuationRoute; +using flm::phi4::ForcedContinuationRoute; +using flm::phi4::SelectContinuationRoute; +using flm::phi4::testing::Phi4FrontendTestAccess; + +struct FactoryScope final { + FactoryScope() { + Phi4FrontendTestAccess::InstallFactory(); + } + + ~FactoryScope() { + Phi4FrontendTestAccess::RemoveFactory(); + } +}; + +std::unique_ptr Load( + const TempModelPackage& package, + nlohmann::ordered_json model_info, + int requested_context = -1, + bool preemption = false, + flm_rt::device* device = + reinterpret_cast(std::uintptr_t{1})) { + auto model = std::make_unique(device); + model->load_model( + package.path().string(), + std::move(model_info), + requested_context, + preemption); + return model; +} + +void TestContinuationSelector() { + CHECK( + SelectContinuationRoute( + 0, + ForcedContinuationRoute::Automatic) == + ContinuationRoute::Append); + CHECK( + SelectContinuationRoute( + 1, + ForcedContinuationRoute::Automatic) == + ContinuationRoute::Reprefill); + CHECK( + SelectContinuationRoute( + 999, + ForcedContinuationRoute::Append) == + ContinuationRoute::Append); + CHECK( + SelectContinuationRoute( + 0, + ForcedContinuationRoute::Reprefill) == + ContinuationRoute::Reprefill); +} + +void TestLegacyRoutingAndUnknownBackend() { + TempModelPackage package({200020}); + FactoryScope factory; + + auto legacy = Load(package, ModelInfo(1024)); + CHECK(g_factory.calls == 1); + CHECK(!g_factory.last_was_corelib); + CHECK(Phi4FrontendTestAccess::HasLegacyNpu(*legacy)); + + FakeEngine* legacy_engine = g_factory.engine; + g_encoded_tokens = {1, 2}; + auto meta = Meta(); + auto input = Prompt(); + CHECK(legacy->insert(meta, input)); + legacy_engine->prefill_calls.clear(); + g_encoded_tokens = {1, 2, 3, 4}; + CHECK(legacy->insert(meta, input)); + CHECK(legacy_engine->prefill_calls.size() == 1); + CHECK( + legacy_engine->prefill_calls[0] == + std::vector({3, 4})); + + legacy->set_max_length(512); + CHECK(legacy->get_max_length() == 1024); + CHECK(legacy_engine->max_length == 512); + CHECK( + legacy->show_profile() == + legacy->AutoModel::show_profile()); + + CheckThrowsContains( + [&] { + auto model = Load( + package, + ModelInfo(64, "invented_backend")); + (void)model; + }, + "invented_backend"); + CHECK(g_factory.calls == 1); +} + +#if !defined(FLM_ENABLE_CORELIB_AIE4) + +void TestFeatureOffRejectsCorelibTag() { + TempModelPackage package; + FactoryScope factory; + + CheckThrowsContains( + [&] { + auto model = Load( + package, + ModelInfo(64, "corelib_aie4"), + -1, + false, + nullptr); + (void)model; + }, + "without Phi-4 AIE4 corelib support"); + CHECK(g_factory.calls == 0); +} + +#else + +void ConfigureFakeCorelibDll() { + const auto fake_dll = + CurrentExecutablePath().parent_path() / + "fake_ryzenai_corelib.dll"; + CHECK(std::filesystem::exists(fake_dll)); + if (_wputenv_s( + L"RYZENAI_CORELIB_PATH", + fake_dll.c_str()) != 0) { + throw std::runtime_error( + "failed to configure RYZENAI_CORELIB_PATH"); + } +} + +void TestCorelibRoutingAndPreemption() { + TempModelPackage package; + FactoryScope factory; + + CheckThrowsContains( + [&] { + auto model = Load( + package, + ModelInfo(64, "corelib_aie4"), + -1, + true, + nullptr); + (void)model; + }, + "preemption"); + CHECK(g_factory.calls == 0); + + auto model = Load( + package, + ModelInfo(64, "corelib_aie4"), + -1, + false, + nullptr); + CHECK(g_factory.calls == 1); + CHECK(g_factory.last_was_corelib); + CHECK(!Phi4FrontendTestAccess::HasLegacyNpu(*model)); + CHECK(Phi4FrontendTestAccess::HasRuntime(*model)); +} + +void TestInitialAndAtomicCaps() { + TempModelPackage package; + FactoryScope factory; + + CheckThrowsContains( + [&] { + auto model = Load( + package, + ModelInfo(4097, "corelib_aie4"), + -1, + false, + nullptr); + (void)model; + }, + "1..4096"); + CHECK(g_factory.calls == 0); + + auto model = Load( + package, + ModelInfo(1024, "corelib_aie4"), + -1, + false, + nullptr); + FakeEngine* engine = g_factory.engine; + CHECK(model->get_max_length() == 1024); + CHECK(engine->max_length == 1024); + + CheckThrowsContains( + [&] { model->set_max_length(4097); }, + "1..4096"); + CHECK(model->get_max_length() == 1024); + CHECK(engine->max_length == 1024); + + model->clear_context(); + CHECK(model->get_current_context_length() == 0); + model->set_max_length(512); + CHECK(model->get_max_length() == 512); + CHECK(engine->max_length == 512); + + g_encoded_tokens.assign(10, 11); + auto meta = Meta(); + auto input = Prompt(); + CHECK(model->insert(meta, input)); + CHECK(model->get_current_context_length() == 10); + CHECK(engine->position == 10); + + CheckThrowsContains( + [&] { model->set_max_length(9); }, + "current"); + CHECK(model->get_max_length() == 512); + CHECK(engine->max_length == 512); + CHECK(engine->position == 10); +} + +void TestRenderedCapacityIsAtomic() { + TempModelPackage package; + FactoryScope factory; + auto model = Load( + package, + ModelInfo(512, "corelib_aie4"), + -1, + false, + nullptr); + FakeEngine* engine = g_factory.engine; + engine->prefill_calls.clear(); + const int clear_count = engine->clear_count; + + g_encoded_tokens.assign(500, 12); + auto meta = Meta(); + auto input = Prompt(13); + CheckRequestError( + [&] { (void)model->insert(meta, input); }, + 400, + false, + "500"); + CHECK(engine->position == 0); + CHECK(engine->clear_count == clear_count); + CHECK(engine->prefill_calls.empty()); + CHECK(Phi4FrontendTestAccess::History(*model).empty()); + + input.requested_max_new_tokens = std::nullopt; + CHECK(model->insert(meta, input)); + CHECK(engine->position == 500); + + Phi4FrontendTestAccess::ForceRoute( + *model, + ForcedContinuationRoute::Append); + g_encoded_tokens.push_back(13); + input.requested_max_new_tokens = 12; + const auto calls_before_append_rejection = + engine->prefill_calls.size(); + CheckRequestError( + [&] { (void)model->insert(meta, input); }, + 400, + false, + "501"); + CHECK(engine->position == 500); + CHECK( + engine->prefill_calls.size() == + calls_before_append_rejection); + CHECK(Phi4FrontendTestAccess::History(*model).size() == 500); +} + +void TestForcedAppendAndCancellationAlignment() { + TempModelPackage package; + FactoryScope factory; + auto model = Load( + package, + ModelInfo(64, "corelib_aie4"), + -1, + false, + nullptr); + FakeEngine* engine = g_factory.engine; + + g_encoded_tokens = {1, 2}; + auto first_meta = Meta(); + auto first = Prompt(); + CHECK(model->insert(first_meta, first)); + engine->prefill_calls.clear(); + + Phi4FrontendTestAccess::ForceRoute( + *model, + ForcedContinuationRoute::Append); + g_encoded_tokens = {1, 2, 3, 4, 5}; + auto second_meta = Meta(); + auto second = Prompt(); + int cancellation_checks = 0; + const bool inserted = model->insert( + second_meta, + second, + [&] { return cancellation_checks++ == 1; }); + CHECK(!inserted); + CHECK(second_meta.stop_reason == CANCEL_DETECTED); + CHECK(engine->prefill_calls.size() == 1); + CHECK(engine->prefill_calls[0] == std::vector({3})); + CHECK(engine->position == 3); + CHECK(model->get_current_context_length() == 3); + CHECK( + Phi4FrontendTestAccess::History(*model) == + std::vector({1, 2, 3})); +} + +void TestForcedAndAutomaticReprefill() { + TempModelPackage package; + FactoryScope factory; + auto model = Load( + package, + ModelInfo(64, "corelib_aie4"), + -1, + false, + nullptr); + FakeEngine* engine = g_factory.engine; + + g_encoded_tokens = {1, 2}; + auto meta = Meta(); + auto input = Prompt(); + CHECK(model->insert(meta, input)); + + engine->prefill_calls.clear(); + const int clear_before_forced = engine->clear_count; + Phi4FrontendTestAccess::ForceRoute( + *model, + ForcedContinuationRoute::Reprefill); + g_encoded_tokens = {1, 2, 3, 4}; + CHECK(model->insert(meta, input)); + CHECK(engine->clear_count == clear_before_forced + 1); + CHECK(engine->prefill_calls.size() == 1); + CHECK( + engine->prefill_calls[0] == + std::vector({1, 2, 3, 4})); + CHECK(engine->position == 4); + + engine->prefill_calls.clear(); + const int clear_before_automatic = engine->clear_count; + Phi4FrontendTestAccess::ForceRoute( + *model, + ForcedContinuationRoute::Automatic); + g_encoded_tokens = {1, 2, 3, 4, 5}; + CHECK(model->insert(meta, input)); + CHECK(engine->clear_count == clear_before_automatic + 1); + CHECK(engine->prefill_calls.size() == 1); + CHECK( + engine->prefill_calls[0] == + std::vector({1, 2, 3, 4, 5})); +} + +void TestEosValidationAndFrontendStop() { + FactoryScope factory; + TempModelPackage invalid_package({200020}); + CheckThrowsContains( + [&] { + auto model = Load( + invalid_package, + ModelInfo(64, "corelib_aie4"), + -1, + false, + nullptr); + (void)model; + }, + "199999"); + + TempModelPackage package; + auto model = Load( + package, + ModelInfo(64, "corelib_aie4"), + -1, + false, + nullptr); + FakeEngine* engine = g_factory.engine; + g_encoded_tokens = {10, 11}; + g_sample_token = 200020; + auto meta = Meta(); + auto input = Prompt(); + CHECK(model->insert(meta, input)); + std::ostringstream output; + CHECK(model->generate(meta, 8, output).empty()); + CHECK(engine->forward_tokens.empty()); + g_sample_token = 7; +} + +void TestRecoverableFailuresClearSession() { + TempModelPackage package; + FactoryScope factory; + auto model = Load( + package, + ModelInfo(64, "corelib_aie4"), + -1, + false, + nullptr); + FakeEngine* engine = g_factory.engine; + + g_encoded_tokens = {20}; + auto meta = Meta(); + auto input = Prompt(); + CHECK(model->insert(meta, input)); + Phi4FrontendTestAccess::ForceRoute( + *model, + ForcedContinuationRoute::Append); + engine->checkpoint(); + engine->fail_next_prefill = true; + const int resets_before_insert = g_sampler_reset_count; + g_encoded_tokens = {20, 21}; + CheckRequestError( + [&] { (void)model->insert(meta, input); }, + 500, + true, + "current conversation was cleared"); + CHECK(engine->position == 0); + CHECK(!engine->checkpoint_position.has_value()); + CHECK(Phi4FrontendTestAccess::History(*model).empty()); + CHECK(g_sampler_reset_count > resets_before_insert); + + g_encoded_tokens = {31, 32}; + CHECK(model->insert(meta, input)); + engine->checkpoint(); + engine->fail_next_forward = true; + const int resets_before_generate = g_sampler_reset_count; + std::ostringstream output; + CheckRequestError( + [&] { (void)model->generate(meta, 8, output); }, + 500, + true, + "current conversation was cleared"); + CHECK(engine->position == 0); + CHECK(!engine->checkpoint_position.has_value()); + CHECK(Phi4FrontendTestAccess::History(*model).empty()); + CHECK(g_sampler_reset_count > resets_before_generate); +} + +void TestProfileSplit() { + TempModelPackage package; + FactoryScope factory; + auto model = Load( + package, + ModelInfo(64, "corelib_aie4"), + -1, + false, + nullptr); + + flm::phi4::Phi4Aie4Metrics metrics; + metrics.model_load_ns = 101; + metrics.weight_pack_ns = 202; + metrics.dispatch_count = 303; + metrics.synchronize_count = 404; + metrics.helper_transition_counts = {1, 2, 3, 4, 5, 6}; + Phi4FrontendTestAccess::SetMetrics(*model, metrics); + + g_encoded_tokens = {1}; + auto meta = Meta(); + auto input = Prompt(); + CHECK(model->insert(meta, input)); + Phi4FrontendTestAccess::ForceRoute( + *model, + ForcedContinuationRoute::Append); + g_encoded_tokens = {1, 2}; + CHECK(model->insert(meta, input)); + + const std::string profile = model->show_profile(); + CHECK(profile.find("corelib_aie4") != std::string::npos); + CHECK(profile.find("Continuation route: append") != std::string::npos); + CHECK(profile.find("Append threshold: 0") != std::string::npos); + CHECK( + profile.find("fake_ryzenai_corelib.dll") != + std::string::npos); + CHECK(profile.find("Dispatches: 303") != std::string::npos); + CHECK(profile.find("Synchronizations: 404") != std::string::npos); + CHECK(profile.find("Helper transitions: 1/2/3/4/5/6") != + std::string::npos); + CHECK(profile.find("Cold model load: 101 ns") != + std::string::npos); + CHECK(profile.find("Cold weight pack: 202 ns") != + std::string::npos); + CHECK(profile.find("Continuation time:") != std::string::npos); +} + +#endif + +} // namespace + +int main() { + try { + TestContinuationSelector(); + TestLegacyRoutingAndUnknownBackend(); +#if defined(FLM_ENABLE_CORELIB_AIE4) + ConfigureFakeCorelibDll(); + TestCorelibRoutingAndPreemption(); + TestInitialAndAtomicCaps(); + TestRenderedCapacityIsAtomic(); + TestForcedAppendAndCancellationAlignment(); + TestForcedAndAutomaticReprefill(); + TestEosValidationAndFrontendStop(); + TestRecoverableFailuresClearSession(); + TestProfileSplit(); + Phi4FrontendTestAccess::RemoveFactory(); + flm::corelib::CorelibRuntime::ShutdownProcess(); + std::cout << "test_phi4_frontend_on: PASS\n"; +#else + TestFeatureOffRejectsCorelibTag(); + std::cout << "test_phi4_frontend_off: PASS\n"; +#endif + return 0; + } catch (const std::exception& error) { +#if defined(FLM_ENABLE_CORELIB_AIE4) + Phi4FrontendTestAccess::RemoveFactory(); + try { + flm::corelib::CorelibRuntime::ShutdownProcess(); + } catch (...) { + } +#endif + std::cerr << "test_phi4_frontend: FAIL: " + << error.what() << '\n'; + return 1; + } +} From e52f60effe371d993e67b3cfb5e2f4e4c3a64067 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CChi?= Date: Tue, 1 Sep 2026 03:28:45 -0700 Subject: [PATCH 017/117] fix: align Phi-4 AIE4 generation state Co-authored-by: Cursor --- src/common/AutoModel/automodel.cpp | 6 - src/common/AutoModel/modeling_phi4.cpp | 156 +++++- src/include/AutoModel/modeling_phi4.hpp | 5 + .../models/phi4/phi4_corelib_aie4_tuning.hpp | 3 + .../phi4_corelib_aie4/test_phi4_frontend.cpp | 520 ++++++++++++++---- src/test/phi4_corelib_aie4/test_support.hpp | 165 ++++++ 6 files changed, 712 insertions(+), 143 deletions(-) diff --git a/src/common/AutoModel/automodel.cpp b/src/common/AutoModel/automodel.cpp index 7a084773..636e4dcc 100644 --- a/src/common/AutoModel/automodel.cpp +++ b/src/common/AutoModel/automodel.cpp @@ -248,12 +248,6 @@ bool AutoModel::_shared_insert( return false; } - if (tokens.empty()) { - meta_info.prefill_duration = 0; - meta_info.prompt_tokens = 0; - return this->last_token != -1; - } - buffer y; auto prefill_start_time = this->profiler_list[PREFILL_TIME].start(); diff --git a/src/common/AutoModel/modeling_phi4.cpp b/src/common/AutoModel/modeling_phi4.cpp index b35843ef..04f647ca 100644 --- a/src/common/AutoModel/modeling_phi4.cpp +++ b/src/common/AutoModel/modeling_phi4.cpp @@ -350,6 +350,97 @@ void Phi4::validate_aie4_capacity( void Phi4::clear_after_corelib_error() { AutoModel::clear_context(); } + +std::string Phi4::generate_aie4( + chat_meta_info_t& meta_info, + int length_limit, + std::ostream& os, + std::function is_cancelled) { + std::string result; + stop_reason_t reason = EOT_DETECTED; + int generated_this_call = 0; + + this->profiler_list[DECODING_TIME].reset(); + this->profiler_list[TKOEN_DECODE_TIME].reset(); + + if (this->last_token == -1) { + throw std::logic_error( + "Phi-4 AIE4 generation has no sampled token"); + } + + while ( + this->last_token != -1 && + this->total_tokens < this->MAX_L) { + if (is_cancelled()) { + reason = CANCEL_DETECTED; + buffer_.clear(); + current_mode_ = StreamEventType::CONTENT; + tool_name_.clear(); + is_in_tool_block_ = false; + break; + } + + const int committed_token = this->last_token; + this->profiler_list[DECODING_TIME].start(); + buffer logits = + this->lm_engine->forward(committed_token); + this->profiler_list[DECODING_TIME].stop(1); + + this->token_history.push_back(committed_token); + ++this->total_tokens; + ++meta_info.generated_tokens; + ++generated_this_call; + + this->profiler_list[TKOEN_DECODE_TIME].start(); + if (this->is_normal_token(committed_token)) { + const std::string token_str = + this->tokenizer->run_time_decoder(committed_token); + os << token_str << std::flush; + result += token_str; + } + this->profiler_list[TKOEN_DECODE_TIME].stop(1); + + if (this->is_eos(committed_token)) { + this->last_token = -1; + break; + } + if ( + length_limit > 0 && + generated_this_call >= length_limit) { + this->last_token = -1; + reason = MAX_LENGTH_REACHED; + break; + } + if (this->total_tokens >= this->MAX_L) { + this->last_token = -1; + reason = MAX_LENGTH_REACHED; + break; + } + + this->profiler_list[SAMPLING_TIME].start(); + this->last_token = this->sampler->sample(logits); + this->profiler_list[SAMPLING_TIME].stop(1); + } + + if (this->total_tokens >= this->MAX_L) { + this->last_token = -1; + reason = MAX_LENGTH_REACHED; + header_print( + "WARNING", + "Max length reached, stopping generation..."); + } + meta_info.decoding_duration = + static_cast( + time_utils::cast_to_us( + this->profiler_list[DECODING_TIME] + .get_total_time()) + .first) * + 1e3; + meta_info.stop_reason = reason; + std::cout << std::endl; + header_print("FLM", "Model RAW Output: \n" + result); + return result; +} #endif bool Phi4::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std::function is_cancelled) { @@ -377,6 +468,7 @@ bool Phi4::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std::f this->validate_aie4_capacity( tokens.size(), input.requested_max_new_tokens); + meta_info.stop_reason = EOT_DETECTED; const size_t history_size = this->token_history.size(); const size_t matched = this->_matching_prefix_length(tokens); @@ -405,6 +497,8 @@ bool Phi4::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std::f nullptr, 0, action); + } catch (const ModelRequestError&) { + throw; } catch (const flm::corelib::CorelibError&) { this->clear_after_corelib_error(); throw ModelRequestError( @@ -412,6 +506,20 @@ bool Phi4::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std::f true, "AIE4 inference failed before submission; the " "current conversation was cleared."); + } catch (const std::exception&) { + this->clear_after_corelib_error(); + throw ModelRequestError( + 500, + true, + "AIE4 inference failed before submission; the " + "current conversation was cleared."); + } catch (...) { + this->clear_after_corelib_error(); + throw ModelRequestError( + 500, + true, + "AIE4 inference failed before submission; the " + "current conversation was cleared."); } const auto elapsed = @@ -444,11 +552,13 @@ std::string Phi4::generate(chat_meta_info_t& meta_info, int length_limit, std::o #if defined(FLM_ENABLE_CORELIB_AIE4) if (this->uses_corelib_aie4_) { try { - return this->_shared_generate( + return this->generate_aie4( meta_info, length_limit, os, std::move(is_cancelled)); + } catch (const ModelRequestError&) { + throw; } catch (const flm::corelib::CorelibError&) { this->clear_after_corelib_error(); throw ModelRequestError( @@ -456,6 +566,20 @@ std::string Phi4::generate(chat_meta_info_t& meta_info, int length_limit, std::o true, "AIE4 inference failed before submission; the " "current conversation was cleared."); + } catch (const std::exception&) { + this->clear_after_corelib_error(); + throw ModelRequestError( + 500, + true, + "AIE4 inference failed before submission; the " + "current conversation was cleared."); + } catch (...) { + this->clear_after_corelib_error(); + throw ModelRequestError( + 500, + true, + "AIE4 inference failed before submission; the " + "current conversation was cleared."); } } #endif @@ -467,24 +591,9 @@ std::string Phi4::generate(chat_meta_info_t& meta_info, int length_limit, std::o } std::string Phi4::generate_with_prompt(chat_meta_info_t& meta_info, lm_uniform_input_t& input, int length_limit, std::ostream& os) { - const std::optional caller_limit = - input.requested_max_new_tokens; - if ( - !input.requested_max_new_tokens.has_value() && - length_limit >= 0) { - input.requested_max_new_tokens = length_limit; - } - - try { - if (!this->insert(meta_info, input)) { - input.requested_max_new_tokens = caller_limit; - return ""; - } - } catch (...) { - input.requested_max_new_tokens = caller_limit; - throw; + if (!this->insert(meta_info, input)) { + return ""; } - input.requested_max_new_tokens = caller_limit; return this->generate(meta_info, length_limit, os); } @@ -497,21 +606,14 @@ void Phi4::set_max_length(unsigned int requested_max_length) { throw std::out_of_range( "Phi-4 AIE4 maximum length must be in 1..4096"); } - const int frontend_position = - AutoModel::get_current_context_length(); const int engine_position = this->lm_engine->get_current_context_length(); - if (frontend_position != engine_position) { - throw std::logic_error( - "Phi-4 AIE4 frontend and engine context positions " - "are inconsistent"); - } if ( requested_max_length < - static_cast(frontend_position)) { + static_cast(engine_position)) { throw std::out_of_range( "Phi-4 AIE4 maximum length cannot be below the " - "current logical position"); + "current engine position"); } this->lm_engine->update_max_length(requested_max_length); diff --git a/src/include/AutoModel/modeling_phi4.hpp b/src/include/AutoModel/modeling_phi4.hpp index dcce242d..cedfe004 100644 --- a/src/include/AutoModel/modeling_phi4.hpp +++ b/src/include/AutoModel/modeling_phi4.hpp @@ -37,6 +37,11 @@ class Phi4 : public AutoModel { size_t rendered_tokens, std::optional requested_max_new_tokens) const; void clear_after_corelib_error(); + std::string generate_aie4( + chat_meta_info_t& meta_info, + int length_limit, + std::ostream& os, + std::function is_cancelled); const flm::phi4::Phi4Aie4Metrics& aie4_metrics() const; bool uses_corelib_aie4_ = false; diff --git a/src/include/models/phi4/phi4_corelib_aie4_tuning.hpp b/src/include/models/phi4/phi4_corelib_aie4_tuning.hpp index 4a4bcba9..dd19a176 100644 --- a/src/include/models/phi4/phi4_corelib_aie4_tuning.hpp +++ b/src/include/models/phi4/phi4_corelib_aie4_tuning.hpp @@ -21,6 +21,9 @@ inline constexpr std::uint32_t kContinuationAppendThreshold = 0; inline constexpr ContinuationRoute SelectContinuationRoute( std::size_t suffix_tokens, ForcedContinuationRoute forced) noexcept { + if (suffix_tokens == 0) { + return ContinuationRoute::Reprefill; + } switch (forced) { case ForcedContinuationRoute::Append: return ContinuationRoute::Append; diff --git a/src/test/phi4_corelib_aie4/test_phi4_frontend.cpp b/src/test/phi4_corelib_aie4/test_phi4_frontend.cpp index ea003f80..fad4fc2d 100644 --- a/src/test/phi4_corelib_aie4/test_phi4_frontend.cpp +++ b/src/test/phi4_corelib_aie4/test_phi4_frontend.cpp @@ -1,5 +1,3 @@ -#include "test_support.hpp" - #include #include @@ -8,10 +6,14 @@ #include #endif +#define FLM_PHI4_FRONTEND_TEST_SUPPORT +#include "test_support.hpp" + #include #include #include +#include #include #include #include @@ -29,6 +31,7 @@ namespace { std::vector g_encoded_tokens; int g_sample_token = 7; +std::deque g_sample_tokens; int g_sampler_reset_count = 0; int g_sampler_sample_count = 0; @@ -126,110 +129,6 @@ class TempModelPackage final { std::filesystem::path path_; }; -class FakeEngine final : public causal_lm { -public: - explicit FakeEngine(std::uint32_t max_length) - : max_length(max_length) {} - - buffer forward(int id) override { -#if defined(FLM_ENABLE_CORELIB_AIE4) - if (fail_next_forward) { - fail_next_forward = false; - throw flm::corelib::CorelibError( - ryzenai_corelib_status_failure, - "fake_forward", - "injected pre-submit forward failure", - "failure"); - } -#endif - forward_tokens.push_back(id); - ++position; - return MakeLogits(); - } - - buffer prefill( - std::vector& ids, - void*) override { -#if defined(FLM_ENABLE_CORELIB_AIE4) - if (fail_next_prefill) { - fail_next_prefill = false; - throw flm::corelib::CorelibError( - ryzenai_corelib_status_failure, - "fake_prefill", - "injected pre-submit prefill failure", - "failure"); - } -#endif - prefill_calls.push_back(ids); - position += static_cast(ids.size()); - return MakeLogits(); - } - - void set_context_length(int length) override { - position = length; - } - - void load_weights(Q4NX&) override {} - - void update_max_length(std::uint32_t requested) override { - if (requested == 0 || requested > 4096) { - throw std::out_of_range( - "fake AIE4 maximum length must be in 1..4096"); - } - if (requested < static_cast(position)) { - throw std::out_of_range( - "fake AIE4 maximum length cannot be below current"); - } - max_length = requested; - } - - void clear_context() override { - ++clear_count; - position = 0; - checkpoint_position.reset(); - } - - buffer get_k_cache(int, int) override { - return MakeLogits(); - } - - buffer get_v_cache(int, int) override { - return MakeLogits(); - } - - int get_current_context_length() override { - return position; - } - - int checkpoint() override { - checkpoint_position = position; - return position; - } - - int restore() override { - if (!checkpoint_position.has_value()) { - throw std::logic_error("fake engine has no checkpoint"); - } - position = *checkpoint_position; - return position; - } - - static buffer MakeLogits() { - return buffer(1); - } - - std::uint32_t max_length; - int position = 0; - int clear_count = 0; - std::optional checkpoint_position; - std::vector> prefill_calls; - std::vector forward_tokens; -#if defined(FLM_ENABLE_CORELIB_AIE4) - bool fail_next_prefill = false; - bool fail_next_forward = false; -#endif -}; - struct FactoryState { int calls = 0; bool last_was_corelib = false; @@ -267,6 +166,10 @@ lm_uniform_input_t Prompt( return input; } +void SetSamples(std::initializer_list samples) { + g_sample_tokens.assign(samples.begin(), samples.end()); +} + template void CheckRequestError( Function&& function, @@ -339,8 +242,14 @@ void Sampler::reset_penalties() { int Sampler::sample(buffer&) { ++g_sampler_sample_count; ++total_tokens; - token_history.push_back(g_sample_token); - return g_sample_token; + const int sampled_token = g_sample_tokens.empty() + ? g_sample_token + : g_sample_tokens.front(); + if (!g_sample_tokens.empty()) { + g_sample_tokens.pop_front(); + } + token_history.push_back(sampled_token); + return sampled_token; } namespace utils { @@ -409,6 +318,23 @@ class Phi4FrontendTestAccess final { static const std::vector& History(const Phi4& model) { return model.token_history; } + + static int LastToken(const Phi4& model) { + return model.last_token; + } + + static bool SharedInsert( + Phi4& model, + chat_meta_info_t& meta, + std::vector& tokens, + void* payload) { + return model._shared_insert( + meta, + tokens, + [] { return false; }, + payload, + 0); + } }; } // namespace flm::phi4::testing @@ -451,12 +377,17 @@ void TestContinuationSelector() { SelectContinuationRoute( 0, ForcedContinuationRoute::Automatic) == - ContinuationRoute::Append); + ContinuationRoute::Reprefill); CHECK( SelectContinuationRoute( 1, ForcedContinuationRoute::Automatic) == ContinuationRoute::Reprefill); + CHECK( + SelectContinuationRoute( + 0, + ForcedContinuationRoute::Append) == + ContinuationRoute::Reprefill); CHECK( SelectContinuationRoute( 999, @@ -509,6 +440,35 @@ void TestLegacyRoutingAndUnknownBackend() { CHECK(g_factory.calls == 1); } +void TestLegacyExactRepeatPreservesEmptyPrefillPayload() { + TempModelPackage package({200020}); + FactoryScope factory; + auto model = Load(package, ModelInfo(64)); + FakeEngine* engine = g_factory.engine; + + int first_payload = 1; + int repeated_payload = 2; + auto meta = Meta(); + std::vector first{1, 2}; + CHECK(Phi4FrontendTestAccess::SharedInsert( + *model, + meta, + first, + &first_payload)); + const int samples_before_repeat = g_sampler_sample_count; + + std::vector repeated{1, 2}; + CHECK(Phi4FrontendTestAccess::SharedInsert( + *model, + meta, + repeated, + &repeated_payload)); + CHECK(engine->prefill_calls.size() == 2); + CHECK(engine->prefill_calls.back().empty()); + CHECK(engine->prefill_payloads.back() == &repeated_payload); + CHECK(g_sampler_sample_count == samples_before_repeat + 1); +} + #if !defined(FLM_ENABLE_CORELIB_AIE4) void TestFeatureOffRejectsCorelibTag() { @@ -627,6 +587,34 @@ void TestInitialAndAtomicCaps() { CHECK(engine->position == 10); } +void TestEnginePositionIsAuthoritativeForCapUpdate() { + TempModelPackage package; + FactoryScope factory; + auto model = Load( + package, + ModelInfo(64, "corelib_aie4"), + -1, + false, + nullptr); + FakeEngine* engine = g_factory.engine; + + g_encoded_tokens.assign(10, 11); + auto meta = Meta(); + auto input = Prompt(); + CHECK(model->insert(meta, input)); + engine->position = 11; + + model->set_max_length(11); + CHECK(model->get_max_length() == 11); + CHECK(engine->max_length == 11); + + CheckThrowsContains( + [&] { model->set_max_length(10); }, + "engine position"); + CHECK(model->get_max_length() == 11); + CHECK(engine->max_length == 11); +} + void TestRenderedCapacityIsAtomic() { TempModelPackage package; FactoryScope factory; @@ -676,6 +664,222 @@ void TestRenderedCapacityIsAtomic() { CHECK(Phi4FrontendTestAccess::History(*model).size() == 500); } +void CheckAligned( + Phi4& model, + const FakeEngine& engine, + const std::vector& expected_history) { + CHECK(Phi4FrontendTestAccess::History(model) == expected_history); + CHECK( + model.get_current_context_length() == + static_cast(expected_history.size())); + CHECK( + engine.position == + static_cast(expected_history.size())); +} + +void TestDefaultChatLimitWithoutExplicitRequestIsAdmitted() { + TempModelPackage package; + FactoryScope factory; + auto model = Load( + package, + ModelInfo(4096, "corelib_aie4"), + -1, + false, + nullptr); + + g_encoded_tokens = {41}; + SetSamples({200020}); + auto meta = Meta(); + auto input = Prompt(); + std::ostringstream output; + CHECK( + model->generate_with_prompt( + meta, + input, + 4096, + output) + .empty()); + CHECK(!input.requested_max_new_tokens.has_value()); + CheckAligned(*model, *g_factory.engine, {41, 200020}); +} + +void TestLengthStopCommitsTokensBeforeForcedAppend() { + TempModelPackage package; + FactoryScope factory; + auto model = Load( + package, + ModelInfo(64, "corelib_aie4"), + -1, + false, + nullptr); + FakeEngine* engine = g_factory.engine; + + g_encoded_tokens = {1, 2}; + SetSamples({101, 102}); + auto meta = Meta(); + auto input = Prompt(); + CHECK(model->insert(meta, input)); + std::ostringstream output; + CHECK( + model->generate(meta, 2, output) == + "token-101token-102"); + CHECK(meta.generated_tokens == 2); + CHECK(meta.stop_reason == MAX_LENGTH_REACHED); + CheckAligned(*model, *engine, {1, 2, 101, 102}); + + model->set_max_length(16); + CHECK(model->get_max_length() == 16); + CHECK(engine->max_length == 16); + + Phi4FrontendTestAccess::ForceRoute( + *model, + ForcedContinuationRoute::Append); + engine->prefill_calls.clear(); + g_encoded_tokens = {1, 2, 101, 102, 3}; + SetSamples({103}); + auto append_meta = Meta(); + CHECK(model->insert(append_meta, input)); + CHECK(engine->prefill_calls == std::vector>({{3}})); + CheckAligned(*model, *engine, {1, 2, 101, 102, 3}); +} + +void TestEosStopCommitsTokenBeforeCapUpdateAndAppend() { + TempModelPackage package; + FactoryScope factory; + auto model = Load( + package, + ModelInfo(64, "corelib_aie4"), + -1, + false, + nullptr); + FakeEngine* engine = g_factory.engine; + + g_encoded_tokens = {10}; + SetSamples({101, 200020}); + auto meta = Meta(); + auto input = Prompt(); + CHECK(model->insert(meta, input)); + std::ostringstream output; + CHECK(model->generate(meta, 8, output) == "token-101"); + CHECK(meta.generated_tokens == 2); + CHECK(meta.stop_reason == EOT_DETECTED); + CheckAligned(*model, *engine, {10, 101, 200020}); + + model->set_max_length(8); + CHECK(model->get_max_length() == 8); + CHECK(engine->max_length == 8); + + Phi4FrontendTestAccess::ForceRoute( + *model, + ForcedContinuationRoute::Append); + engine->prefill_calls.clear(); + g_encoded_tokens = {10, 101, 200020, 11}; + SetSamples({102}); + auto append_meta = Meta(); + CHECK(model->insert(append_meta, input)); + CHECK(engine->prefill_calls == std::vector>({{11}})); + CheckAligned(*model, *engine, {10, 101, 200020, 11}); +} + +void TestActiveCapNeverEmitsUncommittedToken() { + TempModelPackage package; + FactoryScope factory; + auto model = Load( + package, + ModelInfo(3, "corelib_aie4"), + -1, + false, + nullptr); + FakeEngine* engine = g_factory.engine; + + g_encoded_tokens = {1, 2}; + SetSamples({101, 102}); + auto meta = Meta(); + auto input = Prompt(); + CHECK(model->insert(meta, input)); + std::ostringstream output; + CHECK(model->generate(meta, 8, output) == "token-101"); + CHECK(meta.generated_tokens == 1); + CHECK(meta.stop_reason == MAX_LENGTH_REACHED); + CHECK(engine->forward_tokens == std::vector({101})); + CHECK(g_sample_tokens.size() == 1); + CheckAligned(*model, *engine, {1, 2, 101}); +} + +void TestCancellationLeavesOnlyCommittedTokensVisible() { + TempModelPackage package; + FactoryScope factory; + auto model = Load( + package, + ModelInfo(64, "corelib_aie4"), + -1, + false, + nullptr); + FakeEngine* engine = g_factory.engine; + + g_encoded_tokens = {1, 2}; + SetSamples({101, 102}); + auto meta = Meta(); + auto input = Prompt(); + CHECK(model->insert(meta, input)); + int cancellation_checks = 0; + std::ostringstream output; + CHECK( + model->generate( + meta, + 8, + output, + [&] { return cancellation_checks++ == 1; }) == + "token-101"); + CHECK(meta.generated_tokens == 1); + CHECK(meta.stop_reason == CANCEL_DETECTED); + CHECK(engine->forward_tokens == std::vector({101})); + CheckAligned(*model, *engine, {1, 2, 101}); +} + +void TestExactRepeatReprefillsAndSamplesFreshToken() { + TempModelPackage package; + FactoryScope factory; + auto model = Load( + package, + ModelInfo(64, "corelib_aie4"), + -1, + false, + nullptr); + FakeEngine* engine = g_factory.engine; + + g_encoded_tokens = {1}; + SetSamples({101}); + auto first_meta = Meta(); + auto input = Prompt(); + CHECK(model->insert(first_meta, input)); + std::ostringstream first_output; + CHECK(model->generate(first_meta, 1, first_output) == "token-101"); + CheckAligned(*model, *engine, {1, 101}); + + const int clear_before = engine->clear_count; + engine->prefill_calls.clear(); + Phi4FrontendTestAccess::ForceRoute( + *model, + ForcedContinuationRoute::Append); + g_encoded_tokens = {1, 101}; + SetSamples({202}); + auto repeated_meta = Meta(); + CHECK(model->insert(repeated_meta, input)); + CHECK(engine->clear_count == clear_before + 1); + CHECK( + engine->prefill_calls == + std::vector>({{1, 101}})); + std::ostringstream repeated_output; + CHECK( + model->generate(repeated_meta, 1, repeated_output) == + "token-202"); + CHECK( + engine->forward_tokens == + std::vector({101, 202})); + CheckAligned(*model, *engine, {1, 101, 202}); +} + void TestForcedAppendAndCancellationAlignment() { TempModelPackage package; FactoryScope factory; @@ -789,7 +993,8 @@ void TestEosValidationAndFrontendStop() { CHECK(model->insert(meta, input)); std::ostringstream output; CHECK(model->generate(meta, 8, output).empty()); - CHECK(engine->forward_tokens.empty()); + CHECK(engine->forward_tokens == std::vector({200020})); + CheckAligned(*model, *engine, {10, 11, 200020}); g_sample_token = 7; } @@ -842,6 +1047,90 @@ void TestRecoverableFailuresClearSession() { CHECK(g_sampler_reset_count > resets_before_generate); } +void CheckPartialAppendFailureClearsSession( + FakeEngine::FailureKind failure) { + TempModelPackage package; + FactoryScope factory; + auto model = Load( + package, + ModelInfo(64, "corelib_aie4"), + -1, + false, + nullptr); + FakeEngine* engine = g_factory.engine; + + g_encoded_tokens = {1}; + SetSamples({101}); + auto meta = Meta(); + auto input = Prompt(); + CHECK(model->insert(meta, input)); + engine->checkpoint(); + Phi4FrontendTestAccess::ForceRoute( + *model, + ForcedContinuationRoute::Append); + engine->prefill_failure = failure; + engine->successful_prefills_before_failure = 1; + const int resets_before = g_sampler_reset_count; + g_encoded_tokens = {1, 2, 3, 4}; + + CheckRequestError( + [&] { (void)model->insert(meta, input); }, + 500, + true, + "current conversation was cleared"); + CHECK(engine->prefill_calls.back() == std::vector({2})); + CHECK(engine->position == 0); + CHECK(!engine->checkpoint_position.has_value()); + CHECK(model->get_current_context_length() == 0); + CHECK(Phi4FrontendTestAccess::History(*model).empty()); + CHECK(Phi4FrontendTestAccess::LastToken(*model) == -1); + CHECK(g_sampler_reset_count > resets_before); +} + +void TestPartialAppendStandardFailureClearsCommittedPrefix() { + CheckPartialAppendFailureClearsSession( + FakeEngine::FailureKind::Standard); +} + +void TestPartialAppendUnknownFailureClearsCommittedPrefix() { + CheckPartialAppendFailureClearsSession( + FakeEngine::FailureKind::Unknown); +} + +void TestStandardGenerateFailureClearsSession() { + TempModelPackage package; + FactoryScope factory; + auto model = Load( + package, + ModelInfo(64, "corelib_aie4"), + -1, + false, + nullptr); + FakeEngine* engine = g_factory.engine; + + g_encoded_tokens = {31, 32}; + SetSamples({101}); + auto meta = Meta(); + auto input = Prompt(); + CHECK(model->insert(meta, input)); + engine->checkpoint(); + engine->forward_failure = FakeEngine::FailureKind::Standard; + const int resets_before = g_sampler_reset_count; + std::ostringstream output; + + CheckRequestError( + [&] { (void)model->generate(meta, 8, output); }, + 500, + true, + "current conversation was cleared"); + CHECK(engine->position == 0); + CHECK(!engine->checkpoint_position.has_value()); + CHECK(model->get_current_context_length() == 0); + CHECK(Phi4FrontendTestAccess::History(*model).empty()); + CHECK(Phi4FrontendTestAccess::LastToken(*model) == -1); + CHECK(g_sampler_reset_count > resets_before); +} + void TestProfileSplit() { TempModelPackage package; FactoryScope factory; @@ -896,15 +1185,26 @@ int main() { try { TestContinuationSelector(); TestLegacyRoutingAndUnknownBackend(); + TestLegacyExactRepeatPreservesEmptyPrefillPayload(); #if defined(FLM_ENABLE_CORELIB_AIE4) ConfigureFakeCorelibDll(); TestCorelibRoutingAndPreemption(); TestInitialAndAtomicCaps(); + TestEnginePositionIsAuthoritativeForCapUpdate(); TestRenderedCapacityIsAtomic(); + TestDefaultChatLimitWithoutExplicitRequestIsAdmitted(); + TestLengthStopCommitsTokensBeforeForcedAppend(); + TestEosStopCommitsTokenBeforeCapUpdateAndAppend(); + TestActiveCapNeverEmitsUncommittedToken(); + TestCancellationLeavesOnlyCommittedTokensVisible(); + TestExactRepeatReprefillsAndSamplesFreshToken(); TestForcedAppendAndCancellationAlignment(); TestForcedAndAutomaticReprefill(); TestEosValidationAndFrontendStop(); TestRecoverableFailuresClearSession(); + TestPartialAppendStandardFailureClearsCommittedPrefix(); + TestPartialAppendUnknownFailureClearsCommittedPrefix(); + TestStandardGenerateFailureClearsSession(); TestProfileSplit(); Phi4FrontendTestAccess::RemoveFactory(); flm::corelib::CorelibRuntime::ShutdownProcess(); diff --git a/src/test/phi4_corelib_aie4/test_support.hpp b/src/test/phi4_corelib_aie4/test_support.hpp index cd1b5b10..b472fd19 100644 --- a/src/test/phi4_corelib_aie4/test_support.hpp +++ b/src/test/phi4_corelib_aie4/test_support.hpp @@ -31,3 +31,168 @@ void CheckThrowsContains(Function&& function, std::string_view expected) { } throw std::runtime_error("expected exception was not thrown"); } + +#if defined(FLM_PHI4_FRONTEND_TEST_SUPPORT) + +#include +#include +#include +#include +#include + +class FakeEngine final : public causal_lm { +public: + enum class FailureKind { + None, + Corelib, + Standard, + Unknown, + }; + + explicit FakeEngine(std::uint32_t max_length) + : max_length(max_length) {} + + buffer forward(int id) override { +#if defined(FLM_ENABLE_CORELIB_AIE4) + if (fail_next_forward) { + fail_next_forward = false; + throw flm::corelib::CorelibError( + ryzenai_corelib_status_failure, + "fake_forward", + "injected pre-submit forward failure", + "failure"); + } + MaybeFail( + forward_failure, + successful_forwards_before_failure, + "fake_forward"); +#endif + forward_tokens.push_back(id); + ++position; + return MakeLogits(); + } + + buffer prefill( + std::vector& ids, + void* payload) override { +#if defined(FLM_ENABLE_CORELIB_AIE4) + if (fail_next_prefill) { + fail_next_prefill = false; + throw flm::corelib::CorelibError( + ryzenai_corelib_status_failure, + "fake_prefill", + "injected pre-submit prefill failure", + "failure"); + } + MaybeFail( + prefill_failure, + successful_prefills_before_failure, + "fake_prefill"); +#endif + prefill_calls.push_back(ids); + prefill_payloads.push_back(payload); + position += static_cast(ids.size()); + return MakeLogits(); + } + + void set_context_length(int length) override { + position = length; + } + + void load_weights(Q4NX&) override {} + + void update_max_length(std::uint32_t requested) override { + if (requested == 0 || requested > 4096) { + throw std::out_of_range( + "fake AIE4 maximum length must be in 1..4096"); + } + if (requested < static_cast(position)) { + throw std::out_of_range( + "fake AIE4 maximum length cannot be below current"); + } + max_length = requested; + } + + void clear_context() override { + ++clear_count; + position = 0; + checkpoint_position.reset(); + } + + buffer get_k_cache(int, int) override { + return MakeLogits(); + } + + buffer get_v_cache(int, int) override { + return MakeLogits(); + } + + int get_current_context_length() override { + return position; + } + + int checkpoint() override { + checkpoint_position = position; + return position; + } + + int restore() override { + if (!checkpoint_position.has_value()) { + throw std::logic_error("fake engine has no checkpoint"); + } + position = *checkpoint_position; + return position; + } + + static buffer MakeLogits() { + return buffer(1); + } + +#if defined(FLM_ENABLE_CORELIB_AIE4) + static void MaybeFail( + FailureKind& failure, + int& successful_calls_before_failure, + const char* operation) { + if (failure == FailureKind::None) { + return; + } + if (successful_calls_before_failure > 0) { + --successful_calls_before_failure; + return; + } + + const FailureKind injected = std::exchange( + failure, + FailureKind::None); + if (injected == FailureKind::Corelib) { + throw flm::corelib::CorelibError( + ryzenai_corelib_status_failure, + operation, + "injected pre-submit corelib failure", + "failure"); + } + if (injected == FailureKind::Standard) { + throw std::bad_alloc(); + } + throw 17; + } +#endif + + std::uint32_t max_length; + int position = 0; + int clear_count = 0; + std::optional checkpoint_position; + std::vector> prefill_calls; + std::vector prefill_payloads; + std::vector forward_tokens; +#if defined(FLM_ENABLE_CORELIB_AIE4) + bool fail_next_prefill = false; + bool fail_next_forward = false; + FailureKind prefill_failure = FailureKind::None; + FailureKind forward_failure = FailureKind::None; + int successful_prefills_before_failure = 0; + int successful_forwards_before_failure = 0; +#endif +}; + +#endif From 2fb9302eae2b95c1ee41215fe3ed40f28427a447 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CChi?= Date: Tue, 1 Sep 2026 04:13:45 -0700 Subject: [PATCH 018/117] fix: enforce Phi-4 AIE4 request limits Co-authored-by: Cursor --- src/CMakeLists.txt | 2 +- src/common/corelib/corelib_fatal_record.cpp | 8 + src/include/corelib/corelib_fatal_record.hpp | 2 + src/include/server/generation_limit.hpp | 53 +++ src/include/server/npu_access_manager.hpp | 15 + src/runner/runner.cpp | 45 ++- src/server/generation_limit.cpp | 139 ++++++++ src/server/npu_access_manager.cpp | 50 +++ src/server/rest_handler.cpp | 198 ++++++++++- src/server/rest_handler.hpp | 1 + src/server/server.cpp | 70 +--- src/server/server.hpp | 18 +- src/src/main.cpp | 246 ++++++++++++-- src/test/phi4_corelib_aie4/CMakeLists.txt | 80 ++++- .../test_generation_limit.cpp | 308 ++++++++++++++++++ .../phi4_corelib_aie4/test_phi4_frontend.cpp | 68 ++++ 16 files changed, 1169 insertions(+), 134 deletions(-) create mode 100644 src/include/server/generation_limit.hpp create mode 100644 src/include/server/npu_access_manager.hpp create mode 100644 src/server/generation_limit.cpp create mode 100644 src/server/npu_access_manager.cpp create mode 100644 src/test/phi4_corelib_aie4/test_generation_limit.cpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 2d7f3275..ec5a7d7f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -271,7 +271,7 @@ add_subdirectory(${CMAKE_SOURCE_DIR}/../third_party/tokenizers-cpp # ——————————————————————————————————————————————— file(GLOB SOURCES "src/*.cpp" "runner/*.cpp" "common/*.cpp" "common/*/*.cpp" "server/*.cpp" "pull/*.cpp" ) list(FILTER SOURCES EXCLUDE REGEX ".*/common/corelib/.*\\.cpp$") -file(GLOB HEADERS "include/*.hpp" "runner/*.hpp" "common/*.hpp" "common/*/*.hpp" "server/*.hpp" "pull/*.hpp") +file(GLOB HEADERS "include/*.hpp" "include/server/*.hpp" "runner/*.hpp" "common/*.hpp" "common/*/*.hpp" "server/*.hpp" "pull/*.hpp") # Exclude files that depend on missing libraries for Linux if(NOT WIN32) diff --git a/src/common/corelib/corelib_fatal_record.cpp b/src/common/corelib/corelib_fatal_record.cpp index 767e8443..ddfda968 100644 --- a/src/common/corelib/corelib_fatal_record.cpp +++ b/src/common/corelib/corelib_fatal_record.cpp @@ -575,6 +575,14 @@ std::filesystem::path FatalRecordStore::Persist( return final_path; } +std::vector FatalRecordStore::DrainPriorRecords( + std::ostream& output) { + return DrainPriorRecords( + LocalAppDataLogRoot(), + ProbeProcessStart, + output); +} + std::vector FatalRecordStore::DrainPriorRecords( const std::filesystem::path& root, ProcessProbe process_probe, diff --git a/src/include/corelib/corelib_fatal_record.hpp b/src/include/corelib/corelib_fatal_record.hpp index 72c8c0c6..2b8464c5 100644 --- a/src/include/corelib/corelib_fatal_record.hpp +++ b/src/include/corelib/corelib_fatal_record.hpp @@ -46,6 +46,8 @@ class FatalRecordStore { void Prepare(); std::filesystem::path Persist(const FailureContext& failure); + static std::vector DrainPriorRecords( + std::ostream& output); static std::vector DrainPriorRecords( const std::filesystem::path& root, ProcessProbe process_probe, diff --git a/src/include/server/generation_limit.hpp b/src/include/server/generation_limit.hpp new file mode 100644 index 00000000..cc836872 --- /dev/null +++ b/src/include/server/generation_limit.hpp @@ -0,0 +1,53 @@ +#pragma once + +#include + +#include +#include +#include + +enum class GenerationEndpoint { + Generate, + OpenAiChatCompletion, + OpenAiCompletion +}; + +struct ParsedGenerationLimit { + bool explicit_limit; + int value; +}; + +ParsedGenerationLimit ParseGenerationLimit( + const nlohmann::ordered_json& request, + GenerationEndpoint endpoint); + +int GenerationLoopLimit( + const ParsedGenerationLimit& parsed, + bool uses_corelib_aie4) noexcept; + +std::optional RequestedMaxNewTokens( + const ParsedGenerationLimit& parsed) noexcept; + +int OllamaChatGenerationLoopLimit( + const nlohmann::ordered_json& request); + +nlohmann::ordered_json ModelErrorResponse( + std::string_view message, + int http_code, + bool session_cleared); + +int HttpStatusForResponse( + const nlohmann::ordered_json& response) noexcept; + +bool UseFinalStreamingErrorChunk( + bool stream_started) noexcept; + +std::optional CliRequestedMaxNewTokens( + int generate_limit) noexcept; + +std::string CliModelErrorNotice( + std::string_view message, + bool session_cleared); + +bool IsCorelibAie4ModelInfo( + const nlohmann::ordered_json& model_info) noexcept; diff --git a/src/include/server/npu_access_manager.hpp b/src/include/server/npu_access_manager.hpp new file mode 100644 index 00000000..6245cc4b --- /dev/null +++ b/src/include/server/npu_access_manager.hpp @@ -0,0 +1,15 @@ +#pragma once + +#include + +class NPUAccessManager { +public: + static bool try_acquire_npu_access(); + static void release_npu_access(); + static bool is_npu_available(); + static int get_active_npu_requests(); +}; + +bool requires_npu_access( + const std::string& method, + const std::string& path); diff --git a/src/runner/runner.cpp b/src/runner/runner.cpp index d38e5e9c..3fb3bf70 100644 --- a/src/runner/runner.cpp +++ b/src/runner/runner.cpp @@ -8,6 +8,7 @@ */ #include "runner.hpp" #include "harmony_filter.hpp" +#include #ifndef FASTFLOWLM_LINUX_LIMITED_MODELS #include "AutoEmbeddingModel/all_embedding_model.hpp" #endif @@ -18,9 +19,22 @@ #include #include #include +#include #include #include +namespace { + +void PrintModelRequestError(const ModelRequestError& error) { + std::cerr + << CliModelErrorNotice( + error.what(), + error.session_cleared()) + << '\n'; +} + +} // namespace + /// \brief Command map for command line input std::map cmd_map = { {"/set", CMD_SET}, @@ -66,7 +80,9 @@ Runner::Runner(model_list& supported_models, ModelDownloader& downloader, progra this->downloader.pull_model(this->tag, this->modelscope); break; case ModelDownloader::ModelStatus::Incompatible: - exit(EXIT_FAILURE); + throw std::runtime_error( + "Model is incompatible with this version of FastFlowLM: " + + this->tag); } auto [new_tag, model_info] = this->supported_models.get_model_info(this->tag); this->asr_supported = model_info.contains("asr") && model_info["asr"]; @@ -77,7 +93,7 @@ Runner::Runner(model_list& supported_models, ModelDownloader& downloader, progra } catch (const std::exception& e) { header_print("ERROR", "Failed to load model: " + std::string(e.what())); - exit(EXIT_FAILURE); + throw; } try { @@ -127,7 +143,7 @@ Runner::Runner(model_list& supported_models, ModelDownloader& downloader, progra } catch (const std::exception& e) { header_print("WARNING", "Failed to load ASR model: " + std::string(e.what())); - exit(EXIT_FAILURE); + throw; } } else { @@ -358,6 +374,8 @@ void Runner::run() { chat_meta_info_t meta_info; meta_info.max_prefill_len = this->prefill_chunk_len; uniformed_input.prompt = input; + uniformed_input.requested_max_new_tokens = + CliRequestedMaxNewTokens(this->generate_limit); this->auto_chat_engine->start_total_timer(); @@ -369,6 +387,10 @@ void Runner::run() { break; } } + catch (const ModelRequestError& error) { + PrintModelRequestError(error); + continue; + } catch (const std::exception& e) { header_print("ERROR", "Insertion error: " + std::string(e.what())); this->auto_chat_engine->clear_context(); @@ -383,6 +405,10 @@ void Runner::run() { try { this->auto_chat_engine->generate(meta_info, this->generate_limit, harmony_filter_ostream); } + catch (const ModelRequestError& error) { + PrintModelRequestError(error); + continue; + } catch (const std::exception& e) { header_print("ERROR", "Generation error: " + std::string(e.what())); this->auto_chat_engine->clear_context(); @@ -392,6 +418,10 @@ void Runner::run() { try { this->auto_chat_engine->generate(meta_info, this->generate_limit, base_ostream); } + catch (const ModelRequestError& error) { + PrintModelRequestError(error); + continue; + } catch (const std::exception& e) { header_print("ERROR", "Generation error: " + std::string(e.what())); this->auto_chat_engine->clear_context(); @@ -440,7 +470,10 @@ void Runner::cmd_load(std::vector& input_list) { break; case ModelDownloader::ModelStatus::Incompatible: header_print("ERROR", "Model is incompatible with this version of FastFlowLM: " + this->tag); - exit(EXIT_FAILURE); + throw std::runtime_error( + "Model is incompatible with this version of " + "FastFlowLM: " + + this->tag); } auto_chat_engine.reset(); if(model_name=="gpt-oss:20b") @@ -454,7 +487,7 @@ void Runner::cmd_load(std::vector& input_list) { } catch (const std::exception& e) { header_print("ERROR", "Failed to load model: " + std::string(e.what())); - exit(EXIT_FAILURE); + throw; } this->auto_chat_engine->configure_parameter("system_prompt", this->system_prompt); @@ -604,7 +637,7 @@ void Runner::cmd_set(std::vector& input_list) { } catch (const std::exception& e) { header_print("ERROR", "Failed to set context length: " + std::string(e.what())); - exit(EXIT_FAILURE); + return; } } else if (set_context == "gen-lim"){ diff --git a/src/server/generation_limit.cpp b/src/server/generation_limit.cpp new file mode 100644 index 00000000..24efce82 --- /dev/null +++ b/src/server/generation_limit.cpp @@ -0,0 +1,139 @@ +#include + +namespace { + +constexpr int kLegacyDefaultGenerationLimit = 4096; +constexpr int kNoExplicitGenerationLimit = -1; + +ParsedGenerationLimit ParseField( + const nlohmann::ordered_json& request, + std::string_view field) { + if (!request.contains(field)) { + return {false, kNoExplicitGenerationLimit}; + } + return {true, request.at(field).get()}; +} + +} // namespace + +ParsedGenerationLimit ParseGenerationLimit( + const nlohmann::ordered_json& request, + GenerationEndpoint endpoint) { + switch (endpoint) { + case GenerationEndpoint::Generate: + case GenerationEndpoint::OpenAiCompletion: + return ParseField(request, "max_tokens"); + case GenerationEndpoint::OpenAiChatCompletion: { + const ParsedGenerationLimit max_tokens = + ParseField(request, "max_tokens"); + if (max_tokens.explicit_limit) { + return max_tokens; + } + return ParseField(request, "max_completion_tokens"); + } + } + return {false, kNoExplicitGenerationLimit}; +} + +int GenerationLoopLimit( + const ParsedGenerationLimit& parsed, + bool uses_corelib_aie4) noexcept { + if (parsed.explicit_limit) { + return parsed.value; + } + return uses_corelib_aie4 + ? kNoExplicitGenerationLimit + : kLegacyDefaultGenerationLimit; +} + +std::optional RequestedMaxNewTokens( + const ParsedGenerationLimit& parsed) noexcept { + if (!parsed.explicit_limit) { + return std::nullopt; + } + return parsed.value; +} + +int OllamaChatGenerationLoopLimit( + const nlohmann::ordered_json& request) { + const nlohmann::ordered_json options = + request.value( + "options", + nlohmann::ordered_json::object()); + return options.value( + "num_predict", + kLegacyDefaultGenerationLimit); +} + +nlohmann::ordered_json ModelErrorResponse( + std::string_view message, + int http_code, + bool session_cleared) { + return { + {"error", { + {"message", std::string(message)}, + {"type", http_code == 400 + ? "invalid_request_error" + : "server_error"}, + {"code", http_code}, + {"session_cleared", session_cleared}, + }}, + }; +} + +int HttpStatusForResponse( + const nlohmann::ordered_json& response) noexcept { + try { + if (!response.is_object() || !response.contains("error")) { + return 200; + } + const auto& error = response.at("error"); + if (!error.is_object() || !error.contains("code")) { + return 200; + } + const int code = error.at("code").get(); + return code == 400 || code == 500 ? code : 200; + } catch (...) { + return 200; + } +} + +bool UseFinalStreamingErrorChunk( + bool stream_started) noexcept { + return stream_started; +} + +std::optional CliRequestedMaxNewTokens( + int generate_limit) noexcept { + if (generate_limit <= 0) { + return std::nullopt; + } + return generate_limit; +} + +std::string CliModelErrorNotice( + std::string_view message, + bool session_cleared) { + if (session_cleared) { + return + "ERROR: AIE4 inference failed before submission; the current " + "conversation was cleared."; + } + return "ERROR: " + std::string(message); +} + +bool IsCorelibAie4ModelInfo( + const nlohmann::ordered_json& model_info) noexcept { + try { + if (!model_info.is_object() || + !model_info.contains("details")) { + return false; + } + const auto& details = model_info.at("details"); + return details.is_object() && + details.value("execution_backend", std::string{}) == + "corelib_aie4"; + } catch (...) { + return false; + } +} diff --git a/src/server/npu_access_manager.cpp b/src/server/npu_access_manager.cpp new file mode 100644 index 00000000..8690d7ce --- /dev/null +++ b/src/server/npu_access_manager.cpp @@ -0,0 +1,50 @@ +#include + +#include +#include + +namespace { + +std::mutex g_npu_access_mutex; +std::atomic g_npu_in_use{false}; +std::atomic g_npu_active_requests{0}; + +} // namespace + +bool NPUAccessManager::try_acquire_npu_access() { + std::lock_guard lock(g_npu_access_mutex); + if (g_npu_in_use.load(std::memory_order_relaxed)) { + return false; + } + g_npu_in_use.store(true, std::memory_order_release); + g_npu_active_requests.fetch_add(1, std::memory_order_relaxed); + return true; +} + +void NPUAccessManager::release_npu_access() { + std::lock_guard lock(g_npu_access_mutex); + g_npu_in_use.store(false, std::memory_order_release); + g_npu_active_requests.fetch_sub(1, std::memory_order_relaxed); +} + +bool NPUAccessManager::is_npu_available() { + return !g_npu_in_use.load(std::memory_order_acquire); +} + +int NPUAccessManager::get_active_npu_requests() { + return g_npu_active_requests.load(std::memory_order_relaxed); +} + +bool requires_npu_access( + const std::string& method, + const std::string& path) { + if (method != "POST") { + return false; + } + return path == "/api/generate" || + path == "/api/chat" || + path == "/v1/chat/completions" || + path == "/v1/completions" || + path == "/v1/audio/transcriptions" || + path == "/v1/embeddings"; +} diff --git a/src/server/rest_handler.cpp b/src/server/rest_handler.cpp index 95e90a3a..7da97e00 100644 --- a/src/server/rest_handler.cpp +++ b/src/server/rest_handler.cpp @@ -11,6 +11,7 @@ #include "streaming_ostream.hpp" #include "streaming_ostream_openai.hpp" #include "image/image_reader.hpp" +#include #include #include #include @@ -20,6 +21,17 @@ #include #include "server.hpp" +namespace { + +json ModelErrorResponse(const ModelRequestError& error) { + return ::ModelErrorResponse( + error.what(), + error.http_code(), + error.session_cleared()); +} + +} // namespace + ///@brief Normalize messages by merging consecutive user messages (like Ollama does) ///@param messages the original messages ///@return normalized messages with consecutive user messages merged @@ -419,6 +431,16 @@ bool RestHandler::ensure_model_loaded(const std::string& model_tag) { return true; } +bool RestHandler::uses_corelib_aie4(const std::string& model_tag) { + if (!supported_models.is_model_supported(model_tag)) { + return false; + } + const auto [resolved_tag, model_info] = + supported_models.get_model_info(model_tag); + (void)resolved_tag; + return IsCorelibAie4ModelInfo(model_info); +} + ///@brief Ensure the asr model is loaded ///@param model_tag the model tag void RestHandler::ensure_asr_model_loaded(const std::string& model_tag) { @@ -638,12 +660,17 @@ void RestHandler::handle_generate(const json& request, StreamResponseCallback send_streaming_response, std::shared_ptr cancellation_token) { try { + const ParsedGenerationLimit parsed_limit = + ParseGenerationLimit( + request, + GenerationEndpoint::Generate); std::string prompt = request["prompt"]; bool stream = request.value("stream", true); std::string model = request.value("model", current_model_tag); json options = request.value("options", json::object()); - - int length_limit = request.value("max_tokens", 4096); + const bool corelib_aie4 = uses_corelib_aie4(model); + const int length_limit = + GenerationLoopLimit(parsed_limit, corelib_aie4); auto load_start_time = time_utils::now(); // TODO: Use Another Check Function avoid loading again if (!ensure_model_loaded(model)) { @@ -655,6 +682,8 @@ void RestHandler::handle_generate(const json& request, chat_meta_info_t meta_info; lm_uniform_input_t uniformed_input; + uniformed_input.requested_max_new_tokens = + RequestedMaxNewTokens(parsed_limit); meta_info.max_prefill_len = this->prefill_chunk_len; meta_info.load_duration = (uint64_t)time_utils::duration_ns(load_start_time, load_end_time).first; header_print("FLM", "Start generating..."); @@ -662,7 +691,18 @@ void RestHandler::handle_generate(const json& request, if (stream) { // Streaming response using streaming_ostream auto total_start_time = time_utils::now(); - streaming_ostream ostream(model, send_streaming_response, false); + bool stream_started = false; + auto tracked_stream_response = + [&stream_started, &send_streaming_response]( + const json& data, + bool is_final) { + stream_started = true; + send_streaming_response(data, is_final); + }; + streaming_ostream ostream( + model, + tracked_stream_response, + false); uniformed_input.prompt = prompt; try { bool success = auto_chat_engine->insert(meta_info, uniformed_input); @@ -672,6 +712,9 @@ void RestHandler::handle_generate(const json& request, this->auto_chat_engine->clear_context(); return; } + } catch (const ModelRequestError& error) { + send_response(ModelErrorResponse(error)); + return; } catch (const std::exception& e) { json error_response = {{"error", e.what()}}; send_response(error_response); @@ -680,6 +723,14 @@ void RestHandler::handle_generate(const json& request, } try { auto_chat_engine->generate(meta_info, length_limit, ostream); + } catch (const ModelRequestError& error) { + const json error_response = ModelErrorResponse(error); + if (UseFinalStreamingErrorChunk(stream_started)) { + send_streaming_response(error_response, true); + } else { + send_response(error_response); + } + return; } catch (const std::exception& e) { json error_response = {{"error", e.what()}}; send_response(error_response); @@ -705,6 +756,9 @@ void RestHandler::handle_generate(const json& request, this->auto_chat_engine->clear_context(); return; } + } catch (const ModelRequestError& error) { + send_response(ModelErrorResponse(error)); + return; } catch (const std::exception& e) { json error_response = {{"error", e.what()}}; send_response(error_response); @@ -713,6 +767,9 @@ void RestHandler::handle_generate(const json& request, } try { auto_chat_engine->generate(meta_info, length_limit, ostream); + } catch (const ModelRequestError& error) { + send_response(ModelErrorResponse(error)); + return; } catch (const std::exception& e) { json error_response = {{"error", e.what()}}; send_response(error_response); @@ -737,6 +794,8 @@ void RestHandler::handle_generate(const json& request, // std::cout << "history: " << history.first << std::endl; send_response(response); } + } catch (const ModelRequestError& error) { + send_response(ModelErrorResponse(error)); } catch (const std::exception& e) { json error_response = {{"error", e.what()}}; send_response(error_response); @@ -756,7 +815,7 @@ void RestHandler::handle_chat(const json& request, bool stream = request.value("stream", false); std::string model = request.value("model", current_model_tag); json options = request.value("options", json::object()); - int length_limit = options.value("num_predict", 4096); + int length_limit = OllamaChatGenerationLoopLimit(request); auto load_start_time = time_utils::now(); if (!ensure_model_loaded(model)) { @@ -772,13 +831,26 @@ void RestHandler::handle_chat(const json& request, chat_meta_info_t meta_info; lm_uniform_input_t uniformed_input; + // options.num_predict is a soft loop bound only. It must not reserve + // AIE4 context capacity through requested_max_new_tokens. meta_info.load_duration = (uint64_t)time_utils::duration_ns(load_start_time, load_end_time).first; meta_info.max_prefill_len = this->prefill_chunk_len; header_print("FLM", "Start generating..."); if (stream) { // Streaming response using streaming_ostream auto total_start_time = time_utils::now(); - streaming_ostream ostream(model, send_streaming_response, true); // true for chat format + bool stream_started = false; + auto tracked_stream_response = + [&stream_started, &send_streaming_response]( + const json& data, + bool is_final) { + stream_started = true; + send_streaming_response(data, is_final); + }; + streaming_ostream ostream( + model, + tracked_stream_response, + true); // true for chat format uniformed_input.messages = messages; try { bool success = auto_chat_engine->insert(meta_info, uniformed_input); @@ -788,6 +860,9 @@ void RestHandler::handle_chat(const json& request, this->auto_chat_engine->clear_context(); return; } + } catch (const ModelRequestError& error) { + send_response(ModelErrorResponse(error)); + return; } catch (const std::exception& e) { json error_response = {{"error", e.what()}}; send_response(error_response); @@ -795,13 +870,18 @@ void RestHandler::handle_chat(const json& request, return; } try { - bool success = auto_chat_engine->insert(meta_info, uniformed_input); - if (!success){ - json error_response = {{"error", "Max length reached"}}; + auto_chat_engine->generate( + meta_info, + length_limit, + ostream); + } catch (const ModelRequestError& error) { + const json error_response = ModelErrorResponse(error); + if (UseFinalStreamingErrorChunk(stream_started)) { + send_streaming_response(error_response, true); + } else { send_response(error_response); - this->auto_chat_engine->clear_context(); - return; } + return; } catch (const std::exception& e) { json error_response = {{"error", e.what()}}; send_response(error_response); @@ -824,6 +904,9 @@ void RestHandler::handle_chat(const json& request, std::string response_text; try { response_text = auto_chat_engine->generate_with_prompt(meta_info, uniformed_input, length_limit, nstream); + } catch (const ModelRequestError& error) { + send_response(ModelErrorResponse(error)); + return; } catch (const std::exception& e) { json error_response = {{"error", e.what()}}; send_response(error_response); @@ -856,6 +939,8 @@ void RestHandler::handle_chat(const json& request, // std::cout << "history: " << history.first << std::endl; this->auto_chat_engine->clear_context(); } + } catch (const ModelRequestError& error) { + send_response(ModelErrorResponse(error)); } catch (const std::exception& e) { json error_response = {{"error", e.what()}}; send_response(error_response); @@ -1091,10 +1176,16 @@ void RestHandler::handle_openai_chat_completion(const json& request, static std::string model_used_for_last_message = "model-faker"; try { // Extract OpenAI-style parameters + const ParsedGenerationLimit parsed_limit = + ParseGenerationLimit( + request, + GenerationEndpoint::OpenAiChatCompletion); json current_messages = request["messages"]; std::string model = request.value("model", current_model_tag); bool stream = request.value("stream", false); - int length_limit = request.value("max_tokens", request.value("max_completion_tokens", 4096)); + const bool corelib_aie4 = uses_corelib_aie4(model); + const int length_limit = + GenerationLoopLimit(parsed_limit, corelib_aie4); json tools = request.value("tools", json::array()); json options = request.value("options", json::object()); @@ -1119,7 +1210,7 @@ void RestHandler::handle_openai_chat_completion(const json& request, this->prompt_cache.update_tool_checksum(tools); model_used_for_last_message = model; } - else { + else if (!corelib_aie4) { cache_match_info_t cache_info; can_use_prompt_cache = prompt_cache.can_use_cache(current_messages, auto_chat_engine->get_chat_template_type(), tools, cache_info); if (can_use_prompt_cache) { @@ -1146,13 +1237,21 @@ void RestHandler::handle_openai_chat_completion(const json& request, lm_uniform_input_t uniformed_input; uniformed_input.messages = current_messages; uniformed_input.tools = tools; + uniformed_input.requested_max_new_tokens = + RequestedMaxNewTokens(parsed_limit); meta_info.load_duration = (uint64_t)time_utils::duration_ns(load_start_time, load_end_time).first; meta_info.max_prefill_len = this->prefill_chunk_len; if (stream){ // Create a wrapper callback that passes the pre-formatted SSE string directly cancellation_token->reset(); auto_chat_engine->reset_parser(); - auto openai_stream_callback = [&send_streaming_response](const std::string& data, bool is_final) { + bool stream_started = false; + auto openai_stream_callback = [ + &send_streaming_response, + &stream_started]( + const std::string& data, + bool is_final) { + stream_started = true; json data_json = data; send_streaming_response(data_json, is_final); }; @@ -1183,6 +1282,12 @@ void RestHandler::handle_openai_chat_completion(const json& request, this->prompt_cache.reset(); return; } + } catch (const ModelRequestError& error) { + send_response(ModelErrorResponse(error)); + if (error.session_cleared()) { + this->prompt_cache.reset(); + } + return; } catch (const std::exception& e) { json error_response = {{"error", e.what()}}; send_response(error_response); @@ -1193,6 +1298,17 @@ void RestHandler::handle_openai_chat_completion(const json& request, header_print("FLM", "Start generating..."); try { auto_chat_engine->generate(meta_info, length_limit, ostream, [&] { return cancellation_token->cancelled(); }); + } catch (const ModelRequestError& error) { + const json error_response = ModelErrorResponse(error); + if (UseFinalStreamingErrorChunk(stream_started)) { + send_streaming_response(error_response, true); + } else { + send_response(error_response); + } + if (error.session_cleared()) { + this->prompt_cache.reset(); + } + return; } catch (const std::exception& e) { json error_response = {{"error", e.what()}}; send_response(error_response); @@ -1235,6 +1351,12 @@ void RestHandler::handle_openai_chat_completion(const json& request, this->prompt_cache.reset(); return; } + } catch (const ModelRequestError& error) { + send_response(ModelErrorResponse(error)); + if (error.session_cleared()) { + this->prompt_cache.reset(); + } + return; } catch (const std::exception& e) { json error_response = {{"error", e.what()}}; send_response(error_response); @@ -1245,6 +1367,12 @@ void RestHandler::handle_openai_chat_completion(const json& request, header_print("FLM", "Start generating..."); try { response_text = auto_chat_engine->generate(meta_info, length_limit, nstream, [&] { return cancellation_token->cancelled(); }); + } catch (const ModelRequestError& error) { + send_response(ModelErrorResponse(error)); + if (error.session_cleared()) { + this->prompt_cache.reset(); + } + return; } catch (const std::exception& e) { json error_response = {{"error", e.what()}}; send_response(error_response); @@ -1280,6 +1408,11 @@ void RestHandler::handle_openai_chat_completion(const json& request, send_response(response); } + } catch (const ModelRequestError& error) { + send_response(ModelErrorResponse(error)); + if (error.session_cleared()) { + this->prompt_cache.reset(); + } } catch (const std::exception& e) { json error_response = { {"error", { @@ -1365,18 +1498,23 @@ void RestHandler::handle_openai_completion(const json& request, std::shared_ptr cancellation_token) { try { // Extract OpenAI-style parameters + const ParsedGenerationLimit parsed_limit = + ParseGenerationLimit( + request, + GenerationEndpoint::OpenAiCompletion); std::string prompt = request["prompt"]; std::string model = request.value("model", current_model_tag); std::string reasoning_effort = request.value("reasoning_effort", "medium"); bool stream = request.value("stream", false); json options = request.value("options", json::object()); + const bool corelib_aie4 = uses_corelib_aie4(model); + const int length_limit = + GenerationLoopLimit(parsed_limit, corelib_aie4); // direct return if model not supported if (!supported_models.is_model_supported(model)) { throw std::runtime_error("Model " + model + " is not supported."); } - - int length_limit = request.value("max_tokens", 4096); if (!ensure_model_loaded(model)) { json error_response = {{"error", "Failed to load " + model + " model!"}}; @@ -1389,11 +1527,19 @@ void RestHandler::handle_openai_completion(const json& request, chat_meta_info_t meta_info; meta_info.max_prefill_len = this->prefill_chunk_len; lm_uniform_input_t uniformed_input; + uniformed_input.requested_max_new_tokens = + RequestedMaxNewTokens(parsed_limit); header_print("FLM", "Start generating..."); if (stream) { // Create a wrapper callback that passes the pre-formatted SSE string directly - auto openai_stream_callback = [&send_streaming_response](const std::string& data, bool is_final) { + bool stream_started = false; + auto openai_stream_callback = [ + &send_streaming_response, + &stream_started]( + const std::string& data, + bool is_final) { + stream_started = true; json data_json = data; send_streaming_response(data_json, is_final); }; @@ -1407,6 +1553,9 @@ void RestHandler::handle_openai_completion(const json& request, this->auto_chat_engine->clear_context(); return; } + } catch (const ModelRequestError& error) { + send_response(ModelErrorResponse(error)); + return; } catch (const std::exception& e) { json error_response = {{"error", e.what()}}; send_response(error_response); @@ -1415,6 +1564,14 @@ void RestHandler::handle_openai_completion(const json& request, } try { auto_chat_engine->generate(meta_info, length_limit, ostream); + } catch (const ModelRequestError& error) { + const json error_response = ModelErrorResponse(error); + if (UseFinalStreamingErrorChunk(stream_started)) { + send_streaming_response(error_response, true); + } else { + send_response(error_response); + } + return; } catch (const std::exception& e) { json error_response = {{"error", e.what()}}; send_response(error_response); @@ -1438,6 +1595,9 @@ void RestHandler::handle_openai_completion(const json& request, this->auto_chat_engine->clear_context(); return; } + } catch (const ModelRequestError& error) { + send_response(ModelErrorResponse(error)); + return; } catch (const std::exception& e) { json error_response = {{"error", e.what()}}; send_response(error_response); @@ -1446,6 +1606,9 @@ void RestHandler::handle_openai_completion(const json& request, } try { auto_chat_engine->generate(meta_info, length_limit, ostream); + } catch (const ModelRequestError& error) { + send_response(ModelErrorResponse(error)); + return; } catch (const std::exception& e) { json error_response = {{"error", e.what()}}; send_response(error_response); @@ -1477,6 +1640,9 @@ void RestHandler::handle_openai_completion(const json& request, send_response(response); } } + catch (const ModelRequestError& error) { + send_response(ModelErrorResponse(error)); + } catch (const std::exception& e) { json error_response = { {"error", { diff --git a/src/server/rest_handler.hpp b/src/server/rest_handler.hpp index 9e5fcad7..51b520bc 100644 --- a/src/server/rest_handler.hpp +++ b/src/server/rest_handler.hpp @@ -109,6 +109,7 @@ class RestHandler { private: bool ensure_model_loaded(const std::string& model_tag); + bool uses_corelib_aie4(const std::string& model_tag); void ensure_asr_model_loaded(const std::string& model_tag); void ensure_embed_model_loaded(const std::string& model_tag); void configure_chat_engine_parameters(const json& options, const json& request); diff --git a/src/server/server.cpp b/src/server/server.cpp index bc612211..bc0a5d66 100644 --- a/src/server/server.cpp +++ b/src/server/server.cpp @@ -8,19 +8,13 @@ */ #include "server.hpp" #include "rest_handler.hpp" +#include #include #include #include #include #include - -// Global NPU access control -std::mutex g_npu_access_mutex; -std::atomic g_npu_in_use{false}; - -std::atomic g_npu_active_requests{0}; - ///@brief get current time string, format: hh:mm:ss mm:dd:yyyy ///@return the current time string std::string get_current_time_string() { @@ -129,46 +123,6 @@ void brief_print_message_response(nlohmann::json request) { } -// NPU Access Manager implementation -bool NPUAccessManager::try_acquire_npu_access() { - std::lock_guard lock(g_npu_access_mutex); - if (g_npu_in_use.load()) { - return false; // NPU is already in use - } - g_npu_in_use.store(true); - g_npu_active_requests.fetch_add(1); - return true; -} - -void NPUAccessManager::release_npu_access() { - - header_print("🔵 ", "NPU Lock Released!" ); - std::lock_guard lock(g_npu_access_mutex); - g_npu_in_use.store(false); - g_npu_active_requests.fetch_sub(1); -} - -bool NPUAccessManager::is_npu_available() { - return !g_npu_in_use.load(); -} - -int NPUAccessManager::get_active_npu_requests() { - return g_npu_active_requests.load(); -} - -// Helper function to check if an endpoint requires NPU access -bool requires_npu_access(const std::string& method, const std::string& path) { - // NPU-intensive endpoints that should be restricted to one user at a time - if (method == "POST") { - return path == "/api/generate" || - path == "/api/chat" || - path == "/v1/chat/completions" || - path == "/v1/audio/transcriptions" || - path == "/v1/embeddings"; - } - return false; -} - ///@brief HttpSession class implementation ///@param socket the socket ///@param server the server @@ -728,20 +682,14 @@ bool WebServer::handle_request(http::request& req, // catch is_deferred auto send_response = [res_ptr, session, this, request_id, needs_npu, is_deferred, cancellation_token](const json& response_data) { auto& response_ref = *res_ptr; - http::status status = http::status::ok; - - if (response_data.contains("error") && - response_data["error"].contains("code")) - { - int code = response_data["error"]["code"].get(); - - if (code == 400) { - status = http::status::bad_request; - } - //else if () { - - //} - } + const int status_code = + HttpStatusForResponse(response_data); + const http::status status = + status_code == 400 + ? http::status::bad_request + : status_code == 500 + ? http::status::internal_server_error + : http::status::ok; response_ref.result(status); response_ref.body() = response_data.dump(); diff --git a/src/server/server.hpp b/src/server/server.hpp index 910c5884..95cdf053 100644 --- a/src/server/server.hpp +++ b/src/server/server.hpp @@ -29,6 +29,7 @@ #include "streaming_ostream.hpp" #include "model_downloader.hpp" #include "multipart.hpp" +#include #include #include @@ -42,27 +43,10 @@ using json = nlohmann::ordered_json; class RestHandler; class HttpSession; -// Global NPU access control -extern std::mutex g_npu_access_mutex; -extern std::atomic g_npu_in_use; -extern std::atomic g_npu_active_requests; - -// Helper function to check if an endpoint requires NPU access -bool requires_npu_access(const std::string& method, const std::string& path); - ///@brief get current time string, format: hh:mm:ss mm:dd:yyyy ///@return the current time string std::string get_current_time_string(); -// NPU access manager class -class NPUAccessManager { -public: - static bool try_acquire_npu_access(); - static void release_npu_access(); - static bool is_npu_available(); - static int get_active_npu_requests(); -}; - // Stream response callback type for handling streaming responses using StreamCallback = std::function; diff --git a/src/src/main.cpp b/src/src/main.cpp index 446b907b..d9d4ec76 100644 --- a/src/src/main.cpp +++ b/src/src/main.cpp @@ -14,6 +14,7 @@ #include "program_args.hpp" #include "minja/chat-template.hpp" #include +#include #include #include #include @@ -22,6 +23,7 @@ #include #include #include +#include #include #include #ifdef _WIN32 @@ -49,6 +51,11 @@ #include "AutoModel/automodel.hpp" +#if defined(FLM_ENABLE_CORELIB_AIE4) +#include +#include +#endif + #ifndef _WIN32 #include #endif @@ -207,7 +214,10 @@ std::string identify_npu_arch() { #endif -static bool sanity_check_npu_stack(bool quiet, bool json_output = false) { +static bool sanity_check_npu_stack( + bool quiet, + bool json_output = false, + nlohmann::json* report_output = nullptr) { bool print_human = !quiet && !json_output; #ifndef _WIN32 nlohmann::json validation_json = { @@ -221,6 +231,17 @@ static bool sanity_check_npu_stack(bool quiet, bool json_output = false) { {"devices", nlohmann::json::array()}, {"ready", true} }; + const auto finish_validation = + [&](bool ready) { + validation_json["ready"] = ready; + if (report_output != nullptr) { + *report_output = validation_json; + } + if (json_output) { + std::cout << validation_json.dump(4) << std::endl; + } + return ready; + }; validation_json["platform"] = "linux"; // Check kernel version struct utsname u_name; @@ -228,11 +249,7 @@ static bool sanity_check_npu_stack(bool quiet, bool json_output = false) { if (print_human) perror("Failed to get kernel version"); validation_json["kernel_ok"] = false; - validation_json["ready"] = false; - if (json_output) { - std::cout << validation_json.dump(4) << std::endl; - } - return false; + return finish_validation(false); } int major, minor; sscanf(u_name.release, "%d.%d", &major, &minor); @@ -243,11 +260,7 @@ static bool sanity_check_npu_stack(bool quiet, bool json_output = false) { if (print_human) { header_print_r("ERROR", "Kernel version incompatible with this version of FLM. Please update your kernel!"); } - validation_json["ready"] = false; - if (json_output) { - std::cout << validation_json.dump(4) << std::endl; - } - return false; + return finish_validation(false); } if (print_human) { header_print("Linux", "Kernel: " << u_name.release); @@ -398,12 +411,7 @@ static bool sanity_check_npu_stack(bool quiet, bool json_output = false) { validation_json["memlock_ok"] = memlock_ok; bool overall_ok = amd_device_found && kernel_ok && all_fw_ok && enough_cols && memlock_ok; - validation_json["ready"] = overall_ok; - if (json_output) { - std::cout << validation_json.dump(4) << std::endl; - } - - return overall_ok; + return finish_validation(overall_ok); #else nlohmann::json validation_json = { {"object", "npu_stack_validation"}, @@ -412,16 +420,23 @@ static bool sanity_check_npu_stack(bool quiet, bool json_output = false) { {"npu_driver_ok", true}, {"ready", true} }; + const auto finish_validation = + [&](bool ready) { + validation_json["ready"] = ready; + if (report_output != nullptr) { + *report_output = validation_json; + } + if (json_output) { + std::cout << validation_json.dump(4) << std::endl; + } + return ready; + }; std::string npu_arch = identify_npu_arch(); if (npu_arch.empty()) { if (print_human) header_print("Error", "No XDNA2 NPU hardware detected"); - validation_json["ready"] = false; validation_json["amd_device_found"] = false; - if (json_output) { - std::cout << validation_json.dump(4) << std::endl; - } - return false; + return finish_validation(false); } std::string min_drv = __NPU_VERSION__; @@ -435,12 +450,8 @@ static bool sanity_check_npu_stack(bool quiet, bool json_output = false) { if (print_human) { header_print("Error", "NPU driver version doesn't meet the minimum!"); } - validation_json["ready"] = false; validation_json["npu_driver_ok"] = false; - if (json_output) { - std::cout << validation_json.dump(4) << std::endl; - } - return false; + return finish_validation(false); } if (print_human) { @@ -448,13 +459,97 @@ static bool sanity_check_npu_stack(bool quiet, bool json_output = false) { header_print_g("Windows", "NPU dirver version: " << drv); } - if (json_output) { - std::cout << validation_json.dump(4) << std::endl; - } - return true; + return finish_validation(true); #endif } +#if defined(FLM_ENABLE_CORELIB_AIE4) +static void drain_prior_corelib_fatal_records() noexcept { + try { + (void)flm::corelib::FatalRecordStore::DrainPriorRecords( + std::cerr); + } catch (const std::exception& error) { + std::cerr + << "AIE4 fatal record warning: startup drain failed: " + << error.what() << '\n'; + } catch (...) { + std::cerr + << "AIE4 fatal record warning: startup drain failed with an " + "unknown error\n"; + } +} + +static bool shutdown_corelib_process() noexcept { + try { + flm::corelib::CorelibRuntime::ShutdownProcess(); + return true; + } catch (const std::exception& error) { + std::cerr << "Error: failed to shut down AIE4 corelib runtime: " + << error.what() << '\n'; + } catch (...) { + std::cerr + << "Error: failed to shut down AIE4 corelib runtime with an " + "unknown error\n"; + } + return false; +} + +static nlohmann::json validate_corelib_aie4( + const std::filesystem::path& executable_dir, + bool print_human) { + nlohmann::json report = { + {"available_in_build", true}, + {"loader_ok", false}, + {"dependencies_ok", false}, + {"device_context_ok", false}, + {"fatal_log_writable", false}, + {"shutdown_ok", false}, + {"ready", false}, + }; + + std::shared_ptr runtime; + try { + runtime = + flm::corelib::CorelibRuntime::GetOrCreate( + executable_dir); + report["loader_ok"] = true; + report["dependencies_ok"] = true; + report["device_context_ok"] = true; + report["fatal_log_writable"] = true; + runtime.reset(); + const bool shutdown_ok = shutdown_corelib_process(); + report["shutdown_ok"] = shutdown_ok; + report["ready"] = shutdown_ok; + } catch (const std::exception& error) { + runtime.reset(); + (void)shutdown_corelib_process(); + report["error"] = error.what(); + } catch (...) { + runtime.reset(); + (void)shutdown_corelib_process(); + report["error"] = "unknown AIE4 validation error"; + } + + if (print_human) { + if (report["ready"].get()) { + header_print_g( + "Windows", + "Corelib AIE4: ready"); + } else { + header_print_r( + "ERROR", + "Corelib AIE4: not ready" + << (report.contains("error") + ? " (" + + report["error"].get() + + ")" + : "")); + } + } + return report; +} +#endif + ///@brief main function ///@param argc the number of arguments @@ -468,6 +563,7 @@ int main(int argc, char* argv[]) { // XRT backend: preload bundled XRT libraries from the executable directory. preload_bundled_libraries(); #endif + std::signal(SIGINT, signal_handler); // Parse command line arguments using Boost Program Options program_args_t parsed_args; @@ -475,6 +571,10 @@ int main(int argc, char* argv[]) { return 1; // Help was already printed by Boost Program Options } +#if defined(FLM_ENABLE_CORELIB_AIE4) + drain_prior_corelib_fatal_records(); +#endif + // Get the command, model tag, and force flag std::string exe_dir = utils::get_executable_directory(); @@ -505,7 +605,75 @@ int main(int argc, char* argv[]) { // Check if the commands and args valid if (parsed_args.command == "validate") { - stable_stack = sanity_check_npu_stack(parsed_args.command != "validate", parsed_args.command == "validate" && parsed_args.json_output); + nlohmann::json legacy_report; + bool legacy_ready = false; + try { + legacy_ready = + sanity_check_npu_stack( + parsed_args.json_output, + false, + &legacy_report); + } catch (const std::exception& error) { + legacy_report = { + {"object", "npu_stack_validation"}, + {"ready", false}, + {"error", error.what()}, + }; +#ifdef _WIN32 + legacy_report["platform"] = "windows"; +#else + legacy_report["platform"] = "linux"; +#endif + if (!parsed_args.json_output) { + header_print_r( + "ERROR", + "Legacy XDNA2 validation failed: " + << error.what()); + } + } catch (...) { + legacy_report = { + {"object", "npu_stack_validation"}, + {"ready", false}, + {"error", "unknown legacy XDNA2 validation error"}, + }; +#ifdef _WIN32 + legacy_report["platform"] = "windows"; +#else + legacy_report["platform"] = "linux"; +#endif + if (!parsed_args.json_output) { + header_print_r( + "ERROR", + "Legacy XDNA2 validation failed with an " + "unknown error"); + } + } + nlohmann::json aie4_report = { + {"available_in_build", false}, + {"ready", false}, + }; + bool aie4_ready = false; +#if defined(FLM_ENABLE_CORELIB_AIE4) + aie4_report = + validate_corelib_aie4( + exe_dir, + !parsed_args.json_output); + aie4_ready = aie4_report["ready"].get(); +#else + if (!parsed_args.json_output) { + header_print( + "FLM", + "Corelib AIE4: unavailable in this build"); + } +#endif + stable_stack = legacy_ready || aie4_ready; + if (parsed_args.json_output) { + nlohmann::json validation_report = legacy_report; + validation_report["legacy_xdna2"] = legacy_report; + validation_report["corelib_aie4"] = aie4_report; + validation_report["ready"] = stable_stack; + std::cout << validation_report.dump(4) << std::endl; + } return stable_stack ? 0 : 1; } @@ -715,11 +883,25 @@ int main(int argc, char* argv[]) { std::cerr << "Use --help for usage information" << std::endl; return 1; } +#if defined(FLM_ENABLE_CORELIB_AIE4) + if (!shutdown_corelib_process()) { + return 1; + } +#endif // Return 0 if the command is valid return 0; } catch (const std::exception& e) { // If an error occurs, this will be used to show the error std::cerr << "Error: " << e.what() << std::endl; +#if defined(FLM_ENABLE_CORELIB_AIE4) + (void)shutdown_corelib_process(); +#endif + return 1; + } catch (...) { + std::cerr << "Error: unknown command failure" << std::endl; +#if defined(FLM_ENABLE_CORELIB_AIE4) + (void)shutdown_corelib_process(); +#endif return 1; } } diff --git a/src/test/phi4_corelib_aie4/CMakeLists.txt b/src/test/phi4_corelib_aie4/CMakeLists.txt index 9c7da669..ea9a6590 100644 --- a/src/test/phi4_corelib_aie4/CMakeLists.txt +++ b/src/test/phi4_corelib_aie4/CMakeLists.txt @@ -84,6 +84,14 @@ function(add_corelib_host_test TEST_NAME TEST_SOURCE) endfunction() enable_testing() +add_executable(test_generation_limit + test_generation_limit.cpp + ${FASTFLOW_SOURCE_DIR}/server/generation_limit.cpp + ${FASTFLOW_SOURCE_DIR}/server/npu_access_manager.cpp) +target_include_directories(test_generation_limit PRIVATE + ${FASTFLOW_SOURCE_DIR}/include) +add_test(NAME test_generation_limit COMMAND test_generation_limit) + add_corelib_host_test(test_corelib_api test_corelib_api.cpp) add_corelib_host_test(test_phi4_manifest test_phi4_manifest.cpp) add_corelib_host_test(test_phi4_shape_plan test_phi4_shape_plan.cpp) @@ -103,7 +111,8 @@ target_link_libraries(test_corelib_fatal_record PRIVATE set(PHI4_FRONTEND_SOURCES ${FASTFLOW_SOURCE_DIR}/common/AutoModel/automodel.cpp - ${FASTFLOW_SOURCE_DIR}/common/AutoModel/modeling_phi4.cpp) + ${FASTFLOW_SOURCE_DIR}/common/AutoModel/modeling_phi4.cpp + ${FASTFLOW_SOURCE_DIR}/server/generation_limit.cpp) function(add_phi4_frontend_test TEST_NAME ENABLE_AIE4) add_executable(${TEST_NAME} @@ -186,3 +195,72 @@ add_phi4_frontend_compile_check( add_phi4_frontend_compile_check( phi4_frontend_compile_on TRUE) + +set(TASK10_PRODUCTION_SOURCES + ${FASTFLOW_SOURCE_DIR}/server/generation_limit.cpp + ${FASTFLOW_SOURCE_DIR}/server/npu_access_manager.cpp + ${FASTFLOW_SOURCE_DIR}/server/rest_handler.cpp + ${FASTFLOW_SOURCE_DIR}/server/server.cpp + ${FASTFLOW_SOURCE_DIR}/runner/runner.cpp + ${FASTFLOW_SOURCE_DIR}/src/main.cpp) + +set(TASK10_STUB_INCLUDE_DIR + "${CMAKE_CURRENT_BINARY_DIR}/task10-compile-stubs") +# These production translation units include declaration-only FFmpeg headers +# transitively. The standalone host suite does not link or execute FFmpeg. +foreach(TASK10_STUB_HEADER IN ITEMS + libavcodec/avcodec.h + libswscale/swscale.h + libavutil/imgutils.h + libavutil/frame.h + libavutil/pixfmt.h) + get_filename_component( + TASK10_STUB_PARENT + "${TASK10_STUB_INCLUDE_DIR}/${TASK10_STUB_HEADER}" + DIRECTORY) + file(MAKE_DIRECTORY "${TASK10_STUB_PARENT}") + file(WRITE + "${TASK10_STUB_INCLUDE_DIR}/${TASK10_STUB_HEADER}" + "#pragma once\n") +endforeach() + +function(add_task10_production_compile_check TARGET_NAME ENABLE_AIE4) + add_library(${TARGET_NAME} OBJECT + ${TASK10_PRODUCTION_SOURCES}) + target_include_directories(${TARGET_NAME} PRIVATE + ${FASTFLOW_SOURCE_DIR}/include + ${FASTFLOW_SOURCE_DIR}/runner + ${FASTFLOW_SOURCE_DIR}/server + ${FASTFLOW_SOURCE_DIR}/pull + ${FASTFLOW_SOURCE_DIR}/../third_party/tokenizers-cpp/include + ${TASK10_STUB_INCLUDE_DIR} + ${BOOST_INCLUDE_DIR} + ${XRT_INCLUDE_DIR}) + target_compile_definitions(${TARGET_NAME} PRIVATE + DEV_BUILD=1 + __WINDOWS__ + USEAVX2=1 + DISABLE_ABI_CHECK=1 + _ENABLE_EXTENDED_ALIGNED_STORAGE + CMAKE_INSTALL_PREFIX="${FASTFLOW_SOURCE_DIR}/build/phi4_corelib_aie4-tests" + CMAKE_XCLBIN_PREFIX="${FASTFLOW_SOURCE_DIR}/xclbins" + __FLM_VERSION__="task10-test" + __NPU_VERSION__="0.0.0.0" + WIN32_LEAN_AND_MEAN + NOMINMAX) + target_compile_options(${TARGET_NAME} PRIVATE + $<$:/arch:AVX2 /fp:precise>) + if(ENABLE_AIE4) + target_compile_definitions(${TARGET_NAME} PRIVATE + FLM_ENABLE_CORELIB_AIE4=1) + target_include_directories(${TARGET_NAME} PRIVATE + ${RYZENAI_CORELIB_INCLUDE_DIR}) + endif() +endfunction() + +add_task10_production_compile_check( + task10_production_compile_off + FALSE) +add_task10_production_compile_check( + task10_production_compile_on + TRUE) diff --git a/src/test/phi4_corelib_aie4/test_generation_limit.cpp b/src/test/phi4_corelib_aie4/test_generation_limit.cpp new file mode 100644 index 00000000..1ff466d8 --- /dev/null +++ b/src/test/phi4_corelib_aie4/test_generation_limit.cpp @@ -0,0 +1,308 @@ +#include +#include + +#include "test_support.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using nlohmann::ordered_json; + +void CheckParsed( + const ParsedGenerationLimit& parsed, + bool explicit_limit, + int value) { + CHECK(parsed.explicit_limit == explicit_limit); + CHECK(parsed.value == value); +} + +void TestHandlerSpecificPresence() { + CheckParsed( + ParseGenerationLimit( + ordered_json{ + {"max_tokens", 11}, + {"max_completion_tokens", 22}}, + GenerationEndpoint::Generate), + true, + 11); + CheckParsed( + ParseGenerationLimit( + ordered_json{{"max_completion_tokens", 22}}, + GenerationEndpoint::Generate), + false, + -1); + + CheckParsed( + ParseGenerationLimit( + ordered_json{ + {"max_tokens", 31}, + {"max_completion_tokens", 32}}, + GenerationEndpoint::OpenAiChatCompletion), + true, + 31); + CheckParsed( + ParseGenerationLimit( + ordered_json{{"max_completion_tokens", 32}}, + GenerationEndpoint::OpenAiChatCompletion), + true, + 32); + + CheckParsed( + ParseGenerationLimit( + ordered_json{ + {"max_tokens", 41}, + {"max_completion_tokens", 42}}, + GenerationEndpoint::OpenAiCompletion), + true, + 41); + CheckParsed( + ParseGenerationLimit( + ordered_json{{"max_completion_tokens", 42}}, + GenerationEndpoint::OpenAiCompletion), + false, + -1); + + for (const GenerationEndpoint endpoint : { + GenerationEndpoint::Generate, + GenerationEndpoint::OpenAiChatCompletion, + GenerationEndpoint::OpenAiCompletion}) { + CheckParsed( + ParseGenerationLimit(ordered_json::object(), endpoint), + false, + -1); + } +} + +void TestEndpointDefaultsAndPropagation() { + const ParsedGenerationLimit omitted{false, -1}; + const ParsedGenerationLimit explicit_limit{true, 73}; + + CHECK(GenerationLoopLimit(omitted, true) == -1); + CHECK(GenerationLoopLimit(omitted, false) == 4096); + CHECK(GenerationLoopLimit(explicit_limit, true) == 73); + CHECK(GenerationLoopLimit(explicit_limit, false) == 73); + + CHECK(!RequestedMaxNewTokens(omitted).has_value()); + CHECK(RequestedMaxNewTokens(explicit_limit) == std::optional(73)); +} + +void TestOllamaChatLimitStaysSoftOnly() { + const ordered_json default_request = { + {"options", ordered_json::object()}, + }; + CHECK(OllamaChatGenerationLoopLimit(default_request) == 4096); + + const ordered_json explicit_request = { + {"options", {{"num_predict", 91}}}, + }; + CHECK(OllamaChatGenerationLoopLimit(explicit_request) == 91); +} + +void TestNestedModelErrorAndHttpStatus() { + const ordered_json bad_request = + ModelErrorResponse("too many tokens", 400, false); + CHECK(bad_request["error"]["message"] == "too many tokens"); + CHECK(bad_request["error"]["type"] == "invalid_request_error"); + CHECK(bad_request["error"]["code"] == 400); + CHECK(bad_request["error"]["session_cleared"] == false); + CHECK(HttpStatusForResponse(bad_request) == 400); + + const ordered_json server_error = + ModelErrorResponse("submission failed", 500, true); + CHECK(server_error["error"]["message"] == "submission failed"); + CHECK(server_error["error"]["type"] == "server_error"); + CHECK(server_error["error"]["code"] == 500); + CHECK(server_error["error"]["session_cleared"] == true); + CHECK(HttpStatusForResponse(server_error) == 500); + + CHECK(HttpStatusForResponse(ordered_json{{"ok", true}}) == 200); + CHECK( + HttpStatusForResponse( + ordered_json{{"error", "legacy string error"}}) == 200); + CHECK(!UseFinalStreamingErrorChunk(false)); + CHECK(UseFinalStreamingErrorChunk(true)); +} + +void TestCliLimitAndRecoverableNotice() { + CHECK(!CliRequestedMaxNewTokens(-1).has_value()); + CHECK(!CliRequestedMaxNewTokens(0).has_value()); + CHECK(CliRequestedMaxNewTokens(17) == std::optional(17)); + + CHECK( + CliModelErrorNotice("capacity exceeded", false) == + "ERROR: capacity exceeded"); + CHECK( + CliModelErrorNotice("ignored", true) == + "ERROR: AIE4 inference failed before submission; the current " + "conversation was cleared."); +} + +void TestAie4ModelInfoDetection() { + CHECK( + IsCorelibAie4ModelInfo( + ordered_json{ + {"details", {{"execution_backend", "corelib_aie4"}}}})); + CHECK( + !IsCorelibAie4ModelInfo( + ordered_json{{"details", {{"family", "phi4"}}}})); + CHECK(!IsCorelibAie4ModelInfo(ordered_json::object())); +} + +void RunForcedGateInterleaving(bool queue_during_insert) { + CHECK(NPUAccessManager::is_npu_available()); + CHECK(NPUAccessManager::get_active_npu_requests() == 0); + + std::mutex mutex; + std::condition_variable ready; + bool insert_entered = false; + bool generate_entered = false; + bool second_attempted = false; + bool second_attempt_rejected = false; + bool generate_completed = false; + bool gate_released = false; + bool second_insert_entered = false; + bool second_entered_early = false; + std::exception_ptr first_error; + std::exception_ptr second_error; + + std::thread first([&] { + try { + if (!NPUAccessManager::try_acquire_npu_access()) { + throw std::runtime_error( + "first request did not acquire NPU access"); + } + { + std::unique_lock lock(mutex); + insert_entered = true; + ready.notify_all(); + if (queue_during_insert) { + ready.wait( + lock, + [&] { return second_attempted; }); + } + generate_entered = true; + ready.notify_all(); + if (!queue_during_insert) { + ready.wait( + lock, + [&] { return second_attempted; }); + } + generate_completed = true; + } + NPUAccessManager::release_npu_access(); + { + std::lock_guard lock(mutex); + gate_released = true; + } + ready.notify_all(); + } catch (...) { + first_error = std::current_exception(); + ready.notify_all(); + } + }); + + std::thread second([&] { + try { + { + std::unique_lock lock(mutex); + ready.wait( + lock, + [&] { + return queue_during_insert + ? insert_entered + : generate_entered; + }); + } + + const bool acquired_early = + NPUAccessManager::try_acquire_npu_access(); + { + std::lock_guard lock(mutex); + second_attempt_rejected = !acquired_early; + if (acquired_early) { + second_insert_entered = true; + second_entered_early = !generate_completed; + } + second_attempted = true; + } + ready.notify_all(); + if (acquired_early) { + NPUAccessManager::release_npu_access(); + return; + } + + { + std::unique_lock lock(mutex); + ready.wait(lock, [&] { return gate_released; }); + } + if (!NPUAccessManager::try_acquire_npu_access()) { + throw std::runtime_error( + "queued request did not acquire released NPU gate"); + } + { + std::lock_guard lock(mutex); + second_insert_entered = true; + second_entered_early = !generate_completed; + } + NPUAccessManager::release_npu_access(); + } catch (...) { + second_error = std::current_exception(); + ready.notify_all(); + } + }); + + first.join(); + second.join(); + if (first_error) { + std::rethrow_exception(first_error); + } + if (second_error) { + std::rethrow_exception(second_error); + } + + CHECK(second_attempt_rejected); + CHECK(generate_completed); + CHECK(second_insert_entered); + CHECK(!second_entered_early); + CHECK(NPUAccessManager::is_npu_available()); + CHECK(NPUAccessManager::get_active_npu_requests() == 0); +} + +void TestCompleteRequestGate() { + CHECK(requires_npu_access("POST", "/api/generate")); + CHECK(requires_npu_access("POST", "/api/chat")); + CHECK(requires_npu_access("POST", "/v1/chat/completions")); + CHECK(requires_npu_access("POST", "/v1/completions")); + CHECK(!requires_npu_access("GET", "/v1/completions")); + + RunForcedGateInterleaving(true); + RunForcedGateInterleaving(false); +} + +} // namespace + +int main() { + try { + TestHandlerSpecificPresence(); + TestEndpointDefaultsAndPropagation(); + TestOllamaChatLimitStaysSoftOnly(); + TestNestedModelErrorAndHttpStatus(); + TestCliLimitAndRecoverableNotice(); + TestAie4ModelInfoDetection(); + TestCompleteRequestGate(); + std::cout << "test_generation_limit: PASS\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << "test_generation_limit: FAIL: " + << error.what() << '\n'; + return 1; + } +} diff --git a/src/test/phi4_corelib_aie4/test_phi4_frontend.cpp b/src/test/phi4_corelib_aie4/test_phi4_frontend.cpp index fad4fc2d..829c86d5 100644 --- a/src/test/phi4_corelib_aie4/test_phi4_frontend.cpp +++ b/src/test/phi4_corelib_aie4/test_phi4_frontend.cpp @@ -1,5 +1,6 @@ #include #include +#include #if defined(FLM_ENABLE_CORELIB_AIE4) #include @@ -703,6 +704,72 @@ void TestDefaultChatLimitWithoutExplicitRequestIsAdmitted() { CheckAligned(*model, *g_factory.engine, {41, 200020}); } +void TestEndpointLimitsAtLoweredCap() { + TempModelPackage package; + FactoryScope factory; + auto model = Load( + package, + ModelInfo(512, "corelib_aie4"), + -1, + false, + nullptr); + FakeEngine* engine = g_factory.engine; + + for (const GenerationEndpoint endpoint : { + GenerationEndpoint::Generate, + GenerationEndpoint::OpenAiChatCompletion, + GenerationEndpoint::OpenAiCompletion}) { + const ParsedGenerationLimit omitted = + ParseGenerationLimit( + nlohmann::ordered_json::object(), + endpoint); + CHECK(!omitted.explicit_limit); + CHECK(GenerationLoopLimit(omitted, true) == -1); + + g_encoded_tokens = {41}; + g_sample_tokens.clear(); + g_sample_token = 7; + auto meta = Meta(); + auto input = Prompt(RequestedMaxNewTokens(omitted)); + CHECK(model->insert(meta, input)); + std::ostringstream output; + (void)model->generate( + meta, + GenerationLoopLimit(omitted, true), + output); + CHECK(meta.generated_tokens == 511); + CHECK(model->get_current_context_length() == 512); + CHECK(engine->position == 512); + + model->clear_context(); + nlohmann::ordered_json explicit_request; + if (endpoint == + GenerationEndpoint::OpenAiChatCompletion) { + explicit_request["max_completion_tokens"] = 512; + } else { + explicit_request["max_tokens"] = 512; + } + const ParsedGenerationLimit explicit_limit = + ParseGenerationLimit(explicit_request, endpoint); + CHECK(explicit_limit.explicit_limit); + CHECK(explicit_limit.value == 512); + + meta = Meta(); + input = Prompt(RequestedMaxNewTokens(explicit_limit)); + const int clear_count = engine->clear_count; + const auto prefill_count = engine->prefill_calls.size(); + CheckRequestError( + [&] { (void)model->insert(meta, input); }, + 400, + false, + "512"); + CHECK(engine->position == 0); + CHECK(engine->clear_count == clear_count); + CHECK(engine->prefill_calls.size() == prefill_count); + CHECK(Phi4FrontendTestAccess::History(*model).empty()); + } +} + void TestLengthStopCommitsTokensBeforeForcedAppend() { TempModelPackage package; FactoryScope factory; @@ -1193,6 +1260,7 @@ int main() { TestEnginePositionIsAuthoritativeForCapUpdate(); TestRenderedCapacityIsAtomic(); TestDefaultChatLimitWithoutExplicitRequestIsAdmitted(); + TestEndpointLimitsAtLoweredCap(); TestLengthStopCommitsTokensBeforeForcedAppend(); TestEosStopCommitsTokenBeforeCapUpdateAndAppend(); TestActiveCapNeverEmitsUncommittedToken(); From ed97a8ef586c19fef2e05dbee4fdaaa853d4ee62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CChi?= Date: Tue, 1 Sep 2026 04:44:20 -0700 Subject: [PATCH 019/117] fix: serialize AIE4 response completion Co-authored-by: Cursor --- .../task-10-report.md | 128 ++++++++++ src/include/AutoModel/automodel.hpp | 5 + src/include/AutoModel/modeling_phi4.hpp | 7 + src/include/server/generation_limit.hpp | 11 + src/include/server/npu_access_manager.hpp | 29 +++ src/include/server/serve_lifecycle.hpp | 44 ++++ src/server/generation_limit.cpp | 23 ++ src/server/rest_handler.cpp | 39 ++- src/server/rest_handler.hpp | 1 - src/server/server.cpp | 28 +- src/src/main.cpp | 35 ++- .../test_generation_limit.cpp | 240 +++++++++++++----- .../phi4_corelib_aie4/test_phi4_frontend.cpp | 61 +++++ 13 files changed, 537 insertions(+), 114 deletions(-) create mode 100644 .superpowers/sdd/2026-08-31-phi4-aie4-corelib-fastflow/task-10-report.md create mode 100644 src/include/server/serve_lifecycle.hpp diff --git a/.superpowers/sdd/2026-08-31-phi4-aie4-corelib-fastflow/task-10-report.md b/.superpowers/sdd/2026-08-31-phi4-aie4-corelib-fastflow/task-10-report.md new file mode 100644 index 00000000..1ebc5f6f --- /dev/null +++ b/.superpowers/sdd/2026-08-31-phi4-aie4-corelib-fastflow/task-10-report.md @@ -0,0 +1,128 @@ +# Task 10 Report: REST, CLI, validation, and shutdown + +## Status + +Implemented, verified, self-reviewed, and committed locally. Nothing was +pushed. + +## Implementation + +- Added handler-specific generation-limit parsing. `/api/generate` and + `/v1/completions` recognize only `max_tokens`; `/v1/chat/completions` + prefers `max_tokens` and then `max_completion_tokens`. +- Preserved the legacy omitted limit of 4096 while passing `-1` to the AIE4 + generation loop and leaving `requested_max_new_tokens` empty. Explicit + supported fields are propagated before insertion. +- Kept Ollama `/api/chat` `options.num_predict` solely as a soft generation + loop bound. The streaming path now calls `generate` rather than performing a + duplicate insertion, and never reserves AIE4 output capacity. +- Added nested `ModelRequestError` serialization with code, type, and + `session_cleared`. Nested 400 and 500 responses map to HTTP 400 and 500. + Streaming failures after output begins send a final nested error chunk; + failures before the first chunk remain ordinary HTTP errors. +- Typed 400/false errors preserve REST and CLI session state. Typed 500/true + errors report the frontend clear without calling `clear_context` again. +- Runner propagates positive `/set gen-lim` values to + `requested_max_new_tokens`; unbounded/nonpositive values remain empty. + Invalid `/set ctx-len` reports and returns to the prompt. Recoverable + session-cleared failures print the required exact notice. +- Runner model-load exceptions now unwind to `main` instead of exiting, so + owned models are destroyed before explicit process shutdown. +- Split the existing `NPUAccessManager` implementation into a lightweight, + testable source without adding another production mutex. Added the missing + `/v1/completions` gate and two forced interleaving tests spanning insert and + generate. +- `flm validate` now reports legacy XDNA2 and optional corelib AIE4 readiness + independently. AIE4 readiness is obtained through `CorelibRuntime`, which + checks DLL loading, dependency self-test, device context, and writable fatal + logging without applying `__NPU_VERSION__`. +- Added best-effort prior fatal-record draining, SIGINT command shutdown, and + explicit `CorelibRuntime::ShutdownProcess()` after Runner/WebServer + destruction and on error paths. Validation also shuts down its runtime. +- Feature-OFF main/Runner/REST/server compile checks do not receive corelib + headers or `FLM_ENABLE_CORELIB_AIE4`. + +## RED evidence + +- The first `test_generation_limit` build failed with MSVC C1083 because + `server/generation_limit.hpp` did not exist. + +## Verification + +- Focused generation-limit, frontend ON/OFF, fatal-record/runtime, and gate + tests passed. +- Feature-OFF and feature-ON production compile-check targets passed for the + changed generation, gate, REST, server, Runner, and main translation units. +- Complete standalone Release CTest: 10/10 passed, 0 failed. +- CRLF-aware staged whitespace check passed; the post-commit worktree is clean + and `git diff --check` passes. +- IDE diagnostics reported no errors. +- Broad `flm` build remains blocked before FastFlow sources by the existing + tokenizer custom target (`tokenizers_c.lib`: `no such file or directory`, + Cargo unavailable). + +## Self-review + +- Rechecked every Task 10 brief item, handler field precedence, omitted and + explicit lowered-cap behavior, pre/post-stream error delivery, clear + ownership, gate duration, validation independence, feature guards, and + destruction-before-shutdown ordering. +- No catalog, installer, or parked-minor files were changed. + +## Concerns + +- Full HTTP socket-level and interactive CLI execution still require the broad + product dependency set; focused policy, frontend, runtime, and production + translation-unit checks cover the behavior available without it. +- Real AIE4 hardware/package validation remains deferred to the integration + tasks. Existing XRT `NOMINMAX` and duration-conversion warnings remain. + +## Commit + +- `2fb9302e fix: enforce Phi-4 AIE4 request limits` +- Push: not performed + +## Fix round 1/5 + +### Important findings resolved + +- OpenAI chat/completion post-output `ModelRequestError` delivery now writes + exactly `data: \n\n` with `is_final=false`, then + `data: [DONE]\n\n` with `is_final=true`. Ollama streaming remains NDJSON. + The byte-exact callback test parses the transmitted SSE data and observes + `error.session_cleared=true`. +- SIGINT registration is scoped to `serve` and restores the prior handler. + Non-serve commands retain their ordinary first-Ctrl+C behavior. The tested + serve shutdown sequence stops admission and joins synchronous handlers, + destroys WebServer/routes/RestHandler/AutoModel, then performs healthy + `CorelibRuntime::ShutdownProcess`. +- NPU queue advancement was removed from response callbacks. A single + scope-safe completion guard in `process_task` advances/releases the gate + only after the handler and all trailing context/cache work return, including + parse failures and typed, standard, unknown, cancellation, and final paths. + The forced interleaving test covers nonstream, streaming, exception, and + cancellation-final exits and verifies no early second insert or gate leak. +- Omitted-limit routing now queries `AutoModel::uses_corelib_aie4()` only after + successful model load. The default is false; Phi4 reports true only for its + actual loaded corelib backend. Tests prove misleading non-Phi metadata and + feature-OFF/legacy Phi4 retain 4096 while loaded AIE4 uses `-1`. + +### RED evidence + +- `test_generation_limit` first failed with C1083 for the not-yet-created + `server/serve_lifecycle.hpp`. +- `test_phi4_frontend_off` first failed because + `AutoModel/Phi4::uses_corelib_aie4()` did not exist. + +### Verification and self-review + +- Focused Release CTest passed: generation/gate/lifecycle plus frontend + OFF/ON, 3/3. +- Complete Release build passed, including production and frontend + feature-OFF/ON compile checks. +- Complete Release CTest passed: 10/10, 0 failed. +- IDE diagnostics reported no errors. Review confirmed callback releases are + gone, all three limit decisions occur after load, SIGINT is serve-only, and + both OpenAI typed streaming catches use the shared SSE sender. +- Unrelated Minor findings were not changed. Existing broad-build dependency + and hardware-validation concerns remain as documented above. diff --git a/src/include/AutoModel/automodel.hpp b/src/include/AutoModel/automodel.hpp index eabaa0a5..eb0d3d84 100644 --- a/src/include/AutoModel/automodel.hpp +++ b/src/include/AutoModel/automodel.hpp @@ -261,6 +261,11 @@ class AutoModel { /// \return the current model std::string get_current_model(); + /// \brief Whether the loaded model uses the corelib AIE4 backend + virtual bool uses_corelib_aie4() const noexcept { + return false; + } + /// \brief Get the current context length /// \return the current context length virtual int get_current_context_length(); diff --git a/src/include/AutoModel/modeling_phi4.hpp b/src/include/AutoModel/modeling_phi4.hpp index cedfe004..4d44d63e 100644 --- a/src/include/AutoModel/modeling_phi4.hpp +++ b/src/include/AutoModel/modeling_phi4.hpp @@ -77,6 +77,13 @@ class Phi4 : public AutoModel { Phi4(flm_rt::device* npu_device_inst); void load_model(std::string model_path, json model_inf, int default_context_length = -1, bool enable_preemption = false) override; + bool uses_corelib_aie4() const noexcept override { +#if defined(FLM_ENABLE_CORELIB_AIE4) + return uses_corelib_aie4_; +#else + return false; +#endif + } //void toggle_enable_think() override; bool insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std::function is_cancelled = [] { return false; }) override; std::string generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled = [] { return false; }) override; diff --git a/src/include/server/generation_limit.hpp b/src/include/server/generation_limit.hpp index cc836872..45364301 100644 --- a/src/include/server/generation_limit.hpp +++ b/src/include/server/generation_limit.hpp @@ -2,6 +2,8 @@ #include +#include +#include #include #include #include @@ -39,6 +41,15 @@ nlohmann::ordered_json ModelErrorResponse( int HttpStatusForResponse( const nlohmann::ordered_json& response) noexcept; +std::array OpenAiStreamingErrorFrames( + const nlohmann::ordered_json& error_response); + +void SendOpenAiStreamingError( + const nlohmann::ordered_json& error_response, + const std::function& send_streaming_response); + bool UseFinalStreamingErrorChunk( bool stream_started) noexcept; diff --git a/src/include/server/npu_access_manager.hpp b/src/include/server/npu_access_manager.hpp index 6245cc4b..7627be86 100644 --- a/src/include/server/npu_access_manager.hpp +++ b/src/include/server/npu_access_manager.hpp @@ -1,6 +1,8 @@ #pragma once +#include #include +#include class NPUAccessManager { public: @@ -10,6 +12,33 @@ class NPUAccessManager { static int get_active_npu_requests(); }; +class NPURequestCompletionGuard final { +public: + explicit NPURequestCompletionGuard( + std::function completion) + : completion_(std::move(completion)) {} + + ~NPURequestCompletionGuard() noexcept { + try { + if (completion_) { + completion_(); + } + } catch (...) { + if (!NPUAccessManager::is_npu_available()) { + NPUAccessManager::release_npu_access(); + } + } + } + + NPURequestCompletionGuard( + const NPURequestCompletionGuard&) = delete; + NPURequestCompletionGuard& operator=( + const NPURequestCompletionGuard&) = delete; + +private: + std::function completion_; +}; + bool requires_npu_access( const std::string& method, const std::string& path); diff --git a/src/include/server/serve_lifecycle.hpp b/src/include/server/serve_lifecycle.hpp new file mode 100644 index 00000000..f7c78b4a --- /dev/null +++ b/src/include/server/serve_lifecycle.hpp @@ -0,0 +1,44 @@ +#pragma once + +#include +#include +#include + +class ScopedSignalHandler final { +public: + using Handler = void (*)(int); + + ScopedSignalHandler(int signal_number, Handler handler) + : signal_number_(signal_number), + previous_(std::signal(signal_number, handler)) { + if (previous_ == SIG_ERR) { + throw std::runtime_error("failed to register signal handler"); + } + } + + ~ScopedSignalHandler() { + if (previous_ != SIG_ERR) { + (void)std::signal(signal_number_, previous_); + } + } + + ScopedSignalHandler(const ScopedSignalHandler&) = delete; + ScopedSignalHandler& operator=(const ScopedSignalHandler&) = delete; + +private: + int signal_number_; + Handler previous_; +}; + +template < + typename StopAndWait, + typename DestroyServer, + typename ShutdownRuntime> +bool CompleteServeShutdown( + StopAndWait&& stop_and_wait, + DestroyServer&& destroy_server, + ShutdownRuntime&& shutdown_runtime) { + std::forward(stop_and_wait)(); + std::forward(destroy_server)(); + return std::forward(shutdown_runtime)(); +} diff --git a/src/server/generation_limit.cpp b/src/server/generation_limit.cpp index 24efce82..9cf9832c 100644 --- a/src/server/generation_limit.cpp +++ b/src/server/generation_limit.cpp @@ -98,6 +98,29 @@ int HttpStatusForResponse( } } +std::array OpenAiStreamingErrorFrames( + const nlohmann::ordered_json& error_response) { + return { + "data: " + error_response.dump() + "\n\n", + "data: [DONE]\n\n", + }; +} + +void SendOpenAiStreamingError( + const nlohmann::ordered_json& error_response, + const std::function& send_streaming_response) { + const auto frames = + OpenAiStreamingErrorFrames(error_response); + send_streaming_response( + nlohmann::ordered_json(frames[0]), + false); + send_streaming_response( + nlohmann::ordered_json(frames[1]), + true); +} + bool UseFinalStreamingErrorChunk( bool stream_started) noexcept { return stream_started; diff --git a/src/server/rest_handler.cpp b/src/server/rest_handler.cpp index 7da97e00..a352e944 100644 --- a/src/server/rest_handler.cpp +++ b/src/server/rest_handler.cpp @@ -431,16 +431,6 @@ bool RestHandler::ensure_model_loaded(const std::string& model_tag) { return true; } -bool RestHandler::uses_corelib_aie4(const std::string& model_tag) { - if (!supported_models.is_model_supported(model_tag)) { - return false; - } - const auto [resolved_tag, model_info] = - supported_models.get_model_info(model_tag); - (void)resolved_tag; - return IsCorelibAie4ModelInfo(model_info); -} - ///@brief Ensure the asr model is loaded ///@param model_tag the model tag void RestHandler::ensure_asr_model_loaded(const std::string& model_tag) { @@ -668,9 +658,6 @@ void RestHandler::handle_generate(const json& request, bool stream = request.value("stream", true); std::string model = request.value("model", current_model_tag); json options = request.value("options", json::object()); - const bool corelib_aie4 = uses_corelib_aie4(model); - const int length_limit = - GenerationLoopLimit(parsed_limit, corelib_aie4); auto load_start_time = time_utils::now(); // TODO: Use Another Check Function avoid loading again if (!ensure_model_loaded(model)) { @@ -678,6 +665,10 @@ void RestHandler::handle_generate(const json& request, send_response(error_response); return; } + const int length_limit = + GenerationLoopLimit( + parsed_limit, + auto_chat_engine->uses_corelib_aie4()); auto load_end_time = time_utils::now(); chat_meta_info_t meta_info; @@ -1183,9 +1174,6 @@ void RestHandler::handle_openai_chat_completion(const json& request, json current_messages = request["messages"]; std::string model = request.value("model", current_model_tag); bool stream = request.value("stream", false); - const bool corelib_aie4 = uses_corelib_aie4(model); - const int length_limit = - GenerationLoopLimit(parsed_limit, corelib_aie4); json tools = request.value("tools", json::array()); json options = request.value("options", json::object()); @@ -1195,6 +1183,10 @@ void RestHandler::handle_openai_chat_completion(const json& request, send_response(error_response); return; } + const bool corelib_aie4 = + auto_chat_engine->uses_corelib_aie4(); + const int length_limit = + GenerationLoopLimit(parsed_limit, corelib_aie4); auto load_end_time = time_utils::now(); configure_chat_engine_parameters(options, request); @@ -1301,7 +1293,9 @@ void RestHandler::handle_openai_chat_completion(const json& request, } catch (const ModelRequestError& error) { const json error_response = ModelErrorResponse(error); if (UseFinalStreamingErrorChunk(stream_started)) { - send_streaming_response(error_response, true); + SendOpenAiStreamingError( + error_response, + send_streaming_response); } else { send_response(error_response); } @@ -1507,9 +1501,6 @@ void RestHandler::handle_openai_completion(const json& request, std::string reasoning_effort = request.value("reasoning_effort", "medium"); bool stream = request.value("stream", false); json options = request.value("options", json::object()); - const bool corelib_aie4 = uses_corelib_aie4(model); - const int length_limit = - GenerationLoopLimit(parsed_limit, corelib_aie4); // direct return if model not supported if (!supported_models.is_model_supported(model)) { @@ -1521,6 +1512,10 @@ void RestHandler::handle_openai_completion(const json& request, send_response(error_response); return; } + const int length_limit = + GenerationLoopLimit( + parsed_limit, + auto_chat_engine->uses_corelib_aie4()); configure_chat_engine_parameters(options, request); @@ -1567,7 +1562,9 @@ void RestHandler::handle_openai_completion(const json& request, } catch (const ModelRequestError& error) { const json error_response = ModelErrorResponse(error); if (UseFinalStreamingErrorChunk(stream_started)) { - send_streaming_response(error_response, true); + SendOpenAiStreamingError( + error_response, + send_streaming_response); } else { send_response(error_response); } diff --git a/src/server/rest_handler.hpp b/src/server/rest_handler.hpp index 51b520bc..9e5fcad7 100644 --- a/src/server/rest_handler.hpp +++ b/src/server/rest_handler.hpp @@ -109,7 +109,6 @@ class RestHandler { private: bool ensure_model_loaded(const std::string& model_tag); - bool uses_corelib_aie4(const std::string& model_tag); void ensure_asr_model_loaded(const std::string& model_tag); void ensure_embed_model_loaded(const std::string& model_tag); void configure_chat_engine_parameters(const json& options, const json& request); diff --git a/src/server/server.cpp b/src/server/server.cpp index bc0a5d66..cfcaadb7 100644 --- a/src/server/server.cpp +++ b/src/server/server.cpp @@ -638,6 +638,11 @@ bool WebServer::handle_request(http::request& req, auto process_task = [this, it, req_ptr, res_ptr, session, needs_npu, key, is_json](bool is_deferred) { auto& req_ref = *req_ptr; auto& res_ref = *res_ptr; + NPURequestCompletionGuard npu_completion([this, needs_npu] { + if (needs_npu) { + this->process_next_npu_request(); + } + }); // Parse JSON request body json request_json; @@ -659,9 +664,6 @@ bool WebServer::handle_request(http::request& req, // Only write from callback when deferred if (is_deferred && session) session->write_response_from_callback(); - if (needs_npu) { - this->process_next_npu_request(); - } return; } @@ -680,7 +682,7 @@ bool WebServer::handle_request(http::request& req, register_active_request(request_id, cancellation_token); // catch is_deferred - auto send_response = [res_ptr, session, this, request_id, needs_npu, is_deferred, cancellation_token](const json& response_data) { + auto send_response = [res_ptr, session, this, request_id, is_deferred, cancellation_token](const json& response_data) { auto& response_ref = *res_ptr; const int status_code = HttpStatusForResponse(response_data); @@ -698,16 +700,12 @@ bool WebServer::handle_request(http::request& req, cancellation_token->complete(); unregister_active_request(request_id); - if (needs_npu) { - this->process_next_npu_request(); - } - if (is_deferred && session) { session->write_response_from_callback(); } }; - auto send_streaming_response = [session, this, request_id, needs_npu, cancellation_token](const json& data, bool is_final) { + auto send_streaming_response = [session, this, request_id, cancellation_token](const json& data, bool is_final) { if (is_final) { cancellation_token->complete(); } @@ -717,10 +715,6 @@ bool WebServer::handle_request(http::request& req, } if (is_final) { unregister_active_request(request_id); - - if (needs_npu) { - this->process_next_npu_request(); - } } }; @@ -736,10 +730,6 @@ bool WebServer::handle_request(http::request& req, res_ref.set(http::field::content_type, "application/json"); res_ref.prepare_payload(); - if (needs_npu) { - this->process_next_npu_request(); - } - if (is_deferred && session) { session->write_response_from_callback(); } @@ -753,10 +743,6 @@ bool WebServer::handle_request(http::request& req, res_ref.set(http::field::content_type, "application/json"); res_ref.prepare_payload(); - if (needs_npu) { - this->process_next_npu_request(); - } - if (is_deferred && session) { session->write_response_from_callback(); } diff --git a/src/src/main.cpp b/src/src/main.cpp index d9d4ec76..d9e91461 100644 --- a/src/src/main.cpp +++ b/src/src/main.cpp @@ -7,6 +7,7 @@ #pragma once #include "runner.hpp" #include "server.hpp" +#include #include "model_list.hpp" #include "model_downloader.hpp" #include "update.hpp" @@ -563,8 +564,6 @@ int main(int argc, char* argv[]) { // XRT backend: preload bundled XRT libraries from the executable directory. preload_bundled_libraries(); #endif - std::signal(SIGINT, signal_handler); - // Parse command line arguments using Boost Program Options program_args_t parsed_args; if (!arg_utils::parse_options(argc, argv, parsed_args)) { @@ -770,6 +769,9 @@ int main(int argc, char* argv[]) { try { +#if defined(FLM_ENABLE_CORELIB_AIE4) + bool corelib_shutdown_complete = false; +#endif // Load the model list with the models directory as the base ModelDownloader downloader(availble_models); @@ -788,6 +790,8 @@ int main(int argc, char* argv[]) { runner.run(); } else if (parsed_args.command == "serve") { + running.store(true); + ScopedSignalHandler sigint_scope(SIGINT, signal_handler); check_and_notify_new_version(); // Create the server int port = utils::get_server_port(parsed_args.port); @@ -810,8 +814,28 @@ int main(int argc, char* argv[]) { std::unique_lock lock(mtx); cv.wait(lock, [] { return !running.load(); }); } - // header_print("FLM", "Stopping server..."); - // server->stop(); + const bool healthy_shutdown = CompleteServeShutdown( + [&] { + // Stop admission and join all synchronous request handlers. + server->stop(); + }, + [&] { + // Destroy routes, RestHandler, and the loaded AutoModel. + server.reset(); + }, + [&] { +#if defined(FLM_ENABLE_CORELIB_AIE4) + return shutdown_corelib_process(); +#else + return true; +#endif + }); + if (!healthy_shutdown) { + return 1; + } +#if defined(FLM_ENABLE_CORELIB_AIE4) + corelib_shutdown_complete = true; +#endif } else if (parsed_args.command == "pull") { bool success = downloader.pull_model(parsed_args.model_tag, parsed_args.modelscope, parsed_args.force_redownload); @@ -884,7 +908,8 @@ int main(int argc, char* argv[]) { return 1; } #if defined(FLM_ENABLE_CORELIB_AIE4) - if (!shutdown_corelib_process()) { + if (!corelib_shutdown_complete && + !shutdown_corelib_process()) { return 1; } #endif diff --git a/src/test/phi4_corelib_aie4/test_generation_limit.cpp b/src/test/phi4_corelib_aie4/test_generation_limit.cpp index 1ff466d8..25a9f4c9 100644 --- a/src/test/phi4_corelib_aie4/test_generation_limit.cpp +++ b/src/test/phi4_corelib_aie4/test_generation_limit.cpp @@ -1,16 +1,21 @@ #include #include +#include #include "test_support.hpp" +#include +#include #include #include +#include #include #include #include #include #include #include +#include namespace { @@ -131,6 +136,39 @@ void TestNestedModelErrorAndHttpStatus() { CHECK(UseFinalStreamingErrorChunk(true)); } +void TestOpenAiStreamingErrorFramingAndParsing() { + const ordered_json error = + ModelErrorResponse("submission failed", 500, true); + std::vector> transmitted; + SendOpenAiStreamingError( + error, + [&](const ordered_json& data, bool is_final) { + CHECK(data.is_string()); + transmitted.emplace_back( + data.get(), + is_final); + }); + const std::string expected_error = + "data: " + error.dump() + "\n\n"; + + CHECK(transmitted.size() == 2); + CHECK(transmitted[0].first == expected_error); + CHECK(!transmitted[0].second); + CHECK(transmitted[1].first == "data: [DONE]\n\n"); + CHECK(transmitted[1].second); + + const auto parse_sse_data = [](const std::string& frame) { + CHECK(frame.starts_with("data: ")); + CHECK(frame.ends_with("\n\n")); + return frame.substr(6, frame.size() - 8); + }; + const ordered_json parsed = + ordered_json::parse( + parse_sse_data(transmitted[0].first)); + CHECK(parsed["error"]["session_cleared"] == true); + CHECK(parse_sse_data(transmitted[1].first) == "[DONE]"); +} + void TestCliLimitAndRecoverableNotice() { CHECK(!CliRequestedMaxNewTokens(-1).has_value()); CHECK(!CliRequestedMaxNewTokens(0).has_value()); @@ -156,82 +194,45 @@ void TestAie4ModelInfoDetection() { CHECK(!IsCorelibAie4ModelInfo(ordered_json::object())); } -void RunForcedGateInterleaving(bool queue_during_insert) { +enum class HandlerExit { + NonStreaming, + Streaming, + Exception, + CancellationFinal +}; + +void RunProductionShapedGateInterleaving(HandlerExit exit) { CHECK(NPUAccessManager::is_npu_available()); CHECK(NPUAccessManager::get_active_npu_requests() == 0); + CHECK(NPUAccessManager::try_acquire_npu_access()); std::mutex mutex; std::condition_variable ready; - bool insert_entered = false; - bool generate_entered = false; - bool second_attempted = false; + bool final_response_sent = false; + bool second_attempted_while_handler_active = false; bool second_attempt_rejected = false; - bool generate_completed = false; - bool gate_released = false; + bool response_was_streaming = false; + bool cancellation_finalized = false; + bool post_response_cleanup_completed = false; + bool handler_returned = false; + bool completion_point_reached = false; bool second_insert_entered = false; - bool second_entered_early = false; - std::exception_ptr first_error; + bool second_entered_before_cleanup = false; std::exception_ptr second_error; - std::thread first([&] { - try { - if (!NPUAccessManager::try_acquire_npu_access()) { - throw std::runtime_error( - "first request did not acquire NPU access"); - } - { - std::unique_lock lock(mutex); - insert_entered = true; - ready.notify_all(); - if (queue_during_insert) { - ready.wait( - lock, - [&] { return second_attempted; }); - } - generate_entered = true; - ready.notify_all(); - if (!queue_during_insert) { - ready.wait( - lock, - [&] { return second_attempted; }); - } - generate_completed = true; - } - NPUAccessManager::release_npu_access(); - { - std::lock_guard lock(mutex); - gate_released = true; - } - ready.notify_all(); - } catch (...) { - first_error = std::current_exception(); - ready.notify_all(); - } - }); - std::thread second([&] { try { { std::unique_lock lock(mutex); - ready.wait( - lock, - [&] { - return queue_during_insert - ? insert_entered - : generate_entered; - }); + ready.wait(lock, [&] { return final_response_sent; }); } const bool acquired_early = NPUAccessManager::try_acquire_npu_access(); { std::lock_guard lock(mutex); + second_attempted_while_handler_active = true; second_attempt_rejected = !acquired_early; - if (acquired_early) { - second_insert_entered = true; - second_entered_early = !generate_completed; - } - second_attempted = true; } ready.notify_all(); if (acquired_early) { @@ -241,7 +242,7 @@ void RunForcedGateInterleaving(bool queue_during_insert) { { std::unique_lock lock(mutex); - ready.wait(lock, [&] { return gate_released; }); + ready.wait(lock, [&] { return completion_point_reached; }); } if (!NPUAccessManager::try_acquire_npu_access()) { throw std::runtime_error( @@ -250,7 +251,9 @@ void RunForcedGateInterleaving(bool queue_during_insert) { { std::lock_guard lock(mutex); second_insert_entered = true; - second_entered_early = !generate_completed; + second_entered_before_cleanup = + !post_response_cleanup_completed || + !handler_returned; } NPUAccessManager::release_npu_access(); } catch (...) { @@ -259,19 +262,67 @@ void RunForcedGateInterleaving(bool queue_during_insert) { } }); - first.join(); - second.join(); - if (first_error) { - std::rethrow_exception(first_error); + { + NPURequestCompletionGuard completion([&] { + NPUAccessManager::release_npu_access(); + { + std::lock_guard lock(mutex); + completion_point_reached = true; + } + ready.notify_all(); + }); + + try { + { + std::lock_guard lock(mutex); + response_was_streaming = + exit == HandlerExit::Streaming || + exit == HandlerExit::CancellationFinal; + cancellation_finalized = + exit == HandlerExit::CancellationFinal; + final_response_sent = true; + } + ready.notify_all(); + { + std::unique_lock lock(mutex); + ready.wait( + lock, + [&] { + return second_attempted_while_handler_active; + }); + } + + if (exit == HandlerExit::Exception) { + throw std::runtime_error("forced handler failure"); + } + } catch (const std::exception&) { + // Production process_task catches before its completion guard exits. + } + + { + std::lock_guard lock(mutex); + post_response_cleanup_completed = true; + handler_returned = true; + } } + + second.join(); if (second_error) { std::rethrow_exception(second_error); } CHECK(second_attempt_rejected); - CHECK(generate_completed); + CHECK( + response_was_streaming == + (exit == HandlerExit::Streaming || + exit == HandlerExit::CancellationFinal)); + CHECK( + cancellation_finalized == + (exit == HandlerExit::CancellationFinal)); + CHECK(post_response_cleanup_completed); + CHECK(handler_returned); CHECK(second_insert_entered); - CHECK(!second_entered_early); + CHECK(!second_entered_before_cleanup); CHECK(NPUAccessManager::is_npu_available()); CHECK(NPUAccessManager::get_active_npu_requests() == 0); } @@ -283,8 +334,63 @@ void TestCompleteRequestGate() { CHECK(requires_npu_access("POST", "/v1/completions")); CHECK(!requires_npu_access("GET", "/v1/completions")); - RunForcedGateInterleaving(true); - RunForcedGateInterleaving(false); + for (const HandlerExit exit : { + HandlerExit::NonStreaming, + HandlerExit::Streaming, + HandlerExit::Exception, + HandlerExit::CancellationFinal}) { + RunProductionShapedGateInterleaving(exit); + } + + CHECK(NPUAccessManager::try_acquire_npu_access()); + { + NPURequestCompletionGuard completion([] { + throw std::runtime_error("forced queue handoff failure"); + }); + } + CHECK(NPUAccessManager::is_npu_available()); + CHECK(NPUAccessManager::get_active_npu_requests() == 0); +} + +volatile std::sig_atomic_t g_signal_observation = 0; + +void PriorSignalHandler(int) { + g_signal_observation = 1; +} + +void ServeSignalHandler(int) { + g_signal_observation = 2; +} + +void TestServeSignalScopeAndShutdownOrder() { + const auto original = std::signal(SIGINT, PriorSignalHandler); + CHECK(original != SIG_ERR); + { + ScopedSignalHandler scope(SIGINT, ServeSignalHandler); + g_signal_observation = 0; + CHECK(std::raise(SIGINT) == 0); + CHECK(g_signal_observation == 2); + } + + g_signal_observation = 0; + CHECK(std::raise(SIGINT) == 0); + CHECK(g_signal_observation == 1); + CHECK(std::signal(SIGINT, original) != SIG_ERR); + + std::vector order; + const bool healthy = CompleteServeShutdown( + [&] { order.push_back("stop-admission-and-wait-inflight"); }, + [&] { order.push_back("destroy-server-handler-engine"); }, + [&] { + order.push_back("shutdown-corelib"); + return true; + }); + CHECK(healthy); + CHECK( + order == std::vector({ + "stop-admission-and-wait-inflight", + "destroy-server-handler-engine", + "shutdown-corelib"})); } } // namespace @@ -295,9 +401,11 @@ int main() { TestEndpointDefaultsAndPropagation(); TestOllamaChatLimitStaysSoftOnly(); TestNestedModelErrorAndHttpStatus(); + TestOpenAiStreamingErrorFramingAndParsing(); TestCliLimitAndRecoverableNotice(); TestAie4ModelInfoDetection(); TestCompleteRequestGate(); + TestServeSignalScopeAndShutdownOrder(); std::cout << "test_generation_limit: PASS\n"; return 0; } catch (const std::exception& error) { diff --git a/src/test/phi4_corelib_aie4/test_phi4_frontend.cpp b/src/test/phi4_corelib_aie4/test_phi4_frontend.cpp index 829c86d5..f178d65d 100644 --- a/src/test/phi4_corelib_aie4/test_phi4_frontend.cpp +++ b/src/test/phi4_corelib_aie4/test_phi4_frontend.cpp @@ -342,6 +342,40 @@ class Phi4FrontendTestAccess final { namespace { +class NonPhiModel final : public AutoModel { +public: + NonPhiModel() : AutoModel(nullptr, "non-phi-test") {} + + std::string generate( + chat_meta_info_t&, + int, + std::ostream&, + std::function) override { + return {}; + } + + bool insert( + chat_meta_info_t&, + lm_uniform_input_t&, + std::function) override { + return true; + } + + std::string generate_with_prompt( + chat_meta_info_t&, + lm_uniform_input_t&, + int, + std::ostream&) override { + return {}; + } + + std::string apply_chat_template( + nlohmann::ordered_json&, + nlohmann::ordered_json) override { + return {}; + } +}; + using flm::phi4::ContinuationRoute; using flm::phi4::ForcedContinuationRoute; using flm::phi4::SelectContinuationRoute; @@ -408,6 +442,7 @@ void TestLegacyRoutingAndUnknownBackend() { auto legacy = Load(package, ModelInfo(1024)); CHECK(g_factory.calls == 1); CHECK(!g_factory.last_was_corelib); + CHECK(!legacy->uses_corelib_aie4()); CHECK(Phi4FrontendTestAccess::HasLegacyNpu(*legacy)); FakeEngine* legacy_engine = g_factory.engine; @@ -441,6 +476,30 @@ void TestLegacyRoutingAndUnknownBackend() { CHECK(g_factory.calls == 1); } +void TestLoadedBackendControlsOmittedLimit() { + const ParsedGenerationLimit omitted{false, -1}; + const nlohmann::ordered_json misleading_catalog_info = { + {"details", {{"execution_backend", "corelib_aie4"}}}, + }; + CHECK(IsCorelibAie4ModelInfo(misleading_catalog_info)); + + NonPhiModel non_phi; + CHECK(!non_phi.uses_corelib_aie4()); + CHECK( + GenerationLoopLimit( + omitted, + non_phi.uses_corelib_aie4()) == 4096); + + TempModelPackage package({200020}); + FactoryScope factory; + auto legacy_phi = Load(package, ModelInfo(1024)); + CHECK(!legacy_phi->uses_corelib_aie4()); + CHECK( + GenerationLoopLimit( + omitted, + legacy_phi->uses_corelib_aie4()) == 4096); +} + void TestLegacyExactRepeatPreservesEmptyPrefillPayload() { TempModelPackage package({200020}); FactoryScope factory; @@ -530,6 +589,7 @@ void TestCorelibRoutingAndPreemption() { nullptr); CHECK(g_factory.calls == 1); CHECK(g_factory.last_was_corelib); + CHECK(model->uses_corelib_aie4()); CHECK(!Phi4FrontendTestAccess::HasLegacyNpu(*model)); CHECK(Phi4FrontendTestAccess::HasRuntime(*model)); } @@ -1252,6 +1312,7 @@ int main() { try { TestContinuationSelector(); TestLegacyRoutingAndUnknownBackend(); + TestLoadedBackendControlsOmittedLimit(); TestLegacyExactRepeatPreservesEmptyPrefillPayload(); #if defined(FLM_ENABLE_CORELIB_AIE4) ConfigureFakeCorelibDll(); From e1bae2d3d30dfd316887cace3b6b0f4cd862e1fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CChi?= Date: Tue, 1 Sep 2026 04:51:47 -0700 Subject: [PATCH 020/117] chore: untrack SDD task report Co-authored-by: Cursor --- .../task-10-report.md | 128 ------------------ 1 file changed, 128 deletions(-) delete mode 100644 .superpowers/sdd/2026-08-31-phi4-aie4-corelib-fastflow/task-10-report.md diff --git a/.superpowers/sdd/2026-08-31-phi4-aie4-corelib-fastflow/task-10-report.md b/.superpowers/sdd/2026-08-31-phi4-aie4-corelib-fastflow/task-10-report.md deleted file mode 100644 index 1ebc5f6f..00000000 --- a/.superpowers/sdd/2026-08-31-phi4-aie4-corelib-fastflow/task-10-report.md +++ /dev/null @@ -1,128 +0,0 @@ -# Task 10 Report: REST, CLI, validation, and shutdown - -## Status - -Implemented, verified, self-reviewed, and committed locally. Nothing was -pushed. - -## Implementation - -- Added handler-specific generation-limit parsing. `/api/generate` and - `/v1/completions` recognize only `max_tokens`; `/v1/chat/completions` - prefers `max_tokens` and then `max_completion_tokens`. -- Preserved the legacy omitted limit of 4096 while passing `-1` to the AIE4 - generation loop and leaving `requested_max_new_tokens` empty. Explicit - supported fields are propagated before insertion. -- Kept Ollama `/api/chat` `options.num_predict` solely as a soft generation - loop bound. The streaming path now calls `generate` rather than performing a - duplicate insertion, and never reserves AIE4 output capacity. -- Added nested `ModelRequestError` serialization with code, type, and - `session_cleared`. Nested 400 and 500 responses map to HTTP 400 and 500. - Streaming failures after output begins send a final nested error chunk; - failures before the first chunk remain ordinary HTTP errors. -- Typed 400/false errors preserve REST and CLI session state. Typed 500/true - errors report the frontend clear without calling `clear_context` again. -- Runner propagates positive `/set gen-lim` values to - `requested_max_new_tokens`; unbounded/nonpositive values remain empty. - Invalid `/set ctx-len` reports and returns to the prompt. Recoverable - session-cleared failures print the required exact notice. -- Runner model-load exceptions now unwind to `main` instead of exiting, so - owned models are destroyed before explicit process shutdown. -- Split the existing `NPUAccessManager` implementation into a lightweight, - testable source without adding another production mutex. Added the missing - `/v1/completions` gate and two forced interleaving tests spanning insert and - generate. -- `flm validate` now reports legacy XDNA2 and optional corelib AIE4 readiness - independently. AIE4 readiness is obtained through `CorelibRuntime`, which - checks DLL loading, dependency self-test, device context, and writable fatal - logging without applying `__NPU_VERSION__`. -- Added best-effort prior fatal-record draining, SIGINT command shutdown, and - explicit `CorelibRuntime::ShutdownProcess()` after Runner/WebServer - destruction and on error paths. Validation also shuts down its runtime. -- Feature-OFF main/Runner/REST/server compile checks do not receive corelib - headers or `FLM_ENABLE_CORELIB_AIE4`. - -## RED evidence - -- The first `test_generation_limit` build failed with MSVC C1083 because - `server/generation_limit.hpp` did not exist. - -## Verification - -- Focused generation-limit, frontend ON/OFF, fatal-record/runtime, and gate - tests passed. -- Feature-OFF and feature-ON production compile-check targets passed for the - changed generation, gate, REST, server, Runner, and main translation units. -- Complete standalone Release CTest: 10/10 passed, 0 failed. -- CRLF-aware staged whitespace check passed; the post-commit worktree is clean - and `git diff --check` passes. -- IDE diagnostics reported no errors. -- Broad `flm` build remains blocked before FastFlow sources by the existing - tokenizer custom target (`tokenizers_c.lib`: `no such file or directory`, - Cargo unavailable). - -## Self-review - -- Rechecked every Task 10 brief item, handler field precedence, omitted and - explicit lowered-cap behavior, pre/post-stream error delivery, clear - ownership, gate duration, validation independence, feature guards, and - destruction-before-shutdown ordering. -- No catalog, installer, or parked-minor files were changed. - -## Concerns - -- Full HTTP socket-level and interactive CLI execution still require the broad - product dependency set; focused policy, frontend, runtime, and production - translation-unit checks cover the behavior available without it. -- Real AIE4 hardware/package validation remains deferred to the integration - tasks. Existing XRT `NOMINMAX` and duration-conversion warnings remain. - -## Commit - -- `2fb9302e fix: enforce Phi-4 AIE4 request limits` -- Push: not performed - -## Fix round 1/5 - -### Important findings resolved - -- OpenAI chat/completion post-output `ModelRequestError` delivery now writes - exactly `data: \n\n` with `is_final=false`, then - `data: [DONE]\n\n` with `is_final=true`. Ollama streaming remains NDJSON. - The byte-exact callback test parses the transmitted SSE data and observes - `error.session_cleared=true`. -- SIGINT registration is scoped to `serve` and restores the prior handler. - Non-serve commands retain their ordinary first-Ctrl+C behavior. The tested - serve shutdown sequence stops admission and joins synchronous handlers, - destroys WebServer/routes/RestHandler/AutoModel, then performs healthy - `CorelibRuntime::ShutdownProcess`. -- NPU queue advancement was removed from response callbacks. A single - scope-safe completion guard in `process_task` advances/releases the gate - only after the handler and all trailing context/cache work return, including - parse failures and typed, standard, unknown, cancellation, and final paths. - The forced interleaving test covers nonstream, streaming, exception, and - cancellation-final exits and verifies no early second insert or gate leak. -- Omitted-limit routing now queries `AutoModel::uses_corelib_aie4()` only after - successful model load. The default is false; Phi4 reports true only for its - actual loaded corelib backend. Tests prove misleading non-Phi metadata and - feature-OFF/legacy Phi4 retain 4096 while loaded AIE4 uses `-1`. - -### RED evidence - -- `test_generation_limit` first failed with C1083 for the not-yet-created - `server/serve_lifecycle.hpp`. -- `test_phi4_frontend_off` first failed because - `AutoModel/Phi4::uses_corelib_aie4()` did not exist. - -### Verification and self-review - -- Focused Release CTest passed: generation/gate/lifecycle plus frontend - OFF/ON, 3/3. -- Complete Release build passed, including production and frontend - feature-OFF/ON compile checks. -- Complete Release CTest passed: 10/10, 0 failed. -- IDE diagnostics reported no errors. Review confirmed callback releases are - gone, all three limit decisions occur after load, SIGINT is serve-only, and - both OpenAI typed streaming catches use the shared SSE sender. -- Unrelated Minor findings were not changed. Existing broad-build dependency - and hardware-validation concerns remain as documented above. From dc15d66fdfee07998dc52dc97dc886f665095d44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CChi?= Date: Tue, 1 Sep 2026 17:18:34 -0700 Subject: [PATCH 021/117] build: package the optional corelib AIE4 runtime Co-authored-by: Cursor --- docs/docs/models/phi.md | 20 +- src/CMakeLists.txt | 28 + src/cmake/ConfigureAie4Runtime.cmake | 113 + src/common/corelib/phi4_corelib_manifest.cpp | 36 +- src/common/utils.cpp | 44 + src/include/pull/model_overlay.hpp | 35 + src/include/utils/utils.hpp | 4 + src/inno/flm.iss | 10 +- src/inno/get_files.bat | 8 + src/model_info.json | 83 + src/model_list.json | 55 + .../phi4-mini-it-aie4/config.json | 12 + .../corelib_phi4_manifest.json | 10556 ++++++++++++++++ .../phi4-mini-it-aie4/provenance.json | 117 + .../phi4-mini-it-aie4/tokenizer_config.json | 117 + src/pull/download_model.cpp | 1 + src/pull/download_model.hpp | 3 +- src/pull/model_downloader.cpp | 51 +- src/pull/model_overlay.cpp | 302 + src/test/phi4_corelib_aie4/CMakeLists.txt | 21 + .../phi4_corelib_aie4/test_model_catalog.cpp | 285 + .../test_packaged_runtime.ps1 | 314 + .../phi4_corelib_aie4/test_phi4_manifest.cpp | 36 + src/wix/flm.wxs | 38 +- src/wix/get_files.bat | 8 + tools/generate_phi4_corelib_manifest.py | 140 +- tools/package_phi4_corelib_aie4.py | 582 + tools/tests/test_package_phi4_corelib_aie4.py | 557 + 28 files changed, 13552 insertions(+), 24 deletions(-) create mode 100644 src/cmake/ConfigureAie4Runtime.cmake create mode 100644 src/include/pull/model_overlay.hpp create mode 100644 src/model_overlays/phi4-mini-it-aie4/config.json create mode 100644 src/model_overlays/phi4-mini-it-aie4/corelib_phi4_manifest.json create mode 100644 src/model_overlays/phi4-mini-it-aie4/provenance.json create mode 100644 src/model_overlays/phi4-mini-it-aie4/tokenizer_config.json create mode 100644 src/pull/model_overlay.cpp create mode 100644 src/test/phi4_corelib_aie4/test_model_catalog.cpp create mode 100644 src/test/phi4_corelib_aie4/test_packaged_runtime.ps1 create mode 100644 tools/package_phi4_corelib_aie4.py create mode 100644 tools/tests/test_package_phi4_corelib_aie4.py diff --git a/docs/docs/models/phi.md b/docs/docs/models/phi.md index 9b2c59bf..effcd05a 100644 --- a/docs/docs/models/phi.md +++ b/docs/docs/models/phi.md @@ -22,4 +22,22 @@ parent: Models flm run phi4-mini-it:4b ``` ---- \ No newline at end of file +--- + +## Phi-4 mini on Ryzen AI AIE4 + +The `phi4-mini-it-aie4:4b` tag uses the optional `corelib_aie4` +backend and the pinned AMD +[OGA DML package](https://huggingface.co/amd/phi-4-mini-instruct-oga-dml). +It has a 4096-token default and maximum prefill length and requires +FastFlowLM 1.0.4 or newer. + +Select **Phi-4 AIE4 corelib runtime** in the Windows installer, then use: + +```shell +flm pull phi4-mini-it-aie4:4b +flm run phi4-mini-it-aie4:4b +``` + +This package is hosted only on Hugging Face. `--modelscope` is rejected +before any download starts. \ No newline at end of file diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ec5a7d7f..17e85d5c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -50,6 +50,7 @@ option(FLM_USE_HRX "Use the HRX amdxdna NPU runtime instead of XRT (0=XRT defaul option(FLM_PORTABLE_BUILD "Build portable distribution with bundled runtime libraries" OFF) option(FLM_ENABLE_CORELIB_AIE4 "Enable optional Phi-4 AIE4 execution through ryzenai-corelib" OFF) +include("${CMAKE_SOURCE_DIR}/cmake/ConfigureAie4Runtime.cmake") if(FLM_ENABLE_CORELIB_AIE4 AND (NOT WIN32 OR FLM_USE_HRX)) message(FATAL_ERROR @@ -303,6 +304,7 @@ endif() add_executable(flm ${SOURCES} ${HEADERS}) +flm_collect_aie4_runtime_files(FLM_AIE4_RUNTIME_FILES) if(FLM_ENABLE_CORELIB_AIE4) target_compile_definitions(flm PRIVATE @@ -603,6 +605,23 @@ if(WIN32) ${CMAKE_SOURCE_DIR}/out/flm.exe ) endif() +add_custom_command(TARGET flm POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_directory + "${CMAKE_SOURCE_DIR}/model_overlays" + "$/model_overlays") +if(WIN32 AND FLM_ENABLE_CORELIB_AIE4) + add_custom_command(TARGET flm POST_BUILD + COMMAND ${CMAKE_COMMAND} -E make_directory + "$/aie4" + COMMAND ${CMAKE_COMMAND} -E copy_if_different + ${FLM_AIE4_RUNTIME_FILES} + "$/aie4" + COMMAND ${CMAKE_COMMAND} -E make_directory + "${CMAKE_SOURCE_DIR}/out/aie4" + COMMAND ${CMAKE_COMMAND} -E copy_if_different + ${FLM_AIE4_RUNTIME_FILES} + "${CMAKE_SOURCE_DIR}/out/aie4") +endif() # Default install location for model_list.json / xclbins (matches the app's # relocatable "/../share/flm" lookup). Overridden to the prefix root for @@ -740,6 +759,12 @@ if(WIN32) "[Pp][Dd][Mm][Uu][Tt][Ii][Ll][Ii][Tt][Ii][Ee][Ss].*" "[Ww][Pp][Aa][Xx][Hh][Oo][Ll][Dd][Ee][Rr].*" POST_EXCLUDE_REGEXES ".*[Ww]indows[/\\\\][Ss]ystem32[/\\\\].*") + if(FLM_ENABLE_CORELIB_AIE4) + install( + FILES ${FLM_AIE4_RUNTIME_FILES} + DESTINATION bin/aie4 + COMPONENT AIE4) + endif() elseif(NOT FLM_USE_HRX AND FLM_PORTABLE_BUILD) # Portable XRT: bundle the XRT runtime explicitly. XRT is deliberately # handled here instead of via the dependency closure below: flm only lists @@ -916,6 +941,9 @@ endif() install(FILES model_list.json DESTINATION "${FLM_SHARE_DESTINATION}") install(FILES model_info.json DESTINATION "${FLM_SHARE_DESTINATION}") +install( + DIRECTORY model_overlays + DESTINATION "${FLM_SHARE_DESTINATION}") # xclbins, which are loaded by shared libraries need to be in location # relative to the executable, so we install them relative to the binary. diff --git a/src/cmake/ConfigureAie4Runtime.cmake b/src/cmake/ConfigureAie4Runtime.cmake new file mode 100644 index 00000000..c405e593 --- /dev/null +++ b/src/cmake/ConfigureAie4Runtime.cmake @@ -0,0 +1,113 @@ +set( + RYZENAI_CORELIB_RUNTIME_DIR + "" + CACHE PATH + "Packaging-only directory containing the ryzenai-corelib runtime DLLs") +set( + XRT_RUNTIME_DIR + "" + CACHE PATH + "Packaging-only directory containing the XRT runtime DLLs") +set( + FLM_AIE4_DEPENDENCY_DIRS + "" + CACHE STRING + "Additional directories searched for the optional AIE4 DLL closure") + +function(_flm_aie4_require_directory variable description) + if(NOT ${variable}) + message(FATAL_ERROR + "${variable} is required when FLM_ENABLE_CORELIB_AIE4=ON " + "(${description})") + endif() + if(NOT IS_DIRECTORY "${${variable}}") + message(FATAL_ERROR + "${variable} does not name a directory: ${${variable}}") + endif() +endfunction() + +function(_flm_aie4_find_dependency output filename) + set(_search_dirs + "${RYZENAI_CORELIB_RUNTIME_DIR}" + ${FLM_AIE4_DEPENDENCY_DIRS}) + if(CMAKE_SOURCE_DIR) + list(APPEND _search_dirs "${CMAKE_SOURCE_DIR}/lib") + endif() + set(_found "") + foreach(_directory IN LISTS _search_dirs) + if(_directory AND EXISTS "${_directory}/${filename}") + get_filename_component( + _found + "${_directory}/${filename}" + ABSOLUTE) + break() + endif() + endforeach() + set(${output} "${_found}" PARENT_SCOPE) +endfunction() + +function(flm_collect_aie4_runtime_files output) + if(NOT FLM_ENABLE_CORELIB_AIE4) + set(${output} "" PARENT_SCOPE) + return() + endif() + if(NOT WIN32) + message(FATAL_ERROR + "FLM_ENABLE_CORELIB_AIE4 runtime packaging is Windows-only") + endif() + + _flm_aie4_require_directory( + RYZENAI_CORELIB_RUNTIME_DIR + "ryzenai_corelib.dll, ryzen_mm.dll, and dyn_bins.dll") + _flm_aie4_require_directory( + XRT_RUNTIME_DIR + "xrt_coreutil.dll and the XRT device runtime") + + set(_runtime_files "") + foreach(_filename IN ITEMS + ryzenai_corelib.dll + ryzen_mm.dll + dyn_bins.dll + spdlog.dll + fmt.dll + libprotobuf.dll + zlib.dll + libutf8_validity.dll + abseil_dll.dll) + _flm_aie4_find_dependency(_dependency "${_filename}") + if(NOT _dependency) + message(FATAL_ERROR + "The AIE4 runtime closure is incomplete: ${_filename} " + "was not found in RYZENAI_CORELIB_RUNTIME_DIR or " + "FLM_AIE4_DEPENDENCY_DIRS") + endif() + list(APPEND _runtime_files "${_dependency}") + endforeach() + + foreach(_filename IN ITEMS zlib1.dll) + _flm_aie4_find_dependency(_dependency "${_filename}") + if(_dependency) + list(APPEND _runtime_files "${_dependency}") + endif() + endforeach() + + foreach(_filename IN ITEMS + xrt_coreutil.dll + xrt_core.dll + xrt_umddml.dll) + if(NOT EXISTS "${XRT_RUNTIME_DIR}/${_filename}") + message(FATAL_ERROR + "The AIE4 XRT closure is incomplete: ${_filename} " + "was not found in XRT_RUNTIME_DIR") + endif() + endforeach() + file(GLOB _xrt_runtime_dlls "${XRT_RUNTIME_DIR}/*.dll") + if(NOT _xrt_runtime_dlls) + message(FATAL_ERROR + "XRT_RUNTIME_DIR contains no runtime DLLs: ${XRT_RUNTIME_DIR}") + endif() + list(APPEND _runtime_files ${_xrt_runtime_dlls}) + list(REMOVE_DUPLICATES _runtime_files) + list(SORT _runtime_files) + set(${output} "${_runtime_files}" PARENT_SCOPE) +endfunction() diff --git a/src/common/corelib/phi4_corelib_manifest.cpp b/src/common/corelib/phi4_corelib_manifest.cpp index c1c2f389..50e6b911 100644 --- a/src/common/corelib/phi4_corelib_manifest.cpp +++ b/src/common/corelib/phi4_corelib_manifest.cpp @@ -326,6 +326,21 @@ void RequireShape( } } +void RequireOneOfShapes( + const InitializerView& view, + const std::vector>& expected, + std::string_view context) { + if ( + std::none_of( + expected.begin(), + expected.end(), + [&](const std::vector& shape) { + return view.shape == shape; + })) { + Throw(context, "has an invalid shape"); + } +} + void RequireFloating( const InitializerView& view, std::string_view context) { @@ -507,7 +522,7 @@ void ValidateQuantizedProjection( const auto validate = [&]( std::string_view component, SourceDType dtype, - std::initializer_list shape, + std::vector> shapes, std::string semantic) { const std::string role_name = std::string(component_prefix) + std::string(component); @@ -523,7 +538,7 @@ void ValidateQuantizedProjection( if (view.dtype != dtype) { Throw(context, "has an invalid dtype"); } - RequireShape(view, shape, context); + RequireOneOfShapes(view, shapes, context); RequireSemanticRole( semantic_roles, found->second, @@ -533,7 +548,13 @@ void ValidateQuantizedProjection( validate( "qweight", SourceDType::UInt8, - {n, k / 2}, + { + {n, k / 2}, + { + n, + groups, + static_cast( + constants::kGroupSize / 2)}}, std::string(role_prefix) + ".qweight"); const std::string scales_name = @@ -550,7 +571,10 @@ void ValidateQuantizedProjection( scales_name, scales_component->second); RequireFloating(scales, scales_context); - RequireShape(scales, {n, groups}, scales_context); + RequireOneOfShapes( + scales, + {{n, groups}, {n * groups}}, + scales_context); RequireSemanticRole( semantic_roles, scales_component->second, @@ -559,7 +583,9 @@ void ValidateQuantizedProjection( validate( "qzeros", SourceDType::UInt8, - {n, (groups + 1) / 2}, + { + {n, (groups + 1) / 2}, + {n * ((groups + 1) / 2)}}, std::string(role_prefix) + ".qzeros"); } diff --git a/src/common/utils.cpp b/src/common/utils.cpp index 5e16386e..224417a3 100644 --- a/src/common/utils.cpp +++ b/src/common/utils.cpp @@ -111,12 +111,56 @@ std::string find_model_info() { if (std::filesystem::exists(exe_relative_path)) { return exe_relative_path; } + for (const std::filesystem::path bundle : { + std::filesystem::path(exe_dir) / "share" / "flm" / + "model_info.json", + std::filesystem::path(exe_dir) / ".." / "share" / "flm" / + "model_info.json"}) { + if (std::filesystem::exists(bundle)) { + return bundle.lexically_normal().string(); + } + } #endif // If not found, throw an error throw std::runtime_error("model_info.json not found. Please set FLM_MODELINFO_PATH or place it next to the executable."); } +std::string find_model_overlay_root() { + const char* configured = std::getenv("FLM_MODEL_OVERLAY_PATH"); + if (configured && *configured) { + const std::filesystem::path path(configured); + if (std::filesystem::is_directory(path)) { + return std::filesystem::absolute(path).lexically_normal().string(); + } + throw std::runtime_error( + "FLM_MODEL_OVERLAY_PATH does not name an installed " + "model_overlays directory: " + + path.string()); + } + + const std::filesystem::path executable_dir( + get_executable_directory()); + const std::filesystem::path install_prefix(CMAKE_INSTALL_PREFIX); + const std::filesystem::path candidates[] = { + executable_dir / "model_overlays", + executable_dir / "share" / "flm" / "model_overlays", + executable_dir / ".." / "share" / "flm" / "model_overlays", + std::filesystem::current_path() / "model_overlays", + install_prefix / "share" / "flm" / "model_overlays", + }; + for (const auto& candidate : candidates) { + if (std::filesystem::is_directory(candidate)) { + return std::filesystem::absolute(candidate) + .lexically_normal() + .string(); + } + } + throw std::runtime_error( + "FastFlow model overlay root not found. Reinstall the model " + "overlay package or set FLM_MODEL_OVERLAY_PATH."); +} + std::string find_xclbin_path() { std::string xclbin_prefix = CMAKE_XCLBIN_PREFIX; diff --git a/src/include/pull/model_overlay.hpp b/src/include/pull/model_overlay.hpp new file mode 100644 index 00000000..25461d52 --- /dev/null +++ b/src/include/pull/model_overlay.hpp @@ -0,0 +1,35 @@ +#pragma once + +#include + +#include +#include +#include +#include + +namespace flm::pull { + +void RequireSupportedModelSource( + const nlohmann::json& model_info, + bool use_modelscope); + +std::vector RemoteModelFiles( + const nlohmann::json& model_info); + +std::string BuildRemoteFileUrl( + const nlohmann::json& model_info, + std::string_view filename); + +std::string CalculateFileSha256(const std::filesystem::path& path); + +void StageBundledOverlays( + const nlohmann::json& model_info, + const std::filesystem::path& overlay_root, + const std::filesystem::path& model_dir); + +bool VerifyBundledOverlayTarget( + const nlohmann::json& model_info, + std::string_view filename, + const std::filesystem::path& model_dir); + +} // namespace flm::pull diff --git a/src/include/utils/utils.hpp b/src/include/utils/utils.hpp index 6766c9cb..3d69cf37 100644 --- a/src/include/utils/utils.hpp +++ b/src/include/utils/utils.hpp @@ -370,6 +370,10 @@ std::string find_model_list(); std::string find_model_info(); +///@brief find the root containing FastFlow-owned model overlays +///@return path to the model_overlays directory +std::string find_model_overlay_root(); + ///@brief get the path to the xclbin directory ///@return path to the xclbin directory diff --git a/src/inno/flm.iss b/src/inno/flm.iss index b5e6e886..1f90babd 100644 --- a/src/inno/flm.iss +++ b/src/inno/flm.iss @@ -4,7 +4,7 @@ AppName=flm -AppVersion=1.0.3 +AppVersion=1.0.4 AppPublisher=FastFlowLM @@ -98,6 +98,13 @@ Source: "logo.ico"; DestDir: "{app}"; Flags: ignoreversion Source: "model_list.json"; DestDir: "{app}"; Flags: ignoreversion Source: "model_info.json"; DestDir: "{app}"; Flags: ignoreversion +; Optional Phi-4 AIE4 runtime and FastFlow-owned model overlays +Source: "aie4\*"; DestDir: "{app}\aie4"; Flags: ignoreversion recursesubdirs createallsubdirs; Tasks: aie4runtime +Source: "..\model_overlays\phi4-mini-it-aie4\config.json"; DestDir: "{app}\share\flm\model_overlays\phi4-mini-it-aie4"; Flags: ignoreversion; Tasks: aie4runtime +Source: "..\model_overlays\phi4-mini-it-aie4\corelib_phi4_manifest.json"; DestDir: "{app}\share\flm\model_overlays\phi4-mini-it-aie4"; Flags: ignoreversion; Tasks: aie4runtime +Source: "..\model_overlays\phi4-mini-it-aie4\tokenizer_config.json"; DestDir: "{app}\share\flm\model_overlays\phi4-mini-it-aie4"; Flags: ignoreversion; Tasks: aie4runtime +Source: "..\model_overlays\phi4-mini-it-aie4\provenance.json"; DestDir: "{app}\share\flm\model_overlays\phi4-mini-it-aie4"; Flags: ignoreversion; Tasks: aie4runtime + ; xclbins directory - recursively include all files Source: "..\xclbins\*"; DestDir: "{app}\xclbins"; Flags: ignoreversion recursesubdirs createallsubdirs @@ -136,6 +143,7 @@ Name: "{commondesktop}\flm serve"; \ ; Optional desktop icon task Name: "desktopicon"; Description: "Create a desktop icon"; GroupDescription: "Additional icons:"; Flags: unchecked +Name: "aie4runtime"; Description: "Install optional Phi-4 AIE4 corelib runtime"; GroupDescription: "Optional features:"; Flags: unchecked [Code] var diff --git a/src/inno/get_files.bat b/src/inno/get_files.bat index d0692535..a351d1a5 100644 --- a/src/inno/get_files.bat +++ b/src/inno/get_files.bat @@ -15,4 +15,12 @@ echo Copying model_list.json... copy "..\model_list.json" "model_list.json" copy "..\model_info.json" "model_info.json" +REM Copy the validated optional AIE4 runtime closure +if not exist "..\build\aie4\ryzenai_corelib.dll" ( + echo ERROR: Build src with FLM_ENABLE_CORELIB_AIE4=ON before packaging. + exit /b 1 +) +if not exist "aie4" mkdir "aie4" +xcopy "..\build\aie4\*" "aie4\" /E /I /Y + echo Done! diff --git a/src/model_info.json b/src/model_info.json index cb62db94..126a31c1 100644 --- a/src/model_info.json +++ b/src/model_info.json @@ -3300,5 +3300,88 @@ "xetHash": "7c4d2da22b3de2ed3f3eae66c7034386df6a3c5d81039ad4a1c8067e7eaf0069", "path": "vision_weights.q4nx" } + ], + "phi4-mini-it-aie4:4b": [ + { + "type": "file", + "oid": "f78df3bfb43291872abf78496110e5856b24de73", + "size": 1622, + "path": ".gitattributes" + }, + { + "type": "file", + "oid": "77e6ed402865af3c38b5e44c2d0be68f76f0784f", + "size": 261, + "path": "added_tokens.json" + }, + { + "type": "file", + "oid": "a9c00dd9bbd97e117371168e9d62af65b9f0e725", + "size": 423, + "path": "chat_template.jinja" + }, + { + "type": "file", + "oid": "55b698edc57fe52593dce4423beb51ded982dc96", + "size": 1720, + "path": "genai_config.json" + }, + { + "type": "file", + "oid": "dcecc4524288b351bbd0da8028e74e9b5bcdb9b5", + "size": 2418348, + "path": "merges.txt" + }, + { + "type": "file", + "oid": "df6d4309745d4627b82fd92c615589193ea528db", + "size": 378325, + "lfs": { + "oid": "e80b9d83e018784eda6263d09fa2ab7729087722c6073c72123366f2dbec4529", + "size": 378325, + "pointerSize": 131 + }, + "path": "model.onnx" + }, + { + "type": "file", + "oid": "5ef8f7f74cf6ac6411b8918e8a850bd2c769dd45", + "size": 3248488448, + "lfs": { + "oid": "c48fd647beb02866e68d6dd1fbc05809a439283501d8abb7f6d6b950f92b7b60", + "size": 3248488448, + "pointerSize": 135 + }, + "path": "model.onnx.data" + }, + { + "type": "file", + "oid": "18eba67aba3ef71b01ed13c16b3feed78e83001f", + "size": 617, + "path": "special_tokens_map.json" + }, + { + "type": "file", + "oid": "3a12dacd8e86802d0229d810d3cc69ab548adf0a", + "size": 15524095, + "lfs": { + "oid": "382cc235b56c725945e149cc25f191da667c836655efd0857b004320e90e91ea", + "size": 15524095, + "pointerSize": 133 + }, + "path": "tokenizer.json" + }, + { + "type": "file", + "oid": "c182c54743fe1735b0d5eb3959b9757a160879e8", + "size": 2654, + "path": "tokenizer_config.json" + }, + { + "type": "file", + "oid": "ea953a43348cdb3776cb7fd9ea02e3784febde34", + "size": 3910310, + "path": "vocab.json" + } ] } \ No newline at end of file diff --git a/src/model_list.json b/src/model_list.json index d2c7b656..659f6241 100644 --- a/src/model_list.json +++ b/src/model_list.json @@ -469,6 +469,61 @@ "footprint": 3.4 } }, + "phi4-mini-it-aie4": { + "4b": { + "name": "Phi-4-mini-instruct-oga-dml-AIE4", + "url": "https://huggingface.co/amd/phi-4-mini-instruct-oga-dml", + "revision": "e751fb68c2cfffe6b0d32942118f75ac0a0365bb", + "file_url": "https://huggingface.co/api/models/amd/phi-4-mini-instruct-oga-dml/tree/e751fb68c2cfffe6b0d32942118f75ac0a0365bb?recursive=true&expand=false", + "size": 3271001989, + "default_context_length": 4096, + "max_prefill_len": 4096, + "details": { + "family": "phi4", + "think": false, + "think_toggleable": false, + "parameter_size": "4B", + "quantization_level": "MatMulNBits Q4", + "execution_backend": "corelib_aie4" + }, + "flm_min_version": "1.0.4", + "vlm": false, + "modelscope_supported": false, + "files": [ + ".gitattributes", + "added_tokens.json", + "chat_template.jinja", + "config.json", + "corelib_phi4_manifest.json", + "genai_config.json", + "merges.txt", + "model.onnx", + "model.onnx.data", + "special_tokens_map.json", + "tokenizer.json", + "tokenizer_config.json", + "vocab.json" + ], + "bundled_overlays": { + "config.json": { + "path": "phi4-mini-it-aie4/config.json", + "size": 257, + "sha256": "1b3e74125a109c05f53c8383def18359d8581619f998c4f91b3b5bb78bf2919f" + }, + "corelib_phi4_manifest.json": { + "path": "phi4-mini-it-aie4/corelib_phi4_manifest.json", + "size": 274527, + "sha256": "09cee6efafc513a2048c89e75b40d92d096144f0eed3b2459c138032b69dc045" + }, + "tokenizer_config.json": { + "path": "phi4-mini-it-aie4/tokenizer_config.json", + "size": 3036, + "sha256": "274d22c3cd28c042f28a681832722536663b06e8c9e88b6616622dceab922ca6" + } + }, + "footprint": 3.05 + } + }, "embed-gemma": { "300m": { "name": "Embedding-Gemma-300M-NPU2", diff --git a/src/model_overlays/phi4-mini-it-aie4/config.json b/src/model_overlays/phi4-mini-it-aie4/config.json new file mode 100644 index 00000000..abf9a3d7 --- /dev/null +++ b/src/model_overlays/phi4-mini-it-aie4/config.json @@ -0,0 +1,12 @@ +{ + "flm_version": "1.0.4", + "head_dim": 128, + "hidden_size": 3072, + "intermediate_size": 8192, + "model_type": "phi4", + "num_attention_heads": 24, + "num_hidden_layers": 32, + "num_key_value_heads": 8, + "rms_norm_eps": 1e-05, + "vocab_size": 200064 +} diff --git a/src/model_overlays/phi4-mini-it-aie4/corelib_phi4_manifest.json b/src/model_overlays/phi4-mini-it-aie4/corelib_phi4_manifest.json new file mode 100644 index 00000000..10935371 --- /dev/null +++ b/src/model_overlays/phi4-mini-it-aie4/corelib_phi4_manifest.json @@ -0,0 +1,10556 @@ +{ + "backend": { + "max_seq": 4096 + }, + "execution_backend": "corelib_aie4", + "files": { + "model.onnx": { + "sha256": "e80b9d83e018784eda6263d09fa2ab7729087722c6073c72123366f2dbec4529", + "size": 378325 + }, + "model.onnx.data": { + "sha256": "c48fd647beb02866e68d6dd1fbc05809a439283501d8abb7f6d6b950f92b7b60", + "size": 3248488448 + } + }, + "initializers": { + "cos_cache": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 12976128, + "offset": 1686044672, + "role": "cos_cache", + "shape": [ + 135168, + 48 + ] + }, + "lm_head.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 307298304, + "offset": 1711996928, + "role": "matmul.qweight", + "shape": [ + 200064, + 24, + 64 + ] + }, + "lm_head.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 2400768, + "offset": 164036608, + "role": "matmul.qzeros", + "shape": [ + 2400768 + ] + }, + "lm_head.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 9603072, + "offset": 468451328, + "role": "matmul.scales", + "shape": [ + 4801536 + ] + }, + "model.embed_tokens.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 1229193216, + "offset": 2019295232, + "role": "embedding", + "shape": [ + 200064, + 3072 + ] + }, + "model.layers.0.attn.k_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 63373312, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.0.attn.k_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 399360, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.0.attn.k_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 3545088, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.0.attn.o_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 171180032, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.0.attn.o_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 1222656, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.0.attn.o_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 16275456, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.0.attn.q_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 166461440, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.0.attn.q_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 1185792, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.0.attn.q_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 16128000, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.0.attn.v_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 64946176, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.0.attn.v_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 411648, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.0.attn.v_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 3594240, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.0.input_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 0, + "role": "input_norm", + "shape": [ + 3072 + ] + }, + "model.layers.0.mlp.down_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 503250944, + "role": "ssmlp.down.qweight", + "shape": [ + 3072, + 64, + 64 + ] + }, + "model.layers.0.mlp.down_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 6887424, + "role": "ssmlp.down.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.0.mlp.down_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 26351616, + "role": "ssmlp.down.scales", + "shape": [ + 196608 + ] + }, + "model.layers.0.mlp.gate_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 478085120, + "role": "ssmlp.gate.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.0.mlp.gate_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 6690816, + "role": "ssmlp.gate.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.0.mlp.gate_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 25565184, + "role": "ssmlp.gate.scales", + "shape": [ + 196608 + ] + }, + "model.layers.0.mlp.up_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 490668032, + "role": "ssmlp.up.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.0.mlp.up_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 6789120, + "role": "ssmlp.up.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.0.mlp.up_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 25958400, + "role": "ssmlp.up.scales", + "shape": [ + 196608 + ] + }, + "model.layers.0.post_attention_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 6144, + "role": "ssmlp.norm0", + "shape": [ + 3072 + ] + }, + "model.layers.1.attn.k_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 66519040, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.1.attn.k_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 423936, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.1.attn.k_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 3643392, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.1.attn.o_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 180617216, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.1.attn.o_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 1296384, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.1.attn.o_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 16570368, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.1.attn.q_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 175898624, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.1.attn.q_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 1259520, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.1.attn.q_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 16422912, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.1.attn.v_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 68091904, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.1.attn.v_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 436224, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.1.attn.v_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 3692544, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.1.input_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 12288, + "role": "ssmlp.norm1", + "shape": [ + 3072 + ] + }, + "model.layers.1.mlp.down_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 540999680, + "role": "ssmlp.down.qweight", + "shape": [ + 3072, + 64, + 64 + ] + }, + "model.layers.1.mlp.down_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 7182336, + "role": "ssmlp.down.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.1.mlp.down_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 27531264, + "role": "ssmlp.down.scales", + "shape": [ + 196608 + ] + }, + "model.layers.1.mlp.gate_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 515833856, + "role": "ssmlp.gate.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.1.mlp.gate_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 6985728, + "role": "ssmlp.gate.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.1.mlp.gate_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 26744832, + "role": "ssmlp.gate.scales", + "shape": [ + 196608 + ] + }, + "model.layers.1.mlp.up_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 528416768, + "role": "ssmlp.up.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.1.mlp.up_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 7084032, + "role": "ssmlp.up.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.1.mlp.up_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 27138048, + "role": "ssmlp.up.scales", + "shape": [ + 196608 + ] + }, + "model.layers.1.post_attention_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 18432, + "role": "ssmlp.norm0", + "shape": [ + 3072 + ] + }, + "model.layers.10.attn.k_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 94830592, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.10.attn.k_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 645120, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.10.attn.k_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 4528128, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.10.attn.o_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 265551872, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.10.attn.o_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 1959936, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.10.attn.o_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 19224576, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.10.attn.q_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 260833280, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.10.attn.q_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 1923072, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.10.attn.q_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 19077120, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.10.attn.v_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 96403456, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.10.attn.v_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 657408, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.10.attn.v_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 4577280, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.10.input_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 122880, + "role": "ssmlp.norm1", + "shape": [ + 3072 + ] + }, + "model.layers.10.mlp.down_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 880738304, + "role": "ssmlp.down.qweight", + "shape": [ + 3072, + 64, + 64 + ] + }, + "model.layers.10.mlp.down_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 9836544, + "role": "ssmlp.down.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.10.mlp.down_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 38148096, + "role": "ssmlp.down.scales", + "shape": [ + 196608 + ] + }, + "model.layers.10.mlp.gate_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 855572480, + "role": "ssmlp.gate.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.10.mlp.gate_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 9639936, + "role": "ssmlp.gate.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.10.mlp.gate_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 37361664, + "role": "ssmlp.gate.scales", + "shape": [ + 196608 + ] + }, + "model.layers.10.mlp.up_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 868155392, + "role": "ssmlp.up.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.10.mlp.up_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 9738240, + "role": "ssmlp.up.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.10.mlp.up_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 37754880, + "role": "ssmlp.up.scales", + "shape": [ + 196608 + ] + }, + "model.layers.10.post_attention_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 129024, + "role": "ssmlp.norm0", + "shape": [ + 3072 + ] + }, + "model.layers.11.attn.k_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 97976320, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.11.attn.k_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 669696, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.11.attn.k_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 4626432, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.11.attn.o_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 274989056, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.11.attn.o_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 2033664, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.11.attn.o_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 19519488, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.11.attn.q_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 270270464, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.11.attn.q_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 1996800, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.11.attn.q_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 19372032, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.11.attn.v_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 99549184, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.11.attn.v_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 681984, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.11.attn.v_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 4675584, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.11.input_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 135168, + "role": "ssmlp.norm1", + "shape": [ + 3072 + ] + }, + "model.layers.11.mlp.down_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 918487040, + "role": "ssmlp.down.qweight", + "shape": [ + 3072, + 64, + 64 + ] + }, + "model.layers.11.mlp.down_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 10131456, + "role": "ssmlp.down.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.11.mlp.down_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 39327744, + "role": "ssmlp.down.scales", + "shape": [ + 196608 + ] + }, + "model.layers.11.mlp.gate_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 893321216, + "role": "ssmlp.gate.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.11.mlp.gate_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 9934848, + "role": "ssmlp.gate.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.11.mlp.gate_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 38541312, + "role": "ssmlp.gate.scales", + "shape": [ + 196608 + ] + }, + "model.layers.11.mlp.up_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 905904128, + "role": "ssmlp.up.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.11.mlp.up_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 10033152, + "role": "ssmlp.up.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.11.mlp.up_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 38934528, + "role": "ssmlp.up.scales", + "shape": [ + 196608 + ] + }, + "model.layers.11.post_attention_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 141312, + "role": "ssmlp.norm0", + "shape": [ + 3072 + ] + }, + "model.layers.12.attn.k_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 101122048, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.12.attn.k_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 694272, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.12.attn.k_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 4724736, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.12.attn.o_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 284426240, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.12.attn.o_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 2107392, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.12.attn.o_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 19814400, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.12.attn.q_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 279707648, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.12.attn.q_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 2070528, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.12.attn.q_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 19666944, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.12.attn.v_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 102694912, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.12.attn.v_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 706560, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.12.attn.v_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 4773888, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.12.input_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 147456, + "role": "ssmlp.norm1", + "shape": [ + 3072 + ] + }, + "model.layers.12.mlp.down_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 956235776, + "role": "ssmlp.down.qweight", + "shape": [ + 3072, + 64, + 64 + ] + }, + "model.layers.12.mlp.down_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 10426368, + "role": "ssmlp.down.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.12.mlp.down_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 40507392, + "role": "ssmlp.down.scales", + "shape": [ + 196608 + ] + }, + "model.layers.12.mlp.gate_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 931069952, + "role": "ssmlp.gate.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.12.mlp.gate_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 10229760, + "role": "ssmlp.gate.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.12.mlp.gate_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 39720960, + "role": "ssmlp.gate.scales", + "shape": [ + 196608 + ] + }, + "model.layers.12.mlp.up_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 943652864, + "role": "ssmlp.up.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.12.mlp.up_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 10328064, + "role": "ssmlp.up.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.12.mlp.up_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 40114176, + "role": "ssmlp.up.scales", + "shape": [ + 196608 + ] + }, + "model.layers.12.post_attention_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 153600, + "role": "ssmlp.norm0", + "shape": [ + 3072 + ] + }, + "model.layers.13.attn.k_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 104267776, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.13.attn.k_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 718848, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.13.attn.k_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 4823040, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.13.attn.o_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 293863424, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.13.attn.o_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 2181120, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.13.attn.o_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 20109312, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.13.attn.q_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 289144832, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.13.attn.q_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 2144256, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.13.attn.q_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 19961856, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.13.attn.v_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 105840640, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.13.attn.v_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 731136, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.13.attn.v_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 4872192, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.13.input_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 159744, + "role": "ssmlp.norm1", + "shape": [ + 3072 + ] + }, + "model.layers.13.mlp.down_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 993984512, + "role": "ssmlp.down.qweight", + "shape": [ + 3072, + 64, + 64 + ] + }, + "model.layers.13.mlp.down_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 10721280, + "role": "ssmlp.down.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.13.mlp.down_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 41687040, + "role": "ssmlp.down.scales", + "shape": [ + 196608 + ] + }, + "model.layers.13.mlp.gate_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 968818688, + "role": "ssmlp.gate.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.13.mlp.gate_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 10524672, + "role": "ssmlp.gate.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.13.mlp.gate_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 40900608, + "role": "ssmlp.gate.scales", + "shape": [ + 196608 + ] + }, + "model.layers.13.mlp.up_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 981401600, + "role": "ssmlp.up.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.13.mlp.up_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 10622976, + "role": "ssmlp.up.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.13.mlp.up_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 41293824, + "role": "ssmlp.up.scales", + "shape": [ + 196608 + ] + }, + "model.layers.13.post_attention_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 165888, + "role": "ssmlp.norm0", + "shape": [ + 3072 + ] + }, + "model.layers.14.attn.k_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 107413504, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.14.attn.k_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 743424, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.14.attn.k_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 4921344, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.14.attn.o_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 303300608, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.14.attn.o_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 2254848, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.14.attn.o_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 20404224, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.14.attn.q_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 298582016, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.14.attn.q_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 2217984, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.14.attn.q_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 20256768, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.14.attn.v_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 108986368, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.14.attn.v_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 755712, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.14.attn.v_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 4970496, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.14.input_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 172032, + "role": "ssmlp.norm1", + "shape": [ + 3072 + ] + }, + "model.layers.14.mlp.down_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1031733248, + "role": "ssmlp.down.qweight", + "shape": [ + 3072, + 64, + 64 + ] + }, + "model.layers.14.mlp.down_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 11016192, + "role": "ssmlp.down.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.14.mlp.down_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 42866688, + "role": "ssmlp.down.scales", + "shape": [ + 196608 + ] + }, + "model.layers.14.mlp.gate_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1006567424, + "role": "ssmlp.gate.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.14.mlp.gate_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 10819584, + "role": "ssmlp.gate.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.14.mlp.gate_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 42080256, + "role": "ssmlp.gate.scales", + "shape": [ + 196608 + ] + }, + "model.layers.14.mlp.up_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1019150336, + "role": "ssmlp.up.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.14.mlp.up_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 10917888, + "role": "ssmlp.up.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.14.mlp.up_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 42473472, + "role": "ssmlp.up.scales", + "shape": [ + 196608 + ] + }, + "model.layers.14.post_attention_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 178176, + "role": "ssmlp.norm0", + "shape": [ + 3072 + ] + }, + "model.layers.15.attn.k_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 110559232, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.15.attn.k_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 768000, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.15.attn.k_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 5019648, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.15.attn.o_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 312737792, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.15.attn.o_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 2328576, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.15.attn.o_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 20699136, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.15.attn.q_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 308019200, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.15.attn.q_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 2291712, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.15.attn.q_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 20551680, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.15.attn.v_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 112132096, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.15.attn.v_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 780288, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.15.attn.v_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 5068800, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.15.input_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 184320, + "role": "ssmlp.norm1", + "shape": [ + 3072 + ] + }, + "model.layers.15.mlp.down_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1069481984, + "role": "ssmlp.down.qweight", + "shape": [ + 3072, + 64, + 64 + ] + }, + "model.layers.15.mlp.down_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 11311104, + "role": "ssmlp.down.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.15.mlp.down_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 44046336, + "role": "ssmlp.down.scales", + "shape": [ + 196608 + ] + }, + "model.layers.15.mlp.gate_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1044316160, + "role": "ssmlp.gate.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.15.mlp.gate_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 11114496, + "role": "ssmlp.gate.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.15.mlp.gate_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 43259904, + "role": "ssmlp.gate.scales", + "shape": [ + 196608 + ] + }, + "model.layers.15.mlp.up_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1056899072, + "role": "ssmlp.up.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.15.mlp.up_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 11212800, + "role": "ssmlp.up.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.15.mlp.up_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 43653120, + "role": "ssmlp.up.scales", + "shape": [ + 196608 + ] + }, + "model.layers.15.post_attention_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 190464, + "role": "ssmlp.norm0", + "shape": [ + 3072 + ] + }, + "model.layers.16.attn.k_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 113704960, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.16.attn.k_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 792576, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.16.attn.k_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 5117952, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.16.attn.o_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 322174976, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.16.attn.o_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 2402304, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.16.attn.o_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 20994048, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.16.attn.q_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 317456384, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.16.attn.q_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 2365440, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.16.attn.q_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 20846592, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.16.attn.v_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 115277824, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.16.attn.v_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 804864, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.16.attn.v_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 5167104, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.16.input_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 196608, + "role": "ssmlp.norm1", + "shape": [ + 3072 + ] + }, + "model.layers.16.mlp.down_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1107230720, + "role": "ssmlp.down.qweight", + "shape": [ + 3072, + 64, + 64 + ] + }, + "model.layers.16.mlp.down_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 11606016, + "role": "ssmlp.down.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.16.mlp.down_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 45225984, + "role": "ssmlp.down.scales", + "shape": [ + 196608 + ] + }, + "model.layers.16.mlp.gate_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1082064896, + "role": "ssmlp.gate.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.16.mlp.gate_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 11409408, + "role": "ssmlp.gate.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.16.mlp.gate_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 44439552, + "role": "ssmlp.gate.scales", + "shape": [ + 196608 + ] + }, + "model.layers.16.mlp.up_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1094647808, + "role": "ssmlp.up.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.16.mlp.up_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 11507712, + "role": "ssmlp.up.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.16.mlp.up_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 44832768, + "role": "ssmlp.up.scales", + "shape": [ + 196608 + ] + }, + "model.layers.16.post_attention_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 202752, + "role": "ssmlp.norm0", + "shape": [ + 3072 + ] + }, + "model.layers.17.attn.k_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 116850688, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.17.attn.k_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 817152, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.17.attn.k_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 5216256, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.17.attn.o_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 331612160, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.17.attn.o_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 2476032, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.17.attn.o_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 21288960, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.17.attn.q_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 326893568, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.17.attn.q_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 2439168, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.17.attn.q_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 21141504, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.17.attn.v_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 118423552, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.17.attn.v_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 829440, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.17.attn.v_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 5265408, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.17.input_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 208896, + "role": "ssmlp.norm1", + "shape": [ + 3072 + ] + }, + "model.layers.17.mlp.down_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1144979456, + "role": "ssmlp.down.qweight", + "shape": [ + 3072, + 64, + 64 + ] + }, + "model.layers.17.mlp.down_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 11900928, + "role": "ssmlp.down.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.17.mlp.down_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 46405632, + "role": "ssmlp.down.scales", + "shape": [ + 196608 + ] + }, + "model.layers.17.mlp.gate_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1119813632, + "role": "ssmlp.gate.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.17.mlp.gate_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 11704320, + "role": "ssmlp.gate.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.17.mlp.gate_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 45619200, + "role": "ssmlp.gate.scales", + "shape": [ + 196608 + ] + }, + "model.layers.17.mlp.up_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1132396544, + "role": "ssmlp.up.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.17.mlp.up_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 11802624, + "role": "ssmlp.up.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.17.mlp.up_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 46012416, + "role": "ssmlp.up.scales", + "shape": [ + 196608 + ] + }, + "model.layers.17.post_attention_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 215040, + "role": "ssmlp.norm0", + "shape": [ + 3072 + ] + }, + "model.layers.18.attn.k_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 119996416, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.18.attn.k_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 841728, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.18.attn.k_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 5314560, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.18.attn.o_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 341049344, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.18.attn.o_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 2549760, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.18.attn.o_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 21583872, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.18.attn.q_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 336330752, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.18.attn.q_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 2512896, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.18.attn.q_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 21436416, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.18.attn.v_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 121569280, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.18.attn.v_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 854016, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.18.attn.v_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 5363712, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.18.input_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 221184, + "role": "ssmlp.norm1", + "shape": [ + 3072 + ] + }, + "model.layers.18.mlp.down_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1182728192, + "role": "ssmlp.down.qweight", + "shape": [ + 3072, + 64, + 64 + ] + }, + "model.layers.18.mlp.down_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 12195840, + "role": "ssmlp.down.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.18.mlp.down_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 47585280, + "role": "ssmlp.down.scales", + "shape": [ + 196608 + ] + }, + "model.layers.18.mlp.gate_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1157562368, + "role": "ssmlp.gate.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.18.mlp.gate_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 11999232, + "role": "ssmlp.gate.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.18.mlp.gate_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 46798848, + "role": "ssmlp.gate.scales", + "shape": [ + 196608 + ] + }, + "model.layers.18.mlp.up_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1170145280, + "role": "ssmlp.up.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.18.mlp.up_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 12097536, + "role": "ssmlp.up.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.18.mlp.up_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 47192064, + "role": "ssmlp.up.scales", + "shape": [ + 196608 + ] + }, + "model.layers.18.post_attention_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 227328, + "role": "ssmlp.norm0", + "shape": [ + 3072 + ] + }, + "model.layers.19.attn.k_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 123142144, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.19.attn.k_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 866304, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.19.attn.k_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 5412864, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.19.attn.o_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 350486528, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.19.attn.o_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 2623488, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.19.attn.o_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 21878784, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.19.attn.q_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 345767936, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.19.attn.q_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 2586624, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.19.attn.q_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 21731328, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.19.attn.v_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 124715008, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.19.attn.v_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 878592, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.19.attn.v_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 5462016, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.19.input_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 233472, + "role": "ssmlp.norm1", + "shape": [ + 3072 + ] + }, + "model.layers.19.mlp.down_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1220476928, + "role": "ssmlp.down.qweight", + "shape": [ + 3072, + 64, + 64 + ] + }, + "model.layers.19.mlp.down_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 12490752, + "role": "ssmlp.down.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.19.mlp.down_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 48764928, + "role": "ssmlp.down.scales", + "shape": [ + 196608 + ] + }, + "model.layers.19.mlp.gate_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1195311104, + "role": "ssmlp.gate.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.19.mlp.gate_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 12294144, + "role": "ssmlp.gate.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.19.mlp.gate_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 47978496, + "role": "ssmlp.gate.scales", + "shape": [ + 196608 + ] + }, + "model.layers.19.mlp.up_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1207894016, + "role": "ssmlp.up.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.19.mlp.up_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 12392448, + "role": "ssmlp.up.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.19.mlp.up_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 48371712, + "role": "ssmlp.up.scales", + "shape": [ + 196608 + ] + }, + "model.layers.19.post_attention_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 239616, + "role": "ssmlp.norm0", + "shape": [ + 3072 + ] + }, + "model.layers.2.attn.k_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 69664768, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.2.attn.k_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 448512, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.2.attn.k_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 3741696, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.2.attn.o_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 190054400, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.2.attn.o_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 1370112, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.2.attn.o_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 16865280, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.2.attn.q_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 185335808, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.2.attn.q_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 1333248, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.2.attn.q_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 16717824, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.2.attn.v_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 71237632, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.2.attn.v_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 460800, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.2.attn.v_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 3790848, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.2.input_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 24576, + "role": "ssmlp.norm1", + "shape": [ + 3072 + ] + }, + "model.layers.2.mlp.down_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 578748416, + "role": "ssmlp.down.qweight", + "shape": [ + 3072, + 64, + 64 + ] + }, + "model.layers.2.mlp.down_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 7477248, + "role": "ssmlp.down.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.2.mlp.down_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 28710912, + "role": "ssmlp.down.scales", + "shape": [ + 196608 + ] + }, + "model.layers.2.mlp.gate_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 553582592, + "role": "ssmlp.gate.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.2.mlp.gate_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 7280640, + "role": "ssmlp.gate.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.2.mlp.gate_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 27924480, + "role": "ssmlp.gate.scales", + "shape": [ + 196608 + ] + }, + "model.layers.2.mlp.up_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 566165504, + "role": "ssmlp.up.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.2.mlp.up_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 7378944, + "role": "ssmlp.up.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.2.mlp.up_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 28317696, + "role": "ssmlp.up.scales", + "shape": [ + 196608 + ] + }, + "model.layers.2.post_attention_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 30720, + "role": "ssmlp.norm0", + "shape": [ + 3072 + ] + }, + "model.layers.20.attn.k_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 126287872, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.20.attn.k_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 890880, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.20.attn.k_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 5511168, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.20.attn.o_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 359923712, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.20.attn.o_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 2697216, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.20.attn.o_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 22173696, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.20.attn.q_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 355205120, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.20.attn.q_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 2660352, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.20.attn.q_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 22026240, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.20.attn.v_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 127860736, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.20.attn.v_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 903168, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.20.attn.v_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 5560320, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.20.input_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 245760, + "role": "ssmlp.norm1", + "shape": [ + 3072 + ] + }, + "model.layers.20.mlp.down_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1258225664, + "role": "ssmlp.down.qweight", + "shape": [ + 3072, + 64, + 64 + ] + }, + "model.layers.20.mlp.down_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 12785664, + "role": "ssmlp.down.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.20.mlp.down_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 49944576, + "role": "ssmlp.down.scales", + "shape": [ + 196608 + ] + }, + "model.layers.20.mlp.gate_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1233059840, + "role": "ssmlp.gate.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.20.mlp.gate_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 12589056, + "role": "ssmlp.gate.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.20.mlp.gate_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 49158144, + "role": "ssmlp.gate.scales", + "shape": [ + 196608 + ] + }, + "model.layers.20.mlp.up_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1245642752, + "role": "ssmlp.up.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.20.mlp.up_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 12687360, + "role": "ssmlp.up.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.20.mlp.up_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 49551360, + "role": "ssmlp.up.scales", + "shape": [ + 196608 + ] + }, + "model.layers.20.post_attention_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 251904, + "role": "ssmlp.norm0", + "shape": [ + 3072 + ] + }, + "model.layers.21.attn.k_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 129433600, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.21.attn.k_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 915456, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.21.attn.k_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 5609472, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.21.attn.o_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 369360896, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.21.attn.o_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 2770944, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.21.attn.o_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 22468608, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.21.attn.q_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 364642304, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.21.attn.q_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 2734080, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.21.attn.q_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 22321152, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.21.attn.v_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 131006464, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.21.attn.v_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 927744, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.21.attn.v_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 5658624, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.21.input_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 258048, + "role": "ssmlp.norm1", + "shape": [ + 3072 + ] + }, + "model.layers.21.mlp.down_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1295974400, + "role": "ssmlp.down.qweight", + "shape": [ + 3072, + 64, + 64 + ] + }, + "model.layers.21.mlp.down_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 13080576, + "role": "ssmlp.down.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.21.mlp.down_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 51124224, + "role": "ssmlp.down.scales", + "shape": [ + 196608 + ] + }, + "model.layers.21.mlp.gate_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1270808576, + "role": "ssmlp.gate.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.21.mlp.gate_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 12883968, + "role": "ssmlp.gate.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.21.mlp.gate_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 50337792, + "role": "ssmlp.gate.scales", + "shape": [ + 196608 + ] + }, + "model.layers.21.mlp.up_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1283391488, + "role": "ssmlp.up.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.21.mlp.up_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 12982272, + "role": "ssmlp.up.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.21.mlp.up_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 50731008, + "role": "ssmlp.up.scales", + "shape": [ + 196608 + ] + }, + "model.layers.21.post_attention_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 264192, + "role": "ssmlp.norm0", + "shape": [ + 3072 + ] + }, + "model.layers.22.attn.k_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 132579328, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.22.attn.k_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 940032, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.22.attn.k_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 5707776, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.22.attn.o_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 378798080, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.22.attn.o_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 2844672, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.22.attn.o_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 22763520, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.22.attn.q_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 374079488, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.22.attn.q_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 2807808, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.22.attn.q_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 22616064, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.22.attn.v_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 134152192, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.22.attn.v_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 952320, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.22.attn.v_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 5756928, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.22.input_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 270336, + "role": "ssmlp.norm1", + "shape": [ + 3072 + ] + }, + "model.layers.22.mlp.down_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1333723136, + "role": "ssmlp.down.qweight", + "shape": [ + 3072, + 64, + 64 + ] + }, + "model.layers.22.mlp.down_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 13375488, + "role": "ssmlp.down.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.22.mlp.down_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 52303872, + "role": "ssmlp.down.scales", + "shape": [ + 196608 + ] + }, + "model.layers.22.mlp.gate_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1308557312, + "role": "ssmlp.gate.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.22.mlp.gate_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 13178880, + "role": "ssmlp.gate.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.22.mlp.gate_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 51517440, + "role": "ssmlp.gate.scales", + "shape": [ + 196608 + ] + }, + "model.layers.22.mlp.up_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1321140224, + "role": "ssmlp.up.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.22.mlp.up_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 13277184, + "role": "ssmlp.up.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.22.mlp.up_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 51910656, + "role": "ssmlp.up.scales", + "shape": [ + 196608 + ] + }, + "model.layers.22.post_attention_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 276480, + "role": "ssmlp.norm0", + "shape": [ + 3072 + ] + }, + "model.layers.23.attn.k_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 135725056, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.23.attn.k_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 964608, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.23.attn.k_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 5806080, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.23.attn.o_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 388235264, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.23.attn.o_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 2918400, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.23.attn.o_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 23058432, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.23.attn.q_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 383516672, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.23.attn.q_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 2881536, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.23.attn.q_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 22910976, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.23.attn.v_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 137297920, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.23.attn.v_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 976896, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.23.attn.v_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 5855232, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.23.input_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 282624, + "role": "ssmlp.norm1", + "shape": [ + 3072 + ] + }, + "model.layers.23.mlp.down_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1371471872, + "role": "ssmlp.down.qweight", + "shape": [ + 3072, + 64, + 64 + ] + }, + "model.layers.23.mlp.down_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 13670400, + "role": "ssmlp.down.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.23.mlp.down_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 53483520, + "role": "ssmlp.down.scales", + "shape": [ + 196608 + ] + }, + "model.layers.23.mlp.gate_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1346306048, + "role": "ssmlp.gate.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.23.mlp.gate_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 13473792, + "role": "ssmlp.gate.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.23.mlp.gate_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 52697088, + "role": "ssmlp.gate.scales", + "shape": [ + 196608 + ] + }, + "model.layers.23.mlp.up_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1358888960, + "role": "ssmlp.up.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.23.mlp.up_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 13572096, + "role": "ssmlp.up.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.23.mlp.up_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 53090304, + "role": "ssmlp.up.scales", + "shape": [ + 196608 + ] + }, + "model.layers.23.post_attention_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 288768, + "role": "ssmlp.norm0", + "shape": [ + 3072 + ] + }, + "model.layers.24.attn.k_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 138870784, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.24.attn.k_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 989184, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.24.attn.k_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 5904384, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.24.attn.o_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 397672448, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.24.attn.o_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 2992128, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.24.attn.o_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 23353344, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.24.attn.q_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 392953856, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.24.attn.q_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 2955264, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.24.attn.q_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 23205888, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.24.attn.v_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 140443648, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.24.attn.v_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 1001472, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.24.attn.v_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 5953536, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.24.input_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 294912, + "role": "ssmlp.norm1", + "shape": [ + 3072 + ] + }, + "model.layers.24.mlp.down_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1409220608, + "role": "ssmlp.down.qweight", + "shape": [ + 3072, + 64, + 64 + ] + }, + "model.layers.24.mlp.down_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 13965312, + "role": "ssmlp.down.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.24.mlp.down_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 54663168, + "role": "ssmlp.down.scales", + "shape": [ + 196608 + ] + }, + "model.layers.24.mlp.gate_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1384054784, + "role": "ssmlp.gate.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.24.mlp.gate_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 13768704, + "role": "ssmlp.gate.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.24.mlp.gate_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 53876736, + "role": "ssmlp.gate.scales", + "shape": [ + 196608 + ] + }, + "model.layers.24.mlp.up_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1396637696, + "role": "ssmlp.up.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.24.mlp.up_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 13867008, + "role": "ssmlp.up.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.24.mlp.up_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 54269952, + "role": "ssmlp.up.scales", + "shape": [ + 196608 + ] + }, + "model.layers.24.post_attention_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 301056, + "role": "ssmlp.norm0", + "shape": [ + 3072 + ] + }, + "model.layers.25.attn.k_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 142016512, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.25.attn.k_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 1013760, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.25.attn.k_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 6002688, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.25.attn.o_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 407109632, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.25.attn.o_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 3065856, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.25.attn.o_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 23648256, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.25.attn.q_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 402391040, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.25.attn.q_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 3028992, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.25.attn.q_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 23500800, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.25.attn.v_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 143589376, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.25.attn.v_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 1026048, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.25.attn.v_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 6051840, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.25.input_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 307200, + "role": "ssmlp.norm1", + "shape": [ + 3072 + ] + }, + "model.layers.25.mlp.down_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1446969344, + "role": "ssmlp.down.qweight", + "shape": [ + 3072, + 64, + 64 + ] + }, + "model.layers.25.mlp.down_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 14260224, + "role": "ssmlp.down.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.25.mlp.down_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 55842816, + "role": "ssmlp.down.scales", + "shape": [ + 196608 + ] + }, + "model.layers.25.mlp.gate_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1421803520, + "role": "ssmlp.gate.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.25.mlp.gate_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 14063616, + "role": "ssmlp.gate.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.25.mlp.gate_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 55056384, + "role": "ssmlp.gate.scales", + "shape": [ + 196608 + ] + }, + "model.layers.25.mlp.up_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1434386432, + "role": "ssmlp.up.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.25.mlp.up_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 14161920, + "role": "ssmlp.up.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.25.mlp.up_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 55449600, + "role": "ssmlp.up.scales", + "shape": [ + 196608 + ] + }, + "model.layers.25.post_attention_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 313344, + "role": "ssmlp.norm0", + "shape": [ + 3072 + ] + }, + "model.layers.26.attn.k_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 145162240, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.26.attn.k_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 1038336, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.26.attn.k_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 6100992, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.26.attn.o_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 416546816, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.26.attn.o_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 3139584, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.26.attn.o_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 23943168, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.26.attn.q_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 411828224, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.26.attn.q_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 3102720, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.26.attn.q_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 23795712, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.26.attn.v_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 146735104, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.26.attn.v_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 1050624, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.26.attn.v_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 6150144, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.26.input_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 319488, + "role": "ssmlp.norm1", + "shape": [ + 3072 + ] + }, + "model.layers.26.mlp.down_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1484718080, + "role": "ssmlp.down.qweight", + "shape": [ + 3072, + 64, + 64 + ] + }, + "model.layers.26.mlp.down_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 14555136, + "role": "ssmlp.down.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.26.mlp.down_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 57022464, + "role": "ssmlp.down.scales", + "shape": [ + 196608 + ] + }, + "model.layers.26.mlp.gate_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1459552256, + "role": "ssmlp.gate.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.26.mlp.gate_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 14358528, + "role": "ssmlp.gate.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.26.mlp.gate_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 56236032, + "role": "ssmlp.gate.scales", + "shape": [ + 196608 + ] + }, + "model.layers.26.mlp.up_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1472135168, + "role": "ssmlp.up.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.26.mlp.up_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 14456832, + "role": "ssmlp.up.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.26.mlp.up_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 56629248, + "role": "ssmlp.up.scales", + "shape": [ + 196608 + ] + }, + "model.layers.26.post_attention_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 325632, + "role": "ssmlp.norm0", + "shape": [ + 3072 + ] + }, + "model.layers.27.attn.k_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 148307968, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.27.attn.k_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 1062912, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.27.attn.k_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 6199296, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.27.attn.o_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 425984000, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.27.attn.o_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 3213312, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.27.attn.o_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 24238080, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.27.attn.q_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 421265408, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.27.attn.q_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 3176448, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.27.attn.q_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 24090624, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.27.attn.v_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 149880832, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.27.attn.v_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 1075200, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.27.attn.v_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 6248448, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.27.input_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 331776, + "role": "ssmlp.norm1", + "shape": [ + 3072 + ] + }, + "model.layers.27.mlp.down_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1522466816, + "role": "ssmlp.down.qweight", + "shape": [ + 3072, + 64, + 64 + ] + }, + "model.layers.27.mlp.down_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 14850048, + "role": "ssmlp.down.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.27.mlp.down_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 58202112, + "role": "ssmlp.down.scales", + "shape": [ + 196608 + ] + }, + "model.layers.27.mlp.gate_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1497300992, + "role": "ssmlp.gate.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.27.mlp.gate_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 14653440, + "role": "ssmlp.gate.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.27.mlp.gate_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 57415680, + "role": "ssmlp.gate.scales", + "shape": [ + 196608 + ] + }, + "model.layers.27.mlp.up_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1509883904, + "role": "ssmlp.up.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.27.mlp.up_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 14751744, + "role": "ssmlp.up.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.27.mlp.up_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 57808896, + "role": "ssmlp.up.scales", + "shape": [ + 196608 + ] + }, + "model.layers.27.post_attention_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 337920, + "role": "ssmlp.norm0", + "shape": [ + 3072 + ] + }, + "model.layers.28.attn.k_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 151453696, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.28.attn.k_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 1087488, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.28.attn.k_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 6297600, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.28.attn.o_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 435421184, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.28.attn.o_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 3287040, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.28.attn.o_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 24532992, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.28.attn.q_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 430702592, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.28.attn.q_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 3250176, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.28.attn.q_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 24385536, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.28.attn.v_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 153026560, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.28.attn.v_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 1099776, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.28.attn.v_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 6346752, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.28.input_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 344064, + "role": "ssmlp.norm1", + "shape": [ + 3072 + ] + }, + "model.layers.28.mlp.down_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1560215552, + "role": "ssmlp.down.qweight", + "shape": [ + 3072, + 64, + 64 + ] + }, + "model.layers.28.mlp.down_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 15144960, + "role": "ssmlp.down.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.28.mlp.down_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 59381760, + "role": "ssmlp.down.scales", + "shape": [ + 196608 + ] + }, + "model.layers.28.mlp.gate_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1535049728, + "role": "ssmlp.gate.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.28.mlp.gate_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 14948352, + "role": "ssmlp.gate.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.28.mlp.gate_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 58595328, + "role": "ssmlp.gate.scales", + "shape": [ + 196608 + ] + }, + "model.layers.28.mlp.up_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1547632640, + "role": "ssmlp.up.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.28.mlp.up_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 15046656, + "role": "ssmlp.up.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.28.mlp.up_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 58988544, + "role": "ssmlp.up.scales", + "shape": [ + 196608 + ] + }, + "model.layers.28.post_attention_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 350208, + "role": "ssmlp.norm0", + "shape": [ + 3072 + ] + }, + "model.layers.29.attn.k_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 154599424, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.29.attn.k_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 1112064, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.29.attn.k_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 6395904, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.29.attn.o_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 444858368, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.29.attn.o_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 3360768, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.29.attn.o_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 24827904, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.29.attn.q_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 440139776, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.29.attn.q_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 3323904, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.29.attn.q_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 24680448, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.29.attn.v_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 156172288, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.29.attn.v_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 1124352, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.29.attn.v_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 6445056, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.29.input_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 356352, + "role": "ssmlp.norm1", + "shape": [ + 3072 + ] + }, + "model.layers.29.mlp.down_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1597964288, + "role": "ssmlp.down.qweight", + "shape": [ + 3072, + 64, + 64 + ] + }, + "model.layers.29.mlp.down_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 15439872, + "role": "ssmlp.down.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.29.mlp.down_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 60561408, + "role": "ssmlp.down.scales", + "shape": [ + 196608 + ] + }, + "model.layers.29.mlp.gate_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1572798464, + "role": "ssmlp.gate.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.29.mlp.gate_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 15243264, + "role": "ssmlp.gate.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.29.mlp.gate_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 59774976, + "role": "ssmlp.gate.scales", + "shape": [ + 196608 + ] + }, + "model.layers.29.mlp.up_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1585381376, + "role": "ssmlp.up.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.29.mlp.up_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 15341568, + "role": "ssmlp.up.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.29.mlp.up_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 60168192, + "role": "ssmlp.up.scales", + "shape": [ + 196608 + ] + }, + "model.layers.29.post_attention_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 362496, + "role": "ssmlp.norm0", + "shape": [ + 3072 + ] + }, + "model.layers.3.attn.k_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 72810496, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.3.attn.k_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 473088, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.3.attn.k_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 3840000, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.3.attn.o_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 199491584, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.3.attn.o_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 1443840, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.3.attn.o_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 17160192, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.3.attn.q_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 194772992, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.3.attn.q_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 1406976, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.3.attn.q_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 17012736, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.3.attn.v_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 74383360, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.3.attn.v_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 485376, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.3.attn.v_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 3889152, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.3.input_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 36864, + "role": "ssmlp.norm1", + "shape": [ + 3072 + ] + }, + "model.layers.3.mlp.down_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 616497152, + "role": "ssmlp.down.qweight", + "shape": [ + 3072, + 64, + 64 + ] + }, + "model.layers.3.mlp.down_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 7772160, + "role": "ssmlp.down.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.3.mlp.down_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 29890560, + "role": "ssmlp.down.scales", + "shape": [ + 196608 + ] + }, + "model.layers.3.mlp.gate_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 591331328, + "role": "ssmlp.gate.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.3.mlp.gate_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 7575552, + "role": "ssmlp.gate.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.3.mlp.gate_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 29104128, + "role": "ssmlp.gate.scales", + "shape": [ + 196608 + ] + }, + "model.layers.3.mlp.up_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 603914240, + "role": "ssmlp.up.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.3.mlp.up_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 7673856, + "role": "ssmlp.up.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.3.mlp.up_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 29497344, + "role": "ssmlp.up.scales", + "shape": [ + 196608 + ] + }, + "model.layers.3.post_attention_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 43008, + "role": "ssmlp.norm0", + "shape": [ + 3072 + ] + }, + "model.layers.30.attn.k_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 157745152, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.30.attn.k_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 1136640, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.30.attn.k_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 6494208, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.30.attn.o_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 454295552, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.30.attn.o_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 3434496, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.30.attn.o_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 25122816, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.30.attn.q_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 449576960, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.30.attn.q_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 3397632, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.30.attn.q_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 24975360, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.30.attn.v_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 159318016, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.30.attn.v_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 1148928, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.30.attn.v_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 6543360, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.30.input_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 368640, + "role": "ssmlp.norm1", + "shape": [ + 3072 + ] + }, + "model.layers.30.mlp.down_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1635713024, + "role": "ssmlp.down.qweight", + "shape": [ + 3072, + 64, + 64 + ] + }, + "model.layers.30.mlp.down_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 15734784, + "role": "ssmlp.down.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.30.mlp.down_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 61741056, + "role": "ssmlp.down.scales", + "shape": [ + 196608 + ] + }, + "model.layers.30.mlp.gate_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1610547200, + "role": "ssmlp.gate.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.30.mlp.gate_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 15538176, + "role": "ssmlp.gate.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.30.mlp.gate_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 60954624, + "role": "ssmlp.gate.scales", + "shape": [ + 196608 + ] + }, + "model.layers.30.mlp.up_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1623130112, + "role": "ssmlp.up.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.30.mlp.up_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 15636480, + "role": "ssmlp.up.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.30.mlp.up_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 61347840, + "role": "ssmlp.up.scales", + "shape": [ + 196608 + ] + }, + "model.layers.30.post_attention_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 374784, + "role": "ssmlp.norm0", + "shape": [ + 3072 + ] + }, + "model.layers.31.attn.k_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 160890880, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.31.attn.k_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 1161216, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.31.attn.k_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 6592512, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.31.attn.o_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 463732736, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.31.attn.o_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 3508224, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.31.attn.o_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 25417728, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.31.attn.q_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 459014144, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.31.attn.q_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 3471360, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.31.attn.q_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 25270272, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.31.attn.v_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 162463744, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.31.attn.v_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 1173504, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.31.attn.v_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 6641664, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.31.input_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 380928, + "role": "ssmlp.norm1", + "shape": [ + 3072 + ] + }, + "model.layers.31.mlp.down_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1673461760, + "role": "ssmlp.down.qweight", + "shape": [ + 3072, + 64, + 64 + ] + }, + "model.layers.31.mlp.down_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 16029696, + "role": "ssmlp.down.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.31.mlp.down_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 62920704, + "role": "ssmlp.down.scales", + "shape": [ + 196608 + ] + }, + "model.layers.31.mlp.gate_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1648295936, + "role": "ssmlp.gate.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.31.mlp.gate_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 15833088, + "role": "ssmlp.gate.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.31.mlp.gate_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 62134272, + "role": "ssmlp.gate.scales", + "shape": [ + 196608 + ] + }, + "model.layers.31.mlp.up_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1660878848, + "role": "ssmlp.up.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.31.mlp.up_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 15931392, + "role": "ssmlp.up.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.31.mlp.up_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 62527488, + "role": "ssmlp.up.scales", + "shape": [ + 196608 + ] + }, + "model.layers.31.post_attention_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 387072, + "role": "ssmlp.norm0", + "shape": [ + 3072 + ] + }, + "model.layers.32.final_norm_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 393216, + "role": "ssmlp.norm1", + "shape": [ + 3072 + ] + }, + "model.layers.4.attn.k_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 75956224, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.4.attn.k_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 497664, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.4.attn.k_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 3938304, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.4.attn.o_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 208928768, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.4.attn.o_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 1517568, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.4.attn.o_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 17455104, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.4.attn.q_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 204210176, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.4.attn.q_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 1480704, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.4.attn.q_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 17307648, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.4.attn.v_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 77529088, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.4.attn.v_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 509952, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.4.attn.v_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 3987456, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.4.input_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 49152, + "role": "ssmlp.norm1", + "shape": [ + 3072 + ] + }, + "model.layers.4.mlp.down_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 654245888, + "role": "ssmlp.down.qweight", + "shape": [ + 3072, + 64, + 64 + ] + }, + "model.layers.4.mlp.down_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 8067072, + "role": "ssmlp.down.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.4.mlp.down_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 31070208, + "role": "ssmlp.down.scales", + "shape": [ + 196608 + ] + }, + "model.layers.4.mlp.gate_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 629080064, + "role": "ssmlp.gate.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.4.mlp.gate_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 7870464, + "role": "ssmlp.gate.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.4.mlp.gate_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 30283776, + "role": "ssmlp.gate.scales", + "shape": [ + 196608 + ] + }, + "model.layers.4.mlp.up_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 641662976, + "role": "ssmlp.up.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.4.mlp.up_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 7968768, + "role": "ssmlp.up.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.4.mlp.up_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 30676992, + "role": "ssmlp.up.scales", + "shape": [ + 196608 + ] + }, + "model.layers.4.post_attention_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 55296, + "role": "ssmlp.norm0", + "shape": [ + 3072 + ] + }, + "model.layers.5.attn.k_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 79101952, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.5.attn.k_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 522240, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.5.attn.k_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 4036608, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.5.attn.o_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 218365952, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.5.attn.o_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 1591296, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.5.attn.o_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 17750016, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.5.attn.q_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 213647360, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.5.attn.q_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 1554432, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.5.attn.q_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 17602560, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.5.attn.v_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 80674816, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.5.attn.v_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 534528, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.5.attn.v_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 4085760, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.5.input_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 61440, + "role": "ssmlp.norm1", + "shape": [ + 3072 + ] + }, + "model.layers.5.mlp.down_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 691994624, + "role": "ssmlp.down.qweight", + "shape": [ + 3072, + 64, + 64 + ] + }, + "model.layers.5.mlp.down_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 8361984, + "role": "ssmlp.down.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.5.mlp.down_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 32249856, + "role": "ssmlp.down.scales", + "shape": [ + 196608 + ] + }, + "model.layers.5.mlp.gate_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 666828800, + "role": "ssmlp.gate.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.5.mlp.gate_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 8165376, + "role": "ssmlp.gate.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.5.mlp.gate_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 31463424, + "role": "ssmlp.gate.scales", + "shape": [ + 196608 + ] + }, + "model.layers.5.mlp.up_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 679411712, + "role": "ssmlp.up.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.5.mlp.up_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 8263680, + "role": "ssmlp.up.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.5.mlp.up_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 31856640, + "role": "ssmlp.up.scales", + "shape": [ + 196608 + ] + }, + "model.layers.5.post_attention_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 67584, + "role": "ssmlp.norm0", + "shape": [ + 3072 + ] + }, + "model.layers.6.attn.k_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 82247680, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.6.attn.k_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 546816, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.6.attn.k_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 4134912, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.6.attn.o_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 227803136, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.6.attn.o_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 1665024, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.6.attn.o_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 18044928, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.6.attn.q_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 223084544, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.6.attn.q_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 1628160, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.6.attn.q_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 17897472, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.6.attn.v_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 83820544, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.6.attn.v_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 559104, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.6.attn.v_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 4184064, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.6.input_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 73728, + "role": "ssmlp.norm1", + "shape": [ + 3072 + ] + }, + "model.layers.6.mlp.down_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 729743360, + "role": "ssmlp.down.qweight", + "shape": [ + 3072, + 64, + 64 + ] + }, + "model.layers.6.mlp.down_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 8656896, + "role": "ssmlp.down.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.6.mlp.down_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 33429504, + "role": "ssmlp.down.scales", + "shape": [ + 196608 + ] + }, + "model.layers.6.mlp.gate_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 704577536, + "role": "ssmlp.gate.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.6.mlp.gate_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 8460288, + "role": "ssmlp.gate.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.6.mlp.gate_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 32643072, + "role": "ssmlp.gate.scales", + "shape": [ + 196608 + ] + }, + "model.layers.6.mlp.up_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 717160448, + "role": "ssmlp.up.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.6.mlp.up_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 8558592, + "role": "ssmlp.up.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.6.mlp.up_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 33036288, + "role": "ssmlp.up.scales", + "shape": [ + 196608 + ] + }, + "model.layers.6.post_attention_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 79872, + "role": "ssmlp.norm0", + "shape": [ + 3072 + ] + }, + "model.layers.7.attn.k_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 85393408, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.7.attn.k_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 571392, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.7.attn.k_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 4233216, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.7.attn.o_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 237240320, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.7.attn.o_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 1738752, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.7.attn.o_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 18339840, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.7.attn.q_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 232521728, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.7.attn.q_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 1701888, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.7.attn.q_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 18192384, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.7.attn.v_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 86966272, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.7.attn.v_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 583680, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.7.attn.v_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 4282368, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.7.input_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 86016, + "role": "ssmlp.norm1", + "shape": [ + 3072 + ] + }, + "model.layers.7.mlp.down_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 767492096, + "role": "ssmlp.down.qweight", + "shape": [ + 3072, + 64, + 64 + ] + }, + "model.layers.7.mlp.down_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 8951808, + "role": "ssmlp.down.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.7.mlp.down_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 34609152, + "role": "ssmlp.down.scales", + "shape": [ + 196608 + ] + }, + "model.layers.7.mlp.gate_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 742326272, + "role": "ssmlp.gate.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.7.mlp.gate_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 8755200, + "role": "ssmlp.gate.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.7.mlp.gate_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 33822720, + "role": "ssmlp.gate.scales", + "shape": [ + 196608 + ] + }, + "model.layers.7.mlp.up_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 754909184, + "role": "ssmlp.up.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.7.mlp.up_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 8853504, + "role": "ssmlp.up.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.7.mlp.up_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 34215936, + "role": "ssmlp.up.scales", + "shape": [ + 196608 + ] + }, + "model.layers.7.post_attention_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 92160, + "role": "ssmlp.norm0", + "shape": [ + 3072 + ] + }, + "model.layers.8.attn.k_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 88539136, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.8.attn.k_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 595968, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.8.attn.k_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 4331520, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.8.attn.o_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 246677504, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.8.attn.o_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 1812480, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.8.attn.o_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 18634752, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.8.attn.q_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 241958912, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.8.attn.q_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 1775616, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.8.attn.q_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 18487296, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.8.attn.v_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 90112000, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.8.attn.v_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 608256, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.8.attn.v_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 4380672, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.8.input_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 98304, + "role": "ssmlp.norm1", + "shape": [ + 3072 + ] + }, + "model.layers.8.mlp.down_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 805240832, + "role": "ssmlp.down.qweight", + "shape": [ + 3072, + 64, + 64 + ] + }, + "model.layers.8.mlp.down_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 9246720, + "role": "ssmlp.down.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.8.mlp.down_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 35788800, + "role": "ssmlp.down.scales", + "shape": [ + 196608 + ] + }, + "model.layers.8.mlp.gate_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 780075008, + "role": "ssmlp.gate.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.8.mlp.gate_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 9050112, + "role": "ssmlp.gate.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.8.mlp.gate_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 35002368, + "role": "ssmlp.gate.scales", + "shape": [ + 196608 + ] + }, + "model.layers.8.mlp.up_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 792657920, + "role": "ssmlp.up.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.8.mlp.up_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 9148416, + "role": "ssmlp.up.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.8.mlp.up_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 35395584, + "role": "ssmlp.up.scales", + "shape": [ + 196608 + ] + }, + "model.layers.8.post_attention_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 104448, + "role": "ssmlp.norm0", + "shape": [ + 3072 + ] + }, + "model.layers.9.attn.k_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 91684864, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.9.attn.k_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 620544, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.9.attn.k_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 4429824, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.9.attn.o_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 256114688, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.9.attn.o_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 1886208, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.9.attn.o_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 18929664, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.9.attn.q_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 251396096, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.9.attn.q_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 1849344, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.9.attn.q_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 18782208, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.9.attn.v_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 93257728, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.9.attn.v_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 632832, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.9.attn.v_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 4478976, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.9.input_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 110592, + "role": "ssmlp.norm1", + "shape": [ + 3072 + ] + }, + "model.layers.9.mlp.down_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 842989568, + "role": "ssmlp.down.qweight", + "shape": [ + 3072, + 64, + 64 + ] + }, + "model.layers.9.mlp.down_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 9541632, + "role": "ssmlp.down.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.9.mlp.down_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 36968448, + "role": "ssmlp.down.scales", + "shape": [ + 196608 + ] + }, + "model.layers.9.mlp.gate_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 817823744, + "role": "ssmlp.gate.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.9.mlp.gate_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 9345024, + "role": "ssmlp.gate.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.9.mlp.gate_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 36182016, + "role": "ssmlp.gate.scales", + "shape": [ + 196608 + ] + }, + "model.layers.9.mlp.up_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 830406656, + "role": "ssmlp.up.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.9.mlp.up_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 9443328, + "role": "ssmlp.up.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.9.mlp.up_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 36575232, + "role": "ssmlp.up.scales", + "shape": [ + 196608 + ] + }, + "model.layers.9.post_attention_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 116736, + "role": "ssmlp.norm0", + "shape": [ + 3072 + ] + }, + "sin_cache": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 12976128, + "offset": 1699020800, + "role": "sin_cache", + "shape": [ + 135168, + 48 + ] + } + }, + "model": { + "family": "phi4", + "group_size": 128, + "head_size": 128, + "hidden_size": 3072, + "intermediate_size": 8192, + "kv_heads": 8, + "layers": 32, + "num_heads": 24, + "rms_epsilon": 1e-05, + "rope_dim": 96, + "vocab_size": 200064 + }, + "schema_version": 1, + "weight_objects": [ + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.0.attn.q_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.0.attn.q_proj.MatMulNBits.qweight", + "qzeros": "model.layers.0.attn.q_proj.MatMulNBits.qzeros", + "scales": "model.layers.0.attn.q_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.0.attn.k_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.0.attn.k_proj.MatMulNBits.qweight", + "qzeros": "model.layers.0.attn.k_proj.MatMulNBits.qzeros", + "scales": "model.layers.0.attn.k_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.0.attn.v_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.0.attn.v_proj.MatMulNBits.qweight", + "qzeros": "model.layers.0.attn.v_proj.MatMulNBits.qzeros", + "scales": "model.layers.0.attn.v_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.0.attn.o_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.0.attn.o_proj.MatMulNBits.qweight", + "qzeros": "model.layers.0.attn.o_proj.MatMulNBits.qzeros", + "scales": "model.layers.0.attn.o_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "k": 3072, + "n": 8192 + }, + "kind": "ssmlp", + "name": "model.layers.0.ssmlp", + "roles": { + "down_qweight": "model.layers.0.mlp.down_proj.MatMulNBits.qweight", + "down_qzeros": "model.layers.0.mlp.down_proj.MatMulNBits.qzeros", + "down_scales": "model.layers.0.mlp.down_proj.MatMulNBits.scales", + "gate_qweight": "model.layers.0.mlp.gate_proj.MatMulNBits.qweight", + "gate_qzeros": "model.layers.0.mlp.gate_proj.MatMulNBits.qzeros", + "gate_scales": "model.layers.0.mlp.gate_proj.MatMulNBits.scales", + "norm0": "model.layers.0.post_attention_layernorm.weight", + "norm1": "model.layers.1.input_layernorm.weight", + "up_qweight": "model.layers.0.mlp.up_proj.MatMulNBits.qweight", + "up_qzeros": "model.layers.0.mlp.up_proj.MatMulNBits.qzeros", + "up_scales": "model.layers.0.mlp.up_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.1.attn.q_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.1.attn.q_proj.MatMulNBits.qweight", + "qzeros": "model.layers.1.attn.q_proj.MatMulNBits.qzeros", + "scales": "model.layers.1.attn.q_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.1.attn.k_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.1.attn.k_proj.MatMulNBits.qweight", + "qzeros": "model.layers.1.attn.k_proj.MatMulNBits.qzeros", + "scales": "model.layers.1.attn.k_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.1.attn.v_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.1.attn.v_proj.MatMulNBits.qweight", + "qzeros": "model.layers.1.attn.v_proj.MatMulNBits.qzeros", + "scales": "model.layers.1.attn.v_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.1.attn.o_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.1.attn.o_proj.MatMulNBits.qweight", + "qzeros": "model.layers.1.attn.o_proj.MatMulNBits.qzeros", + "scales": "model.layers.1.attn.o_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "k": 3072, + "n": 8192 + }, + "kind": "ssmlp", + "name": "model.layers.1.ssmlp", + "roles": { + "down_qweight": "model.layers.1.mlp.down_proj.MatMulNBits.qweight", + "down_qzeros": "model.layers.1.mlp.down_proj.MatMulNBits.qzeros", + "down_scales": "model.layers.1.mlp.down_proj.MatMulNBits.scales", + "gate_qweight": "model.layers.1.mlp.gate_proj.MatMulNBits.qweight", + "gate_qzeros": "model.layers.1.mlp.gate_proj.MatMulNBits.qzeros", + "gate_scales": "model.layers.1.mlp.gate_proj.MatMulNBits.scales", + "norm0": "model.layers.1.post_attention_layernorm.weight", + "norm1": "model.layers.2.input_layernorm.weight", + "up_qweight": "model.layers.1.mlp.up_proj.MatMulNBits.qweight", + "up_qzeros": "model.layers.1.mlp.up_proj.MatMulNBits.qzeros", + "up_scales": "model.layers.1.mlp.up_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.2.attn.q_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.2.attn.q_proj.MatMulNBits.qweight", + "qzeros": "model.layers.2.attn.q_proj.MatMulNBits.qzeros", + "scales": "model.layers.2.attn.q_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.2.attn.k_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.2.attn.k_proj.MatMulNBits.qweight", + "qzeros": "model.layers.2.attn.k_proj.MatMulNBits.qzeros", + "scales": "model.layers.2.attn.k_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.2.attn.v_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.2.attn.v_proj.MatMulNBits.qweight", + "qzeros": "model.layers.2.attn.v_proj.MatMulNBits.qzeros", + "scales": "model.layers.2.attn.v_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.2.attn.o_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.2.attn.o_proj.MatMulNBits.qweight", + "qzeros": "model.layers.2.attn.o_proj.MatMulNBits.qzeros", + "scales": "model.layers.2.attn.o_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "k": 3072, + "n": 8192 + }, + "kind": "ssmlp", + "name": "model.layers.2.ssmlp", + "roles": { + "down_qweight": "model.layers.2.mlp.down_proj.MatMulNBits.qweight", + "down_qzeros": "model.layers.2.mlp.down_proj.MatMulNBits.qzeros", + "down_scales": "model.layers.2.mlp.down_proj.MatMulNBits.scales", + "gate_qweight": "model.layers.2.mlp.gate_proj.MatMulNBits.qweight", + "gate_qzeros": "model.layers.2.mlp.gate_proj.MatMulNBits.qzeros", + "gate_scales": "model.layers.2.mlp.gate_proj.MatMulNBits.scales", + "norm0": "model.layers.2.post_attention_layernorm.weight", + "norm1": "model.layers.3.input_layernorm.weight", + "up_qweight": "model.layers.2.mlp.up_proj.MatMulNBits.qweight", + "up_qzeros": "model.layers.2.mlp.up_proj.MatMulNBits.qzeros", + "up_scales": "model.layers.2.mlp.up_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.3.attn.q_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.3.attn.q_proj.MatMulNBits.qweight", + "qzeros": "model.layers.3.attn.q_proj.MatMulNBits.qzeros", + "scales": "model.layers.3.attn.q_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.3.attn.k_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.3.attn.k_proj.MatMulNBits.qweight", + "qzeros": "model.layers.3.attn.k_proj.MatMulNBits.qzeros", + "scales": "model.layers.3.attn.k_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.3.attn.v_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.3.attn.v_proj.MatMulNBits.qweight", + "qzeros": "model.layers.3.attn.v_proj.MatMulNBits.qzeros", + "scales": "model.layers.3.attn.v_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.3.attn.o_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.3.attn.o_proj.MatMulNBits.qweight", + "qzeros": "model.layers.3.attn.o_proj.MatMulNBits.qzeros", + "scales": "model.layers.3.attn.o_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "k": 3072, + "n": 8192 + }, + "kind": "ssmlp", + "name": "model.layers.3.ssmlp", + "roles": { + "down_qweight": "model.layers.3.mlp.down_proj.MatMulNBits.qweight", + "down_qzeros": "model.layers.3.mlp.down_proj.MatMulNBits.qzeros", + "down_scales": "model.layers.3.mlp.down_proj.MatMulNBits.scales", + "gate_qweight": "model.layers.3.mlp.gate_proj.MatMulNBits.qweight", + "gate_qzeros": "model.layers.3.mlp.gate_proj.MatMulNBits.qzeros", + "gate_scales": "model.layers.3.mlp.gate_proj.MatMulNBits.scales", + "norm0": "model.layers.3.post_attention_layernorm.weight", + "norm1": "model.layers.4.input_layernorm.weight", + "up_qweight": "model.layers.3.mlp.up_proj.MatMulNBits.qweight", + "up_qzeros": "model.layers.3.mlp.up_proj.MatMulNBits.qzeros", + "up_scales": "model.layers.3.mlp.up_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.4.attn.q_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.4.attn.q_proj.MatMulNBits.qweight", + "qzeros": "model.layers.4.attn.q_proj.MatMulNBits.qzeros", + "scales": "model.layers.4.attn.q_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.4.attn.k_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.4.attn.k_proj.MatMulNBits.qweight", + "qzeros": "model.layers.4.attn.k_proj.MatMulNBits.qzeros", + "scales": "model.layers.4.attn.k_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.4.attn.v_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.4.attn.v_proj.MatMulNBits.qweight", + "qzeros": "model.layers.4.attn.v_proj.MatMulNBits.qzeros", + "scales": "model.layers.4.attn.v_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.4.attn.o_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.4.attn.o_proj.MatMulNBits.qweight", + "qzeros": "model.layers.4.attn.o_proj.MatMulNBits.qzeros", + "scales": "model.layers.4.attn.o_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "k": 3072, + "n": 8192 + }, + "kind": "ssmlp", + "name": "model.layers.4.ssmlp", + "roles": { + "down_qweight": "model.layers.4.mlp.down_proj.MatMulNBits.qweight", + "down_qzeros": "model.layers.4.mlp.down_proj.MatMulNBits.qzeros", + "down_scales": "model.layers.4.mlp.down_proj.MatMulNBits.scales", + "gate_qweight": "model.layers.4.mlp.gate_proj.MatMulNBits.qweight", + "gate_qzeros": "model.layers.4.mlp.gate_proj.MatMulNBits.qzeros", + "gate_scales": "model.layers.4.mlp.gate_proj.MatMulNBits.scales", + "norm0": "model.layers.4.post_attention_layernorm.weight", + "norm1": "model.layers.5.input_layernorm.weight", + "up_qweight": "model.layers.4.mlp.up_proj.MatMulNBits.qweight", + "up_qzeros": "model.layers.4.mlp.up_proj.MatMulNBits.qzeros", + "up_scales": "model.layers.4.mlp.up_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.5.attn.q_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.5.attn.q_proj.MatMulNBits.qweight", + "qzeros": "model.layers.5.attn.q_proj.MatMulNBits.qzeros", + "scales": "model.layers.5.attn.q_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.5.attn.k_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.5.attn.k_proj.MatMulNBits.qweight", + "qzeros": "model.layers.5.attn.k_proj.MatMulNBits.qzeros", + "scales": "model.layers.5.attn.k_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.5.attn.v_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.5.attn.v_proj.MatMulNBits.qweight", + "qzeros": "model.layers.5.attn.v_proj.MatMulNBits.qzeros", + "scales": "model.layers.5.attn.v_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.5.attn.o_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.5.attn.o_proj.MatMulNBits.qweight", + "qzeros": "model.layers.5.attn.o_proj.MatMulNBits.qzeros", + "scales": "model.layers.5.attn.o_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "k": 3072, + "n": 8192 + }, + "kind": "ssmlp", + "name": "model.layers.5.ssmlp", + "roles": { + "down_qweight": "model.layers.5.mlp.down_proj.MatMulNBits.qweight", + "down_qzeros": "model.layers.5.mlp.down_proj.MatMulNBits.qzeros", + "down_scales": "model.layers.5.mlp.down_proj.MatMulNBits.scales", + "gate_qweight": "model.layers.5.mlp.gate_proj.MatMulNBits.qweight", + "gate_qzeros": "model.layers.5.mlp.gate_proj.MatMulNBits.qzeros", + "gate_scales": "model.layers.5.mlp.gate_proj.MatMulNBits.scales", + "norm0": "model.layers.5.post_attention_layernorm.weight", + "norm1": "model.layers.6.input_layernorm.weight", + "up_qweight": "model.layers.5.mlp.up_proj.MatMulNBits.qweight", + "up_qzeros": "model.layers.5.mlp.up_proj.MatMulNBits.qzeros", + "up_scales": "model.layers.5.mlp.up_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.6.attn.q_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.6.attn.q_proj.MatMulNBits.qweight", + "qzeros": "model.layers.6.attn.q_proj.MatMulNBits.qzeros", + "scales": "model.layers.6.attn.q_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.6.attn.k_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.6.attn.k_proj.MatMulNBits.qweight", + "qzeros": "model.layers.6.attn.k_proj.MatMulNBits.qzeros", + "scales": "model.layers.6.attn.k_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.6.attn.v_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.6.attn.v_proj.MatMulNBits.qweight", + "qzeros": "model.layers.6.attn.v_proj.MatMulNBits.qzeros", + "scales": "model.layers.6.attn.v_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.6.attn.o_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.6.attn.o_proj.MatMulNBits.qweight", + "qzeros": "model.layers.6.attn.o_proj.MatMulNBits.qzeros", + "scales": "model.layers.6.attn.o_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "k": 3072, + "n": 8192 + }, + "kind": "ssmlp", + "name": "model.layers.6.ssmlp", + "roles": { + "down_qweight": "model.layers.6.mlp.down_proj.MatMulNBits.qweight", + "down_qzeros": "model.layers.6.mlp.down_proj.MatMulNBits.qzeros", + "down_scales": "model.layers.6.mlp.down_proj.MatMulNBits.scales", + "gate_qweight": "model.layers.6.mlp.gate_proj.MatMulNBits.qweight", + "gate_qzeros": "model.layers.6.mlp.gate_proj.MatMulNBits.qzeros", + "gate_scales": "model.layers.6.mlp.gate_proj.MatMulNBits.scales", + "norm0": "model.layers.6.post_attention_layernorm.weight", + "norm1": "model.layers.7.input_layernorm.weight", + "up_qweight": "model.layers.6.mlp.up_proj.MatMulNBits.qweight", + "up_qzeros": "model.layers.6.mlp.up_proj.MatMulNBits.qzeros", + "up_scales": "model.layers.6.mlp.up_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.7.attn.q_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.7.attn.q_proj.MatMulNBits.qweight", + "qzeros": "model.layers.7.attn.q_proj.MatMulNBits.qzeros", + "scales": "model.layers.7.attn.q_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.7.attn.k_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.7.attn.k_proj.MatMulNBits.qweight", + "qzeros": "model.layers.7.attn.k_proj.MatMulNBits.qzeros", + "scales": "model.layers.7.attn.k_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.7.attn.v_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.7.attn.v_proj.MatMulNBits.qweight", + "qzeros": "model.layers.7.attn.v_proj.MatMulNBits.qzeros", + "scales": "model.layers.7.attn.v_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.7.attn.o_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.7.attn.o_proj.MatMulNBits.qweight", + "qzeros": "model.layers.7.attn.o_proj.MatMulNBits.qzeros", + "scales": "model.layers.7.attn.o_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "k": 3072, + "n": 8192 + }, + "kind": "ssmlp", + "name": "model.layers.7.ssmlp", + "roles": { + "down_qweight": "model.layers.7.mlp.down_proj.MatMulNBits.qweight", + "down_qzeros": "model.layers.7.mlp.down_proj.MatMulNBits.qzeros", + "down_scales": "model.layers.7.mlp.down_proj.MatMulNBits.scales", + "gate_qweight": "model.layers.7.mlp.gate_proj.MatMulNBits.qweight", + "gate_qzeros": "model.layers.7.mlp.gate_proj.MatMulNBits.qzeros", + "gate_scales": "model.layers.7.mlp.gate_proj.MatMulNBits.scales", + "norm0": "model.layers.7.post_attention_layernorm.weight", + "norm1": "model.layers.8.input_layernorm.weight", + "up_qweight": "model.layers.7.mlp.up_proj.MatMulNBits.qweight", + "up_qzeros": "model.layers.7.mlp.up_proj.MatMulNBits.qzeros", + "up_scales": "model.layers.7.mlp.up_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.8.attn.q_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.8.attn.q_proj.MatMulNBits.qweight", + "qzeros": "model.layers.8.attn.q_proj.MatMulNBits.qzeros", + "scales": "model.layers.8.attn.q_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.8.attn.k_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.8.attn.k_proj.MatMulNBits.qweight", + "qzeros": "model.layers.8.attn.k_proj.MatMulNBits.qzeros", + "scales": "model.layers.8.attn.k_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.8.attn.v_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.8.attn.v_proj.MatMulNBits.qweight", + "qzeros": "model.layers.8.attn.v_proj.MatMulNBits.qzeros", + "scales": "model.layers.8.attn.v_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.8.attn.o_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.8.attn.o_proj.MatMulNBits.qweight", + "qzeros": "model.layers.8.attn.o_proj.MatMulNBits.qzeros", + "scales": "model.layers.8.attn.o_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "k": 3072, + "n": 8192 + }, + "kind": "ssmlp", + "name": "model.layers.8.ssmlp", + "roles": { + "down_qweight": "model.layers.8.mlp.down_proj.MatMulNBits.qweight", + "down_qzeros": "model.layers.8.mlp.down_proj.MatMulNBits.qzeros", + "down_scales": "model.layers.8.mlp.down_proj.MatMulNBits.scales", + "gate_qweight": "model.layers.8.mlp.gate_proj.MatMulNBits.qweight", + "gate_qzeros": "model.layers.8.mlp.gate_proj.MatMulNBits.qzeros", + "gate_scales": "model.layers.8.mlp.gate_proj.MatMulNBits.scales", + "norm0": "model.layers.8.post_attention_layernorm.weight", + "norm1": "model.layers.9.input_layernorm.weight", + "up_qweight": "model.layers.8.mlp.up_proj.MatMulNBits.qweight", + "up_qzeros": "model.layers.8.mlp.up_proj.MatMulNBits.qzeros", + "up_scales": "model.layers.8.mlp.up_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.9.attn.q_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.9.attn.q_proj.MatMulNBits.qweight", + "qzeros": "model.layers.9.attn.q_proj.MatMulNBits.qzeros", + "scales": "model.layers.9.attn.q_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.9.attn.k_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.9.attn.k_proj.MatMulNBits.qweight", + "qzeros": "model.layers.9.attn.k_proj.MatMulNBits.qzeros", + "scales": "model.layers.9.attn.k_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.9.attn.v_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.9.attn.v_proj.MatMulNBits.qweight", + "qzeros": "model.layers.9.attn.v_proj.MatMulNBits.qzeros", + "scales": "model.layers.9.attn.v_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.9.attn.o_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.9.attn.o_proj.MatMulNBits.qweight", + "qzeros": "model.layers.9.attn.o_proj.MatMulNBits.qzeros", + "scales": "model.layers.9.attn.o_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "k": 3072, + "n": 8192 + }, + "kind": "ssmlp", + "name": "model.layers.9.ssmlp", + "roles": { + "down_qweight": "model.layers.9.mlp.down_proj.MatMulNBits.qweight", + "down_qzeros": "model.layers.9.mlp.down_proj.MatMulNBits.qzeros", + "down_scales": "model.layers.9.mlp.down_proj.MatMulNBits.scales", + "gate_qweight": "model.layers.9.mlp.gate_proj.MatMulNBits.qweight", + "gate_qzeros": "model.layers.9.mlp.gate_proj.MatMulNBits.qzeros", + "gate_scales": "model.layers.9.mlp.gate_proj.MatMulNBits.scales", + "norm0": "model.layers.9.post_attention_layernorm.weight", + "norm1": "model.layers.10.input_layernorm.weight", + "up_qweight": "model.layers.9.mlp.up_proj.MatMulNBits.qweight", + "up_qzeros": "model.layers.9.mlp.up_proj.MatMulNBits.qzeros", + "up_scales": "model.layers.9.mlp.up_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.10.attn.q_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.10.attn.q_proj.MatMulNBits.qweight", + "qzeros": "model.layers.10.attn.q_proj.MatMulNBits.qzeros", + "scales": "model.layers.10.attn.q_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.10.attn.k_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.10.attn.k_proj.MatMulNBits.qweight", + "qzeros": "model.layers.10.attn.k_proj.MatMulNBits.qzeros", + "scales": "model.layers.10.attn.k_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.10.attn.v_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.10.attn.v_proj.MatMulNBits.qweight", + "qzeros": "model.layers.10.attn.v_proj.MatMulNBits.qzeros", + "scales": "model.layers.10.attn.v_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.10.attn.o_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.10.attn.o_proj.MatMulNBits.qweight", + "qzeros": "model.layers.10.attn.o_proj.MatMulNBits.qzeros", + "scales": "model.layers.10.attn.o_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "k": 3072, + "n": 8192 + }, + "kind": "ssmlp", + "name": "model.layers.10.ssmlp", + "roles": { + "down_qweight": "model.layers.10.mlp.down_proj.MatMulNBits.qweight", + "down_qzeros": "model.layers.10.mlp.down_proj.MatMulNBits.qzeros", + "down_scales": "model.layers.10.mlp.down_proj.MatMulNBits.scales", + "gate_qweight": "model.layers.10.mlp.gate_proj.MatMulNBits.qweight", + "gate_qzeros": "model.layers.10.mlp.gate_proj.MatMulNBits.qzeros", + "gate_scales": "model.layers.10.mlp.gate_proj.MatMulNBits.scales", + "norm0": "model.layers.10.post_attention_layernorm.weight", + "norm1": "model.layers.11.input_layernorm.weight", + "up_qweight": "model.layers.10.mlp.up_proj.MatMulNBits.qweight", + "up_qzeros": "model.layers.10.mlp.up_proj.MatMulNBits.qzeros", + "up_scales": "model.layers.10.mlp.up_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.11.attn.q_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.11.attn.q_proj.MatMulNBits.qweight", + "qzeros": "model.layers.11.attn.q_proj.MatMulNBits.qzeros", + "scales": "model.layers.11.attn.q_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.11.attn.k_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.11.attn.k_proj.MatMulNBits.qweight", + "qzeros": "model.layers.11.attn.k_proj.MatMulNBits.qzeros", + "scales": "model.layers.11.attn.k_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.11.attn.v_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.11.attn.v_proj.MatMulNBits.qweight", + "qzeros": "model.layers.11.attn.v_proj.MatMulNBits.qzeros", + "scales": "model.layers.11.attn.v_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.11.attn.o_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.11.attn.o_proj.MatMulNBits.qweight", + "qzeros": "model.layers.11.attn.o_proj.MatMulNBits.qzeros", + "scales": "model.layers.11.attn.o_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "k": 3072, + "n": 8192 + }, + "kind": "ssmlp", + "name": "model.layers.11.ssmlp", + "roles": { + "down_qweight": "model.layers.11.mlp.down_proj.MatMulNBits.qweight", + "down_qzeros": "model.layers.11.mlp.down_proj.MatMulNBits.qzeros", + "down_scales": "model.layers.11.mlp.down_proj.MatMulNBits.scales", + "gate_qweight": "model.layers.11.mlp.gate_proj.MatMulNBits.qweight", + "gate_qzeros": "model.layers.11.mlp.gate_proj.MatMulNBits.qzeros", + "gate_scales": "model.layers.11.mlp.gate_proj.MatMulNBits.scales", + "norm0": "model.layers.11.post_attention_layernorm.weight", + "norm1": "model.layers.12.input_layernorm.weight", + "up_qweight": "model.layers.11.mlp.up_proj.MatMulNBits.qweight", + "up_qzeros": "model.layers.11.mlp.up_proj.MatMulNBits.qzeros", + "up_scales": "model.layers.11.mlp.up_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.12.attn.q_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.12.attn.q_proj.MatMulNBits.qweight", + "qzeros": "model.layers.12.attn.q_proj.MatMulNBits.qzeros", + "scales": "model.layers.12.attn.q_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.12.attn.k_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.12.attn.k_proj.MatMulNBits.qweight", + "qzeros": "model.layers.12.attn.k_proj.MatMulNBits.qzeros", + "scales": "model.layers.12.attn.k_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.12.attn.v_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.12.attn.v_proj.MatMulNBits.qweight", + "qzeros": "model.layers.12.attn.v_proj.MatMulNBits.qzeros", + "scales": "model.layers.12.attn.v_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.12.attn.o_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.12.attn.o_proj.MatMulNBits.qweight", + "qzeros": "model.layers.12.attn.o_proj.MatMulNBits.qzeros", + "scales": "model.layers.12.attn.o_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "k": 3072, + "n": 8192 + }, + "kind": "ssmlp", + "name": "model.layers.12.ssmlp", + "roles": { + "down_qweight": "model.layers.12.mlp.down_proj.MatMulNBits.qweight", + "down_qzeros": "model.layers.12.mlp.down_proj.MatMulNBits.qzeros", + "down_scales": "model.layers.12.mlp.down_proj.MatMulNBits.scales", + "gate_qweight": "model.layers.12.mlp.gate_proj.MatMulNBits.qweight", + "gate_qzeros": "model.layers.12.mlp.gate_proj.MatMulNBits.qzeros", + "gate_scales": "model.layers.12.mlp.gate_proj.MatMulNBits.scales", + "norm0": "model.layers.12.post_attention_layernorm.weight", + "norm1": "model.layers.13.input_layernorm.weight", + "up_qweight": "model.layers.12.mlp.up_proj.MatMulNBits.qweight", + "up_qzeros": "model.layers.12.mlp.up_proj.MatMulNBits.qzeros", + "up_scales": "model.layers.12.mlp.up_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.13.attn.q_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.13.attn.q_proj.MatMulNBits.qweight", + "qzeros": "model.layers.13.attn.q_proj.MatMulNBits.qzeros", + "scales": "model.layers.13.attn.q_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.13.attn.k_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.13.attn.k_proj.MatMulNBits.qweight", + "qzeros": "model.layers.13.attn.k_proj.MatMulNBits.qzeros", + "scales": "model.layers.13.attn.k_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.13.attn.v_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.13.attn.v_proj.MatMulNBits.qweight", + "qzeros": "model.layers.13.attn.v_proj.MatMulNBits.qzeros", + "scales": "model.layers.13.attn.v_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.13.attn.o_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.13.attn.o_proj.MatMulNBits.qweight", + "qzeros": "model.layers.13.attn.o_proj.MatMulNBits.qzeros", + "scales": "model.layers.13.attn.o_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "k": 3072, + "n": 8192 + }, + "kind": "ssmlp", + "name": "model.layers.13.ssmlp", + "roles": { + "down_qweight": "model.layers.13.mlp.down_proj.MatMulNBits.qweight", + "down_qzeros": "model.layers.13.mlp.down_proj.MatMulNBits.qzeros", + "down_scales": "model.layers.13.mlp.down_proj.MatMulNBits.scales", + "gate_qweight": "model.layers.13.mlp.gate_proj.MatMulNBits.qweight", + "gate_qzeros": "model.layers.13.mlp.gate_proj.MatMulNBits.qzeros", + "gate_scales": "model.layers.13.mlp.gate_proj.MatMulNBits.scales", + "norm0": "model.layers.13.post_attention_layernorm.weight", + "norm1": "model.layers.14.input_layernorm.weight", + "up_qweight": "model.layers.13.mlp.up_proj.MatMulNBits.qweight", + "up_qzeros": "model.layers.13.mlp.up_proj.MatMulNBits.qzeros", + "up_scales": "model.layers.13.mlp.up_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.14.attn.q_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.14.attn.q_proj.MatMulNBits.qweight", + "qzeros": "model.layers.14.attn.q_proj.MatMulNBits.qzeros", + "scales": "model.layers.14.attn.q_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.14.attn.k_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.14.attn.k_proj.MatMulNBits.qweight", + "qzeros": "model.layers.14.attn.k_proj.MatMulNBits.qzeros", + "scales": "model.layers.14.attn.k_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.14.attn.v_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.14.attn.v_proj.MatMulNBits.qweight", + "qzeros": "model.layers.14.attn.v_proj.MatMulNBits.qzeros", + "scales": "model.layers.14.attn.v_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.14.attn.o_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.14.attn.o_proj.MatMulNBits.qweight", + "qzeros": "model.layers.14.attn.o_proj.MatMulNBits.qzeros", + "scales": "model.layers.14.attn.o_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "k": 3072, + "n": 8192 + }, + "kind": "ssmlp", + "name": "model.layers.14.ssmlp", + "roles": { + "down_qweight": "model.layers.14.mlp.down_proj.MatMulNBits.qweight", + "down_qzeros": "model.layers.14.mlp.down_proj.MatMulNBits.qzeros", + "down_scales": "model.layers.14.mlp.down_proj.MatMulNBits.scales", + "gate_qweight": "model.layers.14.mlp.gate_proj.MatMulNBits.qweight", + "gate_qzeros": "model.layers.14.mlp.gate_proj.MatMulNBits.qzeros", + "gate_scales": "model.layers.14.mlp.gate_proj.MatMulNBits.scales", + "norm0": "model.layers.14.post_attention_layernorm.weight", + "norm1": "model.layers.15.input_layernorm.weight", + "up_qweight": "model.layers.14.mlp.up_proj.MatMulNBits.qweight", + "up_qzeros": "model.layers.14.mlp.up_proj.MatMulNBits.qzeros", + "up_scales": "model.layers.14.mlp.up_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.15.attn.q_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.15.attn.q_proj.MatMulNBits.qweight", + "qzeros": "model.layers.15.attn.q_proj.MatMulNBits.qzeros", + "scales": "model.layers.15.attn.q_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.15.attn.k_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.15.attn.k_proj.MatMulNBits.qweight", + "qzeros": "model.layers.15.attn.k_proj.MatMulNBits.qzeros", + "scales": "model.layers.15.attn.k_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.15.attn.v_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.15.attn.v_proj.MatMulNBits.qweight", + "qzeros": "model.layers.15.attn.v_proj.MatMulNBits.qzeros", + "scales": "model.layers.15.attn.v_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.15.attn.o_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.15.attn.o_proj.MatMulNBits.qweight", + "qzeros": "model.layers.15.attn.o_proj.MatMulNBits.qzeros", + "scales": "model.layers.15.attn.o_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "k": 3072, + "n": 8192 + }, + "kind": "ssmlp", + "name": "model.layers.15.ssmlp", + "roles": { + "down_qweight": "model.layers.15.mlp.down_proj.MatMulNBits.qweight", + "down_qzeros": "model.layers.15.mlp.down_proj.MatMulNBits.qzeros", + "down_scales": "model.layers.15.mlp.down_proj.MatMulNBits.scales", + "gate_qweight": "model.layers.15.mlp.gate_proj.MatMulNBits.qweight", + "gate_qzeros": "model.layers.15.mlp.gate_proj.MatMulNBits.qzeros", + "gate_scales": "model.layers.15.mlp.gate_proj.MatMulNBits.scales", + "norm0": "model.layers.15.post_attention_layernorm.weight", + "norm1": "model.layers.16.input_layernorm.weight", + "up_qweight": "model.layers.15.mlp.up_proj.MatMulNBits.qweight", + "up_qzeros": "model.layers.15.mlp.up_proj.MatMulNBits.qzeros", + "up_scales": "model.layers.15.mlp.up_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.16.attn.q_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.16.attn.q_proj.MatMulNBits.qweight", + "qzeros": "model.layers.16.attn.q_proj.MatMulNBits.qzeros", + "scales": "model.layers.16.attn.q_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.16.attn.k_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.16.attn.k_proj.MatMulNBits.qweight", + "qzeros": "model.layers.16.attn.k_proj.MatMulNBits.qzeros", + "scales": "model.layers.16.attn.k_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.16.attn.v_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.16.attn.v_proj.MatMulNBits.qweight", + "qzeros": "model.layers.16.attn.v_proj.MatMulNBits.qzeros", + "scales": "model.layers.16.attn.v_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.16.attn.o_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.16.attn.o_proj.MatMulNBits.qweight", + "qzeros": "model.layers.16.attn.o_proj.MatMulNBits.qzeros", + "scales": "model.layers.16.attn.o_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "k": 3072, + "n": 8192 + }, + "kind": "ssmlp", + "name": "model.layers.16.ssmlp", + "roles": { + "down_qweight": "model.layers.16.mlp.down_proj.MatMulNBits.qweight", + "down_qzeros": "model.layers.16.mlp.down_proj.MatMulNBits.qzeros", + "down_scales": "model.layers.16.mlp.down_proj.MatMulNBits.scales", + "gate_qweight": "model.layers.16.mlp.gate_proj.MatMulNBits.qweight", + "gate_qzeros": "model.layers.16.mlp.gate_proj.MatMulNBits.qzeros", + "gate_scales": "model.layers.16.mlp.gate_proj.MatMulNBits.scales", + "norm0": "model.layers.16.post_attention_layernorm.weight", + "norm1": "model.layers.17.input_layernorm.weight", + "up_qweight": "model.layers.16.mlp.up_proj.MatMulNBits.qweight", + "up_qzeros": "model.layers.16.mlp.up_proj.MatMulNBits.qzeros", + "up_scales": "model.layers.16.mlp.up_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.17.attn.q_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.17.attn.q_proj.MatMulNBits.qweight", + "qzeros": "model.layers.17.attn.q_proj.MatMulNBits.qzeros", + "scales": "model.layers.17.attn.q_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.17.attn.k_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.17.attn.k_proj.MatMulNBits.qweight", + "qzeros": "model.layers.17.attn.k_proj.MatMulNBits.qzeros", + "scales": "model.layers.17.attn.k_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.17.attn.v_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.17.attn.v_proj.MatMulNBits.qweight", + "qzeros": "model.layers.17.attn.v_proj.MatMulNBits.qzeros", + "scales": "model.layers.17.attn.v_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.17.attn.o_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.17.attn.o_proj.MatMulNBits.qweight", + "qzeros": "model.layers.17.attn.o_proj.MatMulNBits.qzeros", + "scales": "model.layers.17.attn.o_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "k": 3072, + "n": 8192 + }, + "kind": "ssmlp", + "name": "model.layers.17.ssmlp", + "roles": { + "down_qweight": "model.layers.17.mlp.down_proj.MatMulNBits.qweight", + "down_qzeros": "model.layers.17.mlp.down_proj.MatMulNBits.qzeros", + "down_scales": "model.layers.17.mlp.down_proj.MatMulNBits.scales", + "gate_qweight": "model.layers.17.mlp.gate_proj.MatMulNBits.qweight", + "gate_qzeros": "model.layers.17.mlp.gate_proj.MatMulNBits.qzeros", + "gate_scales": "model.layers.17.mlp.gate_proj.MatMulNBits.scales", + "norm0": "model.layers.17.post_attention_layernorm.weight", + "norm1": "model.layers.18.input_layernorm.weight", + "up_qweight": "model.layers.17.mlp.up_proj.MatMulNBits.qweight", + "up_qzeros": "model.layers.17.mlp.up_proj.MatMulNBits.qzeros", + "up_scales": "model.layers.17.mlp.up_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.18.attn.q_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.18.attn.q_proj.MatMulNBits.qweight", + "qzeros": "model.layers.18.attn.q_proj.MatMulNBits.qzeros", + "scales": "model.layers.18.attn.q_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.18.attn.k_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.18.attn.k_proj.MatMulNBits.qweight", + "qzeros": "model.layers.18.attn.k_proj.MatMulNBits.qzeros", + "scales": "model.layers.18.attn.k_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.18.attn.v_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.18.attn.v_proj.MatMulNBits.qweight", + "qzeros": "model.layers.18.attn.v_proj.MatMulNBits.qzeros", + "scales": "model.layers.18.attn.v_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.18.attn.o_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.18.attn.o_proj.MatMulNBits.qweight", + "qzeros": "model.layers.18.attn.o_proj.MatMulNBits.qzeros", + "scales": "model.layers.18.attn.o_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "k": 3072, + "n": 8192 + }, + "kind": "ssmlp", + "name": "model.layers.18.ssmlp", + "roles": { + "down_qweight": "model.layers.18.mlp.down_proj.MatMulNBits.qweight", + "down_qzeros": "model.layers.18.mlp.down_proj.MatMulNBits.qzeros", + "down_scales": "model.layers.18.mlp.down_proj.MatMulNBits.scales", + "gate_qweight": "model.layers.18.mlp.gate_proj.MatMulNBits.qweight", + "gate_qzeros": "model.layers.18.mlp.gate_proj.MatMulNBits.qzeros", + "gate_scales": "model.layers.18.mlp.gate_proj.MatMulNBits.scales", + "norm0": "model.layers.18.post_attention_layernorm.weight", + "norm1": "model.layers.19.input_layernorm.weight", + "up_qweight": "model.layers.18.mlp.up_proj.MatMulNBits.qweight", + "up_qzeros": "model.layers.18.mlp.up_proj.MatMulNBits.qzeros", + "up_scales": "model.layers.18.mlp.up_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.19.attn.q_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.19.attn.q_proj.MatMulNBits.qweight", + "qzeros": "model.layers.19.attn.q_proj.MatMulNBits.qzeros", + "scales": "model.layers.19.attn.q_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.19.attn.k_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.19.attn.k_proj.MatMulNBits.qweight", + "qzeros": "model.layers.19.attn.k_proj.MatMulNBits.qzeros", + "scales": "model.layers.19.attn.k_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.19.attn.v_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.19.attn.v_proj.MatMulNBits.qweight", + "qzeros": "model.layers.19.attn.v_proj.MatMulNBits.qzeros", + "scales": "model.layers.19.attn.v_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.19.attn.o_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.19.attn.o_proj.MatMulNBits.qweight", + "qzeros": "model.layers.19.attn.o_proj.MatMulNBits.qzeros", + "scales": "model.layers.19.attn.o_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "k": 3072, + "n": 8192 + }, + "kind": "ssmlp", + "name": "model.layers.19.ssmlp", + "roles": { + "down_qweight": "model.layers.19.mlp.down_proj.MatMulNBits.qweight", + "down_qzeros": "model.layers.19.mlp.down_proj.MatMulNBits.qzeros", + "down_scales": "model.layers.19.mlp.down_proj.MatMulNBits.scales", + "gate_qweight": "model.layers.19.mlp.gate_proj.MatMulNBits.qweight", + "gate_qzeros": "model.layers.19.mlp.gate_proj.MatMulNBits.qzeros", + "gate_scales": "model.layers.19.mlp.gate_proj.MatMulNBits.scales", + "norm0": "model.layers.19.post_attention_layernorm.weight", + "norm1": "model.layers.20.input_layernorm.weight", + "up_qweight": "model.layers.19.mlp.up_proj.MatMulNBits.qweight", + "up_qzeros": "model.layers.19.mlp.up_proj.MatMulNBits.qzeros", + "up_scales": "model.layers.19.mlp.up_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.20.attn.q_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.20.attn.q_proj.MatMulNBits.qweight", + "qzeros": "model.layers.20.attn.q_proj.MatMulNBits.qzeros", + "scales": "model.layers.20.attn.q_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.20.attn.k_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.20.attn.k_proj.MatMulNBits.qweight", + "qzeros": "model.layers.20.attn.k_proj.MatMulNBits.qzeros", + "scales": "model.layers.20.attn.k_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.20.attn.v_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.20.attn.v_proj.MatMulNBits.qweight", + "qzeros": "model.layers.20.attn.v_proj.MatMulNBits.qzeros", + "scales": "model.layers.20.attn.v_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.20.attn.o_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.20.attn.o_proj.MatMulNBits.qweight", + "qzeros": "model.layers.20.attn.o_proj.MatMulNBits.qzeros", + "scales": "model.layers.20.attn.o_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "k": 3072, + "n": 8192 + }, + "kind": "ssmlp", + "name": "model.layers.20.ssmlp", + "roles": { + "down_qweight": "model.layers.20.mlp.down_proj.MatMulNBits.qweight", + "down_qzeros": "model.layers.20.mlp.down_proj.MatMulNBits.qzeros", + "down_scales": "model.layers.20.mlp.down_proj.MatMulNBits.scales", + "gate_qweight": "model.layers.20.mlp.gate_proj.MatMulNBits.qweight", + "gate_qzeros": "model.layers.20.mlp.gate_proj.MatMulNBits.qzeros", + "gate_scales": "model.layers.20.mlp.gate_proj.MatMulNBits.scales", + "norm0": "model.layers.20.post_attention_layernorm.weight", + "norm1": "model.layers.21.input_layernorm.weight", + "up_qweight": "model.layers.20.mlp.up_proj.MatMulNBits.qweight", + "up_qzeros": "model.layers.20.mlp.up_proj.MatMulNBits.qzeros", + "up_scales": "model.layers.20.mlp.up_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.21.attn.q_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.21.attn.q_proj.MatMulNBits.qweight", + "qzeros": "model.layers.21.attn.q_proj.MatMulNBits.qzeros", + "scales": "model.layers.21.attn.q_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.21.attn.k_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.21.attn.k_proj.MatMulNBits.qweight", + "qzeros": "model.layers.21.attn.k_proj.MatMulNBits.qzeros", + "scales": "model.layers.21.attn.k_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.21.attn.v_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.21.attn.v_proj.MatMulNBits.qweight", + "qzeros": "model.layers.21.attn.v_proj.MatMulNBits.qzeros", + "scales": "model.layers.21.attn.v_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.21.attn.o_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.21.attn.o_proj.MatMulNBits.qweight", + "qzeros": "model.layers.21.attn.o_proj.MatMulNBits.qzeros", + "scales": "model.layers.21.attn.o_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "k": 3072, + "n": 8192 + }, + "kind": "ssmlp", + "name": "model.layers.21.ssmlp", + "roles": { + "down_qweight": "model.layers.21.mlp.down_proj.MatMulNBits.qweight", + "down_qzeros": "model.layers.21.mlp.down_proj.MatMulNBits.qzeros", + "down_scales": "model.layers.21.mlp.down_proj.MatMulNBits.scales", + "gate_qweight": "model.layers.21.mlp.gate_proj.MatMulNBits.qweight", + "gate_qzeros": "model.layers.21.mlp.gate_proj.MatMulNBits.qzeros", + "gate_scales": "model.layers.21.mlp.gate_proj.MatMulNBits.scales", + "norm0": "model.layers.21.post_attention_layernorm.weight", + "norm1": "model.layers.22.input_layernorm.weight", + "up_qweight": "model.layers.21.mlp.up_proj.MatMulNBits.qweight", + "up_qzeros": "model.layers.21.mlp.up_proj.MatMulNBits.qzeros", + "up_scales": "model.layers.21.mlp.up_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.22.attn.q_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.22.attn.q_proj.MatMulNBits.qweight", + "qzeros": "model.layers.22.attn.q_proj.MatMulNBits.qzeros", + "scales": "model.layers.22.attn.q_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.22.attn.k_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.22.attn.k_proj.MatMulNBits.qweight", + "qzeros": "model.layers.22.attn.k_proj.MatMulNBits.qzeros", + "scales": "model.layers.22.attn.k_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.22.attn.v_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.22.attn.v_proj.MatMulNBits.qweight", + "qzeros": "model.layers.22.attn.v_proj.MatMulNBits.qzeros", + "scales": "model.layers.22.attn.v_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.22.attn.o_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.22.attn.o_proj.MatMulNBits.qweight", + "qzeros": "model.layers.22.attn.o_proj.MatMulNBits.qzeros", + "scales": "model.layers.22.attn.o_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "k": 3072, + "n": 8192 + }, + "kind": "ssmlp", + "name": "model.layers.22.ssmlp", + "roles": { + "down_qweight": "model.layers.22.mlp.down_proj.MatMulNBits.qweight", + "down_qzeros": "model.layers.22.mlp.down_proj.MatMulNBits.qzeros", + "down_scales": "model.layers.22.mlp.down_proj.MatMulNBits.scales", + "gate_qweight": "model.layers.22.mlp.gate_proj.MatMulNBits.qweight", + "gate_qzeros": "model.layers.22.mlp.gate_proj.MatMulNBits.qzeros", + "gate_scales": "model.layers.22.mlp.gate_proj.MatMulNBits.scales", + "norm0": "model.layers.22.post_attention_layernorm.weight", + "norm1": "model.layers.23.input_layernorm.weight", + "up_qweight": "model.layers.22.mlp.up_proj.MatMulNBits.qweight", + "up_qzeros": "model.layers.22.mlp.up_proj.MatMulNBits.qzeros", + "up_scales": "model.layers.22.mlp.up_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.23.attn.q_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.23.attn.q_proj.MatMulNBits.qweight", + "qzeros": "model.layers.23.attn.q_proj.MatMulNBits.qzeros", + "scales": "model.layers.23.attn.q_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.23.attn.k_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.23.attn.k_proj.MatMulNBits.qweight", + "qzeros": "model.layers.23.attn.k_proj.MatMulNBits.qzeros", + "scales": "model.layers.23.attn.k_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.23.attn.v_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.23.attn.v_proj.MatMulNBits.qweight", + "qzeros": "model.layers.23.attn.v_proj.MatMulNBits.qzeros", + "scales": "model.layers.23.attn.v_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.23.attn.o_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.23.attn.o_proj.MatMulNBits.qweight", + "qzeros": "model.layers.23.attn.o_proj.MatMulNBits.qzeros", + "scales": "model.layers.23.attn.o_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "k": 3072, + "n": 8192 + }, + "kind": "ssmlp", + "name": "model.layers.23.ssmlp", + "roles": { + "down_qweight": "model.layers.23.mlp.down_proj.MatMulNBits.qweight", + "down_qzeros": "model.layers.23.mlp.down_proj.MatMulNBits.qzeros", + "down_scales": "model.layers.23.mlp.down_proj.MatMulNBits.scales", + "gate_qweight": "model.layers.23.mlp.gate_proj.MatMulNBits.qweight", + "gate_qzeros": "model.layers.23.mlp.gate_proj.MatMulNBits.qzeros", + "gate_scales": "model.layers.23.mlp.gate_proj.MatMulNBits.scales", + "norm0": "model.layers.23.post_attention_layernorm.weight", + "norm1": "model.layers.24.input_layernorm.weight", + "up_qweight": "model.layers.23.mlp.up_proj.MatMulNBits.qweight", + "up_qzeros": "model.layers.23.mlp.up_proj.MatMulNBits.qzeros", + "up_scales": "model.layers.23.mlp.up_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.24.attn.q_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.24.attn.q_proj.MatMulNBits.qweight", + "qzeros": "model.layers.24.attn.q_proj.MatMulNBits.qzeros", + "scales": "model.layers.24.attn.q_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.24.attn.k_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.24.attn.k_proj.MatMulNBits.qweight", + "qzeros": "model.layers.24.attn.k_proj.MatMulNBits.qzeros", + "scales": "model.layers.24.attn.k_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.24.attn.v_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.24.attn.v_proj.MatMulNBits.qweight", + "qzeros": "model.layers.24.attn.v_proj.MatMulNBits.qzeros", + "scales": "model.layers.24.attn.v_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.24.attn.o_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.24.attn.o_proj.MatMulNBits.qweight", + "qzeros": "model.layers.24.attn.o_proj.MatMulNBits.qzeros", + "scales": "model.layers.24.attn.o_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "k": 3072, + "n": 8192 + }, + "kind": "ssmlp", + "name": "model.layers.24.ssmlp", + "roles": { + "down_qweight": "model.layers.24.mlp.down_proj.MatMulNBits.qweight", + "down_qzeros": "model.layers.24.mlp.down_proj.MatMulNBits.qzeros", + "down_scales": "model.layers.24.mlp.down_proj.MatMulNBits.scales", + "gate_qweight": "model.layers.24.mlp.gate_proj.MatMulNBits.qweight", + "gate_qzeros": "model.layers.24.mlp.gate_proj.MatMulNBits.qzeros", + "gate_scales": "model.layers.24.mlp.gate_proj.MatMulNBits.scales", + "norm0": "model.layers.24.post_attention_layernorm.weight", + "norm1": "model.layers.25.input_layernorm.weight", + "up_qweight": "model.layers.24.mlp.up_proj.MatMulNBits.qweight", + "up_qzeros": "model.layers.24.mlp.up_proj.MatMulNBits.qzeros", + "up_scales": "model.layers.24.mlp.up_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.25.attn.q_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.25.attn.q_proj.MatMulNBits.qweight", + "qzeros": "model.layers.25.attn.q_proj.MatMulNBits.qzeros", + "scales": "model.layers.25.attn.q_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.25.attn.k_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.25.attn.k_proj.MatMulNBits.qweight", + "qzeros": "model.layers.25.attn.k_proj.MatMulNBits.qzeros", + "scales": "model.layers.25.attn.k_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.25.attn.v_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.25.attn.v_proj.MatMulNBits.qweight", + "qzeros": "model.layers.25.attn.v_proj.MatMulNBits.qzeros", + "scales": "model.layers.25.attn.v_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.25.attn.o_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.25.attn.o_proj.MatMulNBits.qweight", + "qzeros": "model.layers.25.attn.o_proj.MatMulNBits.qzeros", + "scales": "model.layers.25.attn.o_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "k": 3072, + "n": 8192 + }, + "kind": "ssmlp", + "name": "model.layers.25.ssmlp", + "roles": { + "down_qweight": "model.layers.25.mlp.down_proj.MatMulNBits.qweight", + "down_qzeros": "model.layers.25.mlp.down_proj.MatMulNBits.qzeros", + "down_scales": "model.layers.25.mlp.down_proj.MatMulNBits.scales", + "gate_qweight": "model.layers.25.mlp.gate_proj.MatMulNBits.qweight", + "gate_qzeros": "model.layers.25.mlp.gate_proj.MatMulNBits.qzeros", + "gate_scales": "model.layers.25.mlp.gate_proj.MatMulNBits.scales", + "norm0": "model.layers.25.post_attention_layernorm.weight", + "norm1": "model.layers.26.input_layernorm.weight", + "up_qweight": "model.layers.25.mlp.up_proj.MatMulNBits.qweight", + "up_qzeros": "model.layers.25.mlp.up_proj.MatMulNBits.qzeros", + "up_scales": "model.layers.25.mlp.up_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.26.attn.q_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.26.attn.q_proj.MatMulNBits.qweight", + "qzeros": "model.layers.26.attn.q_proj.MatMulNBits.qzeros", + "scales": "model.layers.26.attn.q_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.26.attn.k_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.26.attn.k_proj.MatMulNBits.qweight", + "qzeros": "model.layers.26.attn.k_proj.MatMulNBits.qzeros", + "scales": "model.layers.26.attn.k_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.26.attn.v_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.26.attn.v_proj.MatMulNBits.qweight", + "qzeros": "model.layers.26.attn.v_proj.MatMulNBits.qzeros", + "scales": "model.layers.26.attn.v_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.26.attn.o_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.26.attn.o_proj.MatMulNBits.qweight", + "qzeros": "model.layers.26.attn.o_proj.MatMulNBits.qzeros", + "scales": "model.layers.26.attn.o_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "k": 3072, + "n": 8192 + }, + "kind": "ssmlp", + "name": "model.layers.26.ssmlp", + "roles": { + "down_qweight": "model.layers.26.mlp.down_proj.MatMulNBits.qweight", + "down_qzeros": "model.layers.26.mlp.down_proj.MatMulNBits.qzeros", + "down_scales": "model.layers.26.mlp.down_proj.MatMulNBits.scales", + "gate_qweight": "model.layers.26.mlp.gate_proj.MatMulNBits.qweight", + "gate_qzeros": "model.layers.26.mlp.gate_proj.MatMulNBits.qzeros", + "gate_scales": "model.layers.26.mlp.gate_proj.MatMulNBits.scales", + "norm0": "model.layers.26.post_attention_layernorm.weight", + "norm1": "model.layers.27.input_layernorm.weight", + "up_qweight": "model.layers.26.mlp.up_proj.MatMulNBits.qweight", + "up_qzeros": "model.layers.26.mlp.up_proj.MatMulNBits.qzeros", + "up_scales": "model.layers.26.mlp.up_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.27.attn.q_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.27.attn.q_proj.MatMulNBits.qweight", + "qzeros": "model.layers.27.attn.q_proj.MatMulNBits.qzeros", + "scales": "model.layers.27.attn.q_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.27.attn.k_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.27.attn.k_proj.MatMulNBits.qweight", + "qzeros": "model.layers.27.attn.k_proj.MatMulNBits.qzeros", + "scales": "model.layers.27.attn.k_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.27.attn.v_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.27.attn.v_proj.MatMulNBits.qweight", + "qzeros": "model.layers.27.attn.v_proj.MatMulNBits.qzeros", + "scales": "model.layers.27.attn.v_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.27.attn.o_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.27.attn.o_proj.MatMulNBits.qweight", + "qzeros": "model.layers.27.attn.o_proj.MatMulNBits.qzeros", + "scales": "model.layers.27.attn.o_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "k": 3072, + "n": 8192 + }, + "kind": "ssmlp", + "name": "model.layers.27.ssmlp", + "roles": { + "down_qweight": "model.layers.27.mlp.down_proj.MatMulNBits.qweight", + "down_qzeros": "model.layers.27.mlp.down_proj.MatMulNBits.qzeros", + "down_scales": "model.layers.27.mlp.down_proj.MatMulNBits.scales", + "gate_qweight": "model.layers.27.mlp.gate_proj.MatMulNBits.qweight", + "gate_qzeros": "model.layers.27.mlp.gate_proj.MatMulNBits.qzeros", + "gate_scales": "model.layers.27.mlp.gate_proj.MatMulNBits.scales", + "norm0": "model.layers.27.post_attention_layernorm.weight", + "norm1": "model.layers.28.input_layernorm.weight", + "up_qweight": "model.layers.27.mlp.up_proj.MatMulNBits.qweight", + "up_qzeros": "model.layers.27.mlp.up_proj.MatMulNBits.qzeros", + "up_scales": "model.layers.27.mlp.up_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.28.attn.q_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.28.attn.q_proj.MatMulNBits.qweight", + "qzeros": "model.layers.28.attn.q_proj.MatMulNBits.qzeros", + "scales": "model.layers.28.attn.q_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.28.attn.k_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.28.attn.k_proj.MatMulNBits.qweight", + "qzeros": "model.layers.28.attn.k_proj.MatMulNBits.qzeros", + "scales": "model.layers.28.attn.k_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.28.attn.v_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.28.attn.v_proj.MatMulNBits.qweight", + "qzeros": "model.layers.28.attn.v_proj.MatMulNBits.qzeros", + "scales": "model.layers.28.attn.v_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.28.attn.o_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.28.attn.o_proj.MatMulNBits.qweight", + "qzeros": "model.layers.28.attn.o_proj.MatMulNBits.qzeros", + "scales": "model.layers.28.attn.o_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "k": 3072, + "n": 8192 + }, + "kind": "ssmlp", + "name": "model.layers.28.ssmlp", + "roles": { + "down_qweight": "model.layers.28.mlp.down_proj.MatMulNBits.qweight", + "down_qzeros": "model.layers.28.mlp.down_proj.MatMulNBits.qzeros", + "down_scales": "model.layers.28.mlp.down_proj.MatMulNBits.scales", + "gate_qweight": "model.layers.28.mlp.gate_proj.MatMulNBits.qweight", + "gate_qzeros": "model.layers.28.mlp.gate_proj.MatMulNBits.qzeros", + "gate_scales": "model.layers.28.mlp.gate_proj.MatMulNBits.scales", + "norm0": "model.layers.28.post_attention_layernorm.weight", + "norm1": "model.layers.29.input_layernorm.weight", + "up_qweight": "model.layers.28.mlp.up_proj.MatMulNBits.qweight", + "up_qzeros": "model.layers.28.mlp.up_proj.MatMulNBits.qzeros", + "up_scales": "model.layers.28.mlp.up_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.29.attn.q_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.29.attn.q_proj.MatMulNBits.qweight", + "qzeros": "model.layers.29.attn.q_proj.MatMulNBits.qzeros", + "scales": "model.layers.29.attn.q_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.29.attn.k_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.29.attn.k_proj.MatMulNBits.qweight", + "qzeros": "model.layers.29.attn.k_proj.MatMulNBits.qzeros", + "scales": "model.layers.29.attn.k_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.29.attn.v_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.29.attn.v_proj.MatMulNBits.qweight", + "qzeros": "model.layers.29.attn.v_proj.MatMulNBits.qzeros", + "scales": "model.layers.29.attn.v_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.29.attn.o_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.29.attn.o_proj.MatMulNBits.qweight", + "qzeros": "model.layers.29.attn.o_proj.MatMulNBits.qzeros", + "scales": "model.layers.29.attn.o_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "k": 3072, + "n": 8192 + }, + "kind": "ssmlp", + "name": "model.layers.29.ssmlp", + "roles": { + "down_qweight": "model.layers.29.mlp.down_proj.MatMulNBits.qweight", + "down_qzeros": "model.layers.29.mlp.down_proj.MatMulNBits.qzeros", + "down_scales": "model.layers.29.mlp.down_proj.MatMulNBits.scales", + "gate_qweight": "model.layers.29.mlp.gate_proj.MatMulNBits.qweight", + "gate_qzeros": "model.layers.29.mlp.gate_proj.MatMulNBits.qzeros", + "gate_scales": "model.layers.29.mlp.gate_proj.MatMulNBits.scales", + "norm0": "model.layers.29.post_attention_layernorm.weight", + "norm1": "model.layers.30.input_layernorm.weight", + "up_qweight": "model.layers.29.mlp.up_proj.MatMulNBits.qweight", + "up_qzeros": "model.layers.29.mlp.up_proj.MatMulNBits.qzeros", + "up_scales": "model.layers.29.mlp.up_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.30.attn.q_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.30.attn.q_proj.MatMulNBits.qweight", + "qzeros": "model.layers.30.attn.q_proj.MatMulNBits.qzeros", + "scales": "model.layers.30.attn.q_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.30.attn.k_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.30.attn.k_proj.MatMulNBits.qweight", + "qzeros": "model.layers.30.attn.k_proj.MatMulNBits.qzeros", + "scales": "model.layers.30.attn.k_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.30.attn.v_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.30.attn.v_proj.MatMulNBits.qweight", + "qzeros": "model.layers.30.attn.v_proj.MatMulNBits.qzeros", + "scales": "model.layers.30.attn.v_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.30.attn.o_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.30.attn.o_proj.MatMulNBits.qweight", + "qzeros": "model.layers.30.attn.o_proj.MatMulNBits.qzeros", + "scales": "model.layers.30.attn.o_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "k": 3072, + "n": 8192 + }, + "kind": "ssmlp", + "name": "model.layers.30.ssmlp", + "roles": { + "down_qweight": "model.layers.30.mlp.down_proj.MatMulNBits.qweight", + "down_qzeros": "model.layers.30.mlp.down_proj.MatMulNBits.qzeros", + "down_scales": "model.layers.30.mlp.down_proj.MatMulNBits.scales", + "gate_qweight": "model.layers.30.mlp.gate_proj.MatMulNBits.qweight", + "gate_qzeros": "model.layers.30.mlp.gate_proj.MatMulNBits.qzeros", + "gate_scales": "model.layers.30.mlp.gate_proj.MatMulNBits.scales", + "norm0": "model.layers.30.post_attention_layernorm.weight", + "norm1": "model.layers.31.input_layernorm.weight", + "up_qweight": "model.layers.30.mlp.up_proj.MatMulNBits.qweight", + "up_qzeros": "model.layers.30.mlp.up_proj.MatMulNBits.qzeros", + "up_scales": "model.layers.30.mlp.up_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.31.attn.q_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.31.attn.q_proj.MatMulNBits.qweight", + "qzeros": "model.layers.31.attn.q_proj.MatMulNBits.qzeros", + "scales": "model.layers.31.attn.q_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.31.attn.k_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.31.attn.k_proj.MatMulNBits.qweight", + "qzeros": "model.layers.31.attn.k_proj.MatMulNBits.qzeros", + "scales": "model.layers.31.attn.k_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.31.attn.v_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.31.attn.v_proj.MatMulNBits.qweight", + "qzeros": "model.layers.31.attn.v_proj.MatMulNBits.qzeros", + "scales": "model.layers.31.attn.v_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.31.attn.o_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.31.attn.o_proj.MatMulNBits.qweight", + "qzeros": "model.layers.31.attn.o_proj.MatMulNBits.qzeros", + "scales": "model.layers.31.attn.o_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "k": 3072, + "n": 8192 + }, + "kind": "ssmlp", + "name": "model.layers.31.ssmlp", + "roles": { + "down_qweight": "model.layers.31.mlp.down_proj.MatMulNBits.qweight", + "down_qzeros": "model.layers.31.mlp.down_proj.MatMulNBits.qzeros", + "down_scales": "model.layers.31.mlp.down_proj.MatMulNBits.scales", + "gate_qweight": "model.layers.31.mlp.gate_proj.MatMulNBits.qweight", + "gate_qzeros": "model.layers.31.mlp.gate_proj.MatMulNBits.qzeros", + "gate_scales": "model.layers.31.mlp.gate_proj.MatMulNBits.scales", + "norm0": "model.layers.31.post_attention_layernorm.weight", + "norm1": "model.layers.32.final_norm_layernorm.weight", + "up_qweight": "model.layers.31.mlp.up_proj.MatMulNBits.qweight", + "up_qzeros": "model.layers.31.mlp.up_proj.MatMulNBits.qzeros", + "up_scales": "model.layers.31.mlp.up_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 200064 + }, + "kind": "matmul", + "name": "lm_head.MatMulNBits", + "roles": { + "qweight": "lm_head.MatMulNBits.qweight", + "qzeros": "lm_head.MatMulNBits.qzeros", + "scales": "lm_head.MatMulNBits.scales" + } + } + ] +} diff --git a/src/model_overlays/phi4-mini-it-aie4/provenance.json b/src/model_overlays/phi4-mini-it-aie4/provenance.json new file mode 100644 index 00000000..430229aa --- /dev/null +++ b/src/model_overlays/phi4-mini-it-aie4/provenance.json @@ -0,0 +1,117 @@ +{ + "generated": { + "config.json": { + "sha256": "1b3e74125a109c05f53c8383def18359d8581619f998c4f91b3b5bb78bf2919f", + "size": 257 + }, + "corelib_phi4_manifest.json": { + "sha256": "09cee6efafc513a2048c89e75b40d92d096144f0eed3b2459c138032b69dc045", + "size": 274527 + }, + "tokenizer_config.json": { + "sha256": "274d22c3cd28c042f28a681832722536663b06e8c9e88b6616622dceab922ca6", + "size": 3036 + } + }, + "upstream": { + "commit": "e751fb68c2cfffe6b0d32942118f75ac0a0365bb", + "git_files": [ + { + "oid": "f78df3bfb43291872abf78496110e5856b24de73", + "path": ".gitattributes", + "size": 1622, + "type": "file" + }, + { + "oid": "77e6ed402865af3c38b5e44c2d0be68f76f0784f", + "path": "added_tokens.json", + "size": 261, + "type": "file" + }, + { + "oid": "a9c00dd9bbd97e117371168e9d62af65b9f0e725", + "path": "chat_template.jinja", + "size": 423, + "type": "file" + }, + { + "oid": "55b698edc57fe52593dce4423beb51ded982dc96", + "path": "genai_config.json", + "size": 1720, + "type": "file" + }, + { + "oid": "dcecc4524288b351bbd0da8028e74e9b5bcdb9b5", + "path": "merges.txt", + "size": 2418348, + "type": "file" + }, + { + "lfs": { + "oid": "e80b9d83e018784eda6263d09fa2ab7729087722c6073c72123366f2dbec4529", + "pointerSize": 131, + "size": 378325 + }, + "oid": "df6d4309745d4627b82fd92c615589193ea528db", + "path": "model.onnx", + "size": 378325, + "type": "file" + }, + { + "lfs": { + "oid": "c48fd647beb02866e68d6dd1fbc05809a439283501d8abb7f6d6b950f92b7b60", + "pointerSize": 135, + "size": 3248488448 + }, + "oid": "5ef8f7f74cf6ac6411b8918e8a850bd2c769dd45", + "path": "model.onnx.data", + "size": 3248488448, + "type": "file" + }, + { + "oid": "18eba67aba3ef71b01ed13c16b3feed78e83001f", + "path": "special_tokens_map.json", + "size": 617, + "type": "file" + }, + { + "lfs": { + "oid": "382cc235b56c725945e149cc25f191da667c836655efd0857b004320e90e91ea", + "pointerSize": 133, + "size": 15524095 + }, + "oid": "3a12dacd8e86802d0229d810d3cc69ab548adf0a", + "path": "tokenizer.json", + "size": 15524095, + "type": "file" + }, + { + "oid": "c182c54743fe1735b0d5eb3959b9757a160879e8", + "path": "tokenizer_config.json", + "size": 2654, + "type": "file" + }, + { + "oid": "ea953a43348cdb3776cb7fd9ea02e3784febde34", + "path": "vocab.json", + "size": 3910310, + "type": "file" + } + ], + "inputs": { + "chat_template.jinja": { + "sha256": "febf589225c9728ab791f52e8897d7607a823d45368f0a4c92fa68997b40cce9", + "size": 423 + }, + "genai_config.json": { + "sha256": "8b7206f5f94e84cc0f80d90458cbb6a77caa61ce97e2d694661b9ed0286c820b", + "size": 1720 + }, + "tokenizer_config.json": { + "sha256": "35aab11f13510ccb463fe7e2e54b86eb29406e66b801a66e90e85b89e6da988c", + "size": 2654 + } + }, + "repository": "https://huggingface.co/amd/phi-4-mini-instruct-oga-dml" + } +} diff --git a/src/model_overlays/phi4-mini-it-aie4/tokenizer_config.json b/src/model_overlays/phi4-mini-it-aie4/tokenizer_config.json new file mode 100644 index 00000000..6675e217 --- /dev/null +++ b/src/model_overlays/phi4-mini-it-aie4/tokenizer_config.json @@ -0,0 +1,117 @@ +{ + "add_bos_token": false, + "add_eos_token": false, + "add_prefix_space": false, + "added_tokens_decoder": { + "199999": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "200018": { + "content": "<|endofprompt|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "200019": { + "content": "<|assistant|>", + "lstrip": false, + "normalized": false, + "rstrip": true, + "single_word": false, + "special": true + }, + "200020": { + "content": "<|end|>", + "lstrip": false, + "normalized": false, + "rstrip": true, + "single_word": false, + "special": true + }, + "200021": { + "content": "<|user|>", + "lstrip": false, + "normalized": false, + "rstrip": true, + "single_word": false, + "special": true + }, + "200022": { + "content": "<|system|>", + "lstrip": false, + "normalized": false, + "rstrip": true, + "single_word": false, + "special": true + }, + "200023": { + "content": "<|tool|>", + "lstrip": false, + "normalized": false, + "rstrip": true, + "single_word": false, + "special": false + }, + "200024": { + "content": "<|/tool|>", + "lstrip": false, + "normalized": false, + "rstrip": true, + "single_word": false, + "special": false + }, + "200025": { + "content": "<|tool_call|>", + "lstrip": false, + "normalized": false, + "rstrip": true, + "single_word": false, + "special": false + }, + "200026": { + "content": "<|/tool_call|>", + "lstrip": false, + "normalized": false, + "rstrip": true, + "single_word": false, + "special": false + }, + "200027": { + "content": "<|tool_response|>", + "lstrip": false, + "normalized": false, + "rstrip": true, + "single_word": false, + "special": false + }, + "200028": { + "content": "<|tag|>", + "lstrip": false, + "normalized": false, + "rstrip": true, + "single_word": false, + "special": true + } + }, + "bos_token": "<|endoftext|>", + "chat_template": "{% for message in messages %}{% if message['role'] == 'system' and 'tools' in message and message['tools'] is not none %}{{ '<|' + message['role'] + '|>' + message['content'] + '<|tool|>' + message['tools'] + '<|/tool|>' + '<|end|>' }}{% else %}{{ '<|' + message['role'] + '|>' + message['content'] + '<|end|>' }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>' }}{% else %}{{ eos_token }}{% endif %}", + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "eos_token_id": [ + 200020, + 199999 + ], + "extra_special_tokens": {}, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "padding_side": "left", + "tokenizer_class": "GPT2Tokenizer", + "unk_token": "<|endoftext|>" +} diff --git a/src/pull/download_model.cpp b/src/pull/download_model.cpp index 38ccf44a..b691dd38 100644 --- a/src/pull/download_model.cpp +++ b/src/pull/download_model.cpp @@ -5,6 +5,7 @@ /// \version 0.9.24 /// \note This class for curl download #include "download_model.hpp" +#include #include #include #include diff --git a/src/pull/download_model.hpp b/src/pull/download_model.hpp index 12e9a1a1..28351cc5 100644 --- a/src/pull/download_model.hpp +++ b/src/pull/download_model.hpp @@ -6,11 +6,12 @@ /// \note This class for curl download #pragma once +#include +#include #include #include #include #include -#include #include "nlohmann/json.hpp" namespace download_utils { diff --git a/src/pull/model_downloader.cpp b/src/pull/model_downloader.cpp index 24fc98d4..a5adfbdc 100644 --- a/src/pull/model_downloader.cpp +++ b/src/pull/model_downloader.cpp @@ -5,6 +5,7 @@ /// \version 0.9.24 /// \note This class is used to download models from the huggingface #include "model_downloader.hpp" +#include #include "utils/utils.hpp" #include "download_model.hpp" #include @@ -87,6 +88,15 @@ bool ModelDownloader::pull_model(const std::string& model_tag, bool use_modelsco try { // Get model info auto [new_model_tag, model_info] = supported_models.get_model_info(model_tag); + flm::pull::RequireSupportedModelSource( + model_info, + use_modelscope); + if (model_info.contains("bundled_overlays")) { + flm::pull::StageBundledOverlays( + model_info, + utils::find_model_overlay_root(), + supported_models.get_model_path(new_model_tag)); + } std::string model_name = model_info["name"]; std::string model_server = use_modelscope ? "ModelScope" : "HuggingFace"; @@ -292,10 +302,12 @@ std::pair ModelDownloader::build_download_list(const std: try { auto [new_model_tag, model_info] = supported_models.get_model_info(model_tag); + flm::pull::RequireSupportedModelSource(model_info, modelscope); std::string base_url = modelscope ? model_info["ms_url"] : model_info["url"]; std::string model_name = model_info["name"]; std::string file_url = model_info["file_url"]; - std::vector model_files = model_info["files"]; + std::vector model_files = + flm::pull::RemoteModelFiles(model_info); // Create model directory std::string model_path = supported_models.get_model_path(new_model_tag); @@ -334,6 +346,11 @@ std::pair ModelDownloader::build_download_list(const std: if (std::string(base_url).find("resolve") != std::string::npos) { // resolve provided , may from a specific branch url = base_url + "/" + filename + "?download=true"; } + else if (!modelscope) { + url = flm::pull::BuildRemoteFileUrl( + model_info, + filename); + } else { url = base_url + "/resolve/main/" + filename + "?download=true"; } @@ -421,6 +438,7 @@ bool ModelDownloader::remove_model(const std::string& model_tag, bool sub_proces /// \return true if all files are present and compatible, false otherwise bool ModelDownloader::check_model(const std::string& model_tag, bool use_modelscope, bool sub_process_mode) { auto [new_model_tag, model_info] = supported_models.get_model_info(model_tag); + flm::pull::RequireSupportedModelSource(model_info, use_modelscope); header_print("FLM", "Checking model: " + new_model_tag + "...\n"); ModelStatus status = is_model_downloaded(new_model_tag, sub_process_mode); @@ -455,6 +473,9 @@ bool ModelDownloader::verify_and_clean_files(const std::string& model_tag, bool bool any_error = false; try { auto [new_model_tag, model_info] = supported_models.get_model_info(model_tag); + flm::pull::RequireSupportedModelSource( + model_info, + use_modelscope); std::vector model_files = model_info["files"]; std::string model_path = supported_models.get_model_path(new_model_tag); std::string file_url = model_info["file_url"]; @@ -477,6 +498,33 @@ bool ModelDownloader::verify_and_clean_files(const std::string& model_tag, bool header_print("FLM", "Checking file: " + filename + "..."); } + std::string local_path = + get_model_file_path(model_path, filename); + if ( + model_info.contains("bundled_overlays") && + model_info["bundled_overlays"].contains(filename)) { + if (flm::pull::VerifyBundledOverlayTarget( + model_info, + filename, + model_path)) { + if (!sub_process_mode) { + header_print("FLM", "Success!"); + } + } else { + if (!sub_process_mode) { + header_print("FLM", "Fail!"); + header_print( + "FLM", + "Removing corrupted bundled overlay: " + + filename + "..."); + } + std::error_code ignored; + std::filesystem::remove(local_path, ignored); + any_error = true; + } + continue; + } + auto it = std::find_if( hf_model_infos.begin(), hf_model_infos.end(), @@ -488,7 +536,6 @@ bool ModelDownloader::verify_and_clean_files(const std::string& model_tag, bool continue; } const auto& file = *it; - std::string local_path = get_model_file_path(model_path, filename); // If the file isn't present locally, there's nothing to verify or // remove; treat as an error so the caller knows a re-pull is needed. diff --git a/src/pull/model_overlay.cpp b/src/pull/model_overlay.cpp new file mode 100644 index 00000000..b086dc9f --- /dev/null +++ b/src/pull/model_overlay.cpp @@ -0,0 +1,302 @@ +#include + +#include "picosha2.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +#include +#endif + +namespace flm::pull { +namespace { + +struct OverlayRecord final { + std::string target; + std::filesystem::path source; + std::uint64_t size; + std::string sha256; +}; + +bool IsSha256(std::string_view value) { + return value.size() == 64 && + std::all_of( + value.begin(), + value.end(), + [](unsigned char character) { + return std::isxdigit(character) != 0; + }); +} + +std::filesystem::path SafeRelativePath( + const std::string& value, + std::string_view field) { + std::filesystem::path path(value); + if ( + value.empty() || + path.is_absolute() || + path.has_root_name() || + path.has_root_directory()) { + throw std::invalid_argument( + std::string(field) + " must be a non-empty relative path"); + } + for (const auto& component : path) { + if (component == "..") { + throw std::invalid_argument( + std::string(field) + " must not contain path traversal"); + } + } + return path.lexically_normal(); +} + +std::vector ReadOverlayRecords( + const nlohmann::json& model_info, + const std::filesystem::path& overlay_root) { + const auto found = model_info.find("bundled_overlays"); + if (found == model_info.end()) { + return {}; + } + if (!found->is_object()) { + throw std::invalid_argument("bundled_overlays must be an object"); + } + + std::vector records; + records.reserve(found->size()); + for (const auto& [target, value] : found->items()) { + if (!value.is_object()) { + throw std::invalid_argument( + "bundled overlay record must be an object: " + target); + } + if ( + !value.contains("path") || + !value.contains("size") || + !value.contains("sha256") || + value.size() != 3) { + throw std::invalid_argument( + "bundled overlay record must contain path, size, and " + "sha256: " + + target); + } + const auto target_path = + SafeRelativePath(target, "bundled overlay target"); + if ( + target_path.parent_path() != std::filesystem::path() || + target_path.filename().string() != target) { + throw std::invalid_argument( + "bundled overlay target must be a model-root filename"); + } + const auto source_relative = SafeRelativePath( + value.at("path").get(), + "bundled overlay source"); + const auto size = value.at("size").get(); + std::string sha256 = value.at("sha256").get(); + if (!IsSha256(sha256)) { + throw std::invalid_argument( + "bundled overlay SHA-256 is invalid: " + target); + } + std::transform( + sha256.begin(), + sha256.end(), + sha256.begin(), + [](unsigned char character) { + return static_cast(std::tolower(character)); + }); + records.push_back( + OverlayRecord{ + target, + overlay_root / source_relative, + size, + std::move(sha256)}); + } + return records; +} + +void VerifySource(const OverlayRecord& record) { + std::error_code error; + const auto size = std::filesystem::file_size(record.source, error); + if (error) { + throw std::runtime_error( + "bundled overlay source is missing or unreadable: " + + record.source.string()); + } + if (size != record.size) { + throw std::runtime_error( + "bundled overlay size mismatch: " + + record.source.string()); + } + if (CalculateFileSha256(record.source) != record.sha256) { + throw std::runtime_error( + "bundled overlay SHA-256 mismatch: " + + record.source.string()); + } +} + +bool TargetMatches( + const OverlayRecord& record, + const std::filesystem::path& model_dir) { + const auto target = model_dir / record.target; + std::error_code error; + const auto size = std::filesystem::file_size(target, error); + return !error && + size == record.size && + CalculateFileSha256(target) == record.sha256; +} + +void CopyAtomically( + const OverlayRecord& record, + const std::filesystem::path& model_dir) { + static std::atomic sequence{0}; + const auto target = model_dir / record.target; + const auto temporary = + model_dir / + ("." + record.target + ".flm-overlay-" + +#ifdef _WIN32 + std::to_string(GetCurrentProcessId()) + "-" + +#else + std::string("process-") + +#endif + std::to_string( + sequence.fetch_add(1, std::memory_order_relaxed)) + + ".tmp"); + + std::error_code ignored; + std::filesystem::remove(temporary, ignored); + try { +#ifdef _WIN32 + if (!CopyFileW( + record.source.c_str(), + temporary.c_str(), + TRUE)) { + throw std::system_error( + static_cast(GetLastError()), + std::system_category(), + "failed to copy bundled overlay to temporary file"); + } +#else + std::filesystem::copy_file( + record.source, + temporary, + std::filesystem::copy_options::none); +#endif + OverlayRecord temporary_record = record; + temporary_record.source = temporary; + VerifySource(temporary_record); +#ifdef _WIN32 + if (!MoveFileExW( + temporary.c_str(), + target.c_str(), + MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH)) { + throw std::system_error( + static_cast(GetLastError()), + std::system_category(), + "failed to atomically install bundled overlay"); + } +#else + std::filesystem::rename(temporary, target); +#endif + } catch (...) { + std::filesystem::remove(temporary, ignored); + throw; + } +} + +} // namespace + +void RequireSupportedModelSource( + const nlohmann::json& model_info, + bool use_modelscope) { + if (!use_modelscope) { + return; + } + const auto found = model_info.find("modelscope_supported"); + if ( + found != model_info.end() && + (!found->is_boolean() || !found->get())) { + throw std::invalid_argument( + "this model does not support --modelscope; use the " + "Hugging Face source"); + } +} + +std::vector RemoteModelFiles( + const nlohmann::json& model_info) { + const auto overlays = model_info.find("bundled_overlays"); + std::vector files; + for (const auto& value : model_info.at("files")) { + const std::string filename = value.get(); + if ( + overlays == model_info.end() || + !overlays->is_object() || + !overlays->contains(filename)) { + files.push_back(filename); + } + } + return files; +} + +std::string BuildRemoteFileUrl( + const nlohmann::json& model_info, + std::string_view filename) { + const std::string base_url = model_info.at("url").get(); + if (base_url.find("resolve") != std::string::npos) { + return base_url + "/" + std::string(filename) + "?download=true"; + } + const std::string revision = + model_info.value("revision", std::string("main")); + return base_url + "/resolve/" + revision + "/" + + std::string(filename) + "?download=true"; +} + +std::string CalculateFileSha256(const std::filesystem::path& path) { + std::ifstream input(path, std::ios::binary); + if (!input) { + throw std::runtime_error( + "failed to open file for SHA-256: " + path.string()); + } + std::vector hash(picosha2::k_digest_size); + picosha2::hash256(input, hash.begin(), hash.end()); + return picosha2::bytes_to_hex_string(hash.begin(), hash.end()); +} + +void StageBundledOverlays( + const nlohmann::json& model_info, + const std::filesystem::path& overlay_root, + const std::filesystem::path& model_dir) { + const auto records = ReadOverlayRecords(model_info, overlay_root); + for (const auto& record : records) { + VerifySource(record); + } + if (records.empty()) { + return; + } + std::filesystem::create_directories(model_dir); + for (const auto& record : records) { + if (!TargetMatches(record, model_dir)) { + CopyAtomically(record, model_dir); + } + } +} + +bool VerifyBundledOverlayTarget( + const nlohmann::json& model_info, + std::string_view filename, + const std::filesystem::path& model_dir) { + const auto records = ReadOverlayRecords(model_info, {}); + const auto found = std::find_if( + records.begin(), + records.end(), + [&](const OverlayRecord& record) { + return record.target == filename; + }); + return found != records.end() && TargetMatches(*found, model_dir); +} + +} // namespace flm::pull diff --git a/src/test/phi4_corelib_aie4/CMakeLists.txt b/src/test/phi4_corelib_aie4/CMakeLists.txt index ea9a6590..aaafd307 100644 --- a/src/test/phi4_corelib_aie4/CMakeLists.txt +++ b/src/test/phi4_corelib_aie4/CMakeLists.txt @@ -84,6 +84,27 @@ function(add_corelib_host_test TEST_NAME TEST_SOURCE) endfunction() enable_testing() +add_executable(test_model_catalog + test_model_catalog.cpp + ${FASTFLOW_SOURCE_DIR}/pull/model_overlay.cpp) +target_include_directories(test_model_catalog PRIVATE + ${FASTFLOW_SOURCE_DIR}/include + ${FASTFLOW_SOURCE_DIR}/pull) +target_compile_definitions(test_model_catalog PRIVATE + FLM_TEST_SOURCE_DIR="${FASTFLOW_SOURCE_DIR}") +add_test(NAME test_model_catalog COMMAND test_model_catalog) + +add_library(test_model_downloader_compile OBJECT + ${FASTFLOW_SOURCE_DIR}/pull/model_downloader.cpp) +target_include_directories(test_model_downloader_compile PRIVATE + ${FASTFLOW_SOURCE_DIR}/include + ${FASTFLOW_SOURCE_DIR}/pull + ${FASTFLOW_SOURCE_DIR}/../third_party/tokenizers-cpp/include + ${BOOST_INCLUDE_DIR} + ${XRT_INCLUDE_DIR}) +target_compile_definitions(test_model_downloader_compile PRIVATE + __FLM_VERSION__="${FLM_VERSION}") + add_executable(test_generation_limit test_generation_limit.cpp ${FASTFLOW_SOURCE_DIR}/server/generation_limit.cpp diff --git a/src/test/phi4_corelib_aie4/test_model_catalog.cpp b/src/test/phi4_corelib_aie4/test_model_catalog.cpp new file mode 100644 index 00000000..1304b8b6 --- /dev/null +++ b/src/test/phi4_corelib_aie4/test_model_catalog.cpp @@ -0,0 +1,285 @@ +#include "test_support.hpp" + +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace { + +using nlohmann::json; + +constexpr std::string_view kTag = "phi4-mini-it-aie4:4b"; +constexpr std::string_view kCommit = + "e751fb68c2cfffe6b0d32942118f75ac0a0365bb"; +constexpr std::string_view kRepository = + "https://huggingface.co/amd/phi-4-mini-instruct-oga-dml"; + +json ReadJson(const std::filesystem::path& path) { + std::ifstream input(path, std::ios::binary); + if (!input) { + throw std::runtime_error("failed to open " + path.string()); + } + return json::parse(input); +} + +std::uint64_t FileSize(const std::filesystem::path& path) { + return static_cast( + std::filesystem::file_size(path)); +} + +std::set JsonStringSet(const json& values) { + std::set result; + for (const auto& value : values) { + result.insert(value.get()); + } + return result; +} + +class TempDirectory final { +public: + TempDirectory() { + const auto parent = std::filesystem::temp_directory_path(); + for (int attempt = 0; attempt < 100; ++attempt) { + path_ = parent / + ("flm-model-overlay-" + + std::to_string(GetCurrentProcessId()) + "-" + + std::to_string(GetTickCount64()) + "-" + + std::to_string(attempt)); + std::error_code error; + if (std::filesystem::create_directory(path_, error)) { + return; + } + } + throw std::runtime_error("failed to create temporary directory"); + } + + ~TempDirectory() { + std::error_code ignored; + std::filesystem::remove_all(path_, ignored); + } + + const std::filesystem::path& path() const noexcept { + return path_; + } + +private: + std::filesystem::path path_; +}; + +void TestCatalogContract() { + const std::filesystem::path source(FLM_TEST_SOURCE_DIR); + const json catalog = ReadJson(source / "model_list.json"); + const json metadata = ReadJson(source / "model_info.json"); + const auto& model = + catalog.at("models").at("phi4-mini-it-aie4").at("4b"); + + CHECK(model.at("url").get() == kRepository); + CHECK(model.at("revision").get() == kCommit); + const std::string expected_api = + "https://huggingface.co/api/models/amd/" + "phi-4-mini-instruct-oga-dml/tree/" + + std::string(kCommit) + + "?recursive=true&expand=false"; + CHECK(model.at("file_url") == expected_api); + CHECK(model.at("flm_min_version") == "1.0.4"); + CHECK(model.at("default_context_length") == 4096); + CHECK(model.at("max_prefill_len") == 4096); + CHECK(model.at("vlm") == false); + CHECK(model.at("modelscope_supported") == false); + CHECK(!model.contains("ms_url")); + CHECK(model.at("details").at("family") == "phi4"); + CHECK( + model.at("details").at("execution_backend") == + "corelib_aie4"); + + const std::set expected_files{ + ".gitattributes", + "added_tokens.json", + "chat_template.jinja", + "config.json", + "corelib_phi4_manifest.json", + "genai_config.json", + "merges.txt", + "model.onnx", + "model.onnx.data", + "special_tokens_map.json", + "tokenizer.json", + "tokenizer_config.json", + "vocab.json", + }; + CHECK(JsonStringSet(model.at("files")) == expected_files); + + const auto& overlays = model.at("bundled_overlays"); + CHECK(overlays.size() == 3); + CHECK(overlays.contains("config.json")); + CHECK(overlays.contains("corelib_phi4_manifest.json")); + CHECK(overlays.contains("tokenizer_config.json")); + + const std::filesystem::path overlay_root = + source / "model_overlays"; + std::uint64_t overlay_size = 0; + for (const auto& [target, record] : overlays.items()) { + const auto path = overlay_root / record.at("path").get(); + CHECK(std::filesystem::is_regular_file(path)); + CHECK(record.at("size").get() == FileSize(path)); + CHECK( + record.at("sha256") == + flm::pull::CalculateFileSha256(path)); + overlay_size += FileSize(path); + } + + const auto& remote = metadata.at(kTag); + CHECK(remote.size() == 11); + CHECK( + std::is_sorted( + remote.begin(), + remote.end(), + [](const json& left, const json& right) { + return left.at("path").get() < + right.at("path").get(); + })); + const auto find_record = [&](std::string_view path) -> const json& { + const auto found = std::find_if( + remote.begin(), + remote.end(), + [&](const json& record) { + return record.at("path").get() == path; + }); + CHECK(found != remote.end()); + return *found; + }; + CHECK( + find_record("model.onnx").at("oid") == + "df6d4309745d4627b82fd92c615589193ea528db"); + CHECK( + find_record("model.onnx").at("lfs").at("oid") == + "e80b9d83e018784eda6263d09fa2ab7729087722c6073c72123366f2dbec4529"); + CHECK( + find_record("model.onnx.data").at("lfs").at("size") == + UINT64_C(3248488448)); + CHECK( + find_record("tokenizer.json").at("lfs").at("oid") == + "382cc235b56c725945e149cc25f191da667c836655efd0857b004320e90e91ea"); + + constexpr std::uint64_t kRemoteLogicalBytes = UINT64_C(3270726823); + constexpr std::uint64_t kReplacedTokenizerConfigBytes = 2654; + const std::uint64_t expected_size = + kRemoteLogicalBytes - kReplacedTokenizerConfigBytes + overlay_size; + CHECK(model.at("size").get() == expected_size); + const double expected_footprint = + std::round( + static_cast(expected_size) / + static_cast(UINT64_C(1024) * 1024 * 1024) * + 100.0) / + 100.0; + CHECK(model.at("footprint").get() == expected_footprint); +} + +void TestSourcePolicyAndPinnedUrls() { + const std::filesystem::path source(FLM_TEST_SOURCE_DIR); + const json catalog = ReadJson(source / "model_list.json"); + const auto& model = + catalog.at("models").at("phi4-mini-it-aie4").at("4b"); + + flm::pull::RequireSupportedModelSource(model, false); + CheckThrowsContains( + [&] { flm::pull::RequireSupportedModelSource(model, true); }, + "does not support --modelscope"); + CHECK( + flm::pull::BuildRemoteFileUrl(model, "model.onnx") == + std::string(kRepository) + "/resolve/" + + std::string(kCommit) + "/model.onnx?download=true"); + + const auto remote_files = flm::pull::RemoteModelFiles(model); + CHECK( + std::find( + remote_files.begin(), + remote_files.end(), + "tokenizer_config.json") == remote_files.end()); + CHECK( + std::find( + remote_files.begin(), + remote_files.end(), + "model.onnx") != remote_files.end()); +} + +void TestBundledOverlayCopyAndIntegrity() { + const std::filesystem::path source(FLM_TEST_SOURCE_DIR); + const json catalog = ReadJson(source / "model_list.json"); + const auto& model = + catalog.at("models").at("phi4-mini-it-aie4").at("4b"); + const auto overlay_root = source / "model_overlays"; + + TempDirectory temporary; + const auto model_dir = temporary.path() / "model"; + flm::pull::StageBundledOverlays(model, overlay_root, model_dir); + + for (const auto& [target, record] : + model.at("bundled_overlays").items()) { + const auto installed = model_dir / target; + CHECK(std::filesystem::is_regular_file(installed)); + CHECK( + flm::pull::CalculateFileSha256(installed) == + record.at("sha256").get()); + } + + const auto corrupt_root = temporary.path() / "corrupt"; + std::filesystem::create_directories( + corrupt_root / "phi4-mini-it-aie4"); + for (const auto& [target, record] : + model.at("bundled_overlays").items()) { + std::filesystem::copy_file( + overlay_root / record.at("path").get(), + corrupt_root / record.at("path").get()); + } + { + std::ofstream corrupt( + corrupt_root / "phi4-mini-it-aie4" / "config.json", + std::ios::binary | std::ios::trunc); + corrupt << std::string( + model.at("bundled_overlays") + .at("config.json") + .at("size") + .get(), + 'x'); + } + CheckThrowsContains( + [&] { + flm::pull::StageBundledOverlays( + model, + corrupt_root, + temporary.path() / "rejected"); + }, + "bundled overlay SHA-256 mismatch"); + CHECK( + !std::filesystem::exists( + temporary.path() / "rejected" / "config.json")); +} + +} // namespace + +int main() { + try { + TestCatalogContract(); + TestSourcePolicyAndPinnedUrls(); + TestBundledOverlayCopyAndIntegrity(); + std::cout << "model catalog tests passed\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/src/test/phi4_corelib_aie4/test_packaged_runtime.ps1 b/src/test/phi4_corelib_aie4/test_packaged_runtime.ps1 new file mode 100644 index 00000000..f89b695a --- /dev/null +++ b/src/test/phi4_corelib_aie4/test_packaged_runtime.ps1 @@ -0,0 +1,314 @@ +param( + [string]$FlmExe = "", + [string]$CorelibRuntimeDir = "", + [string]$XrtRuntimeDir = "", + [string]$DependencyDir = "", + [switch]$RunAie4ModelLoad +) + +$ErrorActionPreference = "Stop" +$sourceRoot = (Resolve-Path (Join-Path $PSScriptRoot "../..")).Path +$modulePath = Join-Path $sourceRoot "cmake/ConfigureAie4Runtime.cmake" +$temporary = Join-Path ([System.IO.Path]::GetTempPath()) ( + "flm-aie4-package-{0}-{1}" -f $PID, [DateTime]::UtcNow.Ticks) + +function Write-Bytes { + param([string]$Path, [int]$Count) + [System.IO.File]::WriteAllBytes($Path, [byte[]]::new($Count)) +} + +function Invoke-Configure { + param( + [string]$Build, + [bool]$Enabled, + [string]$Corelib = "", + [string]$Xrt = "", + [string]$Dependency = "", + [bool]$ExpectSuccess = $true + ) + $arguments = @( + "-S", (Join-Path $temporary "fixture"), + "-B", $Build, + "-DFLM_ENABLE_CORELIB_AIE4=$( + if ($Enabled) { "ON" } else { "OFF" })" + ) + if ($Corelib) { + $arguments += "-DRYZENAI_CORELIB_RUNTIME_DIR=$Corelib" + } + if ($Xrt) { + $arguments += "-DXRT_RUNTIME_DIR=$Xrt" + } + if ($Dependency) { + $arguments += "-DFLM_AIE4_DEPENDENCY_DIRS=$Dependency" + } + $previousPreference = $ErrorActionPreference + $ErrorActionPreference = "Continue" + $output = (& cmake @arguments 2>&1 | + ForEach-Object { $_.ToString() } | + Out-String) + $exitCode = $LASTEXITCODE + $ErrorActionPreference = $previousPreference + $succeeded = $exitCode -eq 0 + if ($succeeded -ne $ExpectSuccess) { + throw "Unexpected CMake result.`n$output" + } + return $output +} + +try { + $inno = Get-Content (Join-Path $sourceRoot "inno/flm.iss") -Raw + foreach ($required in @( + 'AppVersion=1.0.4', + 'Name: "aie4runtime"', + 'Source: "aie4\*"', + 'corelib_phi4_manifest.json', + 'tokenizer_config.json', + 'config.json' + )) { + if ($inno -notmatch [regex]::Escape($required)) { + throw "Inno manifest is missing: $required" + } + } + $wix = Get-Content (Join-Path $sourceRoot "wix/flm.wxs") -Raw + foreach ($required in @( + 'Version="1.0.4"', + 'Feature Id="Aie4Feature"', + 'ComponentGroup Id="Aie4RuntimeComponents"', + 'ComponentGroup Id="Aie4OverlayComponents"', + 'corelib_phi4_manifest.json', + 'tokenizer_config.json', + 'config.json' + )) { + if ($wix -notmatch [regex]::Escape($required)) { + throw "WiX manifest is missing: $required" + } + } + + New-Item -ItemType Directory -Path $temporary | Out-Null + $fixture = Join-Path $temporary "fixture" + New-Item -ItemType Directory -Path $fixture | Out-Null + @" +cmake_minimum_required(VERSION 3.24) +project(flm_aie4_packaging NONE) +option(FLM_ENABLE_CORELIB_AIE4 "" OFF) +include("$($modulePath.Replace('\', '/'))") +flm_collect_aie4_runtime_files(_flm_aie4_files) +install(FILES `${_flm_aie4_files} DESTINATION bin/aie4) +"@ | Set-Content -Path (Join-Path $fixture "CMakeLists.txt") -Encoding utf8 + + $offBuild = Join-Path $temporary "off" + Invoke-Configure -Build $offBuild -Enabled $false | Out-Null + & cmake --install $offBuild --prefix (Join-Path $temporary "off-stage") + if ($LASTEXITCODE -ne 0) { + throw "Feature-OFF install failed" + } + if (Test-Path (Join-Path $temporary "off-stage/bin/aie4")) { + throw "Feature OFF unexpectedly staged an AIE4 directory" + } + + $missingOutput = Invoke-Configure ` + -Build (Join-Path $temporary "missing") ` + -Enabled $true ` + -ExpectSuccess $false + if ($missingOutput -notmatch "RYZENAI_CORELIB_RUNTIME_DIR") { + throw "Missing runtime path failure was not actionable" + } + + $corelib = Join-Path $temporary "corelib" + $xrt = Join-Path $temporary "xrt" + New-Item -ItemType Directory -Path $corelib, $xrt | Out-Null + foreach ($name in @( + "ryzenai_corelib.dll", + "ryzen_mm.dll", + "dyn_bins.dll", + "spdlog.dll", + "libprotobuf.dll", + "fmt.dll", + "zlib.dll", + "zlib1.dll", + "libutf8_validity.dll", + "abseil_dll.dll" + )) { + Write-Bytes -Path (Join-Path $corelib $name) -Count 17 + } + foreach ($name in @( + "xrt_coreutil.dll", + "xrt_core.dll", + "xrt_umddml.dll", + "xdp_native_plugin.dll" + )) { + Write-Bytes -Path (Join-Path $xrt $name) -Count 19 + } + + $onBuild = Join-Path $temporary "on" + Invoke-Configure ` + -Build $onBuild ` + -Enabled $true ` + -Corelib $corelib ` + -Xrt $xrt | Out-Null + $stage = Join-Path $temporary "on-stage" + & cmake --install $onBuild --prefix $stage + if ($LASTEXITCODE -ne 0) { + throw "Feature-ON install failed" + } + $actual = @( + Get-ChildItem (Join-Path $stage "bin/aie4") -File | + ForEach-Object Name | + Sort-Object + ) + $expected = @( + "dyn_bins.dll", + "abseil_dll.dll", + "fmt.dll", + "libprotobuf.dll", + "libutf8_validity.dll", + "ryzen_mm.dll", + "ryzenai_corelib.dll", + "spdlog.dll", + "xdp_native_plugin.dll", + "xrt_core.dll", + "xrt_coreutil.dll", + "xrt_umddml.dll", + "zlib.dll", + "zlib1.dll" + ) | Sort-Object + if (Compare-Object $expected $actual) { + throw "Installed AIE4 closure did not match the collected files" + } + + if ($CorelibRuntimeDir -or $XrtRuntimeDir -or $DependencyDir) { + if ( + -not $CorelibRuntimeDir -or + -not $XrtRuntimeDir -or + -not $DependencyDir + ) { + throw "Real closure check requires all three runtime directories" + } + $realBuild = Join-Path $temporary "real" + Invoke-Configure ` + -Build $realBuild ` + -Enabled $true ` + -Corelib (Resolve-Path $CorelibRuntimeDir).Path ` + -Xrt (Resolve-Path $XrtRuntimeDir).Path ` + -Dependency (Resolve-Path $DependencyDir).Path | Out-Null + $realStage = Join-Path $temporary "real-stage" + & cmake --install $realBuild --prefix $realStage + if ($LASTEXITCODE -ne 0) { + throw "Real AIE4 closure staging failed" + } + + Add-Type @" +using System; +using System.Runtime.InteropServices; +public static class FlmAie4Loader { + [DllImport("kernel32", CharSet = CharSet.Unicode, SetLastError = true)] + public static extern IntPtr LoadLibraryEx( + string path, IntPtr file, uint flags); + [DllImport("kernel32", SetLastError = true)] + public static extern bool FreeLibrary(IntPtr module); +} +"@ + $savedPathForLoad = $env:PATH + $savedCorelibForLoad = $env:RYZENAI_CORELIB_PATH + $savedXrtForLoad = $env:XILINX_XRT + try { + $env:PATH = "$env:SystemRoot\System32;$env:SystemRoot" + Remove-Item Env:RYZENAI_CORELIB_PATH ` + -ErrorAction SilentlyContinue + Remove-Item Env:XILINX_XRT -ErrorAction SilentlyContinue + $library = Join-Path ` + $realStage ` + "bin/aie4/ryzenai_corelib.dll" + $module = [FlmAie4Loader]::LoadLibraryEx( + $library, + [IntPtr]::Zero, + 0x00000100 -bor 0x00001000) + if ($module -eq [IntPtr]::Zero) { + $errorCode = [Runtime.InteropServices.Marshal]:: + GetLastWin32Error() + throw "Clean-environment corelib load failed: $errorCode" + } + [FlmAie4Loader]::FreeLibrary($module) | Out-Null + } finally { + $env:PATH = $savedPathForLoad + if ($null -eq $savedCorelibForLoad) { + Remove-Item Env:RYZENAI_CORELIB_PATH ` + -ErrorAction SilentlyContinue + } else { + $env:RYZENAI_CORELIB_PATH = $savedCorelibForLoad + } + if ($null -eq $savedXrtForLoad) { + Remove-Item Env:XILINX_XRT ` + -ErrorAction SilentlyContinue + } else { + $env:XILINX_XRT = $savedXrtForLoad + } + } + } + + if ($FlmExe) { + $resolvedFlm = (Resolve-Path $FlmExe).Path + $imports = (& dumpbin /nologo /dependents $resolvedFlm | Out-String) + if ($imports -match "(?im)^\s*ryzenai_corelib\.dll\s*$") { + throw "flm.exe has an unexpected ryzenai_corelib import" + } + + $savedPath = $env:PATH + $savedCorelib = $env:RYZENAI_CORELIB_PATH + $savedXrt = $env:XILINX_XRT + try { + Remove-Item Env:RYZENAI_CORELIB_PATH -ErrorAction SilentlyContinue + Remove-Item Env:XILINX_XRT -ErrorAction SilentlyContinue + $env:PATH = "$env:SystemRoot\System32;$env:SystemRoot" + $validation = (& $resolvedFlm validate --json 2>&1 | Out-String) + if ($LASTEXITCODE -ne 0) { + throw "Clean-environment flm validate failed.`n$validation" + } + if ($RunAie4ModelLoad) { + $smoke = "/bye`r`n" | + & $resolvedFlm run phi4-mini-it-aie4:4b 2>&1 | + Out-String + if ( + $LASTEXITCODE -ne 0 -or + $smoke -notmatch "phi4-mini-it-aie4" + ) { + throw "Clean-environment AIE4 model load failed.`n$smoke" + } + } + } finally { + $env:PATH = $savedPath + if ($null -eq $savedCorelib) { + Remove-Item Env:RYZENAI_CORELIB_PATH ` + -ErrorAction SilentlyContinue + } else { + $env:RYZENAI_CORELIB_PATH = $savedCorelib + } + if ($null -eq $savedXrt) { + Remove-Item Env:XILINX_XRT -ErrorAction SilentlyContinue + } else { + $env:XILINX_XRT = $savedXrt + } + } + + $portable = Join-Path $temporary "without-aie4" + Copy-Item ` + -Path (Split-Path $resolvedFlm -Parent) ` + -Destination $portable ` + -Recurse + Remove-Item (Join-Path $portable "aie4") ` + -Recurse ` + -Force ` + -ErrorAction SilentlyContinue + & (Join-Path $portable (Split-Path $resolvedFlm -Leaf)) list | + Out-Null + if ($LASTEXITCODE -ne 0) { + throw "Non-AIE4 command failed without the AIE4 directory" + } + } + + Write-Output "packaged runtime tests passed" +} finally { + if (Test-Path $temporary) { + Remove-Item $temporary -Recurse -Force + } +} diff --git a/src/test/phi4_corelib_aie4/test_phi4_manifest.cpp b/src/test/phi4_corelib_aie4/test_phi4_manifest.cpp index d9600234..427c8eda 100644 --- a/src/test/phi4_corelib_aie4/test_phi4_manifest.cpp +++ b/src/test/phi4_corelib_aie4/test_phi4_manifest.cpp @@ -783,6 +783,41 @@ void TestValidMappingAndExplicitRoles( "missing initializer"); } +void TestOgaQuantizedLayoutsAccepted( + const SyntheticPackage& fixture, + const std::shared_ptr& api) { + json manifest = fixture.manifest(); + for (auto& [name, initializer] : + manifest["initializers"].items()) { + if (name.ends_with(".qweight")) { + const auto logical = + initializer["shape"].get>(); + CHECK(logical.size() == 2); + CHECK(logical[1] % 64 == 0); + initializer["shape"] = { + logical[0], + logical[1] / 64, + 64}; + } else if ( + name.ends_with(".scales") || + name.ends_with(".qzeros")) { + const auto logical = + initializer["shape"].get>(); + CHECK(logical.size() == 2); + initializer["shape"] = {logical[0] * logical[1]}; + } + } + fixture.Write(manifest); + + auto package = Phi4Package::Load(fixture.path(), api, false); + CHECK(package.weight_objects().size() == 161); + CHECK( + package.Require( + "model.layers.0.attn.q_proj.MatMulNBits.qweight") + .shape == + std::vector({3072, 24, 64})); +} + void TestMappedOwnerOutlivesPackage( const SyntheticPackage& fixture, const std::shared_ptr& api) { @@ -1285,6 +1320,7 @@ int main() { SyntheticPackage fixture; auto api = ResolveRecordingCorelib(); TestValidMappingAndExplicitRoles(fixture, api); + TestOgaQuantizedLayoutsAccepted(fixture, api); TestMappedOwnerOutlivesPackage(fixture, api); TestPathRangeAndHashRejections(fixture, api); TestWeightObjectRejections(fixture, api); diff --git a/src/wix/flm.wxs b/src/wix/flm.wxs index bbd49a8d..fecca26b 100644 --- a/src/wix/flm.wxs +++ b/src/wix/flm.wxs @@ -18,12 +18,14 @@ - + + + + + + + + + + + @@ -69,6 +79,11 @@ + + + + + @@ -219,6 +234,25 @@ + + + + + + + + + + + + + + + + + + + diff --git a/src/wix/get_files.bat b/src/wix/get_files.bat index a04b986f..aea21667 100644 --- a/src/wix/get_files.bat +++ b/src/wix/get_files.bat @@ -22,4 +22,12 @@ echo Copying static assets... copy "..\inno\logo.ico" "package\logo.ico" copy "..\inno\terms.rtf" "package\terms.rtf" +REM Copy the validated optional AIE4 runtime closure +if not exist "..\build\aie4\ryzenai_corelib.dll" ( + echo ERROR: Build src with FLM_ENABLE_CORELIB_AIE4=ON before packaging. + exit /b 1 +) +if not exist "package\aie4" mkdir "package\aie4" +xcopy "..\build\aie4\*" "package\aie4\" /E /I /Y + echo Done! diff --git a/tools/generate_phi4_corelib_manifest.py b/tools/generate_phi4_corelib_manifest.py index f2431d18..6b2bbcc4 100644 --- a/tools/generate_phi4_corelib_manifest.py +++ b/tools/generate_phi4_corelib_manifest.py @@ -3,6 +3,7 @@ import argparse import hashlib import json +import re from pathlib import Path, PurePosixPath, PureWindowsPath import onnx @@ -452,10 +453,30 @@ def _validate_contract( shape = [int(dimension) for dimension in tensor.dims] if "shape" in contract: expected_shape = contract["shape"] - if shape != expected_shape: + accepted_shapes = [expected_shape] + role = contract["role"] + if ( + role.endswith(".qweight") + and len(expected_shape) == 2 + and expected_shape[1] % 64 == 0 + ): + accepted_shapes.append( + [expected_shape[0], expected_shape[1] // 64, 64] + ) + elif ( + len(expected_shape) == 2 + and ( + role.endswith(".scales") + or role.endswith(".qzeros") + ) + ): + accepted_shapes.append( + [expected_shape[0] * expected_shape[1]] + ) + if shape not in accepted_shapes: raise ValueError( - f"{initializer}: shape {shape} does not match " - f"{expected_shape}" + f"{initializer}: shape {shape} does not match any " + f"accepted ONNX layout {accepted_shapes}" ) else: rank = contract.get("rank") @@ -489,13 +510,79 @@ def _sha256(path: Path) -> str: return digest.hexdigest() -def _file_record(path: Path, full_hash: bool) -> dict[str, object]: - size = path.stat().st_size +def _verified_file_metadata( + path: Path, + metadata: dict[str, object], +) -> tuple[int, str]: + if set(metadata) != {"size", "sha256"}: + raise ValueError( + f"verified file metadata must contain size and sha256: {path}" + ) + size = metadata["size"] + sha256 = metadata["sha256"] + if ( + isinstance(size, bool) + or not isinstance(size, int) + or size < 0 + or size > MAX_U64 + ): + raise ValueError(f"verified file size exceeds uint64: {path}") + if ( + not isinstance(sha256, str) + or re.fullmatch(r"[0-9a-fA-F]{64}", sha256) is None + ): + raise ValueError(f"verified file SHA-256 is invalid: {path}") + return size, sha256.lower() + + +def _lfs_pointer_metadata(path: Path) -> tuple[int, str] | None: + if path.stat().st_size > 1024: + return None + try: + text = path.read_text(encoding="ascii") + except (OSError, UnicodeDecodeError): + return None + match = re.fullmatch( + r"version https://git-lfs\.github\.com/spec/v1\r?\n" + r"oid sha256:([0-9a-fA-F]{64})\r?\n" + r"size ([0-9]+)\r?\n?", + text, + ) + if match is None: + return None + size = int(match.group(2)) + if size > MAX_U64: + raise ValueError(f"Git LFS pointer size exceeds uint64: {path}") + return size, match.group(1).lower() + + +def _file_record( + path: Path, + full_hash: bool, + verified_metadata: dict[str, object] | None = None, +) -> dict[str, object]: + physical_size = path.stat().st_size + if verified_metadata is not None: + size, sha256 = _verified_file_metadata(path, verified_metadata) + pointer = _lfs_pointer_metadata(path) + if physical_size == size: + if _sha256(path) != sha256: + raise ValueError( + f"verified file SHA-256 does not match: {path}" + ) + elif pointer != (size, sha256): + raise ValueError( + f"file is neither the verified payload nor its Git LFS pointer: " + f"{path}" + ) + else: + size = physical_size + sha256 = _sha256(path) if full_hash else "" if size < 0 or size > MAX_U64: raise ValueError(f"file size exceeds uint64: {path}") record: dict[str, object] = {"size": size} if full_hash: - record["sha256"] = _sha256(path) + record["sha256"] = sha256 return record @@ -606,6 +693,8 @@ def _generate_manifest( full_hash: bool, roles: dict[str, dict[str, object]], weight_objects: list[dict[str, object]] | None = None, + *, + file_metadata: dict[str, dict[str, object]] | None = None, ) -> dict[str, object]: """Generate a manifest using an explicit role map. @@ -619,6 +708,10 @@ def _generate_manifest( raise ValueError(f"model directory is not a directory: {model_dir}") if not isinstance(full_hash, bool): raise ValueError("full_hash must be a boolean") + if file_metadata is None: + file_metadata = {} + if not isinstance(file_metadata, dict): + raise ValueError("file_metadata must be an object") model_path = model_dir / "model.onnx" if not model_path.is_file(): @@ -636,6 +729,7 @@ def _generate_manifest( records: dict[str, dict[str, object]] = {} external_files: dict[str, Path] = {} + external_file_records: dict[str, dict[str, object]] = {} embedded: list[tuple[str, bytes]] = [] embedded_offset = 0 @@ -666,9 +760,12 @@ def _generate_manifest( raise ValueError( f"{name}: external offset is not dtype-aligned" ) - size = path.stat().st_size - if size > MAX_U64: - raise ValueError(f"{name}: external file size exceeds uint64") + file_record = _file_record( + path, + full_hash, + file_metadata.get(location), + ) + size = file_record["size"] if offset > size or length > size - offset: raise ValueError( f"{name}: external range exceeds file size" @@ -678,6 +775,7 @@ def _generate_manifest( raise ValueError( f"{name}: external location resolves inconsistently" ) + external_file_records[location] = file_record records[name] = _initializer_record( contract, dtype=dtype, @@ -731,15 +829,25 @@ def _generate_manifest( stream.write(raw_data) files: dict[str, dict[str, object]] = { - "model.onnx": _file_record(model_path, full_hash) + "model.onnx": _file_record( + model_path, + full_hash, + file_metadata.get("model.onnx"), + ) } for location in sorted(external_files): - files[location] = _file_record(external_files[location], full_hash) + files[location] = external_file_records[location] if embedded: files[EMBEDDED_INITIALIZERS_FILE] = _file_record( sidecar_path, full_hash, ) + unused_metadata = sorted(set(file_metadata) - set(files)) + if unused_metadata: + raise ValueError( + "verified metadata does not describe a manifest file: " + + unused_metadata[0] + ) emitted_weight_objects = ( [] if weight_objects is None else weight_objects @@ -768,14 +876,22 @@ def generate_manifest( model_dir: Path, output: Path, full_hash: bool, + *, + file_metadata: dict[str, dict[str, object]] | None = None, ) -> dict[str, object]: - return _generate_manifest( + arguments = ( model_dir, output, full_hash, required_initializer_roles(), required_weight_objects(), ) + if file_metadata is None: + return _generate_manifest(*arguments) + return _generate_manifest( + *arguments, + file_metadata=file_metadata, + ) def main() -> int: diff --git a/tools/package_phi4_corelib_aie4.py b/tools/package_phi4_corelib_aie4.py new file mode 100644 index 00000000..c3d59004 --- /dev/null +++ b/tools/package_phi4_corelib_aie4.py @@ -0,0 +1,582 @@ +from __future__ import annotations + +import argparse +import copy +import hashlib +import json +import re +import subprocess +import urllib.error +import urllib.request +from pathlib import Path + +from tools import generate_phi4_corelib_manifest as manifest_tool + + +UPSTREAM_COMMIT = "e751fb68c2cfffe6b0d32942118f75ac0a0365bb" +FLM_MIN_VERSION = "1.0.4" +EXPECTED_EOS_IDS = [200020, 199999] +UPSTREAM_REPOSITORY = ( + "https://huggingface.co/amd/phi-4-mini-instruct-oga-dml" +) +UPSTREAM_API_URL = ( + "https://huggingface.co/api/models/amd/" + f"phi-4-mini-instruct-oga-dml/tree/{UPSTREAM_COMMIT}" + "?recursive=true&expand=false" +) + +_EXPECTED_DECODER = { + "head_size": 128, + "hidden_size": 3072, + "num_attention_heads": 24, + "num_hidden_layers": 32, + "num_key_value_heads": 8, +} +_EXPECTED_MODEL = { + "vocab_size": 200064, +} + + +def _require_mapping(value: object, field: str) -> dict[str, object]: + if not isinstance(value, dict): + raise ValueError(f"{field} must be an object") + return value + + +def _require_exact_value( + mapping: dict[str, object], + field: str, + expected: object, + context: str, +) -> object: + value = mapping.get(field) + if value != expected: + raise ValueError( + f"{context}.{field} must be {expected!r}, got {value!r}" + ) + return value + + +def normalize_config(genai_config: dict[str, object]) -> dict[str, object]: + model = _require_mapping(genai_config.get("model"), "model") + decoder = _require_mapping(model.get("decoder"), "model.decoder") + for field, expected in _EXPECTED_DECODER.items(): + _require_exact_value(decoder, field, expected, "model.decoder") + for field, expected in _EXPECTED_MODEL.items(): + _require_exact_value(model, field, expected, "model") + + return { + "flm_version": FLM_MIN_VERSION, + "head_dim": decoder["head_size"], + "hidden_size": decoder["hidden_size"], + "intermediate_size": 8192, + "model_type": "phi4", + "num_attention_heads": decoder["num_attention_heads"], + "num_hidden_layers": decoder["num_hidden_layers"], + "num_key_value_heads": decoder["num_key_value_heads"], + "rms_norm_eps": 1.0e-5, + "vocab_size": model["vocab_size"], + } + + +def normalize_tokenizer_config( + tokenizer_config: dict[str, object], + chat_template: str, + genai_config: dict[str, object], +) -> dict[str, object]: + if not isinstance(chat_template, str) or not chat_template: + raise ValueError("chat template must be a non-empty string") + model = _require_mapping(genai_config.get("model"), "model") + _require_exact_value( + model, + "eos_token_id", + EXPECTED_EOS_IDS, + "model", + ) + normalized = dict(tokenizer_config) + normalized["chat_template"] = chat_template + normalized["eos_token_id"] = list(EXPECTED_EOS_IDS) + return normalized + + +def catalog_measurements( + model_dir: Path, + logical_sizes: dict[str, int] | None = None, +) -> tuple[int, float]: + model_dir = Path(model_dir) + logical_sizes = {} if logical_sizes is None else dict(logical_sizes) + files = [path for path in model_dir.rglob("*") if path.is_file()] + relative_paths = { + path.relative_to(model_dir).as_posix(): path for path in files + } + unknown = sorted(set(logical_sizes) - set(relative_paths)) + if unknown: + raise ValueError( + f"logical size has no matching package file: {unknown[0]}" + ) + size = 0 + for relative, path in relative_paths.items(): + logical_size = logical_sizes.get(relative, path.stat().st_size) + if ( + isinstance(logical_size, bool) + or not isinstance(logical_size, int) + or logical_size < 0 + ): + raise ValueError(f"invalid logical size for {relative}") + size += logical_size + footprint_gib = round(size / (1024**3), 2) + return size, footprint_gib + + +def _sha256_record(path: Path) -> dict[str, object]: + data = path.read_bytes() + return { + "size": len(data), + "sha256": hashlib.sha256(data).hexdigest(), + } + + +def build_provenance( + upstream_dir: Path, + overlay_dir: Path, + upstream_commit: str, + generated_files: list[str], + git_files: list[dict[str, object]] | None = None, +) -> dict[str, object]: + if upstream_commit != UPSTREAM_COMMIT: + raise ValueError( + f"upstream commit must be pinned to {UPSTREAM_COMMIT}" + ) + upstream_dir = Path(upstream_dir) + overlay_dir = Path(overlay_dir) + input_names = ( + "chat_template.jinja", + "genai_config.json", + "tokenizer_config.json", + ) + provenance: dict[str, object] = { + "upstream": { + "repository": ( + UPSTREAM_REPOSITORY + ), + "commit": upstream_commit, + "inputs": { + name: _sha256_record(upstream_dir / name) + for name in input_names + }, + }, + "generated": { + name: _sha256_record(overlay_dir / name) + for name in sorted(generated_files) + }, + } + if git_files is not None: + provenance["upstream"]["git_files"] = sorted( + git_files, + key=lambda record: record["path"], + ) + return provenance + + +def _git_blob_oid(data: bytes) -> str: + header = f"blob {len(data)}\0".encode("ascii") + return hashlib.sha1(header + data).hexdigest() + + +def _index_git_records( + records: list[dict[str, object]], +) -> dict[str, dict[str, object]]: + indexed: dict[str, dict[str, object]] = {} + for record in records: + if not isinstance(record, dict): + raise ValueError("Git metadata record must be an object") + path = record.get("path") + if not isinstance(path, str) or not path: + raise ValueError("Git metadata record has an invalid path") + if path in indexed: + raise ValueError(f"duplicate Git metadata path: {path}") + if record.get("type") != "file": + raise ValueError(f"Git metadata path is not a file: {path}") + indexed[path] = record + return indexed + + +def _validate_input_git_record( + upstream_dir: Path, + record: dict[str, object], +) -> None: + path = upstream_dir / str(record["path"]) + data = path.read_bytes() + if record.get("size") != len(data): + raise ValueError(f"Git metadata size does not match {record['path']}") + if "lfs" in record: + lfs = _require_mapping(record["lfs"], f"{record['path']}.lfs") + if lfs.get("size") != len(data): + raise ValueError( + f"Git LFS size does not match {record['path']}" + ) + if lfs.get("oid") != hashlib.sha256(data).hexdigest(): + raise ValueError( + f"Git LFS SHA-256 does not match {record['path']}" + ) + elif record.get("oid") != _git_blob_oid(data): + raise ValueError(f"Git blob OID does not match {record['path']}") + + +def _manifest_file_metadata( + indexed: dict[str, dict[str, object]], +) -> dict[str, dict[str, object]]: + metadata: dict[str, dict[str, object]] = {} + for name in ("model.onnx", "model.onnx.data"): + record = indexed.get(name) + if record is None: + raise ValueError(f"Git metadata is missing {name}") + lfs = _require_mapping(record.get("lfs"), f"{name}.lfs") + if record.get("size") != lfs.get("size"): + raise ValueError(f"Git LFS logical size does not match {name}") + metadata[name] = { + "size": lfs.get("size"), + "sha256": lfs.get("oid"), + } + return metadata + + +def _write_json(path: Path, value: object) -> None: + path.write_text( + json.dumps( + value, + ensure_ascii=False, + indent=2, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + newline="\n", + ) + + +def generate_overlay( + upstream_dir: Path, + overlay_dir: Path, + upstream_commit: str, + git_files: list[dict[str, object]], +) -> dict[str, object]: + if upstream_commit != UPSTREAM_COMMIT: + raise ValueError( + f"upstream commit must be pinned to {UPSTREAM_COMMIT}" + ) + upstream_dir = Path(upstream_dir).resolve(strict=True) + overlay_dir = Path(overlay_dir) + overlay_dir.mkdir(parents=True, exist_ok=True) + indexed = _index_git_records(git_files) + + input_names = ( + "chat_template.jinja", + "genai_config.json", + "tokenizer_config.json", + ) + for name in input_names: + record = indexed.get(name) + if record is None: + raise ValueError(f"Git metadata is missing {name}") + _validate_input_git_record(upstream_dir, record) + + genai_config = json.loads( + (upstream_dir / "genai_config.json").read_text(encoding="utf-8") + ) + tokenizer_config = json.loads( + (upstream_dir / "tokenizer_config.json").read_text(encoding="utf-8") + ) + chat_template = (upstream_dir / "chat_template.jinja").read_text( + encoding="utf-8" + ) + + _write_json( + overlay_dir / "config.json", + normalize_config(genai_config), + ) + _write_json( + overlay_dir / "tokenizer_config.json", + normalize_tokenizer_config( + tokenizer_config, + chat_template, + genai_config, + ), + ) + manifest_tool.generate_manifest( + upstream_dir, + overlay_dir / "corelib_phi4_manifest.json", + True, + file_metadata=_manifest_file_metadata(indexed), + ) + + generated_files = [ + "config.json", + "corelib_phi4_manifest.json", + "tokenizer_config.json", + ] + provenance = build_provenance( + upstream_dir, + overlay_dir, + upstream_commit, + generated_files, + git_files, + ) + _write_json(overlay_dir / "provenance.json", provenance) + return provenance + + +def build_catalog_entry( + overlay_dir: Path, + git_files: list[dict[str, object]], +) -> dict[str, object]: + overlay_dir = Path(overlay_dir) + overlay_names = ( + "config.json", + "corelib_phi4_manifest.json", + "tokenizer_config.json", + ) + overlays = { + name: { + "path": f"{overlay_dir.name}/{name}", + **_sha256_record(overlay_dir / name), + } + for name in overlay_names + } + + indexed = _index_git_records(git_files) + final_files = sorted(set(indexed) | set(overlays)) + remote_size = sum( + int(record["size"]) + for path, record in indexed.items() + if path not in overlays + ) + overlay_size = sum(int(record["size"]) for record in overlays.values()) + size = remote_size + overlay_size + footprint = round(size / (1024**3), 2) + return { + "name": "Phi-4-mini-instruct-oga-dml-AIE4", + "url": UPSTREAM_REPOSITORY, + "revision": UPSTREAM_COMMIT, + "file_url": UPSTREAM_API_URL, + "size": size, + "default_context_length": 4096, + "max_prefill_len": 4096, + "details": { + "family": "phi4", + "think": False, + "think_toggleable": False, + "parameter_size": "4B", + "quantization_level": "MatMulNBits Q4", + "execution_backend": "corelib_aie4", + }, + "flm_min_version": FLM_MIN_VERSION, + "vlm": False, + "modelscope_supported": False, + "files": final_files, + "bundled_overlays": overlays, + "footprint": footprint, + } + + +def updated_catalog_documents( + model_list: dict[str, object], + model_info: dict[str, object], + entry: dict[str, object], + git_files: list[dict[str, object]], +) -> tuple[dict[str, object], dict[str, object]]: + updated_list = copy.deepcopy(model_list) + models = _require_mapping(updated_list.get("models"), "models") + reordered: dict[str, object] = {} + inserted = False + for name, value in models.items(): + reordered[name] = value + if name == "phi4-mini-it": + reordered["phi4-mini-it-aie4"] = {"4b": copy.deepcopy(entry)} + inserted = True + if not inserted: + reordered["phi4-mini-it-aie4"] = {"4b": copy.deepcopy(entry)} + updated_list["models"] = reordered + + updated_info = copy.deepcopy(model_info) + updated_info["phi4-mini-it-aie4:4b"] = sorted( + copy.deepcopy(git_files), + key=lambda record: record["path"], + ) + return updated_list, updated_info + + +_LFS_POINTER = re.compile( + rb"version https://git-lfs\.github\.com/spec/v1\r?\n" + rb"oid sha256:([0-9a-fA-F]{64})\r?\n" + rb"size ([0-9]+)\r?\n?" +) + + +def git_metadata_records( + git_dir: Path, + commit: str, +) -> list[dict[str, object]]: + if commit != UPSTREAM_COMMIT: + raise ValueError(f"upstream commit must be pinned to {UPSTREAM_COMMIT}") + repository = str(Path(git_dir)) + resolved = subprocess.check_output( + ["git", "-C", repository, "rev-parse", commit], + text=True, + ).strip() + if resolved != commit: + raise ValueError( + f"metadata checkout resolved {resolved}, expected {commit}" + ) + tree = subprocess.check_output( + ["git", "-C", repository, "ls-tree", "-r", "--long", commit], + text=True, + ) + records: list[dict[str, object]] = [] + line_pattern = re.compile( + r"^[0-9]+ blob ([0-9a-f]{40})\s+([0-9]+)\t(.+)$" + ) + for line in tree.splitlines(): + match = line_pattern.fullmatch(line) + if match is None: + raise ValueError(f"unexpected git ls-tree record: {line}") + oid, pointer_size_text, path = match.groups() + pointer_size = int(pointer_size_text) + content = subprocess.check_output( + ["git", "-C", repository, "show", f"{commit}:{path}"] + ) + lfs_match = _LFS_POINTER.fullmatch(content) + record: dict[str, object] = { + "type": "file", + "oid": oid, + "size": pointer_size, + "path": path, + } + if lfs_match is not None: + logical_size = int(lfs_match.group(2)) + record["size"] = logical_size + record["lfs"] = { + "oid": lfs_match.group(1).decode("ascii").lower(), + "size": logical_size, + "pointerSize": pointer_size, + } + records.append(record) + return sorted(records, key=lambda record: record["path"]) + + +def huggingface_metadata_records( + git_dir: Path, + commit: str, +) -> list[dict[str, object]]: + request = urllib.request.Request( + UPSTREAM_API_URL, + headers={"User-Agent": "FastFlowLM-model-packager/1"}, + ) + try: + with urllib.request.urlopen(request) as response: + payload = json.load(response) + except urllib.error.HTTPError as error: + if error.code not in {401, 403}: + raise + return git_metadata_records(git_dir, commit) + + if not isinstance(payload, list): + raise ValueError("Hugging Face tree response must be an array") + records = [ + record + for record in payload + if isinstance(record, dict) and record.get("type") == "file" + ] + if len(records) != len(payload): + raise ValueError("Hugging Face recursive tree contains non-file records") + normalized: list[dict[str, object]] = [] + for record in records: + value: dict[str, object] = { + "type": "file", + "oid": record["oid"], + "size": record["size"], + "path": record["path"], + } + if "lfs" in record: + lfs = _require_mapping(record["lfs"], f"{record['path']}.lfs") + value["lfs"] = { + "oid": lfs["oid"], + "size": lfs["size"], + "pointerSize": lfs["pointerSize"], + } + normalized.append(value) + return sorted(normalized, key=lambda record: record["path"]) + + +def update_catalog_files( + model_list_path: Path, + model_info_path: Path, + overlay_dir: Path, + git_files: list[dict[str, object]], +) -> None: + model_list = json.loads( + Path(model_list_path).read_text(encoding="utf-8") + ) + model_info = json.loads( + Path(model_info_path).read_text(encoding="utf-8") + ) + entry = build_catalog_entry(overlay_dir, git_files) + updated_list, updated_info = updated_catalog_documents( + model_list, + model_info, + entry, + git_files, + ) + _write_json(Path(model_list_path), updated_list) + _write_json(Path(model_info_path), updated_info) + + +def main() -> int: + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers(dest="command", required=True) + + overlay = subparsers.add_parser("generate-overlay") + overlay.add_argument("--upstream-dir", type=Path, required=True) + overlay.add_argument("--git-dir", type=Path, required=True) + overlay.add_argument("--output-dir", type=Path, required=True) + overlay.add_argument( + "--upstream-commit", + default=UPSTREAM_COMMIT, + choices=[UPSTREAM_COMMIT], + ) + + catalog = subparsers.add_parser("refresh-catalog") + catalog.add_argument("--git-dir", type=Path, required=True) + catalog.add_argument("--overlay-dir", type=Path, required=True) + catalog.add_argument("--model-list", type=Path, required=True) + catalog.add_argument("--model-info", type=Path, required=True) + catalog.add_argument( + "--upstream-commit", + default=UPSTREAM_COMMIT, + choices=[UPSTREAM_COMMIT], + ) + + args = parser.parse_args() + records = huggingface_metadata_records( + args.git_dir, + args.upstream_commit, + ) + if args.command == "generate-overlay": + generate_overlay( + args.upstream_dir, + args.output_dir, + args.upstream_commit, + records, + ) + else: + update_catalog_files( + args.model_list, + args.model_info, + args.overlay_dir, + records, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/tests/test_package_phi4_corelib_aie4.py b/tools/tests/test_package_phi4_corelib_aie4.py new file mode 100644 index 00000000..1185b62d --- /dev/null +++ b/tools/tests/test_package_phi4_corelib_aie4.py @@ -0,0 +1,557 @@ +from __future__ import annotations + +import hashlib +import io +import json +import tempfile +import unittest +from pathlib import Path +from urllib.error import HTTPError +from unittest.mock import patch + +import onnx +from onnx import TensorProto, helper + +from tools import generate_phi4_corelib_manifest as manifest_tool +from tools import package_phi4_corelib_aie4 as package_tool + + +class Phi4CorelibOverlayTests(unittest.TestCase): + def _genai_config(self) -> dict[str, object]: + return { + "model": { + "bos_token_id": 199999, + "context_length": 131072, + "decoder": { + "head_size": 128, + "hidden_size": 3072, + "num_attention_heads": 24, + "num_hidden_layers": 32, + "num_key_value_heads": 8, + }, + "eos_token_id": [200020, 199999], + "type": "phi3", + "vocab_size": 200064, + }, + "search": {"max_length": 131072}, + } + + def _git_blob_oid(self, data: bytes) -> str: + header = f"blob {len(data)}\0".encode("ascii") + return hashlib.sha1(header + data).hexdigest() + + def _write_accepted_upstream( + self, + root: Path, + ) -> list[dict[str, object]]: + genai_data = ( + json.dumps(self._genai_config(), indent=2, sort_keys=True) + "\n" + ).encode() + tokenizer_data = b'{"model_max_length":131072}\n' + template_data = b"{{ messages | length }}\n" + for name, data in ( + ("genai_config.json", genai_data), + ("tokenizer_config.json", tokenizer_data), + ("chat_template.jinja", template_data), + ): + (root / name).write_bytes(data) + + item_sizes = { + "uint8": (TensorProto.UINT8, 1), + "float16": (TensorProto.FLOAT16, 2), + "float32": (TensorProto.FLOAT, 4), + "int64": (TensorProto.INT64, 8), + } + tensors: list[TensorProto] = [] + offset = 0 + for name, contract in sorted( + manifest_tool.required_initializer_roles().items() + ): + accepted = sorted(contract["dtypes"]) + dtype = "float16" if "float16" in accepted else accepted[0] + data_type, item_size = item_sizes[dtype] + shape = list( + contract.get("shape", contract.get("minimum_shape")) + ) + if name.endswith(".qweight"): + shape = [shape[0], shape[1] // 64, 64] + elif ( + name.endswith(".scales") or + name.endswith(".qzeros") + ): + shape = [shape[0] * shape[1]] + length = item_size + for dimension in shape: + length *= dimension + tensor = TensorProto() + tensor.name = name + tensor.data_type = data_type + tensor.dims.extend(shape) + tensor.data_location = TensorProto.EXTERNAL + for key, value in ( + ("location", "model.onnx.data"), + ("offset", str(offset)), + ("length", str(length)), + ): + item = tensor.external_data.add() + item.key = key + item.value = value + tensors.append(tensor) + offset += length + + logical_sha = "b" * 64 + (root / "model.onnx.data").write_text( + "version https://git-lfs.github.com/spec/v1\n" + f"oid sha256:{logical_sha}\n" + f"size {offset}\n", + encoding="ascii", + newline="\n", + ) + model = helper.make_model( + helper.make_graph([], "accepted-phi4", [], [], tensors) + ) + model_data = model.SerializeToString() + (root / "model.onnx").write_bytes(model_data) + + records: list[dict[str, object]] = [] + for name, data in ( + ("chat_template.jinja", template_data), + ("genai_config.json", genai_data), + ("tokenizer_config.json", tokenizer_data), + ): + records.append( + { + "type": "file", + "oid": self._git_blob_oid(data), + "size": len(data), + "path": name, + } + ) + records.extend( + [ + { + "type": "file", + "oid": "1" * 40, + "size": len(model_data), + "lfs": { + "oid": hashlib.sha256(model_data).hexdigest(), + "size": len(model_data), + "pointerSize": 131, + }, + "path": "model.onnx", + }, + { + "type": "file", + "oid": "2" * 40, + "size": offset, + "lfs": { + "oid": logical_sha, + "size": offset, + "pointerSize": 135, + }, + "path": "model.onnx.data", + }, + ] + ) + return records + + def test_normalized_config_is_derived_from_oga_and_backend_contract(self): + self.assertEqual( + package_tool.normalize_config(self._genai_config()), + { + "flm_version": "1.0.4", + "head_dim": 128, + "hidden_size": 3072, + "intermediate_size": 8192, + "model_type": "phi4", + "num_attention_heads": 24, + "num_hidden_layers": 32, + "num_key_value_heads": 8, + "rms_norm_eps": 1.0e-5, + "vocab_size": 200064, + }, + ) + + def test_normalized_tokenizer_preserves_upstream_and_adds_exact_sources(self): + upstream = { + "add_bos_token": False, + "model_max_length": 131072, + "tokenizer_class": "GPT2Tokenizer", + } + template = "{{ messages | length }}" + + normalized = package_tool.normalize_tokenizer_config( + upstream, + template, + self._genai_config(), + ) + + self.assertEqual(normalized["add_bos_token"], False) + self.assertEqual(normalized["model_max_length"], 131072) + self.assertEqual(normalized["tokenizer_class"], "GPT2Tokenizer") + self.assertEqual(normalized["chat_template"], template) + self.assertEqual(normalized["eos_token_id"], [200020, 199999]) + + def test_normalization_rejects_unapproved_oga_identity(self): + bad = self._genai_config() + bad["model"]["decoder"]["hidden_size"] = 4096 + with self.assertRaisesRegex(ValueError, "hidden_size"): + package_tool.normalize_config(bad) + + bad = self._genai_config() + bad["model"]["eos_token_id"] = [199999] + with self.assertRaisesRegex(ValueError, "eos_token_id"): + package_tool.normalize_tokenizer_config({}, "template", bad) + + def test_manifest_accepts_verified_lfs_pointer_as_logical_external_file(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + logical_sha = "a" * 64 + logical_size = 4 + (root / "weights.bin").write_text( + "version https://git-lfs.github.com/spec/v1\n" + f"oid sha256:{logical_sha}\n" + f"size {logical_size}\n", + encoding="ascii", + newline="\n", + ) + + tensor = TensorProto() + tensor.name = "test.weight" + tensor.data_type = TensorProto.UINT8 + tensor.dims.extend([2, 2]) + tensor.data_location = TensorProto.EXTERNAL + for key, value in ( + ("location", "weights.bin"), + ("offset", "0"), + ("length", "4"), + ): + item = tensor.external_data.add() + item.key = key + item.value = value + model = helper.make_model( + helper.make_graph([], "logical-lfs", [], [], [tensor]) + ) + model_path = root / "model.onnx" + model_path.write_bytes(model.SerializeToString()) + + manifest = manifest_tool._generate_manifest( + root, + root / "manifest.json", + True, + { + "test.weight": { + "role": "test.tensor", + "dtypes": {"uint8"}, + "shape": [2, 2], + } + }, + file_metadata={ + "weights.bin": { + "size": logical_size, + "sha256": logical_sha, + } + }, + ) + + self.assertEqual( + manifest["files"]["weights.bin"], + {"size": logical_size, "sha256": logical_sha}, + ) + self.assertEqual( + manifest["files"]["model.onnx"], + { + "size": model_path.stat().st_size, + "sha256": hashlib.sha256(model_path.read_bytes()).hexdigest(), + }, + ) + + def test_catalog_measurements_use_remote_logical_and_overlay_sizes(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "remote-lfs.bin").write_bytes(b"pointer") + (root / "config.json").write_bytes(b"{}") + size, footprint = package_tool.catalog_measurements( + root, + {"remote-lfs.bin": 1024**3}, + ) + self.assertEqual(size, 1024**3 + 2) + self.assertEqual(footprint, 1.0) + + def test_provenance_records_pinned_inputs_and_generated_outputs(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + upstream = root / "upstream" + overlay = root / "overlay" + upstream.mkdir() + overlay.mkdir() + (upstream / "genai_config.json").write_bytes(b'{"source":1}\n') + (upstream / "tokenizer_config.json").write_bytes(b'{"source":2}\n') + (upstream / "chat_template.jinja").write_bytes(b"template\n") + (overlay / "config.json").write_bytes(b'{"output":1}\n') + + provenance = package_tool.build_provenance( + upstream, + overlay, + "e751fb68c2cfffe6b0d32942118f75ac0a0365bb", + ["config.json"], + ) + + self.assertEqual( + provenance["upstream"]["commit"], + "e751fb68c2cfffe6b0d32942118f75ac0a0365bb", + ) + self.assertEqual( + set(provenance["upstream"]["inputs"]), + { + "chat_template.jinja", + "genai_config.json", + "tokenizer_config.json", + }, + ) + self.assertEqual( + provenance["generated"]["config.json"]["size"], + len(b'{"output":1}\n'), + ) + self.assertEqual( + len(provenance["generated"]["config.json"]["sha256"]), + 64, + ) + + def test_overlay_generation_is_complete_and_deterministic(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + upstream = root / "upstream" + first = root / "first" + second = root / "second" + upstream.mkdir() + records = self._write_accepted_upstream(upstream) + + package_tool.generate_overlay( + upstream, + first, + package_tool.UPSTREAM_COMMIT, + records, + ) + package_tool.generate_overlay( + upstream, + second, + package_tool.UPSTREAM_COMMIT, + records, + ) + + expected = { + "config.json", + "corelib_phi4_manifest.json", + "provenance.json", + "tokenizer_config.json", + } + self.assertEqual( + {path.name for path in first.iterdir()}, + expected, + ) + self.assertEqual( + { + path.name: path.read_bytes() + for path in first.iterdir() + }, + { + path.name: path.read_bytes() + for path in second.iterdir() + }, + ) + manifest = json.loads( + (first / "corelib_phi4_manifest.json").read_text() + ) + self.assertEqual(len(manifest["initializers"]), 743) + self.assertEqual(len(manifest["weight_objects"]), 161) + self.assertEqual( + set(manifest["files"]), + {"model.onnx", "model.onnx.data"}, + ) + self.assertNotIn( + "corelib_embedded_initializers.bin", + manifest["files"], + ) + provenance = json.loads( + (first / "provenance.json").read_text() + ) + self.assertEqual( + provenance["upstream"]["commit"], + package_tool.UPSTREAM_COMMIT, + ) + self.assertEqual( + provenance["upstream"]["git_files"], + sorted(records, key=lambda record: record["path"]), + ) + + def test_catalog_entry_uses_exact_remote_and_overlay_sizes(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + overlay = root / "phi4-mini-it-aie4" + overlay.mkdir() + for name, data in ( + ("config.json", b"config"), + ("corelib_phi4_manifest.json", b"manifest"), + ("tokenizer_config.json", b"normalized-tokenizer"), + ): + (overlay / name).write_bytes(data) + records = [ + { + "type": "file", + "oid": "1" * 40, + "size": 100, + "path": "model.onnx", + }, + { + "type": "file", + "oid": "2" * 40, + "size": 200, + "path": "tokenizer_config.json", + }, + ] + + entry = package_tool.build_catalog_entry(overlay, records) + + expected_size = ( + 100 + len(b"config") + len(b"manifest") + + len(b"normalized-tokenizer") + ) + self.assertEqual(entry["size"], expected_size) + self.assertEqual( + entry["footprint"], + round(expected_size / (1024**3), 2), + ) + self.assertEqual( + set(entry["files"]), + { + "config.json", + "corelib_phi4_manifest.json", + "model.onnx", + "tokenizer_config.json", + }, + ) + self.assertEqual( + set(entry["bundled_overlays"]), + { + "config.json", + "corelib_phi4_manifest.json", + "tokenizer_config.json", + }, + ) + + def test_metadata_refresh_preserves_unrelated_entries(self): + model_list = { + "models": { + "existing": {"1b": {"name": "keep"}}, + } + } + model_info = {"existing:1b": [{"path": "keep"}]} + entry = {"name": "generated"} + records = [{"path": "model.onnx", "type": "file"}] + + updated_list, updated_info = package_tool.updated_catalog_documents( + model_list, + model_info, + entry, + records, + ) + + self.assertEqual( + updated_list["models"]["existing"], + {"1b": {"name": "keep"}}, + ) + self.assertEqual( + updated_info["existing:1b"], + [{"path": "keep"}], + ) + self.assertEqual( + updated_list["models"]["phi4-mini-it-aie4"]["4b"], + entry, + ) + self.assertEqual( + updated_info["phi4-mini-it-aie4:4b"], + records, + ) + + def test_http_unauthorized_falls_back_to_pinned_git_metadata(self): + unauthorized = HTTPError( + package_tool.UPSTREAM_API_URL, + 401, + "Unauthorized", + {}, + None, + ) + expected = [{"path": "model.onnx", "type": "file"}] + with ( + patch.object( + package_tool.urllib.request, + "urlopen", + side_effect=unauthorized, + ), + patch.object( + package_tool, + "git_metadata_records", + return_value=expected, + ) as git_records, + ): + actual = package_tool.huggingface_metadata_records( + Path("metadata checkout"), + package_tool.UPSTREAM_COMMIT, + ) + + self.assertEqual(actual, expected) + git_records.assert_called_once_with( + Path("metadata checkout"), + package_tool.UPSTREAM_COMMIT, + ) + + def test_http_metadata_is_normalized_to_git_fallback_schema(self): + response = io.BytesIO( + json.dumps( + [ + { + "type": "file", + "oid": "1" * 40, + "size": 4, + "lfs": { + "oid": "a" * 64, + "size": 4, + "pointerSize": 127, + }, + "xetHash": "environment-specific", + "path": "model.onnx", + } + ] + ).encode() + ) + with patch.object( + package_tool.urllib.request, + "urlopen", + return_value=response, + ): + records = package_tool.huggingface_metadata_records( + Path("unused"), + package_tool.UPSTREAM_COMMIT, + ) + + self.assertEqual( + records, + [ + { + "type": "file", + "oid": "1" * 40, + "size": 4, + "lfs": { + "oid": "a" * 64, + "size": 4, + "pointerSize": 127, + }, + "path": "model.onnx", + } + ], + ) + + +if __name__ == "__main__": + unittest.main() From e70f270976912a26ffb001f5c9e2081ad9782697 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CChi?= Date: Tue, 1 Sep 2026 18:55:02 -0700 Subject: [PATCH 022/117] fix: rebase corelib integration onto e5258d2 The upstream C ABI changed in ways that a rename sweep alone would not have caught, so this lands the whole adapter boundary at once: nothing between the version gate and the element-unit conversion compiles in isolation. - Gate the load on ryzenai_corelib_get_version, resolved and checked before any other symbol. While corelib is pre-1.0 all three version components must match exactly, because the header says the API may change in any release. Resolving the rest first would have reported a missing renamed symbol instead of the skew that caused it. - tensor_write / tensor_read now count ELEMENTS of the tensor's own dtype. Every call site moves to CorelibApi::WriteElements / ReadElements, and no byte-taking spelling remains: the two counts differ by 2x when FP32 crosses into a BF16 tensor, so the wrong one half-fills or overruns rather than failing. - ryzenai_corelib_convert and _convert_strided are gone. The RoPE slice becomes a bounds-checked host gather in the source dtype, and the activation path stays in FP32 and lets tensor_write narrow, which leaves exactly one BF16 rounding implementation on that path. The two host conversions design API-6 still permits move into corelib/host_convert.hpp. - An FP32 scales array is now rejected with an actionable error rather than narrowed, since narrowing it would need a third host converter. - The fake corelib exports the e5258d2 symbol set and models tensors with a real dtype and element count, so a byte-sized count is rejected. A fake that accepted both conventions would have let this whole change pass while moving twice the data. Co-Authored-By: Claude Opus 5 (1M context) --- src/common/corelib/corelib_api.cpp | 142 ++++- src/common/corelib/phi4_corelib_aie4.cpp | 288 +++++----- src/common/corelib/phi4_corelib_host.cpp | 114 ++-- src/common/corelib/phi4_corelib_manifest.cpp | 135 +++-- src/common/corelib/phi4_corelib_weights.cpp | 36 +- src/include/corelib/corelib_api.hpp | 67 ++- src/include/corelib/host_convert.hpp | 93 +++ src/include/models/phi4/phi4_corelib_host.hpp | 14 +- .../models/phi4/phi4_corelib_manifest.hpp | 16 +- src/test/phi4_corelib_aie4/fake_corelib.cpp | 279 ++++++--- src/test/phi4_corelib_aie4/fake_corelib.hpp | 6 + .../phi4_corelib_aie4/test_corelib_api.cpp | 200 ++++++- .../phi4_corelib_aie4/test_phi4_engine.cpp | 237 ++++---- src/test/phi4_corelib_aie4/test_phi4_host.cpp | 534 +++++++++--------- .../phi4_corelib_aie4/test_phi4_manifest.cpp | 298 ++-------- .../phi4_corelib_aie4/test_phi4_weights.cpp | 203 +++---- 16 files changed, 1502 insertions(+), 1160 deletions(-) create mode 100644 src/include/corelib/host_convert.hpp diff --git a/src/common/corelib/corelib_api.cpp b/src/common/corelib/corelib_api.cpp index 2123ce49..84575eca 100644 --- a/src/common/corelib/corelib_api.cpp +++ b/src/common/corelib/corelib_api.cpp @@ -76,8 +76,37 @@ Function ResolveRequired( return reinterpret_cast(address); } -CorelibFunctions ResolveFunctions(const CorelibApi::Resolver& resolver) { +// Resolves and calls the version entry point BEFORE any other symbol is +// looked up. A runtime built from a different corelib revision renames and +// removes entry points, so resolving the rest first would report a missing +// symbol instead of the version skew that actually caused it. +CorelibVersion GateVersion( + const CorelibApi::Resolver& resolver, + decltype(&::ryzenai_corelib_get_version)& out_get_version) { + out_get_version = + ResolveRequired( + resolver, + "ryzenai_corelib_get_version"); + + CorelibVersion runtime{}; + out_get_version(&runtime.major, &runtime.minor, &runtime.patch); + + const auto compiled = CompiledCorelibVersion(); + if (!IsCorelibVersionCompatible(compiled, runtime)) { + throw std::runtime_error( + FormatCorelibVersionMismatch(compiled, runtime)); + } + return runtime; +} + +CorelibFunctions ResolveFunctions( + const CorelibApi::Resolver& resolver, + CorelibVersion& out_runtime_version) { + decltype(&::ryzenai_corelib_get_version) get_version = nullptr; + out_runtime_version = GateVersion(resolver, get_version); + return CorelibFunctions{ + get_version, ResolveRequired< decltype(&::ryzenai_corelib_status_to_string)>( resolver, @@ -123,21 +152,17 @@ CorelibFunctions ResolveFunctions(const CorelibApi::Resolver& resolver) { resolver, "ryzenai_corelib_tensor_get_byte_size"), ResolveRequired< - decltype(&::ryzenai_corelib_convert)>( + decltype(&::ryzenai_corelib_tensor_get_data_type)>( resolver, - "ryzenai_corelib_convert"), - ResolveRequired< - decltype(&::ryzenai_corelib_convert_strided)>( - resolver, - "ryzenai_corelib_convert_strided"), + "ryzenai_corelib_tensor_get_data_type"), ResolveRequired< decltype(&::ryzenai_corelib_matmul_bf16_pad_shape)>( resolver, "ryzenai_corelib_matmul_bf16_pad_shape"), ResolveRequired( + &::ryzenai_corelib_matmul_bf16_weights_create_onnx)>( resolver, - "ryzenai_corelib_matmul_bf16_weights_create_from_onnx_components"), + "ryzenai_corelib_matmul_bf16_weights_create_onnx"), ResolveRequired( resolver, @@ -151,9 +176,9 @@ CorelibFunctions ResolveFunctions(const CorelibApi::Resolver& resolver) { resolver, "ryzenai_corelib_ssmlp_bf16_pad_rows"), ResolveRequired( + &::ryzenai_corelib_ssmlp_bf16_weights_create_onnx)>( resolver, - "ryzenai_corelib_ssmlp_bf16_weights_create_from_onnx_components"), + "ryzenai_corelib_ssmlp_bf16_weights_create_onnx"), ResolveRequired( resolver, @@ -186,6 +211,44 @@ std::string LoadFailureMessage( } // namespace +bool IsCorelibVersionCompatible( + const CorelibVersion& compiled, + const CorelibVersion& runtime) noexcept { + if (compiled.major == 0) { + // Pre-1.0 corelib may change the API in any release, including a + // patch one, so every component is part of the contract. + return compiled.major == runtime.major && + compiled.minor == runtime.minor && + compiled.patch == runtime.patch; + } + return compiled.major == runtime.major && + runtime.minor >= compiled.minor; +} + +std::string FormatCorelibVersion(const CorelibVersion& value) { + return std::to_string(value.major) + '.' + + std::to_string(value.minor) + '.' + + std::to_string(value.patch); +} + +std::string FormatCorelibVersionMismatch( + const CorelibVersion& compiled, + const CorelibVersion& runtime) { + return "ryzenai-corelib version mismatch: the loaded runtime reports " + "version " + + FormatCorelibVersion(runtime) + + " but FastFlowLM was compiled against version " + + FormatCorelibVersion(compiled) + + "; install the matching ryzenai_corelib.dll"; +} + +CorelibVersion CompiledCorelibVersion() noexcept { + return CorelibVersion{ + static_cast(RYZENAI_CORELIB_VERSION_MAJOR), + static_cast(RYZENAI_CORELIB_VERSION_MINOR), + static_cast(RYZENAI_CORELIB_VERSION_PATCH)}; +} + CorelibError::CorelibError( ryzenai_corelib_status status_value, std::string call_value, @@ -237,12 +300,14 @@ std::shared_ptr CorelibApi::Load( return reinterpret_cast( GetProcAddress(module, symbol.c_str())); }; - auto functions = ResolveFunctions(resolver); + CorelibVersion runtime_version{}; + auto functions = ResolveFunctions(resolver, runtime_version); return std::shared_ptr( new CorelibApi( module, absolute_path.lexically_normal(), - std::move(functions))); + std::move(functions), + runtime_version)); } catch (...) { FreeLibrary(module); throw; @@ -251,9 +316,14 @@ std::shared_ptr CorelibApi::Load( std::shared_ptr CorelibApi::ResolveForTest( Resolver resolver) { - auto functions = ResolveFunctions(resolver); + CorelibVersion runtime_version{}; + auto functions = ResolveFunctions(resolver, runtime_version); return std::shared_ptr( - new CorelibApi(nullptr, {}, std::move(functions))); + new CorelibApi( + nullptr, + {}, + std::move(functions), + runtime_version)); } std::filesystem::path CorelibApi::ResolveLibraryPath( @@ -281,10 +351,12 @@ std::filesystem::path CorelibApi::ResolveLibraryPath( CorelibApi::CorelibApi( void* module, std::filesystem::path library_path, - CorelibFunctions functions) + CorelibFunctions functions, + CorelibVersion runtime_version) : module_(module), library_path_(std::move(library_path)), - functions_(std::move(functions)) {} + functions_(std::move(functions)), + runtime_version_(runtime_version) {} CorelibApi::~CorelibApi() { if (module_ != nullptr) { @@ -296,6 +368,42 @@ const CorelibFunctions& CorelibApi::functions() const noexcept { return functions_; } +const CorelibVersion& CorelibApi::runtime_version() const noexcept { + return runtime_version_; +} + +void CorelibApi::WriteElements( + ryzenai_corelib_tensor_ptr tensor, + ryzenai_corelib_data_type source_type, + const void* source, + std::size_t count, + std::size_t offset) const { + Check( + functions_.tensor_write( + tensor, + source_type, + source, + count, + offset), + "ryzenai_corelib_tensor_write"); +} + +void CorelibApi::ReadElements( + ryzenai_corelib_tensor_ptr tensor, + ryzenai_corelib_data_type destination_type, + void* destination, + std::size_t count, + std::size_t offset) const { + Check( + functions_.tensor_read( + tensor, + destination_type, + destination, + count, + offset), + "ryzenai_corelib_tensor_read"); +} + void CorelibApi::Check( ryzenai_corelib_status status, std::string_view call) const { diff --git a/src/common/corelib/phi4_corelib_aie4.cpp b/src/common/corelib/phi4_corelib_aie4.cpp index b2318737..8ca059c1 100644 --- a/src/common/corelib/phi4_corelib_aie4.cpp +++ b/src/common/corelib/phi4_corelib_aie4.cpp @@ -1,6 +1,7 @@ #include #include +#include #include #include #include @@ -40,14 +41,10 @@ constexpr std::string_view kCreateTensorCall = "ryzenai_corelib_create_device_tensor"; constexpr std::string_view kTensorByteSizeCall = "ryzenai_corelib_tensor_get_byte_size"; -constexpr std::string_view kTensorWriteCall = - "ryzenai_corelib_tensor_write"; -constexpr std::string_view kTensorReadCall = - "ryzenai_corelib_tensor_read"; constexpr std::string_view kSynchronizeCall = "ryzenai_corelib_stream_synchronize"; -constexpr std::string_view kConvertCall = - "ryzenai_corelib_convert"; +constexpr std::string_view kTensorDataTypeCall = + "ryzenai_corelib_tensor_get_data_type"; static_assert(sizeof(bf16) == sizeof(std::uint16_t)); @@ -173,23 +170,6 @@ std::size_t CheckedElements( return static_cast(row_count * column_count); } -std::size_t CheckedBytes( - std::int64_t rows, - std::int64_t width, - std::size_t element_size, - std::string_view context) { - const std::size_t elements = - CheckedElements(rows, width, context); - if ( - element_size != 0 && - elements > - std::numeric_limits::max() / element_size) { - throw std::overflow_error( - std::string(context) + " byte size overflows size_t"); - } - return elements * element_size; -} - std::size_t TensorByteCount( ryzenai_corelib_data_type data_type, std::span shape) { @@ -407,22 +387,30 @@ struct phi4_corelib_aie4::Impl final { embedding_view.data), embedding_view.size / sizeof(std::uint16_t)); + // The layer-0 norm feeds the host RMSNorm directly and never + // reaches a tensor, so widening it is the `API-6` FP16-to-FP32 + // helper; an FP32 source is a plain copy. const auto& norm_view = package->Require(kInputNormName); input_norm.resize( static_cast(constants::kHiddenSize)); - api->Check( - api->functions().convert( - SourceDataType(norm_view.dtype, kInputNormName), - norm_view.data, - ryzenai_corelib_data_type_fp32, - input_norm.data(), - input_norm.size()), - kConvertCall); + if ( + SourceDataType(norm_view.dtype, kInputNormName) == + ryzenai_corelib_data_type_fp32) { + std::copy_n( + reinterpret_cast(norm_view.data), + input_norm.size(), + input_norm.begin()); + } else { + corelib::WidenFp16Array( + reinterpret_cast(norm_view.data), + input_norm.size(), + input_norm.data()); + } const auto cos_host = - package->MaterializeRopeFp32(kCosName); + package->MaterializeRopeGather(kCosName); const auto sin_host = - package->MaterializeRopeFp32(kSinName); + package->MaterializeRopeGather(kSinName); const auto weight_pack_started = std::chrono::steady_clock::now(); @@ -442,7 +430,7 @@ struct phi4_corelib_aie4::Impl final { "Phi-4 host layer staging"); embedding_fp32.resize(layer_elements); normalized_fp32.resize(layer_elements); - bf16_staging.resize(layer_elements); + fp32_staging.resize(layer_elements); const std::size_t lm_head_elements = CheckedElements( capacities.lm_head_rows, constants::kHiddenSize, @@ -528,25 +516,31 @@ struct phi4_corelib_aie4::Impl final { true); } - const std::size_t rope_bytes = CheckedBytes( + // One write per table, in the source dtype, with `count` in FP32 + // elements of the destination tensor. tensor_write is the only + // conversion boundary corelib now offers. + const std::size_t rope_elements = CheckedElements( constants::kMaxSequenceLength, constants::kRopeDimension / 2, - sizeof(float), "Phi-4 RoPE upload"); - api->Check( - api->functions().tensor_write( - cos_tensor.get(), - cos_host.data(), - rope_bytes, - 0), - kTensorWriteCall); - api->Check( - api->functions().tensor_write( - sin_tensor.get(), - sin_host.data(), - rope_bytes, - 0), - kTensorWriteCall); + if ( + cos_host.count != rope_elements || + sin_host.count != rope_elements) { + throw std::logic_error( + "Phi-4 RoPE gather produced the wrong element count"); + } + api->WriteElements( + cos_tensor.get(), + cos_host.dtype, + cos_host.data, + rope_elements, + 0); + api->WriteElements( + sin_tensor.get(), + sin_host.dtype, + sin_host.data, + rope_elements, + 0); current_hidden = &hidden_tensors[0]; next_hidden = &hidden_tensors[1]; @@ -642,6 +636,21 @@ struct phi4_corelib_aie4::Impl final { "corelib device tensor byte size does not match " "the requested Phi-4 shape"); } + + // Every subsequent write and read counts in elements of THIS + // dtype, so confirm the tensor holds what was asked for rather + // than inferring it from the byte size, which FP16 and BF16 share. + ryzenai_corelib_data_type actual_type{}; + api->Check( + api->functions().tensor_get_data_type( + result.get(), + &actual_type), + kTensorDataTypeCall); + if (actual_type != data_type) { + throw std::runtime_error( + "corelib device tensor dtype does not match the " + "requested Phi-4 dtype"); + } ++metrics.device_tensor_create_count; if (is_kv) { metrics.kv_bytes += actual_bytes; @@ -750,19 +759,12 @@ struct phi4_corelib_aie4::Impl final { throw std::out_of_range( "Phi-4 padding write has a negative row offset"); } + // Offsets and counts are BF16 elements of the destination tensor, + // not bytes; the staging buffer is BF16 too, so this is a copy. const std::size_t offset = first_row == 0 ? 0 - : CheckedBytes( - first_row, - width, - sizeof(std::uint16_t), - context); - const std::size_t bytes = CheckedBytes( - row_count, - width, - sizeof(std::uint16_t), - context); + : CheckedElements(first_row, width, context); const std::size_t words = CheckedElements( row_count, width, @@ -771,15 +773,14 @@ struct phi4_corelib_aie4::Impl final { throw std::out_of_range( "Phi-4 padding write exceeds host staging capacity"); } - api->Check( - api->functions().tensor_write( - tensor.get(), - padding_zero_staging.data(), - bytes, - offset), - kTensorWriteCall); + api->WriteElements( + tensor.get(), + ryzenai_corelib_data_type_bf16, + padding_zero_staging.data(), + words, + offset); ++metrics.padding_write_calls; - metrics.padding_bytes += bytes; + metrics.padding_bytes += words * sizeof(std::uint16_t); } void BridgePadding( @@ -819,7 +820,6 @@ struct phi4_corelib_aie4::Impl final { const auto normalized_output = std::span(normalized_fp32).first(live_elements); GatherEmbedding( - *api, embedding, token_ids, embedding_output); @@ -831,48 +831,46 @@ struct phi4_corelib_aie4::Impl final { static_cast(constants::kRmsEpsilon), normalized_output); + // Design Section 10.2: the host stays in FP32 and writes FP32 into + // the BF16 tensors. Corelib narrows inside tensor_write, so there + // is one BF16 rounding implementation on this path, not two that + // have to agree. `count` is in BF16 elements of the destination. const std::int64_t hidden_rows = extents.ProjectionInput(); - StageBf16( - *api, + StageFp32( normalized_output, rows, hidden_rows, constants::kHiddenSize, - bf16_staging); - const std::size_t hidden_bytes = CheckedBytes( + fp32_staging); + const std::size_t hidden_elements = CheckedElements( hidden_rows, constants::kHiddenSize, - sizeof(std::uint16_t), "Phi-4 hidden staging"); - api->Check( - api->functions().tensor_write( - current_hidden->get(), - bf16_staging.data(), - hidden_bytes, - 0), - kTensorWriteCall); + api->WriteElements( + current_hidden->get(), + ryzenai_corelib_data_type_fp32, + fp32_staging.data(), + hidden_elements, + 0); const std::int64_t residual_rows = extents.ssmlp; - StageBf16( - *api, + StageFp32( embedding_output, rows, residual_rows, constants::kHiddenSize, - bf16_staging); - const std::size_t residual_bytes = CheckedBytes( + fp32_staging); + const std::size_t residual_elements = CheckedElements( residual_rows, constants::kHiddenSize, - sizeof(std::uint16_t), "Phi-4 residual staging"); - api->Check( - api->functions().tensor_write( - current_residual->get(), - bf16_staging.data(), - residual_bytes, - 0), - kTensorWriteCall); + api->WriteElements( + current_residual->get(), + ryzenai_corelib_data_type_fp32, + fp32_staging.data(), + residual_elements, + 0); } ryzenai_corelib_status SubmitMatMul( @@ -944,51 +942,50 @@ struct phi4_corelib_aie4::Impl final { void PrepareLastHidden( std::int64_t rows, std::int64_t lm_head_rows) { - const std::size_t row_bytes = CheckedBytes( + // Design Section 10.5: both tensors are BF16 and so is the caller + // dtype, so this is a straight copy with no FP32 round trip. + const std::size_t row_elements = CheckedElements( 1, constants::kHiddenSize, - sizeof(std::uint16_t), "Phi-4 last hidden"); const std::size_t source_offset = - static_cast(rows - 1) * row_bytes; + static_cast(rows - 1) * row_elements; WriteZeroRows( lm_input_tensor, 0, lm_head_rows, constants::kHiddenSize, "Phi-4 LM-head input initialization"); - api->Check( - api->functions().tensor_read( - current_hidden->get(), - last_hidden_staging.data(), - row_bytes, - source_offset), - kTensorReadCall); - api->Check( - api->functions().tensor_write( - lm_input_tensor.get(), - last_hidden_staging.data(), - row_bytes, - 0), - kTensorWriteCall); + api->ReadElements( + current_hidden->get(), + ryzenai_corelib_data_type_bf16, + last_hidden_staging.data(), + row_elements, + source_offset); + api->WriteElements( + lm_input_tensor.get(), + ryzenai_corelib_data_type_bf16, + last_hidden_staging.data(), + row_elements, + 0); } void ReadLogits(buffer& output) { - constexpr std::size_t logits_bytes = - static_cast(constants::kVocabularySize) * - sizeof(std::uint16_t); - if (output.size() != - static_cast(constants::kVocabularySize)) { + constexpr std::size_t logits_elements = + static_cast(constants::kVocabularySize); + if (output.size() != logits_elements) { throw std::logic_error( "Phi-4 logits buffer has the wrong size"); } - api->Check( - api->functions().tensor_read( - lm_output_tensor.get(), - output.data(), - logits_bytes, - 0), - kTensorReadCall); + // Straight into the returned buffer: the return type is + // already BF16, so widening and narrowing again would be a round + // trip for nothing. + api->ReadElements( + lm_output_tensor.get(), + ryzenai_corelib_data_type_bf16, + output.data(), + logits_elements, + 0); } buffer RunRows(std::span token_ids) { @@ -1195,8 +1192,7 @@ struct phi4_corelib_aie4::Impl final { std::vector result( static_cast(constants::kKvHeadCount) * live_rows * head_width); - const std::size_t bytes_per_head = - live_rows * head_width * sizeof(std::uint16_t); + const std::size_t elements_per_head = live_rows * head_width; for (std::size_t head = 0; head < static_cast(constants::kKvHeadCount); @@ -1205,14 +1201,13 @@ struct phi4_corelib_aie4::Impl final { head * static_cast( constants::kMaxSequenceLength) * - head_width * sizeof(std::uint16_t); - api->Check( - api->functions().tensor_read( - cache.get(), - result.data() + head * live_rows * head_width, - bytes_per_head, - source_offset), - kTensorReadCall); + head_width; + api->ReadElements( + cache.get(), + ryzenai_corelib_data_type_bf16, + result.data() + head * elements_per_head, + elements_per_head, + source_offset); } return result; } @@ -1231,21 +1226,18 @@ struct phi4_corelib_aie4::Impl final { static_cast(constants::kHiddenSize)); snapshot.logits.resize( static_cast(constants::kVocabularySize)); - api->Check( - api->functions().tensor_read( - lm_input_tensor.get(), - snapshot.last_hidden.data(), - snapshot.last_hidden.size() * - sizeof(std::uint16_t), - 0), - kTensorReadCall); - api->Check( - api->functions().tensor_read( - lm_output_tensor.get(), - snapshot.logits.data(), - snapshot.logits.size() * sizeof(std::uint16_t), - 0), - kTensorReadCall); + api->ReadElements( + lm_input_tensor.get(), + ryzenai_corelib_data_type_bf16, + snapshot.last_hidden.data(), + snapshot.last_hidden.size(), + 0); + api->ReadElements( + lm_output_tensor.get(), + ryzenai_corelib_data_type_bf16, + snapshot.logits.data(), + snapshot.logits.size(), + 0); return snapshot; } #endif @@ -1260,7 +1252,7 @@ struct phi4_corelib_aie4::Impl final { std::vector input_norm; std::vector embedding_fp32; std::vector normalized_fp32; - std::vector bf16_staging; + std::vector fp32_staging; std::vector padding_zero_staging; std::vector v_staging; std::vector last_hidden_staging; diff --git a/src/common/corelib/phi4_corelib_host.cpp b/src/common/corelib/phi4_corelib_host.cpp index 46ae2c00..e8a70c65 100644 --- a/src/common/corelib/phi4_corelib_host.cpp +++ b/src/common/corelib/phi4_corelib_host.cpp @@ -1,6 +1,8 @@ #include #include +#include + #include #include #include @@ -15,13 +17,6 @@ namespace flm::phi4 { namespace { -constexpr std::string_view kConvertCall = - "ryzenai_corelib_convert"; -constexpr std::string_view kTensorReadCall = - "ryzenai_corelib_tensor_read"; -constexpr std::string_view kTensorWriteCall = - "ryzenai_corelib_tensor_write"; - std::size_t CheckedExtent( std::int64_t rows, std::int64_t width, @@ -44,7 +39,6 @@ std::size_t CheckedExtent( } // namespace void GatherEmbedding( - const corelib::CorelibApi& api, std::span embedding_fp16, std::span token_ids, std::span output) { @@ -69,33 +63,31 @@ void GatherEmbedding( return; } + // Validate every token before touching the mapping, so an out-of-range + // ID fails instead of reading a row that is not there. const std::size_t vocabulary_rows = embedding_fp16.size() / width; - thread_local std::vector staging; - staging.resize(output_count); - for (std::size_t row = 0; row < token_ids.size(); ++row) { - const int token_id = token_ids[row]; + for (const int token_id : token_ids) { if (token_id < 0 || static_cast(token_id) >= vocabulary_rows) { throw std::out_of_range( "Phi-4 embedding token ID is outside the mapped table"); } + } + + // The gathered rows never reach a tensor before the host RMSNorm + // consumes them, so this is the `API-6` FP16-to-FP32 widening. It is + // scalar and per-element on purpose: the embedding table is a + // read-only file mapping, and a vectorized widening over-reads its + // source by up to 14 bytes, which faults on a page boundary. + for (std::size_t row = 0; row < token_ids.size(); ++row) { const std::size_t source_offset = - static_cast(token_id) * width; - std::copy_n( - embedding_fp16.begin() + source_offset, + static_cast(token_ids[row]) * width; + corelib::WidenFp16Array( + embedding_fp16.data() + source_offset, width, - staging.begin() + row * width); + output.data() + row * width); } - - api.Check( - api.functions().convert( - ryzenai_corelib_data_type_fp16, - staging.data(), - ryzenai_corelib_data_type_fp32, - output.data(), - output_count), - kConvertCall); } void RmsNorm( @@ -144,41 +136,34 @@ void RmsNorm( } } -void StageBf16( - const corelib::CorelibApi& api, +void StageFp32( std::span input, std::int64_t live_rows, std::int64_t padded_rows, std::int64_t width, - std::span output) { + std::span output) { if (padded_rows < live_rows) { throw std::invalid_argument( - "Phi-4 BF16 staging padded rows are smaller than live rows"); + "Phi-4 FP32 staging padded rows are smaller than live rows"); } const std::size_t live_count = - CheckedExtent(live_rows, width, "Phi-4 BF16 staging"); + CheckedExtent(live_rows, width, "Phi-4 FP32 staging"); const std::size_t padded_count = - CheckedExtent(padded_rows, width, "Phi-4 BF16 staging"); + CheckedExtent(padded_rows, width, "Phi-4 FP32 staging"); if (input.size() != live_count || output.size() < padded_count) { throw std::invalid_argument( - "Phi-4 BF16 staging shape mismatch"); + "Phi-4 FP32 staging shape mismatch"); } - thread_local std::vector staging; - staging.resize(padded_count); - std::copy(input.begin(), input.end(), staging.begin()); + // Design Section 10.2: the host stays in FP32 and never produces BF16 + // activations. Corelib narrows FP32 to BF16 inside `tensor_write`, so + // there is exactly one BF16 rounding implementation on this path and + // FastFlow does not have to match it. + std::copy(input.begin(), input.end(), output.begin()); std::fill( - staging.begin() + live_count, - staging.end(), + output.begin() + live_count, + output.begin() + padded_count, 0.0f); - api.Check( - api.functions().convert( - ryzenai_corelib_data_type_fp32, - staging.data(), - ryzenai_corelib_data_type_bf16, - output.data(), - padded_count), - kConvertCall); } void ScatterV( @@ -213,22 +198,20 @@ void ScatterV( staging.resize(source_count + head_staging_count); const auto started = std::chrono::steady_clock::now(); - const std::size_t source_bytes = - source_count * sizeof(std::uint16_t); - api.Check( - api.functions().tensor_read( - source, - staging.data(), - source_bytes, - 0), - kTensorReadCall); + // `count` and `offset` are BF16 ELEMENTS of the cache's own dtype, not + // bytes (`API-7`). Both tensors are BF16 and so is the host staging + // buffer, so every transfer here is a straight copy. + api.ReadElements( + source, + ryzenai_corelib_data_type_bf16, + staging.data(), + source_count, + 0); ++metrics.read_calls; - metrics.bytes += source_bytes; + metrics.bytes += source_count * sizeof(std::uint16_t); std::uint16_t* const head_staging = staging.data() + source_count; - const std::size_t head_bytes = - head_staging_count * sizeof(std::uint16_t); for (std::size_t head = 0; head < head_count; ++head) { for (std::size_t row = 0; row < live_rows; ++row) { const std::size_t source_offset = @@ -243,16 +226,15 @@ void ScatterV( static_cast( constants::kMaxSequenceLength)) + static_cast(position)) * - head_width * sizeof(std::uint16_t); - api.Check( - api.functions().tensor_write( - value_cache, - head_staging, - head_bytes, - cache_offset), - kTensorWriteCall); + head_width; + api.WriteElements( + value_cache, + ryzenai_corelib_data_type_bf16, + head_staging, + head_staging_count, + cache_offset); ++metrics.write_calls; - metrics.bytes += head_bytes; + metrics.bytes += head_staging_count * sizeof(std::uint16_t); } const auto elapsed = std::chrono::duration_cast( diff --git a/src/common/corelib/phi4_corelib_manifest.cpp b/src/common/corelib/phi4_corelib_manifest.cpp index 50e6b911..68811c73 100644 --- a/src/common/corelib/phi4_corelib_manifest.cpp +++ b/src/common/corelib/phi4_corelib_manifest.cpp @@ -3,6 +3,7 @@ #include "../../pull/picosha2.h" +#include #include #include @@ -11,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -1353,19 +1355,27 @@ std::span Phi4Package::MaterializeFp16( } const auto& source = Require(name); - const auto source_type = CorelibDType(source.dtype, name); + // Design Section 9.3: the ONNX-component contract specifies FP16 + // scales, and narrowing an FP32 source would require a second host + // rounding converter that `API-6` does not permit. Reject rather than + // convert, and say what the package must contain. + if (source.dtype != SourceDType::Float16) { + Throw( + name, + "MatMulNBits scales must be FP16 in an accepted AIE4 " + "package; an FP32 scales array is rejected rather than " + "narrowed, so repackage the model with FP16 scales"); + } const std::size_t count = ElementCount(source, name); auto [buffer, inserted] = fp16_buffers_.try_emplace(std::string(name), count); try { - api_->Check( - api_->functions().convert( - source_type, - source.data, - ryzenai_corelib_data_type_fp16, - buffer->second.data(), - count), - "ryzenai_corelib_convert"); + // An element-wise copy, not a conversion: this exists so that a + // strided or non-contiguous source view still yields the + // contiguous model-owned buffer `WEIGHT-2` requires. + const auto* elements = + reinterpret_cast(source.data); + std::copy_n(elements, count, buffer->second.begin()); } catch (...) { if (inserted) { fp16_buffers_.erase(buffer); @@ -1388,14 +1398,27 @@ std::span Phi4Package::MaterializeBf16( auto [buffer, inserted] = bf16_buffers_.try_emplace(std::string(name), count); try { - api_->Check( - api_->functions().convert( - source_type, - source.data, - ryzenai_corelib_data_type_bf16, - buffer->second.data(), - count), - "ryzenai_corelib_convert"); + // SSMLP norms are raw BF16 blobs handed to the packer, with no + // tensor boundary to convert through, so this is the `API-6` + // FP32-to-BF16 helper. An FP16 source is first widened losslessly, + // which composes the two permitted conversions rather than adding + // a third. + auto* destination = buffer->second.data(); + if (source_type == ryzenai_corelib_data_type_fp32) { + const auto* elements = + reinterpret_cast(source.data); + corelib::NarrowFp32ToBf16Array( + elements, + count, + destination); + } else { + const auto* elements = + reinterpret_cast(source.data); + for (std::size_t index = 0; index < count; ++index) { + destination[index] = corelib::NarrowFp32ToBf16( + corelib::WidenFp16(elements[index])); + } + } } catch (...) { if (inserted) { bf16_buffers_.erase(buffer); @@ -1405,13 +1428,8 @@ std::span Phi4Package::MaterializeBf16( return buffer->second; } -std::span Phi4Package::MaterializeRopeFp32( +RopeSourceView Phi4Package::MaterializeRopeGather( std::string_view name) { - if (const auto found = fp32_buffers_.find(name); - found != fp32_buffers_.end()) { - return found->second; - } - const auto& source = Require(name); const auto source_type = CorelibDType(source.dtype, name); if ( @@ -1422,32 +1440,69 @@ std::span Phi4Package::MaterializeRopeFp32( name, "RoPE source must be rank 2 and at least [4096,48]"); } + + constexpr std::size_t rows = + static_cast(constants::kMaxSequenceLength); + constexpr std::size_t columns = + static_cast(kRopeColumns); + constexpr std::size_t count = rows * columns; + + if (const auto found = rope_buffers_.find(name); + found != rope_buffers_.end()) { + return RopeSourceView{ + source_type, + found->second.data(), + count}; + } + + const std::size_t item_size = ItemSize(source.dtype); const auto source_columns = static_cast(source.shape[1]); - constexpr std::size_t count = - static_cast( - constants::kMaxSequenceLength * kRopeColumns); + const std::size_t source_row_bytes = source_columns * item_size; + const std::size_t gathered_row_bytes = columns * item_size; + + // Corelib `e5258d2` removed `convert_strided`, so the slice is + // FastFlow's own element-wise copy. It stays in the SOURCE dtype and + // lets `tensor_write` widen to FP32: the RoPE tables have a tensor to + // write into, so this path must not use the `API-6` widening helper. + // + // Every row is bounds-checked against the mapped extent before it is + // read, and only [row_start, row_start + 48) of each row is touched. + // The source is a read-only file mapping where a tail over-read faults + // rather than returning garbage, and the final row commonly sits + // immediately before an inaccessible page. auto [buffer, inserted] = - fp32_buffers_.try_emplace(std::string(name), count); + rope_buffers_.try_emplace( + std::string(name), + count * item_size); try { - api_->Check( - api_->functions().convert_strided( - source_type, - source.data, - source_columns, - ryzenai_corelib_data_type_fp32, - buffer->second.data(), - static_cast(kRopeColumns), - count, - static_cast(kRopeColumns)), - "ryzenai_corelib_convert_strided"); + const auto* const base = source.data; + auto* destination = buffer->second.data(); + for (std::size_t row = 0; row < rows; ++row) { + const std::size_t row_start = row * source_row_bytes; + if ( + row_start > source.size || + gathered_row_bytes > source.size - row_start) { + Throw( + name, + "RoPE gather row exceeds the mapped initializer " + "extent"); + } + std::memcpy( + destination + row * gathered_row_bytes, + base + row_start, + gathered_row_bytes); + } } catch (...) { if (inserted) { - fp32_buffers_.erase(buffer); + rope_buffers_.erase(buffer); } throw; } - return buffer->second; + return RopeSourceView{ + source_type, + buffer->second.data(), + count}; } } // namespace flm::phi4 diff --git a/src/common/corelib/phi4_corelib_weights.cpp b/src/common/corelib/phi4_corelib_weights.cpp index 9c9f0ced..a77a30b9 100644 --- a/src/common/corelib/phi4_corelib_weights.cpp +++ b/src/common/corelib/phi4_corelib_weights.cpp @@ -1,5 +1,7 @@ #include +#include + #include #include #include @@ -9,14 +11,12 @@ namespace flm::phi4 { namespace { -constexpr std::string_view kConvertCall = - "ryzenai_corelib_convert"; constexpr std::string_view kMatMulCreateCall = - "ryzenai_corelib_matmul_bf16_weights_create_from_onnx_components"; + "ryzenai_corelib_matmul_bf16_weights_create_onnx"; constexpr std::string_view kMatMulGetDataCall = "ryzenai_corelib_matmul_bf16_weights_get_data"; constexpr std::string_view kSsMlpCreateCall = - "ryzenai_corelib_ssmlp_bf16_weights_create_from_onnx_components"; + "ryzenai_corelib_ssmlp_bf16_weights_create_onnx"; constexpr std::string_view kSsMlpGetDataCall = "ryzenai_corelib_ssmlp_bf16_weights_get_data"; @@ -83,7 +83,7 @@ corelib::UniqueMatMulWeights CreateMatMul( object.n, constants::kGroupSize, false}; - const ryzenai_corelib_matmul_bf16_onnx_weights_components + const ryzenai_corelib_matmul_bf16_onnx_components components{ qweight.data, scales.data(), @@ -94,6 +94,7 @@ corelib::UniqueMatMulWeights CreateMatMul( api->functions().matmul_weights_from_onnx( &descriptor, &components, + corelib::kPackingThreads, &raw); corelib::UniqueMatMulWeights weights(api, raw); api->Check(status, kMatMulCreateCall); @@ -172,7 +173,7 @@ corelib::UniqueSsMlpWeights CreateSsMlp( object.k, object.n, constants::kGroupSize}; - const ryzenai_corelib_ssmlp_bf16_onnx_weights_components + const ryzenai_corelib_ssmlp_bf16_onnx_components components{ epsilon, norm0.data(), @@ -192,6 +193,7 @@ corelib::UniqueSsMlpWeights CreateSsMlp( api->functions().ssmlp_weights_from_onnx( &descriptor, &components, + corelib::kPackingThreads, &raw); corelib::UniqueSsMlpWeights weights(api, raw); api->Check(status, kSsMlpCreateCall); @@ -234,23 +236,11 @@ Phi4Weights Phi4Weights::Load( Phi4Weights result; result.package_ = std::move(package); - auto epsilon = std::make_shared(); - const float epsilon_fp32 = - static_cast(constants::kRmsEpsilon); - try { - api->Check( - api->functions().convert( - ryzenai_corelib_data_type_fp32, - &epsilon_fp32, - ryzenai_corelib_data_type_bf16, - epsilon.get(), - 1), - kConvertCall); - } catch (const std::exception& error) { - throw std::runtime_error( - "failed to materialize Phi-4 RMS epsilon: " + - std::string(error.what())); - } + // The packer takes epsilon as a raw BF16 blob, not as a tensor, so + // this is one of the two host conversions design `API-6` permits. + auto epsilon = std::make_shared( + corelib::NarrowFp32ToBf16( + static_cast(constants::kRmsEpsilon))); result.epsilon_bf16_ = std::move(epsilon); const auto& objects = result.package_->weight_objects(); diff --git a/src/include/corelib/corelib_api.hpp b/src/include/corelib/corelib_api.hpp index 960903af..d85c6eb4 100644 --- a/src/include/corelib/corelib_api.hpp +++ b/src/include/corelib/corelib_api.hpp @@ -4,6 +4,7 @@ #include #include +#include #include #include #include @@ -13,6 +14,30 @@ namespace flm::corelib { +// The loaded library's version, and the version this build compiled +// against. Corelib is pre-1.0, and the header states that below 1.0 the +// API may change in any release, so the patch component is load-bearing. +struct CorelibVersion final { + std::uint32_t major = 0; + std::uint32_t minor = 0; + std::uint32_t patch = 0; +}; + +// API-5. While the compiled-against major is 0 all three components must +// match exactly. From corelib 1.0 the rule relaxes to major equality with +// the runtime minor at least the compiled minor. +bool IsCorelibVersionCompatible( + const CorelibVersion& compiled, + const CorelibVersion& runtime) noexcept; + +std::string FormatCorelibVersion(const CorelibVersion& value); + +std::string FormatCorelibVersionMismatch( + const CorelibVersion& compiled, + const CorelibVersion& runtime); + +CorelibVersion CompiledCorelibVersion() noexcept; + struct CorelibError final : std::runtime_error { CorelibError( ryzenai_corelib_status status, @@ -31,6 +56,10 @@ struct CorelibError final : std::runtime_error { }; struct CorelibFunctions { + // Resolved and called first: the version gate runs before any other + // symbol is looked up, so a mismatched runtime is reported instead of + // producing a confusing "missing symbol" for a renamed entry point. + decltype(&::ryzenai_corelib_get_version) get_version; decltype(&::ryzenai_corelib_status_to_string) status_to_string; decltype(&::ryzenai_corelib_get_last_error_message) get_last_error_message; @@ -44,18 +73,15 @@ struct CorelibFunctions { decltype(&::ryzenai_corelib_tensor_write) tensor_write; decltype(&::ryzenai_corelib_tensor_read) tensor_read; decltype(&::ryzenai_corelib_tensor_get_byte_size) tensor_get_byte_size; - decltype(&::ryzenai_corelib_convert) convert; - decltype(&::ryzenai_corelib_convert_strided) convert_strided; + decltype(&::ryzenai_corelib_tensor_get_data_type) tensor_get_data_type; decltype(&::ryzenai_corelib_matmul_bf16_pad_shape) matmul_pad_shape; - decltype( - &::ryzenai_corelib_matmul_bf16_weights_create_from_onnx_components) + decltype(&::ryzenai_corelib_matmul_bf16_weights_create_onnx) matmul_weights_from_onnx; decltype(&::ryzenai_corelib_matmul_bf16_weights_get_data) matmul_weights_get_data; decltype(&::ryzenai_corelib_matmul_bf16) matmul; decltype(&::ryzenai_corelib_ssmlp_bf16_pad_rows) ssmlp_pad_rows; - decltype( - &::ryzenai_corelib_ssmlp_bf16_weights_create_from_onnx_components) + decltype(&::ryzenai_corelib_ssmlp_bf16_weights_create_onnx) ssmlp_weights_from_onnx; decltype(&::ryzenai_corelib_ssmlp_bf16_weights_get_data) ssmlp_weights_get_data; @@ -83,9 +109,30 @@ class CorelibApi final { CorelibApi& operator=(CorelibApi&&) = delete; const CorelibFunctions& functions() const noexcept; + const CorelibVersion& runtime_version() const noexcept; void Check( ryzenai_corelib_status status, std::string_view call) const; + + // API-7. `count` and `offset` are ELEMENTS of the tensor's own dtype, + // never bytes. These are the only spellings FastFlow uses; there is + // deliberately no byte-taking overload, because the byte and element + // counts differ by 2x when writing FP32 into a BF16 tensor and the + // wrong one would half-fill or overrun the tensor instead of failing. + void WriteElements( + ryzenai_corelib_tensor_ptr tensor, + ryzenai_corelib_data_type source_type, + const void* source, + std::size_t count, + std::size_t offset) const; + + void ReadElements( + ryzenai_corelib_tensor_ptr tensor, + ryzenai_corelib_data_type destination_type, + void* destination, + std::size_t count, + std::size_t offset) const; + void RegisterObject() const noexcept; void Release(void* value) const noexcept; std::size_t live_object_count() const noexcept; @@ -95,12 +142,18 @@ class CorelibApi final { CorelibApi( void* module, std::filesystem::path library_path, - CorelibFunctions functions); + CorelibFunctions functions, + CorelibVersion runtime_version); void* module_ = nullptr; std::filesystem::path library_path_; CorelibFunctions functions_; + CorelibVersion runtime_version_; mutable std::atomic live_object_count_{0}; }; +// The header's "one thread" hint for the ONNX packing entry points. +// Design Section 19 defers concurrent packing; FastFlow does not adopt it. +inline constexpr std::uint32_t kPackingThreads = 0; + } // namespace flm::corelib diff --git a/src/include/corelib/host_convert.hpp b/src/include/corelib/host_convert.hpp new file mode 100644 index 00000000..d65295ef --- /dev/null +++ b/src/include/corelib/host_convert.hpp @@ -0,0 +1,93 @@ +#pragma once + +// The only two host conversions FastFlow is permitted to implement +// (design `API-6`). Corelib `e5258d2` removed `ryzenai_corelib_convert` +// and `ryzenai_corelib_convert_strided`: every conversion with a tensor on +// either side now crosses `tensor_write` / `tensor_read`. What remains here +// is data that never touches a tensor. +// +// 1. FP16 -> FP32 widening, for the embedding rows gathered before the +// host RMSNorm and for an FP16 layer-0 norm. Lossless, so it has no +// rounding policy and cannot disagree with corelib. It is a scalar +// loop on purpose: the source is a read-only file mapping, and the +// naive vectorized widening reads up to 14 bytes past its source, +// which faults on a page boundary rather than returning garbage. +// +// 2. FP32 -> BF16 round-to-nearest-even, for the SSMLP epsilon / norm0 / +// norm1 blobs, which are packer inputs rather than tensors. Bit- +// compatible with the reference driver's `to_bf16`. +// +// There is deliberately no third converter. In particular there is no host +// FP32-to-FP16 narrowing, which is why an FP32 `scales` array is rejected +// rather than converted. + +#include +#include +#include + +namespace flm::corelib { + +// Exact: every FP16 value is representable in FP32. +inline float WidenFp16(std::uint16_t bits) noexcept { + const std::uint32_t sign = + static_cast(bits & 0x8000u) << 16; + const std::uint32_t exponent = + (static_cast(bits) >> 10) & 0x1Fu; + const std::uint32_t mantissa = + static_cast(bits) & 0x3FFu; + + if (exponent == 0u) { + if (mantissa == 0u) { + return std::bit_cast(sign); + } + std::uint32_t significand = mantissa; + std::uint32_t shift = 0u; + while ((significand & 0x400u) == 0u) { + significand <<= 1; + ++shift; + } + significand &= 0x3FFu; + const std::uint32_t widened_exponent = 127u - 15u - shift + 1u; + return std::bit_cast( + sign | (widened_exponent << 23) | (significand << 13)); + } + if (exponent == 0x1Fu) { + return std::bit_cast( + sign | 0x7F800000u | (mantissa << 13)); + } + return std::bit_cast( + sign | ((exponent - 15u + 127u) << 23) | (mantissa << 13)); +} + +// Scalar and tail-guarded: reads exactly `count` halfwords from `source` +// and touches nothing beyond them. +inline void WidenFp16Array( + const std::uint16_t* source, + std::size_t count, + float* destination) noexcept { + for (std::size_t index = 0; index < count; ++index) { + destination[index] = WidenFp16(source[index]); + } +} + +// Bit-compatible with the reference driver's `to_bf16`: +// rounded = bits + 0x7FFF + ((bits >> 16) & 1); return rounded >> 16 +// The driver accumulates in uint64 and narrows to uint16; the wrapped +// uint32 sum has the same low 16 bits, so the two agree on every input. +inline std::uint16_t NarrowFp32ToBf16(float value) noexcept { + const std::uint32_t bits = std::bit_cast(value); + const std::uint32_t rounded = + bits + 0x7FFFu + ((bits >> 16) & 1u); + return static_cast(rounded >> 16); +} + +inline void NarrowFp32ToBf16Array( + const float* source, + std::size_t count, + std::uint16_t* destination) noexcept { + for (std::size_t index = 0; index < count; ++index) { + destination[index] = NarrowFp32ToBf16(source[index]); + } +} + +} // namespace flm::corelib diff --git a/src/include/models/phi4/phi4_corelib_host.hpp b/src/include/models/phi4/phi4_corelib_host.hpp index 2b88333b..806bb4f2 100644 --- a/src/include/models/phi4/phi4_corelib_host.hpp +++ b/src/include/models/phi4/phi4_corelib_host.hpp @@ -16,8 +16,10 @@ struct VScatterMetrics { std::uint64_t nanoseconds = 0; }; +// Widens the gathered FP16 rows to FP32 with the `API-6` bounds-safe +// scalar helper. There is no tensor on either side of this conversion: +// the host RMSNorm consumes the result directly. void GatherEmbedding( - const corelib::CorelibApi& api, std::span embedding_fp16, std::span token_ids, std::span output); @@ -30,15 +32,15 @@ void RmsNorm( float epsilon, std::span output); -// Stages only the helper-required initial hidden/residual prefix. -// Elements beyond padded_rows * width are intentionally untouched. -void StageBf16( - const corelib::CorelibApi& api, +// Stages only the helper-required initial hidden/residual prefix, in FP32. +// Elements beyond padded_rows * width are intentionally untouched, and the +// FP32-to-BF16 narrowing is corelib's, inside tensor_write. +void StageFp32( std::span input, std::int64_t live_rows, std::int64_t padded_rows, std::int64_t width, - std::span output); + std::span output); // Precondition: the caller has successfully synchronized the Stream after // V projection and before this host read. ScatterV deliberately owns no diff --git a/src/include/models/phi4/phi4_corelib_manifest.hpp b/src/include/models/phi4/phi4_corelib_manifest.hpp index f82f560d..bbb71b50 100644 --- a/src/include/models/phi4/phi4_corelib_manifest.hpp +++ b/src/include/models/phi4/phi4_corelib_manifest.hpp @@ -77,6 +77,15 @@ struct WeightObjectView { std::map components; }; +// A contiguous [4096, 48] slice of a RoPE table, kept in the SOURCE dtype. +// `tensor_write` performs the widening to the FP32 device tensor, which is +// the only conversion boundary corelib `e5258d2` offers. +struct RopeSourceView { + ryzenai_corelib_data_type dtype; + const void* data; + std::size_t count; +}; + class Phi4Package final { public: static Phi4Package Load( @@ -97,8 +106,7 @@ class Phi4Package final { std::string_view name); std::span MaterializeBf16( std::string_view name); - std::span MaterializeRopeFp32( - std::string_view name); + RopeSourceView MaterializeRopeGather(std::string_view name); private: Phi4Package() = default; @@ -116,8 +124,8 @@ class Phi4Package final { fp16_buffers_; std::map, std::less<>> bf16_buffers_; - std::map, std::less<>> - fp32_buffers_; + std::map, std::less<>> + rope_buffers_; std::map> initializers_; diff --git a/src/test/phi4_corelib_aie4/fake_corelib.cpp b/src/test/phi4_corelib_aie4/fake_corelib.cpp index 8db8ecec..d444ff95 100644 --- a/src/test/phi4_corelib_aie4/fake_corelib.cpp +++ b/src/test/phi4_corelib_aie4/fake_corelib.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -12,6 +13,9 @@ namespace flm::test { namespace detail { thread_local std::string g_last_error; +std::uint32_t g_version_major = RYZENAI_CORELIB_VERSION_MAJOR; +std::uint32_t g_version_minor = RYZENAI_CORELIB_VERSION_MINOR; +std::uint32_t g_version_patch = RYZENAI_CORELIB_VERSION_PATCH; std::unordered_map g_release_counts; std::size_t g_release_count = 0; void* g_last_released_object = nullptr; @@ -21,6 +25,89 @@ bool g_has_device_context = true; std::size_t g_cleanup_count = 0; std::vector g_events; +void FakeGetVersion( + std::uint32_t* major, + std::uint32_t* minor, + std::uint32_t* patch) { + if (major != nullptr) { + *major = g_version_major; + } + if (minor != nullptr) { + *minor = g_version_minor; + } + if (patch != nullptr) { + *patch = g_version_patch; + } +} + +// A device tensor with a real dtype and element count, because that is +// what tensor_write / tensor_read are bounded by in e5258d2. A fake that +// accepted either unit convention would let a byte-sized count through, +// which is precisely the regression the element rebase must not reproduce. +struct FakeTensor { + ryzenai_corelib_data_type data_type; + std::size_t element_count; +}; + +std::unordered_map> g_tensors; + +bool IsConvertibleDataType(ryzenai_corelib_data_type value) noexcept { + return value == ryzenai_corelib_data_type_fp32 || + value == ryzenai_corelib_data_type_fp16 || + value == ryzenai_corelib_data_type_bf16; +} + +std::size_t DataTypeByteSize(ryzenai_corelib_data_type value) noexcept { + return value == ryzenai_corelib_data_type_fp32 ? 4u : 2u; +} + +FakeTensor* FindTensor(ryzenai_corelib_tensor_ptr tensor) noexcept { + const auto found = g_tensors.find(tensor); + return found == g_tensors.end() ? nullptr : found->second.get(); +} + +// Shared by write and read: count and offset are ELEMENTS of the tensor's +// own dtype, so offset + count must lie inside it. A caller still passing +// bytes overruns a BF16 tensor by exactly 2x and is rejected here rather +// than silently moving twice the data. +ryzenai_corelib_status CheckTensorRange( + ryzenai_corelib_tensor_ptr tensor, + ryzenai_corelib_data_type caller_type, + const void* host, + std::size_t count, + std::size_t offset) { + if (host == nullptr) { + g_last_error = "host buffer is null"; + return ryzenai_corelib_status_bad_argument; + } + if (!IsConvertibleDataType(caller_type)) { + g_last_error = + "only FP32, FP16 and BF16 are accepted for the caller type"; + return ryzenai_corelib_status_bad_argument; + } + const FakeTensor* record = FindTensor(tensor); + if (record == nullptr) { + // Tests that supply their own tensor handles opt out of the range + // model; they assert on the recorded arguments instead. + return ryzenai_corelib_status_success; + } + if (!IsConvertibleDataType(record->data_type)) { + g_last_error = "tensor dtype has no host float representation"; + return ryzenai_corelib_status_bad_argument; + } + if ( + offset > record->element_count || + count > record->element_count - offset) { + g_last_error = + "offset + count (" + std::to_string(offset) + " + " + + std::to_string(count) + ") exceeds the tensor's " + + std::to_string(record->element_count) + + " elements; count and offset are ELEMENTS, not bytes"; + return ryzenai_corelib_status_bad_argument; + } + return ryzenai_corelib_status_success; +} + const char* FakeStatusToString(ryzenai_corelib_status status) { g_last_error = "overwritten by status_to_string"; switch (status) { @@ -53,6 +140,7 @@ void FakeObjectRelease(ryzenai_corelib_object_ptr object) { ++g_release_counts[object]; g_last_released_object = object; g_events.emplace_back("release"); + g_tensors.erase(object); } ryzenai_corelib_status FakeCreateStream(ryzenai_corelib_stream_ptr* out) { @@ -68,59 +156,80 @@ ryzenai_corelib_status FakeStreamSynchronize( } ryzenai_corelib_status FakeCreateDeviceTensor( - ryzenai_corelib_data_type, - const int64_t*, - std::size_t, + ryzenai_corelib_data_type data_type, + const int64_t* shape, + std::size_t shape_len, ryzenai_corelib_tensor_ptr* out) { if (out != nullptr) { *out = nullptr; } + if (out == nullptr || shape == nullptr || shape_len == 0) { + g_last_error = "create_device_tensor requires a shape and an out"; + return ryzenai_corelib_status_bad_argument; + } + std::size_t elements = 1; + for (std::size_t index = 0; index < shape_len; ++index) { + if (shape[index] <= 0) { + g_last_error = "tensor dimensions must be positive"; + return ryzenai_corelib_status_bad_argument; + } + elements *= static_cast(shape[index]); + } + auto record = std::make_unique( + FakeTensor{data_type, elements}); + void* handle = record.get(); + g_tensors.emplace(handle, std::move(record)); + *out = handle; return ryzenai_corelib_status_success; } ryzenai_corelib_status FakeTensorWrite( - ryzenai_corelib_tensor_ptr, - const void*, - std::size_t, - std::size_t) { - return ryzenai_corelib_status_success; + ryzenai_corelib_tensor_ptr tensor, + ryzenai_corelib_data_type source_type, + const void* source, + std::size_t count, + std::size_t offset) { + return CheckTensorRange(tensor, source_type, source, count, offset); } ryzenai_corelib_status FakeTensorRead( - ryzenai_corelib_tensor_ptr, - void*, - std::size_t, - std::size_t) { - return ryzenai_corelib_status_success; + ryzenai_corelib_tensor_ptr tensor, + ryzenai_corelib_data_type destination_type, + void* destination, + std::size_t count, + std::size_t offset) { + return CheckTensorRange( + tensor, + destination_type, + destination, + count, + offset); } ryzenai_corelib_status FakeTensorGetByteSize( - ryzenai_corelib_tensor_ptr, + ryzenai_corelib_tensor_ptr tensor, std::size_t* out) { - if (out != nullptr) { - *out = 0; + if (out == nullptr) { + return ryzenai_corelib_status_bad_argument; } + const FakeTensor* record = FindTensor(tensor); + *out = record == nullptr + ? 0 + : record->element_count * + DataTypeByteSize(record->data_type); return ryzenai_corelib_status_success; } -ryzenai_corelib_status FakeConvert( - ryzenai_corelib_data_type, - const void*, - ryzenai_corelib_data_type, - void*, - std::size_t) { - return ryzenai_corelib_status_success; -} - -ryzenai_corelib_status FakeConvertStrided( - ryzenai_corelib_data_type, - const void*, - std::size_t, - ryzenai_corelib_data_type, - void*, - std::size_t, - std::size_t, - std::size_t) { +ryzenai_corelib_status FakeTensorGetDataType( + ryzenai_corelib_tensor_ptr tensor, + ryzenai_corelib_data_type* out) { + if (out == nullptr) { + return ryzenai_corelib_status_bad_argument; + } + const FakeTensor* record = FindTensor(tensor); + *out = record == nullptr + ? ryzenai_corelib_data_type_bf16 + : record->data_type; return ryzenai_corelib_status_success; } @@ -134,7 +243,8 @@ ryzenai_corelib_status FakeMatmulPadShape( ryzenai_corelib_status FakeMatmulWeightsFromOnnx( const ryzenai_corelib_matmul_bf16_weights_desc*, - const ryzenai_corelib_matmul_bf16_onnx_weights_components*, + const ryzenai_corelib_matmul_bf16_onnx_components*, + uint32_t, ryzenai_corelib_matmul_bf16_weights_ptr* out) { if (out != nullptr) { *out = nullptr; @@ -174,7 +284,8 @@ ryzenai_corelib_status FakeSsmlpPadRows( ryzenai_corelib_status FakeSsmlpWeightsFromOnnx( const ryzenai_corelib_ssmlp_bf16_weights_desc*, - const ryzenai_corelib_ssmlp_bf16_onnx_weights_components*, + const ryzenai_corelib_ssmlp_bf16_onnx_components*, + uint32_t, ryzenai_corelib_ssmlp_bf16_weights_ptr* out) { if (out != nullptr) { *out = nullptr; @@ -259,6 +370,13 @@ void FakeCleanup() { #if defined(FLM_FAKE_CORELIB_DLL) +void ryzenai_corelib_get_version( + uint32_t* major, + uint32_t* minor, + uint32_t* patch) { + flm::test::detail::FakeGetVersion(major, minor, patch); +} + const char* ryzenai_corelib_status_to_string( ryzenai_corelib_status status) { return flm::test::detail::FakeStatusToString(status); @@ -304,25 +422,29 @@ ryzenai_corelib_status ryzenai_corelib_create_device_tensor( ryzenai_corelib_status ryzenai_corelib_tensor_write( ryzenai_corelib_tensor_ptr tensor, + ryzenai_corelib_data_type source_type, const void* source, - std::size_t size, + std::size_t count, std::size_t offset) { return flm::test::detail::FakeTensorWrite( tensor, + source_type, source, - size, + count, offset); } ryzenai_corelib_status ryzenai_corelib_tensor_read( ryzenai_corelib_tensor_ptr tensor, + ryzenai_corelib_data_type destination_type, void* destination, - std::size_t size, + std::size_t count, std::size_t offset) { return flm::test::detail::FakeTensorRead( tensor, + destination_type, destination, - size, + count, offset); } @@ -332,38 +454,10 @@ ryzenai_corelib_status ryzenai_corelib_tensor_get_byte_size( return flm::test::detail::FakeTensorGetByteSize(tensor, out); } -ryzenai_corelib_status ryzenai_corelib_convert( - ryzenai_corelib_data_type source_type, - const void* source, - ryzenai_corelib_data_type destination_type, - void* destination, - std::size_t count) { - return flm::test::detail::FakeConvert( - source_type, - source, - destination_type, - destination, - count); -} - -ryzenai_corelib_status ryzenai_corelib_convert_strided( - ryzenai_corelib_data_type source_type, - const void* source, - std::size_t source_stride, - ryzenai_corelib_data_type destination_type, - void* destination, - std::size_t destination_stride, - std::size_t count, - std::size_t row) { - return flm::test::detail::FakeConvertStrided( - source_type, - source, - source_stride, - destination_type, - destination, - destination_stride, - count, - row); +ryzenai_corelib_status ryzenai_corelib_tensor_get_data_type( + ryzenai_corelib_tensor_ptr tensor, + ryzenai_corelib_data_type* out) { + return flm::test::detail::FakeTensorGetDataType(tensor, out); } ryzenai_corelib_status ryzenai_corelib_matmul_bf16_pad_shape( @@ -378,14 +472,15 @@ ryzenai_corelib_status ryzenai_corelib_matmul_bf16_pad_shape( group_size); } -ryzenai_corelib_status -ryzenai_corelib_matmul_bf16_weights_create_from_onnx_components( +ryzenai_corelib_status ryzenai_corelib_matmul_bf16_weights_create_onnx( const ryzenai_corelib_matmul_bf16_weights_desc* desc, - const ryzenai_corelib_matmul_bf16_onnx_weights_components* components, + const ryzenai_corelib_matmul_bf16_onnx_components* components, + uint32_t threads, ryzenai_corelib_matmul_bf16_weights_ptr* out) { return flm::test::detail::FakeMatmulWeightsFromOnnx( desc, components, + threads, out); } @@ -425,14 +520,15 @@ ryzenai_corelib_status ryzenai_corelib_ssmlp_bf16_pad_rows( group_size); } -ryzenai_corelib_status -ryzenai_corelib_ssmlp_bf16_weights_create_from_onnx_components( +ryzenai_corelib_status ryzenai_corelib_ssmlp_bf16_weights_create_onnx( const ryzenai_corelib_ssmlp_bf16_weights_desc* desc, - const ryzenai_corelib_ssmlp_bf16_onnx_weights_components* components, + const ryzenai_corelib_ssmlp_bf16_onnx_components* components, + uint32_t threads, ryzenai_corelib_ssmlp_bf16_weights_ptr* out) { return flm::test::detail::FakeSsmlpWeightsFromOnnx( desc, components, + threads, out); } @@ -521,6 +617,9 @@ void* FunctionAddress(Function function) { std::unordered_map CompleteCorelibResolver() { return { + FLM_FAKE_ENTRY( + ryzenai_corelib_get_version, + FakeGetVersion), FLM_FAKE_ENTRY( ryzenai_corelib_status_to_string, FakeStatusToString), @@ -555,16 +654,13 @@ std::unordered_map CompleteCorelibResolver() { ryzenai_corelib_tensor_get_byte_size, FakeTensorGetByteSize), FLM_FAKE_ENTRY( - ryzenai_corelib_convert, - FakeConvert), - FLM_FAKE_ENTRY( - ryzenai_corelib_convert_strided, - FakeConvertStrided), + ryzenai_corelib_tensor_get_data_type, + FakeTensorGetDataType), FLM_FAKE_ENTRY( ryzenai_corelib_matmul_bf16_pad_shape, FakeMatmulPadShape), FLM_FAKE_ENTRY( - ryzenai_corelib_matmul_bf16_weights_create_from_onnx_components, + ryzenai_corelib_matmul_bf16_weights_create_onnx, FakeMatmulWeightsFromOnnx), FLM_FAKE_ENTRY( ryzenai_corelib_matmul_bf16_weights_get_data, @@ -576,7 +672,7 @@ std::unordered_map CompleteCorelibResolver() { ryzenai_corelib_ssmlp_bf16_pad_rows, FakeSsmlpPadRows), FLM_FAKE_ENTRY( - ryzenai_corelib_ssmlp_bf16_weights_create_from_onnx_components, + ryzenai_corelib_ssmlp_bf16_weights_create_onnx, FakeSsmlpWeightsFromOnnx), FLM_FAKE_ENTRY( ryzenai_corelib_ssmlp_bf16_weights_get_data, @@ -600,6 +696,10 @@ std::unordered_map CompleteCorelibResolver() { void ResetFakeCorelib() { detail::g_last_error.clear(); + detail::g_version_major = RYZENAI_CORELIB_VERSION_MAJOR; + detail::g_version_minor = RYZENAI_CORELIB_VERSION_MINOR; + detail::g_version_patch = RYZENAI_CORELIB_VERSION_PATCH; + detail::g_tensors.clear(); detail::g_release_counts.clear(); detail::g_release_count = 0; detail::g_last_released_object = nullptr; @@ -613,6 +713,15 @@ void SetLastErrorMessage(std::string message) { detail::g_last_error = std::move(message); } +void SetFakeCorelibVersion( + std::uint32_t major, + std::uint32_t minor, + std::uint32_t patch) noexcept { + detail::g_version_major = major; + detail::g_version_minor = minor; + detail::g_version_patch = patch; +} + void SetSelftestStatus(ryzenai_corelib_status status) noexcept { detail::g_selftest_status = status; } diff --git a/src/test/phi4_corelib_aie4/fake_corelib.hpp b/src/test/phi4_corelib_aie4/fake_corelib.hpp index 58906464..bc030088 100644 --- a/src/test/phi4_corelib_aie4/fake_corelib.hpp +++ b/src/test/phi4_corelib_aie4/fake_corelib.hpp @@ -3,6 +3,7 @@ #include #include +#include #include #include #include @@ -13,6 +14,11 @@ std::unordered_map CompleteCorelibResolver(); void ResetFakeCorelib(); void SetLastErrorMessage(std::string message); +// Drives the API-5 load-time version gate. +void SetFakeCorelibVersion( + std::uint32_t major, + std::uint32_t minor, + std::uint32_t patch) noexcept; void SetSelftestStatus(ryzenai_corelib_status status) noexcept; void SetHasDeviceContext(bool value) noexcept; std::size_t ObjectReleaseCount() noexcept; diff --git a/src/test/phi4_corelib_aie4/test_corelib_api.cpp b/src/test/phi4_corelib_aie4/test_corelib_api.cpp index e47d4353..d2d01ae3 100644 --- a/src/test/phi4_corelib_aie4/test_corelib_api.cpp +++ b/src/test/phi4_corelib_aie4/test_corelib_api.cpp @@ -25,6 +25,7 @@ namespace { using flm::corelib::CorelibApi; constexpr std::array kRequiredSymbols{ + "ryzenai_corelib_get_version", "ryzenai_corelib_status_to_string", "ryzenai_corelib_get_last_error_message", "ryzenai_corelib_selftest_dependencies", @@ -36,14 +37,13 @@ constexpr std::array kRequiredSymbols{ "ryzenai_corelib_tensor_write", "ryzenai_corelib_tensor_read", "ryzenai_corelib_tensor_get_byte_size", - "ryzenai_corelib_convert", - "ryzenai_corelib_convert_strided", + "ryzenai_corelib_tensor_get_data_type", "ryzenai_corelib_matmul_bf16_pad_shape", - "ryzenai_corelib_matmul_bf16_weights_create_from_onnx_components", + "ryzenai_corelib_matmul_bf16_weights_create_onnx", "ryzenai_corelib_matmul_bf16_weights_get_data", "ryzenai_corelib_matmul_bf16", "ryzenai_corelib_ssmlp_bf16_pad_rows", - "ryzenai_corelib_ssmlp_bf16_weights_create_from_onnx_components", + "ryzenai_corelib_ssmlp_bf16_weights_create_onnx", "ryzenai_corelib_ssmlp_bf16_weights_get_data", "ryzenai_corelib_ssmlp_bf16", "ryzenai_corelib_flat_mha_bf16_pad_rows", @@ -198,6 +198,7 @@ void TestCompleteResolution() { #define CHECK_MEMBER_IDENTITY(member, symbol) \ CHECK(reinterpret_cast(functions.member) == \ resolver.at(#symbol)) + CHECK_MEMBER_IDENTITY(get_version, ryzenai_corelib_get_version); CHECK_MEMBER_IDENTITY( status_to_string, ryzenai_corelib_status_to_string); @@ -225,16 +226,15 @@ void TestCompleteResolution() { CHECK_MEMBER_IDENTITY( tensor_get_byte_size, ryzenai_corelib_tensor_get_byte_size); - CHECK_MEMBER_IDENTITY(convert, ryzenai_corelib_convert); CHECK_MEMBER_IDENTITY( - convert_strided, - ryzenai_corelib_convert_strided); + tensor_get_data_type, + ryzenai_corelib_tensor_get_data_type); CHECK_MEMBER_IDENTITY( matmul_pad_shape, ryzenai_corelib_matmul_bf16_pad_shape); CHECK_MEMBER_IDENTITY( matmul_weights_from_onnx, - ryzenai_corelib_matmul_bf16_weights_create_from_onnx_components); + ryzenai_corelib_matmul_bf16_weights_create_onnx); CHECK_MEMBER_IDENTITY( matmul_weights_get_data, ryzenai_corelib_matmul_bf16_weights_get_data); @@ -244,7 +244,7 @@ void TestCompleteResolution() { ryzenai_corelib_ssmlp_bf16_pad_rows); CHECK_MEMBER_IDENTITY( ssmlp_weights_from_onnx, - ryzenai_corelib_ssmlp_bf16_weights_create_from_onnx_components); + ryzenai_corelib_ssmlp_bf16_weights_create_onnx); CHECK_MEMBER_IDENTITY( ssmlp_weights_get_data, ryzenai_corelib_ssmlp_bf16_weights_get_data); @@ -257,6 +257,182 @@ void TestCompleteResolution() { #undef CHECK_MEMBER_IDENTITY } +void TestMatchingVersionLoadsAndIsRecorded() { + flm::test::ResetFakeCorelib(); + auto api = ResolveCompleteCorelib(); + CHECK(api != nullptr); + const auto compiled = flm::corelib::CompiledCorelibVersion(); + CHECK(api->runtime_version().major == compiled.major); + CHECK(api->runtime_version().minor == compiled.minor); + CHECK(api->runtime_version().patch == compiled.patch); +} + +void TestMinorMismatchFailsNamingBothVersions() { + flm::test::ResetFakeCorelib(); + const auto compiled = flm::corelib::CompiledCorelibVersion(); + flm::test::SetFakeCorelibVersion( + compiled.major, + compiled.minor + 1, + compiled.patch); + + try { + (void)ResolveCompleteCorelib(); + } catch (const std::exception& error) { + const std::string_view message(error.what()); + CHECK(message.find( + flm::corelib::FormatCorelibVersion(compiled)) != + std::string_view::npos); + CHECK(message.find( + flm::corelib::FormatCorelibVersion( + flm::corelib::CorelibVersion{ + compiled.major, + compiled.minor + 1, + compiled.patch})) != std::string_view::npos); + CHECK(message.find("version") != std::string_view::npos); + flm::test::ResetFakeCorelib(); + return; + } + flm::test::ResetFakeCorelib(); + throw std::runtime_error( + "a corelib minor-version mismatch must fail the load"); +} + +void TestPatchMismatchFailsWhileCorelibIsPreOneDotZero() { + flm::test::ResetFakeCorelib(); + const auto compiled = flm::corelib::CompiledCorelibVersion(); + if (compiled.major != 0) { + return; + } + flm::test::SetFakeCorelibVersion( + compiled.major, + compiled.minor, + compiled.patch + 1); + CheckThrowsContains( + [&] { (void)ResolveCompleteCorelib(); }, + "version"); + flm::test::ResetFakeCorelib(); +} + +void TestVersionIsCheckedBeforeAnyOtherSymbolIsResolved() { + flm::test::ResetFakeCorelib(); + auto resolver = flm::test::CompleteCorelibResolver(); + std::vector requested; + const auto resolve = [&resolver, &requested]( + std::string_view name) -> void* { + requested.emplace_back(name); + const auto found = resolver.find(std::string(name)); + return found == resolver.end() ? nullptr : found->second; + }; + + (void)CorelibApi::ResolveForTest(resolve); + CHECK(!requested.empty()); + CHECK(requested.front() == "ryzenai_corelib_get_version"); + + // With an incompatible runtime the gate must stop there. Resolving the + // rest first would report a missing renamed symbol and hide the skew + // that actually caused it. + const auto compiled = flm::corelib::CompiledCorelibVersion(); + flm::test::SetFakeCorelibVersion( + compiled.major + 1, + compiled.minor, + compiled.patch); + requested.clear(); + CheckThrowsContains( + [&] { (void)CorelibApi::ResolveForTest(resolve); }, + "version"); + CHECK(requested.size() == 1); + CHECK(requested.front() == "ryzenai_corelib_get_version"); + flm::test::ResetFakeCorelib(); +} + +void TestVersionCompatibilityRule() { + using flm::corelib::CorelibVersion; + using flm::corelib::IsCorelibVersionCompatible; + + // Below 1.0 the header says the API may change in any release, so all + // three components are part of the contract. + const CorelibVersion pre{0, 1, 0}; + CHECK(IsCorelibVersionCompatible(pre, CorelibVersion{0, 1, 0})); + CHECK(!IsCorelibVersionCompatible(pre, CorelibVersion{0, 1, 1})); + CHECK(!IsCorelibVersionCompatible(pre, CorelibVersion{0, 2, 0})); + CHECK(!IsCorelibVersionCompatible(pre, CorelibVersion{1, 1, 0})); + + // From 1.0 the C ABI is additive within a major version. + const CorelibVersion stable{1, 3, 2}; + CHECK(IsCorelibVersionCompatible(stable, CorelibVersion{1, 3, 2})); + CHECK(IsCorelibVersionCompatible(stable, CorelibVersion{1, 3, 0})); + CHECK(IsCorelibVersionCompatible(stable, CorelibVersion{1, 4, 0})); + CHECK(!IsCorelibVersionCompatible(stable, CorelibVersion{1, 2, 9})); + CHECK(!IsCorelibVersionCompatible(stable, CorelibVersion{2, 3, 2})); +} + +// API-7: the wrapper takes elements, and there is no byte-taking overload +// to reach for by mistake. A count in bytes is 2x too large for a BF16 +// tensor and the fake rejects it rather than moving twice the data. +void TestElementCountsAreBoundedByTheTensorNotItsBytes() { + flm::test::ResetFakeCorelib(); + auto api = ResolveCompleteCorelib(); + + const std::array shape{4, 8}; + ryzenai_corelib_tensor_ptr raw = nullptr; + api->Check( + api->functions().create_device_tensor( + ryzenai_corelib_data_type_bf16, + shape.data(), + shape.size(), + &raw), + "ryzenai_corelib_create_device_tensor"); + flm::corelib::UniqueTensor tensor(api, raw); + + std::size_t byte_size = 0; + api->Check( + api->functions().tensor_get_byte_size(tensor.get(), &byte_size), + "ryzenai_corelib_tensor_get_byte_size"); + CHECK(byte_size == 32u * sizeof(std::uint16_t)); + + ryzenai_corelib_data_type data_type{}; + api->Check( + api->functions().tensor_get_data_type(tensor.get(), &data_type), + "ryzenai_corelib_tensor_get_data_type"); + CHECK(data_type == ryzenai_corelib_data_type_bf16); + + std::vector source(32, 0.0f); + api->WriteElements( + tensor.get(), + ryzenai_corelib_data_type_fp32, + source.data(), + 32, + 0); + api->WriteElements( + tensor.get(), + ryzenai_corelib_data_type_fp32, + source.data(), + 8, + 24); + + // The old spelling: 32 BF16 elements is 64 bytes, and 64 must fail. + CheckThrowsContains( + [&] { + api->WriteElements( + tensor.get(), + ryzenai_corelib_data_type_fp32, + source.data(), + byte_size, + 0); + }, + "ELEMENTS"); + CheckThrowsContains( + [&] { + api->ReadElements( + tensor.get(), + ryzenai_corelib_data_type_bf16, + source.data(), + 32, + 1); + }, + "ELEMENTS"); +} + void TestTypeIdenticalGetDataSymbolsCannotBeSwapped() { auto expected = flm::test::CompleteCorelibResolver(); const auto matmul_name = @@ -507,6 +683,12 @@ static_assert( int main() { try { TestCompleteResolution(); + TestMatchingVersionLoadsAndIsRecorded(); + TestMinorMismatchFailsNamingBothVersions(); + TestPatchMismatchFailsWhileCorelibIsPreOneDotZero(); + TestVersionIsCheckedBeforeAnyOtherSymbolIsResolved(); + TestVersionCompatibilityRule(); + TestElementCountsAreBoundedByTheTensorNotItsBytes(); TestTypeIdenticalGetDataSymbolsCannotBeSwapped(); TestMissingSymbolFailsAtomically(); TestErrorDetailSurvivesStatusConversion(); diff --git a/src/test/phi4_corelib_aie4/test_phi4_engine.cpp b/src/test/phi4_corelib_aie4/test_phi4_engine.cpp index 874e27a4..7866ac58 100644 --- a/src/test/phi4_corelib_aie4/test_phi4_engine.cpp +++ b/src/test/phi4_corelib_aie4/test_phi4_engine.cpp @@ -736,7 +736,8 @@ struct MhaCall { struct TensorWriteCall { FakeTensor* tensor; - std::size_t size; + // API-7: elements of the tensor's own dtype, never bytes. + std::size_t count; std::size_t offset; }; @@ -933,66 +934,6 @@ std::int64_t MatMulPaddedRows( "engine fake received an unknown matmul shape"); } -ryzenai_corelib_status RecordingConvert( - ryzenai_corelib_data_type source_type, - const void* source, - ryzenai_corelib_data_type destination_type, - void* destination, - std::size_t count) { - auto& state = State(); - state.ObserveLoadThread(); - if ( - state.failure == FailurePoint::StageBadAlloc && - source_type == ryzenai_corelib_data_type_fp16 && - destination_type == ryzenai_corelib_data_type_fp32 && - count != static_cast(constants::kHiddenSize)) { - throw std::bad_alloc{}; - } - if ((source == nullptr || destination == nullptr) && count != 0) { - return ryzenai_corelib_status_bad_argument; - } - for (std::size_t index = 0; index < count; ++index) { - WriteElement( - destination_type, - destination, - index, - ReadElement(source_type, source, index)); - } - return ryzenai_corelib_status_success; -} - -ryzenai_corelib_status RecordingConvertStrided( - ryzenai_corelib_data_type source_type, - const void* source, - std::size_t source_stride, - ryzenai_corelib_data_type destination_type, - void* destination, - std::size_t destination_stride, - std::size_t count, - std::size_t row) { - State().ObserveLoadThread(); - if ( - source == nullptr || destination == nullptr || row == 0 || - count % row != 0 || source_stride < row || - destination_stride < row) { - return ryzenai_corelib_status_bad_argument; - } - const std::size_t rows = count / row; - for (std::size_t row_index = 0; row_index < rows; ++row_index) { - for (std::size_t column = 0; column < row; ++column) { - WriteElement( - destination_type, - destination, - row_index * destination_stride + column, - ReadElement( - source_type, - source, - row_index * source_stride + column)); - } - } - return ryzenai_corelib_status_success; -} - ryzenai_corelib_status RecordingMatMulPadShape( std::int64_t* m, std::int64_t* k, @@ -1085,16 +1026,38 @@ ryzenai_corelib_status RecordingTensorGetByteSize( return ryzenai_corelib_status_success; } +ryzenai_corelib_status RecordingTensorGetDataType( + ryzenai_corelib_tensor_ptr tensor, + ryzenai_corelib_data_type* out) { + auto* value = State().Tensor(tensor); + if (value == nullptr || out == nullptr) { + return ryzenai_corelib_status_bad_argument; + } + *out = value->data_type; + return ryzenai_corelib_status_success; +} + +// `count` and `offset` are ELEMENTS of the TENSOR's dtype. The caller may +// hold a different dtype -- FP32 activations written into a BF16 tensor -- +// and corelib converts at this boundary, which is the only conversion +// path e5258d2 offers. ryzenai_corelib_status RecordingTensorWrite( ryzenai_corelib_tensor_ptr tensor, + ryzenai_corelib_data_type source_type, const void* source, - std::size_t size, + std::size_t count, std::size_t offset) { auto& state = State(); auto* value = state.Tensor(tensor); - if (value == nullptr) { + if (value == nullptr || source == nullptr) { + return ryzenai_corelib_status_bad_argument; + } + const std::size_t element_size = TypeSize(value->data_type); + const std::size_t tensor_elements = value->byte_size / element_size; + if (offset > tensor_elements || count > tensor_elements - offset) { return ryzenai_corelib_status_bad_argument; } + const std::size_t size = count * element_size; const bool is_cache = value->shape == @@ -1111,16 +1074,14 @@ ryzenai_corelib_status RecordingTensorWrite( } const auto* words = static_cast(source); - const std::size_t word_count = size / sizeof(std::uint16_t); state.cache_publish_poison_observed = state.cache_publish_poison_observed || std::any_of( words, - words + word_count, + words + count, [](std::uint16_t word) { return word == kPoison; }); - constexpr std::size_t head_pitch_bytes = - 4096u * 128u * sizeof(std::uint16_t); - const std::size_t head = offset / head_pitch_bytes; + constexpr std::size_t head_pitch_elements = 4096u * 128u; + const std::size_t head = offset / head_pitch_elements; state.events.push_back( "tensor_write_v_head_" + std::to_string(head)); } else if ( @@ -1141,7 +1102,17 @@ ryzenai_corelib_status RecordingTensorWrite( value->data_type == ryzenai_corelib_data_type_bf16 && value->shape == std::vector{4096, 3072}) { - state.stage_writes.push_back({value, size, offset}); + if (state.failure == FailurePoint::StageBadAlloc) { + // The staging write is the first corelib call of a pass, so + // this stands in for the host allocation that used to fail + // inside the removed converter. + throw std::bad_alloc{}; + } + // Design 10.2: the host stays in FP32 and lets corelib narrow. + if (source_type != ryzenai_corelib_data_type_fp32) { + return ryzenai_corelib_status_bad_argument; + } + state.stage_writes.push_back({value, count, offset}); if (state.stage_writes.size() % 2 == 1) { state.staged_hidden = value; } else { @@ -1149,21 +1120,36 @@ ryzenai_corelib_status RecordingTensorWrite( } } - return value->Write(source, size, offset) + std::vector staged(size); + for (std::size_t index = 0; index < count; ++index) { + WriteElement( + value->data_type, + staged.data(), + index, + ReadElement(source_type, source, index)); + } + return value->Write(staged.data(), size, offset * element_size) ? ryzenai_corelib_status_success : ryzenai_corelib_status_bad_argument; } ryzenai_corelib_status RecordingTensorRead( ryzenai_corelib_tensor_ptr tensor, + ryzenai_corelib_data_type destination_type, void* destination, - std::size_t size, + std::size_t count, std::size_t offset) { auto& state = State(); auto* value = state.Tensor(tensor); - if (value == nullptr) { + if (value == nullptr || destination == nullptr) { + return ryzenai_corelib_status_bad_argument; + } + const std::size_t element_size = TypeSize(value->data_type); + const std::size_t tensor_elements = value->byte_size / element_size; + if (offset > tensor_elements || count > tensor_elements - offset) { return ryzenai_corelib_status_bad_argument; } + const std::size_t size = count * element_size; if (state.v_tensors.contains(value)) { if (state.failure == FailurePoint::ScatterBadAlloc) { @@ -1175,9 +1161,8 @@ ryzenai_corelib_status RecordingTensorRead( state.events.emplace_back("tensor_read_v"); const std::size_t expected = static_cast(state.active_rows) * - static_cast(constants::kKvDimension) * - sizeof(std::uint16_t); - if (offset != 0 || size != expected) { + static_cast(constants::kKvDimension); + if (offset != 0 || count != expected) { state.host_read_poison_observed = true; } } else if ( @@ -1192,23 +1177,36 @@ ryzenai_corelib_status RecordingTensorRead( state.host_read_poison_observed = state.host_read_poison_observed || - value->ContainsPoisonWords( - offset / sizeof(std::uint16_t), - size / sizeof(std::uint16_t)); - return value->Read(destination, size, offset) - ? ryzenai_corelib_status_success - : ryzenai_corelib_status_bad_argument; + (element_size == sizeof(std::uint16_t) && + value->ContainsPoisonWords(offset, count)); + + std::vector staged(size); + if (!value->Read(staged.data(), size, offset * element_size)) { + return ryzenai_corelib_status_bad_argument; + } + for (std::size_t index = 0; index < count; ++index) { + WriteElement( + destination_type, + destination, + index, + ReadElement(value->data_type, staged.data(), index)); + } + return ryzenai_corelib_status_success; } ryzenai_corelib_status RecordingMatMulWeightsCreate( const ryzenai_corelib_matmul_bf16_weights_desc* desc, - const ryzenai_corelib_matmul_bf16_onnx_weights_components* components, + const ryzenai_corelib_matmul_bf16_onnx_components* components, + uint32_t threads, ryzenai_corelib_matmul_bf16_weights_ptr* out) { auto& state = State(); state.ObserveLoadThread(); if (desc == nullptr || components == nullptr || out == nullptr) { return ryzenai_corelib_status_bad_argument; } + if (threads != 0) { + return ryzenai_corelib_status_bad_argument; + } const auto ordinal = state.matmul_weight_count++; *out = state.Create( MatMulLabel(ordinal), @@ -1218,13 +1216,17 @@ ryzenai_corelib_status RecordingMatMulWeightsCreate( ryzenai_corelib_status RecordingSsMlpWeightsCreate( const ryzenai_corelib_ssmlp_bf16_weights_desc* desc, - const ryzenai_corelib_ssmlp_bf16_onnx_weights_components* components, + const ryzenai_corelib_ssmlp_bf16_onnx_components* components, + uint32_t threads, ryzenai_corelib_ssmlp_bf16_weights_ptr* out) { auto& state = State(); state.ObserveLoadThread(); if (desc == nullptr || components == nullptr || out == nullptr) { return ryzenai_corelib_status_bad_argument; } + if (threads != 0) { + return ryzenai_corelib_status_bad_argument; + } const auto ordinal = state.ssmlp_weight_count++; *out = state.Create( "ssmlp_" + std::to_string(ordinal), @@ -1552,12 +1554,9 @@ std::shared_ptr ResolveRecordingCorelib( resolver["ryzenai_corelib_tensor_get_byte_size"] = FunctionAddress( static_cast( &RecordingTensorGetByteSize)); - resolver["ryzenai_corelib_convert"] = FunctionAddress( - static_cast( - &RecordingConvert)); - resolver["ryzenai_corelib_convert_strided"] = FunctionAddress( - static_cast( - &RecordingConvertStrided)); + resolver["ryzenai_corelib_tensor_get_data_type"] = FunctionAddress( + static_cast( + &RecordingTensorGetDataType)); resolver["ryzenai_corelib_matmul_bf16_pad_shape"] = FunctionAddress( static_cast< @@ -1573,23 +1572,21 @@ std::shared_ptr ResolveRecordingCorelib( static_cast< decltype(&::ryzenai_corelib_flat_mha_bf16_pad_rows)>( &RecordingMhaPadRows)); - resolver - ["ryzenai_corelib_matmul_bf16_weights_create_from_onnx_components"] = - FunctionAddress( - static_cast( - &RecordingMatMulWeightsCreate)); + resolver["ryzenai_corelib_matmul_bf16_weights_create_onnx"] = + FunctionAddress( + static_cast( + &RecordingMatMulWeightsCreate)); resolver["ryzenai_corelib_matmul_bf16_weights_get_data"] = FunctionAddress( static_cast( &RecordingMatMulWeightsGetData)); - resolver - ["ryzenai_corelib_ssmlp_bf16_weights_create_from_onnx_components"] = - FunctionAddress( - static_cast( - &RecordingSsMlpWeightsCreate)); + resolver["ryzenai_corelib_ssmlp_bf16_weights_create_onnx"] = + FunctionAddress( + static_cast( + &RecordingSsMlpWeightsCreate)); resolver["ryzenai_corelib_ssmlp_bf16_weights_get_data"] = FunctionAddress( static_cast(PaddedRows(3)) * - static_cast(constants::kHiddenSize) * - sizeof(std::uint16_t); - CHECK(fixture.state.stage_writes[0].size == - padded_hidden_bytes); - CHECK(fixture.state.stage_writes[1].size == - padded_hidden_bytes); + static_cast(constants::kHiddenSize); + CHECK(fixture.state.stage_writes[0].count == + padded_hidden_elements); + CHECK(fixture.state.stage_writes[1].count == + padded_hidden_elements); CHECK(!fixture.state.input_poison_observed); CHECK(!fixture.state.host_read_poison_observed); CHECK(!fixture.state.cache_publish_poison_observed); @@ -2052,9 +2048,8 @@ void TestOrderBuffersTailsStateAndMetrics( })); CHECK(fixture.state.stage_writes.size() == 2); CHECK( - fixture.state.stage_writes[0].size == - static_cast(constants::kHiddenSize) * - sizeof(std::uint16_t)); + fixture.state.stage_writes[0].count == + static_cast(constants::kHiddenSize)); fixture.engine->clear_context(); fixture.state.ResetExecutionRecords(); @@ -2152,13 +2147,11 @@ void TestDivergentPaddingGrids(const SyntheticPackage& package) { 1); CHECK(fixture.state.stage_writes.size() == 2); CHECK( - fixture.state.stage_writes[0].size == - 4u * static_cast(constants::kHiddenSize) * - sizeof(std::uint16_t)); + fixture.state.stage_writes[0].count == + 4u * static_cast(constants::kHiddenSize)); CHECK( - fixture.state.stage_writes[1].size == - 8u * static_cast(constants::kHiddenSize) * - sizeof(std::uint16_t)); + fixture.state.stage_writes[1].count == + 8u * static_cast(constants::kHiddenSize)); const auto& metrics = fixture.engine->metrics(); CHECK(metrics.padding_write_calls == 97); CHECK(metrics.attention_extent_queries == 1); @@ -2207,13 +2200,11 @@ void TestDivergentPaddingGrids(const SyntheticPackage& package) { 1); CHECK(fixture.state.stage_writes.size() == 2); CHECK( - fixture.state.stage_writes[0].size == - 8u * static_cast(constants::kHiddenSize) * - sizeof(std::uint16_t)); + fixture.state.stage_writes[0].count == + 8u * static_cast(constants::kHiddenSize)); CHECK( - fixture.state.stage_writes[1].size == - 4u * static_cast(constants::kHiddenSize) * - sizeof(std::uint16_t)); + fixture.state.stage_writes[1].count == + 4u * static_cast(constants::kHiddenSize)); const auto& metrics = fixture.engine->metrics(); CHECK(metrics.padding_write_calls == 64); CHECK(metrics.attention_extent_queries == 1); diff --git a/src/test/phi4_corelib_aie4/test_phi4_host.cpp b/src/test/phi4_corelib_aie4/test_phi4_host.cpp index 7da7c865..24f62dc8 100644 --- a/src/test/phi4_corelib_aie4/test_phi4_host.cpp +++ b/src/test/phi4_corelib_aie4/test_phi4_host.cpp @@ -1,9 +1,12 @@ #include "fake_corelib.hpp" #include "test_support.hpp" +#include #include #include +#include + #include #include #include @@ -27,39 +30,30 @@ using flm::phi4::ArgmaxLowest; using flm::phi4::GatherEmbedding; using flm::phi4::RmsNorm; using flm::phi4::ScatterV; -using flm::phi4::StageBf16; +using flm::phi4::StageFp32; using flm::phi4::VScatterMetrics; using bf16 = biovault::bfloat16_t; constexpr std::uint16_t kPoison = 0xDEADu; -struct ConvertCall { - ryzenai_corelib_data_type source_type; - const void* source; - ryzenai_corelib_data_type destination_type; - void* destination; - std::size_t count; - std::vector fp16_source; - std::vector fp32_source_bits; -}; - struct ReadCall { ryzenai_corelib_tensor_ptr tensor; + ryzenai_corelib_data_type destination_type; void* destination; - std::size_t size; + std::size_t count; std::size_t offset; }; struct WriteCall { ryzenai_corelib_tensor_ptr tensor; + ryzenai_corelib_data_type source_type; const void* source; - std::size_t size; + std::size_t count; std::size_t offset; std::vector values; }; struct RecordingState { - std::vector converts; std::vector reads; std::vector writes; std::size_t synchronize_calls = 0; @@ -76,7 +70,6 @@ struct RecordingState { } void ResetCalls() { - converts.clear(); reads.clear(); writes.clear(); synchronize_calls = 0; @@ -85,133 +78,27 @@ struct RecordingState { RecordingState g_recording; -bool FixtureFp16ToFp32(std::uint16_t bits, float& value) { - switch (bits) { - case 0x0000u: - value = 0.0f; - return true; - case 0x3800u: - value = 0.5f; - return true; - case 0x3C00u: - value = 1.0f; - return true; - case 0x4000u: - value = 2.0f; - return true; - case 0x4200u: - value = 3.0f; - return true; - case 0xBC00u: - value = -1.0f; - return true; - case 0xC000u: - value = -2.0f; - return true; - default: - return false; - } -} - -bool FixtureFp32ToBf16(std::uint32_t bits, std::uint16_t& value) { - switch (bits) { - case 0x00000000u: - value = 0x0000u; - return true; - case 0x3F000000u: - value = 0x3F00u; - return true; - case 0x3F800000u: - value = 0x3F80u; - return true; - case 0x3F808000u: - value = 0x3F80u; - return true; - case 0x3F818000u: - value = 0x3F82u; - return true; - case 0x40000000u: - value = 0x4000u; - return true; - case 0xBF800000u: - value = 0xBF80u; - return true; - case 0xC0000000u: - value = 0xC000u; - return true; - default: - return false; - } -} - -ryzenai_corelib_status RecordingConvert( - ryzenai_corelib_data_type source_type, - const void* source, - ryzenai_corelib_data_type destination_type, - void* destination, - std::size_t count) { - if ((source == nullptr || destination == nullptr) && count != 0) { - return ryzenai_corelib_status_bad_argument; - } - - ConvertCall call{ - source_type, - source, - destination_type, - destination, - count, - {}, - {}}; - if (source_type == ryzenai_corelib_data_type_fp16 && - destination_type == ryzenai_corelib_data_type_fp32) { - const auto* input = - static_cast(source); - auto* output = static_cast(destination); - call.fp16_source.assign(input, input + count); - for (std::size_t index = 0; index < count; ++index) { - if (!FixtureFp16ToFp32(input[index], output[index])) { - return ryzenai_corelib_status_bad_argument; - } - } - } else if ( - source_type == ryzenai_corelib_data_type_fp32 && - destination_type == ryzenai_corelib_data_type_bf16) { - const auto* input = static_cast(source); - auto* output = static_cast(destination); - call.fp32_source_bits.reserve(count); - for (std::size_t index = 0; index < count; ++index) { - const auto bits = std::bit_cast(input[index]); - call.fp32_source_bits.push_back(bits); - if (!FixtureFp32ToBf16(bits, output[index])) { - return ryzenai_corelib_status_bad_argument; - } - } - } else { - return ryzenai_corelib_status_bad_argument; - } - g_recording.converts.push_back(std::move(call)); - return ryzenai_corelib_status_success; -} - +// `count` and `offset` are BF16 ELEMENTS of the V tensor, never bytes. A +// caller that still passes bytes asks for twice the elements and is +// rejected here rather than reading past the source. ryzenai_corelib_status RecordingTensorRead( ryzenai_corelib_tensor_ptr tensor, + ryzenai_corelib_data_type destination_type, void* destination, - std::size_t size, + std::size_t count, std::size_t offset) { if (tensor != g_recording.v_tensor() || destination == nullptr || - offset > g_recording.v_source.size() * sizeof(std::uint16_t) || - size > g_recording.v_source.size() * sizeof(std::uint16_t) - - offset) { + destination_type != ryzenai_corelib_data_type_bf16 || + offset > g_recording.v_source.size() || + count > g_recording.v_source.size() - offset) { return ryzenai_corelib_status_bad_argument; } g_recording.reads.push_back( - ReadCall{tensor, destination, size, offset}); + ReadCall{tensor, destination_type, destination, count, offset}); std::memcpy( destination, - reinterpret_cast( - g_recording.v_source.data()) + - offset, - size); + g_recording.v_source.data() + offset, + count * sizeof(std::uint16_t)); const auto tick = std::chrono::steady_clock::now(); while (std::chrono::steady_clock::now() == tick) { @@ -221,22 +108,28 @@ ryzenai_corelib_status RecordingTensorRead( ryzenai_corelib_status RecordingTensorWrite( ryzenai_corelib_tensor_ptr tensor, + ryzenai_corelib_data_type source_type, const void* source, - std::size_t size, + std::size_t count, std::size_t offset) { + constexpr std::size_t cache_elements = + static_cast(flm::phi4::constants::kKvHeadCount) * + static_cast( + flm::phi4::constants::kMaxSequenceLength) * + static_cast(flm::phi4::constants::kHeadSize); if (tensor != g_recording.v_cache() || source == nullptr || - size % sizeof(std::uint16_t) != 0) { + source_type != ryzenai_corelib_data_type_bf16 || + offset > cache_elements || count > cache_elements - offset) { return ryzenai_corelib_status_bad_argument; } const auto* values = static_cast(source); g_recording.writes.push_back(WriteCall{ tensor, + source_type, source, - size, + count, offset, - std::vector( - values, - values + size / sizeof(std::uint16_t))}); + std::vector(values, values + count)}); return ryzenai_corelib_status_success; } @@ -253,10 +146,6 @@ void* FunctionAddress(Function function) { std::shared_ptr ResolveRecordingCorelib() { auto resolver = flm::test::CompleteCorelibResolver(); - resolver["ryzenai_corelib_convert"] = - FunctionAddress( - static_cast( - &RecordingConvert)); resolver["ryzenai_corelib_tensor_read"] = FunctionAddress( static_cast( @@ -278,8 +167,121 @@ std::shared_ptr ResolveRecordingCorelib() { }); } -void TestGatherEmbeddingUsesOneReusableContiguousConversion( - const std::shared_ptr& api) { +// The two host conversions design API-6 permits, and only those two. +// This one is lossless, so it is checked against exact values. +void TestWidenFp16IsExactAcrossTheFp16Range() { + struct Case { + std::uint16_t bits; + float value; + }; + constexpr Case kCases[]{ + {0x0000u, 0.0f}, + {0x8000u, -0.0f}, + {0x3C00u, 1.0f}, + {0xBC00u, -1.0f}, + {0x4000u, 2.0f}, + {0xC000u, -2.0f}, + {0x4200u, 3.0f}, + {0x3800u, 0.5f}, + {0x0001u, 5.9604645e-8f}, // smallest subnormal + {0x03FFu, 6.0975552e-5f}, // largest subnormal + {0x0400u, 6.1035156e-5f}, // smallest normal + {0x7BFFu, 65504.0f}, // largest finite + }; + for (const auto& item : kCases) { + CHECK(flm::corelib::WidenFp16(item.bits) == item.value); + } + CHECK(std::isinf(flm::corelib::WidenFp16(0x7C00u))); + CHECK(flm::corelib::WidenFp16(0x7C00u) > 0.0f); + CHECK(std::isinf(flm::corelib::WidenFp16(0xFC00u))); + CHECK(std::isnan(flm::corelib::WidenFp16(0x7E00u))); + + // Every representable FP16 round-trips through float, which is what + // "lossless, so it has no rounding policy" means in practice. + for (std::uint32_t bits = 0; bits <= 0xFFFFu; ++bits) { + const auto narrow = static_cast(bits); + const float widened = flm::corelib::WidenFp16(narrow); + if (std::isnan(widened)) { + continue; + } + const auto exponent = (narrow >> 10) & 0x1Fu; + const auto mantissa = narrow & 0x3FFu; + const bool negative = (narrow & 0x8000u) != 0; + double expected = 0.0; + if (exponent == 0u) { + expected = std::ldexp(static_cast(mantissa), -24); + } else if (exponent == 0x1Fu) { + continue; + } else { + expected = std::ldexp( + 1.0 + static_cast(mantissa) / 1024.0, + static_cast(exponent) - 15); + } + if (negative) { + expected = -expected; + } + CHECK(static_cast(widened) == expected); + } +} + +// The reference driver's to_bf16, transcribed. FastFlow's helper must +// agree with it bit for bit; there is no second BF16 rounding policy. +std::uint16_t ReferenceToBf16(std::uint32_t bits) { + const std::uint64_t rounded = + static_cast(bits) + 0x7FFFull + + ((static_cast(bits) >> 16) & 1ull); + return static_cast(rounded >> 16); +} + +void TestNarrowFp32ToBf16MatchesTheReferenceDriverBitForBit() { + constexpr std::uint32_t kExact[]{ + 0x00000000u, + 0x80000000u, + 0x3F800000u, + 0x3F808000u, // exact tie, rounds to even + 0x3F818000u, // exact tie, rounds up to even + 0xBF800000u, + 0x40000000u, + 0xC0000000u, + 0x3F000000u, + 0x322BCC77u, // 1e-8f + 0x3727C5ACu, // 1e-5f, the Phi-4 RMS epsilon + 0x7F800000u, + 0xFF800000u, + 0x7FC00000u, + 0xFFFFFFFFu, // the wrap the driver hides in uint64 + 0xFFFF8000u, + }; + for (const std::uint32_t bits : kExact) { + CHECK( + flm::corelib::NarrowFp32ToBf16(std::bit_cast(bits)) == + ReferenceToBf16(bits)); + } + + // A dense sweep of the low mantissa bits, where the tie-breaking + // actually differs between truncation and round-to-nearest-even. + for (std::uint32_t low = 0; low < 0x20000u; ++low) { + const std::uint32_t bits = 0x3F800000u + low; + CHECK( + flm::corelib::NarrowFp32ToBf16(std::bit_cast(bits)) == + ReferenceToBf16(bits)); + } + // And a stride across the whole exponent range. + for (std::uint64_t bits = 0; bits <= 0xFFFFFFFFull; bits += 65413ull) { + const auto value = static_cast(bits); + CHECK( + flm::corelib::NarrowFp32ToBf16(std::bit_cast(value)) == + ReferenceToBf16(value)); + } + + CHECK( + flm::corelib::NarrowFp32ToBf16( + static_cast(flm::phi4::constants::kRmsEpsilon)) == + ReferenceToBf16(std::bit_cast( + static_cast(flm::phi4::constants::kRmsEpsilon)))); +} + +void TestGatherEmbeddingWidensWithoutCorelib() { constexpr std::size_t width = static_cast( flm::phi4::constants::kHiddenSize); std::vector embedding(3u * width); @@ -290,26 +292,8 @@ void TestGatherEmbeddingUsesOneReusableContiguousConversion( const std::array ids{2, 0}; std::vector output(ids.size() * width); g_recording.ResetCalls(); - GatherEmbedding(*api, embedding, ids, output); + GatherEmbedding(embedding, ids, output); - CHECK(g_recording.converts.size() == 1); - const auto& first_call = g_recording.converts.front(); - CHECK( - first_call.source_type == - ryzenai_corelib_data_type_fp16); - CHECK( - first_call.destination_type == - ryzenai_corelib_data_type_fp32); - CHECK(first_call.count == ids.size() * width); - CHECK(first_call.fp16_source.size() == ids.size() * width); - CHECK(std::all_of( - first_call.fp16_source.begin(), - first_call.fp16_source.begin() + width, - [](std::uint16_t value) { return value == 0x4200u; })); - CHECK(std::all_of( - first_call.fp16_source.begin() + width, - first_call.fp16_source.end(), - [](std::uint16_t value) { return value == 0x3C00u; })); CHECK(std::all_of( output.begin(), output.begin() + width, @@ -319,17 +303,61 @@ void TestGatherEmbeddingUsesOneReusableContiguousConversion( output.end(), [](float value) { return value == 1.0f; })); - const void* first_staging = first_call.source; const std::array second_ids{1}; - output.resize(width); - g_recording.ResetCalls(); - GatherEmbedding(*api, embedding, second_ids, output); - CHECK(g_recording.converts.size() == 1); - CHECK(g_recording.converts.front().source == first_staging); + output.assign(width, 0.0f); + GatherEmbedding(embedding, second_ids, output); CHECK(std::all_of( output.begin(), output.end(), [](float value) { return value == -2.0f; })); + + // No corelib call is made at all: the widening is FastFlow's own. + CHECK(g_recording.reads.empty()); + CHECK(g_recording.writes.empty()); +} + +// The embedding table is a read-only file mapping. A vectorized widening +// reads up to 14 bytes past its source, which faults instead of returning +// garbage -- so the gather must touch nothing beyond the last row. +void TestGatherEmbeddingStopsAtAGuardPage() { + constexpr std::size_t width = static_cast( + flm::phi4::constants::kHiddenSize); + constexpr std::size_t rows = 4; + constexpr std::size_t table_bytes = rows * width * sizeof(std::uint16_t); + static_assert(table_bytes % 4096u == 0u); + + auto* base = static_cast(VirtualAlloc( + nullptr, + table_bytes + 4096u, + MEM_RESERVE, + PAGE_NOACCESS)); + if (base == nullptr) { + throw std::runtime_error("failed to reserve guard-page range"); + } + if (VirtualAlloc(base, table_bytes, MEM_COMMIT, PAGE_READWRITE) == + nullptr) { + VirtualFree(base, 0, MEM_RELEASE); + throw std::runtime_error("failed to commit guard-page table"); + } + + auto* table = reinterpret_cast(base); + for (std::size_t index = 0; index < rows * width; ++index) { + table[index] = 0x3C00u; + } + const std::span embedding(table, rows * width); + + // The last row ends exactly at the boundary of the reserved, + // never-committed page that follows. + const std::array ids{static_cast(rows) - 1}; + std::vector output(width); + GatherEmbedding(embedding, ids, output); + const bool all_one = std::all_of( + output.begin(), + output.end(), + [](float value) { return value == 1.0f; }); + + VirtualFree(base, 0, MEM_RELEASE); + CHECK(all_one); } void TestRmsNormUsesFp32AccumulationAndSharedEpsilon() { @@ -365,8 +393,7 @@ void TestRmsNormUsesFp32AccumulationAndSharedEpsilon() { } } -void TestStageBf16UsesRneAndZerosOnlyInitialInputPrefixes( - const std::shared_ptr& api) { +void TestStageFp32ZerosOnlyInitialInputPrefixes() { const std::array normalized{ std::bit_cast(0x3F800000u), std::bit_cast(0x3F808000u), @@ -377,8 +404,9 @@ void TestStageBf16UsesRneAndZerosOnlyInitialInputPrefixes( std::bit_cast(0xC0000000u), std::bit_cast(0x00000000u), std::bit_cast(0x3F000000u)}; - std::vector hidden(8, kPoison); - std::vector residual_device(8, kPoison); + constexpr float kPoisonFloat = -12345.0f; + std::vector hidden(8, kPoisonFloat); + std::vector residual_device(8, kPoisonFloat); // Task 7 has no production consumer for q/k/attention/skip-sum/ // next-hidden padded tails. Task 8's dispatch test must poison those @@ -386,77 +414,42 @@ void TestStageBf16UsesRneAndZerosOnlyInitialInputPrefixes( // regions exclude stale values. g_recording.ResetCalls(); - StageBf16(*api, normalized, 2, 3, 2, hidden); - StageBf16(*api, residual, 2, 3, 2, residual_device); - - CHECK(g_recording.converts.size() == 2); - CHECK(g_recording.converts[0].count == 6); - CHECK(g_recording.converts[1].count == 6); - CHECK( - g_recording.converts[0].source_type == - ryzenai_corelib_data_type_fp32); - CHECK( - g_recording.converts[0].destination_type == - ryzenai_corelib_data_type_bf16); - CHECK( - g_recording.converts[0].source == - g_recording.converts[1].source); + StageFp32(normalized, 2, 3, 2, hidden); + StageFp32(residual, 2, 3, 2, residual_device); + + // Design 10.2: the host stays in FP32 and never rounds to BF16, so + // the staged bits are the input bits, unchanged. + const std::array expected_hidden{ + normalized[0], + normalized[1], + normalized[2], + normalized[3], + 0.0f, + 0.0f, + kPoisonFloat, + kPoisonFloat}; + const std::array expected_residual{ + residual[0], + residual[1], + residual[2], + residual[3], + 0.0f, + 0.0f, + kPoisonFloat, + kPoisonFloat}; + for (std::size_t index = 0; index < hidden.size(); ++index) { + CHECK( + std::bit_cast(hidden[index]) == + std::bit_cast(expected_hidden[index])); + CHECK( + std::bit_cast(residual_device[index]) == + std::bit_cast(expected_residual[index])); + } - constexpr std::array expected_hidden_source{ - 0x3F800000u, - 0x3F808000u, - 0x3F818000u, - 0xBF800000u, - 0x00000000u, - 0x00000000u}; - constexpr std::array expected_residual_source{ - 0x40000000u, - 0xC0000000u, - 0x00000000u, - 0x3F000000u, - 0x00000000u, - 0x00000000u}; - CHECK( - std::equal( - g_recording.converts[0].fp32_source_bits.begin(), - g_recording.converts[0].fp32_source_bits.end(), - expected_hidden_source.begin(), - expected_hidden_source.end())); - CHECK( - std::equal( - g_recording.converts[1].fp32_source_bits.begin(), - g_recording.converts[1].fp32_source_bits.end(), - expected_residual_source.begin(), - expected_residual_source.end())); - - constexpr std::array expected_hidden{ - 0x3F80u, - 0x3F80u, - 0x3F82u, - 0xBF80u, - 0x0000u, - 0x0000u, - kPoison, - kPoison}; - constexpr std::array expected_residual{ - 0x4000u, - 0xC000u, - 0x0000u, - 0x3F00u, - 0x0000u, - 0x0000u, - kPoison, - kPoison}; - CHECK(std::equal( - hidden.begin(), - hidden.end(), - expected_hidden.begin(), - expected_hidden.end())); - CHECK(std::equal( - residual_device.begin(), - residual_device.end(), - expected_residual.begin(), - expected_residual.end())); + // Nothing crossed the corelib boundary: the FP32-to-BF16 narrowing on + // this path is corelib's, inside tensor_write. + CHECK(g_recording.reads.empty()); + CHECK(g_recording.writes.empty()); } std::uint16_t VValue( @@ -493,16 +486,18 @@ void CheckScatterWrites( CHECK(g_recording.writes.size() == heads); for (std::size_t head = 0; head < heads; ++head) { const auto& write = g_recording.writes[head]; + // API-7: elements of the cache's BF16 dtype, not bytes. The + // previous spelling multiplied both by sizeof(uint16_t) and was + // wrong by exactly 2x. const std::size_t expected_offset = ((head * 4096u) + static_cast(position)) * - width * sizeof(std::uint16_t); + width; CHECK(write.tensor == g_recording.v_cache()); + CHECK(write.source_type == ryzenai_corelib_data_type_bf16); CHECK(write.offset == expected_offset); CHECK( - write.size == - static_cast(rows) * width * - sizeof(std::uint16_t)); + write.count == static_cast(rows) * width); CHECK( write.values.size() == static_cast(rows) * width); @@ -540,12 +535,14 @@ void TestScatterVReadsOnlyLiveRowsWithoutHiddenSynchronize( staging, metrics); - const std::size_t read_size = - 3u * heads * width * sizeof(std::uint16_t); + const std::size_t read_elements = 3u * heads * width; CHECK(g_recording.reads.size() == 1); CHECK(g_recording.reads.front().tensor == g_recording.v_tensor()); - CHECK(g_recording.reads.front().size == read_size); + CHECK( + g_recording.reads.front().destination_type == + ryzenai_corelib_data_type_bf16); + CHECK(g_recording.reads.front().count == read_elements); CHECK(g_recording.reads.front().offset == 0); CHECK(g_recording.reads.front().destination == staging.data()); CHECK(g_recording.synchronize_calls == 0); @@ -557,14 +554,14 @@ void TestScatterVReadsOnlyLiveRowsWithoutHiddenSynchronize( const auto* source = static_cast(write.source); CHECK(source >= staging_begin); - CHECK( - source + - write.size / sizeof(std::uint16_t) <= - staging_end); + CHECK(source + write.count <= staging_end); } CHECK(metrics.read_calls == 1); CHECK(metrics.write_calls == 8); - CHECK(metrics.bytes == 2u * read_size); + // Metrics stay in bytes; only the transfer arguments are elements. + CHECK( + metrics.bytes == + 2u * read_elements * sizeof(std::uint16_t)); CHECK(metrics.nanoseconds > 0); CHECK(std::all_of( g_recording.v_source.begin() + 3u * heads * width, @@ -586,17 +583,15 @@ void TestScatterVReadsOnlyLiveRowsWithoutHiddenSynchronize( CHECK(staging.data() == first_data); CHECK(staging.capacity() == first_capacity); CHECK(g_recording.reads.size() == 1); - CHECK( - g_recording.reads.front().size == - heads * width * sizeof(std::uint16_t)); + CHECK(g_recording.reads.front().count == heads * width); CHECK(g_recording.synchronize_calls == 0); CheckScatterWrites(1, 20); CHECK(metrics.read_calls == 2); CHECK(metrics.write_calls == 16); CHECK( metrics.bytes == - 2u * read_size + - 2u * heads * width * sizeof(std::uint16_t)); + (2u * read_elements + 2u * heads * width) * + sizeof(std::uint16_t)); CHECK(metrics.nanoseconds > first_ns); } @@ -621,24 +616,19 @@ void TestInvalidHostArgumentsFailBeforeCorelib( std::vector embedding(width); const std::array invalid_id{1}; std::vector embedding_output(width); - std::vector staged(4); + std::vector staged(4); std::vector scatter_staging; VScatterMetrics metrics{}; g_recording.ResetCalls(); CheckThrowsContains( [&] { - GatherEmbedding( - *api, - embedding, - invalid_id, - embedding_output); + GatherEmbedding(embedding, invalid_id, embedding_output); }, "token ID"); CheckThrowsContains( [&] { - StageBf16( - *api, + StageFp32( std::span{embedding_output}.first(2), 2, 1, @@ -677,7 +667,6 @@ void TestInvalidHostArgumentsFailBeforeCorelib( (void)ArgmaxLowest(empty); }, "empty"); - CHECK(g_recording.converts.empty()); CHECK(g_recording.reads.empty()); CHECK(g_recording.writes.empty()); } @@ -687,9 +676,12 @@ void TestInvalidHostArgumentsFailBeforeCorelib( int main() { try { const auto api = ResolveRecordingCorelib(); - TestGatherEmbeddingUsesOneReusableContiguousConversion(api); + TestWidenFp16IsExactAcrossTheFp16Range(); + TestNarrowFp32ToBf16MatchesTheReferenceDriverBitForBit(); + TestGatherEmbeddingWidensWithoutCorelib(); + TestGatherEmbeddingStopsAtAGuardPage(); TestRmsNormUsesFp32AccumulationAndSharedEpsilon(); - TestStageBf16UsesRneAndZerosOnlyInitialInputPrefixes(api); + TestStageFp32ZerosOnlyInitialInputPrefixes(); TestScatterVReadsOnlyLiveRowsWithoutHiddenSynchronize(api); TestArgmaxLowestChoosesLowestTokenOnTie(); TestInvalidHostArgumentsFailBeforeCorelib(api); diff --git a/src/test/phi4_corelib_aie4/test_phi4_manifest.cpp b/src/test/phi4_corelib_aie4/test_phi4_manifest.cpp index 427c8eda..af525a6c 100644 --- a/src/test/phi4_corelib_aie4/test_phi4_manifest.cpp +++ b/src/test/phi4_corelib_aie4/test_phi4_manifest.cpp @@ -60,194 +60,6 @@ constexpr std::string_view kFp16Scale = constexpr std::string_view kFp32Norm = "model.layers.0.post_attention_layernorm.weight"; -struct ConvertRecord { - ryzenai_corelib_data_type source_type = - ryzenai_corelib_data_type_fp32; - const void* source = nullptr; - std::size_t src_stride = 0; - ryzenai_corelib_data_type destination_type = - ryzenai_corelib_data_type_fp32; - void* destination = nullptr; - std::size_t dst_stride = 0; - std::size_t count = 0; - std::size_t row = 0; -}; - -ConvertRecord g_last_convert; -std::size_t g_convert_calls = 0; - -float HalfToFloat(std::uint16_t value) { - const bool negative = (value & 0x8000u) != 0; - const unsigned exponent = (value >> 10) & 0x1fu; - const unsigned mantissa = value & 0x03ffu; - float result = 0.0f; - if (exponent == 0) { - result = std::ldexp(static_cast(mantissa), -24); - } else if (exponent == 31) { - result = mantissa == 0 - ? std::numeric_limits::infinity() - : std::numeric_limits::quiet_NaN(); - } else { - result = std::ldexp( - 1.0f + static_cast(mantissa) / 1024.0f, - static_cast(exponent) - 15); - } - return negative ? -result : result; -} - -std::uint16_t FloatToHalf(float value) { - const std::uint32_t bits = std::bit_cast(value); - const std::uint16_t sign = - static_cast((bits >> 16) & 0x8000u); - const std::uint32_t source_exponent = (bits >> 23) & 0xffu; - const std::uint32_t source_mantissa = bits & 0x007fffffu; - - if (source_exponent == 0xffu) { - return static_cast( - sign | (source_mantissa == 0 ? 0x7c00u : 0x7e00u)); - } - - const int exponent = static_cast(source_exponent) - 127 + 15; - if (exponent >= 31) { - return static_cast(sign | 0x7c00u); - } - if (exponent <= 0) { - if (exponent < -10) { - return sign; - } - const std::uint32_t mantissa = source_mantissa | 0x00800000u; - const unsigned shift = static_cast(14 - exponent); - const std::uint32_t halfway = 1u << (shift - 1); - const std::uint32_t rounded = - (mantissa + halfway - 1u + ((mantissa >> shift) & 1u)) >> - shift; - return static_cast(sign | rounded); - } - - const std::uint32_t rounded = - source_mantissa + 0x00000fffu + - ((source_mantissa >> 13) & 1u); - if ((rounded & 0x00800000u) != 0) { - if (exponent + 1 >= 31) { - return static_cast(sign | 0x7c00u); - } - return static_cast( - sign | (static_cast(exponent + 1) << 10)); - } - return static_cast( - sign | (static_cast(exponent) << 10) | - (rounded >> 13)); -} - -std::uint16_t FloatToBf16(float value) { - std::uint32_t bits = std::bit_cast(value); - bits += 0x7fffu + ((bits >> 16) & 1u); - return static_cast(bits >> 16); -} - -float ReadElement( - ryzenai_corelib_data_type type, - const void* source, - std::size_t index) { - if (type == ryzenai_corelib_data_type_fp16) { - return HalfToFloat( - static_cast(source)[index]); - } - if (type == ryzenai_corelib_data_type_fp32) { - return static_cast(source)[index]; - } - throw std::runtime_error("test converter received unsupported source type"); -} - -void WriteElement( - ryzenai_corelib_data_type type, - void* destination, - std::size_t index, - float value) { - if (type == ryzenai_corelib_data_type_fp16) { - static_cast(destination)[index] = - FloatToHalf(value); - return; - } - if (type == ryzenai_corelib_data_type_bf16) { - static_cast(destination)[index] = - FloatToBf16(value); - return; - } - if (type == ryzenai_corelib_data_type_fp32) { - static_cast(destination)[index] = value; - return; - } - throw std::runtime_error( - "test converter received unsupported destination type"); -} - -ryzenai_corelib_status RecordingConvert( - ryzenai_corelib_data_type source_type, - const void* source, - ryzenai_corelib_data_type destination_type, - void* destination, - std::size_t count) { - g_last_convert = { - source_type, - source, - 0, - destination_type, - destination, - 0, - count, - 0}; - ++g_convert_calls; - for (std::size_t index = 0; index < count; ++index) { - WriteElement( - destination_type, - destination, - index, - ReadElement(source_type, source, index)); - } - return ryzenai_corelib_status_success; -} - -ryzenai_corelib_status RecordingConvertStrided( - ryzenai_corelib_data_type source_type, - const void* source, - std::size_t source_stride, - ryzenai_corelib_data_type destination_type, - void* destination, - std::size_t destination_stride, - std::size_t count, - std::size_t row) { - g_last_convert = { - source_type, - source, - source_stride, - destination_type, - destination, - destination_stride, - count, - row}; - ++g_convert_calls; - if ( - row == 0 || count % row != 0 || source_stride < row || - destination_stride < row) { - return ryzenai_corelib_status_bad_argument; - } - const std::size_t rows = count / row; - for (std::size_t source_row = 0; source_row < rows; ++source_row) { - for (std::size_t column = 0; column < row; ++column) { - WriteElement( - destination_type, - destination, - source_row * destination_stride + column, - ReadElement( - source_type, - source, - source_row * source_stride + column)); - } - } - return ryzenai_corelib_status_success; -} - template void* FunctionAddress(Function function) { return reinterpret_cast(function); @@ -255,12 +67,6 @@ void* FunctionAddress(Function function) { std::shared_ptr ResolveRecordingCorelib() { auto resolver = flm::test::CompleteCorelibResolver(); - resolver["ryzenai_corelib_convert"] = FunctionAddress( - static_cast( - &RecordingConvert)); - resolver["ryzenai_corelib_convert_strided"] = FunctionAddress( - static_cast( - &RecordingConvertStrided)); return CorelibApi::ResolveForTest( [resolver = std::move(resolver)](std::string_view name) mutable -> void* { @@ -1164,44 +970,36 @@ void TestOwnedScaleAndNormConversions( fixture.Write(fixture.manifest()); auto package = Phi4Package::Load(fixture.path(), api, false); - g_convert_calls = 0; + // FP16 scales are an element-wise copy into a contiguous model-owned + // buffer, not a conversion. WEIGHT-2 still holds. const auto fp16 = package.MaterializeFp16(kFp16Scale); - CHECK(g_convert_calls == 1); - CHECK(g_last_convert.source_type == - ryzenai_corelib_data_type_fp16); - CHECK(g_last_convert.destination_type == - ryzenai_corelib_data_type_fp16); - CHECK(g_last_convert.count == 1024u * 24u); CHECK(fp16.size() == 1024u * 24u); CHECK(fp16[0] == 0x3c00u); CHECK(fp16[1] == 0xc000u); CHECK( reinterpret_cast(fp16.data()) != - g_last_convert.source); + reinterpret_cast( + package.Require(kFp16Scale).data)); const auto* fp16_address = fp16.data(); const auto same_fp16 = package.MaterializeFp16(kFp16Scale); - CHECK(g_convert_calls == 1); CHECK(same_fp16.data() == fp16_address); - const auto converted = package.MaterializeFp16(kFp32Scale); - CHECK(g_convert_calls == 2); - CHECK(g_last_convert.source_type == - ryzenai_corelib_data_type_fp32); - CHECK(g_last_convert.destination_type == - ryzenai_corelib_data_type_fp16); - CHECK(g_last_convert.count == 3072u * 24u); - CHECK(converted[0] == 0x3c00u); - CHECK(converted[1] == 0xc000u); + // Design Section 9.3: an FP32 scales array is REJECTED, not narrowed. + // Narrowing it would need a host FP32-to-FP16 converter, which API-6 + // does not permit, so admitting one is a spec change. + CheckThrowsContains( + [&] { (void)package.MaterializeFp16(kFp32Scale); }, + "FP16"); + CheckThrowsContains( + [&] { (void)package.MaterializeFp16(kFp32Scale); }, + "rejected"); CHECK(fp16.data() == fp16_address); + // Norms are raw BF16 packer blobs, so they keep the API-6 host + // round-to-nearest-even helper. const auto bf16 = package.MaterializeBf16(kFp32Norm); - CHECK(g_convert_calls == 3); - CHECK(g_last_convert.source_type == - ryzenai_corelib_data_type_fp32); - CHECK(g_last_convert.destination_type == - ryzenai_corelib_data_type_bf16); - CHECK(g_last_convert.count == 3072); + CHECK(bf16.size() == 3072u); CHECK(bf16[0] == 0x3f80u); CHECK(bf16[1] == 0xc000u); @@ -1210,6 +1008,12 @@ void TestOwnedScaleAndNormConversions( (void)package.MaterializeFp16( "model.layers.0.attn.q_proj.MatMulNBits.qweight"); }, + "FP16"); + CheckThrowsContains( + [&] { + (void)package.MaterializeBf16( + "model.layers.0.attn.q_proj.MatMulNBits.qweight"); + }, "floating"); } @@ -1254,7 +1058,11 @@ class NoAccessGuard final { DWORD old_protection_ = 0; }; -void TestRopeUsesExactStridedContractAtGuardPage( +// Corelib e5258d2 removed convert_strided, so this slice is now +// FastFlow's own code -- which makes the guard page more important, not +// less. The last source row sits immediately before an inaccessible page, +// and the gather must take its 48 columns without touching the tail. +void TestRopeGatherStaysInSourceDtypeAtGuardPage( const SyntheticPackage& fixture, const std::shared_ptr& api) { fixture.Write(fixture.manifest()); @@ -1268,26 +1076,22 @@ void TestRopeUsesExactStridedContractAtGuardPage( const_cast(source.data + source.size); NoAccessGuard guard(one_past); - g_convert_calls = 0; - const auto rope = package.MaterializeRopeFp32("cos_cache"); - CHECK(g_convert_calls == 1); - CHECK(g_last_convert.source_type == - ryzenai_corelib_data_type_fp16); - CHECK(g_last_convert.destination_type == - ryzenai_corelib_data_type_fp32); - CHECK(g_last_convert.row == 48); - CHECK(g_last_convert.src_stride == kRopeColumns); - CHECK(g_last_convert.dst_stride == 48); - CHECK(g_last_convert.count == 4096u * 48u); - CHECK(rope.size() == 4096u * 48u); - CHECK(rope[0] == 1.0f); - CHECK(rope[48] == 2.0f); - CHECK(rope.back() == 3.0f); - - const auto* address = rope.data(); - const auto again = package.MaterializeRopeFp32("cos_cache"); - CHECK(g_convert_calls == 1); - CHECK(again.data() == address); + const auto rope = package.MaterializeRopeGather("cos_cache"); + // The gather preserves the SOURCE dtype: tensor_write does the + // widening to the FP32 device tensor, and this path performs no + // conversion of its own. + CHECK(rope.dtype == ryzenai_corelib_data_type_fp16); + CHECK(rope.count == 4096u * 48u); + const auto* elements = + static_cast(rope.data); + CHECK(elements[0] == 0x3c00u); + CHECK(elements[48] == 0x4000u); + CHECK(elements[4096u * 48u - 1u] == 0x4200u); + + const auto again = package.MaterializeRopeGather("cos_cache"); + CHECK(again.data == rope.data); + CHECK(again.dtype == rope.dtype); + CHECK(again.count == rope.count); } void TestFp32RopeSource( @@ -1296,16 +1100,10 @@ void TestFp32RopeSource( fixture.Write(fixture.manifest()); auto package = Phi4Package::Load(fixture.path(), api, false); - g_convert_calls = 0; - const auto rope = package.MaterializeRopeFp32("sin_cache"); - CHECK(g_convert_calls == 1); - CHECK(g_last_convert.source_type == - ryzenai_corelib_data_type_fp32); - CHECK(g_last_convert.src_stride == 48); - CHECK(g_last_convert.dst_stride == 48); - CHECK(g_last_convert.row == 48); - CHECK(g_last_convert.count == 196608); - CHECK(rope[0] == 4.0f); + const auto rope = package.MaterializeRopeGather("sin_cache"); + CHECK(rope.dtype == ryzenai_corelib_data_type_fp32); + CHECK(rope.count == 196608u); + CHECK(static_cast(rope.data)[0] == 4.0f); } static_assert(!std::is_copy_constructible_v); @@ -1328,7 +1126,7 @@ int main() { TestExactSourceValidation(fixture, api); TestComponentDiagnosticsIdentifyWeightObjects(fixture, api); TestOwnedScaleAndNormConversions(fixture, api); - TestRopeUsesExactStridedContractAtGuardPage(fixture, api); + TestRopeGatherStaysInSourceDtypeAtGuardPage(fixture, api); TestFp32RopeSource(fixture, api); std::cout << "test_phi4_manifest: PASS\n"; return 0; diff --git a/src/test/phi4_corelib_aie4/test_phi4_weights.cpp b/src/test/phi4_corelib_aie4/test_phi4_weights.cpp index 2c736610..506cda57 100644 --- a/src/test/phi4_corelib_aie4/test_phi4_weights.cpp +++ b/src/test/phi4_corelib_aie4/test_phi4_weights.cpp @@ -48,7 +48,6 @@ constexpr std::uint64_t kDataBytes = 200064ull * 3072ull * sizeof(std::uint16_t); constexpr std::size_t kMatMulPackedBytes = 17; constexpr std::size_t kSsMlpPackedBytes = 29; -constexpr std::uint16_t kBf16One = 0x3f80u; constexpr std::uint16_t kBf16Epsilon = 0x3728u; enum class FailurePoint { @@ -65,30 +64,20 @@ struct FakeWeightHandle { struct MatMulCreateRecord { ryzenai_corelib_matmul_bf16_weights_desc desc{}; - ryzenai_corelib_matmul_bf16_onnx_weights_components components{}; + ryzenai_corelib_matmul_bf16_onnx_components components{}; + std::uint32_t threads = 0; void* object = nullptr; std::thread::id thread; }; struct SsMlpCreateRecord { ryzenai_corelib_ssmlp_bf16_weights_desc desc{}; - ryzenai_corelib_ssmlp_bf16_onnx_weights_components components{}; + ryzenai_corelib_ssmlp_bf16_onnx_components components{}; + std::uint32_t threads = 0; void* object = nullptr; std::thread::id thread; }; -struct ConvertRecord { - ryzenai_corelib_data_type source_type = - ryzenai_corelib_data_type_fp32; - ryzenai_corelib_data_type destination_type = - ryzenai_corelib_data_type_fp32; - const void* source = nullptr; - void* destination = nullptr; - std::size_t count = 0; - float first_source_value = 0.0f; - std::thread::id thread; -}; - struct RecordingState { std::mutex mutex; std::weak_ptr current_package; @@ -100,7 +89,6 @@ struct RecordingState { std::size_t ssmlp_get_attempts = 0; std::vector matmul_creates; std::vector ssmlp_creates; - std::vector converts; std::vector creation_order; std::vector release_order; std::vector package_alive_at_release; @@ -129,55 +117,10 @@ bool ShouldFail( ordinal == State().failure_ordinal; } -ryzenai_corelib_status RecordingConvert( - ryzenai_corelib_data_type source_type, - const void* source, - ryzenai_corelib_data_type destination_type, - void* destination, - std::size_t count) { - if (source == nullptr || destination == nullptr || count == 0) { - return ryzenai_corelib_status_bad_argument; - } - - ConvertRecord record{ - source_type, - destination_type, - source, - destination, - count, - 0.0f, - std::this_thread::get_id()}; - if (source_type == ryzenai_corelib_data_type_fp32) { - record.first_source_value = - *static_cast(source); - } - - auto& state = State(); - { - std::lock_guard lock(state.mutex); - state.converts.push_back(record); - } - - auto* output = static_cast(destination); - if ( - destination_type == ryzenai_corelib_data_type_bf16 && - source_type == ryzenai_corelib_data_type_fp32 && - count == 1 && - std::abs(record.first_source_value - 1.0e-5f) < 1.0e-10f) { - output[0] = kBf16Epsilon; - } else if ( - destination_type == ryzenai_corelib_data_type_fp16 || - destination_type == ryzenai_corelib_data_type_bf16) { - output[0] = kBf16One; - } else { - return ryzenai_corelib_status_bad_argument; - } - return ryzenai_corelib_status_success; -} - ryzenai_corelib_status RecordingMatMulCreate( const ryzenai_corelib_matmul_bf16_weights_desc* desc, - const ryzenai_corelib_matmul_bf16_onnx_weights_components* components, + const ryzenai_corelib_matmul_bf16_onnx_components* components, + uint32_t threads, ryzenai_corelib_matmul_bf16_weights_ptr* out) { if (desc == nullptr || components == nullptr || out == nullptr) { return ryzenai_corelib_status_bad_argument; @@ -200,7 +143,11 @@ ryzenai_corelib_status RecordingMatMulCreate( auto* handle = new FakeWeightHandle{state.current_package}; *out = handle; state.matmul_creates.push_back( - {*desc, *components, handle, std::this_thread::get_id()}); + {*desc, + *components, + threads, + handle, + std::this_thread::get_id()}); state.creation_order.push_back(handle); return ryzenai_corelib_status_success; } @@ -238,7 +185,8 @@ ryzenai_corelib_status RecordingMatMulGetData( ryzenai_corelib_status RecordingSsMlpCreate( const ryzenai_corelib_ssmlp_bf16_weights_desc* desc, - const ryzenai_corelib_ssmlp_bf16_onnx_weights_components* components, + const ryzenai_corelib_ssmlp_bf16_onnx_components* components, + uint32_t threads, ryzenai_corelib_ssmlp_bf16_weights_ptr* out) { if (desc == nullptr || components == nullptr || out == nullptr) { return ryzenai_corelib_status_bad_argument; @@ -261,7 +209,11 @@ ryzenai_corelib_status RecordingSsMlpCreate( auto* handle = new FakeWeightHandle{state.current_package}; *out = handle; state.ssmlp_creates.push_back( - {*desc, *components, handle, std::this_thread::get_id()}); + {*desc, + *components, + threads, + handle, + std::this_thread::get_id()}); state.creation_order.push_back(handle); return ryzenai_corelib_status_success; } @@ -322,26 +274,21 @@ std::shared_ptr ResolveRecordingCorelib( resolver["ryzenai_corelib_object_release"] = FunctionAddress( static_cast( &RecordingRelease)); - resolver["ryzenai_corelib_convert"] = FunctionAddress( - static_cast( - &RecordingConvert)); - resolver - ["ryzenai_corelib_matmul_bf16_weights_create_from_onnx_components"] = - FunctionAddress( - static_cast( - &RecordingMatMulCreate)); + resolver["ryzenai_corelib_matmul_bf16_weights_create_onnx"] = + FunctionAddress( + static_cast( + &RecordingMatMulCreate)); resolver["ryzenai_corelib_matmul_bf16_weights_get_data"] = FunctionAddress( static_cast( &RecordingMatMulGetData)); - resolver - ["ryzenai_corelib_ssmlp_bf16_weights_create_from_onnx_components"] = - FunctionAddress( - static_cast( - &RecordingSsMlpCreate)); + resolver["ryzenai_corelib_ssmlp_bf16_weights_create_onnx"] = + FunctionAddress( + static_cast( + &RecordingSsMlpCreate)); resolver["ryzenai_corelib_ssmlp_bf16_weights_get_data"] = FunctionAddress( static_cast(actual.components.qzeros) == std::byte{0}); + // The synthetic data file is sparse, so a faithful element-wise copy + // reproduces its zeros. The old expectation was an artifact of the + // recording converter writing a constant. CHECK( *static_cast(actual.components.scales) == - kBf16One); + 0x0000u); + CHECK(actual.threads == 0u); } void CheckSsMlpComponents( @@ -771,7 +726,9 @@ void CheckSsMlpComponents( CHECK(actual.desc.k == 3072); CHECK(actual.desc.n == 8192); CHECK(actual.desc.group_size == 128); + CHECK(actual.threads == 0u); CHECK(actual.components.epsilon != nullptr); + // The API-6 host round-to-nearest-even helper, on 1e-5f. CHECK( *static_cast( actual.components.epsilon) == kBf16Epsilon); @@ -785,7 +742,7 @@ void CheckSsMlpComponents( CHECK(actual_pointer != package.Require(name).data); CHECK( *static_cast(actual_pointer) == - kBf16One); + 0x0000u); }; const auto check_projection = [&]( const void* qweight, @@ -917,26 +874,19 @@ void TestExactConstructionAndLifetime( "model.layers.32.final_norm_layernorm.weight") .data()); - CHECK(state.converts.size() == 290); - CHECK(std::count_if( - state.converts.begin(), - state.converts.end(), - [](const ConvertRecord& call) { - return call.destination_type == - ryzenai_corelib_data_type_fp16; - }) == 225); - CHECK(std::count_if( - state.converts.begin(), - state.converts.end(), - [](const ConvertRecord& call) { - return call.destination_type == - ryzenai_corelib_data_type_bf16; - }) == 65); + // Every packing call takes the header's "one thread" hint: design + // Section 19 defers concurrent packing to the caller. CHECK(std::all_of( - state.converts.begin(), - state.converts.end(), - [load_thread](const ConvertRecord& call) { - return call.thread == load_thread; + state.matmul_creates.begin(), + state.matmul_creates.end(), + [](const MatMulCreateRecord& call) { + return call.threads == 0u; + })); + CHECK(std::all_of( + state.ssmlp_creates.begin(), + state.ssmlp_creates.end(), + [](const SsMlpCreateRecord& call) { + return call.threads == 0u; })); CHECK(std::all_of( state.matmul_creates.begin(), @@ -984,7 +934,6 @@ void TestExactConstructionAndLifetime( *retained); retained.reset(); CHECK(!package_lifetime.expired()); - CHECK(state.converts.size() == 290); (void)moved; CHECK(creation_order.size() == 161); } @@ -1076,7 +1025,6 @@ void CheckLoadFailure( state.ssmlp_get_attempts = 0; state.matmul_creates.clear(); state.ssmlp_creates.clear(); - state.converts.clear(); state.creation_order.clear(); state.release_order.clear(); state.package_alive_at_release.clear(); @@ -1164,7 +1112,7 @@ void TestActionableFailures(const SyntheticPackage& fixture) { 70, 86, "model.layers.17.attn.k_proj.MatMulNBits", - "ryzenai_corelib_matmul_bf16_weights_create_from_onnx_components", + "ryzenai_corelib_matmul_bf16_weights_create_onnx", "intentional Task 6 MatMul create failure"); CheckLoadFailure( state, @@ -1184,7 +1132,7 @@ void TestActionableFailures(const SyntheticPackage& fixture) { 24, 119, "model.layers.23.ssmlp", - "ryzenai_corelib_ssmlp_bf16_weights_create_from_onnx_components", + "ryzenai_corelib_ssmlp_bf16_weights_create_onnx", "intentional Task 6 SSMLP create failure"); CheckLoadFailure( state, @@ -1241,6 +1189,38 @@ void TestNonCorelibObjectFailureRemainsDistinct( "expected packed-byte validation failure"); } +// Design Section 9.3: scales must be FP16 in an accepted package. An FP32 +// scales array is rejected at load with an actionable error rather than +// narrowed, because narrowing needs a host FP32-to-FP16 converter that +// API-6 does not permit. Admitting FP32 scales is a spec change. +void TestFp32ScalesAreRejectedWithAnActionableError() { + SyntheticPackage fp32_scales(true); + RecordingState state; + auto api = ResolveRecordingCorelib(state); + auto package = fp32_scales.Load(api); + state.current_package = package; + + try { + (void)Phi4Weights::Load(api, package); + } catch (const std::exception& error) { + const std::string_view message(error.what()); + CHECK( + message.find("model.layers.0.attn.q_proj.MatMulNBits") != + std::string_view::npos); + CHECK(message.find("FP16") != std::string_view::npos); + CHECK(message.find("rejected") != std::string_view::npos); + CHECK(message.find("repackage") != std::string_view::npos); + // The first object fails, so nothing is left behind. + CHECK(api->live_object_count() == 0); + package.reset(); + g_recording = nullptr; + return; + } + g_recording = nullptr; + throw std::runtime_error( + "an FP32 scales array must fail the Phi-4 weight load"); +} + static_assert(!std::is_copy_constructible_v); static_assert(!std::is_copy_assignable_v); static_assert(std::is_nothrow_move_constructible_v); @@ -1255,6 +1235,7 @@ int main() { TestMoveAssignmentReleasesBeforeOwners(fixture); TestActionableFailures(fixture); TestNonCorelibObjectFailureRemainsDistinct(fixture); + TestFp32ScalesAreRejectedWithAnActionableError(); std::cout << "test_phi4_weights: PASS\n"; return 0; } catch (const std::exception& error) { From e43468feb62c02dc5fae8fb8914e03a70bf1dc25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CChi?= Date: Tue, 1 Sep 2026 19:13:58 -0700 Subject: [PATCH 023/117] test: validate the corelib adapter against the real library Every other test in this suite runs against fake_ryzenai_corelib, which is FastFlow's own code. That validates FastFlow against FastFlow's model of corelib rather than against corelib -- and the model stayed green through a full ABI break, which is the gap this closes without needing hardware. The header documents three things as NPU-free, and those are what runs: get_version and selftest_dependencies; the three shape helpers; and has_device_context, which is recorded rather than asserted because the dev box has an NPU but is not the AIE4 target. The shape check is the load-bearing one. Against the real e5258d2 library the padded K/N equals the logical K/N for all three Phi-4 MatMul descriptors, and every helper leaves rows = 1 at 1 -- so MEM-5 and the Task 5 capacity model hold against the shipped kernel set, not just against a fake that returned what FastFlow expected. It also found that the fake did: identity padding, where the real grid is {1, 64, 128, 256, 512, 1024, 2048, 3072, 4096}, and an LM head that ships only 1 and 128 and REJECTS anything larger rather than rounding up. The fake now reproduces both, and the real test asserts the two agree, so a future kernel-set change surfaces here instead of on hardware. Configured-but-missing is a failure rather than a skip: a silently skipped ABI check is how a stale DLL stays hidden. Co-Authored-By: Claude Opus 5 (1M context) --- src/test/phi4_corelib_aie4/CMakeLists.txt | 26 ++ src/test/phi4_corelib_aie4/fake_corelib.cpp | 93 ++++- .../phi4_corelib_aie4/test_real_corelib.cpp | 386 ++++++++++++++++++ 3 files changed, 499 insertions(+), 6 deletions(-) create mode 100644 src/test/phi4_corelib_aie4/test_real_corelib.cpp diff --git a/src/test/phi4_corelib_aie4/CMakeLists.txt b/src/test/phi4_corelib_aie4/CMakeLists.txt index aaafd307..bbbdcdf7 100644 --- a/src/test/phi4_corelib_aie4/CMakeLists.txt +++ b/src/test/phi4_corelib_aie4/CMakeLists.txt @@ -111,8 +111,34 @@ add_executable(test_generation_limit ${FASTFLOW_SOURCE_DIR}/server/npu_access_manager.cpp) target_include_directories(test_generation_limit PRIVATE ${FASTFLOW_SOURCE_DIR}/include) +target_compile_definitions(test_generation_limit PRIVATE + FLM_TEST_SOURCE_DIR="${FASTFLOW_SOURCE_DIR}") add_test(NAME test_generation_limit COMMAND test_generation_limit) +# Validates FastFlow against the REAL corelib rather than the fake. It is +# a normal CTest test, and it SKIPS (passing, with a notice) only when the +# runtime directory is unset -- a configured-but-missing DLL fails, so a +# stale or absent runtime cannot hide behind a green suite. +set(RYZENAI_CORELIB_RUNTIME_DIR + "" + CACHE PATH + "Directory holding the real ryzenai_corelib.dll and its closure") +set(RYZENAI_CORELIB_EXTRA_DLL_DIRS + "" + CACHE STRING + "Semicolon-separated directories completing corelib's DLL closure") +string(REPLACE ";" "\\;" RYZENAI_CORELIB_EXTRA_DLL_DIRS_ESCAPED + "${RYZENAI_CORELIB_EXTRA_DLL_DIRS}") +add_corelib_host_test(test_real_corelib test_real_corelib.cpp) +target_compile_definitions(test_real_corelib PRIVATE + FLM_REAL_CORELIB_RUNTIME_DIR="${RYZENAI_CORELIB_RUNTIME_DIR}" + FLM_REAL_CORELIB_EXTRA_DLL_DIRS="${RYZENAI_CORELIB_EXTRA_DLL_DIRS_ESCAPED}") +if(RYZENAI_CORELIB_RUNTIME_DIR) + set_tests_properties(test_real_corelib PROPERTIES + ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:${XRT_LIB_DIR};PATH=path_list_prepend:${RYZENAI_CORELIB_RUNTIME_DIR}") +endif() + add_corelib_host_test(test_corelib_api test_corelib_api.cpp) add_corelib_host_test(test_phi4_manifest test_phi4_manifest.cpp) add_corelib_host_test(test_phi4_shape_plan test_phi4_shape_plan.cpp) diff --git a/src/test/phi4_corelib_aie4/fake_corelib.cpp b/src/test/phi4_corelib_aie4/fake_corelib.cpp index d444ff95..519d52a2 100644 --- a/src/test/phi4_corelib_aie4/fake_corelib.cpp +++ b/src/test/phi4_corelib_aie4/fake_corelib.cpp @@ -233,11 +233,65 @@ ryzenai_corelib_status FakeTensorGetDataType( return ryzenai_corelib_status_success; } +// The shipped AIE4 kernel grid, measured against the real e5258d2 library +// (see test_real_corelib, which asserts the two still agree). Padding is a +// pure lookup, so the fake can reproduce it exactly -- and it must: every +// other test in this suite sizes its buffers from these answers. +constexpr int64_t kShippedRowGrid[] = { + 1, 64, 128, 256, 512, 1024, 2048, 3072, 4096}; + +// The LM head ships far fewer M: 1, then 128, and nothing above it. An +// out-of-grid M is an ERROR rather than a larger answer, which is what +// stops a caller allocating for a shape no kernel serves. +constexpr int64_t kLmHeadRowGrid[] = {1, 128}; + +bool PadToGrid( + int64_t* m, + const int64_t* grid, + std::size_t grid_size) { + for (std::size_t index = 0; index < grid_size; ++index) { + if (*m <= grid[index]) { + *m = grid[index]; + return true; + } + } + return false; +} + ryzenai_corelib_status FakeMatmulPadShape( - int64_t*, - int64_t*, - int64_t*, + int64_t* m, + int64_t* k, + int64_t* n, uint32_t) { + if (k == nullptr || n == nullptr || *k <= 0 || *n <= 0) { + g_last_error = "matmul_bf16_pad_shape requires k and n"; + return ryzenai_corelib_status_bad_argument; + } + // K and N are never padded for the Phi-4 shapes; MEM-5 rests on that. + if (m == nullptr) { + return ryzenai_corelib_status_success; + } + if (*m <= 0) { + g_last_error = "matmul_bf16_pad_shape requires positive m"; + return ryzenai_corelib_status_bad_argument; + } + const bool is_lm_head = *n >= 200064; + const bool padded = + is_lm_head + ? PadToGrid( + m, + kLmHeadRowGrid, + sizeof(kLmHeadRowGrid) / sizeof(int64_t)) + : PadToGrid( + m, + kShippedRowGrid, + sizeof(kShippedRowGrid) / sizeof(int64_t)); + if (!padded) { + g_last_error = + "no valid padded M shape for AIE4 at m=" + + std::to_string(*m); + return ryzenai_corelib_status_unsupported; + } return ryzenai_corelib_status_success; } @@ -275,10 +329,21 @@ ryzenai_corelib_status FakeMatmul( } ryzenai_corelib_status FakeSsmlpPadRows( - int64_t*, + int64_t* m, int64_t, int64_t, uint32_t) { + if (m == nullptr || *m <= 0) { + g_last_error = "ssmlp_bf16_pad_rows requires positive m"; + return ryzenai_corelib_status_bad_argument; + } + if (!PadToGrid( + m, + kShippedRowGrid, + sizeof(kShippedRowGrid) / sizeof(int64_t))) { + g_last_error = "no valid padded row count for AIE4"; + return ryzenai_corelib_status_unsupported; + } return ryzenai_corelib_status_success; } @@ -318,8 +383,24 @@ ryzenai_corelib_status FakeSsmlp( } ryzenai_corelib_status FakeFlatMhaPadRows( - int64_t*, - const ryzenai_corelib_flat_mha_bf16_desc*) { + int64_t* m, + const ryzenai_corelib_flat_mha_bf16_desc* desc) { + if (m == nullptr || desc == nullptr || *m <= 0) { + g_last_error = "flat_mha_bf16_pad_rows requires m and a desc"; + return ryzenai_corelib_status_bad_argument; + } + // Decode is not padded: the token kernel runs one row and pads its KV + // window internally. + if (*m == 1) { + return ryzenai_corelib_status_success; + } + if (!PadToGrid( + m, + kShippedRowGrid, + sizeof(kShippedRowGrid) / sizeof(int64_t))) { + g_last_error = "no valid padded row count for AIE4"; + return ryzenai_corelib_status_unsupported; + } return ryzenai_corelib_status_success; } diff --git a/src/test/phi4_corelib_aie4/test_real_corelib.cpp b/src/test/phi4_corelib_aie4/test_real_corelib.cpp new file mode 100644 index 00000000..756022a7 --- /dev/null +++ b/src/test/phi4_corelib_aie4/test_real_corelib.cpp @@ -0,0 +1,386 @@ +// Validates FastFlow against the REAL ryzenai_corelib, off-hardware. +// +// Everything else in this suite runs against fake_ryzenai_corelib, which is +// FastFlow's own code -- so it validates FastFlow against FastFlow's model +// of corelib, not against corelib. That model stayed green through a full +// ABI break, which is exactly the gap this file closes. +// +// The header documents three things as needing no NPU, and those are what +// runs here: +// +// 1. selftest_dependencies -- "Allocates host memory only, no NPU." +// 2. matmul_bf16_pad_shape / ssmlp_bf16_pad_rows / flat_mha_bf16_pad_rows +// -- "Needs no NPU: it is a lookup over the shipped kernel set." +// 3. has_device_context -- recorded, not asserted: the development box +// has an NPU but is not the AIE4 target. +// +// The shape-plan check is the load-bearing one. The whole allocation +// strategy rests on MatMul padded K/N equalling logical K/N, and until now +// that was only "confirmed" against a fake that returns whatever FastFlow +// expects. + +#include "fake_corelib.hpp" +#include "test_support.hpp" + +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +namespace constants = flm::phi4::constants; + +using flm::corelib::CorelibApi; +using flm::phi4::Phi4ShapePlan; +using flm::phi4::RowUse; + +#if !defined(FLM_REAL_CORELIB_RUNTIME_DIR) +#define FLM_REAL_CORELIB_RUNTIME_DIR "" +#endif + +// Semicolon-separated directories holding parts of corelib's dependency +// closure that do not sit beside the DLL. On the dev box that is the conda +// prefix the build linked protobuf from; Task 12 is where the closure +// becomes self-contained. +#if !defined(FLM_REAL_CORELIB_EXTRA_DLL_DIRS) +#define FLM_REAL_CORELIB_EXTRA_DLL_DIRS "" +#endif + +// CorelibApi::Load uses LOAD_LIBRARY_SEARCH_DEFAULT_DIRS, which honours +// directories added here and deliberately ignores PATH. +void AddExtraDllDirectories(std::string_view directories) { + std::size_t start = 0; + while (start <= directories.size()) { + const std::size_t end = directories.find(';', start); + const std::string_view entry = directories.substr( + start, + end == std::string_view::npos ? std::string_view::npos + : end - start); + if (!entry.empty()) { + const std::filesystem::path directory(entry); + if (!std::filesystem::exists(directory)) { + throw std::runtime_error( + "extra corelib DLL directory does not exist: " + + directory.string()); + } + if (AddDllDirectory(directory.c_str()) == nullptr) { + throw std::runtime_error( + "AddDllDirectory failed for " + directory.string()); + } + std::cout << "added DLL directory " << directory.string() + << '\n'; + } + if (end == std::string_view::npos) { + break; + } + start = end + 1; + } +} + +struct NamedRowUse { + RowUse use; + std::string_view name; +}; + +constexpr std::array kRowUses{{ + {RowUse::QueryProjection, "query_projection"}, + {RowUse::KvProjection, "kv_projection"}, + {RowUse::Attention, "attention"}, + {RowUse::OutputProjection, "output_projection"}, + {RowUse::SsMlp, "ssmlp"}, + {RowUse::LmHead, "lm_head"}, +}}; + +// The three MatMul shapes a Phi-4 forward pass dispatches. +struct MatMulDescriptor { + std::int64_t k; + std::int64_t n; + std::string_view name; + // The LM head is only ever dispatched at one row (design 10.5), and + // the real kernel set has no padded M for it beyond that -- asking is + // an error, not a larger answer. Phi4ShapePlan queries it at 1 only. + bool multi_row; +}; + +constexpr std::array kMatMulDescriptors{{ + {constants::kHiddenSize, constants::kQueryDimension, "q/o_proj", true}, + {constants::kHiddenSize, constants::kKvDimension, "k/v_proj", true}, + {constants::kHiddenSize, + constants::kVocabularySize, + "lm_head", + false}, +}}; + +std::shared_ptr ResolveFakeCorelib() { + auto resolver = flm::test::CompleteCorelibResolver(); + return CorelibApi::ResolveForTest( + [resolver = std::move(resolver)](std::string_view name) mutable + -> void* { + const auto found = resolver.find(std::string(name)); + return found == resolver.end() ? nullptr : found->second; + }); +} + +void PrintTransitions( + std::string_view label, + const std::vector>& + transitions) { + std::cout << " " << label << ": " << transitions.size() + << " transitions"; + const std::size_t shown = + transitions.size() < 12u ? transitions.size() : 12u; + for (std::size_t index = 0; index < shown; ++index) { + std::cout << (index == 0 ? " [" : " ") << transitions[index].first + << "->" << transitions[index].second; + } + if (shown != 0) { + std::cout << (shown < transitions.size() ? " ...]" : "]"); + } + std::cout << " last=" << transitions.back().first << "->" + << transitions.back().second << '\n'; +} + +// 1. Dependency self-test, against the real DynamicDispatch and RyzenMM. +void CheckVersionAndSelftest(const std::shared_ptr& api) { + const auto compiled = flm::corelib::CompiledCorelibVersion(); + const auto& runtime = api->runtime_version(); + std::cout << "real corelib version: " + << flm::corelib::FormatCorelibVersion(runtime) + << " (compiled against " + << flm::corelib::FormatCorelibVersion(compiled) << ")\n"; + // Loading at all means the API-5 gate passed; restate it so a future + // relaxation of the gate cannot make this file vacuous. + CHECK(flm::corelib::IsCorelibVersionCompatible(compiled, runtime)); + + api->Check( + api->functions().selftest_dependencies(), + "ryzenai_corelib_selftest_dependencies"); + std::cout << "selftest_dependencies: ok\n"; +} + +// 3. Missing-device path, recorded rather than asserted. +void RecordDeviceContext(const std::shared_ptr& api) { + const bool has_context = api->functions().has_device_context(); + std::cout << "has_device_context: " + << (has_context ? "true" : "false") + << " (recorded, not asserted: this box is not the AIE4 " + "target)\n"; + if (!has_context) { + std::cout + << " note: tensors, weights and dispatch will fail with " + "unsupported here; padding and packing still work.\n"; + } +} + +// 2. Shape plan against the real kernel set. +void CheckRealShapePlan(const std::shared_ptr& api) { + // MEM-5 and the Task 5 capacity model rest on this: padded K/N must + // equal logical K/N for every Phi-4 MatMul, at every live row class. + // Phi4ShapePlan::Build enforces it over 1..4096 and throws by name if + // it ever fails, so building it IS the assertion. Restate it directly + // at the boundaries so the check does not depend on Build's internals. + for (const auto& descriptor : kMatMulDescriptors) { + for (const std::int64_t rows : + {std::int64_t{1}, + std::int64_t{2}, + std::int64_t{63}, + std::int64_t{64}, + std::int64_t{65}, + std::int64_t{2048}, + std::int64_t{4095}, + std::int64_t{4096}}) { + if (rows != 1 && !descriptor.multi_row) { + continue; + } + std::int64_t m = rows; + std::int64_t k = descriptor.k; + std::int64_t n = descriptor.n; + api->Check( + api->functions().matmul_pad_shape( + &m, + &k, + &n, + constants::kGroupSize), + "ryzenai_corelib_matmul_bf16_pad_shape"); + if (k != descriptor.k || n != descriptor.n) { + throw std::runtime_error( + "real corelib padded " + std::string(descriptor.name) + + " K/N at rows " + std::to_string(rows) + ": K " + + std::to_string(descriptor.k) + "->" + + std::to_string(k) + ", N " + + std::to_string(descriptor.n) + "->" + + std::to_string(n) + + "; MEM-5 and the capacity model assume K/N are " + "unchanged"); + } + CHECK(m >= rows); + } + } + std::cout << "matmul padded K/N equals logical K/N for all three " + "Phi-4 descriptors\n"; + + // rows = 1 must stay 1 on every helper: decode allocates a single row + // and the header says flat MHA pads its KV window instead. + for (const auto& descriptor : kMatMulDescriptors) { + std::int64_t m = 1; + std::int64_t k = descriptor.k; + std::int64_t n = descriptor.n; + api->Check( + api->functions().matmul_pad_shape( + &m, + &k, + &n, + constants::kGroupSize), + "ryzenai_corelib_matmul_bf16_pad_shape"); + std::cout << " matmul " << descriptor.name << " rows 1 -> " << m + << '\n'; + CHECK(m == 1); + } + { + std::int64_t m = 1; + api->Check( + api->functions().ssmlp_pad_rows( + &m, + constants::kHiddenSize, + constants::kIntermediateSize, + constants::kGroupSize), + "ryzenai_corelib_ssmlp_bf16_pad_rows"); + std::cout << " ssmlp rows 1 -> " << m << '\n'; + CHECK(m == 1); + } + { + const ryzenai_corelib_flat_mha_bf16_desc desc{ + constants::kQueryHeadCount, + constants::kKvHeadCount, + constants::kHeadSize, + constants::kMaxSequenceLength, + constants::kRopeDimension}; + std::int64_t m = 1; + api->Check( + api->functions().flat_mha_pad_rows(&m, &desc), + "ryzenai_corelib_flat_mha_bf16_pad_rows"); + std::cout << " flat_mha rows 1 -> " << m << '\n'; + CHECK(m == 1); + } + + const Phi4ShapePlan real_plan = Phi4ShapePlan::Build(api); + std::cout << "real Phi4ShapePlan over rows 1..4096:\n"; + for (const auto& row_use : kRowUses) { + PrintTransitions( + row_use.name, + real_plan.Transitions(row_use.use)); + } + std::cout << " capacities: layer_rows=" + << real_plan.capacities().layer_rows + << " lm_head_rows=" << real_plan.capacities().lm_head_rows + << '\n'; + CHECK(real_plan.capacities().layer_rows >= constants::kMaxSequenceLength); + CHECK(real_plan.capacities().lm_head_rows == 1); + CHECK(real_plan.RowsFor(RowUse::LmHead, 1) == 1); + + // The fake must reproduce the real library's answers exactly. A fake + // that cannot is not a test double, it is a second implementation of + // the same guess -- and every other file in this suite trusts it. + const Phi4ShapePlan fake_plan = + Phi4ShapePlan::Build(ResolveFakeCorelib()); + for (const auto& row_use : kRowUses) { + const auto& real_transitions = real_plan.Transitions(row_use.use); + const auto& fake_transitions = fake_plan.Transitions(row_use.use); + if (real_transitions != fake_transitions) { + std::cout << " DIVERGENCE " << row_use.name + << ": real has " << real_transitions.size() + << " transitions, fake has " + << fake_transitions.size() << '\n'; + PrintTransitions(" real", real_transitions); + PrintTransitions(" fake", fake_transitions); + throw std::runtime_error( + "fake_ryzenai_corelib disagrees with the real shipped " + "kernel set for " + std::string(row_use.name) + + "; fix the fake to match, then re-run the host suite"); + } + } + CHECK( + fake_plan.capacities().layer_rows == + real_plan.capacities().layer_rows); + CHECK( + fake_plan.capacities().lm_head_rows == + real_plan.capacities().lm_head_rows); + std::cout << "fake transition lists match the real ones\n"; + + // The engine only ever reads RowsFor(), so the property it depends on + // is that padding is monotonic and never below the live rows -- which + // is what makes a single peak allocation safe. + for (const auto& row_use : kRowUses) { + if (row_use.use == RowUse::LmHead) { + continue; + } + std::int64_t previous = 0; + for (std::int64_t rows = 1; + rows <= constants::kMaxSequenceLength; + ++rows) { + const std::int64_t padded = + real_plan.RowsFor(row_use.use, rows); + CHECK(padded >= rows); + CHECK(padded >= previous); + CHECK(padded <= real_plan.capacities().layer_rows); + previous = padded; + } + } + std::cout << "real padding is monotonic, never below live rows, and " + "never above the planned capacity\n"; +} + +} // namespace + +int main() { + const std::string runtime_dir(FLM_REAL_CORELIB_RUNTIME_DIR); + if (runtime_dir.empty()) { + std::cout + << "test_real_corelib: SKIPPED -- configure with " + "-DRYZENAI_CORELIB_RUNTIME_DIR= to run it.\n"; + return 0; + } + + try { + const std::filesystem::path library = + std::filesystem::absolute( + std::filesystem::path(runtime_dir) / + "ryzenai_corelib.dll") + .lexically_normal(); + if (!std::filesystem::exists(library)) { + // Configured but absent is a failure, not a skip: a silently + // skipped ABI check is how a stale DLL stays hidden. + throw std::runtime_error( + "RYZENAI_CORELIB_RUNTIME_DIR is set but " + + library.string() + " does not exist"); + } + AddExtraDllDirectories(FLM_REAL_CORELIB_EXTRA_DLL_DIRS); + std::cout << "loading " << library.string() << '\n'; + + auto api = CorelibApi::Load(library); + CheckVersionAndSelftest(api); + RecordDeviceContext(api); + CheckRealShapePlan(api); + + api->functions().cleanup(); + std::cout << "test_real_corelib: PASS\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << "test_real_corelib: FAIL: " << error.what() << '\n'; + return 1; + } +} From 96b008544644f2994be7d1af7ea47659f53265b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CChi?= Date: Tue, 1 Sep 2026 19:14:08 -0700 Subject: [PATCH 024/117] fix: route /api/chat through the shared generation limit Task 10 wired ParseGenerationLimit into handle_generate, handle_openai_chat_completion and handle_openai_completion. It missed handle_chat, which serves /api/chat and computed its limit through a separate path, OllamaChatGenerationLoopLimit. /api/chat is the Ollama-compatible endpoint and so likely the most used one. On the AIE4 tag it was bypassing the explicit-limit detection, the prompt_tokens + requested_max_new_tokens admission rule, the HTTP 400 path and session_cleared reporting -- silently capping at the legacy 4096 instead of running to the context cap, and never checking that an explicit num_predict fits. ParseGenerationLimit gains an OllamaChat case reading options.num_predict, and OllamaChatGenerationLoopLimit is now defined in terms of it, so the legacy and shared rules cannot drift. Legacy behaviour is unchanged: an omitted num_predict is still 4096 and an explicit one is still honoured. Three-of-four was found by inspection, so this also adds the rule. GenerationRoutes() declares every route that reaches the causal engine, each handler resolves its endpoint through it, and a test derives the set independently -- reading rest_handler.cpp for handlers that call generate()/generate_with_prompt(), mapping them to their server.cpp registrations, and failing if any is undeclared. A new generation endpoint now cannot reach the engine without declaring how its limit is parsed. Co-Authored-By: Claude Opus 5 (1M context) --- src/include/server/generation_limit.hpp | 28 +++ src/server/generation_limit.cpp | 61 ++++- src/server/rest_handler.cpp | 29 ++- .../test_generation_limit.cpp | 235 +++++++++++++++++- 4 files changed, 336 insertions(+), 17 deletions(-) diff --git a/src/include/server/generation_limit.hpp b/src/include/server/generation_limit.hpp index 45364301..4dc65d4e 100644 --- a/src/include/server/generation_limit.hpp +++ b/src/include/server/generation_limit.hpp @@ -5,15 +5,43 @@ #include #include #include +#include #include #include enum class GenerationEndpoint { Generate, + OllamaChat, OpenAiChatCompletion, OpenAiCompletion }; +// Every route whose handler reaches the causal engine's generate() or +// generate_with_prompt(), and the field each one reads its limit from. +// +// This table exists because /api/chat was missed. Three of the four +// endpoints were wired to ParseGenerationLimit by inspection, and +// inspection is what let the fourth keep its own private limit path -- +// bypassing explicit-limit detection, the admission rule, the HTTP 400 +// path and session_cleared reporting on the AIE4 tag. +struct GenerationRoute { + std::string_view method; + std::string_view path; + GenerationEndpoint endpoint; +}; + +std::span GenerationRoutes() noexcept; + +std::optional GenerationEndpointForRoute( + std::string_view method, + std::string_view path) noexcept; + +// Throws when the route is absent from the table, so a generation route +// cannot reach the engine without declaring how its limit is parsed. +GenerationEndpoint RequireGenerationEndpoint( + std::string_view method, + std::string_view path); + struct ParsedGenerationLimit { bool explicit_limit; int value; diff --git a/src/server/generation_limit.cpp b/src/server/generation_limit.cpp index 9cf9832c..b3a26410 100644 --- a/src/server/generation_limit.cpp +++ b/src/server/generation_limit.cpp @@ -1,10 +1,21 @@ #include +#include + namespace { constexpr int kLegacyDefaultGenerationLimit = 4096; constexpr int kNoExplicitGenerationLimit = -1; +constexpr std::array kGenerationRoutes{{ + {"POST", "/api/generate", GenerationEndpoint::Generate}, + {"POST", "/api/chat", GenerationEndpoint::OllamaChat}, + {"POST", + "/v1/chat/completions", + GenerationEndpoint::OpenAiChatCompletion}, + {"POST", "/v1/completions", GenerationEndpoint::OpenAiCompletion}, +}}; + ParsedGenerationLimit ParseField( const nlohmann::ordered_json& request, std::string_view field) { @@ -16,6 +27,35 @@ ParsedGenerationLimit ParseField( } // namespace +std::span GenerationRoutes() noexcept { + return kGenerationRoutes; +} + +std::optional GenerationEndpointForRoute( + std::string_view method, + std::string_view path) noexcept { + for (const GenerationRoute& route : kGenerationRoutes) { + if (route.method == method && route.path == path) { + return route.endpoint; + } + } + return std::nullopt; +} + +GenerationEndpoint RequireGenerationEndpoint( + std::string_view method, + std::string_view path) { + const auto endpoint = GenerationEndpointForRoute(method, path); + if (!endpoint.has_value()) { + throw std::logic_error( + "generation route " + std::string(method) + " " + + std::string(path) + + " is missing from GenerationRoutes(); add it there so its " + "limit field and admission rule are declared"); + } + return *endpoint; +} + ParsedGenerationLimit ParseGenerationLimit( const nlohmann::ordered_json& request, GenerationEndpoint endpoint) { @@ -23,6 +63,15 @@ ParsedGenerationLimit ParseGenerationLimit( case GenerationEndpoint::Generate: case GenerationEndpoint::OpenAiCompletion: return ParseField(request, "max_tokens"); + case GenerationEndpoint::OllamaChat: { + // Ollama nests the limit, and reads it with the same + // presence/absence rule as the flat endpoints. + const nlohmann::ordered_json options = + request.value( + "options", + nlohmann::ordered_json::object()); + return ParseField(options, "num_predict"); + } case GenerationEndpoint::OpenAiChatCompletion: { const ParsedGenerationLimit max_tokens = ParseField(request, "max_tokens"); @@ -56,13 +105,11 @@ std::optional RequestedMaxNewTokens( int OllamaChatGenerationLoopLimit( const nlohmann::ordered_json& request) { - const nlohmann::ordered_json options = - request.value( - "options", - nlohmann::ordered_json::object()); - return options.value( - "num_predict", - kLegacyDefaultGenerationLimit); + // Retained as the legacy (non-AIE4) spelling, and defined in terms of + // the shared rule so the two cannot drift apart. + return GenerationLoopLimit( + ParseGenerationLimit(request, GenerationEndpoint::OllamaChat), + false); } nlohmann::ordered_json ModelErrorResponse( diff --git a/src/server/rest_handler.cpp b/src/server/rest_handler.cpp index a352e944..43dcb136 100644 --- a/src/server/rest_handler.cpp +++ b/src/server/rest_handler.cpp @@ -653,7 +653,7 @@ void RestHandler::handle_generate(const json& request, const ParsedGenerationLimit parsed_limit = ParseGenerationLimit( request, - GenerationEndpoint::Generate); + RequireGenerationEndpoint("POST", "/api/generate")); std::string prompt = request["prompt"]; bool stream = request.value("stream", true); std::string model = request.value("model", current_model_tag); @@ -806,7 +806,10 @@ void RestHandler::handle_chat(const json& request, bool stream = request.value("stream", false); std::string model = request.value("model", current_model_tag); json options = request.value("options", json::object()); - int length_limit = OllamaChatGenerationLoopLimit(request); + const ParsedGenerationLimit parsed_limit = + ParseGenerationLimit( + request, + RequireGenerationEndpoint("POST", "/api/chat")); auto load_start_time = time_utils::now(); if (!ensure_model_loaded(model)) { @@ -814,16 +817,24 @@ void RestHandler::handle_chat(const json& request, send_response(error_response); return; } + // Same rule as the other three generation endpoints. On a legacy + // model this is still options.num_predict defaulting to 4096; on + // AIE4 an omitted limit means "until the context cap", and an + // explicit one takes part in the admission check below. + const int length_limit = + GenerationLoopLimit( + parsed_limit, + auto_chat_engine->uses_corelib_aie4()); auto load_end_time = time_utils::now(); - + configure_chat_engine_parameters(options, request); // messages = normalize_messages(messages); - + chat_meta_info_t meta_info; lm_uniform_input_t uniformed_input; - // options.num_predict is a soft loop bound only. It must not reserve - // AIE4 context capacity through requested_max_new_tokens. + uniformed_input.requested_max_new_tokens = + RequestedMaxNewTokens(parsed_limit); meta_info.load_duration = (uint64_t)time_utils::duration_ns(load_start_time, load_end_time).first; meta_info.max_prefill_len = this->prefill_chunk_len; header_print("FLM", "Start generating..."); @@ -1170,7 +1181,9 @@ void RestHandler::handle_openai_chat_completion(const json& request, const ParsedGenerationLimit parsed_limit = ParseGenerationLimit( request, - GenerationEndpoint::OpenAiChatCompletion); + RequireGenerationEndpoint( + "POST", + "/v1/chat/completions")); json current_messages = request["messages"]; std::string model = request.value("model", current_model_tag); bool stream = request.value("stream", false); @@ -1495,7 +1508,7 @@ void RestHandler::handle_openai_completion(const json& request, const ParsedGenerationLimit parsed_limit = ParseGenerationLimit( request, - GenerationEndpoint::OpenAiCompletion); + RequireGenerationEndpoint("POST", "/v1/completions")); std::string prompt = request["prompt"]; std::string model = request.value("model", current_model_tag); std::string reasoning_effort = request.value("reasoning_effort", "medium"); diff --git a/src/test/phi4_corelib_aie4/test_generation_limit.cpp b/src/test/phi4_corelib_aie4/test_generation_limit.cpp index 25a9f4c9..d1d661b9 100644 --- a/src/test/phi4_corelib_aie4/test_generation_limit.cpp +++ b/src/test/phi4_corelib_aie4/test_generation_limit.cpp @@ -8,10 +8,15 @@ #include #include #include +#include +#include #include #include +#include #include #include +#include +#include #include #include #include @@ -75,8 +80,31 @@ void TestHandlerSpecificPresence() { false, -1); + // /api/chat nests its limit, and it is now parsed by the same rule + // rather than by a private one. + CheckParsed( + ParseGenerationLimit( + ordered_json{{"options", {{"num_predict", 51}}}}, + GenerationEndpoint::OllamaChat), + true, + 51); + CheckParsed( + ParseGenerationLimit( + ordered_json{{"options", ordered_json::object()}}, + GenerationEndpoint::OllamaChat), + false, + -1); + // A flat max_tokens is not the Ollama field and must not be read. + CheckParsed( + ParseGenerationLimit( + ordered_json{{"max_tokens", 52}}, + GenerationEndpoint::OllamaChat), + false, + -1); + for (const GenerationEndpoint endpoint : { GenerationEndpoint::Generate, + GenerationEndpoint::OllamaChat, GenerationEndpoint::OpenAiChatCompletion, GenerationEndpoint::OpenAiCompletion}) { CheckParsed( @@ -99,16 +127,217 @@ void TestEndpointDefaultsAndPropagation() { CHECK(RequestedMaxNewTokens(explicit_limit) == std::optional(73)); } -void TestOllamaChatLimitStaysSoftOnly() { +// Legacy behaviour on /api/chat must not change: an omitted num_predict +// is still the 4096 soft bound, and an explicit one is still honoured. +void TestOllamaChatLegacyLimitIsUnchanged() { const ordered_json default_request = { {"options", ordered_json::object()}, }; CHECK(OllamaChatGenerationLoopLimit(default_request) == 4096); + CHECK(OllamaChatGenerationLoopLimit(ordered_json::object()) == 4096); const ordered_json explicit_request = { {"options", {{"num_predict", 91}}}, }; CHECK(OllamaChatGenerationLoopLimit(explicit_request) == 91); + + // The legacy spelling and the shared rule are the same function. + for (const ordered_json& request : + {default_request, explicit_request, ordered_json::object()}) { + const ParsedGenerationLimit parsed = + ParseGenerationLimit( + request, + GenerationEndpoint::OllamaChat); + CHECK( + GenerationLoopLimit(parsed, false) == + OllamaChatGenerationLoopLimit(request)); + } +} + +// On AIE4 an omitted /api/chat limit must mean "until the context cap", +// not the legacy 4096, and an explicit one must reach the admission rule +// through requested_max_new_tokens. Both were bypassed before. +void TestOllamaChatAie4LimitAndAdmission() { + const ordered_json omitted = {{"options", ordered_json::object()}}; + const ParsedGenerationLimit omitted_parsed = + ParseGenerationLimit(omitted, GenerationEndpoint::OllamaChat); + CHECK(GenerationLoopLimit(omitted_parsed, true) == -1); + CHECK(!RequestedMaxNewTokens(omitted_parsed).has_value()); + + const ordered_json over_limit = { + {"options", {{"num_predict", 8192}}}, + }; + const ParsedGenerationLimit over_parsed = + ParseGenerationLimit(over_limit, GenerationEndpoint::OllamaChat); + CHECK(GenerationLoopLimit(over_parsed, true) == 8192); + CHECK( + RequestedMaxNewTokens(over_parsed) == std::optional(8192)); +} + +// The rule the /api/chat gap was missing. Three of four endpoints were +// covered by inspection; this derives the set instead of restating it. +// +// It reads the two production sources, finds every RestHandler::handle_* +// whose body reaches the causal engine's generate() or +// generate_with_prompt(), maps those handlers to the routes server.cpp +// registers for them, and requires each such route to be declared in +// GenerationRoutes(). A new generation endpoint fails this test until it +// is. +std::string ReadSource(const char* relative) { + const std::filesystem::path path = + std::filesystem::path(FLM_TEST_SOURCE_DIR) / relative; + std::ifstream file(path, std::ios::binary); + if (!file) { + throw std::runtime_error( + "failed to read " + path.string()); + } + std::ostringstream buffer; + buffer << file.rdbuf(); + return buffer.str(); +} + +std::set GeneratingHandlerNames() { + const std::string source = ReadSource("server/rest_handler.cpp"); + std::set generating; + constexpr std::string_view kDefinition = "void RestHandler::handle_"; + for (std::size_t at = source.find(kDefinition); + at != std::string::npos; + at = source.find(kDefinition, at + 1)) { + const std::size_t name_start = at + std::string("void RestHandler::").size(); + const std::size_t name_end = source.find('(', name_start); + if (name_end == std::string::npos) { + continue; + } + const std::string name = + source.substr(name_start, name_end - name_start); + const std::size_t body_end = + source.find(kDefinition, at + 1); + const std::string body = source.substr( + at, + body_end == std::string::npos + ? std::string::npos + : body_end - at); + if ( + body.find("auto_chat_engine->generate(") != + std::string::npos || + body.find("auto_chat_engine->generate_with_prompt(") != + std::string::npos) { + generating.insert(name); + } + } + return generating; +} + +std::map> +RegisteredRoutesByHandler() { + const std::string source = ReadSource("server/server.cpp"); + std::map> routes; + constexpr std::string_view kRegister = "register_handler(\""; + for (std::size_t at = source.find(kRegister); + at != std::string::npos; + at = source.find(kRegister, at + 1)) { + std::size_t cursor = at + kRegister.size(); + const std::size_t method_end = source.find('"', cursor); + if (method_end == std::string::npos) { + continue; + } + const std::string method = + source.substr(cursor, method_end - cursor); + const std::size_t path_start = source.find('"', method_end + 1); + if (path_start == std::string::npos) { + continue; + } + const std::size_t path_end = source.find('"', path_start + 1); + if (path_end == std::string::npos) { + continue; + } + const std::string path = + source.substr(path_start + 1, path_end - path_start - 1); + + // The handler this route dispatches to, before the next + // registration begins. + const std::size_t next = source.find(kRegister, at + 1); + const std::string block = source.substr( + path_end, + next == std::string::npos + ? std::string::npos + : next - path_end); + constexpr std::string_view kCall = "rest_handler->"; + const std::size_t call_at = block.find(kCall); + if (call_at == std::string::npos) { + continue; + } + const std::size_t call_start = call_at + kCall.size(); + const std::size_t call_end = block.find('(', call_start); + if (call_end == std::string::npos) { + continue; + } + routes.emplace( + block.substr(call_start, call_end - call_start), + std::pair(method, path)); + } + return routes; +} + +void TestEveryGenerationRouteIsDeclared() { + const auto generating = GeneratingHandlerNames(); + const auto registered = RegisteredRoutesByHandler(); + CHECK(!generating.empty()); + CHECK(!registered.empty()); + + std::set declared; + for (const GenerationRoute& route : GenerationRoutes()) { + declared.insert( + std::string(route.method) + " " + std::string(route.path)); + CHECK( + GenerationEndpointForRoute(route.method, route.path) == + std::optional(route.endpoint)); + CHECK( + requires_npu_access( + std::string(route.method), + std::string(route.path))); + } + + std::set discovered; + for (const std::string& handler : generating) { + const auto found = registered.find(handler); + if (found == registered.end()) { + throw std::runtime_error( + "generation handler " + handler + + " is not registered on any route in server.cpp"); + } + const std::string route = + found->second.first + " " + found->second.second; + discovered.insert(route); + if (!declared.contains(route)) { + throw std::runtime_error( + "route " + route + " reaches the causal engine but is " + "missing from GenerationRoutes(); it would bypass the " + "AIE4 admission rule"); + } + } + // And no declared route is stale. + CHECK(discovered == declared); + CHECK(declared.size() == 4); + CHECK(declared.contains("POST /api/chat")); + + // A route that does not generate has no endpoint, and an unknown one + // fails loudly rather than silently defaulting. + CHECK( + !GenerationEndpointForRoute("POST", "/v1/embeddings") + .has_value()); + CHECK( + !GenerationEndpointForRoute("GET", "/api/chat").has_value()); + bool threw = false; + try { + (void)RequireGenerationEndpoint("POST", "/api/not-a-route"); + } catch (const std::logic_error&) { + threw = true; + } + CHECK(threw); + CHECK( + RequireGenerationEndpoint("POST", "/api/chat") == + GenerationEndpoint::OllamaChat); } void TestNestedModelErrorAndHttpStatus() { @@ -399,7 +628,9 @@ int main() { try { TestHandlerSpecificPresence(); TestEndpointDefaultsAndPropagation(); - TestOllamaChatLimitStaysSoftOnly(); + TestOllamaChatLegacyLimitIsUnchanged(); + TestOllamaChatAie4LimitAndAdmission(); + TestEveryGenerationRouteIsDeclared(); TestNestedModelErrorAndHttpStatus(); TestOpenAiStreamingErrorFramingAndParsing(); TestCliLimitAndRecoverableNotice(); From ac95f5474c840d77d005049e0d49d711253880c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CChi?= Date: Tue, 1 Sep 2026 19:42:45 -0700 Subject: [PATCH 025/117] test: make the 10R guards fire Review round 1. The code the guards protect was right; the guards were not. Each of these was verified by mutation, not by inspection. test_real_corelib returned 0 when no runtime directory was configured, so CTest reported Passed. The suite's highest-value check was green and inert by default, which reads as coverage it does not have. It now returns CTest's SKIP_RETURN_CODE and reports Skipped. Configured-but-missing stays a hard failure. The corrected pad grid was consumed only by that skippable test -- every other host test overrides the pad helpers -- and the LM-head refusal branch had no caller at all. test_phi4_shape_plan now builds a plan against the unmodified fake and asserts the grid, the capacities and the refusal directly. The two tests form a chain: test_real_corelib asserts fake == library, this asserts fake == the numbers written down, so a regression at either end fails something that runs without a runtime directory. The real library's refusal above M = 128 on the LM head is now asserted rather than avoided. That refusal is what makes Phi4ShapePlan correct in querying the LM head at one row and having RowsFor throw above it; if the library ever started rounding out-of-grid M up instead, the design would be resting on a property it no longer had. The route-coverage rule caught an undeclared fifth endpoint but not a declared one computing its limit privately -- which is exactly the shape of the /api/chat defect it was written for. handle_chat was declared and routed all along. Each generating handler must now be shown to call ParseGenerationLimit, GenerationLoopLimit, RequestedMaxNewTokens and RequireGenerationEndpoint, to call no other *GenerationLoopLimit, and to declare the route it is actually registered on. Co-Authored-By: Claude Opus 5 (1M context) --- src/test/phi4_corelib_aie4/CMakeLists.txt | 12 +- .../test_generation_limit.cpp | 140 ++++++++++++++++-- .../test_phi4_shape_plan.cpp | 132 +++++++++++++++++ .../phi4_corelib_aie4/test_real_corelib.cpp | 55 ++++++- 4 files changed, 322 insertions(+), 17 deletions(-) diff --git a/src/test/phi4_corelib_aie4/CMakeLists.txt b/src/test/phi4_corelib_aie4/CMakeLists.txt index bbbdcdf7..949ad6e6 100644 --- a/src/test/phi4_corelib_aie4/CMakeLists.txt +++ b/src/test/phi4_corelib_aie4/CMakeLists.txt @@ -115,10 +115,13 @@ target_compile_definitions(test_generation_limit PRIVATE FLM_TEST_SOURCE_DIR="${FASTFLOW_SOURCE_DIR}") add_test(NAME test_generation_limit COMMAND test_generation_limit) -# Validates FastFlow against the REAL corelib rather than the fake. It is -# a normal CTest test, and it SKIPS (passing, with a notice) only when the -# runtime directory is unset -- a configured-but-missing DLL fails, so a -# stale or absent runtime cannot hide behind a green suite. +# Validates FastFlow against the REAL corelib rather than the fake. +# +# With no runtime directory it reports SKIPPED to CTest via +# SKIP_RETURN_CODE, NOT Passed: this is the suite's highest-value check and +# a green-but-inert result would read as coverage it does not have. A +# configured-but-missing DLL is still a hard failure, so a stale or absent +# runtime cannot hide behind a green suite either. set(RYZENAI_CORELIB_RUNTIME_DIR "" CACHE PATH @@ -133,6 +136,7 @@ add_corelib_host_test(test_real_corelib test_real_corelib.cpp) target_compile_definitions(test_real_corelib PRIVATE FLM_REAL_CORELIB_RUNTIME_DIR="${RYZENAI_CORELIB_RUNTIME_DIR}" FLM_REAL_CORELIB_EXTRA_DLL_DIRS="${RYZENAI_CORELIB_EXTRA_DLL_DIRS_ESCAPED}") +set_tests_properties(test_real_corelib PROPERTIES SKIP_RETURN_CODE 77) if(RYZENAI_CORELIB_RUNTIME_DIR) set_tests_properties(test_real_corelib PROPERTIES ENVIRONMENT_MODIFICATION diff --git a/src/test/phi4_corelib_aie4/test_generation_limit.cpp b/src/test/phi4_corelib_aie4/test_generation_limit.cpp index d1d661b9..6ad87369 100644 --- a/src/test/phi4_corelib_aie4/test_generation_limit.cpp +++ b/src/test/phi4_corelib_aie4/test_generation_limit.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include namespace { @@ -179,10 +180,21 @@ void TestOllamaChatAie4LimitAndAdmission() { // // It reads the two production sources, finds every RestHandler::handle_* // whose body reaches the causal engine's generate() or -// generate_with_prompt(), maps those handlers to the routes server.cpp -// registers for them, and requires each such route to be declared in -// GenerationRoutes(). A new generation endpoint fails this test until it -// is. +// generate_with_prompt(), and maps those handlers to the routes server.cpp +// registers for them. Each such handler must then satisfy two conditions, +// because the defect that motivated this rule had BOTH shapes available +// and took the second: +// +// 1. its route is declared in GenerationRoutes() -- catches a fifth +// endpoint appearing with no declaration at all; and +// 2. it obtains its limit through the shared functions -- +// ParseGenerationLimit, GenerationLoopLimit and RequestedMaxNewTokens +// -- and calls no other *GenerationLoopLimit. +// +// handle_chat was declared and routed all along. What it did wrong was +// compute its limit privately, through OllamaChatGenerationLoopLimit, and +// a rule that only checked the route set would have passed it. Condition 2 +// is what makes this guard able to catch the bug it exists for. std::string ReadSource(const char* relative) { const std::filesystem::path path = std::filesystem::path(FLM_TEST_SOURCE_DIR) / relative; @@ -196,14 +208,20 @@ std::string ReadSource(const char* relative) { return buffer.str(); } -std::set GeneratingHandlerNames() { +struct GeneratingHandler { + std::string name; + std::string body; +}; + +std::vector GeneratingHandlers() { const std::string source = ReadSource("server/rest_handler.cpp"); - std::set generating; + std::vector generating; constexpr std::string_view kDefinition = "void RestHandler::handle_"; for (std::size_t at = source.find(kDefinition); at != std::string::npos; at = source.find(kDefinition, at + 1)) { - const std::size_t name_start = at + std::string("void RestHandler::").size(); + const std::size_t name_start = + at + std::string("void RestHandler::").size(); const std::size_t name_end = source.find('(', name_start); if (name_end == std::string::npos) { continue; @@ -222,12 +240,63 @@ std::set GeneratingHandlerNames() { std::string::npos || body.find("auto_chat_engine->generate_with_prompt(") != std::string::npos) { - generating.insert(name); + generating.push_back({name, body}); } } return generating; } +bool IsIdentifierCharacter(char value) noexcept { + return value == '_' || + (value >= '0' && value <= '9') || + (value >= 'a' && value <= 'z') || + (value >= 'A' && value <= 'Z'); +} + +// True when `body` calls something whose name ENDS in the given suffix but +// is not exactly it -- OllamaChatGenerationLoopLimit for +// GenerationLoopLimit, say. That is the private-path shape. +std::string FindQualifiedVariantCall( + const std::string& body, + const std::string& call) { + for (std::size_t at = body.find(call); + at != std::string::npos; + at = body.find(call, at + 1)) { + if (at == 0 || !IsIdentifierCharacter(body[at - 1])) { + continue; + } + std::size_t start = at; + while (start > 0 && IsIdentifierCharacter(body[start - 1])) { + --start; + } + return body.substr(start, at + call.size() - start - 1); + } + return {}; +} + +// The (method, path) the handler declares to RequireGenerationEndpoint. +std::pair DeclaredRoute( + const std::string& body) { + constexpr std::string_view kCall = "RequireGenerationEndpoint("; + const std::size_t at = body.find(kCall); + if (at == std::string::npos) { + return {}; + } + const std::size_t method_start = body.find('"', at + kCall.size()); + if (method_start == std::string::npos) { + return {}; + } + const std::size_t method_end = body.find('"', method_start + 1); + const std::size_t path_start = body.find('"', method_end + 1); + const std::size_t path_end = body.find('"', path_start + 1); + if (path_end == std::string::npos) { + return {}; + } + return { + body.substr(method_start + 1, method_end - method_start - 1), + body.substr(path_start + 1, path_end - path_start - 1)}; +} + std::map> RegisteredRoutesByHandler() { const std::string source = ReadSource("server/server.cpp"); @@ -280,7 +349,7 @@ RegisteredRoutesByHandler() { } void TestEveryGenerationRouteIsDeclared() { - const auto generating = GeneratingHandlerNames(); + const auto generating = GeneratingHandlers(); const auto registered = RegisteredRoutesByHandler(); CHECK(!generating.empty()); CHECK(!registered.empty()); @@ -299,22 +368,69 @@ void TestEveryGenerationRouteIsDeclared() { } std::set discovered; - for (const std::string& handler : generating) { - const auto found = registered.find(handler); + for (const GeneratingHandler& handler : generating) { + const auto found = registered.find(handler.name); if (found == registered.end()) { throw std::runtime_error( - "generation handler " + handler + + "generation handler " + handler.name + " is not registered on any route in server.cpp"); } const std::string route = found->second.first + " " + found->second.second; discovered.insert(route); + + // Condition 1: the route is declared. if (!declared.contains(route)) { throw std::runtime_error( "route " + route + " reaches the causal engine but is " "missing from GenerationRoutes(); it would bypass the " "AIE4 admission rule"); } + + // Condition 2: the handler goes through the shared rule. This is + // the one that catches the /api/chat defect, which was declared + // and routed but parsed its own limit. + for (const std::string& required : { + std::string("ParseGenerationLimit("), + std::string("GenerationLoopLimit("), + std::string("RequestedMaxNewTokens("), + std::string("RequireGenerationEndpoint(")}) { + if (handler.body.find(required) == std::string::npos) { + throw std::runtime_error( + handler.name + " serves " + route + + " but never calls " + required + + "; a generation handler must obtain its limit " + "through the shared rule, not privately"); + } + } + const std::string variant = + FindQualifiedVariantCall( + handler.body, + "GenerationLoopLimit("); + if (!variant.empty()) { + throw std::runtime_error( + handler.name + " serves " + route + " and calls " + + variant + + "; that is the private limit path /api/chat used, and it " + "bypasses the admission rule and the HTTP 400 response"); + } + + // The route it declares must be the route it is registered on, so + // a copy-pasted path cannot silently select another endpoint's + // limit field. + const auto declared_route = DeclaredRoute(handler.body); + if (declared_route != found->second) { + throw std::runtime_error( + handler.name + " is registered on " + route + + " but declares RequireGenerationEndpoint(\"" + + declared_route.first + "\", \"" + + declared_route.second + "\")"); + } + CHECK( + GenerationEndpointForRoute( + declared_route.first, + declared_route.second) + .has_value()); } // And no declared route is stale. CHECK(discovered == declared); diff --git a/src/test/phi4_corelib_aie4/test_phi4_shape_plan.cpp b/src/test/phi4_corelib_aie4/test_phi4_shape_plan.cpp index 0dda64fc..8027c4fa 100644 --- a/src/test/phi4_corelib_aie4/test_phi4_shape_plan.cpp +++ b/src/test/phi4_corelib_aie4/test_phi4_shape_plan.cpp @@ -538,6 +538,136 @@ void TestInvalidInputsRejectWithoutHelperCalls( CHECK(g_helpers.TotalCalls() == calls_after_build); } +// Everything above drives Phi4ShapePlan through a synthetic padding grid, +// which is what lets it test the plan's own logic. This one does the +// opposite: it builds against the UNMODIFIED fake, whose pad helpers encode +// the grid measured from the real e5258d2 library. +// +// It exists because that grid was otherwise consumed only by +// test_real_corelib, which skips without a runtime directory. The two +// together form a chain -- test_real_corelib asserts fake == library, and +// this asserts fake == the numbers written down here -- so a regression in +// either end fails something that runs by default. +std::shared_ptr ResolveUnmodifiedFake() { + auto resolver = flm::test::CompleteCorelibResolver(); + return CorelibApi::ResolveForTest( + [resolver = std::move(resolver)](std::string_view name) mutable + -> void* { + const auto found = resolver.find(std::string(name)); + return found == resolver.end() ? nullptr : found->second; + }); +} + +void TestFakeReproducesTheShippedKernelGrid() { + auto api = ResolveUnmodifiedFake(); + const Phi4ShapePlan plan = Phi4ShapePlan::Build(api); + + const std::vector> expected{ + {1, 1}, + {2, 64}, + {65, 128}, + {129, 256}, + {257, 512}, + {513, 1024}, + {1025, 2048}, + {2049, 3072}, + {3073, 4096}}; + for (const RowUse use : { + RowUse::QueryProjection, + RowUse::KvProjection, + RowUse::Attention, + RowUse::OutputProjection, + RowUse::SsMlp}) { + CHECK(plan.Transitions(use) == expected); + } + CHECK( + plan.Transitions(RowUse::LmHead) == + (std::vector>{{1, 1}})); + CHECK(plan.capacities().layer_rows == 4096); + CHECK(plan.capacities().lm_head_rows == 1); + + // Decode stays unpadded on every helper. + for (const RowUse use : { + RowUse::QueryProjection, + RowUse::KvProjection, + RowUse::Attention, + RowUse::OutputProjection, + RowUse::SsMlp, + RowUse::LmHead}) { + CHECK(plan.RowsFor(use, 1) == 1); + } + CHECK(plan.RowsFor(RowUse::SsMlp, 2) == 64); + CHECK(plan.RowsFor(RowUse::Attention, 64) == 64); + CHECK(plan.RowsFor(RowUse::QueryProjection, 65) == 128); + CHECK(plan.RowsFor(RowUse::QueryProjection, 3072) == 3072); + CHECK(plan.RowsFor(RowUse::QueryProjection, 4096) == 4096); +} + +// The LM head ships M = 1 and M = 128 and refuses anything larger rather +// than rounding up. Phi4ShapePlan never asks for more, so this branch of +// the fake had no other caller -- and an unexercised branch is not a model +// of the library, it is dead code that happens to be written down. +void TestFakeRefusesOutOfGridLmHeadRows() { + auto api = ResolveUnmodifiedFake(); + const auto& functions = api->functions(); + + for (const auto& expectation : std::vector< + std::pair>{ + {1, 1}, + {2, 128}, + {128, 128}}) { + std::int64_t m = expectation.first; + std::int64_t k = 3072; + std::int64_t n = 200064; + CHECK( + functions.matmul_pad_shape(&m, &k, &n, 128) == + ryzenai_corelib_status_success); + CHECK(m == expectation.second); + // K and N are never padded; MEM-5 rests on that. + CHECK(k == 3072); + CHECK(n == 200064); + } + + for (const std::int64_t rows : + {std::int64_t{129}, std::int64_t{256}, std::int64_t{4096}}) { + std::int64_t m = rows; + std::int64_t k = 3072; + std::int64_t n = 200064; + CHECK( + functions.matmul_pad_shape(&m, &k, &n, 128) == + ryzenai_corelib_status_unsupported); + } + + // The layer shapes have no such ceiling: 4096 is on their grid. + std::int64_t m = 4096; + std::int64_t k = 3072; + std::int64_t n = 3072; + CHECK( + functions.matmul_pad_shape(&m, &k, &n, 128) == + ryzenai_corelib_status_success); + CHECK(m == 4096); + + // And nothing beyond the grid is silently accepted anywhere. + m = 4097; + CHECK( + functions.matmul_pad_shape(&m, &k, &n, 128) == + ryzenai_corelib_status_unsupported); + m = 4097; + CHECK( + functions.ssmlp_pad_rows(&m, 3072, 8192, 128) == + ryzenai_corelib_status_unsupported); + const ryzenai_corelib_flat_mha_bf16_desc desc{ + 24, + 8, + 128, + 4096, + 96}; + m = 4097; + CHECK( + functions.flat_mha_pad_rows(&m, &desc) == + ryzenai_corelib_status_unsupported); +} + } // namespace int main() { @@ -548,6 +678,8 @@ int main() { TestUnsupportedRowsRejectBuild(api); TestInvalidPaddedRowsRejectBuild(api); TestInvalidInputsRejectWithoutHelperCalls(api); + TestFakeReproducesTheShippedKernelGrid(); + TestFakeRefusesOutOfGridLmHeadRows(); std::cout << "test_phi4_shape_plan: PASS\n"; return 0; } catch (const std::exception& error) { diff --git a/src/test/phi4_corelib_aie4/test_real_corelib.cpp b/src/test/phi4_corelib_aie4/test_real_corelib.cpp index 756022a7..58d31245 100644 --- a/src/test/phi4_corelib_aie4/test_real_corelib.cpp +++ b/src/test/phi4_corelib_aie4/test_real_corelib.cpp @@ -232,6 +232,54 @@ void CheckRealShapePlan(const std::shared_ptr& api) { std::cout << "matmul padded K/N equals logical K/N for all three " "Phi-4 descriptors\n"; + // The LM head ships M = 1 and M = 128 and NOTHING ABOVE, and asking for + // more is an error rather than a larger answer. This is the constraint + // that makes Phi4ShapePlan correct: it queries the LM head at m = 1 + // only, and RowsFor(LmHead, n>1) throws. If the real library ever + // started rounding an out-of-grid M up instead of refusing, that design + // would be resting on a property the library no longer has -- so assert + // the refusal here rather than inferring it from the code that avoids + // it. + for (const std::int64_t rows : + {std::int64_t{1}, std::int64_t{2}, std::int64_t{128}}) { + std::int64_t m = rows; + std::int64_t k = constants::kHiddenSize; + std::int64_t n = constants::kVocabularySize; + api->Check( + api->functions().matmul_pad_shape( + &m, + &k, + &n, + constants::kGroupSize), + "ryzenai_corelib_matmul_bf16_pad_shape"); + CHECK(m == (rows == 1 ? 1 : 128)); + } + for (const std::int64_t rows : + {std::int64_t{129}, + std::int64_t{256}, + std::int64_t{2048}, + std::int64_t{4096}}) { + std::int64_t m = rows; + std::int64_t k = constants::kHiddenSize; + std::int64_t n = constants::kVocabularySize; + const auto status = api->functions().matmul_pad_shape( + &m, + &k, + &n, + constants::kGroupSize); + if (status == ryzenai_corelib_status_success) { + throw std::runtime_error( + "real corelib accepted LM-head rows " + + std::to_string(rows) + " and padded to " + + std::to_string(m) + + "; Phi4ShapePlan assumes the LM head is single-row and " + "RowsFor(LmHead) throws above 1, which is only safe while " + "the library refuses"); + } + } + std::cout << "LM head pads 1->1 and 2..128->128, and refuses every M " + "above 128\n"; + // rows = 1 must stay 1 on every helper: decode allocates a single row // and the header says flat MHA pads its KV window instead. for (const auto& descriptor : kMatMulDescriptors) { @@ -345,6 +393,11 @@ void CheckRealShapePlan(const std::shared_ptr& api) { } // namespace +// CTest's SKIP_RETURN_CODE. Returning 0 here would report Passed, and a +// green-and-inert check is worse than an absent one because it reads as +// coverage. +constexpr int kCTestSkipReturnCode = 77; + int main() { const std::string runtime_dir(FLM_REAL_CORELIB_RUNTIME_DIR); if (runtime_dir.empty()) { @@ -352,7 +405,7 @@ int main() { << "test_real_corelib: SKIPPED -- configure with " "-DRYZENAI_CORELIB_RUNTIME_DIR= to run it.\n"; - return 0; + return kCTestSkipReturnCode; } try { From cb7c33729f3cad50e6bcde80860a48f86238fd06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CChi?= Date: Tue, 1 Sep 2026 20:06:02 -0700 Subject: [PATCH 026/117] build: derive the AIE4 runtime closure instead of transcribing it The staged closure is now enumerated by a dependency walker run against the exact ryzenai_corelib.dll being shipped, per design CLOSURE-1. The previous hardcoded list was wrong in both directions on this box: it demanded xrt_core.dll and xrt_umddml.dll, which do not exist here and which the real DLL does not import, and it named zlib1.dll while conda's libprotobuf.dll actually imports zlib.dll. RYZENAI_CORELIB_RUNTIME_DIR is now required only by the install/package step. flm.exe resolves the corelib DLL at run time by absolute path and never links ryzenai_corelib.lib, so a feature-ON configure warns rather than failing. The packaging test now proves the closure per CLOSURE-2 with a negative control: each derived DLL is hidden in turn and the load must fail. Without that, a green result only shows the machine had the DLL somewhere, which is exactly the failure mode that produced Win32 error 126 on a clean box. Both get_files.bat scripts no longer abort when no AIE4 closure is staged. Requiring one made the optional feature a precondition of shipping the ordinary NPU2 installer. Co-Authored-By: Claude Opus 5 (1M context) --- src/CMakeLists.txt | 25 +- src/cmake/ConfigureAie4Runtime.cmake | 171 ++++----- src/cmake/StageAie4Runtime.cmake | 177 +++++++++ src/inno/flm.iss | 21 +- src/inno/get_files.bat | 20 +- .../test_packaged_runtime.ps1 | 344 +++++++++++------- src/wix/flm.wxs | 8 +- src/wix/get_files.bat | 25 +- 8 files changed, 527 insertions(+), 264 deletions(-) create mode 100644 src/cmake/StageAie4Runtime.cmake diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 17e85d5c..ce2ef9e5 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -304,7 +304,7 @@ endif() add_executable(flm ${SOURCES} ${HEADERS}) -flm_collect_aie4_runtime_files(FLM_AIE4_RUNTIME_FILES) +flm_aie4_warn_if_unstageable() if(FLM_ENABLE_CORELIB_AIE4) target_compile_definitions(flm PRIVATE @@ -609,19 +609,9 @@ add_custom_command(TARGET flm POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_directory "${CMAKE_SOURCE_DIR}/model_overlays" "$/model_overlays") -if(WIN32 AND FLM_ENABLE_CORELIB_AIE4) - add_custom_command(TARGET flm POST_BUILD - COMMAND ${CMAKE_COMMAND} -E make_directory - "$/aie4" - COMMAND ${CMAKE_COMMAND} -E copy_if_different - ${FLM_AIE4_RUNTIME_FILES} - "$/aie4" - COMMAND ${CMAKE_COMMAND} -E make_directory - "${CMAKE_SOURCE_DIR}/out/aie4" - COMMAND ${CMAKE_COMMAND} -E copy_if_different - ${FLM_AIE4_RUNTIME_FILES} - "${CMAKE_SOURCE_DIR}/out/aie4") -endif() +flm_aie4_stage_for_target(flm + "$/aie4" + "${CMAKE_SOURCE_DIR}/out/aie4") # Default install location for model_list.json / xclbins (matches the app's # relocatable "/../share/flm" lookup). Overridden to the prefix root for @@ -759,12 +749,7 @@ if(WIN32) "[Pp][Dd][Mm][Uu][Tt][Ii][Ll][Ii][Tt][Ii][Ee][Ss].*" "[Ww][Pp][Aa][Xx][Hh][Oo][Ll][Dd][Ee][Rr].*" POST_EXCLUDE_REGEXES ".*[Ww]indows[/\\\\][Ss]ystem32[/\\\\].*") - if(FLM_ENABLE_CORELIB_AIE4) - install( - FILES ${FLM_AIE4_RUNTIME_FILES} - DESTINATION bin/aie4 - COMPONENT AIE4) - endif() + flm_aie4_install_runtime(DESTINATION bin/aie4 COMPONENT AIE4) elseif(NOT FLM_USE_HRX AND FLM_PORTABLE_BUILD) # Portable XRT: bundle the XRT runtime explicitly. XRT is deliberately # handled here instead of via the dependency closure below: flm only lists diff --git a/src/cmake/ConfigureAie4Runtime.cmake b/src/cmake/ConfigureAie4Runtime.cmake index c405e593..a4f1afc0 100644 --- a/src/cmake/ConfigureAie4Runtime.cmake +++ b/src/cmake/ConfigureAie4Runtime.cmake @@ -1,3 +1,16 @@ +# Packaging support for the optional Phi-4 AIE4 corelib runtime. +# +# The corelib DLL is resolved at run time by absolute path and `flm.exe` never +# links `ryzenai_corelib.lib`, so building the product with +# `FLM_ENABLE_CORELIB_AIE4=ON` needs the corelib *include* directory and +# nothing else. The runtime directory is therefore a packaging input, required +# by the install/package step and never by a feature-ON configure. Making it a +# configure-time requirement would contradict calling it packaging-only and +# would break the ordinary developer build. +# +# The file list itself is derived, never transcribed: see StageAie4Runtime.cmake +# and design `CLOSURE-1`. + set( RYZENAI_CORELIB_RUNTIME_DIR "" @@ -7,107 +20,103 @@ set( XRT_RUNTIME_DIR "" CACHE PATH - "Packaging-only directory containing the XRT runtime DLLs") + "Packaging-only directory containing stageable XRT runtime DLLs") set( FLM_AIE4_DEPENDENCY_DIRS "" CACHE STRING "Additional directories searched for the optional AIE4 DLL closure") -function(_flm_aie4_require_directory variable description) - if(NOT ${variable}) - message(FATAL_ERROR - "${variable} is required when FLM_ENABLE_CORELIB_AIE4=ON " - "(${description})") +set(FLM_AIE4_STAGE_SCRIPT + "${CMAKE_CURRENT_LIST_DIR}/StageAie4Runtime.cmake") + +function(flm_aie4_runtime_configured output) + if(FLM_ENABLE_CORELIB_AIE4 + AND RYZENAI_CORELIB_RUNTIME_DIR + AND EXISTS "${RYZENAI_CORELIB_RUNTIME_DIR}/ryzenai_corelib.dll") + set(${output} TRUE PARENT_SCOPE) + else() + set(${output} FALSE PARENT_SCOPE) endif() - if(NOT IS_DIRECTORY "${${variable}}") - message(FATAL_ERROR - "${variable} does not name a directory: ${${variable}}") +endfunction() + +# Warns at configure time, at most. A developer who only wants to compile the +# AIE4 code paths should not be stopped here; the install step is where the +# missing input actually matters, and that is where it fails. +function(flm_aie4_warn_if_unstageable) + if(NOT FLM_ENABLE_CORELIB_AIE4) + return() + endif() + flm_aie4_runtime_configured(_configured) + if(NOT _configured) + message(WARNING + "FLM_ENABLE_CORELIB_AIE4=ON without a usable " + "RYZENAI_CORELIB_RUNTIME_DIR. flm.exe will still build, because " + "it resolves ryzenai_corelib.dll at run time by absolute path. " + "Installing or packaging the AIE4 feature will fail until " + "RYZENAI_CORELIB_RUNTIME_DIR points at the directory holding the " + "ryzenai_corelib.dll you intend to ship.") endif() endfunction() -function(_flm_aie4_find_dependency output filename) - set(_search_dirs - "${RYZENAI_CORELIB_RUNTIME_DIR}" - ${FLM_AIE4_DEPENDENCY_DIRS}) - if(CMAKE_SOURCE_DIR) - list(APPEND _search_dirs "${CMAKE_SOURCE_DIR}/lib") +function(flm_aie4_stage_command output stage_dir report) + set(${output} + "${CMAKE_COMMAND}" + "-DFLM_AIE4_CORELIB_DIR=${RYZENAI_CORELIB_RUNTIME_DIR}" + "-DFLM_AIE4_XRT_DIR=${XRT_RUNTIME_DIR}" + "-DFLM_AIE4_EXTRA_DIRS=${FLM_AIE4_DEPENDENCY_DIRS}" + "-DFLM_AIE4_DESTINATION=${stage_dir}" + "-DFLM_AIE4_REPORT=${report}" + -P "${FLM_AIE4_STAGE_SCRIPT}" + PARENT_SCOPE) +endfunction() + +# Stages the derived closure beside a just-built binary so a developer can run +# the AIE4 path without an install. Skipped, with no error, when the runtime +# directory is not configured. +function(flm_aie4_stage_for_target target) + if(NOT FLM_ENABLE_CORELIB_AIE4 OR NOT WIN32) + return() + endif() + flm_aie4_runtime_configured(_configured) + if(NOT _configured) + return() endif() - set(_found "") - foreach(_directory IN LISTS _search_dirs) - if(_directory AND EXISTS "${_directory}/${filename}") - get_filename_component( - _found - "${_directory}/${filename}" - ABSOLUTE) - break() - endif() + foreach(_stage_dir IN LISTS ARGN) + string(MD5 _stage_id "${_stage_dir}") + flm_aie4_stage_command(_command + "${_stage_dir}" + "${CMAKE_BINARY_DIR}/aie4-closure-${_stage_id}.txt") + add_custom_command(TARGET ${target} POST_BUILD + COMMAND ${_command} + COMMENT "Deriving the Phi-4 AIE4 runtime closure") endforeach() - set(${output} "${_found}" PARENT_SCOPE) endfunction() -function(flm_collect_aie4_runtime_files output) +# Emits the install rule. The derivation runs at install time, against the +# ryzenai_corelib.dll actually being shipped, so the packaged closure can never +# be a stale list captured when the project was configured. +function(flm_aie4_install_runtime) + cmake_parse_arguments(_arg "" "DESTINATION;COMPONENT" "" ${ARGN}) if(NOT FLM_ENABLE_CORELIB_AIE4) - set(${output} "" PARENT_SCOPE) return() endif() if(NOT WIN32) message(FATAL_ERROR "FLM_ENABLE_CORELIB_AIE4 runtime packaging is Windows-only") endif() - - _flm_aie4_require_directory( - RYZENAI_CORELIB_RUNTIME_DIR - "ryzenai_corelib.dll, ryzen_mm.dll, and dyn_bins.dll") - _flm_aie4_require_directory( - XRT_RUNTIME_DIR - "xrt_coreutil.dll and the XRT device runtime") - - set(_runtime_files "") - foreach(_filename IN ITEMS - ryzenai_corelib.dll - ryzen_mm.dll - dyn_bins.dll - spdlog.dll - fmt.dll - libprotobuf.dll - zlib.dll - libutf8_validity.dll - abseil_dll.dll) - _flm_aie4_find_dependency(_dependency "${_filename}") - if(NOT _dependency) - message(FATAL_ERROR - "The AIE4 runtime closure is incomplete: ${_filename} " - "was not found in RYZENAI_CORELIB_RUNTIME_DIR or " - "FLM_AIE4_DEPENDENCY_DIRS") - endif() - list(APPEND _runtime_files "${_dependency}") - endforeach() - - foreach(_filename IN ITEMS zlib1.dll) - _flm_aie4_find_dependency(_dependency "${_filename}") - if(_dependency) - list(APPEND _runtime_files "${_dependency}") - endif() - endforeach() - - foreach(_filename IN ITEMS - xrt_coreutil.dll - xrt_core.dll - xrt_umddml.dll) - if(NOT EXISTS "${XRT_RUNTIME_DIR}/${_filename}") - message(FATAL_ERROR - "The AIE4 XRT closure is incomplete: ${_filename} " - "was not found in XRT_RUNTIME_DIR") - endif() - endforeach() - file(GLOB _xrt_runtime_dlls "${XRT_RUNTIME_DIR}/*.dll") - if(NOT _xrt_runtime_dlls) - message(FATAL_ERROR - "XRT_RUNTIME_DIR contains no runtime DLLs: ${XRT_RUNTIME_DIR}") + set(_component_args "") + if(_arg_COMPONENT) + set(_component_args COMPONENT ${_arg_COMPONENT}) endif() - list(APPEND _runtime_files ${_xrt_runtime_dlls}) - list(REMOVE_DUPLICATES _runtime_files) - list(SORT _runtime_files) - set(${output} "${_runtime_files}" PARENT_SCOPE) + install(CODE " +set(FLM_AIE4_CORELIB_DIR [[${RYZENAI_CORELIB_RUNTIME_DIR}]]) +set(FLM_AIE4_XRT_DIR [[${XRT_RUNTIME_DIR}]]) +set(FLM_AIE4_EXTRA_DIRS [[${FLM_AIE4_DEPENDENCY_DIRS}]]) +set(FLM_AIE4_DESTINATION [[${_arg_DESTINATION}]]) +set(FLM_AIE4_REPORT + \"\${CMAKE_INSTALL_PREFIX}/${_arg_DESTINATION}/aie4-closure.txt\") +" + ${_component_args}) + install(SCRIPT "${FLM_AIE4_STAGE_SCRIPT}" ${_component_args}) endfunction() diff --git a/src/cmake/StageAie4Runtime.cmake b/src/cmake/StageAie4Runtime.cmake new file mode 100644 index 00000000..15440d8f --- /dev/null +++ b/src/cmake/StageAie4Runtime.cmake @@ -0,0 +1,177 @@ +# Derives and stages the optional Phi-4 AIE4 corelib runtime closure. +# +# Design `CLOSURE-1`: the closure is defined by what the shipped +# `ryzenai_corelib.dll` actually imports, enumerated with a dependency walker +# against that exact binary. It is never transcribed, because different +# DynamicDispatch linkages and different dependency builds import different +# sets: the 223 MB dev-box binary statically links DynamicDispatch while the +# 0.8 MB target binary loads it as separate DLLs, and neither closure +# validates the other. +# +# This file is a standalone script. It runs both under `cmake -P` (developer +# staging beside `flm.exe`) and under `install(SCRIPT)` (packaging), so the +# packaged closure and the closure a developer runs against are produced by +# the same derivation rather than two lists that can drift apart. +# +# Inputs: +# FLM_AIE4_CORELIB_DIR directory holding ryzenai_corelib.dll (required) +# FLM_AIE4_XRT_DIR directory holding stageable XRT DLLs (optional) +# FLM_AIE4_EXTRA_DIRS additional dependency search directories +# FLM_AIE4_DESTINATION directory to stage into; relative paths resolve +# against CMAKE_INSTALL_PREFIX +# FLM_AIE4_REPORT optional path for the derived closure report + +cmake_minimum_required(VERSION 3.24) + +if(NOT WIN32 AND NOT CMAKE_HOST_WIN32) + message(FATAL_ERROR + "The Phi-4 AIE4 runtime closure is Windows-only.") +endif() + +if(NOT FLM_AIE4_CORELIB_DIR) + message(FATAL_ERROR + "RYZENAI_CORELIB_RUNTIME_DIR is required to install or package the " + "Phi-4 AIE4 feature (FLM_ENABLE_CORELIB_AIE4=ON). Point it at the " + "directory holding the ryzenai_corelib.dll you intend to ship, then " + "re-run the install step. Building flm.exe does not need it: the " + "corelib DLL is resolved at runtime by absolute path and flm.exe " + "never links ryzenai_corelib.lib.") +endif() + +if(NOT IS_DIRECTORY "${FLM_AIE4_CORELIB_DIR}") + message(FATAL_ERROR + "RYZENAI_CORELIB_RUNTIME_DIR does not name a directory: " + "${FLM_AIE4_CORELIB_DIR}") +endif() + +set(_flm_aie4_root "${FLM_AIE4_CORELIB_DIR}/ryzenai_corelib.dll") +if(NOT EXISTS "${_flm_aie4_root}") + message(FATAL_ERROR + "RYZENAI_CORELIB_RUNTIME_DIR contains no ryzenai_corelib.dll: " + "${FLM_AIE4_CORELIB_DIR}") +endif() + +# Search order matters. Directories supplied for this package win over +# anything the machine happens to provide, so a build box with an ambient +# conda or toolchain prefix stages the DLLs we chose rather than the ones it +# stumbled across. +set(_flm_aie4_search_dirs "${FLM_AIE4_CORELIB_DIR}") +if(FLM_AIE4_XRT_DIR AND IS_DIRECTORY "${FLM_AIE4_XRT_DIR}") + list(APPEND _flm_aie4_search_dirs "${FLM_AIE4_XRT_DIR}") +elseif(FLM_AIE4_XRT_DIR) + message(FATAL_ERROR + "XRT_RUNTIME_DIR does not name a directory: ${FLM_AIE4_XRT_DIR}") +endif() +foreach(_flm_aie4_dir IN LISTS FLM_AIE4_EXTRA_DIRS) + if(_flm_aie4_dir) + if(NOT IS_DIRECTORY "${_flm_aie4_dir}") + message(FATAL_ERROR + "FLM_AIE4_DEPENDENCY_DIRS entry is not a directory: " + "${_flm_aie4_dir}") + endif() + list(APPEND _flm_aie4_search_dirs "${_flm_aie4_dir}") + endif() +endforeach() +list(REMOVE_DUPLICATES _flm_aie4_search_dirs) + +# `dyn_bins.dll` holds DynamicDispatch's precompiled binaries and is opened by +# name at runtime, so it is never an import and a walker cannot find it. It is +# staged when the selected linkage ships it and omitted when DynamicDispatch is +# statically linked, which is why it is discovered by presence rather than +# demanded unconditionally. +set(_flm_aie4_runtime_loaded "") +foreach(_flm_aie4_name IN ITEMS dyn_bins.dll) + if(EXISTS "${FLM_AIE4_CORELIB_DIR}/${_flm_aie4_name}") + list(APPEND _flm_aie4_runtime_loaded + "${FLM_AIE4_CORELIB_DIR}/${_flm_aie4_name}") + endif() +endforeach() + +# The Visual C++ runtime is deliberately not staged. `flm.exe` itself imports +# MSVCP140/VCRUNTIME140, so the redistributable is already a product-wide +# prerequisite and the AIE4 feature adds no new one. Copying a build machine's +# conda or toolchain copy beside ryzenai_corelib.dll would ship a second, +# possibly older, runtime next to the one the rest of the process already +# loaded. +set(_flm_aie4_pre_exclude + "^api-ms-win-.*" + "^ext-ms-.*" + "^[Mm][Ss][Vv][Cc][Pp]1[0-9]+.*\\.dll$" + "^[Vv][Cc][Rr][Uu][Nn][Tt][Ii][Mm][Ee]1[0-9]+.*\\.dll$" + "^[Cc][Oo][Nn][Cc][Rr][Tt]1[0-9]+.*\\.dll$") +set(_flm_aie4_post_exclude + "^[A-Za-z]:[\\\\/][Ww][Ii][Nn][Dd][Oo][Ww][Ss][\\\\/].*" + "^.*[\\\\/][Ss][Yy][Ss][Tt][Ee][Mm]32[\\\\/].*" + "^.*[\\\\/][Ss][Yy][Ss][Ww][Oo][Ww]64[\\\\/].*") + +file(GET_RUNTIME_DEPENDENCIES + LIBRARIES + "${_flm_aie4_root}" + ${_flm_aie4_runtime_loaded} + RESOLVED_DEPENDENCIES_VAR _flm_aie4_resolved + UNRESOLVED_DEPENDENCIES_VAR _flm_aie4_unresolved + CONFLICTING_DEPENDENCIES_PREFIX _flm_aie4_conflicting + DIRECTORIES ${_flm_aie4_search_dirs} + PRE_EXCLUDE_REGEXES ${_flm_aie4_pre_exclude} + POST_EXCLUDE_REGEXES ${_flm_aie4_post_exclude}) + +if(_flm_aie4_unresolved) + list(JOIN _flm_aie4_unresolved "\n " _flm_aie4_unresolved_text) + message(FATAL_ERROR + "The Phi-4 AIE4 runtime closure is incomplete. " + "${_flm_aie4_root} imports DLLs that were not found in " + "RYZENAI_CORELIB_RUNTIME_DIR, XRT_RUNTIME_DIR, " + "FLM_AIE4_DEPENDENCY_DIRS, or the approved system directories:\n" + " ${_flm_aie4_unresolved_text}\n" + "Add the directory that provides them to FLM_AIE4_DEPENDENCY_DIRS. " + "Do not rely on them being on PATH: a closure that only loads " + "because a build machine had a conda or toolchain prefix on PATH " + "fails on the target with Win32 error 126.") +endif() + +if(_flm_aie4_conflicting_FILENAMES) + list(JOIN _flm_aie4_conflicting_FILENAMES ", " _flm_aie4_conflict_text) + message(FATAL_ERROR + "The Phi-4 AIE4 runtime closure resolved conflicting copies of: " + "${_flm_aie4_conflict_text}. Narrow the search directories so each " + "DLL has one unambiguous source.") +endif() + +set(_flm_aie4_files ${_flm_aie4_root} ${_flm_aie4_runtime_loaded}) +list(APPEND _flm_aie4_files ${_flm_aie4_resolved}) +list(REMOVE_DUPLICATES _flm_aie4_files) +list(SORT _flm_aie4_files) + +set(_flm_aie4_destination "${FLM_AIE4_DESTINATION}") +if(NOT _flm_aie4_destination) + message(FATAL_ERROR "FLM_AIE4_DESTINATION was not set") +endif() +if(NOT IS_ABSOLUTE "${_flm_aie4_destination}") + set(_flm_aie4_destination + "${CMAKE_INSTALL_PREFIX}/${_flm_aie4_destination}") +endif() + +file(MAKE_DIRECTORY "${_flm_aie4_destination}") +foreach(_flm_aie4_file IN LISTS _flm_aie4_files) + message(STATUS "Staging AIE4 runtime: ${_flm_aie4_file}") + file(COPY "${_flm_aie4_file}" + DESTINATION "${_flm_aie4_destination}" + FOLLOW_SYMLINK_CHAIN) +endforeach() + +# The report records both the derived closure and the directories it was +# derived from. The source path of every staged DLL is the audit trail: it is +# what shows, after the fact, that a shipped dependency came from the intended +# package rather than from whatever the build machine happened to have. +if(FLM_AIE4_REPORT) + set(_flm_aie4_report_text "root\t${_flm_aie4_root}\n") + foreach(_flm_aie4_dir IN LISTS _flm_aie4_search_dirs) + string(APPEND _flm_aie4_report_text "search\t${_flm_aie4_dir}\n") + endforeach() + foreach(_flm_aie4_file IN LISTS _flm_aie4_files) + get_filename_component(_flm_aie4_leaf "${_flm_aie4_file}" NAME) + string(APPEND _flm_aie4_report_text + "staged\t${_flm_aie4_leaf}\t${_flm_aie4_file}\n") + endforeach() + file(WRITE "${FLM_AIE4_REPORT}" "${_flm_aie4_report_text}") +endif() diff --git a/src/inno/flm.iss b/src/inno/flm.iss index 1f90babd..d225cc81 100644 --- a/src/inno/flm.iss +++ b/src/inno/flm.iss @@ -4,7 +4,7 @@ AppName=flm -AppVersion=1.0.4 +AppVersion=1.0.4 AppPublisher=FastFlowLM @@ -98,13 +98,16 @@ Source: "logo.ico"; DestDir: "{app}"; Flags: ignoreversion Source: "model_list.json"; DestDir: "{app}"; Flags: ignoreversion Source: "model_info.json"; DestDir: "{app}"; Flags: ignoreversion -; Optional Phi-4 AIE4 runtime and FastFlow-owned model overlays -Source: "aie4\*"; DestDir: "{app}\aie4"; Flags: ignoreversion recursesubdirs createallsubdirs; Tasks: aie4runtime -Source: "..\model_overlays\phi4-mini-it-aie4\config.json"; DestDir: "{app}\share\flm\model_overlays\phi4-mini-it-aie4"; Flags: ignoreversion; Tasks: aie4runtime -Source: "..\model_overlays\phi4-mini-it-aie4\corelib_phi4_manifest.json"; DestDir: "{app}\share\flm\model_overlays\phi4-mini-it-aie4"; Flags: ignoreversion; Tasks: aie4runtime -Source: "..\model_overlays\phi4-mini-it-aie4\tokenizer_config.json"; DestDir: "{app}\share\flm\model_overlays\phi4-mini-it-aie4"; Flags: ignoreversion; Tasks: aie4runtime -Source: "..\model_overlays\phi4-mini-it-aie4\provenance.json"; DestDir: "{app}\share\flm\model_overlays\phi4-mini-it-aie4"; Flags: ignoreversion; Tasks: aie4runtime - +; Optional Phi-4 AIE4 runtime and FastFlow-owned model overlays. +; skipifsourcedoesntexist keeps the ordinary NPU2 installer buildable on a +; machine that never staged an AIE4 closure: the feature is optional, so its +; absence is a skipped entry rather than a build failure. +Source: "aie4\*"; DestDir: "{app}\aie4"; Flags: ignoreversion recursesubdirs createallsubdirs skipifsourcedoesntexist; Tasks: aie4runtime +Source: "..\model_overlays\phi4-mini-it-aie4\config.json"; DestDir: "{app}\share\flm\model_overlays\phi4-mini-it-aie4"; Flags: ignoreversion; Tasks: aie4runtime +Source: "..\model_overlays\phi4-mini-it-aie4\corelib_phi4_manifest.json"; DestDir: "{app}\share\flm\model_overlays\phi4-mini-it-aie4"; Flags: ignoreversion; Tasks: aie4runtime +Source: "..\model_overlays\phi4-mini-it-aie4\tokenizer_config.json"; DestDir: "{app}\share\flm\model_overlays\phi4-mini-it-aie4"; Flags: ignoreversion; Tasks: aie4runtime +Source: "..\model_overlays\phi4-mini-it-aie4\provenance.json"; DestDir: "{app}\share\flm\model_overlays\phi4-mini-it-aie4"; Flags: ignoreversion; Tasks: aie4runtime + ; xclbins directory - recursively include all files Source: "..\xclbins\*"; DestDir: "{app}\xclbins"; Flags: ignoreversion recursesubdirs createallsubdirs @@ -143,7 +146,7 @@ Name: "{commondesktop}\flm serve"; \ ; Optional desktop icon task Name: "desktopicon"; Description: "Create a desktop icon"; GroupDescription: "Additional icons:"; Flags: unchecked -Name: "aie4runtime"; Description: "Install optional Phi-4 AIE4 corelib runtime"; GroupDescription: "Optional features:"; Flags: unchecked +Name: "aie4runtime"; Description: "Install optional Phi-4 AIE4 corelib runtime"; GroupDescription: "Optional features:"; Flags: unchecked [Code] var diff --git a/src/inno/get_files.bat b/src/inno/get_files.bat index a351d1a5..327db80a 100644 --- a/src/inno/get_files.bat +++ b/src/inno/get_files.bat @@ -15,12 +15,16 @@ echo Copying model_list.json... copy "..\model_list.json" "model_list.json" copy "..\model_info.json" "model_info.json" -REM Copy the validated optional AIE4 runtime closure -if not exist "..\build\aie4\ryzenai_corelib.dll" ( - echo ERROR: Build src with FLM_ENABLE_CORELIB_AIE4=ON before packaging. - exit /b 1 -) -if not exist "aie4" mkdir "aie4" -xcopy "..\build\aie4\*" "aie4\" /E /I /Y - +REM Copy the optional, derived AIE4 runtime closure when one has been staged. +REM The AIE4 feature is optional, so a missing closure is a skip and not an +REM error: requiring it here would make the AIE4 build a precondition of +REM shipping the ordinary NPU2 product. +if exist "..\build\aie4\ryzenai_corelib.dll" ( + echo Copying optional AIE4 runtime closure... + if not exist "aie4" mkdir "aie4" + xcopy "..\build\aie4\*" "aie4\" /E /I /Y +) else ( + echo No AIE4 runtime closure found; building without the AIE4 feature. +) + echo Done! diff --git a/src/test/phi4_corelib_aie4/test_packaged_runtime.ps1 b/src/test/phi4_corelib_aie4/test_packaged_runtime.ps1 index f89b695a..1faae990 100644 --- a/src/test/phi4_corelib_aie4/test_packaged_runtime.ps1 +++ b/src/test/phi4_corelib_aie4/test_packaged_runtime.ps1 @@ -1,3 +1,13 @@ +# Packaging and clean-environment tests for the optional Phi-4 AIE4 feature. +# +# `-CorelibRuntimeDir` (with `-DependencyDir`) enables the checks that matter +# most, and they are the ones that cannot be faked: the closure is derived from +# the real ryzenai_corelib.dll, loaded from the staged directory with +# development paths removed, and then re-loaded with one staged DLL at a time +# hidden. Without the negative control a green result proves nothing, because +# an ambient conda or toolchain prefix supplies the missing DLL on precisely +# the machine that built the binary. + param( [string]$FlmExe = "", [string]$CorelibRuntimeDir = "", @@ -9,13 +19,21 @@ param( $ErrorActionPreference = "Stop" $sourceRoot = (Resolve-Path (Join-Path $PSScriptRoot "../..")).Path $modulePath = Join-Path $sourceRoot "cmake/ConfigureAie4Runtime.cmake" +$stageScript = Join-Path $sourceRoot "cmake/StageAie4Runtime.cmake" $temporary = Join-Path ([System.IO.Path]::GetTempPath()) ( "flm-aie4-package-{0}-{1}" -f $PID, [DateTime]::UtcNow.Ticks) -function Write-Bytes { - param([string]$Path, [int]$Count) - [System.IO.File]::WriteAllBytes($Path, [byte[]]::new($Count)) +Add-Type @" +using System; +using System.Runtime.InteropServices; +public static class FlmAie4Loader { + [DllImport("kernel32", CharSet = CharSet.Unicode, SetLastError = true)] + public static extern IntPtr LoadLibraryEx( + string path, IntPtr file, uint flags); + [DllImport("kernel32", SetLastError = true)] + public static extern bool FreeLibrary(IntPtr module); } +"@ function Invoke-Configure { param( @@ -48,22 +66,78 @@ function Invoke-Configure { Out-String) $exitCode = $LASTEXITCODE $ErrorActionPreference = $previousPreference - $succeeded = $exitCode -eq 0 - if ($succeeded -ne $ExpectSuccess) { - throw "Unexpected CMake result.`n$output" + if (($exitCode -eq 0) -ne $ExpectSuccess) { + throw "Unexpected CMake configure result.`n$output" + } + return $output +} + +function Invoke-Install { + param( + [string]$Build, + [string]$Prefix, + [bool]$ExpectSuccess = $true + ) + $previousPreference = $ErrorActionPreference + $ErrorActionPreference = "Continue" + $output = (& cmake --install $Build --prefix $Prefix 2>&1 | + ForEach-Object { $_.ToString() } | + Out-String) + $exitCode = $LASTEXITCODE + $ErrorActionPreference = $previousPreference + if (($exitCode -eq 0) -ne $ExpectSuccess) { + throw "Unexpected CMake install result.`n$output" } return $output } +# Loads the corelib DLL by absolute path from $Dir with development paths +# removed. LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR | LOAD_LIBRARY_SEARCH_DEFAULT_DIRS +# resolves dependencies from the staged directory and the approved system +# directories only; PATH is not consulted at all, which is the product's own +# search behaviour and the point of design CLOSURE-2. +function Invoke-CleanEnvironmentLoad { + param([string]$Dir) + $savedPath = $env:PATH + $savedCorelib = $env:RYZENAI_CORELIB_PATH + $savedXrt = $env:XILINX_XRT + try { + $env:PATH = "$env:SystemRoot\System32;$env:SystemRoot" + Remove-Item Env:RYZENAI_CORELIB_PATH -ErrorAction SilentlyContinue + Remove-Item Env:XILINX_XRT -ErrorAction SilentlyContinue + $module = [FlmAie4Loader]::LoadLibraryEx( + (Join-Path $Dir "ryzenai_corelib.dll"), + [IntPtr]::Zero, + 0x00000100 -bor 0x00001000) + if ($module -eq [IntPtr]::Zero) { + return [Runtime.InteropServices.Marshal]::GetLastWin32Error() + } + [FlmAie4Loader]::FreeLibrary($module) | Out-Null + return 0 + } finally { + $env:PATH = $savedPath + if ($null -eq $savedCorelib) { + Remove-Item Env:RYZENAI_CORELIB_PATH -ErrorAction SilentlyContinue + } else { + $env:RYZENAI_CORELIB_PATH = $savedCorelib + } + if ($null -eq $savedXrt) { + Remove-Item Env:XILINX_XRT -ErrorAction SilentlyContinue + } else { + $env:XILINX_XRT = $savedXrt + } + } +} + try { $inno = Get-Content (Join-Path $sourceRoot "inno/flm.iss") -Raw foreach ($required in @( 'AppVersion=1.0.4', 'Name: "aie4runtime"', - 'Source: "aie4\*"', 'corelib_phi4_manifest.json', 'tokenizer_config.json', - 'config.json' + 'config.json', + 'provenance.json' )) { if ($inno -notmatch [regex]::Escape($required)) { throw "Inno manifest is missing: $required" @@ -73,16 +147,27 @@ try { foreach ($required in @( 'Version="1.0.4"', 'Feature Id="Aie4Feature"', - 'ComponentGroup Id="Aie4RuntimeComponents"', 'ComponentGroup Id="Aie4OverlayComponents"', 'corelib_phi4_manifest.json', 'tokenizer_config.json', - 'config.json' + 'config.json', + 'provenance.json' )) { if ($wix -notmatch [regex]::Escape($required)) { throw "WiX manifest is missing: $required" } } + [xml]$parsedWix = $wix + + # The main installer must still build with no AIE4 closure present. + # Hard-failing here would make the AIE4 feature a precondition of shipping + # the ordinary NPU2 product, which is the opposite of optional. + foreach ($script in @("wix/get_files.bat", "inno/get_files.bat")) { + $text = Get-Content (Join-Path $sourceRoot $script) -Raw + if ($text -match "exit /b 1") { + throw "$script still fails the non-AIE4 package build" + } + } New-Item -ItemType Directory -Path $temporary | Out-Null $fixture = Join-Path $temporary "fixture" @@ -92,157 +177,144 @@ cmake_minimum_required(VERSION 3.24) project(flm_aie4_packaging NONE) option(FLM_ENABLE_CORELIB_AIE4 "" OFF) include("$($modulePath.Replace('\', '/'))") -flm_collect_aie4_runtime_files(_flm_aie4_files) -install(FILES `${_flm_aie4_files} DESTINATION bin/aie4) +flm_aie4_warn_if_unstageable() +flm_aie4_install_runtime(DESTINATION bin/aie4 COMPONENT AIE4) "@ | Set-Content -Path (Join-Path $fixture "CMakeLists.txt") -Encoding utf8 + # Feature OFF: no corelib runtime is required and nothing is staged. $offBuild = Join-Path $temporary "off" Invoke-Configure -Build $offBuild -Enabled $false | Out-Null - & cmake --install $offBuild --prefix (Join-Path $temporary "off-stage") - if ($LASTEXITCODE -ne 0) { - throw "Feature-OFF install failed" - } + Invoke-Install -Build $offBuild -Prefix (Join-Path $temporary "off-stage") | + Out-Null if (Test-Path (Join-Path $temporary "off-stage/bin/aie4")) { throw "Feature OFF unexpectedly staged an AIE4 directory" } - $missingOutput = Invoke-Configure ` - -Build (Join-Path $temporary "missing") ` - -Enabled $true ` + # Feature ON without a runtime directory: configuring must SUCCEED, because + # flm.exe resolves the corelib DLL at run time by absolute path and never + # links its import library. Only the install step needs the runtime. + $devBuild = Join-Path $temporary "dev" + $devOutput = Invoke-Configure -Build $devBuild -Enabled $true + if ($devOutput -notmatch "RYZENAI_CORELIB_RUNTIME_DIR") { + throw "Feature-ON configure did not warn about the missing runtime" + } + $devInstall = Invoke-Install ` + -Build $devBuild ` + -Prefix (Join-Path $temporary "dev-stage") ` -ExpectSuccess $false - if ($missingOutput -notmatch "RYZENAI_CORELIB_RUNTIME_DIR") { - throw "Missing runtime path failure was not actionable" + if ($devInstall -notmatch "RYZENAI_CORELIB_RUNTIME_DIR") { + throw "Install-time failure was not actionable.`n$devInstall" } - $corelib = Join-Path $temporary "corelib" - $xrt = Join-Path $temporary "xrt" - New-Item -ItemType Directory -Path $corelib, $xrt | Out-Null - foreach ($name in @( - "ryzenai_corelib.dll", - "ryzen_mm.dll", - "dyn_bins.dll", - "spdlog.dll", - "libprotobuf.dll", - "fmt.dll", - "zlib.dll", - "zlib1.dll", - "libutf8_validity.dll", - "abseil_dll.dll" - )) { - Write-Bytes -Path (Join-Path $corelib $name) -Count 17 - } - foreach ($name in @( - "xrt_coreutil.dll", - "xrt_core.dll", - "xrt_umddml.dll", - "xdp_native_plugin.dll" - )) { - Write-Bytes -Path (Join-Path $xrt $name) -Count 19 + # A configured but wrong runtime directory must fail at install, loudly. + $badBuild = Join-Path $temporary "bad" + Invoke-Configure ` + -Build $badBuild ` + -Enabled $true ` + -Corelib (Join-Path $temporary "does-not-exist") | Out-Null + $badInstall = Invoke-Install ` + -Build $badBuild ` + -Prefix (Join-Path $temporary "bad-stage") ` + -ExpectSuccess $false + if ($badInstall -notmatch "RYZENAI_CORELIB_RUNTIME_DIR") { + throw "Missing runtime directory was not reported.`n$badInstall" } - $onBuild = Join-Path $temporary "on" + # A directory that exists but holds no ryzenai_corelib.dll is the mistake a + # packager is most likely to make, so it gets its own named failure rather + # than an unresolved-import message later. + $emptyCorelib = Join-Path $temporary "empty-corelib" + New-Item -ItemType Directory -Path $emptyCorelib | Out-Null + $emptyBuild = Join-Path $temporary "empty" Invoke-Configure ` - -Build $onBuild ` + -Build $emptyBuild ` -Enabled $true ` - -Corelib $corelib ` - -Xrt $xrt | Out-Null - $stage = Join-Path $temporary "on-stage" - & cmake --install $onBuild --prefix $stage - if ($LASTEXITCODE -ne 0) { - throw "Feature-ON install failed" - } - $actual = @( - Get-ChildItem (Join-Path $stage "bin/aie4") -File | - ForEach-Object Name | - Sort-Object - ) - $expected = @( - "dyn_bins.dll", - "abseil_dll.dll", - "fmt.dll", - "libprotobuf.dll", - "libutf8_validity.dll", - "ryzen_mm.dll", - "ryzenai_corelib.dll", - "spdlog.dll", - "xdp_native_plugin.dll", - "xrt_core.dll", - "xrt_coreutil.dll", - "xrt_umddml.dll", - "zlib.dll", - "zlib1.dll" - ) | Sort-Object - if (Compare-Object $expected $actual) { - throw "Installed AIE4 closure did not match the collected files" + -Corelib $emptyCorelib | Out-Null + $emptyInstall = Invoke-Install ` + -Build $emptyBuild ` + -Prefix (Join-Path $temporary "empty-stage") ` + -ExpectSuccess $false + if ($emptyInstall -notmatch "no ryzenai_corelib\.dll") { + throw "Empty runtime directory was not reported.`n$emptyInstall" } - if ($CorelibRuntimeDir -or $XrtRuntimeDir -or $DependencyDir) { - if ( - -not $CorelibRuntimeDir -or - -not $XrtRuntimeDir -or - -not $DependencyDir - ) { - throw "Real closure check requires all three runtime directories" - } + if ($CorelibRuntimeDir) { + $resolvedCorelib = (Resolve-Path $CorelibRuntimeDir).Path $realBuild = Join-Path $temporary "real" Invoke-Configure ` -Build $realBuild ` -Enabled $true ` - -Corelib (Resolve-Path $CorelibRuntimeDir).Path ` - -Xrt (Resolve-Path $XrtRuntimeDir).Path ` - -Dependency (Resolve-Path $DependencyDir).Path | Out-Null + -Corelib $resolvedCorelib ` + -Xrt $(if ($XrtRuntimeDir) { + (Resolve-Path $XrtRuntimeDir).Path } else { "" }) ` + -Dependency $(if ($DependencyDir) { + (Resolve-Path $DependencyDir).Path } else { "" }) | Out-Null $realStage = Join-Path $temporary "real-stage" - & cmake --install $realBuild --prefix $realStage - if ($LASTEXITCODE -ne 0) { - throw "Real AIE4 closure staging failed" + Invoke-Install -Build $realBuild -Prefix $realStage | Out-Null + $stagedDir = Join-Path $realStage "bin/aie4" + + # CLOSURE-1: the staged set is whatever the walker derived from this + # exact binary, so the test asserts properties of the derivation rather + # than a transcribed list that would silently disagree with the other + # DynamicDispatch linkage. + $report = Join-Path $stagedDir "aie4-closure.txt" + if (-not (Test-Path $report)) { + throw "The install did not record a derived closure report" + } + $derived = @( + Get-Content $report | + Where-Object { $_ -like "staged`t*" } | + ForEach-Object { ($_ -split "`t")[1] } + ) + if ($derived -notcontains "ryzenai_corelib.dll") { + throw "The derived closure does not contain ryzenai_corelib.dll" + } + $staged = @( + Get-ChildItem $stagedDir -File | + Where-Object { $_.Extension -eq ".dll" } | + ForEach-Object Name + ) + if (Compare-Object ($derived | Sort-Object) ($staged | Sort-Object)) { + throw "Staged AIE4 files do not match the derived closure" + } + if ($derived -contains "msvcp140.dll") { + throw "The closure staged a build-machine Visual C++ runtime" } - Add-Type @" -using System; -using System.Runtime.InteropServices; -public static class FlmAie4Loader { - [DllImport("kernel32", CharSet = CharSet.Unicode, SetLastError = true)] - public static extern IntPtr LoadLibraryEx( - string path, IntPtr file, uint flags); - [DllImport("kernel32", SetLastError = true)] - public static extern bool FreeLibrary(IntPtr module); -} -"@ - $savedPathForLoad = $env:PATH - $savedCorelibForLoad = $env:RYZENAI_CORELIB_PATH - $savedXrtForLoad = $env:XILINX_XRT - try { - $env:PATH = "$env:SystemRoot\System32;$env:SystemRoot" - Remove-Item Env:RYZENAI_CORELIB_PATH ` - -ErrorAction SilentlyContinue - Remove-Item Env:XILINX_XRT -ErrorAction SilentlyContinue - $library = Join-Path ` - $realStage ` - "bin/aie4/ryzenai_corelib.dll" - $module = [FlmAie4Loader]::LoadLibraryEx( - $library, - [IntPtr]::Zero, - 0x00000100 -bor 0x00001000) - if ($module -eq [IntPtr]::Zero) { - $errorCode = [Runtime.InteropServices.Marshal]:: - GetLastWin32Error() - throw "Clean-environment corelib load failed: $errorCode" - } - [FlmAie4Loader]::FreeLibrary($module) | Out-Null - } finally { - $env:PATH = $savedPathForLoad - if ($null -eq $savedCorelibForLoad) { - Remove-Item Env:RYZENAI_CORELIB_PATH ` - -ErrorAction SilentlyContinue - } else { - $env:RYZENAI_CORELIB_PATH = $savedCorelibForLoad + # CLOSURE-2, positive control. + $code = Invoke-CleanEnvironmentLoad -Dir $stagedDir + if ($code -ne 0) { + throw "Clean-environment corelib load failed: Win32 error $code" + } + + # CLOSURE-2, negative control. Every derived import must be + # load-bearing from the staged directory. If hiding one still loads, + # the environment supplied it and the positive control certified + # nothing. `dyn_bins.dll` is exempt: it is opened by name at run time + # rather than imported, so it is discovered by presence, not by the + # walker, and its absence does not break LoadLibrary. + $exempt = @("dyn_bins.dll") + $proved = 0 + foreach ($name in $derived) { + if ($exempt -contains $name) { continue } + $path = Join-Path $stagedDir $name + $hidden = "$path.hidden" + Rename-Item -Path $path -NewName "$name.hidden" + try { + $missingCode = Invoke-CleanEnvironmentLoad -Dir $stagedDir + } finally { + Rename-Item -Path $hidden -NewName $name } - if ($null -eq $savedXrtForLoad) { - Remove-Item Env:XILINX_XRT ` - -ErrorAction SilentlyContinue - } else { - $env:XILINX_XRT = $savedXrtForLoad + if ($missingCode -eq 0) { + throw ( + "Removing $name from the staged closure still loaded. " + + "The load resolved it from outside the staged directory, " + + "so this closure is not proven.") } + $proved += 1 + } + if ($proved -lt 1) { + throw "No staged dependency was proven load-bearing" } } @@ -309,6 +381,6 @@ public static class FlmAie4Loader { Write-Output "packaged runtime tests passed" } finally { if (Test-Path $temporary) { - Remove-Item $temporary -Recurse -Force + Remove-Item $temporary -Recurse -Force -ErrorAction SilentlyContinue } } diff --git a/src/wix/flm.wxs b/src/wix/flm.wxs index fecca26b..4648a536 100644 --- a/src/wix/flm.wxs +++ b/src/wix/flm.wxs @@ -20,7 +20,6 @@ - - - - + + diff --git a/src/wix/get_files.bat b/src/wix/get_files.bat index aea21667..945b49ba 100644 --- a/src/wix/get_files.bat +++ b/src/wix/get_files.bat @@ -22,12 +22,25 @@ echo Copying static assets... copy "..\inno\logo.ico" "package\logo.ico" copy "..\inno\terms.rtf" "package\terms.rtf" -REM Copy the validated optional AIE4 runtime closure -if not exist "..\build\aie4\ryzenai_corelib.dll" ( - echo ERROR: Build src with FLM_ENABLE_CORELIB_AIE4=ON before packaging. - exit /b 1 +REM Copy the optional, derived AIE4 runtime closure when one has been staged. +REM The AIE4 feature is optional, so a missing closure is a skip and not an +REM error: requiring it here would make the AIE4 build a precondition of +REM shipping the ordinary NPU2 product. +if exist "..\build\aie4\ryzenai_corelib.dll" ( + echo Copying optional AIE4 runtime closure... + if not exist "package\aie4" mkdir "package\aie4" + xcopy "..\build\aie4\*" "package\aie4\" /E /I /Y + > "package\aie4.wxi" echo ^ + >> "package\aie4.wxi" echo ^ + >> "package\aie4.wxi" echo ^ + >> "package\aie4.wxi" echo ^ + >> "package\aie4.wxi" echo ^ +) else ( + echo No AIE4 runtime closure found; building without the AIE4 feature. + if not exist "package\aie4" mkdir "package\aie4" + > "package\aie4.wxi" echo ^ + >> "package\aie4.wxi" echo ^ + >> "package\aie4.wxi" echo ^ ) -if not exist "package\aie4" mkdir "package\aie4" -xcopy "..\build\aie4\*" "package\aie4\" /E /I /Y echo Done! From a8e4db479ae5964b4cf8438ab13652aaa1d389b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CChi?= Date: Tue, 1 Sep 2026 20:10:47 -0700 Subject: [PATCH 027/117] fix: enforce MODEL-2 on the Phi-4 AIE4 overlay config The overlay config.json restated the Section 5.1 constants but nothing failed the load when it disagreed with them, so a hand-edited or stale overlay could silently redefine layer count, head geometry or vocabulary size for a model whose weights say otherwise. The AIE4 load path now compares every declared value against flm::phi4::constants, which is the same set the manifest loader validates the ONNX initializers against. A disagreement is a hard load failure in either direction: the overlay restates the contract and is never the source of truth. The test corrupts each field in turn and requires the load to throw before the engine is constructed. Verified RED by neutering the check, which fails the test, and GREEN with it restored. Co-Authored-By: Claude Opus 5 (1M context) --- src/common/AutoModel/modeling_phi4.cpp | 90 +++++++++++++++++++ .../phi4_corelib_aie4/test_phi4_frontend.cpp | 69 +++++++++++++- 2 files changed, 158 insertions(+), 1 deletion(-) diff --git a/src/common/AutoModel/modeling_phi4.cpp b/src/common/AutoModel/modeling_phi4.cpp index 04f647ca..1863afae 100644 --- a/src/common/AutoModel/modeling_phi4.cpp +++ b/src/common/AutoModel/modeling_phi4.cpp @@ -7,6 +7,10 @@ #include "AutoModel/modeling_phi4.hpp" +#if defined(FLM_ENABLE_CORELIB_AIE4) +#include +#endif + #include #include #include @@ -78,6 +82,91 @@ void RequireAie4FrontendFile( } } +#if defined(FLM_ENABLE_CORELIB_AIE4) + +// Design `MODEL-2`. The overlay `config.json` exists only because the upstream +// repository ships none, and it restates the Section 5.1 constants so +// FastFlow's existing readers keep working. It is never an independent source +// of truth. `flm::phi4::constants` holds those same Section 5.1 values and is +// what the manifest loader validates the ONNX initializers against, so +// requiring the overlay to equal them is what makes a disagreement between the +// overlay and the real weights a hard load failure instead of a silent +// reconfiguration of the model. +void RequireAie4OverlayMatchesModelConstants( + const std::filesystem::path& model_path) { + namespace constants = flm::phi4::constants; + const auto path = model_path / "config.json"; + json config; + try { + std::ifstream input(path, std::ios::binary); + input.exceptions(std::ios::badbit); + config = json::parse(input); + } catch (const std::exception& error) { + throw std::runtime_error( + "Phi-4 AIE4 config.json could not be parsed: " + path.string() + + ": " + error.what()); + } + if (!config.is_object()) { + throw std::runtime_error( + "Phi-4 AIE4 config.json must be a JSON object: " + path.string()); + } + + const auto require_integer = + [&](std::string_view field, std::int64_t expected) { + const auto found = config.find(field); + if (found == config.end() || !found->is_number_integer()) { + throw std::runtime_error( + "Phi-4 AIE4 config.json is missing integer field '" + + std::string(field) + "': " + path.string()); + } + const std::int64_t actual = found->get(); + if (actual != expected) { + throw std::runtime_error( + "Phi-4 AIE4 config.json disagrees with the validated " + "model constants: " + + std::string(field) + " is " + std::to_string(actual) + + ", expected " + std::to_string(expected) + + ". The overlay restates the model contract and cannot " + "redefine it; regenerate it from the packaged model " + "rather than editing it."); + } + }; + + require_integer("num_hidden_layers", constants::kLayerCount); + require_integer("hidden_size", constants::kHiddenSize); + require_integer("intermediate_size", constants::kIntermediateSize); + require_integer("num_attention_heads", constants::kQueryHeadCount); + require_integer("num_key_value_heads", constants::kKvHeadCount); + require_integer("head_dim", constants::kHeadSize); + require_integer("vocab_size", constants::kVocabularySize); + + const auto model_type = config.find("model_type"); + if ( + model_type == config.end() || + !model_type->is_string() || + model_type->get() != "phi4") { + throw std::runtime_error( + "Phi-4 AIE4 config.json must declare model_type \"phi4\": " + + path.string()); + } + + const auto epsilon = config.find("rms_norm_eps"); + if (epsilon == config.end() || !epsilon->is_number()) { + throw std::runtime_error( + "Phi-4 AIE4 config.json is missing numeric field " + "'rms_norm_eps': " + + path.string()); + } + if (epsilon->get() != constants::kRmsEpsilon) { + throw std::runtime_error( + "Phi-4 AIE4 config.json disagrees with the validated model " + "constants: rms_norm_eps does not equal the packed epsilon. " + "The overlay restates the model contract and cannot redefine it."); + } +} + +#endif // FLM_ENABLE_CORELIB_AIE4 + void ConfigureDefaultSampler(Phi4& model) { sampler_config config; config.top_k = 40; @@ -118,6 +207,7 @@ void Phi4::load_model(std::string model_path, json model_info, int default_conte RequireAie4FrontendFile( package_path, "tokenizer_config.json"); + RequireAie4OverlayMatchesModelConstants(package_path); this->_shared_initialize_model_state( model_path, diff --git a/src/test/phi4_corelib_aie4/test_phi4_frontend.cpp b/src/test/phi4_corelib_aie4/test_phi4_frontend.cpp index f178d65d..e2709d1f 100644 --- a/src/test/phi4_corelib_aie4/test_phi4_frontend.cpp +++ b/src/test/phi4_corelib_aie4/test_phi4_frontend.cpp @@ -53,7 +53,8 @@ class TempModelPackage final { public: explicit TempModelPackage( std::vector eos_ids = {200020, 199999}, - std::optional hidden_size = 3072) { + std::optional hidden_size = 3072, + nlohmann::json config_overrides = nlohmann::json::object()) { const auto base = std::filesystem::temp_directory_path(); for (int attempt = 0; attempt < 100; ++attempt) { path_ = base / @@ -82,6 +83,9 @@ class TempModelPackage final { {"vocab_size", 200064}, {"rms_norm_eps", 1.0e-5}, }; + for (const auto& [key, value] : config_overrides.items()) { + config[key] = value; + } WriteJson(path_ / "config.json", config); nlohmann::json tokenizer_config = { @@ -107,6 +111,12 @@ class TempModelPackage final { return path_; } + void OverwriteFile( + std::string_view name, + std::string_view contents) const { + WriteText(path_ / name, contents); + } + private: static void WriteJson( const std::filesystem::path& path, @@ -564,6 +574,62 @@ void ConfigureFakeCorelibDll() { } } +// Design `MODEL-2`. The overlay config.json is a restatement of the model +// contract, so any disagreement with the validated constants must fail the +// load rather than quietly reconfigure the model. Each corruption below is a +// value the AIE4 frontend and weight loader would otherwise trust. +void TestCorruptOverlayConfigFailsLoad() { + const nlohmann::json corruptions[] = { + {{"num_hidden_layers", 28}}, + {{"hidden_size", 4096}}, + {{"intermediate_size", 8960}}, + {{"num_attention_heads", 32}}, + {{"num_key_value_heads", 4}}, + {{"head_dim", 64}}, + {{"vocab_size", 200065}}, + {{"rms_norm_eps", 1.0e-6}}, + {{"model_type", "phi3"}}, + }; + for (const auto& corruption : corruptions) { + TempModelPackage package({200020, 199999}, 3072, corruption); + FactoryScope factory; + CheckThrowsContains( + [&] { + auto model = Load( + package, + ModelInfo(64, "corelib_aie4"), + -1, + false, + nullptr); + (void)model; + }, + "config.json"); + // The engine must never be constructed from a package whose declared + // contract does not match the validated one. + CHECK(g_factory.calls == 0); + } + + // A config.json that is not even parseable must fail the same way rather + // than falling through to a partially initialized model. + { + TempModelPackage package; + package.OverwriteFile("config.json", "{not json"); + FactoryScope factory; + CheckThrowsContains( + [&] { + auto model = Load( + package, + ModelInfo(64, "corelib_aie4"), + -1, + false, + nullptr); + (void)model; + }, + "config.json"); + CHECK(g_factory.calls == 0); + } +} + void TestCorelibRoutingAndPreemption() { TempModelPackage package; FactoryScope factory; @@ -1316,6 +1382,7 @@ int main() { TestLegacyExactRepeatPreservesEmptyPrefillPayload(); #if defined(FLM_ENABLE_CORELIB_AIE4) ConfigureFakeCorelibDll(); + TestCorruptOverlayConfigFailsLoad(); TestCorelibRoutingAndPreemption(); TestInitialAndAtomicCaps(); TestEnginePositionIsAuthoritativeForCapUpdate(); From b821450c2fd40f660cc40c5fb7b9c6902a673a6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CChi?= Date: Tue, 1 Sep 2026 20:17:54 -0700 Subject: [PATCH 028/117] fix: validate the AIE4 catalog's two provenances separately Step 4's rule as written could not hold. Requiring every model_list.json file to have a matching Hugging Face record fails by construction: config.json, corelib_phi4_manifest.json and provenance.json are FastFlow-authored and do not exist upstream. Per PACKAGE-1 the two sets are now validated apart. Upstream files must each carry a metadata record at the pinned revision. Authored overlays must exist in the installed overlay directory and must have NO upstream record; acquiring one means FastFlow's contract was published to the model repository. tokenizer_config.json is the documented exception: it exists upstream and the overlay shadows it, because the published file carries neither a chat template nor eos_token_id. Three catalog corrections fall out of reading the spec against the entry: - genai_config.json was being downloaded. MODEL-2 excludes it, because flm.exe runs no ORT or genai graph and an unused config invites a future reader to treat it as authoritative. - .gitattributes was being downloaded. It is a Git repository artifact. - provenance.json was installed but never staged into the model directory, so the record that makes the two provenances checkable was missing from the place that needs it. It is now the fourth bundled overlay. Size and footprint now state what they cover -- the assembled on-disk directory -- and both the tool and the C++ test derive them instead of restating a literal. CMakePresets FLM_VERSION moves to 1.0.4. The overlay declares flm_version 1.0.4 and the catalog demands flm_min_version 1.0.4, so a 1.0.3 binary reports Incompatible for its own catalog entry. A new test pins that three-way relationship; it fails when built as 1.0.3, which is how it was verified. Co-Authored-By: Claude Opus 5 (1M context) --- docs/docs/models/phi.md | 12 +- src/CMakePresets.json | 2 +- src/model_list.json | 2287 +++++++++-------- src/test/phi4_corelib_aie4/CMakeLists.txt | 3 +- .../phi4_corelib_aie4/test_model_catalog.cpp | 130 +- tools/package_phi4_corelib_aie4.py | 199 +- tools/tests/test_package_phi4_corelib_aie4.py | 147 +- 7 files changed, 1610 insertions(+), 1170 deletions(-) diff --git a/docs/docs/models/phi.md b/docs/docs/models/phi.md index effcd05a..0c74285a 100644 --- a/docs/docs/models/phi.md +++ b/docs/docs/models/phi.md @@ -40,4 +40,14 @@ flm run phi4-mini-it-aie4:4b ``` This package is hosted only on Hugging Face. `--modelscope` is rejected -before any download starts. \ No newline at end of file +before any download starts. + +The assembled model directory has two provenances. The weights, tokenizer and +vocabulary are downloaded from the pinned upstream revision and hash-checked +against its published metadata. Four files -- `config.json`, +`tokenizer_config.json`, `corelib_phi4_manifest.json` and `provenance.json` -- +are authored by FastFlowLM and installed with the product, because the upstream +repository ships no `config.json` and its tokenizer configuration carries +neither a chat template nor the EOS token IDs this backend needs. Those files +restate the model contract for FastFlowLM's existing readers; they never +override it, and a model whose weights disagree with them fails to load. diff --git a/src/CMakePresets.json b/src/CMakePresets.json index 39a07cdd..b8eb74bc 100644 --- a/src/CMakePresets.json +++ b/src/CMakePresets.json @@ -5,7 +5,7 @@ "name": "common-default", "hidden": true, "cacheVariables": { - "FLM_VERSION": "1.0.3", + "FLM_VERSION": "1.0.4", "NPU_VERSION": "32.0.203.304" } }, diff --git a/src/model_list.json b/src/model_list.json index 659f6241..9b0ea222 100644 --- a/src/model_list.json +++ b/src/model_list.json @@ -1,1171 +1,1174 @@ { - "model_path": "models", - "models": { - "nanbeige4.1": { - "3b": { - "name": "Nanbeige4.1-3B-NPU2", - "url": "https://huggingface.co/FastFlowLM/Nanbeige4.1-3B-NPU2", - "file_url": "https://huggingface.co/api/models/FastFlowLM/Nanbeige4.1-3B-NPU2/tree/main", - "ms_url": "https://modelscope.cn/models/amd/Nanbeige4.1-3B-NPU2", - "modified_at": "2025-05-30T00:00:00Z", - "size": 1000000000, - "flm_min_version": "0.9.38", - "default_context_length": 8192, - "max_prefill_len": 4096, - "files": [ - "config.json", - "model.q4nx", - "tokenizer.json", - "tokenizer_config.json" - ], - "details": { - "family": "nanbeige", - "think": false, - "parameter_size": "3B", - "quantization_level": "Q4_1" - }, - "label":[ - "reasoning" - ], - "footprint": 3.1 - } + "model_path": "models", + "models": { + "deepseek-r1": { + "8b": { + "default_context_length": 16384, + "details": { + "family": "deepseek-r1", + "parameter_size": "8B", + "quantization_level": "Q4_1", + "think": true }, - "qwen3vl-it": { - "4b": { - "name": "Qwen3-VL-4B-Instruct-NPU2", - "url": "https://huggingface.co/FastFlowLM/Qwen3-VL-4B-Instruct-NPU2", - "file_url": "https://huggingface.co/api/models/FastFlowLM/Qwen3-VL-4B-Instruct-NPU2/tree/main", - "ms_url": "https://modelscope.cn/models/amd/Qwen3-VL-4B-Instruct-NPU2", - "size": 4000000000, - "flm_min_version": "0.9.22", - "files": [ - "config.json", - "model.q4nx", - "tokenizer.json", - "tokenizer_config.json", - "vision_weight.q4nx" - ], - "vlm": true, - "default_context_length": 32768, - "max_prefill_len": 4096, - "details": { - "format": "NPU2", - "family": "qwen3vl", - "think": false, - "parameter_size": "4B", - "quantization_level": "Q4_1" - }, - "label": [ - "vision", - "tool-calling" - ], - "footprint": 3.9 - } + "file_url": "https://huggingface.co/api/models/FastFlowLM/Deepseek-R1-Distill-Llama-8B-NPU2/tree/main", + "files": [ + "config.json", + "model.q4nx", + "tokenizer.json", + "tokenizer_config.json" + ], + "flm_min_version": "0.9.43", + "footprint": 5.4, + "label": [ + "reasoning" + ], + "max_prefill_len": 4096, + "modified_at": "2025-05-30T00:00:00Z", + "ms_url": "https://modelscope.cn/models/amd/Deepseek-R1-Distill-Llama-8B-NPU2", + "name": "Deepseek-R1-Distill-Llama-8B-NPU2", + "size": 8000000000, + "url": "https://huggingface.co/FastFlowLM/Deepseek-R1-Distill-Llama-8B-NPU2/resolve/main" + } + }, + "deepseek-r1-0528": { + "8b": { + "default_context_length": 16384, + "details": { + "family": "deepseek-r1-0528", + "parameter_size": "8B", + "quantization_level": "Q4_1", + "think": true, + "think_toggleable": false }, - "gemma4-it": { - "e2b": { - "name": "Gemma4-E2B-IT-NPU2", - "url": "https://huggingface.co/FastFlowLM/Gemma4-E2B-IT-NPU2", - "file_url": "https://huggingface.co/api/models/FastFlowLM/Gemma4-E2B-IT-NPU2/tree/main", - "ms_url": "https://modelscope.cn/models/amd/Gemma4-E2B-IT-NPU2", - "size": 2000000000, - "flm_min_version": "0.9.43", - "files": [ - "config.json", - "model.q4nx", - "tokenizer.json", - "tokenizer_config.json", - "vision_weight.q4nx", - "audio_weight.q4nx", - "chat_template.jinja" - ], - "vlm": true, - "asr": true, - "default_context_length": 32768, - "max_prefill_len": 4096, - "details": { - "format": "NPU2", - "family": "gemma4e", - "think": true, - "parameter_size": "5B", - "quantization_level": "Q4_1" - }, - "label": [ - "audio", - "vision", - "reasoning", - "tool-calling", - "chat-transcription" - ], - "footprint": 6.0 - }, - "e4b": { - "name": "Gemma4-E4B-IT-NPU2", - "url": "https://huggingface.co/FastFlowLM/Gemma4-E4B-IT-NPU2", - "file_url": "https://huggingface.co/api/models/FastFlowLM/Gemma4-E4B-IT-NPU2/tree/main", - "ms_url": "https://modelscope.cn/models/amd/Gemma4-E4B-IT-NPU2", - "size": 4000000000, - "flm_min_version": "0.9.43", - "files": [ - "config.json", - "model.q4nx", - "tokenizer.json", - "tokenizer_config.json", - "vision_weight.q4nx", - "audio_weight.q4nx", - "chat_template.jinja" - ], - "vlm": true, - "asr": true, - "default_context_length": 32768, - "max_prefill_len": 4096, - "details": { - "format": "NPU2", - "family": "gemma4e", - "think": true, - "parameter_size": "8B", - "quantization_level": "Q4_1" - }, - "label": [ - "audio", - "vision", - "reasoning", - "tool-calling", - "chat-transcription" - ], - "footprint": 9.1 - } - }, - "qwen3.5": { - "0.8b": { - "name": "Qwen3.5-0.8B-NPU2", - "url": "https://huggingface.co/FastFlowLM/Qwen3.5-0.8B-NPU2/resolve/flm_q4k_high_precision", - "file_url": "https://huggingface.co/api/models/FastFlowLM/Qwen3.5-0.8B-NPU2/tree/flm_q4k_high_precision", - "ms_url": "https://modelscope.cn/models/amd/Qwen3.5-0.8B-NPU2/resolve/flm_q4k_high_precision", - "size": 800000000, - "flm_min_version": "1.0.3", - "files": [ - "config.json", - "model.q4nx", - "tokenizer.json", - "tokenizer_config.json", - "vision_weight.q4nx", - "chat_template.jinja" - ], - "vlm": true, - "default_context_length": 32768, - "max_prefill_len": 4096, - "details": { - "format": "NPU2", - "family": "qwen3.5", - "think": true, - "parameter_size": "0.8B", - "quantization_level": "Q4_1" - }, - "label": [ - "vision", - "reasoning" - ], - "footprint": 1.4 - }, - "2b": { - "name": "Qwen3.5-2B-NPU2", - "url": "https://huggingface.co/FastFlowLM/Qwen3.5-2B-NPU2/resolve/flm_q4k_high_precision", - "file_url": "https://huggingface.co/api/models/FastFlowLM/Qwen3.5-2B-NPU2/tree/flm_q4k_high_precision", - "ms_url": "https://modelscope.cn/models/amd/Qwen3.5-2B-NPU2/resolve/flm_q4k_high_precision", - "size": 2000000000, - "flm_min_version": "1.0.3", - "files": [ - "config.json", - "model.q4nx", - "tokenizer.json", - "tokenizer_config.json", - "vision_weight.q4nx", - "chat_template.jinja" - ], - "vlm": true, - "default_context_length": 32768, - "max_prefill_len": 4096, - "details": { - "format": "NPU2", - "family": "qwen3.5", - "think": true, - "parameter_size": "2B", - "quantization_level": "Q4_1" - }, - "label": [ - "vision", - "reasoning", - "tool-calling" - ], - "footprint": 3.2 - }, - "4b": { - "name": "Qwen3.5-4B-NPU2", - "url": "https://huggingface.co/FastFlowLM/Qwen3.5-4B-NPU2/resolve/flm_q4k_high_precision", - "file_url": "https://huggingface.co/api/models/FastFlowLM/Qwen3.5-4B-NPU2/tree/flm_q4k_high_precision", - "ms_url": "https://modelscope.cn/models/amd/Qwen3.5-4B-NPU2/resolve/flm_q4k_high_precision", - "size": 4000000000, - "flm_min_version": "1.0.3", - "files": [ - "config.json", - "model.q4nx", - "tokenizer.json", - "tokenizer_config.json", - "vision_weight.q4nx", - "chat_template.jinja" - ], - "vlm": true, - "default_context_length": 32768, - "max_prefill_len": 4096, - "details": { - "format": "NPU2", - "family": "qwen3.5", - "think": true, - "parameter_size": "4B", - "quantization_level": "Q4_1" - }, - "label": [ - "vision", - "reasoning", - "tool-calling" - ], - "footprint": 5.2 - }, - "9b":{ - "name": "Qwen3.5-9B-NPU2", - "url": "https://huggingface.co/FastFlowLM/Qwen3.5-9B-NPU2/resolve/flm_q4k_high_precision", - "file_url": "https://huggingface.co/api/models/FastFlowLM/Qwen3.5-9B-NPU2/tree/flm_q4k_high_precision", - "ms_url": "https://modelscope.cn/models/amd/Qwen3.5-9B-NPU2/resolve/flm_q4k_high_precision", - "size": 9000000000, - "flm_min_version": "1.0.3", - "files": [ - "config.json", - "model.q4nx", - "tokenizer.json", - "tokenizer_config.json", - "vision_weight.q4nx", - "chat_template.jinja" - ], - "vlm": true, - "default_context_length": 32768, - "max_prefill_len": 4096, - "details": { - "format": "NPU2", - "family": "qwen3.5", - "think": true, - "parameter_size": "9B", - "quantization_level": "Q4_1" - }, - "label": [ - "vision", - "reasoning", - "tool-calling" - ], - "footprint": 8.94 - } + "file_url": "https://huggingface.co/api/models/FastFlowLM/DeepSeek-R1-0528-Qwen3-8B-NPU2/tree/main", + "files": [ + "config.json", + "model.q4nx", + "tokenizer.json", + "tokenizer_config.json" + ], + "flm_min_version": "0.9.22", + "footprint": 5.6, + "label": [ + "reasoning" + ], + "max_prefill_len": 4096, + "modified_at": "2025-05-30T00:00:00Z", + "ms_url": "https://modelscope.cn/models/amd/DeepSeek-R1-0528-Qwen3-8B-NPU2", + "name": "DeepSeek-R1-0528-Qwen3-8B-NPU2", + "size": 8000000000, + "url": "https://huggingface.co/FastFlowLM/DeepSeek-R1-0528-Qwen3-8B-NPU2" + } + }, + "embed-gemma": { + "300m": { + "default_context_length": 2048, + "details": { + "family": "embed-gemma", + "format": "NPU2", + "parameter_size": "300M", + "quantization_level": "none", + "think": false }, - - "qwen3.6-moe": { - "35b-a3b": { - "name": "Qwen3.6-35B-A3B-NPU2", - "url": "https://huggingface.co/FastFlowLM/Qwen3.6-35B-A3B-NPU2/resolve/flm_q4k_high_precision", - "file_url": "https://huggingface.co/api/models/FastFlowLM/Qwen3.6-35B-A3B-NPU2/tree/flm_q4k_high_precision", - "ms_url": "https://modelscope.cn/models/amd/Qwen3.6-35B-A3B-NPU2/resolve/flm_q4k_high_precision", - "size": 35000000000, - "flm_min_version": "1.0.3", - "files": [ - "config.json", - "model.q4nx", - "tokenizer.json", - "tokenizer_config.json", - "vision_weight.q4nx", - "chat_template.jinja" - ], - "vlm": true, - "default_context_length": 32768, - "max_prefill_len": 4096, - "details": { - "format": "NPU2", - "family": "qwen3.6-moe", - "think": true, - "parameter_size": "35B", - "quantization_level": "Q4_K_S" - }, - "label": [ - "vision", - "reasoning", - "tool-calling" - ], - "footprint": 24.3 - } + "file_url": "https://huggingface.co/api/models/FastFlowLM/Embedding-Gemma-300M-NPU2/tree/main", + "files": [ + "config.json", + "tokenizer.json", + "tokenizer_config.json", + "model.q4nx" + ], + "flm_min_version": "0.9.15", + "footprint": 0.62, + "label": [ + "embeddings" + ], + "ms_url": "https://modelscope.cn/models/amd/Embedding-Gemma-300M-NPU2", + "name": "Embedding-Gemma-300M-NPU2", + "size": 300000000, + "url": "https://huggingface.co/FastFlowLM/Embedding-Gemma-300M-NPU2" + } + }, + "gemma3": { + "1b": { + "default_context_length": 32768, + "details": { + "family": "gemma3-text", + "parameter_size": "1B", + "quantization_level": "Q4_1", + "think": false, + "think_toggleable": false }, - "lfm2": { - "1.2b": { - "name": "LFM2-1.2B-NPU2", - "url": "https://huggingface.co/FastFlowLM/LFM2-1.2B-NPU2", - "file_url": "https://huggingface.co/api/models/FastFlowLM/LFM2-1.2B-NPU2/tree/main", - "ms_url": "https://modelscope.cn/models/amd/LFM2-1.2B-NPU2", - "size": 1200000000, - "default_context_length": 32768, - "max_prefill_len": 4096, - "details": { - "format": "NPU2", - "family": "lfm2", - "think": false, - "think_toggleable": false, - "parameter_size": "1.2B", - "quantization_level": "Q4_0" - }, - "flm_min_version": "0.9.24", - "vlm": false, - "files": [ - "config.json", - "model.q4nx", - "tokenizer.json", - "tokenizer_config.json" - ], - "footprint": 0.96 - }, - "2.6b": { - "name": "LFM2-2.6B-NPU2", - "url": "https://huggingface.co/FastFlowLM/LFM2-2.6B-NPU2", - "file_url": "https://huggingface.co/api/models/FastFlowLM/LFM2-2.6B-NPU2/tree/main", - "ms_url": "https://modelscope.cn/models/amd/LFM2-2.6B-NPU2", - "size": 2600000000, - "default_context_length": 32768, - "max_prefill_len": 4096, - "details": { - "format": "NPU2", - "family": "lfm2", - "think": false, - "think_toggleable": false, - "parameter_size": "2.6B", - "quantization_level": "Q4_0" - }, - "flm_min_version": "0.9.24", - "vlm": false, - "files": [ - "config.json", - "model.q4nx", - "tokenizer.json", - "tokenizer_config.json" - ], - "footprint": 1.8 - } + "file_url": "https://huggingface.co/api/models/FastFlowLM/Gemma3-1B-NPU2/tree/main", + "files": [ + "config.json", + "model.q4nx", + "tokenizer.json", + "tokenizer_config.json" + ], + "flm_min_version": "0.9.20", + "footprint": 1.2, + "max_prefill_len": 4096, + "modified_at": "2025-05-30T00:00:00Z", + "ms_url": "https://modelscope.cn/models/amd/Gemma3-1B-NPU2", + "name": "Gemma3-1B-NPU2", + "size": 1000000000, + "url": "https://huggingface.co/FastFlowLM/Gemma3-1B-NPU2" + }, + "4b": { + "default_context_length": 65536, + "details": { + "family": "gemma3", + "parameter_size": "4B", + "quantization_level": "Q4_1", + "think": false, + "think_toggleable": false }, - "lfm2-trans": { - "2.6b": { - "name": "LFM2-2.6B-Transcript-NPU2", - "url": "https://huggingface.co/FastFlowLM/LFM2-2.6B-Transcript-NPU2", - "file_url": "https://huggingface.co/api/models/FastFlowLM/LFM2-2.6B-Transcript-NPU2/tree/main", - "ms_url": "https://modelscope.cn/models/amd/LFM2-2.6B-Transcript-NPU2", - "size": 2600000000, - "default_context_length": 32768, - "max_prefill_len": 4096, - "details": { - "format": "NPU2", - "family": "lfm2", - "think": false, - "think_toggleable": false, - "parameter_size": "2.6B", - "quantization_level": "Q4_0" - }, - "flm_min_version": "0.9.24", - "vlm": false, - "files": [ - "config.json", - "model.q4nx", - "tokenizer.json", - "tokenizer_config.json" - ], - "footprint": 1.8 - } + "file_url": "https://huggingface.co/api/models/FastFlowLM/Gemma3-4B-NPU2/tree/main", + "files": [ + "config.json", + "model.q4nx", + "tokenizer.json", + "tokenizer_config.json", + "vision_weight.q4nx" + ], + "flm_min_version": "0.9.23", + "footprint": 4.5, + "label": [ + "vision" + ], + "max_prefill_len": 4096, + "modified_at": "2025-05-30T00:00:00Z", + "ms_url": "https://modelscope.cn/models/amd/Gemma3-4B-NPU2", + "name": "Gemma3-4B-NPU2", + "size": 4000000000, + "url": "https://huggingface.co/FastFlowLM/Gemma3-4B-NPU2", + "vlm": true + } + }, + "gemma4-it": { + "e2b": { + "asr": true, + "default_context_length": 32768, + "details": { + "family": "gemma4e", + "format": "NPU2", + "parameter_size": "5B", + "quantization_level": "Q4_1", + "think": true }, - "lfm2.5-it": { - "1.2b": { - "name": "LFM2.5-1.2B-NPU2", - "url": "https://huggingface.co/FastFlowLM/LFM2.5-1.2B-NPU2", - "file_url": "https://huggingface.co/api/models/FastFlowLM/LFM2.5-1.2B-NPU2/tree/main", - "ms_url": "https://modelscope.cn/models/amd/LFM2.5-1.2B-NPU2", - "size": 1200000000, - "default_context_length": 32768, - "max_prefill_len": 4096, - "details": { - "format": "NPU2", - "family": "lfm2", - "think": false, - "think_toggleable": false, - "parameter_size": "1.2B", - "quantization_level": "Q4_0" - }, - "flm_min_version": "0.9.25", - "vlm": false, - "files": [ - "config.json", - "model.q4nx", - "tokenizer.json", - "tokenizer_config.json" - ], - "footprint": 0.96 - } + "file_url": "https://huggingface.co/api/models/FastFlowLM/Gemma4-E2B-IT-NPU2/tree/main", + "files": [ + "config.json", + "model.q4nx", + "tokenizer.json", + "tokenizer_config.json", + "vision_weight.q4nx", + "audio_weight.q4nx", + "chat_template.jinja" + ], + "flm_min_version": "0.9.43", + "footprint": 6.0, + "label": [ + "audio", + "vision", + "reasoning", + "tool-calling", + "chat-transcription" + ], + "max_prefill_len": 4096, + "ms_url": "https://modelscope.cn/models/amd/Gemma4-E2B-IT-NPU2", + "name": "Gemma4-E2B-IT-NPU2", + "size": 2000000000, + "url": "https://huggingface.co/FastFlowLM/Gemma4-E2B-IT-NPU2", + "vlm": true + }, + "e4b": { + "asr": true, + "default_context_length": 32768, + "details": { + "family": "gemma4e", + "format": "NPU2", + "parameter_size": "8B", + "quantization_level": "Q4_1", + "think": true }, - "lfm2.5-tk": { - "1.2b": { - "name": "LFM2.5-1.2B-Thinking-NPU2", - "url": "https://huggingface.co/FastFlowLM/LFM2.5-1.2B-Thinking-NPU2", - "file_url": "https://huggingface.co/api/models/FastFlowLM/LFM2.5-1.2B-Thinking-NPU2/tree/main", - "ms_url": "https://modelscope.cn/models/amd/LFM2.5-1.2B-Thinking-NPU2", - "size": 1200000000, - "default_context_length": 32768, - "max_prefill_len": 4096, - "details": { - "format": "NPU2", - "family": "lfm2.5-tk", - "think": true, - "think_toggleable": false, - "parameter_size": "1.2B", - "quantization_level": "Q4_0" - }, - "flm_min_version": "0.9.27", - "vlm": false, - "files": [ - "config.json", - "model.q4nx", - "tokenizer.json", - "tokenizer_config.json", - "chat_template.jinja" - ], - "label":[ - "reasoning" - ], - "footprint": 0.96 - } + "file_url": "https://huggingface.co/api/models/FastFlowLM/Gemma4-E4B-IT-NPU2/tree/main", + "files": [ + "config.json", + "model.q4nx", + "tokenizer.json", + "tokenizer_config.json", + "vision_weight.q4nx", + "audio_weight.q4nx", + "chat_template.jinja" + ], + "flm_min_version": "0.9.43", + "footprint": 9.1, + "label": [ + "audio", + "vision", + "reasoning", + "tool-calling", + "chat-transcription" + ], + "max_prefill_len": 4096, + "ms_url": "https://modelscope.cn/models/amd/Gemma4-E4B-IT-NPU2", + "name": "Gemma4-E4B-IT-NPU2", + "size": 4000000000, + "url": "https://huggingface.co/FastFlowLM/Gemma4-E4B-IT-NPU2", + "vlm": true + } + }, + "gpt-oss": { + "20b": { + "default_context_length": 8192, + "details": { + "family": "gpt-oss", + "parameter_size": "20B", + "quantization_level": "Q4_1", + "think": true, + "think_toggleable": false }, - "phi4-mini-it": { - "4b": { - "name": "Phi4-mini-Instruct-NPU2", - "url": "https://huggingface.co/FastFlowLM/Phi4-mini-Instruct-NPU2", - "file_url": "https://huggingface.co/api/models/FastFlowLM/Phi4-mini-Instruct-NPU2/tree/main", - "ms_url": "https://modelscope.cn/models/amd/Phi4-mini-Instruct-NPU2", - "size": 4000000000, - "default_context_length": 32768, - "max_prefill_len": 4096, - "details": { - "family": "phi4", - "think": false, - "think_toggleable": false, - "parameter_size": "4B", - "quantization_level": "Q4_1" - }, - "flm_min_version": "0.9.25", - "vlm": false, - "files": [ - "config.json", - "model.q4nx", - "tokenizer.json", - "tokenizer_config.json" - ], - "footprint": 3.4 - } + "file_url": "https://huggingface.co/api/models/FastFlowLM/GPT-OSS-20B-NPU2/tree/main", + "files": [ + "config.json", + "model.q4nx", + "tokenizer.json", + "tokenizer_config.json" + ], + "flm_min_version": "0.9.20", + "footprint": 14.0, + "label": [ + "reasoning" + ], + "max_prefill_len": 4096, + "modified_at": "2025-05-30T00:00:00Z", + "ms_url": "https://modelscope.cn/models/amd/GPT-OSS-20B-NPU2", + "name": "GPT-OSS-20B-NPU2", + "size": 20000000000, + "url": "https://huggingface.co/FastFlowLM/GPT-OSS-20B-NPU2" + } + }, + "gpt-oss-sg": { + "20b": { + "default_context_length": 8192, + "details": { + "family": "gpt-oss", + "parameter_size": "20B", + "quantization_level": "Q4_1", + "think": true, + "think_toggleable": false }, - "phi4-mini-it-aie4": { - "4b": { - "name": "Phi-4-mini-instruct-oga-dml-AIE4", - "url": "https://huggingface.co/amd/phi-4-mini-instruct-oga-dml", - "revision": "e751fb68c2cfffe6b0d32942118f75ac0a0365bb", - "file_url": "https://huggingface.co/api/models/amd/phi-4-mini-instruct-oga-dml/tree/e751fb68c2cfffe6b0d32942118f75ac0a0365bb?recursive=true&expand=false", - "size": 3271001989, - "default_context_length": 4096, - "max_prefill_len": 4096, - "details": { - "family": "phi4", - "think": false, - "think_toggleable": false, - "parameter_size": "4B", - "quantization_level": "MatMulNBits Q4", - "execution_backend": "corelib_aie4" - }, - "flm_min_version": "1.0.4", - "vlm": false, - "modelscope_supported": false, - "files": [ - ".gitattributes", - "added_tokens.json", - "chat_template.jinja", - "config.json", - "corelib_phi4_manifest.json", - "genai_config.json", - "merges.txt", - "model.onnx", - "model.onnx.data", - "special_tokens_map.json", - "tokenizer.json", - "tokenizer_config.json", - "vocab.json" - ], - "bundled_overlays": { - "config.json": { - "path": "phi4-mini-it-aie4/config.json", - "size": 257, - "sha256": "1b3e74125a109c05f53c8383def18359d8581619f998c4f91b3b5bb78bf2919f" - }, - "corelib_phi4_manifest.json": { - "path": "phi4-mini-it-aie4/corelib_phi4_manifest.json", - "size": 274527, - "sha256": "09cee6efafc513a2048c89e75b40d92d096144f0eed3b2459c138032b69dc045" - }, - "tokenizer_config.json": { - "path": "phi4-mini-it-aie4/tokenizer_config.json", - "size": 3036, - "sha256": "274d22c3cd28c042f28a681832722536663b06e8c9e88b6616622dceab922ca6" - } - }, - "footprint": 3.05 - } + "file_url": "https://huggingface.co/api/models/FastFlowLM/GPT-OSS-Safeguard-20b-NPU2/tree/main", + "files": [ + "config.json", + "model.q4nx", + "tokenizer.json", + "tokenizer_config.json" + ], + "flm_min_version": "0.9.20", + "footprint": 14.0, + "label": [ + "reasoning" + ], + "max_prefill_len": 4096, + "modified_at": "2025-05-30T00:00:00Z", + "ms_url": "https://modelscope.cn/models/amd/GPT-OSS-Safeguard-20b-NPU2", + "name": "GPT-OSS-Safeguard-20b-NPU2", + "size": 20000000000, + "url": "https://huggingface.co/FastFlowLM/GPT-OSS-Safeguard-20b-NPU2" + } + }, + "lfm2": { + "1.2b": { + "default_context_length": 32768, + "details": { + "family": "lfm2", + "format": "NPU2", + "parameter_size": "1.2B", + "quantization_level": "Q4_0", + "think": false, + "think_toggleable": false }, - "embed-gemma": { - "300m": { - "name": "Embedding-Gemma-300M-NPU2", - "url": "https://huggingface.co/FastFlowLM/Embedding-Gemma-300M-NPU2", - "file_url": "https://huggingface.co/api/models/FastFlowLM/Embedding-Gemma-300M-NPU2/tree/main", - "ms_url": "https://modelscope.cn/models/amd/Embedding-Gemma-300M-NPU2", - "size": 300000000, - "flm_min_version": "0.9.15", - "files": [ - "config.json", - "tokenizer.json", - "tokenizer_config.json", - "model.q4nx" - ], - "default_context_length": 2048, - "details": { - "format": "NPU2", - "family": "embed-gemma", - "think": false, - "parameter_size": "300M", - "quantization_level": "none" - }, - "label":[ - "embeddings" - ], - "footprint": 0.62 - } + "file_url": "https://huggingface.co/api/models/FastFlowLM/LFM2-1.2B-NPU2/tree/main", + "files": [ + "config.json", + "model.q4nx", + "tokenizer.json", + "tokenizer_config.json" + ], + "flm_min_version": "0.9.24", + "footprint": 0.96, + "max_prefill_len": 4096, + "ms_url": "https://modelscope.cn/models/amd/LFM2-1.2B-NPU2", + "name": "LFM2-1.2B-NPU2", + "size": 1200000000, + "url": "https://huggingface.co/FastFlowLM/LFM2-1.2B-NPU2", + "vlm": false + }, + "2.6b": { + "default_context_length": 32768, + "details": { + "family": "lfm2", + "format": "NPU2", + "parameter_size": "2.6B", + "quantization_level": "Q4_0", + "think": false, + "think_toggleable": false }, - "whisper-v3": { - "turbo": { - "name": "Whisper-V3-Turbo-NPU2", - "url": "https://huggingface.co/FastFlowLM/Whisper-V3-Turbo-NPU2", - "file_url": "https://huggingface.co/api/models/FastFlowLM/Whisper-V3-Turbo-NPU2/tree/main", - "ms_url": "https://modelscope.cn/models/amd/Whisper-V3-Turbo-NPU2", - "size": 1000000000, - "flm_min_version": "0.9.14", - "files": [ - "config.json", - "tokenizer.json", - "tokenizer_config.json", - "model.q4nx" - ], - "asr": true, - "default_context_length": 448, - "details": { - "format": "NPU2", - "family": "whisper-v3", - "think": false, - "parameter_size": "1B", - "quantization_level": "Q4_1" - }, - "label":[ - "audio", - "realtime-transcription", - "transcription" - ], - "footprint": 0.62 - } + "file_url": "https://huggingface.co/api/models/FastFlowLM/LFM2-2.6B-NPU2/tree/main", + "files": [ + "config.json", + "model.q4nx", + "tokenizer.json", + "tokenizer_config.json" + ], + "flm_min_version": "0.9.24", + "footprint": 1.8, + "max_prefill_len": 4096, + "ms_url": "https://modelscope.cn/models/amd/LFM2-2.6B-NPU2", + "name": "LFM2-2.6B-NPU2", + "size": 2600000000, + "url": "https://huggingface.co/FastFlowLM/LFM2-2.6B-NPU2", + "vlm": false + } + }, + "lfm2-trans": { + "2.6b": { + "default_context_length": 32768, + "details": { + "family": "lfm2", + "format": "NPU2", + "parameter_size": "2.6B", + "quantization_level": "Q4_0", + "think": false, + "think_toggleable": false }, - "gemma3": { - "1b": { - "name": "Gemma3-1B-NPU2", - "url": "https://huggingface.co/FastFlowLM/Gemma3-1B-NPU2", - "file_url": "https://huggingface.co/api/models/FastFlowLM/Gemma3-1B-NPU2/tree/main", - "ms_url": "https://modelscope.cn/models/amd/Gemma3-1B-NPU2", - "modified_at": "2025-05-30T00:00:00Z", - "size": 1000000000, - "default_context_length": 32768, - "max_prefill_len": 4096, - "flm_min_version": "0.9.20", - "files": [ - "config.json", - "model.q4nx", - "tokenizer.json", - "tokenizer_config.json" - ], - "details": { - "family": "gemma3-text", - "think": false, - "think_toggleable": false, - "parameter_size": "1B", - "quantization_level": "Q4_1" - }, - "footprint": 1.2 - }, - "4b": { - "name": "Gemma3-4B-NPU2", - "url": "https://huggingface.co/FastFlowLM/Gemma3-4B-NPU2", - "file_url": "https://huggingface.co/api/models/FastFlowLM/Gemma3-4B-NPU2/tree/main", - "ms_url": "https://modelscope.cn/models/amd/Gemma3-4B-NPU2", - "modified_at": "2025-05-30T00:00:00Z", - "size": 4000000000, - "default_context_length": 65536, - "max_prefill_len": 4096, - "flm_min_version": "0.9.23", - "vlm": true, - "files": [ - "config.json", - "model.q4nx", - "tokenizer.json", - "tokenizer_config.json", - "vision_weight.q4nx" - ], - "details": { - "family": "gemma3", - "think": false, - "think_toggleable": false, - "parameter_size": "4B", - "quantization_level": "Q4_1" - }, - "label":[ - "vision" - ], - "footprint": 4.5 - } + "file_url": "https://huggingface.co/api/models/FastFlowLM/LFM2-2.6B-Transcript-NPU2/tree/main", + "files": [ + "config.json", + "model.q4nx", + "tokenizer.json", + "tokenizer_config.json" + ], + "flm_min_version": "0.9.24", + "footprint": 1.8, + "max_prefill_len": 4096, + "ms_url": "https://modelscope.cn/models/amd/LFM2-2.6B-Transcript-NPU2", + "name": "LFM2-2.6B-Transcript-NPU2", + "size": 2600000000, + "url": "https://huggingface.co/FastFlowLM/LFM2-2.6B-Transcript-NPU2", + "vlm": false + } + }, + "lfm2.5-it": { + "1.2b": { + "default_context_length": 32768, + "details": { + "family": "lfm2", + "format": "NPU2", + "parameter_size": "1.2B", + "quantization_level": "Q4_0", + "think": false, + "think_toggleable": false }, - "translategemma": { - "4b": { - "name": "Translategemma-4B-Instruct-NPU2", - "url": "https://huggingface.co/FastFlowLM/Translategemma-4B-Instruct-NPU2/resolve/main", - "file_url": "https://huggingface.co/api/models/FastFlowLM/Translategemma-4B-Instruct-NPU2/tree/main", - "ms_url": "https://modelscope.cn/models/amd/Translategemma-4B-Instruct-NPU2", - "modified_at": "2025-05-30T00:00:00Z", - "size": 4000000000, - "default_context_length": 65536, - "max_prefill_len": 4096, - "vlm": true, - "flm_min_version": "0.9.34", - "files": [ - "config.json", - "model.q4nx", - "tokenizer.json", - "tokenizer_config.json", - "vision_weight.q4nx" - ], - "details": { - "family": "gemma3", - "think": false, - "think_toggleable": false, - "parameter_size": "4B", - "quantization_level": "Q4_1" - }, - "label":[ - "vision" - ], - "footprint": 4.5 - } + "file_url": "https://huggingface.co/api/models/FastFlowLM/LFM2.5-1.2B-NPU2/tree/main", + "files": [ + "config.json", + "model.q4nx", + "tokenizer.json", + "tokenizer_config.json" + ], + "flm_min_version": "0.9.25", + "footprint": 0.96, + "max_prefill_len": 4096, + "ms_url": "https://modelscope.cn/models/amd/LFM2.5-1.2B-NPU2", + "name": "LFM2.5-1.2B-NPU2", + "size": 1200000000, + "url": "https://huggingface.co/FastFlowLM/LFM2.5-1.2B-NPU2", + "vlm": false + } + }, + "lfm2.5-tk": { + "1.2b": { + "default_context_length": 32768, + "details": { + "family": "lfm2.5-tk", + "format": "NPU2", + "parameter_size": "1.2B", + "quantization_level": "Q4_0", + "think": true, + "think_toggleable": false }, - "medgemma": { - "4b": { - "name": "Medgemma-4B-NPU2", - "url": "https://huggingface.co/FastFlowLM/medgemma-4b-it-NPU2/resolve/main", - "file_url": "https://huggingface.co/api/models/FastFlowLM/medgemma-4b-it-NPU2/tree/main", - "ms_url": "https://modelscope.cn/models/amd/Medgemma-4B-NPU2", - "modified_at": "2025-05-30T00:00:00Z", - "size": 4000000000, - "default_context_length": 65536, - "max_prefill_len": 4096, - "vlm": true, - "flm_min_version": "0.9.43", - "files": [ - "config.json", - "model.q4nx", - "tokenizer.json", - "tokenizer_config.json", - "vision_weight.q4nx" - ], - "details": { - "family": "gemma3", - "think": false, - "think_toggleable": false, - "parameter_size": "4B", - "quantization_level": "Q4_1" - }, - "label":[ - "vision" - ], - "footprint": 4.5 - } + "file_url": "https://huggingface.co/api/models/FastFlowLM/LFM2.5-1.2B-Thinking-NPU2/tree/main", + "files": [ + "config.json", + "model.q4nx", + "tokenizer.json", + "tokenizer_config.json", + "chat_template.jinja" + ], + "flm_min_version": "0.9.27", + "footprint": 0.96, + "label": [ + "reasoning" + ], + "max_prefill_len": 4096, + "ms_url": "https://modelscope.cn/models/amd/LFM2.5-1.2B-Thinking-NPU2", + "name": "LFM2.5-1.2B-Thinking-NPU2", + "size": 1200000000, + "url": "https://huggingface.co/FastFlowLM/LFM2.5-1.2B-Thinking-NPU2", + "vlm": false + } + }, + "llama3.1": { + "8b": { + "default_context_length": 16384, + "details": { + "family": "llama3", + "parameter_size": "8B", + "quantization_level": "Q4_1", + "think": false }, - "medgemma1.5": { - "4b": { - "name": "Medgemma-1.5-4B-NPU2", - "url": "https://huggingface.co/FastFlowLM/medgemma-1.5-4b-it-NPU2", - "file_url": "https://huggingface.co/api/models/FastFlowLM/medgemma-1.5-4b-it-NPU2/tree/main", - "ms_url": "https://modelscope.cn/models/amd/Medgemma-1.5-4B-NPU2", - "modified_at": "2025-05-30T00:00:00Z", - "size": 4000000000, - "default_context_length": 65536, - "max_prefill_len": 4096, - "vlm": true, - "flm_min_version": "0.9.43", - "files": [ - "config.json", - "model.q4nx", - "tokenizer.json", - "tokenizer_config.json", - "vision_weight.q4nx" - ], - "details": { - "family": "gemma3", - "think": false, - "think_toggleable": false, - "parameter_size": "4B", - "quantization_level": "Q4_1" - }, - "label":[ - "vision" - ], - "footprint": 4.5 - } + "file_url": "https://huggingface.co/api/models/FastFlowLM/Llama-3.1-8B-NPU2/tree/main", + "files": [ + "config.json", + "model.q4nx", + "tokenizer.json", + "tokenizer_config.json" + ], + "flm_min_version": "0.9.21", + "footprint": 5.4, + "max_prefill_len": 4096, + "modified_at": "2025-05-30T00:00:00Z", + "ms_url": "https://modelscope.cn/models/amd/Llama-3.1-8B-NPU2", + "name": "Llama-3.1-8B-NPU2", + "size": 8000000000, + "url": "https://huggingface.co/FastFlowLM/Llama-3.1-8B-NPU2" + } + }, + "llama3.2": { + "1b": { + "default_context_length": 131072, + "details": { + "family": "llama3", + "parameter_size": "1B", + "quantization_level": "Q4_1", + "think": false }, - "llama3.2": { - "1b": { - "name": "Llama-3.2-1B-NPU2", - "url": "https://huggingface.co/FastFlowLM/Llama-3.2-1B-NPU2", - "file_url": "https://huggingface.co/api/models/FastFlowLM/Llama-3.2-1B-NPU2/tree/main", - "ms_url": "https://modelscope.cn/models/amd/Llama-3.2-1B-NPU2", - "modified_at": "2025-05-30T00:00:00Z", - "size": 1000000000, - "flm_min_version": "0.9.21", - "default_context_length": 131072, - "max_prefill_len": 4096, - "files": [ - "config.json", - "model.q4nx", - "tokenizer.json", - "tokenizer_config.json" - ], - "details": { - "family": "llama3", - "think": false, - "parameter_size": "1B", - "quantization_level": "Q4_1" - }, - "footprint": 1.3 - }, - "3b": { - "name": "Llama-3.2-3B-NPU2", - "url": "https://huggingface.co/FastFlowLM/Llama-3.2-3B-NPU2", - "file_url": "https://huggingface.co/api/models/FastFlowLM/Llama-3.2-3B-NPU2/tree/main", - "ms_url": "https://modelscope.cn/models/amd/Llama-3.2-3B-NPU2", - "modified_at": "2025-05-30T00:00:00Z", - "size": 3000000000, - "flm_min_version": "0.9.21", - "files": [ - "config.json", - "model.q4nx", - "tokenizer.json", - "tokenizer_config.json" - ], - "default_context_length": 65536, - "max_prefill_len": 4096, - "details": { - "family": "llama3", - "think": false, - "parameter_size": "3B", - "quantization_level": "Q4_1" - }, - "footprint": 2.7 - } + "file_url": "https://huggingface.co/api/models/FastFlowLM/Llama-3.2-1B-NPU2/tree/main", + "files": [ + "config.json", + "model.q4nx", + "tokenizer.json", + "tokenizer_config.json" + ], + "flm_min_version": "0.9.21", + "footprint": 1.3, + "max_prefill_len": 4096, + "modified_at": "2025-05-30T00:00:00Z", + "ms_url": "https://modelscope.cn/models/amd/Llama-3.2-1B-NPU2", + "name": "Llama-3.2-1B-NPU2", + "size": 1000000000, + "url": "https://huggingface.co/FastFlowLM/Llama-3.2-1B-NPU2" + }, + "3b": { + "default_context_length": 65536, + "details": { + "family": "llama3", + "parameter_size": "3B", + "quantization_level": "Q4_1", + "think": false }, - "llama3.1": { - "8b": { - "name": "Llama-3.1-8B-NPU2", - "url": "https://huggingface.co/FastFlowLM/Llama-3.1-8B-NPU2", - "file_url": "https://huggingface.co/api/models/FastFlowLM/Llama-3.1-8B-NPU2/tree/main", - "ms_url": "https://modelscope.cn/models/amd/Llama-3.1-8B-NPU2", - "modified_at": "2025-05-30T00:00:00Z", - "size": 8000000000, - "flm_min_version": "0.9.21", - "files": [ - "config.json", - "model.q4nx", - "tokenizer.json", - "tokenizer_config.json" - ], - "default_context_length": 16384, - "max_prefill_len": 4096, - "details": { - "family": "llama3", - "think": false, - "parameter_size": "8B", - "quantization_level": "Q4_1" - }, - "footprint": 5.4 - } + "file_url": "https://huggingface.co/api/models/FastFlowLM/Llama-3.2-3B-NPU2/tree/main", + "files": [ + "config.json", + "model.q4nx", + "tokenizer.json", + "tokenizer_config.json" + ], + "flm_min_version": "0.9.21", + "footprint": 2.7, + "max_prefill_len": 4096, + "modified_at": "2025-05-30T00:00:00Z", + "ms_url": "https://modelscope.cn/models/amd/Llama-3.2-3B-NPU2", + "name": "Llama-3.2-3B-NPU2", + "size": 3000000000, + "url": "https://huggingface.co/FastFlowLM/Llama-3.2-3B-NPU2" + } + }, + "medgemma": { + "4b": { + "default_context_length": 65536, + "details": { + "family": "gemma3", + "parameter_size": "4B", + "quantization_level": "Q4_1", + "think": false, + "think_toggleable": false }, - "deepseek-r1": { - "8b": { - "name": "Deepseek-R1-Distill-Llama-8B-NPU2", - "url": "https://huggingface.co/FastFlowLM/Deepseek-R1-Distill-Llama-8B-NPU2/resolve/main", - "file_url": "https://huggingface.co/api/models/FastFlowLM/Deepseek-R1-Distill-Llama-8B-NPU2/tree/main", - "ms_url": "https://modelscope.cn/models/amd/Deepseek-R1-Distill-Llama-8B-NPU2", - "modified_at": "2025-05-30T00:00:00Z", - "size": 8000000000, - "flm_min_version": "0.9.43", - "files": [ - "config.json", - "model.q4nx", - "tokenizer.json", - "tokenizer_config.json" - ], - "default_context_length": 16384, - "max_prefill_len": 4096, - "details": { - "family": "deepseek-r1", - "think": true, - "parameter_size": "8B", - "quantization_level": "Q4_1" - }, - "label":[ - "reasoning" - ], - "footprint": 5.4 - } + "file_url": "https://huggingface.co/api/models/FastFlowLM/medgemma-4b-it-NPU2/tree/main", + "files": [ + "config.json", + "model.q4nx", + "tokenizer.json", + "tokenizer_config.json", + "vision_weight.q4nx" + ], + "flm_min_version": "0.9.43", + "footprint": 4.5, + "label": [ + "vision" + ], + "max_prefill_len": 4096, + "modified_at": "2025-05-30T00:00:00Z", + "ms_url": "https://modelscope.cn/models/amd/Medgemma-4B-NPU2", + "name": "Medgemma-4B-NPU2", + "size": 4000000000, + "url": "https://huggingface.co/FastFlowLM/medgemma-4b-it-NPU2/resolve/main", + "vlm": true + } + }, + "medgemma1.5": { + "4b": { + "default_context_length": 65536, + "details": { + "family": "gemma3", + "parameter_size": "4B", + "quantization_level": "Q4_1", + "think": false, + "think_toggleable": false }, - "deepseek-r1-0528": { - "8b": { - "name": "DeepSeek-R1-0528-Qwen3-8B-NPU2", - "url": "https://huggingface.co/FastFlowLM/DeepSeek-R1-0528-Qwen3-8B-NPU2", - "file_url": "https://huggingface.co/api/models/FastFlowLM/DeepSeek-R1-0528-Qwen3-8B-NPU2/tree/main", - "ms_url": "https://modelscope.cn/models/amd/DeepSeek-R1-0528-Qwen3-8B-NPU2", - "modified_at": "2025-05-30T00:00:00Z", - "size": 8000000000, - "flm_min_version": "0.9.22", - "files": [ - "config.json", - "model.q4nx", - "tokenizer.json", - "tokenizer_config.json" - ], - "default_context_length": 16384, - "max_prefill_len": 4096, - "details": { - "family": "deepseek-r1-0528", - "think": true, - "think_toggleable": false, - "parameter_size": "8B", - "quantization_level": "Q4_1" - }, - "label":[ - "reasoning" - ], - "footprint": 5.6 - } + "file_url": "https://huggingface.co/api/models/FastFlowLM/medgemma-1.5-4b-it-NPU2/tree/main", + "files": [ + "config.json", + "model.q4nx", + "tokenizer.json", + "tokenizer_config.json", + "vision_weight.q4nx" + ], + "flm_min_version": "0.9.43", + "footprint": 4.5, + "label": [ + "vision" + ], + "max_prefill_len": 4096, + "modified_at": "2025-05-30T00:00:00Z", + "ms_url": "https://modelscope.cn/models/amd/Medgemma-1.5-4B-NPU2", + "name": "Medgemma-1.5-4B-NPU2", + "size": 4000000000, + "url": "https://huggingface.co/FastFlowLM/medgemma-1.5-4b-it-NPU2", + "vlm": true + } + }, + "nanbeige4.1": { + "3b": { + "default_context_length": 8192, + "details": { + "family": "nanbeige", + "parameter_size": "3B", + "quantization_level": "Q4_1", + "think": false }, - "qwen3": { - "0.6b": { - "name": "Qwen3-0.6B-NPU2", - "url": "https://huggingface.co/FastFlowLM/Qwen3-0.6B-NPU2", - "file_url": "https://huggingface.co/api/models/FastFlowLM/Qwen3-0.6B-NPU2/tree/main", - "ms_url": "https://modelscope.cn/models/amd/Qwen3-0.6B-NPU2", - "modified_at": "2025-05-30T00:00:00Z", - "size": 600000000, - "flm_min_version": "0.9.22", - "files": [ - "config.json", - "model.q4nx", - "tokenizer.json", - "tokenizer_config.json" - ], - "default_context_length": 32768, - "max_prefill_len": 4096, - "details": { - "family": "qwen3", - "think": false, - "think_toggleable": true, - "parameter_size": "0.6B", - "quantization_level": "Q4_1" - }, - "label":[ - "reasoning" - ], - "footprint": 0.66 - }, - "1.7b": { - "name": "Qwen3-1.7B-NPU2", - "url": "https://huggingface.co/FastFlowLM/Qwen3-1.7B-NPU2", - "file_url": "https://huggingface.co/api/models/FastFlowLM/Qwen3-1.7B-NPU2/tree/main", - "ms_url": "https://modelscope.cn/models/amd/Qwen3-1.7B-NPU2", - "modified_at": "2025-05-30T00:00:00Z", - "size": 1700000000, - "flm_min_version": "0.9.22", - "files": [ - "config.json", - "model.q4nx", - "tokenizer.json", - "tokenizer_config.json" - ], - "default_context_length": 32768, - "max_prefill_len": 4096, - "details": { - "family": "qwen3", - "think": true, - "think_toggleable": true, - "parameter_size": "1.7B", - "quantization_level": "Q4_1" - }, - "label":[ - "reasoning" - ], - "footprint": 1.6 - }, - "4b": { - "name": "Qwen3-4B-NPU2", - "url": "https://huggingface.co/FastFlowLM/Qwen3-4B-NPU2", - "file_url": "https://huggingface.co/api/models/FastFlowLM/Qwen3-4B-NPU2/tree/main", - "ms_url": "https://modelscope.cn/models/amd/Qwen3-4B-NPU2", - "modified_at": "2025-05-30T00:00:00Z", - "size": 4000000000, - "flm_min_version": "0.9.22", - "files": [ - "config.json", - "model.q4nx", - "tokenizer.json", - "tokenizer_config.json" - ], - "default_context_length": 32768, - "max_prefill_len": 4096, - "details": { - "family": "qwen3", - "think": true, - "think_toggleable": true, - "parameter_size": "4B", - "quantization_level": "Q4_1" - }, - "label":[ - "reasoning", - "tool-calling" - ], - "footprint": 3.1 - }, - "8b": { - "name": "Qwen3-8B-NPU2", - "url": "https://huggingface.co/FastFlowLM/Qwen3-8B-NPU2", - "file_url": "https://huggingface.co/api/models/FastFlowLM/Qwen3-8B-NPU2/tree/main", - "ms_url": "https://modelscope.cn/models/amd/Qwen3-8B-NPU2", - "modified_at": "2025-05-30T00:00:00Z", - "size": 8000000000, - "flm_min_version": "0.9.22", - "files": [ - "config.json", - "model.q4nx", - "tokenizer.json", - "tokenizer_config.json" - ], - "default_context_length": 16384, - "max_prefill_len": 4096, - "details": { - "family": "qwen3", - "think": true, - "think_toggleable": true, - "parameter_size": "8B", - "quantization_level": "Q4_1" - }, - "label":[ - "reasoning", - "tool-calling" - ], - "footprint": 5.6 - } + "file_url": "https://huggingface.co/api/models/FastFlowLM/Nanbeige4.1-3B-NPU2/tree/main", + "files": [ + "config.json", + "model.q4nx", + "tokenizer.json", + "tokenizer_config.json" + ], + "flm_min_version": "0.9.38", + "footprint": 3.1, + "label": [ + "reasoning" + ], + "max_prefill_len": 4096, + "modified_at": "2025-05-30T00:00:00Z", + "ms_url": "https://modelscope.cn/models/amd/Nanbeige4.1-3B-NPU2", + "name": "Nanbeige4.1-3B-NPU2", + "size": 1000000000, + "url": "https://huggingface.co/FastFlowLM/Nanbeige4.1-3B-NPU2" + } + }, + "phi4-mini-it": { + "4b": { + "default_context_length": 32768, + "details": { + "family": "phi4", + "parameter_size": "4B", + "quantization_level": "Q4_1", + "think": false, + "think_toggleable": false }, - "qwen3-tk": { - "4b": { - "name": "Qwen3-4B-Thinking-2507-NPU2", - "url": "https://huggingface.co/FastFlowLM/Qwen3-4B-Thinking-2507-NPU2", - "file_url": "https://huggingface.co/api/models/FastFlowLM/Qwen3-4B-Thinking-2507-NPU2/tree/main", - "ms_url": "https://modelscope.cn/models/amd/Qwen3-4B-Thinking-2507-NPU2", - "modified_at": "2025-05-30T00:00:00Z", - "size": 4000000000, - "flm_min_version": "0.9.22", - "files": [ - "config.json", - "model.q4nx", - "tokenizer.json", - "tokenizer_config.json" - ], - "default_context_length": 32768, - "max_prefill_len": 4096, - "details": { - "family": "qwen3-tk", - "think": true, - "think_toggleable": false, - "parameter_size": "4B", - "quantization_level": "Q4_1" - }, - "label":[ - "reasoning", - "tool-calling" - ], - "footprint": 3.1 - } + "file_url": "https://huggingface.co/api/models/FastFlowLM/Phi4-mini-Instruct-NPU2/tree/main", + "files": [ + "config.json", + "model.q4nx", + "tokenizer.json", + "tokenizer_config.json" + ], + "flm_min_version": "0.9.25", + "footprint": 3.4, + "max_prefill_len": 4096, + "ms_url": "https://modelscope.cn/models/amd/Phi4-mini-Instruct-NPU2", + "name": "Phi4-mini-Instruct-NPU2", + "size": 4000000000, + "url": "https://huggingface.co/FastFlowLM/Phi4-mini-Instruct-NPU2", + "vlm": false + } + }, + "phi4-mini-it-aie4": { + "4b": { + "bundled_overlays": { + "config.json": { + "path": "phi4-mini-it-aie4/config.json", + "sha256": "1b3e74125a109c05f53c8383def18359d8581619f998c4f91b3b5bb78bf2919f", + "size": 257 + }, + "corelib_phi4_manifest.json": { + "path": "phi4-mini-it-aie4/corelib_phi4_manifest.json", + "sha256": "09cee6efafc513a2048c89e75b40d92d096144f0eed3b2459c138032b69dc045", + "size": 274527 + }, + "provenance.json": { + "path": "phi4-mini-it-aie4/provenance.json", + "sha256": "f610a719bece3c40cad47e15049be2ab285495c5ea56d1b2b73ad6293f53cc86", + "size": 3330 + }, + "tokenizer_config.json": { + "path": "phi4-mini-it-aie4/tokenizer_config.json", + "sha256": "274d22c3cd28c042f28a681832722536663b06e8c9e88b6616622dceab922ca6", + "size": 3036 + } }, - "qwen3-it": { - "4b": { - "name": "Qwen3-4B-Instruct-2507-NPU2", - "url": "https://huggingface.co/FastFlowLM/Qwen3-4B-Instruct-2507-NPU2", - "file_url": "https://huggingface.co/api/models/FastFlowLM/Qwen3-4B-Instruct-2507-NPU2/tree/main", - "ms_url": "https://modelscope.cn/models/amd/Qwen3-4B-Instruct-2507-NPU2", - "modified_at": "2025-05-30T00:00:00Z", - "size": 4000000000, - "flm_min_version": "0.9.22", - "files": [ - "config.json", - "model.q4nx", - "tokenizer.json", - "tokenizer_config.json" - ], - "default_context_length": 32768, - "max_prefill_len": 4096, - "details": { - "family": "qwen3-it", - "think": false, - "think_toggleable": false, - "parameter_size": "4B", - "quantization_level": "Q4_1" - }, - "label":[ - "tool-calling" - ], - "footprint": 3.1 - } + "default_context_length": 4096, + "details": { + "execution_backend": "corelib_aie4", + "family": "phi4", + "parameter_size": "4B", + "quantization_level": "MatMulNBits Q4", + "think": false, + "think_toggleable": false }, - "gpt-oss": { - "20b": { - "name": "GPT-OSS-20B-NPU2", - "url": "https://huggingface.co/FastFlowLM/GPT-OSS-20B-NPU2", - "file_url": "https://huggingface.co/api/models/FastFlowLM/GPT-OSS-20B-NPU2/tree/main", - "ms_url": "https://modelscope.cn/models/amd/GPT-OSS-20B-NPU2", - "modified_at": "2025-05-30T00:00:00Z", - "size": 20000000000, - "flm_min_version": "0.9.20", - "files": [ - "config.json", - "model.q4nx", - "tokenizer.json", - "tokenizer_config.json" - ], - "default_context_length": 8192, - "max_prefill_len": 4096, - "details": { - "family": "gpt-oss", - "think": true, - "think_toggleable": false, - "parameter_size": "20B", - "quantization_level": "Q4_1" - }, - "label":[ - "reasoning" - ], - "footprint": 14.0 - } + "file_url": "https://huggingface.co/api/models/amd/phi-4-mini-instruct-oga-dml/tree/e751fb68c2cfffe6b0d32942118f75ac0a0365bb?recursive=true&expand=false", + "files": [ + "added_tokens.json", + "chat_template.jinja", + "config.json", + "corelib_phi4_manifest.json", + "merges.txt", + "model.onnx", + "model.onnx.data", + "provenance.json", + "special_tokens_map.json", + "tokenizer.json", + "tokenizer_config.json", + "vocab.json" + ], + "flm_min_version": "1.0.4", + "footprint": 3.05, + "max_prefill_len": 4096, + "modelscope_supported": false, + "name": "Phi-4-mini-instruct-oga-dml-AIE4", + "revision": "e751fb68c2cfffe6b0d32942118f75ac0a0365bb", + "size": 3271001977, + "url": "https://huggingface.co/amd/phi-4-mini-instruct-oga-dml", + "vlm": false + } + }, + "qwen2.5-it": { + "3b": { + "default_context_length": 32768, + "details": { + "family": "qwen2", + "parameter_size": "3B", + "quantization_level": "Q4_0", + "think": true, + "think_toggleable": false }, - "gpt-oss-sg": { - "20b": { - "name": "GPT-OSS-Safeguard-20b-NPU2", - "url": "https://huggingface.co/FastFlowLM/GPT-OSS-Safeguard-20b-NPU2", - "file_url": "https://huggingface.co/api/models/FastFlowLM/GPT-OSS-Safeguard-20b-NPU2/tree/main", - "ms_url": "https://modelscope.cn/models/amd/GPT-OSS-Safeguard-20b-NPU2", - "modified_at": "2025-05-30T00:00:00Z", - "size": 20000000000, - "flm_min_version": "0.9.20", - "files": [ - "config.json", - "model.q4nx", - "tokenizer.json", - "tokenizer_config.json" - ], - "default_context_length": 8192, - "max_prefill_len": 4096, - "details": { - "family": "gpt-oss", - "think": true, - "think_toggleable": false, - "parameter_size": "20B", - "quantization_level": "Q4_1" - }, - "label":[ - "reasoning" - ], - "footprint": 14.0 - } + "file_url": "https://huggingface.co/api/models/FastFlowLM/Qwen2.5-3B-Instruct-NPU2/tree/main", + "files": [ + "config.json", + "model.q4nx", + "tokenizer.json", + "tokenizer_config.json" + ], + "flm_min_version": "0.9.32", + "footprint": 2.5, + "max_prefill_len": 4096, + "modified_at": "2025-05-30T00:00:00Z", + "ms_url": "https://modelscope.cn/models/amd/Qwen2.5-3B-Instruct-NPU2", + "name": "Qwen2.5-3B-Instruct-NPU2", + "size": 3000000000, + "url": "https://huggingface.co/FastFlowLM/Qwen2.5-3B-Instruct-NPU2/resolve/main" + } + }, + "qwen2.5vl-it": { + "3b": { + "default_context_length": 32768, + "details": { + "family": "qwen2vl", + "parameter_size": "3B", + "quantization_level": "Q4_1", + "think": true, + "think_toggleable": false }, - "qwen2.5-it": { - "3b": { - "name": "Qwen2.5-3B-Instruct-NPU2", - "url": "https://huggingface.co/FastFlowLM/Qwen2.5-3B-Instruct-NPU2/resolve/main", - "file_url": "https://huggingface.co/api/models/FastFlowLM/Qwen2.5-3B-Instruct-NPU2/tree/main", - "ms_url": "https://modelscope.cn/models/amd/Qwen2.5-3B-Instruct-NPU2", - "modified_at": "2025-05-30T00:00:00Z", - "size": 3000000000, - "flm_min_version": "0.9.32", - "files": [ - "config.json", - "model.q4nx", - "tokenizer.json", - "tokenizer_config.json" - ], - "default_context_length": 32768, - "max_prefill_len": 4096, - "details": { - "family": "qwen2", - "think": true, - "think_toggleable": false, - "parameter_size": "3B", - "quantization_level": "Q4_0" - }, - "footprint": 2.5 - } + "file_url": "https://huggingface.co/api/models/FastFlowLM/Qwen2.5-VL-3B-Instruct-NPU2/tree/main", + "files": [ + "config.json", + "model.q4nx", + "tokenizer.json", + "tokenizer_config.json", + "vision_weights.q4nx" + ], + "flm_min_version": "0.9.32", + "footprint": 3.8, + "label": [ + "vision" + ], + "max_prefill_len": 4096, + "modified_at": "2025-05-30T00:00:00Z", + "ms_url": "https://modelscope.cn/models/amd/Qwen2.5-VL-3B-Instruct-NPU2", + "name": "Qwen2.5-VL-3B-Instruct-NPU2", + "size": 3000000000, + "url": "https://huggingface.co/FastFlowLM/Qwen2.5-VL-3B-Instruct-NPU2/resolve/main", + "vlm": true + } + }, + "qwen3": { + "0.6b": { + "default_context_length": 32768, + "details": { + "family": "qwen3", + "parameter_size": "0.6B", + "quantization_level": "Q4_1", + "think": false, + "think_toggleable": true }, - "qwen2.5vl-it": { - "3b": { - "name": "Qwen2.5-VL-3B-Instruct-NPU2", - "url": "https://huggingface.co/FastFlowLM/Qwen2.5-VL-3B-Instruct-NPU2/resolve/main", - "file_url": "https://huggingface.co/api/models/FastFlowLM/Qwen2.5-VL-3B-Instruct-NPU2/tree/main", - "ms_url": "https://modelscope.cn/models/amd/Qwen2.5-VL-3B-Instruct-NPU2", - "modified_at": "2025-05-30T00:00:00Z", - "size": 3000000000, - "flm_min_version": "0.9.32", - "files": [ - "config.json", - "model.q4nx", - "tokenizer.json", - "tokenizer_config.json", - "vision_weights.q4nx" - ], - "vlm": true, - "default_context_length": 32768, - "max_prefill_len": 4096, - "details": { - "family": "qwen2vl", - "think": true, - "think_toggleable": false, - "parameter_size": "3B", - "quantization_level": "Q4_1" - }, - "label":[ - "vision" - ], - "footprint": 3.8 - } - } + "file_url": "https://huggingface.co/api/models/FastFlowLM/Qwen3-0.6B-NPU2/tree/main", + "files": [ + "config.json", + "model.q4nx", + "tokenizer.json", + "tokenizer_config.json" + ], + "flm_min_version": "0.9.22", + "footprint": 0.66, + "label": [ + "reasoning" + ], + "max_prefill_len": 4096, + "modified_at": "2025-05-30T00:00:00Z", + "ms_url": "https://modelscope.cn/models/amd/Qwen3-0.6B-NPU2", + "name": "Qwen3-0.6B-NPU2", + "size": 600000000, + "url": "https://huggingface.co/FastFlowLM/Qwen3-0.6B-NPU2" + }, + "1.7b": { + "default_context_length": 32768, + "details": { + "family": "qwen3", + "parameter_size": "1.7B", + "quantization_level": "Q4_1", + "think": true, + "think_toggleable": true + }, + "file_url": "https://huggingface.co/api/models/FastFlowLM/Qwen3-1.7B-NPU2/tree/main", + "files": [ + "config.json", + "model.q4nx", + "tokenizer.json", + "tokenizer_config.json" + ], + "flm_min_version": "0.9.22", + "footprint": 1.6, + "label": [ + "reasoning" + ], + "max_prefill_len": 4096, + "modified_at": "2025-05-30T00:00:00Z", + "ms_url": "https://modelscope.cn/models/amd/Qwen3-1.7B-NPU2", + "name": "Qwen3-1.7B-NPU2", + "size": 1700000000, + "url": "https://huggingface.co/FastFlowLM/Qwen3-1.7B-NPU2" + }, + "4b": { + "default_context_length": 32768, + "details": { + "family": "qwen3", + "parameter_size": "4B", + "quantization_level": "Q4_1", + "think": true, + "think_toggleable": true + }, + "file_url": "https://huggingface.co/api/models/FastFlowLM/Qwen3-4B-NPU2/tree/main", + "files": [ + "config.json", + "model.q4nx", + "tokenizer.json", + "tokenizer_config.json" + ], + "flm_min_version": "0.9.22", + "footprint": 3.1, + "label": [ + "reasoning", + "tool-calling" + ], + "max_prefill_len": 4096, + "modified_at": "2025-05-30T00:00:00Z", + "ms_url": "https://modelscope.cn/models/amd/Qwen3-4B-NPU2", + "name": "Qwen3-4B-NPU2", + "size": 4000000000, + "url": "https://huggingface.co/FastFlowLM/Qwen3-4B-NPU2" + }, + "8b": { + "default_context_length": 16384, + "details": { + "family": "qwen3", + "parameter_size": "8B", + "quantization_level": "Q4_1", + "think": true, + "think_toggleable": true + }, + "file_url": "https://huggingface.co/api/models/FastFlowLM/Qwen3-8B-NPU2/tree/main", + "files": [ + "config.json", + "model.q4nx", + "tokenizer.json", + "tokenizer_config.json" + ], + "flm_min_version": "0.9.22", + "footprint": 5.6, + "label": [ + "reasoning", + "tool-calling" + ], + "max_prefill_len": 4096, + "modified_at": "2025-05-30T00:00:00Z", + "ms_url": "https://modelscope.cn/models/amd/Qwen3-8B-NPU2", + "name": "Qwen3-8B-NPU2", + "size": 8000000000, + "url": "https://huggingface.co/FastFlowLM/Qwen3-8B-NPU2" + } + }, + "qwen3-it": { + "4b": { + "default_context_length": 32768, + "details": { + "family": "qwen3-it", + "parameter_size": "4B", + "quantization_level": "Q4_1", + "think": false, + "think_toggleable": false + }, + "file_url": "https://huggingface.co/api/models/FastFlowLM/Qwen3-4B-Instruct-2507-NPU2/tree/main", + "files": [ + "config.json", + "model.q4nx", + "tokenizer.json", + "tokenizer_config.json" + ], + "flm_min_version": "0.9.22", + "footprint": 3.1, + "label": [ + "tool-calling" + ], + "max_prefill_len": 4096, + "modified_at": "2025-05-30T00:00:00Z", + "ms_url": "https://modelscope.cn/models/amd/Qwen3-4B-Instruct-2507-NPU2", + "name": "Qwen3-4B-Instruct-2507-NPU2", + "size": 4000000000, + "url": "https://huggingface.co/FastFlowLM/Qwen3-4B-Instruct-2507-NPU2" + } + }, + "qwen3-tk": { + "4b": { + "default_context_length": 32768, + "details": { + "family": "qwen3-tk", + "parameter_size": "4B", + "quantization_level": "Q4_1", + "think": true, + "think_toggleable": false + }, + "file_url": "https://huggingface.co/api/models/FastFlowLM/Qwen3-4B-Thinking-2507-NPU2/tree/main", + "files": [ + "config.json", + "model.q4nx", + "tokenizer.json", + "tokenizer_config.json" + ], + "flm_min_version": "0.9.22", + "footprint": 3.1, + "label": [ + "reasoning", + "tool-calling" + ], + "max_prefill_len": 4096, + "modified_at": "2025-05-30T00:00:00Z", + "ms_url": "https://modelscope.cn/models/amd/Qwen3-4B-Thinking-2507-NPU2", + "name": "Qwen3-4B-Thinking-2507-NPU2", + "size": 4000000000, + "url": "https://huggingface.co/FastFlowLM/Qwen3-4B-Thinking-2507-NPU2" + } + }, + "qwen3.5": { + "0.8b": { + "default_context_length": 32768, + "details": { + "family": "qwen3.5", + "format": "NPU2", + "parameter_size": "0.8B", + "quantization_level": "Q4_1", + "think": true + }, + "file_url": "https://huggingface.co/api/models/FastFlowLM/Qwen3.5-0.8B-NPU2/tree/flm_q4k_high_precision", + "files": [ + "config.json", + "model.q4nx", + "tokenizer.json", + "tokenizer_config.json", + "vision_weight.q4nx", + "chat_template.jinja" + ], + "flm_min_version": "1.0.3", + "footprint": 1.4, + "label": [ + "vision", + "reasoning" + ], + "max_prefill_len": 4096, + "ms_url": "https://modelscope.cn/models/amd/Qwen3.5-0.8B-NPU2/resolve/flm_q4k_high_precision", + "name": "Qwen3.5-0.8B-NPU2", + "size": 800000000, + "url": "https://huggingface.co/FastFlowLM/Qwen3.5-0.8B-NPU2/resolve/flm_q4k_high_precision", + "vlm": true + }, + "2b": { + "default_context_length": 32768, + "details": { + "family": "qwen3.5", + "format": "NPU2", + "parameter_size": "2B", + "quantization_level": "Q4_1", + "think": true + }, + "file_url": "https://huggingface.co/api/models/FastFlowLM/Qwen3.5-2B-NPU2/tree/flm_q4k_high_precision", + "files": [ + "config.json", + "model.q4nx", + "tokenizer.json", + "tokenizer_config.json", + "vision_weight.q4nx", + "chat_template.jinja" + ], + "flm_min_version": "1.0.3", + "footprint": 3.2, + "label": [ + "vision", + "reasoning", + "tool-calling" + ], + "max_prefill_len": 4096, + "ms_url": "https://modelscope.cn/models/amd/Qwen3.5-2B-NPU2/resolve/flm_q4k_high_precision", + "name": "Qwen3.5-2B-NPU2", + "size": 2000000000, + "url": "https://huggingface.co/FastFlowLM/Qwen3.5-2B-NPU2/resolve/flm_q4k_high_precision", + "vlm": true + }, + "4b": { + "default_context_length": 32768, + "details": { + "family": "qwen3.5", + "format": "NPU2", + "parameter_size": "4B", + "quantization_level": "Q4_1", + "think": true + }, + "file_url": "https://huggingface.co/api/models/FastFlowLM/Qwen3.5-4B-NPU2/tree/flm_q4k_high_precision", + "files": [ + "config.json", + "model.q4nx", + "tokenizer.json", + "tokenizer_config.json", + "vision_weight.q4nx", + "chat_template.jinja" + ], + "flm_min_version": "1.0.3", + "footprint": 5.2, + "label": [ + "vision", + "reasoning", + "tool-calling" + ], + "max_prefill_len": 4096, + "ms_url": "https://modelscope.cn/models/amd/Qwen3.5-4B-NPU2/resolve/flm_q4k_high_precision", + "name": "Qwen3.5-4B-NPU2", + "size": 4000000000, + "url": "https://huggingface.co/FastFlowLM/Qwen3.5-4B-NPU2/resolve/flm_q4k_high_precision", + "vlm": true + }, + "9b": { + "default_context_length": 32768, + "details": { + "family": "qwen3.5", + "format": "NPU2", + "parameter_size": "9B", + "quantization_level": "Q4_1", + "think": true + }, + "file_url": "https://huggingface.co/api/models/FastFlowLM/Qwen3.5-9B-NPU2/tree/flm_q4k_high_precision", + "files": [ + "config.json", + "model.q4nx", + "tokenizer.json", + "tokenizer_config.json", + "vision_weight.q4nx", + "chat_template.jinja" + ], + "flm_min_version": "1.0.3", + "footprint": 8.94, + "label": [ + "vision", + "reasoning", + "tool-calling" + ], + "max_prefill_len": 4096, + "ms_url": "https://modelscope.cn/models/amd/Qwen3.5-9B-NPU2/resolve/flm_q4k_high_precision", + "name": "Qwen3.5-9B-NPU2", + "size": 9000000000, + "url": "https://huggingface.co/FastFlowLM/Qwen3.5-9B-NPU2/resolve/flm_q4k_high_precision", + "vlm": true + } + }, + "qwen3.6-moe": { + "35b-a3b": { + "default_context_length": 32768, + "details": { + "family": "qwen3.6-moe", + "format": "NPU2", + "parameter_size": "35B", + "quantization_level": "Q4_K_S", + "think": true + }, + "file_url": "https://huggingface.co/api/models/FastFlowLM/Qwen3.6-35B-A3B-NPU2/tree/flm_q4k_high_precision", + "files": [ + "config.json", + "model.q4nx", + "tokenizer.json", + "tokenizer_config.json", + "vision_weight.q4nx", + "chat_template.jinja" + ], + "flm_min_version": "1.0.3", + "footprint": 24.3, + "label": [ + "vision", + "reasoning", + "tool-calling" + ], + "max_prefill_len": 4096, + "ms_url": "https://modelscope.cn/models/amd/Qwen3.6-35B-A3B-NPU2/resolve/flm_q4k_high_precision", + "name": "Qwen3.6-35B-A3B-NPU2", + "size": 35000000000, + "url": "https://huggingface.co/FastFlowLM/Qwen3.6-35B-A3B-NPU2/resolve/flm_q4k_high_precision", + "vlm": true + } + }, + "qwen3vl-it": { + "4b": { + "default_context_length": 32768, + "details": { + "family": "qwen3vl", + "format": "NPU2", + "parameter_size": "4B", + "quantization_level": "Q4_1", + "think": false + }, + "file_url": "https://huggingface.co/api/models/FastFlowLM/Qwen3-VL-4B-Instruct-NPU2/tree/main", + "files": [ + "config.json", + "model.q4nx", + "tokenizer.json", + "tokenizer_config.json", + "vision_weight.q4nx" + ], + "flm_min_version": "0.9.22", + "footprint": 3.9, + "label": [ + "vision", + "tool-calling" + ], + "max_prefill_len": 4096, + "ms_url": "https://modelscope.cn/models/amd/Qwen3-VL-4B-Instruct-NPU2", + "name": "Qwen3-VL-4B-Instruct-NPU2", + "size": 4000000000, + "url": "https://huggingface.co/FastFlowLM/Qwen3-VL-4B-Instruct-NPU2", + "vlm": true + } + }, + "translategemma": { + "4b": { + "default_context_length": 65536, + "details": { + "family": "gemma3", + "parameter_size": "4B", + "quantization_level": "Q4_1", + "think": false, + "think_toggleable": false + }, + "file_url": "https://huggingface.co/api/models/FastFlowLM/Translategemma-4B-Instruct-NPU2/tree/main", + "files": [ + "config.json", + "model.q4nx", + "tokenizer.json", + "tokenizer_config.json", + "vision_weight.q4nx" + ], + "flm_min_version": "0.9.34", + "footprint": 4.5, + "label": [ + "vision" + ], + "max_prefill_len": 4096, + "modified_at": "2025-05-30T00:00:00Z", + "ms_url": "https://modelscope.cn/models/amd/Translategemma-4B-Instruct-NPU2", + "name": "Translategemma-4B-Instruct-NPU2", + "size": 4000000000, + "url": "https://huggingface.co/FastFlowLM/Translategemma-4B-Instruct-NPU2/resolve/main", + "vlm": true + } + }, + "whisper-v3": { + "turbo": { + "asr": true, + "default_context_length": 448, + "details": { + "family": "whisper-v3", + "format": "NPU2", + "parameter_size": "1B", + "quantization_level": "Q4_1", + "think": false + }, + "file_url": "https://huggingface.co/api/models/FastFlowLM/Whisper-V3-Turbo-NPU2/tree/main", + "files": [ + "config.json", + "tokenizer.json", + "tokenizer_config.json", + "model.q4nx" + ], + "flm_min_version": "0.9.14", + "footprint": 0.62, + "label": [ + "audio", + "realtime-transcription", + "transcription" + ], + "ms_url": "https://modelscope.cn/models/amd/Whisper-V3-Turbo-NPU2", + "name": "Whisper-V3-Turbo-NPU2", + "size": 1000000000, + "url": "https://huggingface.co/FastFlowLM/Whisper-V3-Turbo-NPU2" + } } + } } diff --git a/src/test/phi4_corelib_aie4/CMakeLists.txt b/src/test/phi4_corelib_aie4/CMakeLists.txt index 949ad6e6..cec1eec4 100644 --- a/src/test/phi4_corelib_aie4/CMakeLists.txt +++ b/src/test/phi4_corelib_aie4/CMakeLists.txt @@ -91,7 +91,8 @@ target_include_directories(test_model_catalog PRIVATE ${FASTFLOW_SOURCE_DIR}/include ${FASTFLOW_SOURCE_DIR}/pull) target_compile_definitions(test_model_catalog PRIVATE - FLM_TEST_SOURCE_DIR="${FASTFLOW_SOURCE_DIR}") + FLM_TEST_SOURCE_DIR="${FASTFLOW_SOURCE_DIR}" + __FLM_VERSION__="${FLM_VERSION}") add_test(NAME test_model_catalog COMMAND test_model_catalog) add_library(test_model_downloader_compile OBJECT diff --git a/src/test/phi4_corelib_aie4/test_model_catalog.cpp b/src/test/phi4_corelib_aie4/test_model_catalog.cpp index 1304b8b6..774eecb4 100644 --- a/src/test/phi4_corelib_aie4/test_model_catalog.cpp +++ b/src/test/phi4_corelib_aie4/test_model_catalog.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -105,16 +106,21 @@ void TestCatalogContract() { model.at("details").at("execution_backend") == "corelib_aie4"); + // Design 8.1. `.gitattributes` is a Git repository artifact and + // `genai_config.json` is excluded by MODEL-2: flm.exe runs no ORT or genai + // graph, and an unused configuration invites a future reader to believe it + // is authoritative. `chat_template.jinja` is kept deliberately, because it + // is the verbatim source of the template the overlay inlines and is what + // makes the overlay auditable on the target machine. const std::set expected_files{ - ".gitattributes", "added_tokens.json", "chat_template.jinja", "config.json", "corelib_phi4_manifest.json", - "genai_config.json", "merges.txt", "model.onnx", "model.onnx.data", + "provenance.json", "special_tokens_map.json", "tokenizer.json", "tokenizer_config.json", @@ -123,9 +129,10 @@ void TestCatalogContract() { CHECK(JsonStringSet(model.at("files")) == expected_files); const auto& overlays = model.at("bundled_overlays"); - CHECK(overlays.size() == 3); + CHECK(overlays.size() == 4); CHECK(overlays.contains("config.json")); CHECK(overlays.contains("corelib_phi4_manifest.json")); + CHECK(overlays.contains("provenance.json")); CHECK(overlays.contains("tokenizer_config.json")); const std::filesystem::path overlay_root = @@ -174,10 +181,19 @@ void TestCatalogContract() { find_record("tokenizer.json").at("lfs").at("oid") == "382cc235b56c725945e149cc25f191da667c836655efd0857b004320e90e91ea"); - constexpr std::uint64_t kRemoteLogicalBytes = UINT64_C(3270726823); - constexpr std::uint64_t kReplacedTokenizerConfigBytes = 2654; - const std::uint64_t expected_size = - kRemoteLogicalBytes - kReplacedTokenizerConfigBytes + overlay_size; + // `size` and `footprint` cover the assembled on-disk directory: the + // upstream files actually downloaded plus the overlay files shipped inside + // FastFlow. Deriving it from the metadata here, rather than restating a + // literal, is what keeps the number honest when the file set changes. + std::uint64_t upstream_size = 0; + for (const auto& file : model.at("files")) { + const auto name = file.get(); + if (overlays.contains(name)) { + continue; + } + upstream_size += find_record(name).at("size").get(); + } + const std::uint64_t expected_size = upstream_size + overlay_size; CHECK(model.at("size").get() == expected_size); const double expected_footprint = std::round( @@ -188,6 +204,104 @@ void TestCatalogContract() { CHECK(model.at("footprint").get() == expected_footprint); } +// Design `PACKAGE-1`. The assembled package has two provenances and they are +// checked separately. Requiring every catalog file to carry a Hugging Face +// metadata record cannot hold: the overlay files are FastFlow-authored and do +// not exist upstream by construction. The check that matters is the reverse +// one. +void TestSplitProvenance() { + const std::filesystem::path source(FLM_TEST_SOURCE_DIR); + const json catalog = ReadJson(source / "model_list.json"); + const json metadata = ReadJson(source / "model_info.json"); + const auto& model = + catalog.at("models").at("phi4-mini-it-aie4").at("4b"); + const auto& overlays = model.at("bundled_overlays"); + const auto& remote = metadata.at(kTag); + + std::set upstream_paths; + for (const auto& record : remote) { + upstream_paths.insert(record.at("path").get()); + } + + // Upstream files: every one must have a metadata record to download and + // hash-check against. + for (const auto& file : model.at("files")) { + const auto name = file.get(); + if (overlays.contains(name)) { + continue; + } + CHECK(upstream_paths.count(name) == 1); + } + + // Overlay files: each must be installed beside FastFlow. Three of them + // must have no upstream record at all; an upstream record for one would + // mean FastFlow's contract was published to the model repository, which + // makes the two provenances indistinguishable. + const std::filesystem::path overlay_root = source / "model_overlays"; + for (const auto& name : { + std::string("config.json"), + std::string("corelib_phi4_manifest.json"), + std::string("provenance.json")}) { + CHECK(overlays.contains(name)); + CHECK(upstream_paths.count(name) == 0); + CHECK( + std::filesystem::is_regular_file( + overlay_root / + overlays.at(name).at("path").get())); + } + + // tokenizer_config.json is the one overlay that also exists upstream. The + // overlay shadows it because the published file carries neither a chat + // template nor eos_token_id, so its upstream record is expected. + CHECK(overlays.contains("tokenizer_config.json")); + CHECK(upstream_paths.count("tokenizer_config.json") == 1); + + // Nothing upstream is silently dropped: every record is either packaged or + // one of the two files excluded on purpose. + const std::set excluded{".gitattributes", "genai_config.json"}; + const auto packaged = JsonStringSet(model.at("files")); + for (const auto& path : upstream_paths) { + CHECK(packaged.count(path) == 1 || excluded.count(path) == 1); + } + for (const auto& name : excluded) { + CHECK(upstream_paths.count(name) == 1); + CHECK(packaged.count(name) == 0); + } +} + +// `ModelDownloader::check_model_compatibility` compares three versions: the +// overlay config.json's flm_version, the catalog's flm_min_version, and the +// binary's own __FLM_VERSION__. A binary built as 1.0.3 reports Incompatible +// for a 1.0.4 overlay and refuses its own catalog entry, which is a failure +// that only appears at first real load. Pin the relationship here instead. +void TestVersionGateIsSelfConsistent() { + const std::filesystem::path source(FLM_TEST_SOURCE_DIR); + const json catalog = ReadJson(source / "model_list.json"); + const auto& model = + catalog.at("models").at("phi4-mini-it-aie4").at("4b"); + const json overlay_config = + ReadJson(source / "model_overlays" / "phi4-mini-it-aie4" / + "config.json"); + + const auto encode = [](const std::string& version) -> std::uint32_t { + int major = -1; + int minor = -1; + int patch = -1; + CHECK( + std::sscanf(version.c_str(), "%d.%d.%d", &major, &minor, &patch) == + 3); + CHECK(major >= 0 && minor >= 0 && patch >= 0); + return static_cast( + major * 1000000 + minor * 1000 + patch); + }; + + const auto minimum = model.at("flm_min_version").get(); + const auto packaged = overlay_config.at("flm_version").get(); + CHECK(packaged == minimum); + CHECK(encode(packaged) >= encode(minimum)); + CHECK(encode(std::string(__FLM_VERSION__)) >= encode(packaged)); +} + void TestSourcePolicyAndPinnedUrls() { const std::filesystem::path source(FLM_TEST_SOURCE_DIR); const json catalog = ReadJson(source / "model_list.json"); @@ -274,6 +388,8 @@ void TestBundledOverlayCopyAndIntegrity() { int main() { try { TestCatalogContract(); + TestSplitProvenance(); + TestVersionGateIsSelfConsistent(); TestSourcePolicyAndPinnedUrls(); TestBundledOverlayCopyAndIntegrity(); std::cout << "model catalog tests passed\n"; diff --git a/tools/package_phi4_corelib_aie4.py b/tools/package_phi4_corelib_aie4.py index c3d59004..244f2379 100644 --- a/tools/package_phi4_corelib_aie4.py +++ b/tools/package_phi4_corelib_aie4.py @@ -25,6 +25,44 @@ "?recursive=true&expand=false" ) +# Design 8.1 / `PACKAGE-1`. The assembled model directory has two provenances +# and they are validated separately, because a rule requiring every catalog +# file to carry Hugging Face metadata cannot succeed: these files are +# FastFlow-authored and do not exist upstream by construction. +# +# `tokenizer_config.json` is the one overlay that also exists upstream. The +# overlay shadows it because the published file carries neither a chat template +# nor eos_token_id, so it is downloaded from nowhere and its upstream size is +# not counted, but it legitimately has an upstream record. The other three must +# never acquire one: an upstream record for `config.json`, +# `corelib_phi4_manifest.json` or `provenance.json` would mean FastFlow's +# contract was published to the model repository, which is a provenance error +# and not a convenience. +OVERLAY_FILES = ( + "config.json", + "corelib_phi4_manifest.json", + "provenance.json", + "tokenizer_config.json", +) +OVERLAY_FILES_WITHOUT_UPSTREAM = ( + "config.json", + "corelib_phi4_manifest.json", + "provenance.json", +) + +# Upstream files deliberately not carried into the assembled package. +# +# `genai_config.json` is excluded by `MODEL-2`: flm.exe runs no ORT or genai +# graph, and shipping an unused configuration invites a future reader to treat +# it as authoritative. `.gitattributes` is a Git repository artifact rather +# than a model file. `chat_template.jinja` is deliberately NOT excluded: it is +# the verbatim source of the template the overlay inlines, and keeping it is +# what makes the overlay auditable on the target machine. +EXCLUDED_UPSTREAM_FILES = ( + ".gitattributes", + "genai_config.json", +) + _EXPECTED_DECODER = { "head_size": 128, "hidden_size": 3072, @@ -331,26 +369,26 @@ def build_catalog_entry( git_files: list[dict[str, object]], ) -> dict[str, object]: overlay_dir = Path(overlay_dir) - overlay_names = ( - "config.json", - "corelib_phi4_manifest.json", - "tokenizer_config.json", - ) overlays = { name: { "path": f"{overlay_dir.name}/{name}", **_sha256_record(overlay_dir / name), } - for name in overlay_names + for name in OVERLAY_FILES } indexed = _index_git_records(git_files) - final_files = sorted(set(indexed) | set(overlays)) - remote_size = sum( - int(record["size"]) - for path, record in indexed.items() - if path not in overlays - ) + upstream_files = { + path + for path in indexed + if path not in overlays and path not in EXCLUDED_UPSTREAM_FILES + } + final_files = sorted(upstream_files | set(overlays)) + # `size` and `footprint` cover the assembled on-disk directory: the + # upstream files actually downloaded plus the overlay files shipped inside + # FastFlow. They are not the upstream repository's size, which is larger, + # and not the overlay's, which is negligible. + remote_size = sum(int(indexed[path]["size"]) for path in upstream_files) overlay_size = sum(int(record["size"]) for record in overlays.values()) size = remote_size + overlay_size footprint = round(size / (1024**3), 2) @@ -379,6 +417,104 @@ def build_catalog_entry( } +def validate_catalog_provenance( + entry: dict[str, object], + upstream_records: list[dict[str, object]], + overlay_dir: Path, +) -> None: + """Check the two provenances of the assembled package separately. + + Design `PACKAGE-1`. Upstream files must each carry a Hugging Face metadata + record at the pinned revision and are what the downloader fetches and + hash-checks. Overlay files must exist in the shipped overlay directory and, + unless they shadow a published file, must have no upstream record at all. + """ + overlay_dir = Path(overlay_dir) + indexed = _index_git_records(upstream_records) + + if entry.get("revision") != UPSTREAM_COMMIT: + raise ValueError( + f"catalog revision must be pinned to {UPSTREAM_COMMIT}" + ) + if UPSTREAM_COMMIT not in str(entry.get("file_url", "")): + raise ValueError("catalog file_url must reference the pinned revision") + if entry.get("flm_min_version") != FLM_MIN_VERSION: + raise ValueError( + f"catalog flm_min_version must be {FLM_MIN_VERSION}" + ) + if entry.get("modelscope_supported") is not False: + raise ValueError("this tag has no ModelScope publication") + if "ms_url" in entry: + raise ValueError("a tag without a ModelScope publication has no ms_url") + + overlays = _require_mapping(entry.get("bundled_overlays"), "bundled_overlays") + if set(overlays) != set(OVERLAY_FILES): + raise ValueError( + "bundled_overlays must be exactly " + f"{sorted(OVERLAY_FILES)}, got {sorted(overlays)}" + ) + + catalog_files = entry.get("files") + if not isinstance(catalog_files, list): + raise ValueError("catalog files must be a list") + if sorted(catalog_files) != list(catalog_files): + raise ValueError("catalog files must be sorted") + if len(set(catalog_files)) != len(catalog_files): + raise ValueError("catalog files contains duplicates") + + for name, record in overlays.items(): + if name not in catalog_files: + raise ValueError(f"overlay {name} is missing from catalog files") + path = overlay_dir.parent / str(record["path"]) + if not path.is_file(): + raise ValueError(f"overlay file is not installed: {path}") + actual = _sha256_record(path) + if actual != {"size": record["size"], "sha256": record["sha256"]}: + raise ValueError(f"overlay {name} does not match its catalog record") + if name in OVERLAY_FILES_WITHOUT_UPSTREAM and name in indexed: + raise ValueError( + f"overlay {name} has an upstream metadata record. FastFlow's " + "own package contract appears to have been published to " + f"{UPSTREAM_REPOSITORY}, which makes the two provenances " + "indistinguishable." + ) + + for name in catalog_files: + if name in overlays: + continue + if name not in indexed: + raise ValueError( + f"upstream file {name} has no Hugging Face metadata record" + ) + if name in EXCLUDED_UPSTREAM_FILES: + raise ValueError( + f"{name} is excluded from the package but is still listed" + ) + + for name in indexed: + if name in catalog_files or name in EXCLUDED_UPSTREAM_FILES: + continue + raise ValueError( + f"upstream file {name} is neither packaged nor explicitly excluded" + ) + + expected_size = sum( + int(indexed[name]["size"]) + for name in catalog_files + if name not in overlays + ) + sum(int(record["size"]) for record in overlays.values()) + if entry.get("size") != expected_size: + raise ValueError( + f"catalog size must be {expected_size}, got {entry.get('size')}" + ) + expected_footprint = round(expected_size / (1024**3), 2) + if entry.get("footprint") != expected_footprint: + raise ValueError( + f"catalog footprint must be {expected_footprint}, " + f"got {entry.get('footprint')}" + ) + + def updated_catalog_documents( model_list: dict[str, object], model_info: dict[str, object], @@ -521,6 +657,7 @@ def update_catalog_files( Path(model_info_path).read_text(encoding="utf-8") ) entry = build_catalog_entry(overlay_dir, git_files) + validate_catalog_provenance(entry, git_files, Path(overlay_dir)) updated_list, updated_info = updated_catalog_documents( model_list, model_info, @@ -531,6 +668,31 @@ def update_catalog_files( _write_json(Path(model_info_path), updated_info) +def validate_catalog_files( + model_list_path: Path, + model_info_path: Path, + overlay_dir: Path, +) -> None: + """Re-check the committed catalog without contacting the network. + + Regeneration is not always possible on a machine without a metadata + checkout, but the committed documents can still be held to the same + contract, which is what keeps a hand edit from slipping through. + """ + model_list = json.loads( + Path(model_list_path).read_text(encoding="utf-8") + ) + model_info = json.loads( + Path(model_info_path).read_text(encoding="utf-8") + ) + entry = model_list["models"]["phi4-mini-it-aie4"]["4b"] + validate_catalog_provenance( + entry, + model_info["phi4-mini-it-aie4:4b"], + Path(overlay_dir), + ) + + def main() -> int: parser = argparse.ArgumentParser() subparsers = parser.add_subparsers(dest="command", required=True) @@ -556,7 +718,20 @@ def main() -> int: choices=[UPSTREAM_COMMIT], ) + validate = subparsers.add_parser("validate-catalog") + validate.add_argument("--overlay-dir", type=Path, required=True) + validate.add_argument("--model-list", type=Path, required=True) + validate.add_argument("--model-info", type=Path, required=True) + args = parser.parse_args() + if args.command == "validate-catalog": + validate_catalog_files( + args.model_list, + args.model_info, + args.overlay_dir, + ) + return 0 + records = huggingface_metadata_records( args.git_dir, args.upstream_commit, diff --git a/tools/tests/test_package_phi4_corelib_aie4.py b/tools/tests/test_package_phi4_corelib_aie4.py index 1185b62d..39fbabf9 100644 --- a/tools/tests/test_package_phi4_corelib_aie4.py +++ b/tools/tests/test_package_phi4_corelib_aie4.py @@ -393,6 +393,7 @@ def test_catalog_entry_uses_exact_remote_and_overlay_sizes(self): for name, data in ( ("config.json", b"config"), ("corelib_phi4_manifest.json", b"manifest"), + ("provenance.json", b"provenance"), ("tokenizer_config.json", b"normalized-tokenizer"), ): (overlay / name).write_bytes(data) @@ -409,13 +410,29 @@ def test_catalog_entry_uses_exact_remote_and_overlay_sizes(self): "size": 200, "path": "tokenizer_config.json", }, + # Present upstream, deliberately not carried into the package. + { + "type": "file", + "oid": "3" * 40, + "size": 400, + "path": "genai_config.json", + }, + { + "type": "file", + "oid": "4" * 40, + "size": 800, + "path": ".gitattributes", + }, ] entry = package_tool.build_catalog_entry(overlay, records) + # The shadowed upstream tokenizer_config.json and the excluded + # files contribute nothing: size covers the assembled directory, + # not the upstream repository. expected_size = ( 100 + len(b"config") + len(b"manifest") + - len(b"normalized-tokenizer") + len(b"provenance") + len(b"normalized-tokenizer") ) self.assertEqual(entry["size"], expected_size) self.assertEqual( @@ -428,17 +445,16 @@ def test_catalog_entry_uses_exact_remote_and_overlay_sizes(self): "config.json", "corelib_phi4_manifest.json", "model.onnx", + "provenance.json", "tokenizer_config.json", }, ) self.assertEqual( set(entry["bundled_overlays"]), - { - "config.json", - "corelib_phi4_manifest.json", - "tokenizer_config.json", - }, + set(package_tool.OVERLAY_FILES), ) + self.assertNotIn("genai_config.json", entry["files"]) + self.assertNotIn(".gitattributes", entry["files"]) def test_metadata_refresh_preserves_unrelated_entries(self): model_list = { @@ -553,5 +569,124 @@ def test_http_metadata_is_normalized_to_git_fallback_schema(self): ) +class Phi4CatalogProvenanceTests(unittest.TestCase): + """Design `PACKAGE-1`: the two provenances are validated separately. + + A rule requiring every catalog file to carry Hugging Face metadata cannot + hold, because the overlay files do not exist upstream by construction. The + interesting failure is the opposite one: an overlay file that acquires an + upstream record means FastFlow's package contract was published to the + model repository, and the two provenances stop being distinguishable. + """ + + SOURCE = Path(__file__).resolve().parents[2] / "src" + + def _committed(self): + model_list = json.loads( + (self.SOURCE / "model_list.json").read_text(encoding="utf-8") + ) + model_info = json.loads( + (self.SOURCE / "model_info.json").read_text(encoding="utf-8") + ) + return ( + model_list["models"]["phi4-mini-it-aie4"]["4b"], + model_info["phi4-mini-it-aie4:4b"], + self.SOURCE / "model_overlays" / "phi4-mini-it-aie4", + ) + + def test_committed_catalog_satisfies_both_provenances(self): + entry, records, overlay = self._committed() + package_tool.validate_catalog_provenance(entry, records, overlay) + + def test_every_upstream_file_has_a_metadata_record(self): + entry, records, overlay = self._committed() + overlays = set(entry["bundled_overlays"]) + indexed = {record["path"] for record in records} + for name in entry["files"]: + if name in overlays: + continue + self.assertIn(name, indexed, f"{name} has no upstream record") + + def test_authored_overlays_have_no_upstream_record(self): + entry, records, overlay = self._committed() + indexed = {record["path"] for record in records} + for name in package_tool.OVERLAY_FILES_WITHOUT_UPSTREAM: + self.assertIn(name, entry["bundled_overlays"]) + self.assertNotIn(name, indexed) + self.assertTrue((overlay / name).is_file()) + + def test_shadowed_overlay_keeps_its_upstream_record(self): + # tokenizer_config.json is the one overlay that also exists upstream. + # The overlay replaces it because the published file carries neither a + # chat template nor eos_token_id, so its upstream record is expected + # and must not be mistaken for a published FastFlow contract. + entry, records, _ = self._committed() + indexed = {record["path"] for record in records} + self.assertIn("tokenizer_config.json", entry["bundled_overlays"]) + self.assertIn("tokenizer_config.json", indexed) + + def test_published_overlay_contract_is_rejected(self): + entry, records, overlay = self._committed() + for name in package_tool.OVERLAY_FILES_WITHOUT_UPSTREAM: + polluted = list(records) + [ + {"type": "file", "oid": "0" * 40, "size": 1, "path": name} + ] + with self.assertRaises(ValueError) as caught: + package_tool.validate_catalog_provenance( + entry, polluted, overlay + ) + self.assertIn("upstream metadata record", str(caught.exception)) + + def test_unaccounted_upstream_file_is_rejected(self): + entry, records, overlay = self._committed() + extra = list(records) + [ + {"type": "file", "oid": "0" * 40, "size": 7, "path": "surprise.bin"} + ] + with self.assertRaises(ValueError) as caught: + package_tool.validate_catalog_provenance(entry, extra, overlay) + self.assertIn("explicitly excluded", str(caught.exception)) + + def test_excluded_upstream_files_are_not_packaged(self): + entry, records, _ = self._committed() + indexed = {record["path"] for record in records} + for name in package_tool.EXCLUDED_UPSTREAM_FILES: + self.assertIn(name, indexed, f"{name} should exist upstream") + self.assertNotIn(name, entry["files"]) + + def test_overlay_hash_drift_is_rejected(self): + entry, records, overlay = self._committed() + drifted = json.loads(json.dumps(entry)) + drifted["bundled_overlays"]["config.json"]["sha256"] = "0" * 64 + with self.assertRaises(ValueError) as caught: + package_tool.validate_catalog_provenance(drifted, records, overlay) + self.assertIn("does not match its catalog record", str(caught.exception)) + + def test_size_and_footprint_cover_the_assembled_directory(self): + entry, records, overlay = self._committed() + indexed = {record["path"]: record for record in records} + overlays = entry["bundled_overlays"] + expected = sum( + indexed[name]["size"] + for name in entry["files"] + if name not in overlays + ) + sum(record["size"] for record in overlays.values()) + self.assertEqual(entry["size"], expected) + self.assertEqual(entry["footprint"], round(expected / (1024**3), 2)) + + def test_catalog_pins_version_and_rejects_modelscope(self): + entry, _, _ = self._committed() + self.assertEqual(entry["flm_min_version"], "1.0.4") + self.assertEqual(entry["revision"], package_tool.UPSTREAM_COMMIT) + self.assertIs(entry["modelscope_supported"], False) + self.assertNotIn("ms_url", entry) + + def test_genai_config_is_not_carried_through(self): + # MODEL-2: flm.exe runs no ORT or genai graph, and an unused config + # invites a future reader to believe it is authoritative. + entry, _, overlay = self._committed() + self.assertNotIn("genai_config.json", entry["files"]) + self.assertFalse((overlay / "genai_config.json").exists()) + + if __name__ == "__main__": unittest.main() From b05c24fcff4bc3fe84211a1509873812cfb1c3bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CChi?= Date: Tue, 1 Sep 2026 20:19:30 -0700 Subject: [PATCH 029/117] test: guard the no-import rule and the measured catalog size Step 8 checks dumpbin /DEPENDENTS on a packaged flm.exe, which only runs where an exe exists. The packaging test now also rejects any build file that links ryzenai_corelib, catching the regression at its source rather than at the end of a release build. Verified by adding such a link, which fails the test. Step 2 forbids hand-estimating the catalog numbers. The entry is computed from upstream metadata because nobody assembles a 3 GiB directory to write a catalog line, so a new test measures a stand-in assembled directory with catalog_measurements and requires the same size and footprint. If the two derivations drift, the published number stops describing the directory a user actually gets. Co-Authored-By: Claude Opus 5 (1M context) --- .../test_packaged_runtime.ps1 | 22 +++++++++++++++ tools/tests/test_package_phi4_corelib_aie4.py | 27 +++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/src/test/phi4_corelib_aie4/test_packaged_runtime.ps1 b/src/test/phi4_corelib_aie4/test_packaged_runtime.ps1 index 1faae990..b8064e0c 100644 --- a/src/test/phi4_corelib_aie4/test_packaged_runtime.ps1 +++ b/src/test/phi4_corelib_aie4/test_packaged_runtime.ps1 @@ -169,6 +169,28 @@ try { } } + # flm.exe must never gain a link-time dependency on ryzenai_corelib. The + # DLL is resolved at run time by absolute path precisely so that a binary + # without the AIE4 runtime installed still starts. dumpbin on the packaged + # exe is the end check, but it only runs where an exe exists; this catches + # the regression at its source. + foreach ($buildFile in @( + "CMakeLists.txt", + "cmake/ConfigureAie4Runtime.cmake", + "common/corelib/corelib_sources.cmake" + )) { + $path = Join-Path $sourceRoot $buildFile + if (-not (Test-Path $path)) { continue } + foreach ($line in Get-Content $path) { + if ( + $line -match "target_link_libraries" -and + $line -match "ryzenai_corelib" + ) { + throw "$buildFile links ryzenai_corelib: $line" + } + } + } + New-Item -ItemType Directory -Path $temporary | Out-Null $fixture = Join-Path $temporary "fixture" New-Item -ItemType Directory -Path $fixture | Out-Null diff --git a/tools/tests/test_package_phi4_corelib_aie4.py b/tools/tests/test_package_phi4_corelib_aie4.py index 39fbabf9..6612fe71 100644 --- a/tools/tests/test_package_phi4_corelib_aie4.py +++ b/tools/tests/test_package_phi4_corelib_aie4.py @@ -680,6 +680,33 @@ def test_catalog_pins_version_and_rejects_modelscope(self): self.assertIs(entry["modelscope_supported"], False) self.assertNotIn("ms_url", entry) + def test_catalog_size_matches_a_measured_assembled_directory(self): + # Step 2 forbids hand-estimating the catalog numbers. The entry is + # computed from metadata because nobody assembles a 3 GiB directory to + # write a catalog line, so this pins the two derivations together: if + # they ever disagree, the published number stops describing the + # directory a user actually gets. + entry, records, overlay = self._committed() + indexed = {record["path"]: record for record in records} + overlays = entry["bundled_overlays"] + with tempfile.TemporaryDirectory() as directory: + assembled = Path(directory) + logical_sizes = {} + for name in entry["files"]: + target = assembled / name + if name in overlays: + target.write_bytes((overlay / name).read_bytes()) + else: + # Stand in for the payload; the logical size comes from the + # upstream record, exactly as it does for an LFS pointer. + target.write_bytes(b"") + logical_sizes[name] = indexed[name]["size"] + size, footprint = package_tool.catalog_measurements( + assembled, logical_sizes + ) + self.assertEqual(size, entry["size"]) + self.assertEqual(footprint, entry["footprint"]) + def test_genai_config_is_not_carried_through(self): # MODEL-2: flm.exe runs no ORT or genai graph, and an unused config # invites a future reader to believe it is authoritative. From 4c9a9baae71b2f4f6fcc42dc129cc8b0de8e3e42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CChi?= Date: Tue, 1 Sep 2026 20:52:50 -0700 Subject: [PATCH 030/117] fix: make the AIE4 packaging guards actually fire Review round 1. Four Important findings shared one theme: guards that do not fire, and a script reporting success for work it never did. I1. The no-corelib-import guard matched line by line, so it missed the multi-line target_link_libraries() form -- the only form the AIE4 target uses. With Step 8's dumpbin unrunnable here, that guard is the sole cover for a binding constraint, and it was inert against the exact regression it names. It now parses balanced-paren command bodies across every CMake file in the tree (48 link calls scanned) and fails if the scan matches nothing. Verified by linking ryzenai_corelib on its own line: previously passed, now fails. I2. A multi-entry FLM_AIE4_DEPENDENCY_DIRS was interpolated unescaped into the POST_BUILD command, so "a;b" became two argv entries and everything after the first directory was silently dropped -- then staging failed telling the developer to add a directory they had already added. Reproduced, fixed by escaping, and re-verified with the needed DLLs reachable only via the second entry. The ctest registration passes the list through, so CI exercises it. I3. test_packaged_runtime.ps1 was not registered anywhere. Every guard in it ran only when someone remembered the command. It is now a ctest test with SKIP_RETURN_CODE 77 and inherits the configured runtime directories. I4. The script printed a success line when the real-closure and flm.exe blocks were skipped. That is the defect Task 10R had to fix in test_real_corelib. It now enumerates RAN and SKIPPED blocks and exits 77 when anything was skipped, so a partial run shows as Skipped rather than Passed. M-esc-1. aie4-closure.txt shipped absolute build-machine paths into bin/aie4. The shipped report now lists names and SHA-256 only, which is what a recipient can act on; the path-bearing audit record stays in the build tree. The test rejects any absolute path in the shipped report and verifies every recorded hash against the staged file. M-esc-2. The docs called all four overlay files authored. Three are authored with no upstream counterpart; tokenizer_config.json shadows a published file and legitimately has an upstream record. Blurring that is what caused the bug. Also, per the coordinator's note: AutoModel overwrites the tokenizer_config chat_template with a standalone chat_template.jinja when present, so the inlined copy is dead unless byte-identical. They are identical today (423 bytes, sha256 febf5892...) but nothing asserted it. Generation and offline validation now require the equality, and overlay generation reads the template as bytes so newline translation cannot break it. Co-Authored-By: Claude Opus 5 (1M context) --- docs/docs/models/phi.md | 25 ++- src/cmake/ConfigureAie4Runtime.cmake | 20 ++- src/cmake/StageAie4Runtime.cmake | 41 +++-- src/test/phi4_corelib_aie4/CMakeLists.txt | 39 +++++ .../test_packaged_runtime.ps1 | 153 ++++++++++++++++-- tools/package_phi4_corelib_aie4.py | 58 ++++++- tools/tests/test_package_phi4_corelib_aie4.py | 38 +++++ 7 files changed, 336 insertions(+), 38 deletions(-) diff --git a/docs/docs/models/phi.md b/docs/docs/models/phi.md index 0c74285a..de585398 100644 --- a/docs/docs/models/phi.md +++ b/docs/docs/models/phi.md @@ -44,10 +44,21 @@ before any download starts. The assembled model directory has two provenances. The weights, tokenizer and vocabulary are downloaded from the pinned upstream revision and hash-checked -against its published metadata. Four files -- `config.json`, -`tokenizer_config.json`, `corelib_phi4_manifest.json` and `provenance.json` -- -are authored by FastFlowLM and installed with the product, because the upstream -repository ships no `config.json` and its tokenizer configuration carries -neither a chat template nor the EOS token IDs this backend needs. Those files -restate the model contract for FastFlowLM's existing readers; they never -override it, and a model whose weights disagree with them fails to load. +against its published metadata. Four further files are shipped inside +FastFlowLM rather than fetched, and they are not all the same kind of file: + +- **Authored, with no upstream counterpart.** `config.json`, + `corelib_phi4_manifest.json` and `provenance.json` do not exist in the + upstream repository at all. The upstream package ships no `config.json`, and + the other two describe FastFlowLM's own packaging. +- **Shadowing an upstream file.** `tokenizer_config.json` *does* exist + upstream, and the shipped copy replaces it, because the published one carries + neither a chat template nor the EOS token IDs this backend requires. + +The distinction is not cosmetic: an upstream metadata record is expected for +the shadowing file and is a provenance error for the authored ones, since it +would mean FastFlowLM's own package contract had been published to the model +repository. + +These files restate the model contract for FastFlowLM's existing readers; they +never override it, and a model whose weights disagree with them fails to load. diff --git a/src/cmake/ConfigureAie4Runtime.cmake b/src/cmake/ConfigureAie4Runtime.cmake index a4f1afc0..797f65dd 100644 --- a/src/cmake/ConfigureAie4Runtime.cmake +++ b/src/cmake/ConfigureAie4Runtime.cmake @@ -59,14 +59,22 @@ function(flm_aie4_warn_if_unstageable) endif() endfunction() -function(flm_aie4_stage_command output stage_dir report) +function(flm_aie4_stage_command output stage_dir report audit) + # FLM_AIE4_DEPENDENCY_DIRS is a CMake list. Interpolating it into a + # command argument unescaped turns each `;` into an argument separator, so + # `-DFLM_AIE4_EXTRA_DIRS=a;b` reaches cmake as two argv entries: the + # variable silently loses everything after the first directory, and the + # staging then fails telling the developer to add a directory they already + # added. Escaping keeps the whole list in one argument. + string(REPLACE ";" "\\;" _extra_dirs "${FLM_AIE4_DEPENDENCY_DIRS}") set(${output} "${CMAKE_COMMAND}" "-DFLM_AIE4_CORELIB_DIR=${RYZENAI_CORELIB_RUNTIME_DIR}" "-DFLM_AIE4_XRT_DIR=${XRT_RUNTIME_DIR}" - "-DFLM_AIE4_EXTRA_DIRS=${FLM_AIE4_DEPENDENCY_DIRS}" + "-DFLM_AIE4_EXTRA_DIRS=${_extra_dirs}" "-DFLM_AIE4_DESTINATION=${stage_dir}" "-DFLM_AIE4_REPORT=${report}" + "-DFLM_AIE4_AUDIT=${audit}" -P "${FLM_AIE4_STAGE_SCRIPT}" PARENT_SCOPE) endfunction() @@ -74,6 +82,10 @@ endfunction() # Stages the derived closure beside a just-built binary so a developer can run # the AIE4 path without an install. Skipped, with no error, when the runtime # directory is not configured. +# +# The staged directory is what the installer scripts copy, so it gets the +# shippable report. The audit record, which names build-machine absolute +# paths, stays in the build tree. function(flm_aie4_stage_for_target target) if(NOT FLM_ENABLE_CORELIB_AIE4 OR NOT WIN32) return() @@ -86,7 +98,8 @@ function(flm_aie4_stage_for_target target) string(MD5 _stage_id "${_stage_dir}") flm_aie4_stage_command(_command "${_stage_dir}" - "${CMAKE_BINARY_DIR}/aie4-closure-${_stage_id}.txt") + "${_stage_dir}/aie4-closure.txt" + "${CMAKE_BINARY_DIR}/aie4-closure-audit-${_stage_id}.txt") add_custom_command(TARGET ${target} POST_BUILD COMMAND ${_command} COMMENT "Deriving the Phi-4 AIE4 runtime closure") @@ -116,6 +129,7 @@ set(FLM_AIE4_EXTRA_DIRS [[${FLM_AIE4_DEPENDENCY_DIRS}]]) set(FLM_AIE4_DESTINATION [[${_arg_DESTINATION}]]) set(FLM_AIE4_REPORT \"\${CMAKE_INSTALL_PREFIX}/${_arg_DESTINATION}/aie4-closure.txt\") +set(FLM_AIE4_AUDIT [[${CMAKE_BINARY_DIR}/aie4-closure-audit-install.txt]]) " ${_component_args}) install(SCRIPT "${FLM_AIE4_STAGE_SCRIPT}" ${_component_args}) diff --git a/src/cmake/StageAie4Runtime.cmake b/src/cmake/StageAie4Runtime.cmake index 15440d8f..790a57f7 100644 --- a/src/cmake/StageAie4Runtime.cmake +++ b/src/cmake/StageAie4Runtime.cmake @@ -19,7 +19,11 @@ # FLM_AIE4_EXTRA_DIRS additional dependency search directories # FLM_AIE4_DESTINATION directory to stage into; relative paths resolve # against CMAKE_INSTALL_PREFIX -# FLM_AIE4_REPORT optional path for the derived closure report +# FLM_AIE4_REPORT optional path for the SHIPPABLE closure report: +# file names and hashes only, no build-machine paths +# FLM_AIE4_AUDIT optional path for the build-side audit record, which +# does name absolute source paths and must stay in the +# build tree cmake_minimum_required(VERSION 3.24) @@ -159,19 +163,38 @@ foreach(_flm_aie4_file IN LISTS _flm_aie4_files) FOLLOW_SYMLINK_CHAIN) endforeach() -# The report records both the derived closure and the directories it was -# derived from. The source path of every staged DLL is the audit trail: it is -# what shows, after the fact, that a shipped dependency came from the intended -# package rather than from whatever the build machine happened to have. +# Two records, deliberately different. +# +# The shippable report lists the derived closure by name and SHA-256 only. It +# is written into the staged directory, which the installer scripts copy +# verbatim, so it must not carry absolute paths from the machine that built it: +# a customer artifact naming a developer's conda prefix leaks build-machine +# layout for no benefit to the reader. The hashes are what a recipient can +# actually act on, since they verify the staged bits. if(FLM_AIE4_REPORT) - set(_flm_aie4_report_text "root\t${_flm_aie4_root}\n") + set(_flm_aie4_report_text "") + foreach(_flm_aie4_file IN LISTS _flm_aie4_files) + get_filename_component(_flm_aie4_leaf "${_flm_aie4_file}" NAME) + file(SHA256 "${_flm_aie4_file}" _flm_aie4_hash) + string(APPEND _flm_aie4_report_text + "staged\t${_flm_aie4_leaf}\t${_flm_aie4_hash}\n") + endforeach() + file(WRITE "${FLM_AIE4_REPORT}" "${_flm_aie4_report_text}") +endif() + +# The audit record stays in the build tree and does name the source path of +# every staged DLL. That path is what shows, after the fact, that a shipped +# dependency came from the intended package rather than from whatever the +# build machine happened to have. +if(FLM_AIE4_AUDIT) + set(_flm_aie4_audit_text "root\t${_flm_aie4_root}\n") foreach(_flm_aie4_dir IN LISTS _flm_aie4_search_dirs) - string(APPEND _flm_aie4_report_text "search\t${_flm_aie4_dir}\n") + string(APPEND _flm_aie4_audit_text "search\t${_flm_aie4_dir}\n") endforeach() foreach(_flm_aie4_file IN LISTS _flm_aie4_files) get_filename_component(_flm_aie4_leaf "${_flm_aie4_file}" NAME) - string(APPEND _flm_aie4_report_text + string(APPEND _flm_aie4_audit_text "staged\t${_flm_aie4_leaf}\t${_flm_aie4_file}\n") endforeach() - file(WRITE "${FLM_AIE4_REPORT}" "${_flm_aie4_report_text}") + file(WRITE "${FLM_AIE4_AUDIT}" "${_flm_aie4_audit_text}") endif() diff --git a/src/test/phi4_corelib_aie4/CMakeLists.txt b/src/test/phi4_corelib_aie4/CMakeLists.txt index cec1eec4..f9c7fb4a 100644 --- a/src/test/phi4_corelib_aie4/CMakeLists.txt +++ b/src/test/phi4_corelib_aie4/CMakeLists.txt @@ -144,6 +144,45 @@ if(RYZENAI_CORELIB_RUNTIME_DIR) "PATH=path_list_prepend:${XRT_LIB_DIR};PATH=path_list_prepend:${RYZENAI_CORELIB_RUNTIME_DIR}") endif() +# The packaging and clean-environment checks run by default rather than only +# when someone remembers the command. Every regression guard in that script -- +# the derived closure, the CLOSURE-2 negative control, the no-corelib-import +# rule, and the requirement that the ordinary NPU2 installer still builds +# without an AIE4 closure -- is invisible if it is never invoked. +# +# It reports SKIPPED via SKIP_RETURN_CODE when its optional inputs are absent, +# so a partial run never reads as full coverage. +find_program(FLM_POWERSHELL_EXECUTABLE + NAMES pwsh powershell + DOC "PowerShell used to run the packaging tests") +if(FLM_POWERSHELL_EXECUTABLE) + set(_packaged_runtime_args "") + if(RYZENAI_CORELIB_RUNTIME_DIR) + list(APPEND _packaged_runtime_args + -CorelibRuntimeDir "${RYZENAI_CORELIB_RUNTIME_DIR}") + endif() + if(RYZENAI_CORELIB_EXTRA_DLL_DIRS) + list(JOIN RYZENAI_CORELIB_EXTRA_DLL_DIRS ";" _packaged_runtime_deps) + list(APPEND _packaged_runtime_args + -DependencyDir "${_packaged_runtime_deps}") + endif() + add_test(NAME test_packaged_runtime + COMMAND "${FLM_POWERSHELL_EXECUTABLE}" + -NoProfile + -ExecutionPolicy Bypass + -File + "${CMAKE_CURRENT_LIST_DIR}/test_packaged_runtime.ps1" + ${_packaged_runtime_args}) + set_tests_properties(test_packaged_runtime PROPERTIES + SKIP_RETURN_CODE 77 + TIMEOUT 1800) +else() + message(WARNING + "PowerShell was not found, so test_packaged_runtime is not " + "registered. The AIE4 packaging and clean-environment guards will " + "not run in this build.") +endif() + add_corelib_host_test(test_corelib_api test_corelib_api.cpp) add_corelib_host_test(test_phi4_manifest test_phi4_manifest.cpp) add_corelib_host_test(test_phi4_shape_plan test_phi4_shape_plan.cpp) diff --git a/src/test/phi4_corelib_aie4/test_packaged_runtime.ps1 b/src/test/phi4_corelib_aie4/test_packaged_runtime.ps1 index b8064e0c..53ea7070 100644 --- a/src/test/phi4_corelib_aie4/test_packaged_runtime.ps1 +++ b/src/test/phi4_corelib_aie4/test_packaged_runtime.ps1 @@ -17,6 +17,16 @@ param( ) $ErrorActionPreference = "Stop" + +# Skipped work is reported as skipped, never as passed. A script that prints a +# success line for blocks it never entered reads as coverage it does not have, +# which is the defect test_real_corelib had to fix in Task 10R. Exit code 77 is +# CTest's SKIP_RETURN_CODE, so a partial run shows as Skipped rather than +# Passed. +$ran = @() +$skipped = @() +$SKIP_EXIT = 77 + $sourceRoot = (Resolve-Path (Join-Path $PSScriptRoot "../..")).Path $modulePath = Join-Path $sourceRoot "cmake/ConfigureAie4Runtime.cmake" $stageScript = Join-Path $sourceRoot "cmake/StageAie4Runtime.cmake" @@ -35,6 +45,27 @@ public static class FlmAie4Loader { } "@ +# Get-FileHash is not reliably resolvable in the constrained -NoProfile host +# CTest launches, so hash through .NET directly rather than depending on module +# autoloading. +function Get-Sha256Hex { + param([string]$Path) + $algorithm = [System.Security.Cryptography.SHA256]::Create() + try { + $stream = [System.IO.File]::OpenRead($Path) + try { + $bytes = $algorithm.ComputeHash($stream) + } finally { + $stream.Dispose() + } + } finally { + $algorithm.Dispose() + } + return ( + -join ($bytes | ForEach-Object { $_.ToString("x2") }) + ) +} + function Invoke-Configure { param( [string]$Build, @@ -158,6 +189,7 @@ try { } } [xml]$parsedWix = $wix + $ran += "installer-manifests" # The main installer must still build with no AIE4 closure present. # Hard-failing here would make the AIE4 feature a precondition of shipping @@ -168,28 +200,63 @@ try { throw "$script still fails the non-AIE4 package build" } } + $ran += "optional-feature-packaging" # flm.exe must never gain a link-time dependency on ryzenai_corelib. The # DLL is resolved at run time by absolute path precisely so that a binary # without the AIE4 runtime installed still starts. dumpbin on the packaged # exe is the end check, but it only runs where an exe exists; this catches # the regression at its source. - foreach ($buildFile in @( - "CMakeLists.txt", - "cmake/ConfigureAie4Runtime.cmake", - "common/corelib/corelib_sources.cmake" - )) { - $path = Join-Path $sourceRoot $buildFile - if (-not (Test-Path $path)) { continue } - foreach ($line in Get-Content $path) { - if ( - $line -match "target_link_libraries" -and - $line -match "ryzenai_corelib" - ) { - throw "$buildFile links ryzenai_corelib: $line" + # + # The scan reads whole balanced-paren command invocations, not lines. Every + # link call in this tree spans multiple lines -- the AIE4 target's own does + # -- so a line-at-a-time match would miss the exact form the regression + # would take and report success for a check it never performed. + $cmakeFiles = @( + Get-ChildItem -Path $sourceRoot -Recurse -File -Include @( + "CMakeLists.txt", "*.cmake" + ) | Where-Object { + $_.FullName -notmatch "[\\/](build|out|third_party)[^\\/]*[\\/]" + } + ) + if ($cmakeFiles.Count -lt 3) { + throw "The link-guard scan found only $($cmakeFiles.Count) CMake files" + } + $scannedLinkCalls = 0 + foreach ($file in $cmakeFiles) { + $text = Get-Content $file.FullName -Raw + if ($null -eq $text) { continue } + # Strip line comments so a commented-out example cannot trip the guard + # and, more importantly, cannot hide a real call behind an unbalanced + # parenthesis inside a comment. + $text = [regex]::Replace($text, '(?m)#.*$', '') + foreach ($match in [regex]::Matches( + $text, '(?i)\b(target_link_libraries|link_libraries)\s*\(')) { + $depth = 1 + $index = $match.Index + $match.Length + while ($index -lt $text.Length -and $depth -gt 0) { + if ($text[$index] -eq '(') { $depth++ } + elseif ($text[$index] -eq ')') { $depth-- } + $index++ + } + if ($depth -ne 0) { + throw "Unbalanced $($match.Value) in $($file.FullName)" + } + $scannedLinkCalls++ + $body = $text.Substring( + $match.Index, $index - $match.Index) + if ($body -match '(?i)\bryzenai_corelib\b') { + $relative = $file.FullName.Substring($sourceRoot.Length + 1) + throw ( + "$relative links ryzenai_corelib: " + + ($body -replace '\s+', ' ')) } } } + if ($scannedLinkCalls -lt 1) { + throw "The link-guard scan matched no link calls; it is inert" + } + $ran += "no-corelib-import-guard ($scannedLinkCalls link calls)" New-Item -ItemType Directory -Path $temporary | Out-Null $fixture = Join-Path $temporary "fixture" @@ -259,9 +326,19 @@ flm_aie4_install_runtime(DESTINATION bin/aie4 COMPONENT AIE4) if ($emptyInstall -notmatch "no ryzenai_corelib\.dll") { throw "Empty runtime directory was not reported.`n$emptyInstall" } + $ran += "packaging-configure-and-install-contract" if ($CorelibRuntimeDir) { $resolvedCorelib = (Resolve-Path $CorelibRuntimeDir).Path + # -DependencyDir is a semicolon-separated list, matching the CMake + # cache variable it feeds. Resolving each entry separately is also what + # exercises the multi-entry path end to end: a single interpolated + # string silently loses everything after the first `;`. + $resolvedDependencies = @( + $DependencyDir -split ';' | + Where-Object { $_ } | + ForEach-Object { (Resolve-Path $_).Path } + ) $realBuild = Join-Path $temporary "real" Invoke-Configure ` -Build $realBuild ` @@ -269,8 +346,7 @@ flm_aie4_install_runtime(DESTINATION bin/aie4 COMPONENT AIE4) -Corelib $resolvedCorelib ` -Xrt $(if ($XrtRuntimeDir) { (Resolve-Path $XrtRuntimeDir).Path } else { "" }) ` - -Dependency $(if ($DependencyDir) { - (Resolve-Path $DependencyDir).Path } else { "" }) | Out-Null + -Dependency ($resolvedDependencies -join ';') | Out-Null $realStage = Join-Path $temporary "real-stage" Invoke-Install -Build $realBuild -Prefix $realStage | Out-Null $stagedDir = Join-Path $realStage "bin/aie4" @@ -302,6 +378,24 @@ flm_aie4_install_runtime(DESTINATION bin/aie4 COMPONENT AIE4) if ($derived -contains "msvcp140.dll") { throw "The closure staged a build-machine Visual C++ runtime" } + # The report ships inside bin/aie4, so it must not carry absolute + # paths from the machine that built it. The audit record that does + # name them stays in the build tree. + foreach ($line in Get-Content $report) { + if ($line -match '(?i)[a-z]:[\\/]') { + throw "The shipped closure report leaks a build path: $line" + } + } + foreach ($line in Get-Content $report) { + $fields = $line -split "`t" + if ($fields.Count -ne 3 -or $fields[2] -notmatch '^[0-9a-f]{64}$') { + throw "The shipped closure report lacks a SHA-256: $line" + } + $actual = Get-Sha256Hex (Join-Path $stagedDir $fields[1]) + if ($actual -ne $fields[2]) { + throw "Staged $($fields[1]) does not match its recorded hash" + } + } # CLOSURE-2, positive control. $code = Invoke-CleanEnvironmentLoad -Dir $stagedDir @@ -338,6 +432,11 @@ flm_aie4_install_runtime(DESTINATION bin/aie4 COMPONENT AIE4) if ($proved -lt 1) { throw "No staged dependency was proven load-bearing" } + $ran += "real-closure (CLOSURE-1/2, $proved load-bearing DLLs proven)" + } else { + $skipped += ( + "real-closure (CLOSURE-1/2): pass -CorelibRuntimeDir, and " + + "-DependencyDir where the corelib's own dependencies live") } if ($FlmExe) { @@ -398,11 +497,33 @@ flm_aie4_install_runtime(DESTINATION bin/aie4 COMPONENT AIE4) if ($LASTEXITCODE -ne 0) { throw "Non-AIE4 command failed without the AIE4 directory" } + $ran += "flm-exe (Step 7 clean environment, Step 8 dumpbin)" + } else { + $skipped += ( + "flm-exe (Step 7 clean environment, Step 8 dumpbin): " + + "pass -FlmExe, and -RunAie4ModelLoad on AIE4 hardware") } - Write-Output "packaged runtime tests passed" + foreach ($item in $ran) { + Write-Output "RAN : $item" + } + foreach ($item in $skipped) { + Write-Output "SKIPPED : $item" + } + if ($skipped.Count -gt 0) { + Write-Output ( + "packaged runtime tests INCOMPLETE: " + + "$($ran.Count) block(s) ran, $($skipped.Count) skipped") + $script:exitCode = $SKIP_EXIT + } else { + Write-Output ( + "packaged runtime tests passed: $($ran.Count) block(s), " + + "none skipped") + $script:exitCode = 0 + } } finally { if (Test-Path $temporary) { Remove-Item $temporary -Recurse -Force -ErrorAction SilentlyContinue } } +exit $script:exitCode diff --git a/tools/package_phi4_corelib_aie4.py b/tools/package_phi4_corelib_aie4.py index 244f2379..9aaecdc1 100644 --- a/tools/package_phi4_corelib_aie4.py +++ b/tools/package_phi4_corelib_aie4.py @@ -325,9 +325,12 @@ def generate_overlay( tokenizer_config = json.loads( (upstream_dir / "tokenizer_config.json").read_text(encoding="utf-8") ) - chat_template = (upstream_dir / "chat_template.jinja").read_text( - encoding="utf-8" - ) + # Read bytes and decode explicitly. `read_text` applies universal newline + # translation, which would turn a CRLF template into an LF string and break + # the byte-equality that `_require_inlined_template_matches_upstream` + # depends on. + chat_template_bytes = (upstream_dir / "chat_template.jinja").read_bytes() + chat_template = chat_template_bytes.decode("utf-8") _write_json( overlay_dir / "config.json", @@ -361,6 +364,7 @@ def generate_overlay( git_files, ) _write_json(overlay_dir / "provenance.json", provenance) + _require_inlined_template_matches_upstream(overlay_dir) return provenance @@ -417,6 +421,52 @@ def build_catalog_entry( } +def _require_inlined_template_matches_upstream(overlay_dir: Path) -> None: + """Require the inlined chat template to equal the upstream jinja file. + + `AutoModel::setup_tokenizer` prefers a standalone `chat_template.jinja` + over the `chat_template` key in `tokenizer_config.json`: when the file is + present it *overwrites* the key. Both are in the package, so the upstream + file wins at run time and the overlay's inlined copy is dead code unless + the two are byte-identical. + + That makes any drift silent and one-directional -- the overlay would look + edited while the model kept using the upstream template -- so the equality + is asserted rather than assumed. The check is offline: `provenance.json` + records the upstream file's SHA-256, so no download is needed. + """ + overlay_dir = Path(overlay_dir) + tokenizer_config = json.loads( + (overlay_dir / "tokenizer_config.json").read_text(encoding="utf-8") + ) + provenance = json.loads( + (overlay_dir / "provenance.json").read_text(encoding="utf-8") + ) + inlined = tokenizer_config.get("chat_template") + if not isinstance(inlined, str) or not inlined: + raise ValueError( + "overlay tokenizer_config.json has no string chat_template" + ) + record = _require_mapping( + _require_mapping( + _require_mapping(provenance.get("upstream"), "upstream").get( + "inputs" + ), + "upstream.inputs", + ).get("chat_template.jinja"), + "upstream.inputs['chat_template.jinja']", + ) + encoded = inlined.encode("utf-8") + actual = hashlib.sha256(encoded).hexdigest() + if actual != record.get("sha256") or len(encoded) != record.get("size"): + raise ValueError( + "the overlay's inlined chat_template does not match the upstream " + "chat_template.jinja it was generated from. AutoModel prefers the " + "standalone .jinja file, so the inlined copy would be silently " + "ignored: regenerate the overlay instead of editing it." + ) + + def validate_catalog_provenance( entry: dict[str, object], upstream_records: list[dict[str, object]], @@ -447,6 +497,8 @@ def validate_catalog_provenance( if "ms_url" in entry: raise ValueError("a tag without a ModelScope publication has no ms_url") + _require_inlined_template_matches_upstream(overlay_dir) + overlays = _require_mapping(entry.get("bundled_overlays"), "bundled_overlays") if set(overlays) != set(OVERLAY_FILES): raise ValueError( diff --git a/tools/tests/test_package_phi4_corelib_aie4.py b/tools/tests/test_package_phi4_corelib_aie4.py index 6612fe71..47f3a2ee 100644 --- a/tools/tests/test_package_phi4_corelib_aie4.py +++ b/tools/tests/test_package_phi4_corelib_aie4.py @@ -707,6 +707,44 @@ def test_catalog_size_matches_a_measured_assembled_directory(self): self.assertEqual(size, entry["size"]) self.assertEqual(footprint, entry["footprint"]) + def test_inlined_chat_template_matches_upstream_jinja(self): + # AutoModel prefers the standalone chat_template.jinja and overwrites + # the tokenizer_config key with it, so the inlined copy is dead unless + # the two are byte-identical. Drift would be silent. + _, _, overlay = self._committed() + package_tool._require_inlined_template_matches_upstream(overlay) + + tokenizer_config = json.loads( + (overlay / "tokenizer_config.json").read_text(encoding="utf-8") + ) + provenance = json.loads( + (overlay / "provenance.json").read_text(encoding="utf-8") + ) + record = provenance["upstream"]["inputs"]["chat_template.jinja"] + encoded = tokenizer_config["chat_template"].encode("utf-8") + self.assertEqual(len(encoded), record["size"]) + self.assertEqual(hashlib.sha256(encoded).hexdigest(), record["sha256"]) + + def test_drifted_inlined_chat_template_is_rejected(self): + _, _, overlay = self._committed() + with tempfile.TemporaryDirectory() as directory: + drifted = Path(directory) / "phi4-mini-it-aie4" + drifted.mkdir() + for name in package_tool.OVERLAY_FILES: + (drifted / name).write_bytes((overlay / name).read_bytes()) + tokenizer_config = json.loads( + (drifted / "tokenizer_config.json").read_text(encoding="utf-8") + ) + # A single trailing newline is enough: it is the kind of edit that + # looks harmless and would be silently discarded at run time. + tokenizer_config["chat_template"] += "\n" + (drifted / "tokenizer_config.json").write_text( + json.dumps(tokenizer_config), encoding="utf-8" + ) + with self.assertRaises(ValueError) as caught: + package_tool._require_inlined_template_matches_upstream(drifted) + self.assertIn("silently ignored", str(caught.exception)) + def test_genai_config_is_not_carried_through(self): # MODEL-2: flm.exe runs no ORT or genai graph, and an unused config # invites a future reader to believe it is authoritative. From d069aca12b3247c06e6e0c30383c79e689def53c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CChi?= Date: Tue, 1 Sep 2026 21:10:10 -0700 Subject: [PATCH 031/117] fix: stop the ctest wiring from re-creating the I2 argv defect The ctest registration added in round 1 to fix I3 reintroduced the exact defect round 1 had just fixed, one function away from the correct idiom. list(JOIN ... ";" out) followed by list(APPEND args "${out}") does not produce one argument. The join builds a string that append immediately re-splits, so a two-directory value yielded five argv entries instead of four. Measured: BROKEN length=5 [-CorelibRuntimeDir;C:/core;-DependencyDir;C:/one;C:/two] FIXED length=4 [-CorelibRuntimeDir;C:/core;-DependencyDir;C:/one\;C:/two] In PowerShell the consequence is worse than the lost directory. The script's positional order is FlmExe=pos0, CorelibRuntimeDir=pos1, XrtRuntimeDir=pos2, DependencyDir=pos3, so the orphaned second directory binds to $FlmExe. That both drops a dependency directory and drags the run into the flm.exe/dumpbin block with a directory standing in for the executable, converting an honest SKIPPED into a spurious run: the I4 failure mode through a different door. A single-entry cache masks it entirely, which is why the round 1 transcript looked clean. The argument vector is now built by one function used by both the real registration and a configure-time self-check that feeds it a synthetic two-entry value and asserts four arguments with both directories intact. The self-check does not depend on how a given machine's cache is populated, so this cannot regress a third time. Also, dumpbin's exit code was ignored. A dumpbin that runs but errors produces output that simply fails to match the import pattern, so the Step 8 guard passed while checking nothing; on a non-PE input dumpbin exits 1107 and the old check reported no match. The exit code is now checked, the dependency listing must actually be present, and -FlmExe must name a file rather than a directory. Co-Authored-By: Claude Opus 5 (1M context) --- src/test/phi4_corelib_aie4/CMakeLists.txt | 56 ++++++++++++++++--- .../test_packaged_runtime.ps1 | 22 +++++++- 2 files changed, 68 insertions(+), 10 deletions(-) diff --git a/src/test/phi4_corelib_aie4/CMakeLists.txt b/src/test/phi4_corelib_aie4/CMakeLists.txt index f9c7fb4a..de4ed1be 100644 --- a/src/test/phi4_corelib_aie4/CMakeLists.txt +++ b/src/test/phi4_corelib_aie4/CMakeLists.txt @@ -155,17 +155,55 @@ endif() find_program(FLM_POWERSHELL_EXECUTABLE NAMES pwsh powershell DOC "PowerShell used to run the packaging tests") -if(FLM_POWERSHELL_EXECUTABLE) - set(_packaged_runtime_args "") - if(RYZENAI_CORELIB_RUNTIME_DIR) - list(APPEND _packaged_runtime_args - -CorelibRuntimeDir "${RYZENAI_CORELIB_RUNTIME_DIR}") + +# Builds the argument vector for test_packaged_runtime.ps1. +# +# `-DependencyDir` takes one argument holding a semicolon-separated list, so +# the separator must be escaped before it is appended. `list(JOIN ... ";" ...)` +# followed by `list(APPEND)` does NOT achieve that: the join produces a string +# that append immediately re-splits, yielding one argv entry per directory. +# +# The consequence in PowerShell is worse than a lost directory. `-DependencyDir` +# binds only the first entry, and each orphan then binds POSITIONALLY -- the +# next positional parameter is `$FlmExe` -- so a second dependency directory +# silently becomes the path to the executable under test, turning an honest +# SKIPPED into a spurious flm.exe run against a directory. +function(flm_packaged_runtime_args output corelib_dir dependency_dirs) + set(_args "") + if(corelib_dir) + list(APPEND _args -CorelibRuntimeDir "${corelib_dir}") endif() - if(RYZENAI_CORELIB_EXTRA_DLL_DIRS) - list(JOIN RYZENAI_CORELIB_EXTRA_DLL_DIRS ";" _packaged_runtime_deps) - list(APPEND _packaged_runtime_args - -DependencyDir "${_packaged_runtime_deps}") + if(dependency_dirs) + string(REPLACE ";" "\\;" _escaped "${dependency_dirs}") + list(APPEND _args -DependencyDir "${_escaped}") endif() + set(${output} "${_args}" PARENT_SCOPE) +endfunction() + +# Self-check on the escaping, run at configure time against a synthetic +# multi-entry value so it does not depend on how this machine's cache happens +# to be populated. A single-entry cache masks the defect completely, which is +# how it survived the previous round. +flm_packaged_runtime_args(_flm_argv_probe "C:/core" "C:/one;C:/two") +list(LENGTH _flm_argv_probe _flm_argv_probe_length) +if(NOT _flm_argv_probe_length EQUAL 4) + message(FATAL_ERROR + "test_packaged_runtime argument construction flattened a multi-entry " + "dependency list into ${_flm_argv_probe_length} arguments instead of " + "4. The extra entries bind positionally in PowerShell, so -FlmExe " + "would receive a directory: [${_flm_argv_probe}]") +endif() +list(GET _flm_argv_probe 3 _flm_argv_probe_deps) +if(NOT _flm_argv_probe_deps STREQUAL "C:/one;C:/two") + message(FATAL_ERROR + "test_packaged_runtime dropped a dependency directory: " + "[${_flm_argv_probe_deps}]") +endif() + +if(FLM_POWERSHELL_EXECUTABLE) + flm_packaged_runtime_args(_packaged_runtime_args + "${RYZENAI_CORELIB_RUNTIME_DIR}" + "${RYZENAI_CORELIB_EXTRA_DLL_DIRS}") add_test(NAME test_packaged_runtime COMMAND "${FLM_POWERSHELL_EXECUTABLE}" -NoProfile diff --git a/src/test/phi4_corelib_aie4/test_packaged_runtime.ps1 b/src/test/phi4_corelib_aie4/test_packaged_runtime.ps1 index 53ea7070..766a7a64 100644 --- a/src/test/phi4_corelib_aie4/test_packaged_runtime.ps1 +++ b/src/test/phi4_corelib_aie4/test_packaged_runtime.ps1 @@ -441,7 +441,27 @@ flm_aie4_install_runtime(DESTINATION bin/aie4 COMPONENT AIE4) if ($FlmExe) { $resolvedFlm = (Resolve-Path $FlmExe).Path - $imports = (& dumpbin /nologo /dependents $resolvedFlm | Out-String) + # -FlmExe is the first positional parameter, so a stray argument from a + # mis-quoted argument vector lands here. Refusing a non-file keeps that + # mistake from entering this block and reporting a run it never made. + if (-not (Test-Path -LiteralPath $resolvedFlm -PathType Leaf)) { + throw "-FlmExe is not a file: $resolvedFlm" + } + + # dumpbin's exit code is load-bearing. If it runs but errors -- wrong + # architecture, missing tool, unreadable image -- its output simply + # fails to match the import pattern, and the Step 8 guard passes + # vacuously while checking nothing. + $imports = (& dumpbin /nologo /dependents $resolvedFlm 2>&1 | + Out-String) + if ($LASTEXITCODE -ne 0) { + throw "dumpbin failed with exit code ${LASTEXITCODE}.`n$imports" + } + if ($imports -notmatch "(?i)Image has the following dependencies") { + throw ( + "dumpbin produced no dependency listing, so the no-import " + + "check would pass vacuously.`n$imports") + } if ($imports -match "(?im)^\s*ryzenai_corelib\.dll\s*$") { throw "flm.exe has an unexpected ryzenai_corelib import" } From b993a04df90931be398df77734c68f1cfc0aab57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CChi?= Date: Tue, 1 Sep 2026 21:43:34 -0700 Subject: [PATCH 032/117] test: validate Phi-4 AIE4 end to end Adds the AIE4 hardware acceptance suite: a device smoke that asserts a real device context and pushes the RoPE gather through one real tensor_write at a guard page, an end-to-end harness that emits explicit-token checkpoints for both forced continuation routes, a comparator against the corelib reference driver, and a fatal child that really exits with 0xE0040001 and whose record a parent really reads back. The synthetic package fixture moves out of test_phi4_manifest.cpp into a shared header so both files build the same on-disk package rather than two copies that can drift. The hardware tests are gated on FLM_AIE4_HARDWARE=1 as well as on a configured runtime directory. A runtime directory names a DLL; it does not assert that the machine is the AIE4 target, and the development box has a real corelib but no AIE4 device. With the flag set, an absent device context is a hard failure rather than a skip. Co-Authored-By: Claude Opus 5 (1M context) --- src/test/phi4_corelib_aie4/CMakeLists.txt | 48 ++ .../phi4_package_fixture.hpp | 519 +++++++++++++++ .../phi4_corelib_aie4/run_hardware_suite.ps1 | 267 ++++++++ .../phi4_corelib_aie4/test_fatal_child.cpp | 491 ++++++++++++++ src/test/phi4_corelib_aie4/test_phi4_e2e.cpp | 615 ++++++++++++++++++ .../phi4_corelib_aie4/test_phi4_hardware.cpp | 568 ++++++++++++++++ .../phi4_corelib_aie4/test_phi4_manifest.cpp | 496 +------------- src/tools/compare_phi4_corelib_e2e.py | 424 ++++++++++++ 8 files changed, 2942 insertions(+), 486 deletions(-) create mode 100644 src/test/phi4_corelib_aie4/phi4_package_fixture.hpp create mode 100644 src/test/phi4_corelib_aie4/run_hardware_suite.ps1 create mode 100644 src/test/phi4_corelib_aie4/test_fatal_child.cpp create mode 100644 src/test/phi4_corelib_aie4/test_phi4_e2e.cpp create mode 100644 src/test/phi4_corelib_aie4/test_phi4_hardware.cpp create mode 100644 src/tools/compare_phi4_corelib_e2e.py diff --git a/src/test/phi4_corelib_aie4/CMakeLists.txt b/src/test/phi4_corelib_aie4/CMakeLists.txt index de4ed1be..0ef381ec 100644 --- a/src/test/phi4_corelib_aie4/CMakeLists.txt +++ b/src/test/phi4_corelib_aie4/CMakeLists.txt @@ -144,6 +144,54 @@ if(RYZENAI_CORELIB_RUNTIME_DIR) "PATH=path_list_prepend:${XRT_LIB_DIR};PATH=path_list_prepend:${RYZENAI_CORELIB_RUNTIME_DIR}") endif() +# The AIE4 hardware suite. These need a real NPU as well as a real DLL, so +# unlike test_real_corelib they cannot run anywhere but the target -- and like +# it, they report SKIPPED rather than Passed when the runtime directory is +# unset, because a green-and-inert hardware check reads as coverage it does not +# have. +function(add_corelib_hardware_test TEST_NAME TEST_SOURCE) + add_executable(${TEST_NAME} ${TEST_SOURCE}) + target_link_libraries(${TEST_NAME} PRIVATE + flm_corelib_aie4_testlib + shell32 + ole32 + advapi32) + target_compile_definitions(${TEST_NAME} PRIVATE + FLM_REAL_CORELIB_RUNTIME_DIR="${RYZENAI_CORELIB_RUNTIME_DIR}" + FLM_REAL_CORELIB_EXTRA_DLL_DIRS="${RYZENAI_CORELIB_EXTRA_DLL_DIRS_ESCAPED}") + add_test(NAME ${TEST_NAME} COMMAND ${TEST_NAME}) + set_tests_properties(${TEST_NAME} PROPERTIES + SKIP_RETURN_CODE 77 + # A hardware run loads a 3 GB model and packs its weights, and two + # AIE4 contexts at once fail in ways that look like defects. RUN_ + # SERIAL keeps ctest from overlapping them. + RUN_SERIAL ON + TIMEOUT 3600) + if(RYZENAI_CORELIB_RUNTIME_DIR) + set_tests_properties(${TEST_NAME} PROPERTIES + ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:${XRT_LIB_DIR};PATH=path_list_prepend:${RYZENAI_CORELIB_RUNTIME_DIR}") + endif() +endfunction() + +add_corelib_hardware_test(test_phi4_hardware test_phi4_hardware.cpp) +add_corelib_hardware_test(test_fatal_child test_fatal_child.cpp) + +# The end-to-end harness is NOT registered with add_test. It requires a model +# directory and explicit token IDs, and the reference comparator must run to +# completion before it starts; run_hardware_suite.ps1 owns that sequencing. +# Registering it here with no arguments would produce a test that fails for +# want of a flag on every machine, which is worse than one that is absent. +add_executable(test_phi4_e2e test_phi4_e2e.cpp) +target_link_libraries(test_phi4_e2e PRIVATE + flm_corelib_aie4_testlib + shell32 + ole32 + advapi32) +target_compile_definitions(test_phi4_e2e PRIVATE + FLM_REAL_CORELIB_RUNTIME_DIR="${RYZENAI_CORELIB_RUNTIME_DIR}" + FLM_REAL_CORELIB_EXTRA_DLL_DIRS="${RYZENAI_CORELIB_EXTRA_DLL_DIRS_ESCAPED}") + # The packaging and clean-environment checks run by default rather than only # when someone remembers the command. Every regression guard in that script -- # the derived closure, the CLOSURE-2 negative control, the no-corelib-import diff --git a/src/test/phi4_corelib_aie4/phi4_package_fixture.hpp b/src/test/phi4_corelib_aie4/phi4_package_fixture.hpp new file mode 100644 index 00000000..8f10a7d6 --- /dev/null +++ b/src/test/phi4_corelib_aie4/phi4_package_fixture.hpp @@ -0,0 +1,519 @@ +#pragma once + +// Shared on-disk synthetic Phi-4 package fixture. Both the manifest tests +// and the hardware tests build the same package, so the builders live here +// instead of being duplicated per test translation unit. + +#include "test_support.hpp" + +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace flm::test::phi4fixture { + +using nlohmann::json; + +inline constexpr std::string_view kManifestName = + "corelib_phi4_manifest.json"; +inline constexpr std::string_view kDataFile = "z-data.bin"; +inline constexpr std::string_view kRopeFile = "z-rope.bin"; +inline constexpr std::uint64_t kDataBytes = + 200064ull * 3072ull * sizeof(std::uint16_t); +inline constexpr std::size_t kRopeRows = 4096; +inline constexpr std::size_t kRopeColumns = 64; +inline constexpr std::uint64_t kRopeBytes = + kRopeRows * kRopeColumns * sizeof(std::uint16_t); +inline constexpr std::uint64_t kRopeMappedBytes = kRopeBytes + 4096; +inline constexpr std::uint64_t kFp32ScaleOffset = 4096; +inline constexpr std::uint64_t kNormOffset = 1024 * 1024; +inline constexpr std::uint64_t kSinOffset = 2 * 1024 * 1024; + +inline constexpr std::string_view kFp32Scale = + "model.layers.0.attn.q_proj.MatMulNBits.scales"; +inline constexpr std::string_view kFp16Scale = + "model.layers.0.attn.k_proj.MatMulNBits.scales"; +inline constexpr std::string_view kFp32Norm = + "model.layers.0.post_attention_layernorm.weight"; + +class TempDirectory final { +public: + TempDirectory() { + const auto nonce = + std::chrono::steady_clock::now().time_since_epoch().count(); + path_ = std::filesystem::temp_directory_path() / + ("fastflowlm-phi4-manifest-" + + std::to_string(GetCurrentProcessId()) + "-" + + std::to_string(nonce)); + std::filesystem::create_directories(path_); + } + + ~TempDirectory() noexcept { + std::error_code error; + std::filesystem::remove_all(path_, error); + } + + TempDirectory(const TempDirectory&) = delete; + TempDirectory& operator=(const TempDirectory&) = delete; + + const std::filesystem::path& path() const noexcept { + return path_; + } + +private: + std::filesystem::path path_; +}; + +inline void CreateSparseFile( + const std::filesystem::path& path, + std::uint64_t size) { + HANDLE file = CreateFileW( + path.c_str(), + GENERIC_READ | GENERIC_WRITE, + FILE_SHARE_READ, + nullptr, + CREATE_ALWAYS, + FILE_ATTRIBUTE_NORMAL, + nullptr); + if (file == INVALID_HANDLE_VALUE) { + throw std::runtime_error("failed to create sparse test file"); + } + + DWORD ignored = 0; + DeviceIoControl( + file, + FSCTL_SET_SPARSE, + nullptr, + 0, + nullptr, + 0, + &ignored, + nullptr); + LARGE_INTEGER end{}; + end.QuadPart = static_cast(size); + const bool success = + SetFilePointerEx(file, end, nullptr, FILE_BEGIN) != FALSE && + SetEndOfFile(file) != FALSE; + CloseHandle(file); + if (!success) { + throw std::runtime_error("failed to size sparse test file"); + } +} + +template +void WriteValues( + const std::filesystem::path& path, + std::uint64_t offset, + std::span values) { + std::fstream file( + path, + std::ios::in | std::ios::out | std::ios::binary); + if (!file) { + throw std::runtime_error("failed to open synthetic data file"); + } + file.seekp(static_cast(offset)); + file.write( + reinterpret_cast(values.data()), + static_cast(values.size_bytes())); + if (!file) { + throw std::runtime_error("failed to write synthetic data"); + } +} + +inline std::uint64_t ItemSize(std::string_view dtype) { + if (dtype == "uint8") { + return 1; + } + if (dtype == "float16") { + return 2; + } + if (dtype == "float32") { + return 4; + } + if (dtype == "int64") { + return 8; + } + throw std::runtime_error("unsupported synthetic dtype"); +} + +inline std::uint64_t ByteLength( + std::string_view dtype, + const std::vector& shape) { + std::uint64_t elements = 1; + for (const std::int64_t dimension : shape) { + elements *= static_cast(dimension); + } + return elements * ItemSize(dtype); +} + +inline void AddInitializer( + json& initializers, + std::string name, + std::string dtype, + std::vector shape, + std::string role, + std::string file = std::string(kDataFile), + std::uint64_t offset = 0) { + CHECK(!initializers.contains(name)); + const std::uint64_t length = ByteLength(dtype, shape); + initializers[std::move(name)] = { + {"file", std::move(file)}, + {"offset", offset}, + {"length", length}, + {"dtype", std::move(dtype)}, + {"shape", std::move(shape)}, + {"role", std::move(role)}}; +} + +inline void AddMatMul( + json& manifest, + const std::string& name, + std::int64_t k, + std::int64_t n) { + json roles = { + {"qweight", name + ".qweight"}, + {"scales", name + ".scales"}, + {"qzeros", name + ".qzeros"}}; + manifest["weight_objects"].push_back({ + {"name", name}, + {"kind", "matmul"}, + {"descriptor", + { + {"k", k}, + {"n", n}, + {"group_size", 128}, + {"has_bias", false}, + }}, + {"roles", roles}}); + + auto& initializers = manifest["initializers"]; + AddInitializer( + initializers, + roles["qweight"].get(), + "uint8", + {n, k / 2}, + "matmul.qweight"); + + std::string scale_dtype = "float16"; + std::uint64_t scale_offset = 0; + if (name == "model.layers.0.attn.q_proj.MatMulNBits") { + scale_dtype = "float32"; + scale_offset = kFp32ScaleOffset; + } + AddInitializer( + initializers, + roles["scales"].get(), + scale_dtype, + {n, k / 128}, + "matmul.scales", + std::string(kDataFile), + scale_offset); + AddInitializer( + initializers, + roles["qzeros"].get(), + "uint8", + {n, ((k / 128) + 1) / 2}, + "matmul.qzeros"); +} + +inline void AddSsMlp(json& manifest, int layer) { + const std::string base = + "model.layers." + std::to_string(layer); + const std::string object_name = base + ".ssmlp"; + const std::string norm0 = + base + ".post_attention_layernorm.weight"; + const std::string norm1 = + layer == 31 + ? "model.layers.32.final_norm_layernorm.weight" + : "model.layers." + std::to_string(layer + 1) + + ".input_layernorm.weight"; + + json roles = { + {"norm0", norm0}, + {"norm1", norm1}, + }; + auto& initializers = manifest["initializers"]; + for (const std::string projection : {"gate", "up", "down"}) { + const std::int64_t k = projection == "down" ? 8192 : 3072; + const std::int64_t n = projection == "down" ? 3072 : 8192; + const std::string prefix = + base + ".mlp." + projection + "_proj.MatMulNBits"; + for (const std::string component : + {"qweight", "scales", "qzeros"}) { + roles[projection + "_" + component] = + prefix + "." + component; + } + AddInitializer( + initializers, + prefix + ".qweight", + "uint8", + {n, k / 2}, + "ssmlp." + projection + ".qweight"); + AddInitializer( + initializers, + prefix + ".scales", + "float16", + {n, k / 128}, + "ssmlp." + projection + ".scales"); + AddInitializer( + initializers, + prefix + ".qzeros", + "uint8", + {n, ((k / 128) + 1) / 2}, + "ssmlp." + projection + ".qzeros"); + } + + AddInitializer( + initializers, + norm0, + layer == 0 ? "float32" : "float16", + {3072}, + "ssmlp.norm0", + std::string(kDataFile), + layer == 0 ? kNormOffset : 0); + AddInitializer( + initializers, + norm1, + "float16", + {3072}, + "ssmlp.norm1"); + + manifest["weight_objects"].push_back({ + {"name", object_name}, + {"kind", "ssmlp"}, + {"descriptor", + { + {"k", 3072}, + {"n", 8192}, + {"group_size", 128}, + }}, + {"roles", std::move(roles)}}); +} + +inline json BuildManifest(std::uint64_t model_size) { + json manifest = { + {"schema_version", 1}, + {"execution_backend", "corelib_aie4"}, + {"model", + { + {"family", "phi4"}, + {"layers", 32}, + {"hidden_size", 3072}, + {"intermediate_size", 8192}, + {"num_heads", 24}, + {"kv_heads", 8}, + {"head_size", 128}, + {"vocab_size", 200064}, + {"group_size", 128}, + {"rope_dim", 96}, + {"rms_epsilon", 0.00001}, + }}, + {"backend", {{"max_seq", 4096}}}, + {"files", + { + {"model.onnx", {{"size", model_size}}}, + {std::string(kDataFile), {{"size", kDataBytes}}}, + {std::string(kRopeFile), {{"size", kRopeMappedBytes}}}, + }}, + {"initializers", json::object()}, + {"weight_objects", json::array()}, + }; + + for (int layer = 0; layer < 32; ++layer) { + const std::string base = + "model.layers." + std::to_string(layer) + ".attn."; + AddMatMul( + manifest, + base + "q_proj.MatMulNBits", + 3072, + 3072); + AddMatMul( + manifest, + base + "k_proj.MatMulNBits", + 3072, + 1024); + AddMatMul( + manifest, + base + "v_proj.MatMulNBits", + 3072, + 1024); + AddMatMul( + manifest, + base + "o_proj.MatMulNBits", + 3072, + 3072); + AddSsMlp(manifest, layer); + } + AddMatMul(manifest, "lm_head.MatMulNBits", 3072, 200064); + + AddInitializer( + manifest["initializers"], + "model.embed_tokens.weight", + "float16", + {200064, 3072}, + "embedding"); + AddInitializer( + manifest["initializers"], + "model.layers.0.input_layernorm.weight", + "float32", + {3072}, + "input_norm", + std::string(kDataFile), + kNormOffset); + AddInitializer( + manifest["initializers"], + "cos_cache", + "float16", + {4096, 64}, + "cos_cache", + std::string(kRopeFile)); + AddInitializer( + manifest["initializers"], + "sin_cache", + "float32", + {4096, 48}, + "sin_cache", + std::string(kDataFile), + kSinOffset); + + CHECK(manifest["weight_objects"].size() == 161); + CHECK(manifest["initializers"].size() == 743); + return manifest; +} + +class SyntheticPackage final { +public: + SyntheticPackage() { + const auto model_path = temp_.path() / "model.onnx"; + { + std::ofstream model(model_path, std::ios::binary); + model << "model"; + } + CreateSparseFile(temp_.path() / kDataFile, kDataBytes); + CreateSparseFile( + temp_.path() / kRopeFile, + kRopeMappedBytes); + + const std::array half_values{ + 0x3c00u, + 0xc000u}; + const std::array float_values{1.0f, -2.0f}; + WriteValues( + temp_.path() / kDataFile, + 0, + std::span(half_values)); + WriteValues( + temp_.path() / kDataFile, + kFp32ScaleOffset, + std::span(float_values)); + WriteValues( + temp_.path() / kDataFile, + kNormOffset, + std::span(float_values)); + const std::array sin_value{4.0f}; + WriteValues( + temp_.path() / kDataFile, + kSinOffset, + std::span(sin_value)); + + const std::array one{0x3c00u}; + const std::array two{0x4000u}; + const std::array three{0x4200u}; + WriteValues( + temp_.path() / kRopeFile, + 0, + std::span(one)); + WriteValues( + temp_.path() / kRopeFile, + kRopeColumns * sizeof(std::uint16_t), + std::span(two)); + WriteValues( + temp_.path() / kRopeFile, + ((kRopeRows - 1) * kRopeColumns + 47) * + sizeof(std::uint16_t), + std::span(three)); + + manifest_ = BuildManifest( + std::filesystem::file_size(model_path)); + Write(manifest_); + } + + const std::filesystem::path& path() const noexcept { + return temp_.path(); + } + + json manifest() const { + return manifest_; + } + + void Write(const json& manifest) const { + std::ofstream stream( + temp_.path() / kManifestName, + std::ios::binary | std::ios::trunc); + if (!stream) { + throw std::runtime_error("failed to write synthetic manifest"); + } + stream << manifest.dump(2) << '\n'; + } + +private: + TempDirectory temp_; + json manifest_; +}; + +class NoAccessGuard final { +public: + explicit NoAccessGuard(void* address) { + MEMORY_BASIC_INFORMATION region{}; + if ( + VirtualQuery(address, ®ion, sizeof(region)) != + sizeof(region) || + region.State != MEM_COMMIT) { + throw std::runtime_error("failed to query guard-page address"); + } + address_ = address; + if ( + VirtualProtect( + address, + 4096, + PAGE_NOACCESS, + &old_protection_) == FALSE) { + throw std::runtime_error( + "failed to protect no-access guard page"); + } + } + + ~NoAccessGuard() noexcept { + if (address_ != nullptr) { + DWORD ignored = 0; + VirtualProtect( + address_, + 4096, + old_protection_, + &ignored); + } + } + + NoAccessGuard(const NoAccessGuard&) = delete; + NoAccessGuard& operator=(const NoAccessGuard&) = delete; + +private: + void* address_ = nullptr; + DWORD old_protection_ = 0; +}; + +} // namespace flm::test::phi4fixture diff --git a/src/test/phi4_corelib_aie4/run_hardware_suite.ps1 b/src/test/phi4_corelib_aie4/run_hardware_suite.ps1 new file mode 100644 index 00000000..77d5ee99 --- /dev/null +++ b/src/test/phi4_corelib_aie4/run_hardware_suite.ps1 @@ -0,0 +1,267 @@ +# Task 12 Step 8: the single entry point for the AIE4 hardware acceptance run. +# +# Everything this suite needs to do on the target is here rather than in a +# session transcript, because a sequence that only exists in somebody's shell +# history is not reproducible and cannot be reviewed. +# +# The ordering is load-bearing, not stylistic. Two processes holding AIE4 +# device contexts at once fail in ways that look like defects, so the Python +# reference is run to completion -- including corelib.cleanup() -- before the +# C++ harness starts, and the harness finishes before the next reference run +# begins. Nothing here runs in parallel. +# +# Skipped work is reported as SKIPPED, never as passed, and exit code 77 is +# CTest's SKIP_RETURN_CODE so a partial run reads as Skipped rather than +# Passed. + +[CmdletBinding()] +param( + # The accepted OGA DML Phi-4 package. Without it the numeric and boundary + # blocks cannot run; the device smoke and the fatal child still can. + [string]$ModelDir, + + # Directory holding the ryzenai_corelib.dll under test, with its derived + # runtime closure staged beside it. Design CLOSURE-1: derive that closure + # with cmake -P cmake/StageAie4Runtime.cmake against the exact shipped + # DLL. Never transcribe it. + [string]$CorelibRuntimeDir, + + # Additional directories completing the closure, if the staged directory + # is not self-contained. Semicolon-separated. + [string]$DependencyDir = "", + + # The ryzenai-corelib checkout whose python/ holds phi4_driver.py. Read + # only: no --continuation-route option is added to that driver and that + # repository is never modified. + [string]$CorelibSource = $env:RYZENAI_CORELIB_SOURCE, + + [string]$BuildDir, + [string]$Cmake, + [string]$Python = "python", + [int]$DecodeSteps = 16, + + # Rows the boundary sweep runs REAL prefills at. Off by default because a + # 4096-row prefill is the slowest thing in the suite; the helper-table + # half of Step 5 runs unconditionally inside test_phi4_hardware. + [switch]$BoundarySweep +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest + +$suiteDir = Split-Path -Parent $MyInvocation.MyCommand.Path +$sourceDir = Resolve-Path (Join-Path $suiteDir '..' '..') +$ran = @() +$skipped = @() + +function Write-Section { + param([string]$Name) + Write-Output '' + Write-Output "=== $Name ===" +} + +function Invoke-Checked { + param( + [string]$Label, + [string]$Exe, + [string[]]$Arguments + ) + Write-Output "> $Exe $($Arguments -join ' ')" + & $Exe @Arguments + $code = $LASTEXITCODE + if ($code -ne 0) { + throw "$Label failed with exit code $code" + } +} + +# --------------------------------------------------------------------------- +# Preconditions +# --------------------------------------------------------------------------- + +if (-not $Cmake) { + # The Cygwin cmake on PATH cannot drive a Visual Studio generator. Prefer + # the one Visual Studio ships, and only fall back to PATH if it is absent. + $bundled = Join-Path $env:ProgramFiles ('Microsoft Visual Studio\2022\Community\Common7\IDE\' + + 'CommonExtensions\Microsoft\CMake\CMake\bin\cmake.exe') + if (Test-Path $bundled) { + $Cmake = $bundled + } else { + $Cmake = 'cmake' + } +} +if (-not $BuildDir) { + $BuildDir = Join-Path $sourceDir 'build/phi4-hardware' +} + +if (-not $CorelibRuntimeDir) { + Write-Output 'run_hardware_suite: SKIPPED -- -CorelibRuntimeDir is required.' + Write-Output ' Stage the derived closure first, for example:' + Write-Output ' cmake -DFLM_AIE4_CORELIB_DIR= \' + Write-Output ' -DFLM_AIE4_XRT_DIR= \' + Write-Output ' -DFLM_AIE4_EXTRA_DIRS=
\' + Write-Output ' -DFLM_AIE4_DESTINATION= \' + Write-Output ' -P src/cmake/StageAie4Runtime.cmake' + exit 77 +} +$CorelibRuntimeDir = (Resolve-Path $CorelibRuntimeDir).Path +$corelibDll = Join-Path $CorelibRuntimeDir 'ryzenai_corelib.dll' +if (-not (Test-Path $corelibDll)) { + throw "no ryzenai_corelib.dll in $CorelibRuntimeDir" +} + +$hash = (Get-FileHash -Algorithm SHA256 -Path $corelibDll).Hash +Write-Section 'Runtime under test' +Write-Output "corelib : $corelibDll" +Write-Output "sha256 : $hash" +Write-Output "size : $((Get-Item $corelibDll).Length) bytes" +Get-ChildItem $CorelibRuntimeDir -Filter *.dll | + Sort-Object Name | + ForEach-Object { Write-Output ("staged : {0} ({1} bytes)" -f $_.Name, $_.Length) } + +# --------------------------------------------------------------------------- +# Configure and build +# --------------------------------------------------------------------------- + +Write-Section 'Configure and build' +$configureArgs = @( + '-S', $suiteDir, + '-B', $BuildDir, + '-G', 'Visual Studio 17 2022', + '-A', 'x64', + "-DRYZENAI_CORELIB_RUNTIME_DIR=$CorelibRuntimeDir" +) +if ($DependencyDir) { + $configureArgs += "-DRYZENAI_CORELIB_EXTRA_DLL_DIRS=$DependencyDir" +} +Invoke-Checked 'configure' $Cmake $configureArgs +Invoke-Checked 'build' $Cmake @('--build', $BuildDir, '--config', 'Release') + +$binDir = Join-Path $BuildDir 'Release' +$e2eExe = Join-Path $binDir 'test_phi4_e2e.exe' + +# --------------------------------------------------------------------------- +# Standalone suite, then the device smoke, then the fatal child +# --------------------------------------------------------------------------- + +Write-Section 'CTest suite (host tests, real-corelib check, device smoke, fatal child)' +# The hardware tests skip unless this is set, so that a development box with a +# configured runtime directory but no AIE4 device does not go red for a reason +# that is not a defect. With it set, an absent device context is a hard +# failure -- which is the behaviour that matters on the one machine where this +# script is meant to run. +$env:FLM_AIE4_HARDWARE = '1' +# Serial on purpose: several of these hold an AIE4 device context. +Invoke-Checked 'ctest' 'ctest' @( + '--test-dir', $BuildDir, + '-C', 'Release', + '--output-on-failure', + '--no-tests=error' +) +$ran += 'ctest suite (includes test_phi4_hardware and test_fatal_child)' + +# --------------------------------------------------------------------------- +# Numeric goldens, one per forced continuation route +# --------------------------------------------------------------------------- + +$comparator = Join-Path $sourceDir 'tools/compare_phi4_corelib_e2e.py' + +if (-not $ModelDir) { + $skipped += 'numeric goldens and boundary sweep (-ModelDir not supplied)' +} elseif (-not $CorelibSource) { + $skipped += 'numeric goldens (-CorelibSource / RYZENAI_CORELIB_SOURCE not set)' +} else { + $ModelDir = (Resolve-Path $ModelDir).Path + $CorelibSource = (Resolve-Path $CorelibSource).Path + $artifacts = Join-Path $BuildDir 'artifacts' + New-Item -ItemType Directory -Force -Path $artifacts | Out-Null + + # Explicit token IDs, never a tokenizer. The point of design Section 12.4's + # explicit-token checkpoints is that both sides consume the SAME integers, + # and a tokenizer in the loop would make a mismatch ambiguous between the + # model and the encoder. The prefix/suffix split is what makes a + # continuation route meaningful: with no suffix both routes degenerate to + # one prefill. + $tokenPlan = Join-Path $artifacts 'tokens.json' + @{ + prefix = @(200022, 882, 200024, 3923, 374, 279, 6864, 315, 9822, 30, 200021, 200022, 78191, 200024) + suffix = @(791, 6864, 315, 9822, 374) + } | ConvertTo-Json -Compress | Set-Content -Path $tokenPlan -Encoding ascii + + $env:RYZENAI_CORELIB_SOURCE = $CorelibSource + # The harness resolves the DLL through this, so it loads the staged + # closure rather than whatever happens to be on PATH. + $env:RYZENAI_CORELIB_PATH = $corelibDll + + foreach ($route in @('force_reprefill', 'force_append')) { + Write-Section "Numeric golden: $route" + $referenceJson = Join-Path $artifacts "reference-$route.json" + $fastflowJson = Join-Path $artifacts "fastflow-$route.json" + + # 1. Reference FIRST, run to completion including corelib.cleanup(). + Invoke-Checked "reference ($route)" $Python @( + $comparator, 'emit-reference', + '--model-dir', $ModelDir, + '--token-ids-json', $tokenPlan, + '--decode-steps', "$DecodeSteps", + '--continuation-route', $route, + '--output-json', $referenceJson + ) + + # 2. Only then the C++ harness. Never both at once. + $e2eArgs = @( + '--model-dir', $ModelDir, + '--token-ids-json', $tokenPlan, + '--decode-steps', "$DecodeSteps", + '--continuation-route', $route, + '--output-json', $fastflowJson + ) + if ($BoundarySweep -and $route -eq 'force_reprefill') { + # Once, not once per route: the sweep is route-independent and a + # 4096-row prefill is the slowest thing in the suite. + $e2eArgs += '--boundary-sweep' + } + Invoke-Checked "fastflow ($route)" $e2eExe $e2eArgs + + # 3. Comparison holds no device context at all. + Invoke-Checked "compare ($route)" $Python @( + $comparator, 'compare', + '--fastflow-json', $fastflowJson, + '--reference-json', $referenceJson + ) + $ran += "numeric golden $route" + } + if ($BoundarySweep) { + $ran += 'boundary sweep (real prefills at every helper transition)' + } else { + $skipped += 'boundary sweep real prefills (-BoundarySweep not passed; the helper-table half still ran inside test_phi4_hardware)' + } +} + +# --------------------------------------------------------------------------- +# Step 7 -- CLI and server endpoints +# --------------------------------------------------------------------------- + +Write-Section 'CLI and server endpoints (Step 7)' +# BLOCKED, and reported as blocked rather than skipped-and-forgotten. +# FastFlow depends on tokenizers-cpp, which needs Cargo, and neither cargo nor +# rustc exists on the development box or on the AIE4 target. flm.exe therefore +# cannot be built anywhere in this environment, so `flm run`, /api/generate, +# /api/chat, /v1/chat/completions and /v1/completions are unverified. Installing +# a Rust toolchain on a shared lab box is a human decision, not something this +# script should take. +Write-Output 'BLOCKED: flm.exe cannot be built (no Rust toolchain for tokenizers-cpp).' +Write-Output ' Unverified: flm run phi4-mini-it-aie4:4b, /api/generate, /api/chat,' +Write-Output ' /v1/chat/completions, /v1/completions.' +$skipped += 'Step 7 CLI and server endpoints (BLOCKED: no Rust toolchain, flm.exe unbuildable)' + +# --------------------------------------------------------------------------- + +Write-Section 'Summary' +foreach ($item in $ran) { Write-Output "ran : $item" } +foreach ($item in $skipped) { Write-Output "skipped: $item" } +if ($ran.Count -eq 0) { + Write-Output 'run_hardware_suite: SKIPPED -- nothing ran.' + exit 77 +} +Write-Output 'run_hardware_suite: PASS' +exit 0 diff --git a/src/test/phi4_corelib_aie4/test_fatal_child.cpp b/src/test/phi4_corelib_aie4/test_fatal_child.cpp new file mode 100644 index 00000000..432b26cc --- /dev/null +++ b/src/test/phi4_corelib_aie4/test_fatal_child.cpp @@ -0,0 +1,491 @@ +// Task 12 Step 6: the terminal-failure path, in a real process that really +// dies. +// +// Everything before this ran the failure policy in-process against an +// intercepted terminator, so nothing had ever confirmed that a FastFlow process +// actually exits with 0xE0040001, that the detailed record actually reaches +// %LOCALAPPDATA%\FastFlowLM\logs, or that a parent can actually read it back. +// Those are the three things this file establishes, by forking itself. +// +// The record root is deliberately the REAL one. FatalRecordStore resolves it +// with SHGetKnownFolderPath rather than from the environment, so it cannot be +// redirected -- and redirecting it would remove the very precondition design +// Section 12.1 asks about, which is whether that directory is writable on the +// target. +// +// What is injected and what is not, stated plainly because it bounds the +// claim: the FAILURE is real -- a genuine call into the real corelib that the +// real library genuinely refuses, carrying its own status code and its own +// message. The SUBMISSION STATE is synthetic. There is no way to make the +// device fail after a successful submit on demand without corrupting the +// device, so the irrevocable-boundary cases set that flag directly and then +// let the unmodified policy decide. The policy, the record, the exit code and +// the parent-side drain are all the production ones. + +#include "test_support.hpp" + +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +namespace constants = flm::phi4::constants; + +using flm::corelib::CorelibApi; +using flm::corelib::CorelibError; +using flm::corelib::CorelibRuntime; +using flm::corelib::FatalRecordStore; +using flm::corelib::StepSubmissionState; + +constexpr unsigned int kFatalExitCode = 0xE0040001u; + +// Printed by the child ONLY on the healthy shutdown path. Its ABSENCE from a +// hard-exit child's output is an assertion in its own right: a process that +// terminated after an irrevocable failure must not have run normal cleanup, +// and "the record exists" alone would not show that. +constexpr std::string_view kCleanupMarker = "FATAL_CHILD_CLEANUP_OK"; +constexpr std::string_view kSurvivedMarker = "FATAL_CHILD_SURVIVED"; + +#if !defined(FLM_REAL_CORELIB_RUNTIME_DIR) +#define FLM_REAL_CORELIB_RUNTIME_DIR "" +#endif + +#if !defined(FLM_REAL_CORELIB_EXTRA_DLL_DIRS) +#define FLM_REAL_CORELIB_EXTRA_DLL_DIRS "" +#endif + +void AddExtraDllDirectories(std::string_view directories) { + std::size_t start = 0; + while (start <= directories.size()) { + const std::size_t end = directories.find(';', start); + const std::string_view entry = directories.substr( + start, + end == std::string_view::npos ? std::string_view::npos + : end - start); + if (!entry.empty()) { + const std::filesystem::path directory(entry); + if ( + !std::filesystem::exists(directory) || + AddDllDirectory(directory.c_str()) == nullptr) { + throw std::runtime_error( + "cannot add corelib DLL directory " + + directory.string()); + } + } + if (end == std::string_view::npos) { + break; + } + start = end + 1; + } +} + +std::filesystem::path CorelibLibraryPath() { + const std::string runtime_dir(FLM_REAL_CORELIB_RUNTIME_DIR); + return std::filesystem::absolute( + std::filesystem::path(runtime_dir) / + "ryzenai_corelib.dll") + .lexically_normal(); +} + +// A real corelib call that the real library really refuses. +// +// The LM-head MatMul shape ships kernels at M in {1, 128} and ERRORS above +// 128 rather than rounding up -- measured, and asserted separately in +// test_phi4_hardware. Using it here means the CorelibError carries the +// library's own status code and its own last-error message, so the record the +// parent reads back is a real diagnostic rather than a string this file made +// up. +CorelibError RealCorelibFailure(const std::shared_ptr& api) { + std::int64_t m = constants::kMaxSequenceLength; + std::int64_t k = constants::kHiddenSize; + std::int64_t n = constants::kVocabularySize; + try { + api->Check( + api->functions().matmul_pad_shape( + &m, + &k, + &n, + constants::kGroupSize), + "ryzenai_corelib_matmul_bf16_pad_shape"); + } catch (const CorelibError& error) { + return error; + } + throw std::runtime_error( + "the real corelib accepted an LM-head M of " + + std::to_string(constants::kMaxSequenceLength) + + ", so this file has no genuine failure to drive the policy with. " + "That is a finding about the library, not a defect here."); +} + +// --------------------------------------------------------------------------- +// Child +// --------------------------------------------------------------------------- + +int RunChild(std::string_view scenario) { + AddExtraDllDirectories(FLM_REAL_CORELIB_EXTRA_DLL_DIRS); + + // GetOrCreate builds the production runtime: the real DLL, the real + // FatalRecordStore rooted in LocalAppData, and the real terminator that + // calls TerminateProcess. Nothing is substituted. + const auto library = CorelibLibraryPath(); + SetEnvironmentVariableW( + L"RYZENAI_CORELIB_PATH", + library.c_str()); + auto runtime = CorelibRuntime::GetOrCreate(library.parent_path()); + CHECK( + runtime->state() == flm::corelib::ProcessState::Healthy); + + const CorelibError failure = RealCorelibFailure(runtime->api()); + std::cout << "child scenario=" << scenario + << " status=" << static_cast(failure.status) + << " call=" << failure.call << '\n'; + + StepSubmissionState submission; + bool synchronize_in_progress = false; + std::string phase = "qkv"; + if (scenario == "before_submit") { + // Nothing has been submitted and no synchronize is running, so the + // policy must RETHROW and leave the process alive. + phase = "qkv"; + } else if (scenario == "after_submit") { + // q submitted successfully, k failed. Past the irrevocable boundary. + submission.MarkSuccessfulSubmit(); + phase = "qkv"; + } else if (scenario == "synchronize") { + // A failing synchronize is irrevocable even with nothing marked as + // submitted: the device may already be mid-flight. + synchronize_in_progress = true; + phase = "flat_mha"; + } else { + throw std::runtime_error( + "unknown child scenario: " + std::string(scenario)); + } + + bool rethrown = false; + try { + flm::phi4::testing::ApplyCorelibFailurePolicyForTest( + runtime, + failure, + synchronize_in_progress, + submission, + phase, + /*layer=*/7, + /*rows=*/13, + /*position=*/29); + } catch (const CorelibError&) { + rethrown = true; + } + + // Only "before_submit" can reach here. The other two scenarios died + // inside the policy, so if control returns for them the policy did not do + // what design Section 12.1 requires and the child fails loudly rather + // than exiting 0. + if (scenario != "before_submit") { + std::cerr << "child scenario=" << scenario + << " survived an irrevocable failure\n"; + return 2; + } + CHECK(rethrown); + + // The session clears and the process stays usable: admission is still + // open and the runtime is still healthy after a recoverable failure. + CHECK(runtime->admission_open()); + CHECK(runtime->state() == flm::corelib::ProcessState::Healthy); + std::cout << kSurvivedMarker << '\n'; + + CorelibRuntime::ShutdownProcess(); + std::cout << kCleanupMarker << '\n'; + std::cout.flush(); + return 0; +} + +// --------------------------------------------------------------------------- +// Parent +// --------------------------------------------------------------------------- + +class TempDirectory final { +public: + TempDirectory() { + path_ = std::filesystem::temp_directory_path() / + ("fastflowlm-fatal-child-" + + std::to_string(GetCurrentProcessId())); + std::filesystem::create_directories(path_); + } + + ~TempDirectory() noexcept { + std::error_code error; + std::filesystem::remove_all(path_, error); + } + + TempDirectory(const TempDirectory&) = delete; + TempDirectory& operator=(const TempDirectory&) = delete; + + const std::filesystem::path& path() const noexcept { + return path_; + } + +private: + std::filesystem::path path_; +}; + +struct ChildResult { + DWORD exit_code = 0; + std::string output; +}; + +ChildResult RunScenario( + const std::filesystem::path& executable, + const std::filesystem::path& log_path, + std::string_view scenario) { + SECURITY_ATTRIBUTES inheritable{}; + inheritable.nLength = sizeof(inheritable); + inheritable.bInheritHandle = TRUE; + + HANDLE log = CreateFileW( + log_path.c_str(), + GENERIC_WRITE, + FILE_SHARE_READ, + &inheritable, + CREATE_ALWAYS, + FILE_ATTRIBUTE_NORMAL, + nullptr); + if (log == INVALID_HANDLE_VALUE) { + throw std::runtime_error( + "cannot create child log " + log_path.string()); + } + + std::wstring command = L"\"" + executable.wstring() + + L"\" --child " + + std::wstring( + scenario.begin(), + scenario.end()); + STARTUPINFOW startup{}; + startup.cb = sizeof(startup); + startup.dwFlags = STARTF_USESTDHANDLES; + startup.hStdOutput = log; + startup.hStdError = log; + startup.hStdInput = nullptr; + + PROCESS_INFORMATION process{}; + const BOOL created = CreateProcessW( + executable.c_str(), + command.data(), + nullptr, + nullptr, + TRUE, + 0, + nullptr, + nullptr, + &startup, + &process); + CloseHandle(log); + if (created == FALSE) { + throw std::runtime_error( + "CreateProcessW failed for scenario " + + std::string(scenario) + " (error " + + std::to_string(GetLastError()) + ")"); + } + + ChildResult result; + // Generous, but bounded: the child loads the real corelib, and a hang + // must fail the suite rather than wedge it. + if (WaitForSingleObject(process.hProcess, 300000) != WAIT_OBJECT_0) { + TerminateProcess(process.hProcess, 1); + CloseHandle(process.hThread); + CloseHandle(process.hProcess); + throw std::runtime_error( + "child scenario " + std::string(scenario) + " did not exit"); + } + GetExitCodeProcess(process.hProcess, &result.exit_code); + CloseHandle(process.hThread); + CloseHandle(process.hProcess); + + std::ifstream stream(log_path, std::ios::binary); + std::ostringstream buffer; + buffer << stream.rdbuf(); + result.output = buffer.str(); + return result; +} + +bool Contains(std::string_view haystack, std::string_view needle) { + return haystack.find(needle) != std::string_view::npos; +} + +// Field extraction over the record's own on-disk text rather than a JSON +// parser, so the check is against exactly the bytes a support engineer would +// receive. The record writer emits compact JSON with no spaces around the +// colon, which is what makes this reliable. +std::string FieldValue( + std::string_view record, + std::string_view key) { + const std::string needle = "\"" + std::string(key) + "\":"; + const auto start = record.find(needle); + if (start == std::string_view::npos) { + throw std::runtime_error( + "fatal record is missing the field \"" + std::string(key) + + "\": " + std::string(record)); + } + auto cursor = start + needle.size(); + if (cursor < record.size() && record[cursor] == '"') { + ++cursor; + const auto end = record.find('"', cursor); + return std::string(record.substr(cursor, end - cursor)); + } + const auto end = record.find_first_of(",}", cursor); + return std::string(record.substr(cursor, end - cursor)); +} + +void CheckDetailedRecord( + std::string_view record, + std::string_view expected_phase) { + // "Complete" means every field design Section 12.1 lists is present and + // carries the value the child produced -- not merely that a file exists. + CHECK(FieldValue(record, "phase") == expected_phase); + CHECK(FieldValue(record, "layer") == "7"); + CHECK(FieldValue(record, "rows") == "13"); + CHECK(FieldValue(record, "position") == "29"); + CHECK( + FieldValue(record, "call") == + "ryzenai_corelib_matmul_bf16_pad_shape"); + CHECK(!FieldValue(record, "status").empty()); + CHECK(!FieldValue(record, "detail").empty()); + CHECK(!FieldValue(record, "pid").empty()); + CHECK(!FieldValue(record, "failure_utc").empty()); + CHECK(!FieldValue(record, "process_start_utc").empty()); +} + +int RunParent(const std::filesystem::path& executable) { + TempDirectory logs; + + // Start from a clean slate so a record left by an earlier run cannot be + // mistaken for this run's evidence. + std::ostringstream discarded; + FatalRecordStore::DrainPriorRecords(discarded); + + { + const auto result = RunScenario( + executable, + logs.path() / "before_submit.log", + "before_submit"); + std::cout << "before_submit exit=0x" << std::hex + << result.exit_code << std::dec << '\n'; + CHECK(result.exit_code == 0); + CHECK(Contains(result.output, kSurvivedMarker)); + CHECK(Contains(result.output, kCleanupMarker)); + + // A recoverable failure writes NO record. The child's healthy + // shutdown removes its own pending file, so a drain here must come + // back empty. + std::ostringstream drained; + const auto records = + FatalRecordStore::DrainPriorRecords(drained); + if (!records.empty()) { + throw std::runtime_error( + "a pre-submit failure left a fatal record behind: " + + records.front()); + } + } + + for (const auto& [scenario, phase] : + std::array, 2>{ + {{"after_submit", "qkv"}, {"synchronize", "flat_mha"}}}) { + const auto result = RunScenario( + executable, + logs.path() / (std::string(scenario) + ".log"), + scenario); + std::cout << scenario << " exit=0x" << std::hex + << result.exit_code << std::dec << '\n'; + CHECK(result.exit_code == kFatalExitCode); + + // Normal cleanup must NOT have run. Both markers are printed only on + // the healthy path, so their absence is the evidence. + CHECK(!Contains(result.output, kCleanupMarker)); + CHECK(!Contains(result.output, kSurvivedMarker)); + // The terminal diagnostic goes to stderr before the process dies, + // and it must name the record it wrote. + CHECK(Contains(result.output, "AIE4 terminal failure")); + CHECK(Contains(result.output, "AIE4 fatal record:")); + + std::ostringstream drained; + const auto records = + FatalRecordStore::DrainPriorRecords(drained); + if (records.size() != 1) { + throw std::runtime_error( + "expected exactly one fatal record from scenario " + + std::string(scenario) + ", drained " + + std::to_string(records.size())); + } + CheckDetailedRecord(records.front(), phase); + // The drain reports what it removed, so the record reaches an + // operator rather than only the filesystem. + CHECK(Contains(drained.str(), "\"phase\":\"" + std::string(phase))); + std::cout << " record: " << records.front(); + } + + std::cout << "test_fatal_child: PASS\n"; + return 0; +} + +} // namespace + +constexpr int kCTestSkipReturnCode = 77; + +// Same opt-in as test_phi4_hardware, and for the same reason: a configured +// runtime directory names a DLL, it does not assert that this machine is the +// AIE4 target. GetOrCreate refuses to build a runtime without a device +// context, so without the flag this would fail on every development box. +bool HardwareRunRequested() { + char value[8] = {}; + const DWORD length = GetEnvironmentVariableA( + "FLM_AIE4_HARDWARE", + value, + sizeof(value)); + return length != 0 && length < sizeof(value) && + std::string_view(value) == "1"; +} + +int main(int argc, char** argv) { + const std::string runtime_dir(FLM_REAL_CORELIB_RUNTIME_DIR); + if (runtime_dir.empty()) { + std::cout + << "test_fatal_child: SKIPPED -- configure with " + "-DRYZENAI_CORELIB_RUNTIME_DIR and run on the AIE4 " + "target.\n"; + return kCTestSkipReturnCode; + } + if (!HardwareRunRequested()) { + std::cout + << "test_fatal_child: SKIPPED -- set FLM_AIE4_HARDWARE=1 to " + "run this on the AIE4 target.\n"; + return kCTestSkipReturnCode; + } + + try { + if (argc >= 3 && std::string_view(argv[1]) == "--child") { + return RunChild(argv[2]); + } + return RunParent( + std::filesystem::absolute(argv[0]).lexically_normal()); + } catch (const std::exception& error) { + std::cerr << "test_fatal_child: FAIL: " << error.what() << '\n'; + return 1; + } +} diff --git a/src/test/phi4_corelib_aie4/test_phi4_e2e.cpp b/src/test/phi4_corelib_aie4/test_phi4_e2e.cpp new file mode 100644 index 00000000..47411ef9 --- /dev/null +++ b/src/test/phi4_corelib_aie4/test_phi4_e2e.cpp @@ -0,0 +1,615 @@ +// Task 12 Steps 2, 4 and 5: explicit-token checkpoints from the real engine on +// real hardware, and a real boundary sweep. +// +// This is a harness, not a self-judging test. It takes explicit token IDs and a +// FORCED continuation route, drives `phi4_corelib_aie4` through the real +// corelib on the real device, and writes every checkpoint the Python reference +// comparator needs to a JSON file. It deliberately holds no expected values of +// its own for the numeric checks: the comparison against the reference driver +// belongs in `tools/compare_phi4_corelib_e2e.py`, and duplicating a golden here +// would just be a second copy of the same guess. +// +// The invariants it DOES assert are the ones the reference cannot supply, +// because they are properties of FastFlow's own schedule rather than of the +// model: +// +// * exactly 129 synchronizes per model step (32 layers x 4, plus the LM +// head). The reference driver still uses the collapsed two-synchronize +// schedule that design Section 10.4 no longer considers sound, so this +// count must never be compared against it; +// * exactly 193 dispatches per model step (32 x 6, plus the LM head); +// * 32 V-cache tensor reads and 256 per-head tensor writes per step. +// +// Two processes must never hold AIE4 device contexts at once. The suite script +// runs the Python reference to completion, including corelib.cleanup(), before +// this executable starts. + +#include "test_support.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +namespace constants = flm::phi4::constants; + +using flm::phi4::ContinuationRoute; +using flm::phi4::phi4_corelib_aie4; +using nlohmann::json; + +// Design Section 10.4's schedule, stated as numbers so a change to it fails +// here rather than being absorbed silently. +constexpr std::uint64_t kSynchronizesPerStep = + static_cast(constants::kLayerCount) * 4u + 1u; +constexpr std::uint64_t kDispatchesPerStep = + static_cast(constants::kLayerCount) * 6u + 1u; +constexpr std::uint64_t kVReadsPerStep = + static_cast(constants::kLayerCount); +constexpr std::uint64_t kVWritesPerStep = + static_cast(constants::kLayerCount) * + static_cast(constants::kKvHeadCount); + +static_assert(kSynchronizesPerStep == 129u); +static_assert(kDispatchesPerStep == 193u); +static_assert(kVReadsPerStep == 32u); +static_assert(kVWritesPerStep == 256u); + +struct Options { + std::filesystem::path model_dir; + std::filesystem::path token_ids_json; + std::filesystem::path output_json; + int decode_steps = 16; + ContinuationRoute route = ContinuationRoute::Reprefill; + bool boundary_sweep = false; +}; + +[[noreturn]] void Usage(std::string_view problem) { + throw std::runtime_error( + std::string(problem) + + "\nusage: test_phi4_e2e" + "\n --model-dir " + "\n --token-ids-json " + "\n --decode-steps " + "\n --continuation-route force_append|force_reprefill" + "\n --output-json " + "\n [--boundary-sweep]"); +} + +std::string_view RequireValue( + int argc, + char** argv, + int& index, + std::string_view flag) { + if (index + 1 >= argc) { + Usage(std::string(flag) + " requires a value"); + } + ++index; + return argv[index]; +} + +Options ParseOptions(int argc, char** argv) { + Options options; + bool route_seen = false; + for (int index = 1; index < argc; ++index) { + const std::string_view argument(argv[index]); + if (argument == "--model-dir") { + options.model_dir = + std::filesystem::path( + RequireValue(argc, argv, index, argument)); + } else if (argument == "--token-ids-json") { + options.token_ids_json = + std::filesystem::path( + RequireValue(argc, argv, index, argument)); + } else if (argument == "--output-json") { + options.output_json = + std::filesystem::path( + RequireValue(argc, argv, index, argument)); + } else if (argument == "--decode-steps") { + options.decode_steps = std::stoi( + std::string( + RequireValue(argc, argv, index, argument))); + } else if (argument == "--continuation-route") { + const auto value = + RequireValue(argc, argv, index, argument); + if (value == "force_append") { + options.route = ContinuationRoute::Append; + } else if (value == "force_reprefill") { + options.route = ContinuationRoute::Reprefill; + } else { + Usage( + "--continuation-route must be force_append or " + "force_reprefill"); + } + route_seen = true; + } else if (argument == "--boundary-sweep") { + options.boundary_sweep = true; + } else { + Usage("unrecognized argument: " + std::string(argument)); + } + } + + if (options.model_dir.empty()) { + Usage("--model-dir is required"); + } + if (options.token_ids_json.empty()) { + Usage("--token-ids-json is required"); + } + if (options.output_json.empty()) { + Usage("--output-json is required"); + } + // The route is FORCED, never inferred. Design 12.4 wants one golden per + // route, and a default here would let a run silently produce two goldens + // for the same route. + if (!route_seen) { + Usage("--continuation-route is required"); + } + if (options.decode_steps < 1) { + Usage("--decode-steps must be at least 1"); + } + return options; +} + +struct TokenPlan { + std::vector prefix; + std::vector suffix; +}; + +// Accepts either a flat array of IDs (all prefix, no continuation) or an +// object with "prefix" and "suffix". The two-part form is what makes a route +// meaningful: with no suffix there is nothing to append and both routes +// degenerate to the same single prefill. +TokenPlan LoadTokenPlan(const std::filesystem::path& path) { + std::ifstream stream(path); + if (!stream) { + throw std::runtime_error( + "cannot open --token-ids-json: " + path.string()); + } + json document; + stream >> document; + + TokenPlan plan; + if (document.is_array()) { + plan.prefix = document.get>(); + } else if (document.is_object()) { + plan.prefix = document.at("prefix").get>(); + if (document.contains("suffix")) { + plan.suffix = document.at("suffix").get>(); + } + } else { + throw std::runtime_error( + "--token-ids-json must hold an array or an object with " + "\"prefix\" and \"suffix\""); + } + if (plan.prefix.empty()) { + throw std::runtime_error( + "--token-ids-json prefix must hold at least one token"); + } + return plan; +} + +std::vector LogitBits(const buffer& logits) { + std::vector bits(logits.size()); + const auto* const raw = + reinterpret_cast(logits.data()); + std::copy_n(raw, logits.size(), bits.begin()); + return bits; +} + +// Widened here rather than in the comparator only for the ranking below; the +// full BF16 bit pattern is what the JSON carries, so the comparator does its +// own widening and nothing depends on this conversion being reproduced. +float WidenBf16(std::uint16_t bits) { + const std::uint32_t widened = static_cast(bits) << 16; + float value = 0.0f; + std::memcpy(&value, &widened, sizeof(value)); + return value; +} + +struct Ranking { + std::vector ids; + std::vector values; +}; + +// Ties break toward the LOWEST ID, matching flm::phi4::ArgmaxLowest. The +// comparator asserts that behaviour explicitly, so the ordering used to +// produce the top-k here has to be the same one or the check would be +// comparing two different conventions. +Ranking TopK(const std::vector& bits, std::size_t k) { + std::vector order(bits.size()); + std::iota(order.begin(), order.end(), 0); + const std::size_t count = std::min(k, order.size()); + std::partial_sort( + order.begin(), + order.begin() + static_cast(count), + order.end(), + [&](int left, int right) { + const float left_value = + WidenBf16(bits[static_cast(left)]); + const float right_value = + WidenBf16(bits[static_cast(right)]); + if (left_value != right_value) { + return left_value > right_value; + } + return left < right; + }); + order.resize(count); + + Ranking ranking; + ranking.ids = order; + ranking.values.reserve(count); + for (const int id : order) { + ranking.values.push_back( + WidenBf16(bits[static_cast(id)])); + } + return ranking; +} + +json StepRecord( + const buffer& logits, + const flm::phi4::Phi4Aie4Metrics& before, + const flm::phi4::Phi4Aie4Metrics& after, + std::uint64_t model_steps_in_this_call) { + const auto bits = LogitBits(logits); + const Ranking top32 = TopK(bits, 32); + const Ranking top5 = TopK(bits, 5); + + const std::span logit_span( + logits.data(), + logits.size()); + const int argmax = flm::phi4::ArgmaxLowest(logit_span); + + // The top-1 from the ranking and ArgmaxLowest must agree. If they ever + // did not, every downstream comparison would be against a token the + // engine would not actually have emitted. + CHECK(!top32.ids.empty()); + CHECK(top32.ids.front() == argmax); + + json record; + record["top1_id"] = argmax; + record["top5_ids"] = top5.ids; + record["top32_ids"] = top32.ids; + record["top32_values"] = top32.values; + record["logits_bf16"] = bits; + record["dispatch_delta"] = + after.dispatch_count - before.dispatch_count; + record["synchronize_delta"] = + after.synchronize_count - before.synchronize_count; + record["v_read_delta"] = + after.v_read_calls - before.v_read_calls; + record["v_write_delta"] = + after.v_write_calls - before.v_write_calls; + record["model_steps"] = model_steps_in_this_call; + + // Design Section 10.4 and Section 5.2, asserted rather than merely + // reported. A run that emitted the wrong count and left the judgement to + // a human reading JSON would be a report, not a test. + CHECK( + record["synchronize_delta"].get() == + kSynchronizesPerStep * model_steps_in_this_call); + CHECK( + record["dispatch_delta"].get() == + kDispatchesPerStep * model_steps_in_this_call); + CHECK( + record["v_read_delta"].get() == + kVReadsPerStep * model_steps_in_this_call); + CHECK( + record["v_write_delta"].get() == + kVWritesPerStep * model_steps_in_this_call); + return record; +} + +// One "model step" is one RunRows call. Append walks the suffix one token at a +// time, which is one step each; re-prefill recomputes the whole history in a +// single step. That difference is the whole point of the two routes, so the +// harness counts them explicitly rather than inferring them from the metrics +// it is trying to check. +struct RouteResult { + buffer logits; + std::uint64_t model_steps = 0; +}; + +RouteResult RunForcedRoute( + phi4_corelib_aie4& engine, + const TokenPlan& plan, + ContinuationRoute route) { + RouteResult result; + if (route == ContinuationRoute::Append) { + // The prefix is already the model's history in the product; here it + // is established by one prefill, and only the suffix exercises the + // append path. `prefill` itself walks a multi-token continuation one + // row at a time, so the suffix is fed through `forward` to keep the + // step count unambiguous. + std::vector prefix = plan.prefix; + result.logits = engine.prefill(prefix); + ++result.model_steps; + for (const int token : plan.suffix) { + result.logits = engine.forward(token); + ++result.model_steps; + } + return result; + } + + // Re-prefill clears the conversation and recomputes the full rendered + // history from position zero, which is one step over prefix+suffix rows. + engine.clear_context(); + std::vector full = plan.prefix; + full.insert(full.end(), plan.suffix.begin(), plan.suffix.end()); + result.logits = engine.prefill(full); + ++result.model_steps; + return result; +} + +json SnapshotRecord(const phi4_corelib_aie4& engine) { +#ifdef DEV_BUILD + const auto snapshot = engine.debug_snapshot(); + json record; + record["live_rows"] = snapshot.live_rows; + record["position"] = snapshot.position; + record["layer0_k"] = snapshot.layer0_k; + record["layer0_v"] = snapshot.layer0_v; + record["layer31_k"] = snapshot.layer31_k; + record["layer31_v"] = snapshot.layer31_v; + record["last_hidden"] = snapshot.last_hidden; + + // Live K/V only. The caches are allocated at the full 4096-row window but + // only [0, position) has been written; the rest is uninitialised device + // memory, and comparing it against the reference would be comparing + // garbage to garbage and calling it agreement. + const std::size_t live = + static_cast(constants::kKvHeadCount) * + static_cast(snapshot.position) * + static_cast(constants::kHeadSize); + CHECK(snapshot.layer0_k.size() == live); + CHECK(snapshot.layer0_v.size() == live); + CHECK(snapshot.layer31_k.size() == live); + CHECK(snapshot.layer31_v.size() == live); + return record; +#else + (void)engine; + throw std::runtime_error( + "test_phi4_e2e must be built with DEV_BUILD=1; without it " + "debug_snapshot() does not exist and there are no K/V " + "checkpoints to emit"); +#endif +} + +json MetricsRecord(const flm::phi4::Phi4Aie4Metrics& metrics) { + json record; + record["dispatch_count"] = metrics.dispatch_count; + record["synchronize_count"] = metrics.synchronize_count; + record["v_read_calls"] = metrics.v_read_calls; + record["v_write_calls"] = metrics.v_write_calls; + record["v_bytes"] = metrics.v_bytes; + record["device_tensor_create_count"] = + metrics.device_tensor_create_count; + record["weight_create_count"] = metrics.weight_create_count; + record["padding_write_calls"] = metrics.padding_write_calls; + record["padding_bytes"] = metrics.padding_bytes; + record["packed_weight_bytes"] = metrics.packed_weight_bytes; + record["mapped_source_bytes"] = metrics.mapped_source_bytes; + record["kv_bytes"] = metrics.kv_bytes; + record["scratch_bytes"] = metrics.scratch_bytes; + record["helper_transition_counts"] = + metrics.helper_transition_counts; + record["attention_extent_queries"] = + metrics.attention_extent_queries; + record["output_projection_extent_queries"] = + metrics.output_projection_extent_queries; + record["lm_head_extent_queries"] = metrics.lm_head_extent_queries; + return record; +} + +// Step 5 on real hardware. The probe rows come from the running helper table, +// not from a transcribed grid, and each one is a REAL prefill: a pad-shape +// query alone would not show that the device accepts the padded extent the +// plan chose. +json RunBoundarySweep( + phi4_corelib_aie4& engine, + const std::vector& vocabulary_sample, + const std::vector& probe_rows) { + json sweep = json::array(); + for (const std::int64_t rows : probe_rows) { + engine.clear_context(); + std::vector tokens( + static_cast(rows)); + for (std::size_t index = 0; index < tokens.size(); ++index) { + tokens[index] = + vocabulary_sample[index % vocabulary_sample.size()]; + } + + const auto before = engine.metrics(); + const auto logits = engine.prefill(tokens); + const auto after = engine.metrics(); + + const auto bits = LogitBits(logits); + const Ranking top5 = TopK(bits, 5); + json record; + record["rows"] = rows; + record["top1_id"] = top5.ids.front(); + record["top5_ids"] = top5.ids; + record["synchronize_delta"] = + after.synchronize_count - before.synchronize_count; + record["dispatch_delta"] = + after.dispatch_count - before.dispatch_count; + // Every prefill is ONE model step regardless of how many rows it + // carries, so the schedule counts do not vary with the row count. + CHECK( + record["synchronize_delta"].get() == + kSynchronizesPerStep); + CHECK( + record["dispatch_delta"].get() == + kDispatchesPerStep); + CHECK(engine.get_current_context_length() == rows); + sweep.push_back(std::move(record)); + std::cout << " boundary rows " << rows << ": ok\n"; + } + return sweep; +} + +// One below, at, and above every transition the running helper table reports, +// plus a fresh row 1 and the low-level row 4096. The transitions come from +// Phi4ShapePlan, which derived them from the library; nothing here is a +// transcribed grid, so a library that changed its buckets changes the probes +// rather than slipping past them. +std::vector BoundaryProbeRows( + const flm::phi4::Phi4ShapePlan& plan) { + std::vector rows{1, constants::kMaxSequenceLength}; + for (const auto& [live_rows, padded_rows] : + plan.Transitions(flm::phi4::RowUse::Attention)) { + (void)padded_rows; + for (const std::int64_t offset : {-1, 0, 1}) { + const std::int64_t probe = live_rows + offset; + if (probe >= 1 && probe <= constants::kMaxSequenceLength) { + rows.push_back(probe); + } + } + } + std::sort(rows.begin(), rows.end()); + rows.erase(std::unique(rows.begin(), rows.end()), rows.end()); + return rows; +} + +} // namespace + +int main(int argc, char** argv) { + try { + const Options options = ParseOptions(argc, argv); + if (!std::filesystem::is_directory(options.model_dir)) { + throw std::runtime_error( + "--model-dir is not a directory: " + + options.model_dir.string()); + } + const TokenPlan plan = LoadTokenPlan(options.token_ids_json); + + // RYZENAI_CORELIB_PATH selects the DLL. The suite script points it at + // the staged closure; there is deliberately no fallback that would + // load whatever DLL happened to be on PATH. + const std::filesystem::path executable_dir = + std::filesystem::absolute(argv[0]).parent_path(); + auto runtime = + flm::corelib::CorelibRuntime::GetOrCreate(executable_dir); + std::cout << "corelib runtime ready: " + << runtime->api()->library_path().string() << '\n'; + + LM_Config config; + config._json_config = json::object(); + config.model_path = options.model_dir.string(); + config.model_name = options.model_dir.filename().string(); + + json document; + document["model_dir"] = options.model_dir.string(); + document["corelib_library"] = + runtime->api()->library_path().string(); + document["corelib_version"] = + flm::corelib::FormatCorelibVersion( + runtime->api()->runtime_version()); + document["continuation_route"] = + flm::phi4::ContinuationRouteName(options.route); + document["prefix_ids"] = plan.prefix; + document["suffix_ids"] = plan.suffix; + document["decode_steps_requested"] = options.decode_steps; + + { + phi4_corelib_aie4 engine( + config, + options.model_dir, + runtime, + static_cast( + constants::kMaxSequenceLength)); + + document["load_metrics"] = MetricsRecord(engine.metrics()); + + if (options.boundary_sweep) { + const auto sweep_plan = + flm::phi4::Phi4ShapePlan::Build(runtime->api()); + // Rows are sampled out of the prompt so each one carries a + // real embedding: a constant token would satisfy every shape + // check while masking a row-indexing defect. + document["boundary_sweep"] = RunBoundarySweep( + engine, + plan.prefix, + BoundaryProbeRows(sweep_plan)); + engine.clear_context(); + } + + const auto before_route = engine.metrics(); + RouteResult route_result = + RunForcedRoute(engine, plan, options.route); + const auto after_route = engine.metrics(); + document["continuation"] = StepRecord( + route_result.logits, + before_route, + after_route, + route_result.model_steps); + + json decode = json::array(); + int token = flm::phi4::ArgmaxLowest( + std::span( + route_result.logits.data(), + route_result.logits.size())); + for (int step = 0; step < options.decode_steps; ++step) { + const auto before = engine.metrics(); + const auto logits = engine.forward(token); + const auto after = engine.metrics(); + json record = StepRecord(logits, before, after, 1u); + record["input_id"] = token; + token = record["top1_id"].get(); + decode.push_back(std::move(record)); + } + document["decode"] = std::move(decode); + document["final_snapshot"] = SnapshotRecord(engine); + document["final_metrics"] = MetricsRecord(engine.metrics()); + document["final_position"] = + engine.get_current_context_length(); + } + + // The engine is destroyed before the runtime shuts down, so the + // healthy path really runs: CorelibRuntime refuses cleanup() while + // live corelib objects remain, and a leak would fail here rather + // than pass quietly. + flm::corelib::CorelibRuntime::ShutdownProcess(); + + std::ofstream output(options.output_json, std::ios::trunc); + if (!output) { + throw std::runtime_error( + "cannot write --output-json: " + + options.output_json.string()); + } + output << document.dump(); + if (!output) { + throw std::runtime_error( + "failed while writing --output-json"); + } + output.close(); + + std::cout << "test_phi4_e2e: PASS (" + << options.output_json.string() << ")\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << "test_phi4_e2e: FAIL: " << error.what() << '\n'; + return 1; + } +} diff --git a/src/test/phi4_corelib_aie4/test_phi4_hardware.cpp b/src/test/phi4_corelib_aie4/test_phi4_hardware.cpp new file mode 100644 index 00000000..296bcfbc --- /dev/null +++ b/src/test/phi4_corelib_aie4/test_phi4_hardware.cpp @@ -0,0 +1,568 @@ +// Task 12 Steps 1 and 5: the real corelib, on a real AIE4 device. +// +// test_real_corelib covers the entry points the header documents as needing no +// NPU. Everything here needs one. `has_device_context()` is ASSERTED rather +// than recorded, a Stream and a DeviceTensor are really created on the device, +// and the RoPE upload is a real `tensor_write` rather than a fake that records +// its arguments. +// +// The RoPE case is the one that covers FastFlow rather than corelib. Corelib +// `e5258d2` removed `convert_strided`, so the [4096, 48] slice out of a wider +// table is now FastFlow's own bounds-checked loop over a read-only file +// mapping. The fixture ends the last source row immediately before an +// inaccessible page, so an off-by-one in that loop faults instead of silently +// reading a neighbouring page -- and the gathered rows are then pushed through +// one real FP16-source write into a real FP32 device tensor, which is the pair +// of steps the product actually performs at load. + +#include "phi4_package_fixture.hpp" +#include "test_support.hpp" + +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +namespace constants = flm::phi4::constants; +namespace fixture = flm::test::phi4fixture; + +using flm::corelib::CorelibApi; +using flm::phi4::Phi4Package; +using flm::phi4::Phi4ShapePlan; +using flm::phi4::RowUse; + +#if !defined(FLM_REAL_CORELIB_RUNTIME_DIR) +#define FLM_REAL_CORELIB_RUNTIME_DIR "" +#endif + +#if !defined(FLM_REAL_CORELIB_EXTRA_DLL_DIRS) +#define FLM_REAL_CORELIB_EXTRA_DLL_DIRS "" +#endif + +// CorelibApi::Load uses LOAD_LIBRARY_SEARCH_DEFAULT_DIRS, which honours +// directories added here and deliberately ignores PATH. Design CLOSURE-2: a +// pass that depended on PATH would certify the build machine, not the closure. +void AddExtraDllDirectories(std::string_view directories) { + std::size_t start = 0; + while (start <= directories.size()) { + const std::size_t end = directories.find(';', start); + const std::string_view entry = directories.substr( + start, + end == std::string_view::npos ? std::string_view::npos + : end - start); + if (!entry.empty()) { + const std::filesystem::path directory(entry); + if (!std::filesystem::exists(directory)) { + throw std::runtime_error( + "extra corelib DLL directory does not exist: " + + directory.string()); + } + if (AddDllDirectory(directory.c_str()) == nullptr) { + throw std::runtime_error( + "AddDllDirectory failed for " + directory.string()); + } + std::cout << "added DLL directory " << directory.string() + << '\n'; + } + if (end == std::string_view::npos) { + break; + } + start = end + 1; + } +} + +struct NamedRowUse { + RowUse use; + std::string_view name; +}; + +constexpr std::array kMultiRowUses{{ + {RowUse::QueryProjection, "query_projection"}, + {RowUse::KvProjection, "kv_projection"}, + {RowUse::Attention, "attention"}, + {RowUse::OutputProjection, "output_projection"}, + {RowUse::SsMlp, "ssmlp"}, +}}; + +// 1a. Version identity. API-5 requires an exact major.minor.patch match while +// the compiled-against major is 0, and `CorelibApi::Load` enforces it before +// resolving any other symbol -- so loading at all means the gate passed. +// Restating it here keeps this file from going vacuous if the gate is ever +// relaxed, and prints both sides so a mismatch is diagnosable from the log. +void CheckVersionIdentity(const std::shared_ptr& api) { + const auto compiled = flm::corelib::CompiledCorelibVersion(); + const auto& runtime = api->runtime_version(); + std::cout << "runtime corelib version " + << flm::corelib::FormatCorelibVersion(runtime) + << ", compiled against " + << flm::corelib::FormatCorelibVersion(compiled) << '\n'; + CHECK(runtime.major == compiled.major); + CHECK(runtime.minor == compiled.minor); + CHECK(runtime.patch == compiled.patch); + CHECK(flm::corelib::IsCorelibVersionCompatible(compiled, runtime)); +} + +// 1b. Dependency self-test and device context. On this machine the device +// context is REQUIRED, not recorded: the whole point of the target box is that +// there is an AIE4 device, and a run that quietly proceeded without one would +// report every later check as passed while testing nothing. +void CheckDependenciesAndDevice( + const std::shared_ptr& api) { + api->Check( + api->functions().selftest_dependencies(), + "ryzenai_corelib_selftest_dependencies"); + std::cout << "selftest_dependencies: ok\n"; + + const bool has_context = api->functions().has_device_context(); + std::cout << "has_device_context: " + << (has_context ? "true" : "false") << '\n'; + if (!has_context) { + throw std::runtime_error( + "ryzenai_corelib_has_device_context reported no AIE4 device " + "context. This suite must run on the AIE4 target; a pass " + "without a device would certify nothing. Check that no other " + "process is holding a device context."); + } +} + +// 1c. A real Stream and a real DeviceTensor, with a bounded element round +// trip. `count` and `offset` are ELEMENTS of the tensor's own dtype (API-7), +// so a BF16 tensor written from an FP32 source consumes half as many source +// bytes as destination bytes -- the asymmetry that makes a byte-taking +// overload dangerous enough that FastFlow does not offer one. +void CheckStreamAndTensorRoundTrip( + const std::shared_ptr& api) { + ryzenai_corelib_stream_ptr raw_stream = nullptr; + api->Check( + api->functions().create_stream(&raw_stream), + "ryzenai_corelib_create_stream"); + flm::corelib::UniqueStream stream(api, raw_stream); + CHECK(static_cast(stream)); + + constexpr std::int64_t kRows = 4; + constexpr std::int64_t kWidth = 128; + constexpr std::size_t kElements = + static_cast(kRows * kWidth); + const std::array shape{kRows, kWidth}; + + ryzenai_corelib_tensor_ptr raw_tensor = nullptr; + api->Check( + api->functions().create_device_tensor( + ryzenai_corelib_data_type_bf16, + shape.data(), + shape.size(), + &raw_tensor), + "ryzenai_corelib_create_device_tensor"); + flm::corelib::UniqueTensor tensor(api, raw_tensor); + CHECK(static_cast(tensor)); + + std::size_t byte_size = 0; + api->Check( + api->functions().tensor_get_byte_size( + tensor.get(), + &byte_size), + "ryzenai_corelib_tensor_get_byte_size"); + CHECK(byte_size == kElements * sizeof(std::uint16_t)); + + ryzenai_corelib_data_type data_type{}; + api->Check( + api->functions().tensor_get_data_type( + tensor.get(), + &data_type), + "ryzenai_corelib_tensor_get_data_type"); + CHECK(data_type == ryzenai_corelib_data_type_bf16); + + // FP32 in, BF16 stored, FP32 back out. The values are chosen to survive + // BF16 exactly -- powers of two and small sums of them -- so a mismatch + // means the transfer moved the wrong elements, not that rounding lost a + // bit. That distinction is why this does not compare with a tolerance. + std::vector source(kElements); + for (std::size_t index = 0; index < kElements; ++index) { + source[index] = + static_cast((index % 64) + 1) * 0.5f; + } + api->WriteElements( + tensor.get(), + ryzenai_corelib_data_type_fp32, + source.data(), + kElements, + 0); + + std::vector destination(kElements, -1.0f); + api->ReadElements( + tensor.get(), + ryzenai_corelib_data_type_fp32, + destination.data(), + kElements, + 0); + CHECK(destination == source); + + // A bounded interior window, to show that a non-zero element offset lands + // where it is asked to and leaves its neighbours untouched. Reading the + // whole tensor back afterwards is what proves the "leaves neighbours + // untouched" half; checking only the window would pass for a write that + // clobbered the rest. + constexpr std::size_t kWindowOffset = kWidth; + constexpr std::size_t kWindowCount = 16; + std::vector window(kWindowCount, 8.0f); + api->WriteElements( + tensor.get(), + ryzenai_corelib_data_type_fp32, + window.data(), + kWindowCount, + kWindowOffset); + + std::vector expected = source; + std::copy( + window.begin(), + window.end(), + expected.begin() + kWindowOffset); + std::fill(destination.begin(), destination.end(), -1.0f); + api->ReadElements( + tensor.get(), + ryzenai_corelib_data_type_fp32, + destination.data(), + kElements, + 0); + CHECK(destination == expected); + + // Released before the healthy cleanup below. CorelibRuntime refuses to + // call cleanup() while live objects remain, so a leak here would surface + // as a shutdown failure rather than as a quiet leak. + tensor.reset(); + stream.reset(); + CHECK(api->live_object_count() == 0); + std::cout << "stream and device tensor round trip: ok\n"; +} + +// 1d. The RoPE gather at a guard page, followed by ONE real tensor_write. +// +// The gather is FastFlow's code and the write is corelib's, and this is the +// only place the two meet on real hardware. The fixture's last source row ends +// exactly at the mapped extent, with PAGE_NOACCESS immediately after, so the +// process dies on an over-read rather than passing with garbage. +void CheckRopeGatherAndRealUpload( + const std::shared_ptr& api) { + fixture::SyntheticPackage synthetic; + auto package = Phi4Package::Load(synthetic.path(), api, false); + + const auto& source = package.Require("cos_cache"); + CHECK(source.size == fixture::kRopeBytes); + CHECK(source.owner->size() == fixture::kRopeMappedBytes); + + constexpr std::size_t kRopeElements = + static_cast(constants::kMaxSequenceLength) * + static_cast(constants::kRopeDimension / 2); + static_assert(kRopeElements == 4096u * 48u); + + const std::array shape{ + constants::kMaxSequenceLength, + constants::kRopeDimension / 2}; + ryzenai_corelib_tensor_ptr raw_tensor = nullptr; + api->Check( + api->functions().create_device_tensor( + ryzenai_corelib_data_type_fp32, + shape.data(), + shape.size(), + &raw_tensor), + "ryzenai_corelib_create_device_tensor"); + flm::corelib::UniqueTensor cos_tensor(api, raw_tensor); + + flm::phi4::RopeSourceView rope{}; + { + auto* one_past = + const_cast(source.data + source.size); + fixture::NoAccessGuard guard(one_past); + rope = package.MaterializeRopeGather("cos_cache"); + } + CHECK(rope.dtype == ryzenai_corelib_data_type_fp16); + CHECK(rope.count == kRopeElements); + + // One write, in the SOURCE dtype, with `count` in elements of the + // DESTINATION tensor. tensor_write is the only widening boundary corelib + // e5258d2 offers, and this is the call the engine makes at load. + api->WriteElements( + cos_tensor.get(), + rope.dtype, + rope.data, + kRopeElements, + 0); + + // Read back the three rows the fixture seeded, so the write is shown to + // have landed rather than merely to have returned success. Row 4095's + // column 47 is the last element before the guard page, which is the + // element an over- or under-reading gather would get wrong. + const auto read_element = [&](std::size_t index) { + float value = 0.0f; + api->ReadElements( + cos_tensor.get(), + ryzenai_corelib_data_type_fp32, + &value, + 1, + index); + return value; + }; + CHECK(read_element(0) == 1.0f); + CHECK(read_element(48) == 2.0f); + CHECK(read_element(kRopeElements - 1) == 3.0f); + + cos_tensor.reset(); + CHECK(api->live_object_count() == 0); + std::cout << "RoPE gather at guard page plus real tensor_write: ok\n"; +} + +// 5. Helper boundaries, discovered rather than transcribed. +// +// The row grid is a property of the SHIPPED kernel set, so the transitions are +// read back out of the running helper table and the boundary rows are derived +// from them. A hard-coded {1, 64, 128, ...} would keep passing against a +// library that had changed its grid, which is exactly the failure this is here +// to catch. +void CheckHelperBoundaries(const std::shared_ptr& api) { + const Phi4ShapePlan plan = Phi4ShapePlan::Build(api); + + for (const auto& row_use : kMultiRowUses) { + const auto& transitions = plan.Transitions(row_use.use); + CHECK(!transitions.empty()); + + std::vector probes{ + 1, + constants::kMaxSequenceLength}; + for (const auto& [live_rows, padded_rows] : transitions) { + for (const std::int64_t offset : {-1, 0, 1}) { + const std::int64_t probe = live_rows + offset; + if ( + probe >= 1 && + probe <= constants::kMaxSequenceLength) { + probes.push_back(probe); + } + } + } + std::sort(probes.begin(), probes.end()); + probes.erase( + std::unique(probes.begin(), probes.end()), + probes.end()); + + std::cout << " " << row_use.name << ": " + << transitions.size() << " transitions, " + << probes.size() << " boundary probes"; + + std::int64_t previous_padded = 0; + for (const std::int64_t rows : probes) { + const std::int64_t padded = + plan.RowsFor(row_use.use, rows); + // Never below the live rows, never above the single peak + // allocation, and never decreasing: those three together are + // what make one capacity-sized tensor safe for every row count. + CHECK(padded >= rows); + CHECK(padded <= plan.capacities().layer_rows); + CHECK(padded >= previous_padded); + previous_padded = padded; + + // And the plan must still agree with the library it was built + // from, asked directly at this row count. A plan that had + // memoised a stale answer would pass every check above. + std::int64_t m = rows; + std::int64_t k = constants::kHiddenSize; + std::int64_t n = row_use.use == RowUse::KvProjection + ? constants::kKvDimension + : constants::kQueryDimension; + if (row_use.use == RowUse::SsMlp) { + m = rows; + api->Check( + api->functions().ssmlp_pad_rows( + &m, + constants::kHiddenSize, + constants::kIntermediateSize, + constants::kGroupSize), + "ryzenai_corelib_ssmlp_bf16_pad_rows"); + } else if (row_use.use == RowUse::Attention) { + api->Check( + api->functions().flat_mha_pad_rows( + &m, + &plan.attention_desc()), + "ryzenai_corelib_flat_mha_bf16_pad_rows"); + } else { + api->Check( + api->functions().matmul_pad_shape( + &m, + &k, + &n, + constants::kGroupSize), + "ryzenai_corelib_matmul_bf16_pad_shape"); + // MEM-5: the capacity model assumes padding touches M only. + CHECK(k == constants::kHiddenSize); + } + CHECK(m == padded); + } + std::cout << ", padded 1 -> " << plan.RowsFor(row_use.use, 1) + << ", " << constants::kMaxSequenceLength << " -> " + << plan.RowsFor( + row_use.use, + constants::kMaxSequenceLength) + << '\n'; + + // Fresh row 1 must stay 1 on every multi-row helper: decode + // allocates a single row and flat MHA pads its KV window instead. + CHECK(plan.RowsFor(row_use.use, 1) == 1); + } + + // The LM head is not a multi-row helper. It ships M in {1, 128} and + // ERRORS above 128 rather than rounding up, which is the property that + // makes Phi4ShapePlan's single-row query correct. Assert the refusal + // rather than inferring it from the code that avoids it. + CHECK(plan.capacities().lm_head_rows == 1); + CHECK(plan.RowsFor(RowUse::LmHead, 1) == 1); + for (const std::int64_t rows : + {std::int64_t{1}, std::int64_t{2}, std::int64_t{128}}) { + std::int64_t m = rows; + std::int64_t k = constants::kHiddenSize; + std::int64_t n = constants::kVocabularySize; + api->Check( + api->functions().matmul_pad_shape( + &m, + &k, + &n, + constants::kGroupSize), + "ryzenai_corelib_matmul_bf16_pad_shape"); + CHECK(m == (rows == 1 ? 1 : 128)); + } + for (const std::int64_t rows : + {std::int64_t{129}, + std::int64_t{256}, + std::int64_t{4096}}) { + std::int64_t m = rows; + std::int64_t k = constants::kHiddenSize; + std::int64_t n = constants::kVocabularySize; + const auto status = api->functions().matmul_pad_shape( + &m, + &k, + &n, + constants::kGroupSize); + if (status == ryzenai_corelib_status_success) { + throw std::runtime_error( + "the LM-head shape accepted M=" + std::to_string(rows) + + " and padded to " + std::to_string(m) + + ". Phi4ShapePlan queries the LM head at m=1 only because " + "the shipped kernel set refuses everything above 128; if " + "that changed, the single-row assumption needs revisiting " + "rather than this assertion needs relaxing."); + } + } + std::cout << "helper boundaries agree with the running kernel set\n"; + + // The peak allocation must really be creatable on the device. The + // capacity model is only sound if a tensor of that extent exists, and a + // pad-shape query alone would not show that. + const std::array peak_shape{ + plan.capacities().layer_rows, + constants::kQueryDimension}; + ryzenai_corelib_tensor_ptr raw_tensor = nullptr; + api->Check( + api->functions().create_device_tensor( + ryzenai_corelib_data_type_bf16, + peak_shape.data(), + peak_shape.size(), + &raw_tensor), + "ryzenai_corelib_create_device_tensor"); + flm::corelib::UniqueTensor peak(api, raw_tensor); + CHECK(static_cast(peak)); + peak.reset(); + CHECK(api->live_object_count() == 0); + std::cout << "peak layer allocation (" + << plan.capacities().layer_rows << " x " + << constants::kQueryDimension + << " BF16) is creatable on the device\n"; +} + +} // namespace + +// CTest's SKIP_RETURN_CODE. Returning 0 without a runtime directory would +// report Passed, and a green-and-inert hardware check reads as coverage it +// does not have. +constexpr int kCTestSkipReturnCode = 77; + +// Opt-in, and deliberately separate from RYZENAI_CORELIB_RUNTIME_DIR. +// +// A configured runtime directory says which DLL to load; it does not say that +// this machine is the AIE4 target. The development box has an NPU and a real +// corelib but is not AIE4, so keying the hardware run off the directory alone +// would turn every dev-box `ctest` red for a reason that is not a defect. +// Keying it off nothing would be worse: an absent device on the target would +// then read as a skip, and the one machine where this must run would be the +// one machine where a silent skip goes unnoticed. With the flag set, a missing +// device context is a hard failure. +bool HardwareRunRequested() { + char value[8] = {}; + const DWORD length = GetEnvironmentVariableA( + "FLM_AIE4_HARDWARE", + value, + sizeof(value)); + return length != 0 && length < sizeof(value) && + std::string_view(value) == "1"; +} + +int main() { + const std::string runtime_dir(FLM_REAL_CORELIB_RUNTIME_DIR); + if (runtime_dir.empty()) { + std::cout + << "test_phi4_hardware: SKIPPED -- configure with " + "-DRYZENAI_CORELIB_RUNTIME_DIR= and run on the AIE4 target.\n"; + return kCTestSkipReturnCode; + } + if (!HardwareRunRequested()) { + std::cout + << "test_phi4_hardware: SKIPPED -- set FLM_AIE4_HARDWARE=1 to " + "run this on the AIE4 target. run_hardware_suite.ps1 sets " + "it; nothing else should.\n"; + return kCTestSkipReturnCode; + } + + try { + const std::filesystem::path library = + std::filesystem::absolute( + std::filesystem::path(runtime_dir) / + "ryzenai_corelib.dll") + .lexically_normal(); + if (!std::filesystem::exists(library)) { + throw std::runtime_error( + "RYZENAI_CORELIB_RUNTIME_DIR is set but " + + library.string() + " does not exist"); + } + AddExtraDllDirectories(FLM_REAL_CORELIB_EXTRA_DLL_DIRS); + std::cout << "loading " << library.string() << '\n'; + + auto api = CorelibApi::Load(library); + CheckVersionIdentity(api); + CheckDependenciesAndDevice(api); + CheckStreamAndTensorRoundTrip(api); + CheckRopeGatherAndRealUpload(api); + CheckHelperBoundaries(api); + + api->functions().cleanup(); + std::cout << "test_phi4_hardware: PASS\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << "test_phi4_hardware: FAIL: " << error.what() << '\n'; + return 1; + } +} diff --git a/src/test/phi4_corelib_aie4/test_phi4_manifest.cpp b/src/test/phi4_corelib_aie4/test_phi4_manifest.cpp index af525a6c..e8c26952 100644 --- a/src/test/phi4_corelib_aie4/test_phi4_manifest.cpp +++ b/src/test/phi4_corelib_aie4/test_phi4_manifest.cpp @@ -1,4 +1,5 @@ #include "fake_corelib.hpp" +#include "phi4_package_fixture.hpp" #include "test_support.hpp" #include @@ -39,26 +40,15 @@ using flm::phi4::SourceDType; using flm::phi4::WeightObjectKind; using nlohmann::json; -constexpr std::string_view kManifestName = "corelib_phi4_manifest.json"; -constexpr std::string_view kDataFile = "z-data.bin"; -constexpr std::string_view kRopeFile = "z-rope.bin"; -constexpr std::uint64_t kDataBytes = - 200064ull * 3072ull * sizeof(std::uint16_t); -constexpr std::size_t kRopeRows = 4096; -constexpr std::size_t kRopeColumns = 64; -constexpr std::uint64_t kRopeBytes = - kRopeRows * kRopeColumns * sizeof(std::uint16_t); -constexpr std::uint64_t kRopeMappedBytes = kRopeBytes + 4096; -constexpr std::uint64_t kFp32ScaleOffset = 4096; -constexpr std::uint64_t kNormOffset = 1024 * 1024; -constexpr std::uint64_t kSinOffset = 2 * 1024 * 1024; - -constexpr std::string_view kFp32Scale = - "model.layers.0.attn.q_proj.MatMulNBits.scales"; -constexpr std::string_view kFp16Scale = - "model.layers.0.attn.k_proj.MatMulNBits.scales"; -constexpr std::string_view kFp32Norm = - "model.layers.0.post_attention_layernorm.weight"; +using flm::test::phi4fixture::kDataBytes; +using flm::test::phi4fixture::kDataFile; +using flm::test::phi4fixture::kFp16Scale; +using flm::test::phi4fixture::kFp32Norm; +using flm::test::phi4fixture::kFp32Scale; +using flm::test::phi4fixture::kRopeBytes; +using flm::test::phi4fixture::kRopeMappedBytes; +using flm::test::phi4fixture::NoAccessGuard; +using flm::test::phi4fixture::SyntheticPackage; template void* FunctionAddress(Function function) { @@ -75,431 +65,6 @@ std::shared_ptr ResolveRecordingCorelib() { }); } -class TempDirectory final { -public: - TempDirectory() { - const auto nonce = - std::chrono::steady_clock::now().time_since_epoch().count(); - path_ = std::filesystem::temp_directory_path() / - ("fastflowlm-phi4-manifest-" + - std::to_string(GetCurrentProcessId()) + "-" + - std::to_string(nonce)); - std::filesystem::create_directories(path_); - } - - ~TempDirectory() noexcept { - std::error_code error; - std::filesystem::remove_all(path_, error); - } - - TempDirectory(const TempDirectory&) = delete; - TempDirectory& operator=(const TempDirectory&) = delete; - - const std::filesystem::path& path() const noexcept { - return path_; - } - -private: - std::filesystem::path path_; -}; - -void CreateSparseFile( - const std::filesystem::path& path, - std::uint64_t size) { - HANDLE file = CreateFileW( - path.c_str(), - GENERIC_READ | GENERIC_WRITE, - FILE_SHARE_READ, - nullptr, - CREATE_ALWAYS, - FILE_ATTRIBUTE_NORMAL, - nullptr); - if (file == INVALID_HANDLE_VALUE) { - throw std::runtime_error("failed to create sparse test file"); - } - - DWORD ignored = 0; - DeviceIoControl( - file, - FSCTL_SET_SPARSE, - nullptr, - 0, - nullptr, - 0, - &ignored, - nullptr); - LARGE_INTEGER end{}; - end.QuadPart = static_cast(size); - const bool success = - SetFilePointerEx(file, end, nullptr, FILE_BEGIN) != FALSE && - SetEndOfFile(file) != FALSE; - CloseHandle(file); - if (!success) { - throw std::runtime_error("failed to size sparse test file"); - } -} - -template -void WriteValues( - const std::filesystem::path& path, - std::uint64_t offset, - std::span values) { - std::fstream file( - path, - std::ios::in | std::ios::out | std::ios::binary); - if (!file) { - throw std::runtime_error("failed to open synthetic data file"); - } - file.seekp(static_cast(offset)); - file.write( - reinterpret_cast(values.data()), - static_cast(values.size_bytes())); - if (!file) { - throw std::runtime_error("failed to write synthetic data"); - } -} - -std::uint64_t ItemSize(std::string_view dtype) { - if (dtype == "uint8") { - return 1; - } - if (dtype == "float16") { - return 2; - } - if (dtype == "float32") { - return 4; - } - if (dtype == "int64") { - return 8; - } - throw std::runtime_error("unsupported synthetic dtype"); -} - -std::uint64_t ByteLength( - std::string_view dtype, - const std::vector& shape) { - std::uint64_t elements = 1; - for (const std::int64_t dimension : shape) { - elements *= static_cast(dimension); - } - return elements * ItemSize(dtype); -} - -void AddInitializer( - json& initializers, - std::string name, - std::string dtype, - std::vector shape, - std::string role, - std::string file = std::string(kDataFile), - std::uint64_t offset = 0) { - CHECK(!initializers.contains(name)); - const std::uint64_t length = ByteLength(dtype, shape); - initializers[std::move(name)] = { - {"file", std::move(file)}, - {"offset", offset}, - {"length", length}, - {"dtype", std::move(dtype)}, - {"shape", std::move(shape)}, - {"role", std::move(role)}}; -} - -void AddMatMul( - json& manifest, - const std::string& name, - std::int64_t k, - std::int64_t n) { - json roles = { - {"qweight", name + ".qweight"}, - {"scales", name + ".scales"}, - {"qzeros", name + ".qzeros"}}; - manifest["weight_objects"].push_back({ - {"name", name}, - {"kind", "matmul"}, - {"descriptor", - { - {"k", k}, - {"n", n}, - {"group_size", 128}, - {"has_bias", false}, - }}, - {"roles", roles}}); - - auto& initializers = manifest["initializers"]; - AddInitializer( - initializers, - roles["qweight"].get(), - "uint8", - {n, k / 2}, - "matmul.qweight"); - - std::string scale_dtype = "float16"; - std::uint64_t scale_offset = 0; - if (name == "model.layers.0.attn.q_proj.MatMulNBits") { - scale_dtype = "float32"; - scale_offset = kFp32ScaleOffset; - } - AddInitializer( - initializers, - roles["scales"].get(), - scale_dtype, - {n, k / 128}, - "matmul.scales", - std::string(kDataFile), - scale_offset); - AddInitializer( - initializers, - roles["qzeros"].get(), - "uint8", - {n, ((k / 128) + 1) / 2}, - "matmul.qzeros"); -} - -void AddSsMlp(json& manifest, int layer) { - const std::string base = - "model.layers." + std::to_string(layer); - const std::string object_name = base + ".ssmlp"; - const std::string norm0 = - base + ".post_attention_layernorm.weight"; - const std::string norm1 = - layer == 31 - ? "model.layers.32.final_norm_layernorm.weight" - : "model.layers." + std::to_string(layer + 1) + - ".input_layernorm.weight"; - - json roles = { - {"norm0", norm0}, - {"norm1", norm1}, - }; - auto& initializers = manifest["initializers"]; - for (const std::string projection : {"gate", "up", "down"}) { - const std::int64_t k = projection == "down" ? 8192 : 3072; - const std::int64_t n = projection == "down" ? 3072 : 8192; - const std::string prefix = - base + ".mlp." + projection + "_proj.MatMulNBits"; - for (const std::string component : - {"qweight", "scales", "qzeros"}) { - roles[projection + "_" + component] = - prefix + "." + component; - } - AddInitializer( - initializers, - prefix + ".qweight", - "uint8", - {n, k / 2}, - "ssmlp." + projection + ".qweight"); - AddInitializer( - initializers, - prefix + ".scales", - "float16", - {n, k / 128}, - "ssmlp." + projection + ".scales"); - AddInitializer( - initializers, - prefix + ".qzeros", - "uint8", - {n, ((k / 128) + 1) / 2}, - "ssmlp." + projection + ".qzeros"); - } - - AddInitializer( - initializers, - norm0, - layer == 0 ? "float32" : "float16", - {3072}, - "ssmlp.norm0", - std::string(kDataFile), - layer == 0 ? kNormOffset : 0); - AddInitializer( - initializers, - norm1, - "float16", - {3072}, - "ssmlp.norm1"); - - manifest["weight_objects"].push_back({ - {"name", object_name}, - {"kind", "ssmlp"}, - {"descriptor", - { - {"k", 3072}, - {"n", 8192}, - {"group_size", 128}, - }}, - {"roles", std::move(roles)}}); -} - -json BuildManifest(std::uint64_t model_size) { - json manifest = { - {"schema_version", 1}, - {"execution_backend", "corelib_aie4"}, - {"model", - { - {"family", "phi4"}, - {"layers", 32}, - {"hidden_size", 3072}, - {"intermediate_size", 8192}, - {"num_heads", 24}, - {"kv_heads", 8}, - {"head_size", 128}, - {"vocab_size", 200064}, - {"group_size", 128}, - {"rope_dim", 96}, - {"rms_epsilon", 0.00001}, - }}, - {"backend", {{"max_seq", 4096}}}, - {"files", - { - {"model.onnx", {{"size", model_size}}}, - {std::string(kDataFile), {{"size", kDataBytes}}}, - {std::string(kRopeFile), {{"size", kRopeMappedBytes}}}, - }}, - {"initializers", json::object()}, - {"weight_objects", json::array()}, - }; - - for (int layer = 0; layer < 32; ++layer) { - const std::string base = - "model.layers." + std::to_string(layer) + ".attn."; - AddMatMul( - manifest, - base + "q_proj.MatMulNBits", - 3072, - 3072); - AddMatMul( - manifest, - base + "k_proj.MatMulNBits", - 3072, - 1024); - AddMatMul( - manifest, - base + "v_proj.MatMulNBits", - 3072, - 1024); - AddMatMul( - manifest, - base + "o_proj.MatMulNBits", - 3072, - 3072); - AddSsMlp(manifest, layer); - } - AddMatMul(manifest, "lm_head.MatMulNBits", 3072, 200064); - - AddInitializer( - manifest["initializers"], - "model.embed_tokens.weight", - "float16", - {200064, 3072}, - "embedding"); - AddInitializer( - manifest["initializers"], - "model.layers.0.input_layernorm.weight", - "float32", - {3072}, - "input_norm", - std::string(kDataFile), - kNormOffset); - AddInitializer( - manifest["initializers"], - "cos_cache", - "float16", - {4096, 64}, - "cos_cache", - std::string(kRopeFile)); - AddInitializer( - manifest["initializers"], - "sin_cache", - "float32", - {4096, 48}, - "sin_cache", - std::string(kDataFile), - kSinOffset); - - CHECK(manifest["weight_objects"].size() == 161); - CHECK(manifest["initializers"].size() == 743); - return manifest; -} - -class SyntheticPackage final { -public: - SyntheticPackage() { - const auto model_path = temp_.path() / "model.onnx"; - { - std::ofstream model(model_path, std::ios::binary); - model << "model"; - } - CreateSparseFile(temp_.path() / kDataFile, kDataBytes); - CreateSparseFile( - temp_.path() / kRopeFile, - kRopeMappedBytes); - - const std::array half_values{ - 0x3c00u, - 0xc000u}; - const std::array float_values{1.0f, -2.0f}; - WriteValues( - temp_.path() / kDataFile, - 0, - std::span(half_values)); - WriteValues( - temp_.path() / kDataFile, - kFp32ScaleOffset, - std::span(float_values)); - WriteValues( - temp_.path() / kDataFile, - kNormOffset, - std::span(float_values)); - const std::array sin_value{4.0f}; - WriteValues( - temp_.path() / kDataFile, - kSinOffset, - std::span(sin_value)); - - const std::array one{0x3c00u}; - const std::array two{0x4000u}; - const std::array three{0x4200u}; - WriteValues( - temp_.path() / kRopeFile, - 0, - std::span(one)); - WriteValues( - temp_.path() / kRopeFile, - kRopeColumns * sizeof(std::uint16_t), - std::span(two)); - WriteValues( - temp_.path() / kRopeFile, - ((kRopeRows - 1) * kRopeColumns + 47) * - sizeof(std::uint16_t), - std::span(three)); - - manifest_ = BuildManifest( - std::filesystem::file_size(model_path)); - Write(manifest_); - } - - const std::filesystem::path& path() const noexcept { - return temp_.path(); - } - - json manifest() const { - return manifest_; - } - - void Write(const json& manifest) const { - std::ofstream stream( - temp_.path() / kManifestName, - std::ios::binary | std::ios::trunc); - if (!stream) { - throw std::runtime_error("failed to write synthetic manifest"); - } - stream << manifest.dump(2) << '\n'; - } - -private: - TempDirectory temp_; - json manifest_; -}; - void RenameFile( json& manifest, std::string_view old_name, @@ -1017,47 +582,6 @@ void TestOwnedScaleAndNormConversions( "floating"); } -class NoAccessGuard final { -public: - explicit NoAccessGuard(void* address) { - MEMORY_BASIC_INFORMATION region{}; - if ( - VirtualQuery(address, ®ion, sizeof(region)) != - sizeof(region) || - region.State != MEM_COMMIT) { - throw std::runtime_error("failed to query guard-page address"); - } - address_ = address; - if ( - VirtualProtect( - address, - 4096, - PAGE_NOACCESS, - &old_protection_) == FALSE) { - throw std::runtime_error( - "failed to protect no-access guard page"); - } - } - - ~NoAccessGuard() noexcept { - if (address_ != nullptr) { - DWORD ignored = 0; - VirtualProtect( - address_, - 4096, - old_protection_, - &ignored); - } - } - - NoAccessGuard(const NoAccessGuard&) = delete; - NoAccessGuard& operator=(const NoAccessGuard&) = delete; - -private: - void* address_ = nullptr; - DWORD old_protection_ = 0; -}; - // Corelib e5258d2 removed convert_strided, so this slice is now // FastFlow's own code -- which makes the guard page more important, not // less. The last source row sits immediately before an inaccessible page, diff --git a/src/tools/compare_phi4_corelib_e2e.py b/src/tools/compare_phi4_corelib_e2e.py new file mode 100644 index 00000000..fe9cdab8 --- /dev/null +++ b/src/tools/compare_phi4_corelib_e2e.py @@ -0,0 +1,424 @@ +#!/usr/bin/env python3 +"""Task 12 Step 3: compare FastFlow's Phi-4 AIE4 engine against the corelib +reference driver, on the same explicit token IDs and the same forced route. + +Two modes, and they are separate PROCESSES on purpose: + + emit-reference loads the reference driver, runs it, writes JSON, and + calls corelib.cleanup() before exiting. + compare reads two JSON files and holds no device context at all. + +Nothing here ever runs while the C++ harness is running. Two processes holding +AIE4 device contexts at once fail in ways that look like defects, so the suite +script serialises them and this file makes that easy to honour rather than +something to remember. + +Three things about the `e5258d2` reference are worth stating, because each one +would otherwise look like a bug in this comparator: + + * `Phi4.forward` and `Phi4.logits_for` return FP32 arrays directly. Corelib + widens on `tensor_read`, so there is no `from_bf16` unpacking left to + mirror. It is FASTFLOW's BF16 logits that get widened here. + * The reference driver still uses the collapsed two-synchronize-per-layer + schedule. Design Section 10.4 no longer considers that sound and FastFlow + deliberately uses four. Its VALUES are the reference; its SCHEDULE is not, + so synchronize counts are never compared. + * The driver is read-only. No `--continuation-route` option is added to it + and that repository is not modified; the routes are composed here out of + `Phi4.forward` calls. +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from pathlib import Path + +import numpy as np + +# Thresholds from design Section 12.4. They are named rather than inlined so a +# report can quote the number that actually ran. +MIN_CORRELATION = 0.9999 +MAX_TOP32_ABS_DIFF = 0.25 +MIN_DECODE_STEPS = 16 +TOP_K = 32 +TOP_5 = 5 + + +def _import_reference(): + """Import the corelib reference driver from RYZENAI_CORELIB_SOURCE. + + Imported by path rather than vendored: a copy would drift, and the whole + value of this comparison is that the reference is the corelib repository's + own driver rather than a second transcription of the same guess. + """ + source = os.environ.get("RYZENAI_CORELIB_SOURCE") + if not source: + raise SystemExit( + "RYZENAI_CORELIB_SOURCE is not set. Point it at the " + "ryzenai-corelib checkout whose python/ holds phi4_driver.py." + ) + python_dir = Path(source) / "python" + if not (python_dir / "phi4_driver.py").is_file(): + raise SystemExit(f"no phi4_driver.py under {python_dir}") + sys.path.insert(0, str(python_dir)) + import phi4_driver # noqa: E402 + import ryzenai_corelib as corelib # noqa: E402 + + return phi4_driver, corelib + + +def _argmax_lowest(values) -> int: + """The lowest ID among ties. + + `np.argmax` already returns the first maximal index, which for a dense + logit vector IS the lowest ID. It is spelled out because FastFlow's + `ArgmaxLowest` promises the same thing and the comparison below is only + meaningful if both sides use one convention. + """ + return int(np.argmax(np.asarray(values))) + + +def _widen_bf16(bits) -> np.ndarray: + """FastFlow emits raw BF16 bit patterns; the reference is already FP32. + + BF16 to FP32 is a 16-bit left shift and nothing else, so this is exact and + introduces no error of its own into the comparison below. + """ + raw = np.asarray(bits, dtype=np.uint16).astype(np.uint32) << np.uint32(16) + return np.ascontiguousarray(raw).view(np.float32) + + +# --------------------------------------------------------------------------- +# Reference +# --------------------------------------------------------------------------- + + +def _read_live_cache(tensor, position: int, driver) -> list[int]: + """Live rows only, in FastFlow's [head][row][head_size] order. + + The cache is allocated at the full 4096-row window but only [0, position) + has ever been written. Comparing the tail would be comparing uninitialised + device memory on both sides and calling the agreement a result. + """ + if position <= 0: + return [] + head_size = driver.HEAD_SIZE + max_seq = driver.MAX_SEQ + out = [] + for head in range(driver.KV_HEADS): + offset = (head * max_seq) * head_size + raw = tensor.read(position * head_size, offset, driver.DataType.BF16) + out.extend(np.frombuffer(raw, dtype=np.uint16).tolist()) + return out + + +def emit_reference(args) -> int: + driver, corelib = _import_reference() + + if not corelib.load_library().ryzenai_corelib_has_device_context(): + raise SystemExit( + "no AIE4 device context for the reference driver. Check that " + "nothing else is holding one; the C++ harness must not be " + "running concurrently." + ) + + plan = json.loads(Path(args.token_ids_json).read_text(encoding="utf-8")) + if isinstance(plan, list): + prefix, suffix = list(plan), [] + else: + prefix = list(plan["prefix"]) + suffix = list(plan.get("suffix", [])) + + model_onnx = Path(args.model_dir) / "model.onnx" + if not model_onnx.is_file(): + raise SystemExit(f"no model.onnx under {args.model_dir}") + + record = { + "continuation_route": args.continuation_route, + "prefix_ids": prefix, + "suffix_ids": suffix, + } + try: + model = driver.Phi4(model_onnx) + + # Routes composed here, out of Phi4.forward calls, exactly as design + # Section 12.4 describes them. + if args.continuation_route == "force_append": + hidden = model.forward( + model.embed_rows(prefix), len(prefix), 0 + ) + position = len(prefix) + for token in suffix: + hidden = model.forward( + model.embed_rows([token]), 1, position + ) + position += 1 + elif args.continuation_route == "force_reprefill": + # A fresh process is a cleared reference state, and the full + # rendered history is recomputed from position zero in one call. + full = prefix + suffix + hidden = model.forward(model.embed_rows(full), len(full), 0) + position = len(full) + else: + raise SystemExit( + "--continuation-route must be force_append or " + "force_reprefill" + ) + + logits = model.logits_for(hidden) + record["continuation"] = { + "logits": logits.tolist(), + "top1_id": _argmax_lowest(logits), + } + + decode = [] + token = _argmax_lowest(logits) + for _ in range(args.decode_steps): + hidden = model.forward(model.embed_rows([token]), 1, position) + position += 1 + logits = model.logits_for(hidden) + step_top1 = _argmax_lowest(logits) + decode.append( + { + "input_id": token, + "logits": logits.tolist(), + "top1_id": step_top1, + } + ) + token = step_top1 + record["decode"] = decode + record["final_position"] = position + record["final_snapshot"] = { + "position": position, + "layer0_k": _read_live_cache(model.k_cache[0], position, driver), + "layer0_v": _read_live_cache(model.v_cache[0], position, driver), + "layer31_k": _read_live_cache(model.k_cache[-1], position, driver), + "layer31_v": _read_live_cache(model.v_cache[-1], position, driver), + } + finally: + # Released before the C++ harness starts. The suite runs this process + # to completion for exactly this reason. + corelib.cleanup() + + Path(args.output_json).write_text(json.dumps(record), encoding="utf-8") + print(f"reference written to {args.output_json}") + return 0 + + +# --------------------------------------------------------------------------- +# Comparison +# --------------------------------------------------------------------------- + + +class Failures: + def __init__(self) -> None: + self.messages: list[str] = [] + + def check(self, condition: bool, message: str) -> None: + if not condition: + self.messages.append(message) + + def report(self, label: str) -> int: + if not self.messages: + print(f"{label}: PASS") + return 0 + print(f"{label}: FAIL") + for message in self.messages: + print(f" - {message}") + return 1 + + +def _correlation(left: np.ndarray, right: np.ndarray) -> float: + left = left.astype(np.float64) + right = right.astype(np.float64) + left -= left.mean() + right -= right.mean() + denominator = np.linalg.norm(left) * np.linalg.norm(right) + if denominator == 0.0: + return 0.0 + return float(np.dot(left, right) / denominator) + + +def _compare_step( + failures: Failures, + label: str, + mine_bits, + theirs, +) -> None: + mine = _widen_bf16(mine_bits) + reference = np.asarray(theirs, dtype=np.float32) + if mine.shape != reference.shape: + failures.check( + False, + f"{label}: logit vector length {mine.shape} vs " + f"{reference.shape}", + ) + return + + if not np.all(np.isfinite(mine)) or not np.all(np.isfinite(reference)): + failures.check(False, f"{label}: non-finite logits") + return + + correlation = _correlation(mine, reference) + failures.check( + correlation >= MIN_CORRELATION, + f"{label}: correlation {correlation:.8f} < {MIN_CORRELATION}", + ) + + mine_top1 = _argmax_lowest(mine) + reference_top1 = _argmax_lowest(reference) + failures.check( + mine_top1 == reference_top1, + f"{label}: top-1 {mine_top1} vs {reference_top1}", + ) + + # Ties resolve to the lowest ID on both sides. Asserting it directly + # matters because a tie is exactly where two argmax conventions diverge, + # and BF16 logits tie far more often than FP32 ones. + for name, values, chosen in ( + ("fastflow", mine, mine_top1), + ("reference", reference, reference_top1), + ): + tied = np.flatnonzero(values == values[chosen]) + failures.check( + int(tied[0]) == chosen, + f"{label}: {name} argmax {chosen} is not the lowest tied ID " + f"{int(tied[0])}", + ) + + mine_top5 = set(np.argsort(-mine, kind="stable")[:TOP_5].tolist()) + reference_top5 = set( + np.argsort(-reference, kind="stable")[:TOP_5].tolist() + ) + failures.check( + mine_top5 == reference_top5, + f"{label}: top-5 {sorted(mine_top5)} vs {sorted(reference_top5)}", + ) + + union = sorted( + set(np.argsort(-mine, kind="stable")[:TOP_K].tolist()) + | set(np.argsort(-reference, kind="stable")[:TOP_K].tolist()) + ) + index = np.asarray(union, dtype=np.int64) + max_abs = float(np.max(np.abs(mine[index] - reference[index]))) + failures.check( + max_abs <= MAX_TOP32_ABS_DIFF, + f"{label}: max |diff| over the union top-{TOP_K} is {max_abs:.4f} " + f"> {MAX_TOP32_ABS_DIFF}", + ) + + +def compare(args) -> int: + mine = json.loads(Path(args.fastflow_json).read_text(encoding="utf-8")) + theirs = json.loads(Path(args.reference_json).read_text(encoding="utf-8")) + failures = Failures() + + failures.check( + mine["continuation_route"].replace("force_", "") + == theirs["continuation_route"].replace("force_", ""), + f"route {mine['continuation_route']} vs " + f"{theirs['continuation_route']}", + ) + failures.check( + mine["prefix_ids"] == theirs["prefix_ids"] + and mine["suffix_ids"] == theirs["suffix_ids"], + "the two runs did not use the same explicit token IDs", + ) + + _compare_step( + failures, + "continuation", + mine["continuation"]["logits_bf16"], + theirs["continuation"]["logits"], + ) + + steps = min(len(mine["decode"]), len(theirs["decode"])) + failures.check( + steps >= MIN_DECODE_STEPS, + f"only {steps} decode steps compared; design Section 12.4 requires " + f"at least {MIN_DECODE_STEPS}", + ) + for index in range(steps): + mine_step = mine["decode"][index] + reference_step = theirs["decode"][index] + failures.check( + mine_step["input_id"] == reference_step["input_id"], + f"decode[{index}]: fed {mine_step['input_id']} vs " + f"{reference_step['input_id']}", + ) + _compare_step( + failures, + f"decode[{index}]", + mine_step["logits_bf16"], + reference_step["logits"], + ) + + # Live K/V only, and only where both sides reached the same position. + # Comparing beyond `position` would compare uninitialised device memory. + mine_snapshot = mine["final_snapshot"] + reference_snapshot = theirs["final_snapshot"] + failures.check( + mine_snapshot["position"] == reference_snapshot["position"], + f"final position {mine_snapshot['position']} vs " + f"{reference_snapshot['position']}", + ) + for name in ("layer0_k", "layer0_v", "layer31_k", "layer31_v"): + left = _widen_bf16(mine_snapshot[name]) + right = _widen_bf16(reference_snapshot[name]) + if left.shape != right.shape: + failures.check( + False, + f"{name}: live extent {left.shape} vs {right.shape}", + ) + continue + if left.size == 0: + failures.check(False, f"{name}: no live rows to compare") + continue + correlation = _correlation(left, right) + failures.check( + correlation >= MIN_CORRELATION, + f"{name}: live-cache correlation {correlation:.8f} < " + f"{MIN_CORRELATION}", + ) + + # Deliberately NOT compared: synchronize counts. FastFlow uses four + # synchronizes per layer by design and the reference still uses two, so + # equality there would mean FastFlow had regressed to a schedule design + # Section 10.4 rejected. The 129-per-step count is asserted inside the + # C++ harness against FastFlow's own contract instead. + if "final_metrics" in mine: + print( + "fastflow synchronize_count=" + f"{mine['final_metrics']['synchronize_count']} " + "(not compared against the reference: different schedules)" + ) + + return failures.report(f"compare[{mine['continuation_route']}]") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="mode", required=True) + + emit = subparsers.add_parser("emit-reference") + emit.add_argument("--model-dir", required=True) + emit.add_argument("--token-ids-json", required=True) + emit.add_argument("--decode-steps", type=int, default=MIN_DECODE_STEPS) + emit.add_argument("--continuation-route", required=True) + emit.add_argument("--output-json", required=True) + emit.set_defaults(handler=emit_reference) + + check = subparsers.add_parser("compare") + check.add_argument("--fastflow-json", required=True) + check.add_argument("--reference-json", required=True) + check.set_defaults(handler=compare) + + args = parser.parse_args() + return args.handler(args) + + +if __name__ == "__main__": + sys.exit(main()) From 95f6afad3b7f88f9beace20a43d800ed8a2553f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CChi?= Date: Tue, 1 Sep 2026 21:51:43 -0700 Subject: [PATCH 033/117] test: let the hardware suite take the target's build inputs The suite CMakeLists defaults XRT_INCLUDE_DIR and XRT_LIB_DIR to C:/dev paths that exist on the original development box and nowhere else, and its find_path calls are REQUIRED. Passing them through the one entry point keeps the AIE4 target's layout out of session history. Co-Authored-By: Claude Opus 5 (1M context) --- .../phi4_corelib_aie4/run_hardware_suite.ps1 | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/test/phi4_corelib_aie4/run_hardware_suite.ps1 b/src/test/phi4_corelib_aie4/run_hardware_suite.ps1 index 77d5ee99..8eb81dbe 100644 --- a/src/test/phi4_corelib_aie4/run_hardware_suite.ps1 +++ b/src/test/phi4_corelib_aie4/run_hardware_suite.ps1 @@ -35,6 +35,15 @@ param( # repository is never modified. [string]$CorelibSource = $env:RYZENAI_CORELIB_SOURCE, + # Build inputs. These have no usable defaults on the AIE4 target: the + # suite CMakeLists falls back to C:/dev paths that exist on the original + # development box and nowhere else, and its find_path calls are REQUIRED, + # so a wrong guess fails configure loudly rather than building something + # subtly different. + [string]$XrtDir, + [string]$CorelibIncludeDir, + [string]$BoostIncludeDir, + [string]$BuildDir, [string]$Cmake, [string]$Python = "python", @@ -133,6 +142,17 @@ $configureArgs = @( if ($DependencyDir) { $configureArgs += "-DRYZENAI_CORELIB_EXTRA_DLL_DIRS=$DependencyDir" } +if ($XrtDir) { + $XrtDir = (Resolve-Path $XrtDir).Path + $configureArgs += "-DXRT_INCLUDE_DIR=$XrtDir/include" + $configureArgs += "-DXRT_LIB_DIR=$XrtDir/lib" +} +if ($CorelibIncludeDir) { + $configureArgs += "-DRYZENAI_CORELIB_INCLUDE_DIR=$((Resolve-Path $CorelibIncludeDir).Path)" +} +if ($BoostIncludeDir) { + $configureArgs += "-DBOOST_INCLUDE_DIR=$((Resolve-Path $BoostIncludeDir).Path)" +} Invoke-Checked 'configure' $Cmake $configureArgs Invoke-Checked 'build' $Cmake @('--build', $BuildDir, '--config', 'Release') From 71cabefb8ee01da64c9e6085f0eac790ea5d2267 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CChi?= Date: Tue, 1 Sep 2026 21:52:51 -0700 Subject: [PATCH 034/117] fix: keep the hardware suite runnable on Windows PowerShell 5.1 Join-Path's multi-segment form is PowerShell 6+. On the AIE4 target the shell is 5.1, where the extra segment binds positionally and the script dies with a parameter-binding error before doing anything. Co-Authored-By: Claude Opus 5 (1M context) --- src/test/phi4_corelib_aie4/run_hardware_suite.ps1 | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/test/phi4_corelib_aie4/run_hardware_suite.ps1 b/src/test/phi4_corelib_aie4/run_hardware_suite.ps1 index 8eb81dbe..79d2d8c7 100644 --- a/src/test/phi4_corelib_aie4/run_hardware_suite.ps1 +++ b/src/test/phi4_corelib_aie4/run_hardware_suite.ps1 @@ -59,7 +59,11 @@ $ErrorActionPreference = 'Stop' Set-StrictMode -Version Latest $suiteDir = Split-Path -Parent $MyInvocation.MyCommand.Path -$sourceDir = Resolve-Path (Join-Path $suiteDir '..' '..') +# Nested rather than a three-argument Join-Path: Windows PowerShell 5.1 is the +# shell on the AIE4 target and its Join-Path takes only -Path and -ChildPath, +# so the multi-segment form binds the extra segments positionally and fails +# with a parameter-binding error before the script has done anything. +$sourceDir = (Resolve-Path (Join-Path (Join-Path $suiteDir '..') '..')).Path $ran = @() $skipped = @() From 11ea3085a6cb9707969e5f6f5db0958d7835d7d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CChi?= Date: Tue, 1 Sep 2026 21:58:17 -0700 Subject: [PATCH 035/117] fix: stage dyn_bins.dll for a shared DynamicDispatch too Measured on the AIE4 target. The closure derivation looked for dyn_bins.dll only beside ryzenai_corelib.dll, which is where a statically linked DynamicDispatch keeps it. The target links DD as a shared library, so dyn_bins.dll lives beside dyn_dispatch_core.dll and the derived closure omitted 512 MB of precompiled binaries. Nothing failed to load. Every shape query came back "Shape list size: 0" and the first visible symptom was matmul_bf16_weights_create_onnx refusing a (K:3072, N:3072, Gs:128) MatMul at weight-packing time, which reads as an unsupported shape rather than as a missing file. Design 12.3 asked the packaging to handle both linkages; it handled one. Also runs ctest from the chosen cmake's directory. PATH on the target finds Cygwin's ctest, which mangles the Windows --test-dir and reports an empty suite instead of an error. Co-Authored-By: Claude Opus 5 (1M context) --- src/cmake/StageAie4Runtime.cmake | 33 +++++++++++++++---- .../phi4_corelib_aie4/run_hardware_suite.ps1 | 9 ++++- 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/src/cmake/StageAie4Runtime.cmake b/src/cmake/StageAie4Runtime.cmake index 790a57f7..e000b23e 100644 --- a/src/cmake/StageAie4Runtime.cmake +++ b/src/cmake/StageAie4Runtime.cmake @@ -79,17 +79,36 @@ endforeach() list(REMOVE_DUPLICATES _flm_aie4_search_dirs) # `dyn_bins.dll` holds DynamicDispatch's precompiled binaries and is opened by -# name at runtime, so it is never an import and a walker cannot find it. It is -# staged when the selected linkage ships it and omitted when DynamicDispatch is -# statically linked, which is why it is discovered by presence rather than -# demanded unconditionally. +# NAME at runtime, so it is never an import and a dependency walker cannot see +# it. It has to be found by presence, and WHERE it lives depends on the +# DynamicDispatch linkage: +# +# * statically linked DD -- the dev box -- puts it beside +# ryzenai_corelib.dll, because Transaction resolves it against the +# directory of the module that linked DD in; +# * shared DD -- the AIE4 target -- puts it beside dyn_dispatch_core.dll. +# +# Searching only the corelib directory is therefore correct on one box and +# silently wrong on the other. It is silent because nothing fails to load: the +# process starts, and every shape query then comes back "Shape list size: 0", +# which surfaces as `matmul_bf16_weights_create_onnx failed ... not supported +# in this supported shape list` at weight-packing time. That is a long way from +# the missing file, which is why this searches every directory the closure is +# allowed to draw from rather than assuming a linkage. set(_flm_aie4_runtime_loaded "") -foreach(_flm_aie4_name IN ITEMS dyn_bins.dll) - if(EXISTS "${FLM_AIE4_CORELIB_DIR}/${_flm_aie4_name}") +foreach(_flm_aie4_dir IN LISTS _flm_aie4_search_dirs) + if(EXISTS "${_flm_aie4_dir}/dyn_bins.dll") list(APPEND _flm_aie4_runtime_loaded - "${FLM_AIE4_CORELIB_DIR}/${_flm_aie4_name}") + "${_flm_aie4_dir}/dyn_bins.dll") + break() endif() endforeach() +if(NOT _flm_aie4_runtime_loaded) + message(STATUS + "No dyn_bins.dll in any search directory. That is expected only for " + "a DynamicDispatch build that embeds its binaries; if the staged " + "runtime later reports \"Shape list size: 0\", this is why.") +endif() # The Visual C++ runtime is deliberately not staged. `flm.exe` itself imports # MSVCP140/VCRUNTIME140, so the redistributable is already a product-wide diff --git a/src/test/phi4_corelib_aie4/run_hardware_suite.ps1 b/src/test/phi4_corelib_aie4/run_hardware_suite.ps1 index 79d2d8c7..de8e21a4 100644 --- a/src/test/phi4_corelib_aie4/run_hardware_suite.ps1 +++ b/src/test/phi4_corelib_aie4/run_hardware_suite.ps1 @@ -174,8 +174,15 @@ Write-Section 'CTest suite (host tests, real-corelib check, device smoke, fatal # failure -- which is the behaviour that matters on the one machine where this # script is meant to run. $env:FLM_AIE4_HARDWARE = '1' +# The ctest beside the chosen cmake, never the one on PATH. On the AIE4 target +# PATH finds Cygwin's ctest, which reads a Windows -DVALUE --test-dir as a +# POSIX path, silently reports "Test project /cygdrive/c/Users/chiz" and then +# "No tests were found" -- a result that looks like an empty suite rather than +# like a wrong tool. +$ctest = Join-Path (Split-Path -Parent $Cmake) 'ctest.exe' +if (-not (Test-Path $ctest)) { $ctest = 'ctest' } # Serial on purpose: several of these hold an AIE4 device context. -Invoke-Checked 'ctest' 'ctest' @( +Invoke-Checked 'ctest' $ctest @( '--test-dir', $BuildDir, '-C', 'Release', '--output-on-failure', From 2f3efaa7b3874b1ef8e88c1cf65c2a4b2a6516dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9CChi?= Date: Tue, 1 Sep 2026 22:05:18 -0700 Subject: [PATCH 036/117] fix: make the AIE4 catalog and packaging guards survive a fresh clone Two defects the AIE4 target exposed that no development box could. model_list.json records a byte size and SHA-256 for every shipped overlay, and the installer copies those files as they sit in the checkout. Git's default core.autocrlf=true on Windows rewrites them, so a fresh clone of the same commit has different bytes and the catalog describes a file that does not exist there. test_model_catalog failed on exactly that. Marking src/model_overlays as -text makes the recorded hash a property of the commit rather than of the machine that cloned it. test_packaged_runtime invoked bare Usage cmake [options] cmake [options] cmake [options] -S -B Specify a source directory to (re-)generate a build system for it in the current working directory. Specify an existing build directory to re-generate its build system. Run 'cmake --help' for more information.. On the target that is Cygwin's 3.14.5, below the 3.24 StageAie4Runtime.cmake requires, so the closure guard failed for a reason unrelated to the closure. It now takes the CMake that configured the suite, passed as ${CMAKE_COMMAND}. Worse than the failure was its silence: under Usage: pwsh[.exe] [-Login] [[-File] [args]] [-Command { - | [-args ] | [] } ] [-ConfigurationName ] [-CustomPipeName ] [-EncodedCommand ] [-ExecutionPolicy ] [-InputFormat {Text | XML}] [-Interactive] [-MTA] [-NoExit] [-NoLogo] [-NonInteractive] [-NoProfile] [-OutputFormat {Text | XML}] [-SettingsFile ] [-STA] [-Version] [-WindowStyle