diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..29d48445 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,52 @@ +# The AIE4 model overlays are shipped verbatim and their bytes are hashed. +# +# `model_list.json` records a byte size and a SHA-256 for every file under +# `src/model_overlays/`, and the installer copies those files as they sit in +# the checkout. Git's default `core.autocrlf=true` on Windows rewrites text +# files on checkout, so a fresh clone gets different bytes, a different size +# and a different hash from the ones recorded -- and the catalog then describes +# a file that does not exist on that machine. +# +# Measured on the AIE4 target: `test_model_catalog` failed on +# `record.at("size") == FileSize(path)` for the overlay files, on a clean +# checkout of the same commit that passes on a development box whose worktree +# happened to be LF. Nothing about the product was wrong; the checkout was. +# +# `-text` disables all end-of-line conversion for these paths in both +# directions, which is the only setting that makes the recorded hash a property +# of the commit rather than of the machine that cloned it. +src/model_overlays/** -text + +# `corelib_phi4_manifest.json` is 274 KB -- 10,556 lines -- of generated +# per-initializer records, one entry per weight tensor in the ONNX. It is +# written by `tools/generate_phi4_corelib_manifest.py`, consumed by +# `phi4_corelib_manifest.cpp`, and read by no human: reviewing it line by line +# is not a thing anyone does or should do, and at that size it crowds out the +# rest of a diff. +# +# `-diff` makes git treat it as binary for display only. `git diff` reports +# that the file changed without printing its contents, and `git diff --numstat` +# reports `-` instead of a line count. NOTHING about the file's bytes changes: +# the size and SHA-256 recorded in `model_list.json` still describe it, and +# `git log -p -- ` with `--text` still shows the contents when someone +# genuinely needs them. +# +# The alternative considered and rejected was re-serialising it compactly to +# one line. That shrinks the diff the same amount but changes every byte, +# which invalidates the recorded hash and forces a matching edit to +# `model_list.json` -- churn in a checked artefact, to buy a presentation fix. +src/model_overlays/phi4-mini-it-aie4/corelib_phi4_manifest.json -text -diff + +# The same reasoning for the three AIE4 benchmark records. Together they are +# ~5,900 lines of per-sample timing arrays emitted by the benchmark +# instrument. They are NOT dead evidence -- `test_phi4_continuation_calibration` +# reads all three, and they are the fixtures roughly fifteen of that module's +# assertions are written against, so the calibrator is tested against the real +# measurement rather than against invented numbers. They stay, and they stay +# byte-for-byte, because that test is green and depends on their contents. +# +# But nobody reviews a timing array line by line. `-diff` keeps them out of the +# rendered diff without touching a byte. +docs/docs/benchmarks/phi4_aie4_baseline.json -diff +docs/docs/benchmarks/phi4_aie4_baseline_task15_rerun.json -diff +docs/docs/benchmarks/phi4_aie4_crossover_history.json -diff diff --git a/.gitignore b/.gitignore index 8bf22b3e..0db37d18 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,12 @@ -build -build_*/ -.vscode -__pycache__ -_site -.jekyll-metadata -*.csv +build +build_*/ +# Not root-anchored: scratch build trees get dropped next to the sources they +# build, e.g. src/test/phi4_corelib_aie4/.build-task11/. +.build-*/ +/src/build-*/ +.vscode +__pycache__ +_site +.jekyll-metadata +*.csv +/docs/superpowers/ diff --git a/docs/docs/benchmarks/phi4_aie4_baseline.json b/docs/docs/benchmarks/phi4_aie4_baseline.json new file mode 100644 index 00000000..5a919741 --- /dev/null +++ b/docs/docs/benchmarks/phi4_aie4_baseline.json @@ -0,0 +1,2990 @@ +{ + "continuation": { + "crossover": { + "2048": { + "append_wins_up_to": 12, + "bracket_is_tight": false, + "crossover_bracket": [ + 12, + 64 + ], + "decisions": [ + { + "append_p50_ns": 54678400, + "decided": true, + "drift_across_point_ns": 17639700, + "gap_ns": 1106326100, + "gap_over_uncertainty": 62.71796572504068, + "reprefill_p50_ns": 1161004500, + "suffix": 1, + "uncertainty_ns": 17639700, + "winner": "append", + "within_route_spread_ns": 15020600 + }, + { + "append_p50_ns": 104797800, + "decided": true, + "drift_across_point_ns": 20926500, + "gap_ns": 1047481400, + "gap_over_uncertainty": 50.055260076935944, + "reprefill_p50_ns": 1152279200, + "suffix": 2, + "uncertainty_ns": 20926500, + "winner": "append", + "within_route_spread_ns": 20926500 + }, + { + "append_p50_ns": 196408300, + "decided": true, + "drift_across_point_ns": 8498200, + "gap_ns": 959665100, + "gap_over_uncertainty": 112.9256901461486, + "reprefill_p50_ns": 1156073400, + "suffix": 4, + "uncertainty_ns": 8498200, + "winner": "append", + "within_route_spread_ns": 7311000 + }, + { + "append_p50_ns": 370469400, + "decided": true, + "drift_across_point_ns": 11231200, + "gap_ns": 781608900, + "gap_over_uncertainty": 48.65381239612318, + "reprefill_p50_ns": 1152078300, + "suffix": 8, + "uncertainty_ns": 16064700, + "winner": "append", + "within_route_spread_ns": 16064700 + }, + { + "append_p50_ns": 547347800, + "decided": true, + "drift_across_point_ns": 32324300, + "gap_ns": 601326500, + "gap_over_uncertainty": 14.283221932437376, + "reprefill_p50_ns": 1148674300, + "suffix": 12, + "uncertainty_ns": 42100200, + "winner": "append", + "within_route_spread_ns": 42100200 + }, + { + "append_p50_ns": 783355500, + "decided": false, + "drift_across_point_ns": 81245700, + "gap_ns": 374902200, + "gap_over_uncertainty": 0.4980778479549567, + "reprefill_p50_ns": 1158257700, + "suffix": 16, + "uncertainty_ns": 752698000, + "winner": "undecided", + "within_route_spread_ns": 752698000 + }, + { + "append_p50_ns": 1229600900, + "decided": false, + "drift_across_point_ns": 171406900, + "gap_ns": 82340200, + "gap_over_uncertainty": 0.07141318315034581, + "reprefill_p50_ns": 1147260700, + "suffix": 24, + "uncertainty_ns": 1153011200, + "winner": "undecided", + "within_route_spread_ns": 1153011200 + }, + { + "append_p50_ns": 1545251300, + "decided": false, + "drift_across_point_ns": 67346300, + "gap_ns": 399120500, + "gap_over_uncertainty": 0.5791064314221432, + "reprefill_p50_ns": 1146130800, + "suffix": 32, + "uncertainty_ns": 689200600, + "winner": "undecided", + "within_route_spread_ns": 689200600 + }, + { + "append_p50_ns": 4158156400, + "decided": true, + "drift_across_point_ns": 80888300, + "gap_ns": 3001837700, + "gap_over_uncertainty": 6.439819932727917, + "reprefill_p50_ns": 1156318700, + "suffix": 64, + "uncertainty_ns": 466136900, + "winner": "reprefill", + "within_route_spread_ns": 466136900 + }, + { + "append_p50_ns": 5775075800, + "decided": true, + "drift_across_point_ns": 28206600, + "gap_ns": 4624494100, + "gap_over_uncertainty": 20.231562327466047, + "reprefill_p50_ns": 1150581700, + "suffix": 128, + "uncertainty_ns": 228578200, + "winner": "reprefill", + "within_route_spread_ns": 228578200 + }, + { + "append_p50_ns": 11376518800, + "decided": true, + "drift_across_point_ns": 76340700, + "gap_ns": 10215834400, + "gap_over_uncertainty": 6.085364223530441, + "reprefill_p50_ns": 1160684400, + "suffix": 256, + "uncertainty_ns": 1678754800, + "winner": "reprefill", + "within_route_spread_ns": 1678754800 + } + ], + "reprefill_wins_from": 64 + }, + "512": { + "append_wins_up_to": 4, + "bracket_is_tight": false, + "crossover_bracket": [ + 4, + 12 + ], + "decisions": [ + { + "append_p50_ns": 48896600, + "decided": true, + "drift_across_point_ns": 50007300, + "gap_ns": 266819000, + "gap_over_uncertainty": 5.335601002253671, + "reprefill_p50_ns": 315715600, + "suffix": 1, + "uncertainty_ns": 50007300, + "winner": "append", + "within_route_spread_ns": 50007300 + }, + { + "append_p50_ns": 93884400, + "decided": true, + "drift_across_point_ns": 8865200, + "gap_ns": 238085000, + "gap_over_uncertainty": 20.736582646715558, + "reprefill_p50_ns": 331969400, + "suffix": 2, + "uncertainty_ns": 11481400, + "winner": "append", + "within_route_spread_ns": 11481400 + }, + { + "append_p50_ns": 182472300, + "decided": true, + "drift_across_point_ns": 38966100, + "gap_ns": 128621900, + "gap_over_uncertainty": 3.3008666507554003, + "reprefill_p50_ns": 311094200, + "suffix": 4, + "uncertainty_ns": 38966100, + "winner": "append", + "within_route_spread_ns": 38966100 + }, + { + "append_p50_ns": 352560300, + "decided": false, + "drift_across_point_ns": 9537000, + "gap_ns": 18448400, + "gap_over_uncertainty": 0.6281054763971877, + "reprefill_p50_ns": 334111900, + "suffix": 8, + "uncertainty_ns": 29371500, + "winner": "undecided", + "within_route_spread_ns": 29371500 + }, + { + "append_p50_ns": 513162800, + "decided": true, + "drift_across_point_ns": 37012000, + "gap_ns": 214604900, + "gap_over_uncertainty": 5.798251918296769, + "reprefill_p50_ns": 298557900, + "suffix": 12, + "uncertainty_ns": 37012000, + "winner": "reprefill", + "within_route_spread_ns": 37012000 + }, + { + "append_p50_ns": 702220300, + "decided": true, + "drift_across_point_ns": 72983100, + "gap_ns": 384194300, + "gap_over_uncertainty": 5.264154304215634, + "reprefill_p50_ns": 318026000, + "suffix": 16, + "uncertainty_ns": 72983100, + "winner": "reprefill", + "within_route_spread_ns": 72983100 + }, + { + "append_p50_ns": 1006874800, + "decided": true, + "drift_across_point_ns": 49695700, + "gap_ns": 709993900, + "gap_over_uncertainty": 14.286827632974282, + "reprefill_p50_ns": 296880900, + "suffix": 24, + "uncertainty_ns": 49695700, + "winner": "reprefill", + "within_route_spread_ns": 43690500 + }, + { + "append_p50_ns": 1429619200, + "decided": true, + "drift_across_point_ns": 147407100, + "gap_ns": 1121318200, + "gap_over_uncertainty": 7.6069483762993775, + "reprefill_p50_ns": 308301000, + "suffix": 32, + "uncertainty_ns": 147407100, + "winner": "reprefill", + "within_route_spread_ns": 79392000 + }, + { + "append_p50_ns": 2671497000, + "decided": true, + "drift_across_point_ns": 80057900, + "gap_ns": 2370787800, + "gap_over_uncertainty": 29.61341479104498, + "reprefill_p50_ns": 300709200, + "suffix": 64, + "uncertainty_ns": 80057900, + "winner": "reprefill", + "within_route_spread_ns": 79324800 + }, + { + "append_p50_ns": 5301284600, + "decided": true, + "drift_across_point_ns": 113316300, + "gap_ns": 4990078800, + "gap_over_uncertainty": 44.03672551962957, + "reprefill_p50_ns": 311205800, + "suffix": 128, + "uncertainty_ns": 113316300, + "winner": "reprefill", + "within_route_spread_ns": 54891600 + }, + { + "append_p50_ns": 10624495300, + "decided": true, + "drift_across_point_ns": 328146300, + "gap_ns": 10329683100, + "gap_over_uncertainty": 31.478895541409425, + "reprefill_p50_ns": 294812200, + "suffix": 256, + "uncertainty_ns": 328146300, + "winner": "reprefill", + "within_route_spread_ns": 216946200 + } + ], + "reprefill_wins_from": 12 + } + }, + "decision_rule": "append and re-prefill samples are INTERLEAVED, so a machine regime shift moves both routes together; a point is decided only when the gap between the two routes' p50 exceeds BOTH the larger within-point p50-to-p95 spread AND the larger drift between a route's first and last sample at that point. The reported figure is a BRACKET, not a threshold.", + "histories": [ + 512, + 2048 + ], + "points": [ + { + "count": 5, + "drift_ns": 3955800, + "history_rows": 512, + "interleaved_with_reprefill": true, + "max_ns": 52852400, + "mean_ns": 48566460, + "min_ns": 45039000, + "model_steps": 1, + "p50_ns": 48896600, + "p95_ns": 52852400, + "route": "append", + "sample_starts_ns": [ + 863792100, + 1505012500, + 2018531100, + 2513939300, + 3009945800 + ], + "samples_ns": [ + 48896600, + 49336600, + 45039000, + 46707700, + 52852400 + ], + "suffix": 1, + "tokens_per_second": 20.45131972366177 + }, + { + "count": 5, + "drift_ns": 50007300, + "history_rows": 512, + "interleaved_with_append": true, + "max_ns": 365722900, + "mean_ns": 321373580, + "min_ns": 300870800, + "model_steps": 1, + "p50_ns": 315715600, + "p95_ns": 365722900, + "route": "reprefill", + "sample_starts_ns": [ + 912692200, + 1554353400, + 2063573900, + 2560650200, + 3062802400 + ], + "samples_ns": [ + 365722900, + 319500200, + 300870800, + 305058400, + 315715600 + ], + "suffix": 1, + "tokens_per_second": 1624.8801136212464 + }, + { + "count": 5, + "drift_ns": 2335100, + "history_rows": 512, + "interleaved_with_reprefill": true, + "max_ns": 100923600, + "mean_ns": 94630320, + "min_ns": 88875600, + "model_steps": 2, + "p50_ns": 93884400, + "p95_ns": 100923600, + "route": "append", + "sample_starts_ns": [ + 4054492000, + 4623067600, + 5155935400, + 5751720400, + 6303162800 + ], + "samples_ns": [ + 88875600, + 98257300, + 100923600, + 93884400, + 91210700 + ], + "suffix": 2, + "tokens_per_second": 21.302793648359046 + }, + { + "count": 5, + "drift_ns": 8865200, + "history_rows": 512, + "interleaved_with_append": true, + "max_ns": 343450800, + "mean_ns": 321466000, + "min_ns": 286123000, + "model_steps": 1, + "p50_ns": 331969400, + "p95_ns": 343450800, + "route": "reprefill", + "sample_starts_ns": [ + 4143371000, + 4721328200, + 5256867700, + 5845608000, + 6394377900 + ], + "samples_ns": [ + 334585600, + 286123000, + 331969400, + 311201200, + 343450800 + ], + "suffix": 2, + "tokens_per_second": 1548.3354791134363 + }, + { + "count": 5, + "drift_ns": 9159000, + "history_rows": 512, + "interleaved_with_reprefill": true, + "max_ns": 192344900, + "mean_ns": 183776260, + "min_ns": 173313300, + "model_steps": 4, + "p50_ns": 182472300, + "p95_ns": 192344900, + "route": "append", + "sample_starts_ns": [ + 7512889000, + 8192097400, + 8840390400, + 9508564300, + 10257711100 + ], + "samples_ns": [ + 173313300, + 192030000, + 192344900, + 178720800, + 182472300 + ], + "suffix": 4, + "tokens_per_second": 21.921135427130583 + }, + { + "count": 5, + "drift_ns": 38966100, + "history_rows": 512, + "interleaved_with_append": true, + "max_ns": 350060300, + "mean_ns": 317713680, + "min_ns": 283874100, + "model_steps": 1, + "p50_ns": 311094200, + "p95_ns": 350060300, + "route": "reprefill", + "sample_starts_ns": [ + 7686207000, + 8384132600, + 9032738800, + 9687288600, + 10440188300 + ], + "samples_ns": [ + 350060300, + 283874100, + 295499100, + 348040700, + 311094200 + ], + "suffix": 4, + "tokens_per_second": 1658.661588676356 + }, + { + "count": 5, + "drift_ns": 9537000, + "history_rows": 512, + "interleaved_with_reprefill": true, + "max_ns": 381931800, + "mean_ns": 355445300, + "min_ns": 343023300, + "model_steps": 8, + "p50_ns": 352560300, + "p95_ns": 381931800, + "route": "append", + "sample_starts_ns": [ + 11761809100, + 12594605300, + 13487693900, + 14303797000, + 15098573800 + ], + "samples_ns": [ + 352560300, + 355855400, + 381931800, + 343855700, + 343023300 + ], + "suffix": 8, + "tokens_per_second": 22.691153825317258 + }, + { + "count": 5, + "drift_ns": 2136900, + "history_rows": 512, + "interleaved_with_append": true, + "max_ns": 336248800, + "mean_ns": 319573340, + "min_ns": 284507200, + "model_steps": 1, + "p50_ns": 334111900, + "p95_ns": 336248800, + "route": "reprefill", + "sample_starts_ns": [ + 12114375300, + 12950464200, + 13869629400, + 14647656400, + 15441600900 + ], + "samples_ns": [ + 334111900, + 335865800, + 284507200, + 307133000, + 336248800 + ], + "suffix": 8, + "tokens_per_second": 1556.3647987395839 + }, + { + "count": 5, + "drift_ns": 5441300, + "history_rows": 512, + "interleaved_with_reprefill": true, + "max_ns": 530729500, + "mean_ns": 514071000, + "min_ns": 502992400, + "model_steps": 12, + "p50_ns": 513162800, + "p95_ns": 530729500, + "route": "append", + "sample_starts_ns": [ + 17070413200, + 18026153300, + 19010260300, + 19986431700, + 20932035200 + ], + "samples_ns": [ + 514455800, + 502992400, + 530729500, + 513162800, + 509014500 + ], + "suffix": 12, + "tokens_per_second": 23.384391853813256 + }, + { + "count": 5, + "drift_ns": 37012000, + "history_rows": 512, + "interleaved_with_append": true, + "max_ns": 335569900, + "mean_ns": 302945560, + "min_ns": 284066500, + "model_steps": 1, + "p50_ns": 298557900, + "p95_ns": 335569900, + "route": "reprefill", + "sample_starts_ns": [ + 17584872000, + 18529148500, + 19540993600, + 20499598100, + 21441053300 + ], + "samples_ns": [ + 298557900, + 296453200, + 300080300, + 284066500, + 335569900 + ], + "suffix": 12, + "tokens_per_second": 1755.1034489457488 + }, + { + "count": 5, + "drift_ns": 23888200, + "history_rows": 512, + "interleaved_with_reprefill": true, + "max_ns": 727464700, + "mean_ns": 697649180, + "min_ns": 665209700, + "model_steps": 16, + "p50_ns": 702220300, + "p95_ns": 727464700, + "route": "append", + "sample_starts_ns": [ + 23157500400, + 24517183800, + 25658405400, + 26826717700, + 27924977000 + ], + "samples_ns": [ + 726108500, + 665209700, + 727464700, + 667242700, + 702220300 + ], + "suffix": 16, + "tokens_per_second": 22.784872496565537 + }, + { + "count": 5, + "drift_ns": 72983100, + "history_rows": 512, + "interleaved_with_append": true, + "max_ns": 391009100, + "mean_ns": 323518600, + "min_ns": 284167800, + "model_steps": 1, + "p50_ns": 318026000, + "p95_ns": 391009100, + "route": "reprefill", + "sample_starts_ns": [ + 23883618000, + 25182398600, + 26385873000, + 27493963500, + 28627200400 + ], + "samples_ns": [ + 391009100, + 330075600, + 294314500, + 284167800, + 318026000 + ], + "suffix": 16, + "tokens_per_second": 1660.2416154654022 + }, + { + "count": 5, + "drift_ns": 49695700, + "history_rows": 512, + "interleaved_with_reprefill": true, + "max_ns": 1048872500, + "mean_ns": 1004064020, + "min_ns": 969120500, + "model_steps": 24, + "p50_ns": 1006874800, + "p95_ns": 1048872500, + "route": "append", + "sample_starts_ns": [ + 30590163800, + 32134109200, + 33624535500, + 35087549100, + 36496667700 + ], + "samples_ns": [ + 972878300, + 1048872500, + 1006874800, + 969120500, + 1022574000 + ], + "suffix": 24, + "tokens_per_second": 23.836131364098097 + }, + { + "count": 5, + "drift_ns": 43690500, + "history_rows": 512, + "interleaved_with_append": true, + "max_ns": 340571400, + "mean_ns": 304550620, + "min_ns": 292082200, + "model_steps": 1, + "p50_ns": 296880900, + "p95_ns": 340571400, + "route": "reprefill", + "sample_starts_ns": [ + 31563045200, + 33182984700, + 34631413300, + 36056672700, + 37519244800 + ], + "samples_ns": [ + 340571400, + 292082200, + 299986000, + 293232600, + 296880900 + ], + "suffix": 24, + "tokens_per_second": 1805.4378035097577 + }, + { + "count": 5, + "drift_ns": 147407100, + "history_rows": 512, + "interleaved_with_reprefill": true, + "max_ns": 1509011200, + "mean_ns": 1430987480, + "min_ns": 1356833800, + "model_steps": 32, + "p50_ns": 1429619200, + "p95_ns": 1509011200, + "route": "append", + "sample_starts_ns": [ + 39742400800, + 41772052800, + 43680357800, + 45593296400, + 47530854300 + ], + "samples_ns": [ + 1509011200, + 1356833800, + 1429619200, + 1497869100, + 1361604100 + ], + "suffix": 32, + "tokens_per_second": 22.383582984895558 + }, + { + "count": 5, + "drift_ns": 47266800, + "history_rows": 512, + "interleaved_with_append": true, + "max_ns": 335351800, + "mean_ns": 310585460, + "min_ns": 287041400, + "model_steps": 1, + "p50_ns": 308301000, + "p95_ns": 335351800, + "route": "reprefill", + "sample_starts_ns": [ + 41251415000, + 43128889700, + 45109980100, + 47091168800, + 48892463700 + ], + "samples_ns": [ + 335351800, + 334148100, + 308301000, + 287041400, + 288085000 + ], + "suffix": 32, + "tokens_per_second": 1764.5093593598465 + }, + { + "count": 5, + "drift_ns": 80057900, + "history_rows": 512, + "interleaved_with_reprefill": true, + "max_ns": 2750821800, + "mean_ns": 2697386800, + "min_ns": 2667656800, + "model_steps": 64, + "p50_ns": 2671497000, + "p95_ns": 2750821800, + "route": "append", + "sample_starts_ns": [ + 52497531800, + 55630311500, + 58737185500, + 61995762100, + 65110367900 + ], + "samples_ns": [ + 2670763900, + 2671497000, + 2726194500, + 2667656800, + 2750821800 + ], + "suffix": 64, + "tokens_per_second": 23.956605603524913 + }, + { + "count": 5, + "drift_ns": 12722900, + "history_rows": 512, + "interleaved_with_append": true, + "max_ns": 342830600, + "mean_ns": 305728900, + "min_ns": 282030900, + "model_steps": 1, + "p50_ns": 300709200, + "p95_ns": 342830600, + "route": "reprefill", + "sample_starts_ns": [ + 55168300700, + 58301811600, + 61463383200, + 64663422000, + 67861192800 + ], + "samples_ns": [ + 313432100, + 289641700, + 342830600, + 282030900, + 300709200 + ], + "suffix": 64, + "tokens_per_second": 1915.47182460663 + }, + { + "count": 5, + "drift_ns": 113316300, + "history_rows": 512, + "interleaved_with_reprefill": true, + "max_ns": 5356176200, + "mean_ns": 5288077760, + "min_ns": 5210225600, + "model_steps": 128, + "p50_ns": 5301284600, + "p95_ns": 5356176200, + "route": "append", + "sample_starts_ns": [ + 74141382100, + 79838707300, + 85700744300, + 91360247100, + 97230539000 + ], + "samples_ns": [ + 5242859900, + 5301284600, + 5210225600, + 5329842500, + 5356176200 + ], + "suffix": 128, + "tokens_per_second": 24.145091172807437 + }, + { + "count": 5, + "drift_ns": 18709700, + "history_rows": 512, + "interleaved_with_append": true, + "max_ns": 344912200, + "mean_ns": 315466100, + "min_ns": 292496100, + "model_steps": 1, + "p50_ns": 311205800, + "p95_ns": 344912200, + "route": "reprefill", + "sample_starts_ns": [ + 79384246000, + 85139995900, + 90910973400, + 96690093500, + 102586721500 + ], + "samples_ns": [ + 311205800, + 333147700, + 295568700, + 344912200, + 292496100 + ], + "suffix": 128, + "tokens_per_second": 2056.516941522298 + }, + { + "count": 5, + "drift_ns": 328146300, + "history_rows": 512, + "interleaved_with_reprefill": true, + "max_ns": 10841441500, + "mean_ns": 10665783420, + "min_ns": 10513295200, + "model_steps": 256, + "p50_ns": 10624495300, + "p95_ns": 10841441500, + "route": "append", + "sample_starts_ns": [ + 114412449200, + 125378550000, + 136595044800, + 147705130700, + 158715422800 + ], + "samples_ns": [ + 10513295200, + 10769311700, + 10624495300, + 10580373400, + 10841441500 + ], + "suffix": 256, + "tokens_per_second": 24.095262200360708 + }, + { + "count": 5, + "drift_ns": 58047900, + "history_rows": 512, + "interleaved_with_append": true, + "max_ns": 344248900, + "mean_ns": 310183960, + "min_ns": 286201000, + "model_steps": 1, + "p50_ns": 294812200, + "p95_ns": 344248900, + "route": "reprefill", + "sample_starts_ns": [ + 124925748000, + 136147865700, + 147219543500, + 158285507400, + 169556869700 + ], + "samples_ns": [ + 286201000, + 294812200, + 338278800, + 287378900, + 344248900 + ], + "suffix": 256, + "tokens_per_second": 2605.0482307041566 + }, + { + "count": 5, + "drift_ns": 8365300, + "history_rows": 2048, + "interleaved_with_reprefill": true, + "max_ns": 61660200, + "mean_ns": 55855480, + "min_ns": 53294900, + "model_steps": 1, + "p50_ns": 54678400, + "p95_ns": 61660200, + "route": "append", + "sample_starts_ns": [ + 172656755900, + 174666273900, + 176640379000, + 178618480200, + 180588368900 + ], + "samples_ns": [ + 61660200, + 56141200, + 54678400, + 53502700, + 53294900 + ], + "suffix": 1, + "tokens_per_second": 18.288757534968106 + }, + { + "count": 5, + "drift_ns": 17639700, + "history_rows": 2048, + "interleaved_with_append": true, + "max_ns": 1176025100, + "mean_ns": 1164158880, + "min_ns": 1158385400, + "model_steps": 1, + "p50_ns": 1161004500, + "p95_ns": 1176025100, + "route": "reprefill", + "sample_starts_ns": [ + 172718426800, + 174722425100, + 176695068300, + 178671992400, + 180641673200 + ], + "samples_ns": [ + 1176025100, + 1159272200, + 1161004500, + 1166107200, + 1158385400 + ], + "suffix": 1, + "tokens_per_second": 1764.8510406290416 + }, + { + "count": 5, + "drift_ns": 3489800, + "history_rows": 2048, + "interleaved_with_reprefill": true, + "max_ns": 113827000, + "mean_ns": 106060940, + "min_ns": 101586600, + "model_steps": 2, + "p50_ns": 104797800, + "p95_ns": 113827000, + "route": "append", + "sample_starts_ns": [ + 184595128700, + 186615651500, + 188648191900, + 190640040800, + 192650026900 + ], + "samples_ns": [ + 104797800, + 113827000, + 101586600, + 101805700, + 108287600 + ], + "suffix": 2, + "tokens_per_second": 19.084370091738567 + }, + { + "count": 5, + "drift_ns": 20926500, + "history_rows": 2048, + "interleaved_with_append": true, + "max_ns": 1173205700, + "mean_ns": 1155962660, + "min_ns": 1146046100, + "model_steps": 1, + "p50_ns": 1152279200, + "p95_ns": 1173205700, + "route": "reprefill", + "sample_starts_ns": [ + 184699934200, + 186729487000, + 188749786600, + 190741854600, + 192758323100 + ], + "samples_ns": [ + 1152279200, + 1150895600, + 1146046100, + 1157386700, + 1173205700 + ], + "suffix": 2, + "tokens_per_second": 1779.082708426916 + }, + { + "count": 5, + "drift_ns": 8498200, + "history_rows": 2048, + "interleaved_with_reprefill": true, + "max_ns": 203719300, + "mean_ns": 198083700, + "min_ns": 193027000, + "model_steps": 4, + "p50_ns": 196408300, + "p95_ns": 203719300, + "route": "append", + "sample_starts_ns": [ + 196806005700, + 198914102700, + 201006604300, + 203111200600, + 205240884300 + ], + "samples_ns": [ + 203719300, + 193027000, + 202042800, + 196408300, + 195221100 + ], + "suffix": 4, + "tokens_per_second": 20.36573810780909 + }, + { + "count": 5, + "drift_ns": 8373400, + "history_rows": 2048, + "interleaved_with_append": true, + "max_ns": 1162108600, + "mean_ns": 1153685820, + "min_ns": 1146437900, + "model_steps": 1, + "p50_ns": 1156073400, + "p95_ns": 1162108600, + "route": "reprefill", + "sample_starts_ns": [ + 197009732800, + 199107142700, + 201208654600, + 203307615700, + 205436112100 + ], + "samples_ns": [ + 1147700000, + 1146437900, + 1156109200, + 1162108600, + 1156073400 + ], + "suffix": 4, + "tokens_per_second": 1774.9738035664518 + }, + { + "count": 5, + "drift_ns": 11231200, + "history_rows": 2048, + "interleaved_with_reprefill": true, + "max_ns": 386534100, + "mean_ns": 371713440, + "min_ns": 358359900, + "model_steps": 8, + "p50_ns": 370469400, + "p95_ns": 386534100, + "route": "append", + "sample_starts_ns": [ + 209639292800, + 211903138900, + 214168676000, + 216464269100, + 218744640200 + ], + "samples_ns": [ + 369591100, + 373612700, + 386534100, + 370469400, + 358359900 + ], + "suffix": 8, + "tokens_per_second": 21.594226135815806 + }, + { + "count": 5, + "drift_ns": 4218200, + "history_rows": 2048, + "interleaved_with_append": true, + "max_ns": 1157720600, + "mean_ns": 1150808740, + "min_ns": 1141688400, + "model_steps": 1, + "p50_ns": 1152078300, + "p95_ns": 1157720600, + "route": "reprefill", + "sample_starts_ns": [ + 210008890800, + 212276758500, + 214555216500, + 216834744500, + 219103005900 + ], + "samples_ns": [ + 1149169100, + 1141688400, + 1152078300, + 1157720600, + 1153387300 + ], + "suffix": 8, + "tokens_per_second": 1784.6009251280925 + }, + { + "count": 5, + "drift_ns": 32324300, + "history_rows": 2048, + "interleaved_with_reprefill": true, + "max_ns": 589448000, + "mean_ns": 555860400, + "min_ns": 542346100, + "model_steps": 12, + "p50_ns": 547347800, + "p95_ns": 589448000, + "route": "append", + "sample_starts_ns": [ + 223596189200, + 226078230700, + 228507479500, + 230942179100, + 233387072700 + ], + "samples_ns": [ + 557123700, + 542346100, + 547347800, + 543036400, + 589448000 + ], + "suffix": 12, + "tokens_per_second": 21.92390286395597 + }, + { + "count": 5, + "drift_ns": 7335000, + "history_rows": 2048, + "interleaved_with_append": true, + "max_ns": 1156009300, + "mean_ns": 1149120260, + "min_ns": 1143513000, + "model_steps": 1, + "p50_ns": 1148674300, + "p95_ns": 1156009300, + "route": "reprefill", + "sample_starts_ns": [ + 224153319500, + 226620583400, + 229054833600, + 231485226300, + 233976527800 + ], + "samples_ns": [ + 1148674300, + 1147743100, + 1143513000, + 1149661600, + 1156009300 + ], + "suffix": 12, + "tokens_per_second": 1793.3717155507004 + }, + { + "count": 5, + "drift_ns": 81245700, + "history_rows": 2048, + "interleaved_with_reprefill": true, + "max_ns": 1536053500, + "mean_ns": 978267820, + "min_ns": 702109800, + "model_steps": 16, + "p50_ns": 783355500, + "p95_ns": 1536053500, + "route": "append", + "sample_starts_ns": [ + 238527478400, + 241134241100, + 243809492900, + 246878639300, + 250331068900 + ], + "samples_ns": [ + 702109800, + 763689100, + 1106131200, + 1536053500, + 783355500 + ], + "suffix": 16, + "tokens_per_second": 20.424953932154686 + }, + { + "count": 5, + "drift_ns": 1927100, + "history_rows": 2048, + "interleaved_with_append": true, + "max_ns": 1190825500, + "mean_ns": 1163942620, + "min_ns": 1154446800, + "model_steps": 1, + "p50_ns": 1158257700, + "p95_ns": 1190825500, + "route": "reprefill", + "sample_starts_ns": [ + 239229598600, + 241897946400, + 244915643000, + 248414699700, + 251114431400 + ], + "samples_ns": [ + 1156330600, + 1159852500, + 1190825500, + 1154446800, + 1158257700 + ], + "suffix": 16, + "tokens_per_second": 1781.986858364939 + }, + { + "count": 5, + "drift_ns": 171406900, + "history_rows": 2048, + "interleaved_with_reprefill": true, + "max_ns": 2382612100, + "mean_ns": 1415788800, + "min_ns": 1063457300, + "model_steps": 24, + "p50_ns": 1229600900, + "p95_ns": 2382612100, + "route": "append", + "sample_starts_ns": [ + 255996470500, + 259015269300, + 261984065700, + 265149423500, + 269416755700 + ], + "samples_ns": [ + 1115933400, + 1063457300, + 1229600900, + 2382612100, + 1287340300 + ], + "suffix": 24, + "tokens_per_second": 19.518528328988697 + }, + { + "count": 5, + "drift_ns": 13417600, + "history_rows": 2048, + "interleaved_with_append": true, + "max_ns": 1160678300, + "mean_ns": 1147002180, + "min_ns": 1135883000, + "model_steps": 1, + "p50_ns": 1147260700, + "p95_ns": 1160678300, + "route": "reprefill", + "sample_starts_ns": [ + 257112410200, + 260078732800, + 263213677300, + 267532041900, + 270704103500 + ], + "samples_ns": [ + 1147260700, + 1152262600, + 1135883000, + 1138926300, + 1160678300 + ], + "suffix": 24, + "tokens_per_second": 1806.0411203835363 + }, + { + "count": 5, + "drift_ns": 67346300, + "history_rows": 2048, + "interleaved_with_reprefill": true, + "max_ns": 2234451900, + "mean_ns": 1650273820, + "min_ns": 1448365100, + "model_steps": 32, + "p50_ns": 1545251300, + "p95_ns": 2234451900, + "route": "append", + "sample_starts_ns": [ + 275972960400, + 279380372200, + 282824571500, + 286191778000, + 290344470500 + ], + "samples_ns": [ + 1477905000, + 1545395800, + 1448365100, + 2234451900, + 1545251300 + ], + "suffix": 32, + "tokens_per_second": 20.70860577823167 + }, + { + "count": 5, + "drift_ns": 17253500, + "history_rows": 2048, + "interleaved_with_append": true, + "max_ns": 1173369800, + "mean_ns": 1148691300, + "min_ns": 1126689800, + "model_steps": 1, + "p50_ns": 1146130800, + "p95_ns": 1173369800, + "route": "reprefill", + "sample_starts_ns": [ + 277450872000, + 280925779200, + 284272943400, + 288426237700, + 291889732700 + ], + "samples_ns": [ + 1140006300, + 1146130800, + 1173369800, + 1126689800, + 1157259800 + ], + "suffix": 32, + "tokens_per_second": 1814.8015915809958 + }, + { + "count": 5, + "drift_ns": 80888300, + "history_rows": 2048, + "interleaved_with_reprefill": true, + "max_ns": 4624293300, + "mean_ns": 4187460900, + "min_ns": 3763627000, + "model_steps": 64, + "p50_ns": 4158156400, + "p95_ns": 4624293300, + "route": "append", + "sample_starts_ns": [ + 298600636200, + 304631848700, + 311130230000, + 317352880500, + 322994488100 + ], + "samples_ns": [ + 4158156400, + 4624293300, + 4313959700, + 3763627000, + 4077268100 + ], + "suffix": 64, + "tokens_per_second": 15.391436454867355 + }, + { + "count": 5, + "drift_ns": 19609500, + "history_rows": 2048, + "interleaved_with_append": true, + "max_ns": 1175928200, + "mean_ns": 1160704400, + "min_ns": 1143544900, + "model_steps": 1, + "p50_ns": 1156318700, + "p95_ns": 1175928200, + "route": "reprefill", + "sample_starts_ns": [ + 302758802500, + 309256161000, + 315444196200, + 321116514000, + 327071772300 + ], + "samples_ns": [ + 1156318700, + 1155181000, + 1172549200, + 1143544900, + 1175928200 + ], + "suffix": 64, + "tokens_per_second": 1826.4860717032423 + }, + { + "count": 5, + "drift_ns": 28206600, + "history_rows": 2048, + "interleaved_with_reprefill": true, + "max_ns": 6003654000, + "mean_ns": 5802587620, + "min_ns": 5676571500, + "model_steps": 128, + "p50_ns": 5775075800, + "p95_ns": 6003654000, + "route": "append", + "sample_starts_ns": [ + 336689443300, + 344279960200, + 352037111700, + 359944297200, + 367661084300 + ], + "samples_ns": [ + 5676571500, + 5852858700, + 6003654000, + 5775075800, + 5704778100 + ], + "suffix": 128, + "tokens_per_second": 22.164211247235922 + }, + { + "count": 5, + "drift_ns": 2442300, + "history_rows": 2048, + "interleaved_with_append": true, + "max_ns": 1180523700, + "mean_ns": 1156093960, + "min_ns": 1148139400, + "model_steps": 1, + "p50_ns": 1150581700, + "p95_ns": 1180523700, + "route": "reprefill", + "sample_starts_ns": [ + 342366025300, + 350132825500, + 358040771700, + 365719380600, + 373365868900 + ], + "samples_ns": [ + 1150581700, + 1148525400, + 1152699600, + 1180523700, + 1148139400 + ], + "suffix": 128, + "tokens_per_second": 1891.2172860041142 + }, + { + "count": 5, + "drift_ns": 76340700, + "history_rows": 2048, + "interleaved_with_reprefill": true, + "max_ns": 13055273600, + "mean_ns": 11704220960, + "min_ns": 11211844300, + "model_steps": 256, + "p50_ns": 11376518800, + "p95_ns": 13055273600, + "route": "append", + "sample_starts_ns": [ + 388547989900, + 401754591700, + 415302760100, + 430379768000, + 443665786500 + ], + "samples_ns": [ + 11288185000, + 11589283100, + 13055273600, + 11376518800, + 11211844300 + ], + "suffix": 256, + "tokens_per_second": 22.502489953253537 + }, + { + "count": 5, + "drift_ns": 10665200, + "history_rows": 2048, + "interleaved_with_append": true, + "max_ns": 1245193100, + "mean_ns": 1174774020, + "min_ns": 1143214000, + "model_steps": 1, + "p50_ns": 1160684400, + "p95_ns": 1245193100, + "route": "reprefill", + "sample_starts_ns": [ + 399836203400, + 413343881900, + 428358050000, + 441756298400, + 454877638000 + ], + "samples_ns": [ + 1143214000, + 1170899400, + 1245193100, + 1160684400, + 1153879200 + ], + "suffix": 256, + "tokens_per_second": 1985.0357254736946 + } + ], + "prefix_monotonic": true, + "samples_interleaved": true, + "suffixes": [ + 1, + 2, + 4, + 8, + 12, + 16, + 24, + 32, + 64, + 128, + 256 + ], + "warm_samples_per_point": 5 + }, + "decode": { + "runs": [ + { + "count": 128, + "device_tensor_creates_after_warmup": 0, + "dispatches_per_pass": 193, + "live_corelib_object_delta": 0, + "max_ns": 125699800, + "mean_ns": 48270408, + "min_ns": 37934200, + "p50_ns": 41101000, + "p95_ns": 91776300, + "samples_ns": [ + 47523800, + 44188800, + 42167700, + 41646700, + 51028800, + 45601300, + 53800000, + 41487600, + 40241200, + 40491400, + 49619100, + 46696100, + 45238800, + 40392600, + 39297700, + 40039800, + 46941400, + 41101000, + 40250400, + 40621500, + 39395200, + 40312200, + 49301800, + 43638800, + 41099800, + 40829200, + 40725900, + 40437400, + 41600900, + 125699800, + 125358700, + 74580000, + 87425700, + 81792700, + 62778000, + 91776300, + 89377600, + 100162200, + 42896200, + 41529600, + 55192700, + 40964400, + 40220200, + 39961500, + 57007900, + 94846500, + 101710100, + 95298200, + 51341500, + 49590700, + 64634200, + 42285800, + 45175600, + 41316600, + 40780800, + 40379700, + 40046800, + 72608400, + 48784100, + 50861100, + 41807300, + 41704100, + 41458600, + 44709100, + 41741900, + 73347300, + 41607300, + 40172400, + 41724800, + 38524100, + 40382600, + 39742600, + 39752400, + 38585300, + 43943400, + 39865700, + 40989400, + 39635000, + 38768800, + 39881700, + 40284300, + 42800800, + 38964300, + 37934200, + 39113800, + 39573400, + 42069800, + 39418500, + 48646200, + 40703200, + 39789400, + 38799500, + 53675000, + 44013100, + 39911200, + 40392900, + 40214300, + 39005500, + 40714100, + 39417000, + 41764000, + 38272400, + 51225200, + 40903500, + 40091100, + 40512700, + 39689900, + 39653100, + 39035300, + 40812700, + 54216700, + 51640100, + 73258800, + 41932500, + 41462200, + 45119900, + 39856200, + 40201800, + 40163000, + 38705600, + 40674300, + 39669000, + 44135500, + 40160600, + 38583400, + 39829600, + 39275900, + 51882500 + ], + "start_context": 128, + "synchronizes_per_pass": 129, + "tokens": 128, + "tokens_per_second": 20.035539603648502, + "total_ns": 6388647500, + "v_bytes": 16777216, + "v_scatter_ns": 8271300, + "weight_creates_after_warmup": 0 + }, + { + "count": 128, + "device_tensor_creates_after_warmup": 0, + "dispatches_per_pass": 193, + "live_corelib_object_delta": 0, + "max_ns": 151984200, + "mean_ns": 45700515, + "min_ns": 38596000, + "p50_ns": 41736100, + "p95_ns": 54805900, + "samples_ns": [ + 53431800, + 48541200, + 45575700, + 41734800, + 41683000, + 44640700, + 41568000, + 39893500, + 40339700, + 40842700, + 40546100, + 41024200, + 51780500, + 45587200, + 41526500, + 41121700, + 38596000, + 40378900, + 42138900, + 44959800, + 40821900, + 40473200, + 42027400, + 41209400, + 43782300, + 41621400, + 74343800, + 41661600, + 46580500, + 45818600, + 42002500, + 40118000, + 44431600, + 45354300, + 41253600, + 40394300, + 40190100, + 39828700, + 39504100, + 40321200, + 39454800, + 43313200, + 40931900, + 40764300, + 40452000, + 54028600, + 41736100, + 44713400, + 44772200, + 40939000, + 41273200, + 40947300, + 44317900, + 41623100, + 146525700, + 151984200, + 72646200, + 54805900, + 48703000, + 43869000, + 41844600, + 45210400, + 49115900, + 41151200, + 42453300, + 79269800, + 44292400, + 42643400, + 42732400, + 42805700, + 41383200, + 40274500, + 41941100, + 53696900, + 41023500, + 48462600, + 41837600, + 39943000, + 40135200, + 40956300, + 44123900, + 40349300, + 44154800, + 44201800, + 40965200, + 43495800, + 44378800, + 42533600, + 40711400, + 41355100, + 54618600, + 43529700, + 41398000, + 40959300, + 41009900, + 44705300, + 40890900, + 41538500, + 40743900, + 42862600, + 41182900, + 41417700, + 39435800, + 47134200, + 41561000, + 44013900, + 44015500, + 50790400, + 44384700, + 42235900, + 51012900, + 71842400, + 42971800, + 50631400, + 42681000, + 41904600, + 40851000, + 41628400, + 40309800, + 40812500, + 43878300, + 40920900, + 40754600, + 40827500, + 40722700, + 44201500, + 41276300, + 41116600 + ], + "start_context": 512, + "synchronizes_per_pass": 129, + "tokens": 128, + "tokens_per_second": 21.159218707090076, + "total_ns": 6049372700, + "v_bytes": 16777216, + "v_scatter_ns": 6532100, + "weight_creates_after_warmup": 0 + }, + { + "count": 128, + "device_tensor_creates_after_warmup": 0, + "dispatches_per_pass": 193, + "live_corelib_object_delta": 0, + "max_ns": 92449400, + "mean_ns": 46783816, + "min_ns": 42066600, + "p50_ns": 44157700, + "p95_ns": 55775000, + "samples_ns": [ + 58253600, + 51834500, + 50947200, + 57298500, + 55775000, + 44980400, + 47239200, + 44085900, + 43449800, + 44698100, + 92449400, + 68946500, + 51860200, + 48896500, + 46012200, + 44060300, + 52255500, + 44487700, + 44119100, + 46983600, + 43913300, + 43995100, + 46116200, + 55116600, + 48161300, + 45325600, + 48001900, + 44032200, + 44683100, + 43757300, + 44479900, + 43842400, + 44134800, + 44180000, + 43190900, + 43989900, + 42131800, + 43134800, + 43479000, + 43166100, + 42411300, + 42066600, + 42824800, + 50639900, + 43548600, + 42197700, + 43682600, + 42636900, + 53215500, + 47643100, + 42796800, + 43914200, + 80912000, + 49567200, + 44187700, + 44461300, + 43532400, + 43733100, + 44577100, + 43924900, + 43693800, + 46512700, + 43453900, + 43479400, + 44005100, + 47432000, + 47129300, + 44565700, + 44068400, + 44308700, + 42751100, + 48039000, + 42436600, + 42915800, + 50170200, + 44402000, + 42716300, + 46498900, + 43859200, + 42682800, + 43475600, + 43422900, + 42359700, + 42115600, + 42247200, + 43434300, + 44980400, + 44287900, + 47139700, + 45234100, + 43588200, + 54976500, + 44745300, + 44605300, + 43929100, + 46150400, + 88719100, + 49580600, + 53727700, + 45097200, + 44249800, + 53097300, + 44524600, + 43871100, + 42691400, + 43984400, + 48305300, + 44157700, + 43382000, + 53576800, + 43537700, + 50355800, + 44568700, + 46861000, + 43992000, + 42778000, + 47896600, + 43959900, + 43215000, + 46625500, + 43935600, + 43372600, + 51905900, + 42686100, + 43545400, + 42540000, + 42668700, + 43166800 + ], + "start_context": 2048, + "synchronizes_per_pass": 129, + "tokens": 128, + "tokens_per_second": 20.679732800403695, + "total_ns": 6189635100, + "v_bytes": 16777216, + "v_scatter_ns": 6011700, + "weight_creates_after_warmup": 0 + } + ] + }, + "identity": { + "corelib_compiled_version": "0.1.0", + "corelib_dll_path": "C:\\Users\\chiz\\work\\aie4-runtime-mirrored\\ryzenai_corelib.dll", + "corelib_dll_sha256": "a523b23837c9bea3b06f8f8ae74e1e8292092eb73aaa93791022c448b865ecdb", + "corelib_source_revision": "e5258d29b5cb979d4a538994409b90ceff6e6e7a-untracked-only", + "corelib_version": "0.1.0", + "cpu_sku": "AMD Eng Sample: 100-000001713-33_N", + "dynamic_dispatch_version": "9999.0.0.0 (C:\\Users\\chiz\\work\\aie4-runtime-mirrored\\dyn_dispatch_core.dll)", + "fastflow_revision": "a02a2cf7e6cda62ab21e1383de2674b933ed2b74-untracked-only", + "machine": "xcomedusad-43", + "model_dir": "C:\\Users\\chiz\\work\\FastFlowLM\\src\\build\\phi4-hardware\\model", + "model_files": { + "corelib_phi4_manifest.json": { + "sha256": "09cee6efafc513a2048c89e75b40d92d096144f0eed3b2459c138032b69dc045" + }, + "model.onnx": { + "sha256": "e80b9d83e018784eda6263d09fa2ab7729087722c6073c72123366f2dbec4529", + "size": 378325 + }, + "model.onnx.data": { + "sha256": "c48fd647beb02866e68d6dd1fbc05809a439283501d8abb7f6d6b950f92b7b60", + "size": 3248488448 + } + }, + "model_hash_verified": true, + "model_hash_verify_ns": 22197780100, + "model_sha256": "d6f503a9ea142c8b6320313d6ae341a88049b1b8ef01e641b2313fe42cdc7309", + "npu_driver_version": "32.0.20214.4161", + "npu_sku": "AMD XDNA(TM) NPU", + "prompt_id": "ff6a21dcdcff126f", + "prompt_source": "C:\\Users\\chiz\\work\\FastFlowLM\\src\\test\\phi4_corelib_aie4\\phi4_tokens.json", + "ryzen_mm_version": "0.10.2.0 (C:\\Users\\chiz\\work\\aie4-runtime-mirrored\\ryzen_mm.dll)", + "utc": "2026-09-02T16:32:18Z", + "xrt_version": "32.0.20214.4161 (C:\\Windows\\SYSTEM32\\xrt_coreutil.dll)" + }, + "memory": { + "device_tensor_creates_after_warmup": 0, + "device_tensors_created_after_load": 0, + "device_tensors_created_total": 76, + "embedding_bytes": 1229193216, + "kv_bytes": 536870912, + "live_corelib_object_delta": 0, + "live_corelib_objects_after_load": 238, + "live_corelib_objects_now": 238, + "mapped_source_bytes": 3248488448, + "packed_weight_bytes": 2098112512, + "peak_private_bytes": 8235696128, + "peak_working_set_bytes": 10226159616, + "private_bytes_at_end": 8051359744, + "private_bytes_at_warmup": 8066838528, + "private_bytes_before_load": 13594624, + "private_bytes_delta": -15478784, + "private_bytes_growth": 0, + "private_bytes_slope_exact": -51419.8623793319, + "private_bytes_slope_per_token": -51419, + "samples": [ + { + "private_bytes": 8066838528, + "working_set_bytes": 10121515008 + }, + { + "private_bytes": 8066838528, + "working_set_bytes": 10121523200 + }, + { + "private_bytes": 8066838528, + "working_set_bytes": 10121523200 + }, + { + "private_bytes": 8066838528, + "working_set_bytes": 10121523200 + }, + { + "private_bytes": 8066838528, + "working_set_bytes": 10121523200 + }, + { + "private_bytes": 8066838528, + "working_set_bytes": 10121523200 + }, + { + "private_bytes": 8066838528, + "working_set_bytes": 10121523200 + }, + { + "private_bytes": 8066838528, + "working_set_bytes": 10121523200 + }, + { + "private_bytes": 8066838528, + "working_set_bytes": 10121523200 + }, + { + "private_bytes": 8066838528, + "working_set_bytes": 10121523200 + }, + { + "private_bytes": 8066838528, + "working_set_bytes": 10121531392 + }, + { + "private_bytes": 8066838528, + "working_set_bytes": 10121531392 + }, + { + "private_bytes": 8066838528, + "working_set_bytes": 10121539584 + }, + { + "private_bytes": 8066838528, + "working_set_bytes": 10121539584 + }, + { + "private_bytes": 8066838528, + "working_set_bytes": 10121539584 + }, + { + "private_bytes": 8066838528, + "working_set_bytes": 10121539584 + }, + { + "private_bytes": 8066838528, + "working_set_bytes": 10121539584 + }, + { + "private_bytes": 8066838528, + "working_set_bytes": 10121539584 + }, + { + "private_bytes": 8066838528, + "working_set_bytes": 10121539584 + }, + { + "private_bytes": 8066838528, + "working_set_bytes": 10121539584 + }, + { + "private_bytes": 8066838528, + "working_set_bytes": 10121547776 + }, + { + "private_bytes": 8066838528, + "working_set_bytes": 10121547776 + }, + { + "private_bytes": 8066838528, + "working_set_bytes": 10121555968 + }, + { + "private_bytes": 8066838528, + "working_set_bytes": 10121555968 + }, + { + "private_bytes": 8066838528, + "working_set_bytes": 10121555968 + }, + { + "private_bytes": 8066838528, + "working_set_bytes": 10121555968 + }, + { + "private_bytes": 8066838528, + "working_set_bytes": 10121555968 + }, + { + "private_bytes": 8066838528, + "working_set_bytes": 10121564160 + }, + { + "private_bytes": 8066838528, + "working_set_bytes": 10121572352 + }, + { + "private_bytes": 8066838528, + "working_set_bytes": 10121572352 + }, + { + "private_bytes": 8066838528, + "working_set_bytes": 10121580544 + }, + { + "private_bytes": 8066838528, + "working_set_bytes": 10121588736 + }, + { + "private_bytes": 8066838528, + "working_set_bytes": 10121588736 + }, + { + "private_bytes": 8066838528, + "working_set_bytes": 10121588736 + }, + { + "private_bytes": 8066838528, + "working_set_bytes": 10121588736 + }, + { + "private_bytes": 8066838528, + "working_set_bytes": 10121588736 + }, + { + "private_bytes": 8066838528, + "working_set_bytes": 10121588736 + }, + { + "private_bytes": 8066838528, + "working_set_bytes": 10121588736 + }, + { + "private_bytes": 8066838528, + "working_set_bytes": 10121588736 + }, + { + "private_bytes": 8066838528, + "working_set_bytes": 10121588736 + }, + { + "private_bytes": 8066838528, + "working_set_bytes": 10121588736 + }, + { + "private_bytes": 8066838528, + "working_set_bytes": 10121588736 + }, + { + "private_bytes": 8066838528, + "working_set_bytes": 10121596928 + }, + { + "private_bytes": 8066838528, + "working_set_bytes": 10121605120 + }, + { + "private_bytes": 8066838528, + "working_set_bytes": 10121613312 + }, + { + "private_bytes": 8066838528, + "working_set_bytes": 10121613312 + }, + { + "private_bytes": 8066838528, + "working_set_bytes": 10121621504 + }, + { + "private_bytes": 8066838528, + "working_set_bytes": 10121629696 + }, + { + "private_bytes": 8064471040, + "working_set_bytes": 10121629696 + }, + { + "private_bytes": 8064471040, + "working_set_bytes": 10121629696 + }, + { + "private_bytes": 8064471040, + "working_set_bytes": 10121629696 + }, + { + "private_bytes": 8064471040, + "working_set_bytes": 10121629696 + }, + { + "private_bytes": 8064471040, + "working_set_bytes": 10121637888 + }, + { + "private_bytes": 8064471040, + "working_set_bytes": 10121646080 + }, + { + "private_bytes": 8064471040, + "working_set_bytes": 10121654272 + }, + { + "private_bytes": 8064471040, + "working_set_bytes": 10121654272 + }, + { + "private_bytes": 8064471040, + "working_set_bytes": 10121654272 + }, + { + "private_bytes": 8064471040, + "working_set_bytes": 10121654272 + }, + { + "private_bytes": 8064471040, + "working_set_bytes": 10121654272 + }, + { + "private_bytes": 8064471040, + "working_set_bytes": 10121654272 + }, + { + "private_bytes": 8064471040, + "working_set_bytes": 10121662464 + }, + { + "private_bytes": 8064471040, + "working_set_bytes": 10121662464 + }, + { + "private_bytes": 8064471040, + "working_set_bytes": 10121662464 + }, + { + "private_bytes": 8064471040, + "working_set_bytes": 10121670656 + }, + { + "private_bytes": 8064471040, + "working_set_bytes": 10121670656 + }, + { + "private_bytes": 8064471040, + "working_set_bytes": 10121678848 + }, + { + "private_bytes": 8064471040, + "working_set_bytes": 10121678848 + }, + { + "private_bytes": 8064471040, + "working_set_bytes": 10121678848 + }, + { + "private_bytes": 8064471040, + "working_set_bytes": 10121678848 + }, + { + "private_bytes": 8064471040, + "working_set_bytes": 10121678848 + }, + { + "private_bytes": 8064471040, + "working_set_bytes": 10121678848 + }, + { + "private_bytes": 8064471040, + "working_set_bytes": 10121678848 + }, + { + "private_bytes": 8064471040, + "working_set_bytes": 10121678848 + }, + { + "private_bytes": 8064471040, + "working_set_bytes": 10121678848 + }, + { + "private_bytes": 8064471040, + "working_set_bytes": 10121678848 + }, + { + "private_bytes": 8064471040, + "working_set_bytes": 10121678848 + }, + { + "private_bytes": 8064471040, + "working_set_bytes": 10121687040 + }, + { + "private_bytes": 8064471040, + "working_set_bytes": 10121687040 + }, + { + "private_bytes": 8064471040, + "working_set_bytes": 10121687040 + }, + { + "private_bytes": 8064471040, + "working_set_bytes": 10121687040 + }, + { + "private_bytes": 8064471040, + "working_set_bytes": 10121687040 + }, + { + "private_bytes": 8064471040, + "working_set_bytes": 10121695232 + }, + { + "private_bytes": 8064471040, + "working_set_bytes": 10121695232 + }, + { + "private_bytes": 8064471040, + "working_set_bytes": 10121703424 + }, + { + "private_bytes": 8064471040, + "working_set_bytes": 10121703424 + }, + { + "private_bytes": 8064471040, + "working_set_bytes": 10121703424 + }, + { + "private_bytes": 8064471040, + "working_set_bytes": 10121711616 + }, + { + "private_bytes": 8064471040, + "working_set_bytes": 10121711616 + }, + { + "private_bytes": 8064471040, + "working_set_bytes": 10121711616 + }, + { + "private_bytes": 8064471040, + "working_set_bytes": 10121719808 + }, + { + "private_bytes": 8064471040, + "working_set_bytes": 10121719808 + }, + { + "private_bytes": 8064471040, + "working_set_bytes": 10121719808 + }, + { + "private_bytes": 8064471040, + "working_set_bytes": 10121719808 + }, + { + "private_bytes": 8064471040, + "working_set_bytes": 10121719808 + }, + { + "private_bytes": 8064471040, + "working_set_bytes": 10121719808 + }, + { + "private_bytes": 8064471040, + "working_set_bytes": 10121719808 + }, + { + "private_bytes": 8064471040, + "working_set_bytes": 10121728000 + }, + { + "private_bytes": 8064471040, + "working_set_bytes": 10121728000 + }, + { + "private_bytes": 8064471040, + "working_set_bytes": 10121728000 + }, + { + "private_bytes": 8064471040, + "working_set_bytes": 10121728000 + }, + { + "private_bytes": 8064471040, + "working_set_bytes": 10121728000 + }, + { + "private_bytes": 8064471040, + "working_set_bytes": 10121728000 + }, + { + "private_bytes": 8064471040, + "working_set_bytes": 10121728000 + }, + { + "private_bytes": 8064471040, + "working_set_bytes": 10121736192 + }, + { + "private_bytes": 8064471040, + "working_set_bytes": 10121736192 + }, + { + "private_bytes": 8064471040, + "working_set_bytes": 10121736192 + }, + { + "private_bytes": 8064471040, + "working_set_bytes": 10121744384 + }, + { + "private_bytes": 8064471040, + "working_set_bytes": 10121744384 + }, + { + "private_bytes": 8064471040, + "working_set_bytes": 10121744384 + }, + { + "private_bytes": 8064471040, + "working_set_bytes": 10121744384 + }, + { + "private_bytes": 8064471040, + "working_set_bytes": 10121744384 + }, + { + "private_bytes": 8064471040, + "working_set_bytes": 10121744384 + }, + { + "private_bytes": 8062468096, + "working_set_bytes": 10119753728 + }, + { + "private_bytes": 8062468096, + "working_set_bytes": 10119753728 + }, + { + "private_bytes": 8062468096, + "working_set_bytes": 10119753728 + }, + { + "private_bytes": 8062468096, + "working_set_bytes": 10119753728 + }, + { + "private_bytes": 8062468096, + "working_set_bytes": 10119761920 + }, + { + "private_bytes": 8062468096, + "working_set_bytes": 10119761920 + }, + { + "private_bytes": 8062468096, + "working_set_bytes": 10119761920 + }, + { + "private_bytes": 8062468096, + "working_set_bytes": 10119761920 + }, + { + "private_bytes": 8062468096, + "working_set_bytes": 10119761920 + }, + { + "private_bytes": 8062468096, + "working_set_bytes": 10119761920 + }, + { + "private_bytes": 8062468096, + "working_set_bytes": 10119761920 + }, + { + "private_bytes": 8062468096, + "working_set_bytes": 10119761920 + }, + { + "private_bytes": 8062468096, + "working_set_bytes": 10119770112 + }, + { + "private_bytes": 8051359744, + "working_set_bytes": 10118451200 + }, + { + "private_bytes": 8051359744, + "working_set_bytes": 10118459392 + }, + { + "private_bytes": 8051359744, + "working_set_bytes": 10118459392 + } + ], + "scratch_bytes": 169751296, + "slope_first_token": 9, + "source": "decode@2048", + "tokens": 128, + "weight_creates_after_warmup": 0, + "weight_objects_created_after_load": 0, + "weight_objects_created_total": 161 + }, + "model_load": { + "benchmark_second_shape_plan_ns": 14026636400, + "device_setup_ns": 142306900, + "device_tensors": 76, + "engine_reported_total_ns": 13043901400, + "manifest_map_ns": 4497300, + "mapped_source_bytes": 3248488448, + "packed_weight_bytes": 2098112512, + "shape_plan_ns": 11282606100, + "shape_plan_share": 0.8649709980424125, + "total_ns": 13043912600, + "unaccounted_ns": 11066800, + "weight_objects": 161, + "weight_pack_ns": 1603435500 + }, + "prefill": { + "points": [ + { + "ns": 144725000, + "padded_rows": 1, + "rows": 1, + "tokens_per_second": 6.909656244601831, + "top1_id": 4576, + "v_bytes": 131072, + "v_scatter_ns": 63100 + }, + { + "ns": 50514300, + "padded_rows": 64, + "rows": 2, + "tokens_per_second": 39.59274898395108, + "top1_id": 477, + "v_bytes": 262144, + "v_scatter_ns": 65600 + }, + { + "ns": 70602800, + "padded_rows": 128, + "rows": 65, + "tokens_per_second": 920.6433739171817, + "top1_id": 9822, + "v_bytes": 8519680, + "v_scatter_ns": 276500 + }, + { + "ns": 92661500, + "padded_rows": 256, + "rows": 129, + "tokens_per_second": 1392.163951587229, + "top1_id": 6864, + "v_bytes": 16908288, + "v_scatter_ns": 554300 + }, + { + "ns": 136357900, + "padded_rows": 512, + "rows": 257, + "tokens_per_second": 1884.745951646366, + "top1_id": 200020, + "v_bytes": 33685504, + "v_scatter_ns": 1169600 + }, + { + "ns": 289416400, + "padded_rows": 1024, + "rows": 513, + "tokens_per_second": 1772.5325862667078, + "top1_id": 200019, + "v_bytes": 67239936, + "v_scatter_ns": 2989300 + }, + { + "ns": 731295700, + "padded_rows": 2048, + "rows": 1025, + "tokens_per_second": 1401.6218063363424, + "top1_id": 374, + "v_bytes": 134348800, + "v_scatter_ns": 7683300 + }, + { + "ns": 1171699000, + "padded_rows": 3072, + "rows": 2049, + "tokens_per_second": 1748.7426378276332, + "top1_id": 44643, + "v_bytes": 268566528, + "v_scatter_ns": 16640900 + }, + { + "ns": 1717807600, + "padded_rows": 4096, + "rows": 3073, + "tokens_per_second": 1788.9081408185643, + "top1_id": 791, + "v_bytes": 402784256, + "v_scatter_ns": 28384300 + }, + { + "ns": 1746325400, + "padded_rows": 4096, + "rows": 4096, + "tokens_per_second": 2345.496434971398, + "top1_id": 350, + "v_bytes": 536870912, + "v_scatter_ns": 35780400 + } + ] + }, + "ttft": { + "cold_ns": 3169362500, + "cold_top1_id": 1715, + "prompt_id": "ff6a21dcdcff126f", + "prompt_source": "C:\\Users\\chiz\\work\\FastFlowLM\\src\\test\\phi4_corelib_aie4\\phi4_tokens.json", + "prompt_token_count": 19, + "row_extent": 64, + "stream_rebuilt_for_warm": false, + "warm_ns": 49078200, + "warm_top1_id": 1715 + }, + "v_scatter": { + "bytes": 47889514496, + "counts_are_measured": true, + "design_18_5_expected_reads": 32, + "design_18_5_expected_writes": 256, + "model_steps": 7227, + "nanoseconds": 3315910400, + "reads_per_model_step": 32, + "reads_per_model_step_source": "v_read_calls / (synchronize_count / 129)", + "total_read_calls": 231264, + "total_write_calls": 1850112, + "writes_per_model_step": 256 + }, + "determinism": { + "min_runs_per_route": 20, + "routes": { + "append": { + "runs": 41, + "steps": 697, + "bit_identical_steps": 679, + "bit_identical_runs": 39, + "step_bit_identity_rate": 0.9741750358680057, + "run_bit_identity_rate": 0.9512195121951219, + "observed_max_abs_diff": { + "max": 49.25, + "min": 49.21484375, + "median": 0.0, + "nonzero_runs": 2, + "histogram": { + "49.2148": 1, + "49.25": 1 + } + }, + "runs_with_gate_failures": 2, + "first_divergences": { + "decode[6]": 1, + "decode[8]": 1 + }, + "localisation": { + "measured_runs": 2, + "by_source": { + "model_body": 2 + }, + "elements_by_source": { + "model_body": [ + 2571, + 2754 + ] + }, + "steps_by_source": { + "model_body": [ + "decode[6]", + "decode[8]" + ] + }, + "lm_head_input_elements": 3072, + "instruments": { + "per_step_lm_head_input_capture": 2 + } + }, + "is_baseline": true + }, + "reprefill": { + "runs": 41, + "steps": 697, + "bit_identical_steps": 697, + "bit_identical_runs": 41, + "step_bit_identity_rate": 1.0, + "run_bit_identity_rate": 1.0, + "observed_max_abs_diff": { + "max": 0.0, + "min": 0.0, + "median": 0.0, + "nonzero_runs": 0, + "histogram": {} + }, + "runs_with_gate_failures": 0, + "first_divergences": {}, + "localisation": { + "measured_runs": 0, + "by_source": {}, + "elements_by_source": {}, + "steps_by_source": {}, + "lm_head_input_elements": 3072, + "instruments": {} + }, + "is_baseline": true + } + }, + "observed_rates_differ": true, + "route_dependence_p": 0.49382716049382713, + "route_dependence_alpha": 0.05, + "route_dependent": false, + "gate_failures": 2, + "is_baseline": false, + "sources": [ + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_append-001.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_append-002.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_append-003.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_append-004.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_append-005.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_append-006.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_append-007.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_append-008.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_append-009.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_append-010.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_append-011.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_append-012.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_append-013.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_append-014.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_append-015.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_append-016.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_append-017.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_append-018.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_append-019.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_append-020.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_append-021.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_append-022.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_append-023.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_append-024.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_append-025.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_append-026.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_append-027.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_append-028.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_append-029.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_append-030.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_append-031.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_append-032.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_append-033.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_append-034.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_append-035.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_append-036.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_append-037.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_append-038.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_append-039.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_append-040.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_append.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_reprefill-001.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_reprefill-002.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_reprefill-003.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_reprefill-004.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_reprefill-005.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_reprefill-006.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_reprefill-007.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_reprefill-008.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_reprefill-009.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_reprefill-010.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_reprefill-011.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_reprefill-012.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_reprefill-013.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_reprefill-014.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_reprefill-015.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_reprefill-016.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_reprefill-017.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_reprefill-018.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_reprefill-019.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_reprefill-020.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_reprefill-021.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_reprefill-022.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_reprefill-023.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_reprefill-024.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_reprefill-025.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_reprefill-026.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_reprefill-027.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_reprefill-028.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_reprefill-029.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_reprefill-030.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_reprefill-031.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_reprefill-032.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_reprefill-033.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_reprefill-034.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_reprefill-035.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_reprefill-036.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_reprefill-037.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_reprefill-038.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_reprefill-039.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_reprefill-040.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T140230Z-11788\\determ1-force_reprefill.json" + ], + "excluded_sources": [ + "src/test/phi4_corelib_aie4/determinism_records\\20260902T120919Z-10264\\determ1-force_append-001.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T120919Z-10264\\determ1-force_append-002.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T120919Z-10264\\determ1-force_append-003.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T120919Z-10264\\determ1-force_append-004.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T120919Z-10264\\determ1-force_append-005.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T120919Z-10264\\determ1-force_append-006.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T120919Z-10264\\determ1-force_append-007.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T120919Z-10264\\determ1-force_append-008.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T120919Z-10264\\determ1-force_append-009.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T120919Z-10264\\determ1-force_append-010.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T120919Z-10264\\determ1-force_append-011.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T120919Z-10264\\determ1-force_append-012.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T120919Z-10264\\determ1-force_append-013.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T120919Z-10264\\determ1-force_append-014.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T120919Z-10264\\determ1-force_append-015.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T120919Z-10264\\determ1-force_append-016.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T120919Z-10264\\determ1-force_append-017.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T120919Z-10264\\determ1-force_append-018.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T120919Z-10264\\determ1-force_append-019.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T120919Z-10264\\determ1-force_append-020.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T120919Z-10264\\determ1-force_append.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T120919Z-10264\\determ1-force_reprefill-001.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T120919Z-10264\\determ1-force_reprefill-002.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T120919Z-10264\\determ1-force_reprefill-003.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T120919Z-10264\\determ1-force_reprefill-004.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T120919Z-10264\\determ1-force_reprefill-005.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T120919Z-10264\\determ1-force_reprefill-006.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T120919Z-10264\\determ1-force_reprefill-007.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T120919Z-10264\\determ1-force_reprefill-008.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T120919Z-10264\\determ1-force_reprefill-009.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T120919Z-10264\\determ1-force_reprefill-010.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T120919Z-10264\\determ1-force_reprefill-011.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T120919Z-10264\\determ1-force_reprefill-012.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T120919Z-10264\\determ1-force_reprefill-013.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T120919Z-10264\\determ1-force_reprefill-014.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T120919Z-10264\\determ1-force_reprefill-015.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T120919Z-10264\\determ1-force_reprefill-016.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T120919Z-10264\\determ1-force_reprefill-017.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T120919Z-10264\\determ1-force_reprefill-018.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T120919Z-10264\\determ1-force_reprefill-019.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T120919Z-10264\\determ1-force_reprefill-020.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T120919Z-10264\\determ1-force_reprefill.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T125729Z-14892\\determ1-force_append-001.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T125729Z-14892\\determ1-force_append-002.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T125729Z-14892\\determ1-force_append-003.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T125729Z-14892\\determ1-force_append-004.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T125729Z-14892\\determ1-force_append-005.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T125729Z-14892\\determ1-force_append-006.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T125729Z-14892\\determ1-force_append-007.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T125729Z-14892\\determ1-force_append-008.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T125729Z-14892\\determ1-force_append-009.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T125729Z-14892\\determ1-force_append-010.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T125729Z-14892\\determ1-force_append.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T125729Z-14892\\determ1-force_reprefill-001.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T125729Z-14892\\determ1-force_reprefill-002.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T125729Z-14892\\determ1-force_reprefill-003.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T125729Z-14892\\determ1-force_reprefill-004.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T125729Z-14892\\determ1-force_reprefill-005.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T125729Z-14892\\determ1-force_reprefill-006.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T125729Z-14892\\determ1-force_reprefill-007.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T125729Z-14892\\determ1-force_reprefill-008.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T125729Z-14892\\determ1-force_reprefill-009.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T125729Z-14892\\determ1-force_reprefill-010.json", + "src/test/phi4_corelib_aie4/determinism_records\\20260902T125729Z-14892\\determ1-force_reprefill.json" + ], + "blocking_problems": [], + "problems": [ + "64 record(s) carry no FastFlow harness SHA-256 and were EXCLUDED from the pooled figures: they cannot be shown to describe the same binary as the rest. They remain committed as evidence and are listed under `excluded_sources`.", + "route 'append': 2 run(s) recorded a DETERM-2 gate failure. They ARE counted in the rate above -- dropping them would bias it upward -- but a baseline should not be declared over a window containing a hard-gate failure." + ] + } +} \ No newline at end of file diff --git a/docs/docs/benchmarks/phi4_aie4_baseline_task15_rerun.json b/docs/docs/benchmarks/phi4_aie4_baseline_task15_rerun.json new file mode 100644 index 00000000..7e9d4eb2 --- /dev/null +++ b/docs/docs/benchmarks/phi4_aie4_baseline_task15_rerun.json @@ -0,0 +1,2752 @@ +{ + "continuation": { + "crossover": { + "2048": { + "append_wins_up_to": 24, + "bracket_is_tight": false, + "crossover_bracket": [ + 24, + 32 + ], + "decisions": [ + { + "append_p50_ns": 54186600, + "decided": true, + "drift_across_point_ns": 14727200, + "gap_ns": 1095917300, + "gap_over_uncertainty": 74.41450513335869, + "reprefill_p50_ns": 1150103900, + "suffix": 1, + "uncertainty_ns": 14727200, + "winner": "append", + "within_route_spread_ns": 6312200 + }, + { + "append_p50_ns": 101509100, + "decided": true, + "drift_across_point_ns": 7753000, + "gap_ns": 1049675000, + "gap_over_uncertainty": 135.38952663485102, + "reprefill_p50_ns": 1151184100, + "suffix": 2, + "uncertainty_ns": 7753000, + "winner": "append", + "within_route_spread_ns": 4850800 + }, + { + "append_p50_ns": 188131600, + "decided": true, + "drift_across_point_ns": 2862800, + "gap_ns": 951793600, + "gap_over_uncertainty": 70.99334665990393, + "reprefill_p50_ns": 1139925200, + "suffix": 4, + "uncertainty_ns": 13406800, + "winner": "append", + "within_route_spread_ns": 13406800 + }, + { + "append_p50_ns": 358749200, + "decided": true, + "drift_across_point_ns": 10875400, + "gap_ns": 786213200, + "gap_over_uncertainty": 72.29280762086911, + "reprefill_p50_ns": 1144962400, + "suffix": 8, + "uncertainty_ns": 10875400, + "winner": "append", + "within_route_spread_ns": 7067700 + }, + { + "append_p50_ns": 526311300, + "decided": true, + "drift_across_point_ns": 12969900, + "gap_ns": 619408700, + "gap_over_uncertainty": 47.75739982575039, + "reprefill_p50_ns": 1145720000, + "suffix": 12, + "uncertainty_ns": 12969900, + "winner": "append", + "within_route_spread_ns": 7981700 + }, + { + "append_p50_ns": 682441800, + "decided": true, + "drift_across_point_ns": 21189100, + "gap_ns": 463406600, + "gap_over_uncertainty": 21.870046391776903, + "reprefill_p50_ns": 1145848400, + "suffix": 16, + "uncertainty_ns": 21189100, + "winner": "append", + "within_route_spread_ns": 7494400 + }, + { + "append_p50_ns": 1005245900, + "decided": true, + "drift_across_point_ns": 4466100, + "gap_ns": 137566100, + "gap_over_uncertainty": 30.802288350014553, + "reprefill_p50_ns": 1142812000, + "suffix": 24, + "uncertainty_ns": 4466100, + "winner": "append", + "within_route_spread_ns": 4076300 + }, + { + "append_p50_ns": 1334130600, + "decided": true, + "drift_across_point_ns": 8662700, + "gap_ns": 187255800, + "gap_over_uncertainty": 21.616332090456787, + "reprefill_p50_ns": 1146874800, + "suffix": 32, + "uncertainty_ns": 8662700, + "winner": "reprefill", + "within_route_spread_ns": 7236200 + }, + { + "append_p50_ns": 2626684000, + "decided": true, + "drift_across_point_ns": 3021300, + "gap_ns": 1481928900, + "gap_over_uncertainty": 478.13412273343226, + "reprefill_p50_ns": 1144755100, + "suffix": 64, + "uncertainty_ns": 3099400, + "winner": "reprefill", + "within_route_spread_ns": 3099400 + }, + { + "append_p50_ns": 5250833000, + "decided": true, + "drift_across_point_ns": 10395800, + "gap_ns": 4113118000, + "gap_over_uncertainty": 69.8038996061021, + "reprefill_p50_ns": 1137715000, + "suffix": 128, + "uncertainty_ns": 58923900, + "winner": "reprefill", + "within_route_spread_ns": 58923900 + }, + { + "append_p50_ns": 10422542000, + "decided": true, + "drift_across_point_ns": 43839700, + "gap_ns": 9271308000, + "gap_over_uncertainty": 211.48201287873778, + "reprefill_p50_ns": 1151234000, + "suffix": 256, + "uncertainty_ns": 43839700, + "winner": "reprefill", + "within_route_spread_ns": 41616700 + } + ], + "reprefill_wins_from": 32 + }, + "512": { + "append_wins_up_to": 4, + "bracket_is_tight": false, + "crossover_bracket": [ + 4, + 12 + ], + "decisions": [ + { + "append_p50_ns": 47494700, + "decided": true, + "drift_across_point_ns": 5187200, + "gap_ns": 321650000, + "gap_over_uncertainty": 62.008405305367056, + "reprefill_p50_ns": 369144700, + "suffix": 1, + "uncertainty_ns": 5187200, + "winner": "append", + "within_route_spread_ns": 4650700 + }, + { + "append_p50_ns": 87972900, + "decided": true, + "drift_across_point_ns": 15405900, + "gap_ns": 251701700, + "gap_over_uncertainty": 16.338006867498816, + "reprefill_p50_ns": 339674600, + "suffix": 2, + "uncertainty_ns": 15405900, + "winner": "append", + "within_route_spread_ns": 15405900 + }, + { + "append_p50_ns": 176413400, + "decided": true, + "drift_across_point_ns": 6845300, + "gap_ns": 168902300, + "gap_over_uncertainty": 24.6741998159321, + "reprefill_p50_ns": 345315700, + "suffix": 4, + "uncertainty_ns": 6845300, + "winner": "append", + "within_route_spread_ns": 4959900 + }, + { + "append_p50_ns": 332451000, + "decided": false, + "drift_across_point_ns": 19683000, + "gap_ns": 405800, + "gap_over_uncertainty": 0.020616775897983032, + "reprefill_p50_ns": 332045200, + "suffix": 8, + "uncertainty_ns": 19683000, + "winner": "undecided", + "within_route_spread_ns": 19683000 + }, + { + "append_p50_ns": 488070800, + "decided": true, + "drift_across_point_ns": 25897000, + "gap_ns": 149873400, + "gap_over_uncertainty": 5.787288102869058, + "reprefill_p50_ns": 338197400, + "suffix": 12, + "uncertainty_ns": 25897000, + "winner": "reprefill", + "within_route_spread_ns": 3003100 + }, + { + "append_p50_ns": 635415700, + "decided": true, + "drift_across_point_ns": 6566300, + "gap_ns": 295946200, + "gap_over_uncertainty": 45.07046586357614, + "reprefill_p50_ns": 339469500, + "suffix": 16, + "uncertainty_ns": 6566300, + "winner": "reprefill", + "within_route_spread_ns": 6272100 + }, + { + "append_p50_ns": 937499500, + "decided": true, + "drift_across_point_ns": 46321700, + "gap_ns": 599895100, + "gap_over_uncertainty": 12.950627891463396, + "reprefill_p50_ns": 337604400, + "suffix": 24, + "uncertainty_ns": 46321700, + "winner": "reprefill", + "within_route_spread_ns": 11693000 + }, + { + "append_p50_ns": 1240925900, + "decided": true, + "drift_across_point_ns": 5983300, + "gap_ns": 907233700, + "gap_over_uncertainty": 139.4885762607626, + "reprefill_p50_ns": 333692200, + "suffix": 32, + "uncertainty_ns": 6504000, + "winner": "reprefill", + "within_route_spread_ns": 6504000 + }, + { + "append_p50_ns": 2445478800, + "decided": true, + "drift_across_point_ns": 208800, + "gap_ns": 2104393200, + "gap_over_uncertainty": 16.97173411616691, + "reprefill_p50_ns": 341085600, + "suffix": 64, + "uncertainty_ns": 123994000, + "winner": "reprefill", + "within_route_spread_ns": 123994000 + }, + { + "append_p50_ns": 4869900200, + "decided": true, + "drift_across_point_ns": 6813400, + "gap_ns": 4533190400, + "gap_over_uncertainty": 541.8911481680713, + "reprefill_p50_ns": 336709800, + "suffix": 128, + "uncertainty_ns": 8365500, + "winner": "reprefill", + "within_route_spread_ns": 8365500 + }, + { + "append_p50_ns": 9705341200, + "decided": true, + "drift_across_point_ns": 78552900, + "gap_ns": 9363677900, + "gap_over_uncertainty": 119.20219240791874, + "reprefill_p50_ns": 341663300, + "suffix": 256, + "uncertainty_ns": 78552900, + "winner": "reprefill", + "within_route_spread_ns": 58381500 + } + ], + "reprefill_wins_from": 12 + } + }, + "decision_rule": "append and re-prefill samples are INTERLEAVED, so a machine regime shift moves both routes together; a point is decided only when the gap between the two routes' p50 exceeds BOTH the larger within-point p50-to-p95 spread AND the larger drift between a route's first and last sample at that point. The reported figure is a BRACKET, not a threshold.", + "histories": [ + 512, + 2048 + ], + "points": [ + { + "count": 5, + "drift_ns": 5187200, + "history_rows": 512, + "interleaved_with_reprefill": true, + "max_ns": 52145400, + "mean_ns": 48283480, + "min_ns": 46111000, + "model_steps": 1, + "p50_ns": 47494700, + "p95_ns": 52145400, + "route": "append", + "sample_starts_ns": [ + 874399800, + 1514884900, + 2151859400, + 2796651000, + 3442642300 + ], + "samples_ns": [ + 46958200, + 47494700, + 46111000, + 48708100, + 52145400 + ], + "suffix": 1, + "tokens_per_second": 21.05498087154988 + }, + { + "count": 5, + "drift_ns": 3010400, + "history_rows": 512, + "interleaved_with_append": true, + "max_ns": 370404300, + "mean_ns": 367779280, + "min_ns": 363435300, + "model_steps": 1, + "p50_ns": 369144700, + "p95_ns": 370404300, + "route": "reprefill", + "sample_starts_ns": [ + 921362100, + 1562383500, + 2197974200, + 2845363100, + 3494791400 + ], + "samples_ns": [ + 363435300, + 370404300, + 369144700, + 369466400, + 366445700 + ], + "suffix": 1, + "tokens_per_second": 1389.6989446144019 + }, + { + "count": 5, + "drift_ns": 8283600, + "history_rows": 512, + "interleaved_with_reprefill": true, + "max_ns": 93613400, + "mean_ns": 88346020, + "min_ns": 82725200, + "model_steps": 2, + "p50_ns": 87972900, + "p95_ns": 93613400, + "route": "append", + "sample_starts_ns": [ + 4739268500, + 5323650700, + 5981663100, + 6543002000, + 7068293900 + ], + "samples_ns": [ + 92851100, + 82725200, + 93613400, + 87972900, + 84567500 + ], + "suffix": 2, + "tokens_per_second": 22.734273850242516 + }, + { + "count": 5, + "drift_ns": 15405900, + "history_rows": 512, + "interleaved_with_append": true, + "max_ns": 355080500, + "mean_ns": 327527680, + "min_ns": 286969000, + "model_steps": 1, + "p50_ns": 339674600, + "p95_ns": 355080500, + "route": "reprefill", + "sample_starts_ns": [ + 4832122800, + 5406379400, + 6075282800, + 6630978100, + 7152864600 + ], + "samples_ns": [ + 339674600, + 344375600, + 311538700, + 286969000, + 355080500 + ], + "suffix": 2, + "tokens_per_second": 1513.2129396781509 + }, + { + "count": 5, + "drift_ns": 6845300, + "history_rows": 512, + "interleaved_with_reprefill": true, + "max_ns": 179430300, + "mean_ns": 173938720, + "min_ns": 167535400, + "model_steps": 4, + "p50_ns": 176413400, + "p95_ns": 179430300, + "route": "append", + "sample_starts_ns": [ + 8366448800, + 9079496600, + 9822372900, + 10558958300, + 11295804500 + ], + "samples_ns": [ + 169734600, + 167535400, + 176413400, + 179430300, + 176579900 + ], + "suffix": 4, + "tokens_per_second": 22.674014558984748 + }, + { + "count": 5, + "drift_ns": 4238500, + "history_rows": 512, + "interleaved_with_append": true, + "max_ns": 350275600, + "mean_ns": 343126220, + "min_ns": 329344800, + "model_steps": 1, + "p50_ns": 345315700, + "p95_ns": 350275600, + "route": "reprefill", + "sample_starts_ns": [ + 8536186500, + 9247035500, + 9998789500, + 10738392100, + 11472388000 + ], + "samples_ns": [ + 350275600, + 344657900, + 329344800, + 345315700, + 346037100 + ], + "suffix": 4, + "tokens_per_second": 1494.2847950440712 + }, + { + "count": 5, + "drift_ns": 13652200, + "history_rows": 512, + "interleaved_with_reprefill": true, + "max_ns": 337574700, + "mean_ns": 330979020, + "min_ns": 322933800, + "model_steps": 8, + "p50_ns": 332451000, + "p95_ns": 337574700, + "route": "append", + "sample_starts_ns": [ + 12927017900, + 13830470600, + 14723625700, + 15617666600, + 16517808800 + ], + "samples_ns": [ + 322933800, + 325349600, + 332451000, + 337574700, + 336586000 + ], + "suffix": 8, + "tokens_per_second": 24.063696604913204 + }, + { + "count": 5, + "drift_ns": 19683000, + "history_rows": 512, + "interleaved_with_append": true, + "max_ns": 351728200, + "mean_ns": 336659200, + "min_ns": 330853200, + "model_steps": 1, + "p50_ns": 332045200, + "p95_ns": 351728200, + "route": "reprefill", + "sample_starts_ns": [ + 13249955200, + 14155825900, + 15056080000, + 15955244900, + 16854398300 + ], + "samples_ns": [ + 351728200, + 336915700, + 330853200, + 331753700, + 332045200 + ], + "suffix": 8, + "tokens_per_second": 1566.0518507721238 + }, + { + "count": 5, + "drift_ns": 12005000, + "history_rows": 512, + "interleaved_with_reprefill": true, + "max_ns": 491069200, + "mean_ns": 486239480, + "min_ns": 479064200, + "model_steps": 12, + "p50_ns": 488070800, + "p95_ns": 491069200, + "route": "append", + "sample_starts_ns": [ + 18456083400, + 19493277400, + 20549928700, + 21607800000, + 22661499000 + ], + "samples_ns": [ + 491069200, + 488963800, + 488070800, + 484029400, + 479064200 + ], + "suffix": 12, + "tokens_per_second": 24.58659686258633 + }, + { + "count": 5, + "drift_ns": 25897000, + "history_rows": 512, + "interleaved_with_append": true, + "max_ns": 341200500, + "mean_ns": 334208800, + "min_ns": 315303500, + "model_steps": 1, + "p50_ns": 338197400, + "p95_ns": 341200500, + "route": "reprefill", + "sample_starts_ns": [ + 18947156000, + 19982245500, + 21038004400, + 22091834300, + 23140565900 + ], + "samples_ns": [ + 315303500, + 337053600, + 338197400, + 339289000, + 341200500 + ], + "suffix": 12, + "tokens_per_second": 1549.3909769856302 + }, + { + "count": 5, + "drift_ns": 6566300, + "history_rows": 512, + "interleaved_with_reprefill": true, + "max_ns": 640189900, + "mean_ns": 634781760, + "min_ns": 628849400, + "model_steps": 16, + "p50_ns": 635415700, + "p95_ns": 640189900, + "route": "append", + "sample_starts_ns": [ + 24914520300, + 26086763400, + 27259155400, + 28463051000, + 29671963200 + ], + "samples_ns": [ + 628849400, + 630867900, + 640189900, + 638585900, + 635415700 + ], + "suffix": 16, + "tokens_per_second": 25.18036617603248 + }, + { + "count": 5, + "drift_ns": 2450600, + "history_rows": 512, + "interleaved_with_append": true, + "max_ns": 345741600, + "mean_ns": 339630820, + "min_ns": 334029800, + "model_steps": 1, + "p50_ns": 339469500, + "p95_ns": 345741600, + "route": "reprefill", + "sample_starts_ns": [ + 25543372900, + 26717634100, + 27899348800, + 29101639800, + 30307382100 + ], + "samples_ns": [ + 343291000, + 334029800, + 335622200, + 339469500, + 345741600 + ], + "suffix": 16, + "tokens_per_second": 1555.368008024285 + }, + { + "count": 5, + "drift_ns": 2504500, + "history_rows": 512, + "interleaved_with_reprefill": true, + "max_ns": 940552500, + "mean_ns": 935120120, + "min_ns": 927234400, + "model_steps": 24, + "p50_ns": 937499500, + "p95_ns": 940552500, + "route": "append", + "sample_starts_ns": [ + 32365088800, + 33744455600, + 35161964300, + 36657489600, + 38119898300 + ], + "samples_ns": [ + 940552500, + 927234400, + 932266200, + 937499500, + 938048000 + ], + "suffix": 24, + "tokens_per_second": 25.600013653340614 + }, + { + "count": 5, + "drift_ns": 46321700, + "history_rows": 512, + "interleaved_with_append": true, + "max_ns": 349297400, + "mean_ns": 328000880, + "min_ns": 291596800, + "model_steps": 1, + "p50_ns": 337604400, + "p95_ns": 349297400, + "route": "reprefill", + "sample_starts_ns": [ + 33305644100, + 34671693900, + 36094233800, + 37594993000, + 39057948900 + ], + "samples_ns": [ + 291596800, + 323587300, + 337604400, + 349297400, + 337918500 + ], + "suffix": 24, + "tokens_per_second": 1587.6570329059693 + }, + { + "count": 5, + "drift_ns": 5787100, + "history_rows": 512, + "interleaved_with_reprefill": true, + "max_ns": 1247429900, + "mean_ns": 1233310780, + "min_ns": 1202757200, + "model_steps": 32, + "p50_ns": 1240925900, + "p95_ns": 1247429900, + "route": "append", + "sample_starts_ns": [ + 41389810500, + 43149830700, + 44942913700, + 46714103800, + 48409220600 + ], + "samples_ns": [ + 1247429900, + 1240925900, + 1202757200, + 1233798100, + 1241642800 + ], + "suffix": 32, + "tokens_per_second": 25.78719647966087 + }, + { + "count": 5, + "drift_ns": 5983300, + "history_rows": 512, + "interleaved_with_append": true, + "max_ns": 339675500, + "mean_ns": 330512200, + "min_ns": 318492300, + "model_steps": 1, + "p50_ns": 333692200, + "p95_ns": 339675500, + "route": "reprefill", + "sample_starts_ns": [ + 42637243400, + 44390759300, + 46145674400, + 47947908200, + 49650866600 + ], + "samples_ns": [ + 333692200, + 323793900, + 336907100, + 318492300, + 339675500 + ], + "suffix": 32, + "tokens_per_second": 1630.2448783639534 + }, + { + "count": 5, + "drift_ns": 16600, + "history_rows": 512, + "interleaved_with_reprefill": true, + "max_ns": 2569472800, + "mean_ns": 2469938720, + "min_ns": 2440591700, + "model_steps": 64, + "p50_ns": 2445478800, + "p95_ns": 2569472800, + "route": "append", + "sample_starts_ns": [ + 53180031400, + 56151875700, + 59115343200, + 62089632800, + 65132379900 + ], + "samples_ns": [ + 2445478800, + 2440591700, + 2448688100, + 2569472800, + 2445462200 + ], + "suffix": 64, + "tokens_per_second": 26.170744150388874 + }, + { + "count": 5, + "drift_ns": 208800, + "history_rows": 512, + "interleaved_with_append": true, + "max_ns": 345984600, + "mean_ns": 334382520, + "min_ns": 309997200, + "model_steps": 1, + "p50_ns": 341085600, + "p95_ns": 345984600, + "route": "reprefill", + "sample_starts_ns": [ + 55625513200, + 58592471200, + 61564036600, + 64659109300, + 67577845500 + ], + "samples_ns": [ + 341294400, + 345984600, + 333550800, + 309997200, + 341085600 + ], + "suffix": 64, + "tokens_per_second": 1688.7256454098326 + }, + { + "count": 5, + "drift_ns": 6813400, + "history_rows": 512, + "interleaved_with_reprefill": true, + "max_ns": 4872885600, + "mean_ns": 4864915500, + "min_ns": 4843804800, + "model_steps": 128, + "p50_ns": 4869900200, + "p95_ns": 4872885600, + "route": "append", + "sample_starts_ns": [ + 73503065100, + 78933219200, + 84312750100, + 89749529300, + 95160559500 + ], + "samples_ns": [ + 4866072200, + 4843804800, + 4869900200, + 4871914700, + 4872885600 + ], + "suffix": 128, + "tokens_per_second": 26.28390618764631 + }, + { + "count": 5, + "drift_ns": 3484200, + "history_rows": 512, + "interleaved_with_append": true, + "max_ns": 345075300, + "mean_ns": 335841040, + "min_ns": 322752500, + "model_steps": 1, + "p50_ns": 336709800, + "p95_ns": 345075300, + "route": "reprefill", + "sample_starts_ns": [ + 78369141300, + 83777027600, + 89182653700, + 94621447600, + 100033448200 + ], + "samples_ns": [ + 335591700, + 322752500, + 336709800, + 345075300, + 339075900 + ], + "suffix": 128, + "tokens_per_second": 1900.7465776166896 + }, + { + "count": 5, + "drift_ns": 78552900, + "history_rows": 512, + "interleaved_with_reprefill": true, + "max_ns": 9763722700, + "mean_ns": 9713494040, + "min_ns": 9685169800, + "model_steps": 256, + "p50_ns": 9705341200, + "p95_ns": 9763722700, + "route": "append", + "sample_starts_ns": [ + 110871434700, + 121098560900, + 131366250300, + 141654144000, + 151922659400 + ], + "samples_ns": [ + 9685169800, + 9705341200, + 9716124700, + 9697111800, + 9763722700 + ], + "suffix": 256, + "tokens_per_second": 26.377228242114764 + }, + { + "count": 5, + "drift_ns": 933900, + "history_rows": 512, + "interleaved_with_append": true, + "max_ns": 346643300, + "mean_ns": 341678520, + "min_ns": 333679900, + "model_steps": 1, + "p50_ns": 341663300, + "p95_ns": 346643300, + "route": "reprefill", + "sample_starts_ns": [ + 120556613100, + 130803905700, + 141082378600, + 151351258900, + 161686387500 + ], + "samples_ns": [ + 345709400, + 333679900, + 341663300, + 340696700, + 346643300 + ], + "suffix": 256, + "tokens_per_second": 2247.8270273687576 + }, + { + "count": 5, + "drift_ns": 1744000, + "history_rows": 2048, + "interleaved_with_reprefill": true, + "max_ns": 55042000, + "mean_ns": 54063880, + "min_ns": 53298000, + "model_steps": 1, + "p50_ns": 54186600, + "p95_ns": 55042000, + "route": "append", + "sample_starts_ns": [ + 164699147700, + 166631054900, + 168586312000, + 170535544800, + 172491967200 + ], + "samples_ns": [ + 55042000, + 54400800, + 53392000, + 54186600, + 53298000 + ], + "suffix": 1, + "tokens_per_second": 18.454747114600288 + }, + { + "count": 5, + "drift_ns": 14727200, + "history_rows": 2048, + "interleaved_with_append": true, + "max_ns": 1156416100, + "mean_ns": 1149205220, + "min_ns": 1141688900, + "model_steps": 1, + "p50_ns": 1150103900, + "p95_ns": 1156416100, + "route": "reprefill", + "sample_starts_ns": [ + 164754201100, + 166685465200, + 168639713800, + 170589740900, + 172545274500 + ], + "samples_ns": [ + 1141688900, + 1153156400, + 1144660800, + 1150103900, + 1156416100 + ], + "suffix": 1, + "tokens_per_second": 1781.5781687202348 + }, + { + "count": 5, + "drift_ns": 7753000, + "history_rows": 2048, + "interleaved_with_reprefill": true, + "max_ns": 106148700, + "mean_ns": 101751920, + "min_ns": 98395700, + "model_steps": 2, + "p50_ns": 101509100, + "p95_ns": 106148700, + "route": "append", + "sample_starts_ns": [ + 176456467700, + 178466011600, + 180448568700, + 182456978400, + 184442383200 + ], + "samples_ns": [ + 106148700, + 101841300, + 101509100, + 100864800, + 98395700 + ], + "suffix": 2, + "tokens_per_second": 19.70266705152543 + }, + { + "count": 5, + "drift_ns": 5332400, + "history_rows": 2048, + "interleaved_with_append": true, + "max_ns": 1156034900, + "mean_ns": 1148028160, + "min_ns": 1133600600, + "model_steps": 1, + "p50_ns": 1151184100, + "p95_ns": 1156034900, + "route": "reprefill", + "sample_starts_ns": [ + 176562625600, + 178567861100, + 180550087100, + 182557852500, + 184540789700 + ], + "samples_ns": [ + 1151184100, + 1133600600, + 1156034900, + 1153469500, + 1145851700 + ], + "suffix": 2, + "tokens_per_second": 1780.775116682032 + }, + { + "count": 5, + "drift_ns": 1919700, + "history_rows": 2048, + "interleaved_with_reprefill": true, + "max_ns": 191686400, + "mean_ns": 188900640, + "min_ns": 186211900, + "model_steps": 4, + "p50_ns": 188131600, + "p95_ns": 191686400, + "route": "append", + "sample_starts_ns": [ + 188515227000, + 190586735300, + 192688763300, + 194774822000, + 196846170000 + ], + "samples_ns": [ + 188131600, + 188081300, + 190392000, + 191686400, + 186211900 + ], + "suffix": 4, + "tokens_per_second": 21.26171254589872 + }, + { + "count": 5, + "drift_ns": 2862800, + "history_rows": 2048, + "interleaved_with_append": true, + "max_ns": 1153332000, + "mean_ns": 1143525840, + "min_ns": 1134584700, + "model_steps": 1, + "p50_ns": 1139925200, + "p95_ns": 1153332000, + "route": "reprefill", + "sample_starts_ns": [ + 188703365600, + 190774829200, + 192879162400, + 194966516500, + 197032387800 + ], + "samples_ns": [ + 1137447500, + 1153332000, + 1152339800, + 1139925200, + 1134584700 + ], + "suffix": 4, + "tokens_per_second": 1800.1181130130292 + }, + { + "count": 5, + "drift_ns": 2340000, + "history_rows": 2048, + "interleaved_with_reprefill": true, + "max_ns": 365816900, + "mean_ns": 359201760, + "min_ns": 349629600, + "model_steps": 8, + "p50_ns": 358749200, + "p95_ns": 365816900, + "route": "append", + "sample_starts_ns": [ + 201112399400, + 203353684900, + 205601362600, + 207858182900, + 210111847600 + ], + "samples_ns": [ + 356409200, + 365403900, + 365816900, + 349629600, + 358749200 + ], + "suffix": 8, + "tokens_per_second": 22.299701295501148 + }, + { + "count": 5, + "drift_ns": 10875400, + "history_rows": 2048, + "interleaved_with_append": true, + "max_ns": 1149178300, + "mean_ns": 1142669980, + "min_ns": 1134684500, + "model_steps": 1, + "p50_ns": 1144962400, + "p95_ns": 1149178300, + "route": "reprefill", + "sample_starts_ns": [ + 201468819700, + 203719095600, + 205967187000, + 208207819200, + 210470603600 + ], + "samples_ns": [ + 1145559900, + 1144962400, + 1138964800, + 1149178300, + 1134684500 + ], + "suffix": 8, + "tokens_per_second": 1795.6921554803896 + }, + { + "count": 5, + "drift_ns": 4346500, + "history_rows": 2048, + "interleaved_with_reprefill": true, + "max_ns": 530657800, + "mean_ns": 522967960, + "min_ns": 515031700, + "model_steps": 12, + "p50_ns": 526311300, + "p95_ns": 530657800, + "route": "append", + "sample_starts_ns": [ + 214759913400, + 217161232700, + 219577159600, + 221996819200, + 224405770600 + ], + "samples_ns": [ + 526311300, + 515031700, + 526581200, + 516257800, + 530657800 + ], + "suffix": 12, + "tokens_per_second": 22.800194485658963 + }, + { + "count": 5, + "drift_ns": 12969900, + "history_rows": 2048, + "interleaved_with_append": true, + "max_ns": 1153701700, + "mean_ns": 1142761560, + "min_ns": 1132750100, + "model_steps": 1, + "p50_ns": 1145720000, + "p95_ns": 1153701700, + "route": "reprefill", + "sample_starts_ns": [ + 215286230500, + 217676270800, + 220103747400, + 222513088200, + 224936435300 + ], + "samples_ns": [ + 1132750100, + 1153701700, + 1146357900, + 1135278100, + 1145720000 + ], + "suffix": 12, + "tokens_per_second": 1797.9960199699751 + }, + { + "count": 5, + "drift_ns": 12181700, + "history_rows": 2048, + "interleaved_with_reprefill": true, + "max_ns": 688456400, + "mean_ns": 682527480, + "min_ns": 676274700, + "model_steps": 16, + "p50_ns": 682441800, + "p95_ns": 688456400, + "route": "append", + "sample_starts_ns": [ + 229385515100, + 231972745000, + 234553236400, + 237142510700, + 239719230300 + ], + "samples_ns": [ + 688456400, + 677239000, + 682441800, + 688225500, + 676274700 + ], + "suffix": 16, + "tokens_per_second": 23.44522272815059 + }, + { + "count": 5, + "drift_ns": 21189100, + "history_rows": 2048, + "interleaved_with_append": true, + "max_ns": 1153342800, + "mean_ns": 1143575320, + "min_ns": 1129102500, + "model_steps": 1, + "p50_ns": 1145848400, + "p95_ns": 1153342800, + "route": "reprefill", + "sample_starts_ns": [ + 230073977400, + 232649990400, + 235235684100, + 237830742500, + 240395511100 + ], + "samples_ns": [ + 1150291600, + 1139291300, + 1153342800, + 1145848400, + 1129102500 + ], + "suffix": 16, + "tokens_per_second": 1801.285405643539 + }, + { + "count": 5, + "drift_ns": 4466100, + "history_rows": 2048, + "interleaved_with_reprefill": true, + "max_ns": 1009322200, + "mean_ns": 1006268580, + "min_ns": 1004856100, + "model_steps": 24, + "p50_ns": 1005245900, + "p95_ns": 1009322200, + "route": "append", + "sample_starts_ns": [ + 245151664400, + 248040075700, + 250931531100, + 253831434300, + 256726138500 + ], + "samples_ns": [ + 1004856100, + 1006983500, + 1005245900, + 1004935200, + 1009322200 + ], + "suffix": 24, + "tokens_per_second": 23.874755420539394 + }, + { + "count": 5, + "drift_ns": 1162200, + "history_rows": 2048, + "interleaved_with_append": true, + "max_ns": 1144080200, + "mean_ns": 1141783140, + "min_ns": 1136656900, + "model_steps": 1, + "p50_ns": 1142812000, + "p95_ns": 1144080200, + "route": "reprefill", + "sample_starts_ns": [ + 246156526100, + 249047064800, + 251936784300, + 254836375900, + 257735467100 + ], + "samples_ns": [ + 1143974200, + 1136656900, + 1144080200, + 1141392400, + 1142812000 + ], + "suffix": 24, + "tokens_per_second": 1813.071616328845 + }, + { + "count": 5, + "drift_ns": 8662700, + "history_rows": 2048, + "interleaved_with_reprefill": true, + "max_ns": 1341366800, + "mean_ns": 1335350500, + "min_ns": 1328218200, + "model_steps": 32, + "p50_ns": 1334130600, + "p95_ns": 1341366800, + "route": "append", + "sample_starts_ns": [ + 262825184900, + 266049302800, + 269264805000, + 272481786700, + 275707918500 + ], + "samples_ns": [ + 1332704100, + 1328218200, + 1334130600, + 1340332800, + 1341366800 + ], + "suffix": 32, + "tokens_per_second": 23.985657775932882 + }, + { + "count": 5, + "drift_ns": 1797000, + "history_rows": 2048, + "interleaved_with_append": true, + "max_ns": 1148671800, + "mean_ns": 1142226140, + "min_ns": 1128325700, + "model_steps": 1, + "p50_ns": 1146874800, + "p95_ns": 1148671800, + "route": "reprefill", + "sample_starts_ns": [ + 264157895100, + 267377527200, + 270598941600, + 273822126100, + 277049296400 + ], + "samples_ns": [ + 1146874800, + 1147579800, + 1139678600, + 1128325700, + 1148671800 + ], + "suffix": 32, + "tokens_per_second": 1813.6242944739913 + }, + { + "count": 5, + "drift_ns": 1975500, + "history_rows": 2048, + "interleaved_with_reprefill": true, + "max_ns": 2629783400, + "mean_ns": 2626382720, + "min_ns": 2623542200, + "model_steps": 64, + "p50_ns": 2626684000, + "p95_ns": 2629783400, + "route": "append", + "sample_starts_ns": [ + 283454503100, + 287965724600, + 292482660400, + 296994362200, + 301509005200 + ], + "samples_ns": [ + 2629783400, + 2626684000, + 2624096100, + 2623542200, + 2627807900 + ], + "suffix": 64, + "tokens_per_second": 24.365321447117353 + }, + { + "count": 5, + "drift_ns": 3021300, + "history_rows": 2048, + "interleaved_with_append": true, + "max_ns": 1147200400, + "mean_ns": 1142970560, + "min_ns": 1137190900, + "model_steps": 1, + "p50_ns": 1144755100, + "p95_ns": 1147200400, + "route": "reprefill", + "sample_starts_ns": [ + 286084291300, + 290592414700, + 295106761600, + 299617910700, + 304136818600 + ], + "samples_ns": [ + 1140212200, + 1147200400, + 1145494200, + 1144755100, + 1137190900 + ], + "suffix": 64, + "tokens_per_second": 1844.936091571027 + }, + { + "count": 5, + "drift_ns": 10395800, + "history_rows": 2048, + "interleaved_with_reprefill": true, + "max_ns": 5309756900, + "mean_ns": 5259730300, + "min_ns": 5229183500, + "model_steps": 128, + "p50_ns": 5250833000, + "p95_ns": 5309756900, + "route": "append", + "sample_starts_ns": [ + 313132122900, + 320249198400, + 327393270400, + 334534069300, + 341728765500 + ], + "samples_ns": [ + 5240437200, + 5229183500, + 5268440900, + 5309756900, + 5250833000 + ], + "suffix": 128, + "tokens_per_second": 24.377084550203747 + }, + { + "count": 5, + "drift_ns": 6572000, + "history_rows": 2048, + "interleaved_with_append": true, + "max_ns": 1158051200, + "mean_ns": 1142271860, + "min_ns": 1131143000, + "model_steps": 1, + "p50_ns": 1137715000, + "p95_ns": 1158051200, + "route": "reprefill", + "sample_starts_ns": [ + 318372566900, + 325478387900, + 332661717700, + 339843832700, + 346979605900 + ], + "samples_ns": [ + 1131143000, + 1158051200, + 1134775600, + 1149674500, + 1137715000 + ], + "suffix": 128, + "tokens_per_second": 1912.6055295043134 + }, + { + "count": 5, + "drift_ns": 43839700, + "history_rows": 2048, + "interleaved_with_reprefill": true, + "max_ns": 10464158700, + "mean_ns": 10430809900, + "min_ns": 10395248700, + "model_steps": 256, + "p50_ns": 10422542000, + "p95_ns": 10464158700, + "route": "append", + "sample_starts_ns": [ + 361157130800, + 373495461400, + 385840615400, + 398132869700, + 410448302200 + ], + "samples_ns": [ + 10420319000, + 10451781100, + 10395248700, + 10422542000, + 10464158700 + ], + "suffix": 256, + "tokens_per_second": 24.562146163575065 + }, + { + "count": 5, + "drift_ns": 2036000, + "history_rows": 2048, + "interleaved_with_append": true, + "max_ns": 1153774300, + "mean_ns": 1149306600, + "min_ns": 1141291200, + "model_steps": 1, + "p50_ns": 1151234000, + "p95_ns": 1153774300, + "route": "reprefill", + "sample_starts_ns": [ + 371577456100, + 383947249400, + 396235871000, + 408555418300, + 420912467500 + ], + "samples_ns": [ + 1153270000, + 1153774300, + 1146963500, + 1141291200, + 1151234000 + ], + "suffix": 256, + "tokens_per_second": 2001.3307459647647 + } + ], + "prefix_monotonic": true, + "samples_interleaved": true, + "suffixes": [ + 1, + 2, + 4, + 8, + 12, + 16, + 24, + 32, + 64, + 128, + 256 + ], + "warm_samples_per_point": 5 + }, + "decode": { + "runs": [ + { + "count": 128, + "device_tensor_creates_after_warmup": 0, + "dispatches_per_pass": 193, + "live_corelib_object_delta": 0, + "max_ns": 47158800, + "mean_ns": 38897679, + "min_ns": 34419000, + "p50_ns": 39111100, + "p95_ns": 42440200, + "samples_ns": [ + 47158800, + 46726700, + 42972000, + 41550100, + 39905600, + 39479300, + 39177000, + 39717100, + 39546000, + 39252500, + 39254000, + 39619700, + 38354500, + 43033300, + 38992500, + 38801900, + 38846100, + 40525300, + 35406000, + 35201800, + 35037300, + 35467000, + 37789500, + 36922600, + 38382000, + 38179700, + 43501300, + 39616800, + 38548500, + 39111100, + 38855900, + 38942100, + 39360600, + 37650600, + 37444900, + 39146400, + 39312100, + 39441300, + 38091500, + 41880300, + 39616900, + 39465500, + 40785600, + 37512800, + 36790900, + 37904400, + 35314000, + 35419000, + 37917600, + 39146500, + 39523700, + 39383900, + 41845400, + 38665900, + 39284400, + 39176900, + 39244100, + 39173100, + 39260800, + 39323500, + 39036000, + 39121200, + 39197000, + 39217300, + 43094900, + 39607400, + 39339600, + 40865800, + 35677600, + 34419000, + 36596400, + 36170000, + 36697200, + 38214700, + 36796200, + 38292900, + 39216600, + 38651700, + 38188600, + 39408600, + 38972400, + 38982700, + 39173600, + 39175400, + 39359500, + 38605600, + 38714500, + 39276300, + 39351900, + 39691800, + 42440200, + 39337000, + 39173700, + 40684400, + 35511300, + 36800900, + 39602900, + 35039400, + 35679800, + 37952900, + 39062000, + 37619600, + 37828700, + 40371500, + 40000400, + 39431700, + 38854700, + 39016500, + 39787000, + 37627900, + 38276800, + 37581300, + 37588700, + 38035200, + 38942600, + 39387000, + 41912300, + 39397400, + 40073800, + 36606400, + 36576300, + 38528200, + 37310300, + 37991100, + 38659800, + 38882700, + 39217800, + 42043800 + ], + "start_context": 128, + "synchronizes_per_pass": 129, + "tokens": 128, + "tokens_per_second": 24.88807850539933, + "total_ns": 5143024600, + "v_bytes": 16777216, + "v_scatter_ns": 4816100, + "weight_creates_after_warmup": 0 + }, + { + "count": 128, + "device_tensor_creates_after_warmup": 0, + "dispatches_per_pass": 193, + "live_corelib_object_delta": 0, + "max_ns": 45563100, + "mean_ns": 39646328, + "min_ns": 35276400, + "p50_ns": 39821200, + "p95_ns": 42537100, + "samples_ns": [ + 45563100, + 43718400, + 42537100, + 41509700, + 40482300, + 41507200, + 41064800, + 40778300, + 41192400, + 44201600, + 41377500, + 35899100, + 37392800, + 38999900, + 38844800, + 36407400, + 37823000, + 39006400, + 39501400, + 38633200, + 39806900, + 42522400, + 40194000, + 40238600, + 40439800, + 38920100, + 40130100, + 39912800, + 40197700, + 40135600, + 39114600, + 40169800, + 39855600, + 40539500, + 42725700, + 35757100, + 36006000, + 37076500, + 36784100, + 36547300, + 40163200, + 38201900, + 39878300, + 39821200, + 38816200, + 40212900, + 40690700, + 39158900, + 39542600, + 40392100, + 40915700, + 40148500, + 39343800, + 40251200, + 39191600, + 40263000, + 40336900, + 39464100, + 40811300, + 43286400, + 35276400, + 36460400, + 36611000, + 42204500, + 38850800, + 39574300, + 39285900, + 38385000, + 39983600, + 39862900, + 39102100, + 39648900, + 39060600, + 40104100, + 40230000, + 40130300, + 39299800, + 39935200, + 38979100, + 40119400, + 40925900, + 39168100, + 40448900, + 40894000, + 42775800, + 35452300, + 36221500, + 37837900, + 39479300, + 39288700, + 40336000, + 38565700, + 39782300, + 39296900, + 40093400, + 40412800, + 41167600, + 38522700, + 40309400, + 40626700, + 39472700, + 39726000, + 38880000, + 38830500, + 39853900, + 39150700, + 40807500, + 39995300, + 40394000, + 39401000, + 36022800, + 37882500, + 39558600, + 41313300, + 38470900, + 38572300, + 38588400, + 39879100, + 39014400, + 40109800, + 41046400, + 40837100, + 39167700, + 40503500, + 41170100, + 39730800, + 38089200, + 39146300 + ], + "start_context": 512, + "synchronizes_per_pass": 129, + "tokens": 128, + "tokens_per_second": 24.383626219736133, + "total_ns": 5249424300, + "v_bytes": 16777216, + "v_scatter_ns": 4792300, + "weight_creates_after_warmup": 0 + }, + { + "count": 128, + "device_tensor_creates_after_warmup": 0, + "dispatches_per_pass": 193, + "live_corelib_object_delta": 0, + "max_ns": 54702100, + "mean_ns": 42710065, + "min_ns": 37277200, + "p50_ns": 42876100, + "p95_ns": 45753600, + "samples_ns": [ + 54702100, + 49840600, + 46295700, + 45668900, + 44009600, + 42854700, + 43532700, + 43459500, + 43092000, + 45173500, + 40554200, + 38261900, + 43154100, + 42363200, + 42872700, + 43963100, + 41225400, + 41440500, + 42727000, + 42817400, + 42560800, + 41925600, + 42087300, + 44966500, + 43219700, + 43530100, + 42578500, + 43051100, + 42979300, + 42954900, + 43484000, + 42170500, + 45272800, + 39279400, + 39478000, + 43232500, + 43176000, + 42746400, + 42855300, + 42606900, + 42590700, + 42287000, + 42870100, + 42974400, + 43272400, + 43139900, + 43199700, + 41838100, + 43359100, + 43758600, + 43117400, + 42149200, + 42832900, + 43036900, + 42414000, + 44341300, + 38918000, + 38214100, + 43359600, + 41972800, + 43255500, + 44823400, + 41412900, + 42536000, + 42677900, + 43212300, + 44287900, + 42762000, + 41683800, + 41568200, + 45753600, + 43238500, + 43374500, + 43037600, + 42948600, + 42927600, + 43002300, + 43098000, + 42458900, + 37964600, + 41862700, + 41184300, + 42556700, + 42509600, + 42115600, + 42803200, + 41871500, + 43038100, + 43280600, + 42769800, + 43820700, + 43875700, + 42251100, + 46442200, + 42353200, + 43254500, + 42876100, + 42968800, + 43158100, + 43070500, + 44964600, + 38563600, + 37703900, + 38472800, + 38882500, + 43461900, + 42757600, + 45836000, + 41780700, + 41570900, + 42568700, + 42673300, + 43033400, + 43756100, + 42472500, + 43249100, + 46365700, + 44127800, + 43070000, + 43338600, + 42218600, + 43112300, + 44760600, + 40828800, + 38304700, + 37277200, + 37568300, + 38270500 + ], + "start_context": 2048, + "synchronizes_per_pass": 129, + "tokens": 128, + "tokens_per_second": 22.677497392176384, + "total_ns": 5644361800, + "v_bytes": 16777216, + "v_scatter_ns": 5193900, + "weight_creates_after_warmup": 0 + } + ] + }, + "identity": { + "corelib_compiled_version": "0.1.0", + "corelib_dll_path": "C:\\Users\\chiz\\work\\aie4-runtime-mirrored\\ryzenai_corelib.dll", + "corelib_dll_sha256": "a523b23837c9bea3b06f8f8ae74e1e8292092eb73aaa93791022c448b865ecdb", + "corelib_source_revision": "e5258d29b5cb979d4a538994409b90ceff6e6e7a-untracked-only", + "corelib_version": "0.1.0", + "cpu_sku": "AMD Eng Sample: 100-000001713-33_N", + "dynamic_dispatch_version": "9999.0.0.0 (C:\\Users\\chiz\\work\\aie4-runtime-mirrored\\dyn_dispatch_core.dll)", + "fastflow_revision": "a975b6394223f322532e158286894894a9c6715a-dirty", + "machine": "xcomedusad-43", + "model_dir": "C:\\Users\\chiz\\work\\FastFlowLM\\src\\build\\phi4-hardware\\model", + "model_files": { + "corelib_phi4_manifest.json": { + "sha256": "09cee6efafc513a2048c89e75b40d92d096144f0eed3b2459c138032b69dc045" + }, + "model.onnx": { + "sha256": "e80b9d83e018784eda6263d09fa2ab7729087722c6073c72123366f2dbec4529", + "size": 378325 + }, + "model.onnx.data": { + "sha256": "c48fd647beb02866e68d6dd1fbc05809a439283501d8abb7f6d6b950f92b7b60", + "size": 3248488448 + } + }, + "model_hash_verified": true, + "model_hash_verify_ns": 21683222400, + "model_sha256": "d6f503a9ea142c8b6320313d6ae341a88049b1b8ef01e641b2313fe42cdc7309", + "npu_driver_version": "32.0.20214.4161", + "npu_sku": "AMD XDNA(TM) NPU", + "prompt_id": "ff6a21dcdcff126f", + "prompt_source": "C:\\Users\\chiz\\work\\FastFlowLM\\src\\test\\phi4_corelib_aie4\\phi4_tokens.json", + "ryzen_mm_version": "0.10.2.0 (C:\\Users\\chiz\\work\\aie4-runtime-mirrored\\ryzen_mm.dll)", + "utc": "2026-09-03T08:20:08Z", + "xrt_version": "32.0.20214.4161 (C:\\Windows\\SYSTEM32\\xrt_coreutil.dll)" + }, + "memory": { + "device_tensor_creates_after_warmup": 0, + "device_tensors_created_after_load": 0, + "device_tensors_created_total": 76, + "embedding_bytes": 1229193216, + "kv_bytes": 536870912, + "live_corelib_object_delta": 0, + "live_corelib_objects_after_load": 238, + "live_corelib_objects_now": 238, + "mapped_source_bytes": 3248488448, + "packed_weight_bytes": 2098112512, + "peak_private_bytes": 8244830208, + "peak_working_set_bytes": 10308235264, + "private_bytes_at_end": 8079872000, + "private_bytes_at_warmup": 8106467328, + "private_bytes_before_load": 13795328, + "private_bytes_delta": -26595328, + "private_bytes_growth": 0, + "private_bytes_slope_exact": -305775.1614973263, + "private_bytes_slope_per_token": -305775, + "samples": [ + { + "private_bytes": 8106467328, + "working_set_bytes": 10169724928 + }, + { + "private_bytes": 8106467328, + "working_set_bytes": 10169733120 + }, + { + "private_bytes": 8106467328, + "working_set_bytes": 10169733120 + }, + { + "private_bytes": 8106467328, + "working_set_bytes": 10169733120 + }, + { + "private_bytes": 8106467328, + "working_set_bytes": 10169733120 + }, + { + "private_bytes": 8106467328, + "working_set_bytes": 10169733120 + }, + { + "private_bytes": 8106467328, + "working_set_bytes": 10169733120 + }, + { + "private_bytes": 8106467328, + "working_set_bytes": 10169733120 + }, + { + "private_bytes": 8106467328, + "working_set_bytes": 10169733120 + }, + { + "private_bytes": 8106467328, + "working_set_bytes": 10169733120 + }, + { + "private_bytes": 8106467328, + "working_set_bytes": 10169741312 + }, + { + "private_bytes": 8106467328, + "working_set_bytes": 10169741312 + }, + { + "private_bytes": 8106467328, + "working_set_bytes": 10169749504 + }, + { + "private_bytes": 8106467328, + "working_set_bytes": 10169749504 + }, + { + "private_bytes": 8106467328, + "working_set_bytes": 10169749504 + }, + { + "private_bytes": 8106467328, + "working_set_bytes": 10169749504 + }, + { + "private_bytes": 8106467328, + "working_set_bytes": 10169749504 + }, + { + "private_bytes": 8106467328, + "working_set_bytes": 10169749504 + }, + { + "private_bytes": 8106467328, + "working_set_bytes": 10169749504 + }, + { + "private_bytes": 8106467328, + "working_set_bytes": 10169749504 + }, + { + "private_bytes": 8106467328, + "working_set_bytes": 10169757696 + }, + { + "private_bytes": 8106467328, + "working_set_bytes": 10169765888 + }, + { + "private_bytes": 8106467328, + "working_set_bytes": 10169765888 + }, + { + "private_bytes": 8106467328, + "working_set_bytes": 10169774080 + }, + { + "private_bytes": 8106467328, + "working_set_bytes": 10169774080 + }, + { + "private_bytes": 8106467328, + "working_set_bytes": 10169774080 + }, + { + "private_bytes": 8106467328, + "working_set_bytes": 10169774080 + }, + { + "private_bytes": 8106467328, + "working_set_bytes": 10169774080 + }, + { + "private_bytes": 8106467328, + "working_set_bytes": 10169782272 + }, + { + "private_bytes": 8106467328, + "working_set_bytes": 10169790464 + }, + { + "private_bytes": 8106467328, + "working_set_bytes": 10169798656 + }, + { + "private_bytes": 8106467328, + "working_set_bytes": 10169798656 + }, + { + "private_bytes": 8106467328, + "working_set_bytes": 10169798656 + }, + { + "private_bytes": 8106467328, + "working_set_bytes": 10169798656 + }, + { + "private_bytes": 8106467328, + "working_set_bytes": 10169798656 + }, + { + "private_bytes": 8106467328, + "working_set_bytes": 10169806848 + }, + { + "private_bytes": 8106467328, + "working_set_bytes": 10169815040 + }, + { + "private_bytes": 8106467328, + "working_set_bytes": 10169815040 + }, + { + "private_bytes": 8106467328, + "working_set_bytes": 10169815040 + }, + { + "private_bytes": 8106467328, + "working_set_bytes": 10169815040 + }, + { + "private_bytes": 8106467328, + "working_set_bytes": 10169815040 + }, + { + "private_bytes": 8106467328, + "working_set_bytes": 10169823232 + }, + { + "private_bytes": 8106467328, + "working_set_bytes": 10169823232 + }, + { + "private_bytes": 8106467328, + "working_set_bytes": 10169823232 + }, + { + "private_bytes": 8106467328, + "working_set_bytes": 10169831424 + }, + { + "private_bytes": 8106467328, + "working_set_bytes": 10169839616 + }, + { + "private_bytes": 8106467328, + "working_set_bytes": 10169839616 + }, + { + "private_bytes": 8106467328, + "working_set_bytes": 10169839616 + }, + { + "private_bytes": 8106467328, + "working_set_bytes": 10169839616 + }, + { + "private_bytes": 8106467328, + "working_set_bytes": 10169839616 + }, + { + "private_bytes": 8106467328, + "working_set_bytes": 10169847808 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153406464 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153414656 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153414656 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153422848 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153431040 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153431040 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153431040 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153431040 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153431040 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153439232 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153447424 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153455616 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153455616 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153455616 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153455616 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153463808 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153463808 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153463808 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153463808 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153463808 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153463808 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153472000 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153472000 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153472000 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153472000 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153472000 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153472000 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153472000 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153472000 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153472000 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153472000 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153472000 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153480192 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153480192 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153480192 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153480192 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153480192 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153480192 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153480192 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153480192 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153480192 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153480192 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153480192 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153480192 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153480192 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153488384 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153496576 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153496576 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153496576 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153496576 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153496576 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153504768 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153504768 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153504768 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153504768 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153504768 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153512960 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153521152 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153521152 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153529344 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153537536 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153537536 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153545728 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153545728 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153545728 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153545728 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153545728 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153545728 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153545728 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153545728 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153545728 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153545728 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153545728 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153553920 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153553920 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153553920 + }, + { + "private_bytes": 8079872000, + "working_set_bytes": 10153553920 + } + ], + "scratch_bytes": 169751296, + "slope_first_token": 9, + "source": "decode@2048", + "tokens": 128, + "weight_creates_after_warmup": 0, + "weight_objects_created_after_load": 0, + "weight_objects_created_total": 161 + }, + "model_load": { + "benchmark_second_shape_plan_ns": 10879859400, + "device_setup_ns": 120906300, + "device_tensors": 76, + "engine_reported_total_ns": 12796321600, + "manifest_map_ns": 6402100, + "mapped_source_bytes": 3248488448, + "packed_weight_bytes": 2098112512, + "shape_plan_ns": 11061580200, + "shape_plan_share": 0.8644332890998437, + "total_ns": 12796337600, + "unaccounted_ns": 11402200, + "weight_objects": 161, + "weight_pack_ns": 1596046800 + }, + "prefill": { + "points": [ + { + "ns": 129379800, + "padded_rows": 1, + "rows": 1, + "tokens_per_second": 7.729181835186019, + "top1_id": 4576, + "v_bytes": 131072, + "v_scatter_ns": 73800 + }, + { + "ns": 50950500, + "padded_rows": 64, + "rows": 2, + "tokens_per_second": 39.25378553694272, + "top1_id": 477, + "v_bytes": 262144, + "v_scatter_ns": 72900 + }, + { + "ns": 70033100, + "padded_rows": 128, + "rows": 65, + "tokens_per_second": 928.1325544635322, + "top1_id": 9822, + "v_bytes": 8519680, + "v_scatter_ns": 264700 + }, + { + "ns": 86177800, + "padded_rows": 256, + "rows": 129, + "tokens_per_second": 1496.9052354550709, + "top1_id": 6864, + "v_bytes": 16908288, + "v_scatter_ns": 536200 + }, + { + "ns": 132412500, + "padded_rows": 512, + "rows": 257, + "tokens_per_second": 1940.904370810913, + "top1_id": 200020, + "v_bytes": 33685504, + "v_scatter_ns": 1237100 + }, + { + "ns": 301184000, + "padded_rows": 1024, + "rows": 513, + "tokens_per_second": 1703.277730556736, + "top1_id": 200019, + "v_bytes": 67239936, + "v_scatter_ns": 2810900 + }, + { + "ns": 740182800, + "padded_rows": 2048, + "rows": 1025, + "tokens_per_second": 1384.7930538240014, + "top1_id": 374, + "v_bytes": 134348800, + "v_scatter_ns": 8227100 + }, + { + "ns": 1158438500, + "padded_rows": 3072, + "rows": 2049, + "tokens_per_second": 1768.760275146242, + "top1_id": 44643, + "v_bytes": 268566528, + "v_scatter_ns": 17153100 + }, + { + "ns": 1698833300, + "padded_rows": 4096, + "rows": 3073, + "tokens_per_second": 1808.8884883525652, + "top1_id": 791, + "v_bytes": 402784256, + "v_scatter_ns": 25868300 + }, + { + "ns": 1748592400, + "padded_rows": 4096, + "rows": 4096, + "tokens_per_second": 2342.4555659740945, + "top1_id": 350, + "v_bytes": 536870912, + "v_scatter_ns": 35805600 + } + ] + }, + "ttft": { + "cold_ns": 3061714800, + "cold_top1_id": 1715, + "prompt_id": "ff6a21dcdcff126f", + "prompt_source": "C:\\Users\\chiz\\work\\FastFlowLM\\src\\test\\phi4_corelib_aie4\\phi4_tokens.json", + "prompt_token_count": 19, + "row_extent": 64, + "stream_rebuilt_for_warm": false, + "warm_ns": 53531800, + "warm_top1_id": 1715 + }, + "v_scatter": { + "bytes": 47889514496, + "counts_are_measured": true, + "design_18_5_expected_reads": 32, + "design_18_5_expected_writes": 256, + "model_steps": 7227, + "nanoseconds": 3081716100, + "reads_per_model_step": 32, + "reads_per_model_step_source": "v_read_calls / (synchronize_count / 129)", + "total_read_calls": 231264, + "total_write_calls": 1850112, + "writes_per_model_step": 256 + } +} \ No newline at end of file diff --git a/docs/docs/benchmarks/phi4_aie4_crossover_history.json b/docs/docs/benchmarks/phi4_aie4_crossover_history.json new file mode 100644 index 00000000..a9802511 --- /dev/null +++ b/docs/docs/benchmarks/phi4_aie4_crossover_history.json @@ -0,0 +1,179 @@ +[ + { + "utc": "2026-09-02T13:12:13Z", + "fastflow_revision": "00332b995ce2da95b2bf0309f1d06615fc58d681", + "corelib_dll_sha256": "a523b23837c9bea3b06f8f8ae74e1e8292092eb73aaa93791022c448b865ecdb", + "samples_interleaved": null, + "suffix_grid": [ + 1, + 2, + 32, + 128, + 256 + ], + "edges": { + "2048": [ + 2, + null + ], + "512": [ + 2, + null + ] + }, + "points_decided": 0, + "points_total": 0, + "undecided_suffixes": {}, + "source": "git:47680585:docs/docs/benchmarks/phi4_aie4_baseline.json", + "note": "sparse five-point grid; reported a single threshold, retracted as a grid artifact (report section 12). Not comparable." + }, + { + "utc": "2026-09-02T14:55:44Z", + "fastflow_revision": "21ad125ffdea96341d7bf898348dcccc1c26e0ad-untracked-only", + "corelib_dll_sha256": "a523b23837c9bea3b06f8f8ae74e1e8292092eb73aaa93791022c448b865ecdb", + "samples_interleaved": null, + "suffix_grid": [ + 1, + 2, + 4, + 8, + 12, + 16, + 24, + 32, + 64, + 128, + 256 + ], + "edges": { + "2048": [ + 12, + 16 + ], + "512": [ + 4, + 8 + ] + }, + "points_decided": 21, + "points_total": 22, + "undecided_suffixes": { + "2048": [ + 32 + ] + }, + "source": "git:3c532e80:docs/docs/benchmarks/phi4_aie4_baseline.json" + }, + { + "utc": "2026-09-02T15:34:01Z", + "fastflow_revision": "26f2dcb5d2d8d8b7cd2915f5a56db94e5a68fb7d-untracked-only", + "corelib_dll_sha256": "a523b23837c9bea3b06f8f8ae74e1e8292092eb73aaa93791022c448b865ecdb", + "samples_interleaved": true, + "suffix_grid": [ + 1, + 2, + 4, + 8, + 12, + 16, + 24, + 32, + 64, + 128, + 256 + ], + "edges": { + "2048": [ + 12, + 16 + ], + "512": [ + 4, + 8 + ] + }, + "points_decided": 22, + "points_total": 22, + "undecided_suffixes": {}, + "source": "git:1d573e30:docs/docs/benchmarks/phi4_aie4_baseline.json" + }, + { + "utc": "2026-09-02T16:32:18Z", + "fastflow_revision": "a02a2cf7e6cda62ab21e1383de2674b933ed2b74-untracked-only", + "corelib_dll_sha256": "a523b23837c9bea3b06f8f8ae74e1e8292092eb73aaa93791022c448b865ecdb", + "samples_interleaved": true, + "suffix_grid": [ + 1, + 2, + 4, + 8, + 12, + 16, + 24, + 32, + 64, + 128, + 256 + ], + "edges": { + "2048": [ + 12, + 64 + ], + "512": [ + 4, + 12 + ] + }, + "points_decided": 18, + "points_total": 22, + "undecided_suffixes": { + "2048": [ + 16, + 24, + 32 + ], + "512": [ + 8 + ] + }, + "source": "git:90300b0a:docs/docs/benchmarks/phi4_aie4_baseline.json" + }, + { + "utc": "2026-09-03T08:20:08Z", + "fastflow_revision": "a975b6394223f322532e158286894894a9c6715a-dirty", + "corelib_dll_sha256": "a523b23837c9bea3b06f8f8ae74e1e8292092eb73aaa93791022c448b865ecdb", + "samples_interleaved": true, + "suffix_grid": [ + 1, + 2, + 4, + 8, + 12, + 16, + 24, + 32, + 64, + 128, + 256 + ], + "edges": { + "2048": [ + 24, + 32 + ], + "512": [ + 4, + 12 + ] + }, + "points_decided": 21, + "points_total": 22, + "undecided_suffixes": { + "512": [ + 8 + ] + }, + "source": "git:1b6db540:docs/docs/benchmarks/phi4_aie4_baseline_task15_rerun.json" + } +] \ No newline at end of file diff --git a/docs/docs/benchmarks/phi4_results.md b/docs/docs/benchmarks/phi4_results.md index 7d7de5ea..30599ff7 100644 --- a/docs/docs/benchmarks/phi4_results.md +++ b/docs/docs/benchmarks/phi4_results.md @@ -41,3 +41,412 @@ AMD Ryzen™ AI 7 350 (Kraken Point) with 32 GB DRAM; performance is comparable | **Model** | **HW** | **1k** | **2k** | **4k** | **8k** | **16k** | **32k** | |------------------|--------------------|--------:|--------:|--------:|--------:|---------:|---------:| | **Phi-4-mini-instruct** | NPU (FLM) | 643 | 787 | 857 | 809 | 644 | 447 | + + +--- + + + +## Phi-4 mini instruct on AIE4 (corelib backend) — recorded baseline + +Recorded per design section 15.6, **without pass/fail thresholds**. Design section 4 makes performance explicitly not a release blocker for this release; these figures exist so a later change can be compared against a measured starting point, not so a number can be defended. + +> **Every latency and throughput figure here comes from ONE run on a shared lab machine.** Task 13 ran this same benchmark three times against the same binary and the same model within two hours and measured decode throughput of 22.5, 22.4 and 12.4 tokens/s at context 128 — a factor of 1.8, with no code change. Within a single run, per-token append latency stepped from 76 ms to 46 ms partway through the continuation sweep and stayed there. The machine runs corporate endpoint agents whose scans are not under this project's control, and the host share of a decode token is large enough for CPU contention to show. + +> **Two different comparisons, and they do not share a rule.** Comparing a figure here against one from a DIFFERENT run — a later revision, another machine, this document a month from now — is subject to that 1.8x instability, so treat a difference below roughly 2x as unresolved unless it is reproduced across runs. The append-versus-re-prefill decision AT EACH POINT is not that comparison: its samples are **interleaved within a single point**, so a regime shift moves both routes together, and each point additionally has to beat the drift measured across it. That is why a single point can resolve differences the 2x rule could not. + +> **The exemption is for a single point, not for the bracket.** Whether a given suffix can be decided depends on how quiet the machine was during that point, so the SET of decided points — and therefore the bracket's width — is subject to the same run-to-run instability as everything else here. The run-to-run table under "Where append stops winning" shows how far it has actually moved. + +### Identity + +| | | +| --- | --- | +| machine | `xcomedusad-43` | +| CPU | `AMD Eng Sample: 100-000001713-33_N` | +| NPU | `AMD XDNA(TM) NPU` | +| NPU driver | `32.0.20214.4161` | +| corelib DLL | `C:\Users\chiz\work\aie4-runtime-mirrored\ryzenai_corelib.dll` | +| corelib SHA-256 | `a523b23837c9bea3b06f8f8ae74e1e8292092eb73aaa93791022c448b865ecdb` | +| corelib version | `0.1.0` | +| corelib source revision | `e5258d29b5cb979d4a538994409b90ceff6e6e7a-untracked-only` | +| DynamicDispatch | `9999.0.0.0 (C:\Users\chiz\work\aie4-runtime-mirrored\dyn_dispatch_core.dll)` | +| RyzenMM | `0.10.2.0 (C:\Users\chiz\work\aie4-runtime-mirrored\ryzen_mm.dll)` | +| XRT | `32.0.20214.4161 (C:\Windows\SYSTEM32\xrt_coreutil.dll)` | +| model directory | `C:\Users\chiz\work\FastFlowLM\src\build\phi4-hardware\model` | +| model SHA-256 | `d6f503a9ea142c8b6320313d6ae341a88049b1b8ef01e641b2313fe42cdc7309` | +| FastFlow revision | `a02a2cf7e6cda62ab21e1383de2674b933ed2b74-untracked-only` | +| measured (UTC) | `2026-09-02T16:32:18Z` | + +### Model load and TTFT + +| | | +| --- | --- | +| manifest parse and file mapping | 4.5 ms | +| **1..4096 helper interrogation (`Phi4ShapePlan::Build`)** | **11282.6 ms** — **86% of load** | +| weight pack/upload (161 objects) | 1603.4 ms | +| stream, 76 device tensors, RoPE upload | 142.3 ms | +| unaccounted | 11.1 ms | +| total model load | 13043.9 ms | +| cold TTFT (19 prompt tokens, row extent 64) | 3169.4 ms | +| warm TTFT, same Stream after `clear_context()` | 49.1 ms | + +### Fresh prefill + +| rows | padded rows | wall time | tokens/s | +| ---: | ---: | ---: | ---: | +| 1 | 1 | 144.7 ms | 6.9 | +| 2 | 64 | 50.5 ms | 39.6 | +| 65 | 128 | 70.6 ms | 920.6 | +| 129 | 256 | 92.7 ms | 1,392.2 | +| 257 | 512 | 136.4 ms | 1,884.7 | +| 513 | 1024 | 289.4 ms | 1,772.5 | +| 1025 | 2048 | 731.3 ms | 1,401.6 | +| 2049 | 3072 | 1171.7 ms | 1,748.7 | +| 3073 | 4096 | 1717.8 ms | 1,788.9 | +| 4096 | 4096 | 1746.3 ms | 2,345.5 | + +### Decode + +| starting context | tokens | tokens/s | p50 | p95 | synchronizes per pass | +| ---: | ---: | ---: | ---: | ---: | ---: | +| 128 | 128 | 20.04 | 41.1 ms | 91.8 ms | 129 | +| 512 | 128 | 21.16 | 41.7 ms | 54.8 ms | 129 | +| 2048 | 128 | 20.68 | 44.2 ms | 55.8 ms | 129 | + +### Continuation routes + +| history rows | suffix | route | samples | p50 | p95 | +| ---: | ---: | --- | ---: | ---: | ---: | +| 512 | 1 | append | 5 | 48.9 ms | 52.9 ms | +| 512 | 1 | reprefill | 5 | 315.7 ms | 365.7 ms | +| 512 | 2 | append | 5 | 93.9 ms | 100.9 ms | +| 512 | 2 | reprefill | 5 | 332.0 ms | 343.5 ms | +| 512 | 4 | append | 5 | 182.5 ms | 192.3 ms | +| 512 | 4 | reprefill | 5 | 311.1 ms | 350.1 ms | +| 512 | 8 | append | 5 | 352.6 ms | 381.9 ms | +| 512 | 8 | reprefill | 5 | 334.1 ms | 336.2 ms | +| 512 | 12 | append | 5 | 513.2 ms | 530.7 ms | +| 512 | 12 | reprefill | 5 | 298.6 ms | 335.6 ms | +| 512 | 16 | append | 5 | 702.2 ms | 727.5 ms | +| 512 | 16 | reprefill | 5 | 318.0 ms | 391.0 ms | +| 512 | 24 | append | 5 | 1006.9 ms | 1048.9 ms | +| 512 | 24 | reprefill | 5 | 296.9 ms | 340.6 ms | +| 512 | 32 | append | 5 | 1429.6 ms | 1509.0 ms | +| 512 | 32 | reprefill | 5 | 308.3 ms | 335.4 ms | +| 512 | 64 | append | 5 | 2671.5 ms | 2750.8 ms | +| 512 | 64 | reprefill | 5 | 300.7 ms | 342.8 ms | +| 512 | 128 | append | 5 | 5301.3 ms | 5356.2 ms | +| 512 | 128 | reprefill | 5 | 311.2 ms | 344.9 ms | +| 512 | 256 | append | 5 | 10624.5 ms | 10841.4 ms | +| 512 | 256 | reprefill | 5 | 294.8 ms | 344.2 ms | +| 2048 | 1 | append | 5 | 54.7 ms | 61.7 ms | +| 2048 | 1 | reprefill | 5 | 1161.0 ms | 1176.0 ms | +| 2048 | 2 | append | 5 | 104.8 ms | 113.8 ms | +| 2048 | 2 | reprefill | 5 | 1152.3 ms | 1173.2 ms | +| 2048 | 4 | append | 5 | 196.4 ms | 203.7 ms | +| 2048 | 4 | reprefill | 5 | 1156.1 ms | 1162.1 ms | +| 2048 | 8 | append | 5 | 370.5 ms | 386.5 ms | +| 2048 | 8 | reprefill | 5 | 1152.1 ms | 1157.7 ms | +| 2048 | 12 | append | 5 | 547.3 ms | 589.4 ms | +| 2048 | 12 | reprefill | 5 | 1148.7 ms | 1156.0 ms | +| 2048 | 16 | append | 5 | 783.4 ms | 1536.1 ms | +| 2048 | 16 | reprefill | 5 | 1158.3 ms | 1190.8 ms | +| 2048 | 24 | append | 5 | 1229.6 ms | 2382.6 ms | +| 2048 | 24 | reprefill | 5 | 1147.3 ms | 1160.7 ms | +| 2048 | 32 | append | 5 | 1545.3 ms | 2234.5 ms | +| 2048 | 32 | reprefill | 5 | 1146.1 ms | 1173.4 ms | +| 2048 | 64 | append | 5 | 4158.2 ms | 4624.3 ms | +| 2048 | 64 | reprefill | 5 | 1156.3 ms | 1175.9 ms | +| 2048 | 128 | append | 5 | 5775.1 ms | 6003.7 ms | +| 2048 | 128 | reprefill | 5 | 1150.6 ms | 1180.5 ms | +| 2048 | 256 | append | 5 | 11376.5 ms | 13055.3 ms | +| 2048 | 256 | reprefill | 5 | 1160.7 ms | 1245.2 ms | + +#### Where append stops winning + +**This is a BRACKET, not a threshold.** Append and re-prefill samples are interleaved within each point, so a machine regime shift moves both together rather than one. A point counts as decided only when the gap between the routes exceeds **both** the larger within-point p50-to-p95 spread **and** the larger drift between a route's first and last sample there; anything else widens the bracket. Task 14 has to choose this constant, so what it needs to see is how much room the measurement leaves — not a number picked because it was the last grid point where append happened to win. + +| rendered history | append decisively wins up to | re-prefill decisively wins from | crossover lies in | margin at each bracket edge | +| ---: | ---: | ---: | :--- | :--- | +| 2048 | 12 | 64 | `(12, 64]` — not resolved further | 12: 14.3x, 64: 6.4x | +| 512 | 4 | 12 | `(4, 12]` — not resolved further | 4: 3.3x, 12: 5.8x | + +18 of 22 sweep points were decided; 4 were not and widen the brackets above. The margin column is the gap between the routes divided by the uncertainty it had to beat at that suffix. + +Undecided points, which is where the bracket's width comes from: history 2048 at suffix 16, 24, 32; history 512 at suffix 8. + +##### The same measurement, run to run + +**⚠ Corrected by hand; this block was NOT re-rendered.** The sentence generated here, and the guidance below the table, both said the bracket's lower edge was stable. A third interleaved run — `2026-09-03T08:20:08Z` — falsifies that. The two paragraphs that follow are what `report_phi4_corelib_baseline.py`'s own `crossover_edge_stability()` / `crossover_stability_narrative()` return when fed the updated history, pasted here verbatim because the generator will not re-render this block. **The table itself is unchanged and still shows only the runs that existed when it was generated**; the newer run is tabulated in [Second measurement, 2026-09-03](#second-measurement-2026-09-03). Read that section before citing anything in this one about edge stability. + +**Neither edge of the bracket is stable between runs.** This table is every render of this document that recorded a crossover, from the committed baseline artifacts. + +| measured (UTC) | interleaved | grid | history 2048 | history 512 | decided | note | +| --- | :---: | ---: | ---: | ---: | ---: | :--- | +| 2026-09-02T13:12:13Z | no | 5 points | threshold `2` | threshold `2` | n/a | sparse five-point grid; reported a single threshold, retracted as a grid artifact (report section 12). Not comparable. | +| 2026-09-02T14:55:44Z | no | 11 points | `(12, 16]` | `(4, 8]` | 21/22 | | +| 2026-09-02T15:34:01Z | yes | 11 points | `(12, 16]` | `(4, 8]` | 22/22 | | +| 2026-09-02T16:32:18Z | yes | 11 points | `(12, 64]` | `(4, 12]` | 18/22 | | + +> **Neither edge can be read as measured.** Across 3 interleaved runs of the same binary the lower edge has taken 12, 24 at history 2048; 4 at history 512 and the upper edge 16, 32, 64 at history 2048; 8, 12 at history 512. Whatever a single run reports is one observation on an unstable machine, and the crossover is not resolved by this data. + +2 of the 4 rows are non-interleaved and are excluded from that comparison, because they measured the routes in blocks rather than paired in time. They are shown for provenance. + +Prefix-monotonic: `True`. Decision rule: append and re-prefill samples are INTERLEAVED, so a machine regime shift moves both routes together; a point is decided only when the gap between the two routes' p50 exceeds BOTH the larger within-point p50-to-p95 spread AND the larger drift between a route's first and last sample at that point. The reported figure is a BRACKET, not a threshold.. + +### V scatter, memory and synchronization + +| | | +| --- | --- | +| V cache reads per model step | 32 | +| per-head V writes per model step | 256 | +| V bytes transferred | 44.60 GiB | +| V scatter wall time | 3315.9 ms | +| FP16 embedding | 1.14 GiB | +| KV cache | 512.00 MiB | +| corelib packed weights | 1.95 GiB | +| scratch and device tensors | 161.89 MiB | +| mapped ONNX source | 3.03 GiB | +| peak host private bytes | 7.67 GiB | +| peak host working set | 9.52 GiB | + +128-token stability window: device tensors created after warmup **0**, weight objects **0**, net live corelib objects **0**, private-byte growth **0 B**, least-squares private-byte slope over tokens 9..128 **-50.21 KiB/token**. + +### DETERM-3 — run-to-run logit bit-identity baseline + +Two runs of the same binary, same device, same explicit token IDs, proven by recorded SHA-256 to have loaded the same corelib DLL. Per `DETERM-3` the routes are reported separately and never averaged: append and re-prefill drive different row extents through the same LM-head shape. + +| route | runs (n) | step bit-identity | run bit-identity | max abs diff: max / median | nonzero runs | ≥ 20 runs? | +| --- | ---: | ---: | ---: | ---: | ---: | --- | +| append | 41 | 97.42% (n = 697) | 95.12% (n = 41) | 49.25 / 0.0 | 2 | yes | +| reprefill | 41 | 100.00% (n = 697) | 100.00% (n = 41) | 0.0 / 0.0 | 0 | yes | + +**This window is not a settled baseline.** The figures above are what has been measured; the problems listed below say why they cannot yet be cited as the floor `DETERM-2`'s "degrades from the recorded baseline" clause needs. + +Observed maximum absolute difference, `append`: `49.2148` × 1, `49.25` × 1. + +#### Where the divergence enters — measured + +At the step whose logits first differ, the harness records the exact row that was fed to the LM head in each run, so this is an observation and not an inference. **2 event(s): `model_body`**. + +> **The instrument perturbs what it measures.** These 2 localisation(s) were measured by capturing the LM-head input after every model step, which adds a host tensor read and a stream acquisition between steps — changing the timing of exactly the window a race would occupy. The phenomenon survives the instrumentation: it still reproduces, and with the same coarse signature. What this data cannot rule out is that the capture shifts the rate, or which step is reached first — the campaigns are far too small to detect either. + +In the **2** `model_body` event(s) the two runs fed the LM head **different rows** — 2,571, 2,754 of 3,072 elements differing, at `decode[6]`, `decode[8]`. **In those events the divergence enters the model body, not the LM-head dispatch.** Each run's LM head was separately measured to be correctly rounded against its own input, so it is faithfully transforming inputs that already differ. + +**The layer at which it enters is not known.** Layer 0's K and V have been bit-identical in the events where the emitted tokens matched, and layer 31's have not, which bounds it to somewhere above layer 0. Narrowing it further needs per-layer capture, and until then this is an open question rather than a characterised one. + +> **2 run(s) in this window breached a `DETERM-2` HARD GATE.** They are counted in the rates above -- dropping them would bias the figures upward, and silently -- and they are not a wobble within the recorded tolerance. Read the run records before citing anything here as a settled baseline. + +The two routes' observed rates differ, but **the n does not support calling the rate route-dependent** (Fisher exact p = 0.494 against 0.05). They are still reported separately, per `DETERM-3`, because pooling them would hide a difference that a larger campaign might resolve — not because this one resolved it. + +- 64 record(s) carry no FastFlow harness SHA-256 and were EXCLUDED from the pooled figures: they cannot be shown to describe the same binary as the rest. They remain committed as evidence and are listed under `excluded_sources`. +- route 'append': 2 run(s) recorded a DETERM-2 gate failure. They ARE counted in the rate above -- dropping them would bias it upward -- but a baseline should not be declared over a window containing a hard-gate failure. + + + + + +## Phi-4 continuation routing — the release-fixed threshold + +Design Section 10.7 fixes ONE integer for the release and applies it at every history length: append when the suffix is at most the threshold, clear and re-prefill when it is larger. It is a FastFlow backend constant — not model-package data, not a runtime calibration — so route choice does not drift with thermal state or load. + +```cpp +inline constexpr std::uint32_t kContinuationAppendThreshold = 4; +``` + +**Selected: 4.** Suffix lengths [1, 2, 4] are the sampled lengths whose append p95 is lower than re-prefill p95 at BOTH measured history lengths, and they form a prefix of the sampled set [1, 2, 4, 8, 12, 16, 24, 32, 64, 128, 256], so the largest of them is the threshold. + +### Which sampled lengths append wins, per history + +| history rows | append p95 wins at | last winning length | +| ---: | :--- | ---: | +| 512 | [1, 2, 4] | 4 | +| 2048 | [1, 2, 4, 8, 12] | 12 | + +The rule is a conjunction: the constant is the largest sampled length in the INTERSECTION of those winner sets — here [1, 2, 4], giving 4. It is not in general the smaller of the two ceilings; that holds only when each winner set is downward-closed, which is a property of the measurement rather than of the rule and is checked further down. It is also NOT a per-history policy: Section 10.7 specifies one integer, and one integer cannot be optimal at two history lengths whose crossovers differ. + +### What the single constant gives up + +**Worst case: history 2048, suffix 8 — 3.0x slower, +771.2 ms.** Every conceded point follows, ordered by slowdown. The widest conceded suffix is not the most expensive one, because append cost grows with suffix length while re-prefill cost does not — so the band's cost must be read off its narrow end. + +| history rows | suffix | append p95 | re-prefill p95 | route taken | penalty | slowdown | +| ---: | ---: | ---: | ---: | :--- | ---: | ---: | +| 2048 | 8 | 386.5 ms | 1,157.7 ms | re-prefill | +771.2 ms | 3.0x | +| 2048 | 12 | 589.4 ms | 1,156.0 ms | re-prefill | +566.6 ms | 2.0x | +| 512 | none | — | — | — | — | — | + +Those are measured losses taken deliberately. The alternative — a threshold above the selected one — is optimal for the longer history and WRONG for the shorter one, where it would append past the point at which append has already lost. Conceding measured throughput at one history is the cheaper error than routing against the measurement at the other. + +There is a second reason to prefer the lower candidate, and it is about confidence rather than cost. Across the 3 interleaved run(s) on record the bracket's LOWER edge has been 4 at history 512; 12, 24 at history 2048. It is NOT a measured constant: at history 2048 it took more than one value across runs of the same binary, so a threshold cannot be justified by pointing at one run's edge. What carries the choice is the inequality rather than the stability: 4 is at or below EVERY recorded lower edge, so it is inside the append-wins region on every interleaved run on record. A larger threshold would be supported by some of them and contradicted by others. + +### The measurement behind it + +| history rows | suffix | append p50 | append p95 | re-prefill p50 | re-prefill p95 | append p95 wins | +| ---: | ---: | ---: | ---: | ---: | ---: | :---: | +| 512 | 1 | 48.9 ms | 52.9 ms | 315.7 ms | 365.7 ms | yes | +| 512 | 2 | 93.9 ms | 100.9 ms | 332.0 ms | 343.5 ms | yes | +| 512 | 4 | 182.5 ms | 192.3 ms | 311.1 ms | 350.1 ms | yes | +| 512 | 8 | 352.6 ms | 381.9 ms | 334.1 ms | 336.2 ms | no | +| 512 | 12 | 513.2 ms | 530.7 ms | 298.6 ms | 335.6 ms | no | +| 512 | 16 | 702.2 ms | 727.5 ms | 318.0 ms | 391.0 ms | no | +| 512 | 24 | 1,006.9 ms | 1,048.9 ms | 296.9 ms | 340.6 ms | no | +| 512 | 32 | 1,429.6 ms | 1,509.0 ms | 308.3 ms | 335.4 ms | no | +| 512 | 64 | 2,671.5 ms | 2,750.8 ms | 300.7 ms | 342.8 ms | no | +| 512 | 128 | 5,301.3 ms | 5,356.2 ms | 311.2 ms | 344.9 ms | no | +| 512 | 256 | 10,624.5 ms | 10,841.4 ms | 294.8 ms | 344.2 ms | no | +| 2048 | 1 | 54.7 ms | 61.7 ms | 1,161.0 ms | 1,176.0 ms | yes | +| 2048 | 2 | 104.8 ms | 113.8 ms | 1,152.3 ms | 1,173.2 ms | yes | +| 2048 | 4 | 196.4 ms | 203.7 ms | 1,156.1 ms | 1,162.1 ms | yes | +| 2048 | 8 | 370.5 ms | 386.5 ms | 1,152.1 ms | 1,157.7 ms | yes | +| 2048 | 12 | 547.3 ms | 589.4 ms | 1,148.7 ms | 1,156.0 ms | yes | +| 2048 | 16 | 783.4 ms | 1,536.1 ms | 1,158.3 ms | 1,190.8 ms | no | +| 2048 | 24 | 1,229.6 ms | 2,382.6 ms | 1,147.3 ms | 1,160.7 ms | no | +| 2048 | 32 | 1,545.3 ms | 2,234.5 ms | 1,146.1 ms | 1,173.4 ms | no | +| 2048 | 64 | 4,158.2 ms | 4,624.3 ms | 1,156.3 ms | 1,175.9 ms | no | +| 2048 | 128 | 5,775.1 ms | 6,003.7 ms | 1,150.6 ms | 1,180.5 ms | no | +| 2048 | 256 | 11,376.5 ms | 13,055.3 ms | 1,160.7 ms | 1,245.2 ms | no | + +At least 5 warm samples per route per point, append and re-prefill interleaved WITHIN each point. p50 and p95 are nearest-rank and were recomputed from the raw samples by the calibrator, not copied. + +### Do the earlier runs permit this constant? + +Not: would they have selected it. Each recorded crossover run carries, per history, the bracket `[lower, upper]`, and the lower edge is that run's per-history winner CEILING. The minimum of those ceilings is an **upper bound** on what Section 10.7 would have selected from that run, and in general only an upper bound: the rule takes the largest suffix in the INTERSECTION of the winner sets, which equals the minimum of the ceilings only when each winner set is downward-closed. A crossover record stores the edges and never the winner sets, so that property cannot be checked from it. What holds with no assumption is the inequality — any suffix winning at every history is at most every ceiling — so that is what is enforced here. + +This bound touches only the lower edges and never the upper ones, so however far an upper edge moves between runs it cannot move the bound. The lower edges are printed per run below rather than reduced away, because their MINIMUM cannot answer whether the edge itself held still: where a history shows different values on different runs the edge moved, the bound is still their minimum, and the inequality is still sound — but nothing here may then call that edge stable. + +| measured (UTC) | interleaved | lower edge per history | bounds the threshold at | permits 4 | meets the bound exactly | +| --- | :---: | :--- | ---: | :---: | :---: | +| 2026-09-02T13:12:13Z | no | 512: 2, 2048: 2 | (2) | excluded — routes not measured against the same machine state | — | +| 2026-09-02T14:55:44Z | no | 512: 4, 2048: 12 | (4) | excluded — routes not measured against the same machine state | — | +| 2026-09-02T15:34:01Z | yes | 512: 4, 2048: 12 | 4 | yes | yes | +| 2026-09-02T16:32:18Z | yes | 512: 4, 2048: 12 | 4 | yes | yes | +| 2026-09-03T08:20:08Z | yes | 512: 4, 2048: 24 | 4 | yes | yes | + +Every interleaved run on record permits 4, and 3 of 3 bound it there exactly. An exactly-met bound is what "that run would have selected the same constant" would need, but only under the downward-closure assumption above, which its record does not attest. + +This run's own per-history winner sets ARE downward-closed at every history, checked directly from the sampled points rather than assumed. That is only a property of this run; it says nothing about the earlier ones, whose winner sets were never recorded. + +The non-interleaved rows are shown for provenance and excluded from the verdict, because their two routes were measured in separate blocks and a machine regime shift there lands on one route and not the other. The first of them bounds the constant lower for a reason that has nothing to do with the machine: it swept only the five suffix lengths Section 10.7 names, and on `{1, 2, 32, 128, 256}` the last winning sampled length is 2 whether the true crossover is at 3 or at 31. **Two earlier answers are withdrawn and must not be reused: the threshold `2` from that sparse grid, and the extrapolated figures `≈9 at history 512, ≈26 at history 2048` from a contended non-interleaved run.** + +**The selection RULE is Section 10.7's, verbatim and unqualified; the measurement GRID is wider than Section 10.7 specifies, deliberately.** Those are two different things and only one of them changed. The rule — largest suffix whose append p95 is lower at both history lengths, after asserting the winners are prefix-contiguous — was applied as written. The grid was not: Section 10.7 names five suffix lengths, and on those five alone this same measurement yields 2, the answer since retracted as a grid artifact. Locating a crossover needs sample points near it, so the five named lengths are treated as a floor and every additional measured length is used. The measurement plan was corrected; the decision rule was not touched. + +### Identity of the run this constant came from + +| | | +| --- | --- | +| machine | `xcomedusad-43` | +| CPU | `AMD Eng Sample: 100-000001713-33_N` | +| NPU | `AMD XDNA(TM) NPU` | +| NPU driver | `32.0.20214.4161` | +| corelib SHA-256 | `a523b23837c9bea3b06f8f8ae74e1e8292092eb73aaa93791022c448b865ecdb` | +| corelib source revision | `e5258d29b5cb979d4a538994409b90ceff6e6e7a-untracked-only` | +| model SHA-256 | `d6f503a9ea142c8b6320313d6ae341a88049b1b8ef01e641b2313fe42cdc7309` | +| FastFlow revision | `a02a2cf7e6cda62ab21e1383de2674b933ed2b74-untracked-only` | +| measured (UTC) | `2026-09-02T16:32:18Z` | + +**"Release-fixed" does not mean hardware-independent.** Every measurement behind this constant comes from the single machine, corelib build and model named above. Nothing here establishes where the crossover sits on different silicon, on a corelib whose append or prefill path changed, or on a different model. Section 10.7 fixes the constant for the release rather than calibrating at run time, so that scope is intended — but a port to other hardware needs this re-measured, not inherited. + +The timing table above is published here and is NOT shipped in the model package: Section 10.7 carries the threshold and no cost table. + + + +## Second measurement, 2026-09-03 + +Hand-written, outside both generated blocks above. The measurements are +committed at `phi4_aie4_baseline_task15_rerun.json` beside this file; every +figure here is derived from that artifact rather than typed. + +Same machine, same corelib DLL (`a523b238…`), same model (`d6f503a9…`). It is a +second independent measurement of the run rendered above, taken a day later. + +| | 2026-09-02 (rendered above) | 2026-09-03 (this run) | +| --- | ---: | ---: | +| model load, total | 13,043.9 ms | 12,796.3 ms | +| — helper interrogation share | 86.5% | 86.4% | +| cold TTFT (19 tokens) | 3,169.4 ms | 3,061.7 ms | +| warm TTFT | 49.1 ms | 53.5 ms | +| prefill 4096 rows | 1,746.3 ms / 2,345.5 tok/s | 1,748.6 ms / 2,342.5 tok/s | +| decode @128 | 20.04 tok/s | 24.89 tok/s | +| decode @512 | 21.16 tok/s | 24.38 tok/s | +| decode @2048 | 20.68 tok/s | 22.68 tok/s | +| peak host private | 7.67 GiB | 7.68 GiB | +| V bytes / time | 44.60 GiB / 3,315.9 ms | 44.60 GiB / 3,081.7 ms | + +Load, prefill, memory and V-scatter reproduce within a few percent. Decode is +**20–24% faster** in this run — within the 1.8x instability this document +already warns about, and in the direction that costs nobody anything. + +### Two things this run changes + +**1. The published crossover bracket's LOWER edge is not as stable as the table +above says.** That table concluded *"the lower edge has been 12 at history +2048; 4 at history 512 every time"*, over two interleaved runs. This is a third +interleaved run, and at history 2048 its lower edge is **24**, not 12. At +history 512 it is 4 again. + +| interleaved run | history 512 | history 2048 | +| --- | ---: | ---: | +| 2026-09-02T15:34:01Z | `(4, 8]` | `(12, 16]` | +| 2026-09-02T16:32:18Z | `(4, 12]` | `(12, 64]` | +| **2026-09-03T08:20:08Z** | **`(4, 12]`** | **`(24, 32]`** | + +So the stability claim holds at history 512 on three runs and **does not hold** +at history 2048. The generated statement above has been corrected in place, by +hand, and says so where it is corrected. + +**This run is in the file the tooling reads.** It is the fifth record in +`phi4_aie4_crossover_history.json`, every field derived from +`phi4_aie4_baseline_task15_rerun.json` by `report_phi4_corelib_baseline.py`'s +own `crossover_entry()` rather than typed in, and a test fails if it ever goes +missing. That matters because a measurement recorded only in prose is not an +input to anything: while it lived only in this paragraph, the calibrator kept +cross-checking the shipped constant against only the two runs that agree. + +**2. On this run alone, Section 10.7's rule would select 8, not the shipped 4 — +and the calibrator refuses on two independent grounds, not one.** Run against +this measurement it selects `8` and then **refuses to write anything**. Both +`failed` conditions fire (`tools/calibrate_phi4_corelib_continuation.py:1177-1180`): + +> diagnostic: history 512: the recorded bracket says append wins up to 4, the Section 10.7 p95 rule says 8 +> +> diagnostic: the constant exceeds what an interleaved run can support: 2026-09-02T15:34:01Z bounds it at 4, 2026-09-02T16:32:18Z bounds it at 4, 2026-09-03T08:20:08Z bounds it at 4, and the selected threshold is 8. At that run, some history had already stopped favouring append below this suffix length. A release-fixed constant that one recorded run contradicts is a decision for a human, not for this script. + +The second is the cross-run check. The **first** is `crossover_disagreements` +(`:339-367`) and needs no other run at all: this run's own recorded bracket says +append wins up to 4 at history 512, and the p95 rule applied to this run's own +samples says 8. Two statistics disagree about one measurement. + +**The 8 rests entirely on one comparison** — the single point history 512, +suffix 8: + +| history 512, suffix 8 | append | re-prefill | gap | +| --- | ---: | ---: | ---: | +| p95 | 337.6 ms | 351.7 ms | 14.2 ms, to **append** | +| p50 | 332.5 ms | 332.0 ms | 0.4 ms, to **re-prefill** | + +The p95 row is the whole of why 8 is a winner. Its p50 counterpart runs the other +way by 0.4 ms against **19.7 ms** of measurement uncertainty at that point — the +run's own record marks it `"decided": false`, `uncertainty_ns` 19,683,000, +`gap_over_uncertainty` 0.02 — i.e. undecided under the bracket rule this document +uses everywhere else, and the reason the recorded bracket stops at 4. "On this +run the rule would select 8" is true, and it is one p95 comparison away from +selecting 4. + +**The shipped constant is unchanged at 4, and it is not wrong.** 4 is inside the +append-wins region on this run too — append still wins at suffixes 1, 2 and 4 at +both histories. What this run says is that 4 concedes more than it needed to +*here*, which is the deliberate trade the selection already documents: the lower +edge is the safe edge. Raising it is a recalibration decision for a human with +more than three runs, not something to change in a regression pass. The +calibrator's `--check` against the committed baseline passed in the same suite +run. + +### Why the block above still shows the earlier run + +The generated baseline is frozen at 2026-09-02 and this run is written beside +it rather than into it. The reason is the `DETERM-3` pooling rule, not a +measurement problem: the records behind it did not all share one harness +binary, and `DETERM-3` forbids pooling records from different binaries into one +baseline. Resolving that needs a decision about how the window is scoped, which +was deferred. See +[caveat 10](/docs/phi4_aie4_caveats/#10-the-determinism-baseline-is-frozen). diff --git a/docs/docs/models/phi.md b/docs/docs/models/phi.md index 9b2c59bf..28ce02fd 100644 --- a/docs/docs/models/phi.md +++ b/docs/docs/models/phi.md @@ -22,4 +22,348 @@ parent: Models flm run phi4-mini-it:4b ``` ---- \ No newline at end of file +--- + +## Phi-4 mini on Ryzen AI AIE4 + +The `phi4-mini-it-aie4:4b` tag uses the optional `corelib_aie4` +backend and the pinned AMD +[OGA DML package](https://huggingface.co/amd/phi-4-mini-instruct-oga-dml). +It requires FastFlowLM 1.0.4 or newer. + +**Size prompts against 4,094 tokens, not 4,096.** The catalog entry lists a +4096-token default and maximum prefill length, and the backend is stricter than +its catalog entry: the prompt and the generated tokens together are capped at +**4,095**, and a prompt is refused once it reaches that figure on its own, so +**the largest prompt this tag will accept is 4,094 tokens** — it needs at least +one token of room to answer in. Sizing a prompt to the advertised 4,096 gets +HTTP 400 before any work is submitted. The rule is +`Phi4::validate_aie4_capacity` (`src/common/AutoModel/modeling_phi4.cpp:420`), +which refuses when the rendered prompt length reaches the cap; see +[Known limitations](#known-limitations) for why the cap is what it is. + +**Windows and AIE4 only.** The backend is compiled in only when FastFlowLM is +built with `FLM_ENABLE_CORELIB_AIE4=ON`, the packaging step is Windows-only, +and the tag runs on an AIE4 NPU. There is no Linux build of it and it does not +run on XDNA2 hardware. + +**Building it against a shared dependency prefix also needs +`-DFLM_WIN_STATIC_DEPS=OFF`.** That option defaults to `ON`, which is the static +Boost and curl layout CI and the MSI ship; a conda or other shared prefix — the +usual way the AIE4 work is built — has to opt out, or the Boost linkage macros +and the library name the build picks describe two different Boosts. Getting it +wrong is a configure-time hard failure naming both halves, not a mystery link +error at the end. + +**The existing NPU2 tag is unchanged.** `phi4-mini-it:4b` still resolves to the +legacy `phi4_npu` engine, still uses the `Q4_1` `model.q4nx` package with a +32k default context, and is unaffected by any of this. The two tags are +separate catalog entries with separate model packages; installing one does not +change the other. + +Select **Phi-4 AIE4 corelib runtime** in the Windows installer, then use: + +```shell +flm pull phi4-mini-it-aie4:4b +flm run phi4-mini-it-aie4:4b +``` + +This package is hosted only on Hugging Face. `--modelscope` is rejected +before any download starts. + +### The runtime this tag needs, and what happens without it + +The AIE4 feature is an optional installer component. It installs +`ryzenai_corelib.dll` and its derived dependency closure into an `aie4` +directory beside `flm.exe`, together with `aie4-closure.txt` — a manifest +naming the corelib version, the corelib DLL's SHA-256, and the name and +SHA-256 of every DLL staged with it. If a result ever looks wrong, that file +is what identifies the runtime that produced it. + +`flm.exe` never links `ryzenai_corelib.lib`. The DLL is resolved at run time by +absolute path, so a FastFlowLM without the AIE4 component installed starts and +runs normally; only this one tag is unavailable. + +**There is no fallback.** If the backend is not there, the tag fails — it does +not quietly run on something else, and it does not silently produce answers +from a different engine: + +- A build without `FLM_ENABLE_CORELIB_AIE4` refuses the tag outright with + `This binary was built without Phi-4 AIE4 corelib support`, and + `flm validate --json` reports `corelib_aie4.available_in_build: false`. +- A build with the feature but without the runtime directory installed fails + to load the library and says so, naming the path it tried and the Win32 + error. + +`flm validate --json` is the way to check before running: it reports the +loader, the dependency closure, the device context, whether the fatal-record +directory is writable, and clean shutdown, each separately. + +The assembled model directory has two provenances. The weights, tokenizer and +vocabulary are downloaded from the pinned upstream revision and hash-checked +against its published metadata. Four further files are shipped inside +FastFlowLM rather than fetched, and they are not all the same kind of file: + +- **Authored, with no upstream counterpart.** `config.json`, + `corelib_phi4_manifest.json` and `provenance.json` do not exist in the + upstream repository at all. The upstream package ships no `config.json`, and + the other two describe FastFlowLM's own packaging. +- **Shadowing an upstream file.** `tokenizer_config.json` *does* exist + upstream, and the shipped copy replaces it, because the published one carries + neither a chat template nor the EOS token IDs this backend requires. + +The distinction is not cosmetic: an upstream metadata record is expected for +the shadowing file and is a provenance error for the authored ones, since it +would mean FastFlowLM's own package contract had been published to the model +repository. + +These files restate the model contract for FastFlowLM's existing readers; they +never override it, and a model whose weights disagree with them fails to load. +That last sentence is checked rather than asserted: editing a single value in +the shipped `config.json` makes `flm run phi4-mini-it-aie4:4b` refuse to load, +naming the field, the value it found and the value the ONNX initializers +require. + +### What has been verified on hardware + +This backend has been run end to end against the published model on an AMD +Ryzen AI NPU: a 20-step acceptance pass whose tally was **16 steps met, 1 not +met, 3 not exercised**, and **12 of 52 criteria met, 12 partial, 2 not met, 26 +not exercised**. + +Read that last number before relying on any claim here — half the criteria were +**not tested at all**. What was and was not established is summarised in +[Phi-4 AIE4 Implementation](/docs/phi4_aie4_implementation/); every known defect +and limit is in [Phi-4 AIE4 Caveats](/docs/phi4_aie4_caveats/). The step-by-step +record itself was an artifact of that one campaign and is not carried in the +repository — see caveat 11. + +### What it costs to run + +Two measured runs on the same machine (`xcomedusad-43`), a day apart, are shown +side by side rather than one being picked. This machine has been measured +moving by up to 1.8x on decode throughput with no code change, so a single +column would read as more precise than the measurement is. Treat a difference +below roughly 2x against any figure here as unresolved. + +| | 2026-09-02 | 2026-09-03 | +| --- | ---: | ---: | +| model load, total | 13,043.9 ms | 12,796.3 ms | +| — of which the 1..4096 helper interrogation | 11,282.6 ms (86%) | 11,061.6 ms (86%) | +| — of which weight pack and upload (161 objects) | 1,603.4 ms | 1,596.0 ms | +| cold TTFT (19-token prompt) | 3,169.4 ms | 3,061.7 ms | +| warm TTFT (same stream, context cleared) | 49.1 ms | 53.5 ms | +| prefill, 4096 rows † | 1,746.3 ms / 2,345.5 tok/s | 1,748.6 ms / 2,342.5 tok/s | +| decode from 128 tokens | 20.04 tok/s | 24.89 tok/s | +| decode from 512 tokens | 21.16 tok/s | 24.38 tok/s | +| decode from 2048 tokens | 20.68 tok/s | 22.68 tok/s | +| peak host private bytes | 7.67 GiB | 7.68 GiB | +| peak host working set | 9.52 GiB | 9.60 GiB | +| V-cache reads per model step | 32 | 32 | +| per-head V writes per model step | 256 | 256 | +| V-cache read calls, whole run | 231,264 | 231,264 | +| per-head V write calls, whole run | 1,850,112 | 1,850,112 | +| model steps those calls span | 7,227 | 7,227 | +| V bytes transferred | 44.60 GiB | 44.60 GiB | +| V scatter wall time | 3,315.9 ms | 3,081.7 ms | + +† The 4096-row prefill is measured by driving the engine directly. It is **not** +a prompt size you can submit — see the cap above: the largest admissible prompt +is 4,094 tokens. + +The three V-scatter call and step figures are counted by the engine, not inferred +from a rate: both baselines record `v_scatter.counts_are_measured: true`, and the +per-step figures in the two rows above them are those counts divided by +`model_steps`. They are identical in both runs because the workload is; only the +wall time moved. + +**Model load is dominated by one thing.** 86% of it is interrogating the +corelib helpers for every row extent from 1 to 4096. That is a fixed startup +cost, paid once per process, and it is why a cold `flm run` takes about 13 +seconds before the first token. + +**Accounted device-side memory**, identical in both runs: FP16 embedding +1.14 GiB, KV cache 512 MiB, corelib packed weights 1.95 GiB, and the ONNX +source mapped at 3.03 GiB. Those account for the bulk of the host peak above. + +**Memory is stable once warm.** Over a 128-token decode window after warmup, +both runs created **0** new device tensors, **0** new weight objects, and +recorded a net change of **0** live corelib objects and **0 bytes** of +private-byte growth. The least-squares slope over that window was negative in +every window measured — memory did not grow. + +**Continuation: append or re-prefill.** When you continue a conversation, the +backend either appends the new tokens to the existing KV cache or clears it and +re-prefills the whole rendered history. The choice is a fixed release constant, +not a runtime measurement: **append when the new suffix is at most 4 tokens, +otherwise re-prefill.** It does not drift with load or thermal state. + +The cost difference is large and runs the opposite way at each end. From the +2026-09-03 run, at a 2048-token history: appending 1 token took **54.2 ms** +against **1,150.1 ms** to re-prefill — 21x in append's favour — while appending +256 tokens took **10,422.5 ms** against the same **1,151.2 ms**. Append cost +grows with the suffix; re-prefill cost does not. + +The constant is deliberately set at the low end. It sits inside the +append-wins region on every run measured, whereas a higher value has not been +supported by every run — on one run the rule would have chosen 8, and on two +earlier ones it would not. The full per-suffix tables, and that disagreement, +are in +[the Phi-4 benchmarks](/docs/benchmarks/phi4_results/). + +**A terminal device failure restarts the process, by design.** If a corelib +call fails while work may still be outstanding on the device — or if a +synchronize itself fails — the state of that work is unknowable, so FastFlowLM +does not try to continue: it writes a diagnostic record and terminates the +process with exit code `0xE0040001`. Under `flm serve` that ends the server and +you restart it; under `flm run` the session ends. + +A failure at a point where nothing is outstanding is recoverable: the session +is cleared, an error is returned, and the process keeps running. That covers +both a failure before anything was submitted *and* a failure after a +synchronize has completed and drained the device, so a host-side error between +two dispatch groups no longer costs you the server — only the conversation. + +**Where the record goes.** `%LOCALAPPDATA%\FastFlowLM\logs`, as +`corelib-fatal--.json`, carrying the corelib status code, the +call that failed, the library's own message, the phase, layer, row count and +position. The next `flm` process on that machine prints any records it finds +and removes them, so the diagnostic reaches you on the next run rather than +only on disk. A record belonging to a process that is still alive is left +alone. + +### Known limitations + +- **Multi-turn conversation in `flm run` does not retain history.** Each turn + is answered as though it were the first: after telling the model a fact and + asking about it on the next turn, the model does not have it. `/history` + confirms it — the conversation it prints contains only the current exchange, + with the previous turn's message and reply absent, so each turn replaces the + history rather than appending to it. This is **not specific to this model or + to the AIE4 backend, and it is not an AIE4 defect.** `Phi4::insert` + (`common/AutoModel/modeling_phi4.cpp:576`) wraps `input.prompt` in a fresh + single-message array whenever `input.messages` is empty, which is every CLI + turn, and that happens at line 589 — *above* the + `#if defined(FLM_ENABLE_CORELIB_AIE4)` branch that starts at line 596. The + history is already gone before any AIE4 code runs. The identical construction + is in the Llama 3, Qwen 2, Qwen 3, Gemma 3, LFM2, GPT-OSS and Nanbeige + frontends: it is the shared CLI path for every text model. Supply the whole + conversation yourself — the `/api/chat` and `/v1/chat/completions` endpoints + take a full `messages` array and count it in full — until this is fixed. +- **Two runs of the same binary, on the same input, have produced different + text.** This backend is **not deterministic run to run**, and the difference + is not a rounding difference in the last bits — it is a different answer. In + the committed determinism campaign, **3 of the 150 recorded run-pair + comparisons** ended with the two runs emitting different token sequences. One + pair parts at the **8th** emitted token, the other two at the 14th. Two + measures of severity are recorded and **they do not rank the three records + the same way**, so each figure below is given against the record it came + from rather than rolled into one "worst": the largest absolute logit + difference is **49.25**, while the largest number of logits moving at a + single step is **196,835 of 200,064** — and those are different records. + Each comparison was a JSON record written by the campaign. The 150 records + are not carried in this branch — they are the output of one campaign on one + machine, and nothing in the tree reads them — so the three that diverged are + summarised here rather than cited by path: + + - `determ1-force_append-010` — max abs diff **48.34**; up to **196,154** of + 200,064 logits differing at one step (`decode[13]`) by more than the + 2-BF16-ULP bound the suite gated on; sequences part at token 8 + - `determ1-force_append-022` — **49.25**; up to **196,748** at `decode[15]`; + part at token 14 + - `determ1-force_append-026` — **49.21**; up to **196,835** at `decode[15]`; + part at token 14 + + All three are on the **append** continuation route. Two re-prefill records + also carry a logit difference, but in both the emitted text still matched. + The pooled rates, the two runs that breached a `DETERM-2` hard gate, and the + measurement showing that the divergence enters the **model body** rather than + the LM-head dispatch are published in + [the Phi-4 benchmarks](/docs/benchmarks/phi4_results/), which reports them + honestly and refuses to call that window a settled baseline. Nothing here is + a fix: the cause is not known, and no run has been made to find it. + + **What this means for you:** do not assume the same prompt gives the same + answer, do not use output equality as a test oracle against this backend, and + do not cache or diff on the assumption that a repeat is a repeat. +- **Long, open-ended generations can collapse. Set an explicit generation + limit — and know which interface gives you one.** There is **no command-line + flag**: nothing in `src/include/utils/vm_args.hpp` takes a generation limit, + so `flm run phi4-mini-it-aie4:4b` cannot be launched with one. Two mechanisms + exist, and they cover both ways of using this tag: + - **In the `flm run` session, type `/set gen-lim `** before the prompt + you want bounded (`src/runner/runner.cpp:643`). It sets the per-round token + limit for the rest of the session, and the AIE4 admission check reads it + (`CliRequestedMaxNewTokens`, `src/server/generation_limit.cpp:176-182`). + It is a REPL command, not a startup option, so it has to be typed each + session; `/set` on its own prints the list it appears in + (`src/runner/runner.cpp:572-586`). + - **Over HTTP, send `max_tokens` (OpenAI and `/api/generate`) or + `options.num_predict` (`/api/chat`)** — `src/server/generation_limit.cpp:65,73`. + + A request with no limit is bounded only by the model emitting an end + token. When it does not, generation runs to the cap. In the acceptance run, + two open-ended turns did exactly that, taking **186.9 s** and **179.9 s**; + neither emitted a stop token and both ended on *"Max length reached, stopping + generation"*. Their prompts were tiny — 13 and 9 tokens — so the 4,095 tokens + were output alone. A limit is the difference between a bounded reply and + three minutes of unusable text. + + **Normal-length output is sound, and that is the larger part of the + evidence.** Five separate single-turn prompts in the same acceptance run — + arithmetic, a list, a Shakespeare quotation, a definition, a factual + question — all returned correct, well-formed answers in about ten seconds + each. The collapse has been seen in long generations only. + + Two distinct things happen, and only the first is ordinary: + + - **Both** long turns entered a **phrase-repetition loop**, early. One locked + on about 3% of the way in and repeated a single clause 375 times; the other + looped on a similar phrase. This is ordinary greedy-decode degeneration and + is what a length limit is for. + - **One** of them additionally, and abruptly, stopped producing language at + all: about 85% of the way through that reply — roughly decode step 3,470 — + it broke **mid-word** and emitted high-entropy punctuation and digits + (`,7&)6E-G.()$G%D:*DC#C1<$25DFC70D1)9%=!C4$+…`) to the cap. The other turn + never did this; its output stayed alphabetic throughout. + + A language model losing the thread repeats, drifts or confabulates. It does + not emit uniform random punctuation. So the second event is a different kind + of failure from the first, and it is recorded as a **known open issue**, not + investigated on this branch. + + **A hypothesis, from operator experience and not verified here:** this + signature has come up several times before on this class of hardware and has + essentially always turned out to be the attention kernel. The entry point a + future investigation should start from is `ryzenai_corelib_flat_mha_bf16` + (bound at `common/corelib/corelib_api.cpp:195-197`, called at + `common/corelib/phi4_corelib_aie4.cpp:941`, with `active_phase = "flat_mha"` + set at `:1107`). Nothing in this project's testing confirms that, and it + should not be read as a finding. + + **This is a single observation.** Whether it reproduces at all, whether the + onset is stable near that step, and whether it depends on the prompt are all + unknown, and no run has been made to find out. + + The acceptance record described both turns with one reason, *"the reply is + dominated by a repeated token"*, which was right about the repetition and + silent about the collapse. The paragraph above is the correction, derived from + that record's own data. +- **The limit is 4095 tokens** for the prompt and the generated tokens + together, counted over the complete rendered conversation, **so the largest + prompt you can submit is 4,094.** This is **by design, not a shortfall**: the + AIE4 operators support at most about 4k input. The KV window is 4096 rows and + the usable cap is one less, because no token-attention kernel ships for a + 4096-token window; and a prompt that fills the cap exactly leaves no room to + answer, so `Phi4::validate_aie4_capacity` + (`src/common/AutoModel/modeling_phi4.cpp:420`) refuses at 4,095 rendered + tokens rather than at 4,096. Requests over the cap are refused before any work + is submitted, with HTTP 400 on the server and a message naming the cap and the + rendered prompt length. The acceptance run verifies the enforcement exactly: + the remaining capacity is admitted and one token more is refused. + + **The catalog says 4096, and it is describing a different thing.** + `src/model_list.json` gives `phi4-mini-it-aie4:4b` a `default_context_length` + and `max_prefill_len` of 4096, which is the KV window the engine allocates. + The admission gate is one token tighter than the window and two tighter than + a full-window prompt; the figure to size against is **4,094**. diff --git a/docs/docs/phi4_aie4_caveats.md b/docs/docs/phi4_aie4_caveats.md new file mode 100644 index 00000000..75f91bc4 --- /dev/null +++ b/docs/docs/phi4_aie4_caveats.md @@ -0,0 +1,307 @@ +--- +layout: docs +title: Phi-4 AIE4 Caveats +nav_order: 6 +--- + +# Phi-4 on AIE4 — what to watch out for + +Every item below is known, recorded, and either measured or explicitly marked +as unverified. Nothing here is speculation dressed as fact; where something is +a hypothesis it says so. + +For what was built, read +[Phi-4 AIE4 Implementation](../phi4_aie4_implementation/). + +--- + +## 1. The MSI is not self-contained. It will not start on a clean machine. + +**Symptom.** `flm.exe` exits immediately with `0xC0000135` +(`STATUS_DLL_NOT_FOUND`). No message, no log. + +**Verified by execution**, not inferred: on a freshly installed machine with a +conda prefix removed from `PATH`, the shipped binary cannot start. + +**Cause.** The installer stages `src/lib/*.dll`, which carries FFmpeg +`avformat-61 / avcodec-61 / avutil-59 / swscale-8 / swresample-5`. This +`flm.exe` links `avformat-63 / avcodec-63 / avutil-61 / swscale-10 / +swresample-7`. Five direct imports are missing, and completing them from a +conda prefix pulls in 54 further transitive DLLs. The version families line up +one for one, so this is **version skew between the vendored `src/lib` set and +a conda-linked build** — an environment artifact. + +A sixth missing import, `boost_program_options.dll`, had a different cause and +**is fixed**: this work had moved the Windows Boost linkage from static to +shared without staging a DLL for it. The static default is restored. + +**Workaround.** Put a matching FFmpeg on `PATH` before running — a conda +prefix's `Library\bin` works. + +**Not fixed here**, because it is a packaging-list problem rather than an AIE4 +one, and the scope for this work was narrow. It is real and it is on the path +of anyone who installs the MSI on a clean box. + +--- + +## 2. `Aie4Feature` is not installed by default + +`Aie4Feature` carries `Level="1000"`. A plain `msiexec /i` — or a double-click +— produces a FastFlowLM **without the AIE4 runtime**, and the failure surfaces +later as a confusing model-load error rather than at install time. + +``` +msiexec /i flm-setup-with-aie4.msi ADDLOCAL=MainFeature,Aie4Feature +``` + +Confirm with `/status` in the REPL: it must report `Engine: corelib_aie4`. + +--- + +## 3. Multi-turn conversation in `flm run` does not retain history + +Each turn is answered as though it were the first. `/history` prints only the +current exchange. + +**This is not an AIE4 defect and not specific to Phi-4.** `Phi4::insert` +(`src/common/AutoModel/modeling_phi4.cpp`, line 576 at the time of writing) +wraps `input.prompt` in a fresh single-message array whenever `input.messages` +is empty — which is every CLI turn. That `messages.push_back` sits at line 589, +*above* the `#if defined(FLM_ENABLE_CORELIB_AIE4)` branch at line 596: the +history is gone before any AIE4 code runs. The identical construction is in the +Llama 3, Qwen 2, Qwen 3, Gemma 3, LFM2, GPT-OSS and Nanbeige frontends. + +Line numbers in this file have already moved once during this work. If they no +longer match, search for the `messages.push_back` inside `Phi4::insert` and +compare its position to the `#if` — the *ordering* is the claim, not the +numbers. + +**Workaround.** Use `/api/chat` or `/v1/chat/completions` and supply the whole +conversation yourself; those endpoints take a full `messages` array and count +it in full. + +--- + +## 4. The same binary, on the same input, has produced different text + +This backend is **not deterministic run to run**, and the difference is not a +rounding difference in the last bits — it is a different answer. + +In the determinism campaign, **3 of 150 recorded run-pair comparisons** ended +with the two runs emitting different token sequences. One pair parts at the 8th +emitted token, the other two at the 14th. + +The 150 JSON records are **not carried in this branch** — see item 11 — so the +three that diverged are summarised below rather than cited by path. + +Two severity measures are recorded and **they do not rank the three records the +same way**, so each figure belongs to its own record rather than to a single +"worst": + +| Record | Max absolute logit difference | Most logits moving at one step | Gate breaches | +|---|---:|---:|---:| +| `determ1-force_append-010` | 48.34 | 196,154 of 200,064 | 17 | +| `determ1-force_append-022` | **49.25** | 196,748 | 8 | +| `determ1-force_append-026` | 49.21 | **196,835** | 10 | + +All three are on the **append** continuation route. All three **breach** the +2-BF16-ULP bound the suite gated on — the bound is not merely approached. + +**Two things follow, and only the second is obvious.** The per-value bound did +not hold. And even inside that bound the effect would not be bounded, because +greedy decoding turns one flipped argmax into an unrelated continuation. +Read "bounded" in the acceptance record as a statement about a number, not +about what a user sees. + +**Current understanding.** The LM head is exonerated — against an FP64 host +reference, 200,059 of 200,064 logits fall within half a BF16 ULP. The +divergence is localised to the 32-layer model body; the specific layer is +unknown. The search is for an operator whose accumulation or work-partitioning +**order** is unpinned, not for the least accurate one: deterministic rounding +of deterministic inputs is reproducible, so varying results additionally +require varying order. + +Carried as a known defect (`DETERM-5`), not a release blocker. + +--- + +## 5. Long, open-ended generations can collapse + +Set an explicit generation limit. In the REPL: + +``` +/set gen-lim 200 +``` + +**What was observed.** In the multi-turn acceptance step, two of four turns ran +to the 4095-token cap — 186.9 s and 179.9 s — because no stop token was ever +emitted. Both began coherently. Both then entered a phrase-repetition loop, +which is ordinary language-model degeneration. + +**One of them did something else.** At roughly decode step 3,470 — about 85 % +of the way through, breaking mid-word immediately after "foot" — the output +switched from English to high-entropy random ASCII and stayed there: + +``` +...want you to scratch Fluffy on Fluffy's foot C(.;%>%A9G<:,&>-@G<*9H.",3;2=A,EC2#6F3F3... +``` + +A language model losing coherence repeats, drifts or confabulates. It does not +emit uniform random punctuation and digits. That signature suggests the logits +themselves became garbage. + +**Status: one observation.** Whether it reproduces, whether the onset is stable +near step 3,470, and whether it is prompt-dependent are all **unknown**. + +**Hypothesis, attributed to operator experience and not verified here**: this +signature has previously turned out to be the attention kernel. Entry point for +anyone investigating: + +``` +ryzenai_corelib_flat_mha_bf16 + bound src/common/corelib/corelib_api.cpp :197 + called src/common/corelib/phi4_corelib_aie4.cpp :966 api->functions().flat_mha( + phase src/common/corelib/phi4_corelib_aie4.cpp :1178 active_phase = "flat_mha" +``` + +Search by symbol rather than by line if these have drifted. + +Not investigated here by decision, not by oversight. + +--- + +## 6. The limit is 4095, not 4096 + +`kMaxDecodeWindow` is `kMaxSequenceLength - 1`. A 4096-row prefill at position +0 dispatches, but a single-row decode step at position 4095 — whose window is +also 4096 — is refused: no token-attention kernel ships for a 4096-token +window. The prefill and decode paths do not share a bound. + +The ~4k ceiling itself is **by design** — the operators support at most 4k +input. Raising `/set ctx-len` beyond it prints an error rather than crashing. + +--- + +## 7. Building against a shared or conda prefix needs a flag + +`FLM_WIN_STATIC_DEPS` defaults to **`ON`**, which is the static layout CI and +the MSI ship. Building against a shared or conda prefix requires: + +``` +-DFLM_WIN_STATIC_DEPS=OFF +``` + +A mismatch is a **configure-time hard failure naming both halves**, not a link +error at the end. + +### How the library is found — recorded, not actioned + +Nothing is required of you here. This is written down only so the next person +to touch that block knows why the explicit `PATHS` are in it. + +Before this work, CMake handed the Boost library's **file name** to the linker, +which resolved it through the linker's own search path — a path +`target_link_directories` contributes to. This work replaced that with +`find_library()`, and **`target_link_directories` contributes nothing to +`find_library`**. That is probed, not assumed: a stage directory known only +that way yields `PROBE-NOTFOUND`, while the same directory reachable through +`LIB` or `CMAKE_PREFIX_PATH` resolves. The two mechanisms search different +places. + +Nothing in the repository sets `CMAKE_PREFIX_PATH` for Boost, and the Windows +workflow sets no environment at all. So the build either leaned on an ambient +machine-level `LIB` on the build host — state that exists nowhere in the +repository — or it would not have configured at all. Which of the two was never +established, and by decision will not be: this backend is not built in CI. + +`src/CMakeLists.txt` now passes the staging directories as explicit `PATHS`, so +it resolves either way and no longer depends on host environment. **Do not +delete those `PATHS` on the grounds that the build works without them** — on +some hosts it will, for a reason that is not in the repository. + +--- + +## 8. Nothing automated compiles this backend or runs any of its tests + +No CI job runs any test, and the release preset never compiles the AIE4 path. +After merge, none of these tests run unless a human runs them. + +**This is worse than an absence.** `debian-portable.yml:122` defines a job +named `test-summary` whose single step echoes four hard-coded ✅ lines into the +step summary, unconditionally, `if: always()`. The absence of testing is +currently reported as four green ticks. + +Out of scope for this work by decision; recorded so it is not mistaken for +coverage. + +--- + +## 9. The legacy NPU2 path was compiled and linked, not run + +No such hardware was available. It builds; whether it works is unestablished. + +--- + +## 10. The determinism baseline is frozen + +`DETERM-3` forbids pooling measurements across harness binaries, and the +records spanned three. The published baseline is therefore frozen at its +original run, later measurements are recorded separately, and the stability +sentences in that block are **hand-corrected rather than re-rendered** — +labelled as such, so a reader can tell the two apart. If the pooling question +is resolved, that block should be re-rendered and the label deleted. + +--- + +## 11. The hardware campaign is not in this branch + +The measurements on this page are real and were taken as described. The +**instruments and raw records are not here.** Removed: the end-to-end and +benchmark harnesses, the PowerShell suite that drove them, the 20-step +acceptance driver and its offline guards, the 150 determinism records, the +acceptance record JSON, and the two pages rendered from it — the step-by-step +acceptance document and its provenance companion. Those two were a record of +one campaign's internal review, published into a navigation menu beside the +model benchmark pages, which is not what that menu is for. + +**Why.** None of it could run anywhere except one lab machine with an AIE4 +device and the real model — not on a developer box, and, per item 8, not in +CI. It was evidence of one campaign rather than a gate on future changes, and +carrying ~19,000 lines of it made the diff unreadable without making anything +safer. + +**What this costs you.** The figures on this page and in the benchmark pages +can no longer be re-derived from anything in the tree. They are reports, not +reproducible artifacts. Treat every number here as *measured once, on +`xcomedusad-43`, by an instrument you cannot run* — which is what it always +was; the records' presence made it look otherwise. + +**What still runs.** `test_phi4_hardware` and `test_fatal_child` exercise the +real corelib on a device when `FLM_AIE4_HARDWARE=1`; +`provision_aie4_target.ps1` still stages a machine to do it. The remaining 16 +registered tests run without a device. The reference tooling +(`compare_phi4_corelib_e2e.py`, `phi4_host_lm_head_reference.py`, +`report_phi4_corelib_baseline.py`) is kept with its own tests, but **nothing +in the tree now produces its inputs.** + +Recoverable from git history at `5c93aadb` if a future campaign wants them. + +--- + +## A note on how to read this page + +The acceptance record was wrong in three places that mattered, and the +corrections rather than the original are what this page carries: two failed +turns were described inaccurately (item 5 has the corrected reason), a source +revision claimed a tree that was not pristine, and the word "bounded" did more +work than the data supports (item 4). + +That list is short only because the recurring failure in this work was **a +record that reads better than the run it describes** — a check reported as +passing for work it did not do, a retraction reaching the code but not the +rendered artifact, a test agreeing with the implementation about which case +exists. Roughly twenty instances were found and fixed across the effort, +several of them by reviews catching earlier reviews. Assume the ones still +undetected exist. + +Trust the cited line numbers over any prose summary, including this page. diff --git a/docs/docs/phi4_aie4_implementation.md b/docs/docs/phi4_aie4_implementation.md new file mode 100644 index 00000000..0cff3d94 --- /dev/null +++ b/docs/docs/phi4_aie4_implementation.md @@ -0,0 +1,161 @@ +--- +layout: docs +title: Phi-4 AIE4 Implementation +nav_order: 5 +--- + +# Phi-4 on AIE4 — what was built + +FastFlowLM had no AIE4 backend. This work adds one: Phi-4-mini runs on an AMD +AIE4 NPU by calling `ryzenai-corelib`'s C ABI. Everything below describes what +ships, not what was attempted. + +For what to watch out for, read [Phi-4 AIE4 Caveats](../phi4_aie4_caveats/). + +--- + +## The shape of it + +``` +flm.exe ──(runtime, absolute path)──► ryzenai_corelib.dll ──► AIE4 NPU + │ + └── phi4_corelib_aie4 ── the backend added by this work +``` + +`flm.exe` **never links** `ryzenai_corelib.lib`. The DLL is resolved at run +time by absolute path, so a build without the AIE4 component installed starts +normally and reports the backend as unavailable. **There is no silent +fallback** — a Phi-4 AIE4 tag either runs on the NPU or fails loudly. + +## The corelib surface actually used + +Pinned to corelib `e5258d2`, ABI version **0.1.0**, gated on an exact +`major.minor.patch` match while the major version is 0. + +| Group | Entry points | +|---|---| +| Compute | `matmul_bf16`, `flat_mha_bf16`, `ssmlp_bf16` | +| Padding | `matmul_bf16_pad_shape`, `flat_mha_bf16_pad_rows`, `ssmlp_bf16_pad_rows` | +| Weights | `matmul_bf16_weights_create_onnx` / `_get_data`, `ssmlp_bf16_weights_create_onnx` / `_get_data` | +| Tensors | `create_device_tensor`, `tensor_write`, `tensor_read`, `tensor_get_byte_size`, `tensor_get_data_type` | +| Stream | `create_stream`, `stream_synchronize` | +| Lifecycle | `get_version`, `has_device_context`, `selftest_dependencies`, `object_release`, `cleanup` | +| Errors | `get_last_error_message`, `status_to_string` | + +Two conventions that bite if forgotten: + +- **Counts and offsets are elements, never bytes.** `tensor_write` and + `tensor_read` take element counts and convert dtype on the way. +- **Exactly two host-side conversions exist.** FP16 → FP32 widening (lossless, + bounds-checked), and FP32 → BF16 round-to-nearest-even for the SSMLP epsilon + and the two norm weights. Everything else crosses the boundary unconverted. + +## Model constants + +From `src/include/models/phi4/phi4_corelib_constants.hpp`, all validated +against the published ONNX: + +| | | +|---|---| +| Layers | 32 | +| Hidden / intermediate | 3072 / 8192 | +| Query heads / KV heads | 24 / 8 | +| Head size | 128 | +| Query dim / KV dim | 3072 / 1024 | +| Vocabulary | 200064 | +| Quantisation group | 128 | +| RoPE dimension | 96 | +| RMS epsilon | 1e-5 | +| Max sequence | 4096 | +| **Max decode window** | **4095** (`kMaxSequenceLength - 1`) | + +The last row is not a typo — see the caveats page. + +## What ships + +**Model tag** `phi4-mini-it-aie4:4b`, built from +[`amd/phi-4-mini-instruct-oga-dml`](https://huggingface.co/amd/phi-4-mini-instruct-oga-dml) +at revision `e751fb68`, plus four overlay files whose SHA-256 values are pinned +in the catalog. + +**Installer**: the AIE4 runtime is an **optional MSI feature** +(`Aie4Feature`). It stages `ryzenai_corelib.dll`, DynamicDispatch and RyzenMM +into an `aie4` directory beside `flm.exe`, together with `aie4-closure.txt` — +a manifest naming the corelib version, the corelib DLL's SHA-256, and the name +and SHA-256 of every DLL staged with it. + +## Execution notes + +- **Four stream synchronizes per layer.** AIE4 completion is not ordered by + submission, so a dispatch whose input another dispatch is still writing must + be separated by an explicit synchronize. +- **RoPE is applied host-side** as a strided gather followed by a single + `tensor_write`. FP32 scale tensors are rejected rather than silently narrowed. +- **The host stays FP32**; corelib narrows on write. +- **Unbounded generation is capped.** `kMaxDecodeWindow` bounds every decode + path, which closed a defect where an ungated generation could exhaust the + server. + +## What was verified + +**On `xcomedusad-43`** — a 20-step acceptance run against the real model in one +uninterrupted pass. Five single-turn probes, five coherent and factually +correct, roughly ten seconds each after a ~20 s model load. + +Final acceptance tally: **16 steps met, 1 not met, 3 not exercised**; +**12 of 52 criteria met, 12 partial, 2 not met, 26 not exercised**. Read the +last figure first: half the criteria were never exercised. + +**On `xcomedusad-44`** — a machine that took no part in development, given the +shipped MSI, the model package, and a conda environment reproduced from an +explicit package list. The model loads and generates. Read the caveats page +before concluding what this does and does not prove. + +**The instruments that produced both are not in this branch.** They ran only on +a machine with an AIE4 device and the real model, so the harnesses, the +PowerShell campaign and its raw records were removed rather than merged; the +paths cited in the two pages above resolve at commit `5c93aadb`. The +consequence — that these figures are reports rather than artifacts you can +re-derive — is +[caveat 11](../phi4_aie4_caveats/#11-the-hardware-campaign-is-not-in-this-branch). + +## Scale + +| | | +|---|---| +| Files changed | 110 (**32 of them tests**, carrying 23,912 of the added lines) | +| Lines | +49,635 / −289 | +| `ctest` on `xcomedusad-43` | **13 passed, 3 skipped, 0 failed** (16 registered) | +| Python tooling | **312 passed** | + +The test share is the point rather than an accident: most real defects in this +work were found by tests and review, not by the compiler. + +The three skips are `test_phi4_hardware` and `test_fatal_child` (they want +`FLM_AIE4_HARDWARE=1`) and `test_packaged_runtime` (it wants a staged +installer). `test_real_corelib` is not among them — it loaded the real +`ryzenai_corelib.dll` and passed in 11 s. + +### Configuring this suite on the AIE4 target + +**You must pass `-DFLM_PYTHON_EXECUTABLE` there.** On that machine `PATH` +resolves `python3` to a Cygwin symlink Windows cannot execute, so the probe +finds no usable interpreter and `test_phi4_continuation_calibration` is +deliberately registered as a *failing* test rather than silently skipped. The +script that used to supply this path was part of the hardware campaign and is +no longer in the branch, so it is now the caller's job: + +``` +cmake -S src/test/phi4_corelib_aie4 -B -G "Visual Studio 17 2022" -A x64 \ + -DRYZENAI_CORELIB_INCLUDE_DIR=/install-mirrored/include \ + -DRYZENAI_CORELIB_RUNTIME_DIR= \ + -DXRT_INCLUDE_DIR=/include -DXRT_LIB_DIR=/lib \ + -DBOOST_INCLUDE_DIR=/Library/include \ + -DFLM_PYTHON_EXECUTABLE=/python.exe +cmake --build --config Release +ctest -C Release +``` + +Omit that last flag and you get 15 passed, 1 failed — and the failure is the +configure, not the code. CMake says so at configure time; the warning is worth +reading. diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index dd4d33fe..42c6cb9c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -13,6 +13,32 @@ if(NOT DEFINED NPU_VERSION) message(FATAL_ERROR "NPU_VERSION must be specified externally. Use -DNPU_VERSION= when running cmake.") endif() +# Both versions must have all three components, and being DEFINED is not +# enough to establish that. +# +# Measured on 2026-09-02: `cmake ... -DFLM_VERSION=1.0.4` written UNQUOTED in a +# Windows PowerShell script reaches CMake as `-DFLM_VERSION=1`. The cache then +# holds FLM_VERSION=1, __FLM_VERSION__ compiles to "1", `flm version` prints +# "FLM v1", and ModelDownloader::check_model_compatibility sscanf's it with +# "%d.%d.%d" against a catalog flm_min_version of "1.0.4" -- leaving the minor +# and patch fields uninitialised in that comparison. Every flm.exe built on the +# AIE4 target before this guard existed had it, and nothing anywhere failed: +# the build succeeded, the binary ran, and the only symptom was a version +# string nobody was checking. +# +# A regex on the value is the cheapest place to stop that, because it fails at +# configure time in the same command that introduced the mistake. +foreach(_flm_version_var FLM_VERSION NPU_VERSION) + if(NOT "${${_flm_version_var}}" MATCHES "^[0-9]+\\.[0-9]+\\.[0-9]+(\\.[0-9]+)?$") + message(FATAL_ERROR + "${_flm_version_var} is '${${_flm_version_var}}', which is not a " + "dotted version. Expected major.minor.patch (a fourth component is " + "allowed). A single number here usually means the argument was " + "written unquoted in PowerShell, which truncates it at the first " + "'.': quote it as \"-D${_flm_version_var}=1.0.4\".") + endif() +endforeach() + # Set build type to Release set(CMAKE_BUILD_TYPE Release) @@ -48,6 +74,75 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY_RELEASE ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) # ——————————————————————————————————————————————— option(FLM_USE_HRX "Use the HRX amdxdna NPU runtime instead of XRT (0=XRT default, 1=HRX)" OFF) option(FLM_PORTABLE_BUILD "Build portable distribution with bundled runtime libraries" OFF) +option(FLM_ENABLE_CORELIB_AIE4 + "Enable optional Phi-4 AIE4 execution through ryzenai-corelib" OFF) +include("${CMAKE_SOURCE_DIR}/cmake/ConfigureAie4Runtime.cmake") + +# Where boost/any.hpp is looked for. ONE list, used by both Boost searches. +# +# There are two find_path(... boost/any.hpp ... REQUIRED) calls below: the +# feature-ON one for flm_corelib_aie4, and the unconditional one for flm on the +# non-vcpkg Windows path. They were written as two literal hint lists that a +# comment claimed were "the same search", and they were not: the second omitted +# $ENV{USERPROFILE}/anaconda3/Library/include. Because they share one cache +# variable (FLM_BOOST_INCLUDE_DIR) and both are REQUIRED, the difference was +# invisible with the feature ON -- the first call populated the cache and the +# second found it already set -- and turned into a hard configure failure with +# the feature OFF on any box whose only Boost is a bare anaconda3 install. The +# feature-OFF build is the shipping one, so the narrower list was the one that +# mattered. Defining the hints once is what makes the comment true. +set(FLM_BOOST_INCLUDE_HINTS + "$ENV{BOOST_INCLUDEDIR}" + "$ENV{BOOST_ROOT}" + "$ENV{CONDA_PREFIX}/Library/include" + "$ENV{USERPROFILE}/anaconda3/Library/include" + "C:/dev/boost_1_88_0") + +if(FLM_ENABLE_CORELIB_AIE4 AND (NOT WIN32 OR FLM_USE_HRX)) + message(FATAL_ERROR + "FLM_ENABLE_CORELIB_AIE4 requires Windows and the XRT backend") +endif() + +if(FLM_ENABLE_CORELIB_AIE4) + find_path(RYZENAI_CORELIB_INCLUDE_DIR + NAMES ryzenai/corelib.h + REQUIRED) + include("${CMAKE_SOURCE_DIR}/common/corelib/corelib_sources.cmake") + add_library(flm_corelib_aie4 STATIC + ${FLM_CORELIB_AIE4_SOURCES}) + # Boost headers, for XRT's sake rather than ours. + # + # `xrt/detail/any.h` picks `boost::any` over `std::any` unless the compiler + # reports `__cplusplus >= 201703L`, which MSVC does not do without + # `/Zc:__cplusplus`. Every source in this target reaches XRT's headers + # transitively, so without a Boost include directory the target fails to + # compile with C1083 on `boost/any.hpp`. + # + # This went undetected through Tasks 1-12 because the AIE4 code had only + # ever been compiled by the standalone test project at + # test/phi4_corelib_aie4/CMakeLists.txt, which does exactly this discovery. + # The product build had no equivalent, and nobody had built the product. + # + # `flm` itself gets Boost from vcpkg's `find_package` or from a hard-coded + # `C:/dev/boost_1_88_0`; neither reaches this target, and the second does + # not exist on either machine here. Searching is what makes the same source + # tree build on a developer box, the AIE4 target and the CI runner. + find_path(FLM_BOOST_INCLUDE_DIR + NAMES boost/any.hpp + HINTS ${FLM_BOOST_INCLUDE_HINTS} + DOC "Directory containing boost/any.hpp, needed by XRT's headers" + REQUIRED) + + target_include_directories(flm_corelib_aie4 PUBLIC + ${CMAKE_SOURCE_DIR}/include + ${RYZENAI_CORELIB_INCLUDE_DIR} + ${FLM_BOOST_INCLUDE_DIR}) + target_compile_definitions(flm_corelib_aie4 PUBLIC + FLM_ENABLE_CORELIB_AIE4=1 + WIN32_LEAN_AND_MEAN + NOMINMAX) + target_compile_options(flm_corelib_aie4 PRIVATE /fp:precise) +endif() if(FLM_USE_HRX) set(FLM_RUNTIME_NAME "hrx") @@ -73,6 +168,52 @@ else() if(WIN32) set(XRT_INCLUDE_DIR C:/dev/XRT/src/runtime_src/core/include CACHE PATH "Where XRT headers live") set(XRT_LIB_DIR C:/dev/xrtNPUfromDLL CACHE PATH "Where XRT libs live") + + # Diagnose at CONFIGURE rather than hundreds of lines into compilation. + # + # These defaults describe one historical machine and exist on neither + # box in use here. CMake accepts a nonexistent include directory and + # the build then dies on C1083 for an XRT header, which reads as a + # missing dependency rather than as a mis-set cache variable. + # + # Both are load-bearing for every Windows XRT build -- every TU reaches + # XRT's headers, and `xrt_coreutil` resolves from XRT_LIB_DIR, not from + # lib/xrt, which ships the engine libraries and aiebu_static but no + # xrt_coreutil.lib. So a wrong value fails either way. + # + # It is FATAL only with the AIE4 feature on, and a warning otherwise. + # That is a deliberate limit of scope: turning a late failure into an + # early one for the feature-OFF product build is a change to a + # configuration this task has no evidence about and which Task 15 + # depends on. The message names the variable in both cases, which is + # the part that was actually missing. + foreach(_flm_xrt_dir IN ITEMS XRT_INCLUDE_DIR XRT_LIB_DIR) + if(NOT IS_DIRECTORY "${${_flm_xrt_dir}}") + set(_flm_xrt_hint + "${_flm_xrt_dir} does not name a directory: " + "'${${_flm_xrt_dir}}'. Pass -D${_flm_xrt_dir}= " + "pointing at your XRT package; the built-in default is a " + "development-box path and is not present here.") + if(FLM_ENABLE_CORELIB_AIE4) + message(FATAL_ERROR ${_flm_xrt_hint}) + else() + message(WARNING ${_flm_xrt_hint} + " The build will fail later on an XRT header or on " + "xrt_coreutil.") + endif() + endif() + endforeach() + + if(FLM_ENABLE_CORELIB_AIE4) + target_include_directories(flm_corelib_aie4 PUBLIC + ${XRT_INCLUDE_DIR}) + target_link_directories(flm_corelib_aie4 PUBLIC + ${XRT_LIB_DIR}) + target_link_libraries(flm_corelib_aie4 PUBLIC + xrt_coreutil + shell32 + ole32) + endif() else() find_package(PkgConfig) if(PkgConfig_FOUND) @@ -236,7 +377,8 @@ add_subdirectory(${CMAKE_SOURCE_DIR}/../third_party/tokenizers-cpp # Gather your sources # ——————————————————————————————————————————————— file(GLOB SOURCES "src/*.cpp" "runner/*.cpp" "common/*.cpp" "common/*/*.cpp" "server/*.cpp" "pull/*.cpp" ) -file(GLOB HEADERS "include/*.hpp" "runner/*.hpp" "common/*.hpp" "common/*/*.hpp" "server/*.hpp" "pull/*.hpp") +list(FILTER SOURCES EXCLUDE REGEX ".*/common/corelib/.*\\.cpp$") +file(GLOB HEADERS "include/*.hpp" "include/server/*.hpp" "runner/*.hpp" "common/*.hpp" "common/*/*.hpp" "server/*.hpp" "pull/*.hpp") # Exclude files that depend on missing libraries for Linux if(NOT WIN32) @@ -268,6 +410,13 @@ endif() add_executable(flm ${SOURCES} ${HEADERS}) +flm_aie4_warn_if_unstageable() + +if(FLM_ENABLE_CORELIB_AIE4) + target_compile_definitions(flm PRIVATE + FLM_ENABLE_CORELIB_AIE4=1) + target_link_libraries(flm PRIVATE flm_corelib_aie4) +endif() if(WIN32) if(VCPKG_TOOLCHAIN) @@ -335,10 +484,66 @@ if(NOT FLM_USE_HRX) endif() if(WIN32 AND NOT VCPKG_TOOLCHAIN) - # Bare CI runner: standalone Boost + vcpkg headers by absolute path. + # Headers for the native dependencies, found rather than hard-coded. + # + # This block used to add `C:/dev/boost_1_88_0` and + # `C:/dev/vcpkg/installed/x64-windows/include/` verbatim. Both are paths on + # one historical CI runner and exist on neither machine in use, so the + # build failed C1083 on `libavformat/avformat.h` with nothing to say about + # where it expected to find it. Searching for a representative header of + # each family gives an error at configure that names what is missing. + find_path(FLM_FFMPEG_INCLUDE_DIR + NAMES libavformat/avformat.h + DOC "Directory containing FFmpeg's libavformat/avformat.h") + if(NOT FLM_FFMPEG_INCLUDE_DIR) + message(FATAL_ERROR + "Could not find libavformat/avformat.h. flm.exe needs FFmpeg " + "headers; add the prefix that provides them to CMAKE_PREFIX_PATH " + "or configure with a vcpkg toolchain.") + endif() + # fftw3.h is VENDORED at src/include/fftw3.h, and src/include is already + # first on flm's include path, so the vendored header is what the compiler + # reads whatever this finds. Searching the tree first keeps the reported + # value honest instead of naming a header that is being shadowed. + find_path(FLM_FFTW_INCLUDE_DIR + NAMES fftw3.h + PATHS "${CMAKE_SOURCE_DIR}/include" + NO_DEFAULT_PATH + DOC "Directory containing fftw3.h") + find_path(FLM_FFTW_INCLUDE_DIR + NAMES fftw3.h + DOC "Directory containing fftw3.h") + if(NOT FLM_FFTW_INCLUDE_DIR) + message(FATAL_ERROR + "Could not find fftw3.h. Add the prefix that provides FFTW3 to " + "CMAKE_PREFIX_PATH or configure with a vcpkg toolchain.") + endif() + find_path(FLM_CURL_INCLUDE_DIR + NAMES curl/curl.h + DOC "Directory containing curl/curl.h") + if(NOT FLM_CURL_INCLUDE_DIR) + message(FATAL_ERROR + "Could not find curl/curl.h. Add the prefix that provides CURL to " + "CMAKE_PREFIX_PATH or configure with a vcpkg toolchain.") + endif() + # Boost: literally the same hint list flm_corelib_aie4 uses + # (FLM_BOOST_INCLUDE_HINTS, defined once near the top), so the two targets + # cannot end up with different Boost headers in one build -- and so this + # search cannot fail on a prefix the other one accepts. + find_path(FLM_BOOST_INCLUDE_DIR + NAMES boost/any.hpp + HINTS ${FLM_BOOST_INCLUDE_HINTS} + DOC "Directory containing boost/any.hpp" + REQUIRED) + message(STATUS " FFmpeg headers: ${FLM_FFMPEG_INCLUDE_DIR}") + message(STATUS " FFTW3 headers: ${FLM_FFTW_INCLUDE_DIR}") + message(STATUS " CURL headers: ${FLM_CURL_INCLUDE_DIR}") + message(STATUS " Boost headers: ${FLM_BOOST_INCLUDE_DIR}") target_include_directories(flm PUBLIC - C:/dev/boost_1_88_0 - C:/dev/vcpkg/installed/x64-windows/include/ + ${FLM_FFMPEG_INCLUDE_DIR} + ${FLM_FFTW_INCLUDE_DIR} + ${FLM_CURL_INCLUDE_DIR} + ${FLM_BOOST_INCLUDE_DIR} ) endif() @@ -366,13 +571,64 @@ if(WIN32) ) endif() +# Whether the Windows non-vcpkg dependencies are STATIC archives. +# +# The CI runner links a standalone Boost b2 build and a static curl, which need +# CURL_STATICLIB and BOOST_ALL_STATIC_LINK. A conda prefix ships both as DLLs, +# where those same macros are actively wrong: CURL_STATICLIB removes the +# __declspec(dllimport) the import library expects. It was previously inferred +# from "no vcpkg toolchain", which is not the same question, so the option now +# says what it means. +# +# Default ON, and the default is the whole point of the option existing. +# +# It shipped as OFF once, and that was a regression rather than a preference. +# Before the option existed this block defined CURL_STATICLIB / +# BOOST_ALL_NO_LIB / BOOST_ALL_STATIC_LINK unconditionally on the +# `WIN32 AND NOT VCPKG_TOOLCHAIN` path, and the Windows link list named +# `libboost_program_options-vc143-mt-x64-1_88` -- the STATIC b2 spelling. So +# the shipped binary had no Boost DLL import at all, which is why src/lib +# vendors no boost_program_options.dll and why src/wix/get_files.bat stages +# none. Defaulting the new option OFF flipped that binary from static Boost to +# shared, silently: FLM_WIN_STATIC_DEPS is set nowhere in the tree, no preset +# sets a toolchainFile (so VCPKG_TOOLCHAIN is false on every Windows preset), +# and .github/workflows/windows-build.yml runs `cmake --preset windows-vs18` +# bare. The MSI was then missing a DLL nothing in the repository produces, and +# the failure was misattributed to upstream packaging. +# +# ON is therefore the configuration that CI and the MSI actually ship, and the +# default has to match it. A conda or otherwise shared prefix -- which is what +# the AIE4 work uses -- opts out with -DFLM_WIN_STATIC_DEPS=OFF, and gets the +# import-library spellings and BOOST_ALL_DYN_LINK described below. +option(FLM_WIN_STATIC_DEPS + "Windows non-vcpkg dependencies are static archives (CI runner layout)" + ON) if(WIN32 AND NOT VCPKG_TOOLCHAIN) - # Bare CI runner: statically link the standalone Boost + curl. - target_compile_definitions(flm PUBLIC - CURL_STATICLIB - BOOST_ALL_NO_LIB - BOOST_ALL_STATIC_LINK - ) + if(FLM_WIN_STATIC_DEPS) + target_compile_definitions(flm PUBLIC + CURL_STATICLIB + BOOST_ALL_NO_LIB + BOOST_ALL_STATIC_LINK + ) + else() + # Shared Boost needs BOOST_ALL_DYN_LINK, and needs it for a reason that + # is easy to misread as a missing library. Boost's headers declare + # symbols without __declspec(dllimport) unless this is set, so the + # FUNCTIONS still link -- the import library satisfies them -- while + # the exported DATA symbols do not. The failure is + # LNK2019: boost::program_options::arg + # LNK2019: options_description::m_default_line_length + # which names two variables and looks like the wrong Boost version + # rather than the wrong linkage. + # + # BOOST_ALL_NO_LIB stays: MSVC auto-linking would otherwise ask for + # decorated names like libboost_program_options-vc143-mt-x64-1_88.lib + # that a conda prefix does not ship, undoing the explicit search above. + target_compile_definitions(flm PUBLIC + BOOST_ALL_NO_LIB + BOOST_ALL_DYN_LINK + ) + endif() endif() target_link_directories(flm PUBLIC @@ -470,20 +726,260 @@ if(WIN32 AND VCPKG_TOOLCHAIN) FFTW3::fftw3l ) elseif(WIN32) - # Bare CI runner: link the native deps by raw name from the standalone Boost - # + vcpkg lib dirs wired above (no CMake package configs there). - target_link_libraries(flm PUBLIC - avformat - avcodec - avutil - swscale - swresample - libcurl - libboost_program_options-vc143-mt-x64-1_88 - libfftw3-3 - libfftw3f-3 - libfftw3l-3 - ) + # No vcpkg toolchain: resolve each native dependency by SEARCHING for it, + # rather than naming one machine's import libraries. + # + # This block used to link `libboost_program_options-vc143-mt-x64-1_88`, + # `libfftw3-3` and friends by raw name. Those are the artifact names of one + # specific standalone Boost b2 build and one specific vcpkg tree, and they + # exist on no other machine -- so the only Windows configuration that could + # build flm.exe without vcpkg was that one CI runner. A conda prefix, which + # is what the AIE4 target has, ships the same libraries as `fftw3.lib` and + # `boost_program_options.lib` and matched nothing here. + # + # `find_library` takes a list of accepted spellings and reports which one it + # picked, so a missing dependency fails at configure naming the library + # rather than at link naming a symbol. Point CMAKE_PREFIX_PATH at whichever + # prefix supplies them. + # + # This does NOT make an ambient prefix acceptable in place of proper target + # wiring (ENV-2). It makes the search explicit and its result auditable, + # which is the opposite of relying on whatever happens to be reachable. + # Only the SINGLE-precision FFTW entry points are called: everything in + # common/audio_process_utils uses fftwf_*, and neither fftw_* (double) nor + # fftwl_* (long double) appears anywhere in this tree. The vendored + # include/fftw3.h declares all three, which is why all three were being + # linked. + # + # Double and long double are marked optional because nothing calls them, + # NOT because they are unavailable. All three import libraries are + # committed in src/lib and all three resolve there. An earlier version of + # this comment said Windows conda-forge does not build the long-double + # variant; that is true and was never the reason the build failed. The + # reason was that the name list was truncated -- see the SUBLIST note below + # -- so `libfftw3l-3` was never searched for under any commit, and the + # in-tree copy went unseen. + # Self-check on the split, against a synthetic entry with two alternative + # spellings. The real entries are easy to get wrong in a way that still + # configures -- dropping a fallback name only shows up on a machine that + # needs it -- so the parsing is verified before it is used, the way the + # test suite's CMakeLists verifies its argv construction. + string(REPLACE "|" ";" _flm_dep_probe "optional|probe|alpha;beta") + list(SUBLIST _flm_dep_probe 2 -1 _flm_dep_probe_names) + list(LENGTH _flm_dep_probe_names _flm_dep_probe_count) + if(NOT _flm_dep_probe_count EQUAL 2) + message(FATAL_ERROR + "Windows dependency name parsing dropped alternative spellings: " + "expected 2 names, got ${_flm_dep_probe_count} " + "[${_flm_dep_probe_names}]. Every fallback library name would be " + "silently ignored.") + endif() + + # Boost's name order follows FLM_WIN_STATIC_DEPS, because the linkage macros + # and the library name are two halves of ONE decision. + # + # b2 encodes linkage in the file name: `libboost_program_options-...lib` is + # the static archive, bare `boost_program_options.lib` is the import library + # for boost_program_options.dll. Before this list existed the name was + # hard-coded to the static spelling, which agreed with the then + # unconditional BOOST_ALL_STATIC_LINK. Replacing it with a shared-first + # search left the two halves free to disagree: with FLM_WIN_STATIC_DEPS OFF + # they were both shared and consistent, but a machine that staged only the + # static archive got BOOST_ALL_DYN_LINK against it, and turning the option + # ON on a machine with both variants would have declared static linkage and + # then linked the import library. Deriving the order from the same variable + # that picks the macros makes disagreement unrepresentable; the guard after + # the loop catches the residual case where only the wrong variant exists. + # + # The fallback spelling stays in the list either way -- a prefix that ships + # only one variant should still build, it just gets the linkage its macros + # do not expect, and that is exactly what the guard reports. + set(_flm_boost_po_static + "libboost_program_options-vc143-mt-x64-1_88;libboost_program_options-vc142-mt-x64-1_88") + set(_flm_boost_po_shared "boost_program_options") + if(FLM_WIN_STATIC_DEPS) + set(_flm_boost_po_names + "${_flm_boost_po_static};${_flm_boost_po_shared}") + else() + set(_flm_boost_po_names + "${_flm_boost_po_shared};${_flm_boost_po_static}") + endif() + # curl gets no such treatment, deliberately: its names do not encode + # linkage. `libcurl.lib` is the static archive in a vcpkg static triplet AND + # the import library conda-forge ships for libcurl.dll -- the same spelling + # means opposite things depending on who built it. Only `libcurl_imp` is + # unambiguous (curl's own CMake build names its import library that), and it + # is already searched. There is no order that carries information here, and + # for the same reason the guard below cannot check curl either. + + set(_flm_win_libs "") + foreach(_flm_dep IN ITEMS + "required|avformat|avformat" + "required|avcodec|avcodec" + "required|avutil|avutil" + "required|swscale|swscale" + "required|swresample|swresample" + "required|curl|libcurl;libcurl_imp;curl" + "required|boost_program_options|${_flm_boost_po_names}" + "required|fftw3f|fftw3f;libfftw3f-3" + "optional|fftw3|fftw3;libfftw3-3" + "optional|fftw3l|fftw3l;libfftw3l-3") + string(REPLACE "|" ";" _flm_dep_parts "${_flm_dep}") + list(GET _flm_dep_parts 0 _flm_dep_need) + list(GET _flm_dep_parts 1 _flm_dep_key) + # SUBLIST, not GET. + # + # The names field is itself a `;`-separated list, so the REPLACE above + # flattens it into the SAME list: "optional|fftw3l|fftw3l;libfftw3l-3" + # becomes four elements, not three. `list(GET ... 2)` therefore + # returned only the FIRST spelling and silently discarded every + # alternative -- which is why libfftw3f-3.lib and libfftw3l-3.lib in + # this repository were never found, and why fftw3l looked absent when + # it is committed at src/lib/libfftw3l-3.lib. Everything from index 2 + # onward is the name list. + list(SUBLIST _flm_dep_parts 2 -1 _flm_dep_names) + # The TREE's own libraries first, and not merely as a HINTS entry. + # + # This repository vendors libfftw3-3.lib, libfftw3f-3.lib and + # libfftw3l-3.lib in src/lib, and src/lib is already on flm's link + # path. Searching only external prefixes therefore did two wrong + # things: it failed outright without one, and where one WAS supplied it + # silently paired conda's import library with the vendored + # src/include/fftw3.h, because src/include is unconditionally first on + # the include path and always wins. A configure line reporting a conda + # .lib while the compiler reads the vendored header describes a build + # that is not happening. + # + # HINTS is not enough: CMAKE_PREFIX_PATH is searched BEFORE HINTS, so + # an external prefix would still take precedence. NO_DEFAULT_PATH makes + # the in-tree pass authoritative, with an ordinary search as fallback. + find_library(FLM_LIB_${_flm_dep_key} + NAMES ${_flm_dep_names} + PATHS "${CMAKE_SOURCE_DIR}/lib" + NO_DEFAULT_PATH + DOC "Import library for ${_flm_dep_key}") + # The bare CI runner's two staging directories, as PATHS. + # + # `target_link_directories(flm PUBLIC C:/dev/...)` above is what made + # the PRE-BRANCH raw-name link work: the linker was handed the + # directory, and the name never had to be resolved at configure time. + # This branch replaced the raw names with find_library, and + # target_link_directories contributes NOTHING to find_library -- probed + # directly: a stage directory known only that way yields NOTFOUND, + # while the same directory on $ENV{LIB} or CMAKE_PREFIX_PATH resolves. + # + # So without this, `cmake --preset windows-vs18` on that runner resolves + # Boost only if the machine happens to carry an ambient LIB covering + # C:/dev/boost_1_88_0/stage/lib. The pre-branch link needed no such + # ambient state and the branch should not quietly start needing it. + # PATHS is the LAST bucket find_library searches, after HINTS and + # CMAKE_PREFIX_PATH, so this cannot displace a prefix that is supplied + # deliberately -- it only restores the fallback the raw names had. + find_library(FLM_LIB_${_flm_dep_key} + NAMES ${_flm_dep_names} + PATHS + C:/dev/boost_1_88_0/stage/lib + C:/dev/vcpkg/installed/x64-windows/lib + DOC "Import library for ${_flm_dep_key}") + if(NOT FLM_LIB_${_flm_dep_key}) + if(_flm_dep_need STREQUAL "required") + message(FATAL_ERROR + "Could not find an import library for '${_flm_dep_key}' " + "(tried: ${_flm_dep_names}). flm.exe needs FFmpeg, CURL, " + "Boost.program_options and single-precision FFTW3. Either " + "configure with a vcpkg toolchain, or add the prefix that " + "provides them to CMAKE_PREFIX_PATH.") + endif() + message(STATUS + " ${_flm_dep_key}: not found (optional, no call sites)") + continue() + endif() + # A build directory configured before the in-tree preference existed + # keeps its cached external path, so the vendored header would compile + # against an external import library -- the exact mismatch this is here + # to prevent, silently, on precisely the machines that already had a + # working build. find_library will not re-search a set cache entry, so + # say so rather than let it pass. + if(EXISTS "${CMAKE_SOURCE_DIR}/lib/${_flm_dep_key}.lib" OR + EXISTS "${CMAKE_SOURCE_DIR}/lib/lib${_flm_dep_key}-3.lib") + string(FIND "${FLM_LIB_${_flm_dep_key}}" + "${CMAKE_SOURCE_DIR}/lib" _flm_dep_in_tree) + if(_flm_dep_in_tree EQUAL -1) + message(WARNING + "${_flm_dep_key} resolved to " + "'${FLM_LIB_${_flm_dep_key}}' although this tree ships its " + "own copy in ${CMAKE_SOURCE_DIR}/lib. That is a stale " + "cache entry from a configure predating the in-tree " + "preference, and it pairs an external import library with " + "the vendored header. Remove FLM_LIB_${_flm_dep_key} from " + "CMakeCache.txt, or configure a fresh build directory.") + endif() + endif() + message(STATUS " ${_flm_dep_key}: ${FLM_LIB_${_flm_dep_key}}") + list(APPEND _flm_win_libs "${FLM_LIB_${_flm_dep_key}}") + endforeach() + + # Boost linkage consistency: the library that was actually RESOLVED has to + # agree with the macros that were actually DEFINED. + # + # This is the check that would have caught the boost_program_options.dll + # regression at configure time instead of at MSI-validation time. The + # macro choice (FLM_WIN_STATIC_DEPS, above) and the name preference (the + # search list, above) are now derived from the same variable, so they + # cannot disagree by construction -- but the SEARCH can still land on the + # other variant, because both spellings stay in the list as fallbacks and + # a stale FLM_LIB_boost_program_options cache entry is never re-searched. + # Either way the symptom is silent: BOOST_ALL_STATIC_LINK against an import + # library, or BOOST_ALL_DYN_LINK against an archive, both link and both + # produce a binary whose Boost dependency is not the one the packaging + # assumes. Fail here, loudly, naming both halves -- a warning in a + # thousand-line configure log is what got missed the first time. + # + # Only a positively identified b2 spelling is judged. An unrecognised name + # (a repackaged prefix, a vcpkg-style artifact) carries no linkage + # information and is left alone rather than guessed at. + get_filename_component(_flm_boost_po_name + "${FLM_LIB_boost_program_options}" NAME) + set(_flm_boost_po_linkage "unknown") + if(_flm_boost_po_name MATCHES "^libboost_program_options") + set(_flm_boost_po_linkage "static") + elseif(_flm_boost_po_name MATCHES "^boost_program_options") + set(_flm_boost_po_linkage "shared") + endif() + if(FLM_WIN_STATIC_DEPS AND _flm_boost_po_linkage STREQUAL "shared") + message(FATAL_ERROR + "Boost linkage is inconsistent. FLM_WIN_STATIC_DEPS=ON defined " + "BOOST_ALL_NO_LIB + BOOST_ALL_STATIC_LINK (static linkage), but " + "the search resolved '${FLM_LIB_boost_program_options}' -- a bare " + "'boost_' name, which is b2's IMPORT LIBRARY for " + "boost_program_options.dll, not an archive. flm.exe would declare " + "static linkage and then depend on a DLL that nothing in this " + "repository stages (src/lib ships none and src/wix/get_files.bat " + "stages none), producing an MSI that cannot start off this " + "machine. Fix ONE of the two: stage the static variant " + "(libboost_program_options-vc143-mt-x64-1_88.lib) on " + "CMAKE_PREFIX_PATH, or configure with -DFLM_WIN_STATIC_DEPS=OFF " + "and ship boost_program_options.dll alongside flm.exe. Delete " + "FLM_LIB_boost_program_options from CMakeCache.txt after either.") + endif() + if(NOT FLM_WIN_STATIC_DEPS AND _flm_boost_po_linkage STREQUAL "static") + message(FATAL_ERROR + "Boost linkage is inconsistent. FLM_WIN_STATIC_DEPS=OFF defined " + "BOOST_ALL_NO_LIB + BOOST_ALL_DYN_LINK (shared linkage), but the " + "search resolved '${FLM_LIB_boost_program_options}' -- a " + "'lib'-prefixed name, which is b2's STATIC archive. Boost's " + "headers would declare every symbol __declspec(dllimport) and the " + "archive exports none of them, so the link fails on data symbols " + "(boost::program_options::arg, " + "options_description::m_default_line_length) in a way that reads " + "as the wrong Boost version. Fix ONE of the two: configure with " + "-DFLM_WIN_STATIC_DEPS=ON to match the archive, or put a prefix " + "shipping boost_program_options.lib/.dll on CMAKE_PREFIX_PATH. " + "Delete FLM_LIB_boost_program_options from CMakeCache.txt after " + "either.") + endif() + + target_link_libraries(flm PUBLIC ${_flm_win_libs}) elseif(FFMPEG_BUILT_FROM_SOURCE) # Portable Linux: link the static FFmpeg archives built above (plus zlib) so # flm carries no libav*/libsw* runtime dependency. @@ -562,6 +1058,13 @@ if(WIN32) ${CMAKE_SOURCE_DIR}/out/flm.exe ) endif() +add_custom_command(TARGET flm POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_directory + "${CMAKE_SOURCE_DIR}/model_overlays" + "$/model_overlays") +flm_aie4_stage_for_target(flm + "$/aie4" + "${CMAKE_SOURCE_DIR}/out/aie4") # Default install location for model_list.json / xclbins (matches the app's # relocatable "/../share/flm" lookup). Overridden to the prefix root for @@ -699,6 +1202,7 @@ if(WIN32) "[Pp][Dd][Mm][Uu][Tt][Ii][Ll][Ii][Tt][Ii][Ee][Ss].*" "[Ww][Pp][Aa][Xx][Hh][Oo][Ll][Dd][Ee][Rr].*" POST_EXCLUDE_REGEXES ".*[Ww]indows[/\\\\][Ss]ystem32[/\\\\].*") + flm_aie4_install_runtime(DESTINATION bin/aie4 COMPONENT AIE4) elseif(NOT FLM_USE_HRX AND FLM_PORTABLE_BUILD) # Portable XRT: bundle the XRT runtime explicitly. XRT is deliberately # handled here instead of via the dependency closure below: flm only lists @@ -875,6 +1379,9 @@ endif() install(FILES model_list.json DESTINATION "${FLM_SHARE_DESTINATION}") install(FILES model_info.json DESTINATION "${FLM_SHARE_DESTINATION}") +install( + DIRECTORY model_overlays + DESTINATION "${FLM_SHARE_DESTINATION}") # xclbins, which are loaded by shared libraries need to be in location # relative to the executable, so we install them relative to the binary. diff --git a/src/CMakePresets.json b/src/CMakePresets.json index 39a07cdd..b8eb74bc 100644 --- a/src/CMakePresets.json +++ b/src/CMakePresets.json @@ -5,7 +5,7 @@ "name": "common-default", "hidden": true, "cacheVariables": { - "FLM_VERSION": "1.0.3", + "FLM_VERSION": "1.0.4", "NPU_VERSION": "32.0.203.304" } }, diff --git a/src/cmake/ConfigureAie4Runtime.cmake b/src/cmake/ConfigureAie4Runtime.cmake new file mode 100644 index 00000000..797f65dd --- /dev/null +++ b/src/cmake/ConfigureAie4Runtime.cmake @@ -0,0 +1,136 @@ +# Packaging support for the optional Phi-4 AIE4 corelib runtime. +# +# The corelib DLL is resolved at run time by absolute path and `flm.exe` never +# links `ryzenai_corelib.lib`, so building the product with +# `FLM_ENABLE_CORELIB_AIE4=ON` needs the corelib *include* directory and +# nothing else. The runtime directory is therefore a packaging input, required +# by the install/package step and never by a feature-ON configure. Making it a +# configure-time requirement would contradict calling it packaging-only and +# would break the ordinary developer build. +# +# The file list itself is derived, never transcribed: see StageAie4Runtime.cmake +# and design `CLOSURE-1`. + +set( + RYZENAI_CORELIB_RUNTIME_DIR + "" + CACHE PATH + "Packaging-only directory containing the ryzenai-corelib runtime DLLs") +set( + XRT_RUNTIME_DIR + "" + CACHE PATH + "Packaging-only directory containing stageable XRT runtime DLLs") +set( + FLM_AIE4_DEPENDENCY_DIRS + "" + CACHE STRING + "Additional directories searched for the optional AIE4 DLL closure") + +set(FLM_AIE4_STAGE_SCRIPT + "${CMAKE_CURRENT_LIST_DIR}/StageAie4Runtime.cmake") + +function(flm_aie4_runtime_configured output) + if(FLM_ENABLE_CORELIB_AIE4 + AND RYZENAI_CORELIB_RUNTIME_DIR + AND EXISTS "${RYZENAI_CORELIB_RUNTIME_DIR}/ryzenai_corelib.dll") + set(${output} TRUE PARENT_SCOPE) + else() + set(${output} FALSE PARENT_SCOPE) + endif() +endfunction() + +# Warns at configure time, at most. A developer who only wants to compile the +# AIE4 code paths should not be stopped here; the install step is where the +# missing input actually matters, and that is where it fails. +function(flm_aie4_warn_if_unstageable) + if(NOT FLM_ENABLE_CORELIB_AIE4) + return() + endif() + flm_aie4_runtime_configured(_configured) + if(NOT _configured) + message(WARNING + "FLM_ENABLE_CORELIB_AIE4=ON without a usable " + "RYZENAI_CORELIB_RUNTIME_DIR. flm.exe will still build, because " + "it resolves ryzenai_corelib.dll at run time by absolute path. " + "Installing or packaging the AIE4 feature will fail until " + "RYZENAI_CORELIB_RUNTIME_DIR points at the directory holding the " + "ryzenai_corelib.dll you intend to ship.") + endif() +endfunction() + +function(flm_aie4_stage_command output stage_dir report audit) + # FLM_AIE4_DEPENDENCY_DIRS is a CMake list. Interpolating it into a + # command argument unescaped turns each `;` into an argument separator, so + # `-DFLM_AIE4_EXTRA_DIRS=a;b` reaches cmake as two argv entries: the + # variable silently loses everything after the first directory, and the + # staging then fails telling the developer to add a directory they already + # added. Escaping keeps the whole list in one argument. + string(REPLACE ";" "\\;" _extra_dirs "${FLM_AIE4_DEPENDENCY_DIRS}") + set(${output} + "${CMAKE_COMMAND}" + "-DFLM_AIE4_CORELIB_DIR=${RYZENAI_CORELIB_RUNTIME_DIR}" + "-DFLM_AIE4_XRT_DIR=${XRT_RUNTIME_DIR}" + "-DFLM_AIE4_EXTRA_DIRS=${_extra_dirs}" + "-DFLM_AIE4_DESTINATION=${stage_dir}" + "-DFLM_AIE4_REPORT=${report}" + "-DFLM_AIE4_AUDIT=${audit}" + -P "${FLM_AIE4_STAGE_SCRIPT}" + PARENT_SCOPE) +endfunction() + +# Stages the derived closure beside a just-built binary so a developer can run +# the AIE4 path without an install. Skipped, with no error, when the runtime +# directory is not configured. +# +# The staged directory is what the installer scripts copy, so it gets the +# shippable report. The audit record, which names build-machine absolute +# paths, stays in the build tree. +function(flm_aie4_stage_for_target target) + if(NOT FLM_ENABLE_CORELIB_AIE4 OR NOT WIN32) + return() + endif() + flm_aie4_runtime_configured(_configured) + if(NOT _configured) + return() + endif() + foreach(_stage_dir IN LISTS ARGN) + string(MD5 _stage_id "${_stage_dir}") + flm_aie4_stage_command(_command + "${_stage_dir}" + "${_stage_dir}/aie4-closure.txt" + "${CMAKE_BINARY_DIR}/aie4-closure-audit-${_stage_id}.txt") + add_custom_command(TARGET ${target} POST_BUILD + COMMAND ${_command} + COMMENT "Deriving the Phi-4 AIE4 runtime closure") + endforeach() +endfunction() + +# Emits the install rule. The derivation runs at install time, against the +# ryzenai_corelib.dll actually being shipped, so the packaged closure can never +# be a stale list captured when the project was configured. +function(flm_aie4_install_runtime) + cmake_parse_arguments(_arg "" "DESTINATION;COMPONENT" "" ${ARGN}) + if(NOT FLM_ENABLE_CORELIB_AIE4) + return() + endif() + if(NOT WIN32) + message(FATAL_ERROR + "FLM_ENABLE_CORELIB_AIE4 runtime packaging is Windows-only") + endif() + set(_component_args "") + if(_arg_COMPONENT) + set(_component_args COMPONENT ${_arg_COMPONENT}) + endif() + install(CODE " +set(FLM_AIE4_CORELIB_DIR [[${RYZENAI_CORELIB_RUNTIME_DIR}]]) +set(FLM_AIE4_XRT_DIR [[${XRT_RUNTIME_DIR}]]) +set(FLM_AIE4_EXTRA_DIRS [[${FLM_AIE4_DEPENDENCY_DIRS}]]) +set(FLM_AIE4_DESTINATION [[${_arg_DESTINATION}]]) +set(FLM_AIE4_REPORT + \"\${CMAKE_INSTALL_PREFIX}/${_arg_DESTINATION}/aie4-closure.txt\") +set(FLM_AIE4_AUDIT [[${CMAKE_BINARY_DIR}/aie4-closure-audit-install.txt]]) +" + ${_component_args}) + install(SCRIPT "${FLM_AIE4_STAGE_SCRIPT}" ${_component_args}) +endfunction() diff --git a/src/cmake/ReadCorelibVersion.ps1 b/src/cmake/ReadCorelibVersion.ps1 new file mode 100644 index 00000000..6529b36d --- /dev/null +++ b/src/cmake/ReadCorelibVersion.ps1 @@ -0,0 +1,115 @@ +# Reads the version out of a ryzenai_corelib.dll by calling into it. +# +# Prints exactly one line, `MAJOR.MINOR.PATCH`, on success. Anything else is a +# failure and exits non-zero with the reason on stderr. +# +# The version is read from the LOADED library rather than from the headers the +# product compiled against, because the question this answers in the field is +# "which runtime produced this result", and the headers cannot answer it. The +# same reasoning gave every acceptance record the resolved DLL's SHA-256: the +# run-to-run divergence investigation was crippled for days because nothing +# recorded which library had run. +# +# Known and deliberately not hidden: through the whole 0.x series +# `ryzenai_corelib_get_version` returns a hard-coded 0.1.0, so the version +# alone does not identify a build. The SHA-256 recorded beside it is the field +# that does. The version is still worth carrying because it is the only +# self-describing identity the ABI offers, and a future 1.x that actually moves +# it will be readable by a consumer that already knows where to look. + +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [string]$Dll +) + +$ErrorActionPreference = 'Stop' + +if (-not (Test-Path -LiteralPath $Dll -PathType Leaf)) { + [Console]::Error.WriteLine("ReadCorelibVersion: not a file: $Dll") + exit 1 +} +$resolved = (Resolve-Path -LiteralPath $Dll).Path + +# Add-Type compiles C#, the C# compiler reads LIB and INCLUDE, and PowerShell +# runs it with warnings-as-errors. Under MSBuild those variables are set to the +# MSVC toolchain's paths, one of which is relative -- so this probe failed with +# +# Warning as Error: Invalid search path 'lib\um\x64' specified in +# 'LIB environment variable' +# +# and took the whole flm build down with it, because the staging step it runs +# from is a POST_BUILD command. It passed every earlier test because those ran +# it from a plain shell, where LIB is unset. Nothing here needs either +# variable, so they are cleared for the compile and restored afterwards. +$savedLib = $env:LIB +$savedInclude = $env:INCLUDE +try { + Remove-Item Env:LIB -ErrorAction SilentlyContinue + Remove-Item Env:INCLUDE -ErrorAction SilentlyContinue + +Add-Type @" +using System; +using System.Runtime.InteropServices; +public static class FlmCorelibVersionProbe { + // LOAD_WITH_ALTERED_SEARCH_PATH (0x8) makes the loader resolve this + // module's own imports out of the directory it was loaded from, which is + // the staged closure. Without it the imports are resolved against the + // PowerShell host's directory and the load fails with Win32 126 even + // though the closure beside the DLL is complete. + public const uint LOAD_WITH_ALTERED_SEARCH_PATH = 0x00000008; + + [DllImport("kernel32", CharSet = CharSet.Unicode, SetLastError = true)] + public static extern IntPtr LoadLibraryEx(string path, IntPtr file, uint flags); + + [DllImport("kernel32", CharSet = CharSet.Ansi, SetLastError = true, + BestFitMapping = false, ThrowOnUnmappableChar = true)] + public static extern IntPtr GetProcAddress(IntPtr module, string name); + + [DllImport("kernel32", SetLastError = true)] + public static extern bool FreeLibrary(IntPtr module); + + // extern "C" void ryzenai_corelib_get_version( + // uint32_t* major, uint32_t* minor, uint32_t* patch); + // x64 Windows has a single native calling convention, so Cdecl here is + // exact rather than an assumption that happens to work. + [UnmanagedFunctionPointer(CallingConvention.Cdecl)] + public delegate void GetVersion(out uint major, out uint minor, out uint patch); +} +"@ +} finally { + if ($null -ne $savedLib) { $env:LIB = $savedLib } + if ($null -ne $savedInclude) { $env:INCLUDE = $savedInclude } +} + +$module = [FlmCorelibVersionProbe]::LoadLibraryEx( + $resolved, [IntPtr]::Zero, + [FlmCorelibVersionProbe]::LOAD_WITH_ALTERED_SEARCH_PATH) +if ($module -eq [IntPtr]::Zero) { + $code = [Runtime.InteropServices.Marshal]::GetLastWin32Error() + [Console]::Error.WriteLine( + "ReadCorelibVersion: LoadLibraryEx failed with Win32 error $code for " + + "$resolved. The staged closure beside it is incomplete.") + exit 1 +} +try { + $entry = [FlmCorelibVersionProbe]::GetProcAddress( + $module, 'ryzenai_corelib_get_version') + if ($entry -eq [IntPtr]::Zero) { + $code = [Runtime.InteropServices.Marshal]::GetLastWin32Error() + [Console]::Error.WriteLine( + "ReadCorelibVersion: $resolved exports no " + + "ryzenai_corelib_get_version (Win32 error $code)") + exit 1 + } + $call = [Runtime.InteropServices.Marshal]::GetDelegateForFunctionPointer( + $entry, [Type][FlmCorelibVersionProbe+GetVersion]) + $major = 0 + $minor = 0 + $patch = 0 + $call.Invoke([ref]$major, [ref]$minor, [ref]$patch) +} finally { + [void][FlmCorelibVersionProbe]::FreeLibrary($module) +} + +Write-Output ("{0}.{1}.{2}" -f $major, $minor, $patch) diff --git a/src/cmake/StageAie4Runtime.cmake b/src/cmake/StageAie4Runtime.cmake new file mode 100644 index 00000000..02333716 --- /dev/null +++ b/src/cmake/StageAie4Runtime.cmake @@ -0,0 +1,274 @@ +# Derives and stages the optional Phi-4 AIE4 corelib runtime closure. +# +# Design `CLOSURE-1`: the closure is defined by what the shipped +# `ryzenai_corelib.dll` actually imports, enumerated with a dependency walker +# against that exact binary. It is never transcribed, because different +# DynamicDispatch linkages and different dependency builds import different +# sets: the 223 MB dev-box binary statically links DynamicDispatch while the +# 0.8 MB target binary loads it as separate DLLs, and neither closure +# validates the other. +# +# This file is a standalone script. It runs both under `cmake -P` (developer +# staging beside `flm.exe`) and under `install(SCRIPT)` (packaging), so the +# packaged closure and the closure a developer runs against are produced by +# the same derivation rather than two lists that can drift apart. +# +# Inputs: +# FLM_AIE4_CORELIB_DIR directory holding ryzenai_corelib.dll (required) +# FLM_AIE4_XRT_DIR directory holding stageable XRT DLLs (optional) +# FLM_AIE4_EXTRA_DIRS additional dependency search directories +# FLM_AIE4_DESTINATION directory to stage into; relative paths resolve +# against CMAKE_INSTALL_PREFIX +# FLM_AIE4_REPORT optional path for the SHIPPABLE closure report: +# file names and hashes only, no build-machine paths +# FLM_AIE4_AUDIT optional path for the build-side audit record, which +# does name absolute source paths and must stay in the +# build tree + +cmake_minimum_required(VERSION 3.24) + +if(NOT WIN32 AND NOT CMAKE_HOST_WIN32) + message(FATAL_ERROR + "The Phi-4 AIE4 runtime closure is Windows-only.") +endif() + +if(NOT FLM_AIE4_CORELIB_DIR) + message(FATAL_ERROR + "RYZENAI_CORELIB_RUNTIME_DIR is required to install or package the " + "Phi-4 AIE4 feature (FLM_ENABLE_CORELIB_AIE4=ON). Point it at the " + "directory holding the ryzenai_corelib.dll you intend to ship, then " + "re-run the install step. Building flm.exe does not need it: the " + "corelib DLL is resolved at runtime by absolute path and flm.exe " + "never links ryzenai_corelib.lib.") +endif() + +if(NOT IS_DIRECTORY "${FLM_AIE4_CORELIB_DIR}") + message(FATAL_ERROR + "RYZENAI_CORELIB_RUNTIME_DIR does not name a directory: " + "${FLM_AIE4_CORELIB_DIR}") +endif() + +set(_flm_aie4_root "${FLM_AIE4_CORELIB_DIR}/ryzenai_corelib.dll") +if(NOT EXISTS "${_flm_aie4_root}") + message(FATAL_ERROR + "RYZENAI_CORELIB_RUNTIME_DIR contains no ryzenai_corelib.dll: " + "${FLM_AIE4_CORELIB_DIR}") +endif() + +# Search order matters. Directories supplied for this package win over +# anything the machine happens to provide, so a build box with an ambient +# conda or toolchain prefix stages the DLLs we chose rather than the ones it +# stumbled across. +set(_flm_aie4_search_dirs "${FLM_AIE4_CORELIB_DIR}") +if(FLM_AIE4_XRT_DIR AND IS_DIRECTORY "${FLM_AIE4_XRT_DIR}") + list(APPEND _flm_aie4_search_dirs "${FLM_AIE4_XRT_DIR}") +elseif(FLM_AIE4_XRT_DIR) + message(FATAL_ERROR + "XRT_RUNTIME_DIR does not name a directory: ${FLM_AIE4_XRT_DIR}") +endif() +foreach(_flm_aie4_dir IN LISTS FLM_AIE4_EXTRA_DIRS) + if(_flm_aie4_dir) + if(NOT IS_DIRECTORY "${_flm_aie4_dir}") + message(FATAL_ERROR + "FLM_AIE4_DEPENDENCY_DIRS entry is not a directory: " + "${_flm_aie4_dir}") + endif() + list(APPEND _flm_aie4_search_dirs "${_flm_aie4_dir}") + endif() +endforeach() +list(REMOVE_DUPLICATES _flm_aie4_search_dirs) + +# `dyn_bins.dll` holds DynamicDispatch's precompiled binaries and is opened by +# NAME at runtime, so it is never an import and a dependency walker cannot see +# it. It has to be found by presence, and WHERE it lives depends on the +# DynamicDispatch linkage: +# +# * statically linked DD -- the dev box -- puts it beside +# ryzenai_corelib.dll, because Transaction resolves it against the +# directory of the module that linked DD in; +# * shared DD -- the AIE4 target -- puts it beside dyn_dispatch_core.dll. +# +# Searching only the corelib directory is therefore correct on one box and +# silently wrong on the other. It is silent because nothing fails to load: the +# process starts, and every shape query then comes back "Shape list size: 0", +# which surfaces as `matmul_bf16_weights_create_onnx failed ... not supported +# in this supported shape list` at weight-packing time. That is a long way from +# the missing file, which is why this searches every directory the closure is +# allowed to draw from rather than assuming a linkage. +set(_flm_aie4_runtime_loaded "") +foreach(_flm_aie4_dir IN LISTS _flm_aie4_search_dirs) + if(EXISTS "${_flm_aie4_dir}/dyn_bins.dll") + list(APPEND _flm_aie4_runtime_loaded + "${_flm_aie4_dir}/dyn_bins.dll") + break() + endif() +endforeach() +if(NOT _flm_aie4_runtime_loaded) + message(STATUS + "No dyn_bins.dll in any search directory. That is expected only for " + "a DynamicDispatch build that embeds its binaries; if the staged " + "runtime later reports \"Shape list size: 0\", this is why.") +endif() + +# The Visual C++ runtime is deliberately not staged. `flm.exe` itself imports +# MSVCP140/VCRUNTIME140, so the redistributable is already a product-wide +# prerequisite and the AIE4 feature adds no new one. Copying a build machine's +# conda or toolchain copy beside ryzenai_corelib.dll would ship a second, +# possibly older, runtime next to the one the rest of the process already +# loaded. +set(_flm_aie4_pre_exclude + "^api-ms-win-.*" + "^ext-ms-.*" + "^[Mm][Ss][Vv][Cc][Pp]1[0-9]+.*\\.dll$" + "^[Vv][Cc][Rr][Uu][Nn][Tt][Ii][Mm][Ee]1[0-9]+.*\\.dll$" + "^[Cc][Oo][Nn][Cc][Rr][Tt]1[0-9]+.*\\.dll$") +set(_flm_aie4_post_exclude + "^[A-Za-z]:[\\\\/][Ww][Ii][Nn][Dd][Oo][Ww][Ss][\\\\/].*" + "^.*[\\\\/][Ss][Yy][Ss][Tt][Ee][Mm]32[\\\\/].*" + "^.*[\\\\/][Ss][Yy][Ss][Ww][Oo][Ww]64[\\\\/].*") + +file(GET_RUNTIME_DEPENDENCIES + LIBRARIES + "${_flm_aie4_root}" + ${_flm_aie4_runtime_loaded} + RESOLVED_DEPENDENCIES_VAR _flm_aie4_resolved + UNRESOLVED_DEPENDENCIES_VAR _flm_aie4_unresolved + CONFLICTING_DEPENDENCIES_PREFIX _flm_aie4_conflicting + DIRECTORIES ${_flm_aie4_search_dirs} + PRE_EXCLUDE_REGEXES ${_flm_aie4_pre_exclude} + POST_EXCLUDE_REGEXES ${_flm_aie4_post_exclude}) + +if(_flm_aie4_unresolved) + list(JOIN _flm_aie4_unresolved "\n " _flm_aie4_unresolved_text) + message(FATAL_ERROR + "The Phi-4 AIE4 runtime closure is incomplete. " + "${_flm_aie4_root} imports DLLs that were not found in " + "RYZENAI_CORELIB_RUNTIME_DIR, XRT_RUNTIME_DIR, " + "FLM_AIE4_DEPENDENCY_DIRS, or the approved system directories:\n" + " ${_flm_aie4_unresolved_text}\n" + "Add the directory that provides them to FLM_AIE4_DEPENDENCY_DIRS. " + "Do not rely on them being on PATH: a closure that only loads " + "because a build machine had a conda or toolchain prefix on PATH " + "fails on the target with Win32 error 126.") +endif() + +if(_flm_aie4_conflicting_FILENAMES) + list(JOIN _flm_aie4_conflicting_FILENAMES ", " _flm_aie4_conflict_text) + message(FATAL_ERROR + "The Phi-4 AIE4 runtime closure resolved conflicting copies of: " + "${_flm_aie4_conflict_text}. Narrow the search directories so each " + "DLL has one unambiguous source.") +endif() + +set(_flm_aie4_files ${_flm_aie4_root} ${_flm_aie4_runtime_loaded}) +list(APPEND _flm_aie4_files ${_flm_aie4_resolved}) +list(REMOVE_DUPLICATES _flm_aie4_files) +list(SORT _flm_aie4_files) + +set(_flm_aie4_destination "${FLM_AIE4_DESTINATION}") +if(NOT _flm_aie4_destination) + message(FATAL_ERROR "FLM_AIE4_DESTINATION was not set") +endif() +if(NOT IS_ABSOLUTE "${_flm_aie4_destination}") + set(_flm_aie4_destination + "${CMAKE_INSTALL_PREFIX}/${_flm_aie4_destination}") +endif() + +file(MAKE_DIRECTORY "${_flm_aie4_destination}") +foreach(_flm_aie4_file IN LISTS _flm_aie4_files) + message(STATUS "Staging AIE4 runtime: ${_flm_aie4_file}") + file(COPY "${_flm_aie4_file}" + DESTINATION "${_flm_aie4_destination}" + FOLLOW_SYMLINK_CHAIN) +endforeach() + +# Two records, deliberately different. +# +# The shippable report lists the derived closure by name and SHA-256 only. It +# is written into the staged directory, which the installer scripts copy +# verbatim, so it must not carry absolute paths from the machine that built it: +# a customer artifact naming a developer's conda prefix leaks build-machine +# layout for no benefit to the reader. The hashes are what a recipient can +# actually act on, since they verify the staged bits. +# +# It also carries the identity of the corelib itself, ahead of the file list. +# The first question asked about a wrong number in the field is "which runtime +# produced it", and until this project put the resolved DLL's SHA-256 into its +# own test artifacts, nothing could answer it -- an investigation ran for days +# on the assumption that two runs had loaded the same library, with no record +# either way. A shipped artifact needs the same answer available to a reader +# who has only the installed directory. +# +# Deliberately bounded: corelib version and corelib hash, nothing else. The +# FastFlow commit, the model revision and the driver identity live in the +# baseline identity block and the acceptance record. Copying them here would +# create a second source that drifts. +if(FLM_AIE4_REPORT) + # Probed against the STAGED copy, not the source copy. The staged file is + # the one that ships, and it is the one whose closure sits beside it -- so + # a load failure here is a real statement about the artifact rather than + # about the build machine's PATH. + set(_flm_aie4_staged_root "${_flm_aie4_destination}/ryzenai_corelib.dll") + if(NOT EXISTS "${_flm_aie4_staged_root}") + message(FATAL_ERROR + "Staging did not produce ${_flm_aie4_staged_root}") + endif() + get_filename_component(_flm_aie4_probe + "${CMAKE_CURRENT_LIST_DIR}/ReadCorelibVersion.ps1" ABSOLUTE) + if(NOT EXISTS "${_flm_aie4_probe}") + message(FATAL_ERROR + "The corelib version probe is missing: ${_flm_aie4_probe}") + endif() + execute_process( + COMMAND powershell -NoProfile -NonInteractive -ExecutionPolicy Bypass + -File "${_flm_aie4_probe}" + -Dll "${_flm_aie4_staged_root}" + OUTPUT_VARIABLE _flm_aie4_version + ERROR_VARIABLE _flm_aie4_version_error + RESULT_VARIABLE _flm_aie4_version_status + OUTPUT_STRIP_TRAILING_WHITESPACE) + # Hard failure, not a placeholder. + # + # The only way this probe fails is that the staged closure cannot be + # loaded or does not export the entry point -- which means the feature + # being packaged does not work. Writing "unknown" into the report and + # carrying on would ship a broken runtime under a record that reads like a + # successful one, which is the exact failure this project keeps finding. + if(NOT _flm_aie4_version_status EQUAL 0 + OR NOT _flm_aie4_version MATCHES "^[0-9]+\\.[0-9]+\\.[0-9]+$") + message(FATAL_ERROR + "Could not read ryzenai_corelib_get_version from the staged " + "${_flm_aie4_staged_root}.\n" + " status: ${_flm_aie4_version_status}\n" + " stdout: ${_flm_aie4_version}\n" + " stderr: ${_flm_aie4_version_error}\n" + "The staged closure must load before it can be shipped.") + endif() + file(SHA256 "${_flm_aie4_staged_root}" _flm_aie4_root_hash) + set(_flm_aie4_report_text + "corelib_version\t${_flm_aie4_version}\ncorelib_sha256\t${_flm_aie4_root_hash}\n") + foreach(_flm_aie4_file IN LISTS _flm_aie4_files) + get_filename_component(_flm_aie4_leaf "${_flm_aie4_file}" NAME) + file(SHA256 "${_flm_aie4_file}" _flm_aie4_hash) + string(APPEND _flm_aie4_report_text + "staged\t${_flm_aie4_leaf}\t${_flm_aie4_hash}\n") + endforeach() + file(WRITE "${FLM_AIE4_REPORT}" "${_flm_aie4_report_text}") +endif() + +# The audit record stays in the build tree and does name the source path of +# every staged DLL. That path is what shows, after the fact, that a shipped +# dependency came from the intended package rather than from whatever the +# build machine happened to have. +if(FLM_AIE4_AUDIT) + set(_flm_aie4_audit_text "root\t${_flm_aie4_root}\n") + foreach(_flm_aie4_dir IN LISTS _flm_aie4_search_dirs) + string(APPEND _flm_aie4_audit_text "search\t${_flm_aie4_dir}\n") + endforeach() + foreach(_flm_aie4_file IN LISTS _flm_aie4_files) + get_filename_component(_flm_aie4_leaf "${_flm_aie4_file}" NAME) + string(APPEND _flm_aie4_audit_text + "staged\t${_flm_aie4_leaf}\t${_flm_aie4_file}\n") + endforeach() + file(WRITE "${FLM_AIE4_AUDIT}" "${_flm_aie4_audit_text}") +endif() diff --git a/src/common/AutoModel/automodel.cpp b/src/common/AutoModel/automodel.cpp index 5df2f2d2..636e4dcc 100644 --- a/src/common/AutoModel/automodel.cpp +++ b/src/common/AutoModel/automodel.cpp @@ -8,6 +8,22 @@ #include "AutoModel/automodel.hpp" +ModelRequestError::ModelRequestError( + int http_code, + bool session_cleared, + std::string message) + : std::runtime_error(std::move(message)), + http_code_(http_code), + session_cleared_(session_cleared) {} + +int ModelRequestError::http_code() const noexcept { + return http_code_; +} + +bool ModelRequestError::session_cleared() const noexcept { + return session_cleared_; +} + AutoModel::AutoModel(flm_rt::device* npu_device_inst, std::string current_model) { this->npu_device_inst = npu_device_inst; this->current_model = current_model; @@ -116,17 +132,21 @@ void AutoModel::_shared_load_model(std::string model_path, json model_info, int header_print("FLM", "Model already loaded: " << this->model_path); return; } + this->_shared_initialize_model_state( + std::move(model_path), + std::move(model_info), + default_context_length); + this->_shared_initialize_legacy_npu(enable_preemption); +} +void AutoModel::_shared_initialize_model_state( + std::string model_path, + json model_info, + int default_context_length) { this->model_path = model_path; header_print("FLM", "Loading model: " << this->model_path); this->lm_config = std::make_unique(); this->lm_config->from_pretrained(this->model_path); - if (this->npu_device_inst == nullptr) { - header_print("ERROR", "NPU device instance is nullptr"); - exit(1); - } - this->npu = std::make_unique(npu_device::device_npu2, this->npu_device_inst, enable_preemption); - this->enable_preemption = enable_preemption; // Set context length: use provided value if not -1, otherwise use model default if (default_context_length != -1) { this->MAX_L = default_context_length; @@ -144,62 +164,140 @@ void AutoModel::_shared_load_model(std::string model_path, json model_info, int this->total_tokens = 0; } -bool AutoModel::_shared_insert(chat_meta_info_t& meta_info, std::vector& tokens, std::function is_cancelled, void* payload, int first_len_run) { - - // print token history - // header_print("DEBUG", "Current token history: "); - // for (size_t i = 0; i < this->token_history.size(); i++) { - // std::cout << this->token_history[i] << " "; - // } - // std::cout << std::endl; - // // print tokens to insert - // header_print("DEBUG", "Tokens to insert: "); - // for (size_t i = 0; i < tokens.size(); i++) { - // std::cout << tokens[i] << " "; - // } - // std::cout << std::endl; - - // prefix check for tokens and token history to see if we can skip some tokens - const size_t idx = this->token_history.size(); - size_t skip_count = 0; - for (size_t i = 0; i < idx; i++) { - if (i < tokens.size() && tokens[i] == this->token_history[i]) { - skip_count++; - } - else { - break; - } +void AutoModel::_shared_initialize_legacy_npu( + bool enable_preemption) { + if (this->npu_device_inst == nullptr) { + header_print("ERROR", "NPU device instance is nullptr"); + exit(1); } - if (skip_count != idx) { + this->npu = std::make_unique( + npu_device::device_npu2, + this->npu_device_inst, + enable_preemption); + this->enable_preemption = enable_preemption; +} + +size_t AutoModel::_matching_prefix_length( + std::span tokens) const { + const size_t compared = + std::min(tokens.size(), this->token_history.size()); + size_t matched = 0; + while ( + matched < compared && + tokens[matched] == this->token_history[matched]) { + ++matched; + } + return matched; +} + +bool AutoModel::_shared_insert( + chat_meta_info_t& meta_info, + std::vector& tokens, + std::function is_cancelled, + void* payload, + int first_len_run) { + return this->_shared_insert( + meta_info, + tokens, + std::move(is_cancelled), + payload, + first_len_run, + PrefixHitAction::AppendSuffixBatched); +} + +bool AutoModel::_shared_insert( + chat_meta_info_t& meta_info, + std::vector& tokens, + std::function is_cancelled, + void* payload, + int first_len_run, + PrefixHitAction prefix_action) { + const auto mark_cancelled = [&] { + meta_info.stop_reason = CANCEL_DETECTED; + buffer_.clear(); + current_mode_ = StreamEventType::CONTENT; + tool_name_.clear(); + is_in_tool_block_ = false; + }; + + const size_t history_size = this->token_history.size(); + const size_t matched = this->_matching_prefix_length(tokens); + const bool prefix_hit = matched == history_size; + const bool recompute = + prefix_action == PrefixHitAction::RecomputeFull; + const bool clear_before_prefill = !prefix_hit || recompute; + const bool fresh_prefill = history_size == 0; + + if ( + (fresh_prefill || clear_before_prefill) && + is_cancelled()) { + mark_cancelled(); + return false; + } + + size_t skip_count = prefix_hit && !recompute + ? history_size + : 0; + if (clear_before_prefill) { clear_context(); - skip_count = 0; } tokens.erase(tokens.begin(), tokens.begin() + skip_count); - - if (this->total_tokens + tokens.size() >= this->MAX_L){ + if (this->total_tokens + tokens.size() >= this->MAX_L) { header_print("WARNING", "Max length reached, stopping prefilling..."); return false; } - for (int token : tokens){ - this->token_history.push_back(token); - } - buffer y; + buffer y; auto prefill_start_time = this->profiler_list[PREFILL_TIME].start(); - - y = _chunked_insert(meta_info, tokens, is_cancelled, payload, first_len_run); - auto prefill_end_time = this->profiler_list[PREFILL_TIME].stop(tokens.size()); + size_t committed_tokens = 0; + if ( + prefix_action == PrefixHitAction::AppendSuffixOneByOne && + !clear_before_prefill) { + for (const int token : tokens) { + if (is_cancelled()) { + mark_cancelled(); + break; + } + std::vector one_token{token}; + y = this->lm_engine->prefill(one_token, payload); + payload = nullptr; + this->token_history.push_back(token); + ++this->total_tokens; + ++committed_tokens; + } + } else { + for (const int token : tokens) { + this->token_history.push_back(token); + } + const bool direct_full_prefill = + clear_before_prefill && + prefix_action != PrefixHitAction::AppendSuffixBatched; + y = direct_full_prefill + ? this->lm_engine->prefill(tokens, payload) + : _chunked_insert( + meta_info, + tokens, + is_cancelled, + payload, + first_len_run); + if (meta_info.stop_reason != CANCEL_DETECTED) { + committed_tokens = tokens.size(); + this->total_tokens += tokens.size(); + } + } + + auto prefill_end_time = + this->profiler_list[PREFILL_TIME].stop(committed_tokens); meta_info.prefill_duration = (uint64_t)time_utils::duration_ns(prefill_start_time, prefill_end_time).first; - meta_info.prompt_tokens = tokens.size(); + meta_info.prompt_tokens = static_cast(committed_tokens); if (meta_info.stop_reason == CANCEL_DETECTED) { return false; } - this->total_tokens += tokens.size(); - if (this->total_tokens >= this->MAX_L){ + if (this->total_tokens >= this->MAX_L) { header_print("WARNING", "Max length reached, stopping prefilling..."); } this->profiler_list[SAMPLING_TIME].start(); diff --git a/src/common/AutoModel/modeling_phi4.cpp b/src/common/AutoModel/modeling_phi4.cpp index 06c164f5..332142ef 100644 --- a/src/common/AutoModel/modeling_phi4.cpp +++ b/src/common/AutoModel/modeling_phi4.cpp @@ -7,74 +7,381 @@ #include "AutoModel/modeling_phi4.hpp" -/************ Phi4 family **************/ -Phi4::Phi4(flm_rt::device* npu_device_inst) : AutoModel(npu_device_inst, "Phi4") {} +#if defined(FLM_ENABLE_CORELIB_AIE4) +#include +#endif -void Phi4::load_model(std::string model_path, json model_info, int default_context_length, bool enable_preemption) { - this->_shared_load_model(model_path, model_info, default_context_length, enable_preemption); - - this->q4nx = std::make_unique(this->model_path); - // model_type == phi4 - this->lm_engine = std::make_unique(*this->lm_config, this->npu.get(), this->MAX_L); - this->lm_engine->load_weights(*this->q4nx); - - //free the q4nx - this->q4nx.reset(); - - this->lm_engine->clear_context(); - this->setup_tokenizer(model_path); - this->sampler.reset(); +#include +#include +#include +#include +#include +#include +#include + +namespace { + +constexpr std::string_view kCorelibBackend = "corelib_aie4"; +constexpr int kPhi4Eos = 200020; +constexpr int kPhi4End = 199999; + +enum class Phi4Backend { + Legacy, + CorelibAie4 +}; + +Phi4Backend ResolveBackend(const json& model_info) { + const auto details = model_info.find("details"); + if ( + details == model_info.end() || + !details->is_object() || + !details->contains("execution_backend")) { + return Phi4Backend::Legacy; + } + + const auto& value = details->at("execution_backend"); + if (!value.is_string()) { + throw std::invalid_argument( + "Phi-4 details.execution_backend must be a string"); + } + const std::string backend = value.get(); + if (backend == kCorelibBackend) { + return Phi4Backend::CorelibAie4; + } + throw std::invalid_argument( + "Unknown Phi-4 execution backend '" + backend + "'"); +} + +std::uint32_t ResolveAie4ContextLength( + const json& model_info, + int requested_context_length) { + std::int64_t value = requested_context_length; + if (requested_context_length == -1) { + if (!model_info.contains("default_context_length")) { + throw std::invalid_argument( + "Phi-4 AIE4 model metadata has no default_context_length"); + } + value = model_info.at("default_context_length").get(); + } + if (value <= 0 || value > 4096) { + throw std::out_of_range( + "Phi-4 AIE4 maximum length must be in 1..4096"); + } + return static_cast(value); +} + +void RequireAie4FrontendFile( + const std::filesystem::path& model_path, + std::string_view filename) { + const auto path = model_path / filename; + std::ifstream input(path, std::ios::binary); + if (!input) { + throw std::runtime_error( + "Phi-4 AIE4 package is missing required frontend file: " + + path.string()); + } +} + +#if defined(FLM_ENABLE_CORELIB_AIE4) + +// Design `MODEL-2`. The overlay `config.json` exists only because the upstream +// repository ships none, and it restates the Section 5.1 constants so +// FastFlow's existing readers keep working. It is never an independent source +// of truth. `flm::phi4::constants` holds those same Section 5.1 values and is +// what the manifest loader validates the ONNX initializers against, so +// requiring the overlay to equal them is what makes a disagreement between the +// overlay and the real weights a hard load failure instead of a silent +// reconfiguration of the model. +void RequireAie4OverlayMatchesModelConstants( + const std::filesystem::path& model_path) { + namespace constants = flm::phi4::constants; + const auto path = model_path / "config.json"; + json config; + try { + std::ifstream input(path, std::ios::binary); + input.exceptions(std::ios::badbit); + config = json::parse(input); + } catch (const std::exception& error) { + throw std::runtime_error( + "Phi-4 AIE4 config.json could not be parsed: " + path.string() + + ": " + error.what()); + } + if (!config.is_object()) { + throw std::runtime_error( + "Phi-4 AIE4 config.json must be a JSON object: " + path.string()); + } + + const auto require_integer = + [&](std::string_view field, std::int64_t expected) { + const auto found = config.find(field); + if (found == config.end() || !found->is_number_integer()) { + throw std::runtime_error( + "Phi-4 AIE4 config.json is missing integer field '" + + std::string(field) + "': " + path.string()); + } + const std::int64_t actual = found->get(); + if (actual != expected) { + throw std::runtime_error( + "Phi-4 AIE4 config.json disagrees with the validated " + "model constants: " + + std::string(field) + " is " + std::to_string(actual) + + ", expected " + std::to_string(expected) + + ". The overlay restates the model contract and cannot " + "redefine it; regenerate it from the packaged model " + "rather than editing it."); + } + }; + + require_integer("num_hidden_layers", constants::kLayerCount); + require_integer("hidden_size", constants::kHiddenSize); + require_integer("intermediate_size", constants::kIntermediateSize); + require_integer("num_attention_heads", constants::kQueryHeadCount); + require_integer("num_key_value_heads", constants::kKvHeadCount); + require_integer("head_dim", constants::kHeadSize); + require_integer("vocab_size", constants::kVocabularySize); + const auto model_type = config.find("model_type"); + if ( + model_type == config.end() || + !model_type->is_string() || + model_type->get() != "phi4") { + throw std::runtime_error( + "Phi-4 AIE4 config.json must declare model_type \"phi4\": " + + path.string()); + } + + const auto epsilon = config.find("rms_norm_eps"); + if (epsilon == config.end() || !epsilon->is_number()) { + throw std::runtime_error( + "Phi-4 AIE4 config.json is missing numeric field " + "'rms_norm_eps': " + + path.string()); + } + if (epsilon->get() != constants::kRmsEpsilon) { + throw std::runtime_error( + "Phi-4 AIE4 config.json disagrees with the validated model " + "constants: rms_norm_eps does not equal the packed epsilon. " + "The overlay restates the model contract and cannot redefine it."); + } +} + +#endif // FLM_ENABLE_CORELIB_AIE4 + +void ConfigureDefaultSampler(Phi4& model) { sampler_config config; config.top_k = 40; config.top_p = 0.9; config.min_p = 0.1; config.temperature = 0.8; + model.set_sampler(config); +} + +} // namespace + +#if defined(FLM_CORELIB_TESTING) +Phi4::EngineFactoryForTesting Phi4::engine_factory_for_testing_; +#endif + +/************ Phi4 family **************/ +Phi4::Phi4(flm_rt::device* npu_device_inst) : AutoModel(npu_device_inst, "Phi4") {} + +void Phi4::load_model(std::string model_path, json model_info, int default_context_length, bool enable_preemption) { + const Phi4Backend backend = ResolveBackend(model_info); + if (backend == Phi4Backend::CorelibAie4) { +#if !defined(FLM_ENABLE_CORELIB_AIE4) + throw std::runtime_error( + "This binary was built without Phi-4 AIE4 corelib support"); +#else + if (enable_preemption) { + throw std::invalid_argument( + "Phi-4 AIE4 corelib execution does not support preemption"); + } + + const std::uint32_t context_length = + ResolveAie4ContextLength( + model_info, + default_context_length); + const std::filesystem::path package_path(model_path); + RequireAie4FrontendFile(package_path, "config.json"); + RequireAie4FrontendFile(package_path, "tokenizer.json"); + RequireAie4FrontendFile( + package_path, + "tokenizer_config.json"); + RequireAie4OverlayMatchesModelConstants(package_path); + + this->_shared_initialize_model_state( + model_path, + model_info, + static_cast(context_length)); + this->npu.reset(); + this->enable_preemption = false; + this->setup_tokenizer(model_path, true); + + auto runtime = + flm::corelib::CorelibRuntime::GetOrCreate( + utils::get_executable_directory()); + std::unique_ptr engine; +#if defined(FLM_CORELIB_TESTING) + if (!engine_factory_for_testing_) { + throw std::logic_error( + "Phi-4 frontend test engine factory is not installed"); + } + engine = engine_factory_for_testing_( + true, + *this->lm_config, + nullptr, + package_path, + context_length); +#else + engine = std::make_unique( + *this->lm_config, + package_path, + runtime, + context_length); +#endif + engine->clear_context(); + + this->lm_engine = std::move(engine); + this->corelib_runtime_ = std::move(runtime); + this->uses_corelib_aie4_ = true; + this->last_continuation_route_.reset(); + this->last_continuation_ns_ = 0; + this->append_continuation_ns_ = 0; + this->reprefill_continuation_ns_ = 0; + this->sampler.reset(); + ConfigureDefaultSampler(*this); +#endif + } else { +#if defined(FLM_ENABLE_CORELIB_AIE4) + this->uses_corelib_aie4_ = false; + this->corelib_runtime_.reset(); + this->last_continuation_route_.reset(); +#if defined(FLM_CORELIB_TESTING) + this->metrics_for_testing_.reset(); +#endif +#endif + this->_shared_load_model( + model_path, + model_info, + default_context_length, + enable_preemption); + + std::unique_ptr engine; +#if defined(FLM_CORELIB_TESTING) + if (!engine_factory_for_testing_) { + throw std::logic_error( + "Phi-4 frontend test engine factory is not installed"); + } + engine = engine_factory_for_testing_( + false, + *this->lm_config, + this->npu.get(), + std::filesystem::path(model_path), + this->MAX_L); +#else + this->q4nx = std::make_unique(this->model_path); + engine = std::make_unique( + *this->lm_config, + this->npu.get(), + this->MAX_L); + engine->load_weights(*this->q4nx); + this->q4nx.reset(); +#endif + + engine->clear_context(); + this->setup_tokenizer(model_path, false); + this->lm_engine = std::move(engine); + this->sampler.reset(); + ConfigureDefaultSampler(*this); + } - this->set_sampler(config); for (size_t i = 0; i < PROFILER_TYPE_NUM; i++) { this->profiler_list[i].reset(); } } -void Phi4::setup_tokenizer(std::string model_path) { - // load tokenizer configurations - #ifdef _WIN32 +void Phi4::setup_tokenizer( + const std::string& model_path, + bool require_aie4_eos) { +#ifdef _WIN32 std::string tokenizer_config_path = model_path + "\\tokenizer_config.json"; - #else +#else std::string tokenizer_config_path = model_path + "/tokenizer_config.json"; - #endif +#endif std::ifstream fs_config(tokenizer_config_path, std::ios::in | std::ios::binary); if (fs_config.fail()) { - std::cerr << "Cannot open " << tokenizer_config_path << std::endl; - exit(1); + throw std::runtime_error( + "Cannot open " + tokenizer_config_path); } std::string data_config; fs_config.seekg(0, std::ios::end); - size_t size_config = static_cast(fs_config.tellg()); + const auto end = fs_config.tellg(); + if (end < 0) { + throw std::runtime_error( + "Cannot read " + tokenizer_config_path); + } + size_t size_config = static_cast(end); fs_config.seekg(0, std::ios::beg); data_config.resize(size_config); - fs_config.read(data_config.data(), size_config); + fs_config.read( + data_config.data(), + static_cast(size_config)); + if (!fs_config && size_config != 0) { + throw std::runtime_error( + "Cannot read " + tokenizer_config_path); + } fs_config.close(); auto tokenizer_config = nlohmann::json::parse(data_config); - this->has_bos_token = false; - // load chat template - this->chat_tmpl = std::make_unique( - tokenizer_config["chat_template"], + + if ( + !tokenizer_config.contains("chat_template") || + !tokenizer_config.at("chat_template").is_string()) { + throw std::invalid_argument( + "Phi-4 tokenizer_config.json requires a string chat_template"); + } + if (!tokenizer_config.contains("eos_token_id")) { + throw std::invalid_argument( + "Phi-4 tokenizer_config.json requires eos_token_id"); + } + + auto chat_template = std::make_unique( + tokenizer_config.at("chat_template").get(), "", - "" - ); + ""); + std::vector parsed_eos_ids; - if (this->has_bos_token) { - this->bos_token_id = tokenizer_config["bos_token_id"].get(); + const auto& eos_ids = tokenizer_config.at("eos_token_id"); + if (eos_ids.is_number_integer()) { + parsed_eos_ids.push_back(eos_ids.get()); + } else if (eos_ids.is_array()) { + for (const auto& token : eos_ids) { + parsed_eos_ids.push_back(token.get()); + } + } else { + throw std::invalid_argument( + "Phi-4 tokenizer_config.json eos_token_id must be " + "an integer or array"); } - else { - this->bos_token_id = -1; + + if (require_aie4_eos) { + const auto has_id = [&](int id) { + return std::find( + parsed_eos_ids.begin(), + parsed_eos_ids.end(), + id) != parsed_eos_ids.end(); + }; + if (!has_id(kPhi4Eos) || !has_id(kPhi4End)) { + throw std::invalid_argument( + "Phi-4 AIE4 tokenizer_config.json must contain " + "EOS token IDs 200020 and 199999"); + } } + + this->has_bos_token = false; + this->chat_tmpl = std::move(chat_template); + this->bos_token_id = -1; this->eos_token = ""; - for (auto& token : tokenizer_config["eos_token_id"]) { - this->eos_token_ids.push_back(token.get()); - } + this->eos_token_ids = std::move(parsed_eos_ids); this->user_system_prompt = ""; this->extra_context["user_system_prompt"] = this->user_system_prompt; } @@ -87,8 +394,186 @@ std::string Phi4::apply_chat_template(nlohmann::ordered_json& messages, nlohmann return this->chat_tmpl->apply(inputs); } +#if defined(FLM_ENABLE_CORELIB_AIE4) +// The largest total the AIE4 path can actually reach. +// +// MAX_L alone is wrong here. Every generation ends in decode steps, and the +// token attention kernel supports no window above kMaxDecodeWindow -- one +// below kMaxSequenceLength. Admitting a request against MAX_L therefore +// admits one that cannot finish, and the step that cannot finish fails inside +// flat_mha, mid-step, after this layer's V cache has already been scattered. +// Measured on hardware that took the whole server down; the engine's +// irrevocable boundary is now per submission group so the same refusal is +// recoverable, but it still costs the caller the entire conversation. This +// bound is what turns it into an ordinary MAX_LENGTH_REACHED truncation. +// +// Design 11.2 / SEQ-4 require an unbounded request to stop before the +// user-visible total would exceed the cap. That bound existed for the explicit +// -limit path and not for this one; only running the product's own generation +// loop could have shown it, which is why Steps 1-6 never did. +size_t Phi4::aie4_active_cap() const { + return std::min( + this->MAX_L, + static_cast(flm::phi4::constants::kMaxDecodeWindow)); +} + +void Phi4::validate_aie4_capacity( + size_t rendered_tokens, + std::optional requested_max_new_tokens) const { + // An engaged optional is a POSITIVE token budget. Both entry points -- + // RequestedMaxNewTokens for REST and CliRequestedMaxNewTokens for the + // console -- now normalize every non-positive value to nullopt, because + // num_predict/max_tokens of -1, -2 and 0 are sentinels the generation + // loops read as "no bound", not counts. + // + // This guard used to be the whole of the divergence reported against + // this branch: it turned Ollama's documented "generate forever" + // sentinel into HTTP 400 on the AIE4 tag while every other backend + // served it 200, and it refused explicitly what omitting the field + // allowed. It stays as a contract check on the optional's invariant so + // a future caller that hands the engine a raw parsed value fails here + // rather than reserving a negative amount of cap. + if ( + requested_max_new_tokens.has_value() && + *requested_max_new_tokens < 0) { + throw ModelRequestError( + 400, + false, + "Phi-4 AIE4 requested_max_new_tokens cannot be negative"); + } + + const size_t active_cap = this->aie4_active_cap(); + const size_t requested = + requested_max_new_tokens.has_value() + ? static_cast(*requested_max_new_tokens) + : 0; + const bool prompt_has_no_generation_room = + rendered_tokens >= active_cap; + const bool explicit_request_exceeds_cap = + requested_max_new_tokens.has_value() && + (rendered_tokens > active_cap || + requested > active_cap - rendered_tokens); + if ( + prompt_has_no_generation_room || + explicit_request_exceeds_cap) { + std::stringstream message; + message + << "Phi-4 AIE4 request exceeds the active context cap " + << active_cap << ": rendered prompt has " + << rendered_tokens << " tokens"; + if (requested_max_new_tokens.has_value()) { + message << " and requested output has " + << requested << " tokens"; + } + throw ModelRequestError( + 400, + false, + message.str()); + } +} + +void Phi4::clear_after_corelib_error() { + AutoModel::clear_context(); +} + +std::string Phi4::generate_aie4( + chat_meta_info_t& meta_info, + int length_limit, + std::ostream& os, + std::function is_cancelled) { + std::string result; + stop_reason_t reason = EOT_DETECTED; + int generated_this_call = 0; + + this->profiler_list[DECODING_TIME].reset(); + this->profiler_list[TKOEN_DECODE_TIME].reset(); + + if (this->last_token == -1) { + throw std::logic_error( + "Phi-4 AIE4 generation has no sampled token"); + } + + while ( + this->last_token != -1 && + // aie4_active_cap(), not MAX_L: the last position MAX_L would + // allow is one the token attention kernel cannot serve, and the + // resulting failure arrives mid-step and costs the caller the whole + // conversation. Stopping one step earlier turns that into an + // ordinary MAX_LENGTH_REACHED truncation. + this->total_tokens < this->aie4_active_cap()) { + if (is_cancelled()) { + reason = CANCEL_DETECTED; + buffer_.clear(); + current_mode_ = StreamEventType::CONTENT; + tool_name_.clear(); + is_in_tool_block_ = false; + break; + } + + const int committed_token = this->last_token; + this->profiler_list[DECODING_TIME].start(); + buffer logits = + this->lm_engine->forward(committed_token); + this->profiler_list[DECODING_TIME].stop(1); + + this->token_history.push_back(committed_token); + ++this->total_tokens; + ++meta_info.generated_tokens; + ++generated_this_call; + + this->profiler_list[TKOEN_DECODE_TIME].start(); + if (this->is_normal_token(committed_token)) { + const std::string token_str = + this->tokenizer->run_time_decoder(committed_token); + os << token_str << std::flush; + result += token_str; + } + this->profiler_list[TKOEN_DECODE_TIME].stop(1); + + if (this->is_eos(committed_token)) { + this->last_token = -1; + break; + } + if ( + length_limit > 0 && + generated_this_call >= length_limit) { + this->last_token = -1; + reason = MAX_LENGTH_REACHED; + break; + } + if (this->total_tokens >= this->aie4_active_cap()) { + this->last_token = -1; + reason = MAX_LENGTH_REACHED; + break; + } + + this->profiler_list[SAMPLING_TIME].start(); + this->last_token = this->sampler->sample(logits); + this->profiler_list[SAMPLING_TIME].stop(1); + } + + if (this->total_tokens >= this->aie4_active_cap()) { + this->last_token = -1; + reason = MAX_LENGTH_REACHED; + header_print( + "WARNING", + "Max length reached, stopping generation..."); + } + meta_info.decoding_duration = + static_cast( + time_utils::cast_to_us( + this->profiler_list[DECODING_TIME] + .get_total_time()) + .first) * + 1e3; + meta_info.stop_reason = reason; + std::cout << std::endl; + header_print("FLM", "Model RAW Output: \n" + result); + return result; +} +#endif + bool Phi4::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std::function is_cancelled) { - // preprocess this->profiler_list[TKOEN_ENCODE_TIME].start(); std::string templated_text; if (input.messages.empty() && input.prompt.empty()) { @@ -107,19 +592,259 @@ bool Phi4::insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std::f std::vector tokens = this->tokenizer->encode(templated_text); this->profiler_list[TKOEN_ENCODE_TIME].stop(tokens.size()); - // hardware - return this->_shared_insert(meta_info, tokens, is_cancelled); +#if defined(FLM_ENABLE_CORELIB_AIE4) + if (this->uses_corelib_aie4_) { + this->validate_aie4_capacity( + tokens.size(), + input.requested_max_new_tokens); + meta_info.stop_reason = EOT_DETECTED; + + const size_t history_size = this->token_history.size(); + const size_t matched = this->_matching_prefix_length(tokens); + const bool prefix_hit = + history_size != 0 && matched == history_size; + const size_t suffix_tokens = + prefix_hit ? tokens.size() - matched : tokens.size(); + const flm::phi4::ContinuationRoute route = + prefix_hit + ? flm::phi4::SelectContinuationRoute( + suffix_tokens, + this->forced_continuation_route_) + : flm::phi4::ContinuationRoute::Reprefill; + const PrefixHitAction action = + route == flm::phi4::ContinuationRoute::Append + ? PrefixHitAction::AppendSuffixOneByOne + : PrefixHitAction::RecomputeFull; + const auto started = std::chrono::steady_clock::now(); + + bool inserted = false; + // "at a recoverable point", NOT "before submission". + // + // These strings said "before submission" for as long as the engine's + // irrevocable boundary was per STEP: reaching a catch here meant + // nothing had been submitted, so the wording was true by + // construction. The boundary is now per SUBMISSION GROUP -- see the + // note in checked_synchronize in phi4_corelib_aie4.cpp -- and a + // failure past a COMPLETED synchronize is recoverable too. A + // bad_alloc in the V scatter of layer 0 arrives here after three + // submits and one synchronize, so "before submission" is now false + // for exactly the cases the boundary change added. + // + // What is still true, and is all the caller can act on, is that the + // engine classified the failure as recoverable and the session was + // discarded. The irrevocable classes never reach this handler: they + // terminate the process. + try { + inserted = this->_shared_insert( + meta_info, + tokens, + std::move(is_cancelled), + nullptr, + 0, + action); + } catch (const ModelRequestError&) { + throw; + } catch (const flm::corelib::CorelibError&) { + this->clear_after_corelib_error(); + throw ModelRequestError( + 500, + true, + "AIE4 inference failed at a recoverable point; the " + "current conversation was cleared."); + } catch (const std::exception&) { + this->clear_after_corelib_error(); + throw ModelRequestError( + 500, + true, + "AIE4 inference failed at a recoverable point; the " + "current conversation was cleared."); + } catch (...) { + this->clear_after_corelib_error(); + throw ModelRequestError( + 500, + true, + "AIE4 inference failed at a recoverable point; the " + "current conversation was cleared."); + } + + const auto elapsed = + std::chrono::duration_cast( + std::chrono::steady_clock::now() - started) + .count(); + this->last_continuation_ns_ = + static_cast( + std::max(elapsed, 0)); + this->last_continuation_route_ = route; + if (route == flm::phi4::ContinuationRoute::Append) { + this->append_continuation_ns_ += + this->last_continuation_ns_; + } else { + this->reprefill_continuation_ns_ += + this->last_continuation_ns_; + } + return inserted; + } +#endif + + return this->_shared_insert( + meta_info, + tokens, + std::move(is_cancelled)); } std::string Phi4::generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled) { - return this->_shared_generate(meta_info, length_limit, os, is_cancelled); +#if defined(FLM_ENABLE_CORELIB_AIE4) + if (this->uses_corelib_aie4_) { + try { + return this->generate_aie4( + meta_info, + length_limit, + os, + std::move(is_cancelled)); + } catch (const ModelRequestError&) { + throw; + } catch (const flm::corelib::CorelibError&) { + this->clear_after_corelib_error(); + throw ModelRequestError( + 500, + true, + "AIE4 inference failed at a recoverable point; the " + "current conversation was cleared."); + } catch (const std::exception&) { + this->clear_after_corelib_error(); + throw ModelRequestError( + 500, + true, + "AIE4 inference failed at a recoverable point; the " + "current conversation was cleared."); + } catch (...) { + this->clear_after_corelib_error(); + throw ModelRequestError( + 500, + true, + "AIE4 inference failed at a recoverable point; the " + "current conversation was cleared."); + } + } +#endif + return this->_shared_generate( + meta_info, + length_limit, + os, + std::move(is_cancelled)); } std::string Phi4::generate_with_prompt(chat_meta_info_t& meta_info, lm_uniform_input_t& input, int length_limit, std::ostream& os) { if (!this->insert(meta_info, input)) { return ""; } - return this->_shared_generate(meta_info, length_limit, os); + return this->generate(meta_info, length_limit, os); +} + +void Phi4::set_max_length(unsigned int requested_max_length) { +#if defined(FLM_ENABLE_CORELIB_AIE4) + if (this->uses_corelib_aie4_) { + if ( + requested_max_length == 0 || + requested_max_length > 4096) { + throw std::out_of_range( + "Phi-4 AIE4 maximum length must be in 1..4096"); + } + const int engine_position = + this->lm_engine->get_current_context_length(); + if ( + requested_max_length < + static_cast(engine_position)) { + throw std::out_of_range( + "Phi-4 AIE4 maximum length cannot be below the " + "current engine position"); + } + + this->lm_engine->update_max_length(requested_max_length); + this->MAX_L = requested_max_length; + return; + } +#endif + AutoModel::set_max_length(requested_max_length); +} + +#if defined(FLM_ENABLE_CORELIB_AIE4) +const flm::phi4::Phi4Aie4Metrics& +Phi4::aie4_metrics() const { +#if defined(FLM_CORELIB_TESTING) + if (this->metrics_for_testing_.has_value()) { + return *this->metrics_for_testing_; + } +#endif + const auto* engine = + dynamic_cast( + this->lm_engine.get()); + if (engine == nullptr) { + throw std::logic_error( + "Phi-4 AIE4 profile requested for a non-corelib engine"); + } + return engine->metrics(); +} +#endif + +std::string Phi4::show_profile() { + const std::string base = AutoModel::show_profile(); +#if defined(FLM_ENABLE_CORELIB_AIE4) + if (this->uses_corelib_aie4_) { + const auto& metrics = this->aie4_metrics(); + std::stringstream profile; + profile << base; + profile << " Phi-4 AIE4:" << std::endl; + profile << " Engine: corelib_aie4" << std::endl; + profile << " Continuation route: " + << (this->last_continuation_route_.has_value() + ? flm::phi4::ContinuationRouteName( + *this->last_continuation_route_) + : "none") + << std::endl; + profile << " Append threshold: " + << flm::phi4::kContinuationAppendThreshold + << std::endl; + profile << " Corelib DLL: " + << this->corelib_runtime_->api()->library_path().string() + << std::endl; + profile << " Helper transitions: " + << metrics.helper_transition_counts[0] << "/" + << metrics.helper_transition_counts[1] << "/" + << metrics.helper_transition_counts[2] << "/" + << metrics.helper_transition_counts[3] << "/" + << metrics.helper_transition_counts[4] << "/" + << metrics.helper_transition_counts[5] + << std::endl; + profile << " Cold model load: " + << metrics.model_load_ns << " ns" << std::endl; + profile << " Cold weight pack: " + << metrics.weight_pack_ns << " ns" << std::endl; + profile << " Continuation time: " + << this->last_continuation_ns_ << " ns" << std::endl; + profile << " Warm append total: " + << this->append_continuation_ns_ << " ns" << std::endl; + profile << " Warm reprefill total: " + << this->reprefill_continuation_ns_ << " ns" + << std::endl; + profile << " Dispatches: " + << metrics.dispatch_count << std::endl; + profile << " Synchronizations: " + << metrics.synchronize_count << std::endl; + profile << " Packed weights: " + << metrics.packed_weight_bytes << " bytes" + << std::endl; + profile << " Mapped source: " + << metrics.mapped_source_bytes << " bytes" + << std::endl; + profile << " KV storage: " + << metrics.kv_bytes << " bytes" << std::endl; + profile << " Scratch storage: " + << metrics.scratch_bytes << " bytes" << std::endl; + return profile.str(); + } +#endif + return base; } \ No newline at end of file diff --git a/src/common/corelib/corelib_api.cpp b/src/common/corelib/corelib_api.cpp new file mode 100644 index 00000000..fe2063ee --- /dev/null +++ b/src/common/corelib/corelib_api.cpp @@ -0,0 +1,460 @@ +#include + +#include + +#include +#include +#include +#include +#include +#include + +namespace flm::corelib { +namespace { + +constexpr wchar_t kCorelibPathEnvironment[] = L"RYZENAI_CORELIB_PATH"; +constexpr wchar_t kCorelibFilename[] = L"ryzenai_corelib.dll"; + +std::string FormatCorelibError( + std::string_view call, + std::string_view status_text, + std::string_view detail) { + std::string message(call); + message += " failed: "; + message += status_text; + if (!detail.empty()) { + message += ": "; + message += detail; + } + return message; +} + +std::optional ReadWideEnvironment(const wchar_t* name) { + SetLastError(ERROR_SUCCESS); + const DWORD required = GetEnvironmentVariableW(name, nullptr, 0); + if (required == 0) { + const DWORD error = GetLastError(); + if (error == ERROR_SUCCESS || error == ERROR_ENVVAR_NOT_FOUND) { + return std::nullopt; + } + throw std::system_error( + static_cast(error), + std::system_category(), + "GetEnvironmentVariableW failed"); + } + + std::wstring value(required, L'\0'); + const DWORD written = + GetEnvironmentVariableW(name, value.data(), required); + if (written == 0 || written >= required) { + throw std::system_error( + static_cast(GetLastError()), + std::system_category(), + "GetEnvironmentVariableW failed"); + } + value.resize(written); + return value; +} + +std::filesystem::path MakeAbsolute(std::filesystem::path path) { + if (!path.is_absolute()) { + path = std::filesystem::absolute(path); + } + return path.lexically_normal(); +} + +template +Function ResolveRequired( + const CorelibApi::Resolver& resolver, + std::string_view name) { + void* address = resolver(name); + if (address == nullptr) { + throw std::runtime_error( + "missing required ryzenai-corelib symbol: " + + std::string(name)); + } + return reinterpret_cast(address); +} + +// Resolves and calls the version entry point BEFORE any other symbol is +// looked up. A runtime built from a different corelib revision renames and +// removes entry points, so resolving the rest first would report a missing +// symbol instead of the version skew that actually caused it. +CorelibVersion GateVersion( + const CorelibApi::Resolver& resolver, + decltype(&::ryzenai_corelib_get_version)& out_get_version) { + out_get_version = + ResolveRequired( + resolver, + "ryzenai_corelib_get_version"); + + CorelibVersion runtime{}; + out_get_version(&runtime.major, &runtime.minor, &runtime.patch); + + const auto compiled = CompiledCorelibVersion(); + if (!IsCorelibVersionCompatible(compiled, runtime)) { + throw std::runtime_error( + FormatCorelibVersionMismatch(compiled, runtime)); + } + return runtime; +} + +CorelibFunctions ResolveFunctions( + const CorelibApi::Resolver& resolver, + CorelibVersion& out_runtime_version) { + decltype(&::ryzenai_corelib_get_version) get_version = nullptr; + out_runtime_version = GateVersion(resolver, get_version); + + return CorelibFunctions{ + get_version, + ResolveRequired< + decltype(&::ryzenai_corelib_status_to_string)>( + resolver, + "ryzenai_corelib_status_to_string"), + ResolveRequired< + decltype(&::ryzenai_corelib_get_last_error_message)>( + resolver, + "ryzenai_corelib_get_last_error_message"), + ResolveRequired< + decltype(&::ryzenai_corelib_selftest_dependencies)>( + resolver, + "ryzenai_corelib_selftest_dependencies"), + ResolveRequired< + decltype(&::ryzenai_corelib_has_device_context)>( + resolver, + "ryzenai_corelib_has_device_context"), + ResolveRequired< + decltype(&::ryzenai_corelib_object_release)>( + resolver, + "ryzenai_corelib_object_release"), + ResolveRequired< + decltype(&::ryzenai_corelib_create_stream)>( + resolver, + "ryzenai_corelib_create_stream"), + ResolveRequired< + decltype(&::ryzenai_corelib_stream_synchronize)>( + resolver, + "ryzenai_corelib_stream_synchronize"), + ResolveRequired< + decltype(&::ryzenai_corelib_create_device_tensor)>( + resolver, + "ryzenai_corelib_create_device_tensor"), + ResolveRequired< + decltype(&::ryzenai_corelib_tensor_write)>( + resolver, + "ryzenai_corelib_tensor_write"), + ResolveRequired< + decltype(&::ryzenai_corelib_tensor_read)>( + resolver, + "ryzenai_corelib_tensor_read"), + ResolveRequired< + decltype(&::ryzenai_corelib_tensor_get_byte_size)>( + resolver, + "ryzenai_corelib_tensor_get_byte_size"), + ResolveRequired< + decltype(&::ryzenai_corelib_tensor_get_data_type)>( + resolver, + "ryzenai_corelib_tensor_get_data_type"), + ResolveRequired< + decltype(&::ryzenai_corelib_matmul_bf16_pad_shape)>( + resolver, + "ryzenai_corelib_matmul_bf16_pad_shape"), + ResolveRequired( + resolver, + "ryzenai_corelib_matmul_bf16_weights_create_onnx"), + ResolveRequired( + resolver, + "ryzenai_corelib_matmul_bf16_weights_get_data"), + ResolveRequired< + decltype(&::ryzenai_corelib_matmul_bf16)>( + resolver, + "ryzenai_corelib_matmul_bf16"), + ResolveRequired< + decltype(&::ryzenai_corelib_ssmlp_bf16_pad_rows)>( + resolver, + "ryzenai_corelib_ssmlp_bf16_pad_rows"), + ResolveRequired( + resolver, + "ryzenai_corelib_ssmlp_bf16_weights_create_onnx"), + ResolveRequired( + resolver, + "ryzenai_corelib_ssmlp_bf16_weights_get_data"), + ResolveRequired< + decltype(&::ryzenai_corelib_ssmlp_bf16)>( + resolver, + "ryzenai_corelib_ssmlp_bf16"), + ResolveRequired< + decltype(&::ryzenai_corelib_flat_mha_bf16_pad_rows)>( + resolver, + "ryzenai_corelib_flat_mha_bf16_pad_rows"), + ResolveRequired< + decltype(&::ryzenai_corelib_flat_mha_bf16)>( + resolver, + "ryzenai_corelib_flat_mha_bf16"), + ResolveRequired< + decltype(&::ryzenai_corelib_cleanup)>( + resolver, + "ryzenai_corelib_cleanup"), + }; +} + +std::string LoadFailureMessage( + const std::filesystem::path& path, + DWORD error) { + return "LoadLibraryExW failed for " + path.string() + + " (Win32 error " + std::to_string(error) + ")"; +} + +} // namespace + +bool IsCorelibVersionCompatible( + const CorelibVersion& compiled, + const CorelibVersion& runtime) noexcept { + if (compiled.major == 0) { + // Pre-1.0 corelib may change the API in any release, including a + // patch one, so every component is part of the contract. + return compiled.major == runtime.major && + compiled.minor == runtime.minor && + compiled.patch == runtime.patch; + } + return compiled.major == runtime.major && + runtime.minor >= compiled.minor; +} + +std::string FormatCorelibVersion(const CorelibVersion& value) { + return std::to_string(value.major) + '.' + + std::to_string(value.minor) + '.' + + std::to_string(value.patch); +} + +std::string FormatCorelibVersionMismatch( + const CorelibVersion& compiled, + const CorelibVersion& runtime) { + return "ryzenai-corelib version mismatch: the loaded runtime reports " + "version " + + FormatCorelibVersion(runtime) + + " but FastFlowLM was compiled against version " + + FormatCorelibVersion(compiled) + + "; install the matching ryzenai_corelib.dll"; +} + +CorelibVersion CompiledCorelibVersion() noexcept { + return CorelibVersion{ + static_cast(RYZENAI_CORELIB_VERSION_MAJOR), + static_cast(RYZENAI_CORELIB_VERSION_MINOR), + static_cast(RYZENAI_CORELIB_VERSION_PATCH)}; +} + +CorelibError::CorelibError( + ryzenai_corelib_status status_value, + std::string call_value, + std::string detail_value, + std::string status_text) + : std::runtime_error(FormatCorelibError( + call_value, + status_text, + detail_value)), + status(status_value), + call(std::move(call_value)), + detail(std::move(detail_value)), + status_text_(std::move(status_text)) {} + +CorelibError CorelibError::WithContext( + std::string_view context) const { + std::string enriched_detail(context); + if (!enriched_detail.empty() && !detail.empty()) { + enriched_detail += ": "; + } + enriched_detail += detail; + return CorelibError{ + status, + call, + std::move(enriched_detail), + status_text_}; +} + +std::shared_ptr CorelibApi::Load( + const std::filesystem::path& absolute_path) { + if (!absolute_path.is_absolute()) { + throw std::invalid_argument( + "CorelibApi::Load requires an absolute DLL path"); + } + + HMODULE module = LoadLibraryExW( + absolute_path.c_str(), + nullptr, + LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR | LOAD_LIBRARY_SEARCH_DEFAULT_DIRS); + if (module == nullptr) { + const DWORD error = GetLastError(); + throw std::runtime_error( + LoadFailureMessage(absolute_path, error)); + } + + try { + Resolver resolver = [module](std::string_view name) -> void* { + const std::string symbol(name); + return reinterpret_cast( + GetProcAddress(module, symbol.c_str())); + }; + CorelibVersion runtime_version{}; + auto functions = ResolveFunctions(resolver, runtime_version); + return std::shared_ptr( + new CorelibApi( + module, + absolute_path.lexically_normal(), + std::move(functions), + runtime_version)); + } catch (...) { + FreeLibrary(module); + throw; + } +} + +std::shared_ptr CorelibApi::ResolveForTest( + Resolver resolver) { + CorelibVersion runtime_version{}; + auto functions = ResolveFunctions(resolver, runtime_version); + return std::shared_ptr( + new CorelibApi( + nullptr, + {}, + std::move(functions), + runtime_version)); +} + +std::filesystem::path CorelibApi::ResolveLibraryPath( + const std::filesystem::path& executable_dir) { + if (const auto configured = + ReadWideEnvironment(kCorelibPathEnvironment)) { + std::filesystem::path path(*configured); + if (!path.is_absolute()) { + throw std::invalid_argument( + "RYZENAI_CORELIB_PATH must be an absolute file or " + "directory path"); + } + path = path.lexically_normal(); + std::error_code error; + if (std::filesystem::is_directory(path, error)) { + path /= kCorelibFilename; + } + return path.lexically_normal(); + } + + return MakeAbsolute( + executable_dir / "aie4" / kCorelibFilename); +} + +CorelibApi::CorelibApi( + void* module, + std::filesystem::path library_path, + CorelibFunctions functions, + CorelibVersion runtime_version) + : module_(module), + library_path_(std::move(library_path)), + functions_(std::move(functions)), + runtime_version_(runtime_version) {} + +CorelibApi::~CorelibApi() { + if (module_ != nullptr) { + FreeLibrary(static_cast(module_)); + } +} + +const CorelibFunctions& CorelibApi::functions() const noexcept { + return functions_; +} + +const CorelibVersion& CorelibApi::runtime_version() const noexcept { + return runtime_version_; +} + +void CorelibApi::WriteElements( + ryzenai_corelib_tensor_ptr tensor, + ryzenai_corelib_data_type source_type, + const void* source, + std::size_t count, + std::size_t offset) const { + Check( + functions_.tensor_write( + tensor, + source_type, + source, + count, + offset), + "ryzenai_corelib_tensor_write"); +} + +void CorelibApi::ReadElements( + ryzenai_corelib_tensor_ptr tensor, + ryzenai_corelib_data_type destination_type, + void* destination, + std::size_t count, + std::size_t offset) const { + Check( + functions_.tensor_read( + tensor, + destination_type, + destination, + count, + offset), + "ryzenai_corelib_tensor_read"); +} + +void CorelibApi::Check( + ryzenai_corelib_status status, + std::string_view call) const { + if (status == ryzenai_corelib_status_success) { + return; + } + const char* raw = functions_.get_last_error_message(); + std::string detail = raw == nullptr ? std::string() : std::string(raw); + const char* status_text = functions_.status_to_string(status); + throw CorelibError{ + status, + std::string(call), + std::move(detail), + status_text == nullptr ? "corelib failure" : status_text}; +} + +void CorelibApi::RegisterObject(CorelibObjectKind kind) const noexcept { + live_object_count_.fetch_add(1, std::memory_order_relaxed); + creation_counts_[static_cast(kind)].fetch_add( + 1, + std::memory_order_relaxed); +} + +std::uint64_t CorelibApi::creation_count( + CorelibObjectKind kind) const noexcept { + return creation_counts_[static_cast(kind)].load( + std::memory_order_relaxed); +} + +std::uint64_t CorelibApi::weight_creation_count() const noexcept { + return creation_count(CorelibObjectKind::MatMulWeights) + + creation_count(CorelibObjectKind::SsMlpWeights); +} + +void CorelibApi::Release(void* value) const noexcept { + if (value == nullptr) { + return; + } + + functions_.object_release(value); + const auto previous = + live_object_count_.fetch_sub(1, std::memory_order_acq_rel); + assert(previous > 0); +} + +std::size_t CorelibApi::live_object_count() const noexcept { + return live_object_count_.load(std::memory_order_acquire); +} + +const std::filesystem::path& CorelibApi::library_path() const noexcept { + return library_path_; +} + +} // namespace flm::corelib diff --git a/src/common/corelib/corelib_fatal_record.cpp b/src/common/corelib/corelib_fatal_record.cpp new file mode 100644 index 00000000..ddfda968 --- /dev/null +++ b/src/common/corelib/corelib_fatal_record.cpp @@ -0,0 +1,726 @@ +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace flm::corelib { +namespace { + +using HundredNanoseconds = + std::chrono::duration>; + +constexpr uint64_t kWindowsToUnixEpochTicks = 116'444'736'000'000'000ULL; +constexpr std::string_view kPendingPrefix = "pending-corelib-fatal-"; +constexpr std::string_view kPendingSuffix = ".tmp"; +constexpr std::string_view kFinalPrefix = "corelib-fatal-"; +constexpr std::string_view kFinalSuffix = ".json"; + +std::runtime_error FatalRecordError( + std::string_view action, + const std::filesystem::path& path, + unsigned long error) { + return std::runtime_error( + "AIE4 fatal record " + std::string(action) + " failed for " + + path.string() + " (error " + std::to_string(error) + ")"); +} + +std::string FormatUtc( + std::chrono::system_clock::time_point value) { + const auto ticks = + std::chrono::duration_cast( + value.time_since_epoch()) + .count(); + constexpr int64_t ticks_per_second = 10'000'000; + int64_t seconds = ticks / ticks_per_second; + int64_t fraction = ticks % ticks_per_second; + if (fraction < 0) { + fraction += ticks_per_second; + --seconds; + } + + const std::time_t calendar_seconds = + static_cast(seconds); + std::tm utc{}; + if (gmtime_s(&utc, &calendar_seconds) != 0) { + throw std::runtime_error( + "AIE4 fatal record timestamp conversion failed"); + } + + std::ostringstream output; + output << std::put_time(&utc, "%Y%m%dT%H%M%S") + << std::setfill('0') << std::setw(7) << fraction << 'Z'; + return output.str(); +} + +std::optional ParseUtc( + std::string_view value) { + if (value.size() != 23 || value[8] != 'T' || value[22] != 'Z') { + return std::nullopt; + } + + const auto parse_part = + [value](std::size_t offset, std::size_t length) + -> std::optional { + int result = 0; + const char* first = value.data() + offset; + const char* last = first + length; + const auto parsed = std::from_chars(first, last, result); + if (parsed.ec != std::errc{} || parsed.ptr != last) { + return std::nullopt; + } + return result; + }; + + const auto year = parse_part(0, 4); + const auto month = parse_part(4, 2); + const auto day = parse_part(6, 2); + const auto hour = parse_part(9, 2); + const auto minute = parse_part(11, 2); + const auto second = parse_part(13, 2); + const auto fraction = parse_part(15, 7); + if (!year || !month || !day || !hour || !minute || !second || + !fraction) { + return std::nullopt; + } + + std::tm utc{}; + utc.tm_year = *year - 1900; + utc.tm_mon = *month - 1; + utc.tm_mday = *day; + utc.tm_hour = *hour; + utc.tm_min = *minute; + utc.tm_sec = *second; + const __time64_t seconds = _mkgmtime64(&utc); + if (seconds == -1) { + return std::nullopt; + } + + const HundredNanoseconds duration{ + static_cast(seconds) * 10'000'000 + *fraction}; + const auto result = std::chrono::system_clock::time_point{ + std::chrono::duration_cast( + duration)}; + if (FormatUtc(result) != value) { + return std::nullopt; + } + return result; +} + +std::chrono::system_clock::time_point FileTimeToSystemClock( + const FILETIME& value) { + ULARGE_INTEGER ticks{}; + ticks.LowPart = value.dwLowDateTime; + ticks.HighPart = value.dwHighDateTime; + if (ticks.QuadPart < kWindowsToUnixEpochTicks) { + throw std::runtime_error( + "AIE4 fatal record process start time predates Unix epoch"); + } + const auto unix_ticks = ticks.QuadPart - kWindowsToUnixEpochTicks; + if (unix_ticks > + static_cast(std::numeric_limits::max())) { + throw std::runtime_error( + "AIE4 fatal record process start time is out of range"); + } + return std::chrono::system_clock::time_point{ + std::chrono::duration_cast( + HundredNanoseconds{static_cast(unix_ticks)})}; +} + +std::chrono::system_clock::time_point QueryProcessStart(HANDLE process) { + FILETIME creation{}; + FILETIME exit{}; + FILETIME kernel{}; + FILETIME user{}; + if (!GetProcessTimes( + process, + &creation, + &exit, + &kernel, + &user)) { + throw FatalRecordError( + "process-time query", + {}, + GetLastError()); + } + return FileTimeToSystemClock(creation); +} + +std::optional ProbeProcessStart( + DWORD pid) { + HANDLE process = OpenProcess( + PROCESS_QUERY_LIMITED_INFORMATION, + FALSE, + pid); + if (process == nullptr) { + if (GetLastError() == ERROR_INVALID_PARAMETER) { + return std::chrono::system_clock::time_point{}; + } + return std::nullopt; + } + + try { + const auto start = QueryProcessStart(process); + CloseHandle(process); + return start; + } catch (...) { + CloseHandle(process); + return std::nullopt; + } +} + +std::filesystem::path LocalAppDataLogRoot() { + PWSTR raw = nullptr; + const HRESULT result = SHGetKnownFolderPath( + FOLDERID_LocalAppData, + KF_FLAG_DEFAULT, + nullptr, + &raw); + if (FAILED(result)) { + throw std::runtime_error( + "AIE4 fatal record LocalAppData resolution failed (HRESULT " + + std::to_string(static_cast(result)) + ")"); + } + + std::filesystem::path root(raw); + CoTaskMemFree(raw); + return root / "FastFlowLM" / "logs"; +} + +std::string EscapeJson(std::string_view value) { + std::ostringstream output; + output << std::hex << std::uppercase; + for (const unsigned char character : value) { + switch (character) { + case '"': + output << "\\\""; + break; + case '\\': + output << "\\\\"; + break; + case '\b': + output << "\\b"; + break; + case '\f': + output << "\\f"; + break; + case '\n': + output << "\\n"; + break; + case '\r': + output << "\\r"; + break; + case '\t': + output << "\\t"; + break; + default: + if (character < 0x20) { + output << "\\u" + << std::setfill('0') << std::setw(4) + << static_cast(character); + } else { + output << static_cast(character); + } + break; + } + } + return output.str(); +} + +std::string SerializeFailure( + const FailureContext& failure, + std::string_view process_start, + std::string_view failure_utc, + DWORD pid) { + std::ostringstream output; + output << '{' + << "\"status\":" << static_cast(failure.status) << ',' + << "\"call\":\"" << EscapeJson(failure.call) << "\"," + << "\"detail\":\"" << EscapeJson(failure.detail) << "\"," + << "\"phase\":\"" << EscapeJson(failure.phase) << "\"," + << "\"layer\":"; + if (failure.layer.has_value()) { + output << *failure.layer; + } else { + output << "null"; + } + output << ',' + << "\"rows\":" << failure.rows << ',' + << "\"position\":" << failure.position << ',' + << "\"process_start_utc\":\"" << process_start << "\"," + << "\"failure_utc\":\"" << failure_utc << "\"," + << "\"pid\":" << pid + << "}\n"; + return output.str(); +} + +void WriteAll(HANDLE file, std::string_view contents) { + std::size_t offset = 0; + while (offset < contents.size()) { + const auto remaining = contents.size() - offset; + const DWORD chunk = static_cast( + (std::min)( + remaining, + static_cast( + std::numeric_limits::max()))); + DWORD written = 0; + if (!WriteFile( + file, + contents.data() + offset, + chunk, + &written, + nullptr) || + written != chunk) { + throw FatalRecordError( + "write", + {}, + GetLastError()); + } + offset += written; + } +} + +void RewindAndTruncate(HANDLE file) { + LARGE_INTEGER beginning{}; + if (!SetFilePointerEx(file, beginning, nullptr, FILE_BEGIN) || + !SetEndOfFile(file)) { + throw FatalRecordError( + "truncate", + {}, + GetLastError()); + } +} + +void EmitRecord(std::ostream& output, std::string_view record) { + output.write( + record.data(), + static_cast(record.size())); + output.flush(); + if (!output) { + throw std::runtime_error( + "AIE4 fatal record startup reporting failed"); + } +} + +bool HasPrefixAndSuffix( + std::string_view value, + std::string_view prefix, + std::string_view suffix) { + return value.starts_with(prefix) && value.ends_with(suffix) && + value.size() > prefix.size() + suffix.size(); +} + +struct PendingIdentity { + std::string timestamp; + DWORD pid; + std::chrono::system_clock::time_point start; +}; + +std::optional ParsePendingIdentity( + const std::filesystem::path& path) { + const std::string filename = path.filename().string(); + if (!HasPrefixAndSuffix( + filename, + kPendingPrefix, + kPendingSuffix)) { + return std::nullopt; + } + + const std::string_view body(filename.data() + kPendingPrefix.size(), + filename.size() - kPendingPrefix.size() - + kPendingSuffix.size()); + const auto separator = body.rfind('-'); + if (separator == std::string_view::npos) { + return std::nullopt; + } + const std::string_view timestamp = body.substr(0, separator); + const std::string_view pid_text = body.substr(separator + 1); + unsigned long pid_value = 0; + const auto parsed = std::from_chars( + pid_text.data(), + pid_text.data() + pid_text.size(), + pid_value); + if (parsed.ec != std::errc{} || + parsed.ptr != pid_text.data() + pid_text.size() || + pid_value > std::numeric_limits::max()) { + return std::nullopt; + } + const auto start = ParseUtc(timestamp); + if (!start.has_value()) { + return std::nullopt; + } + return PendingIdentity{ + std::string(timestamp), + static_cast(pid_value), + *start}; +} + +std::string ReadRecord(const std::filesystem::path& path) { + std::ifstream input(path, std::ios::binary); + if (!input) { + throw FatalRecordError("read", path, GetLastError()); + } + std::string contents{ + std::istreambuf_iterator(input), + std::istreambuf_iterator()}; + if (input.bad()) { + throw FatalRecordError("read", path, GetLastError()); + } + return contents; +} + +void EmitDrainWarning( + std::ostream& output, + std::string_view operation, + const std::filesystem::path& path, + std::string_view detail) noexcept { + try { + output << "AIE4 fatal record warning: failed to " << operation + << ' ' << path.string(); + if (!detail.empty()) { + output << ": " << detail; + } + output << '\n'; + } catch (...) { + } +} + +void RemoveReportedRecord( + const std::filesystem::path& path, + std::ostream& output) noexcept { + try { + std::error_code error; + const bool removed = std::filesystem::remove(path, error); + if (!error && removed) { + return; + } + EmitDrainWarning( + output, + "remove", + path, + error ? error.message() : "record was not removed"); + } catch (const std::exception& exception) { + EmitDrainWarning(output, "remove", path, exception.what()); + } catch (...) { + EmitDrainWarning(output, "remove", path, "unknown error"); + } +} + +} // namespace + +FatalRecordStore::FatalRecordStore( + std::filesystem::path root, + DWORD pid, + std::chrono::system_clock::time_point process_start, + ProcessProbe process_probe) + : root_(std::move(root)), + pid_(pid), + process_start_(process_start), + process_probe_(std::move(process_probe)) {} + +FatalRecordStore::FatalRecordStore( + FatalRecordStore&& other) noexcept + : root_(std::move(other.root_)), + pid_(other.pid_), + process_start_(other.process_start_), + process_probe_(std::move(other.process_probe_)), + pending_path_(std::move(other.pending_path_)), + pending_handle_( + std::exchange(other.pending_handle_, INVALID_HANDLE_VALUE)), + persisted_(other.persisted_) { + other.pending_path_.clear(); + other.persisted_ = true; +} + +FatalRecordStore& FatalRecordStore::operator=( + FatalRecordStore&& other) noexcept { + if (this != &other) { + RemoveUnusedPending(); + root_ = std::move(other.root_); + pid_ = other.pid_; + process_start_ = other.process_start_; + process_probe_ = std::move(other.process_probe_); + pending_path_ = std::move(other.pending_path_); + pending_handle_ = + std::exchange(other.pending_handle_, INVALID_HANDLE_VALUE); + persisted_ = other.persisted_; + other.pending_path_.clear(); + other.persisted_ = true; + } + return *this; +} + +FatalRecordStore::~FatalRecordStore() noexcept { + RemoveUnusedPending(); +} + +FatalRecordStore FatalRecordStore::ForCurrentProcess() { + const DWORD pid = GetCurrentProcessId(); + return FatalRecordStore( + LocalAppDataLogRoot(), + pid, + QueryProcessStart(GetCurrentProcess()), + ProbeProcessStart); +} + +void FatalRecordStore::Prepare() { + if (pending_handle_ != INVALID_HANDLE_VALUE || persisted_) { + return; + } + + std::error_code error; + std::filesystem::create_directories(root_, error); + if (error || !std::filesystem::is_directory(root_, error) || error) { + throw std::runtime_error( + "AIE4 fatal record directory preparation failed for " + + root_.string() + + (error ? ": " + error.message() : "")); + } + + pending_path_ = + root_ / + (std::string(kPendingPrefix) + FormatUtc(process_start_) + "-" + + std::to_string(pid_) + std::string(kPendingSuffix)); + pending_handle_ = CreateFileW( + pending_path_.c_str(), + GENERIC_WRITE, + FILE_SHARE_READ, + nullptr, + CREATE_NEW, + FILE_ATTRIBUTE_NORMAL | FILE_FLAG_WRITE_THROUGH, + nullptr); + if (pending_handle_ == INVALID_HANDLE_VALUE) { + throw FatalRecordError( + "initialization", + pending_path_, + GetLastError()); + } + + try { + WriteAll(pending_handle_, "P"); + if (!FlushFileBuffers(pending_handle_)) { + throw FatalRecordError( + "initial flush", + pending_path_, + GetLastError()); + } + RewindAndTruncate(pending_handle_); + if (!FlushFileBuffers(pending_handle_)) { + throw FatalRecordError( + "initial flush", + pending_path_, + GetLastError()); + } + } catch (...) { + ClosePending(); + DeleteFileW(pending_path_.c_str()); + throw; + } +} + +std::filesystem::path FatalRecordStore::Persist( + const FailureContext& failure) { + if (pending_handle_ == INVALID_HANDLE_VALUE || persisted_) { + throw std::logic_error( + "AIE4 fatal record store was not prepared"); + } + + const auto failure_time = std::chrono::system_clock::now(); + const std::string failure_utc = FormatUtc(failure_time); + const std::string record = SerializeFailure( + failure, + FormatUtc(process_start_), + failure_utc, + pid_); + RewindAndTruncate(pending_handle_); + WriteAll(pending_handle_, record); + if (!FlushFileBuffers(pending_handle_)) { + throw FatalRecordError( + "flush", + pending_path_, + GetLastError()); + } + ClosePending(); + + const auto final_path = + root_ / + (std::string(kFinalPrefix) + failure_utc + "-" + + std::to_string(pid_) + std::string(kFinalSuffix)); + if (!MoveFileExW( + pending_path_.c_str(), + final_path.c_str(), + MOVEFILE_WRITE_THROUGH)) { + throw FatalRecordError( + "atomic rename", + final_path, + GetLastError()); + } + persisted_ = true; + return final_path; +} + +std::vector FatalRecordStore::DrainPriorRecords( + std::ostream& output) { + return DrainPriorRecords( + LocalAppDataLogRoot(), + ProbeProcessStart, + output); +} + +std::vector FatalRecordStore::DrainPriorRecords( + const std::filesystem::path& root, + ProcessProbe process_probe, + std::ostream& output) { + std::error_code error; + if (!std::filesystem::exists(root, error)) { + if (error) { + EmitDrainWarning( + output, + "inspect", + root, + error.message()); + } + return {}; + } + if (!std::filesystem::is_directory(root, error) || error) { + EmitDrainWarning( + output, + "inspect", + root, + error ? error.message() : "not a directory"); + return {}; + } + + std::vector final_paths; + std::vector pending_paths; + try { + for (std::filesystem::directory_iterator iterator(root, error), end; + !error && iterator != end; + iterator.increment(error)) { + const auto filename = iterator->path().filename().string(); + if (HasPrefixAndSuffix( + filename, + kFinalPrefix, + kFinalSuffix)) { + final_paths.push_back(iterator->path()); + } else if (HasPrefixAndSuffix( + filename, + kPendingPrefix, + kPendingSuffix)) { + pending_paths.push_back(iterator->path()); + } + } + } catch (const std::exception& exception) { + EmitDrainWarning( + output, + "enumerate", + root, + exception.what()); + } catch (...) { + EmitDrainWarning( + output, + "enumerate", + root, + "unknown error"); + } + if (error) { + EmitDrainWarning( + output, + "enumerate", + root, + error.message()); + } + std::sort(final_paths.begin(), final_paths.end()); + std::sort(pending_paths.begin(), pending_paths.end()); + + std::vector records; + records.reserve(final_paths.size() + pending_paths.size()); + for (const auto& path : final_paths) { + std::string record; + try { + record = ReadRecord(path); + } catch (const std::exception& exception) { + EmitDrainWarning( + output, + "read", + path, + exception.what()); + continue; + } catch (...) { + EmitDrainWarning( + output, + "read", + path, + "unknown error"); + continue; + } + EmitRecord(output, record); + records.push_back(std::move(record)); + RemoveReportedRecord(path, output); + } + + for (const auto& path : pending_paths) { + const auto identity = ParsePendingIdentity(path); + if (!identity.has_value() || !process_probe) { + continue; + } + + std::optional current_start; + try { + current_start = process_probe(identity->pid); + } catch (...) { + continue; + } + if (!current_start.has_value() || + FormatUtc(*current_start) == identity->timestamp) { + continue; + } + + std::string record = + "incomplete corelib fatal record: " + + path.filename().string() + "\n"; + EmitRecord(output, record); + records.push_back(std::move(record)); + RemoveReportedRecord(path, output); + } + return records; +} + +void FatalRecordStore::RemoveUnusedPending() noexcept { + if (persisted_) { + return; + } + ClosePending(); + if (!pending_path_.empty()) { + DeleteFileW(pending_path_.c_str()); + } +} + +const std::filesystem::path& FatalRecordStore::pending_path() const noexcept { + return pending_path_; +} + +void FatalRecordStore::ClosePending() noexcept { + if (pending_handle_ != INVALID_HANDLE_VALUE) { + CloseHandle(pending_handle_); + pending_handle_ = INVALID_HANDLE_VALUE; + } +} + +} // namespace flm::corelib diff --git a/src/common/corelib/corelib_runtime.cpp b/src/common/corelib/corelib_runtime.cpp new file mode 100644 index 00000000..827de4c9 --- /dev/null +++ b/src/common/corelib/corelib_runtime.cpp @@ -0,0 +1,221 @@ +#include + +#include + +#include +#include +#include +#include + +namespace flm::corelib { +namespace { + +constexpr unsigned int kFatalExitCode = 0xE0040001u; + +struct ProcessRuntimeSlot { + std::mutex mutex; + std::shared_ptr runtime; +}; + +ProcessRuntimeSlot& RuntimeSlot() { + static auto* slot = new ProcessRuntimeSlot; + return *slot; +} + +[[noreturn]] void TerminateCurrentProcess(unsigned int code) { + TerminateProcess(GetCurrentProcess(), code); + std::abort(); +} + +} // namespace + +std::shared_ptr CorelibRuntime::GetOrCreate( + const std::filesystem::path& executable_dir) { + auto& slot = RuntimeSlot(); + std::lock_guard lock(slot.mutex); + if (slot.runtime) { + if (slot.runtime->state() != ProcessState::Healthy) { + throw std::runtime_error( + "corelib process runtime is not healthy"); + } + return slot.runtime; + } + + auto api = CorelibApi::Load( + CorelibApi::ResolveLibraryPath(executable_dir)); + auto runtime = Create( + std::move(api), + FatalRecordStore::ForCurrentProcess(), + TerminateCurrentProcess); + slot.runtime = runtime; + return runtime; +} + +std::shared_ptr CorelibRuntime::Create( + std::shared_ptr api, + FatalRecordStore records, + Terminator terminator) { + if (!api) { + throw std::invalid_argument( + "CorelibRuntime::Create requires a CorelibApi"); + } + if (!terminator) { + throw std::invalid_argument( + "CorelibRuntime::Create requires a terminator"); + } + + auto runtime = std::shared_ptr( + new CorelibRuntime( + std::move(api), + std::move(records), + std::move(terminator))); + try { + runtime->records_.Prepare(); + runtime->api_->Check( + runtime->api_->functions().selftest_dependencies(), + "ryzenai_corelib_selftest_dependencies"); + if (!runtime->api_->functions().has_device_context()) { + throw std::runtime_error( + "ryzenai_corelib_has_device_context reported no " + "AIE4 device context"); + } + runtime->state_.store( + ProcessState::Healthy, + std::memory_order_release); + return runtime; + } catch (...) { + runtime->api_->functions().cleanup(); + runtime->cleanup_called_ = true; + throw; + } +} + +void CorelibRuntime::ShutdownProcess() { + auto& slot = RuntimeSlot(); + std::shared_ptr runtime; + { + std::lock_guard lock(slot.mutex); + if (!slot.runtime) { + return; + } + runtime = slot.runtime; + runtime->ShutdownHealthy(); + slot.runtime.reset(); + } + runtime.reset(); +} + +CorelibRuntime::CorelibRuntime( + std::shared_ptr api, + FatalRecordStore records, + Terminator terminator) + : api_(std::move(api)), + records_(std::move(records)), + terminator_(std::move(terminator)) {} + +ExecutionLease CorelibRuntime::AcquireExecution() { + ExecutionLease lease(execution_mutex_); + if (state_.load(std::memory_order_acquire) != + ProcessState::Healthy) { + throw std::runtime_error( + "corelib process runtime is not accepting execution"); + } + return lease; +} + +bool CorelibRuntime::admission_open() const noexcept { + return state_.load(std::memory_order_acquire) == + ProcessState::Healthy; +} + +ProcessState CorelibRuntime::state() const noexcept { + return state_.load(std::memory_order_acquire); +} + +const std::shared_ptr& CorelibRuntime::api() const noexcept { + return api_; +} + +void CorelibRuntime::ShutdownHealthy() { + std::lock_guard shutdown_lock(shutdown_mutex_); + ProcessState expected = ProcessState::Healthy; + if (!state_.compare_exchange_strong( + expected, + ProcessState::Shutdown, + std::memory_order_acq_rel, + std::memory_order_acquire)) { + if (expected == ProcessState::Shutdown) { + return; + } + throw std::logic_error( + "cannot clean up a terminating corelib process runtime"); + } + + ExecutionLease lease(execution_mutex_); + if (state_.load(std::memory_order_acquire) == + ProcessState::Terminating) { + throw std::logic_error( + "cannot clean up a terminating corelib process runtime"); + } + if (api_->live_object_count() != 0) { +#if defined(FLM_CORELIB_TESTING) + if (before_live_object_rollback_for_test_) { + before_live_object_rollback_for_test_(); + } +#endif + expected = ProcessState::Shutdown; + state_.compare_exchange_strong( + expected, + ProcessState::Healthy, + std::memory_order_acq_rel, + std::memory_order_acquire); + throw std::logic_error( + "cannot clean up corelib while live corelib objects remain"); + } + + if (!cleanup_called_) { + api_->functions().cleanup(); + cleanup_called_ = true; + } + records_.RemoveUnusedPending(); +} + +#if defined(FLM_CORELIB_TESTING) +void CorelibRuntime::SetBeforeLiveObjectRollbackForTest( + std::function hook) { + before_live_object_rollback_for_test_ = std::move(hook); +} +#endif + +[[noreturn]] void CorelibRuntime::TerminateAfterFailure( + const FailureContext& failure) { + state_.store(ProcessState::Terminating, std::memory_order_release); + + std::cerr << "AIE4 terminal failure: call=" << failure.call + << " phase=" << failure.phase + << " layer="; + if (failure.layer.has_value()) { + std::cerr << *failure.layer; + } else { + std::cerr << "none"; + } + std::cerr << " rows=" << failure.rows + << " position=" << failure.position + << " detail=" << failure.detail << '\n'; + try { + const auto path = records_.Persist(failure); + std::cerr << "AIE4 fatal record: " << path.string() << '\n'; + } catch (const std::exception& error) { + std::cerr << "AIE4 fatal record persistence failed: " + << error.what() << '\n'; + } catch (...) { + std::cerr << "AIE4 fatal record persistence failed with an " + "unknown error\n"; + } + std::cerr.flush(); + + terminator_(kFatalExitCode); + std::abort(); +} + +} // namespace flm::corelib diff --git a/src/common/corelib/corelib_sources.cmake b/src/common/corelib/corelib_sources.cmake new file mode 100644 index 00000000..fe990e2a --- /dev/null +++ b/src/common/corelib/corelib_sources.cmake @@ -0,0 +1,9 @@ +set(FLM_CORELIB_AIE4_SOURCES + "${CMAKE_CURRENT_LIST_DIR}/corelib_api.cpp" + "${CMAKE_CURRENT_LIST_DIR}/corelib_fatal_record.cpp" + "${CMAKE_CURRENT_LIST_DIR}/corelib_runtime.cpp" + "${CMAKE_CURRENT_LIST_DIR}/phi4_corelib_aie4.cpp" + "${CMAKE_CURRENT_LIST_DIR}/phi4_corelib_host.cpp" + "${CMAKE_CURRENT_LIST_DIR}/phi4_corelib_manifest.cpp" + "${CMAKE_CURRENT_LIST_DIR}/phi4_corelib_weights.cpp" + "${CMAKE_CURRENT_LIST_DIR}/phi4_corelib_shape_plan.cpp") diff --git a/src/common/corelib/phi4_corelib_aie4.cpp b/src/common/corelib/phi4_corelib_aie4.cpp new file mode 100644 index 00000000..d0f8db7f --- /dev/null +++ b/src/common/corelib/phi4_corelib_aie4.cpp @@ -0,0 +1,1635 @@ +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace flm::phi4 { +namespace { + +constexpr std::string_view kEmbeddingName = + "model.embed_tokens.weight"; +constexpr std::string_view kInputNormName = + "model.layers.0.input_layernorm.weight"; +constexpr std::string_view kCosName = "cos_cache"; +constexpr std::string_view kSinName = "sin_cache"; +constexpr std::string_view kCreateStreamCall = + "ryzenai_corelib_create_stream"; +constexpr std::string_view kCreateTensorCall = + "ryzenai_corelib_create_device_tensor"; +constexpr std::string_view kTensorByteSizeCall = + "ryzenai_corelib_tensor_get_byte_size"; +constexpr std::string_view kSynchronizeCall = + "ryzenai_corelib_stream_synchronize"; +constexpr std::string_view kTensorDataTypeCall = + "ryzenai_corelib_tensor_get_data_type"; + +static_assert(sizeof(bf16) == sizeof(std::uint16_t)); + +std::uint32_t ValidateMaxLength(std::uint32_t max_length) { + if ( + max_length == 0 || + max_length > + static_cast( + constants::kMaxSequenceLength)) { + throw std::out_of_range( + "Phi-4 AIE4 maximum length must be in 1..4096"); + } + return max_length; +} + +void ValidateOptionalIntegerIdentity( + const LM_Config& config, + std::string_view key, + std::int64_t expected) { + const auto found = config._json_config.find(std::string(key)); + if ( + found == config._json_config.end() || + found->is_null()) { + return; + } + if (!found->is_number_integer()) { + throw std::invalid_argument( + "Phi-4 AIE4 LM_Config field '" + + std::string(key) + "' must be an integer"); + } + const std::int64_t actual = found->get(); + if (actual != expected) { + throw std::invalid_argument( + "Phi-4 AIE4 LM_Config field '" + + std::string(key) + + "' does not match the package identity"); + } +} + +void ValidateOptionalFloatingIdentity( + const LM_Config& config, + std::string_view key, + double expected) { + const auto found = config._json_config.find(std::string(key)); + if ( + found == config._json_config.end() || + found->is_null()) { + return; + } + if (!found->is_number()) { + throw std::invalid_argument( + "Phi-4 AIE4 LM_Config field '" + + std::string(key) + "' must be numeric"); + } + const double actual = found->get(); + if ( + !std::isfinite(actual) || + std::abs(actual - expected) > + std::numeric_limits::epsilon() * + std::max(1.0, std::abs(expected)) * 8.0) { + throw std::invalid_argument( + "Phi-4 AIE4 LM_Config field '" + + std::string(key) + + "' does not match the package identity"); + } +} + +void ValidateConfigIdentity(const LM_Config& config) { + // Legacy LM_Config files do not guarantee every fixed field. Validate + // each field they do provide; the package manifest independently remains + // the authority for absent fields. + ValidateOptionalIntegerIdentity( + config, + "num_hidden_layers", + constants::kLayerCount); + ValidateOptionalIntegerIdentity( + config, + "hidden_size", + constants::kHiddenSize); + ValidateOptionalIntegerIdentity( + config, + "intermediate_size", + constants::kIntermediateSize); + ValidateOptionalIntegerIdentity( + config, + "num_attention_heads", + constants::kQueryHeadCount); + ValidateOptionalIntegerIdentity( + config, + "num_key_value_heads", + constants::kKvHeadCount); + ValidateOptionalIntegerIdentity( + config, + "head_dim", + constants::kHeadSize); + ValidateOptionalIntegerIdentity( + config, + "vocab_size", + constants::kVocabularySize); + ValidateOptionalFloatingIdentity( + config, + "rms_norm_eps", + constants::kRmsEpsilon); +} + +std::size_t CheckedElements( + std::int64_t rows, + std::int64_t width, + std::string_view context) { + if (rows <= 0 || width <= 0) { + throw std::invalid_argument( + std::string(context) + + " rows and width must be positive"); + } + const auto row_count = static_cast(rows); + const auto column_count = static_cast(width); + if ( + row_count > + std::numeric_limits::max() / column_count) { + throw std::overflow_error( + std::string(context) + " extent overflows size_t"); + } + return static_cast(row_count * column_count); +} + +std::size_t TensorByteCount( + ryzenai_corelib_data_type data_type, + std::span shape) { + if (shape.empty()) { + throw std::invalid_argument( + "Phi-4 AIE4 tensor shape cannot be empty"); + } + std::size_t elements = 1; + for (const std::int64_t dimension : shape) { + if (dimension <= 0) { + throw std::invalid_argument( + "Phi-4 AIE4 tensor dimensions must be positive"); + } + const auto value = static_cast(dimension); + if ( + elements > + std::numeric_limits::max() / value) { + throw std::overflow_error( + "Phi-4 AIE4 tensor extent overflows size_t"); + } + elements *= value; + } + + std::size_t element_size = 0; + switch (data_type) { + case ryzenai_corelib_data_type_bf16: + case ryzenai_corelib_data_type_fp16: + element_size = sizeof(std::uint16_t); + break; + case ryzenai_corelib_data_type_fp32: + element_size = sizeof(float); + break; + default: + throw std::invalid_argument( + "Phi-4 AIE4 requested an unsupported tensor dtype"); + } + if ( + elements > + std::numeric_limits::max() / element_size) { + throw std::overflow_error( + "Phi-4 AIE4 tensor byte size overflows size_t"); + } + return elements * element_size; +} + +ryzenai_corelib_data_type SourceDataType( + SourceDType data_type, + std::string_view context) { + switch (data_type) { + case SourceDType::Float16: + return ryzenai_corelib_data_type_fp16; + case SourceDType::Float32: + return ryzenai_corelib_data_type_fp32; + case SourceDType::UInt8: + case SourceDType::Int64: + throw std::invalid_argument( + std::string(context) + + " requires an FP16 or FP32 source"); + } + throw std::logic_error("unreachable Phi-4 source dtype"); +} + +std::uint64_t MappedSourceBytes(const Phi4Package& package) { + std::unordered_set owners; + std::uint64_t total = 0; + const auto add = [&](const InitializerView& view) { + if (!view.owner || !owners.insert(view.owner.get()).second) { + return; + } + if ( + view.owner->size() > + std::numeric_limits::max() - total) { + throw std::overflow_error( + "Phi-4 mapped-source byte total overflows uint64_t"); + } + total += view.owner->size(); + }; + + add(package.Require(kEmbeddingName)); + add(package.Require(kInputNormName)); + add(package.Require(kCosName)); + add(package.Require(kSinName)); + for (const auto& object : package.weight_objects()) { + for (const auto& [_, initializer] : object.components) { + add(package.Require(initializer)); + } + } + return total; +} + +std::uint64_t ElapsedNanoseconds( + std::chrono::steady_clock::time_point started) { + const auto elapsed = + std::chrono::duration_cast( + std::chrono::steady_clock::now() - started); + return static_cast( + std::max(elapsed.count(), 0)); +} + +// Named for what it tests, which is no longer "before submit". +// +// It was called RecoverableBeforeSubmit while the irrevocable boundary was +// per STEP: the first successful submit latched it for the rest of the step, +// so "recoverable" and "nothing submitted yet" were the same predicate. They +// are not any more. checked_synchronize clears the latch on a COMPLETED +// synchronize, so this returns true after three submits and a synchronize -- +// the point at which corelib's outstanding list is provably empty and the +// only residue is a partially-written KV cache the frontend discards. +// +// `submission.irrevocable()` therefore means "device work may be outstanding +// or in an unknown state", not "a submit has happened". +bool RecoverableBeforeIrrevocableWork( + bool synchronize_in_progress, + const corelib::StepSubmissionState& submission) noexcept { + return !synchronize_in_progress && !submission.irrevocable(); +} + +[[noreturn]] void TerminateCorelibFailure( + const std::shared_ptr& runtime, + const corelib::CorelibError& error, + std::string phase, + std::optional layer, + std::int64_t rows, + std::int64_t position) { + runtime->TerminateAfterFailure(corelib::FailureContext{ + error.status, + error.call, + error.detail, + std::move(phase), + layer, + rows, + position}); +} + +[[noreturn]] void TerminateHostFailure( + const std::shared_ptr& runtime, + const std::exception& error, + std::string phase, + std::optional layer, + std::int64_t rows, + std::int64_t position) { + runtime->TerminateAfterFailure(corelib::FailureContext{ + ryzenai_corelib_status_failure, + "host_exception", + error.what(), + std::move(phase), + layer, + rows, + position}); +} + +[[noreturn]] void TerminateUnknownFailure( + const std::shared_ptr& runtime, + std::string phase, + std::optional layer, + std::int64_t rows, + std::int64_t position) { + runtime->TerminateAfterFailure(corelib::FailureContext{ + ryzenai_corelib_status_failure, + "unknown_exception", + "non-standard exception after the irrevocable boundary", + std::move(phase), + layer, + rows, + position}); +} + +} // namespace + +struct phi4_corelib_aie4::Impl final { + Impl( + LM_Config config, + std::filesystem::path model_path, + std::shared_ptr supplied_runtime, + std::uint32_t requested_max_length) + : runtime(std::move(supplied_runtime)), + max_length(ValidateMaxLength(requested_max_length)) { + ValidateConfigIdentity(config); + if (!runtime) { + throw std::invalid_argument( + "phi4_corelib_aie4 requires a CorelibRuntime"); + } + api = runtime->api(); + if (!api) { + throw std::invalid_argument( + "phi4_corelib_aie4 runtime has no CorelibApi"); + } + if (model_path.empty()) { + throw std::invalid_argument( + "phi4_corelib_aie4 requires a model path"); + } + + const auto model_load_started = + std::chrono::steady_clock::now(); + auto execution = runtime->AcquireExecution(); + + const auto manifest_started = + std::chrono::steady_clock::now(); + package = std::make_shared( + Phi4Package::Load(model_path, api, false)); + metrics.manifest_map_ns = + ElapsedNanoseconds(manifest_started); + metrics.mapped_source_bytes = + MappedSourceBytes(*package); + const auto shape_plan_started = + std::chrono::steady_clock::now(); + shape_plan.emplace(Phi4ShapePlan::Build(api)); + metrics.shape_plan_ns = + ElapsedNanoseconds(shape_plan_started); + metrics.helper_transition_counts = { + static_cast( + shape_plan->Transitions( + RowUse::QueryProjection).size()), + static_cast( + shape_plan->Transitions( + RowUse::KvProjection).size()), + static_cast( + shape_plan->Transitions( + RowUse::Attention).size()), + static_cast( + shape_plan->Transitions( + RowUse::OutputProjection).size()), + static_cast( + shape_plan->Transitions( + RowUse::SsMlp).size()), + static_cast( + shape_plan->Transitions( + RowUse::LmHead).size()), + }; + + const auto& embedding_view = + package->Require(kEmbeddingName); + embedding = std::span( + reinterpret_cast( + embedding_view.data), + embedding_view.size / sizeof(std::uint16_t)); + + // The layer-0 norm feeds the host RMSNorm directly and never + // reaches a tensor, so widening it is the `API-6` FP16-to-FP32 + // helper; an FP32 source is a plain copy. + const auto& norm_view = package->Require(kInputNormName); + input_norm.resize( + static_cast(constants::kHiddenSize)); + if ( + SourceDataType(norm_view.dtype, kInputNormName) == + ryzenai_corelib_data_type_fp32) { + std::copy_n( + reinterpret_cast(norm_view.data), + input_norm.size(), + input_norm.begin()); + } else { + corelib::WidenFp16Array( + reinterpret_cast(norm_view.data), + input_norm.size(), + input_norm.data()); + } + + const auto cos_host = + package->MaterializeRopeGather(kCosName); + const auto sin_host = + package->MaterializeRopeGather(kSinName); + + const auto weight_pack_started = + std::chrono::steady_clock::now(); + const std::uint64_t weight_creates_before = + api->weight_creation_count(); + weights.emplace(Phi4Weights::Load(api, package)); + metrics.weight_pack_ns = + ElapsedNanoseconds(weight_pack_started); + metrics.packed_weight_bytes = + static_cast(weights->packed_bytes()); + // COUNTED, not asserted from a constant. + // + // This field used to be assigned `kLayerCount * 5 + 1` directly. That + // value is right, and it is also the one thing a constant can never + // tell you: whether a weight object was created somewhere it should + // not have been. Design 18.7 and design 15.4 make post-warm + // allocation a property to MEASURE, and Task 13's stability window + // reads this field to do it, so it has to come from the API's own + // creation counter. + metrics.weight_create_count = + api->weight_creation_count() - weight_creates_before; + + const auto& capacities = shape_plan->capacities(); + const std::size_t layer_elements = CheckedElements( + capacities.layer_rows, + constants::kHiddenSize, + "Phi-4 host layer staging"); + embedding_fp32.resize(layer_elements); + normalized_fp32.resize(layer_elements); + fp32_staging.resize(layer_elements); + const std::size_t lm_head_elements = CheckedElements( + capacities.lm_head_rows, + constants::kHiddenSize, + "Phi-4 LM-head padding staging"); + padding_zero_staging.resize( + std::max(layer_elements, lm_head_elements), + 0); + v_staging.reserve( + CheckedElements( + capacities.layer_rows, + constants::kKvDimension + constants::kHeadSize, + "Phi-4 V staging")); + last_hidden_staging.resize( + static_cast(constants::kHiddenSize)); + + // Everything from here to the end of load is device-object + // construction and the RoPE upload. It was the only unlabelled part + // of the load timeline, and 138 ms of a 12.8 s load went unexplained + // in the first baseline. A breakdown that does not add up invites the + // reader to assume the missing time is somewhere it is not. + const auto device_setup_started = + std::chrono::steady_clock::now(); + stream = CreateStream(); + hidden_tensors[0] = CreateTensor( + ryzenai_corelib_data_type_bf16, + {capacities.layer_rows, constants::kHiddenSize}, + false); + hidden_tensors[1] = CreateTensor( + ryzenai_corelib_data_type_bf16, + {capacities.layer_rows, constants::kHiddenSize}, + false); + residual_tensor = CreateTensor( + ryzenai_corelib_data_type_bf16, + {capacities.layer_rows, constants::kHiddenSize}, + false); + skip_sum_tensor = CreateTensor( + ryzenai_corelib_data_type_bf16, + {capacities.layer_rows, constants::kHiddenSize}, + false); + query_tensor = CreateTensor( + ryzenai_corelib_data_type_bf16, + {capacities.layer_rows, constants::kQueryDimension}, + false); + key_tensor = CreateTensor( + ryzenai_corelib_data_type_bf16, + {capacities.layer_rows, constants::kKvDimension}, + false); + value_tensor = CreateTensor( + ryzenai_corelib_data_type_bf16, + {capacities.layer_rows, constants::kKvDimension}, + false); + attention_tensor = CreateTensor( + ryzenai_corelib_data_type_bf16, + {capacities.layer_rows, constants::kQueryDimension}, + false); + lm_input_tensor = CreateTensor( + ryzenai_corelib_data_type_bf16, + {capacities.lm_head_rows, constants::kHiddenSize}, + false); + lm_output_tensor = CreateTensor( + ryzenai_corelib_data_type_bf16, + {capacities.lm_head_rows, constants::kVocabularySize}, + false); + cos_tensor = CreateTensor( + ryzenai_corelib_data_type_fp32, + {constants::kMaxSequenceLength, + constants::kRopeDimension / 2}, + false); + sin_tensor = CreateTensor( + ryzenai_corelib_data_type_fp32, + {constants::kMaxSequenceLength, + constants::kRopeDimension / 2}, + false); + + for (std::size_t layer = 0; + layer < + static_cast(constants::kLayerCount); + ++layer) { + key_caches[layer] = CreateTensor( + ryzenai_corelib_data_type_bf16, + {constants::kKvHeadCount, + constants::kMaxSequenceLength, + constants::kHeadSize}, + true); + value_caches[layer] = CreateTensor( + ryzenai_corelib_data_type_bf16, + {constants::kKvHeadCount, + constants::kMaxSequenceLength, + constants::kHeadSize}, + true); + } + + // One write per table, in the source dtype, with `count` in FP32 + // elements of the destination tensor. tensor_write is the only + // conversion boundary corelib now offers. + const std::size_t rope_elements = CheckedElements( + constants::kMaxSequenceLength, + constants::kRopeDimension / 2, + "Phi-4 RoPE upload"); + if ( + cos_host.count != rope_elements || + sin_host.count != rope_elements) { + throw std::logic_error( + "Phi-4 RoPE gather produced the wrong element count"); + } + api->WriteElements( + cos_tensor.get(), + cos_host.dtype, + cos_host.data, + rope_elements, + 0); + api->WriteElements( + sin_tensor.get(), + sin_host.dtype, + sin_host.data, + rope_elements, + 0); + + current_hidden = &hidden_tensors[0]; + next_hidden = &hidden_tensors[1]; + current_residual = &residual_tensor; + next_skip_sum = &skip_sum_tensor; + metrics.device_setup_ns = + ElapsedNanoseconds(device_setup_started); + metrics.model_load_ns = + ElapsedNanoseconds(model_load_started); + } + + ~Impl() noexcept { + if (!stream) { + ReleaseResourcesWithoutSynchronization(); + return; + } + + try { + auto execution = runtime->AcquireExecution(); + api->Check( + api->functions().stream_synchronize(stream.get()), + kSynchronizeCall); + stream.reset(); + ReleaseTensorsWeightsAndPackage(); + } catch (const corelib::CorelibError& error) { + TerminateCorelibFailure( + runtime, + error, + "destruction", + std::nullopt, + last_live_rows, + position); + } catch (const std::exception& error) { + TerminateHostFailure( + runtime, + error, + "destruction", + std::nullopt, + last_live_rows, + position); + } catch (...) { + TerminateUnknownFailure( + runtime, + "destruction", + std::nullopt, + last_live_rows, + position); + } + } + + corelib::UniqueStream CreateStream() { + ryzenai_corelib_stream_ptr raw = nullptr; + const auto status = + api->functions().create_stream(&raw); + corelib::UniqueStream result(api, raw); + api->Check(status, kCreateStreamCall); + if (!result) { + throw std::runtime_error( + "ryzenai_corelib_create_stream succeeded with a " + "null object"); + } + return result; + } + + corelib::UniqueTensor CreateTensor( + ryzenai_corelib_data_type data_type, + std::initializer_list shape_values, + bool is_kv) { + const std::vector shape(shape_values); + ryzenai_corelib_tensor_ptr raw = nullptr; + const auto status = + api->functions().create_device_tensor( + data_type, + shape.data(), + shape.size(), + &raw); + corelib::UniqueTensor result(api, raw); + api->Check(status, kCreateTensorCall); + if (!result) { + throw std::runtime_error( + "ryzenai_corelib_create_device_tensor succeeded " + "with a null object"); + } + + std::size_t actual_bytes = 0; + api->Check( + api->functions().tensor_get_byte_size( + result.get(), + &actual_bytes), + kTensorByteSizeCall); + const std::size_t expected_bytes = + TensorByteCount(data_type, shape); + if (actual_bytes != expected_bytes) { + throw std::runtime_error( + "corelib device tensor byte size does not match " + "the requested Phi-4 shape"); + } + + // Every subsequent write and read counts in elements of THIS + // dtype, so confirm the tensor holds what was asked for rather + // than inferring it from the byte size, which FP16 and BF16 share. + ryzenai_corelib_data_type actual_type{}; + api->Check( + api->functions().tensor_get_data_type( + result.get(), + &actual_type), + kTensorDataTypeCall); + if (actual_type != data_type) { + throw std::runtime_error( + "corelib device tensor dtype does not match the " + "requested Phi-4 dtype"); + } + ++metrics.device_tensor_create_count; + if (is_kv) { + metrics.kv_bytes += actual_bytes; + } else { + metrics.scratch_bytes += actual_bytes; + } + return result; + } + + void ReleaseResourcesWithoutSynchronization() noexcept { + stream.reset(); + ReleaseTensorsWeightsAndPackage(); + } + + void ReleaseTensorsWeightsAndPackage() noexcept { + for (auto layer = value_caches.rbegin(); + layer != value_caches.rend(); + ++layer) { + layer->reset(); + } + for (auto layer = key_caches.rbegin(); + layer != key_caches.rend(); + ++layer) { + layer->reset(); + } + sin_tensor.reset(); + cos_tensor.reset(); + lm_output_tensor.reset(); + lm_input_tensor.reset(); + attention_tensor.reset(); + value_tensor.reset(); + key_tensor.reset(); + query_tensor.reset(); + skip_sum_tensor.reset(); + residual_tensor.reset(); + hidden_tensors[1].reset(); + hidden_tensors[0].reset(); + weights.reset(); + embedding = {}; + package.reset(); + } + + void ValidateTokens(std::span token_ids) const { + for (const int token_id : token_ids) { + if ( + token_id < 0 || + token_id >= constants::kVocabularySize) { + throw std::out_of_range( + "Phi-4 token ID is outside the vocabulary"); + } + } + } + + // Refused here, before the step starts, rather than one dispatch later + // inside flat_mha. + // + // Measured on the AIE4 target: letting the step run to flat_mha at + // position 4095 killed the server (exit 0xE0040001), because the whole + // step past its first successful submit was classified irrevocable. That + // classification is now per submission group -- see the note in + // checked_synchronize -- and the flat_mha refusal, which corelib raises + // before it touches the device, lands on the recoverable side. So this + // guard is no longer the difference between an exception and a dead + // server. + // + // It is still the difference between a clean refusal and a wrecked + // session. Reaching flat_mha means the V cache for this layer has already + // been scattered and the caller has to throw the whole conversation away; + // refusing here costs nothing, names the actual limit, and leaves the + // session usable at the position it had reached. It is also the only + // layer that protects a caller which never consults the frontend's + // aie4_active_cap() -- a different frontend, a direct embedder, or a + // future endpoint. + void EnsureDecodeWindow(std::size_t token_count) const { + const auto window = + static_cast(position) + + static_cast(token_count); + if (window > constants::kMaxDecodeWindow) { + throw std::out_of_range( + "Phi-4 AIE4 decode would need a " + + std::to_string(window) + + "-token attention window; the token attention kernel ships " + "no window above " + + std::to_string(constants::kMaxDecodeWindow)); + } + } + + void EnsureCapacity(std::size_t token_count) const { + if (token_count == 0) { + throw std::invalid_argument( + "Phi-4 prefill requires at least one token"); + } + const auto remaining = + static_cast(max_length) - + static_cast(position); + if (token_count > remaining) { + throw std::out_of_range( + "Phi-4 AIE4 maximum context length would be " + "exceeded"); + } + } + + struct RunRowExtents { + std::int64_t query_projection; + std::int64_t kv_projection; + std::int64_t attention; + std::int64_t output_projection; + std::int64_t ssmlp; + std::int64_t lm_head; + + std::int64_t ProjectionInput() const noexcept { + return std::max(query_projection, kv_projection); + } + }; + + RunRowExtents RowsForRun(std::int64_t rows) { + RunRowExtents extents{ + shape_plan->RowsFor(RowUse::QueryProjection, rows), + shape_plan->RowsFor(RowUse::KvProjection, rows), + shape_plan->RowsFor(RowUse::Attention, rows), + shape_plan->RowsFor(RowUse::OutputProjection, rows), + shape_plan->RowsFor(RowUse::SsMlp, rows), + shape_plan->RowsFor(RowUse::LmHead, 1)}; + ++metrics.attention_extent_queries; + ++metrics.output_projection_extent_queries; + ++metrics.lm_head_extent_queries; + return extents; + } + + void WriteZeroRows( + corelib::UniqueTensor& tensor, + std::int64_t first_row, + std::int64_t row_count, + std::int64_t width, + std::string_view context) { + if (row_count <= 0) { + return; + } + if (first_row < 0) { + throw std::out_of_range( + "Phi-4 padding write has a negative row offset"); + } + // Offsets and counts are BF16 elements of the destination tensor, + // not bytes; the staging buffer is BF16 too, so this is a copy. + const std::size_t offset = + first_row == 0 + ? 0 + : CheckedElements(first_row, width, context); + const std::size_t words = CheckedElements( + row_count, + width, + context); + if (words > padding_zero_staging.size()) { + throw std::out_of_range( + "Phi-4 padding write exceeds host staging capacity"); + } + api->WriteElements( + tensor.get(), + ryzenai_corelib_data_type_bf16, + padding_zero_staging.data(), + words, + offset); + ++metrics.padding_write_calls; + metrics.padding_bytes += words * sizeof(std::uint16_t); + } + + void BridgePadding( + corelib::UniqueTensor& tensor, + std::int64_t producer_rows, + std::int64_t consumer_rows, + std::int64_t width, + std::string_view context) { + if (consumer_rows > producer_rows) { + WriteZeroRows( + tensor, + producer_rows, + consumer_rows - producer_rows, + width, + context); + } + } + + void StageInput( + std::span token_ids, + const RunRowExtents& extents) { + const auto rows = + static_cast(token_ids.size()); + const std::size_t live_elements = CheckedElements( + rows, + constants::kHiddenSize, + "Phi-4 input staging"); + if ( + live_elements > embedding_fp32.size() || + live_elements > normalized_fp32.size()) { + throw std::out_of_range( + "Phi-4 input exceeds the planned host capacity"); + } + + const auto embedding_output = + std::span(embedding_fp32).first(live_elements); + const auto normalized_output = + std::span(normalized_fp32).first(live_elements); + GatherEmbedding( + embedding, + token_ids, + embedding_output); + RmsNorm( + embedding_output, + input_norm, + rows, + constants::kHiddenSize, + static_cast(constants::kRmsEpsilon), + normalized_output); + + // Design Section 10.2: the host stays in FP32 and writes FP32 into + // the BF16 tensors. Corelib narrows inside tensor_write, so there + // is one BF16 rounding implementation on this path, not two that + // have to agree. `count` is in BF16 elements of the destination. + const std::int64_t hidden_rows = + extents.ProjectionInput(); + StageFp32( + normalized_output, + rows, + hidden_rows, + constants::kHiddenSize, + fp32_staging); + const std::size_t hidden_elements = CheckedElements( + hidden_rows, + constants::kHiddenSize, + "Phi-4 hidden staging"); + api->WriteElements( + current_hidden->get(), + ryzenai_corelib_data_type_fp32, + fp32_staging.data(), + hidden_elements, + 0); + + const std::int64_t residual_rows = extents.ssmlp; + StageFp32( + embedding_output, + rows, + residual_rows, + constants::kHiddenSize, + fp32_staging); + const std::size_t residual_elements = CheckedElements( + residual_rows, + constants::kHiddenSize, + "Phi-4 residual staging"); + api->WriteElements( + current_residual->get(), + ryzenai_corelib_data_type_fp32, + fp32_staging.data(), + residual_elements, + 0); + } + + ryzenai_corelib_status SubmitMatMul( + const corelib::UniqueTensor& input, + const corelib::UniqueMatMulWeights& operation_weights, + corelib::UniqueTensor& output, + std::int64_t rows) { + return api->functions().matmul( + stream.get(), + input.get(), + rows, + operation_weights.get(), + output.get()); + } + + ryzenai_corelib_status SubmitMha( + std::size_t layer, + std::int64_t rows, + std::int64_t current_position) { + return api->functions().flat_mha( + stream.get(), + &shape_plan->attention_desc(), + query_tensor.get(), + key_tensor.get(), + rows, + current_position, + cos_tensor.get(), + sin_tensor.get(), + key_caches[layer].get(), + value_caches[layer].get(), + attention_tensor.get()); + } + + ryzenai_corelib_status SubmitSsMlp( + const corelib::UniqueTensor& input, + const corelib::UniqueTensor& residual, + const corelib::UniqueSsMlpWeights& operation_weights, + corelib::UniqueTensor& skip_sum, + corelib::UniqueTensor& normalized, + std::int64_t rows) { + return api->functions().ssmlp( + stream.get(), + input.get(), + residual.get(), + rows, + operation_weights.get(), + skip_sum.get(), + normalized.get()); + } + + void ScatterValue( + std::size_t layer, + std::int64_t rows, + std::int64_t current_position) { + flm::phi4::ScatterV( + *api, + value_tensor.get(), + value_caches[layer].get(), + rows, + current_position, + v_staging, + v_metrics); + metrics.v_read_calls = v_metrics.read_calls; + metrics.v_write_calls = v_metrics.write_calls; + metrics.v_bytes = v_metrics.bytes; + metrics.v_scatter_ns = v_metrics.nanoseconds; + } + + void PrepareLastHidden( + std::int64_t rows, + std::int64_t lm_head_rows) { + // Design Section 10.5: both tensors are BF16 and so is the caller + // dtype, so this is a straight copy with no FP32 round trip. + const std::size_t row_elements = CheckedElements( + 1, + constants::kHiddenSize, + "Phi-4 last hidden"); + const std::size_t source_offset = + static_cast(rows - 1) * row_elements; + WriteZeroRows( + lm_input_tensor, + 0, + lm_head_rows, + constants::kHiddenSize, + "Phi-4 LM-head input initialization"); + api->ReadElements( + current_hidden->get(), + ryzenai_corelib_data_type_bf16, + last_hidden_staging.data(), + row_elements, + source_offset); + api->WriteElements( + lm_input_tensor.get(), + ryzenai_corelib_data_type_bf16, + last_hidden_staging.data(), + row_elements, + 0); + } + + void ReadLogits(buffer& output) { + constexpr std::size_t logits_elements = + static_cast(constants::kVocabularySize); + if (output.size() != logits_elements) { + throw std::logic_error( + "Phi-4 logits buffer has the wrong size"); + } + // Straight into the returned buffer: the return type is + // already BF16, so widening and narrowing again would be a round + // trip for nothing. + api->ReadElements( + lm_output_tensor.get(), + ryzenai_corelib_data_type_bf16, + output.data(), + logits_elements, + 0); + } + + buffer RunRows(std::span token_ids) { + const std::int64_t rows = + static_cast(token_ids.size()); + auto execution = runtime->AcquireExecution(); + corelib::StepSubmissionState submission; + std::optional active_layer; + std::string active_phase = "stage_input"; + bool synchronize_in_progress = false; + buffer logits( + static_cast(constants::kVocabularySize)); + + try { + auto checked_submit = [&]( + ryzenai_corelib_status status, + std::string_view call) { + api->Check(status, call); + submission.MarkSuccessfulSubmit(); + ++metrics.dispatch_count; + }; + auto checked_synchronize = [&] { + synchronize_in_progress = true; + api->Check( + api->functions().stream_synchronize( + stream.get()), + kSynchronizeCall); + synchronize_in_progress = false; + // The irrevocable boundary is per SUBMISSION GROUP, not per + // step. + // + // `submission` answers one question: is there dispatched work + // whose completion state this process cannot know? A + // successful stream_synchronize is a full barrier -- corelib + // waits every outstanding command and empties its own + // outstanding list -- so the answer becomes "no", and the + // classification has to say so. + // + // Latching it for the whole step meant that after the first + // matmul of layer 0, all ~100 remaining dispatches AND every + // host operation between them were treated as irrevocable. + // A bad_alloc in the V scatter, a CheckedElements overflow in + // a padding write, or a rejected tensor_write killed the + // server (exit 0xE0040001) at points where the stream was + // provably quiescent and clearing the session would have + // sufficed. The only lasting state at those points is + // partially-updated KV caches, which the frontend already + // discards via clear_after_corelib_error(). + // + // This deliberately also moves a REJECTED DISPATCH that + // follows a completed synchronize -- flat_mha, o, ssmlp and + // lm_head, all four of which are tested by + // TestFailuresPastACompletedSynchronizeAreRecoverable -- + // onto the recoverable side, which is a wider + // change than "host failures only". It is the same rule + // checked_submit already applies to the first dispatch of a + // step: the flag is set only after api->Check succeeds, + // because a rejected dispatch enqueued nothing. Verified + // against the corelib sources rather than assumed: every + // refusal in matmul_bf16, ssmlp_bf16 and flat_mha_bf16 + // precedes that call's single enqueue point, and a failed + // Stream::Submit pops the command back off the outstanding + // list. In particular the 4096-token-window refusal is the + // first statement of flat_mha's Execute(), ahead of the + // kernel lookup, the rope rebuild and the K-cache write, so + // it leaves the device untouched. + // + // A FAILED synchronize is not cleared: the Check above throws + // first, `synchronize_in_progress` stays true, and the policy + // terminates. That is not conservatism -- corelib clears its + // outstanding list on a throwing Wait() too, releasing runs + // and buffers whose work may still be executing, so the + // process genuinely cannot continue. + submission = corelib::StepSubmissionState{}; + ++metrics.synchronize_count; + }; + + const RunRowExtents extents = RowsForRun(rows); + StageInput(token_ids, extents); + + for (int layer = 0; + layer < constants::kLayerCount; + ++layer) { + active_layer = layer; + active_phase = "qkv"; + const auto& layer_weights = + weights->layers()[static_cast(layer)]; + checked_submit( + SubmitMatMul( + *current_hidden, + layer_weights.q, + query_tensor, + rows), + "q"); + checked_submit( + SubmitMatMul( + *current_hidden, + layer_weights.k, + key_tensor, + rows), + "k"); + checked_submit( + SubmitMatMul( + *current_hidden, + layer_weights.v, + value_tensor, + rows), + "v"); + checked_synchronize(); + + active_phase = "v_scatter"; + ScatterValue( + static_cast(layer), + rows, + position); + + active_phase = "flat_mha"; + BridgePadding( + query_tensor, + extents.query_projection, + extents.attention, + constants::kQueryDimension, + "Phi-4 query-to-attention padding"); + BridgePadding( + key_tensor, + extents.kv_projection, + extents.attention, + constants::kKvDimension, + "Phi-4 key-to-attention padding"); + checked_submit( + SubmitMha( + static_cast(layer), + rows, + position), + "flat_mha"); + checked_synchronize(); + + active_phase = "o"; + BridgePadding( + attention_tensor, + extents.attention, + extents.output_projection, + constants::kQueryDimension, + "Phi-4 attention-to-output padding"); + checked_submit( + SubmitMatMul( + attention_tensor, + layer_weights.o, + *current_hidden, + rows), + "o"); + checked_synchronize(); + + active_phase = "ssmlp"; + BridgePadding( + *current_hidden, + extents.output_projection, + extents.ssmlp, + constants::kHiddenSize, + "Phi-4 output-to-SSMLP padding"); + checked_submit( + SubmitSsMlp( + *current_hidden, + *current_residual, + layer_weights.mlp, + *next_skip_sum, + *next_hidden, + rows), + "ssmlp"); + checked_synchronize(); + if (layer + 1 < constants::kLayerCount) { + active_phase = "next_layer_padding"; + BridgePadding( + *next_hidden, + extents.ssmlp, + extents.ProjectionInput(), + constants::kHiddenSize, + "Phi-4 SSMLP-to-projection padding"); + } + std::swap(current_hidden, next_hidden); + std::swap(current_residual, next_skip_sum); + } + + active_layer.reset(); + active_phase = "lm_head"; + PrepareLastHidden(rows, extents.lm_head); + checked_submit( + SubmitMatMul( + lm_input_tensor, + weights->lm_head(), + lm_output_tensor, + 1), + "lm_head"); + checked_synchronize(); + ReadLogits(logits); + } catch (const corelib::CorelibError& error) { + if (RecoverableBeforeIrrevocableWork( + synchronize_in_progress, + submission)) { + throw; + } + TerminateCorelibFailure( + runtime, + error, + active_phase, + active_layer, + rows, + position); + // THE TWO TERMINATE CALLS BELOW ARE NOW UNREACHABLE FROM RunRows, + // and that is recorded rather than fixed. Read before deleting them. + // + // Reaching either needs a non-CorelibError exception thrown while + // `submission.irrevocable()` or `synchronize_in_progress` is true. + // After the boundary became per submission group, walk the windows: + // + // * irrevocable window -- from checked_submit's MarkSuccessfulSubmit + // to the next checked_synchronize's clear. Everything executed in + // it is a SubmitMatMul/SubmitMha/SubmitSsMlp forwarder (a bare + // call through the C function table, no allocation, no container) + // plus api->Check, which throws CorelibError and nothing else. + // * synchronize window -- stream_synchronize plus api->Check. Same. + // + // Every host operation that CAN throw a std::exception -- StageInput, + // ScatterValue, BridgePadding, PrepareLastHidden, ReadLogits, and the + // `active_phase = "..."` assignments, which allocate for the longer + // literals -- sits either before the step's first submit or after a + // completed synchronize. So the guard above is always true when a + // host exception arrives here, and both arms rethrow. + // + // WHAT WOULD MAKE THEM FIRE AGAIN: any host work inserted BETWEEN a + // checked_submit and its checked_synchronize -- a staging copy folded + // into the q/k/v group, a metrics container appended to mid-group, a + // descriptor built between two dispatches. That is exactly the case + // that must still terminate, which is why these stay. Deleting them + // would silently reclassify the next such edit as recoverable. + // + // The one residue: `api->Check` allocating its own message and + // throwing std::bad_alloc inside the synchronize window would reach + // TerminateHostFailure. Not tested, not worth a fake. + // + // Consequence for coverage, stated because it went unmentioned when + // it happened: the two tests that asserted a host failure past a + // submit terminates were DELETED this round, correctly -- they + // asserted the retracted rule. The `phase == "v_scatter"` string + // assertion went with them, and no fatal record can carry that phase + // any more, so `active_phase`'s v_scatter value is now only reachable + // on this dead path. TestFailuresPastACompletedSynchronizeAreRecoverable + // covers the same three failures from the recoverable side instead. + } catch (const std::exception& error) { + if (RecoverableBeforeIrrevocableWork( + synchronize_in_progress, + submission)) { + throw; + } + TerminateHostFailure( + runtime, + error, + active_phase, + active_layer, + rows, + position); + } catch (...) { + if (RecoverableBeforeIrrevocableWork( + synchronize_in_progress, + submission)) { + throw; + } + TerminateUnknownFailure( + runtime, + active_phase, + active_layer, + rows, + position); + } + + position += rows; + last_live_rows = rows; + return logits; + } + + std::vector ReadLiveCache( + const corelib::UniqueTensor& cache) const { + if (position == 0) { + return {}; + } + const std::size_t live_rows = + static_cast(position); + const std::size_t head_width = + static_cast(constants::kHeadSize); + std::vector result( + static_cast(constants::kKvHeadCount) * + live_rows * head_width); + const std::size_t elements_per_head = live_rows * head_width; + for (std::size_t head = 0; + head < + static_cast(constants::kKvHeadCount); + ++head) { + const std::size_t source_offset = + head * + static_cast( + constants::kMaxSequenceLength) * + head_width; + api->ReadElements( + cache.get(), + ryzenai_corelib_data_type_bf16, + result.data() + head * elements_per_head, + elements_per_head, + source_offset); + } + return result; + } + +#ifdef DEV_BUILD + Phi4DebugSnapshot DebugSnapshot() const { + auto execution = runtime->AcquireExecution(); + Phi4DebugSnapshot snapshot; + snapshot.live_rows = last_live_rows; + snapshot.position = position; + snapshot.layer0_k = ReadLiveCache(key_caches.front()); + snapshot.layer0_v = ReadLiveCache(value_caches.front()); + snapshot.layer31_k = ReadLiveCache(key_caches.back()); + snapshot.layer31_v = ReadLiveCache(value_caches.back()); + snapshot.last_hidden.resize( + static_cast(constants::kHiddenSize)); + snapshot.logits.resize( + static_cast(constants::kVocabularySize)); + api->ReadElements( + lm_input_tensor.get(), + ryzenai_corelib_data_type_bf16, + snapshot.last_hidden.data(), + snapshot.last_hidden.size(), + 0); + api->ReadElements( + lm_output_tensor.get(), + ryzenai_corelib_data_type_bf16, + snapshot.logits.data(), + snapshot.logits.size(), + 0); + return snapshot; + } + + // See the header: this is what makes the LM-head localisation a + // measurement rather than an inference. One 3072-element BF16 row, read + // from the exact tensor the LM-head MatMul was given. + std::vector DebugLmHeadInput() const { + auto execution = runtime->AcquireExecution(); + std::vector row( + static_cast(constants::kHiddenSize)); + api->ReadElements( + lm_input_tensor.get(), + ryzenai_corelib_data_type_bf16, + row.data(), + row.size(), + 0); + return row; + } +#endif + + std::shared_ptr runtime; + std::shared_ptr api; + std::optional shape_plan; + std::shared_ptr package; + std::optional weights; + + std::span embedding; + std::vector input_norm; + std::vector embedding_fp32; + std::vector normalized_fp32; + std::vector fp32_staging; + std::vector padding_zero_staging; + std::vector v_staging; + std::vector last_hidden_staging; + + std::array hidden_tensors; + corelib::UniqueTensor residual_tensor; + corelib::UniqueTensor skip_sum_tensor; + corelib::UniqueTensor query_tensor; + corelib::UniqueTensor key_tensor; + corelib::UniqueTensor value_tensor; + corelib::UniqueTensor attention_tensor; + corelib::UniqueTensor lm_input_tensor; + corelib::UniqueTensor lm_output_tensor; + corelib::UniqueTensor cos_tensor; + corelib::UniqueTensor sin_tensor; + std::array< + corelib::UniqueTensor, + static_cast(constants::kLayerCount)> + key_caches; + std::array< + corelib::UniqueTensor, + static_cast(constants::kLayerCount)> + value_caches; + corelib::UniqueTensor* current_hidden = nullptr; + corelib::UniqueTensor* next_hidden = nullptr; + corelib::UniqueTensor* current_residual = nullptr; + corelib::UniqueTensor* next_skip_sum = nullptr; + + Phi4Aie4Metrics metrics; + VScatterMetrics v_metrics; + std::uint32_t max_length; + std::int64_t position = 0; + std::int64_t last_live_rows = 0; + std::optional checkpoint_position; + + // Declared last so constructor rollback also releases the Stream before + // tensors, weights, and package storage. + corelib::UniqueStream stream; +}; + +phi4_corelib_aie4::phi4_corelib_aie4( + LM_Config config, + std::filesystem::path model_path, + std::shared_ptr runtime, + std::uint32_t max_length) + : impl_(std::make_unique( + std::move(config), + std::move(model_path), + std::move(runtime), + max_length)) {} + +phi4_corelib_aie4::~phi4_corelib_aie4() = default; + +buffer phi4_corelib_aie4::forward(int id) { + const std::array token{id}; + impl_->ValidateTokens(token); + impl_->EnsureCapacity(token.size()); + impl_->EnsureDecodeWindow(token.size()); + return impl_->RunRows(token); +} + +buffer phi4_corelib_aie4::prefill( + std::vector& ids, + void* payload) { + (void)payload; + const std::span token_ids(ids); + impl_->ValidateTokens(token_ids); + impl_->EnsureCapacity(token_ids.size()); + // A prefill from position zero uses the PREFILL attention path, whose + // window goes to the full kMaxSequenceLength. Everything else here is a + // rows=1 step on the token path, which stops one short of that. + if (impl_->position == 0 && token_ids.size() != 1) { + return impl_->RunRows(token_ids); + } + if (token_ids.size() == 1) { + impl_->EnsureDecodeWindow(token_ids.size()); + return impl_->RunRows(token_ids); + } + + buffer logits; + for (const int token_id : token_ids) { + const std::array token{token_id}; + impl_->EnsureDecodeWindow(token.size()); + logits = impl_->RunRows(token); + } + return logits; +} + +void phi4_corelib_aie4::set_context_length(int length) { + if (length != impl_->position) { + throw std::invalid_argument( + "phi4_corelib_aie4 set_context_length accepts only " + "the current logical position"); + } +} + +void phi4_corelib_aie4::load_weights(Q4NX& q4nx) { + (void)q4nx; + throw std::runtime_error( + "Q4NX weight loading is unsupported by phi4_corelib_aie4"); +} + +void phi4_corelib_aie4::update_max_length( + std::uint32_t max_length) { + const std::uint32_t validated = + ValidateMaxLength(max_length); + if ( + validated < + static_cast(impl_->position)) { + throw std::out_of_range( + "Phi-4 AIE4 maximum length cannot be below the " + "current logical position"); + } + impl_->max_length = validated; +} + +void phi4_corelib_aie4::clear_context() { + impl_->position = 0; + impl_->last_live_rows = 0; + impl_->checkpoint_position.reset(); +} + +buffer phi4_corelib_aie4::get_k_cache( + int layer, + int index) { + (void)layer; + (void)index; + throw std::runtime_error( + "K-cache getters are unsupported by phi4_corelib_aie4"); +} + +buffer phi4_corelib_aie4::get_v_cache( + int layer, + int index) { + (void)layer; + (void)index; + throw std::runtime_error( + "V-cache getters are unsupported by phi4_corelib_aie4"); +} + +int phi4_corelib_aie4::get_current_context_length() { + return static_cast(impl_->position); +} + +int phi4_corelib_aie4::checkpoint() { + impl_->checkpoint_position = impl_->position; + return static_cast(impl_->position); +} + +int phi4_corelib_aie4::restore() { + if (!impl_->checkpoint_position.has_value()) { + throw std::logic_error( + "phi4_corelib_aie4 has no checkpoint to restore"); + } + impl_->position = *impl_->checkpoint_position; + impl_->last_live_rows = 0; + return static_cast(impl_->position); +} + +const Phi4Aie4Metrics& +phi4_corelib_aie4::metrics() const noexcept { + return impl_->metrics; +} + +#ifdef DEV_BUILD +Phi4DebugSnapshot phi4_corelib_aie4::debug_snapshot() const { + return impl_->DebugSnapshot(); +} + +std::vector +phi4_corelib_aie4::debug_lm_head_input() const { + return impl_->DebugLmHeadInput(); +} +#endif + +#if defined(FLM_CORELIB_TESTING) +namespace testing { + +[[noreturn]] void ApplyCorelibFailurePolicyForTest( + const std::shared_ptr& runtime, + const corelib::CorelibError& error, + bool synchronize_in_progress, + const corelib::StepSubmissionState& submission, + std::string phase, + std::optional layer, + std::int64_t rows, + std::int64_t position) { + if (RecoverableBeforeIrrevocableWork( + synchronize_in_progress, + submission)) { + throw error; + } + TerminateCorelibFailure( + runtime, + error, + std::move(phase), + layer, + rows, + position); +} + +} // namespace testing +#endif + +} // namespace flm::phi4 diff --git a/src/common/corelib/phi4_corelib_host.cpp b/src/common/corelib/phi4_corelib_host.cpp new file mode 100644 index 00000000..cdf04d77 --- /dev/null +++ b/src/common/corelib/phi4_corelib_host.cpp @@ -0,0 +1,289 @@ +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace flm::phi4 { +namespace { + +std::size_t CheckedExtent( + std::int64_t rows, + std::int64_t width, + std::string_view context) { + if (rows <= 0 || width <= 0) { + throw std::invalid_argument( + std::string(context) + + " rows and width must be positive"); + } + const auto row_count = static_cast(rows); + const auto column_count = static_cast(width); + if (column_count > + std::numeric_limits::max() / row_count) { + throw std::overflow_error( + std::string(context) + " extent overflows size_t"); + } + return static_cast(row_count * column_count); +} + +} // namespace + +void GatherEmbedding( + std::span embedding_fp16, + std::span token_ids, + std::span output) { + constexpr std::size_t width = + static_cast(constants::kHiddenSize); + if (embedding_fp16.empty() || + embedding_fp16.size() % width != 0) { + throw std::invalid_argument( + "Phi-4 embedding shape must be [vocabulary, 3072]"); + } + if (token_ids.size() > + std::numeric_limits::max() / width) { + throw std::overflow_error( + "Phi-4 embedding gather extent overflows size_t"); + } + const std::size_t output_count = token_ids.size() * width; + if (output.size() != output_count) { + throw std::invalid_argument( + "Phi-4 embedding output shape does not match token IDs"); + } + if (token_ids.empty()) { + return; + } + + // Validate every token before touching the mapping, so an out-of-range + // ID fails instead of reading a row that is not there. + const std::size_t vocabulary_rows = + embedding_fp16.size() / width; + for (const int token_id : token_ids) { + if (token_id < 0 || + static_cast(token_id) >= vocabulary_rows) { + throw std::out_of_range( + "Phi-4 embedding token ID is outside the mapped table"); + } + } + + // The gathered rows never reach a tensor before the host RMSNorm + // consumes them, so this is the `API-6` FP16-to-FP32 widening. It is + // scalar and per-element on purpose: the embedding table is a + // read-only file mapping, and a vectorized widening over-reads its + // source by up to 14 bytes, which faults on a page boundary. + for (std::size_t row = 0; row < token_ids.size(); ++row) { + const std::size_t source_offset = + static_cast(token_ids[row]) * width; + corelib::WidenFp16Array( + embedding_fp16.data() + source_offset, + width, + output.data() + row * width); + } +} + +void RmsNorm( + std::span input, + std::span scale, + std::int64_t rows, + std::int64_t width, + float epsilon, + std::span output) { + const std::size_t element_count = + CheckedExtent(rows, width, "Phi-4 RMSNorm"); + if (input.size() != element_count || + output.size() != element_count || + scale.size() != static_cast(width)) { + throw std::invalid_argument( + "Phi-4 RMSNorm shape mismatch"); + } + if (!std::isfinite(epsilon) || epsilon < 0.0f) { + throw std::invalid_argument( + "Phi-4 RMSNorm epsilon must be finite and nonnegative"); + } + + const std::size_t row_width = static_cast(width); + for (std::size_t row = 0; + row < static_cast(rows); + ++row) { + const std::size_t base = row * row_width; + // Accumulated in double, then rounded once. + // + // A serial FP32 sum of 3072 squares is the least accurate reduction + // available here, and the error is not academic: measured on the AIE4 + // target against the corelib reference driver, it moved the layer-0 + // input by up to 4.8e-6 relative, which is enough to land 25 of 58368 + // values on a DIFFERENT BF16 number once `tensor_write` narrows them. + // Thirty-two layers of BF16 arithmetic amplify those 25 seeds into a + // logit correlation of 0.9991 against the reference, below the 0.9999 + // design Section 12.4 requires, and into different sampled tokens + // within six steps. + // + // A double accumulator costs one row-length pass per forward pass -- + // this norm runs once per model step, not once per layer, because + // ssmlp absorbs every other one -- and brings the count of differing + // BF16 values to zero against the same reference. The rest of the + // computation stays in FP32 so the result is still exactly what + // design Section 10.2 specifies the host to produce. + double sum_of_squares = 0.0; + for (std::size_t column = 0; + column < row_width; + ++column) { + const double value = input[base + column]; + sum_of_squares += value * value; + } + const float mean_square = static_cast( + sum_of_squares / static_cast(width)); + const float denominator = + std::sqrt(mean_square + epsilon); + for (std::size_t column = 0; + column < row_width; + ++column) { + output[base + column] = + (input[base + column] / denominator) * + scale[column]; + } + } +} + +void StageFp32( + std::span input, + std::int64_t live_rows, + std::int64_t padded_rows, + std::int64_t width, + std::span output) { + if (padded_rows < live_rows) { + throw std::invalid_argument( + "Phi-4 FP32 staging padded rows are smaller than live rows"); + } + const std::size_t live_count = + CheckedExtent(live_rows, width, "Phi-4 FP32 staging"); + const std::size_t padded_count = + CheckedExtent(padded_rows, width, "Phi-4 FP32 staging"); + if (input.size() != live_count || output.size() < padded_count) { + throw std::invalid_argument( + "Phi-4 FP32 staging shape mismatch"); + } + + // Design Section 10.2: the host stays in FP32 and never produces BF16 + // activations. Corelib narrows FP32 to BF16 inside `tensor_write`, so + // there is exactly one BF16 rounding implementation on this path and + // FastFlow does not have to match it. + std::copy(input.begin(), input.end(), output.begin()); + std::fill( + output.begin() + live_count, + output.begin() + padded_count, + 0.0f); +} + +void ScatterV( + const corelib::CorelibApi& api, + ryzenai_corelib_tensor_ptr source, + ryzenai_corelib_tensor_ptr value_cache, + std::int64_t rows, + std::int64_t position, + std::vector& staging, + VScatterMetrics& metrics) { + if (source == nullptr || value_cache == nullptr) { + throw std::invalid_argument( + "Phi-4 V scatter requires source and cache tensors"); + } + if (rows <= 0 || + rows > constants::kMaxSequenceLength || + position < 0 || + position > + constants::kMaxSequenceLength - rows) { + throw std::out_of_range( + "Phi-4 V scatter live rows exceed the cache window"); + } + + constexpr std::size_t head_count = + static_cast(constants::kKvHeadCount); + constexpr std::size_t head_width = + static_cast(constants::kHeadSize); + constexpr std::size_t row_width = head_count * head_width; + const std::size_t live_rows = static_cast(rows); + const std::size_t source_count = live_rows * row_width; + const std::size_t head_staging_count = live_rows * head_width; + staging.resize(source_count + head_staging_count); + + const auto started = std::chrono::steady_clock::now(); + // `count` and `offset` are BF16 ELEMENTS of the cache's own dtype, not + // bytes (`API-7`). Both tensors are BF16 and so is the host staging + // buffer, so every transfer here is a straight copy. + api.ReadElements( + source, + ryzenai_corelib_data_type_bf16, + staging.data(), + source_count, + 0); + ++metrics.read_calls; + metrics.bytes += source_count * sizeof(std::uint16_t); + + std::uint16_t* const head_staging = + staging.data() + source_count; + for (std::size_t head = 0; head < head_count; ++head) { + for (std::size_t row = 0; row < live_rows; ++row) { + const std::size_t source_offset = + (row * head_count + head) * head_width; + std::copy_n( + staging.data() + source_offset, + head_width, + head_staging + row * head_width); + } + const std::size_t cache_offset = + ((head * + static_cast( + constants::kMaxSequenceLength)) + + static_cast(position)) * + head_width; + api.WriteElements( + value_cache, + ryzenai_corelib_data_type_bf16, + head_staging, + head_staging_count, + cache_offset); + ++metrics.write_calls; + metrics.bytes += head_staging_count * sizeof(std::uint16_t); + } + const auto elapsed = + std::chrono::duration_cast( + std::chrono::steady_clock::now() - started); + metrics.nanoseconds += + static_cast(elapsed.count()); +} + +int ArgmaxLowest( + std::span logits) { + if (logits.empty()) { + throw std::invalid_argument( + "Phi-4 argmax cannot consume empty logits"); + } + if (logits.size() > + static_cast( + std::numeric_limits::max())) { + throw std::overflow_error( + "Phi-4 argmax token index exceeds int"); + } + + int best_index = 0; + float best_value = static_cast(logits.front()); + for (std::size_t index = 1; index < logits.size(); ++index) { + const float value = static_cast(logits[index]); + if (value > best_value) { + best_value = value; + best_index = static_cast(index); + } + } + return best_index; +} + +} // namespace flm::phi4 diff --git a/src/common/corelib/phi4_corelib_manifest.cpp b/src/common/corelib/phi4_corelib_manifest.cpp new file mode 100644 index 00000000..68811c73 --- /dev/null +++ b/src/common/corelib/phi4_corelib_manifest.cpp @@ -0,0 +1,1508 @@ +#include +#include + +#include "../../pull/picosha2.h" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace flm::phi4 { +namespace { + +using nlohmann::json; + +constexpr std::string_view kManifestName = + "corelib_phi4_manifest.json"; +constexpr std::size_t kExpectedInitializers = 743; +constexpr std::size_t kExpectedWeightObjects = 161; +constexpr std::int64_t kRopeColumns = + constants::kRopeDimension / 2; + +[[noreturn]] void Throw( + std::string_view context, + std::string_view detail) { + throw std::runtime_error( + std::string(context) + ": " + std::string(detail)); +} + +[[noreturn]] void ThrowWin32( + std::string_view operation, + const std::filesystem::path& path, + DWORD error) { + std::ostringstream message; + message << operation << " failed for " << path.string() + << " (Win32 error " << error << ")"; + throw std::runtime_error(message.str()); +} + +void RequireExactKeys( + const json& value, + std::initializer_list expected, + std::string_view context) { + if (!value.is_object()) { + Throw(context, "must be an object"); + } + if (value.size() != expected.size()) { + Throw(context, "has an invalid field set"); + } + for (const std::string_view key : expected) { + if (!value.contains(std::string(key))) { + Throw( + context, + std::string("is missing field ") + std::string(key)); + } + } +} + +std::string ReadString( + const json& value, + std::string_view context) { + if (!value.is_string()) { + Throw(context, "must be a string"); + } + const std::string result = value.get(); + if (result.empty() || + result.find('\0') != std::string::npos) { + Throw(context, "must be a non-empty string"); + } + return result; +} + +std::uint64_t ReadU64( + const json& value, + std::string_view context) { + if (!value.is_number_unsigned()) { + Throw(context, "must be an unsigned integer"); + } + return value.get(); +} + +std::int64_t ReadPositiveI64( + const json& value, + std::string_view context) { + std::uint64_t unsigned_value = 0; + if (value.is_number_unsigned()) { + unsigned_value = value.get(); + } else if (value.is_number_integer()) { + const std::int64_t signed_value = value.get(); + if (signed_value <= 0) { + Throw(context, "must be a positive integer"); + } + return signed_value; + } else { + Throw(context, "must be a positive integer"); + } + if ( + unsigned_value == 0 || + unsigned_value > + static_cast( + std::numeric_limits::max())) { + Throw(context, "must fit a positive int64"); + } + return static_cast(unsigned_value); +} + +void RequireInteger( + const json& value, + std::int64_t expected, + std::string_view context) { + if (ReadPositiveI64(value, context) != expected) { + Throw(context, "does not match the Phi-4 model identity"); + } +} + +bool ReadBool( + const json& value, + std::string_view context) { + if (!value.is_boolean()) { + Throw(context, "must be a boolean"); + } + return value.get(); +} + +std::filesystem::path ParseRelativePath( + std::string_view raw, + std::string_view context) { + if (raw.empty() || + raw.find('\0') != std::string_view::npos) { + Throw(context, "file path must not be empty"); + } + + const auto path = std::filesystem::u8path( + raw.begin(), + raw.end()); + if ( + path.is_absolute() || path.has_root_name() || + path.has_root_directory()) { + Throw(context, "file path must be relative"); + } + + bool has_component = false; + for (const auto& component : path) { + if (component == "..") { + Throw(context, "file path contains parent traversal"); + } + if (component == "." || component.empty()) { + continue; + } + has_component = true; + } + if (!has_component) { + Throw(context, "file path must name a file"); + } + return path.lexically_normal(); +} + +bool IsWithin( + const std::filesystem::path& root, + const std::filesystem::path& candidate) { + const auto relative = candidate.lexically_relative(root); + if (relative.empty() || relative.is_absolute()) { + return false; + } + const auto first = relative.begin(); + return first != relative.end() && *first != ".."; +} + +std::filesystem::path ResolvePackageFile( + const std::filesystem::path& root, + std::string_view raw, + std::string_view context) { + const auto relative = ParseRelativePath(raw, context); + std::error_code error; + const auto resolved = + std::filesystem::canonical(root / relative, error); + if (error) { + Throw( + context, + std::string("package file does not exist: ") + + std::string(raw)); + } + if (!IsWithin(root, resolved)) { + Throw(context, "package file escapes the model directory"); + } + if (!std::filesystem::is_regular_file(resolved, error) || error) { + Throw(context, "package path is not a regular file"); + } + return resolved; +} + +std::string CalculateSha256( + const std::filesystem::path& path) { + std::ifstream stream(path, std::ios::binary); + if (!stream) { + Throw("SHA-256", "failed to open package file"); + } + std::array digest{}; + picosha2::hash256(stream, digest.begin(), digest.end()); + if (stream.bad()) { + Throw("SHA-256", "failed while reading package file"); + } + return picosha2::bytes_to_hex_string( + digest.begin(), + digest.end()); +} + +bool IsSha256(std::string_view value) { + return value.size() == picosha2::k_digest_size * 2 && + std::all_of( + value.begin(), + value.end(), + [](unsigned char character) { + return std::isxdigit(character) != 0; + }); +} + +std::string NormalizeSha256(std::string value) { + std::transform( + value.begin(), + value.end(), + value.begin(), + [](unsigned char character) { + return static_cast(std::tolower(character)); + }); + return value; +} + +SourceDType ParseDType( + std::string_view value, + std::string_view context) { + if (value == "uint8") { + return SourceDType::UInt8; + } + if (value == "float16") { + return SourceDType::Float16; + } + if (value == "float32") { + return SourceDType::Float32; + } + if (value == "int64") { + return SourceDType::Int64; + } + Throw(context, "has an unsupported dtype"); +} + +std::uint64_t ItemSize(SourceDType dtype) { + switch (dtype) { + case SourceDType::UInt8: + return 1; + case SourceDType::Float16: + return 2; + case SourceDType::Float32: + return 4; + case SourceDType::Int64: + return 8; + } + throw std::logic_error("unreachable SourceDType"); +} + +std::vector ParseShape( + const json& value, + std::string_view context) { + if (!value.is_array() || value.empty()) { + Throw(context, "shape must be a non-empty array"); + } + std::vector shape; + shape.reserve(value.size()); + for (const auto& dimension : value) { + shape.push_back(ReadPositiveI64(dimension, context)); + } + return shape; +} + +std::uint64_t CheckedByteCount( + const std::vector& shape, + SourceDType dtype, + std::string_view context) { + std::uint64_t elements = 1; + for (const std::int64_t dimension : shape) { + const auto unsigned_dimension = + static_cast(dimension); + if ( + elements > + std::numeric_limits::max() / + unsigned_dimension) { + Throw(context, "shape element count overflow"); + } + elements *= unsigned_dimension; + } + const std::uint64_t item_size = ItemSize(dtype); + if ( + elements > + std::numeric_limits::max() / item_size) { + Throw(context, "tensor byte count overflow"); + } + return elements * item_size; +} + +void RequireShape( + const InitializerView& view, + std::initializer_list expected, + std::string_view context) { + if (!std::equal( + view.shape.begin(), + view.shape.end(), + expected.begin(), + expected.end())) { + Throw(context, "has an invalid shape"); + } +} + +void RequireOneOfShapes( + const InitializerView& view, + const std::vector>& expected, + std::string_view context) { + if ( + std::none_of( + expected.begin(), + expected.end(), + [&](const std::vector& shape) { + return view.shape == shape; + })) { + Throw(context, "has an invalid shape"); + } +} + +void RequireFloating( + const InitializerView& view, + std::string_view context) { + if ( + view.dtype != SourceDType::Float16 && + view.dtype != SourceDType::Float32) { + Throw(context, "must use a floating FP16 or FP32 source"); + } +} + +void RequireSemanticRole( + const std::map>& roles, + std::string_view initializer, + std::string_view expected) { + const auto found = roles.find(initializer); + if (found == roles.end() || found->second != expected) { + Throw( + initializer, + std::string("semantic role does not match ") + + std::string(expected)); + } +} + +struct ExpectedWeightObject { + std::string name; + WeightObjectKind kind; + std::int64_t k; + std::int64_t n; + std::map components; +}; + +std::map ExpectedMatMulComponents( + const std::string& object_name) { + return { + {"qweight", object_name + ".qweight"}, + {"scales", object_name + ".scales"}, + {"qzeros", object_name + ".qzeros"}}; +} + +std::map ExpectedSsMlpComponents( + std::int64_t layer) { + const std::string base = + "model.layers." + std::to_string(layer); + std::map components{ + {"norm0", base + ".post_attention_layernorm.weight"}, + {"norm1", + layer + 1 == constants::kLayerCount + ? "model.layers.32.final_norm_layernorm.weight" + : "model.layers." + std::to_string(layer + 1) + + ".input_layernorm.weight"}}; + for (const std::string projection : {"gate", "up", "down"}) { + const std::string initializer = + base + ".mlp." + projection + "_proj.MatMulNBits."; + components.emplace( + projection + "_qweight", + initializer + "qweight"); + components.emplace( + projection + "_scales", + initializer + "scales"); + components.emplace( + projection + "_qzeros", + initializer + "qzeros"); + } + return components; +} + +const std::vector& ExpectedWeightObjects() { + static const std::vector expected = [] { + std::vector values; + values.reserve(kExpectedWeightObjects); + const auto add_matmul = [&]( + std::string name, + std::int64_t k, + std::int64_t n) { + auto components = ExpectedMatMulComponents(name); + values.push_back({ + std::move(name), + WeightObjectKind::MatMul, + k, + n, + std::move(components)}); + }; + for (std::int64_t layer = 0; + layer < constants::kLayerCount; + ++layer) { + const std::string base = + "model.layers." + std::to_string(layer) + ".attn."; + add_matmul( + base + "q_proj.MatMulNBits", + constants::kHiddenSize, + constants::kQueryDimension); + add_matmul( + base + "k_proj.MatMulNBits", + constants::kHiddenSize, + constants::kKvDimension); + add_matmul( + base + "v_proj.MatMulNBits", + constants::kHiddenSize, + constants::kKvDimension); + add_matmul( + base + "o_proj.MatMulNBits", + constants::kQueryDimension, + constants::kHiddenSize); + values.push_back({ + "model.layers." + std::to_string(layer) + ".ssmlp", + WeightObjectKind::SsMlp, + constants::kHiddenSize, + constants::kIntermediateSize, + ExpectedSsMlpComponents(layer)}); + } + add_matmul( + "lm_head.MatMulNBits", + constants::kHiddenSize, + constants::kVocabularySize); + return values; + }(); + return expected; +} + +std::set MatMulRoleNames() { + return {"qweight", "scales", "qzeros"}; +} + +std::set SsMlpRoleNames() { + return { + "norm0", + "norm1", + "gate_qweight", + "gate_scales", + "gate_qzeros", + "up_qweight", + "up_scales", + "up_qzeros", + "down_qweight", + "down_scales", + "down_qzeros"}; +} + +std::set JsonKeys(const json& value) { + std::set keys; + for (const auto& [key, _] : value.items()) { + keys.insert(key); + } + return keys; +} + +std::string ComponentContext( + std::string_view object_name, + std::string_view role_name) { + return std::string(object_name) + "." + std::string(role_name); +} + +std::string ComponentContext( + std::string_view object_name, + std::string_view role_name, + std::string_view initializer_name) { + return ComponentContext(object_name, role_name) + " (" + + std::string(initializer_name) + ")"; +} + +void ValidateQuantizedProjection( + const Phi4Package& package, + std::string_view object_name, + const std::map>& semantic_roles, + const std::map& components, + std::string_view role_prefix, + std::string_view component_prefix, + std::int64_t k, + std::int64_t n) { + if (k <= 0 || n <= 0 || k % 2 != 0 || + k % static_cast( + constants::kGroupSize) != 0) { + Throw(role_prefix, "has an invalid quantized descriptor"); + } + const std::int64_t groups = + k / static_cast( + constants::kGroupSize); + + const auto validate = [&]( + std::string_view component, + SourceDType dtype, + std::vector> shapes, + std::string semantic) { + const std::string role_name = + std::string(component_prefix) + std::string(component); + const auto found = components.find(role_name); + if (found == components.end()) { + Throw( + ComponentContext(object_name, role_name), + "is missing a component role"); + } + const auto& view = package.Require(found->second); + const std::string context = + ComponentContext(object_name, role_name, found->second); + if (view.dtype != dtype) { + Throw(context, "has an invalid dtype"); + } + RequireOneOfShapes(view, shapes, context); + RequireSemanticRole( + semantic_roles, + found->second, + semantic); + }; + + validate( + "qweight", + SourceDType::UInt8, + { + {n, k / 2}, + { + n, + groups, + static_cast( + constants::kGroupSize / 2)}}, + std::string(role_prefix) + ".qweight"); + + const std::string scales_name = + std::string(component_prefix) + "scales"; + const auto scales_component = components.find(scales_name); + if (scales_component == components.end()) { + Throw( + ComponentContext(object_name, scales_name), + "is missing a scales role"); + } + const auto& scales = package.Require(scales_component->second); + const std::string scales_context = ComponentContext( + object_name, + scales_name, + scales_component->second); + RequireFloating(scales, scales_context); + RequireOneOfShapes( + scales, + {{n, groups}, {n * groups}}, + scales_context); + RequireSemanticRole( + semantic_roles, + scales_component->second, + std::string(role_prefix) + ".scales"); + + validate( + "qzeros", + SourceDType::UInt8, + { + {n, (groups + 1) / 2}, + {n * ((groups + 1) / 2)}}, + std::string(role_prefix) + ".qzeros"); +} + +void ValidateHostInitializers( + const Phi4Package& package, + const std::map>& semantic_roles) { + const auto& embedding = + package.Require("model.embed_tokens.weight"); + if (embedding.dtype != SourceDType::Float16) { + Throw("embedding", "must use an FP16 source"); + } + RequireShape( + embedding, + {constants::kVocabularySize, constants::kHiddenSize}, + "embedding"); + RequireSemanticRole( + semantic_roles, + "model.embed_tokens.weight", + "embedding"); + + const auto& input_norm = + package.Require("model.layers.0.input_layernorm.weight"); + RequireFloating(input_norm, "input_norm"); + RequireShape( + input_norm, + {constants::kHiddenSize}, + "input_norm"); + RequireSemanticRole( + semantic_roles, + "model.layers.0.input_layernorm.weight", + "input_norm"); + + for (const std::string_view name : {"cos_cache", "sin_cache"}) { + const auto& rope = package.Require(name); + RequireFloating(rope, name); + if ( + rope.shape.size() != 2 || + rope.shape[0] < constants::kMaxSequenceLength || + rope.shape[1] < kRopeColumns) { + Throw(name, "must be rank 2 and at least [4096,48]"); + } + RequireSemanticRole(semantic_roles, name, name); + } +} + +ryzenai_corelib_data_type CorelibDType( + SourceDType dtype, + std::string_view context) { + switch (dtype) { + case SourceDType::Float16: + return ryzenai_corelib_data_type_fp16; + case SourceDType::Float32: + return ryzenai_corelib_data_type_fp32; + case SourceDType::UInt8: + case SourceDType::Int64: + Throw(context, "requires a floating FP16 or FP32 source"); + } + throw std::logic_error("unreachable SourceDType"); +} + +std::size_t ElementCount( + const InitializerView& view, + std::string_view context) { + const std::uint64_t count = + static_cast(view.size) / ItemSize(view.dtype); + if ( + count > + static_cast( + std::numeric_limits::max())) { + Throw(context, "element count exceeds addressable memory"); + } + return static_cast(count); +} + +void ValidateModelIdentity(const json& manifest) { + if ( + manifest.is_object() && + !manifest.contains("weight_objects")) { + Throw("weight_objects", "section is missing"); + } + RequireExactKeys( + manifest, + { + "schema_version", + "execution_backend", + "model", + "backend", + "files", + "initializers", + "weight_objects", + }, + "manifest"); + RequireInteger( + manifest.at("schema_version"), + 1, + "schema_version"); + if ( + ReadString( + manifest.at("execution_backend"), + "execution_backend") != "corelib_aie4") { + Throw( + "execution_backend", + "does not match corelib_aie4"); + } + + const auto& model = manifest.at("model"); + RequireExactKeys( + model, + { + "family", + "layers", + "hidden_size", + "intermediate_size", + "num_heads", + "kv_heads", + "head_size", + "vocab_size", + "group_size", + "rope_dim", + "rms_epsilon", + }, + "model"); + if (ReadString(model.at("family"), "model.family") != "phi4") { + Throw("model.family", "does not match phi4"); + } + RequireInteger( + model.at("layers"), + constants::kLayerCount, + "model.layers"); + RequireInteger( + model.at("hidden_size"), + constants::kHiddenSize, + "model.hidden_size"); + RequireInteger( + model.at("intermediate_size"), + constants::kIntermediateSize, + "model.intermediate_size"); + RequireInteger( + model.at("num_heads"), + constants::kQueryHeadCount, + "model.num_heads"); + RequireInteger( + model.at("kv_heads"), + constants::kKvHeadCount, + "model.kv_heads"); + RequireInteger( + model.at("head_size"), + constants::kHeadSize, + "model.head_size"); + RequireInteger( + model.at("vocab_size"), + constants::kVocabularySize, + "model.vocab_size"); + RequireInteger( + model.at("group_size"), + constants::kGroupSize, + "model.group_size"); + RequireInteger( + model.at("rope_dim"), + constants::kRopeDimension, + "model.rope_dim"); + if ( + !model.at("rms_epsilon").is_number() || + model.at("rms_epsilon").get() != + constants::kRmsEpsilon) { + Throw( + "model.rms_epsilon", + "does not match the Phi-4 model identity"); + } + + const auto& backend = manifest.at("backend"); + RequireExactKeys(backend, {"max_seq"}, "backend"); + RequireInteger( + backend.at("max_seq"), + constants::kMaxSequenceLength, + "backend.max_seq"); +} + +} // namespace + +MappedFile::MappedFile( + std::filesystem::path path, + void* file, + void* mapping, + const std::byte* data, + std::uint64_t size) noexcept + : path_(std::move(path)), + file_(file), + mapping_(mapping), + data_(data), + size_(size) {} + +std::shared_ptr MappedFile::OpenReadOnly( + const std::filesystem::path& path) { + // Copy before acquiring Win32 resources so an allocation failure cannot + // strand handles that have not yet been transferred to MappedFile. + std::filesystem::path owned_path = path; + HANDLE file = CreateFileW( + path.c_str(), + GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_DELETE, + nullptr, + OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, + nullptr); + if (file == INVALID_HANDLE_VALUE) { + ThrowWin32("CreateFileW", path, GetLastError()); + } + + LARGE_INTEGER raw_size{}; + if (GetFileSizeEx(file, &raw_size) == FALSE) { + const DWORD error = GetLastError(); + CloseHandle(file); + ThrowWin32("GetFileSizeEx", path, error); + } + if (raw_size.QuadPart <= 0) { + CloseHandle(file); + Throw(path.string(), "mapped file must not be empty"); + } + const auto size = static_cast(raw_size.QuadPart); + + HANDLE mapping = CreateFileMappingW( + file, + nullptr, + PAGE_READONLY, + 0, + 0, + nullptr); + if (mapping == nullptr) { + const DWORD error = GetLastError(); + CloseHandle(file); + ThrowWin32("CreateFileMappingW", path, error); + } + + const void* raw_data = + MapViewOfFile(mapping, FILE_MAP_READ, 0, 0, 0); + if (raw_data == nullptr) { + const DWORD error = GetLastError(); + CloseHandle(mapping); + CloseHandle(file); + ThrowWin32("MapViewOfFile", path, error); + } + + std::unique_ptr owner; + try { + owner.reset(new MappedFile( + std::move(owned_path), + file, + mapping, + static_cast(raw_data), + size)); + } catch (...) { + UnmapViewOfFile(raw_data); + CloseHandle(mapping); + CloseHandle(file); + throw; + } + return std::shared_ptr(std::move(owner)); +} + +MappedFile::MappedFile(MappedFile&& other) noexcept + : path_(std::move(other.path_)), + file_(std::exchange(other.file_, nullptr)), + mapping_(std::exchange(other.mapping_, nullptr)), + data_(std::exchange(other.data_, nullptr)), + size_(std::exchange(other.size_, 0)) {} + +MappedFile& MappedFile::operator=(MappedFile&& other) noexcept { + if (this != &other) { + Reset(); + path_ = std::move(other.path_); + file_ = std::exchange(other.file_, nullptr); + mapping_ = std::exchange(other.mapping_, nullptr); + data_ = std::exchange(other.data_, nullptr); + size_ = std::exchange(other.size_, 0); + } + return *this; +} + +MappedFile::~MappedFile() noexcept { + Reset(); +} + +void MappedFile::Reset() noexcept { + if (data_ != nullptr) { + UnmapViewOfFile(data_); + data_ = nullptr; + } + if (mapping_ != nullptr) { + CloseHandle(static_cast(mapping_)); + mapping_ = nullptr; + } + if (file_ != nullptr) { + CloseHandle(static_cast(file_)); + file_ = nullptr; + } + size_ = 0; +} + +const std::byte* MappedFile::data() const noexcept { + return data_; +} + +std::uint64_t MappedFile::size() const noexcept { + return size_; +} + +const std::filesystem::path& MappedFile::path() const noexcept { + return path_; +} + +Phi4Package Phi4Package::Load( + const std::filesystem::path& model_dir, + std::shared_ptr api, + bool verify_full_hash) { + if (!api) { + throw std::invalid_argument( + "Phi4Package::Load requires a CorelibApi"); + } + + std::error_code error; + const auto root = std::filesystem::canonical(model_dir, error); + if ( + error || !std::filesystem::is_directory(root, error) || + error) { + throw std::runtime_error( + "Phi4 model directory does not exist or is not a directory"); + } + + std::ifstream manifest_stream( + root / kManifestName, + std::ios::binary); + if (!manifest_stream) { + throw std::runtime_error( + "Phi-4 package is missing corelib_phi4_manifest.json"); + } + + json manifest; + try { + manifest_stream >> manifest; + } catch (const json::exception& parse_error) { + throw std::runtime_error( + std::string("failed to parse corelib_phi4_manifest.json: ") + + parse_error.what()); + } + ValidateModelIdentity(manifest); + + Phi4Package package; + package.api_ = std::move(api); + + const auto& files = manifest.at("files"); + if (!files.is_object() || files.empty()) { + Throw("files", "must be a non-empty object"); + } + if (!files.contains("model.onnx")) { + Throw("files", "is missing model.onnx"); + } + for (const auto& [name, record] : files.items()) { + const std::string context = "files." + name; + if (!record.is_object()) { + Throw(context, "must be an object"); + } + const bool has_hash = record.contains("sha256"); + RequireExactKeys( + record, + has_hash + ? std::initializer_list{ + "size", + "sha256"} + : std::initializer_list{"size"}, + context); + + const std::uint64_t expected_size = + ReadU64(record.at("size"), context + ".size"); + if (expected_size == 0) { + Throw(context, "file size must be positive"); + } + const auto resolved = + ResolvePackageFile(root, name, context); + auto mapped = MappedFile::OpenReadOnly(resolved); + if (mapped->size() != expected_size) { + Throw(context, "file size does not match the manifest"); + } + + if (has_hash) { + std::string expected_hash = ReadString( + record.at("sha256"), + context + ".sha256"); + if (!IsSha256(expected_hash)) { + Throw(context, "SHA-256 must contain 64 hexadecimal digits"); + } + expected_hash = NormalizeSha256(std::move(expected_hash)); + if ( + verify_full_hash && + CalculateSha256(resolved) != expected_hash) { + Throw(context, "SHA-256 does not match the package file"); + } + } else if (verify_full_hash) { + Throw(context, "SHA-256 is required for full verification"); + } + + package.mapped_files_.emplace(name, std::move(mapped)); + } + + const auto& initializer_records = manifest.at("initializers"); + if ( + !initializer_records.is_object() || + initializer_records.size() != kExpectedInitializers) { + Throw( + "initializers", + "must contain exactly 743 initializer records"); + } + + std::map> + semantic_roles; + for (const auto& [name, record] : initializer_records.items()) { + const std::string context = "initializers." + name; + RequireExactKeys( + record, + {"file", "offset", "length", "dtype", "shape", "role"}, + context); + const std::string file_name = + ReadString(record.at("file"), context + ".file"); + (void)ParseRelativePath(file_name, context + ".file"); + + const std::uint64_t offset = + ReadU64(record.at("offset"), context + ".offset"); + const std::uint64_t length = + ReadU64(record.at("length"), context + ".length"); + if (length == 0) { + Throw(context, "initializer length must be positive"); + } + if ( + offset > + std::numeric_limits::max() - length) { + Throw(context, "initializer range overflow"); + } + + const auto owner = + package.mapped_files_.find(file_name); + if (owner == package.mapped_files_.end()) { + Throw(context, "initializer references an unlisted file"); + } + if ( + offset > owner->second->size() || + length > owner->second->size() - offset) { + Throw(context, "initializer range exceeds file size"); + } + + const std::string dtype_name = + ReadString(record.at("dtype"), context + ".dtype"); + const SourceDType dtype = + ParseDType(dtype_name, context + ".dtype"); + if (offset % ItemSize(dtype) != 0) { + Throw(context, "initializer offset is not dtype-aligned"); + } + const auto shape = + ParseShape(record.at("shape"), context + ".shape"); + if (CheckedByteCount(shape, dtype, context) != length) { + Throw( + context, + "dtype/shape byte count does not match length"); + } + if ( + length > + static_cast( + std::numeric_limits::max()) || + offset > + static_cast( + std::numeric_limits::max())) { + Throw(context, "initializer range is not addressable"); + } + + const std::string semantic_role = + ReadString(record.at("role"), context + ".role"); + semantic_roles.emplace(name, semantic_role); + package.initializers_.emplace( + name, + InitializerView{ + dtype, + shape, + owner->second->data() + + static_cast(offset), + static_cast(length), + owner->second}); + } + + ValidateHostInitializers(package, semantic_roles); + + if (!manifest.contains("weight_objects")) { + Throw("weight_objects", "section is missing"); + } + const auto& object_records = manifest.at("weight_objects"); + if ( + !object_records.is_array() || + object_records.size() != kExpectedWeightObjects) { + Throw( + "weight_objects", + "must be a non-empty list of exactly 161 entries"); + } + + const auto& expected_objects = ExpectedWeightObjects(); + std::set object_names; + std::set referenced_initializers; + package.weight_objects_.reserve(kExpectedWeightObjects); + for (std::size_t index = 0; index < object_records.size(); ++index) { + const auto& record = object_records.at(index); + const std::string index_context = + "weight_objects[" + std::to_string(index) + "]"; + RequireExactKeys( + record, + {"name", "kind", "descriptor", "roles"}, + index_context); + + const std::string name = + ReadString(record.at("name"), index_context + ".name"); + if (!object_names.insert(name).second) { + Throw(name, "duplicate weight object name"); + } + + const std::string kind_name = + ReadString(record.at("kind"), name + ".kind"); + WeightObjectKind kind; + if (kind_name == "matmul") { + kind = WeightObjectKind::MatMul; + } else if (kind_name == "ssmlp") { + kind = WeightObjectKind::SsMlp; + } else { + Throw(name, "invalid weight object kind"); + } + + const auto& expected = expected_objects.at(index); + if (name != expected.name) { + Throw( + name, + "weight object name/order does not match the fixed " + "Phi-4 plan"); + } + if (kind != expected.kind) { + Throw( + name, + "weight object kind does not match the fixed Phi-4 plan"); + } + + const auto& descriptor = record.at("descriptor"); + RequireExactKeys( + descriptor, + kind == WeightObjectKind::MatMul + ? std::initializer_list{ + "k", + "n", + "group_size", + "has_bias"} + : std::initializer_list{ + "k", + "n", + "group_size"}, + name + ".descriptor"); + const std::int64_t k = + ReadPositiveI64(descriptor.at("k"), name + ".descriptor.k"); + const std::int64_t n = + ReadPositiveI64(descriptor.at("n"), name + ".descriptor.n"); + const std::int64_t raw_group_size = ReadPositiveI64( + descriptor.at("group_size"), + name + ".descriptor.group_size"); + if ( + raw_group_size > + static_cast( + std::numeric_limits::max())) { + Throw(name + ".descriptor", "group_size exceeds uint32"); + } + const auto group_size = + static_cast(raw_group_size); + bool has_bias = false; + if (kind == WeightObjectKind::MatMul) { + has_bias = ReadBool( + descriptor.at("has_bias"), + name + ".descriptor.has_bias"); + if (has_bias) { + Throw( + name + ".descriptor", + "MatMul has_bias must be false"); + } + } + if ( + k != expected.k || n != expected.n || + group_size != constants::kGroupSize) { + Throw( + name + ".descriptor", + "does not match the fixed Phi-4 descriptor"); + } + + const auto& role_map = record.at("roles"); + if (!role_map.is_object()) { + Throw(name, "weight object role map must be an object"); + } + const auto expected_roles = + kind == WeightObjectKind::MatMul + ? MatMulRoleNames() + : SsMlpRoleNames(); + for (const auto& role : expected_roles) { + if (!role_map.contains(role)) { + Throw( + ComponentContext(name, role), + "is missing a component role"); + } + } + if (JsonKeys(role_map) != expected_roles) { + Throw(name, "invalid weight object role map"); + } + + std::map components; + std::set local_references; + for (const auto& [role, initializer] : role_map.items()) { + const std::string initializer_name = ReadString( + initializer, + name + ".roles." + role); + if (!local_references.insert(initializer_name).second) { + Throw(name, "duplicate initializer role reference"); + } + if (!package.initializers_.contains(initializer_name)) { + Throw( + name, + "unresolved initializer " + initializer_name); + } + if (!referenced_initializers.insert(initializer_name).second) { + Throw( + name, + "duplicate initializer reference across weight objects"); + } + components.emplace(role, initializer_name); + } + for (const auto& [role, expected_initializer] : + expected.components) { + const auto component = components.find(role); + if ( + component == components.end() || + component->second != expected_initializer) { + Throw( + component == components.end() + ? ComponentContext(name, role) + : ComponentContext( + name, + role, + component->second), + "does not match fixed Phi-4 initializer " + + expected_initializer); + } + } + + if (kind == WeightObjectKind::MatMul) { + ValidateQuantizedProjection( + package, + name, + semantic_roles, + components, + "matmul", + "", + k, + n); + } else { + const auto validate_norm = [&]( + std::string_view role, + std::string_view semantic) { + const auto component = + components.find(std::string(role)); + const auto& view = + package.Require(component->second); + const std::string context = ComponentContext( + name, + role, + component->second); + RequireFloating(view, context); + RequireShape( + view, + {constants::kHiddenSize}, + context); + RequireSemanticRole( + semantic_roles, + component->second, + semantic); + }; + validate_norm("norm0", "ssmlp.norm0"); + validate_norm("norm1", "ssmlp.norm1"); + ValidateQuantizedProjection( + package, + name, + semantic_roles, + components, + "ssmlp.gate", + "gate_", + k, + n); + ValidateQuantizedProjection( + package, + name, + semantic_roles, + components, + "ssmlp.up", + "up_", + k, + n); + ValidateQuantizedProjection( + package, + name, + semantic_roles, + components, + "ssmlp.down", + "down_", + n, + k); + } + + package.weight_objects_.push_back({ + name, + kind, + k, + n, + group_size, + has_bias, + std::move(components)}); + } + + const std::set host_initializers{ + "model.embed_tokens.weight", + "model.layers.0.input_layernorm.weight", + "cos_cache", + "sin_cache"}; + for (const auto& [name, _] : package.initializers_) { + const bool is_host = host_initializers.contains(name); + const bool is_referenced = + referenced_initializers.contains(name); + if (is_host == is_referenced) { + Throw( + name, + is_host + ? "host initializer must not belong to a weight object" + : "initializer is not referenced by weight_objects"); + } + } + + return package; +} + +const InitializerView& Phi4Package::Require( + std::string_view name) const { + const auto found = initializers_.find(name); + if (found == initializers_.end()) { + throw std::runtime_error( + "missing initializer: " + std::string(name)); + } + return found->second; +} + +const std::vector& +Phi4Package::weight_objects() const noexcept { + return weight_objects_; +} + +std::span Phi4Package::MaterializeFp16( + std::string_view name) { + if (const auto found = fp16_buffers_.find(name); + found != fp16_buffers_.end()) { + return found->second; + } + + const auto& source = Require(name); + // Design Section 9.3: the ONNX-component contract specifies FP16 + // scales, and narrowing an FP32 source would require a second host + // rounding converter that `API-6` does not permit. Reject rather than + // convert, and say what the package must contain. + if (source.dtype != SourceDType::Float16) { + Throw( + name, + "MatMulNBits scales must be FP16 in an accepted AIE4 " + "package; an FP32 scales array is rejected rather than " + "narrowed, so repackage the model with FP16 scales"); + } + const std::size_t count = ElementCount(source, name); + auto [buffer, inserted] = + fp16_buffers_.try_emplace(std::string(name), count); + try { + // An element-wise copy, not a conversion: this exists so that a + // strided or non-contiguous source view still yields the + // contiguous model-owned buffer `WEIGHT-2` requires. + const auto* elements = + reinterpret_cast(source.data); + std::copy_n(elements, count, buffer->second.begin()); + } catch (...) { + if (inserted) { + fp16_buffers_.erase(buffer); + } + throw; + } + return buffer->second; +} + +std::span Phi4Package::MaterializeBf16( + std::string_view name) { + if (const auto found = bf16_buffers_.find(name); + found != bf16_buffers_.end()) { + return found->second; + } + + const auto& source = Require(name); + const auto source_type = CorelibDType(source.dtype, name); + const std::size_t count = ElementCount(source, name); + auto [buffer, inserted] = + bf16_buffers_.try_emplace(std::string(name), count); + try { + // SSMLP norms are raw BF16 blobs handed to the packer, with no + // tensor boundary to convert through, so this is the `API-6` + // FP32-to-BF16 helper. An FP16 source is first widened losslessly, + // which composes the two permitted conversions rather than adding + // a third. + auto* destination = buffer->second.data(); + if (source_type == ryzenai_corelib_data_type_fp32) { + const auto* elements = + reinterpret_cast(source.data); + corelib::NarrowFp32ToBf16Array( + elements, + count, + destination); + } else { + const auto* elements = + reinterpret_cast(source.data); + for (std::size_t index = 0; index < count; ++index) { + destination[index] = corelib::NarrowFp32ToBf16( + corelib::WidenFp16(elements[index])); + } + } + } catch (...) { + if (inserted) { + bf16_buffers_.erase(buffer); + } + throw; + } + return buffer->second; +} + +RopeSourceView Phi4Package::MaterializeRopeGather( + std::string_view name) { + const auto& source = Require(name); + const auto source_type = CorelibDType(source.dtype, name); + if ( + source.shape.size() != 2 || + source.shape[0] < constants::kMaxSequenceLength || + source.shape[1] < kRopeColumns) { + Throw( + name, + "RoPE source must be rank 2 and at least [4096,48]"); + } + + constexpr std::size_t rows = + static_cast(constants::kMaxSequenceLength); + constexpr std::size_t columns = + static_cast(kRopeColumns); + constexpr std::size_t count = rows * columns; + + if (const auto found = rope_buffers_.find(name); + found != rope_buffers_.end()) { + return RopeSourceView{ + source_type, + found->second.data(), + count}; + } + + const std::size_t item_size = ItemSize(source.dtype); + const auto source_columns = + static_cast(source.shape[1]); + const std::size_t source_row_bytes = source_columns * item_size; + const std::size_t gathered_row_bytes = columns * item_size; + + // Corelib `e5258d2` removed `convert_strided`, so the slice is + // FastFlow's own element-wise copy. It stays in the SOURCE dtype and + // lets `tensor_write` widen to FP32: the RoPE tables have a tensor to + // write into, so this path must not use the `API-6` widening helper. + // + // Every row is bounds-checked against the mapped extent before it is + // read, and only [row_start, row_start + 48) of each row is touched. + // The source is a read-only file mapping where a tail over-read faults + // rather than returning garbage, and the final row commonly sits + // immediately before an inaccessible page. + auto [buffer, inserted] = + rope_buffers_.try_emplace( + std::string(name), + count * item_size); + try { + const auto* const base = source.data; + auto* destination = buffer->second.data(); + for (std::size_t row = 0; row < rows; ++row) { + const std::size_t row_start = row * source_row_bytes; + if ( + row_start > source.size || + gathered_row_bytes > source.size - row_start) { + Throw( + name, + "RoPE gather row exceeds the mapped initializer " + "extent"); + } + std::memcpy( + destination + row * gathered_row_bytes, + base + row_start, + gathered_row_bytes); + } + } catch (...) { + if (inserted) { + rope_buffers_.erase(buffer); + } + throw; + } + return RopeSourceView{ + source_type, + buffer->second.data(), + count}; +} + +} // namespace flm::phi4 diff --git a/src/common/corelib/phi4_corelib_shape_plan.cpp b/src/common/corelib/phi4_corelib_shape_plan.cpp new file mode 100644 index 00000000..b199d310 --- /dev/null +++ b/src/common/corelib/phi4_corelib_shape_plan.cpp @@ -0,0 +1,253 @@ +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace flm::phi4 { +namespace { + +constexpr std::int64_t kShapePrecomputeMaxLiveRows = 4096; +static_assert( + kShapePrecomputeMaxLiveRows == + constants::kMaxSequenceLength, + "Phi4ShapePlan v1 queries every live row through the " + "physical attention window"); + +constexpr ryzenai_corelib_flat_mha_bf16_desc kAttentionDesc{ + constants::kQueryHeadCount, + constants::kKvHeadCount, + constants::kHeadSize, + constants::kMaxSequenceLength, + constants::kRopeDimension}; + +std::size_t RowUseIndex(RowUse use) { + switch (use) { + case RowUse::QueryProjection: + return 0; + case RowUse::KvProjection: + return 1; + case RowUse::Attention: + return 2; + case RowUse::OutputProjection: + return 3; + case RowUse::SsMlp: + return 4; + case RowUse::LmHead: + return 5; + } + throw std::invalid_argument("unknown Phi-4 RowUse"); +} + +void ValidatePaddedRows( + std::string_view context, + std::int64_t live_rows, + std::int64_t padded_rows) { + if (padded_rows < live_rows) { + throw std::runtime_error( + std::string(context) + + " returned invalid padded rows: live=" + + std::to_string(live_rows) + + ", padded=" + std::to_string(padded_rows)); + } +} + +MatMulPaddedShape QueryMatMul( + const corelib::CorelibApi& api, + std::int64_t live_rows, + std::int64_t k, + std::int64_t n, + std::string_view context) { + MatMulPaddedShape shape{live_rows, k, n}; + api.Check( + api.functions().matmul_pad_shape( + &shape.m, + &shape.k, + &shape.n, + constants::kGroupSize), + "ryzenai_corelib_matmul_bf16_pad_shape"); + if (shape.k != k || shape.n != n) { + throw std::runtime_error( + std::string(context) + + " MatMul K/N mismatch at live row " + + std::to_string(live_rows) + + ": requested K=" + std::to_string(k) + + ", N=" + std::to_string(n) + + "; returned K=" + std::to_string(shape.k) + + ", N=" + std::to_string(shape.n)); + } + ValidatePaddedRows(context, live_rows, shape.m); + return shape; +} + +std::int64_t QuerySsMlp( + const corelib::CorelibApi& api, + std::int64_t live_rows) { + std::int64_t padded_rows = live_rows; + api.Check( + api.functions().ssmlp_pad_rows( + &padded_rows, + constants::kHiddenSize, + constants::kIntermediateSize, + constants::kGroupSize), + "ryzenai_corelib_ssmlp_bf16_pad_rows"); + ValidatePaddedRows("SSMLP", live_rows, padded_rows); + return padded_rows; +} + +std::int64_t QueryAttention( + const corelib::CorelibApi& api, + std::int64_t live_rows, + const ryzenai_corelib_flat_mha_bf16_desc& desc) { + std::int64_t padded_rows = live_rows; + api.Check( + api.functions().flat_mha_pad_rows(&padded_rows, &desc), + "ryzenai_corelib_flat_mha_bf16_pad_rows"); + ValidatePaddedRows("attention", live_rows, padded_rows); + return padded_rows; +} + +void AppendTransition( + std::vector>& transitions, + std::int64_t live_rows, + std::int64_t padded_rows) { + if (transitions.empty() || + transitions.back().second != padded_rows) { + transitions.emplace_back(live_rows, padded_rows); + } +} + +} // namespace + +Phi4ShapePlan Phi4ShapePlan::Build( + std::shared_ptr api) { + if (!api) { + throw std::invalid_argument( + "Phi4ShapePlan::Build requires a CorelibApi"); + } + + Phi4ShapePlan plan; + plan.attention_desc_ = kAttentionDesc; + + auto& query_transitions = + plan.transitions_[RowUseIndex(RowUse::QueryProjection)]; + auto& kv_transitions = + plan.transitions_[RowUseIndex(RowUse::KvProjection)]; + auto& attention_transitions = + plan.transitions_[RowUseIndex(RowUse::Attention)]; + auto& output_transitions = + plan.transitions_[RowUseIndex(RowUse::OutputProjection)]; + auto& ssmlp_transitions = + plan.transitions_[RowUseIndex(RowUse::SsMlp)]; + + for (std::int64_t live_rows = 1; + live_rows <= kShapePrecomputeMaxLiveRows; + ++live_rows) { + const auto query_shape = QueryMatMul( + *api, + live_rows, + constants::kHiddenSize, + constants::kQueryDimension, + "query/output projection"); + const auto kv_shape = QueryMatMul( + *api, + live_rows, + constants::kHiddenSize, + constants::kKvDimension, + "key/value projection"); + const auto ssmlp_rows = QuerySsMlp(*api, live_rows); + const auto attention_rows = QueryAttention( + *api, + live_rows, + plan.attention_desc_); + + AppendTransition( + query_transitions, + live_rows, + query_shape.m); + AppendTransition( + output_transitions, + live_rows, + query_shape.m); + AppendTransition(kv_transitions, live_rows, kv_shape.m); + AppendTransition( + ssmlp_transitions, + live_rows, + ssmlp_rows); + AppendTransition( + attention_transitions, + live_rows, + attention_rows); + + plan.capacities_.layer_rows = std::max( + {plan.capacities_.layer_rows, + query_shape.m, + kv_shape.m, + ssmlp_rows, + attention_rows}); + } + + const auto lm_head_shape = QueryMatMul( + *api, + 1, + constants::kHiddenSize, + constants::kVocabularySize, + "LM head"); + AppendTransition( + plan.transitions_[RowUseIndex(RowUse::LmHead)], + 1, + lm_head_shape.m); + plan.capacities_.lm_head_rows = lm_head_shape.m; + + return plan; +} + +std::int64_t Phi4ShapePlan::RowsFor( + RowUse use, + std::int64_t live_rows) const { + const auto& transitions = Transitions(use); + const std::int64_t max_live_rows = + use == RowUse::LmHead + ? 1 + : kShapePrecomputeMaxLiveRows; + if (live_rows < 1 || live_rows > max_live_rows) { + throw std::out_of_range( + "Phi4ShapePlan live rows are outside the cached range"); + } + + const auto next = std::upper_bound( + transitions.begin(), + transitions.end(), + live_rows, + [](std::int64_t value, const auto& transition) { + return value < transition.first; + }); + if (next == transitions.begin()) { + throw std::logic_error( + "Phi4ShapePlan has no transition for live rows"); + } + return std::prev(next)->second; +} + +const std::vector>& +Phi4ShapePlan::Transitions(RowUse use) const { + return transitions_[RowUseIndex(use)]; +} + +const Phi4Capacities& Phi4ShapePlan::capacities() const noexcept { + return capacities_; +} + +const ryzenai_corelib_flat_mha_bf16_desc& +Phi4ShapePlan::attention_desc() const noexcept { + return attention_desc_; +} + +} // namespace flm::phi4 diff --git a/src/common/corelib/phi4_corelib_weights.cpp b/src/common/corelib/phi4_corelib_weights.cpp new file mode 100644 index 00000000..a77a30b9 --- /dev/null +++ b/src/common/corelib/phi4_corelib_weights.cpp @@ -0,0 +1,342 @@ +#include + +#include + +#include +#include +#include +#include +#include + +namespace flm::phi4 { +namespace { + +constexpr std::string_view kMatMulCreateCall = + "ryzenai_corelib_matmul_bf16_weights_create_onnx"; +constexpr std::string_view kMatMulGetDataCall = + "ryzenai_corelib_matmul_bf16_weights_get_data"; +constexpr std::string_view kSsMlpCreateCall = + "ryzenai_corelib_ssmlp_bf16_weights_create_onnx"; +constexpr std::string_view kSsMlpGetDataCall = + "ryzenai_corelib_ssmlp_bf16_weights_get_data"; + +const std::string& ComponentName( + const WeightObjectView& object, + std::string_view role) { + const auto component = object.components.find(std::string(role)); + if (component == object.components.end()) { + throw std::runtime_error( + "validated component role is missing: " + + std::string(role)); + } + return component->second; +} + +void AddPackedBytes( + std::size_t packed_size, + std::size_t& total) { + if ( + packed_size > + std::numeric_limits::max() - total) { + throw std::overflow_error( + "packed weight byte total overflows size_t"); + } + total += packed_size; +} + +[[noreturn]] void ThrowObjectError( + const WeightObjectView& object, + const corelib::CorelibError& error) { + throw error.WithContext( + "Phi-4 weight object '" + object.name + "'"); +} + +[[noreturn]] void ThrowObjectError( + const WeightObjectView& object, + const std::exception& error) { + throw std::runtime_error( + "failed to load Phi-4 weight object '" + object.name + + "': " + error.what()); +} + +corelib::UniqueMatMulWeights CreateMatMul( + const std::shared_ptr& api, + Phi4Package& package, + const WeightObjectView& object, + std::size_t& packed_bytes) { + try { + if (object.kind != WeightObjectKind::MatMul) { + throw std::runtime_error( + "validated weight object has the wrong kind"); + } + + const auto& qweight = + package.Require(ComponentName(object, "qweight")); + const auto scales = + package.MaterializeFp16( + ComponentName(object, "scales")); + const auto& qzeros = + package.Require(ComponentName(object, "qzeros")); + + const ryzenai_corelib_matmul_bf16_weights_desc descriptor{ + object.k, + object.n, + constants::kGroupSize, + false}; + const ryzenai_corelib_matmul_bf16_onnx_components + components{ + qweight.data, + scales.data(), + qzeros.data}; + + ryzenai_corelib_matmul_bf16_weights_ptr raw = nullptr; + const auto status = + api->functions().matmul_weights_from_onnx( + &descriptor, + &components, + corelib::kPackingThreads, + &raw); + corelib::UniqueMatMulWeights weights(api, raw); + api->Check(status, kMatMulCreateCall); + if (!weights) { + throw std::runtime_error( + std::string(kMatMulCreateCall) + + " succeeded with a null object"); + } + + std::size_t packed_size = 0; + api->Check( + api->functions().matmul_weights_get_data( + weights.get(), + nullptr, + &packed_size), + kMatMulGetDataCall); + AddPackedBytes(packed_size, packed_bytes); + return weights; + } catch (const corelib::CorelibError& error) { + ThrowObjectError(object, error); + } catch (const std::exception& error) { + ThrowObjectError(object, error); + } +} + +corelib::UniqueSsMlpWeights CreateSsMlp( + const std::shared_ptr& api, + Phi4Package& package, + const std::uint16_t* epsilon, + const WeightObjectView& object, + std::size_t& packed_bytes) { + try { + if (object.kind != WeightObjectKind::SsMlp) { + throw std::runtime_error( + "validated weight object has the wrong kind"); + } + + const auto norm0 = + package.MaterializeBf16( + ComponentName(object, "norm0")); + const auto norm1 = + package.MaterializeBf16( + ComponentName(object, "norm1")); + + const auto& gate_qweight = + package.Require( + ComponentName(object, "gate_qweight")); + const auto gate_scales = + package.MaterializeFp16( + ComponentName(object, "gate_scales")); + const auto& gate_qzeros = + package.Require( + ComponentName(object, "gate_qzeros")); + + const auto& up_qweight = + package.Require( + ComponentName(object, "up_qweight")); + const auto up_scales = + package.MaterializeFp16( + ComponentName(object, "up_scales")); + const auto& up_qzeros = + package.Require( + ComponentName(object, "up_qzeros")); + + const auto& down_qweight = + package.Require( + ComponentName(object, "down_qweight")); + const auto down_scales = + package.MaterializeFp16( + ComponentName(object, "down_scales")); + const auto& down_qzeros = + package.Require( + ComponentName(object, "down_qzeros")); + + const ryzenai_corelib_ssmlp_bf16_weights_desc descriptor{ + object.k, + object.n, + constants::kGroupSize}; + const ryzenai_corelib_ssmlp_bf16_onnx_components + components{ + epsilon, + norm0.data(), + norm1.data(), + gate_qweight.data, + gate_scales.data(), + gate_qzeros.data, + up_qweight.data, + up_scales.data(), + up_qzeros.data, + down_qweight.data, + down_scales.data(), + down_qzeros.data}; + + ryzenai_corelib_ssmlp_bf16_weights_ptr raw = nullptr; + const auto status = + api->functions().ssmlp_weights_from_onnx( + &descriptor, + &components, + corelib::kPackingThreads, + &raw); + corelib::UniqueSsMlpWeights weights(api, raw); + api->Check(status, kSsMlpCreateCall); + if (!weights) { + throw std::runtime_error( + std::string(kSsMlpCreateCall) + + " succeeded with a null object"); + } + + std::size_t packed_size = 0; + api->Check( + api->functions().ssmlp_weights_get_data( + weights.get(), + nullptr, + &packed_size), + kSsMlpGetDataCall); + AddPackedBytes(packed_size, packed_bytes); + return weights; + } catch (const corelib::CorelibError& error) { + ThrowObjectError(object, error); + } catch (const std::exception& error) { + ThrowObjectError(object, error); + } +} + +} // namespace + +Phi4Weights Phi4Weights::Load( + std::shared_ptr api, + std::shared_ptr package) { + if (!api) { + throw std::invalid_argument( + "Phi4Weights::Load requires a CorelibApi"); + } + if (!package) { + throw std::invalid_argument( + "Phi4Weights::Load requires a Phi4Package"); + } + + Phi4Weights result; + result.package_ = std::move(package); + + // The packer takes epsilon as a raw BF16 blob, not as a tensor, so + // this is one of the two host conversions design `API-6` permits. + auto epsilon = std::make_shared( + corelib::NarrowFp32ToBf16( + static_cast(constants::kRmsEpsilon))); + result.epsilon_bf16_ = std::move(epsilon); + + const auto& objects = result.package_->weight_objects(); + constexpr std::size_t objects_per_layer = 5; + constexpr std::size_t expected_objects = + static_cast(constants::kLayerCount) * + objects_per_layer + + 1; + if (objects.size() != expected_objects) { + throw std::runtime_error( + "Phi4Package must contain exactly 161 validated " + "weight objects"); + } + + for (std::size_t layer = 0; + layer < + static_cast(constants::kLayerCount); + ++layer) { + const std::size_t base = layer * objects_per_layer; + auto& destination = result.layers_[layer]; + destination.q = CreateMatMul( + api, + *result.package_, + objects[base], + result.packed_bytes_); + destination.k = CreateMatMul( + api, + *result.package_, + objects[base + 1], + result.packed_bytes_); + destination.v = CreateMatMul( + api, + *result.package_, + objects[base + 2], + result.packed_bytes_); + destination.o = CreateMatMul( + api, + *result.package_, + objects[base + 3], + result.packed_bytes_); + destination.mlp = CreateSsMlp( + api, + *result.package_, + result.epsilon_bf16_.get(), + objects[base + 4], + result.packed_bytes_); + } + + result.lm_head_ = CreateMatMul( + api, + *result.package_, + objects.back(), + result.packed_bytes_); + return result; +} + +Phi4Weights& Phi4Weights::operator=(Phi4Weights&& other) noexcept { + if (this != &other) { + ResetWeightObjects(); + epsilon_bf16_.reset(); + package_.reset(); + + package_ = std::move(other.package_); + epsilon_bf16_ = std::move(other.epsilon_bf16_); + layers_ = std::move(other.layers_); + lm_head_ = std::move(other.lm_head_); + packed_bytes_ = std::exchange(other.packed_bytes_, 0); + } + return *this; +} + +const std::array& +Phi4Weights::layers() const noexcept { + return layers_; +} + +const corelib::UniqueMatMulWeights& +Phi4Weights::lm_head() const noexcept { + return lm_head_; +} + +std::size_t Phi4Weights::packed_bytes() const noexcept { + return packed_bytes_; +} + +void Phi4Weights::ResetWeightObjects() noexcept { + lm_head_.reset(); + for (auto layer = layers_.rbegin(); + layer != layers_.rend(); + ++layer) { + layer->mlp.reset(); + layer->o.reset(); + layer->v.reset(); + layer->k.reset(); + layer->q.reset(); + } +} + +} // namespace flm::phi4 diff --git a/src/common/utils.cpp b/src/common/utils.cpp index 5e16386e..224417a3 100644 --- a/src/common/utils.cpp +++ b/src/common/utils.cpp @@ -111,12 +111,56 @@ std::string find_model_info() { if (std::filesystem::exists(exe_relative_path)) { return exe_relative_path; } + for (const std::filesystem::path bundle : { + std::filesystem::path(exe_dir) / "share" / "flm" / + "model_info.json", + std::filesystem::path(exe_dir) / ".." / "share" / "flm" / + "model_info.json"}) { + if (std::filesystem::exists(bundle)) { + return bundle.lexically_normal().string(); + } + } #endif // If not found, throw an error throw std::runtime_error("model_info.json not found. Please set FLM_MODELINFO_PATH or place it next to the executable."); } +std::string find_model_overlay_root() { + const char* configured = std::getenv("FLM_MODEL_OVERLAY_PATH"); + if (configured && *configured) { + const std::filesystem::path path(configured); + if (std::filesystem::is_directory(path)) { + return std::filesystem::absolute(path).lexically_normal().string(); + } + throw std::runtime_error( + "FLM_MODEL_OVERLAY_PATH does not name an installed " + "model_overlays directory: " + + path.string()); + } + + const std::filesystem::path executable_dir( + get_executable_directory()); + const std::filesystem::path install_prefix(CMAKE_INSTALL_PREFIX); + const std::filesystem::path candidates[] = { + executable_dir / "model_overlays", + executable_dir / "share" / "flm" / "model_overlays", + executable_dir / ".." / "share" / "flm" / "model_overlays", + std::filesystem::current_path() / "model_overlays", + install_prefix / "share" / "flm" / "model_overlays", + }; + for (const auto& candidate : candidates) { + if (std::filesystem::is_directory(candidate)) { + return std::filesystem::absolute(candidate) + .lexically_normal() + .string(); + } + } + throw std::runtime_error( + "FastFlow model overlay root not found. Reinstall the model " + "overlay package or set FLM_MODEL_OVERLAY_PATH."); +} + std::string find_xclbin_path() { std::string xclbin_prefix = CMAKE_XCLBIN_PREFIX; diff --git a/src/include/AutoModel/automodel.hpp b/src/include/AutoModel/automodel.hpp index ed3a6726..eb0d3d84 100644 --- a/src/include/AutoModel/automodel.hpp +++ b/src/include/AutoModel/automodel.hpp @@ -7,9 +7,13 @@ #pragma once #include +#include #include -#include #include +#include +#include +#include +#include #include #include #include @@ -128,10 +132,32 @@ struct lm_uniform_input_t { std::vector audios; std::vector audio_payload_types; nlohmann::ordered_json tools; + std::optional requested_max_new_tokens; }; using json = nlohmann::ordered_json; +class ModelRequestError final : public std::runtime_error { +public: + ModelRequestError( + int http_code, + bool session_cleared, + std::string message); + + int http_code() const noexcept; + bool session_cleared() const noexcept; + +private: + int http_code_; + bool session_cleared_; +}; + +enum class PrefixHitAction { + AppendSuffixBatched, + AppendSuffixOneByOne, + RecomputeFull +}; + class AutoModel { protected: std::string model_path = ""; @@ -188,14 +214,29 @@ class AutoModel { void _shared_load_model(std::string model_path, json model_info, int default_context_length = -1, bool enable_preemption = false); + void _shared_initialize_model_state(std::string model_path, json model_info, int default_context_length); + void _shared_initialize_legacy_npu(bool enable_preemption); nlohmann::json _shared_setup_tokenizer(std::string model_path); + size_t _matching_prefix_length(std::span tokens) const; /// \brief Insert tokens into the model /// \param meta_info the meta information of the chat /// \param tokens the tokens to insert /// \param payload the payload, it shall not be used as this function is only used for chunkwised insertion, no image allowed /// \return true if the tokens were inserted successfully, false otherwise - bool _shared_insert(chat_meta_info_t& meta_info, std::vector& tokens, std::function is_cancelled = [] { return false; }, void* payload = nullptr, int first_len_run = 0); + bool _shared_insert( + chat_meta_info_t& meta_info, + std::vector& tokens, + std::function is_cancelled = [] { return false; }, + void* payload = nullptr, + int first_len_run = 0); + bool _shared_insert( + chat_meta_info_t& meta_info, + std::vector& tokens, + std::function is_cancelled, + void* payload, + int first_len_run, + PrefixHitAction prefix_action); buffer _chunked_insert(chat_meta_info_t& meta_info, std::vector& tokens, std::function is_cancelled = [] { return false; }, void* payload = nullptr, int first_len_run = 0); std::string _shared_generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled = [] { return false; }); @@ -220,6 +261,11 @@ class AutoModel { /// \return the current model std::string get_current_model(); + /// \brief Whether the loaded model uses the corelib AIE4 backend + virtual bool uses_corelib_aie4() const noexcept { + return false; + } + /// \brief Get the current context length /// \return the current context length virtual int get_current_context_length(); diff --git a/src/include/AutoModel/modeling_phi4.hpp b/src/include/AutoModel/modeling_phi4.hpp index 66937d6d..70f4a833 100644 --- a/src/include/AutoModel/modeling_phi4.hpp +++ b/src/include/AutoModel/modeling_phi4.hpp @@ -7,18 +7,98 @@ #pragma once #include "AutoModel/automodel.hpp" +#if defined(FLM_ENABLE_CORELIB_AIE4) +#include "corelib/corelib_runtime.hpp" +#include "models/phi4/phi4_corelib_aie4.hpp" +#include "models/phi4/phi4_corelib_aie4_tuning.hpp" +#include +#endif + +#if defined(FLM_CORELIB_TESTING) +#include +#include +#endif + +#if defined(FLM_CORELIB_TESTING) +namespace flm::phi4::testing { +class Phi4FrontendTestAccess; +} +#endif + /************ phi4 family **************/ class Phi4 : public AutoModel { private: - void setup_tokenizer(std::string model_path); + void setup_tokenizer( + const std::string& model_path, + bool require_aie4_eos); + +#if defined(FLM_ENABLE_CORELIB_AIE4) + // The largest total the AIE4 path can actually reach: MAX_L capped by the + // token attention kernel's window, which is one below the prefill path's. + // Admission and the decode loop both use this, never MAX_L, because the + // step MAX_L would allow fails inside flat_mha, mid-step, after this + // layer's V cache has already been scattered. That used to take the + // process with it; since the engine's irrevocable boundary became per + // submission group it is recoverable, but it still costs the caller the + // whole conversation. See the fuller note on Phi4::aie4_active_cap in + // modeling_phi4.cpp. + size_t aie4_active_cap() const; + void validate_aie4_capacity( + size_t rendered_tokens, + std::optional requested_max_new_tokens) const; + void clear_after_corelib_error(); + std::string generate_aie4( + chat_meta_info_t& meta_info, + int length_limit, + std::ostream& os, + std::function is_cancelled); + const flm::phi4::Phi4Aie4Metrics& aie4_metrics() const; + + bool uses_corelib_aie4_ = false; + std::shared_ptr + corelib_runtime_; + flm::phi4::ForcedContinuationRoute + forced_continuation_route_ = + flm::phi4::ForcedContinuationRoute::Automatic; + std::optional + last_continuation_route_; + std::uint64_t last_continuation_ns_ = 0; + std::uint64_t append_continuation_ns_ = 0; + std::uint64_t reprefill_continuation_ns_ = 0; +#if defined(FLM_CORELIB_TESTING) + std::optional + metrics_for_testing_; +#endif +#endif + +#if defined(FLM_CORELIB_TESTING) + using EngineFactoryForTesting = + std::function( + bool, + const LM_Config&, + npu_xclbin_manager*, + const std::filesystem::path&, + std::uint32_t)>; + static EngineFactoryForTesting engine_factory_for_testing_; + friend class flm::phi4::testing::Phi4FrontendTestAccess; +#endif public: Phi4(flm_rt::device* npu_device_inst); void load_model(std::string model_path, json model_inf, int default_context_length = -1, bool enable_preemption = false) override; + bool uses_corelib_aie4() const noexcept override { +#if defined(FLM_ENABLE_CORELIB_AIE4) + return uses_corelib_aie4_; +#else + return false; +#endif + } //void toggle_enable_think() override; bool insert(chat_meta_info_t& meta_info, lm_uniform_input_t& input, std::function is_cancelled = [] { return false; }) override; std::string generate(chat_meta_info_t& meta_info, int length_limit, std::ostream& os, std::function is_cancelled = [] { return false; }) override; std::string generate_with_prompt(chat_meta_info_t& meta_info, lm_uniform_input_t& input, int length_limit, std::ostream& os = std::cout) override; std::string apply_chat_template(nlohmann::ordered_json& messages, nlohmann::ordered_json tools = nlohmann::ordered_json::object()) override; + void set_max_length(unsigned int MAX_L) override; + std::string show_profile() override; }; diff --git a/src/include/corelib/corelib_api.hpp b/src/include/corelib/corelib_api.hpp new file mode 100644 index 00000000..d4cf53f2 --- /dev/null +++ b/src/include/corelib/corelib_api.hpp @@ -0,0 +1,200 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace flm::corelib { + +// The loaded library's version, and the version this build compiled +// against. Corelib is pre-1.0, and the header states that below 1.0 the +// API may change in any release, so the patch component is load-bearing. +struct CorelibVersion final { + std::uint32_t major = 0; + std::uint32_t minor = 0; + std::uint32_t patch = 0; +}; + +// API-5. While the compiled-against major is 0 all three components must +// match exactly. From corelib 1.0 the rule relaxes to major equality with +// the runtime minor at least the compiled minor. +bool IsCorelibVersionCompatible( + const CorelibVersion& compiled, + const CorelibVersion& runtime) noexcept; + +std::string FormatCorelibVersion(const CorelibVersion& value); + +std::string FormatCorelibVersionMismatch( + const CorelibVersion& compiled, + const CorelibVersion& runtime); + +CorelibVersion CompiledCorelibVersion() noexcept; + +struct CorelibError final : std::runtime_error { + CorelibError( + ryzenai_corelib_status status, + std::string call, + std::string detail, + std::string status_text); + + CorelibError WithContext(std::string_view context) const; + + ryzenai_corelib_status status; + std::string call; + std::string detail; + +private: + std::string status_text_; +}; + +struct CorelibFunctions { + // Resolved and called first: the version gate runs before any other + // symbol is looked up, so a mismatched runtime is reported instead of + // producing a confusing "missing symbol" for a renamed entry point. + decltype(&::ryzenai_corelib_get_version) get_version; + decltype(&::ryzenai_corelib_status_to_string) status_to_string; + decltype(&::ryzenai_corelib_get_last_error_message) + get_last_error_message; + decltype(&::ryzenai_corelib_selftest_dependencies) + selftest_dependencies; + decltype(&::ryzenai_corelib_has_device_context) has_device_context; + decltype(&::ryzenai_corelib_object_release) object_release; + decltype(&::ryzenai_corelib_create_stream) create_stream; + decltype(&::ryzenai_corelib_stream_synchronize) stream_synchronize; + decltype(&::ryzenai_corelib_create_device_tensor) create_device_tensor; + decltype(&::ryzenai_corelib_tensor_write) tensor_write; + decltype(&::ryzenai_corelib_tensor_read) tensor_read; + decltype(&::ryzenai_corelib_tensor_get_byte_size) tensor_get_byte_size; + decltype(&::ryzenai_corelib_tensor_get_data_type) tensor_get_data_type; + decltype(&::ryzenai_corelib_matmul_bf16_pad_shape) matmul_pad_shape; + decltype(&::ryzenai_corelib_matmul_bf16_weights_create_onnx) + matmul_weights_from_onnx; + decltype(&::ryzenai_corelib_matmul_bf16_weights_get_data) + matmul_weights_get_data; + decltype(&::ryzenai_corelib_matmul_bf16) matmul; + decltype(&::ryzenai_corelib_ssmlp_bf16_pad_rows) ssmlp_pad_rows; + decltype(&::ryzenai_corelib_ssmlp_bf16_weights_create_onnx) + ssmlp_weights_from_onnx; + decltype(&::ryzenai_corelib_ssmlp_bf16_weights_get_data) + ssmlp_weights_get_data; + decltype(&::ryzenai_corelib_ssmlp_bf16) ssmlp; + decltype(&::ryzenai_corelib_flat_mha_bf16_pad_rows) flat_mha_pad_rows; + decltype(&::ryzenai_corelib_flat_mha_bf16) flat_mha; + decltype(&::ryzenai_corelib_cleanup) cleanup; +}; + +// Which kind of corelib object a successful creation call produced. +// +// Task 13 Step 4 needs "no device tensor and no weight object was created +// after warmup" to be a MEASUREMENT rather than a restatement of what the +// code is believed to do. Counting at the RAII wrapper -- the single point +// every corelib object passes through on its way to being owned -- makes the +// answer independent of WHICH code path created it, so an allocation +// introduced anywhere in the decode loop is caught. A counter maintained by +// the engine's own tensor helper can only ever confirm that the helper was +// not called. +// +// Before this existed, `Phi4Aie4Metrics::weight_create_count` was assigned +// the constant `kLayerCount * 5 + 1`. That number is correct, and a constant +// cannot detect a weight object being created after warmup, which is the one +// question the field is read for. +enum class CorelibObjectKind : std::size_t { + Stream = 0, + Tensor = 1, + MatMulWeights = 2, + SsMlpWeights = 3, +}; + +inline constexpr std::size_t kCorelibObjectKindCount = 4; + +class CorelibApi final { +public: + using Resolver = std::function; + + static std::shared_ptr Load( + const std::filesystem::path& absolute_path); + static std::shared_ptr ResolveForTest(Resolver resolver); + static std::filesystem::path ResolveLibraryPath( + const std::filesystem::path& executable_dir); + + ~CorelibApi(); + + CorelibApi(const CorelibApi&) = delete; + CorelibApi& operator=(const CorelibApi&) = delete; + CorelibApi(CorelibApi&&) = delete; + CorelibApi& operator=(CorelibApi&&) = delete; + + const CorelibFunctions& functions() const noexcept; + const CorelibVersion& runtime_version() const noexcept; + void Check( + ryzenai_corelib_status status, + std::string_view call) const; + + // API-7. `count` and `offset` are ELEMENTS of the tensor's own dtype, + // never bytes. These are the only spellings FastFlow uses; there is + // deliberately no byte-taking overload, because the byte and element + // counts differ by 2x when writing FP32 into a BF16 tensor and the + // wrong one would half-fill or overrun the tensor instead of failing. + void WriteElements( + ryzenai_corelib_tensor_ptr tensor, + ryzenai_corelib_data_type source_type, + const void* source, + std::size_t count, + std::size_t offset) const; + + void ReadElements( + ryzenai_corelib_tensor_ptr tensor, + ryzenai_corelib_data_type destination_type, + void* destination, + std::size_t count, + std::size_t offset) const; + + void RegisterObject(CorelibObjectKind kind) const noexcept; + void Release(void* value) const noexcept; + std::size_t live_object_count() const noexcept; + + // Cumulative successful creations of `kind` for the process lifetime. + // Never decremented: this answers "was anything created between these two + // points", which a live count cannot, because a create-and-release pair + // leaves the live count where it started. + std::uint64_t creation_count(CorelibObjectKind kind) const noexcept; + + // MatMul plus SSMLP weight objects. Both kinds are "a weight object" for + // the purposes of design 18.7's post-warm allocation property, and a + // caller that summed only one of them would miss half of the model. + std::uint64_t weight_creation_count() const noexcept; + + const std::filesystem::path& library_path() const noexcept; + +private: + CorelibApi( + void* module, + std::filesystem::path library_path, + CorelibFunctions functions, + CorelibVersion runtime_version); + + void* module_ = nullptr; + std::filesystem::path library_path_; + CorelibFunctions functions_; + CorelibVersion runtime_version_; + mutable std::atomic live_object_count_{0}; + mutable std::array< + std::atomic, + kCorelibObjectKindCount> + creation_counts_{}; +}; + +// The header's "one thread" hint for the ONNX packing entry points. +// Design Section 19 defers concurrent packing; FastFlow does not adopt it. +inline constexpr std::uint32_t kPackingThreads = 0; + +} // namespace flm::corelib diff --git a/src/include/corelib/corelib_fatal_record.hpp b/src/include/corelib/corelib_fatal_record.hpp new file mode 100644 index 00000000..2b8464c5 --- /dev/null +++ b/src/include/corelib/corelib_fatal_record.hpp @@ -0,0 +1,70 @@ +#pragma once + +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace flm::corelib { + +struct FailureContext { + ryzenai_corelib_status status; + std::string call; + std::string detail; + std::string phase; + std::optional layer; + int64_t rows; + int64_t position; +}; + +class FatalRecordStore { +public: + using ProcessProbe = + std::function< + std::optional(DWORD)>; + + FatalRecordStore( + std::filesystem::path root, + DWORD pid, + std::chrono::system_clock::time_point process_start, + ProcessProbe process_probe); + FatalRecordStore(FatalRecordStore&& other) noexcept; + FatalRecordStore& operator=(FatalRecordStore&& other) noexcept; + ~FatalRecordStore() noexcept; + FatalRecordStore(const FatalRecordStore&) = delete; + FatalRecordStore& operator=(const FatalRecordStore&) = delete; + + static FatalRecordStore ForCurrentProcess(); + + void Prepare(); + std::filesystem::path Persist(const FailureContext& failure); + static std::vector DrainPriorRecords( + std::ostream& output); + static std::vector DrainPriorRecords( + const std::filesystem::path& root, + ProcessProbe process_probe, + std::ostream& output); + void RemoveUnusedPending() noexcept; + const std::filesystem::path& pending_path() const noexcept; + +private: + void ClosePending() noexcept; + + std::filesystem::path root_; + DWORD pid_ = 0; + std::chrono::system_clock::time_point process_start_; + ProcessProbe process_probe_; + std::filesystem::path pending_path_; + HANDLE pending_handle_ = INVALID_HANDLE_VALUE; + bool persisted_ = false; +}; + +} // namespace flm::corelib diff --git a/src/include/corelib/corelib_object.hpp b/src/include/corelib/corelib_object.hpp new file mode 100644 index 00000000..5229e793 --- /dev/null +++ b/src/include/corelib/corelib_object.hpp @@ -0,0 +1,91 @@ +#pragma once + +#include + +#include +#include +#include + +namespace flm::corelib { + +template +class UniqueObject { +public: + UniqueObject() noexcept = default; + + UniqueObject( + std::shared_ptr api, + void* value) noexcept + : api_(std::move(api)), + value_(value) { + if (value_ != nullptr) { + assert(api_ != nullptr); + api_->RegisterObject(Tag::kKind); + } + } + + UniqueObject(UniqueObject&& other) noexcept + : api_(std::move(other.api_)), + value_(std::exchange(other.value_, nullptr)) {} + + UniqueObject& operator=(UniqueObject&& other) noexcept { + if (this != &other) { + reset(); + api_ = std::move(other.api_); + value_ = std::exchange(other.value_, nullptr); + } + return *this; + } + + ~UniqueObject() noexcept { + reset(); + } + + UniqueObject(const UniqueObject&) = delete; + UniqueObject& operator=(const UniqueObject&) = delete; + + void* get() const noexcept { + return value_; + } + + explicit operator bool() const noexcept { + return value_ != nullptr; + } + + void reset() noexcept { + if (value_ != nullptr) { + api_->Release(std::exchange(value_, nullptr)); + } + api_.reset(); + } + +private: + std::shared_ptr api_; + void* value_ = nullptr; +}; + +// Each tag carries the kind it counts as. Defined rather than forward +// declared so `Tag::kKind` above resolves, and so adding a new corelib object +// type is a compile error here until it is given a kind, rather than a silent +// omission from the post-warm allocation measurement. +struct StreamTag { + static constexpr CorelibObjectKind kKind = CorelibObjectKind::Stream; +}; +struct TensorTag { + static constexpr CorelibObjectKind kKind = CorelibObjectKind::Tensor; +}; +struct MatMulWeightsTag { + static constexpr CorelibObjectKind kKind = + CorelibObjectKind::MatMulWeights; +}; +struct SsMlpWeightsTag { + static constexpr CorelibObjectKind kKind = + CorelibObjectKind::SsMlpWeights; +}; + +using UniqueStream = UniqueObject; +using UniqueTensor = UniqueObject; +using UniqueMatMulWeights = UniqueObject; +using UniqueSsMlpWeights = UniqueObject; + +} // namespace flm::corelib diff --git a/src/include/corelib/corelib_runtime.hpp b/src/include/corelib/corelib_runtime.hpp new file mode 100644 index 00000000..73dc30e2 --- /dev/null +++ b/src/include/corelib/corelib_runtime.hpp @@ -0,0 +1,79 @@ +#pragma once + +#include +#include + +#include +#include +#include +#include +#include + +namespace flm::corelib { + +enum class ProcessState { Healthy, Terminating, Shutdown }; +using ExecutionLease = std::unique_lock; + +class StepSubmissionState { +public: + void MarkSuccessfulSubmit() noexcept { + submitted_ = true; + } + + bool irrevocable() const noexcept { + return submitted_; + } + +private: + bool submitted_ = false; +}; + +class CorelibRuntime final { +public: + using Terminator = std::function; + + static std::shared_ptr GetOrCreate( + const std::filesystem::path& executable_dir); + static std::shared_ptr Create( + std::shared_ptr api, + FatalRecordStore records, + Terminator terminator); + static void ShutdownProcess(); + + ~CorelibRuntime() = default; + CorelibRuntime(const CorelibRuntime&) = delete; + CorelibRuntime& operator=(const CorelibRuntime&) = delete; + CorelibRuntime(CorelibRuntime&&) = delete; + CorelibRuntime& operator=(CorelibRuntime&&) = delete; + + ExecutionLease AcquireExecution(); + bool admission_open() const noexcept; + ProcessState state() const noexcept; + const std::shared_ptr& api() const noexcept; + void ShutdownHealthy(); + [[noreturn]] void TerminateAfterFailure( + const FailureContext& failure); +#if defined(FLM_CORELIB_TESTING) + void SetBeforeLiveObjectRollbackForTest( + std::function hook); +#endif + +private: + CorelibRuntime( + std::shared_ptr api, + FatalRecordStore records, + Terminator terminator); + + std::shared_ptr api_; + FatalRecordStore records_; + Terminator terminator_; + std::mutex shutdown_mutex_; + mutable std::mutex execution_mutex_; + std::atomic state_{ProcessState::Shutdown}; + bool cleanup_called_ = false; +#if defined(FLM_CORELIB_TESTING) + std::function before_live_object_rollback_for_test_; +#endif +}; + +} // namespace flm::corelib diff --git a/src/include/corelib/host_convert.hpp b/src/include/corelib/host_convert.hpp new file mode 100644 index 00000000..d65295ef --- /dev/null +++ b/src/include/corelib/host_convert.hpp @@ -0,0 +1,93 @@ +#pragma once + +// The only two host conversions FastFlow is permitted to implement +// (design `API-6`). Corelib `e5258d2` removed `ryzenai_corelib_convert` +// and `ryzenai_corelib_convert_strided`: every conversion with a tensor on +// either side now crosses `tensor_write` / `tensor_read`. What remains here +// is data that never touches a tensor. +// +// 1. FP16 -> FP32 widening, for the embedding rows gathered before the +// host RMSNorm and for an FP16 layer-0 norm. Lossless, so it has no +// rounding policy and cannot disagree with corelib. It is a scalar +// loop on purpose: the source is a read-only file mapping, and the +// naive vectorized widening reads up to 14 bytes past its source, +// which faults on a page boundary rather than returning garbage. +// +// 2. FP32 -> BF16 round-to-nearest-even, for the SSMLP epsilon / norm0 / +// norm1 blobs, which are packer inputs rather than tensors. Bit- +// compatible with the reference driver's `to_bf16`. +// +// There is deliberately no third converter. In particular there is no host +// FP32-to-FP16 narrowing, which is why an FP32 `scales` array is rejected +// rather than converted. + +#include +#include +#include + +namespace flm::corelib { + +// Exact: every FP16 value is representable in FP32. +inline float WidenFp16(std::uint16_t bits) noexcept { + const std::uint32_t sign = + static_cast(bits & 0x8000u) << 16; + const std::uint32_t exponent = + (static_cast(bits) >> 10) & 0x1Fu; + const std::uint32_t mantissa = + static_cast(bits) & 0x3FFu; + + if (exponent == 0u) { + if (mantissa == 0u) { + return std::bit_cast(sign); + } + std::uint32_t significand = mantissa; + std::uint32_t shift = 0u; + while ((significand & 0x400u) == 0u) { + significand <<= 1; + ++shift; + } + significand &= 0x3FFu; + const std::uint32_t widened_exponent = 127u - 15u - shift + 1u; + return std::bit_cast( + sign | (widened_exponent << 23) | (significand << 13)); + } + if (exponent == 0x1Fu) { + return std::bit_cast( + sign | 0x7F800000u | (mantissa << 13)); + } + return std::bit_cast( + sign | ((exponent - 15u + 127u) << 23) | (mantissa << 13)); +} + +// Scalar and tail-guarded: reads exactly `count` halfwords from `source` +// and touches nothing beyond them. +inline void WidenFp16Array( + const std::uint16_t* source, + std::size_t count, + float* destination) noexcept { + for (std::size_t index = 0; index < count; ++index) { + destination[index] = WidenFp16(source[index]); + } +} + +// Bit-compatible with the reference driver's `to_bf16`: +// rounded = bits + 0x7FFF + ((bits >> 16) & 1); return rounded >> 16 +// The driver accumulates in uint64 and narrows to uint16; the wrapped +// uint32 sum has the same low 16 bits, so the two agree on every input. +inline std::uint16_t NarrowFp32ToBf16(float value) noexcept { + const std::uint32_t bits = std::bit_cast(value); + const std::uint32_t rounded = + bits + 0x7FFFu + ((bits >> 16) & 1u); + return static_cast(rounded >> 16); +} + +inline void NarrowFp32ToBf16Array( + const float* source, + std::size_t count, + std::uint16_t* destination) noexcept { + for (std::size_t index = 0; index < count; ++index) { + destination[index] = NarrowFp32ToBf16(source[index]); + } +} + +} // namespace flm::corelib diff --git a/src/include/models/phi4/phi4_corelib_aie4.hpp b/src/include/models/phi4/phi4_corelib_aie4.hpp new file mode 100644 index 00000000..6c9229aa --- /dev/null +++ b/src/include/models/phi4/phi4_corelib_aie4.hpp @@ -0,0 +1,127 @@ +#pragma once + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace flm::phi4 { + +struct Phi4DebugSnapshot { + std::int64_t live_rows; + std::int64_t position; + std::vector layer0_k; + std::vector layer0_v; + std::vector layer31_k; + std::vector layer31_v; + std::vector last_hidden; + std::vector logits; +}; + +struct Phi4Aie4Metrics { + std::uint64_t model_load_ns = 0; + // Design 18.6 names runtime packing and first-use kernel construction as + // the startup-cost risk, and the mitigation is to "distinguish + // load/cold/warm measurements". A single total cannot say which of the + // three phases dominates, so the manifest/mapping pass and the 1..4096 + // helper interrogation are timed separately from the weight pack. + std::uint64_t manifest_map_ns = 0; + std::uint64_t shape_plan_ns = 0; + std::uint64_t weight_pack_ns = 0; + // Stream creation, the 76 device tensors, and the RoPE table upload. + // Named so the load timeline adds up: an unlabelled remainder is an + // invitation to assume the missing time is somewhere it is not. + std::uint64_t device_setup_ns = 0; + std::uint64_t packed_weight_bytes = 0; + std::uint64_t mapped_source_bytes = 0; + std::uint64_t kv_bytes = 0; + std::uint64_t scratch_bytes = 0; + std::uint64_t device_tensor_create_count = 0; + std::uint64_t weight_create_count = 0; + std::uint64_t dispatch_count = 0; + std::uint64_t synchronize_count = 0; + std::uint64_t padding_write_calls = 0; + std::uint64_t padding_bytes = 0; + std::uint64_t attention_extent_queries = 0; + std::uint64_t output_projection_extent_queries = 0; + std::uint64_t lm_head_extent_queries = 0; + std::array helper_transition_counts{}; + std::uint64_t v_read_calls = 0; + std::uint64_t v_write_calls = 0; + std::uint64_t v_bytes = 0; + std::uint64_t v_scatter_ns = 0; +}; + +class phi4_corelib_aie4 final : public causal_lm { +public: + phi4_corelib_aie4( + LM_Config config, + std::filesystem::path model_path, + std::shared_ptr runtime, + std::uint32_t max_length = 4096); + ~phi4_corelib_aie4() override; + + buffer forward(int id) override; + buffer prefill( + std::vector& ids, + void* payload = nullptr) override; + void set_context_length(int length) override; + void load_weights(Q4NX& q4nx) override; + void update_max_length(std::uint32_t max_length) override; + void clear_context() override; + buffer get_k_cache(int layer, int index) override; + buffer get_v_cache(int layer, int index) override; + int get_current_context_length() override; + int checkpoint() override; + int restore() override; + const Phi4Aie4Metrics& metrics() const noexcept; +#ifdef DEV_BUILD + Phi4DebugSnapshot debug_snapshot() const; + + // The one row that was fed to the LM-head MatMul on the most recent + // model step, in BF16, straight out of `lm_input_tensor`. + // + // `DETERM-1` localised the run-to-run divergence to the LM-head dispatch + // by INFERENCE -- end-of-run state was identical, so the LM-head input + // was assumed identical at the step where the logits differed. That step + // is not the end of the run, and the assumption was never observed. + // Capturing this after every step turns the localisation into a + // measurement: at the first diverging logit vector, either the two runs + // fed the LM head the same row or they did not, and there is nothing left + // to infer. + // + // Deliberately narrower than debug_snapshot(), which also reads four live + // K/V caches. This is called after every step, so it has to be cheap + // enough not to change what it is measuring. + std::vector debug_lm_head_input() const; +#endif + +private: + struct Impl; + std::unique_ptr impl_; +}; + +#if defined(FLM_CORELIB_TESTING) +namespace testing { + +[[noreturn]] void ApplyCorelibFailurePolicyForTest( + const std::shared_ptr& runtime, + const corelib::CorelibError& error, + bool synchronize_in_progress, + const corelib::StepSubmissionState& submission, + std::string phase, + std::optional layer, + std::int64_t rows, + std::int64_t position); + +} // namespace testing +#endif + +} // namespace flm::phi4 diff --git a/src/include/models/phi4/phi4_corelib_aie4_tuning.hpp b/src/include/models/phi4/phi4_corelib_aie4_tuning.hpp new file mode 100644 index 00000000..742df888 --- /dev/null +++ b/src/include/models/phi4/phi4_corelib_aie4_tuning.hpp @@ -0,0 +1,69 @@ +#pragma once + +#include +#include + +namespace flm::phi4 { + +enum class ContinuationRoute { + Append, + Reprefill +}; + +enum class ForcedContinuationRoute { + Automatic, + Append, + Reprefill +}; + +// BEGIN generated by tools/calibrate_phi4_corelib_continuation.py +// +// Generated. Do not edit this block by hand; re-run the calibrator. +// +// Design Section 10.7: the largest sampled suffix length whose append +// p95 is lower than re-prefill p95 at BOTH history lengths [512, 2048], +// after asserting the winning lengths are a prefix of the sampled set. +// +// verdict: selected +// winners: [1, 2, 4] of sampled [1, 2, 4, 8, 12, 16, 24, 32, 64, 128, 256] +// measured: 2026-09-02T16:32:18Z on xcomedusad-43 +// corelib: a523b23837c9bea3b06f8f8ae74e1e8292092eb73aaa93791022c448b865ecdb +// model: d6f503a9ea142c8b6320313d6ae341a88049b1b8ef01e641b2313fe42cdc7309 +// +// RELEASE-FIXED IS NOT HARDWARE-INDEPENDENT. Every measurement behind +// this value comes from the single machine, corelib build and model +// above. Nothing establishes where the crossover sits on different +// silicon, on a corelib whose append or prefill path changed, or on a +// different model. A port needs this re-measured, not inherited. +// +// What this value gives up, measured: history 2048 suffix 8 (3.0x slower, +771.2 ms); history 2048 suffix 12 (2.0x slower, +566.6 ms). +inline constexpr std::uint32_t kContinuationAppendThreshold = 4; +// END generated by tools/calibrate_phi4_corelib_continuation.py + +inline constexpr ContinuationRoute SelectContinuationRoute( + std::size_t suffix_tokens, + ForcedContinuationRoute forced) noexcept { + if (suffix_tokens == 0) { + return ContinuationRoute::Reprefill; + } + switch (forced) { + case ForcedContinuationRoute::Append: + return ContinuationRoute::Append; + case ForcedContinuationRoute::Reprefill: + return ContinuationRoute::Reprefill; + case ForcedContinuationRoute::Automatic: + return suffix_tokens <= kContinuationAppendThreshold + ? ContinuationRoute::Append + : ContinuationRoute::Reprefill; + } + return ContinuationRoute::Reprefill; +} + +inline constexpr const char* ContinuationRouteName( + ContinuationRoute route) noexcept { + return route == ContinuationRoute::Append + ? "append" + : "reprefill"; +} + +} // namespace flm::phi4 diff --git a/src/include/models/phi4/phi4_corelib_constants.hpp b/src/include/models/phi4/phi4_corelib_constants.hpp new file mode 100644 index 00000000..63f250a4 --- /dev/null +++ b/src/include/models/phi4/phi4_corelib_constants.hpp @@ -0,0 +1,55 @@ +#pragma once + +#include + +namespace flm::phi4::constants { + +inline constexpr std::int64_t kLayerCount = 32; +inline constexpr std::int64_t kHiddenSize = 3072; +inline constexpr std::int64_t kIntermediateSize = 8192; +inline constexpr std::int64_t kQueryHeadCount = 24; +inline constexpr std::int64_t kKvHeadCount = 8; +inline constexpr std::int64_t kHeadSize = 128; +inline constexpr std::int64_t kQueryDimension = 3072; +inline constexpr std::int64_t kKvDimension = 1024; +inline constexpr std::int64_t kVocabularySize = 200064; +inline constexpr std::uint32_t kGroupSize = 128; +inline constexpr std::int64_t kRopeDimension = 96; +inline constexpr std::int64_t kMaxSequenceLength = 4096; + +// The TOKEN attention path supports a smaller KV window than the prefill path. +// +// Measured on the AIE4 target 2026-09-02. A 4096-row prefill at position 0 +// dispatches fine -- the boundary sweep runs one -- but a rows=1 step at +// position 4095, whose window is also 4096, is refused by the shipped kernel +// set with "no token attention kernel ships for a 4096-token window". So the +// two paths do not share a bound, and the decode one is `kMaxSequenceLength` +// minus one. +// +// This is not a cosmetic limit. The refusal arrives from `flat_mha` AFTER q, k +// and v have been submitted in the same step. On 2026-09-02 that KILLED THE +// SERVER through a default /api/chat request, because the failure policy was +// per STEP and classified everything past the first submit as irrevocable. +// +// The policy is now per SUBMISSION GROUP: STREAM-2 puts a completed +// synchronize between the q/k/v submits and flat_mha, corelib's outstanding +// list is empty at that point, and the refusal is raised before flat_mha +// touches the device -- so the same failure is now RECOVERABLE and the +// process survives. See the note in checked_synchronize in +// phi4_corelib_aie4.cpp for why, and against which corelib sources. +// +// The bound is not thereby optional. Reaching flat_mha means this layer's V +// cache has already been scattered, so recovery costs the caller the entire +// conversation; and EnsureDecodeWindow / validate_aie4_capacity turn the same +// event into an ordinary MAX_LENGTH_REACHED truncation instead. Anything that +// can reach a decode step must respect this, not kMaxSequenceLength. +inline constexpr std::int64_t kMaxDecodeWindow = kMaxSequenceLength - 1; + +inline constexpr double kRmsEpsilon = 1.0e-5; + +static_assert( + kQueryHeadCount * kHeadSize == kQueryDimension); +static_assert(kQueryDimension == kHiddenSize); +static_assert(kKvHeadCount * kHeadSize == kKvDimension); + +} // namespace flm::phi4::constants diff --git a/src/include/models/phi4/phi4_corelib_host.hpp b/src/include/models/phi4/phi4_corelib_host.hpp new file mode 100644 index 00000000..806bb4f2 --- /dev/null +++ b/src/include/models/phi4/phi4_corelib_host.hpp @@ -0,0 +1,60 @@ +#pragma once + +#include +#include + +#include +#include +#include + +namespace flm::phi4 { + +struct VScatterMetrics { + std::uint64_t read_calls = 0; + std::uint64_t write_calls = 0; + std::uint64_t bytes = 0; + std::uint64_t nanoseconds = 0; +}; + +// Widens the gathered FP16 rows to FP32 with the `API-6` bounds-safe +// scalar helper. There is no tensor on either side of this conversion: +// the host RMSNorm consumes the result directly. +void GatherEmbedding( + std::span embedding_fp16, + std::span token_ids, + std::span output); + +void RmsNorm( + std::span input, + std::span scale, + std::int64_t rows, + std::int64_t width, + float epsilon, + std::span output); + +// Stages only the helper-required initial hidden/residual prefix, in FP32. +// Elements beyond padded_rows * width are intentionally untouched, and the +// FP32-to-BF16 narrowing is corelib's, inside tensor_write. +void StageFp32( + std::span input, + std::int64_t live_rows, + std::int64_t padded_rows, + std::int64_t width, + std::span output); + +// Precondition: the caller has successfully synchronized the Stream after +// V projection and before this host read. ScatterV deliberately owns no +// Stream and performs no synchronization. +void ScatterV( + const corelib::CorelibApi& api, + ryzenai_corelib_tensor_ptr source, + ryzenai_corelib_tensor_ptr value_cache, + std::int64_t rows, + std::int64_t position, + std::vector& staging, + VScatterMetrics& metrics); + +int ArgmaxLowest( + std::span logits); + +} // namespace flm::phi4 diff --git a/src/include/models/phi4/phi4_corelib_manifest.hpp b/src/include/models/phi4/phi4_corelib_manifest.hpp new file mode 100644 index 00000000..bbb71b50 --- /dev/null +++ b/src/include/models/phi4/phi4_corelib_manifest.hpp @@ -0,0 +1,135 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace flm::phi4 { + +class MappedFile final { +public: + static std::shared_ptr OpenReadOnly( + const std::filesystem::path& path); + + MappedFile(MappedFile&& other) noexcept; + MappedFile& operator=(MappedFile&& other) noexcept; + ~MappedFile() noexcept; + + MappedFile(const MappedFile&) = delete; + MappedFile& operator=(const MappedFile&) = delete; + + const std::byte* data() const noexcept; + std::uint64_t size() const noexcept; + const std::filesystem::path& path() const noexcept; + +private: + MappedFile( + std::filesystem::path path, + void* file, + void* mapping, + const std::byte* data, + std::uint64_t size) noexcept; + + void Reset() noexcept; + + std::filesystem::path path_; + void* file_ = nullptr; + void* mapping_ = nullptr; + const std::byte* data_ = nullptr; + std::uint64_t size_ = 0; +}; + +enum class SourceDType { + UInt8, + Float16, + Float32, + Int64 +}; + +struct InitializerView { + SourceDType dtype; + std::vector shape; + const std::byte* data; + std::size_t size; + std::shared_ptr owner; +}; + +enum class WeightObjectKind { + MatMul, + SsMlp +}; + +struct WeightObjectView { + std::string name; + WeightObjectKind kind; + std::int64_t k; + std::int64_t n; + std::uint32_t group_size; + bool has_bias; + std::map components; +}; + +// A contiguous [4096, 48] slice of a RoPE table, kept in the SOURCE dtype. +// `tensor_write` performs the widening to the FP32 device tensor, which is +// the only conversion boundary corelib `e5258d2` offers. +struct RopeSourceView { + ryzenai_corelib_data_type dtype; + const void* data; + std::size_t count; +}; + +class Phi4Package final { +public: + static Phi4Package Load( + const std::filesystem::path& model_dir, + std::shared_ptr api, + bool verify_full_hash); + + Phi4Package(Phi4Package&&) noexcept = default; + Phi4Package& operator=(Phi4Package&&) noexcept = default; + ~Phi4Package() = default; + + Phi4Package(const Phi4Package&) = delete; + Phi4Package& operator=(const Phi4Package&) = delete; + + const InitializerView& Require(std::string_view name) const; + const std::vector& weight_objects() const noexcept; + std::span MaterializeFp16( + std::string_view name); + std::span MaterializeBf16( + std::string_view name); + RopeSourceView MaterializeRopeGather(std::string_view name); + +private: + Phi4Package() = default; + + std::shared_ptr api_; + + // Declare every owner before the non-owning views. Reverse member + // destruction then releases views before their mapped/derived storage. + std::map< + std::string, + std::shared_ptr, + std::less<>> + mapped_files_; + std::map, std::less<>> + fp16_buffers_; + std::map, std::less<>> + bf16_buffers_; + std::map, std::less<>> + rope_buffers_; + + std::map> + initializers_; + std::vector weight_objects_; +}; + +} // namespace flm::phi4 diff --git a/src/include/models/phi4/phi4_corelib_shape_plan.hpp b/src/include/models/phi4/phi4_corelib_shape_plan.hpp new file mode 100644 index 00000000..c4aa8ec1 --- /dev/null +++ b/src/include/models/phi4/phi4_corelib_shape_plan.hpp @@ -0,0 +1,59 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include +#include + +namespace flm::phi4 { + +enum class RowUse { + QueryProjection, + KvProjection, + Attention, + OutputProjection, + SsMlp, + LmHead +}; + +struct MatMulPaddedShape { + std::int64_t m; + std::int64_t k; + std::int64_t n; +}; + +struct Phi4Capacities { + std::int64_t layer_rows; + std::int64_t lm_head_rows; +}; + +class Phi4ShapePlan final { +public: + static Phi4ShapePlan Build( + std::shared_ptr api); + + std::int64_t RowsFor(RowUse use, std::int64_t live_rows) const; + const std::vector>& + Transitions(RowUse use) const; + const Phi4Capacities& capacities() const noexcept; + const ryzenai_corelib_flat_mha_bf16_desc& attention_desc() + const noexcept; + +private: + static constexpr std::size_t kRowUseCount = 6; + + Phi4ShapePlan() = default; + + std::array< + std::vector>, + kRowUseCount> + transitions_; + Phi4Capacities capacities_{}; + ryzenai_corelib_flat_mha_bf16_desc attention_desc_{}; +}; + +} // namespace flm::phi4 diff --git a/src/include/models/phi4/phi4_corelib_weights.hpp b/src/include/models/phi4/phi4_corelib_weights.hpp new file mode 100644 index 00000000..5321b65e --- /dev/null +++ b/src/include/models/phi4/phi4_corelib_weights.hpp @@ -0,0 +1,53 @@ +#pragma once + +#include +#include +#include + +#include +#include +#include +#include + +namespace flm::phi4 { + +struct LayerWeights { + corelib::UniqueMatMulWeights q; + corelib::UniqueMatMulWeights k; + corelib::UniqueMatMulWeights v; + corelib::UniqueMatMulWeights o; + corelib::UniqueSsMlpWeights mlp; +}; + +class Phi4Weights final { +public: + static Phi4Weights Load( + std::shared_ptr api, + std::shared_ptr package); + + Phi4Weights(Phi4Weights&&) noexcept = default; + Phi4Weights& operator=(Phi4Weights&& other) noexcept; + + Phi4Weights(const Phi4Weights&) = delete; + Phi4Weights& operator=(const Phi4Weights&) = delete; + + const std::array& + layers() const noexcept; + const corelib::UniqueMatMulWeights& lm_head() const noexcept; + std::size_t packed_bytes() const noexcept; + +private: + Phi4Weights() = default; + + void ResetWeightObjects() noexcept; + + // Owners precede every weight that receives their addresses. Reverse + // destruction therefore releases corelib objects before source storage. + std::shared_ptr package_; + std::shared_ptr epsilon_bf16_; + std::array layers_; + corelib::UniqueMatMulWeights lm_head_; + std::size_t packed_bytes_ = 0; +}; + +} // namespace flm::phi4 diff --git a/src/include/pull/model_overlay.hpp b/src/include/pull/model_overlay.hpp new file mode 100644 index 00000000..25461d52 --- /dev/null +++ b/src/include/pull/model_overlay.hpp @@ -0,0 +1,35 @@ +#pragma once + +#include + +#include +#include +#include +#include + +namespace flm::pull { + +void RequireSupportedModelSource( + const nlohmann::json& model_info, + bool use_modelscope); + +std::vector RemoteModelFiles( + const nlohmann::json& model_info); + +std::string BuildRemoteFileUrl( + const nlohmann::json& model_info, + std::string_view filename); + +std::string CalculateFileSha256(const std::filesystem::path& path); + +void StageBundledOverlays( + const nlohmann::json& model_info, + const std::filesystem::path& overlay_root, + const std::filesystem::path& model_dir); + +bool VerifyBundledOverlayTarget( + const nlohmann::json& model_info, + std::string_view filename, + const std::filesystem::path& model_dir); + +} // namespace flm::pull diff --git a/src/include/server/generation_limit.hpp b/src/include/server/generation_limit.hpp new file mode 100644 index 00000000..4dc65d4e --- /dev/null +++ b/src/include/server/generation_limit.hpp @@ -0,0 +1,92 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include +#include + +enum class GenerationEndpoint { + Generate, + OllamaChat, + OpenAiChatCompletion, + OpenAiCompletion +}; + +// Every route whose handler reaches the causal engine's generate() or +// generate_with_prompt(), and the field each one reads its limit from. +// +// This table exists because /api/chat was missed. Three of the four +// endpoints were wired to ParseGenerationLimit by inspection, and +// inspection is what let the fourth keep its own private limit path -- +// bypassing explicit-limit detection, the admission rule, the HTTP 400 +// path and session_cleared reporting on the AIE4 tag. +struct GenerationRoute { + std::string_view method; + std::string_view path; + GenerationEndpoint endpoint; +}; + +std::span GenerationRoutes() noexcept; + +std::optional GenerationEndpointForRoute( + std::string_view method, + std::string_view path) noexcept; + +// Throws when the route is absent from the table, so a generation route +// cannot reach the engine without declaring how its limit is parsed. +GenerationEndpoint RequireGenerationEndpoint( + std::string_view method, + std::string_view path); + +struct ParsedGenerationLimit { + bool explicit_limit; + int value; +}; + +ParsedGenerationLimit ParseGenerationLimit( + const nlohmann::ordered_json& request, + GenerationEndpoint endpoint); + +int GenerationLoopLimit( + const ParsedGenerationLimit& parsed, + bool uses_corelib_aie4) noexcept; + +std::optional RequestedMaxNewTokens( + const ParsedGenerationLimit& parsed) noexcept; + +int OllamaChatGenerationLoopLimit( + const nlohmann::ordered_json& request); + +nlohmann::ordered_json ModelErrorResponse( + std::string_view message, + int http_code, + bool session_cleared); + +int HttpStatusForResponse( + const nlohmann::ordered_json& response) noexcept; + +std::array OpenAiStreamingErrorFrames( + const nlohmann::ordered_json& error_response); + +void SendOpenAiStreamingError( + const nlohmann::ordered_json& error_response, + const std::function& send_streaming_response); + +bool UseFinalStreamingErrorChunk( + bool stream_started) noexcept; + +std::optional CliRequestedMaxNewTokens( + int generate_limit) noexcept; + +std::string CliModelErrorNotice( + std::string_view message, + bool session_cleared); + +bool IsCorelibAie4ModelInfo( + const nlohmann::ordered_json& model_info) noexcept; diff --git a/src/include/server/npu_access_manager.hpp b/src/include/server/npu_access_manager.hpp new file mode 100644 index 00000000..7627be86 --- /dev/null +++ b/src/include/server/npu_access_manager.hpp @@ -0,0 +1,44 @@ +#pragma once + +#include +#include +#include + +class NPUAccessManager { +public: + static bool try_acquire_npu_access(); + static void release_npu_access(); + static bool is_npu_available(); + static int get_active_npu_requests(); +}; + +class NPURequestCompletionGuard final { +public: + explicit NPURequestCompletionGuard( + std::function completion) + : completion_(std::move(completion)) {} + + ~NPURequestCompletionGuard() noexcept { + try { + if (completion_) { + completion_(); + } + } catch (...) { + if (!NPUAccessManager::is_npu_available()) { + NPUAccessManager::release_npu_access(); + } + } + } + + NPURequestCompletionGuard( + const NPURequestCompletionGuard&) = delete; + NPURequestCompletionGuard& operator=( + const NPURequestCompletionGuard&) = delete; + +private: + std::function completion_; +}; + +bool requires_npu_access( + const std::string& method, + const std::string& path); diff --git a/src/include/server/serve_lifecycle.hpp b/src/include/server/serve_lifecycle.hpp new file mode 100644 index 00000000..f7c78b4a --- /dev/null +++ b/src/include/server/serve_lifecycle.hpp @@ -0,0 +1,44 @@ +#pragma once + +#include +#include +#include + +class ScopedSignalHandler final { +public: + using Handler = void (*)(int); + + ScopedSignalHandler(int signal_number, Handler handler) + : signal_number_(signal_number), + previous_(std::signal(signal_number, handler)) { + if (previous_ == SIG_ERR) { + throw std::runtime_error("failed to register signal handler"); + } + } + + ~ScopedSignalHandler() { + if (previous_ != SIG_ERR) { + (void)std::signal(signal_number_, previous_); + } + } + + ScopedSignalHandler(const ScopedSignalHandler&) = delete; + ScopedSignalHandler& operator=(const ScopedSignalHandler&) = delete; + +private: + int signal_number_; + Handler previous_; +}; + +template < + typename StopAndWait, + typename DestroyServer, + typename ShutdownRuntime> +bool CompleteServeShutdown( + StopAndWait&& stop_and_wait, + DestroyServer&& destroy_server, + ShutdownRuntime&& shutdown_runtime) { + std::forward(stop_and_wait)(); + std::forward(destroy_server)(); + return std::forward(shutdown_runtime)(); +} diff --git a/src/include/utils/utils.hpp b/src/include/utils/utils.hpp index 6766c9cb..3d69cf37 100644 --- a/src/include/utils/utils.hpp +++ b/src/include/utils/utils.hpp @@ -370,6 +370,10 @@ std::string find_model_list(); std::string find_model_info(); +///@brief find the root containing FastFlow-owned model overlays +///@return path to the model_overlays directory +std::string find_model_overlay_root(); + ///@brief get the path to the xclbin directory ///@return path to the xclbin directory diff --git a/src/inno/flm.iss b/src/inno/flm.iss index b5e6e886..d225cc81 100644 --- a/src/inno/flm.iss +++ b/src/inno/flm.iss @@ -4,7 +4,7 @@ AppName=flm -AppVersion=1.0.3 +AppVersion=1.0.4 AppPublisher=FastFlowLM @@ -98,6 +98,16 @@ Source: "logo.ico"; DestDir: "{app}"; Flags: ignoreversion Source: "model_list.json"; DestDir: "{app}"; Flags: ignoreversion Source: "model_info.json"; DestDir: "{app}"; Flags: ignoreversion +; Optional Phi-4 AIE4 runtime and FastFlow-owned model overlays. +; skipifsourcedoesntexist keeps the ordinary NPU2 installer buildable on a +; machine that never staged an AIE4 closure: the feature is optional, so its +; absence is a skipped entry rather than a build failure. +Source: "aie4\*"; DestDir: "{app}\aie4"; Flags: ignoreversion recursesubdirs createallsubdirs skipifsourcedoesntexist; Tasks: aie4runtime +Source: "..\model_overlays\phi4-mini-it-aie4\config.json"; DestDir: "{app}\share\flm\model_overlays\phi4-mini-it-aie4"; Flags: ignoreversion; Tasks: aie4runtime +Source: "..\model_overlays\phi4-mini-it-aie4\corelib_phi4_manifest.json"; DestDir: "{app}\share\flm\model_overlays\phi4-mini-it-aie4"; Flags: ignoreversion; Tasks: aie4runtime +Source: "..\model_overlays\phi4-mini-it-aie4\tokenizer_config.json"; DestDir: "{app}\share\flm\model_overlays\phi4-mini-it-aie4"; Flags: ignoreversion; Tasks: aie4runtime +Source: "..\model_overlays\phi4-mini-it-aie4\provenance.json"; DestDir: "{app}\share\flm\model_overlays\phi4-mini-it-aie4"; Flags: ignoreversion; Tasks: aie4runtime + ; xclbins directory - recursively include all files Source: "..\xclbins\*"; DestDir: "{app}\xclbins"; Flags: ignoreversion recursesubdirs createallsubdirs @@ -136,6 +146,7 @@ Name: "{commondesktop}\flm serve"; \ ; Optional desktop icon task Name: "desktopicon"; Description: "Create a desktop icon"; GroupDescription: "Additional icons:"; Flags: unchecked +Name: "aie4runtime"; Description: "Install optional Phi-4 AIE4 corelib runtime"; GroupDescription: "Optional features:"; Flags: unchecked [Code] var diff --git a/src/inno/get_files.bat b/src/inno/get_files.bat index d0692535..32bf9a4d 100644 --- a/src/inno/get_files.bat +++ b/src/inno/get_files.bat @@ -15,4 +15,36 @@ echo Copying model_list.json... copy "..\model_list.json" "model_list.json" copy "..\model_info.json" "model_info.json" +REM Copy the optional, derived AIE4 runtime closure when one has been staged. +REM The AIE4 feature is optional, so a missing closure is a skip and not an +REM error: requiring it here would make the AIE4 build a precondition of +REM shipping the ordinary NPU2 product. +if exist "..\build\aie4\ryzenai_corelib.dll" ( + echo Copying optional AIE4 runtime closure... + if not exist "aie4" mkdir "aie4" + xcopy "..\build\aie4\*" "aie4\" /E /I /Y + REM The closure report ships beside the runtime. It is the only thing in + REM an installed tree that says which corelib produced a given result, so + REM it is copied deliberately and its arrival is checked -- the wholesale + REM xcopy above already carries it, and that is exactly why: an incidental + REM dependency breaks silently the day somebody narrows the copy. + copy /Y "..\build\aie4\aie4-closure.txt" "aie4\aie4-closure.txt" >nul + if not exist "aie4\aie4-closure.txt" goto :aie4_missing_closure_report +) else ( + echo No AIE4 runtime closure found; building without the AIE4 feature. +) + echo Done! +exit /b 0 + +REM Reported by jumping out of the block rather than by exiting inside it. +REM cmd.exe discards the exit code of an `exit /b` that runs inside a +REM parenthesised block containing redirections -- the diagnostic prints, +REM the batch stops, and the caller still sees 0. Verified on the target: +REM the same nested `exit /b 1` returns 1 without the redirected lines and +REM 0 with them. +:aie4_missing_closure_report +echo ERROR: the staged AIE4 closure has no aie4-closure.txt, so the +echo installed runtime would carry no provenance. Re-run the AIE4 +echo staging step: cmake -P src/cmake/StageAie4Runtime.cmake +exit /b 1 diff --git a/src/model_info.json b/src/model_info.json index cb62db94..126a31c1 100644 --- a/src/model_info.json +++ b/src/model_info.json @@ -3300,5 +3300,88 @@ "xetHash": "7c4d2da22b3de2ed3f3eae66c7034386df6a3c5d81039ad4a1c8067e7eaf0069", "path": "vision_weights.q4nx" } + ], + "phi4-mini-it-aie4:4b": [ + { + "type": "file", + "oid": "f78df3bfb43291872abf78496110e5856b24de73", + "size": 1622, + "path": ".gitattributes" + }, + { + "type": "file", + "oid": "77e6ed402865af3c38b5e44c2d0be68f76f0784f", + "size": 261, + "path": "added_tokens.json" + }, + { + "type": "file", + "oid": "a9c00dd9bbd97e117371168e9d62af65b9f0e725", + "size": 423, + "path": "chat_template.jinja" + }, + { + "type": "file", + "oid": "55b698edc57fe52593dce4423beb51ded982dc96", + "size": 1720, + "path": "genai_config.json" + }, + { + "type": "file", + "oid": "dcecc4524288b351bbd0da8028e74e9b5bcdb9b5", + "size": 2418348, + "path": "merges.txt" + }, + { + "type": "file", + "oid": "df6d4309745d4627b82fd92c615589193ea528db", + "size": 378325, + "lfs": { + "oid": "e80b9d83e018784eda6263d09fa2ab7729087722c6073c72123366f2dbec4529", + "size": 378325, + "pointerSize": 131 + }, + "path": "model.onnx" + }, + { + "type": "file", + "oid": "5ef8f7f74cf6ac6411b8918e8a850bd2c769dd45", + "size": 3248488448, + "lfs": { + "oid": "c48fd647beb02866e68d6dd1fbc05809a439283501d8abb7f6d6b950f92b7b60", + "size": 3248488448, + "pointerSize": 135 + }, + "path": "model.onnx.data" + }, + { + "type": "file", + "oid": "18eba67aba3ef71b01ed13c16b3feed78e83001f", + "size": 617, + "path": "special_tokens_map.json" + }, + { + "type": "file", + "oid": "3a12dacd8e86802d0229d810d3cc69ab548adf0a", + "size": 15524095, + "lfs": { + "oid": "382cc235b56c725945e149cc25f191da667c836655efd0857b004320e90e91ea", + "size": 15524095, + "pointerSize": 133 + }, + "path": "tokenizer.json" + }, + { + "type": "file", + "oid": "c182c54743fe1735b0d5eb3959b9757a160879e8", + "size": 2654, + "path": "tokenizer_config.json" + }, + { + "type": "file", + "oid": "ea953a43348cdb3776cb7fd9ea02e3784febde34", + "size": 3910310, + "path": "vocab.json" + } ] } \ No newline at end of file diff --git a/src/model_list.json b/src/model_list.json index d2c7b656..9ff5b780 100644 --- a/src/model_list.json +++ b/src/model_list.json @@ -469,6 +469,65 @@ "footprint": 3.4 } }, + "phi4-mini-it-aie4": { + "4b": { + "name": "Phi-4-mini-instruct-oga-dml-AIE4", + "url": "https://huggingface.co/amd/phi-4-mini-instruct-oga-dml", + "revision": "e751fb68c2cfffe6b0d32942118f75ac0a0365bb", + "file_url": "https://huggingface.co/api/models/amd/phi-4-mini-instruct-oga-dml/tree/e751fb68c2cfffe6b0d32942118f75ac0a0365bb?recursive=true&expand=false", + "size": 3271001977, + "default_context_length": 4096, + "max_prefill_len": 4096, + "details": { + "family": "phi4", + "think": false, + "think_toggleable": false, + "parameter_size": "4B", + "quantization_level": "MatMulNBits Q4", + "execution_backend": "corelib_aie4" + }, + "flm_min_version": "1.0.4", + "vlm": false, + "modelscope_supported": false, + "files": [ + "added_tokens.json", + "chat_template.jinja", + "config.json", + "corelib_phi4_manifest.json", + "merges.txt", + "model.onnx", + "model.onnx.data", + "provenance.json", + "special_tokens_map.json", + "tokenizer.json", + "tokenizer_config.json", + "vocab.json" + ], + "bundled_overlays": { + "config.json": { + "path": "phi4-mini-it-aie4/config.json", + "size": 257, + "sha256": "1b3e74125a109c05f53c8383def18359d8581619f998c4f91b3b5bb78bf2919f" + }, + "corelib_phi4_manifest.json": { + "path": "phi4-mini-it-aie4/corelib_phi4_manifest.json", + "size": 274527, + "sha256": "09cee6efafc513a2048c89e75b40d92d096144f0eed3b2459c138032b69dc045" + }, + "provenance.json": { + "path": "phi4-mini-it-aie4/provenance.json", + "size": 3330, + "sha256": "f610a719bece3c40cad47e15049be2ab285495c5ea56d1b2b73ad6293f53cc86" + }, + "tokenizer_config.json": { + "path": "phi4-mini-it-aie4/tokenizer_config.json", + "size": 3036, + "sha256": "274d22c3cd28c042f28a681832722536663b06e8c9e88b6616622dceab922ca6" + } + }, + "footprint": 3.05 + } + }, "embed-gemma": { "300m": { "name": "Embedding-Gemma-300M-NPU2", diff --git a/src/model_overlays/phi4-mini-it-aie4/config.json b/src/model_overlays/phi4-mini-it-aie4/config.json new file mode 100644 index 00000000..abf9a3d7 --- /dev/null +++ b/src/model_overlays/phi4-mini-it-aie4/config.json @@ -0,0 +1,12 @@ +{ + "flm_version": "1.0.4", + "head_dim": 128, + "hidden_size": 3072, + "intermediate_size": 8192, + "model_type": "phi4", + "num_attention_heads": 24, + "num_hidden_layers": 32, + "num_key_value_heads": 8, + "rms_norm_eps": 1e-05, + "vocab_size": 200064 +} diff --git a/src/model_overlays/phi4-mini-it-aie4/corelib_phi4_manifest.json b/src/model_overlays/phi4-mini-it-aie4/corelib_phi4_manifest.json new file mode 100644 index 00000000..10935371 --- /dev/null +++ b/src/model_overlays/phi4-mini-it-aie4/corelib_phi4_manifest.json @@ -0,0 +1,10556 @@ +{ + "backend": { + "max_seq": 4096 + }, + "execution_backend": "corelib_aie4", + "files": { + "model.onnx": { + "sha256": "e80b9d83e018784eda6263d09fa2ab7729087722c6073c72123366f2dbec4529", + "size": 378325 + }, + "model.onnx.data": { + "sha256": "c48fd647beb02866e68d6dd1fbc05809a439283501d8abb7f6d6b950f92b7b60", + "size": 3248488448 + } + }, + "initializers": { + "cos_cache": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 12976128, + "offset": 1686044672, + "role": "cos_cache", + "shape": [ + 135168, + 48 + ] + }, + "lm_head.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 307298304, + "offset": 1711996928, + "role": "matmul.qweight", + "shape": [ + 200064, + 24, + 64 + ] + }, + "lm_head.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 2400768, + "offset": 164036608, + "role": "matmul.qzeros", + "shape": [ + 2400768 + ] + }, + "lm_head.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 9603072, + "offset": 468451328, + "role": "matmul.scales", + "shape": [ + 4801536 + ] + }, + "model.embed_tokens.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 1229193216, + "offset": 2019295232, + "role": "embedding", + "shape": [ + 200064, + 3072 + ] + }, + "model.layers.0.attn.k_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 63373312, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.0.attn.k_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 399360, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.0.attn.k_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 3545088, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.0.attn.o_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 171180032, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.0.attn.o_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 1222656, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.0.attn.o_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 16275456, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.0.attn.q_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 166461440, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.0.attn.q_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 1185792, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.0.attn.q_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 16128000, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.0.attn.v_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 64946176, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.0.attn.v_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 411648, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.0.attn.v_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 3594240, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.0.input_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 0, + "role": "input_norm", + "shape": [ + 3072 + ] + }, + "model.layers.0.mlp.down_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 503250944, + "role": "ssmlp.down.qweight", + "shape": [ + 3072, + 64, + 64 + ] + }, + "model.layers.0.mlp.down_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 6887424, + "role": "ssmlp.down.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.0.mlp.down_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 26351616, + "role": "ssmlp.down.scales", + "shape": [ + 196608 + ] + }, + "model.layers.0.mlp.gate_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 478085120, + "role": "ssmlp.gate.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.0.mlp.gate_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 6690816, + "role": "ssmlp.gate.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.0.mlp.gate_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 25565184, + "role": "ssmlp.gate.scales", + "shape": [ + 196608 + ] + }, + "model.layers.0.mlp.up_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 490668032, + "role": "ssmlp.up.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.0.mlp.up_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 6789120, + "role": "ssmlp.up.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.0.mlp.up_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 25958400, + "role": "ssmlp.up.scales", + "shape": [ + 196608 + ] + }, + "model.layers.0.post_attention_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 6144, + "role": "ssmlp.norm0", + "shape": [ + 3072 + ] + }, + "model.layers.1.attn.k_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 66519040, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.1.attn.k_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 423936, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.1.attn.k_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 3643392, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.1.attn.o_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 180617216, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.1.attn.o_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 1296384, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.1.attn.o_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 16570368, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.1.attn.q_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 175898624, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.1.attn.q_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 1259520, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.1.attn.q_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 16422912, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.1.attn.v_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 68091904, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.1.attn.v_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 436224, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.1.attn.v_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 3692544, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.1.input_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 12288, + "role": "ssmlp.norm1", + "shape": [ + 3072 + ] + }, + "model.layers.1.mlp.down_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 540999680, + "role": "ssmlp.down.qweight", + "shape": [ + 3072, + 64, + 64 + ] + }, + "model.layers.1.mlp.down_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 7182336, + "role": "ssmlp.down.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.1.mlp.down_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 27531264, + "role": "ssmlp.down.scales", + "shape": [ + 196608 + ] + }, + "model.layers.1.mlp.gate_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 515833856, + "role": "ssmlp.gate.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.1.mlp.gate_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 6985728, + "role": "ssmlp.gate.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.1.mlp.gate_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 26744832, + "role": "ssmlp.gate.scales", + "shape": [ + 196608 + ] + }, + "model.layers.1.mlp.up_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 528416768, + "role": "ssmlp.up.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.1.mlp.up_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 7084032, + "role": "ssmlp.up.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.1.mlp.up_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 27138048, + "role": "ssmlp.up.scales", + "shape": [ + 196608 + ] + }, + "model.layers.1.post_attention_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 18432, + "role": "ssmlp.norm0", + "shape": [ + 3072 + ] + }, + "model.layers.10.attn.k_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 94830592, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.10.attn.k_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 645120, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.10.attn.k_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 4528128, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.10.attn.o_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 265551872, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.10.attn.o_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 1959936, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.10.attn.o_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 19224576, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.10.attn.q_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 260833280, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.10.attn.q_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 1923072, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.10.attn.q_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 19077120, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.10.attn.v_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 96403456, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.10.attn.v_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 657408, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.10.attn.v_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 4577280, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.10.input_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 122880, + "role": "ssmlp.norm1", + "shape": [ + 3072 + ] + }, + "model.layers.10.mlp.down_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 880738304, + "role": "ssmlp.down.qweight", + "shape": [ + 3072, + 64, + 64 + ] + }, + "model.layers.10.mlp.down_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 9836544, + "role": "ssmlp.down.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.10.mlp.down_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 38148096, + "role": "ssmlp.down.scales", + "shape": [ + 196608 + ] + }, + "model.layers.10.mlp.gate_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 855572480, + "role": "ssmlp.gate.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.10.mlp.gate_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 9639936, + "role": "ssmlp.gate.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.10.mlp.gate_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 37361664, + "role": "ssmlp.gate.scales", + "shape": [ + 196608 + ] + }, + "model.layers.10.mlp.up_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 868155392, + "role": "ssmlp.up.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.10.mlp.up_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 9738240, + "role": "ssmlp.up.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.10.mlp.up_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 37754880, + "role": "ssmlp.up.scales", + "shape": [ + 196608 + ] + }, + "model.layers.10.post_attention_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 129024, + "role": "ssmlp.norm0", + "shape": [ + 3072 + ] + }, + "model.layers.11.attn.k_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 97976320, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.11.attn.k_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 669696, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.11.attn.k_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 4626432, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.11.attn.o_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 274989056, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.11.attn.o_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 2033664, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.11.attn.o_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 19519488, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.11.attn.q_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 270270464, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.11.attn.q_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 1996800, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.11.attn.q_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 19372032, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.11.attn.v_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 99549184, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.11.attn.v_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 681984, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.11.attn.v_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 4675584, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.11.input_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 135168, + "role": "ssmlp.norm1", + "shape": [ + 3072 + ] + }, + "model.layers.11.mlp.down_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 918487040, + "role": "ssmlp.down.qweight", + "shape": [ + 3072, + 64, + 64 + ] + }, + "model.layers.11.mlp.down_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 10131456, + "role": "ssmlp.down.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.11.mlp.down_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 39327744, + "role": "ssmlp.down.scales", + "shape": [ + 196608 + ] + }, + "model.layers.11.mlp.gate_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 893321216, + "role": "ssmlp.gate.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.11.mlp.gate_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 9934848, + "role": "ssmlp.gate.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.11.mlp.gate_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 38541312, + "role": "ssmlp.gate.scales", + "shape": [ + 196608 + ] + }, + "model.layers.11.mlp.up_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 905904128, + "role": "ssmlp.up.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.11.mlp.up_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 10033152, + "role": "ssmlp.up.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.11.mlp.up_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 38934528, + "role": "ssmlp.up.scales", + "shape": [ + 196608 + ] + }, + "model.layers.11.post_attention_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 141312, + "role": "ssmlp.norm0", + "shape": [ + 3072 + ] + }, + "model.layers.12.attn.k_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 101122048, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.12.attn.k_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 694272, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.12.attn.k_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 4724736, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.12.attn.o_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 284426240, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.12.attn.o_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 2107392, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.12.attn.o_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 19814400, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.12.attn.q_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 279707648, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.12.attn.q_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 2070528, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.12.attn.q_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 19666944, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.12.attn.v_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 102694912, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.12.attn.v_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 706560, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.12.attn.v_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 4773888, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.12.input_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 147456, + "role": "ssmlp.norm1", + "shape": [ + 3072 + ] + }, + "model.layers.12.mlp.down_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 956235776, + "role": "ssmlp.down.qweight", + "shape": [ + 3072, + 64, + 64 + ] + }, + "model.layers.12.mlp.down_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 10426368, + "role": "ssmlp.down.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.12.mlp.down_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 40507392, + "role": "ssmlp.down.scales", + "shape": [ + 196608 + ] + }, + "model.layers.12.mlp.gate_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 931069952, + "role": "ssmlp.gate.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.12.mlp.gate_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 10229760, + "role": "ssmlp.gate.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.12.mlp.gate_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 39720960, + "role": "ssmlp.gate.scales", + "shape": [ + 196608 + ] + }, + "model.layers.12.mlp.up_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 943652864, + "role": "ssmlp.up.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.12.mlp.up_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 10328064, + "role": "ssmlp.up.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.12.mlp.up_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 40114176, + "role": "ssmlp.up.scales", + "shape": [ + 196608 + ] + }, + "model.layers.12.post_attention_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 153600, + "role": "ssmlp.norm0", + "shape": [ + 3072 + ] + }, + "model.layers.13.attn.k_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 104267776, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.13.attn.k_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 718848, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.13.attn.k_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 4823040, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.13.attn.o_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 293863424, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.13.attn.o_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 2181120, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.13.attn.o_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 20109312, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.13.attn.q_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 289144832, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.13.attn.q_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 2144256, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.13.attn.q_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 19961856, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.13.attn.v_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 105840640, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.13.attn.v_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 731136, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.13.attn.v_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 4872192, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.13.input_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 159744, + "role": "ssmlp.norm1", + "shape": [ + 3072 + ] + }, + "model.layers.13.mlp.down_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 993984512, + "role": "ssmlp.down.qweight", + "shape": [ + 3072, + 64, + 64 + ] + }, + "model.layers.13.mlp.down_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 10721280, + "role": "ssmlp.down.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.13.mlp.down_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 41687040, + "role": "ssmlp.down.scales", + "shape": [ + 196608 + ] + }, + "model.layers.13.mlp.gate_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 968818688, + "role": "ssmlp.gate.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.13.mlp.gate_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 10524672, + "role": "ssmlp.gate.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.13.mlp.gate_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 40900608, + "role": "ssmlp.gate.scales", + "shape": [ + 196608 + ] + }, + "model.layers.13.mlp.up_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 981401600, + "role": "ssmlp.up.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.13.mlp.up_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 10622976, + "role": "ssmlp.up.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.13.mlp.up_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 41293824, + "role": "ssmlp.up.scales", + "shape": [ + 196608 + ] + }, + "model.layers.13.post_attention_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 165888, + "role": "ssmlp.norm0", + "shape": [ + 3072 + ] + }, + "model.layers.14.attn.k_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 107413504, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.14.attn.k_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 743424, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.14.attn.k_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 4921344, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.14.attn.o_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 303300608, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.14.attn.o_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 2254848, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.14.attn.o_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 20404224, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.14.attn.q_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 298582016, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.14.attn.q_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 2217984, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.14.attn.q_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 20256768, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.14.attn.v_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 108986368, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.14.attn.v_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 755712, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.14.attn.v_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 4970496, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.14.input_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 172032, + "role": "ssmlp.norm1", + "shape": [ + 3072 + ] + }, + "model.layers.14.mlp.down_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1031733248, + "role": "ssmlp.down.qweight", + "shape": [ + 3072, + 64, + 64 + ] + }, + "model.layers.14.mlp.down_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 11016192, + "role": "ssmlp.down.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.14.mlp.down_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 42866688, + "role": "ssmlp.down.scales", + "shape": [ + 196608 + ] + }, + "model.layers.14.mlp.gate_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1006567424, + "role": "ssmlp.gate.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.14.mlp.gate_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 10819584, + "role": "ssmlp.gate.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.14.mlp.gate_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 42080256, + "role": "ssmlp.gate.scales", + "shape": [ + 196608 + ] + }, + "model.layers.14.mlp.up_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1019150336, + "role": "ssmlp.up.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.14.mlp.up_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 10917888, + "role": "ssmlp.up.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.14.mlp.up_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 42473472, + "role": "ssmlp.up.scales", + "shape": [ + 196608 + ] + }, + "model.layers.14.post_attention_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 178176, + "role": "ssmlp.norm0", + "shape": [ + 3072 + ] + }, + "model.layers.15.attn.k_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 110559232, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.15.attn.k_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 768000, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.15.attn.k_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 5019648, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.15.attn.o_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 312737792, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.15.attn.o_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 2328576, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.15.attn.o_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 20699136, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.15.attn.q_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 308019200, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.15.attn.q_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 2291712, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.15.attn.q_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 20551680, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.15.attn.v_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 112132096, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.15.attn.v_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 780288, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.15.attn.v_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 5068800, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.15.input_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 184320, + "role": "ssmlp.norm1", + "shape": [ + 3072 + ] + }, + "model.layers.15.mlp.down_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1069481984, + "role": "ssmlp.down.qweight", + "shape": [ + 3072, + 64, + 64 + ] + }, + "model.layers.15.mlp.down_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 11311104, + "role": "ssmlp.down.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.15.mlp.down_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 44046336, + "role": "ssmlp.down.scales", + "shape": [ + 196608 + ] + }, + "model.layers.15.mlp.gate_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1044316160, + "role": "ssmlp.gate.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.15.mlp.gate_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 11114496, + "role": "ssmlp.gate.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.15.mlp.gate_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 43259904, + "role": "ssmlp.gate.scales", + "shape": [ + 196608 + ] + }, + "model.layers.15.mlp.up_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1056899072, + "role": "ssmlp.up.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.15.mlp.up_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 11212800, + "role": "ssmlp.up.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.15.mlp.up_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 43653120, + "role": "ssmlp.up.scales", + "shape": [ + 196608 + ] + }, + "model.layers.15.post_attention_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 190464, + "role": "ssmlp.norm0", + "shape": [ + 3072 + ] + }, + "model.layers.16.attn.k_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 113704960, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.16.attn.k_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 792576, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.16.attn.k_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 5117952, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.16.attn.o_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 322174976, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.16.attn.o_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 2402304, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.16.attn.o_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 20994048, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.16.attn.q_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 317456384, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.16.attn.q_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 2365440, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.16.attn.q_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 20846592, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.16.attn.v_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 115277824, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.16.attn.v_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 804864, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.16.attn.v_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 5167104, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.16.input_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 196608, + "role": "ssmlp.norm1", + "shape": [ + 3072 + ] + }, + "model.layers.16.mlp.down_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1107230720, + "role": "ssmlp.down.qweight", + "shape": [ + 3072, + 64, + 64 + ] + }, + "model.layers.16.mlp.down_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 11606016, + "role": "ssmlp.down.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.16.mlp.down_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 45225984, + "role": "ssmlp.down.scales", + "shape": [ + 196608 + ] + }, + "model.layers.16.mlp.gate_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1082064896, + "role": "ssmlp.gate.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.16.mlp.gate_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 11409408, + "role": "ssmlp.gate.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.16.mlp.gate_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 44439552, + "role": "ssmlp.gate.scales", + "shape": [ + 196608 + ] + }, + "model.layers.16.mlp.up_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1094647808, + "role": "ssmlp.up.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.16.mlp.up_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 11507712, + "role": "ssmlp.up.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.16.mlp.up_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 44832768, + "role": "ssmlp.up.scales", + "shape": [ + 196608 + ] + }, + "model.layers.16.post_attention_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 202752, + "role": "ssmlp.norm0", + "shape": [ + 3072 + ] + }, + "model.layers.17.attn.k_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 116850688, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.17.attn.k_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 817152, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.17.attn.k_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 5216256, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.17.attn.o_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 331612160, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.17.attn.o_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 2476032, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.17.attn.o_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 21288960, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.17.attn.q_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 326893568, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.17.attn.q_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 2439168, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.17.attn.q_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 21141504, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.17.attn.v_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 118423552, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.17.attn.v_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 829440, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.17.attn.v_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 5265408, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.17.input_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 208896, + "role": "ssmlp.norm1", + "shape": [ + 3072 + ] + }, + "model.layers.17.mlp.down_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1144979456, + "role": "ssmlp.down.qweight", + "shape": [ + 3072, + 64, + 64 + ] + }, + "model.layers.17.mlp.down_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 11900928, + "role": "ssmlp.down.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.17.mlp.down_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 46405632, + "role": "ssmlp.down.scales", + "shape": [ + 196608 + ] + }, + "model.layers.17.mlp.gate_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1119813632, + "role": "ssmlp.gate.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.17.mlp.gate_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 11704320, + "role": "ssmlp.gate.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.17.mlp.gate_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 45619200, + "role": "ssmlp.gate.scales", + "shape": [ + 196608 + ] + }, + "model.layers.17.mlp.up_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1132396544, + "role": "ssmlp.up.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.17.mlp.up_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 11802624, + "role": "ssmlp.up.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.17.mlp.up_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 46012416, + "role": "ssmlp.up.scales", + "shape": [ + 196608 + ] + }, + "model.layers.17.post_attention_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 215040, + "role": "ssmlp.norm0", + "shape": [ + 3072 + ] + }, + "model.layers.18.attn.k_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 119996416, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.18.attn.k_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 841728, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.18.attn.k_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 5314560, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.18.attn.o_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 341049344, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.18.attn.o_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 2549760, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.18.attn.o_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 21583872, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.18.attn.q_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 336330752, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.18.attn.q_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 2512896, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.18.attn.q_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 21436416, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.18.attn.v_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 121569280, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.18.attn.v_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 854016, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.18.attn.v_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 5363712, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.18.input_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 221184, + "role": "ssmlp.norm1", + "shape": [ + 3072 + ] + }, + "model.layers.18.mlp.down_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1182728192, + "role": "ssmlp.down.qweight", + "shape": [ + 3072, + 64, + 64 + ] + }, + "model.layers.18.mlp.down_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 12195840, + "role": "ssmlp.down.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.18.mlp.down_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 47585280, + "role": "ssmlp.down.scales", + "shape": [ + 196608 + ] + }, + "model.layers.18.mlp.gate_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1157562368, + "role": "ssmlp.gate.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.18.mlp.gate_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 11999232, + "role": "ssmlp.gate.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.18.mlp.gate_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 46798848, + "role": "ssmlp.gate.scales", + "shape": [ + 196608 + ] + }, + "model.layers.18.mlp.up_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1170145280, + "role": "ssmlp.up.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.18.mlp.up_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 12097536, + "role": "ssmlp.up.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.18.mlp.up_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 47192064, + "role": "ssmlp.up.scales", + "shape": [ + 196608 + ] + }, + "model.layers.18.post_attention_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 227328, + "role": "ssmlp.norm0", + "shape": [ + 3072 + ] + }, + "model.layers.19.attn.k_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 123142144, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.19.attn.k_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 866304, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.19.attn.k_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 5412864, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.19.attn.o_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 350486528, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.19.attn.o_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 2623488, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.19.attn.o_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 21878784, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.19.attn.q_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 345767936, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.19.attn.q_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 2586624, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.19.attn.q_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 21731328, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.19.attn.v_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 124715008, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.19.attn.v_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 878592, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.19.attn.v_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 5462016, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.19.input_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 233472, + "role": "ssmlp.norm1", + "shape": [ + 3072 + ] + }, + "model.layers.19.mlp.down_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1220476928, + "role": "ssmlp.down.qweight", + "shape": [ + 3072, + 64, + 64 + ] + }, + "model.layers.19.mlp.down_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 12490752, + "role": "ssmlp.down.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.19.mlp.down_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 48764928, + "role": "ssmlp.down.scales", + "shape": [ + 196608 + ] + }, + "model.layers.19.mlp.gate_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1195311104, + "role": "ssmlp.gate.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.19.mlp.gate_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 12294144, + "role": "ssmlp.gate.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.19.mlp.gate_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 47978496, + "role": "ssmlp.gate.scales", + "shape": [ + 196608 + ] + }, + "model.layers.19.mlp.up_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1207894016, + "role": "ssmlp.up.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.19.mlp.up_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 12392448, + "role": "ssmlp.up.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.19.mlp.up_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 48371712, + "role": "ssmlp.up.scales", + "shape": [ + 196608 + ] + }, + "model.layers.19.post_attention_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 239616, + "role": "ssmlp.norm0", + "shape": [ + 3072 + ] + }, + "model.layers.2.attn.k_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 69664768, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.2.attn.k_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 448512, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.2.attn.k_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 3741696, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.2.attn.o_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 190054400, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.2.attn.o_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 1370112, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.2.attn.o_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 16865280, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.2.attn.q_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 185335808, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.2.attn.q_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 1333248, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.2.attn.q_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 16717824, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.2.attn.v_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 71237632, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.2.attn.v_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 460800, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.2.attn.v_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 3790848, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.2.input_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 24576, + "role": "ssmlp.norm1", + "shape": [ + 3072 + ] + }, + "model.layers.2.mlp.down_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 578748416, + "role": "ssmlp.down.qweight", + "shape": [ + 3072, + 64, + 64 + ] + }, + "model.layers.2.mlp.down_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 7477248, + "role": "ssmlp.down.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.2.mlp.down_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 28710912, + "role": "ssmlp.down.scales", + "shape": [ + 196608 + ] + }, + "model.layers.2.mlp.gate_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 553582592, + "role": "ssmlp.gate.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.2.mlp.gate_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 7280640, + "role": "ssmlp.gate.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.2.mlp.gate_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 27924480, + "role": "ssmlp.gate.scales", + "shape": [ + 196608 + ] + }, + "model.layers.2.mlp.up_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 566165504, + "role": "ssmlp.up.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.2.mlp.up_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 7378944, + "role": "ssmlp.up.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.2.mlp.up_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 28317696, + "role": "ssmlp.up.scales", + "shape": [ + 196608 + ] + }, + "model.layers.2.post_attention_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 30720, + "role": "ssmlp.norm0", + "shape": [ + 3072 + ] + }, + "model.layers.20.attn.k_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 126287872, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.20.attn.k_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 890880, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.20.attn.k_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 5511168, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.20.attn.o_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 359923712, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.20.attn.o_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 2697216, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.20.attn.o_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 22173696, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.20.attn.q_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 355205120, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.20.attn.q_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 2660352, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.20.attn.q_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 22026240, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.20.attn.v_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 127860736, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.20.attn.v_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 903168, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.20.attn.v_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 5560320, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.20.input_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 245760, + "role": "ssmlp.norm1", + "shape": [ + 3072 + ] + }, + "model.layers.20.mlp.down_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1258225664, + "role": "ssmlp.down.qweight", + "shape": [ + 3072, + 64, + 64 + ] + }, + "model.layers.20.mlp.down_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 12785664, + "role": "ssmlp.down.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.20.mlp.down_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 49944576, + "role": "ssmlp.down.scales", + "shape": [ + 196608 + ] + }, + "model.layers.20.mlp.gate_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1233059840, + "role": "ssmlp.gate.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.20.mlp.gate_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 12589056, + "role": "ssmlp.gate.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.20.mlp.gate_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 49158144, + "role": "ssmlp.gate.scales", + "shape": [ + 196608 + ] + }, + "model.layers.20.mlp.up_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1245642752, + "role": "ssmlp.up.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.20.mlp.up_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 12687360, + "role": "ssmlp.up.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.20.mlp.up_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 49551360, + "role": "ssmlp.up.scales", + "shape": [ + 196608 + ] + }, + "model.layers.20.post_attention_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 251904, + "role": "ssmlp.norm0", + "shape": [ + 3072 + ] + }, + "model.layers.21.attn.k_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 129433600, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.21.attn.k_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 915456, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.21.attn.k_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 5609472, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.21.attn.o_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 369360896, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.21.attn.o_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 2770944, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.21.attn.o_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 22468608, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.21.attn.q_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 364642304, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.21.attn.q_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 2734080, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.21.attn.q_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 22321152, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.21.attn.v_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 131006464, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.21.attn.v_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 927744, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.21.attn.v_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 5658624, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.21.input_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 258048, + "role": "ssmlp.norm1", + "shape": [ + 3072 + ] + }, + "model.layers.21.mlp.down_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1295974400, + "role": "ssmlp.down.qweight", + "shape": [ + 3072, + 64, + 64 + ] + }, + "model.layers.21.mlp.down_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 13080576, + "role": "ssmlp.down.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.21.mlp.down_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 51124224, + "role": "ssmlp.down.scales", + "shape": [ + 196608 + ] + }, + "model.layers.21.mlp.gate_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1270808576, + "role": "ssmlp.gate.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.21.mlp.gate_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 12883968, + "role": "ssmlp.gate.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.21.mlp.gate_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 50337792, + "role": "ssmlp.gate.scales", + "shape": [ + 196608 + ] + }, + "model.layers.21.mlp.up_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1283391488, + "role": "ssmlp.up.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.21.mlp.up_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 12982272, + "role": "ssmlp.up.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.21.mlp.up_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 50731008, + "role": "ssmlp.up.scales", + "shape": [ + 196608 + ] + }, + "model.layers.21.post_attention_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 264192, + "role": "ssmlp.norm0", + "shape": [ + 3072 + ] + }, + "model.layers.22.attn.k_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 132579328, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.22.attn.k_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 940032, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.22.attn.k_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 5707776, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.22.attn.o_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 378798080, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.22.attn.o_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 2844672, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.22.attn.o_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 22763520, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.22.attn.q_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 374079488, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.22.attn.q_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 2807808, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.22.attn.q_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 22616064, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.22.attn.v_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 134152192, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.22.attn.v_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 952320, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.22.attn.v_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 5756928, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.22.input_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 270336, + "role": "ssmlp.norm1", + "shape": [ + 3072 + ] + }, + "model.layers.22.mlp.down_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1333723136, + "role": "ssmlp.down.qweight", + "shape": [ + 3072, + 64, + 64 + ] + }, + "model.layers.22.mlp.down_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 13375488, + "role": "ssmlp.down.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.22.mlp.down_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 52303872, + "role": "ssmlp.down.scales", + "shape": [ + 196608 + ] + }, + "model.layers.22.mlp.gate_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1308557312, + "role": "ssmlp.gate.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.22.mlp.gate_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 13178880, + "role": "ssmlp.gate.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.22.mlp.gate_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 51517440, + "role": "ssmlp.gate.scales", + "shape": [ + 196608 + ] + }, + "model.layers.22.mlp.up_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1321140224, + "role": "ssmlp.up.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.22.mlp.up_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 13277184, + "role": "ssmlp.up.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.22.mlp.up_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 51910656, + "role": "ssmlp.up.scales", + "shape": [ + 196608 + ] + }, + "model.layers.22.post_attention_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 276480, + "role": "ssmlp.norm0", + "shape": [ + 3072 + ] + }, + "model.layers.23.attn.k_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 135725056, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.23.attn.k_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 964608, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.23.attn.k_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 5806080, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.23.attn.o_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 388235264, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.23.attn.o_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 2918400, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.23.attn.o_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 23058432, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.23.attn.q_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 383516672, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.23.attn.q_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 2881536, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.23.attn.q_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 22910976, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.23.attn.v_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 137297920, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.23.attn.v_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 976896, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.23.attn.v_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 5855232, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.23.input_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 282624, + "role": "ssmlp.norm1", + "shape": [ + 3072 + ] + }, + "model.layers.23.mlp.down_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1371471872, + "role": "ssmlp.down.qweight", + "shape": [ + 3072, + 64, + 64 + ] + }, + "model.layers.23.mlp.down_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 13670400, + "role": "ssmlp.down.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.23.mlp.down_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 53483520, + "role": "ssmlp.down.scales", + "shape": [ + 196608 + ] + }, + "model.layers.23.mlp.gate_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1346306048, + "role": "ssmlp.gate.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.23.mlp.gate_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 13473792, + "role": "ssmlp.gate.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.23.mlp.gate_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 52697088, + "role": "ssmlp.gate.scales", + "shape": [ + 196608 + ] + }, + "model.layers.23.mlp.up_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1358888960, + "role": "ssmlp.up.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.23.mlp.up_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 13572096, + "role": "ssmlp.up.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.23.mlp.up_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 53090304, + "role": "ssmlp.up.scales", + "shape": [ + 196608 + ] + }, + "model.layers.23.post_attention_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 288768, + "role": "ssmlp.norm0", + "shape": [ + 3072 + ] + }, + "model.layers.24.attn.k_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 138870784, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.24.attn.k_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 989184, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.24.attn.k_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 5904384, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.24.attn.o_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 397672448, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.24.attn.o_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 2992128, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.24.attn.o_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 23353344, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.24.attn.q_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 392953856, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.24.attn.q_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 2955264, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.24.attn.q_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 23205888, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.24.attn.v_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 140443648, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.24.attn.v_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 1001472, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.24.attn.v_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 5953536, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.24.input_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 294912, + "role": "ssmlp.norm1", + "shape": [ + 3072 + ] + }, + "model.layers.24.mlp.down_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1409220608, + "role": "ssmlp.down.qweight", + "shape": [ + 3072, + 64, + 64 + ] + }, + "model.layers.24.mlp.down_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 13965312, + "role": "ssmlp.down.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.24.mlp.down_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 54663168, + "role": "ssmlp.down.scales", + "shape": [ + 196608 + ] + }, + "model.layers.24.mlp.gate_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1384054784, + "role": "ssmlp.gate.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.24.mlp.gate_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 13768704, + "role": "ssmlp.gate.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.24.mlp.gate_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 53876736, + "role": "ssmlp.gate.scales", + "shape": [ + 196608 + ] + }, + "model.layers.24.mlp.up_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1396637696, + "role": "ssmlp.up.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.24.mlp.up_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 13867008, + "role": "ssmlp.up.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.24.mlp.up_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 54269952, + "role": "ssmlp.up.scales", + "shape": [ + 196608 + ] + }, + "model.layers.24.post_attention_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 301056, + "role": "ssmlp.norm0", + "shape": [ + 3072 + ] + }, + "model.layers.25.attn.k_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 142016512, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.25.attn.k_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 1013760, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.25.attn.k_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 6002688, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.25.attn.o_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 407109632, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.25.attn.o_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 3065856, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.25.attn.o_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 23648256, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.25.attn.q_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 402391040, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.25.attn.q_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 3028992, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.25.attn.q_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 23500800, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.25.attn.v_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 143589376, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.25.attn.v_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 1026048, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.25.attn.v_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 6051840, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.25.input_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 307200, + "role": "ssmlp.norm1", + "shape": [ + 3072 + ] + }, + "model.layers.25.mlp.down_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1446969344, + "role": "ssmlp.down.qweight", + "shape": [ + 3072, + 64, + 64 + ] + }, + "model.layers.25.mlp.down_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 14260224, + "role": "ssmlp.down.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.25.mlp.down_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 55842816, + "role": "ssmlp.down.scales", + "shape": [ + 196608 + ] + }, + "model.layers.25.mlp.gate_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1421803520, + "role": "ssmlp.gate.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.25.mlp.gate_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 14063616, + "role": "ssmlp.gate.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.25.mlp.gate_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 55056384, + "role": "ssmlp.gate.scales", + "shape": [ + 196608 + ] + }, + "model.layers.25.mlp.up_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1434386432, + "role": "ssmlp.up.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.25.mlp.up_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 14161920, + "role": "ssmlp.up.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.25.mlp.up_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 55449600, + "role": "ssmlp.up.scales", + "shape": [ + 196608 + ] + }, + "model.layers.25.post_attention_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 313344, + "role": "ssmlp.norm0", + "shape": [ + 3072 + ] + }, + "model.layers.26.attn.k_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 145162240, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.26.attn.k_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 1038336, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.26.attn.k_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 6100992, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.26.attn.o_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 416546816, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.26.attn.o_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 3139584, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.26.attn.o_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 23943168, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.26.attn.q_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 411828224, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.26.attn.q_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 3102720, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.26.attn.q_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 23795712, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.26.attn.v_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 146735104, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.26.attn.v_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 1050624, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.26.attn.v_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 6150144, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.26.input_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 319488, + "role": "ssmlp.norm1", + "shape": [ + 3072 + ] + }, + "model.layers.26.mlp.down_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1484718080, + "role": "ssmlp.down.qweight", + "shape": [ + 3072, + 64, + 64 + ] + }, + "model.layers.26.mlp.down_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 14555136, + "role": "ssmlp.down.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.26.mlp.down_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 57022464, + "role": "ssmlp.down.scales", + "shape": [ + 196608 + ] + }, + "model.layers.26.mlp.gate_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1459552256, + "role": "ssmlp.gate.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.26.mlp.gate_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 14358528, + "role": "ssmlp.gate.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.26.mlp.gate_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 56236032, + "role": "ssmlp.gate.scales", + "shape": [ + 196608 + ] + }, + "model.layers.26.mlp.up_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1472135168, + "role": "ssmlp.up.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.26.mlp.up_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 14456832, + "role": "ssmlp.up.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.26.mlp.up_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 56629248, + "role": "ssmlp.up.scales", + "shape": [ + 196608 + ] + }, + "model.layers.26.post_attention_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 325632, + "role": "ssmlp.norm0", + "shape": [ + 3072 + ] + }, + "model.layers.27.attn.k_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 148307968, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.27.attn.k_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 1062912, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.27.attn.k_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 6199296, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.27.attn.o_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 425984000, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.27.attn.o_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 3213312, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.27.attn.o_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 24238080, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.27.attn.q_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 421265408, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.27.attn.q_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 3176448, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.27.attn.q_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 24090624, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.27.attn.v_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 149880832, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.27.attn.v_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 1075200, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.27.attn.v_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 6248448, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.27.input_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 331776, + "role": "ssmlp.norm1", + "shape": [ + 3072 + ] + }, + "model.layers.27.mlp.down_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1522466816, + "role": "ssmlp.down.qweight", + "shape": [ + 3072, + 64, + 64 + ] + }, + "model.layers.27.mlp.down_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 14850048, + "role": "ssmlp.down.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.27.mlp.down_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 58202112, + "role": "ssmlp.down.scales", + "shape": [ + 196608 + ] + }, + "model.layers.27.mlp.gate_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1497300992, + "role": "ssmlp.gate.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.27.mlp.gate_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 14653440, + "role": "ssmlp.gate.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.27.mlp.gate_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 57415680, + "role": "ssmlp.gate.scales", + "shape": [ + 196608 + ] + }, + "model.layers.27.mlp.up_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1509883904, + "role": "ssmlp.up.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.27.mlp.up_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 14751744, + "role": "ssmlp.up.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.27.mlp.up_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 57808896, + "role": "ssmlp.up.scales", + "shape": [ + 196608 + ] + }, + "model.layers.27.post_attention_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 337920, + "role": "ssmlp.norm0", + "shape": [ + 3072 + ] + }, + "model.layers.28.attn.k_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 151453696, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.28.attn.k_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 1087488, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.28.attn.k_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 6297600, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.28.attn.o_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 435421184, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.28.attn.o_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 3287040, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.28.attn.o_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 24532992, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.28.attn.q_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 430702592, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.28.attn.q_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 3250176, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.28.attn.q_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 24385536, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.28.attn.v_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 153026560, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.28.attn.v_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 1099776, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.28.attn.v_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 6346752, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.28.input_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 344064, + "role": "ssmlp.norm1", + "shape": [ + 3072 + ] + }, + "model.layers.28.mlp.down_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1560215552, + "role": "ssmlp.down.qweight", + "shape": [ + 3072, + 64, + 64 + ] + }, + "model.layers.28.mlp.down_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 15144960, + "role": "ssmlp.down.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.28.mlp.down_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 59381760, + "role": "ssmlp.down.scales", + "shape": [ + 196608 + ] + }, + "model.layers.28.mlp.gate_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1535049728, + "role": "ssmlp.gate.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.28.mlp.gate_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 14948352, + "role": "ssmlp.gate.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.28.mlp.gate_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 58595328, + "role": "ssmlp.gate.scales", + "shape": [ + 196608 + ] + }, + "model.layers.28.mlp.up_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1547632640, + "role": "ssmlp.up.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.28.mlp.up_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 15046656, + "role": "ssmlp.up.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.28.mlp.up_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 58988544, + "role": "ssmlp.up.scales", + "shape": [ + 196608 + ] + }, + "model.layers.28.post_attention_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 350208, + "role": "ssmlp.norm0", + "shape": [ + 3072 + ] + }, + "model.layers.29.attn.k_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 154599424, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.29.attn.k_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 1112064, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.29.attn.k_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 6395904, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.29.attn.o_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 444858368, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.29.attn.o_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 3360768, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.29.attn.o_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 24827904, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.29.attn.q_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 440139776, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.29.attn.q_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 3323904, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.29.attn.q_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 24680448, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.29.attn.v_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 156172288, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.29.attn.v_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 1124352, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.29.attn.v_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 6445056, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.29.input_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 356352, + "role": "ssmlp.norm1", + "shape": [ + 3072 + ] + }, + "model.layers.29.mlp.down_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1597964288, + "role": "ssmlp.down.qweight", + "shape": [ + 3072, + 64, + 64 + ] + }, + "model.layers.29.mlp.down_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 15439872, + "role": "ssmlp.down.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.29.mlp.down_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 60561408, + "role": "ssmlp.down.scales", + "shape": [ + 196608 + ] + }, + "model.layers.29.mlp.gate_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1572798464, + "role": "ssmlp.gate.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.29.mlp.gate_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 15243264, + "role": "ssmlp.gate.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.29.mlp.gate_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 59774976, + "role": "ssmlp.gate.scales", + "shape": [ + 196608 + ] + }, + "model.layers.29.mlp.up_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1585381376, + "role": "ssmlp.up.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.29.mlp.up_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 15341568, + "role": "ssmlp.up.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.29.mlp.up_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 60168192, + "role": "ssmlp.up.scales", + "shape": [ + 196608 + ] + }, + "model.layers.29.post_attention_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 362496, + "role": "ssmlp.norm0", + "shape": [ + 3072 + ] + }, + "model.layers.3.attn.k_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 72810496, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.3.attn.k_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 473088, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.3.attn.k_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 3840000, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.3.attn.o_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 199491584, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.3.attn.o_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 1443840, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.3.attn.o_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 17160192, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.3.attn.q_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 194772992, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.3.attn.q_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 1406976, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.3.attn.q_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 17012736, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.3.attn.v_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 74383360, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.3.attn.v_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 485376, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.3.attn.v_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 3889152, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.3.input_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 36864, + "role": "ssmlp.norm1", + "shape": [ + 3072 + ] + }, + "model.layers.3.mlp.down_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 616497152, + "role": "ssmlp.down.qweight", + "shape": [ + 3072, + 64, + 64 + ] + }, + "model.layers.3.mlp.down_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 7772160, + "role": "ssmlp.down.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.3.mlp.down_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 29890560, + "role": "ssmlp.down.scales", + "shape": [ + 196608 + ] + }, + "model.layers.3.mlp.gate_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 591331328, + "role": "ssmlp.gate.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.3.mlp.gate_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 7575552, + "role": "ssmlp.gate.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.3.mlp.gate_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 29104128, + "role": "ssmlp.gate.scales", + "shape": [ + 196608 + ] + }, + "model.layers.3.mlp.up_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 603914240, + "role": "ssmlp.up.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.3.mlp.up_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 7673856, + "role": "ssmlp.up.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.3.mlp.up_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 29497344, + "role": "ssmlp.up.scales", + "shape": [ + 196608 + ] + }, + "model.layers.3.post_attention_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 43008, + "role": "ssmlp.norm0", + "shape": [ + 3072 + ] + }, + "model.layers.30.attn.k_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 157745152, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.30.attn.k_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 1136640, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.30.attn.k_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 6494208, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.30.attn.o_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 454295552, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.30.attn.o_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 3434496, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.30.attn.o_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 25122816, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.30.attn.q_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 449576960, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.30.attn.q_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 3397632, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.30.attn.q_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 24975360, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.30.attn.v_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 159318016, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.30.attn.v_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 1148928, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.30.attn.v_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 6543360, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.30.input_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 368640, + "role": "ssmlp.norm1", + "shape": [ + 3072 + ] + }, + "model.layers.30.mlp.down_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1635713024, + "role": "ssmlp.down.qweight", + "shape": [ + 3072, + 64, + 64 + ] + }, + "model.layers.30.mlp.down_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 15734784, + "role": "ssmlp.down.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.30.mlp.down_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 61741056, + "role": "ssmlp.down.scales", + "shape": [ + 196608 + ] + }, + "model.layers.30.mlp.gate_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1610547200, + "role": "ssmlp.gate.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.30.mlp.gate_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 15538176, + "role": "ssmlp.gate.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.30.mlp.gate_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 60954624, + "role": "ssmlp.gate.scales", + "shape": [ + 196608 + ] + }, + "model.layers.30.mlp.up_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1623130112, + "role": "ssmlp.up.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.30.mlp.up_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 15636480, + "role": "ssmlp.up.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.30.mlp.up_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 61347840, + "role": "ssmlp.up.scales", + "shape": [ + 196608 + ] + }, + "model.layers.30.post_attention_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 374784, + "role": "ssmlp.norm0", + "shape": [ + 3072 + ] + }, + "model.layers.31.attn.k_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 160890880, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.31.attn.k_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 1161216, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.31.attn.k_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 6592512, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.31.attn.o_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 463732736, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.31.attn.o_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 3508224, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.31.attn.o_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 25417728, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.31.attn.q_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 459014144, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.31.attn.q_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 3471360, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.31.attn.q_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 25270272, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.31.attn.v_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 162463744, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.31.attn.v_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 1173504, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.31.attn.v_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 6641664, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.31.input_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 380928, + "role": "ssmlp.norm1", + "shape": [ + 3072 + ] + }, + "model.layers.31.mlp.down_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1673461760, + "role": "ssmlp.down.qweight", + "shape": [ + 3072, + 64, + 64 + ] + }, + "model.layers.31.mlp.down_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 16029696, + "role": "ssmlp.down.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.31.mlp.down_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 62920704, + "role": "ssmlp.down.scales", + "shape": [ + 196608 + ] + }, + "model.layers.31.mlp.gate_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1648295936, + "role": "ssmlp.gate.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.31.mlp.gate_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 15833088, + "role": "ssmlp.gate.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.31.mlp.gate_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 62134272, + "role": "ssmlp.gate.scales", + "shape": [ + 196608 + ] + }, + "model.layers.31.mlp.up_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 1660878848, + "role": "ssmlp.up.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.31.mlp.up_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 15931392, + "role": "ssmlp.up.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.31.mlp.up_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 62527488, + "role": "ssmlp.up.scales", + "shape": [ + 196608 + ] + }, + "model.layers.31.post_attention_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 387072, + "role": "ssmlp.norm0", + "shape": [ + 3072 + ] + }, + "model.layers.32.final_norm_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 393216, + "role": "ssmlp.norm1", + "shape": [ + 3072 + ] + }, + "model.layers.4.attn.k_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 75956224, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.4.attn.k_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 497664, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.4.attn.k_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 3938304, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.4.attn.o_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 208928768, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.4.attn.o_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 1517568, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.4.attn.o_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 17455104, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.4.attn.q_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 204210176, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.4.attn.q_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 1480704, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.4.attn.q_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 17307648, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.4.attn.v_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 77529088, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.4.attn.v_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 509952, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.4.attn.v_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 3987456, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.4.input_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 49152, + "role": "ssmlp.norm1", + "shape": [ + 3072 + ] + }, + "model.layers.4.mlp.down_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 654245888, + "role": "ssmlp.down.qweight", + "shape": [ + 3072, + 64, + 64 + ] + }, + "model.layers.4.mlp.down_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 8067072, + "role": "ssmlp.down.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.4.mlp.down_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 31070208, + "role": "ssmlp.down.scales", + "shape": [ + 196608 + ] + }, + "model.layers.4.mlp.gate_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 629080064, + "role": "ssmlp.gate.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.4.mlp.gate_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 7870464, + "role": "ssmlp.gate.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.4.mlp.gate_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 30283776, + "role": "ssmlp.gate.scales", + "shape": [ + 196608 + ] + }, + "model.layers.4.mlp.up_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 641662976, + "role": "ssmlp.up.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.4.mlp.up_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 7968768, + "role": "ssmlp.up.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.4.mlp.up_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 30676992, + "role": "ssmlp.up.scales", + "shape": [ + 196608 + ] + }, + "model.layers.4.post_attention_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 55296, + "role": "ssmlp.norm0", + "shape": [ + 3072 + ] + }, + "model.layers.5.attn.k_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 79101952, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.5.attn.k_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 522240, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.5.attn.k_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 4036608, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.5.attn.o_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 218365952, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.5.attn.o_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 1591296, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.5.attn.o_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 17750016, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.5.attn.q_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 213647360, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.5.attn.q_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 1554432, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.5.attn.q_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 17602560, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.5.attn.v_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 80674816, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.5.attn.v_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 534528, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.5.attn.v_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 4085760, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.5.input_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 61440, + "role": "ssmlp.norm1", + "shape": [ + 3072 + ] + }, + "model.layers.5.mlp.down_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 691994624, + "role": "ssmlp.down.qweight", + "shape": [ + 3072, + 64, + 64 + ] + }, + "model.layers.5.mlp.down_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 8361984, + "role": "ssmlp.down.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.5.mlp.down_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 32249856, + "role": "ssmlp.down.scales", + "shape": [ + 196608 + ] + }, + "model.layers.5.mlp.gate_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 666828800, + "role": "ssmlp.gate.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.5.mlp.gate_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 8165376, + "role": "ssmlp.gate.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.5.mlp.gate_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 31463424, + "role": "ssmlp.gate.scales", + "shape": [ + 196608 + ] + }, + "model.layers.5.mlp.up_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 679411712, + "role": "ssmlp.up.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.5.mlp.up_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 8263680, + "role": "ssmlp.up.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.5.mlp.up_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 31856640, + "role": "ssmlp.up.scales", + "shape": [ + 196608 + ] + }, + "model.layers.5.post_attention_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 67584, + "role": "ssmlp.norm0", + "shape": [ + 3072 + ] + }, + "model.layers.6.attn.k_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 82247680, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.6.attn.k_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 546816, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.6.attn.k_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 4134912, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.6.attn.o_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 227803136, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.6.attn.o_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 1665024, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.6.attn.o_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 18044928, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.6.attn.q_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 223084544, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.6.attn.q_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 1628160, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.6.attn.q_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 17897472, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.6.attn.v_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 83820544, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.6.attn.v_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 559104, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.6.attn.v_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 4184064, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.6.input_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 73728, + "role": "ssmlp.norm1", + "shape": [ + 3072 + ] + }, + "model.layers.6.mlp.down_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 729743360, + "role": "ssmlp.down.qweight", + "shape": [ + 3072, + 64, + 64 + ] + }, + "model.layers.6.mlp.down_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 8656896, + "role": "ssmlp.down.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.6.mlp.down_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 33429504, + "role": "ssmlp.down.scales", + "shape": [ + 196608 + ] + }, + "model.layers.6.mlp.gate_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 704577536, + "role": "ssmlp.gate.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.6.mlp.gate_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 8460288, + "role": "ssmlp.gate.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.6.mlp.gate_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 32643072, + "role": "ssmlp.gate.scales", + "shape": [ + 196608 + ] + }, + "model.layers.6.mlp.up_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 717160448, + "role": "ssmlp.up.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.6.mlp.up_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 8558592, + "role": "ssmlp.up.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.6.mlp.up_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 33036288, + "role": "ssmlp.up.scales", + "shape": [ + 196608 + ] + }, + "model.layers.6.post_attention_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 79872, + "role": "ssmlp.norm0", + "shape": [ + 3072 + ] + }, + "model.layers.7.attn.k_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 85393408, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.7.attn.k_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 571392, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.7.attn.k_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 4233216, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.7.attn.o_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 237240320, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.7.attn.o_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 1738752, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.7.attn.o_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 18339840, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.7.attn.q_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 232521728, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.7.attn.q_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 1701888, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.7.attn.q_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 18192384, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.7.attn.v_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 86966272, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.7.attn.v_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 583680, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.7.attn.v_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 4282368, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.7.input_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 86016, + "role": "ssmlp.norm1", + "shape": [ + 3072 + ] + }, + "model.layers.7.mlp.down_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 767492096, + "role": "ssmlp.down.qweight", + "shape": [ + 3072, + 64, + 64 + ] + }, + "model.layers.7.mlp.down_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 8951808, + "role": "ssmlp.down.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.7.mlp.down_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 34609152, + "role": "ssmlp.down.scales", + "shape": [ + 196608 + ] + }, + "model.layers.7.mlp.gate_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 742326272, + "role": "ssmlp.gate.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.7.mlp.gate_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 8755200, + "role": "ssmlp.gate.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.7.mlp.gate_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 33822720, + "role": "ssmlp.gate.scales", + "shape": [ + 196608 + ] + }, + "model.layers.7.mlp.up_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 754909184, + "role": "ssmlp.up.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.7.mlp.up_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 8853504, + "role": "ssmlp.up.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.7.mlp.up_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 34215936, + "role": "ssmlp.up.scales", + "shape": [ + 196608 + ] + }, + "model.layers.7.post_attention_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 92160, + "role": "ssmlp.norm0", + "shape": [ + 3072 + ] + }, + "model.layers.8.attn.k_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 88539136, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.8.attn.k_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 595968, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.8.attn.k_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 4331520, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.8.attn.o_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 246677504, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.8.attn.o_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 1812480, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.8.attn.o_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 18634752, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.8.attn.q_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 241958912, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.8.attn.q_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 1775616, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.8.attn.q_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 18487296, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.8.attn.v_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 90112000, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.8.attn.v_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 608256, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.8.attn.v_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 4380672, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.8.input_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 98304, + "role": "ssmlp.norm1", + "shape": [ + 3072 + ] + }, + "model.layers.8.mlp.down_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 805240832, + "role": "ssmlp.down.qweight", + "shape": [ + 3072, + 64, + 64 + ] + }, + "model.layers.8.mlp.down_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 9246720, + "role": "ssmlp.down.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.8.mlp.down_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 35788800, + "role": "ssmlp.down.scales", + "shape": [ + 196608 + ] + }, + "model.layers.8.mlp.gate_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 780075008, + "role": "ssmlp.gate.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.8.mlp.gate_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 9050112, + "role": "ssmlp.gate.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.8.mlp.gate_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 35002368, + "role": "ssmlp.gate.scales", + "shape": [ + 196608 + ] + }, + "model.layers.8.mlp.up_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 792657920, + "role": "ssmlp.up.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.8.mlp.up_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 9148416, + "role": "ssmlp.up.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.8.mlp.up_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 35395584, + "role": "ssmlp.up.scales", + "shape": [ + 196608 + ] + }, + "model.layers.8.post_attention_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 104448, + "role": "ssmlp.norm0", + "shape": [ + 3072 + ] + }, + "model.layers.9.attn.k_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 91684864, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.9.attn.k_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 620544, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.9.attn.k_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 4429824, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.9.attn.o_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 256114688, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.9.attn.o_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 1886208, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.9.attn.o_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 18929664, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.9.attn.q_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 4718592, + "offset": 251396096, + "role": "matmul.qweight", + "shape": [ + 3072, + 24, + 64 + ] + }, + "model.layers.9.attn.q_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 36864, + "offset": 1849344, + "role": "matmul.qzeros", + "shape": [ + 36864 + ] + }, + "model.layers.9.attn.q_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 147456, + "offset": 18782208, + "role": "matmul.scales", + "shape": [ + 73728 + ] + }, + "model.layers.9.attn.v_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 1572864, + "offset": 93257728, + "role": "matmul.qweight", + "shape": [ + 1024, + 24, + 64 + ] + }, + "model.layers.9.attn.v_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12288, + "offset": 632832, + "role": "matmul.qzeros", + "shape": [ + 12288 + ] + }, + "model.layers.9.attn.v_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 49152, + "offset": 4478976, + "role": "matmul.scales", + "shape": [ + 24576 + ] + }, + "model.layers.9.input_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 110592, + "role": "ssmlp.norm1", + "shape": [ + 3072 + ] + }, + "model.layers.9.mlp.down_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 842989568, + "role": "ssmlp.down.qweight", + "shape": [ + 3072, + 64, + 64 + ] + }, + "model.layers.9.mlp.down_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 9541632, + "role": "ssmlp.down.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.9.mlp.down_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 36968448, + "role": "ssmlp.down.scales", + "shape": [ + 196608 + ] + }, + "model.layers.9.mlp.gate_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 817823744, + "role": "ssmlp.gate.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.9.mlp.gate_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 9345024, + "role": "ssmlp.gate.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.9.mlp.gate_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 36182016, + "role": "ssmlp.gate.scales", + "shape": [ + 196608 + ] + }, + "model.layers.9.mlp.up_proj.MatMulNBits.qweight": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 12582912, + "offset": 830406656, + "role": "ssmlp.up.qweight", + "shape": [ + 8192, + 24, + 64 + ] + }, + "model.layers.9.mlp.up_proj.MatMulNBits.qzeros": { + "dtype": "uint8", + "file": "model.onnx.data", + "length": 98304, + "offset": 9443328, + "role": "ssmlp.up.qzeros", + "shape": [ + 98304 + ] + }, + "model.layers.9.mlp.up_proj.MatMulNBits.scales": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 393216, + "offset": 36575232, + "role": "ssmlp.up.scales", + "shape": [ + 196608 + ] + }, + "model.layers.9.post_attention_layernorm.weight": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 6144, + "offset": 116736, + "role": "ssmlp.norm0", + "shape": [ + 3072 + ] + }, + "sin_cache": { + "dtype": "float16", + "file": "model.onnx.data", + "length": 12976128, + "offset": 1699020800, + "role": "sin_cache", + "shape": [ + 135168, + 48 + ] + } + }, + "model": { + "family": "phi4", + "group_size": 128, + "head_size": 128, + "hidden_size": 3072, + "intermediate_size": 8192, + "kv_heads": 8, + "layers": 32, + "num_heads": 24, + "rms_epsilon": 1e-05, + "rope_dim": 96, + "vocab_size": 200064 + }, + "schema_version": 1, + "weight_objects": [ + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.0.attn.q_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.0.attn.q_proj.MatMulNBits.qweight", + "qzeros": "model.layers.0.attn.q_proj.MatMulNBits.qzeros", + "scales": "model.layers.0.attn.q_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.0.attn.k_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.0.attn.k_proj.MatMulNBits.qweight", + "qzeros": "model.layers.0.attn.k_proj.MatMulNBits.qzeros", + "scales": "model.layers.0.attn.k_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.0.attn.v_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.0.attn.v_proj.MatMulNBits.qweight", + "qzeros": "model.layers.0.attn.v_proj.MatMulNBits.qzeros", + "scales": "model.layers.0.attn.v_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.0.attn.o_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.0.attn.o_proj.MatMulNBits.qweight", + "qzeros": "model.layers.0.attn.o_proj.MatMulNBits.qzeros", + "scales": "model.layers.0.attn.o_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "k": 3072, + "n": 8192 + }, + "kind": "ssmlp", + "name": "model.layers.0.ssmlp", + "roles": { + "down_qweight": "model.layers.0.mlp.down_proj.MatMulNBits.qweight", + "down_qzeros": "model.layers.0.mlp.down_proj.MatMulNBits.qzeros", + "down_scales": "model.layers.0.mlp.down_proj.MatMulNBits.scales", + "gate_qweight": "model.layers.0.mlp.gate_proj.MatMulNBits.qweight", + "gate_qzeros": "model.layers.0.mlp.gate_proj.MatMulNBits.qzeros", + "gate_scales": "model.layers.0.mlp.gate_proj.MatMulNBits.scales", + "norm0": "model.layers.0.post_attention_layernorm.weight", + "norm1": "model.layers.1.input_layernorm.weight", + "up_qweight": "model.layers.0.mlp.up_proj.MatMulNBits.qweight", + "up_qzeros": "model.layers.0.mlp.up_proj.MatMulNBits.qzeros", + "up_scales": "model.layers.0.mlp.up_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.1.attn.q_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.1.attn.q_proj.MatMulNBits.qweight", + "qzeros": "model.layers.1.attn.q_proj.MatMulNBits.qzeros", + "scales": "model.layers.1.attn.q_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.1.attn.k_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.1.attn.k_proj.MatMulNBits.qweight", + "qzeros": "model.layers.1.attn.k_proj.MatMulNBits.qzeros", + "scales": "model.layers.1.attn.k_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.1.attn.v_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.1.attn.v_proj.MatMulNBits.qweight", + "qzeros": "model.layers.1.attn.v_proj.MatMulNBits.qzeros", + "scales": "model.layers.1.attn.v_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.1.attn.o_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.1.attn.o_proj.MatMulNBits.qweight", + "qzeros": "model.layers.1.attn.o_proj.MatMulNBits.qzeros", + "scales": "model.layers.1.attn.o_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "k": 3072, + "n": 8192 + }, + "kind": "ssmlp", + "name": "model.layers.1.ssmlp", + "roles": { + "down_qweight": "model.layers.1.mlp.down_proj.MatMulNBits.qweight", + "down_qzeros": "model.layers.1.mlp.down_proj.MatMulNBits.qzeros", + "down_scales": "model.layers.1.mlp.down_proj.MatMulNBits.scales", + "gate_qweight": "model.layers.1.mlp.gate_proj.MatMulNBits.qweight", + "gate_qzeros": "model.layers.1.mlp.gate_proj.MatMulNBits.qzeros", + "gate_scales": "model.layers.1.mlp.gate_proj.MatMulNBits.scales", + "norm0": "model.layers.1.post_attention_layernorm.weight", + "norm1": "model.layers.2.input_layernorm.weight", + "up_qweight": "model.layers.1.mlp.up_proj.MatMulNBits.qweight", + "up_qzeros": "model.layers.1.mlp.up_proj.MatMulNBits.qzeros", + "up_scales": "model.layers.1.mlp.up_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.2.attn.q_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.2.attn.q_proj.MatMulNBits.qweight", + "qzeros": "model.layers.2.attn.q_proj.MatMulNBits.qzeros", + "scales": "model.layers.2.attn.q_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.2.attn.k_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.2.attn.k_proj.MatMulNBits.qweight", + "qzeros": "model.layers.2.attn.k_proj.MatMulNBits.qzeros", + "scales": "model.layers.2.attn.k_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.2.attn.v_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.2.attn.v_proj.MatMulNBits.qweight", + "qzeros": "model.layers.2.attn.v_proj.MatMulNBits.qzeros", + "scales": "model.layers.2.attn.v_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.2.attn.o_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.2.attn.o_proj.MatMulNBits.qweight", + "qzeros": "model.layers.2.attn.o_proj.MatMulNBits.qzeros", + "scales": "model.layers.2.attn.o_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "k": 3072, + "n": 8192 + }, + "kind": "ssmlp", + "name": "model.layers.2.ssmlp", + "roles": { + "down_qweight": "model.layers.2.mlp.down_proj.MatMulNBits.qweight", + "down_qzeros": "model.layers.2.mlp.down_proj.MatMulNBits.qzeros", + "down_scales": "model.layers.2.mlp.down_proj.MatMulNBits.scales", + "gate_qweight": "model.layers.2.mlp.gate_proj.MatMulNBits.qweight", + "gate_qzeros": "model.layers.2.mlp.gate_proj.MatMulNBits.qzeros", + "gate_scales": "model.layers.2.mlp.gate_proj.MatMulNBits.scales", + "norm0": "model.layers.2.post_attention_layernorm.weight", + "norm1": "model.layers.3.input_layernorm.weight", + "up_qweight": "model.layers.2.mlp.up_proj.MatMulNBits.qweight", + "up_qzeros": "model.layers.2.mlp.up_proj.MatMulNBits.qzeros", + "up_scales": "model.layers.2.mlp.up_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.3.attn.q_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.3.attn.q_proj.MatMulNBits.qweight", + "qzeros": "model.layers.3.attn.q_proj.MatMulNBits.qzeros", + "scales": "model.layers.3.attn.q_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.3.attn.k_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.3.attn.k_proj.MatMulNBits.qweight", + "qzeros": "model.layers.3.attn.k_proj.MatMulNBits.qzeros", + "scales": "model.layers.3.attn.k_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.3.attn.v_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.3.attn.v_proj.MatMulNBits.qweight", + "qzeros": "model.layers.3.attn.v_proj.MatMulNBits.qzeros", + "scales": "model.layers.3.attn.v_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.3.attn.o_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.3.attn.o_proj.MatMulNBits.qweight", + "qzeros": "model.layers.3.attn.o_proj.MatMulNBits.qzeros", + "scales": "model.layers.3.attn.o_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "k": 3072, + "n": 8192 + }, + "kind": "ssmlp", + "name": "model.layers.3.ssmlp", + "roles": { + "down_qweight": "model.layers.3.mlp.down_proj.MatMulNBits.qweight", + "down_qzeros": "model.layers.3.mlp.down_proj.MatMulNBits.qzeros", + "down_scales": "model.layers.3.mlp.down_proj.MatMulNBits.scales", + "gate_qweight": "model.layers.3.mlp.gate_proj.MatMulNBits.qweight", + "gate_qzeros": "model.layers.3.mlp.gate_proj.MatMulNBits.qzeros", + "gate_scales": "model.layers.3.mlp.gate_proj.MatMulNBits.scales", + "norm0": "model.layers.3.post_attention_layernorm.weight", + "norm1": "model.layers.4.input_layernorm.weight", + "up_qweight": "model.layers.3.mlp.up_proj.MatMulNBits.qweight", + "up_qzeros": "model.layers.3.mlp.up_proj.MatMulNBits.qzeros", + "up_scales": "model.layers.3.mlp.up_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.4.attn.q_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.4.attn.q_proj.MatMulNBits.qweight", + "qzeros": "model.layers.4.attn.q_proj.MatMulNBits.qzeros", + "scales": "model.layers.4.attn.q_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.4.attn.k_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.4.attn.k_proj.MatMulNBits.qweight", + "qzeros": "model.layers.4.attn.k_proj.MatMulNBits.qzeros", + "scales": "model.layers.4.attn.k_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.4.attn.v_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.4.attn.v_proj.MatMulNBits.qweight", + "qzeros": "model.layers.4.attn.v_proj.MatMulNBits.qzeros", + "scales": "model.layers.4.attn.v_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.4.attn.o_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.4.attn.o_proj.MatMulNBits.qweight", + "qzeros": "model.layers.4.attn.o_proj.MatMulNBits.qzeros", + "scales": "model.layers.4.attn.o_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "k": 3072, + "n": 8192 + }, + "kind": "ssmlp", + "name": "model.layers.4.ssmlp", + "roles": { + "down_qweight": "model.layers.4.mlp.down_proj.MatMulNBits.qweight", + "down_qzeros": "model.layers.4.mlp.down_proj.MatMulNBits.qzeros", + "down_scales": "model.layers.4.mlp.down_proj.MatMulNBits.scales", + "gate_qweight": "model.layers.4.mlp.gate_proj.MatMulNBits.qweight", + "gate_qzeros": "model.layers.4.mlp.gate_proj.MatMulNBits.qzeros", + "gate_scales": "model.layers.4.mlp.gate_proj.MatMulNBits.scales", + "norm0": "model.layers.4.post_attention_layernorm.weight", + "norm1": "model.layers.5.input_layernorm.weight", + "up_qweight": "model.layers.4.mlp.up_proj.MatMulNBits.qweight", + "up_qzeros": "model.layers.4.mlp.up_proj.MatMulNBits.qzeros", + "up_scales": "model.layers.4.mlp.up_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.5.attn.q_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.5.attn.q_proj.MatMulNBits.qweight", + "qzeros": "model.layers.5.attn.q_proj.MatMulNBits.qzeros", + "scales": "model.layers.5.attn.q_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.5.attn.k_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.5.attn.k_proj.MatMulNBits.qweight", + "qzeros": "model.layers.5.attn.k_proj.MatMulNBits.qzeros", + "scales": "model.layers.5.attn.k_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.5.attn.v_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.5.attn.v_proj.MatMulNBits.qweight", + "qzeros": "model.layers.5.attn.v_proj.MatMulNBits.qzeros", + "scales": "model.layers.5.attn.v_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.5.attn.o_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.5.attn.o_proj.MatMulNBits.qweight", + "qzeros": "model.layers.5.attn.o_proj.MatMulNBits.qzeros", + "scales": "model.layers.5.attn.o_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "k": 3072, + "n": 8192 + }, + "kind": "ssmlp", + "name": "model.layers.5.ssmlp", + "roles": { + "down_qweight": "model.layers.5.mlp.down_proj.MatMulNBits.qweight", + "down_qzeros": "model.layers.5.mlp.down_proj.MatMulNBits.qzeros", + "down_scales": "model.layers.5.mlp.down_proj.MatMulNBits.scales", + "gate_qweight": "model.layers.5.mlp.gate_proj.MatMulNBits.qweight", + "gate_qzeros": "model.layers.5.mlp.gate_proj.MatMulNBits.qzeros", + "gate_scales": "model.layers.5.mlp.gate_proj.MatMulNBits.scales", + "norm0": "model.layers.5.post_attention_layernorm.weight", + "norm1": "model.layers.6.input_layernorm.weight", + "up_qweight": "model.layers.5.mlp.up_proj.MatMulNBits.qweight", + "up_qzeros": "model.layers.5.mlp.up_proj.MatMulNBits.qzeros", + "up_scales": "model.layers.5.mlp.up_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.6.attn.q_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.6.attn.q_proj.MatMulNBits.qweight", + "qzeros": "model.layers.6.attn.q_proj.MatMulNBits.qzeros", + "scales": "model.layers.6.attn.q_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.6.attn.k_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.6.attn.k_proj.MatMulNBits.qweight", + "qzeros": "model.layers.6.attn.k_proj.MatMulNBits.qzeros", + "scales": "model.layers.6.attn.k_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.6.attn.v_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.6.attn.v_proj.MatMulNBits.qweight", + "qzeros": "model.layers.6.attn.v_proj.MatMulNBits.qzeros", + "scales": "model.layers.6.attn.v_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.6.attn.o_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.6.attn.o_proj.MatMulNBits.qweight", + "qzeros": "model.layers.6.attn.o_proj.MatMulNBits.qzeros", + "scales": "model.layers.6.attn.o_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "k": 3072, + "n": 8192 + }, + "kind": "ssmlp", + "name": "model.layers.6.ssmlp", + "roles": { + "down_qweight": "model.layers.6.mlp.down_proj.MatMulNBits.qweight", + "down_qzeros": "model.layers.6.mlp.down_proj.MatMulNBits.qzeros", + "down_scales": "model.layers.6.mlp.down_proj.MatMulNBits.scales", + "gate_qweight": "model.layers.6.mlp.gate_proj.MatMulNBits.qweight", + "gate_qzeros": "model.layers.6.mlp.gate_proj.MatMulNBits.qzeros", + "gate_scales": "model.layers.6.mlp.gate_proj.MatMulNBits.scales", + "norm0": "model.layers.6.post_attention_layernorm.weight", + "norm1": "model.layers.7.input_layernorm.weight", + "up_qweight": "model.layers.6.mlp.up_proj.MatMulNBits.qweight", + "up_qzeros": "model.layers.6.mlp.up_proj.MatMulNBits.qzeros", + "up_scales": "model.layers.6.mlp.up_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.7.attn.q_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.7.attn.q_proj.MatMulNBits.qweight", + "qzeros": "model.layers.7.attn.q_proj.MatMulNBits.qzeros", + "scales": "model.layers.7.attn.q_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.7.attn.k_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.7.attn.k_proj.MatMulNBits.qweight", + "qzeros": "model.layers.7.attn.k_proj.MatMulNBits.qzeros", + "scales": "model.layers.7.attn.k_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.7.attn.v_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.7.attn.v_proj.MatMulNBits.qweight", + "qzeros": "model.layers.7.attn.v_proj.MatMulNBits.qzeros", + "scales": "model.layers.7.attn.v_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.7.attn.o_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.7.attn.o_proj.MatMulNBits.qweight", + "qzeros": "model.layers.7.attn.o_proj.MatMulNBits.qzeros", + "scales": "model.layers.7.attn.o_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "k": 3072, + "n": 8192 + }, + "kind": "ssmlp", + "name": "model.layers.7.ssmlp", + "roles": { + "down_qweight": "model.layers.7.mlp.down_proj.MatMulNBits.qweight", + "down_qzeros": "model.layers.7.mlp.down_proj.MatMulNBits.qzeros", + "down_scales": "model.layers.7.mlp.down_proj.MatMulNBits.scales", + "gate_qweight": "model.layers.7.mlp.gate_proj.MatMulNBits.qweight", + "gate_qzeros": "model.layers.7.mlp.gate_proj.MatMulNBits.qzeros", + "gate_scales": "model.layers.7.mlp.gate_proj.MatMulNBits.scales", + "norm0": "model.layers.7.post_attention_layernorm.weight", + "norm1": "model.layers.8.input_layernorm.weight", + "up_qweight": "model.layers.7.mlp.up_proj.MatMulNBits.qweight", + "up_qzeros": "model.layers.7.mlp.up_proj.MatMulNBits.qzeros", + "up_scales": "model.layers.7.mlp.up_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.8.attn.q_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.8.attn.q_proj.MatMulNBits.qweight", + "qzeros": "model.layers.8.attn.q_proj.MatMulNBits.qzeros", + "scales": "model.layers.8.attn.q_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.8.attn.k_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.8.attn.k_proj.MatMulNBits.qweight", + "qzeros": "model.layers.8.attn.k_proj.MatMulNBits.qzeros", + "scales": "model.layers.8.attn.k_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.8.attn.v_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.8.attn.v_proj.MatMulNBits.qweight", + "qzeros": "model.layers.8.attn.v_proj.MatMulNBits.qzeros", + "scales": "model.layers.8.attn.v_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.8.attn.o_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.8.attn.o_proj.MatMulNBits.qweight", + "qzeros": "model.layers.8.attn.o_proj.MatMulNBits.qzeros", + "scales": "model.layers.8.attn.o_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "k": 3072, + "n": 8192 + }, + "kind": "ssmlp", + "name": "model.layers.8.ssmlp", + "roles": { + "down_qweight": "model.layers.8.mlp.down_proj.MatMulNBits.qweight", + "down_qzeros": "model.layers.8.mlp.down_proj.MatMulNBits.qzeros", + "down_scales": "model.layers.8.mlp.down_proj.MatMulNBits.scales", + "gate_qweight": "model.layers.8.mlp.gate_proj.MatMulNBits.qweight", + "gate_qzeros": "model.layers.8.mlp.gate_proj.MatMulNBits.qzeros", + "gate_scales": "model.layers.8.mlp.gate_proj.MatMulNBits.scales", + "norm0": "model.layers.8.post_attention_layernorm.weight", + "norm1": "model.layers.9.input_layernorm.weight", + "up_qweight": "model.layers.8.mlp.up_proj.MatMulNBits.qweight", + "up_qzeros": "model.layers.8.mlp.up_proj.MatMulNBits.qzeros", + "up_scales": "model.layers.8.mlp.up_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.9.attn.q_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.9.attn.q_proj.MatMulNBits.qweight", + "qzeros": "model.layers.9.attn.q_proj.MatMulNBits.qzeros", + "scales": "model.layers.9.attn.q_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.9.attn.k_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.9.attn.k_proj.MatMulNBits.qweight", + "qzeros": "model.layers.9.attn.k_proj.MatMulNBits.qzeros", + "scales": "model.layers.9.attn.k_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.9.attn.v_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.9.attn.v_proj.MatMulNBits.qweight", + "qzeros": "model.layers.9.attn.v_proj.MatMulNBits.qzeros", + "scales": "model.layers.9.attn.v_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.9.attn.o_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.9.attn.o_proj.MatMulNBits.qweight", + "qzeros": "model.layers.9.attn.o_proj.MatMulNBits.qzeros", + "scales": "model.layers.9.attn.o_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "k": 3072, + "n": 8192 + }, + "kind": "ssmlp", + "name": "model.layers.9.ssmlp", + "roles": { + "down_qweight": "model.layers.9.mlp.down_proj.MatMulNBits.qweight", + "down_qzeros": "model.layers.9.mlp.down_proj.MatMulNBits.qzeros", + "down_scales": "model.layers.9.mlp.down_proj.MatMulNBits.scales", + "gate_qweight": "model.layers.9.mlp.gate_proj.MatMulNBits.qweight", + "gate_qzeros": "model.layers.9.mlp.gate_proj.MatMulNBits.qzeros", + "gate_scales": "model.layers.9.mlp.gate_proj.MatMulNBits.scales", + "norm0": "model.layers.9.post_attention_layernorm.weight", + "norm1": "model.layers.10.input_layernorm.weight", + "up_qweight": "model.layers.9.mlp.up_proj.MatMulNBits.qweight", + "up_qzeros": "model.layers.9.mlp.up_proj.MatMulNBits.qzeros", + "up_scales": "model.layers.9.mlp.up_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.10.attn.q_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.10.attn.q_proj.MatMulNBits.qweight", + "qzeros": "model.layers.10.attn.q_proj.MatMulNBits.qzeros", + "scales": "model.layers.10.attn.q_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.10.attn.k_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.10.attn.k_proj.MatMulNBits.qweight", + "qzeros": "model.layers.10.attn.k_proj.MatMulNBits.qzeros", + "scales": "model.layers.10.attn.k_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.10.attn.v_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.10.attn.v_proj.MatMulNBits.qweight", + "qzeros": "model.layers.10.attn.v_proj.MatMulNBits.qzeros", + "scales": "model.layers.10.attn.v_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.10.attn.o_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.10.attn.o_proj.MatMulNBits.qweight", + "qzeros": "model.layers.10.attn.o_proj.MatMulNBits.qzeros", + "scales": "model.layers.10.attn.o_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "k": 3072, + "n": 8192 + }, + "kind": "ssmlp", + "name": "model.layers.10.ssmlp", + "roles": { + "down_qweight": "model.layers.10.mlp.down_proj.MatMulNBits.qweight", + "down_qzeros": "model.layers.10.mlp.down_proj.MatMulNBits.qzeros", + "down_scales": "model.layers.10.mlp.down_proj.MatMulNBits.scales", + "gate_qweight": "model.layers.10.mlp.gate_proj.MatMulNBits.qweight", + "gate_qzeros": "model.layers.10.mlp.gate_proj.MatMulNBits.qzeros", + "gate_scales": "model.layers.10.mlp.gate_proj.MatMulNBits.scales", + "norm0": "model.layers.10.post_attention_layernorm.weight", + "norm1": "model.layers.11.input_layernorm.weight", + "up_qweight": "model.layers.10.mlp.up_proj.MatMulNBits.qweight", + "up_qzeros": "model.layers.10.mlp.up_proj.MatMulNBits.qzeros", + "up_scales": "model.layers.10.mlp.up_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.11.attn.q_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.11.attn.q_proj.MatMulNBits.qweight", + "qzeros": "model.layers.11.attn.q_proj.MatMulNBits.qzeros", + "scales": "model.layers.11.attn.q_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.11.attn.k_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.11.attn.k_proj.MatMulNBits.qweight", + "qzeros": "model.layers.11.attn.k_proj.MatMulNBits.qzeros", + "scales": "model.layers.11.attn.k_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.11.attn.v_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.11.attn.v_proj.MatMulNBits.qweight", + "qzeros": "model.layers.11.attn.v_proj.MatMulNBits.qzeros", + "scales": "model.layers.11.attn.v_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.11.attn.o_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.11.attn.o_proj.MatMulNBits.qweight", + "qzeros": "model.layers.11.attn.o_proj.MatMulNBits.qzeros", + "scales": "model.layers.11.attn.o_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "k": 3072, + "n": 8192 + }, + "kind": "ssmlp", + "name": "model.layers.11.ssmlp", + "roles": { + "down_qweight": "model.layers.11.mlp.down_proj.MatMulNBits.qweight", + "down_qzeros": "model.layers.11.mlp.down_proj.MatMulNBits.qzeros", + "down_scales": "model.layers.11.mlp.down_proj.MatMulNBits.scales", + "gate_qweight": "model.layers.11.mlp.gate_proj.MatMulNBits.qweight", + "gate_qzeros": "model.layers.11.mlp.gate_proj.MatMulNBits.qzeros", + "gate_scales": "model.layers.11.mlp.gate_proj.MatMulNBits.scales", + "norm0": "model.layers.11.post_attention_layernorm.weight", + "norm1": "model.layers.12.input_layernorm.weight", + "up_qweight": "model.layers.11.mlp.up_proj.MatMulNBits.qweight", + "up_qzeros": "model.layers.11.mlp.up_proj.MatMulNBits.qzeros", + "up_scales": "model.layers.11.mlp.up_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.12.attn.q_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.12.attn.q_proj.MatMulNBits.qweight", + "qzeros": "model.layers.12.attn.q_proj.MatMulNBits.qzeros", + "scales": "model.layers.12.attn.q_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.12.attn.k_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.12.attn.k_proj.MatMulNBits.qweight", + "qzeros": "model.layers.12.attn.k_proj.MatMulNBits.qzeros", + "scales": "model.layers.12.attn.k_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.12.attn.v_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.12.attn.v_proj.MatMulNBits.qweight", + "qzeros": "model.layers.12.attn.v_proj.MatMulNBits.qzeros", + "scales": "model.layers.12.attn.v_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.12.attn.o_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.12.attn.o_proj.MatMulNBits.qweight", + "qzeros": "model.layers.12.attn.o_proj.MatMulNBits.qzeros", + "scales": "model.layers.12.attn.o_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "k": 3072, + "n": 8192 + }, + "kind": "ssmlp", + "name": "model.layers.12.ssmlp", + "roles": { + "down_qweight": "model.layers.12.mlp.down_proj.MatMulNBits.qweight", + "down_qzeros": "model.layers.12.mlp.down_proj.MatMulNBits.qzeros", + "down_scales": "model.layers.12.mlp.down_proj.MatMulNBits.scales", + "gate_qweight": "model.layers.12.mlp.gate_proj.MatMulNBits.qweight", + "gate_qzeros": "model.layers.12.mlp.gate_proj.MatMulNBits.qzeros", + "gate_scales": "model.layers.12.mlp.gate_proj.MatMulNBits.scales", + "norm0": "model.layers.12.post_attention_layernorm.weight", + "norm1": "model.layers.13.input_layernorm.weight", + "up_qweight": "model.layers.12.mlp.up_proj.MatMulNBits.qweight", + "up_qzeros": "model.layers.12.mlp.up_proj.MatMulNBits.qzeros", + "up_scales": "model.layers.12.mlp.up_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.13.attn.q_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.13.attn.q_proj.MatMulNBits.qweight", + "qzeros": "model.layers.13.attn.q_proj.MatMulNBits.qzeros", + "scales": "model.layers.13.attn.q_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.13.attn.k_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.13.attn.k_proj.MatMulNBits.qweight", + "qzeros": "model.layers.13.attn.k_proj.MatMulNBits.qzeros", + "scales": "model.layers.13.attn.k_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.13.attn.v_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.13.attn.v_proj.MatMulNBits.qweight", + "qzeros": "model.layers.13.attn.v_proj.MatMulNBits.qzeros", + "scales": "model.layers.13.attn.v_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.13.attn.o_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.13.attn.o_proj.MatMulNBits.qweight", + "qzeros": "model.layers.13.attn.o_proj.MatMulNBits.qzeros", + "scales": "model.layers.13.attn.o_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "k": 3072, + "n": 8192 + }, + "kind": "ssmlp", + "name": "model.layers.13.ssmlp", + "roles": { + "down_qweight": "model.layers.13.mlp.down_proj.MatMulNBits.qweight", + "down_qzeros": "model.layers.13.mlp.down_proj.MatMulNBits.qzeros", + "down_scales": "model.layers.13.mlp.down_proj.MatMulNBits.scales", + "gate_qweight": "model.layers.13.mlp.gate_proj.MatMulNBits.qweight", + "gate_qzeros": "model.layers.13.mlp.gate_proj.MatMulNBits.qzeros", + "gate_scales": "model.layers.13.mlp.gate_proj.MatMulNBits.scales", + "norm0": "model.layers.13.post_attention_layernorm.weight", + "norm1": "model.layers.14.input_layernorm.weight", + "up_qweight": "model.layers.13.mlp.up_proj.MatMulNBits.qweight", + "up_qzeros": "model.layers.13.mlp.up_proj.MatMulNBits.qzeros", + "up_scales": "model.layers.13.mlp.up_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.14.attn.q_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.14.attn.q_proj.MatMulNBits.qweight", + "qzeros": "model.layers.14.attn.q_proj.MatMulNBits.qzeros", + "scales": "model.layers.14.attn.q_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.14.attn.k_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.14.attn.k_proj.MatMulNBits.qweight", + "qzeros": "model.layers.14.attn.k_proj.MatMulNBits.qzeros", + "scales": "model.layers.14.attn.k_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.14.attn.v_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.14.attn.v_proj.MatMulNBits.qweight", + "qzeros": "model.layers.14.attn.v_proj.MatMulNBits.qzeros", + "scales": "model.layers.14.attn.v_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.14.attn.o_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.14.attn.o_proj.MatMulNBits.qweight", + "qzeros": "model.layers.14.attn.o_proj.MatMulNBits.qzeros", + "scales": "model.layers.14.attn.o_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "k": 3072, + "n": 8192 + }, + "kind": "ssmlp", + "name": "model.layers.14.ssmlp", + "roles": { + "down_qweight": "model.layers.14.mlp.down_proj.MatMulNBits.qweight", + "down_qzeros": "model.layers.14.mlp.down_proj.MatMulNBits.qzeros", + "down_scales": "model.layers.14.mlp.down_proj.MatMulNBits.scales", + "gate_qweight": "model.layers.14.mlp.gate_proj.MatMulNBits.qweight", + "gate_qzeros": "model.layers.14.mlp.gate_proj.MatMulNBits.qzeros", + "gate_scales": "model.layers.14.mlp.gate_proj.MatMulNBits.scales", + "norm0": "model.layers.14.post_attention_layernorm.weight", + "norm1": "model.layers.15.input_layernorm.weight", + "up_qweight": "model.layers.14.mlp.up_proj.MatMulNBits.qweight", + "up_qzeros": "model.layers.14.mlp.up_proj.MatMulNBits.qzeros", + "up_scales": "model.layers.14.mlp.up_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.15.attn.q_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.15.attn.q_proj.MatMulNBits.qweight", + "qzeros": "model.layers.15.attn.q_proj.MatMulNBits.qzeros", + "scales": "model.layers.15.attn.q_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.15.attn.k_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.15.attn.k_proj.MatMulNBits.qweight", + "qzeros": "model.layers.15.attn.k_proj.MatMulNBits.qzeros", + "scales": "model.layers.15.attn.k_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.15.attn.v_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.15.attn.v_proj.MatMulNBits.qweight", + "qzeros": "model.layers.15.attn.v_proj.MatMulNBits.qzeros", + "scales": "model.layers.15.attn.v_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.15.attn.o_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.15.attn.o_proj.MatMulNBits.qweight", + "qzeros": "model.layers.15.attn.o_proj.MatMulNBits.qzeros", + "scales": "model.layers.15.attn.o_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "k": 3072, + "n": 8192 + }, + "kind": "ssmlp", + "name": "model.layers.15.ssmlp", + "roles": { + "down_qweight": "model.layers.15.mlp.down_proj.MatMulNBits.qweight", + "down_qzeros": "model.layers.15.mlp.down_proj.MatMulNBits.qzeros", + "down_scales": "model.layers.15.mlp.down_proj.MatMulNBits.scales", + "gate_qweight": "model.layers.15.mlp.gate_proj.MatMulNBits.qweight", + "gate_qzeros": "model.layers.15.mlp.gate_proj.MatMulNBits.qzeros", + "gate_scales": "model.layers.15.mlp.gate_proj.MatMulNBits.scales", + "norm0": "model.layers.15.post_attention_layernorm.weight", + "norm1": "model.layers.16.input_layernorm.weight", + "up_qweight": "model.layers.15.mlp.up_proj.MatMulNBits.qweight", + "up_qzeros": "model.layers.15.mlp.up_proj.MatMulNBits.qzeros", + "up_scales": "model.layers.15.mlp.up_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.16.attn.q_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.16.attn.q_proj.MatMulNBits.qweight", + "qzeros": "model.layers.16.attn.q_proj.MatMulNBits.qzeros", + "scales": "model.layers.16.attn.q_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.16.attn.k_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.16.attn.k_proj.MatMulNBits.qweight", + "qzeros": "model.layers.16.attn.k_proj.MatMulNBits.qzeros", + "scales": "model.layers.16.attn.k_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.16.attn.v_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.16.attn.v_proj.MatMulNBits.qweight", + "qzeros": "model.layers.16.attn.v_proj.MatMulNBits.qzeros", + "scales": "model.layers.16.attn.v_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.16.attn.o_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.16.attn.o_proj.MatMulNBits.qweight", + "qzeros": "model.layers.16.attn.o_proj.MatMulNBits.qzeros", + "scales": "model.layers.16.attn.o_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "k": 3072, + "n": 8192 + }, + "kind": "ssmlp", + "name": "model.layers.16.ssmlp", + "roles": { + "down_qweight": "model.layers.16.mlp.down_proj.MatMulNBits.qweight", + "down_qzeros": "model.layers.16.mlp.down_proj.MatMulNBits.qzeros", + "down_scales": "model.layers.16.mlp.down_proj.MatMulNBits.scales", + "gate_qweight": "model.layers.16.mlp.gate_proj.MatMulNBits.qweight", + "gate_qzeros": "model.layers.16.mlp.gate_proj.MatMulNBits.qzeros", + "gate_scales": "model.layers.16.mlp.gate_proj.MatMulNBits.scales", + "norm0": "model.layers.16.post_attention_layernorm.weight", + "norm1": "model.layers.17.input_layernorm.weight", + "up_qweight": "model.layers.16.mlp.up_proj.MatMulNBits.qweight", + "up_qzeros": "model.layers.16.mlp.up_proj.MatMulNBits.qzeros", + "up_scales": "model.layers.16.mlp.up_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.17.attn.q_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.17.attn.q_proj.MatMulNBits.qweight", + "qzeros": "model.layers.17.attn.q_proj.MatMulNBits.qzeros", + "scales": "model.layers.17.attn.q_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.17.attn.k_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.17.attn.k_proj.MatMulNBits.qweight", + "qzeros": "model.layers.17.attn.k_proj.MatMulNBits.qzeros", + "scales": "model.layers.17.attn.k_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.17.attn.v_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.17.attn.v_proj.MatMulNBits.qweight", + "qzeros": "model.layers.17.attn.v_proj.MatMulNBits.qzeros", + "scales": "model.layers.17.attn.v_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.17.attn.o_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.17.attn.o_proj.MatMulNBits.qweight", + "qzeros": "model.layers.17.attn.o_proj.MatMulNBits.qzeros", + "scales": "model.layers.17.attn.o_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "k": 3072, + "n": 8192 + }, + "kind": "ssmlp", + "name": "model.layers.17.ssmlp", + "roles": { + "down_qweight": "model.layers.17.mlp.down_proj.MatMulNBits.qweight", + "down_qzeros": "model.layers.17.mlp.down_proj.MatMulNBits.qzeros", + "down_scales": "model.layers.17.mlp.down_proj.MatMulNBits.scales", + "gate_qweight": "model.layers.17.mlp.gate_proj.MatMulNBits.qweight", + "gate_qzeros": "model.layers.17.mlp.gate_proj.MatMulNBits.qzeros", + "gate_scales": "model.layers.17.mlp.gate_proj.MatMulNBits.scales", + "norm0": "model.layers.17.post_attention_layernorm.weight", + "norm1": "model.layers.18.input_layernorm.weight", + "up_qweight": "model.layers.17.mlp.up_proj.MatMulNBits.qweight", + "up_qzeros": "model.layers.17.mlp.up_proj.MatMulNBits.qzeros", + "up_scales": "model.layers.17.mlp.up_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.18.attn.q_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.18.attn.q_proj.MatMulNBits.qweight", + "qzeros": "model.layers.18.attn.q_proj.MatMulNBits.qzeros", + "scales": "model.layers.18.attn.q_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.18.attn.k_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.18.attn.k_proj.MatMulNBits.qweight", + "qzeros": "model.layers.18.attn.k_proj.MatMulNBits.qzeros", + "scales": "model.layers.18.attn.k_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.18.attn.v_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.18.attn.v_proj.MatMulNBits.qweight", + "qzeros": "model.layers.18.attn.v_proj.MatMulNBits.qzeros", + "scales": "model.layers.18.attn.v_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.18.attn.o_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.18.attn.o_proj.MatMulNBits.qweight", + "qzeros": "model.layers.18.attn.o_proj.MatMulNBits.qzeros", + "scales": "model.layers.18.attn.o_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "k": 3072, + "n": 8192 + }, + "kind": "ssmlp", + "name": "model.layers.18.ssmlp", + "roles": { + "down_qweight": "model.layers.18.mlp.down_proj.MatMulNBits.qweight", + "down_qzeros": "model.layers.18.mlp.down_proj.MatMulNBits.qzeros", + "down_scales": "model.layers.18.mlp.down_proj.MatMulNBits.scales", + "gate_qweight": "model.layers.18.mlp.gate_proj.MatMulNBits.qweight", + "gate_qzeros": "model.layers.18.mlp.gate_proj.MatMulNBits.qzeros", + "gate_scales": "model.layers.18.mlp.gate_proj.MatMulNBits.scales", + "norm0": "model.layers.18.post_attention_layernorm.weight", + "norm1": "model.layers.19.input_layernorm.weight", + "up_qweight": "model.layers.18.mlp.up_proj.MatMulNBits.qweight", + "up_qzeros": "model.layers.18.mlp.up_proj.MatMulNBits.qzeros", + "up_scales": "model.layers.18.mlp.up_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.19.attn.q_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.19.attn.q_proj.MatMulNBits.qweight", + "qzeros": "model.layers.19.attn.q_proj.MatMulNBits.qzeros", + "scales": "model.layers.19.attn.q_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.19.attn.k_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.19.attn.k_proj.MatMulNBits.qweight", + "qzeros": "model.layers.19.attn.k_proj.MatMulNBits.qzeros", + "scales": "model.layers.19.attn.k_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.19.attn.v_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.19.attn.v_proj.MatMulNBits.qweight", + "qzeros": "model.layers.19.attn.v_proj.MatMulNBits.qzeros", + "scales": "model.layers.19.attn.v_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.19.attn.o_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.19.attn.o_proj.MatMulNBits.qweight", + "qzeros": "model.layers.19.attn.o_proj.MatMulNBits.qzeros", + "scales": "model.layers.19.attn.o_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "k": 3072, + "n": 8192 + }, + "kind": "ssmlp", + "name": "model.layers.19.ssmlp", + "roles": { + "down_qweight": "model.layers.19.mlp.down_proj.MatMulNBits.qweight", + "down_qzeros": "model.layers.19.mlp.down_proj.MatMulNBits.qzeros", + "down_scales": "model.layers.19.mlp.down_proj.MatMulNBits.scales", + "gate_qweight": "model.layers.19.mlp.gate_proj.MatMulNBits.qweight", + "gate_qzeros": "model.layers.19.mlp.gate_proj.MatMulNBits.qzeros", + "gate_scales": "model.layers.19.mlp.gate_proj.MatMulNBits.scales", + "norm0": "model.layers.19.post_attention_layernorm.weight", + "norm1": "model.layers.20.input_layernorm.weight", + "up_qweight": "model.layers.19.mlp.up_proj.MatMulNBits.qweight", + "up_qzeros": "model.layers.19.mlp.up_proj.MatMulNBits.qzeros", + "up_scales": "model.layers.19.mlp.up_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.20.attn.q_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.20.attn.q_proj.MatMulNBits.qweight", + "qzeros": "model.layers.20.attn.q_proj.MatMulNBits.qzeros", + "scales": "model.layers.20.attn.q_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.20.attn.k_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.20.attn.k_proj.MatMulNBits.qweight", + "qzeros": "model.layers.20.attn.k_proj.MatMulNBits.qzeros", + "scales": "model.layers.20.attn.k_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.20.attn.v_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.20.attn.v_proj.MatMulNBits.qweight", + "qzeros": "model.layers.20.attn.v_proj.MatMulNBits.qzeros", + "scales": "model.layers.20.attn.v_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.20.attn.o_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.20.attn.o_proj.MatMulNBits.qweight", + "qzeros": "model.layers.20.attn.o_proj.MatMulNBits.qzeros", + "scales": "model.layers.20.attn.o_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "k": 3072, + "n": 8192 + }, + "kind": "ssmlp", + "name": "model.layers.20.ssmlp", + "roles": { + "down_qweight": "model.layers.20.mlp.down_proj.MatMulNBits.qweight", + "down_qzeros": "model.layers.20.mlp.down_proj.MatMulNBits.qzeros", + "down_scales": "model.layers.20.mlp.down_proj.MatMulNBits.scales", + "gate_qweight": "model.layers.20.mlp.gate_proj.MatMulNBits.qweight", + "gate_qzeros": "model.layers.20.mlp.gate_proj.MatMulNBits.qzeros", + "gate_scales": "model.layers.20.mlp.gate_proj.MatMulNBits.scales", + "norm0": "model.layers.20.post_attention_layernorm.weight", + "norm1": "model.layers.21.input_layernorm.weight", + "up_qweight": "model.layers.20.mlp.up_proj.MatMulNBits.qweight", + "up_qzeros": "model.layers.20.mlp.up_proj.MatMulNBits.qzeros", + "up_scales": "model.layers.20.mlp.up_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.21.attn.q_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.21.attn.q_proj.MatMulNBits.qweight", + "qzeros": "model.layers.21.attn.q_proj.MatMulNBits.qzeros", + "scales": "model.layers.21.attn.q_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.21.attn.k_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.21.attn.k_proj.MatMulNBits.qweight", + "qzeros": "model.layers.21.attn.k_proj.MatMulNBits.qzeros", + "scales": "model.layers.21.attn.k_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.21.attn.v_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.21.attn.v_proj.MatMulNBits.qweight", + "qzeros": "model.layers.21.attn.v_proj.MatMulNBits.qzeros", + "scales": "model.layers.21.attn.v_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.21.attn.o_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.21.attn.o_proj.MatMulNBits.qweight", + "qzeros": "model.layers.21.attn.o_proj.MatMulNBits.qzeros", + "scales": "model.layers.21.attn.o_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "k": 3072, + "n": 8192 + }, + "kind": "ssmlp", + "name": "model.layers.21.ssmlp", + "roles": { + "down_qweight": "model.layers.21.mlp.down_proj.MatMulNBits.qweight", + "down_qzeros": "model.layers.21.mlp.down_proj.MatMulNBits.qzeros", + "down_scales": "model.layers.21.mlp.down_proj.MatMulNBits.scales", + "gate_qweight": "model.layers.21.mlp.gate_proj.MatMulNBits.qweight", + "gate_qzeros": "model.layers.21.mlp.gate_proj.MatMulNBits.qzeros", + "gate_scales": "model.layers.21.mlp.gate_proj.MatMulNBits.scales", + "norm0": "model.layers.21.post_attention_layernorm.weight", + "norm1": "model.layers.22.input_layernorm.weight", + "up_qweight": "model.layers.21.mlp.up_proj.MatMulNBits.qweight", + "up_qzeros": "model.layers.21.mlp.up_proj.MatMulNBits.qzeros", + "up_scales": "model.layers.21.mlp.up_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.22.attn.q_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.22.attn.q_proj.MatMulNBits.qweight", + "qzeros": "model.layers.22.attn.q_proj.MatMulNBits.qzeros", + "scales": "model.layers.22.attn.q_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.22.attn.k_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.22.attn.k_proj.MatMulNBits.qweight", + "qzeros": "model.layers.22.attn.k_proj.MatMulNBits.qzeros", + "scales": "model.layers.22.attn.k_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.22.attn.v_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.22.attn.v_proj.MatMulNBits.qweight", + "qzeros": "model.layers.22.attn.v_proj.MatMulNBits.qzeros", + "scales": "model.layers.22.attn.v_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.22.attn.o_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.22.attn.o_proj.MatMulNBits.qweight", + "qzeros": "model.layers.22.attn.o_proj.MatMulNBits.qzeros", + "scales": "model.layers.22.attn.o_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "k": 3072, + "n": 8192 + }, + "kind": "ssmlp", + "name": "model.layers.22.ssmlp", + "roles": { + "down_qweight": "model.layers.22.mlp.down_proj.MatMulNBits.qweight", + "down_qzeros": "model.layers.22.mlp.down_proj.MatMulNBits.qzeros", + "down_scales": "model.layers.22.mlp.down_proj.MatMulNBits.scales", + "gate_qweight": "model.layers.22.mlp.gate_proj.MatMulNBits.qweight", + "gate_qzeros": "model.layers.22.mlp.gate_proj.MatMulNBits.qzeros", + "gate_scales": "model.layers.22.mlp.gate_proj.MatMulNBits.scales", + "norm0": "model.layers.22.post_attention_layernorm.weight", + "norm1": "model.layers.23.input_layernorm.weight", + "up_qweight": "model.layers.22.mlp.up_proj.MatMulNBits.qweight", + "up_qzeros": "model.layers.22.mlp.up_proj.MatMulNBits.qzeros", + "up_scales": "model.layers.22.mlp.up_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.23.attn.q_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.23.attn.q_proj.MatMulNBits.qweight", + "qzeros": "model.layers.23.attn.q_proj.MatMulNBits.qzeros", + "scales": "model.layers.23.attn.q_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.23.attn.k_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.23.attn.k_proj.MatMulNBits.qweight", + "qzeros": "model.layers.23.attn.k_proj.MatMulNBits.qzeros", + "scales": "model.layers.23.attn.k_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.23.attn.v_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.23.attn.v_proj.MatMulNBits.qweight", + "qzeros": "model.layers.23.attn.v_proj.MatMulNBits.qzeros", + "scales": "model.layers.23.attn.v_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.23.attn.o_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.23.attn.o_proj.MatMulNBits.qweight", + "qzeros": "model.layers.23.attn.o_proj.MatMulNBits.qzeros", + "scales": "model.layers.23.attn.o_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "k": 3072, + "n": 8192 + }, + "kind": "ssmlp", + "name": "model.layers.23.ssmlp", + "roles": { + "down_qweight": "model.layers.23.mlp.down_proj.MatMulNBits.qweight", + "down_qzeros": "model.layers.23.mlp.down_proj.MatMulNBits.qzeros", + "down_scales": "model.layers.23.mlp.down_proj.MatMulNBits.scales", + "gate_qweight": "model.layers.23.mlp.gate_proj.MatMulNBits.qweight", + "gate_qzeros": "model.layers.23.mlp.gate_proj.MatMulNBits.qzeros", + "gate_scales": "model.layers.23.mlp.gate_proj.MatMulNBits.scales", + "norm0": "model.layers.23.post_attention_layernorm.weight", + "norm1": "model.layers.24.input_layernorm.weight", + "up_qweight": "model.layers.23.mlp.up_proj.MatMulNBits.qweight", + "up_qzeros": "model.layers.23.mlp.up_proj.MatMulNBits.qzeros", + "up_scales": "model.layers.23.mlp.up_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.24.attn.q_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.24.attn.q_proj.MatMulNBits.qweight", + "qzeros": "model.layers.24.attn.q_proj.MatMulNBits.qzeros", + "scales": "model.layers.24.attn.q_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.24.attn.k_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.24.attn.k_proj.MatMulNBits.qweight", + "qzeros": "model.layers.24.attn.k_proj.MatMulNBits.qzeros", + "scales": "model.layers.24.attn.k_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.24.attn.v_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.24.attn.v_proj.MatMulNBits.qweight", + "qzeros": "model.layers.24.attn.v_proj.MatMulNBits.qzeros", + "scales": "model.layers.24.attn.v_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.24.attn.o_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.24.attn.o_proj.MatMulNBits.qweight", + "qzeros": "model.layers.24.attn.o_proj.MatMulNBits.qzeros", + "scales": "model.layers.24.attn.o_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "k": 3072, + "n": 8192 + }, + "kind": "ssmlp", + "name": "model.layers.24.ssmlp", + "roles": { + "down_qweight": "model.layers.24.mlp.down_proj.MatMulNBits.qweight", + "down_qzeros": "model.layers.24.mlp.down_proj.MatMulNBits.qzeros", + "down_scales": "model.layers.24.mlp.down_proj.MatMulNBits.scales", + "gate_qweight": "model.layers.24.mlp.gate_proj.MatMulNBits.qweight", + "gate_qzeros": "model.layers.24.mlp.gate_proj.MatMulNBits.qzeros", + "gate_scales": "model.layers.24.mlp.gate_proj.MatMulNBits.scales", + "norm0": "model.layers.24.post_attention_layernorm.weight", + "norm1": "model.layers.25.input_layernorm.weight", + "up_qweight": "model.layers.24.mlp.up_proj.MatMulNBits.qweight", + "up_qzeros": "model.layers.24.mlp.up_proj.MatMulNBits.qzeros", + "up_scales": "model.layers.24.mlp.up_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.25.attn.q_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.25.attn.q_proj.MatMulNBits.qweight", + "qzeros": "model.layers.25.attn.q_proj.MatMulNBits.qzeros", + "scales": "model.layers.25.attn.q_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.25.attn.k_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.25.attn.k_proj.MatMulNBits.qweight", + "qzeros": "model.layers.25.attn.k_proj.MatMulNBits.qzeros", + "scales": "model.layers.25.attn.k_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.25.attn.v_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.25.attn.v_proj.MatMulNBits.qweight", + "qzeros": "model.layers.25.attn.v_proj.MatMulNBits.qzeros", + "scales": "model.layers.25.attn.v_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.25.attn.o_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.25.attn.o_proj.MatMulNBits.qweight", + "qzeros": "model.layers.25.attn.o_proj.MatMulNBits.qzeros", + "scales": "model.layers.25.attn.o_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "k": 3072, + "n": 8192 + }, + "kind": "ssmlp", + "name": "model.layers.25.ssmlp", + "roles": { + "down_qweight": "model.layers.25.mlp.down_proj.MatMulNBits.qweight", + "down_qzeros": "model.layers.25.mlp.down_proj.MatMulNBits.qzeros", + "down_scales": "model.layers.25.mlp.down_proj.MatMulNBits.scales", + "gate_qweight": "model.layers.25.mlp.gate_proj.MatMulNBits.qweight", + "gate_qzeros": "model.layers.25.mlp.gate_proj.MatMulNBits.qzeros", + "gate_scales": "model.layers.25.mlp.gate_proj.MatMulNBits.scales", + "norm0": "model.layers.25.post_attention_layernorm.weight", + "norm1": "model.layers.26.input_layernorm.weight", + "up_qweight": "model.layers.25.mlp.up_proj.MatMulNBits.qweight", + "up_qzeros": "model.layers.25.mlp.up_proj.MatMulNBits.qzeros", + "up_scales": "model.layers.25.mlp.up_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.26.attn.q_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.26.attn.q_proj.MatMulNBits.qweight", + "qzeros": "model.layers.26.attn.q_proj.MatMulNBits.qzeros", + "scales": "model.layers.26.attn.q_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.26.attn.k_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.26.attn.k_proj.MatMulNBits.qweight", + "qzeros": "model.layers.26.attn.k_proj.MatMulNBits.qzeros", + "scales": "model.layers.26.attn.k_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.26.attn.v_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.26.attn.v_proj.MatMulNBits.qweight", + "qzeros": "model.layers.26.attn.v_proj.MatMulNBits.qzeros", + "scales": "model.layers.26.attn.v_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.26.attn.o_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.26.attn.o_proj.MatMulNBits.qweight", + "qzeros": "model.layers.26.attn.o_proj.MatMulNBits.qzeros", + "scales": "model.layers.26.attn.o_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "k": 3072, + "n": 8192 + }, + "kind": "ssmlp", + "name": "model.layers.26.ssmlp", + "roles": { + "down_qweight": "model.layers.26.mlp.down_proj.MatMulNBits.qweight", + "down_qzeros": "model.layers.26.mlp.down_proj.MatMulNBits.qzeros", + "down_scales": "model.layers.26.mlp.down_proj.MatMulNBits.scales", + "gate_qweight": "model.layers.26.mlp.gate_proj.MatMulNBits.qweight", + "gate_qzeros": "model.layers.26.mlp.gate_proj.MatMulNBits.qzeros", + "gate_scales": "model.layers.26.mlp.gate_proj.MatMulNBits.scales", + "norm0": "model.layers.26.post_attention_layernorm.weight", + "norm1": "model.layers.27.input_layernorm.weight", + "up_qweight": "model.layers.26.mlp.up_proj.MatMulNBits.qweight", + "up_qzeros": "model.layers.26.mlp.up_proj.MatMulNBits.qzeros", + "up_scales": "model.layers.26.mlp.up_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.27.attn.q_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.27.attn.q_proj.MatMulNBits.qweight", + "qzeros": "model.layers.27.attn.q_proj.MatMulNBits.qzeros", + "scales": "model.layers.27.attn.q_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.27.attn.k_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.27.attn.k_proj.MatMulNBits.qweight", + "qzeros": "model.layers.27.attn.k_proj.MatMulNBits.qzeros", + "scales": "model.layers.27.attn.k_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.27.attn.v_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.27.attn.v_proj.MatMulNBits.qweight", + "qzeros": "model.layers.27.attn.v_proj.MatMulNBits.qzeros", + "scales": "model.layers.27.attn.v_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.27.attn.o_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.27.attn.o_proj.MatMulNBits.qweight", + "qzeros": "model.layers.27.attn.o_proj.MatMulNBits.qzeros", + "scales": "model.layers.27.attn.o_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "k": 3072, + "n": 8192 + }, + "kind": "ssmlp", + "name": "model.layers.27.ssmlp", + "roles": { + "down_qweight": "model.layers.27.mlp.down_proj.MatMulNBits.qweight", + "down_qzeros": "model.layers.27.mlp.down_proj.MatMulNBits.qzeros", + "down_scales": "model.layers.27.mlp.down_proj.MatMulNBits.scales", + "gate_qweight": "model.layers.27.mlp.gate_proj.MatMulNBits.qweight", + "gate_qzeros": "model.layers.27.mlp.gate_proj.MatMulNBits.qzeros", + "gate_scales": "model.layers.27.mlp.gate_proj.MatMulNBits.scales", + "norm0": "model.layers.27.post_attention_layernorm.weight", + "norm1": "model.layers.28.input_layernorm.weight", + "up_qweight": "model.layers.27.mlp.up_proj.MatMulNBits.qweight", + "up_qzeros": "model.layers.27.mlp.up_proj.MatMulNBits.qzeros", + "up_scales": "model.layers.27.mlp.up_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.28.attn.q_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.28.attn.q_proj.MatMulNBits.qweight", + "qzeros": "model.layers.28.attn.q_proj.MatMulNBits.qzeros", + "scales": "model.layers.28.attn.q_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.28.attn.k_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.28.attn.k_proj.MatMulNBits.qweight", + "qzeros": "model.layers.28.attn.k_proj.MatMulNBits.qzeros", + "scales": "model.layers.28.attn.k_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.28.attn.v_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.28.attn.v_proj.MatMulNBits.qweight", + "qzeros": "model.layers.28.attn.v_proj.MatMulNBits.qzeros", + "scales": "model.layers.28.attn.v_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.28.attn.o_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.28.attn.o_proj.MatMulNBits.qweight", + "qzeros": "model.layers.28.attn.o_proj.MatMulNBits.qzeros", + "scales": "model.layers.28.attn.o_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "k": 3072, + "n": 8192 + }, + "kind": "ssmlp", + "name": "model.layers.28.ssmlp", + "roles": { + "down_qweight": "model.layers.28.mlp.down_proj.MatMulNBits.qweight", + "down_qzeros": "model.layers.28.mlp.down_proj.MatMulNBits.qzeros", + "down_scales": "model.layers.28.mlp.down_proj.MatMulNBits.scales", + "gate_qweight": "model.layers.28.mlp.gate_proj.MatMulNBits.qweight", + "gate_qzeros": "model.layers.28.mlp.gate_proj.MatMulNBits.qzeros", + "gate_scales": "model.layers.28.mlp.gate_proj.MatMulNBits.scales", + "norm0": "model.layers.28.post_attention_layernorm.weight", + "norm1": "model.layers.29.input_layernorm.weight", + "up_qweight": "model.layers.28.mlp.up_proj.MatMulNBits.qweight", + "up_qzeros": "model.layers.28.mlp.up_proj.MatMulNBits.qzeros", + "up_scales": "model.layers.28.mlp.up_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.29.attn.q_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.29.attn.q_proj.MatMulNBits.qweight", + "qzeros": "model.layers.29.attn.q_proj.MatMulNBits.qzeros", + "scales": "model.layers.29.attn.q_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.29.attn.k_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.29.attn.k_proj.MatMulNBits.qweight", + "qzeros": "model.layers.29.attn.k_proj.MatMulNBits.qzeros", + "scales": "model.layers.29.attn.k_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.29.attn.v_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.29.attn.v_proj.MatMulNBits.qweight", + "qzeros": "model.layers.29.attn.v_proj.MatMulNBits.qzeros", + "scales": "model.layers.29.attn.v_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.29.attn.o_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.29.attn.o_proj.MatMulNBits.qweight", + "qzeros": "model.layers.29.attn.o_proj.MatMulNBits.qzeros", + "scales": "model.layers.29.attn.o_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "k": 3072, + "n": 8192 + }, + "kind": "ssmlp", + "name": "model.layers.29.ssmlp", + "roles": { + "down_qweight": "model.layers.29.mlp.down_proj.MatMulNBits.qweight", + "down_qzeros": "model.layers.29.mlp.down_proj.MatMulNBits.qzeros", + "down_scales": "model.layers.29.mlp.down_proj.MatMulNBits.scales", + "gate_qweight": "model.layers.29.mlp.gate_proj.MatMulNBits.qweight", + "gate_qzeros": "model.layers.29.mlp.gate_proj.MatMulNBits.qzeros", + "gate_scales": "model.layers.29.mlp.gate_proj.MatMulNBits.scales", + "norm0": "model.layers.29.post_attention_layernorm.weight", + "norm1": "model.layers.30.input_layernorm.weight", + "up_qweight": "model.layers.29.mlp.up_proj.MatMulNBits.qweight", + "up_qzeros": "model.layers.29.mlp.up_proj.MatMulNBits.qzeros", + "up_scales": "model.layers.29.mlp.up_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.30.attn.q_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.30.attn.q_proj.MatMulNBits.qweight", + "qzeros": "model.layers.30.attn.q_proj.MatMulNBits.qzeros", + "scales": "model.layers.30.attn.q_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.30.attn.k_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.30.attn.k_proj.MatMulNBits.qweight", + "qzeros": "model.layers.30.attn.k_proj.MatMulNBits.qzeros", + "scales": "model.layers.30.attn.k_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.30.attn.v_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.30.attn.v_proj.MatMulNBits.qweight", + "qzeros": "model.layers.30.attn.v_proj.MatMulNBits.qzeros", + "scales": "model.layers.30.attn.v_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.30.attn.o_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.30.attn.o_proj.MatMulNBits.qweight", + "qzeros": "model.layers.30.attn.o_proj.MatMulNBits.qzeros", + "scales": "model.layers.30.attn.o_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "k": 3072, + "n": 8192 + }, + "kind": "ssmlp", + "name": "model.layers.30.ssmlp", + "roles": { + "down_qweight": "model.layers.30.mlp.down_proj.MatMulNBits.qweight", + "down_qzeros": "model.layers.30.mlp.down_proj.MatMulNBits.qzeros", + "down_scales": "model.layers.30.mlp.down_proj.MatMulNBits.scales", + "gate_qweight": "model.layers.30.mlp.gate_proj.MatMulNBits.qweight", + "gate_qzeros": "model.layers.30.mlp.gate_proj.MatMulNBits.qzeros", + "gate_scales": "model.layers.30.mlp.gate_proj.MatMulNBits.scales", + "norm0": "model.layers.30.post_attention_layernorm.weight", + "norm1": "model.layers.31.input_layernorm.weight", + "up_qweight": "model.layers.30.mlp.up_proj.MatMulNBits.qweight", + "up_qzeros": "model.layers.30.mlp.up_proj.MatMulNBits.qzeros", + "up_scales": "model.layers.30.mlp.up_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.31.attn.q_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.31.attn.q_proj.MatMulNBits.qweight", + "qzeros": "model.layers.31.attn.q_proj.MatMulNBits.qzeros", + "scales": "model.layers.31.attn.q_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.31.attn.k_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.31.attn.k_proj.MatMulNBits.qweight", + "qzeros": "model.layers.31.attn.k_proj.MatMulNBits.qzeros", + "scales": "model.layers.31.attn.k_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 1024 + }, + "kind": "matmul", + "name": "model.layers.31.attn.v_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.31.attn.v_proj.MatMulNBits.qweight", + "qzeros": "model.layers.31.attn.v_proj.MatMulNBits.qzeros", + "scales": "model.layers.31.attn.v_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 3072 + }, + "kind": "matmul", + "name": "model.layers.31.attn.o_proj.MatMulNBits", + "roles": { + "qweight": "model.layers.31.attn.o_proj.MatMulNBits.qweight", + "qzeros": "model.layers.31.attn.o_proj.MatMulNBits.qzeros", + "scales": "model.layers.31.attn.o_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "k": 3072, + "n": 8192 + }, + "kind": "ssmlp", + "name": "model.layers.31.ssmlp", + "roles": { + "down_qweight": "model.layers.31.mlp.down_proj.MatMulNBits.qweight", + "down_qzeros": "model.layers.31.mlp.down_proj.MatMulNBits.qzeros", + "down_scales": "model.layers.31.mlp.down_proj.MatMulNBits.scales", + "gate_qweight": "model.layers.31.mlp.gate_proj.MatMulNBits.qweight", + "gate_qzeros": "model.layers.31.mlp.gate_proj.MatMulNBits.qzeros", + "gate_scales": "model.layers.31.mlp.gate_proj.MatMulNBits.scales", + "norm0": "model.layers.31.post_attention_layernorm.weight", + "norm1": "model.layers.32.final_norm_layernorm.weight", + "up_qweight": "model.layers.31.mlp.up_proj.MatMulNBits.qweight", + "up_qzeros": "model.layers.31.mlp.up_proj.MatMulNBits.qzeros", + "up_scales": "model.layers.31.mlp.up_proj.MatMulNBits.scales" + } + }, + { + "descriptor": { + "group_size": 128, + "has_bias": false, + "k": 3072, + "n": 200064 + }, + "kind": "matmul", + "name": "lm_head.MatMulNBits", + "roles": { + "qweight": "lm_head.MatMulNBits.qweight", + "qzeros": "lm_head.MatMulNBits.qzeros", + "scales": "lm_head.MatMulNBits.scales" + } + } + ] +} diff --git a/src/model_overlays/phi4-mini-it-aie4/provenance.json b/src/model_overlays/phi4-mini-it-aie4/provenance.json new file mode 100644 index 00000000..430229aa --- /dev/null +++ b/src/model_overlays/phi4-mini-it-aie4/provenance.json @@ -0,0 +1,117 @@ +{ + "generated": { + "config.json": { + "sha256": "1b3e74125a109c05f53c8383def18359d8581619f998c4f91b3b5bb78bf2919f", + "size": 257 + }, + "corelib_phi4_manifest.json": { + "sha256": "09cee6efafc513a2048c89e75b40d92d096144f0eed3b2459c138032b69dc045", + "size": 274527 + }, + "tokenizer_config.json": { + "sha256": "274d22c3cd28c042f28a681832722536663b06e8c9e88b6616622dceab922ca6", + "size": 3036 + } + }, + "upstream": { + "commit": "e751fb68c2cfffe6b0d32942118f75ac0a0365bb", + "git_files": [ + { + "oid": "f78df3bfb43291872abf78496110e5856b24de73", + "path": ".gitattributes", + "size": 1622, + "type": "file" + }, + { + "oid": "77e6ed402865af3c38b5e44c2d0be68f76f0784f", + "path": "added_tokens.json", + "size": 261, + "type": "file" + }, + { + "oid": "a9c00dd9bbd97e117371168e9d62af65b9f0e725", + "path": "chat_template.jinja", + "size": 423, + "type": "file" + }, + { + "oid": "55b698edc57fe52593dce4423beb51ded982dc96", + "path": "genai_config.json", + "size": 1720, + "type": "file" + }, + { + "oid": "dcecc4524288b351bbd0da8028e74e9b5bcdb9b5", + "path": "merges.txt", + "size": 2418348, + "type": "file" + }, + { + "lfs": { + "oid": "e80b9d83e018784eda6263d09fa2ab7729087722c6073c72123366f2dbec4529", + "pointerSize": 131, + "size": 378325 + }, + "oid": "df6d4309745d4627b82fd92c615589193ea528db", + "path": "model.onnx", + "size": 378325, + "type": "file" + }, + { + "lfs": { + "oid": "c48fd647beb02866e68d6dd1fbc05809a439283501d8abb7f6d6b950f92b7b60", + "pointerSize": 135, + "size": 3248488448 + }, + "oid": "5ef8f7f74cf6ac6411b8918e8a850bd2c769dd45", + "path": "model.onnx.data", + "size": 3248488448, + "type": "file" + }, + { + "oid": "18eba67aba3ef71b01ed13c16b3feed78e83001f", + "path": "special_tokens_map.json", + "size": 617, + "type": "file" + }, + { + "lfs": { + "oid": "382cc235b56c725945e149cc25f191da667c836655efd0857b004320e90e91ea", + "pointerSize": 133, + "size": 15524095 + }, + "oid": "3a12dacd8e86802d0229d810d3cc69ab548adf0a", + "path": "tokenizer.json", + "size": 15524095, + "type": "file" + }, + { + "oid": "c182c54743fe1735b0d5eb3959b9757a160879e8", + "path": "tokenizer_config.json", + "size": 2654, + "type": "file" + }, + { + "oid": "ea953a43348cdb3776cb7fd9ea02e3784febde34", + "path": "vocab.json", + "size": 3910310, + "type": "file" + } + ], + "inputs": { + "chat_template.jinja": { + "sha256": "febf589225c9728ab791f52e8897d7607a823d45368f0a4c92fa68997b40cce9", + "size": 423 + }, + "genai_config.json": { + "sha256": "8b7206f5f94e84cc0f80d90458cbb6a77caa61ce97e2d694661b9ed0286c820b", + "size": 1720 + }, + "tokenizer_config.json": { + "sha256": "35aab11f13510ccb463fe7e2e54b86eb29406e66b801a66e90e85b89e6da988c", + "size": 2654 + } + }, + "repository": "https://huggingface.co/amd/phi-4-mini-instruct-oga-dml" + } +} diff --git a/src/model_overlays/phi4-mini-it-aie4/tokenizer_config.json b/src/model_overlays/phi4-mini-it-aie4/tokenizer_config.json new file mode 100644 index 00000000..6675e217 --- /dev/null +++ b/src/model_overlays/phi4-mini-it-aie4/tokenizer_config.json @@ -0,0 +1,117 @@ +{ + "add_bos_token": false, + "add_eos_token": false, + "add_prefix_space": false, + "added_tokens_decoder": { + "199999": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "200018": { + "content": "<|endofprompt|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "200019": { + "content": "<|assistant|>", + "lstrip": false, + "normalized": false, + "rstrip": true, + "single_word": false, + "special": true + }, + "200020": { + "content": "<|end|>", + "lstrip": false, + "normalized": false, + "rstrip": true, + "single_word": false, + "special": true + }, + "200021": { + "content": "<|user|>", + "lstrip": false, + "normalized": false, + "rstrip": true, + "single_word": false, + "special": true + }, + "200022": { + "content": "<|system|>", + "lstrip": false, + "normalized": false, + "rstrip": true, + "single_word": false, + "special": true + }, + "200023": { + "content": "<|tool|>", + "lstrip": false, + "normalized": false, + "rstrip": true, + "single_word": false, + "special": false + }, + "200024": { + "content": "<|/tool|>", + "lstrip": false, + "normalized": false, + "rstrip": true, + "single_word": false, + "special": false + }, + "200025": { + "content": "<|tool_call|>", + "lstrip": false, + "normalized": false, + "rstrip": true, + "single_word": false, + "special": false + }, + "200026": { + "content": "<|/tool_call|>", + "lstrip": false, + "normalized": false, + "rstrip": true, + "single_word": false, + "special": false + }, + "200027": { + "content": "<|tool_response|>", + "lstrip": false, + "normalized": false, + "rstrip": true, + "single_word": false, + "special": false + }, + "200028": { + "content": "<|tag|>", + "lstrip": false, + "normalized": false, + "rstrip": true, + "single_word": false, + "special": true + } + }, + "bos_token": "<|endoftext|>", + "chat_template": "{% for message in messages %}{% if message['role'] == 'system' and 'tools' in message and message['tools'] is not none %}{{ '<|' + message['role'] + '|>' + message['content'] + '<|tool|>' + message['tools'] + '<|/tool|>' + '<|end|>' }}{% else %}{{ '<|' + message['role'] + '|>' + message['content'] + '<|end|>' }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>' }}{% else %}{{ eos_token }}{% endif %}", + "clean_up_tokenization_spaces": false, + "eos_token": "<|endoftext|>", + "eos_token_id": [ + 200020, + 199999 + ], + "extra_special_tokens": {}, + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "padding_side": "left", + "tokenizer_class": "GPT2Tokenizer", + "unk_token": "<|endoftext|>" +} diff --git a/src/pull/download_model.cpp b/src/pull/download_model.cpp index 38ccf44a..b691dd38 100644 --- a/src/pull/download_model.cpp +++ b/src/pull/download_model.cpp @@ -5,6 +5,7 @@ /// \version 0.9.24 /// \note This class for curl download #include "download_model.hpp" +#include #include #include #include diff --git a/src/pull/download_model.hpp b/src/pull/download_model.hpp index 12e9a1a1..28351cc5 100644 --- a/src/pull/download_model.hpp +++ b/src/pull/download_model.hpp @@ -6,11 +6,12 @@ /// \note This class for curl download #pragma once +#include +#include #include #include #include #include -#include #include "nlohmann/json.hpp" namespace download_utils { diff --git a/src/pull/model_downloader.cpp b/src/pull/model_downloader.cpp index 24fc98d4..a5adfbdc 100644 --- a/src/pull/model_downloader.cpp +++ b/src/pull/model_downloader.cpp @@ -5,6 +5,7 @@ /// \version 0.9.24 /// \note This class is used to download models from the huggingface #include "model_downloader.hpp" +#include #include "utils/utils.hpp" #include "download_model.hpp" #include @@ -87,6 +88,15 @@ bool ModelDownloader::pull_model(const std::string& model_tag, bool use_modelsco try { // Get model info auto [new_model_tag, model_info] = supported_models.get_model_info(model_tag); + flm::pull::RequireSupportedModelSource( + model_info, + use_modelscope); + if (model_info.contains("bundled_overlays")) { + flm::pull::StageBundledOverlays( + model_info, + utils::find_model_overlay_root(), + supported_models.get_model_path(new_model_tag)); + } std::string model_name = model_info["name"]; std::string model_server = use_modelscope ? "ModelScope" : "HuggingFace"; @@ -292,10 +302,12 @@ std::pair ModelDownloader::build_download_list(const std: try { auto [new_model_tag, model_info] = supported_models.get_model_info(model_tag); + flm::pull::RequireSupportedModelSource(model_info, modelscope); std::string base_url = modelscope ? model_info["ms_url"] : model_info["url"]; std::string model_name = model_info["name"]; std::string file_url = model_info["file_url"]; - std::vector model_files = model_info["files"]; + std::vector model_files = + flm::pull::RemoteModelFiles(model_info); // Create model directory std::string model_path = supported_models.get_model_path(new_model_tag); @@ -334,6 +346,11 @@ std::pair ModelDownloader::build_download_list(const std: if (std::string(base_url).find("resolve") != std::string::npos) { // resolve provided , may from a specific branch url = base_url + "/" + filename + "?download=true"; } + else if (!modelscope) { + url = flm::pull::BuildRemoteFileUrl( + model_info, + filename); + } else { url = base_url + "/resolve/main/" + filename + "?download=true"; } @@ -421,6 +438,7 @@ bool ModelDownloader::remove_model(const std::string& model_tag, bool sub_proces /// \return true if all files are present and compatible, false otherwise bool ModelDownloader::check_model(const std::string& model_tag, bool use_modelscope, bool sub_process_mode) { auto [new_model_tag, model_info] = supported_models.get_model_info(model_tag); + flm::pull::RequireSupportedModelSource(model_info, use_modelscope); header_print("FLM", "Checking model: " + new_model_tag + "...\n"); ModelStatus status = is_model_downloaded(new_model_tag, sub_process_mode); @@ -455,6 +473,9 @@ bool ModelDownloader::verify_and_clean_files(const std::string& model_tag, bool bool any_error = false; try { auto [new_model_tag, model_info] = supported_models.get_model_info(model_tag); + flm::pull::RequireSupportedModelSource( + model_info, + use_modelscope); std::vector model_files = model_info["files"]; std::string model_path = supported_models.get_model_path(new_model_tag); std::string file_url = model_info["file_url"]; @@ -477,6 +498,33 @@ bool ModelDownloader::verify_and_clean_files(const std::string& model_tag, bool header_print("FLM", "Checking file: " + filename + "..."); } + std::string local_path = + get_model_file_path(model_path, filename); + if ( + model_info.contains("bundled_overlays") && + model_info["bundled_overlays"].contains(filename)) { + if (flm::pull::VerifyBundledOverlayTarget( + model_info, + filename, + model_path)) { + if (!sub_process_mode) { + header_print("FLM", "Success!"); + } + } else { + if (!sub_process_mode) { + header_print("FLM", "Fail!"); + header_print( + "FLM", + "Removing corrupted bundled overlay: " + + filename + "..."); + } + std::error_code ignored; + std::filesystem::remove(local_path, ignored); + any_error = true; + } + continue; + } + auto it = std::find_if( hf_model_infos.begin(), hf_model_infos.end(), @@ -488,7 +536,6 @@ bool ModelDownloader::verify_and_clean_files(const std::string& model_tag, bool continue; } const auto& file = *it; - std::string local_path = get_model_file_path(model_path, filename); // If the file isn't present locally, there's nothing to verify or // remove; treat as an error so the caller knows a re-pull is needed. diff --git a/src/pull/model_overlay.cpp b/src/pull/model_overlay.cpp new file mode 100644 index 00000000..b086dc9f --- /dev/null +++ b/src/pull/model_overlay.cpp @@ -0,0 +1,302 @@ +#include + +#include "picosha2.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +#include +#endif + +namespace flm::pull { +namespace { + +struct OverlayRecord final { + std::string target; + std::filesystem::path source; + std::uint64_t size; + std::string sha256; +}; + +bool IsSha256(std::string_view value) { + return value.size() == 64 && + std::all_of( + value.begin(), + value.end(), + [](unsigned char character) { + return std::isxdigit(character) != 0; + }); +} + +std::filesystem::path SafeRelativePath( + const std::string& value, + std::string_view field) { + std::filesystem::path path(value); + if ( + value.empty() || + path.is_absolute() || + path.has_root_name() || + path.has_root_directory()) { + throw std::invalid_argument( + std::string(field) + " must be a non-empty relative path"); + } + for (const auto& component : path) { + if (component == "..") { + throw std::invalid_argument( + std::string(field) + " must not contain path traversal"); + } + } + return path.lexically_normal(); +} + +std::vector ReadOverlayRecords( + const nlohmann::json& model_info, + const std::filesystem::path& overlay_root) { + const auto found = model_info.find("bundled_overlays"); + if (found == model_info.end()) { + return {}; + } + if (!found->is_object()) { + throw std::invalid_argument("bundled_overlays must be an object"); + } + + std::vector records; + records.reserve(found->size()); + for (const auto& [target, value] : found->items()) { + if (!value.is_object()) { + throw std::invalid_argument( + "bundled overlay record must be an object: " + target); + } + if ( + !value.contains("path") || + !value.contains("size") || + !value.contains("sha256") || + value.size() != 3) { + throw std::invalid_argument( + "bundled overlay record must contain path, size, and " + "sha256: " + + target); + } + const auto target_path = + SafeRelativePath(target, "bundled overlay target"); + if ( + target_path.parent_path() != std::filesystem::path() || + target_path.filename().string() != target) { + throw std::invalid_argument( + "bundled overlay target must be a model-root filename"); + } + const auto source_relative = SafeRelativePath( + value.at("path").get(), + "bundled overlay source"); + const auto size = value.at("size").get(); + std::string sha256 = value.at("sha256").get(); + if (!IsSha256(sha256)) { + throw std::invalid_argument( + "bundled overlay SHA-256 is invalid: " + target); + } + std::transform( + sha256.begin(), + sha256.end(), + sha256.begin(), + [](unsigned char character) { + return static_cast(std::tolower(character)); + }); + records.push_back( + OverlayRecord{ + target, + overlay_root / source_relative, + size, + std::move(sha256)}); + } + return records; +} + +void VerifySource(const OverlayRecord& record) { + std::error_code error; + const auto size = std::filesystem::file_size(record.source, error); + if (error) { + throw std::runtime_error( + "bundled overlay source is missing or unreadable: " + + record.source.string()); + } + if (size != record.size) { + throw std::runtime_error( + "bundled overlay size mismatch: " + + record.source.string()); + } + if (CalculateFileSha256(record.source) != record.sha256) { + throw std::runtime_error( + "bundled overlay SHA-256 mismatch: " + + record.source.string()); + } +} + +bool TargetMatches( + const OverlayRecord& record, + const std::filesystem::path& model_dir) { + const auto target = model_dir / record.target; + std::error_code error; + const auto size = std::filesystem::file_size(target, error); + return !error && + size == record.size && + CalculateFileSha256(target) == record.sha256; +} + +void CopyAtomically( + const OverlayRecord& record, + const std::filesystem::path& model_dir) { + static std::atomic sequence{0}; + const auto target = model_dir / record.target; + const auto temporary = + model_dir / + ("." + record.target + ".flm-overlay-" + +#ifdef _WIN32 + std::to_string(GetCurrentProcessId()) + "-" + +#else + std::string("process-") + +#endif + std::to_string( + sequence.fetch_add(1, std::memory_order_relaxed)) + + ".tmp"); + + std::error_code ignored; + std::filesystem::remove(temporary, ignored); + try { +#ifdef _WIN32 + if (!CopyFileW( + record.source.c_str(), + temporary.c_str(), + TRUE)) { + throw std::system_error( + static_cast(GetLastError()), + std::system_category(), + "failed to copy bundled overlay to temporary file"); + } +#else + std::filesystem::copy_file( + record.source, + temporary, + std::filesystem::copy_options::none); +#endif + OverlayRecord temporary_record = record; + temporary_record.source = temporary; + VerifySource(temporary_record); +#ifdef _WIN32 + if (!MoveFileExW( + temporary.c_str(), + target.c_str(), + MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH)) { + throw std::system_error( + static_cast(GetLastError()), + std::system_category(), + "failed to atomically install bundled overlay"); + } +#else + std::filesystem::rename(temporary, target); +#endif + } catch (...) { + std::filesystem::remove(temporary, ignored); + throw; + } +} + +} // namespace + +void RequireSupportedModelSource( + const nlohmann::json& model_info, + bool use_modelscope) { + if (!use_modelscope) { + return; + } + const auto found = model_info.find("modelscope_supported"); + if ( + found != model_info.end() && + (!found->is_boolean() || !found->get())) { + throw std::invalid_argument( + "this model does not support --modelscope; use the " + "Hugging Face source"); + } +} + +std::vector RemoteModelFiles( + const nlohmann::json& model_info) { + const auto overlays = model_info.find("bundled_overlays"); + std::vector files; + for (const auto& value : model_info.at("files")) { + const std::string filename = value.get(); + if ( + overlays == model_info.end() || + !overlays->is_object() || + !overlays->contains(filename)) { + files.push_back(filename); + } + } + return files; +} + +std::string BuildRemoteFileUrl( + const nlohmann::json& model_info, + std::string_view filename) { + const std::string base_url = model_info.at("url").get(); + if (base_url.find("resolve") != std::string::npos) { + return base_url + "/" + std::string(filename) + "?download=true"; + } + const std::string revision = + model_info.value("revision", std::string("main")); + return base_url + "/resolve/" + revision + "/" + + std::string(filename) + "?download=true"; +} + +std::string CalculateFileSha256(const std::filesystem::path& path) { + std::ifstream input(path, std::ios::binary); + if (!input) { + throw std::runtime_error( + "failed to open file for SHA-256: " + path.string()); + } + std::vector hash(picosha2::k_digest_size); + picosha2::hash256(input, hash.begin(), hash.end()); + return picosha2::bytes_to_hex_string(hash.begin(), hash.end()); +} + +void StageBundledOverlays( + const nlohmann::json& model_info, + const std::filesystem::path& overlay_root, + const std::filesystem::path& model_dir) { + const auto records = ReadOverlayRecords(model_info, overlay_root); + for (const auto& record : records) { + VerifySource(record); + } + if (records.empty()) { + return; + } + std::filesystem::create_directories(model_dir); + for (const auto& record : records) { + if (!TargetMatches(record, model_dir)) { + CopyAtomically(record, model_dir); + } + } +} + +bool VerifyBundledOverlayTarget( + const nlohmann::json& model_info, + std::string_view filename, + const std::filesystem::path& model_dir) { + const auto records = ReadOverlayRecords(model_info, {}); + const auto found = std::find_if( + records.begin(), + records.end(), + [&](const OverlayRecord& record) { + return record.target == filename; + }); + return found != records.end() && TargetMatches(*found, model_dir); +} + +} // namespace flm::pull diff --git a/src/runner/runner.cpp b/src/runner/runner.cpp index d38e5e9c..3fb3bf70 100644 --- a/src/runner/runner.cpp +++ b/src/runner/runner.cpp @@ -8,6 +8,7 @@ */ #include "runner.hpp" #include "harmony_filter.hpp" +#include #ifndef FASTFLOWLM_LINUX_LIMITED_MODELS #include "AutoEmbeddingModel/all_embedding_model.hpp" #endif @@ -18,9 +19,22 @@ #include #include #include +#include #include #include +namespace { + +void PrintModelRequestError(const ModelRequestError& error) { + std::cerr + << CliModelErrorNotice( + error.what(), + error.session_cleared()) + << '\n'; +} + +} // namespace + /// \brief Command map for command line input std::map cmd_map = { {"/set", CMD_SET}, @@ -66,7 +80,9 @@ Runner::Runner(model_list& supported_models, ModelDownloader& downloader, progra this->downloader.pull_model(this->tag, this->modelscope); break; case ModelDownloader::ModelStatus::Incompatible: - exit(EXIT_FAILURE); + throw std::runtime_error( + "Model is incompatible with this version of FastFlowLM: " + + this->tag); } auto [new_tag, model_info] = this->supported_models.get_model_info(this->tag); this->asr_supported = model_info.contains("asr") && model_info["asr"]; @@ -77,7 +93,7 @@ Runner::Runner(model_list& supported_models, ModelDownloader& downloader, progra } catch (const std::exception& e) { header_print("ERROR", "Failed to load model: " + std::string(e.what())); - exit(EXIT_FAILURE); + throw; } try { @@ -127,7 +143,7 @@ Runner::Runner(model_list& supported_models, ModelDownloader& downloader, progra } catch (const std::exception& e) { header_print("WARNING", "Failed to load ASR model: " + std::string(e.what())); - exit(EXIT_FAILURE); + throw; } } else { @@ -358,6 +374,8 @@ void Runner::run() { chat_meta_info_t meta_info; meta_info.max_prefill_len = this->prefill_chunk_len; uniformed_input.prompt = input; + uniformed_input.requested_max_new_tokens = + CliRequestedMaxNewTokens(this->generate_limit); this->auto_chat_engine->start_total_timer(); @@ -369,6 +387,10 @@ void Runner::run() { break; } } + catch (const ModelRequestError& error) { + PrintModelRequestError(error); + continue; + } catch (const std::exception& e) { header_print("ERROR", "Insertion error: " + std::string(e.what())); this->auto_chat_engine->clear_context(); @@ -383,6 +405,10 @@ void Runner::run() { try { this->auto_chat_engine->generate(meta_info, this->generate_limit, harmony_filter_ostream); } + catch (const ModelRequestError& error) { + PrintModelRequestError(error); + continue; + } catch (const std::exception& e) { header_print("ERROR", "Generation error: " + std::string(e.what())); this->auto_chat_engine->clear_context(); @@ -392,6 +418,10 @@ void Runner::run() { try { this->auto_chat_engine->generate(meta_info, this->generate_limit, base_ostream); } + catch (const ModelRequestError& error) { + PrintModelRequestError(error); + continue; + } catch (const std::exception& e) { header_print("ERROR", "Generation error: " + std::string(e.what())); this->auto_chat_engine->clear_context(); @@ -440,7 +470,10 @@ void Runner::cmd_load(std::vector& input_list) { break; case ModelDownloader::ModelStatus::Incompatible: header_print("ERROR", "Model is incompatible with this version of FastFlowLM: " + this->tag); - exit(EXIT_FAILURE); + throw std::runtime_error( + "Model is incompatible with this version of " + "FastFlowLM: " + + this->tag); } auto_chat_engine.reset(); if(model_name=="gpt-oss:20b") @@ -454,7 +487,7 @@ void Runner::cmd_load(std::vector& input_list) { } catch (const std::exception& e) { header_print("ERROR", "Failed to load model: " + std::string(e.what())); - exit(EXIT_FAILURE); + throw; } this->auto_chat_engine->configure_parameter("system_prompt", this->system_prompt); @@ -604,7 +637,7 @@ void Runner::cmd_set(std::vector& input_list) { } catch (const std::exception& e) { header_print("ERROR", "Failed to set context length: " + std::string(e.what())); - exit(EXIT_FAILURE); + return; } } else if (set_context == "gen-lim"){ diff --git a/src/server/generation_limit.cpp b/src/server/generation_limit.cpp new file mode 100644 index 00000000..25e77833 --- /dev/null +++ b/src/server/generation_limit.cpp @@ -0,0 +1,239 @@ +#include + +#include + +namespace { + +constexpr int kLegacyDefaultGenerationLimit = 4096; +constexpr int kNoExplicitGenerationLimit = -1; + +constexpr std::array kGenerationRoutes{{ + {"POST", "/api/generate", GenerationEndpoint::Generate}, + {"POST", "/api/chat", GenerationEndpoint::OllamaChat}, + {"POST", + "/v1/chat/completions", + GenerationEndpoint::OpenAiChatCompletion}, + {"POST", "/v1/completions", GenerationEndpoint::OpenAiCompletion}, +}}; + +ParsedGenerationLimit ParseField( + const nlohmann::ordered_json& request, + std::string_view field) { + if (!request.contains(field)) { + return {false, kNoExplicitGenerationLimit}; + } + return {true, request.at(field).get()}; +} + +} // namespace + +std::span GenerationRoutes() noexcept { + return kGenerationRoutes; +} + +std::optional GenerationEndpointForRoute( + std::string_view method, + std::string_view path) noexcept { + for (const GenerationRoute& route : kGenerationRoutes) { + if (route.method == method && route.path == path) { + return route.endpoint; + } + } + return std::nullopt; +} + +GenerationEndpoint RequireGenerationEndpoint( + std::string_view method, + std::string_view path) { + const auto endpoint = GenerationEndpointForRoute(method, path); + if (!endpoint.has_value()) { + throw std::logic_error( + "generation route " + std::string(method) + " " + + std::string(path) + + " is missing from GenerationRoutes(); add it there so its " + "limit field and admission rule are declared"); + } + return *endpoint; +} + +ParsedGenerationLimit ParseGenerationLimit( + const nlohmann::ordered_json& request, + GenerationEndpoint endpoint) { + switch (endpoint) { + case GenerationEndpoint::Generate: + case GenerationEndpoint::OpenAiCompletion: + return ParseField(request, "max_tokens"); + case GenerationEndpoint::OllamaChat: { + // Ollama nests the limit, and reads it with the same + // presence/absence rule as the flat endpoints. + const nlohmann::ordered_json options = + request.value( + "options", + nlohmann::ordered_json::object()); + return ParseField(options, "num_predict"); + } + case GenerationEndpoint::OpenAiChatCompletion: { + const ParsedGenerationLimit max_tokens = + ParseField(request, "max_tokens"); + if (max_tokens.explicit_limit) { + return max_tokens; + } + return ParseField(request, "max_completion_tokens"); + } + } + return {false, kNoExplicitGenerationLimit}; +} + +int GenerationLoopLimit( + const ParsedGenerationLimit& parsed, + bool uses_corelib_aie4) noexcept { + if (parsed.explicit_limit) { + return parsed.value; + } + return uses_corelib_aie4 + ? kNoExplicitGenerationLimit + : kLegacyDefaultGenerationLimit; +} + +// The admission rule's view of the same parsed field. This is NOT the loop +// limit: it is "how many output tokens must the AIE4 cap reserve room for", +// and the only consumer is Phi4::validate_aie4_capacity. +// +// CHOSEN BEHAVIOUR: a non-positive value is not a token budget, so it is +// reported as "no budget requested" -- exactly as if the field had been +// omitted. Positive values pass through unchanged. +// +// Why, rather than passing the number through: +// +// * Ollama documents num_predict = -1 as "generate forever" and -2 as +// "fill context". Both are sentinels, not counts. GenerationLoopLimit +// already returns them verbatim and both generation loops gate on +// `length_limit > 0`, so the loops correctly read them as "unbounded". +// Passing -1 on to the admission rule as a *count* made +// POST /api/chat {"options":{"num_predict":-1}} return HTTP 400 on the +// AIE4 tag and 200 on every other backend -- and made explicitly asking +// for the unbounded behaviour refuse where omitting the field allows it. +// +// * 0 is deliberately in the same bucket rather than there by accident. +// Both loops test `length_limit > 0`, so an explicit 0 means "no bound" +// to the generation loop; reserving 0 output tokens at admission and +// then generating to the cap would be the admission rule and the loop +// disagreeing about the same request. Whether an explicit 0 *ought* to +// mean "emit nothing" is a separate question about both backends' +// generation loops, not about this function, and is not decided here. +// +// * CliRequestedMaxNewTokens below already maps <= 0 to nullopt. The REST +// path did not, so the same request was admitted differently depending +// on which entry point it arrived through. +std::optional RequestedMaxNewTokens( + const ParsedGenerationLimit& parsed) noexcept { + if (!parsed.explicit_limit || parsed.value <= 0) { + return std::nullopt; + } + return parsed.value; +} + +int OllamaChatGenerationLoopLimit( + const nlohmann::ordered_json& request) { + // Retained as the legacy (non-AIE4) spelling, and defined in terms of + // the shared rule so the two cannot drift apart. + return GenerationLoopLimit( + ParseGenerationLimit(request, GenerationEndpoint::OllamaChat), + false); +} + +nlohmann::ordered_json ModelErrorResponse( + std::string_view message, + int http_code, + bool session_cleared) { + return { + {"error", { + {"message", std::string(message)}, + {"type", http_code == 400 + ? "invalid_request_error" + : "server_error"}, + {"code", http_code}, + {"session_cleared", session_cleared}, + }}, + }; +} + +int HttpStatusForResponse( + const nlohmann::ordered_json& response) noexcept { + try { + if (!response.is_object() || !response.contains("error")) { + return 200; + } + const auto& error = response.at("error"); + if (!error.is_object() || !error.contains("code")) { + return 200; + } + const int code = error.at("code").get(); + return code == 400 || code == 500 ? code : 200; + } catch (...) { + return 200; + } +} + +std::array OpenAiStreamingErrorFrames( + const nlohmann::ordered_json& error_response) { + return { + "data: " + error_response.dump() + "\n\n", + "data: [DONE]\n\n", + }; +} + +void SendOpenAiStreamingError( + const nlohmann::ordered_json& error_response, + const std::function& send_streaming_response) { + const auto frames = + OpenAiStreamingErrorFrames(error_response); + send_streaming_response( + nlohmann::ordered_json(frames[0]), + false); + send_streaming_response( + nlohmann::ordered_json(frames[1]), + true); +} + +bool UseFinalStreamingErrorChunk( + bool stream_started) noexcept { + return stream_started; +} + +std::optional CliRequestedMaxNewTokens( + int generate_limit) noexcept { + if (generate_limit <= 0) { + return std::nullopt; + } + return generate_limit; +} + +std::string CliModelErrorNotice( + std::string_view message, + bool session_cleared) { + if (session_cleared) { + return + "ERROR: AIE4 inference failed at a recoverable point; the " + "current conversation was cleared."; + } + return "ERROR: " + std::string(message); +} + +bool IsCorelibAie4ModelInfo( + const nlohmann::ordered_json& model_info) noexcept { + try { + if (!model_info.is_object() || + !model_info.contains("details")) { + return false; + } + const auto& details = model_info.at("details"); + return details.is_object() && + details.value("execution_backend", std::string{}) == + "corelib_aie4"; + } catch (...) { + return false; + } +} diff --git a/src/server/npu_access_manager.cpp b/src/server/npu_access_manager.cpp new file mode 100644 index 00000000..8690d7ce --- /dev/null +++ b/src/server/npu_access_manager.cpp @@ -0,0 +1,50 @@ +#include + +#include +#include + +namespace { + +std::mutex g_npu_access_mutex; +std::atomic g_npu_in_use{false}; +std::atomic g_npu_active_requests{0}; + +} // namespace + +bool NPUAccessManager::try_acquire_npu_access() { + std::lock_guard lock(g_npu_access_mutex); + if (g_npu_in_use.load(std::memory_order_relaxed)) { + return false; + } + g_npu_in_use.store(true, std::memory_order_release); + g_npu_active_requests.fetch_add(1, std::memory_order_relaxed); + return true; +} + +void NPUAccessManager::release_npu_access() { + std::lock_guard lock(g_npu_access_mutex); + g_npu_in_use.store(false, std::memory_order_release); + g_npu_active_requests.fetch_sub(1, std::memory_order_relaxed); +} + +bool NPUAccessManager::is_npu_available() { + return !g_npu_in_use.load(std::memory_order_acquire); +} + +int NPUAccessManager::get_active_npu_requests() { + return g_npu_active_requests.load(std::memory_order_relaxed); +} + +bool requires_npu_access( + const std::string& method, + const std::string& path) { + if (method != "POST") { + return false; + } + return path == "/api/generate" || + path == "/api/chat" || + path == "/v1/chat/completions" || + path == "/v1/completions" || + path == "/v1/audio/transcriptions" || + path == "/v1/embeddings"; +} diff --git a/src/server/rest_handler.cpp b/src/server/rest_handler.cpp index 95e90a3a..43dcb136 100644 --- a/src/server/rest_handler.cpp +++ b/src/server/rest_handler.cpp @@ -11,6 +11,7 @@ #include "streaming_ostream.hpp" #include "streaming_ostream_openai.hpp" #include "image/image_reader.hpp" +#include #include #include #include @@ -20,6 +21,17 @@ #include #include "server.hpp" +namespace { + +json ModelErrorResponse(const ModelRequestError& error) { + return ::ModelErrorResponse( + error.what(), + error.http_code(), + error.session_cleared()); +} + +} // namespace + ///@brief Normalize messages by merging consecutive user messages (like Ollama does) ///@param messages the original messages ///@return normalized messages with consecutive user messages merged @@ -638,12 +650,14 @@ void RestHandler::handle_generate(const json& request, StreamResponseCallback send_streaming_response, std::shared_ptr cancellation_token) { try { + const ParsedGenerationLimit parsed_limit = + ParseGenerationLimit( + request, + RequireGenerationEndpoint("POST", "/api/generate")); std::string prompt = request["prompt"]; bool stream = request.value("stream", true); std::string model = request.value("model", current_model_tag); json options = request.value("options", json::object()); - - int length_limit = request.value("max_tokens", 4096); auto load_start_time = time_utils::now(); // TODO: Use Another Check Function avoid loading again if (!ensure_model_loaded(model)) { @@ -651,10 +665,16 @@ void RestHandler::handle_generate(const json& request, send_response(error_response); return; } + const int length_limit = + GenerationLoopLimit( + parsed_limit, + auto_chat_engine->uses_corelib_aie4()); auto load_end_time = time_utils::now(); chat_meta_info_t meta_info; lm_uniform_input_t uniformed_input; + uniformed_input.requested_max_new_tokens = + RequestedMaxNewTokens(parsed_limit); meta_info.max_prefill_len = this->prefill_chunk_len; meta_info.load_duration = (uint64_t)time_utils::duration_ns(load_start_time, load_end_time).first; header_print("FLM", "Start generating..."); @@ -662,7 +682,18 @@ void RestHandler::handle_generate(const json& request, if (stream) { // Streaming response using streaming_ostream auto total_start_time = time_utils::now(); - streaming_ostream ostream(model, send_streaming_response, false); + bool stream_started = false; + auto tracked_stream_response = + [&stream_started, &send_streaming_response]( + const json& data, + bool is_final) { + stream_started = true; + send_streaming_response(data, is_final); + }; + streaming_ostream ostream( + model, + tracked_stream_response, + false); uniformed_input.prompt = prompt; try { bool success = auto_chat_engine->insert(meta_info, uniformed_input); @@ -672,6 +703,9 @@ void RestHandler::handle_generate(const json& request, this->auto_chat_engine->clear_context(); return; } + } catch (const ModelRequestError& error) { + send_response(ModelErrorResponse(error)); + return; } catch (const std::exception& e) { json error_response = {{"error", e.what()}}; send_response(error_response); @@ -680,6 +714,14 @@ void RestHandler::handle_generate(const json& request, } try { auto_chat_engine->generate(meta_info, length_limit, ostream); + } catch (const ModelRequestError& error) { + const json error_response = ModelErrorResponse(error); + if (UseFinalStreamingErrorChunk(stream_started)) { + send_streaming_response(error_response, true); + } else { + send_response(error_response); + } + return; } catch (const std::exception& e) { json error_response = {{"error", e.what()}}; send_response(error_response); @@ -705,6 +747,9 @@ void RestHandler::handle_generate(const json& request, this->auto_chat_engine->clear_context(); return; } + } catch (const ModelRequestError& error) { + send_response(ModelErrorResponse(error)); + return; } catch (const std::exception& e) { json error_response = {{"error", e.what()}}; send_response(error_response); @@ -713,6 +758,9 @@ void RestHandler::handle_generate(const json& request, } try { auto_chat_engine->generate(meta_info, length_limit, ostream); + } catch (const ModelRequestError& error) { + send_response(ModelErrorResponse(error)); + return; } catch (const std::exception& e) { json error_response = {{"error", e.what()}}; send_response(error_response); @@ -737,6 +785,8 @@ void RestHandler::handle_generate(const json& request, // std::cout << "history: " << history.first << std::endl; send_response(response); } + } catch (const ModelRequestError& error) { + send_response(ModelErrorResponse(error)); } catch (const std::exception& e) { json error_response = {{"error", e.what()}}; send_response(error_response); @@ -756,7 +806,10 @@ void RestHandler::handle_chat(const json& request, bool stream = request.value("stream", false); std::string model = request.value("model", current_model_tag); json options = request.value("options", json::object()); - int length_limit = options.value("num_predict", 4096); + const ParsedGenerationLimit parsed_limit = + ParseGenerationLimit( + request, + RequireGenerationEndpoint("POST", "/api/chat")); auto load_start_time = time_utils::now(); if (!ensure_model_loaded(model)) { @@ -764,21 +817,42 @@ void RestHandler::handle_chat(const json& request, send_response(error_response); return; } + // Same rule as the other three generation endpoints. On a legacy + // model this is still options.num_predict defaulting to 4096; on + // AIE4 an omitted limit means "until the context cap", and an + // explicit one takes part in the admission check below. + const int length_limit = + GenerationLoopLimit( + parsed_limit, + auto_chat_engine->uses_corelib_aie4()); auto load_end_time = time_utils::now(); - + configure_chat_engine_parameters(options, request); // messages = normalize_messages(messages); - + chat_meta_info_t meta_info; lm_uniform_input_t uniformed_input; + uniformed_input.requested_max_new_tokens = + RequestedMaxNewTokens(parsed_limit); meta_info.load_duration = (uint64_t)time_utils::duration_ns(load_start_time, load_end_time).first; meta_info.max_prefill_len = this->prefill_chunk_len; header_print("FLM", "Start generating..."); if (stream) { // Streaming response using streaming_ostream auto total_start_time = time_utils::now(); - streaming_ostream ostream(model, send_streaming_response, true); // true for chat format + bool stream_started = false; + auto tracked_stream_response = + [&stream_started, &send_streaming_response]( + const json& data, + bool is_final) { + stream_started = true; + send_streaming_response(data, is_final); + }; + streaming_ostream ostream( + model, + tracked_stream_response, + true); // true for chat format uniformed_input.messages = messages; try { bool success = auto_chat_engine->insert(meta_info, uniformed_input); @@ -788,6 +862,9 @@ void RestHandler::handle_chat(const json& request, this->auto_chat_engine->clear_context(); return; } + } catch (const ModelRequestError& error) { + send_response(ModelErrorResponse(error)); + return; } catch (const std::exception& e) { json error_response = {{"error", e.what()}}; send_response(error_response); @@ -795,13 +872,18 @@ void RestHandler::handle_chat(const json& request, return; } try { - bool success = auto_chat_engine->insert(meta_info, uniformed_input); - if (!success){ - json error_response = {{"error", "Max length reached"}}; + auto_chat_engine->generate( + meta_info, + length_limit, + ostream); + } catch (const ModelRequestError& error) { + const json error_response = ModelErrorResponse(error); + if (UseFinalStreamingErrorChunk(stream_started)) { + send_streaming_response(error_response, true); + } else { send_response(error_response); - this->auto_chat_engine->clear_context(); - return; } + return; } catch (const std::exception& e) { json error_response = {{"error", e.what()}}; send_response(error_response); @@ -824,6 +906,9 @@ void RestHandler::handle_chat(const json& request, std::string response_text; try { response_text = auto_chat_engine->generate_with_prompt(meta_info, uniformed_input, length_limit, nstream); + } catch (const ModelRequestError& error) { + send_response(ModelErrorResponse(error)); + return; } catch (const std::exception& e) { json error_response = {{"error", e.what()}}; send_response(error_response); @@ -856,6 +941,8 @@ void RestHandler::handle_chat(const json& request, // std::cout << "history: " << history.first << std::endl; this->auto_chat_engine->clear_context(); } + } catch (const ModelRequestError& error) { + send_response(ModelErrorResponse(error)); } catch (const std::exception& e) { json error_response = {{"error", e.what()}}; send_response(error_response); @@ -1091,10 +1178,15 @@ void RestHandler::handle_openai_chat_completion(const json& request, static std::string model_used_for_last_message = "model-faker"; try { // Extract OpenAI-style parameters + const ParsedGenerationLimit parsed_limit = + ParseGenerationLimit( + request, + RequireGenerationEndpoint( + "POST", + "/v1/chat/completions")); json current_messages = request["messages"]; std::string model = request.value("model", current_model_tag); bool stream = request.value("stream", false); - int length_limit = request.value("max_tokens", request.value("max_completion_tokens", 4096)); json tools = request.value("tools", json::array()); json options = request.value("options", json::object()); @@ -1104,6 +1196,10 @@ void RestHandler::handle_openai_chat_completion(const json& request, send_response(error_response); return; } + const bool corelib_aie4 = + auto_chat_engine->uses_corelib_aie4(); + const int length_limit = + GenerationLoopLimit(parsed_limit, corelib_aie4); auto load_end_time = time_utils::now(); configure_chat_engine_parameters(options, request); @@ -1119,7 +1215,7 @@ void RestHandler::handle_openai_chat_completion(const json& request, this->prompt_cache.update_tool_checksum(tools); model_used_for_last_message = model; } - else { + else if (!corelib_aie4) { cache_match_info_t cache_info; can_use_prompt_cache = prompt_cache.can_use_cache(current_messages, auto_chat_engine->get_chat_template_type(), tools, cache_info); if (can_use_prompt_cache) { @@ -1146,13 +1242,21 @@ void RestHandler::handle_openai_chat_completion(const json& request, lm_uniform_input_t uniformed_input; uniformed_input.messages = current_messages; uniformed_input.tools = tools; + uniformed_input.requested_max_new_tokens = + RequestedMaxNewTokens(parsed_limit); meta_info.load_duration = (uint64_t)time_utils::duration_ns(load_start_time, load_end_time).first; meta_info.max_prefill_len = this->prefill_chunk_len; if (stream){ // Create a wrapper callback that passes the pre-formatted SSE string directly cancellation_token->reset(); auto_chat_engine->reset_parser(); - auto openai_stream_callback = [&send_streaming_response](const std::string& data, bool is_final) { + bool stream_started = false; + auto openai_stream_callback = [ + &send_streaming_response, + &stream_started]( + const std::string& data, + bool is_final) { + stream_started = true; json data_json = data; send_streaming_response(data_json, is_final); }; @@ -1183,6 +1287,12 @@ void RestHandler::handle_openai_chat_completion(const json& request, this->prompt_cache.reset(); return; } + } catch (const ModelRequestError& error) { + send_response(ModelErrorResponse(error)); + if (error.session_cleared()) { + this->prompt_cache.reset(); + } + return; } catch (const std::exception& e) { json error_response = {{"error", e.what()}}; send_response(error_response); @@ -1193,6 +1303,19 @@ void RestHandler::handle_openai_chat_completion(const json& request, header_print("FLM", "Start generating..."); try { auto_chat_engine->generate(meta_info, length_limit, ostream, [&] { return cancellation_token->cancelled(); }); + } catch (const ModelRequestError& error) { + const json error_response = ModelErrorResponse(error); + if (UseFinalStreamingErrorChunk(stream_started)) { + SendOpenAiStreamingError( + error_response, + send_streaming_response); + } else { + send_response(error_response); + } + if (error.session_cleared()) { + this->prompt_cache.reset(); + } + return; } catch (const std::exception& e) { json error_response = {{"error", e.what()}}; send_response(error_response); @@ -1235,6 +1358,12 @@ void RestHandler::handle_openai_chat_completion(const json& request, this->prompt_cache.reset(); return; } + } catch (const ModelRequestError& error) { + send_response(ModelErrorResponse(error)); + if (error.session_cleared()) { + this->prompt_cache.reset(); + } + return; } catch (const std::exception& e) { json error_response = {{"error", e.what()}}; send_response(error_response); @@ -1245,6 +1374,12 @@ void RestHandler::handle_openai_chat_completion(const json& request, header_print("FLM", "Start generating..."); try { response_text = auto_chat_engine->generate(meta_info, length_limit, nstream, [&] { return cancellation_token->cancelled(); }); + } catch (const ModelRequestError& error) { + send_response(ModelErrorResponse(error)); + if (error.session_cleared()) { + this->prompt_cache.reset(); + } + return; } catch (const std::exception& e) { json error_response = {{"error", e.what()}}; send_response(error_response); @@ -1280,6 +1415,11 @@ void RestHandler::handle_openai_chat_completion(const json& request, send_response(response); } + } catch (const ModelRequestError& error) { + send_response(ModelErrorResponse(error)); + if (error.session_cleared()) { + this->prompt_cache.reset(); + } } catch (const std::exception& e) { json error_response = { {"error", { @@ -1365,6 +1505,10 @@ void RestHandler::handle_openai_completion(const json& request, std::shared_ptr cancellation_token) { try { // Extract OpenAI-style parameters + const ParsedGenerationLimit parsed_limit = + ParseGenerationLimit( + request, + RequireGenerationEndpoint("POST", "/v1/completions")); std::string prompt = request["prompt"]; std::string model = request.value("model", current_model_tag); std::string reasoning_effort = request.value("reasoning_effort", "medium"); @@ -1375,25 +1519,35 @@ void RestHandler::handle_openai_completion(const json& request, if (!supported_models.is_model_supported(model)) { throw std::runtime_error("Model " + model + " is not supported."); } - - int length_limit = request.value("max_tokens", 4096); if (!ensure_model_loaded(model)) { json error_response = {{"error", "Failed to load " + model + " model!"}}; send_response(error_response); return; } + const int length_limit = + GenerationLoopLimit( + parsed_limit, + auto_chat_engine->uses_corelib_aie4()); configure_chat_engine_parameters(options, request); chat_meta_info_t meta_info; meta_info.max_prefill_len = this->prefill_chunk_len; lm_uniform_input_t uniformed_input; + uniformed_input.requested_max_new_tokens = + RequestedMaxNewTokens(parsed_limit); header_print("FLM", "Start generating..."); if (stream) { // Create a wrapper callback that passes the pre-formatted SSE string directly - auto openai_stream_callback = [&send_streaming_response](const std::string& data, bool is_final) { + bool stream_started = false; + auto openai_stream_callback = [ + &send_streaming_response, + &stream_started]( + const std::string& data, + bool is_final) { + stream_started = true; json data_json = data; send_streaming_response(data_json, is_final); }; @@ -1407,6 +1561,9 @@ void RestHandler::handle_openai_completion(const json& request, this->auto_chat_engine->clear_context(); return; } + } catch (const ModelRequestError& error) { + send_response(ModelErrorResponse(error)); + return; } catch (const std::exception& e) { json error_response = {{"error", e.what()}}; send_response(error_response); @@ -1415,6 +1572,16 @@ void RestHandler::handle_openai_completion(const json& request, } try { auto_chat_engine->generate(meta_info, length_limit, ostream); + } catch (const ModelRequestError& error) { + const json error_response = ModelErrorResponse(error); + if (UseFinalStreamingErrorChunk(stream_started)) { + SendOpenAiStreamingError( + error_response, + send_streaming_response); + } else { + send_response(error_response); + } + return; } catch (const std::exception& e) { json error_response = {{"error", e.what()}}; send_response(error_response); @@ -1438,6 +1605,9 @@ void RestHandler::handle_openai_completion(const json& request, this->auto_chat_engine->clear_context(); return; } + } catch (const ModelRequestError& error) { + send_response(ModelErrorResponse(error)); + return; } catch (const std::exception& e) { json error_response = {{"error", e.what()}}; send_response(error_response); @@ -1446,6 +1616,9 @@ void RestHandler::handle_openai_completion(const json& request, } try { auto_chat_engine->generate(meta_info, length_limit, ostream); + } catch (const ModelRequestError& error) { + send_response(ModelErrorResponse(error)); + return; } catch (const std::exception& e) { json error_response = {{"error", e.what()}}; send_response(error_response); @@ -1477,6 +1650,9 @@ void RestHandler::handle_openai_completion(const json& request, send_response(response); } } + catch (const ModelRequestError& error) { + send_response(ModelErrorResponse(error)); + } catch (const std::exception& e) { json error_response = { {"error", { diff --git a/src/server/server.cpp b/src/server/server.cpp index bc612211..cfcaadb7 100644 --- a/src/server/server.cpp +++ b/src/server/server.cpp @@ -8,19 +8,13 @@ */ #include "server.hpp" #include "rest_handler.hpp" +#include #include #include #include #include #include - -// Global NPU access control -std::mutex g_npu_access_mutex; -std::atomic g_npu_in_use{false}; - -std::atomic g_npu_active_requests{0}; - ///@brief get current time string, format: hh:mm:ss mm:dd:yyyy ///@return the current time string std::string get_current_time_string() { @@ -129,46 +123,6 @@ void brief_print_message_response(nlohmann::json request) { } -// NPU Access Manager implementation -bool NPUAccessManager::try_acquire_npu_access() { - std::lock_guard lock(g_npu_access_mutex); - if (g_npu_in_use.load()) { - return false; // NPU is already in use - } - g_npu_in_use.store(true); - g_npu_active_requests.fetch_add(1); - return true; -} - -void NPUAccessManager::release_npu_access() { - - header_print("🔵 ", "NPU Lock Released!" ); - std::lock_guard lock(g_npu_access_mutex); - g_npu_in_use.store(false); - g_npu_active_requests.fetch_sub(1); -} - -bool NPUAccessManager::is_npu_available() { - return !g_npu_in_use.load(); -} - -int NPUAccessManager::get_active_npu_requests() { - return g_npu_active_requests.load(); -} - -// Helper function to check if an endpoint requires NPU access -bool requires_npu_access(const std::string& method, const std::string& path) { - // NPU-intensive endpoints that should be restricted to one user at a time - if (method == "POST") { - return path == "/api/generate" || - path == "/api/chat" || - path == "/v1/chat/completions" || - path == "/v1/audio/transcriptions" || - path == "/v1/embeddings"; - } - return false; -} - ///@brief HttpSession class implementation ///@param socket the socket ///@param server the server @@ -684,6 +638,11 @@ bool WebServer::handle_request(http::request& req, auto process_task = [this, it, req_ptr, res_ptr, session, needs_npu, key, is_json](bool is_deferred) { auto& req_ref = *req_ptr; auto& res_ref = *res_ptr; + NPURequestCompletionGuard npu_completion([this, needs_npu] { + if (needs_npu) { + this->process_next_npu_request(); + } + }); // Parse JSON request body json request_json; @@ -705,9 +664,6 @@ bool WebServer::handle_request(http::request& req, // Only write from callback when deferred if (is_deferred && session) session->write_response_from_callback(); - if (needs_npu) { - this->process_next_npu_request(); - } return; } @@ -726,22 +682,16 @@ bool WebServer::handle_request(http::request& req, register_active_request(request_id, cancellation_token); // catch is_deferred - auto send_response = [res_ptr, session, this, request_id, needs_npu, is_deferred, cancellation_token](const json& response_data) { + auto send_response = [res_ptr, session, this, request_id, is_deferred, cancellation_token](const json& response_data) { auto& response_ref = *res_ptr; - http::status status = http::status::ok; - - if (response_data.contains("error") && - response_data["error"].contains("code")) - { - int code = response_data["error"]["code"].get(); - - if (code == 400) { - status = http::status::bad_request; - } - //else if () { - - //} - } + const int status_code = + HttpStatusForResponse(response_data); + const http::status status = + status_code == 400 + ? http::status::bad_request + : status_code == 500 + ? http::status::internal_server_error + : http::status::ok; response_ref.result(status); response_ref.body() = response_data.dump(); @@ -750,16 +700,12 @@ bool WebServer::handle_request(http::request& req, cancellation_token->complete(); unregister_active_request(request_id); - if (needs_npu) { - this->process_next_npu_request(); - } - if (is_deferred && session) { session->write_response_from_callback(); } }; - auto send_streaming_response = [session, this, request_id, needs_npu, cancellation_token](const json& data, bool is_final) { + auto send_streaming_response = [session, this, request_id, cancellation_token](const json& data, bool is_final) { if (is_final) { cancellation_token->complete(); } @@ -769,10 +715,6 @@ bool WebServer::handle_request(http::request& req, } if (is_final) { unregister_active_request(request_id); - - if (needs_npu) { - this->process_next_npu_request(); - } } }; @@ -788,10 +730,6 @@ bool WebServer::handle_request(http::request& req, res_ref.set(http::field::content_type, "application/json"); res_ref.prepare_payload(); - if (needs_npu) { - this->process_next_npu_request(); - } - if (is_deferred && session) { session->write_response_from_callback(); } @@ -805,10 +743,6 @@ bool WebServer::handle_request(http::request& req, res_ref.set(http::field::content_type, "application/json"); res_ref.prepare_payload(); - if (needs_npu) { - this->process_next_npu_request(); - } - if (is_deferred && session) { session->write_response_from_callback(); } diff --git a/src/server/server.hpp b/src/server/server.hpp index 910c5884..95cdf053 100644 --- a/src/server/server.hpp +++ b/src/server/server.hpp @@ -29,6 +29,7 @@ #include "streaming_ostream.hpp" #include "model_downloader.hpp" #include "multipart.hpp" +#include #include #include @@ -42,27 +43,10 @@ using json = nlohmann::ordered_json; class RestHandler; class HttpSession; -// Global NPU access control -extern std::mutex g_npu_access_mutex; -extern std::atomic g_npu_in_use; -extern std::atomic g_npu_active_requests; - -// Helper function to check if an endpoint requires NPU access -bool requires_npu_access(const std::string& method, const std::string& path); - ///@brief get current time string, format: hh:mm:ss mm:dd:yyyy ///@return the current time string std::string get_current_time_string(); -// NPU access manager class -class NPUAccessManager { -public: - static bool try_acquire_npu_access(); - static void release_npu_access(); - static bool is_npu_available(); - static int get_active_npu_requests(); -}; - // Stream response callback type for handling streaming responses using StreamCallback = std::function; diff --git a/src/src/main.cpp b/src/src/main.cpp index 446b907b..d9e91461 100644 --- a/src/src/main.cpp +++ b/src/src/main.cpp @@ -7,6 +7,7 @@ #pragma once #include "runner.hpp" #include "server.hpp" +#include #include "model_list.hpp" #include "model_downloader.hpp" #include "update.hpp" @@ -14,6 +15,7 @@ #include "program_args.hpp" #include "minja/chat-template.hpp" #include +#include #include #include #include @@ -22,6 +24,7 @@ #include #include #include +#include #include #include #ifdef _WIN32 @@ -49,6 +52,11 @@ #include "AutoModel/automodel.hpp" +#if defined(FLM_ENABLE_CORELIB_AIE4) +#include +#include +#endif + #ifndef _WIN32 #include #endif @@ -207,7 +215,10 @@ std::string identify_npu_arch() { #endif -static bool sanity_check_npu_stack(bool quiet, bool json_output = false) { +static bool sanity_check_npu_stack( + bool quiet, + bool json_output = false, + nlohmann::json* report_output = nullptr) { bool print_human = !quiet && !json_output; #ifndef _WIN32 nlohmann::json validation_json = { @@ -221,6 +232,17 @@ static bool sanity_check_npu_stack(bool quiet, bool json_output = false) { {"devices", nlohmann::json::array()}, {"ready", true} }; + const auto finish_validation = + [&](bool ready) { + validation_json["ready"] = ready; + if (report_output != nullptr) { + *report_output = validation_json; + } + if (json_output) { + std::cout << validation_json.dump(4) << std::endl; + } + return ready; + }; validation_json["platform"] = "linux"; // Check kernel version struct utsname u_name; @@ -228,11 +250,7 @@ static bool sanity_check_npu_stack(bool quiet, bool json_output = false) { if (print_human) perror("Failed to get kernel version"); validation_json["kernel_ok"] = false; - validation_json["ready"] = false; - if (json_output) { - std::cout << validation_json.dump(4) << std::endl; - } - return false; + return finish_validation(false); } int major, minor; sscanf(u_name.release, "%d.%d", &major, &minor); @@ -243,11 +261,7 @@ static bool sanity_check_npu_stack(bool quiet, bool json_output = false) { if (print_human) { header_print_r("ERROR", "Kernel version incompatible with this version of FLM. Please update your kernel!"); } - validation_json["ready"] = false; - if (json_output) { - std::cout << validation_json.dump(4) << std::endl; - } - return false; + return finish_validation(false); } if (print_human) { header_print("Linux", "Kernel: " << u_name.release); @@ -398,12 +412,7 @@ static bool sanity_check_npu_stack(bool quiet, bool json_output = false) { validation_json["memlock_ok"] = memlock_ok; bool overall_ok = amd_device_found && kernel_ok && all_fw_ok && enough_cols && memlock_ok; - validation_json["ready"] = overall_ok; - if (json_output) { - std::cout << validation_json.dump(4) << std::endl; - } - - return overall_ok; + return finish_validation(overall_ok); #else nlohmann::json validation_json = { {"object", "npu_stack_validation"}, @@ -412,16 +421,23 @@ static bool sanity_check_npu_stack(bool quiet, bool json_output = false) { {"npu_driver_ok", true}, {"ready", true} }; + const auto finish_validation = + [&](bool ready) { + validation_json["ready"] = ready; + if (report_output != nullptr) { + *report_output = validation_json; + } + if (json_output) { + std::cout << validation_json.dump(4) << std::endl; + } + return ready; + }; std::string npu_arch = identify_npu_arch(); if (npu_arch.empty()) { if (print_human) header_print("Error", "No XDNA2 NPU hardware detected"); - validation_json["ready"] = false; validation_json["amd_device_found"] = false; - if (json_output) { - std::cout << validation_json.dump(4) << std::endl; - } - return false; + return finish_validation(false); } std::string min_drv = __NPU_VERSION__; @@ -435,12 +451,8 @@ static bool sanity_check_npu_stack(bool quiet, bool json_output = false) { if (print_human) { header_print("Error", "NPU driver version doesn't meet the minimum!"); } - validation_json["ready"] = false; validation_json["npu_driver_ok"] = false; - if (json_output) { - std::cout << validation_json.dump(4) << std::endl; - } - return false; + return finish_validation(false); } if (print_human) { @@ -448,13 +460,97 @@ static bool sanity_check_npu_stack(bool quiet, bool json_output = false) { header_print_g("Windows", "NPU dirver version: " << drv); } - if (json_output) { - std::cout << validation_json.dump(4) << std::endl; - } - return true; + return finish_validation(true); #endif } +#if defined(FLM_ENABLE_CORELIB_AIE4) +static void drain_prior_corelib_fatal_records() noexcept { + try { + (void)flm::corelib::FatalRecordStore::DrainPriorRecords( + std::cerr); + } catch (const std::exception& error) { + std::cerr + << "AIE4 fatal record warning: startup drain failed: " + << error.what() << '\n'; + } catch (...) { + std::cerr + << "AIE4 fatal record warning: startup drain failed with an " + "unknown error\n"; + } +} + +static bool shutdown_corelib_process() noexcept { + try { + flm::corelib::CorelibRuntime::ShutdownProcess(); + return true; + } catch (const std::exception& error) { + std::cerr << "Error: failed to shut down AIE4 corelib runtime: " + << error.what() << '\n'; + } catch (...) { + std::cerr + << "Error: failed to shut down AIE4 corelib runtime with an " + "unknown error\n"; + } + return false; +} + +static nlohmann::json validate_corelib_aie4( + const std::filesystem::path& executable_dir, + bool print_human) { + nlohmann::json report = { + {"available_in_build", true}, + {"loader_ok", false}, + {"dependencies_ok", false}, + {"device_context_ok", false}, + {"fatal_log_writable", false}, + {"shutdown_ok", false}, + {"ready", false}, + }; + + std::shared_ptr runtime; + try { + runtime = + flm::corelib::CorelibRuntime::GetOrCreate( + executable_dir); + report["loader_ok"] = true; + report["dependencies_ok"] = true; + report["device_context_ok"] = true; + report["fatal_log_writable"] = true; + runtime.reset(); + const bool shutdown_ok = shutdown_corelib_process(); + report["shutdown_ok"] = shutdown_ok; + report["ready"] = shutdown_ok; + } catch (const std::exception& error) { + runtime.reset(); + (void)shutdown_corelib_process(); + report["error"] = error.what(); + } catch (...) { + runtime.reset(); + (void)shutdown_corelib_process(); + report["error"] = "unknown AIE4 validation error"; + } + + if (print_human) { + if (report["ready"].get()) { + header_print_g( + "Windows", + "Corelib AIE4: ready"); + } else { + header_print_r( + "ERROR", + "Corelib AIE4: not ready" + << (report.contains("error") + ? " (" + + report["error"].get() + + ")" + : "")); + } + } + return report; +} +#endif + ///@brief main function ///@param argc the number of arguments @@ -468,13 +564,16 @@ int main(int argc, char* argv[]) { // XRT backend: preload bundled XRT libraries from the executable directory. preload_bundled_libraries(); #endif - // Parse command line arguments using Boost Program Options program_args_t parsed_args; if (!arg_utils::parse_options(argc, argv, parsed_args)) { return 1; // Help was already printed by Boost Program Options } +#if defined(FLM_ENABLE_CORELIB_AIE4) + drain_prior_corelib_fatal_records(); +#endif + // Get the command, model tag, and force flag std::string exe_dir = utils::get_executable_directory(); @@ -505,7 +604,75 @@ int main(int argc, char* argv[]) { // Check if the commands and args valid if (parsed_args.command == "validate") { - stable_stack = sanity_check_npu_stack(parsed_args.command != "validate", parsed_args.command == "validate" && parsed_args.json_output); + nlohmann::json legacy_report; + bool legacy_ready = false; + try { + legacy_ready = + sanity_check_npu_stack( + parsed_args.json_output, + false, + &legacy_report); + } catch (const std::exception& error) { + legacy_report = { + {"object", "npu_stack_validation"}, + {"ready", false}, + {"error", error.what()}, + }; +#ifdef _WIN32 + legacy_report["platform"] = "windows"; +#else + legacy_report["platform"] = "linux"; +#endif + if (!parsed_args.json_output) { + header_print_r( + "ERROR", + "Legacy XDNA2 validation failed: " + << error.what()); + } + } catch (...) { + legacy_report = { + {"object", "npu_stack_validation"}, + {"ready", false}, + {"error", "unknown legacy XDNA2 validation error"}, + }; +#ifdef _WIN32 + legacy_report["platform"] = "windows"; +#else + legacy_report["platform"] = "linux"; +#endif + if (!parsed_args.json_output) { + header_print_r( + "ERROR", + "Legacy XDNA2 validation failed with an " + "unknown error"); + } + } + nlohmann::json aie4_report = { + {"available_in_build", false}, + {"ready", false}, + }; + bool aie4_ready = false; +#if defined(FLM_ENABLE_CORELIB_AIE4) + aie4_report = + validate_corelib_aie4( + exe_dir, + !parsed_args.json_output); + aie4_ready = aie4_report["ready"].get(); +#else + if (!parsed_args.json_output) { + header_print( + "FLM", + "Corelib AIE4: unavailable in this build"); + } +#endif + stable_stack = legacy_ready || aie4_ready; + if (parsed_args.json_output) { + nlohmann::json validation_report = legacy_report; + validation_report["legacy_xdna2"] = legacy_report; + validation_report["corelib_aie4"] = aie4_report; + validation_report["ready"] = stable_stack; + std::cout << validation_report.dump(4) << std::endl; + } return stable_stack ? 0 : 1; } @@ -602,6 +769,9 @@ int main(int argc, char* argv[]) { try { +#if defined(FLM_ENABLE_CORELIB_AIE4) + bool corelib_shutdown_complete = false; +#endif // Load the model list with the models directory as the base ModelDownloader downloader(availble_models); @@ -620,6 +790,8 @@ int main(int argc, char* argv[]) { runner.run(); } else if (parsed_args.command == "serve") { + running.store(true); + ScopedSignalHandler sigint_scope(SIGINT, signal_handler); check_and_notify_new_version(); // Create the server int port = utils::get_server_port(parsed_args.port); @@ -642,8 +814,28 @@ int main(int argc, char* argv[]) { std::unique_lock lock(mtx); cv.wait(lock, [] { return !running.load(); }); } - // header_print("FLM", "Stopping server..."); - // server->stop(); + const bool healthy_shutdown = CompleteServeShutdown( + [&] { + // Stop admission and join all synchronous request handlers. + server->stop(); + }, + [&] { + // Destroy routes, RestHandler, and the loaded AutoModel. + server.reset(); + }, + [&] { +#if defined(FLM_ENABLE_CORELIB_AIE4) + return shutdown_corelib_process(); +#else + return true; +#endif + }); + if (!healthy_shutdown) { + return 1; + } +#if defined(FLM_ENABLE_CORELIB_AIE4) + corelib_shutdown_complete = true; +#endif } else if (parsed_args.command == "pull") { bool success = downloader.pull_model(parsed_args.model_tag, parsed_args.modelscope, parsed_args.force_redownload); @@ -715,11 +907,26 @@ int main(int argc, char* argv[]) { std::cerr << "Use --help for usage information" << std::endl; return 1; } +#if defined(FLM_ENABLE_CORELIB_AIE4) + if (!corelib_shutdown_complete && + !shutdown_corelib_process()) { + return 1; + } +#endif // Return 0 if the command is valid return 0; } catch (const std::exception& e) { // If an error occurs, this will be used to show the error std::cerr << "Error: " << e.what() << std::endl; +#if defined(FLM_ENABLE_CORELIB_AIE4) + (void)shutdown_corelib_process(); +#endif + return 1; + } catch (...) { + std::cerr << "Error: unknown command failure" << std::endl; +#if defined(FLM_ENABLE_CORELIB_AIE4) + (void)shutdown_corelib_process(); +#endif return 1; } } diff --git a/src/test/phi4_corelib_aie4/CMakeLists.txt b/src/test/phi4_corelib_aie4/CMakeLists.txt new file mode 100644 index 00000000..3b79a1c4 --- /dev/null +++ b/src/test/phi4_corelib_aie4/CMakeLists.txt @@ -0,0 +1,642 @@ +cmake_minimum_required(VERSION 3.22) +project(phi4_corelib_aie4_tests LANGUAGES CXX) + +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +# ryzenai-corelib is OPTIONAL here, and that is not a relaxation. +# +# This suite owns the guards that answer "does FastFlow still build with the +# AIE4 feature OFF" -- phi4_frontend_compile_off and task10_production_compile_off +# compile the real production TUs (main.cpp, rest_handler.cpp, runner.cpp, +# modeling_phi4.cpp) with FLM_ENABLE_CORELIB_AIE4 undefined. With REQUIRED here +# those targets could only be CONFIGURED on a machine that has corelib, i.e. the +# "builds without corelib" proof could never be run anywhere corelib was absent, +# which is the only place it means anything. +# +# So the search is allowed to fail, and the guards-only path below configures +# just those two targets and returns. Every target that genuinely needs corelib +# is either past that return or hard-fails by name inside the functions that add +# it -- the ON path is unchanged and still cannot be configured without corelib. +find_path(RYZENAI_CORELIB_INCLUDE_DIR + NAMES ryzenai/corelib.h + DOC "Directory containing ryzenai/corelib.h") +find_path(BOOST_INCLUDE_DIR + NAMES boost/any.hpp + HINTS + "$ENV{BOOST_INCLUDEDIR}" + "$ENV{BOOST_ROOT}" + "$ENV{CONDA_PREFIX}/Library/include" + "$ENV{USERPROFILE}/anaconda3/Library/include" + "C:/dev/boost_1_88_0" + REQUIRED) + +get_filename_component( + FASTFLOW_SOURCE_DIR + "${CMAKE_CURRENT_LIST_DIR}/../.." + ABSOLUTE) +set(XRT_INCLUDE_DIR + "C:/dev/XRT/src/runtime_src/core/include" + CACHE PATH "Where XRT headers live") +set(XRT_LIB_DIR + "C:/dev/xrtNPUfromDLL" + CACHE PATH "Where XRT libraries live") + +# FLM_VERSION and NPU_VERSION are compiled into several tests, and this file +# never declared either of them. +# +# That is not a harmless omission. test_model_catalog asserts +# `__FLM_VERSION__ >= the version packaged in the AIE4 overlay`, so an empty +# string makes its sscanf return EOF and the test fail. It went unnoticed +# because the one machine the suite had ever been configured on carried +# `FLM_VERSION:UNINITIALIZED=1.0.4` in its cache from a hand-typed -D, so the +# suite passed there and failed on the first fresh configure anywhere else -- +# which is exactly what happened on the AIE4 target. +# +# The values are read from the product's own CMakePresets.json rather than +# retyped here, so the standalone suite cannot drift away from the version the +# product actually ships. +if(NOT DEFINED FLM_VERSION OR NOT DEFINED NPU_VERSION) + set(_flm_presets "${FASTFLOW_SOURCE_DIR}/CMakePresets.json") + if(NOT EXISTS "${_flm_presets}") + message(FATAL_ERROR + "Cannot default FLM_VERSION/NPU_VERSION: ${_flm_presets} is " + "missing. Pass -DFLM_VERSION= and -DNPU_VERSION= explicitly.") + endif() + file(READ "${_flm_presets}" _flm_presets_text) + string(JSON _flm_common GET "${_flm_presets_text}" + "configurePresets" 0 "cacheVariables") + if(NOT DEFINED FLM_VERSION) + string(JSON FLM_VERSION GET "${_flm_common}" "FLM_VERSION") + endif() + if(NOT DEFINED NPU_VERSION) + string(JSON NPU_VERSION GET "${_flm_common}" "NPU_VERSION") + endif() + unset(_flm_presets_text) + unset(_flm_common) +endif() +# Validated rather than trusted: an empty or malformed version reaches the +# tests as a compile definition and fails them at run time with a message that +# says nothing about where the value came from. +if(NOT FLM_VERSION MATCHES "^[0-9]+\\.[0-9]+\\.[0-9]+$") + message(FATAL_ERROR + "FLM_VERSION must be MAJOR.MINOR.PATCH, got '${FLM_VERSION}'") +endif() +if(NOT NPU_VERSION MATCHES "^[0-9]+(\\.[0-9]+)+$") + message(FATAL_ERROR + "NPU_VERSION must be dotted numeric, got '${NPU_VERSION}'") +endif() +message(STATUS + "Phi-4 AIE4 test suite: FLM_VERSION=${FLM_VERSION} " + "NPU_VERSION=${NPU_VERSION}") + +# --------------------------------------------------------------------------- +# The feature-OFF compile guards, and the guards-only early return +# --------------------------------------------------------------------------- +# +# These two functions and their feature-OFF invocations live HERE, ahead of +# everything that needs ryzenai-corelib, for one reason: a machine with no +# corelib must still be able to configure and build the proof that FastFlow +# compiles without it. Their feature-ON twins are invoked further down, past the +# return below, where corelib is known to exist. +# +# ENABLE_AIE4 is what makes a target need corelib, so that is where the hard +# failure lives -- naming the target rather than a header. + +set(PHI4_FRONTEND_SOURCES + ${FASTFLOW_SOURCE_DIR}/common/AutoModel/automodel.cpp + ${FASTFLOW_SOURCE_DIR}/common/AutoModel/modeling_phi4.cpp + ${FASTFLOW_SOURCE_DIR}/server/generation_limit.cpp) + +function(add_phi4_frontend_compile_check TARGET_NAME ENABLE_AIE4) + add_library(${TARGET_NAME} OBJECT + ${PHI4_FRONTEND_SOURCES}) + target_include_directories(${TARGET_NAME} PRIVATE + ${FASTFLOW_SOURCE_DIR}/include + ${FASTFLOW_SOURCE_DIR}/../third_party/tokenizers-cpp/include + ${BOOST_INCLUDE_DIR} + ${XRT_INCLUDE_DIR}) + target_compile_definitions(${TARGET_NAME} PRIVATE + DEV_BUILD=1 + __WINDOWS__ + USEAVX2=1 + DISABLE_ABI_CHECK=1 + _ENABLE_EXTENDED_ALIGNED_STORAGE + CMAKE_INSTALL_PREFIX="${FASTFLOW_SOURCE_DIR}/build/phi4_corelib_aie4-tests" + CMAKE_XCLBIN_PREFIX="${FASTFLOW_SOURCE_DIR}/xclbins" + WIN32_LEAN_AND_MEAN + NOMINMAX) + target_compile_options(${TARGET_NAME} PRIVATE + $<$:/arch:AVX2 /fp:precise>) + if(ENABLE_AIE4) + if(NOT RYZENAI_CORELIB_INCLUDE_DIR) + message(FATAL_ERROR + "${TARGET_NAME} compiles with FLM_ENABLE_CORELIB_AIE4=1 and " + "therefore needs ryzenai/corelib.h, which was not found. Pass " + "-DRYZENAI_CORELIB_INCLUDE_DIR= or add the corelib " + "install prefix to CMAKE_PREFIX_PATH. Only the feature-OFF " + "compile guards can be configured without it.") + endif() + target_compile_definitions(${TARGET_NAME} PRIVATE + FLM_ENABLE_CORELIB_AIE4=1) + target_include_directories(${TARGET_NAME} PRIVATE + ${RYZENAI_CORELIB_INCLUDE_DIR}) + endif() +endfunction() + +add_phi4_frontend_compile_check( + phi4_frontend_compile_off + FALSE) + +set(TASK10_PRODUCTION_SOURCES + ${FASTFLOW_SOURCE_DIR}/server/generation_limit.cpp + ${FASTFLOW_SOURCE_DIR}/server/npu_access_manager.cpp + ${FASTFLOW_SOURCE_DIR}/server/rest_handler.cpp + ${FASTFLOW_SOURCE_DIR}/server/server.cpp + ${FASTFLOW_SOURCE_DIR}/runner/runner.cpp + ${FASTFLOW_SOURCE_DIR}/src/main.cpp) + +set(TASK10_STUB_INCLUDE_DIR + "${CMAKE_CURRENT_BINARY_DIR}/task10-compile-stubs") +# These production translation units include declaration-only FFmpeg headers +# transitively. The standalone host suite does not link or execute FFmpeg. +foreach(TASK10_STUB_HEADER IN ITEMS + libavcodec/avcodec.h + libswscale/swscale.h + libavutil/imgutils.h + libavutil/frame.h + libavutil/pixfmt.h) + get_filename_component( + TASK10_STUB_PARENT + "${TASK10_STUB_INCLUDE_DIR}/${TASK10_STUB_HEADER}" + DIRECTORY) + file(MAKE_DIRECTORY "${TASK10_STUB_PARENT}") + file(WRITE + "${TASK10_STUB_INCLUDE_DIR}/${TASK10_STUB_HEADER}" + "#pragma once\n") +endforeach() + +function(add_task10_production_compile_check TARGET_NAME ENABLE_AIE4) + add_library(${TARGET_NAME} OBJECT + ${TASK10_PRODUCTION_SOURCES}) + target_include_directories(${TARGET_NAME} PRIVATE + ${FASTFLOW_SOURCE_DIR}/include + ${FASTFLOW_SOURCE_DIR}/runner + ${FASTFLOW_SOURCE_DIR}/server + ${FASTFLOW_SOURCE_DIR}/pull + ${FASTFLOW_SOURCE_DIR}/../third_party/tokenizers-cpp/include + ${TASK10_STUB_INCLUDE_DIR} + ${BOOST_INCLUDE_DIR} + ${XRT_INCLUDE_DIR}) + target_compile_definitions(${TARGET_NAME} PRIVATE + DEV_BUILD=1 + __WINDOWS__ + USEAVX2=1 + DISABLE_ABI_CHECK=1 + _ENABLE_EXTENDED_ALIGNED_STORAGE + CMAKE_INSTALL_PREFIX="${FASTFLOW_SOURCE_DIR}/build/phi4_corelib_aie4-tests" + CMAKE_XCLBIN_PREFIX="${FASTFLOW_SOURCE_DIR}/xclbins" + __FLM_VERSION__="task10-test" + __NPU_VERSION__="0.0.0.0" + WIN32_LEAN_AND_MEAN + NOMINMAX) + target_compile_options(${TARGET_NAME} PRIVATE + $<$:/arch:AVX2 /fp:precise>) + if(ENABLE_AIE4) + if(NOT RYZENAI_CORELIB_INCLUDE_DIR) + message(FATAL_ERROR + "${TARGET_NAME} compiles with FLM_ENABLE_CORELIB_AIE4=1 and " + "therefore needs ryzenai/corelib.h, which was not found. Pass " + "-DRYZENAI_CORELIB_INCLUDE_DIR= or add the corelib " + "install prefix to CMAKE_PREFIX_PATH. Only the feature-OFF " + "compile guards can be configured without it.") + endif() + target_compile_definitions(${TARGET_NAME} PRIVATE + FLM_ENABLE_CORELIB_AIE4=1) + target_include_directories(${TARGET_NAME} PRIVATE + ${RYZENAI_CORELIB_INCLUDE_DIR}) + endif() +endfunction() + +add_task10_production_compile_check( + task10_production_compile_off + FALSE) + +# `enable_testing()` and the PowerShell probe are done here, ABOVE the +# corelib-less early return below, so that test_packaged_runtime can still be +# registered on a machine configured without an AIE4 device. Moving either of +# them down past that return would silently drop the one PowerShell test a +# device-less machine can run. +enable_testing() +find_program(FLM_POWERSHELL_EXECUTABLE + NAMES pwsh powershell + DOC "PowerShell used to run the packaging tests") + +if(NOT RYZENAI_CORELIB_INCLUDE_DIR) + # Guards-only configure. Everything below this point either includes + # ryzenai/corelib.h or links something that does, so stopping here is the + # difference between "the without-corelib guard cannot be configured" and + # "the without-corelib guard is the only thing configured". Say which + # targets exist, so a green configure is not mistaken for the full suite. + message(STATUS + "ryzenai/corelib.h not found: configuring the feature-OFF compile " + "guards only (phi4_frontend_compile_off, task10_production_compile_off). " + "The corelib tests, the hardware suite and the feature-ON guards are " + "NOT configured. Pass -DRYZENAI_CORELIB_INCLUDE_DIR= for the full " + "suite.") + return() +endif() + +include("${FASTFLOW_SOURCE_DIR}/common/corelib/corelib_sources.cmake") +add_library(flm_corelib_aie4_testlib STATIC + ${FLM_CORELIB_AIE4_SOURCES}) +target_include_directories(flm_corelib_aie4_testlib PUBLIC + ${FASTFLOW_SOURCE_DIR}/include + ${RYZENAI_CORELIB_INCLUDE_DIR} + ${BOOST_INCLUDE_DIR} + ${XRT_INCLUDE_DIR}) +target_link_directories(flm_corelib_aie4_testlib PUBLIC + ${XRT_LIB_DIR}) +target_compile_definitions(flm_corelib_aie4_testlib PUBLIC + FLM_ENABLE_CORELIB_AIE4=1 + FLM_CORELIB_TESTING=1 + DEV_BUILD=1 + __WINDOWS__ + USEAVX2=1 + DISABLE_ABI_CHECK=1 + _ENABLE_EXTENDED_ALIGNED_STORAGE + CMAKE_INSTALL_PREFIX=\"${FASTFLOW_SOURCE_DIR}/build/phi4_corelib_aie4-tests\" + CMAKE_XCLBIN_PREFIX=\"${FASTFLOW_SOURCE_DIR}/xclbins\" + __FLM_VERSION__=\"${FLM_VERSION}\" + __NPU_VERSION__=\"${NPU_VERSION}\" + WIN32_LEAN_AND_MEAN + NOMINMAX) +if(MSVC) + target_compile_options(flm_corelib_aie4_testlib PRIVATE + /arch:AVX2 + /fp:precise + $<$:/O2>) +endif() +target_link_libraries(flm_corelib_aie4_testlib PUBLIC + xrt_coreutil + shell32 + ole32) + +add_library(fake_ryzenai_corelib SHARED + fake_corelib.cpp) +target_include_directories(fake_ryzenai_corelib PRIVATE + ${RYZENAI_CORELIB_INCLUDE_DIR}) +target_compile_definitions(fake_ryzenai_corelib PRIVATE + RYZENAI_CORELIB_EXPORT=1 + FLM_FAKE_CORELIB_DLL=1) + +function(add_corelib_host_test TEST_NAME TEST_SOURCE) + add_executable(${TEST_NAME} + ${TEST_SOURCE} + fake_corelib.cpp) + target_link_libraries(${TEST_NAME} PRIVATE + flm_corelib_aie4_testlib) + add_test(NAME ${TEST_NAME} COMMAND ${TEST_NAME}) + set_tests_properties(${TEST_NAME} PROPERTIES + ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:${XRT_LIB_DIR}") +endfunction() + +enable_testing() +add_executable(test_model_catalog + test_model_catalog.cpp + ${FASTFLOW_SOURCE_DIR}/pull/model_overlay.cpp) +target_include_directories(test_model_catalog PRIVATE + ${FASTFLOW_SOURCE_DIR}/include + ${FASTFLOW_SOURCE_DIR}/pull) +target_compile_definitions(test_model_catalog PRIVATE + FLM_TEST_SOURCE_DIR="${FASTFLOW_SOURCE_DIR}" + __FLM_VERSION__="${FLM_VERSION}") +add_test(NAME test_model_catalog COMMAND test_model_catalog) + +add_library(test_model_downloader_compile OBJECT + ${FASTFLOW_SOURCE_DIR}/pull/model_downloader.cpp) +target_include_directories(test_model_downloader_compile PRIVATE + ${FASTFLOW_SOURCE_DIR}/include + ${FASTFLOW_SOURCE_DIR}/pull + ${FASTFLOW_SOURCE_DIR}/../third_party/tokenizers-cpp/include + ${BOOST_INCLUDE_DIR} + ${XRT_INCLUDE_DIR}) +target_compile_definitions(test_model_downloader_compile PRIVATE + __FLM_VERSION__="${FLM_VERSION}") + +add_executable(test_generation_limit + test_generation_limit.cpp + ${FASTFLOW_SOURCE_DIR}/server/generation_limit.cpp + ${FASTFLOW_SOURCE_DIR}/server/npu_access_manager.cpp) +target_include_directories(test_generation_limit PRIVATE + ${FASTFLOW_SOURCE_DIR}/include) +target_compile_definitions(test_generation_limit PRIVATE + FLM_TEST_SOURCE_DIR="${FASTFLOW_SOURCE_DIR}") +add_test(NAME test_generation_limit COMMAND test_generation_limit) + +# Validates FastFlow against the REAL corelib rather than the fake. +# +# With no runtime directory it reports SKIPPED to CTest via +# SKIP_RETURN_CODE, NOT Passed: this is the suite's highest-value check and +# a green-but-inert result would read as coverage it does not have. A +# configured-but-missing DLL is still a hard failure, so a stale or absent +# runtime cannot hide behind a green suite either. +set(RYZENAI_CORELIB_RUNTIME_DIR + "" + CACHE PATH + "Directory holding the real ryzenai_corelib.dll and its closure") +set(RYZENAI_CORELIB_EXTRA_DLL_DIRS + "" + CACHE STRING + "Semicolon-separated directories completing corelib's DLL closure") +string(REPLACE ";" "\\;" RYZENAI_CORELIB_EXTRA_DLL_DIRS_ESCAPED + "${RYZENAI_CORELIB_EXTRA_DLL_DIRS}") +add_corelib_host_test(test_real_corelib test_real_corelib.cpp) +target_compile_definitions(test_real_corelib PRIVATE + FLM_REAL_CORELIB_RUNTIME_DIR="${RYZENAI_CORELIB_RUNTIME_DIR}" + FLM_REAL_CORELIB_EXTRA_DLL_DIRS="${RYZENAI_CORELIB_EXTRA_DLL_DIRS_ESCAPED}") +set_tests_properties(test_real_corelib PROPERTIES SKIP_RETURN_CODE 77) +if(RYZENAI_CORELIB_RUNTIME_DIR) + set_tests_properties(test_real_corelib PROPERTIES + ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:${XRT_LIB_DIR};PATH=path_list_prepend:${RYZENAI_CORELIB_RUNTIME_DIR}") +endif() + +# The AIE4 hardware suite. These need a real NPU as well as a real DLL, so +# unlike test_real_corelib they cannot run anywhere but the target -- and like +# it, they report SKIPPED rather than Passed when the runtime directory is +# unset, because a green-and-inert hardware check reads as coverage it does not +# have. +function(add_corelib_hardware_test TEST_NAME TEST_SOURCE) + add_executable(${TEST_NAME} ${TEST_SOURCE}) + target_link_libraries(${TEST_NAME} PRIVATE + flm_corelib_aie4_testlib + shell32 + ole32 + advapi32) + target_compile_definitions(${TEST_NAME} PRIVATE + FLM_REAL_CORELIB_RUNTIME_DIR="${RYZENAI_CORELIB_RUNTIME_DIR}" + FLM_REAL_CORELIB_EXTRA_DLL_DIRS="${RYZENAI_CORELIB_EXTRA_DLL_DIRS_ESCAPED}") + add_test(NAME ${TEST_NAME} COMMAND ${TEST_NAME}) + set_tests_properties(${TEST_NAME} PROPERTIES + SKIP_RETURN_CODE 77 + # A hardware run loads a 3 GB model and packs its weights, and two + # AIE4 contexts at once fail in ways that look like defects. RUN_ + # SERIAL keeps ctest from overlapping them. + RUN_SERIAL ON + TIMEOUT 3600) + if(RYZENAI_CORELIB_RUNTIME_DIR) + set_tests_properties(${TEST_NAME} PROPERTIES + ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:${XRT_LIB_DIR};PATH=path_list_prepend:${RYZENAI_CORELIB_RUNTIME_DIR}") + endif() +endfunction() + +add_corelib_hardware_test(test_phi4_hardware test_phi4_hardware.cpp) +add_corelib_hardware_test(test_fatal_child test_fatal_child.cpp) + +# The end-to-end harness (test_phi4_e2e), the performance/memory baseline +# instrument (benchmark_phi4_aie4) and the PowerShell campaign that drove them +# are not part of this branch. They needed an AIE4 device, the real model and a +# caller-supplied token plan, so nothing here or in CI could run them; the +# records they produced were evidence of one campaign rather than a gate on +# future changes. What remains below is what a machine can actually run: +# test_phi4_hardware and test_fatal_child when a device is present, and the +# packaging checks when it is not. + +# The packaging and clean-environment checks run by default rather than only +# when someone remembers the command. Every regression guard in that script -- +# the derived closure, the CLOSURE-2 negative control, the no-corelib-import +# rule, and the requirement that the ordinary NPU2 installer still builds +# without an AIE4 closure -- is invisible if it is never invoked. +# +# It reports SKIPPED via SKIP_RETURN_CODE when its optional inputs are absent, +# so a partial run never reads as full coverage. +# +# FLM_POWERSHELL_EXECUTABLE is found near the top of this file, above the +# corelib-less early return, so this test survives a device-less configure. + +# Builds the argument vector for test_packaged_runtime.ps1. +# +# `-DependencyDir` takes one argument holding a semicolon-separated list, so +# the separator must be escaped before it is appended. `list(JOIN ... ";" ...)` +# followed by `list(APPEND)` does NOT achieve that: the join produces a string +# that append immediately re-splits, yielding one argv entry per directory. +# +# The consequence in PowerShell is worse than a lost directory. `-DependencyDir` +# binds only the first entry, and each orphan then binds POSITIONALLY -- the +# next positional parameter is `$FlmExe` -- so a second dependency directory +# silently becomes the path to the executable under test, turning an honest +# SKIPPED into a spurious flm.exe run against a directory. +function(flm_packaged_runtime_args output corelib_dir dependency_dirs) + set(_args "") + if(corelib_dir) + list(APPEND _args -CorelibRuntimeDir "${corelib_dir}") + endif() + if(dependency_dirs) + string(REPLACE ";" "\\;" _escaped "${dependency_dirs}") + list(APPEND _args -DependencyDir "${_escaped}") + endif() + set(${output} "${_args}" PARENT_SCOPE) +endfunction() + +# Self-check on the escaping, run at configure time against a synthetic +# multi-entry value so it does not depend on how this machine's cache happens +# to be populated. A single-entry cache masks the defect completely, which is +# how it survived the previous round. +flm_packaged_runtime_args(_flm_argv_probe "C:/core" "C:/one;C:/two") +list(LENGTH _flm_argv_probe _flm_argv_probe_length) +if(NOT _flm_argv_probe_length EQUAL 4) + message(FATAL_ERROR + "test_packaged_runtime argument construction flattened a multi-entry " + "dependency list into ${_flm_argv_probe_length} arguments instead of " + "4. The extra entries bind positionally in PowerShell, so -FlmExe " + "would receive a directory: [${_flm_argv_probe}]") +endif() +list(GET _flm_argv_probe 3 _flm_argv_probe_deps) +if(NOT _flm_argv_probe_deps STREQUAL "C:/one;C:/two") + message(FATAL_ERROR + "test_packaged_runtime dropped a dependency directory: " + "[${_flm_argv_probe_deps}]") +endif() + +if(FLM_POWERSHELL_EXECUTABLE) + flm_packaged_runtime_args(_packaged_runtime_args + "${RYZENAI_CORELIB_RUNTIME_DIR}" + "${RYZENAI_CORELIB_EXTRA_DLL_DIRS}") + add_test(NAME test_packaged_runtime + COMMAND "${FLM_POWERSHELL_EXECUTABLE}" + -NoProfile + -ExecutionPolicy Bypass + -File + "${CMAKE_CURRENT_LIST_DIR}/test_packaged_runtime.ps1" + # The CMake that configured this suite, not whatever + # `cmake` PATH happens to find. On the AIE4 target PATH + # finds Cygwin's 3.14.5, which cannot run + # StageAie4Runtime.cmake's required 3.24 -- so the + # closure guard failed for a reason unrelated to the + # closure. + -CmakeExe "${CMAKE_COMMAND}" + ${_packaged_runtime_args}) + set_tests_properties(test_packaged_runtime PROPERTIES + SKIP_RETURN_CODE 77 + TIMEOUT 1800) +else() + message(WARNING + "PowerShell was not found, so test_packaged_runtime is not " + "registered. The AIE4 packaging and clean-environment guards will " + "not run in this build.") +endif() + +add_corelib_host_test(test_corelib_api test_corelib_api.cpp) +add_corelib_host_test(test_phi4_manifest test_phi4_manifest.cpp) +add_corelib_host_test(test_phi4_shape_plan test_phi4_shape_plan.cpp) +add_corelib_host_test(test_phi4_host test_phi4_host.cpp) +add_corelib_host_test(test_phi4_weights test_phi4_weights.cpp) +add_corelib_host_test(test_phi4_engine test_phi4_engine.cpp) +target_link_libraries(test_phi4_engine PRIVATE + advapi32) +add_corelib_host_test( + test_corelib_fatal_record + test_corelib_fatal_record.cpp) +add_dependencies( + test_corelib_fatal_record + fake_ryzenai_corelib) +target_link_libraries(test_corelib_fatal_record PRIVATE + advapi32) + +function(add_phi4_frontend_test TEST_NAME ENABLE_AIE4) + add_executable(${TEST_NAME} + test_phi4_frontend.cpp + ${PHI4_FRONTEND_SOURCES}) + target_include_directories(${TEST_NAME} PRIVATE + ${FASTFLOW_SOURCE_DIR}/include + ${FASTFLOW_SOURCE_DIR}/../third_party/tokenizers-cpp/include + ${BOOST_INCLUDE_DIR} + ${XRT_INCLUDE_DIR}) + target_compile_definitions(${TEST_NAME} PRIVATE + FLM_CORELIB_TESTING=1 + DEV_BUILD=1 + __WINDOWS__ + USEAVX2=1 + DISABLE_ABI_CHECK=1 + _ENABLE_EXTENDED_ALIGNED_STORAGE + CMAKE_INSTALL_PREFIX="${FASTFLOW_SOURCE_DIR}/build/phi4_corelib_aie4-tests" + CMAKE_XCLBIN_PREFIX="${FASTFLOW_SOURCE_DIR}/xclbins" + WIN32_LEAN_AND_MEAN + NOMINMAX) + target_compile_options(${TEST_NAME} PRIVATE + $<$:/arch:AVX2 /fp:precise>) + target_link_directories(${TEST_NAME} PRIVATE + ${XRT_LIB_DIR}) + target_link_libraries(${TEST_NAME} PRIVATE + xrt_coreutil) + + if(ENABLE_AIE4) + target_compile_definitions(${TEST_NAME} PRIVATE + FLM_ENABLE_CORELIB_AIE4=1) + target_include_directories(${TEST_NAME} PRIVATE + ${RYZENAI_CORELIB_INCLUDE_DIR}) + target_link_libraries(${TEST_NAME} PRIVATE + flm_corelib_aie4_testlib) + add_dependencies(${TEST_NAME} + fake_ryzenai_corelib) + endif() + + add_test(NAME ${TEST_NAME} COMMAND ${TEST_NAME}) + set_tests_properties(${TEST_NAME} PROPERTIES + ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:${XRT_LIB_DIR}") +endfunction() + +add_phi4_frontend_test(test_phi4_frontend_off FALSE) +add_phi4_frontend_test(test_phi4_frontend_on TRUE) + +# The feature-ON halves of the two compile guards. Their feature-OFF twins, and +# the functions themselves, are defined near the top of this file so that they +# survive the guards-only return -- see the corelib find_path comment there. +add_phi4_frontend_compile_check( + phi4_frontend_compile_on + TRUE) + +add_task10_production_compile_check( + task10_production_compile_on + TRUE) + +# --------------------------------------------------------------------------- +# Task 14: the continuation-threshold calibrator, and the artifact-drift guard +# --------------------------------------------------------------------------- +# +# `kContinuationAppendThreshold` in phi4_corelib_aie4_tuning.hpp is GENERATED +# from docs/docs/benchmarks/phi4_aie4_baseline.json. Nothing in the C++ build +# can notice if the header, the published benchmark document and the +# measurement drift apart -- the header would still compile, the tests would +# still pass, and the shipped constant would no longer correspond to any +# measurement. `--check` is the only guard against that, so it has to be +# reachable from a runner rather than only from a human who remembers. +# +# The unittest module registered here runs the calibrator's own suite AND, in +# CheckedInArtefactsTest, invokes the calibrator with `--check` against the +# committed header, baseline, crossover history and benchmark document. +# +# FLM_PYTHON_EXECUTABLE may be set explicitly, and on an AIE4 target it should +# be, for the same reason test_packaged_runtime takes -CmakeExe: on that PATH +# resolves `python3` to C:/cygwin/bin/python3, a Cygwin symlink that Windows +# cannot execute at all. A found-but-unusable interpreter is worse than none, +# because it registers a test that can never pass for a reason having nothing +# to do with what the test checks -- which is exactly what happened on the +# first run of this block. +if(NOT FLM_PYTHON_EXECUTABLE) + find_program(FLM_PYTHON_EXECUTABLE + NAMES python3 python + DOC "Python used to run the offline tooling tests") +endif() + +# Whatever was found or supplied, prove it RUNS before trusting it. `-E` and a +# trivial expression rather than `--version`, so a wrapper that prints a banner +# and exits zero without an interpreter behind it still fails here. +set(FLM_PYTHON_USABLE FALSE) +if(FLM_PYTHON_EXECUTABLE) + execute_process( + COMMAND "${FLM_PYTHON_EXECUTABLE}" -c "import sys; sys.exit(0)" + RESULT_VARIABLE _flm_python_probe + OUTPUT_QUIET + ERROR_QUIET) + if(_flm_python_probe EQUAL 0) + set(FLM_PYTHON_USABLE TRUE) + else() + message(WARNING + "FLM_PYTHON_EXECUTABLE=${FLM_PYTHON_EXECUTABLE} does not run " + "(exit ${_flm_python_probe}); ignoring it. Pass a working " + "interpreter with -DFLM_PYTHON_EXECUTABLE=.") + endif() +endif() + +if(FLM_PYTHON_USABLE) + # WORKING_DIRECTORY is the repository root because the tests import + # `tools.calibrate_phi4_corelib_continuation` as a package and resolve the + # committed artifacts relative to it. + add_test(NAME test_phi4_continuation_calibration + COMMAND "${FLM_PYTHON_EXECUTABLE}" + -m unittest + tools.tests.test_calibrate_phi4_corelib_continuation + -v) + set_tests_properties(test_phi4_continuation_calibration PROPERTIES + WORKING_DIRECTORY "${CMAKE_CURRENT_LIST_DIR}/../../.." + TIMEOUT 300) +else() + # Not silently omitted. Registering a test that FAILS is the honest + # outcome: without Python this build cannot check that the shipped + # threshold still matches the measurement it claims to come from, and a + # green suite would say it did. Install Python or accept a red test. + add_test(NAME test_phi4_continuation_calibration + COMMAND "${CMAKE_COMMAND}" -E false) + set_tests_properties(test_phi4_continuation_calibration PROPERTIES + TIMEOUT 60) + message(WARNING + "No usable Python. test_phi4_continuation_calibration is registered " + "as a FAILING test: the generated kContinuationAppendThreshold cannot " + "be checked against the measurement it was derived from. Pass " + "-DFLM_PYTHON_EXECUTABLE= to fix it.") +endif() diff --git a/src/test/phi4_corelib_aie4/drive_flm_console.ps1 b/src/test/phi4_corelib_aie4/drive_flm_console.ps1 new file mode 100644 index 00000000..0ca8d536 --- /dev/null +++ b/src/test/phi4_corelib_aie4/drive_flm_console.ps1 @@ -0,0 +1,459 @@ +# Task 16: drive the interactive `flm run` REPL on Windows, for real. +# +# WHY THIS FILE EXISTS +# +# Tasks 11 and 12 recorded interactive `flm run` generation as unverifiable on +# Windows, and the reason was correct as far as it went: runner/cli_wide.cpp +# reads keystrokes with ReadConsoleInput() against a console input handle, and +# ReadConsoleInput cannot see a redirected stdin pipe. Piping a prompt into +# `flm run` therefore produces a REPL that has entered its input loop and will +# sit there until it is killed. Every attempt to smuggle a prompt in through +# stdin failed for that reason, and none of them was a defect in flm. +# +# But "stdin cannot reach it" is not the same claim as "it cannot be driven". +# ReadConsoleInput reads the console INPUT BUFFER, and that buffer is writable +# by any process attached to the same console, through WriteConsoleInput. So a +# driver can: +# +# 1. FreeConsole() then AllocConsole() to get a console of its own -- an SSH +# exec session has no usable console, so one has to be created; +# 2. start flm.exe as a child with -NoNewWindow so it INHERITS that console +# (Start-Process without -NoNewWindow allocates the child a fresh console +# and the injected keystrokes go to the wrong buffer -- measured, and it +# looks exactly like the input being ignored); +# 3. inject KEY_EVENT records into CONIN$, which is the same buffer +# ReadConsoleInput drains; and +# 4. read the completion back out of the console SCREEN buffer with +# ReadConsoleOutputCharacter, because the child's stdout is the console, +# not a pipe we could have captured. +# +# This is the product's own interactive path, unmodified: the same binary, the +# same ReadConsoleInput loop, the same rendering. It is not a stub, and it is +# not `flm serve` wearing a CLI costume. +# +# WHAT IT COSTS, STATED PLAINLY +# +# Reading the screen buffer returns the RENDERED screen, not the byte stream. +# Trailing spaces on each row are not recoverable, a line longer than the +# buffer width is wrapped with no marker distinguishing that from a real +# newline, and anything scrolled off the top is gone. The buffer is therefore +# set to 200x9000 up front so a 512-token completion cannot scroll away, and +# every recorded completion in the acceptance document should be read as +# "the text a user would have seen", which is what Step 7 actually asks for. +# +# HOW OUTPUT GETS BACK +# +# After AllocConsole() this process's stdout IS the new console, so anything +# written to the success stream disappears from the SSH session. Every result +# goes to -OutJson instead. That is also why this is a separate script rather +# than a function in whatever calls it: the caller has to keep its own stdout, +# so the AllocConsole() has to happen in a process the caller does not need. + +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)][string]$FlmExe, + + # Extra arguments after `run `; e.g. @('-c','4096'). + [string[]]$FlmArgs = @(), + + [Parameter(Mandatory = $true)][string]$ModelTag, + + # One entry per turn. Each is typed at a `>>> ` prompt and followed by + # Enter. `/bye` is appended by the driver; do not include it. + [string[]]$Turns = @(), + + # A file with one turn per line, used INSTEAD of -Turns. + # + # `powershell.exe -File script.ps1 -Turns "a b" "c d"` does not bind an + # array: the first quoted value binds to -Turns and the second is offered + # positionally, the script has no positional parameter, and the process + # exits 1 having produced nothing. A single turn works and several do not, + # which is a trap worth closing rather than remembering. Callers that pass + # more than one turn should use this. + [string]$TurnsFile = '', + + [Parameter(Mandatory = $true)][string]$OutJson, + + # Seconds to wait for the first `>>> ` prompt. Model load is ~1-2 minutes + # on this box and a cold file cache makes it worse. + [int]$LoadTimeoutSec = 900, + + # Seconds to wait for one turn's reply to finish. + [int]$TurnTimeoutSec = 900, + + # A turn is finished when the screen has been unchanged for this long AND + # a fresh `>>> ` prompt is present. The idle requirement is what stops a + # pause between two streamed tokens being read as the end of the reply. + [int]$IdleSec = 6, + + [int]$BufferWidth = 200, + [int]$BufferHeight = 9000, + + # Every poll overwrites this file with the current console screen. The only + # window into a session that is hung: the child's stdout is the console, so + # there is no pipe to tail and no log to read. Without it a stall is + # indistinguishable from a slow load -- which is exactly how the first run + # of this driver was misread. + [string]$ScreenLog = '' +) + +$ErrorActionPreference = 'Stop' + +if ($TurnsFile) { + if (-not (Test-Path $TurnsFile)) { throw "TurnsFile not found: $TurnsFile" } + # Not Get-Content's array form: a turn may legitimately be an empty line + # only if the caller wrote one, and -Raw plus an explicit split keeps that + # decision with the caller instead of with the cmdlet. + $Turns = @((Get-Content $TurnsFile -Raw) -split "`r?`n" | Where-Object { $_.Length -gt 0 }) +} +if ($Turns.Count -eq 0) { throw 'no turns supplied: pass -Turns or -TurnsFile' } + +$result = [ordered]@{ + flm_exe = $FlmExe + model_tag = $ModelTag + turns = @() + load_seconds = $null + reached_prompt = $false + exit_code = $null + error = $null + screen_final = '' + driver = 'WriteConsoleInput/ReadConsoleOutputCharacter' + console_screen_info_ok = $null + console_conin = $null + console_conout = $null +} + +function Save-Result { + $dir = Split-Path -Parent $OutJson + if ($dir -and -not (Test-Path $dir)) { New-Item -ItemType Directory -Force -Path $dir | Out-Null } + ($result | ConvertTo-Json -Depth 8) | Set-Content -Path $OutJson -Encoding utf8 +} + +Add-Type -Namespace FlmCon -Name Api -MemberDefinition @' +[DllImport("kernel32.dll", SetLastError=true)] public static extern bool AllocConsole(); +[DllImport("kernel32.dll", SetLastError=true)] public static extern bool FreeConsole(); +[DllImport("kernel32.dll", SetLastError=true)] public static extern bool SetConsoleScreenBufferSize(IntPtr h, COORD size); +[DllImport("kernel32.dll", SetLastError=true)] public static extern bool SetStdHandle(int which, IntPtr handle); + +[StructLayout(LayoutKind.Sequential)] public struct COORD { public short X; public short Y; } +[StructLayout(LayoutKind.Sequential)] public struct SMALL_RECT { public short Left, Top, Right, Bottom; } +[StructLayout(LayoutKind.Sequential)] public struct CONSOLE_SCREEN_BUFFER_INFO { + public COORD dwSize; public COORD dwCursorPosition; public short wAttributes; + public SMALL_RECT srWindow; public COORD dwMaximumWindowSize; } + +[StructLayout(LayoutKind.Explicit)] public struct INPUT_RECORD { + [FieldOffset(0)] public ushort EventType; + [FieldOffset(4)] public int bKeyDown; + [FieldOffset(8)] public ushort wRepeatCount; + [FieldOffset(10)] public ushort wVirtualKeyCode; + [FieldOffset(12)] public ushort wVirtualScanCode; + [FieldOffset(14)] public ushort UnicodeChar; + [FieldOffset(16)] public uint dwControlKeyState; } + +[StructLayout(LayoutKind.Sequential)] public struct SECURITY_ATTRIBUTES { + public int nLength; public IntPtr lpSecurityDescriptor; public int bInheritHandle; } + +[DllImport("kernel32.dll", SetLastError=true, CharSet=CharSet.Unicode)] +public static extern IntPtr CreateFileW(string name, uint access, uint share, ref SECURITY_ATTRIBUTES sa, uint disp, uint flags, IntPtr tmpl); +[DllImport("kernel32.dll", SetLastError=true)] +public static extern bool WriteConsoleInputW(IntPtr h, INPUT_RECORD[] recs, uint n, out uint written); +[DllImport("kernel32.dll", SetLastError=true)] +public static extern bool GetNumberOfConsoleInputEvents(IntPtr h, out uint n); +[DllImport("kernel32.dll", SetLastError=true)] +public static extern bool GetConsoleScreenBufferInfo(IntPtr h, out CONSOLE_SCREEN_BUFFER_INFO info); +[DllImport("kernel32.dll", SetLastError=true, CharSet=CharSet.Unicode)] +public static extern bool ReadConsoleOutputCharacterW(IntPtr h, [Out] char[] buf, uint len, COORD coord, out uint read); +'@ + +$script:conin = [IntPtr]::Zero +$script:conout = [IntPtr]::Zero +$script:child = $null + +function New-KeyRecord { + param([char]$Ch, [uint16]$Vk = 0) + $r = New-Object FlmCon.Api+INPUT_RECORD + $r.EventType = 1 # KEY_EVENT + $r.bKeyDown = 1 + $r.wRepeatCount = 1 + $r.wVirtualKeyCode = $Vk + $r.wVirtualScanCode = 0 + $r.UnicodeChar = [uint16][char]$Ch + $r.dwControlKeyState = 0 + return $r +} + +# One record per call, and the buffer is drained to empty before the next. +# +# cli_wide.cpp treats "more than one event pending" as a PASTE and takes a +# different code path. Feeding a whole line at once therefore exercises the +# paste handler rather than the keystroke handler, which is not the path a +# user takes. Typing one character at a time keeps GetNumberOfConsoleInputEvents +# at 1, so the ordinary path runs. +function Send-Key { + param([char]$Ch, [uint16]$Vk = 0) + $recs = @((New-KeyRecord -Ch $Ch -Vk $Vk)) + $written = 0 + if (-not [FlmCon.Api]::WriteConsoleInputW($script:conin, $recs, [uint32]1, [ref]$written)) { + throw "WriteConsoleInput failed: $([Runtime.InteropServices.Marshal]::GetLastWin32Error())" + } + # Wait for the child to consume it, so the next key does not look like a + # paste. Bounded: if the child is not reading, fall through rather than + # hanging here forever -- the turn timeout is the real guard. + for ($i = 0; $i -lt 200; $i++) { + $n = 0 + [void][FlmCon.Api]::GetNumberOfConsoleInputEvents($script:conin, [ref]$n) + if ($n -eq 0) { return } + Start-Sleep -Milliseconds 10 + } +} + +function Send-Line { + param([string]$Text) + foreach ($c in $Text.ToCharArray()) { Send-Key -Ch $c } + Send-Key -Ch "`r" -Vk 13 # VK_RETURN +} + +function Read-Screen { + $info = New-Object FlmCon.Api+CONSOLE_SCREEN_BUFFER_INFO + if (-not [FlmCon.Api]::GetConsoleScreenBufferInfo($script:conout, [ref]$info)) { return '' } + $cols = [int]$info.dwSize.X + $rows = [int]$info.dwCursorPosition.Y + 1 + if ($rows -le 0 -or $cols -le 0) { return '' } + $total = $rows * $cols + $buf = New-Object char[] $total + $origin = New-Object FlmCon.Api+COORD + $origin.X = 0; $origin.Y = 0 + $read = 0 + if (-not [FlmCon.Api]::ReadConsoleOutputCharacterW($script:conout, $buf, [uint32]$total, $origin, [ref]$read)) { return '' } + $flat = -join $buf + $lines = New-Object System.Collections.Generic.List[string] + for ($i = 0; $i -lt $rows; $i++) { + $lines.Add(($flat.Substring($i * $cols, $cols)).TrimEnd()) + } + return ($lines -join "`n") +} + +# Wait until the screen stops changing for -IdleSec AND satisfies $Predicate. +# +# Both conditions, deliberately. Idle alone fires during a slow prefill; +# the predicate alone fires the instant the string appears, which for a +# `>>> ` prompt is BEFORE the reply to the previous turn has been printed. +function Wait-ForScreen { + param( + [scriptblock]$Predicate, + [int]$TimeoutSec, + [string]$Label + ) + $deadline = (Get-Date).AddSeconds($TimeoutSec) + $lastText = '' + $lastChange = Get-Date + while ((Get-Date) -lt $deadline) { + if ($script:child.HasExited) { + return @{ Ok = $false; Screen = (Read-Screen); Reason = "child exited with $($script:child.ExitCode)" } + } + $text = Read-Screen + if ($ScreenLog) { + try { Set-Content -Path $ScreenLog -Value "[$Label]`n$text" -Encoding utf8 } catch { } + } + if ($text -ne $lastText) { $lastText = $text; $lastChange = Get-Date } + $idle = ((Get-Date) - $lastChange).TotalSeconds + if ($idle -ge $IdleSec -and (& $Predicate $text)) { + return @{ Ok = $true; Screen = $text; Reason = '' } + } + Start-Sleep -Milliseconds 500 + } + return @{ Ok = $false; Screen = (Read-Screen); Reason = "timed out after ${TimeoutSec}s waiting for $Label" } +} + +# Count of `>>> ` prompts on screen. Turn N is complete when prompt N+1 shows. +# +# Two things about this pattern, both learned the same way -- by watching the +# driver wait out a timeout against a REPL that was sitting at its prompt. +# +# No trailing space: Read-Screen TrimEnd()s every row, so the prompt the user +# sees as ">>> " arrives here as ">>>". +# +# Leading whitespace allowed: when a reply ends without a newline -- which +# happens when it contains emoji or other characters the console does not +# render one cell per code unit -- the prompt is written at whatever column the +# cursor stopped at, and the row arrives as spaces followed by ">>>". Anchoring +# hard at column zero misses it, and the driver hangs for the full turn timeout +# on a turn that finished normally. Measured on a reply that ended in emoji. +function Get-PromptCount { + param([string]$Text) + if (-not $Text) { return 0 } + return ([regex]::Matches($Text, '(?m)^[ ]*>>>')).Count +} + +try { + [void][FlmCon.Api]::FreeConsole() + if (-not [FlmCon.Api]::AllocConsole()) { + $result.error = "AllocConsole failed: $([Runtime.InteropServices.Marshal]::GetLastWin32Error())" + Save-Result + exit 2 + } + + $access = [uint32]3221225472 # GENERIC_READ | GENERIC_WRITE, spelled in + # decimal because Windows PowerShell 5.1 + # parses 0xC0000000 as a negative Int32. + + # bInheritHandle = TRUE, and this is the whole ballgame. + # + # .NET's Process.Start with UseShellExecute = false always passes + # STARTF_USESTDHANDLES and hands the child THIS process's standard + # handles. If the handles installed by SetStdHandle below were opened + # without inheritance, the child receives handles it cannot use: + # ReadConsoleInput fails immediately, the REPL spins on the failure at + # 100% CPU, and nothing is ever written to the screen buffer this driver + # reads back. That is precisely the symptom this cost four runs to find, + # and it is invisible from the outside -- the console is real, the console + # is readable, and it is simply not the child's. + $sa = New-Object FlmCon.Api+SECURITY_ATTRIBUTES + $sa.nLength = [System.Runtime.InteropServices.Marshal]::SizeOf($sa) + $sa.lpSecurityDescriptor = [IntPtr]::Zero + $sa.bInheritHandle = 1 + $script:conin = [FlmCon.Api]::CreateFileW('CONIN$', $access, [uint32]3, [ref]$sa, [uint32]3, [uint32]0, [IntPtr]::Zero) + $script:conout = [FlmCon.Api]::CreateFileW('CONOUT$', $access, [uint32]3, [ref]$sa, [uint32]3, [uint32]0, [IntPtr]::Zero) + if ($script:conin -eq [IntPtr]::new(-1) -or $script:conout -eq [IntPtr]::new(-1)) { + $result.error = 'could not open CONIN$/CONOUT$ on the allocated console' + Save-Result + exit 2 + } + + # Point this process's STANDARD HANDLES at the new console. + # + # AllocConsole only rebinds the standard handles that were not already + # redirected. When this driver is launched by another script -- which is + # how the acceptance harness runs it -- its stdin/stdout are pipes, and + # they stay pipes. Start-Process -NoNewWindow then hands those pipes to + # flm.exe, so ReadConsoleInput has nothing to read (the REPL spins at 100% + # CPU on a failing read) and everything flm prints goes into a pipe rather + # than the console screen buffer this driver reads back. Measured: an + # empty screen, an flm.exe at 944 seconds of CPU, and a driver that looked + # like the model had never loaded. + # + # Rebinding them explicitly makes the driver behave the same whether it is + # run from a terminal, from another script, or over SSH. + [void][FlmCon.Api]::SetStdHandle(-10, $script:conin) # STD_INPUT_HANDLE + [void][FlmCon.Api]::SetStdHandle(-11, $script:conout) # STD_OUTPUT_HANDLE + [void][FlmCon.Api]::SetStdHandle(-12, $script:conout) # STD_ERROR_HANDLE + + # Recorded so that a driver that never got a usable console is + # diagnosable from the artifact rather than from a process listing. + $probe = New-Object FlmCon.Api+CONSOLE_SCREEN_BUFFER_INFO + $result.console_screen_info_ok = [bool][FlmCon.Api]::GetConsoleScreenBufferInfo($script:conout, [ref]$probe) + $result.console_conin = [string]$script:conin + $result.console_conout = [string]$script:conout + if (-not $result.console_screen_info_ok) { + $result.error = 'the allocated console has no readable screen buffer; flm output could not have been captured' + Save-Result + exit 2 + } + + $size = New-Object FlmCon.Api+COORD + $size.X = [int16]$BufferWidth + $size.Y = [int16]$BufferHeight + [void][FlmCon.Api]::SetConsoleScreenBufferSize($script:conout, $size) + + $argList = @('run', $ModelTag) + $FlmArgs + $result.flm_argv = (@($FlmExe) + $argList) -join ' ' + $loadStart = Get-Date + + # ProcessStartInfo directly, NOT Start-Process. + # + # flm.exe has to inherit THIS process's console, and the two obvious + # spellings both fail to deliver that: + # + # Start-Process (plain) -> the child gets a brand new console, so + # injected keystrokes go to a buffer + # nobody is reading; + # Start-Process -NoNewWindow -> Windows PowerShell 5.1 sets + # CreateNoWindow, and a console + # application started that way also gets + # a console of its own rather than the + # caller's. + # + # Measured with -NoNewWindow: an empty screen buffer and an flm.exe at + # 100% CPU for 150 seconds, spinning on a ReadConsoleInput that can never + # succeed. UseShellExecute = false with CreateNoWindow = false and no + # redirection is the combination that actually inherits the console. + $psi = New-Object System.Diagnostics.ProcessStartInfo + $psi.FileName = $FlmExe + $psi.Arguments = ($argList | ForEach-Object { if ($_ -match '\s') { '"' + $_ + '"' } else { $_ } }) -join ' ' + $psi.UseShellExecute = $false + $psi.CreateNoWindow = $false + $psi.RedirectStandardInput = $false + $psi.RedirectStandardOutput = $false + $psi.RedirectStandardError = $false + $script:child = [System.Diagnostics.Process]::Start($psi) + + $ready = Wait-ForScreen -TimeoutSec $LoadTimeoutSec -Label 'the first >>> prompt' -Predicate { + param($t) (Get-PromptCount $t) -ge 1 + } + $result.load_seconds = [math]::Round(((Get-Date) - $loadStart).TotalSeconds, 3) + $result.reached_prompt = $ready.Ok + if (-not $ready.Ok) { + $result.error = "model load / REPL entry: $($ready.Reason)" + $result.screen_final = $ready.Screen + Save-Result + exit 3 + } + + $prevScreen = $ready.Screen + $turnIndex = 0 + foreach ($turn in $Turns) { + $turnIndex++ + $expectPrompts = (Get-PromptCount $prevScreen) + 1 + $t0 = Get-Date + Send-Line -Text $turn + $done = Wait-ForScreen -TimeoutSec $TurnTimeoutSec -Label "turn $turnIndex to finish" -Predicate { + param($t) (Get-PromptCount $t) -ge $expectPrompts + } + $elapsed = [math]::Round(((Get-Date) - $t0).TotalSeconds, 3) + # The reply is what this turn ADDED to the screen. Taking the delta + # rather than re-parsing the whole screen keeps a multi-turn session + # from attributing turn 1's reply to turn 4. + $newText = $done.Screen + $reply = if ($newText.StartsWith($prevScreen)) { $newText.Substring($prevScreen.Length) } else { $newText } + $result.turns += [ordered]@{ + index = $turnIndex + prompt = $turn + ok = $done.Ok + reason = $done.Reason + seconds = $elapsed + reply_raw = $reply + } + $prevScreen = $newText + if (-not $done.Ok) { + $result.error = "turn ${turnIndex}: $($done.Reason)" + $result.screen_final = $newText + Save-Result + exit 4 + } + } + + Send-Line -Text '/bye' + for ($i = 0; $i -lt 120; $i++) { + if ($script:child.HasExited) { break } + Start-Sleep -Milliseconds 500 + } + $result.screen_final = Read-Screen + if ($script:child.HasExited) { + $result.exit_code = $script:child.ExitCode + } else { + $result.error = '/bye did not terminate the REPL' + } + Save-Result + if ($result.error) { exit 5 } else { exit 0 } +} +catch { + $result.error = "$($_.Exception.Message)" + try { $result.screen_final = Read-Screen } catch { } + Save-Result + exit 6 +} +finally { + if ($script:child -and -not $script:child.HasExited) { + Stop-Process -Id $script:child.Id -Force -ErrorAction SilentlyContinue + } +} diff --git a/src/test/phi4_corelib_aie4/fake_corelib.cpp b/src/test/phi4_corelib_aie4/fake_corelib.cpp new file mode 100644 index 00000000..519d52a2 --- /dev/null +++ b/src/test/phi4_corelib_aie4/fake_corelib.cpp @@ -0,0 +1,835 @@ +#include "fake_corelib.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace flm::test { +namespace detail { + +thread_local std::string g_last_error; +std::uint32_t g_version_major = RYZENAI_CORELIB_VERSION_MAJOR; +std::uint32_t g_version_minor = RYZENAI_CORELIB_VERSION_MINOR; +std::uint32_t g_version_patch = RYZENAI_CORELIB_VERSION_PATCH; +std::unordered_map g_release_counts; +std::size_t g_release_count = 0; +void* g_last_released_object = nullptr; +ryzenai_corelib_status g_selftest_status = + ryzenai_corelib_status_success; +bool g_has_device_context = true; +std::size_t g_cleanup_count = 0; +std::vector g_events; + +void FakeGetVersion( + std::uint32_t* major, + std::uint32_t* minor, + std::uint32_t* patch) { + if (major != nullptr) { + *major = g_version_major; + } + if (minor != nullptr) { + *minor = g_version_minor; + } + if (patch != nullptr) { + *patch = g_version_patch; + } +} + +// A device tensor with a real dtype and element count, because that is +// what tensor_write / tensor_read are bounded by in e5258d2. A fake that +// accepted either unit convention would let a byte-sized count through, +// which is precisely the regression the element rebase must not reproduce. +struct FakeTensor { + ryzenai_corelib_data_type data_type; + std::size_t element_count; +}; + +std::unordered_map> g_tensors; + +bool IsConvertibleDataType(ryzenai_corelib_data_type value) noexcept { + return value == ryzenai_corelib_data_type_fp32 || + value == ryzenai_corelib_data_type_fp16 || + value == ryzenai_corelib_data_type_bf16; +} + +std::size_t DataTypeByteSize(ryzenai_corelib_data_type value) noexcept { + return value == ryzenai_corelib_data_type_fp32 ? 4u : 2u; +} + +FakeTensor* FindTensor(ryzenai_corelib_tensor_ptr tensor) noexcept { + const auto found = g_tensors.find(tensor); + return found == g_tensors.end() ? nullptr : found->second.get(); +} + +// Shared by write and read: count and offset are ELEMENTS of the tensor's +// own dtype, so offset + count must lie inside it. A caller still passing +// bytes overruns a BF16 tensor by exactly 2x and is rejected here rather +// than silently moving twice the data. +ryzenai_corelib_status CheckTensorRange( + ryzenai_corelib_tensor_ptr tensor, + ryzenai_corelib_data_type caller_type, + const void* host, + std::size_t count, + std::size_t offset) { + if (host == nullptr) { + g_last_error = "host buffer is null"; + return ryzenai_corelib_status_bad_argument; + } + if (!IsConvertibleDataType(caller_type)) { + g_last_error = + "only FP32, FP16 and BF16 are accepted for the caller type"; + return ryzenai_corelib_status_bad_argument; + } + const FakeTensor* record = FindTensor(tensor); + if (record == nullptr) { + // Tests that supply their own tensor handles opt out of the range + // model; they assert on the recorded arguments instead. + return ryzenai_corelib_status_success; + } + if (!IsConvertibleDataType(record->data_type)) { + g_last_error = "tensor dtype has no host float representation"; + return ryzenai_corelib_status_bad_argument; + } + if ( + offset > record->element_count || + count > record->element_count - offset) { + g_last_error = + "offset + count (" + std::to_string(offset) + " + " + + std::to_string(count) + ") exceeds the tensor's " + + std::to_string(record->element_count) + + " elements; count and offset are ELEMENTS, not bytes"; + return ryzenai_corelib_status_bad_argument; + } + return ryzenai_corelib_status_success; +} + +const char* FakeStatusToString(ryzenai_corelib_status status) { + g_last_error = "overwritten by status_to_string"; + switch (status) { + case ryzenai_corelib_status_success: + return "success"; + case ryzenai_corelib_status_failure: + return "failure"; + case ryzenai_corelib_status_bad_argument: + return "bad argument"; + case ryzenai_corelib_status_unsupported: + return "unsupported"; + } + return "unknown"; +} + +const char* FakeGetLastErrorMessage() { + return g_last_error.c_str(); +} + +ryzenai_corelib_status FakeSelftestDependencies() { + return g_selftest_status; +} + +bool FakeHasDeviceContext() { + return g_has_device_context; +} + +void FakeObjectRelease(ryzenai_corelib_object_ptr object) { + ++g_release_count; + ++g_release_counts[object]; + g_last_released_object = object; + g_events.emplace_back("release"); + g_tensors.erase(object); +} + +ryzenai_corelib_status FakeCreateStream(ryzenai_corelib_stream_ptr* out) { + if (out != nullptr) { + *out = nullptr; + } + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status FakeStreamSynchronize( + ryzenai_corelib_stream_ptr) { + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status FakeCreateDeviceTensor( + ryzenai_corelib_data_type data_type, + const int64_t* shape, + std::size_t shape_len, + ryzenai_corelib_tensor_ptr* out) { + if (out != nullptr) { + *out = nullptr; + } + if (out == nullptr || shape == nullptr || shape_len == 0) { + g_last_error = "create_device_tensor requires a shape and an out"; + return ryzenai_corelib_status_bad_argument; + } + std::size_t elements = 1; + for (std::size_t index = 0; index < shape_len; ++index) { + if (shape[index] <= 0) { + g_last_error = "tensor dimensions must be positive"; + return ryzenai_corelib_status_bad_argument; + } + elements *= static_cast(shape[index]); + } + auto record = std::make_unique( + FakeTensor{data_type, elements}); + void* handle = record.get(); + g_tensors.emplace(handle, std::move(record)); + *out = handle; + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status FakeTensorWrite( + ryzenai_corelib_tensor_ptr tensor, + ryzenai_corelib_data_type source_type, + const void* source, + std::size_t count, + std::size_t offset) { + return CheckTensorRange(tensor, source_type, source, count, offset); +} + +ryzenai_corelib_status FakeTensorRead( + ryzenai_corelib_tensor_ptr tensor, + ryzenai_corelib_data_type destination_type, + void* destination, + std::size_t count, + std::size_t offset) { + return CheckTensorRange( + tensor, + destination_type, + destination, + count, + offset); +} + +ryzenai_corelib_status FakeTensorGetByteSize( + ryzenai_corelib_tensor_ptr tensor, + std::size_t* out) { + if (out == nullptr) { + return ryzenai_corelib_status_bad_argument; + } + const FakeTensor* record = FindTensor(tensor); + *out = record == nullptr + ? 0 + : record->element_count * + DataTypeByteSize(record->data_type); + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status FakeTensorGetDataType( + ryzenai_corelib_tensor_ptr tensor, + ryzenai_corelib_data_type* out) { + if (out == nullptr) { + return ryzenai_corelib_status_bad_argument; + } + const FakeTensor* record = FindTensor(tensor); + *out = record == nullptr + ? ryzenai_corelib_data_type_bf16 + : record->data_type; + return ryzenai_corelib_status_success; +} + +// The shipped AIE4 kernel grid, measured against the real e5258d2 library +// (see test_real_corelib, which asserts the two still agree). Padding is a +// pure lookup, so the fake can reproduce it exactly -- and it must: every +// other test in this suite sizes its buffers from these answers. +constexpr int64_t kShippedRowGrid[] = { + 1, 64, 128, 256, 512, 1024, 2048, 3072, 4096}; + +// The LM head ships far fewer M: 1, then 128, and nothing above it. An +// out-of-grid M is an ERROR rather than a larger answer, which is what +// stops a caller allocating for a shape no kernel serves. +constexpr int64_t kLmHeadRowGrid[] = {1, 128}; + +bool PadToGrid( + int64_t* m, + const int64_t* grid, + std::size_t grid_size) { + for (std::size_t index = 0; index < grid_size; ++index) { + if (*m <= grid[index]) { + *m = grid[index]; + return true; + } + } + return false; +} + +ryzenai_corelib_status FakeMatmulPadShape( + int64_t* m, + int64_t* k, + int64_t* n, + uint32_t) { + if (k == nullptr || n == nullptr || *k <= 0 || *n <= 0) { + g_last_error = "matmul_bf16_pad_shape requires k and n"; + return ryzenai_corelib_status_bad_argument; + } + // K and N are never padded for the Phi-4 shapes; MEM-5 rests on that. + if (m == nullptr) { + return ryzenai_corelib_status_success; + } + if (*m <= 0) { + g_last_error = "matmul_bf16_pad_shape requires positive m"; + return ryzenai_corelib_status_bad_argument; + } + const bool is_lm_head = *n >= 200064; + const bool padded = + is_lm_head + ? PadToGrid( + m, + kLmHeadRowGrid, + sizeof(kLmHeadRowGrid) / sizeof(int64_t)) + : PadToGrid( + m, + kShippedRowGrid, + sizeof(kShippedRowGrid) / sizeof(int64_t)); + if (!padded) { + g_last_error = + "no valid padded M shape for AIE4 at m=" + + std::to_string(*m); + return ryzenai_corelib_status_unsupported; + } + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status FakeMatmulWeightsFromOnnx( + const ryzenai_corelib_matmul_bf16_weights_desc*, + const ryzenai_corelib_matmul_bf16_onnx_components*, + uint32_t, + ryzenai_corelib_matmul_bf16_weights_ptr* out) { + if (out != nullptr) { + *out = nullptr; + } + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status FakeMatmulWeightsGetData( + ryzenai_corelib_matmul_bf16_weights_ptr, + const void** data, + std::size_t* size) { + if (data != nullptr) { + *data = nullptr; + } + if (size != nullptr) { + *size = 0x4D4D; + } + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status FakeMatmul( + ryzenai_corelib_stream_ptr, + ryzenai_corelib_tensor_ptr, + int64_t, + ryzenai_corelib_matmul_bf16_weights_ptr, + ryzenai_corelib_tensor_ptr) { + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status FakeSsmlpPadRows( + int64_t* m, + int64_t, + int64_t, + uint32_t) { + if (m == nullptr || *m <= 0) { + g_last_error = "ssmlp_bf16_pad_rows requires positive m"; + return ryzenai_corelib_status_bad_argument; + } + if (!PadToGrid( + m, + kShippedRowGrid, + sizeof(kShippedRowGrid) / sizeof(int64_t))) { + g_last_error = "no valid padded row count for AIE4"; + return ryzenai_corelib_status_unsupported; + } + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status FakeSsmlpWeightsFromOnnx( + const ryzenai_corelib_ssmlp_bf16_weights_desc*, + const ryzenai_corelib_ssmlp_bf16_onnx_components*, + uint32_t, + ryzenai_corelib_ssmlp_bf16_weights_ptr* out) { + if (out != nullptr) { + *out = nullptr; + } + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status FakeSsmlpWeightsGetData( + ryzenai_corelib_ssmlp_bf16_weights_ptr, + const void** data, + std::size_t* size) { + if (data != nullptr) { + *data = nullptr; + } + if (size != nullptr) { + *size = 0x5353; + } + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status FakeSsmlp( + ryzenai_corelib_stream_ptr, + ryzenai_corelib_tensor_ptr, + ryzenai_corelib_tensor_ptr, + int64_t, + ryzenai_corelib_ssmlp_bf16_weights_ptr, + ryzenai_corelib_tensor_ptr, + ryzenai_corelib_tensor_ptr) { + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status FakeFlatMhaPadRows( + int64_t* m, + const ryzenai_corelib_flat_mha_bf16_desc* desc) { + if (m == nullptr || desc == nullptr || *m <= 0) { + g_last_error = "flat_mha_bf16_pad_rows requires m and a desc"; + return ryzenai_corelib_status_bad_argument; + } + // Decode is not padded: the token kernel runs one row and pads its KV + // window internally. + if (*m == 1) { + return ryzenai_corelib_status_success; + } + if (!PadToGrid( + m, + kShippedRowGrid, + sizeof(kShippedRowGrid) / sizeof(int64_t))) { + g_last_error = "no valid padded row count for AIE4"; + return ryzenai_corelib_status_unsupported; + } + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status FakeFlatMha( + ryzenai_corelib_stream_ptr, + const ryzenai_corelib_flat_mha_bf16_desc*, + ryzenai_corelib_tensor_ptr, + ryzenai_corelib_tensor_ptr, + int64_t, + int64_t, + ryzenai_corelib_tensor_ptr, + ryzenai_corelib_tensor_ptr, + ryzenai_corelib_tensor_ptr, + ryzenai_corelib_tensor_ptr, + ryzenai_corelib_tensor_ptr) { + return ryzenai_corelib_status_success; +} + +void FakeCleanup() { + ++g_cleanup_count; + g_events.emplace_back("cleanup"); + + std::size_t required = 0; + if (_wgetenv_s( + &required, + nullptr, + 0, + L"FLM_FAKE_CORELIB_CLEANUP_MARKER") != 0 || + required == 0) { + return; + } + std::vector marker_path(required); + if (_wgetenv_s( + &required, + marker_path.data(), + marker_path.size(), + L"FLM_FAKE_CORELIB_CLEANUP_MARKER") != 0) { + return; + } + std::ofstream marker( + std::filesystem::path(marker_path.data()), + std::ios::app | std::ios::binary); + marker << "cleanup\n"; +} + +} // namespace detail +} // namespace flm::test + +#if defined(FLM_FAKE_CORELIB_DLL) + +void ryzenai_corelib_get_version( + uint32_t* major, + uint32_t* minor, + uint32_t* patch) { + flm::test::detail::FakeGetVersion(major, minor, patch); +} + +const char* ryzenai_corelib_status_to_string( + ryzenai_corelib_status status) { + return flm::test::detail::FakeStatusToString(status); +} + +const char* ryzenai_corelib_get_last_error_message() { + return flm::test::detail::FakeGetLastErrorMessage(); +} + +ryzenai_corelib_status ryzenai_corelib_selftest_dependencies() { + return flm::test::detail::FakeSelftestDependencies(); +} + +bool ryzenai_corelib_has_device_context() { + return flm::test::detail::FakeHasDeviceContext(); +} + +void ryzenai_corelib_object_release(ryzenai_corelib_object_ptr object) { + flm::test::detail::FakeObjectRelease(object); +} + +ryzenai_corelib_status ryzenai_corelib_create_stream( + ryzenai_corelib_stream_ptr* out) { + return flm::test::detail::FakeCreateStream(out); +} + +ryzenai_corelib_status ryzenai_corelib_stream_synchronize( + ryzenai_corelib_stream_ptr stream) { + return flm::test::detail::FakeStreamSynchronize(stream); +} + +ryzenai_corelib_status ryzenai_corelib_create_device_tensor( + ryzenai_corelib_data_type data_type, + const int64_t* shape, + std::size_t shape_len, + ryzenai_corelib_tensor_ptr* out) { + return flm::test::detail::FakeCreateDeviceTensor( + data_type, + shape, + shape_len, + out); +} + +ryzenai_corelib_status ryzenai_corelib_tensor_write( + ryzenai_corelib_tensor_ptr tensor, + ryzenai_corelib_data_type source_type, + const void* source, + std::size_t count, + std::size_t offset) { + return flm::test::detail::FakeTensorWrite( + tensor, + source_type, + source, + count, + offset); +} + +ryzenai_corelib_status ryzenai_corelib_tensor_read( + ryzenai_corelib_tensor_ptr tensor, + ryzenai_corelib_data_type destination_type, + void* destination, + std::size_t count, + std::size_t offset) { + return flm::test::detail::FakeTensorRead( + tensor, + destination_type, + destination, + count, + offset); +} + +ryzenai_corelib_status ryzenai_corelib_tensor_get_byte_size( + ryzenai_corelib_tensor_ptr tensor, + std::size_t* out) { + return flm::test::detail::FakeTensorGetByteSize(tensor, out); +} + +ryzenai_corelib_status ryzenai_corelib_tensor_get_data_type( + ryzenai_corelib_tensor_ptr tensor, + ryzenai_corelib_data_type* out) { + return flm::test::detail::FakeTensorGetDataType(tensor, out); +} + +ryzenai_corelib_status ryzenai_corelib_matmul_bf16_pad_shape( + int64_t* m, + int64_t* k, + int64_t* n, + uint32_t group_size) { + return flm::test::detail::FakeMatmulPadShape( + m, + k, + n, + group_size); +} + +ryzenai_corelib_status ryzenai_corelib_matmul_bf16_weights_create_onnx( + const ryzenai_corelib_matmul_bf16_weights_desc* desc, + const ryzenai_corelib_matmul_bf16_onnx_components* components, + uint32_t threads, + ryzenai_corelib_matmul_bf16_weights_ptr* out) { + return flm::test::detail::FakeMatmulWeightsFromOnnx( + desc, + components, + threads, + out); +} + +ryzenai_corelib_status ryzenai_corelib_matmul_bf16_weights_get_data( + ryzenai_corelib_matmul_bf16_weights_ptr weights, + const void** data, + std::size_t* size) { + return flm::test::detail::FakeMatmulWeightsGetData( + weights, + data, + size); +} + +ryzenai_corelib_status ryzenai_corelib_matmul_bf16( + ryzenai_corelib_stream_ptr stream, + ryzenai_corelib_tensor_ptr input, + int64_t rows, + ryzenai_corelib_matmul_bf16_weights_ptr weights, + ryzenai_corelib_tensor_ptr output) { + return flm::test::detail::FakeMatmul( + stream, + input, + rows, + weights, + output); +} + +ryzenai_corelib_status ryzenai_corelib_ssmlp_bf16_pad_rows( + int64_t* m, + int64_t k, + int64_t n, + uint32_t group_size) { + return flm::test::detail::FakeSsmlpPadRows( + m, + k, + n, + group_size); +} + +ryzenai_corelib_status ryzenai_corelib_ssmlp_bf16_weights_create_onnx( + const ryzenai_corelib_ssmlp_bf16_weights_desc* desc, + const ryzenai_corelib_ssmlp_bf16_onnx_components* components, + uint32_t threads, + ryzenai_corelib_ssmlp_bf16_weights_ptr* out) { + return flm::test::detail::FakeSsmlpWeightsFromOnnx( + desc, + components, + threads, + out); +} + +ryzenai_corelib_status ryzenai_corelib_ssmlp_bf16_weights_get_data( + ryzenai_corelib_ssmlp_bf16_weights_ptr weights, + const void** data, + std::size_t* size) { + return flm::test::detail::FakeSsmlpWeightsGetData( + weights, + data, + size); +} + +ryzenai_corelib_status ryzenai_corelib_ssmlp_bf16( + ryzenai_corelib_stream_ptr stream, + ryzenai_corelib_tensor_ptr input, + ryzenai_corelib_tensor_ptr residual, + int64_t rows, + ryzenai_corelib_ssmlp_bf16_weights_ptr weights, + ryzenai_corelib_tensor_ptr skip_sum, + ryzenai_corelib_tensor_ptr normalized) { + return flm::test::detail::FakeSsmlp( + stream, + input, + residual, + rows, + weights, + skip_sum, + normalized); +} + +ryzenai_corelib_status ryzenai_corelib_flat_mha_bf16_pad_rows( + int64_t* m, + const ryzenai_corelib_flat_mha_bf16_desc* desc) { + return flm::test::detail::FakeFlatMhaPadRows(m, desc); +} + +ryzenai_corelib_status ryzenai_corelib_flat_mha_bf16( + ryzenai_corelib_stream_ptr stream, + const ryzenai_corelib_flat_mha_bf16_desc* desc, + ryzenai_corelib_tensor_ptr query, + ryzenai_corelib_tensor_ptr key, + int64_t rows, + int64_t position, + ryzenai_corelib_tensor_ptr cos, + ryzenai_corelib_tensor_ptr sin, + ryzenai_corelib_tensor_ptr key_cache, + ryzenai_corelib_tensor_ptr value_cache, + ryzenai_corelib_tensor_ptr output) { + return flm::test::detail::FakeFlatMha( + stream, + desc, + query, + key, + rows, + position, + cos, + sin, + key_cache, + value_cache, + output); +} + +void ryzenai_corelib_cleanup() { + flm::test::detail::FakeCleanup(); +} + +#endif + +namespace flm::test { +namespace { + +template +void* FunctionAddress(Function function) { + return reinterpret_cast(function); +} + +#define FLM_FAKE_ENTRY(symbol, replacement) \ + { \ + #symbol, FunctionAddress( \ + static_cast( \ + &detail::replacement)) \ + } + +} // namespace + +std::unordered_map CompleteCorelibResolver() { + return { + FLM_FAKE_ENTRY( + ryzenai_corelib_get_version, + FakeGetVersion), + FLM_FAKE_ENTRY( + ryzenai_corelib_status_to_string, + FakeStatusToString), + FLM_FAKE_ENTRY( + ryzenai_corelib_get_last_error_message, + FakeGetLastErrorMessage), + FLM_FAKE_ENTRY( + ryzenai_corelib_selftest_dependencies, + FakeSelftestDependencies), + FLM_FAKE_ENTRY( + ryzenai_corelib_has_device_context, + FakeHasDeviceContext), + FLM_FAKE_ENTRY( + ryzenai_corelib_object_release, + FakeObjectRelease), + FLM_FAKE_ENTRY( + ryzenai_corelib_create_stream, + FakeCreateStream), + FLM_FAKE_ENTRY( + ryzenai_corelib_stream_synchronize, + FakeStreamSynchronize), + FLM_FAKE_ENTRY( + ryzenai_corelib_create_device_tensor, + FakeCreateDeviceTensor), + FLM_FAKE_ENTRY( + ryzenai_corelib_tensor_write, + FakeTensorWrite), + FLM_FAKE_ENTRY( + ryzenai_corelib_tensor_read, + FakeTensorRead), + FLM_FAKE_ENTRY( + ryzenai_corelib_tensor_get_byte_size, + FakeTensorGetByteSize), + FLM_FAKE_ENTRY( + ryzenai_corelib_tensor_get_data_type, + FakeTensorGetDataType), + FLM_FAKE_ENTRY( + ryzenai_corelib_matmul_bf16_pad_shape, + FakeMatmulPadShape), + FLM_FAKE_ENTRY( + ryzenai_corelib_matmul_bf16_weights_create_onnx, + FakeMatmulWeightsFromOnnx), + FLM_FAKE_ENTRY( + ryzenai_corelib_matmul_bf16_weights_get_data, + FakeMatmulWeightsGetData), + FLM_FAKE_ENTRY( + ryzenai_corelib_matmul_bf16, + FakeMatmul), + FLM_FAKE_ENTRY( + ryzenai_corelib_ssmlp_bf16_pad_rows, + FakeSsmlpPadRows), + FLM_FAKE_ENTRY( + ryzenai_corelib_ssmlp_bf16_weights_create_onnx, + FakeSsmlpWeightsFromOnnx), + FLM_FAKE_ENTRY( + ryzenai_corelib_ssmlp_bf16_weights_get_data, + FakeSsmlpWeightsGetData), + FLM_FAKE_ENTRY( + ryzenai_corelib_ssmlp_bf16, + FakeSsmlp), + FLM_FAKE_ENTRY( + ryzenai_corelib_flat_mha_bf16_pad_rows, + FakeFlatMhaPadRows), + FLM_FAKE_ENTRY( + ryzenai_corelib_flat_mha_bf16, + FakeFlatMha), + FLM_FAKE_ENTRY( + ryzenai_corelib_cleanup, + FakeCleanup), + }; +} + +#undef FLM_FAKE_ENTRY + +void ResetFakeCorelib() { + detail::g_last_error.clear(); + detail::g_version_major = RYZENAI_CORELIB_VERSION_MAJOR; + detail::g_version_minor = RYZENAI_CORELIB_VERSION_MINOR; + detail::g_version_patch = RYZENAI_CORELIB_VERSION_PATCH; + detail::g_tensors.clear(); + detail::g_release_counts.clear(); + detail::g_release_count = 0; + detail::g_last_released_object = nullptr; + detail::g_selftest_status = ryzenai_corelib_status_success; + detail::g_has_device_context = true; + detail::g_cleanup_count = 0; + detail::g_events.clear(); +} + +void SetLastErrorMessage(std::string message) { + detail::g_last_error = std::move(message); +} + +void SetFakeCorelibVersion( + std::uint32_t major, + std::uint32_t minor, + std::uint32_t patch) noexcept { + detail::g_version_major = major; + detail::g_version_minor = minor; + detail::g_version_patch = patch; +} + +void SetSelftestStatus(ryzenai_corelib_status status) noexcept { + detail::g_selftest_status = status; +} + +void SetHasDeviceContext(bool value) noexcept { + detail::g_has_device_context = value; +} + +std::size_t ObjectReleaseCount() noexcept { + return detail::g_release_count; +} + +std::size_t ObjectReleaseCountFor(void* value) noexcept { + const auto found = detail::g_release_counts.find(value); + return found == detail::g_release_counts.end() ? 0 : found->second; +} + +void* LastReleasedObject() noexcept { + return detail::g_last_released_object; +} + +std::size_t CleanupCount() noexcept { + return detail::g_cleanup_count; +} + +std::vector FakeCorelibEvents() { + return detail::g_events; +} + +} // namespace flm::test diff --git a/src/test/phi4_corelib_aie4/fake_corelib.hpp b/src/test/phi4_corelib_aie4/fake_corelib.hpp new file mode 100644 index 00000000..bc030088 --- /dev/null +++ b/src/test/phi4_corelib_aie4/fake_corelib.hpp @@ -0,0 +1,30 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include + +namespace flm::test { + +std::unordered_map CompleteCorelibResolver(); + +void ResetFakeCorelib(); +void SetLastErrorMessage(std::string message); +// Drives the API-5 load-time version gate. +void SetFakeCorelibVersion( + std::uint32_t major, + std::uint32_t minor, + std::uint32_t patch) noexcept; +void SetSelftestStatus(ryzenai_corelib_status status) noexcept; +void SetHasDeviceContext(bool value) noexcept; +std::size_t ObjectReleaseCount() noexcept; +std::size_t ObjectReleaseCountFor(void* value) noexcept; +void* LastReleasedObject() noexcept; +std::size_t CleanupCount() noexcept; +std::vector FakeCorelibEvents(); + +} // namespace flm::test diff --git a/src/test/phi4_corelib_aie4/phi4_expected_tokens.json b/src/test/phi4_corelib_aie4/phi4_expected_tokens.json new file mode 100644 index 00000000..db86c51f --- /dev/null +++ b/src/test/phi4_corelib_aie4/phi4_expected_tokens.json @@ -0,0 +1,47 @@ +{ + "_comment": [ + "Route-specific expected top-1 token IDs, per design 12.4 Step 4.", + "", + "WHY THIS FILE EXISTS. Every other numeric check in this suite compares", + "FastFlow against the corelib reference driver, and both are re-run from", + "scratch each time. That catches FastFlow drifting away from the", + "reference; it cannot catch a corelib change that moves BOTH sides", + "identically, because the comparison would still agree. A committed", + "sequence is the only check here that is anchored to something outside the", + "current pair of runs.", + "", + "Do NOT regenerate these to make a failing run pass. A mismatch means the", + "model's output changed, which is either a defect or a deliberate change", + "that needs recording. Update them only with an explanation of what", + "changed and why the new sequence is correct.", + "", + "PROVENANCE. Measured on xcomedusad-43 (AMD Eng Sample 100-000001713-33_N,", + "AMD XDNA(TM) NPU driver 32.0.20214.4161) against ryzenai_corelib built", + "from pinned e5258d29b5cb979d4a538994409b90ceff6e6e7a with src/common at", + "91da5c76a66ac0fbdf46c6b1abf5685e7319caaf.", + "", + "Deliberately NOT pinned to a DLL SHA-256. MSVC RelWithDebInfo builds are", + "not reproducible -- the PDB path and signature and a build timestamp go", + "into the image -- so two builds of this exact source differ in bytes.", + "Measured: the same revision built against two different dependency", + "prefixes gave a523b238...65ecdb and bc12a285...57fd1c, both 846848 bytes.", + "A hash here would fail on every rebuild for a reason that is not a defect.", + "The per-run DLL hash still goes into each result artifact, where its job", + "is to show that two runs used the SAME binary, not that it is a", + "particular one.", + "", + "Inputs are phi4_tokens.json; the model is amd/phi-4-mini-instruct-oga-dml", + "at revision e751fb68c2cfffe6b0d32942118f75ac0a0365bb overlaid with", + "src/model_overlays/phi4-mini-it-aie4. Each sequence is the top-1 of the", + "continuation step followed by the top-1 of 16 decode steps.", + "", + "THE TWO ROUTES DIFFER, AND THAT IS CORRECT. Append reaches position 19 as", + "a 14-row prefill plus five single-row steps; re-prefill reaches it in one", + "19-row pass. Those are different kernel shapes over the same tokens, so", + "BF16 rounding differs and the sequences separate at step 6. The corelib", + "reference driver reproduces each route bit-for-bit, which is what shows", + "this is a property of the kernels rather than a FastFlow defect." + ], + "force_reprefill": [1715, 6712, 644, 1561, 326, 19766, 1349, 644, 1561, 67096, 2846, 290, 2375, 13, 730, 5024, 11], + "force_append": [1715, 6712, 644, 1561, 326, 19766, 79040, 9831, 1701, 64501, 472, 290, 7334, 18015, 316, 34257, 4194] +} diff --git a/src/test/phi4_corelib_aie4/phi4_package_fixture.hpp b/src/test/phi4_corelib_aie4/phi4_package_fixture.hpp new file mode 100644 index 00000000..8f10a7d6 --- /dev/null +++ b/src/test/phi4_corelib_aie4/phi4_package_fixture.hpp @@ -0,0 +1,519 @@ +#pragma once + +// Shared on-disk synthetic Phi-4 package fixture. Both the manifest tests +// and the hardware tests build the same package, so the builders live here +// instead of being duplicated per test translation unit. + +#include "test_support.hpp" + +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace flm::test::phi4fixture { + +using nlohmann::json; + +inline constexpr std::string_view kManifestName = + "corelib_phi4_manifest.json"; +inline constexpr std::string_view kDataFile = "z-data.bin"; +inline constexpr std::string_view kRopeFile = "z-rope.bin"; +inline constexpr std::uint64_t kDataBytes = + 200064ull * 3072ull * sizeof(std::uint16_t); +inline constexpr std::size_t kRopeRows = 4096; +inline constexpr std::size_t kRopeColumns = 64; +inline constexpr std::uint64_t kRopeBytes = + kRopeRows * kRopeColumns * sizeof(std::uint16_t); +inline constexpr std::uint64_t kRopeMappedBytes = kRopeBytes + 4096; +inline constexpr std::uint64_t kFp32ScaleOffset = 4096; +inline constexpr std::uint64_t kNormOffset = 1024 * 1024; +inline constexpr std::uint64_t kSinOffset = 2 * 1024 * 1024; + +inline constexpr std::string_view kFp32Scale = + "model.layers.0.attn.q_proj.MatMulNBits.scales"; +inline constexpr std::string_view kFp16Scale = + "model.layers.0.attn.k_proj.MatMulNBits.scales"; +inline constexpr std::string_view kFp32Norm = + "model.layers.0.post_attention_layernorm.weight"; + +class TempDirectory final { +public: + TempDirectory() { + const auto nonce = + std::chrono::steady_clock::now().time_since_epoch().count(); + path_ = std::filesystem::temp_directory_path() / + ("fastflowlm-phi4-manifest-" + + std::to_string(GetCurrentProcessId()) + "-" + + std::to_string(nonce)); + std::filesystem::create_directories(path_); + } + + ~TempDirectory() noexcept { + std::error_code error; + std::filesystem::remove_all(path_, error); + } + + TempDirectory(const TempDirectory&) = delete; + TempDirectory& operator=(const TempDirectory&) = delete; + + const std::filesystem::path& path() const noexcept { + return path_; + } + +private: + std::filesystem::path path_; +}; + +inline void CreateSparseFile( + const std::filesystem::path& path, + std::uint64_t size) { + HANDLE file = CreateFileW( + path.c_str(), + GENERIC_READ | GENERIC_WRITE, + FILE_SHARE_READ, + nullptr, + CREATE_ALWAYS, + FILE_ATTRIBUTE_NORMAL, + nullptr); + if (file == INVALID_HANDLE_VALUE) { + throw std::runtime_error("failed to create sparse test file"); + } + + DWORD ignored = 0; + DeviceIoControl( + file, + FSCTL_SET_SPARSE, + nullptr, + 0, + nullptr, + 0, + &ignored, + nullptr); + LARGE_INTEGER end{}; + end.QuadPart = static_cast(size); + const bool success = + SetFilePointerEx(file, end, nullptr, FILE_BEGIN) != FALSE && + SetEndOfFile(file) != FALSE; + CloseHandle(file); + if (!success) { + throw std::runtime_error("failed to size sparse test file"); + } +} + +template +void WriteValues( + const std::filesystem::path& path, + std::uint64_t offset, + std::span values) { + std::fstream file( + path, + std::ios::in | std::ios::out | std::ios::binary); + if (!file) { + throw std::runtime_error("failed to open synthetic data file"); + } + file.seekp(static_cast(offset)); + file.write( + reinterpret_cast(values.data()), + static_cast(values.size_bytes())); + if (!file) { + throw std::runtime_error("failed to write synthetic data"); + } +} + +inline std::uint64_t ItemSize(std::string_view dtype) { + if (dtype == "uint8") { + return 1; + } + if (dtype == "float16") { + return 2; + } + if (dtype == "float32") { + return 4; + } + if (dtype == "int64") { + return 8; + } + throw std::runtime_error("unsupported synthetic dtype"); +} + +inline std::uint64_t ByteLength( + std::string_view dtype, + const std::vector& shape) { + std::uint64_t elements = 1; + for (const std::int64_t dimension : shape) { + elements *= static_cast(dimension); + } + return elements * ItemSize(dtype); +} + +inline void AddInitializer( + json& initializers, + std::string name, + std::string dtype, + std::vector shape, + std::string role, + std::string file = std::string(kDataFile), + std::uint64_t offset = 0) { + CHECK(!initializers.contains(name)); + const std::uint64_t length = ByteLength(dtype, shape); + initializers[std::move(name)] = { + {"file", std::move(file)}, + {"offset", offset}, + {"length", length}, + {"dtype", std::move(dtype)}, + {"shape", std::move(shape)}, + {"role", std::move(role)}}; +} + +inline void AddMatMul( + json& manifest, + const std::string& name, + std::int64_t k, + std::int64_t n) { + json roles = { + {"qweight", name + ".qweight"}, + {"scales", name + ".scales"}, + {"qzeros", name + ".qzeros"}}; + manifest["weight_objects"].push_back({ + {"name", name}, + {"kind", "matmul"}, + {"descriptor", + { + {"k", k}, + {"n", n}, + {"group_size", 128}, + {"has_bias", false}, + }}, + {"roles", roles}}); + + auto& initializers = manifest["initializers"]; + AddInitializer( + initializers, + roles["qweight"].get(), + "uint8", + {n, k / 2}, + "matmul.qweight"); + + std::string scale_dtype = "float16"; + std::uint64_t scale_offset = 0; + if (name == "model.layers.0.attn.q_proj.MatMulNBits") { + scale_dtype = "float32"; + scale_offset = kFp32ScaleOffset; + } + AddInitializer( + initializers, + roles["scales"].get(), + scale_dtype, + {n, k / 128}, + "matmul.scales", + std::string(kDataFile), + scale_offset); + AddInitializer( + initializers, + roles["qzeros"].get(), + "uint8", + {n, ((k / 128) + 1) / 2}, + "matmul.qzeros"); +} + +inline void AddSsMlp(json& manifest, int layer) { + const std::string base = + "model.layers." + std::to_string(layer); + const std::string object_name = base + ".ssmlp"; + const std::string norm0 = + base + ".post_attention_layernorm.weight"; + const std::string norm1 = + layer == 31 + ? "model.layers.32.final_norm_layernorm.weight" + : "model.layers." + std::to_string(layer + 1) + + ".input_layernorm.weight"; + + json roles = { + {"norm0", norm0}, + {"norm1", norm1}, + }; + auto& initializers = manifest["initializers"]; + for (const std::string projection : {"gate", "up", "down"}) { + const std::int64_t k = projection == "down" ? 8192 : 3072; + const std::int64_t n = projection == "down" ? 3072 : 8192; + const std::string prefix = + base + ".mlp." + projection + "_proj.MatMulNBits"; + for (const std::string component : + {"qweight", "scales", "qzeros"}) { + roles[projection + "_" + component] = + prefix + "." + component; + } + AddInitializer( + initializers, + prefix + ".qweight", + "uint8", + {n, k / 2}, + "ssmlp." + projection + ".qweight"); + AddInitializer( + initializers, + prefix + ".scales", + "float16", + {n, k / 128}, + "ssmlp." + projection + ".scales"); + AddInitializer( + initializers, + prefix + ".qzeros", + "uint8", + {n, ((k / 128) + 1) / 2}, + "ssmlp." + projection + ".qzeros"); + } + + AddInitializer( + initializers, + norm0, + layer == 0 ? "float32" : "float16", + {3072}, + "ssmlp.norm0", + std::string(kDataFile), + layer == 0 ? kNormOffset : 0); + AddInitializer( + initializers, + norm1, + "float16", + {3072}, + "ssmlp.norm1"); + + manifest["weight_objects"].push_back({ + {"name", object_name}, + {"kind", "ssmlp"}, + {"descriptor", + { + {"k", 3072}, + {"n", 8192}, + {"group_size", 128}, + }}, + {"roles", std::move(roles)}}); +} + +inline json BuildManifest(std::uint64_t model_size) { + json manifest = { + {"schema_version", 1}, + {"execution_backend", "corelib_aie4"}, + {"model", + { + {"family", "phi4"}, + {"layers", 32}, + {"hidden_size", 3072}, + {"intermediate_size", 8192}, + {"num_heads", 24}, + {"kv_heads", 8}, + {"head_size", 128}, + {"vocab_size", 200064}, + {"group_size", 128}, + {"rope_dim", 96}, + {"rms_epsilon", 0.00001}, + }}, + {"backend", {{"max_seq", 4096}}}, + {"files", + { + {"model.onnx", {{"size", model_size}}}, + {std::string(kDataFile), {{"size", kDataBytes}}}, + {std::string(kRopeFile), {{"size", kRopeMappedBytes}}}, + }}, + {"initializers", json::object()}, + {"weight_objects", json::array()}, + }; + + for (int layer = 0; layer < 32; ++layer) { + const std::string base = + "model.layers." + std::to_string(layer) + ".attn."; + AddMatMul( + manifest, + base + "q_proj.MatMulNBits", + 3072, + 3072); + AddMatMul( + manifest, + base + "k_proj.MatMulNBits", + 3072, + 1024); + AddMatMul( + manifest, + base + "v_proj.MatMulNBits", + 3072, + 1024); + AddMatMul( + manifest, + base + "o_proj.MatMulNBits", + 3072, + 3072); + AddSsMlp(manifest, layer); + } + AddMatMul(manifest, "lm_head.MatMulNBits", 3072, 200064); + + AddInitializer( + manifest["initializers"], + "model.embed_tokens.weight", + "float16", + {200064, 3072}, + "embedding"); + AddInitializer( + manifest["initializers"], + "model.layers.0.input_layernorm.weight", + "float32", + {3072}, + "input_norm", + std::string(kDataFile), + kNormOffset); + AddInitializer( + manifest["initializers"], + "cos_cache", + "float16", + {4096, 64}, + "cos_cache", + std::string(kRopeFile)); + AddInitializer( + manifest["initializers"], + "sin_cache", + "float32", + {4096, 48}, + "sin_cache", + std::string(kDataFile), + kSinOffset); + + CHECK(manifest["weight_objects"].size() == 161); + CHECK(manifest["initializers"].size() == 743); + return manifest; +} + +class SyntheticPackage final { +public: + SyntheticPackage() { + const auto model_path = temp_.path() / "model.onnx"; + { + std::ofstream model(model_path, std::ios::binary); + model << "model"; + } + CreateSparseFile(temp_.path() / kDataFile, kDataBytes); + CreateSparseFile( + temp_.path() / kRopeFile, + kRopeMappedBytes); + + const std::array half_values{ + 0x3c00u, + 0xc000u}; + const std::array float_values{1.0f, -2.0f}; + WriteValues( + temp_.path() / kDataFile, + 0, + std::span(half_values)); + WriteValues( + temp_.path() / kDataFile, + kFp32ScaleOffset, + std::span(float_values)); + WriteValues( + temp_.path() / kDataFile, + kNormOffset, + std::span(float_values)); + const std::array sin_value{4.0f}; + WriteValues( + temp_.path() / kDataFile, + kSinOffset, + std::span(sin_value)); + + const std::array one{0x3c00u}; + const std::array two{0x4000u}; + const std::array three{0x4200u}; + WriteValues( + temp_.path() / kRopeFile, + 0, + std::span(one)); + WriteValues( + temp_.path() / kRopeFile, + kRopeColumns * sizeof(std::uint16_t), + std::span(two)); + WriteValues( + temp_.path() / kRopeFile, + ((kRopeRows - 1) * kRopeColumns + 47) * + sizeof(std::uint16_t), + std::span(three)); + + manifest_ = BuildManifest( + std::filesystem::file_size(model_path)); + Write(manifest_); + } + + const std::filesystem::path& path() const noexcept { + return temp_.path(); + } + + json manifest() const { + return manifest_; + } + + void Write(const json& manifest) const { + std::ofstream stream( + temp_.path() / kManifestName, + std::ios::binary | std::ios::trunc); + if (!stream) { + throw std::runtime_error("failed to write synthetic manifest"); + } + stream << manifest.dump(2) << '\n'; + } + +private: + TempDirectory temp_; + json manifest_; +}; + +class NoAccessGuard final { +public: + explicit NoAccessGuard(void* address) { + MEMORY_BASIC_INFORMATION region{}; + if ( + VirtualQuery(address, ®ion, sizeof(region)) != + sizeof(region) || + region.State != MEM_COMMIT) { + throw std::runtime_error("failed to query guard-page address"); + } + address_ = address; + if ( + VirtualProtect( + address, + 4096, + PAGE_NOACCESS, + &old_protection_) == FALSE) { + throw std::runtime_error( + "failed to protect no-access guard page"); + } + } + + ~NoAccessGuard() noexcept { + if (address_ != nullptr) { + DWORD ignored = 0; + VirtualProtect( + address_, + 4096, + old_protection_, + &ignored); + } + } + + NoAccessGuard(const NoAccessGuard&) = delete; + NoAccessGuard& operator=(const NoAccessGuard&) = delete; + +private: + void* address_ = nullptr; + DWORD old_protection_ = 0; +}; + +} // namespace flm::test::phi4fixture diff --git a/src/test/phi4_corelib_aie4/phi4_tokens.json b/src/test/phi4_corelib_aie4/phi4_tokens.json new file mode 100644 index 00000000..91fcef7a --- /dev/null +++ b/src/test/phi4_corelib_aie4/phi4_tokens.json @@ -0,0 +1,24 @@ +{ + "_comment": [ + "Explicit token IDs for the Phi-4 AIE4 hardware goldens. Committed rather", + "than generated so that phi4_expected_tokens.json means something: an", + "expected output sequence is only a golden if the input that produced it", + "is fixed too.", + "", + "Never a tokenizer. Design 12.4 wants both sides to consume the SAME", + "integers; an encoder in the loop would make a mismatch ambiguous between", + "the model and the tokenizer.", + "", + "The prefix/suffix split is what makes a continuation route meaningful.", + "force_append establishes the prefix with one prefill and then walks the", + "suffix one token at a time at successive positions; force_reprefill", + "clears and recomputes prefix+suffix in a single pass from position zero.", + "With no suffix the two routes degenerate to the same single prefill and", + "the route coverage would be vacuous.", + "", + "Rendered Phi-4 chat turn: a user question and the opening of an assistant", + "reply, so the suffix is a natural continuation rather than arbitrary IDs." + ], + "prefix": [200022, 882, 200024, 3923, 374, 279, 6864, 315, 9822, 30, 200021, 200022, 78191, 200024], + "suffix": [791, 6864, 315, 9822, 374] +} diff --git a/src/test/phi4_corelib_aie4/provision_aie4_target.ps1 b/src/test/phi4_corelib_aie4/provision_aie4_target.ps1 new file mode 100644 index 00000000..b0244926 --- /dev/null +++ b/src/test/phi4_corelib_aie4/provision_aie4_target.ps1 @@ -0,0 +1,183 @@ +# Provisions the AIE4 target with a self-contained build/runtime environment +# and builds ryzenai_corelib from a pinned revision. +# +# This exists because the identity of the corelib under test was previously +# narrative: the report said which revision was built and the artifacts said +# nothing. A recipe in a session transcript is not reproducible and cannot be +# reviewed. Running this file, and recording the SHA-256 it prints, is what +# makes "we tested e5258d2" a checkable statement. +# +# WHY A PRIVATE PREFIX. The AIE4 dependencies (DynamicDispatch, ryzen_mm, +# aie_codegen, AIEBU, cpptrace, XRT) exist on the lab machine only inside +# another user's tree. That tree is read-only to us AND it moves: its +# ryzenai_corelib.dll was rebuilt from an uncommitted WIP patch mid-task, with +# .base/.new A/B variants beside it. Building against it would make our results +# depend on someone else's working directory. So the pieces are copied out once +# into a prefix we own, and everything after that points only at our own paths. +# +# `ryzenai_corelib_get_version` reports a hard-coded 0.1.0 for every 0.x +# revision, so the version symbol cannot identify a build. The SHA-256 printed +# by this script can, which is why it is the thing worth recording: hash the +# DLL the loader actually served, not the one you believe you built. +# +# NOT byte-reproducible, and do not expect it to be. An earlier version of this +# comment claimed the same source built against this prefix and against the +# original gave byte-identical DLLs with a specific SHA-256. That was wrong and +# is retracted: measured 2026-09-02, the two are 846848 bytes each but hash to +# a523b238...65ecdb and bc12a285...57fd1c. MSVC RelWithDebInfo embeds the PDB +# path and signature and a build timestamp, so two builds of identical source +# differ in bytes. That is expected and is not evidence of different source or +# different dependencies. +# +# So the hash this script prints identifies THIS build, and its use is to show +# that two runs loaded the same binary -- not to prove which revision it came +# from. The revision is established by the checkout verification in step 2. + +[CmdletBinding()] +param( + # The read-only reference tree to copy the AIE4 dependencies out of. Only + # read from; never written to. + [string]$ReferenceInstall = 'C:\Users\akholodn\hybrid-llm\install', + + # Our own prefix. Mirrors the reference's shape so a corelib configure can + # point at one directory, which is what the reference build does. + [string]$Prefix = 'C:\Users\chiz\work\hybrid-llm\install', + + # A clean checkout of ryzenai-corelib at the pinned revision, with the + # src/common submodule initialised. + [string]$CorelibSource = 'C:\Users\chiz\work\corelib-e5258d2', + [string]$CorelibRevision = 'e5258d29b5cb979d4a538994409b90ceff6e6e7a', + [string]$CommonRevision = '91da5c76a66ac0fbdf46c6b1abf5685e7319caaf', + [string]$CorelibInstall = 'C:\Users\chiz\work\corelib-e5258d2\install-mirrored', + [string]$CorelibBuild = 'C:\Users\chiz\work\corelib-e5258d2\build-mirrored', + + # Supplies spdlog, fmt, protobuf, abseil, nlohmann_json, zlib and boost. + [string]$CondaPrefix = 'C:\Users\chiz\.conda\envs\hybrid-llm\Library', + + [string]$Cmake, + [switch]$SkipCopy +) + +$ErrorActionPreference = 'Stop' +Set-StrictMode -Version Latest + +if (-not $Cmake) { + # The `cmake` on the target's PATH is Cygwin's 3.14.5, which cannot drive a + # Visual Studio generator and is below the 3.24 the closure derivation + # needs. Prefer the one Visual Studio ships. + $Cmake = Join-Path $env:ProgramFiles ('Microsoft Visual Studio\2022\Community\Common7\IDE\' + + 'CommonExtensions\Microsoft\CMake\CMake\bin\cmake.exe') +} +if (-not (Test-Path $Cmake)) { throw "no usable cmake at $Cmake" } + +# --------------------------------------------------------------------------- +# 1. Copy the dependency subset +# --------------------------------------------------------------------------- + +if (-not $SkipCopy) { + Write-Output "=== copying AIE4 dependencies from $ReferenceInstall ===" + # A SUBSET, deliberately. The reference install is ~79 GB, almost all of it + # ONNX Runtime build output that nothing here uses. These five directories + # plus six DLLs are ~2.9 GB and are what a corelib build and its runtime + # closure actually need. + foreach ($dir in @('lib', 'include', 'share', 'xrt_package', 'aiebu')) { + $source = Join-Path $ReferenceInstall $dir + if (-not (Test-Path $source)) { continue } + # robocopy returns <8 for success; 1 and 3 mean files were copied. + & robocopy $source (Join-Path $Prefix $dir) /E /NFL /NDL /NJH /NJS /NP /MT | Out-Null + if ($LASTEXITCODE -ge 8) { throw "robocopy $dir failed ($LASTEXITCODE)" } + } + New-Item -ItemType Directory -Force -Path (Join-Path $Prefix 'bin') | Out-Null + foreach ($dll in @('dyn_dispatch_core.dll', 'dyn_bins.dll', 'ryzen_mm.dll', + 'spdlog.dll', 'fmt.dll', 'zlib.dll')) { + Copy-Item (Join-Path $ReferenceInstall "bin\$dll") ` + (Join-Path $Prefix "bin\$dll") -Force + } +} + +# A copied CMake package config that still names the source tree would defeat +# the entire exercise silently: the build would succeed and quietly depend on +# the tree we were trying to stop depending on. Fail loudly instead of +# rewriting, so a new absolute path is a decision rather than a guess. +Write-Output '=== checking copied CMake configs for foreign absolute paths ===' +$leaked = @( + Get-ChildItem -Recurse -File -Path ` + (Join-Path $Prefix 'lib\cmake'), (Join-Path $Prefix 'share'), ` + (Join-Path $Prefix 'xrt_package') -Filter *.cmake -ErrorAction SilentlyContinue | + Where-Object { (Get-Content $_.FullName -Raw) -match 'akholodn' } +) +if ($leaked.Count -gt 0) { + $leaked | ForEach-Object { Write-Output " leaked: $($_.FullName)" } + throw ("$($leaked.Count) copied CMake config(s) still reference the source " + + 'tree. Rewrite or drop those entries before building, or the build ' + + 'depends on a directory we do not control.') +} +Write-Output ' none' + +# --------------------------------------------------------------------------- +# 2. Verify the pinned source +# --------------------------------------------------------------------------- + +Write-Output '=== verifying the pinned corelib checkout ===' +$head = (& git -C $CorelibSource rev-parse HEAD).Trim() +if ($head -ne $CorelibRevision) { + throw "corelib HEAD is $head, expected the pinned $CorelibRevision" +} +$common = (& git -C (Join-Path $CorelibSource 'src\common') rev-parse HEAD).Trim() +if ($common -ne $CommonRevision) { + throw "src/common is $common, expected the pinned $CommonRevision" +} +# Tracked files only. Untracked build and install directories are expected +# here and are not source drift. +$dirty = @(& git -C $CorelibSource status --porcelain --untracked-files=no) +if ($dirty.Count -gt 0) { + $dirty | ForEach-Object { Write-Output " dirty: $_" } + throw 'the corelib checkout has modified tracked files; refusing to build' +} +Write-Output " HEAD $head, src/common $common, no modified tracked files" + +# --------------------------------------------------------------------------- +# 3. Build corelib +# --------------------------------------------------------------------------- + +Write-Output '=== configuring corelib ===' +$p = $Prefix -replace '\\', '/' +$c = $CondaPrefix -replace '\\', '/' +# spdlog_DIR and fmt_DIR are load-bearing. Without them the link fails +# LNK2019/LNK1120 on spdlog::logger::log out of +# dyn_dispatch_core.lib(logging.obj). +& $Cmake -S $CorelibSource -B $CorelibBuild -G 'Visual Studio 17 2022' -A x64 ` + "-DCMAKE_INSTALL_PREFIX=$($CorelibInstall -replace '\\','/')" ` + "-DCMAKE_PREFIX_PATH=$c;$p" ` + "-DXRT_DIR=$p/xrt_package/xrt/share/cmake/XRT" ` + "-DDynamicDispatch_DIR=$p/lib/cmake/DynamicDispatch" ` + "-Dryzen_mm_DIR=$p/lib/cmake/ryzen_mm" ` + "-Daie_codegen_DIR=$p/lib/cmake/aie_codegen" ` + "-Dcpptrace_DIR=$p/lib/cmake/cpptrace" ` + "-DAIEBU_DIR=$p/share/cmake/AIEBU" ` + "-Dspdlog_DIR=$c/lib/cmake/spdlog" ` + "-Dfmt_DIR=$c/lib/cmake/fmt" ` + "-Dabsl_DIR=$c/lib/cmake/absl" ` + "-Dnlohmann_json_DIR=$c/share/cmake/nlohmann_json" +if ($LASTEXITCODE -ne 0) { throw "corelib configure failed ($LASTEXITCODE)" } + +Write-Output '=== building corelib ===' +# RelWithDebInfo to match the DynamicDispatch package, which ships only that +# configuration. +& $Cmake --build $CorelibBuild --config RelWithDebInfo --target install -- /m +if ($LASTEXITCODE -ne 0) { throw "corelib build failed ($LASTEXITCODE)" } + +$dll = Join-Path $CorelibInstall 'bin\ryzenai_corelib.dll' +$hash = (Get-FileHash -Algorithm SHA256 -Path $dll).Hash.ToLower() +Write-Output '' +Write-Output '=== corelib under test ===' +Write-Output "path : $dll" +Write-Output "size : $((Get-Item $dll).Length) bytes" +Write-Output "sha256 : $hash" +Write-Output "source : $CorelibRevision (src/common $CommonRevision)" +Write-Output '' +Write-Output 'Next: derive the runtime closure with cmake/StageAie4Runtime.cmake' +Write-Output 'against THIS DLL, then configure the test suite with' +Write-Output 'RYZENAI_CORELIB_RUNTIME_DIR pointed at the staged directory and' +Write-Output 'run ctest with FLM_AIE4_HARDWARE=1.' +Write-Output 'Do not transcribe the closure (CLOSURE-1).' diff --git a/src/test/phi4_corelib_aie4/run_server_endpoints.ps1 b/src/test/phi4_corelib_aie4/run_server_endpoints.ps1 new file mode 100644 index 00000000..18d9d193 --- /dev/null +++ b/src/test/phi4_corelib_aie4/run_server_endpoints.ps1 @@ -0,0 +1,178 @@ +# Task 12 Step 7: the four generation endpoints, on real AIE4 hardware. +# +# Each endpoint is exercised twice: a DEFAULT request carrying no supported +# limit field, and an explicit OVER-LIMIT request that must be refused with +# HTTP 400. The default case is the interesting one. On every other backend an +# absent limit field falls back to kLegacyDefaultGenerationLimit (4096), but +# GenerationLoopLimit returns kNoExplicitGenerationLimit for AIE4, so the +# decode loop is bounded only by EOS. +# +# MEASURED CONSEQUENCE, 2026-09-02: a default /api/chat request once ran the +# decode loop to the context cap and TERMINATED THE SERVER, leaving +# {"status":3,"call":"flat_mha", +# "detail":"no token attention kernel ships for a 4096-token window", +# "phase":"flat_mha","layer":0,"rows":1,"position":4095} +# It did not reproduce on the next run, because whether the model emits EOS +# before position 4095 is data-dependent. That is why this script restarts the +# server when it dies and records the death per case rather than assuming one +# server lifetime -- an earlier version did assume it, and reported the +# remaining eight cases as connection failures against a stale response body. + +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)][string]$FlmExe, + [string]$ModelTag = 'phi4-mini-it-aie4:4b', + [int]$Port = 11434, + [string]$WorkDir = $env:TEMP +) + +$ErrorActionPreference = 'Stop' + + +$port = $Port +$tag = $ModelTag +$work = $WorkDir +$serverOut = Join-Path $work 'flm-serve.log' +$serverErr = Join-Path $work 'flm-serve.err' +$script:results = @() +$script:server = $null + +# Logging goes to the console stream, never the output stream: a PowerShell +# function returns everything written to the output stream, so a Write-Output +# used for progress becomes part of the return value. +function Say { param([string]$Text) [Console]::Out.WriteLine($Text) } + +function Stop-Flm { + if ($script:server -and -not $script:server.HasExited) { + Stop-Process -Id $script:server.Id -Force -ErrorAction SilentlyContinue + } + Get-Process flm -ErrorAction SilentlyContinue | Stop-Process -Force -ErrorAction SilentlyContinue + Start-Sleep -Seconds 4 +} + +function Start-Flm { + Stop-Flm + $script:server = Start-Process -FilePath $FlmExe ` + -ArgumentList @('serve', $tag, '-p', "$port") ` + -NoNewWindow -PassThru ` + -RedirectStandardOutput $serverOut -RedirectStandardError $serverErr + for ($i = 0; $i -lt 150; $i++) { + Start-Sleep -Seconds 4 + if ($script:server.HasExited) { + throw "flm serve exited during startup with $($script:server.ExitCode)" + } + $probe = & curl.exe -s -o NUL -w '%{http_code}' --max-time 10 "http://127.0.0.1:$port/api/tags" + if ($probe -eq '200') { return } + } + throw 'server never became ready' +} + +# `$MustContain` is not optional decoration on the 400 cases. +# +# Asserting only the status code means ANY 400 passes: malformed JSON, an +# unknown model, a field name typo in this very script. The report claimed +# the refusals carry the capacity message while nothing checked that it +# did -- the same shape as the C2 over-claim, reappearing one layer down. +# Matching the body is what makes these cases evidence that the AIE4 +# capacity rule fired, rather than evidence that something went wrong. +function Invoke-Endpoint { + param( + [string]$Name, + [string]$Path, + [string]$Body, + [int]$Expect, + # Mandatory, with no default. + # + # It defaulted to '' , which meant a case added later without it would + # silently revert to the status-only assertion that review finding 4 + # rejected -- any 400 passing, including one from a typo in this file. + # An omission has to be a parameter-binding error, not a quiet + # downgrade. + [Parameter(Mandatory = $true)] + [ValidateNotNullOrEmpty()] + [string]$MustContain + ) + if (-not $script:server -or $script:server.HasExited) { + Say ' (server not running -- restarting)' + Start-Flm + } + $bodyFile = Join-Path $work 'req.json' + $respFile = Join-Path $work 'resp.json' + # Deleted every time. Leaving it meant a failed request silently reported + # the PREVIOUS request's body, which is how eight dead-server cases looked + # like they had returned a 400. + Remove-Item $respFile -Force -ErrorAction SilentlyContinue + Set-Content -Path $bodyFile -Value $Body -Encoding utf8 + # curl.exe, not Invoke-WebRequest: Windows PowerShell 5.1 throws on non-2xx + # and makes the status code awkward to read, and the 400s are the point. + $code = & curl.exe -s -o $respFile -w '%{http_code}' ` + -X POST "http://127.0.0.1:$port$Path" ` + -H 'Content-Type: application/json' ` + --data-binary "@$bodyFile" --max-time 900 + $body = if (Test-Path $respFile) { (Get-Content $respFile -Raw) } else { '(no response body)' } + if ($null -eq $body) { $body = '(empty)' } + $body = ($body -replace '\s+', ' ').Trim() + $died = $script:server.HasExited + $exitCode = if ($died) { '0x{0:X}' -f $script:server.ExitCode } else { '' } + $bodyOk = $body -like "*$MustContain*" + $ok = ($code -eq "$Expect") -and (-not $died) -and $bodyOk + Say ("{0,-56} expect {1} got {2} {3}{4}" -f ` + $Name, $Expect, $code, $(if ($ok) { 'PASS' } else { 'FAIL' }), ` + $(if ($died) { " SERVER DIED exit=$exitCode" } else { '' })) + Say (" " + $body.Substring(0, [Math]::Min(280, $body.Length))) + if (-not $bodyOk) { + Say (" BODY MISMATCH: expected to contain '" + $MustContain + "'") + } + $script:results += [pscustomobject]@{ + Name = $Name; Ok = $ok; Code = $code; Died = $died; + ExitCode = $exitCode; BodyOk = $bodyOk } +} + +Say "=== Step 7: flm serve $tag on port $port ===" +try { + Start-Flm + Say 'server ready' + Say '' + + Invoke-Endpoint '/api/generate (default, no limit field)' '/api/generate' ` + "{`"model`":`"$tag`",`"prompt`":`"What is the capital of France?`",`"stream`":false}" 200 -MustContain '"response"' + Invoke-Endpoint '/api/generate (over-limit max_tokens)' '/api/generate' ` + "{`"model`":`"$tag`",`"prompt`":`"What is the capital of France?`",`"stream`":false,`"max_tokens`":100000}" 400 -MustContain 'exceeds the active context cap' + + Invoke-Endpoint '/api/chat (default, no limit field)' '/api/chat' ` + "{`"model`":`"$tag`",`"messages`":[{`"role`":`"user`",`"content`":`"What is the capital of France?`"}],`"stream`":false}" 200 -MustContain '"message"' + Invoke-Endpoint '/api/chat (over-limit options.num_predict)' '/api/chat' ` + "{`"model`":`"$tag`",`"messages`":[{`"role`":`"user`",`"content`":`"What is the capital of France?`"}],`"stream`":false,`"options`":{`"num_predict`":100000}}" 400 -MustContain 'exceeds the active context cap' + + Invoke-Endpoint '/v1/chat/completions (default, non-streaming)' '/v1/chat/completions' ` + "{`"model`":`"$tag`",`"messages`":[{`"role`":`"user`",`"content`":`"What is the capital of France?`"}],`"stream`":false}" 200 -MustContain '"choices"' + Invoke-Endpoint '/v1/chat/completions (default, STREAMING)' '/v1/chat/completions' ` + "{`"model`":`"$tag`",`"messages`":[{`"role`":`"user`",`"content`":`"What is the capital of France?`"}],`"stream`":true}" 200 -MustContain 'data:' + Invoke-Endpoint '/v1/chat/completions (over-limit max_tokens)' '/v1/chat/completions' ` + "{`"model`":`"$tag`",`"messages`":[{`"role`":`"user`",`"content`":`"Hi`"}],`"stream`":false,`"max_tokens`":100000}" 400 -MustContain 'exceeds the active context cap' + # The OpenAI chat endpoint accepts a SECOND spelling, parsed only when + # max_tokens is absent, so it needs its own case. + Invoke-Endpoint '/v1/chat/completions (over-limit max_completion_tokens)' '/v1/chat/completions' ` + "{`"model`":`"$tag`",`"messages`":[{`"role`":`"user`",`"content`":`"Hi`"}],`"stream`":false,`"max_completion_tokens`":100000}" 400 -MustContain 'exceeds the active context cap' + + Invoke-Endpoint '/v1/completions (default, no limit field)' '/v1/completions' ` + "{`"model`":`"$tag`",`"prompt`":`"What is the capital of France?`",`"stream`":false}" 200 -MustContain '"choices"' + Invoke-Endpoint '/v1/completions (over-limit max_tokens)' '/v1/completions' ` + "{`"model`":`"$tag`",`"prompt`":`"Hi`",`"stream`":false,`"max_tokens`":100000}" 400 -MustContain 'exceeds the active context cap' +} +finally { Stop-Flm } + +Say '' +Say '=== Step 7 summary ===' +foreach ($r in $script:results) { + Say ("{0,-56} {1}{2}" -f $r.Name, ` + $(if ($r.Ok) { 'PASS' } else { "FAIL (http $($r.Code))" }), ` + $(if ($r.Died) { " [server terminated $($r.ExitCode)]" } else { '' })) +} +$failed = @($script:results | Where-Object { -not $_.Ok }) +$died = @($script:results | Where-Object { $_.Died }) +Say "STEP7_TOTAL=$($script:results.Count) STEP7_FAILED=$($failed.Count) STEP7_SERVER_DEATHS=$($died.Count)" +Say '--- fatal records written during this run ---' +Get-ChildItem (Join-Path $env:LOCALAPPDATA 'FastFlowLM\logs\corelib-fatal-*.json') -ErrorAction SilentlyContinue | + ForEach-Object { Say (Get-Content $_.FullName -Raw) } +if ($failed.Count -gt 0) { exit 1 } else { exit 0 } diff --git a/src/test/phi4_corelib_aie4/test_corelib_api.cpp b/src/test/phi4_corelib_aie4/test_corelib_api.cpp new file mode 100644 index 00000000..900d9103 --- /dev/null +++ b/src/test/phi4_corelib_aie4/test_corelib_api.cpp @@ -0,0 +1,781 @@ +#include "fake_corelib.hpp" +#include "test_support.hpp" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using flm::corelib::CorelibApi; + +constexpr std::array kRequiredSymbols{ + "ryzenai_corelib_get_version", + "ryzenai_corelib_status_to_string", + "ryzenai_corelib_get_last_error_message", + "ryzenai_corelib_selftest_dependencies", + "ryzenai_corelib_has_device_context", + "ryzenai_corelib_object_release", + "ryzenai_corelib_create_stream", + "ryzenai_corelib_stream_synchronize", + "ryzenai_corelib_create_device_tensor", + "ryzenai_corelib_tensor_write", + "ryzenai_corelib_tensor_read", + "ryzenai_corelib_tensor_get_byte_size", + "ryzenai_corelib_tensor_get_data_type", + "ryzenai_corelib_matmul_bf16_pad_shape", + "ryzenai_corelib_matmul_bf16_weights_create_onnx", + "ryzenai_corelib_matmul_bf16_weights_get_data", + "ryzenai_corelib_matmul_bf16", + "ryzenai_corelib_ssmlp_bf16_pad_rows", + "ryzenai_corelib_ssmlp_bf16_weights_create_onnx", + "ryzenai_corelib_ssmlp_bf16_weights_get_data", + "ryzenai_corelib_ssmlp_bf16", + "ryzenai_corelib_flat_mha_bf16_pad_rows", + "ryzenai_corelib_flat_mha_bf16", + "ryzenai_corelib_cleanup", +}; + +std::shared_ptr ResolveCompleteCorelib() { + auto resolver = flm::test::CompleteCorelibResolver(); + return CorelibApi::ResolveForTest( + [resolver = std::move(resolver)](std::string_view name) mutable + -> void* { + const auto found = resolver.find(std::string(name)); + return found == resolver.end() ? nullptr : found->second; + }); +} + +std::optional ReadEnvironment(const wchar_t* name) { + std::size_t required = 0; + if (_wgetenv_s(&required, nullptr, 0, name) != 0) { + throw std::runtime_error("failed to read environment variable"); + } + if (required == 0) { + return std::nullopt; + } + + std::vector value(required); + if (_wgetenv_s(&required, value.data(), value.size(), name) != 0) { + throw std::runtime_error("failed to read environment variable"); + } + return std::wstring(value.data()); +} + +class ScopedEnvironment final { +public: + ScopedEnvironment( + std::wstring name, + std::optional value) + : name_(std::move(name)), + original_(ReadEnvironment(name_.c_str())) { + Set(value); + } + + ~ScopedEnvironment() noexcept { + _wputenv_s( + name_.c_str(), + original_.has_value() ? original_->c_str() : L""); + } + + ScopedEnvironment(const ScopedEnvironment&) = delete; + ScopedEnvironment& operator=(const ScopedEnvironment&) = delete; + +private: + void Set(const std::optional& value) { + if (_wputenv_s( + name_.c_str(), + value.has_value() ? value->c_str() : L"") != 0) { + throw std::runtime_error("failed to set environment variable"); + } + } + + std::wstring name_; + std::optional original_; +}; + +class ScopedCurrentPath final { +public: + explicit ScopedCurrentPath(const std::filesystem::path& value) + : original_(std::filesystem::current_path()) { + std::filesystem::current_path(value); + } + + ~ScopedCurrentPath() noexcept { + std::error_code error; + std::filesystem::current_path(original_, error); + } + + ScopedCurrentPath(const ScopedCurrentPath&) = delete; + ScopedCurrentPath& operator=(const ScopedCurrentPath&) = delete; + +private: + std::filesystem::path original_; +}; + +class TempDirectory final { +public: + TempDirectory() { + const auto nonce = + std::chrono::steady_clock::now().time_since_epoch().count(); + path_ = std::filesystem::temp_directory_path() / + ("fastflowlm-corelib-api-" + std::to_string(nonce)); + std::filesystem::create_directories(path_); + } + + ~TempDirectory() noexcept { + std::error_code error; + std::filesystem::remove_all(path_, error); + } + + TempDirectory(const TempDirectory&) = delete; + TempDirectory& operator=(const TempDirectory&) = delete; + + const std::filesystem::path& path() const noexcept { + return path_; + } + +private: + std::filesystem::path path_; +}; + +void Touch(const std::filesystem::path& path) { + std::ofstream file(path, std::ios::binary); + if (!file) { + throw std::runtime_error("failed to create test file"); + } +} + +void CheckGetDataIdentities( + const CorelibApi& api, + const std::unordered_map& expected) { + const auto& functions = api.functions(); + CHECK(reinterpret_cast(functions.matmul_weights_get_data) == + expected.at( + "ryzenai_corelib_matmul_bf16_weights_get_data")); + CHECK(reinterpret_cast(functions.ssmlp_weights_get_data) == + expected.at( + "ryzenai_corelib_ssmlp_bf16_weights_get_data")); +} + +void TestCompleteResolution() { + auto resolver = flm::test::CompleteCorelibResolver(); + CHECK(resolver.size() == kRequiredSymbols.size()); + for (const auto name : kRequiredSymbols) { + CHECK(resolver.contains(std::string(name))); + } + + std::vector requested; + auto api = CorelibApi::ResolveForTest( + [&resolver, &requested](std::string_view name) -> void* { + requested.emplace_back(name); + const auto found = resolver.find(std::string(name)); + return found == resolver.end() ? nullptr : found->second; + }); + + CHECK(api != nullptr); + CHECK(requested.size() == kRequiredSymbols.size()); + for (const auto name : kRequiredSymbols) { + CHECK(std::count(requested.begin(), requested.end(), name) == 1); + } + + const auto& functions = api->functions(); +#define CHECK_MEMBER_IDENTITY(member, symbol) \ + CHECK(reinterpret_cast(functions.member) == \ + resolver.at(#symbol)) + CHECK_MEMBER_IDENTITY(get_version, ryzenai_corelib_get_version); + CHECK_MEMBER_IDENTITY( + status_to_string, + ryzenai_corelib_status_to_string); + CHECK_MEMBER_IDENTITY( + get_last_error_message, + ryzenai_corelib_get_last_error_message); + CHECK_MEMBER_IDENTITY( + selftest_dependencies, + ryzenai_corelib_selftest_dependencies); + CHECK_MEMBER_IDENTITY( + has_device_context, + ryzenai_corelib_has_device_context); + CHECK_MEMBER_IDENTITY( + object_release, + ryzenai_corelib_object_release); + CHECK_MEMBER_IDENTITY(create_stream, ryzenai_corelib_create_stream); + CHECK_MEMBER_IDENTITY( + stream_synchronize, + ryzenai_corelib_stream_synchronize); + CHECK_MEMBER_IDENTITY( + create_device_tensor, + ryzenai_corelib_create_device_tensor); + CHECK_MEMBER_IDENTITY(tensor_write, ryzenai_corelib_tensor_write); + CHECK_MEMBER_IDENTITY(tensor_read, ryzenai_corelib_tensor_read); + CHECK_MEMBER_IDENTITY( + tensor_get_byte_size, + ryzenai_corelib_tensor_get_byte_size); + CHECK_MEMBER_IDENTITY( + tensor_get_data_type, + ryzenai_corelib_tensor_get_data_type); + CHECK_MEMBER_IDENTITY( + matmul_pad_shape, + ryzenai_corelib_matmul_bf16_pad_shape); + CHECK_MEMBER_IDENTITY( + matmul_weights_from_onnx, + ryzenai_corelib_matmul_bf16_weights_create_onnx); + CHECK_MEMBER_IDENTITY( + matmul_weights_get_data, + ryzenai_corelib_matmul_bf16_weights_get_data); + CHECK_MEMBER_IDENTITY(matmul, ryzenai_corelib_matmul_bf16); + CHECK_MEMBER_IDENTITY( + ssmlp_pad_rows, + ryzenai_corelib_ssmlp_bf16_pad_rows); + CHECK_MEMBER_IDENTITY( + ssmlp_weights_from_onnx, + ryzenai_corelib_ssmlp_bf16_weights_create_onnx); + CHECK_MEMBER_IDENTITY( + ssmlp_weights_get_data, + ryzenai_corelib_ssmlp_bf16_weights_get_data); + CHECK_MEMBER_IDENTITY(ssmlp, ryzenai_corelib_ssmlp_bf16); + CHECK_MEMBER_IDENTITY( + flat_mha_pad_rows, + ryzenai_corelib_flat_mha_bf16_pad_rows); + CHECK_MEMBER_IDENTITY(flat_mha, ryzenai_corelib_flat_mha_bf16); + CHECK_MEMBER_IDENTITY(cleanup, ryzenai_corelib_cleanup); +#undef CHECK_MEMBER_IDENTITY +} + +void TestMatchingVersionLoadsAndIsRecorded() { + flm::test::ResetFakeCorelib(); + auto api = ResolveCompleteCorelib(); + CHECK(api != nullptr); + const auto compiled = flm::corelib::CompiledCorelibVersion(); + CHECK(api->runtime_version().major == compiled.major); + CHECK(api->runtime_version().minor == compiled.minor); + CHECK(api->runtime_version().patch == compiled.patch); +} + +void TestMinorMismatchFailsNamingBothVersions() { + flm::test::ResetFakeCorelib(); + const auto compiled = flm::corelib::CompiledCorelibVersion(); + flm::test::SetFakeCorelibVersion( + compiled.major, + compiled.minor + 1, + compiled.patch); + + try { + (void)ResolveCompleteCorelib(); + } catch (const std::exception& error) { + const std::string_view message(error.what()); + CHECK(message.find( + flm::corelib::FormatCorelibVersion(compiled)) != + std::string_view::npos); + CHECK(message.find( + flm::corelib::FormatCorelibVersion( + flm::corelib::CorelibVersion{ + compiled.major, + compiled.minor + 1, + compiled.patch})) != std::string_view::npos); + CHECK(message.find("version") != std::string_view::npos); + flm::test::ResetFakeCorelib(); + return; + } + flm::test::ResetFakeCorelib(); + throw std::runtime_error( + "a corelib minor-version mismatch must fail the load"); +} + +void TestPatchMismatchFailsWhileCorelibIsPreOneDotZero() { + flm::test::ResetFakeCorelib(); + const auto compiled = flm::corelib::CompiledCorelibVersion(); + if (compiled.major != 0) { + return; + } + flm::test::SetFakeCorelibVersion( + compiled.major, + compiled.minor, + compiled.patch + 1); + CheckThrowsContains( + [&] { (void)ResolveCompleteCorelib(); }, + "version"); + flm::test::ResetFakeCorelib(); +} + +void TestVersionIsCheckedBeforeAnyOtherSymbolIsResolved() { + flm::test::ResetFakeCorelib(); + auto resolver = flm::test::CompleteCorelibResolver(); + std::vector requested; + const auto resolve = [&resolver, &requested]( + std::string_view name) -> void* { + requested.emplace_back(name); + const auto found = resolver.find(std::string(name)); + return found == resolver.end() ? nullptr : found->second; + }; + + (void)CorelibApi::ResolveForTest(resolve); + CHECK(!requested.empty()); + CHECK(requested.front() == "ryzenai_corelib_get_version"); + + // With an incompatible runtime the gate must stop there. Resolving the + // rest first would report a missing renamed symbol and hide the skew + // that actually caused it. + const auto compiled = flm::corelib::CompiledCorelibVersion(); + flm::test::SetFakeCorelibVersion( + compiled.major + 1, + compiled.minor, + compiled.patch); + requested.clear(); + CheckThrowsContains( + [&] { (void)CorelibApi::ResolveForTest(resolve); }, + "version"); + CHECK(requested.size() == 1); + CHECK(requested.front() == "ryzenai_corelib_get_version"); + flm::test::ResetFakeCorelib(); +} + +void TestVersionCompatibilityRule() { + using flm::corelib::CorelibVersion; + using flm::corelib::IsCorelibVersionCompatible; + + // Below 1.0 the header says the API may change in any release, so all + // three components are part of the contract. + const CorelibVersion pre{0, 1, 0}; + CHECK(IsCorelibVersionCompatible(pre, CorelibVersion{0, 1, 0})); + CHECK(!IsCorelibVersionCompatible(pre, CorelibVersion{0, 1, 1})); + CHECK(!IsCorelibVersionCompatible(pre, CorelibVersion{0, 2, 0})); + CHECK(!IsCorelibVersionCompatible(pre, CorelibVersion{1, 1, 0})); + + // From 1.0 the C ABI is additive within a major version. + const CorelibVersion stable{1, 3, 2}; + CHECK(IsCorelibVersionCompatible(stable, CorelibVersion{1, 3, 2})); + CHECK(IsCorelibVersionCompatible(stable, CorelibVersion{1, 3, 0})); + CHECK(IsCorelibVersionCompatible(stable, CorelibVersion{1, 4, 0})); + CHECK(!IsCorelibVersionCompatible(stable, CorelibVersion{1, 2, 9})); + CHECK(!IsCorelibVersionCompatible(stable, CorelibVersion{2, 3, 2})); +} + +// API-7: the wrapper takes elements, and there is no byte-taking overload +// to reach for by mistake. A count in bytes is 2x too large for a BF16 +// tensor and the fake rejects it rather than moving twice the data. +void TestElementCountsAreBoundedByTheTensorNotItsBytes() { + flm::test::ResetFakeCorelib(); + auto api = ResolveCompleteCorelib(); + + const std::array shape{4, 8}; + ryzenai_corelib_tensor_ptr raw = nullptr; + api->Check( + api->functions().create_device_tensor( + ryzenai_corelib_data_type_bf16, + shape.data(), + shape.size(), + &raw), + "ryzenai_corelib_create_device_tensor"); + flm::corelib::UniqueTensor tensor(api, raw); + + std::size_t byte_size = 0; + api->Check( + api->functions().tensor_get_byte_size(tensor.get(), &byte_size), + "ryzenai_corelib_tensor_get_byte_size"); + CHECK(byte_size == 32u * sizeof(std::uint16_t)); + + ryzenai_corelib_data_type data_type{}; + api->Check( + api->functions().tensor_get_data_type(tensor.get(), &data_type), + "ryzenai_corelib_tensor_get_data_type"); + CHECK(data_type == ryzenai_corelib_data_type_bf16); + + std::vector source(32, 0.0f); + api->WriteElements( + tensor.get(), + ryzenai_corelib_data_type_fp32, + source.data(), + 32, + 0); + api->WriteElements( + tensor.get(), + ryzenai_corelib_data_type_fp32, + source.data(), + 8, + 24); + + // The old spelling: 32 BF16 elements is 64 bytes, and 64 must fail. + CheckThrowsContains( + [&] { + api->WriteElements( + tensor.get(), + ryzenai_corelib_data_type_fp32, + source.data(), + byte_size, + 0); + }, + "ELEMENTS"); + CheckThrowsContains( + [&] { + api->ReadElements( + tensor.get(), + ryzenai_corelib_data_type_bf16, + source.data(), + 32, + 1); + }, + "ELEMENTS"); +} + +void TestTypeIdenticalGetDataSymbolsCannotBeSwapped() { + auto expected = flm::test::CompleteCorelibResolver(); + const auto matmul_name = + "ryzenai_corelib_matmul_bf16_weights_get_data"; + const auto ssmlp_name = + "ryzenai_corelib_ssmlp_bf16_weights_get_data"; + CHECK(expected.at(matmul_name) != expected.at(ssmlp_name)); + + auto api = CorelibApi::ResolveForTest( + [&expected](std::string_view name) -> void* { + return expected.at(std::string(name)); + }); + std::size_t matmul_size = 0; + std::size_t ssmlp_size = 0; + CHECK(api->functions().matmul_weights_get_data( + nullptr, + nullptr, + &matmul_size) == ryzenai_corelib_status_success); + CHECK(api->functions().ssmlp_weights_get_data( + nullptr, + nullptr, + &ssmlp_size) == ryzenai_corelib_status_success); + CHECK(matmul_size == 0x4D4D); + CHECK(ssmlp_size == 0x5353); + + auto swapped = expected; + std::swap(swapped.at(matmul_name), swapped.at(ssmlp_name)); + auto swapped_api = CorelibApi::ResolveForTest( + [&swapped](std::string_view name) -> void* { + return swapped.at(std::string(name)); + }); + CheckThrowsContains( + [&] { + CheckGetDataIdentities(*swapped_api, expected); + }, + "matmul_weights_get_data"); +} + +void TestMissingSymbolFailsAtomically() { + auto resolver = flm::test::CompleteCorelibResolver(); + resolver.erase("ryzenai_corelib_flat_mha_bf16"); + CheckThrowsContains( + [&] { + flm::corelib::CorelibApi::ResolveForTest( + [resolver](std::string_view name) mutable -> void* { + const auto found = resolver.find(std::string(name)); + return found == resolver.end() ? nullptr : found->second; + }); + }, + "ryzenai_corelib_flat_mha_bf16"); +} + +void TestErrorDetailSurvivesStatusConversion() { + flm::test::ResetFakeCorelib(); + auto api = ResolveCompleteCorelib(); + const std::string expected_detail = + "durable detail copied before the next corelib call"; + flm::test::SetLastErrorMessage(expected_detail); + + try { + api->Check( + ryzenai_corelib_status_bad_argument, + "ryzenai_corelib_test_call"); + } catch (const flm::corelib::CorelibError& error) { + CHECK(error.status == ryzenai_corelib_status_bad_argument); + CHECK(error.call == "ryzenai_corelib_test_call"); + CHECK(error.detail == expected_detail); + CHECK(std::string_view(error.what()).find( + "ryzenai_corelib_test_call") != std::string_view::npos); + CHECK(std::string_view(error.what()).find("bad argument") != + std::string_view::npos); + CHECK(std::string_view(error.what()).find(expected_detail) != + std::string_view::npos); + return; + } + throw std::runtime_error("expected CorelibError was not thrown"); +} + +void TestSuccessfulStatusDoesNotReadErrorState() { + flm::test::ResetFakeCorelib(); + auto api = ResolveCompleteCorelib(); + flm::test::SetLastErrorMessage("must remain untouched"); + + api->Check(ryzenai_corelib_status_success, "successful_call"); + + CHECK(std::string_view( + api->functions().get_last_error_message()) == + "must remain untouched"); +} + +void TestMoveConstructionReleasesExactlyOnce() { + flm::test::ResetFakeCorelib(); + auto api = ResolveCompleteCorelib(); + int storage = 0; + void* handle = &storage; + + { + flm::corelib::UniqueStream original(api, handle); + CHECK(api->live_object_count() == 1); + + flm::corelib::UniqueStream moved(std::move(original)); + CHECK(!original); + CHECK(moved.get() == handle); + CHECK(api->live_object_count() == 1); + CHECK(flm::test::ObjectReleaseCount() == 0); + } + + CHECK(flm::test::ObjectReleaseCount() == 1); + CHECK(flm::test::ObjectReleaseCountFor(handle) == 1); + CHECK(flm::test::LastReleasedObject() == handle); + CHECK(api->live_object_count() == 0); +} + +void TestMoveAssignmentReleasesEachObjectOnce() { + flm::test::ResetFakeCorelib(); + auto api = ResolveCompleteCorelib(); + int source_storage = 0; + int target_storage = 0; + void* source_handle = &source_storage; + void* target_handle = &target_storage; + + { + flm::corelib::UniqueTensor source(api, source_handle); + flm::corelib::UniqueTensor target(api, target_handle); + CHECK(api->live_object_count() == 2); + + target = std::move(source); + CHECK(!source); + CHECK(target.get() == source_handle); + CHECK(flm::test::ObjectReleaseCountFor(target_handle) == 1); + CHECK(flm::test::ObjectReleaseCountFor(source_handle) == 0); + CHECK(api->live_object_count() == 1); + } + + CHECK(flm::test::ObjectReleaseCount() == 2); + CHECK(flm::test::ObjectReleaseCountFor(target_handle) == 1); + CHECK(flm::test::ObjectReleaseCountFor(source_handle) == 1); + CHECK(api->live_object_count() == 0); +} + +void TestNullResetDoesNotRelease() { + flm::test::ResetFakeCorelib(); + auto api = ResolveCompleteCorelib(); + + flm::corelib::UniqueMatMulWeights empty; + empty.reset(); + flm::corelib::UniqueSsMlpWeights null_value(api, nullptr); + null_value.reset(); + + CHECK(flm::test::ObjectReleaseCount() == 0); + CHECK(api->live_object_count() == 0); +} + +// Task 13 Step 4. The post-warm allocation measurement rests on one property +// that `live_object_count()` cannot supply: a create followed by a release +// leaves the live count exactly where it started, so a decode loop that +// allocated and freed a device tensor per token would read as perfectly +// stable. The cumulative creation counters are what make that visible, and +// this is the test that they are cumulative rather than another live count. +void TestCreationCountsAreCumulativePerKind() { + flm::test::ResetFakeCorelib(); + auto api = ResolveCompleteCorelib(); + int storage = 0; + void* handle = &storage; + + using flm::corelib::CorelibObjectKind; + CHECK(api->creation_count(CorelibObjectKind::Tensor) == 0); + CHECK(api->weight_creation_count() == 0); + + { + flm::corelib::UniqueStream stream(api, handle); + flm::corelib::UniqueTensor first(api, handle); + flm::corelib::UniqueMatMulWeights matmul(api, handle); + flm::corelib::UniqueSsMlpWeights ssmlp(api, handle); + CHECK(api->live_object_count() == 4); + CHECK(api->creation_count(CorelibObjectKind::Stream) == 1); + CHECK(api->creation_count(CorelibObjectKind::Tensor) == 1); + CHECK(api->creation_count(CorelibObjectKind::MatMulWeights) == 1); + CHECK(api->creation_count(CorelibObjectKind::SsMlpWeights) == 1); + // Both weight kinds, because both are "a weight object" for design + // 18.7's purposes and summing one of them would miss half the model. + CHECK(api->weight_creation_count() == 2); + } + + // Everything released: live is back to zero, creations are not. + CHECK(api->live_object_count() == 0); + CHECK(api->creation_count(CorelibObjectKind::Tensor) == 1); + CHECK(api->weight_creation_count() == 2); + + // The churn case: a create/release pair inside a measurement window. + const auto tensors_before = + api->creation_count(CorelibObjectKind::Tensor); + const auto live_before = api->live_object_count(); + { + flm::corelib::UniqueTensor churn(api, handle); + } + CHECK(api->live_object_count() == live_before); + CHECK( + api->creation_count(CorelibObjectKind::Tensor) == + tensors_before + 1); + + // A null object is not a creation. Otherwise a failed create would count + // toward a stability window it never allocated in. + const auto before_null = + api->creation_count(CorelibObjectKind::Tensor); + flm::corelib::UniqueTensor null_value(api, nullptr); + CHECK( + api->creation_count(CorelibObjectKind::Tensor) == before_null); + + // Moving an object does not create a second one. + const auto before_move = + api->creation_count(CorelibObjectKind::Stream); + { + flm::corelib::UniqueStream original(api, handle); + flm::corelib::UniqueStream moved(std::move(original)); + CHECK( + api->creation_count(CorelibObjectKind::Stream) == + before_move + 1); + } + CHECK( + api->creation_count(CorelibObjectKind::Stream) == before_move + 1); +} + +void TestExplicitCorelibFileWins() { + TempDirectory temp; + const auto executable_dir = temp.path() / "bin"; + const auto explicit_file = temp.path() / "chosen-corelib.dll"; + std::filesystem::create_directories(executable_dir); + Touch(explicit_file); + ScopedEnvironment override_path{ + L"RYZENAI_CORELIB_PATH", + explicit_file.wstring()}; + + CHECK(CorelibApi::ResolveLibraryPath(executable_dir) == + explicit_file.lexically_normal()); +} + +void TestExplicitCorelibDirectoryWins() { + TempDirectory temp; + const auto executable_dir = temp.path() / "bin"; + const auto explicit_directory = temp.path() / "runtime"; + std::filesystem::create_directories(executable_dir); + std::filesystem::create_directories(explicit_directory); + ScopedEnvironment override_path{ + L"RYZENAI_CORELIB_PATH", + explicit_directory.wstring()}; + + CHECK(CorelibApi::ResolveLibraryPath(executable_dir) == + (explicit_directory / "ryzenai_corelib.dll").lexically_normal()); +} + +void TestRelativeCorelibFileOverrideIsRejected() { + TempDirectory temp; + const auto executable_dir = temp.path() / "bin"; + std::filesystem::create_directories(executable_dir); + Touch(temp.path() / "relative-corelib.dll"); + ScopedCurrentPath current_path{temp.path()}; + ScopedEnvironment override_path{ + L"RYZENAI_CORELIB_PATH", + L"relative-corelib.dll"}; + + CheckThrowsContains( + [&] { + (void)CorelibApi::ResolveLibraryPath(executable_dir); + }, + "absolute"); +} + +void TestRelativeCorelibDirectoryOverrideIsRejected() { + TempDirectory temp; + const auto executable_dir = temp.path() / "bin"; + std::filesystem::create_directories(executable_dir); + std::filesystem::create_directories(temp.path() / "relative-runtime"); + ScopedCurrentPath current_path{temp.path()}; + ScopedEnvironment override_path{ + L"RYZENAI_CORELIB_PATH", + L"relative-runtime"}; + + CheckThrowsContains( + [&] { + (void)CorelibApi::ResolveLibraryPath(executable_dir); + }, + "absolute"); +} + +void TestFallbackIgnoresCurrentDirectoryAndPath() { + TempDirectory temp; + const auto executable_dir = temp.path() / "application"; + const auto trap_directory = temp.path() / "trap"; + std::filesystem::create_directories(executable_dir); + std::filesystem::create_directories(trap_directory); + Touch(trap_directory / "ryzenai_corelib.dll"); + + ScopedEnvironment no_override{ + L"RYZENAI_CORELIB_PATH", + std::nullopt}; + ScopedEnvironment trap_path{ + L"PATH", + trap_directory.wstring()}; + ScopedCurrentPath trap_current_directory{trap_directory}; + + CHECK(CorelibApi::ResolveLibraryPath(executable_dir) == + (executable_dir / "aie4" / "ryzenai_corelib.dll") + .lexically_normal()); +} + +static_assert( + !std::is_copy_constructible_v); +static_assert( + !std::is_copy_assignable_v); +static_assert( + std::is_nothrow_move_constructible_v); +static_assert( + std::is_nothrow_move_assignable_v); + +} // namespace + +int main() { + try { + TestCompleteResolution(); + TestMatchingVersionLoadsAndIsRecorded(); + TestMinorMismatchFailsNamingBothVersions(); + TestPatchMismatchFailsWhileCorelibIsPreOneDotZero(); + TestVersionIsCheckedBeforeAnyOtherSymbolIsResolved(); + TestVersionCompatibilityRule(); + TestElementCountsAreBoundedByTheTensorNotItsBytes(); + TestTypeIdenticalGetDataSymbolsCannotBeSwapped(); + TestMissingSymbolFailsAtomically(); + TestErrorDetailSurvivesStatusConversion(); + TestSuccessfulStatusDoesNotReadErrorState(); + TestMoveConstructionReleasesExactlyOnce(); + TestMoveAssignmentReleasesEachObjectOnce(); + TestNullResetDoesNotRelease(); + TestCreationCountsAreCumulativePerKind(); + TestExplicitCorelibFileWins(); + TestExplicitCorelibDirectoryWins(); + TestRelativeCorelibFileOverrideIsRejected(); + TestRelativeCorelibDirectoryOverrideIsRejected(); + TestFallbackIgnoresCurrentDirectoryAndPath(); + std::cout << "test_corelib_api: PASS\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/src/test/phi4_corelib_aie4/test_corelib_fatal_record.cpp b/src/test/phi4_corelib_aie4/test_corelib_fatal_record.cpp new file mode 100644 index 00000000..315f4187 --- /dev/null +++ b/src/test/phi4_corelib_aie4/test_corelib_fatal_record.cpp @@ -0,0 +1,878 @@ +#include "fake_corelib.hpp" +#include "test_support.hpp" + +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using flm::corelib::CorelibApi; +using flm::corelib::CorelibRuntime; +using flm::corelib::FailureContext; +using flm::corelib::FatalRecordStore; +using flm::corelib::ProcessState; + +class TempDirectory final { +public: + TempDirectory() { + const auto nonce = + std::chrono::steady_clock::now().time_since_epoch().count(); + path_ = std::filesystem::temp_directory_path() / + ("fastflowlm-corelib-runtime-" + std::to_string(nonce)); + std::filesystem::create_directories(path_); + } + + ~TempDirectory() noexcept { + std::error_code error; + std::filesystem::remove_all(path_, error); + } + + TempDirectory(const TempDirectory&) = delete; + TempDirectory& operator=(const TempDirectory&) = delete; + + const std::filesystem::path& path() const noexcept { + return path_; + } + +private: + std::filesystem::path path_; +}; + +class ScopedHandle final { +public: + ScopedHandle( + const std::filesystem::path& path, + DWORD share_mode) + : handle_(CreateFileW( + path.c_str(), + GENERIC_READ, + share_mode, + nullptr, + OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, + nullptr)) { + if (handle_ == INVALID_HANDLE_VALUE) { + throw std::runtime_error( + "failed to lock test record (error " + + std::to_string(GetLastError()) + ")"); + } + } + + ~ScopedHandle() noexcept { + CloseHandle(handle_); + } + + ScopedHandle(const ScopedHandle&) = delete; + ScopedHandle& operator=(const ScopedHandle&) = delete; + +private: + HANDLE handle_; +}; + +std::optional ReadEnvironment(const wchar_t* name) { + std::size_t required = 0; + if (_wgetenv_s(&required, nullptr, 0, name) != 0) { + throw std::runtime_error("failed to read environment variable"); + } + if (required == 0) { + return std::nullopt; + } + + std::vector value(required); + if (_wgetenv_s(&required, value.data(), value.size(), name) != 0) { + throw std::runtime_error("failed to read environment variable"); + } + return std::wstring(value.data()); +} + +class ScopedEnvironment final { +public: + ScopedEnvironment( + std::wstring name, + std::optional value) + : name_(std::move(name)), + original_(ReadEnvironment(name_.c_str())) { + if (_wputenv_s( + name_.c_str(), + value.has_value() ? value->c_str() : L"") != 0) { + throw std::runtime_error("failed to set environment variable"); + } + } + + ~ScopedEnvironment() noexcept { + _wputenv_s( + name_.c_str(), + original_.has_value() ? original_->c_str() : L""); + } + + ScopedEnvironment(const ScopedEnvironment&) = delete; + ScopedEnvironment& operator=(const ScopedEnvironment&) = delete; + +private: + std::wstring name_; + std::optional original_; +}; + +std::chrono::system_clock::time_point KnownStartTime() { + using namespace std::chrono; + return sys_days{year{2026} / August / day{31}} + hours{17}; +} + +std::shared_ptr ResolveCompleteCorelib() { + auto resolver = flm::test::CompleteCorelibResolver(); + return CorelibApi::ResolveForTest( + [resolver = std::move(resolver)](std::string_view name) mutable + -> void* { + const auto found = resolver.find(std::string(name)); + return found == resolver.end() ? nullptr : found->second; + }); +} + +void WriteText( + const std::filesystem::path& path, + std::string_view contents) { + std::ofstream output(path, std::ios::binary); + if (!output) { + throw std::runtime_error("failed to create test record"); + } + output.write( + contents.data(), + static_cast(contents.size())); + if (!output) { + throw std::runtime_error("failed to write test record"); + } +} + +std::string ReadText(const std::filesystem::path& path) { + std::ifstream input(path, std::ios::binary); + if (!input) { + throw std::runtime_error("failed to open test record"); + } + return std::string( + std::istreambuf_iterator(input), + std::istreambuf_iterator()); +} + +std::vector MatchingFiles( + const std::filesystem::path& root, + std::string_view prefix) { + std::vector paths; + std::error_code error; + for (std::filesystem::directory_iterator iterator(root, error), end; + !error && iterator != end; + iterator.increment(error)) { + const auto name = iterator->path().filename().string(); + if (name.starts_with(prefix)) { + paths.push_back(iterator->path()); + } + } + if (error) { + throw std::filesystem::filesystem_error( + "failed to enumerate test records", + root, + error); + } + std::sort(paths.begin(), paths.end()); + return paths; +} + +std::filesystem::path CurrentExecutablePath() { + std::wstring buffer(32768, L'\0'); + const DWORD size = GetModuleFileNameW( + nullptr, + buffer.data(), + static_cast(buffer.size())); + if (size == 0 || size == buffer.size()) { + throw std::runtime_error("GetModuleFileNameW failed"); + } + buffer.resize(size); + return std::filesystem::path(std::move(buffer)); +} + +void CreateReadOnlyDirectory(const std::filesystem::path& path) { + PSECURITY_DESCRIPTOR descriptor = nullptr; + if (!ConvertStringSecurityDescriptorToSecurityDescriptorW( + L"D:P(D;;GW;;;WD)(A;;GRGXSD;;;WD)", + SDDL_REVISION_1, + &descriptor, + nullptr)) { + throw std::runtime_error( + "failed to construct read-only directory ACL"); + } + SECURITY_ATTRIBUTES attributes{ + sizeof(SECURITY_ATTRIBUTES), + descriptor, + FALSE}; + const BOOL created = CreateDirectoryW(path.c_str(), &attributes); + const DWORD error = created ? ERROR_SUCCESS : GetLastError(); + LocalFree(descriptor); + if (!created) { + throw std::runtime_error( + "failed to create read-only test directory (error " + + std::to_string(error) + ")"); + } +} + +void TestPendingNamesUseStartTimeAndPid() { + TempDirectory temp; + const auto start = KnownStartTime(); + auto probe = [](DWORD) + -> std::optional { + return std::nullopt; + }; + FatalRecordStore store_a(temp.path(), 1001, start, probe); + FatalRecordStore store_b(temp.path(), 1002, start, probe); + + store_a.Prepare(); + store_b.Prepare(); + + CHECK(store_a.pending_path().filename() == + "pending-corelib-fatal-20260831T1700000000000Z-1001.tmp"); + CHECK(store_b.pending_path().filename() == + "pending-corelib-fatal-20260831T1700000000000Z-1002.tmp"); + CHECK(store_a.pending_path() != store_b.pending_path()); + CHECK(std::filesystem::exists(store_a.pending_path())); + CHECK(std::filesystem::exists(store_b.pending_path())); + + const auto pending_a = store_a.pending_path(); + const auto pending_b = store_b.pending_path(); + store_a.RemoveUnusedPending(); + store_b.RemoveUnusedPending(); + CHECK(!std::filesystem::exists(pending_a)); + CHECK(!std::filesystem::exists(pending_b)); +} + +void TestPersistWritesCompleteUniqueRecords() { + TempDirectory temp; + const auto start = KnownStartTime(); + auto probe = [](DWORD) + -> std::optional { + return std::nullopt; + }; + FatalRecordStore store_a(temp.path(), 1001, start, probe); + FatalRecordStore store_b(temp.path(), 1002, start, probe); + store_a.Prepare(); + store_b.Prepare(); + + const FailureContext failure{ + ryzenai_corelib_status_bad_argument, + "matmul_q", + "quoted \"detail\"\nnext line", + "qkv", + 7, + 32, + 128}; + const auto final_a = store_a.Persist(failure); + const auto final_b = store_b.Persist(failure); + + CHECK(final_a != final_b); + CHECK(final_a.filename().string().starts_with("corelib-fatal-")); + CHECK(final_a.filename().string().ends_with("-1001.json")); + CHECK(final_b.filename().string().ends_with("-1002.json")); + CHECK(std::filesystem::exists(final_a)); + CHECK(std::filesystem::exists(final_b)); + CHECK(!std::filesystem::exists(store_a.pending_path())); + CHECK(!std::filesystem::exists(store_b.pending_path())); + + const auto record = ReadText(final_a); + CHECK(record.find("\"status\":2") != std::string::npos); + CHECK(record.find("\"call\":\"matmul_q\"") != std::string::npos); + CHECK(record.find("quoted \\\"detail\\\"\\nnext line") != + std::string::npos); + CHECK(record.find("\"phase\":\"qkv\"") != std::string::npos); + CHECK(record.find("\"layer\":7") != std::string::npos); + CHECK(record.find("\"rows\":32") != std::string::npos); + CHECK(record.find("\"position\":128") != std::string::npos); + CHECK(record.find( + "\"process_start_utc\":\"20260831T1700000000000Z\"") != + std::string::npos); + CHECK(record.find("\"failure_utc\":\"") != std::string::npos); + CHECK(record.find("\"pid\":1001") != std::string::npos); +} + +void TestDrainOrdersFinalRecordsAndRemovesAfterEmission() { + TempDirectory temp; + const auto first = + temp.path() / "corelib-fatal-20260831T1700000000000Z-1001.json"; + const auto same_time_second = + temp.path() / "corelib-fatal-20260831T1700000000000Z-1002.json"; + const auto later = + temp.path() / "corelib-fatal-20260831T1800000000000Z-1000.json"; + WriteText(later, "later\n"); + WriteText(same_time_second, "same-time-second\n"); + WriteText(first, "first\n"); + std::size_t probe_calls = 0; + std::ostringstream output; + + const auto records = FatalRecordStore::DrainPriorRecords( + temp.path(), + [&probe_calls](DWORD) + -> std::optional { + ++probe_calls; + return std::nullopt; + }, + output); + + const std::vector expected{ + "first\n", + "same-time-second\n", + "later\n"}; + CHECK(records == expected); + CHECK(output.str() == "first\nsame-time-second\nlater\n"); + CHECK(probe_calls == 0); + CHECK(!std::filesystem::exists(first)); + CHECK(!std::filesystem::exists(same_time_second)); + CHECK(!std::filesystem::exists(later)); +} + +void TestDrainContinuesPastUnreadableAndUnremovableRecords() { + TempDirectory temp; + const auto unreadable = + temp.path() / "corelib-fatal-20260831T1700000000000Z-1001.json"; + const auto unremovable = + temp.path() / "corelib-fatal-20260831T1800000000000Z-1002.json"; + const auto successful = + temp.path() / "corelib-fatal-20260831T1900000000000Z-1003.json"; + WriteText(unreadable, "unreadable\n"); + WriteText(unremovable, "unremovable\n"); + WriteText(successful, "successful\n"); + ScopedHandle deny_read(unreadable, 0); + ScopedHandle deny_delete( + unremovable, + FILE_SHARE_READ | FILE_SHARE_WRITE); + std::ostringstream output; + + const auto records = FatalRecordStore::DrainPriorRecords( + temp.path(), + {}, + output); + + const std::vector expected{ + "unremovable\n", + "successful\n"}; + CHECK(records == expected); + CHECK(output.str().find("warning") != std::string::npos); + CHECK(output.str().find("read") != std::string::npos); + CHECK(output.str().find(unreadable.filename().string()) != + std::string::npos); + CHECK(output.str().find("remove") != std::string::npos); + CHECK(output.str().find(unremovable.filename().string()) != + std::string::npos); + CHECK(output.str().find("unremovable\n") != std::string::npos); + CHECK(output.str().find("successful\n") != std::string::npos); + CHECK(std::filesystem::exists(unreadable)); + CHECK(std::filesystem::exists(unremovable)); + CHECK(!std::filesystem::exists(successful)); +} + +void TestDrainWarnsAndReturnsForInvalidRoot() { + TempDirectory temp; + const auto root_file = temp.path() / "not-a-directory"; + WriteText(root_file, "not a directory"); + std::ostringstream output; + + const auto records = FatalRecordStore::DrainPriorRecords( + root_file, + {}, + output); + + CHECK(records.empty()); + CHECK(output.str().find("warning") != std::string::npos); + CHECK(output.str().find(root_file.string()) != std::string::npos); +} + +void TestDrainPreservesLivePendingRecord() { + TempDirectory temp; + const auto pending = + temp.path() / + "pending-corelib-fatal-20260831T1700000000000Z-1001.tmp"; + WriteText(pending, "pending"); + std::ostringstream output; + + const auto records = FatalRecordStore::DrainPriorRecords( + temp.path(), + [](DWORD pid) + -> std::optional { + CHECK(pid == 1001); + return KnownStartTime(); + }, + output); + + CHECK(records.empty()); + CHECK(output.str().empty()); + CHECK(std::filesystem::exists(pending)); +} + +void TestDrainPreservesPendingWhenProbeFails() { + TempDirectory temp; + const auto pending = + temp.path() / + "pending-corelib-fatal-20260831T1700000000000Z-1001.tmp"; + WriteText(pending, "pending"); + std::ostringstream output; + + const auto records = FatalRecordStore::DrainPriorRecords( + temp.path(), + [](DWORD) + -> std::optional { + return std::nullopt; + }, + output); + + CHECK(records.empty()); + CHECK(output.str().empty()); + CHECK(std::filesystem::exists(pending)); +} + +void TestDrainReportsAndRemovesStalePendingRecord() { + TempDirectory temp; + const auto pending = + temp.path() / + "pending-corelib-fatal-20260831T1700000000000Z-1001.tmp"; + WriteText(pending, "pending"); + std::ostringstream output; + + const auto records = FatalRecordStore::DrainPriorRecords( + temp.path(), + [](DWORD) + -> std::optional { + return KnownStartTime() + std::chrono::seconds{1}; + }, + output); + + CHECK(records.size() == 1); + CHECK(records.front().find("incomplete corelib fatal record") != + std::string::npos); + CHECK(records.front().find(pending.filename().string()) != + std::string::npos); + CHECK(output.str() == records.front()); + CHECK(!std::filesystem::exists(pending)); +} + +void TestPrepareRejectsUnwritableRoot() { + TempDirectory temp; + const auto read_only = temp.path() / "read-only"; + CreateReadOnlyDirectory(read_only); + FatalRecordStore records( + read_only, + 1001, + KnownStartTime(), + [](DWORD) + -> std::optional { + return std::nullopt; + }); + + CheckThrowsContains( + [&] { + records.Prepare(); + }, + "fatal record"); +} + +void TestUnusedPendingIsRemovedByDestructor() { + TempDirectory temp; + std::filesystem::path pending; + { + FatalRecordStore records( + temp.path(), + 1001, + KnownStartTime(), + [](DWORD) + -> std::optional { + return std::nullopt; + }); + records.Prepare(); + pending = records.pending_path(); + CHECK(std::filesystem::exists(pending)); + } + CHECK(!std::filesystem::exists(pending)); +} + +FatalRecordStore MakeTestRecords( + const std::filesystem::path& root, + DWORD pid = 1001) { + return FatalRecordStore( + root, + pid, + KnownStartTime(), + [](DWORD) + -> std::optional { + return std::nullopt; + }); +} + +void TestRuntimePublishesHealthyAndShutsDownOnce() { + TempDirectory temp; + flm::test::ResetFakeCorelib(); + auto api = ResolveCompleteCorelib(); + bool terminated = false; + auto runtime = CorelibRuntime::Create( + api, + MakeTestRecords(temp.path()), + [&terminated](unsigned int) { + terminated = true; + throw std::runtime_error("unexpected termination"); + }); + + CHECK(runtime->state() == ProcessState::Healthy); + CHECK(runtime->admission_open()); + CHECK(runtime->api().get() == api.get()); + { + auto lease = runtime->AcquireExecution(); + CHECK(lease.owns_lock()); + } + + runtime->ShutdownHealthy(); + runtime->ShutdownHealthy(); + CHECK(runtime->state() == ProcessState::Shutdown); + CHECK(!runtime->admission_open()); + CHECK(flm::test::CleanupCount() == 1); + CHECK(!terminated); + CHECK(MatchingFiles( + temp.path(), + "pending-corelib-fatal-").empty()); +} + +void TestHealthyShutdownClosesAdmissionBeforeWaiting() { + TempDirectory temp; + flm::test::ResetFakeCorelib(); + auto runtime = CorelibRuntime::Create( + ResolveCompleteCorelib(), + MakeTestRecords(temp.path()), + [](unsigned int) { + throw std::runtime_error("unexpected termination"); + }); + auto active_execution = runtime->AcquireExecution(); + std::atomic shutdown_started = false; + std::exception_ptr shutdown_error; + std::thread shutdown([&] { + shutdown_started.store(true, std::memory_order_release); + try { + runtime->ShutdownHealthy(); + } catch (...) { + shutdown_error = std::current_exception(); + } + }); + while (!shutdown_started.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + + const auto deadline = + std::chrono::steady_clock::now() + std::chrono::seconds{1}; + while (runtime->admission_open() && + std::chrono::steady_clock::now() < deadline) { + std::this_thread::yield(); + } + const bool closed_before_active_execution_finished = + !runtime->admission_open(); + active_execution.unlock(); + shutdown.join(); + + if (shutdown_error) { + std::rethrow_exception(shutdown_error); + } + CHECK(closed_before_active_execution_finished); + CHECK(runtime->state() == ProcessState::Shutdown); +} + +void TestInitializationFailureCleansUpAndRemovesPending() { + TempDirectory temp; + flm::test::ResetFakeCorelib(); + flm::test::SetLastErrorMessage("dependency probe failed"); + flm::test::SetSelftestStatus(ryzenai_corelib_status_failure); + auto api = ResolveCompleteCorelib(); + + CheckThrowsContains( + [&] { + (void)CorelibRuntime::Create( + api, + MakeTestRecords(temp.path()), + [](unsigned int) { + throw std::runtime_error("unexpected termination"); + }); + }, + "ryzenai_corelib_selftest_dependencies"); + + CHECK(flm::test::CleanupCount() == 1); + CHECK(MatchingFiles( + temp.path(), + "pending-corelib-fatal-").empty()); +} + +void TestMissingDeviceContextCleansUpAndRemovesPending() { + TempDirectory temp; + flm::test::ResetFakeCorelib(); + flm::test::SetHasDeviceContext(false); + auto api = ResolveCompleteCorelib(); + + CheckThrowsContains( + [&] { + (void)CorelibRuntime::Create( + api, + MakeTestRecords(temp.path()), + [](unsigned int) { + throw std::runtime_error("unexpected termination"); + }); + }, + "device context"); + + CHECK(flm::test::CleanupCount() == 1); + CHECK(MatchingFiles( + temp.path(), + "pending-corelib-fatal-").empty()); +} + +void TestHealthyCleanupFollowsLastObjectRelease() { + TempDirectory temp; + flm::test::ResetFakeCorelib(); + auto api = ResolveCompleteCorelib(); + auto runtime = CorelibRuntime::Create( + api, + MakeTestRecords(temp.path()), + [](unsigned int) { + throw std::runtime_error("unexpected termination"); + }); + int storage = 0; + flm::corelib::UniqueTensor tensor(api, &storage); + + CheckThrowsContains( + [&] { + runtime->ShutdownHealthy(); + }, + "live corelib"); + CHECK(runtime->state() == ProcessState::Healthy); + CHECK(flm::test::CleanupCount() == 0); + + tensor.reset(); + runtime->ShutdownHealthy(); + + const std::vector expected{"release", "cleanup"}; + CHECK(flm::test::FakeCorelibEvents() == expected); + CHECK(flm::test::CleanupCount() == 1); +} + +struct TerminationIntercept final {}; + +void TestFailedShutdownRollbackPreservesConcurrentTermination() { + TempDirectory temp; + flm::test::ResetFakeCorelib(); + auto api = ResolveCompleteCorelib(); + auto runtime = CorelibRuntime::Create( + api, + MakeTestRecords(temp.path()), + [](unsigned int) { + throw TerminationIntercept{}; + }); + int storage = 0; + flm::corelib::UniqueTensor tensor(api, &storage); + std::atomic rollback_reached = false; + std::atomic release_rollback = false; + runtime->SetBeforeLiveObjectRollbackForTest([&] { + rollback_reached.store(true, std::memory_order_release); + while (!release_rollback.load(std::memory_order_acquire)) { + std::this_thread::yield(); + } + }); + + std::exception_ptr shutdown_error; + std::thread shutdown([&] { + try { + runtime->ShutdownHealthy(); + } catch (...) { + shutdown_error = std::current_exception(); + } + }); + const auto rollback_deadline = + std::chrono::steady_clock::now() + std::chrono::seconds{1}; + while (!rollback_reached.load(std::memory_order_acquire) && + std::chrono::steady_clock::now() < rollback_deadline) { + std::this_thread::yield(); + } + const bool reached_rollback = + rollback_reached.load(std::memory_order_acquire); + if (!reached_rollback) { + release_rollback.store(true, std::memory_order_release); + shutdown.join(); + CHECK(reached_rollback); + } + + const FailureContext failure{ + ryzenai_corelib_status_failure, + "ryzenai_corelib_matmul_bf16", + "concurrent terminal failure", + "qkv", + 12, + 4, + 256}; + std::exception_ptr termination_error; + std::thread termination([&] { + try { + runtime->TerminateAfterFailure(failure); + } catch (const TerminationIntercept&) { + } catch (...) { + termination_error = std::current_exception(); + } + }); + termination.join(); + CHECK(!termination_error); + CHECK(runtime->state() == ProcessState::Terminating); + + release_rollback.store(true, std::memory_order_release); + shutdown.join(); + + CHECK(shutdown_error); + CheckThrowsContains( + [&] { + std::rethrow_exception(shutdown_error); + }, + "live corelib"); + CHECK(runtime->state() == ProcessState::Terminating); + CHECK(!runtime->admission_open()); + tensor.reset(); +} + +void TestTerminationClosesAdmissionBeforeTerminator() { + TempDirectory temp; + flm::test::ResetFakeCorelib(); + auto api = ResolveCompleteCorelib(); + std::shared_ptr runtime; + bool terminator_called = false; + unsigned int termination_code = 0; + runtime = CorelibRuntime::Create( + api, + MakeTestRecords(temp.path()), + [&](unsigned int code) { + terminator_called = true; + termination_code = code; + CHECK(runtime->state() == ProcessState::Terminating); + CHECK(!runtime->admission_open()); + throw TerminationIntercept{}; + }); + const FailureContext failure{ + ryzenai_corelib_status_failure, + "ryzenai_corelib_matmul_bf16", + "dispatch failed", + "qkv", + 12, + 4, + 256}; + + try { + runtime->TerminateAfterFailure(failure); + } catch (const TerminationIntercept&) { + } + + CHECK(terminator_called); + CHECK(termination_code == 0xE0040001u); + CHECK(runtime->state() == ProcessState::Terminating); + CHECK(!runtime->admission_open()); + CHECK(flm::test::CleanupCount() == 0); + const auto records = MatchingFiles(temp.path(), "corelib-fatal-"); + CHECK(records.size() == 1); + const auto contents = ReadText(records.front()); + CHECK(contents.find("\"call\":\"ryzenai_corelib_matmul_bf16\"") != + std::string::npos); + CHECK(contents.find("\"phase\":\"qkv\"") != std::string::npos); + CHECK(contents.find("\"layer\":12") != std::string::npos); + CHECK(contents.find("\"rows\":4") != std::string::npos); + CHECK(contents.find("\"position\":256") != std::string::npos); +} + +void TestStepSubmissionStateCrossesIrrevocableBoundary() { + flm::corelib::StepSubmissionState submission; + CHECK(!submission.irrevocable()); + submission.MarkSuccessfulSubmit(); + CHECK(submission.irrevocable()); +} + +void TestGetOrCreateKeepsOneProcessRuntimeUntilExplicitShutdown() { + TempDirectory temp; + const auto fake_dll = + CurrentExecutablePath().parent_path() / "fake_ryzenai_corelib.dll"; + const auto cleanup_marker = temp.path() / "cleanup-marker.txt"; + CHECK(std::filesystem::exists(fake_dll)); + ScopedEnvironment corelib_path{ + L"RYZENAI_CORELIB_PATH", + fake_dll.wstring()}; + ScopedEnvironment marker_path{ + L"FLM_FAKE_CORELIB_CLEANUP_MARKER", + cleanup_marker.wstring()}; + + auto first = CorelibRuntime::GetOrCreate( + CurrentExecutablePath().parent_path()); + const auto first_runtime = first.get(); + const auto first_api = first->api().get(); + std::weak_ptr runtime_weak = first; + std::weak_ptr api_weak = first->api(); + first.reset(); + + CHECK(!runtime_weak.expired()); + auto second = CorelibRuntime::GetOrCreate( + CurrentExecutablePath().parent_path()); + CHECK(second.get() == first_runtime); + CHECK(second->api().get() == first_api); + CHECK(!std::filesystem::exists(cleanup_marker)); + second.reset(); + + CorelibRuntime::ShutdownProcess(); + + CHECK(runtime_weak.expired()); + CHECK(api_weak.expired()); + CHECK(ReadText(cleanup_marker) == "cleanup\n"); +} + +} // namespace + +int main() { + try { + TestPendingNamesUseStartTimeAndPid(); + TestPersistWritesCompleteUniqueRecords(); + TestDrainOrdersFinalRecordsAndRemovesAfterEmission(); + TestDrainContinuesPastUnreadableAndUnremovableRecords(); + TestDrainWarnsAndReturnsForInvalidRoot(); + TestDrainPreservesLivePendingRecord(); + TestDrainPreservesPendingWhenProbeFails(); + TestDrainReportsAndRemovesStalePendingRecord(); + TestPrepareRejectsUnwritableRoot(); + TestUnusedPendingIsRemovedByDestructor(); + TestRuntimePublishesHealthyAndShutsDownOnce(); + TestHealthyShutdownClosesAdmissionBeforeWaiting(); + TestInitializationFailureCleansUpAndRemovesPending(); + TestMissingDeviceContextCleansUpAndRemovesPending(); + TestHealthyCleanupFollowsLastObjectRelease(); + TestFailedShutdownRollbackPreservesConcurrentTermination(); + TestTerminationClosesAdmissionBeforeTerminator(); + TestStepSubmissionStateCrossesIrrevocableBoundary(); + TestGetOrCreateKeepsOneProcessRuntimeUntilExplicitShutdown(); + std::cout << "test_corelib_fatal_record: PASS\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } catch (...) { + std::cerr << "unexpected non-standard exception\n"; + return 1; + } +} diff --git a/src/test/phi4_corelib_aie4/test_fatal_child.cpp b/src/test/phi4_corelib_aie4/test_fatal_child.cpp new file mode 100644 index 00000000..a94fd496 --- /dev/null +++ b/src/test/phi4_corelib_aie4/test_fatal_child.cpp @@ -0,0 +1,1034 @@ +// Task 12 Step 6: the terminal-failure path, in a real process that really +// dies. +// +// Everything before this ran the failure policy in-process against an +// intercepted terminator, so nothing had ever confirmed that a FastFlow process +// actually exits with 0xE0040001, that the detailed record actually reaches +// %LOCALAPPDATA%\FastFlowLM\logs, or that a parent can actually read it back. +// Those are the three things this file establishes, by forking itself. +// +// The record root is deliberately the REAL one. FatalRecordStore resolves it +// with SHGetKnownFolderPath rather than from the environment, so it cannot be +// redirected -- and redirecting it would remove the very precondition design +// Section 12.1 asks about, which is whether that directory is writable on the +// target. +// +// What is injected and what is not, stated plainly because it bounds the +// claim: the FAILURE is real -- a genuine call into the real corelib that the +// real library genuinely refuses, carrying its own status code and its own +// message. The SUBMISSION STATE is synthetic. There is no way to make the +// device fail after a successful submit on demand without corrupting the +// device, so the irrevocable-boundary cases set that flag directly and then +// let the unmodified policy decide. The policy, the record, the exit code and +// the parent-side drain are all the production ones. + +#include "test_support.hpp" + +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +namespace constants = flm::phi4::constants; + +using flm::corelib::CorelibApi; +using flm::corelib::CorelibError; +using flm::corelib::CorelibRuntime; +using flm::corelib::FailureContext; +using flm::corelib::FatalRecordStore; +using flm::corelib::StepSubmissionState; + +constexpr unsigned int kFatalExitCode = 0xE0040001u; + +// Markers are FILES, not stdout lines, and each one is written, flushed and +// closed before the child proceeds. +// +// The first version of this test looked for the absence of a `std::cout` line. +// That check could not fail: `TerminateProcess` discards buffered stdout, so a +// child that had run cleanup and printed the marker would still show no marker +// once it was killed. "Absent" and "never written" were indistinguishable, and +// the assertion passed for the wrong reason. +// +// A closed file survives `TerminateProcess`, so absence now means the child +// really did not get there. `kReachedPolicy` is the positive control that +// makes the absence meaningful: without it, a child that died during startup +// would satisfy "no cleanup marker" just as well as one that was correctly +// terminated mid-step. +constexpr std::wstring_view kReachedPolicyMarker = L"reached-policy.marker"; +constexpr std::wstring_view kCleanupMarker = L"cleanup-ok.marker"; +constexpr std::wstring_view kSurvivedMarker = L"survived.marker"; + +// Where the child drops its markers, passed on the command line so the parent +// and child cannot disagree about it. +std::filesystem::path g_marker_dir; + +void WriteMarker(std::wstring_view name, std::string_view contents) { + const auto path = g_marker_dir / std::filesystem::path(name); + std::ofstream stream(path, std::ios::binary | std::ios::trunc); + if (!stream) { + throw std::runtime_error( + "child could not write marker " + path.string()); + } + stream << contents; + stream.flush(); + stream.close(); + if (!stream) { + throw std::runtime_error( + "child could not flush marker " + path.string()); + } +} + +#if !defined(FLM_REAL_CORELIB_RUNTIME_DIR) +#define FLM_REAL_CORELIB_RUNTIME_DIR "" +#endif + +#if !defined(FLM_REAL_CORELIB_EXTRA_DLL_DIRS) +#define FLM_REAL_CORELIB_EXTRA_DLL_DIRS "" +#endif + +void AddExtraDllDirectories(std::string_view directories) { + std::size_t start = 0; + while (start <= directories.size()) { + const std::size_t end = directories.find(';', start); + const std::string_view entry = directories.substr( + start, + end == std::string_view::npos ? std::string_view::npos + : end - start); + if (!entry.empty()) { + const std::filesystem::path directory(entry); + if ( + !std::filesystem::exists(directory) || + AddDllDirectory(directory.c_str()) == nullptr) { + throw std::runtime_error( + "cannot add corelib DLL directory " + + directory.string()); + } + } + if (end == std::string_view::npos) { + break; + } + start = end + 1; + } +} + +std::filesystem::path CorelibLibraryPath() { + const std::string runtime_dir(FLM_REAL_CORELIB_RUNTIME_DIR); + return std::filesystem::absolute( + std::filesystem::path(runtime_dir) / + "ryzenai_corelib.dll") + .lexically_normal(); +} + +// A real corelib call that the real library really refuses. +// +// The LM-head MatMul shape ships kernels at M in {1, 128} and ERRORS above +// 128 rather than rounding up -- measured, and asserted separately in +// test_phi4_hardware. Using it here means the CorelibError carries the +// library's own status code and its own last-error message, so the record the +// parent reads back is a real diagnostic rather than a string this file made +// up. +CorelibError RealCorelibFailure(const std::shared_ptr& api) { + std::int64_t m = constants::kMaxSequenceLength; + std::int64_t k = constants::kHiddenSize; + std::int64_t n = constants::kVocabularySize; + try { + api->Check( + api->functions().matmul_pad_shape( + &m, + &k, + &n, + constants::kGroupSize), + "ryzenai_corelib_matmul_bf16_pad_shape"); + } catch (const CorelibError& error) { + return error; + } + throw std::runtime_error( + "the real corelib accepted an LM-head M of " + + std::to_string(constants::kMaxSequenceLength) + + ", so this file has no genuine failure to drive the policy with. " + "That is a finding about the library, not a defect here."); +} + +// --------------------------------------------------------------------------- +// Child +// --------------------------------------------------------------------------- + +// The record-store half of the terminal path, in a real process, with no +// device. +// +// Design 12.4's concurrency properties -- unique files per process, a pending +// file preserved while its owner is still alive, ordered reporting of every +// completed record -- have only ever been checked in ONE process against +// injected PIDs, start times and process probes. That leaves the production +// probe itself unexercised: nothing had ever confirmed that +// `ProbeProcessStart` recognises a genuinely live sibling process, which is +// the single decision that stands between a live process's pending file and +// its deletion. +// +// This child deliberately does NOT load corelib. Two processes holding AIE4 +// device contexts at once fail in ways that look like defects, and none of the +// four properties under test needs a device -- so the concurrency is real and +// the device contention is not introduced. +int RunRecordOnlyChild() { + auto store = FatalRecordStore::ForCurrentProcess(); + store.Prepare(); + WriteMarker(L"pid.marker", std::to_string(GetCurrentProcessId())); + WriteMarker(L"pending.marker", store.pending_path().string()); + // Written LAST, and empty on purpose. + // + // The other scenarios read a child's markers only after it has exited, so + // they never had to care that `ofstream` CREATES a file before it writes + // to it. This scenario reads while the child is still alive, and polling + // `exists("pending.marker")` returns true the instant the stream is + // opened -- so the parent read a zero-length path, decided the pending + // file did not exist, and failed. It failed intermittently, which is + // worse than failing. Existence of this marker is ordered after the other + // two are flushed and closed, so it means "the markers are complete" + // rather than "a marker has begun". + WriteMarker(L"ready.marker", ""); + + // Both children hold their pending file open until the parent has drained + // once with both of them alive. The go file is a sibling of the marker + // directories so no extra argument has to be threaded through. + const auto go = g_marker_dir.parent_path() / "go.marker"; + const auto deadline = + std::chrono::steady_clock::now() + std::chrono::seconds(120); + while (!std::filesystem::exists(go)) { + if (std::chrono::steady_clock::now() > deadline) { + std::cerr << "record_only child timed out waiting for " + << go.string() << '\n'; + return 3; + } + Sleep(25); + } + + const FailureContext failure{ + ryzenai_corelib_status_bad_argument, + "matmul_q", + "concurrent record child", + "qkv", + 7, + 13, + 29}; + const auto final_path = store.Persist(failure); + WriteMarker(L"final.marker", final_path.string()); + return 0; +} + +int RunChild(std::string_view scenario) { + if (scenario == "record_only") { + return RunRecordOnlyChild(); + } + AddExtraDllDirectories(FLM_REAL_CORELIB_EXTRA_DLL_DIRS); + + // GetOrCreate builds the production runtime: the real DLL, the real + // FatalRecordStore rooted in LocalAppData, and the real terminator that + // calls TerminateProcess. Nothing is substituted. + const auto library = CorelibLibraryPath(); + SetEnvironmentVariableW( + L"RYZENAI_CORELIB_PATH", + library.c_str()); + auto runtime = CorelibRuntime::GetOrCreate(library.parent_path()); + CHECK( + runtime->state() == flm::corelib::ProcessState::Healthy); + + const CorelibError failure = RealCorelibFailure(runtime->api()); + std::cout << "child scenario=" << scenario + << " status=" << static_cast(failure.status) + << " call=" << failure.call << '\n'; + + // The library's OWN status and message, handed to the parent so it can + // check the record against what corelib actually said rather than against + // a literal this test invented. Written before the policy runs, because + // two of the three scenarios never return from it. + WriteMarker( + L"expected-status.marker", + std::to_string(static_cast(failure.status))); + WriteMarker(L"expected-detail.marker", failure.detail); + WriteMarker(L"expected-call.marker", failure.call); + + StepSubmissionState submission; + bool synchronize_in_progress = false; + std::string phase = "qkv"; + if (scenario == "before_submit") { + // Nothing has been submitted and no synchronize is running, so the + // policy must RETHROW and leave the process alive. + phase = "qkv"; + } else if (scenario == "after_submit") { + // q submitted successfully, k failed. Past the irrevocable boundary. + submission.MarkSuccessfulSubmit(); + phase = "qkv"; + } else if (scenario == "synchronize") { + // A failing synchronize is irrevocable even with nothing marked as + // submitted: the device may already be mid-flight. + synchronize_in_progress = true; + phase = "flat_mha"; + } else { + throw std::runtime_error( + "unknown child scenario: " + std::string(scenario)); + } + + // The positive control. Written and closed immediately before the policy + // is entered, so the parent can tell "terminated inside the policy" from + // "died on the way there" -- both of which leave no cleanup marker, and + // only one of which is the behaviour under test. + WriteMarker(kReachedPolicyMarker, std::string(scenario)); + + bool rethrown = false; + try { + flm::phi4::testing::ApplyCorelibFailurePolicyForTest( + runtime, + failure, + synchronize_in_progress, + submission, + phase, + /*layer=*/7, + /*rows=*/13, + /*position=*/29); + } catch (const CorelibError&) { + rethrown = true; + } + + // Only "before_submit" can reach here. The other two scenarios died + // inside the policy, so if control returns for them the policy did not do + // what design Section 12.1 requires and the child fails loudly rather + // than exiting 0. + if (scenario != "before_submit") { + std::cerr << "child scenario=" << scenario + << " survived an irrevocable failure\n"; + return 2; + } + CHECK(rethrown); + + // Admission is still open and the runtime is still healthy. + CHECK(runtime->admission_open()); + CHECK(runtime->state() == flm::corelib::ProcessState::Healthy); + + // "The session clears" has to mean the session is USABLE again, not just + // that a flag says Healthy. So take a fresh execution lease and put a real + // operation through the real library: create a Stream and a DeviceTensor, + // write and read an element, and release them. If a recoverable failure + // had left the runtime wedged, this is where it would show. + { + auto execution = runtime->AcquireExecution(); + const auto& api = runtime->api(); + ryzenai_corelib_stream_ptr raw_stream = nullptr; + api->Check( + api->functions().create_stream(&raw_stream), + "ryzenai_corelib_create_stream"); + flm::corelib::UniqueStream stream(api, raw_stream); + + const std::array shape{1, 64}; + ryzenai_corelib_tensor_ptr raw_tensor = nullptr; + api->Check( + api->functions().create_device_tensor( + ryzenai_corelib_data_type_bf16, + shape.data(), + shape.size(), + &raw_tensor), + "ryzenai_corelib_create_device_tensor"); + flm::corelib::UniqueTensor tensor(api, raw_tensor); + + const std::array source{1.0f, 2.0f, 4.0f, 8.0f}; + api->WriteElements( + tensor.get(), + ryzenai_corelib_data_type_fp32, + source.data(), + source.size(), + 0); + std::array destination{}; + api->ReadElements( + tensor.get(), + ryzenai_corelib_data_type_fp32, + destination.data(), + destination.size(), + 0); + CHECK(destination == source); + + api->Check( + api->functions().stream_synchronize(stream.get()), + "ryzenai_corelib_stream_synchronize"); + tensor.reset(); + stream.reset(); + } + CHECK(runtime->api()->live_object_count() == 0); + WriteMarker(kSurvivedMarker, "session usable after a pre-submit failure"); + + CorelibRuntime::ShutdownProcess(); + WriteMarker(kCleanupMarker, "healthy shutdown completed"); + return 0; +} + +// --------------------------------------------------------------------------- +// Parent +// --------------------------------------------------------------------------- + +class TempDirectory final { +public: + // PID alone is not unique enough. Windows reuses process IDs freely, and + // a run that fails can leave child processes polling this directory for + // up to their timeout; a later run that happens to get the same PID then + // shares a directory with somebody else's children. That produced an + // intermittent failure that only ever appeared in the run immediately + // after a failed one, which is the hardest kind to read. The tick count + // makes reuse impossible in any window that matters. + TempDirectory() { + path_ = std::filesystem::temp_directory_path() / + ("fastflowlm-fatal-child-" + + std::to_string(GetCurrentProcessId()) + "-" + + std::to_string(GetTickCount64())); + std::filesystem::create_directories(path_); + } + + ~TempDirectory() noexcept { + std::error_code error; + std::filesystem::remove_all(path_, error); + } + + TempDirectory(const TempDirectory&) = delete; + TempDirectory& operator=(const TempDirectory&) = delete; + + const std::filesystem::path& path() const noexcept { + return path_; + } + +private: + std::filesystem::path path_; +}; + +struct ChildResult { + DWORD exit_code = 0; + std::string output; +}; + +// A child that has been started but not yet waited for. Needed because the +// concurrency scenario has to have two of them alive at the same moment; every +// other scenario starts one and waits. +struct RunningChild { + PROCESS_INFORMATION process{}; + std::filesystem::path log; + std::filesystem::path marker_dir; + std::string scenario; +}; + +RunningChild StartChild( + const std::filesystem::path& executable, + const std::filesystem::path& log_path, + const std::filesystem::path& marker_dir, + std::string_view scenario); + +ChildResult AwaitChild(RunningChild& child, DWORD timeout_ms); + +ChildResult RunScenario( + const std::filesystem::path& executable, + const std::filesystem::path& log_path, + const std::filesystem::path& marker_dir, + std::string_view scenario) { + RunningChild child = + StartChild(executable, log_path, marker_dir, scenario); + // Generous, but bounded: the child loads the real corelib, and a hang + // must fail the suite rather than wedge it. + return AwaitChild(child, 300000); +} + +RunningChild StartChild( + const std::filesystem::path& executable, + const std::filesystem::path& log_path, + const std::filesystem::path& marker_dir, + std::string_view scenario) { + // Each scenario gets a fresh marker directory, so a marker left by an + // earlier scenario cannot be read as this one's evidence. + std::error_code error; + std::filesystem::remove_all(marker_dir, error); + std::filesystem::create_directories(marker_dir); + + SECURITY_ATTRIBUTES inheritable{}; + inheritable.nLength = sizeof(inheritable); + inheritable.bInheritHandle = TRUE; + + HANDLE log = CreateFileW( + log_path.c_str(), + GENERIC_WRITE, + FILE_SHARE_READ, + &inheritable, + CREATE_ALWAYS, + FILE_ATTRIBUTE_NORMAL, + nullptr); + if (log == INVALID_HANDLE_VALUE) { + throw std::runtime_error( + "cannot create child log " + log_path.string()); + } + + std::wstring command = L"\"" + executable.wstring() + + L"\" --child " + + std::wstring( + scenario.begin(), + scenario.end()) + + L" \"" + marker_dir.wstring() + L"\""; + STARTUPINFOW startup{}; + startup.cb = sizeof(startup); + startup.dwFlags = STARTF_USESTDHANDLES; + startup.hStdOutput = log; + startup.hStdError = log; + startup.hStdInput = nullptr; + + PROCESS_INFORMATION process{}; + const BOOL created = CreateProcessW( + executable.c_str(), + command.data(), + nullptr, + nullptr, + TRUE, + 0, + nullptr, + nullptr, + &startup, + &process); + CloseHandle(log); + if (created == FALSE) { + throw std::runtime_error( + "CreateProcessW failed for scenario " + + std::string(scenario) + " (error " + + std::to_string(GetLastError()) + ")"); + } + + RunningChild child; + child.process = process; + child.log = log_path; + child.marker_dir = marker_dir; + child.scenario = std::string(scenario); + return child; +} + +ChildResult AwaitChild(RunningChild& child, DWORD timeout_ms) { + ChildResult result; + if ( + WaitForSingleObject(child.process.hProcess, timeout_ms) != + WAIT_OBJECT_0 + ) { + TerminateProcess(child.process.hProcess, 1); + CloseHandle(child.process.hThread); + CloseHandle(child.process.hProcess); + child.process = PROCESS_INFORMATION{}; + throw std::runtime_error( + "child scenario " + child.scenario + " did not exit"); + } + GetExitCodeProcess(child.process.hProcess, &result.exit_code); + CloseHandle(child.process.hThread); + CloseHandle(child.process.hProcess); + child.process = PROCESS_INFORMATION{}; + + std::ifstream stream(child.log, std::ios::binary); + std::ostringstream buffer; + buffer << stream.rdbuf(); + result.output = buffer.str(); + return result; +} + +bool Contains(std::string_view haystack, std::string_view needle) { + return haystack.find(needle) != std::string_view::npos; +} + +// Field extraction over the record's own on-disk text rather than a JSON +// parser, so the check is against exactly the bytes a support engineer would +// receive. The record writer emits compact JSON with no spaces around the +// colon, which is what makes this reliable. +std::string FieldValue( + std::string_view record, + std::string_view key) { + const std::string needle = "\"" + std::string(key) + "\":"; + const auto start = record.find(needle); + if (start == std::string_view::npos) { + throw std::runtime_error( + "fatal record is missing the field \"" + std::string(key) + + "\": " + std::string(record)); + } + auto cursor = start + needle.size(); + if (cursor < record.size() && record[cursor] == '"') { + ++cursor; + // Unescaped, because the record writer escapes what it stores and the + // library's own messages carry Windows paths full of backslashes. A + // raw comparison would fail on `C:\\Users\\...` against `C:\Users\...` + // and look like a content mismatch rather than an encoding one. + std::string value; + while (cursor < record.size() && record[cursor] != '"') { + if (record[cursor] == '\\' && cursor + 1 < record.size()) { + ++cursor; + switch (record[cursor]) { + case 'n': value.push_back('\n'); break; + case 'r': value.push_back('\r'); break; + case 't': value.push_back('\t'); break; + case 'b': value.push_back('\b'); break; + case 'f': value.push_back('\f'); break; + case 'u': { + // The writer only emits \u for control characters. + const auto digits = record.substr(cursor + 1, 4); + value.push_back(static_cast( + std::stoi(std::string(digits), nullptr, 16))); + cursor += 4; + break; + } + default: value.push_back(record[cursor]); break; + } + ++cursor; + continue; + } + value.push_back(record[cursor]); + ++cursor; + } + return value; + } + const auto end = record.find_first_of(",}", cursor); + return std::string(record.substr(cursor, end - cursor)); +} + +std::string ReadMarker( + const std::filesystem::path& marker_dir, + std::wstring_view name) { + std::ifstream stream( + marker_dir / std::filesystem::path(name), + std::ios::binary); + if (!stream) { + return {}; + } + std::ostringstream buffer; + buffer << stream.rdbuf(); + return buffer.str(); +} + +bool MarkerExists( + const std::filesystem::path& marker_dir, + std::wstring_view name) { + return std::filesystem::exists( + marker_dir / std::filesystem::path(name)); +} + +void CheckDetailedRecord( + std::string_view record, + std::string_view expected_phase, + const std::filesystem::path& marker_dir) { + // "Complete" means every field design Section 12.1 lists is present and + // carries the right value -- not merely that a file exists. + // + // `status`, `call` and `detail` are checked against what the LIBRARY + // produced, read out of markers the child wrote from the real + // CorelibError. Checking them for non-emptiness, as this did before, would + // pass for a record that carried the wrong diagnostic entirely. + const auto expected_status = + ReadMarker(marker_dir, L"expected-status.marker"); + const auto expected_detail = + ReadMarker(marker_dir, L"expected-detail.marker"); + const auto expected_call = + ReadMarker(marker_dir, L"expected-call.marker"); + CHECK(!expected_status.empty()); + CHECK(!expected_detail.empty()); + CHECK(!expected_call.empty()); + + CHECK(FieldValue(record, "status") == expected_status); + CHECK(FieldValue(record, "call") == expected_call); + CHECK(FieldValue(record, "detail") == expected_detail); + + // phase/layer/rows/position are values this test chose, so what they + // establish is narrower: that FailureContext survives serialisation + // intact, field for field, including the optional layer. They do NOT + // establish that the engine supplies the right ones -- reaching the + // production catch arm needs a fault injection point the engine does not + // have. See the report's note on I4. + CHECK(FieldValue(record, "phase") == expected_phase); + CHECK(FieldValue(record, "layer") == "7"); + CHECK(FieldValue(record, "rows") == "13"); + CHECK(FieldValue(record, "position") == "29"); + + CHECK(!FieldValue(record, "pid").empty()); + CHECK(!FieldValue(record, "failure_utc").empty()); + CHECK(!FieldValue(record, "process_start_utc").empty()); +} + +// Design 12.4 concurrency, with two processes that are genuinely concurrent. +// +// Four properties, and what each one adds over the in-process tests in +// test_corelib_fatal_record: +// +// 1. Unique files. There the two stores were given PIDs 1001 and 1002 by the +// test. Here they are whatever Windows assigned, and the start times are +// whatever the real `GetProcessTimes` reported. +// 2. A live process's pending file is preserved. There the probe was a lambda +// that returned a chosen answer. Here it is the production +// `ProbeProcessStart`, asked about a sibling process that really is +// running -- the only version of this check that can fail if the probe is +// wrong. +// 3. Every completed record is reported, in order. +// 4. The pending files are gone once their owners have persisted. +// +// What this does NOT add: "safe preservation when the process-start query +// fails". A failing Win32 query cannot be produced on demand, so that case +// stays with the injected-probe test and is not claimed here. +void RunConcurrentRecordScenario( + const std::filesystem::path& executable, + TempDirectory& logs) { + std::cout << "concurrent record children:\n"; + + // Every record from the scenarios above has already been drained, but + // drain again so the counts below describe only these two children. + std::ostringstream cleared; + (void)FatalRecordStore::DrainPriorRecords(cleared); + + const auto go = logs.path() / "go.marker"; + std::error_code ignored; + std::filesystem::remove(go, ignored); + + std::array children{ + StartChild( + executable, + logs.path() / "record_a.log", + logs.path() / "record_a", + "record_only"), + StartChild( + executable, + logs.path() / "record_b.log", + logs.path() / "record_b", + "record_only")}; + + // Nothing here may leave a child behind. A child that outlives a failing + // parent keeps polling for a release file that will never appear, holds + // a pending record in the shared root while it does, and is the state + // that made this scenario's first failure intermittent. + struct TerminateOnScopeExit { + std::array* children; + ~TerminateOnScopeExit() noexcept { + for (auto& child : *children) { + if (child.process.hProcess != nullptr) { + TerminateProcess(child.process.hProcess, 1); + CloseHandle(child.process.hThread); + CloseHandle(child.process.hProcess); + child.process = PROCESS_INFORMATION{}; + } + } + } + } terminate_on_scope_exit{&children}; + + std::array pending{}; + std::array pids{}; + const auto deadline = + std::chrono::steady_clock::now() + std::chrono::seconds(60); + for (std::size_t index = 0; index < children.size(); ++index) { + const auto marker = children[index].marker_dir / "ready.marker"; + while (!std::filesystem::exists(marker)) { + if (std::chrono::steady_clock::now() > deadline) { + for (auto& child : children) { + if (child.process.hProcess != nullptr) { + TerminateProcess(child.process.hProcess, 1); + } + } + throw std::runtime_error( + "a record_only child never wrote its pending marker"); + } + Sleep(25); + } + pending[index] = std::filesystem::path( + ReadMarker(children[index].marker_dir, L"pending.marker")); + pids[index] = ReadMarker(children[index].marker_dir, L"pid.marker"); + // An empty marker means the ordering above broke, not that the store + // misbehaved. Say which, or the next reader spends an hour on the + // wrong one -- as this test already cost once. + if (pending[index].empty() || pids[index].empty()) { + for (auto& child : children) { + if (child.process.hProcess != nullptr) { + TerminateProcess(child.process.hProcess, 1); + } + } + throw std::runtime_error( + "a record_only child's markers were readable but empty, so " + "ready.marker is no longer ordered after them"); + } + } + + // 1. Two live processes, two distinct pending files. + CHECK(pending[0] != pending[1]); + CHECK(pids[0] != pids[1]); + for (std::size_t index = 0; index < pending.size(); ++index) { + if (std::filesystem::exists(pending[index])) { + continue; + } + // A bare CHECK here says only "false", and this exact assertion has + // already been chased once on a wrong theory. Report the state that + // distinguishes the candidates: whether the child already persisted + // (a final record carrying its pid), whether the release file it + // waits on somehow exists, and what is actually in the record root. + std::ostringstream detail; + detail << "child " << pids[index] + << " pending file is absent while the child is still running: " + << pending[index].string() + << "\n go file " << go.string() << " exists: " + << std::boolalpha << std::filesystem::exists(go) + << "\n record root now holds:"; + std::error_code listing; + for (const auto& entry : std::filesystem::directory_iterator( + pending[index].parent_path(), listing)) { + detail << "\n " << entry.path().filename().string(); + } + for (auto& child : children) { + if (child.process.hProcess != nullptr) { + DWORD code = 0; + GetExitCodeProcess(child.process.hProcess, &code); + detail << "\n child " << child.scenario << " exit code: " + << (code == STILL_ACTIVE + ? std::string("still running") + : std::to_string(code)); + TerminateProcess(child.process.hProcess, 1); + } + } + throw std::runtime_error(detail.str()); + } + + // 2. Both owners are alive, so the production probe must keep both. + std::ostringstream while_alive; + const auto during = FatalRecordStore::DrainPriorRecords(while_alive); + if (!during.empty()) { + for (auto& child : children) { + if (child.process.hProcess != nullptr) { + TerminateProcess(child.process.hProcess, 1); + } + } + throw std::runtime_error( + "a drain taken while two children were still running reported " + + std::to_string(during.size()) + " completed record(s)"); + } + if ( + !std::filesystem::exists(pending[0]) || + !std::filesystem::exists(pending[1]) + ) { + for (auto& child : children) { + if (child.process.hProcess != nullptr) { + TerminateProcess(child.process.hProcess, 1); + } + } + throw std::runtime_error( + "the drain deleted a pending record whose owner was still alive"); + } + std::cout << " both pendings survived a drain taken while both children " + "were running\n"; + + { + std::ofstream stream(go, std::ios::binary | std::ios::trunc); + stream << "go"; + } + for (auto& child : children) { + const auto result = AwaitChild(child, 180000); + CHECK(result.exit_code == 0); + } + + std::array finals{ + std::filesystem::path( + ReadMarker(children[0].marker_dir, L"final.marker")), + std::filesystem::path( + ReadMarker(children[1].marker_dir, L"final.marker"))}; + CHECK(finals[0] != finals[1]); + for (std::size_t index = 0; index < finals.size(); ++index) { + CHECK(finals[index].filename().string().ends_with( + "-" + pids[index] + ".json")); + // 4. Persisting consumed the pending file. + CHECK(!std::filesystem::exists(pending[index])); + } + + // 3. Both completed records are reported, and in filename order -- which + // is timestamp then PID, so the expected order is derived here rather + // than assumed to be the order the children were started in. + std::vector expected{ + finals[0].string(), finals[1].string()}; + std::sort(expected.begin(), expected.end()); + std::ostringstream reported; + const auto drained = FatalRecordStore::DrainPriorRecords(reported); + if (drained.size() != 2) { + throw std::runtime_error( + "expected 2 completed records from two concurrent children, " + "drained " + std::to_string(drained.size())); + } + std::vector drained_paths; + for (const auto& record : drained) { + // DrainPriorRecords returns each record's CONTENTS, so identify them + // by the PID each one carries rather than by a path it never returns. + CHECK(!record.empty()); + drained_paths.push_back(FieldValue(record, "pid")); + } + std::vector expected_pids; + for (const auto& path : expected) { + const auto name = std::filesystem::path(path).filename().string(); + const auto dash = name.rfind('-'); + expected_pids.push_back( + name.substr(dash + 1, name.size() - dash - 1 - 5)); + } + CHECK(drained_paths == expected_pids); + for (const auto& pid : expected_pids) { + CHECK(reported.str().find("\"pid\":" + pid) != std::string::npos); + } + std::cout << " 2 completed records drained in filename order, pids " + << expected_pids[0] << " and " << expected_pids[1] << '\n'; +} + +int RunParent(const std::filesystem::path& executable) { + TempDirectory logs; + + // Start from a clean slate so a record left by an earlier run cannot be + // mistaken for this run's evidence. + std::ostringstream discarded; + FatalRecordStore::DrainPriorRecords(discarded); + + { + const auto markers = logs.path() / "before_submit.markers"; + const auto result = RunScenario( + executable, + logs.path() / "before_submit.log", + markers, + "before_submit"); + std::cout << "before_submit exit=0x" << std::hex + << result.exit_code << std::dec << '\n'; + CHECK(result.exit_code == 0); + CHECK(MarkerExists(markers, kReachedPolicyMarker)); + // The session is usable again: the child put a real Stream, tensor + // write, read and synchronize through the library after the + // recoverable failure. + CHECK(MarkerExists(markers, kSurvivedMarker)); + CHECK(MarkerExists(markers, kCleanupMarker)); + + // A recoverable failure writes NO record. The child's healthy + // shutdown removes its own pending file, so a drain here must come + // back empty. + std::ostringstream drained; + const auto records = + FatalRecordStore::DrainPriorRecords(drained); + if (!records.empty()) { + throw std::runtime_error( + "a pre-submit failure left a fatal record behind: " + + records.front()); + } + } + + for (const auto& [scenario, phase] : + std::array, 2>{ + {{"after_submit", "qkv"}, {"synchronize", "flat_mha"}}}) { + const auto markers = + logs.path() / (std::string(scenario) + ".markers"); + const auto result = RunScenario( + executable, + logs.path() / (std::string(scenario) + ".log"), + markers, + scenario); + std::cout << scenario << " exit=0x" << std::hex + << result.exit_code << std::dec << '\n'; + CHECK(result.exit_code == kFatalExitCode); + + // The positive control first. Without it, "no cleanup marker" is also + // satisfied by a child that died before it ever reached the policy, + // and the absence below would prove nothing. + CHECK(MarkerExists(markers, kReachedPolicyMarker)); + CHECK( + ReadMarker(markers, kReachedPolicyMarker) == + std::string(scenario)); + + // Normal cleanup must NOT have run. These are closed files, so they + // survive TerminateProcess and their absence is real -- unlike the + // buffered stdout this check used to read, which TerminateProcess + // discards, making the assertion pass whether cleanup had run or not. + CHECK(!MarkerExists(markers, kCleanupMarker)); + CHECK(!MarkerExists(markers, kSurvivedMarker)); + // The terminal diagnostic goes to stderr before the process dies, + // and it must name the record it wrote. + CHECK(Contains(result.output, "AIE4 terminal failure")); + CHECK(Contains(result.output, "AIE4 fatal record:")); + + std::ostringstream drained; + const auto records = + FatalRecordStore::DrainPriorRecords(drained); + if (records.size() != 1) { + throw std::runtime_error( + "expected exactly one fatal record from scenario " + + std::string(scenario) + ", drained " + + std::to_string(records.size())); + } + CheckDetailedRecord(records.front(), phase, markers); + // The drain reports what it removed, so the record reaches an + // operator rather than only the filesystem. + CHECK(Contains(drained.str(), "\"phase\":\"" + std::string(phase))); + std::cout << " record: " << records.front(); + } + + RunConcurrentRecordScenario(executable, logs); + + std::cout << "test_fatal_child: PASS\n"; + return 0; +} + +} // namespace + +constexpr int kCTestSkipReturnCode = 77; + +// Same opt-in as test_phi4_hardware, and for the same reason: a configured +// runtime directory names a DLL, it does not assert that this machine is the +// AIE4 target. GetOrCreate refuses to build a runtime without a device +// context, so without the flag this would fail on every development box. +bool HardwareRunRequested() { + char value[8] = {}; + const DWORD length = GetEnvironmentVariableA( + "FLM_AIE4_HARDWARE", + value, + sizeof(value)); + return length != 0 && length < sizeof(value) && + std::string_view(value) == "1"; +} + +int main(int argc, char** argv) { + const std::string runtime_dir(FLM_REAL_CORELIB_RUNTIME_DIR); + if (runtime_dir.empty()) { + std::cout + << "test_fatal_child: SKIPPED -- configure with " + "-DRYZENAI_CORELIB_RUNTIME_DIR and run on the AIE4 " + "target.\n"; + return kCTestSkipReturnCode; + } + if (!HardwareRunRequested()) { + std::cout + << "test_fatal_child: SKIPPED -- set FLM_AIE4_HARDWARE=1 to " + "run this on the AIE4 target.\n"; + return kCTestSkipReturnCode; + } + + try { + if (argc >= 4 && std::string_view(argv[1]) == "--child") { + g_marker_dir = std::filesystem::path(argv[3]); + if (!std::filesystem::is_directory(g_marker_dir)) { + throw std::runtime_error( + "child marker directory does not exist: " + + g_marker_dir.string()); + } + return RunChild(argv[2]); + } + return RunParent( + std::filesystem::absolute(argv[0]).lexically_normal()); + } catch (const std::exception& error) { + std::cerr << "test_fatal_child: FAIL: " << error.what() << '\n'; + return 1; + } +} diff --git a/src/test/phi4_corelib_aie4/test_generation_limit.cpp b/src/test/phi4_corelib_aie4/test_generation_limit.cpp new file mode 100644 index 00000000..c7a5aa4b --- /dev/null +++ b/src/test/phi4_corelib_aie4/test_generation_limit.cpp @@ -0,0 +1,867 @@ +#include +#include +#include + +#include "test_support.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using nlohmann::ordered_json; + +void CheckParsed( + const ParsedGenerationLimit& parsed, + bool explicit_limit, + int value) { + CHECK(parsed.explicit_limit == explicit_limit); + CHECK(parsed.value == value); +} + +void TestHandlerSpecificPresence() { + CheckParsed( + ParseGenerationLimit( + ordered_json{ + {"max_tokens", 11}, + {"max_completion_tokens", 22}}, + GenerationEndpoint::Generate), + true, + 11); + CheckParsed( + ParseGenerationLimit( + ordered_json{{"max_completion_tokens", 22}}, + GenerationEndpoint::Generate), + false, + -1); + + CheckParsed( + ParseGenerationLimit( + ordered_json{ + {"max_tokens", 31}, + {"max_completion_tokens", 32}}, + GenerationEndpoint::OpenAiChatCompletion), + true, + 31); + CheckParsed( + ParseGenerationLimit( + ordered_json{{"max_completion_tokens", 32}}, + GenerationEndpoint::OpenAiChatCompletion), + true, + 32); + + CheckParsed( + ParseGenerationLimit( + ordered_json{ + {"max_tokens", 41}, + {"max_completion_tokens", 42}}, + GenerationEndpoint::OpenAiCompletion), + true, + 41); + CheckParsed( + ParseGenerationLimit( + ordered_json{{"max_completion_tokens", 42}}, + GenerationEndpoint::OpenAiCompletion), + false, + -1); + + // /api/chat nests its limit, and it is now parsed by the same rule + // rather than by a private one. + CheckParsed( + ParseGenerationLimit( + ordered_json{{"options", {{"num_predict", 51}}}}, + GenerationEndpoint::OllamaChat), + true, + 51); + CheckParsed( + ParseGenerationLimit( + ordered_json{{"options", ordered_json::object()}}, + GenerationEndpoint::OllamaChat), + false, + -1); + // A flat max_tokens is not the Ollama field and must not be read. + CheckParsed( + ParseGenerationLimit( + ordered_json{{"max_tokens", 52}}, + GenerationEndpoint::OllamaChat), + false, + -1); + + for (const GenerationEndpoint endpoint : { + GenerationEndpoint::Generate, + GenerationEndpoint::OllamaChat, + GenerationEndpoint::OpenAiChatCompletion, + GenerationEndpoint::OpenAiCompletion}) { + CheckParsed( + ParseGenerationLimit(ordered_json::object(), endpoint), + false, + -1); + } +} + +void TestEndpointDefaultsAndPropagation() { + const ParsedGenerationLimit omitted{false, -1}; + const ParsedGenerationLimit explicit_limit{true, 73}; + + CHECK(GenerationLoopLimit(omitted, true) == -1); + CHECK(GenerationLoopLimit(omitted, false) == 4096); + CHECK(GenerationLoopLimit(explicit_limit, true) == 73); + CHECK(GenerationLoopLimit(explicit_limit, false) == 73); + + CHECK(!RequestedMaxNewTokens(omitted).has_value()); + CHECK(RequestedMaxNewTokens(explicit_limit) == std::optional(73)); +} + +// I13. The two readers of the same parsed field must not disagree about what +// a non-positive value means. +// +// GenerationLoopLimit returns an explicit value verbatim and both generation +// loops gate on `length_limit > 0`, so -1, -2 and 0 all mean "no bound" to +// the loop. RequestedMaxNewTokens is the ADMISSION rule's reading -- how much +// output the AIE4 cap must reserve room for -- and it used to hand the same +// -1 on as a count, which validate_aie4_capacity then refused with HTTP 400. +// The result was Ollama's documented "generate forever" sentinel answering +// 400 on the AIE4 tag and 200 on every other backend. +// +// This pins the mapping for every sentinel. The end-to-end consequence -- +// admitted or 400 against a real AIE4 Phi4 and a real legacy Phi4 -- is +// pinned by TestNumPredictSentinelsAdmitIdenticallyOnBothBackends in +// test_phi4_frontend.cpp, which is the binary that links modeling_phi4.cpp. +void TestNonPositiveLimitSentinelsRequestNoBudget() { + struct Case { + int num_predict; + std::optional requested; + }; + // -1 is Ollama's "generate forever", -2 its "fill context". 0 is here + // deliberately and not by omission: the loops read it as "no bound" too, + // so the admission rule must not reserve a budget for it either. + const Case cases[] = { + {-2, std::nullopt}, + {-1, std::nullopt}, + {0, std::nullopt}, + {1, std::optional(1)}, + {73, std::optional(73)}, + }; + + for (const Case& test_case : cases) { + const ordered_json request = { + {"options", {{"num_predict", test_case.num_predict}}}, + }; + const ParsedGenerationLimit parsed = + ParseGenerationLimit( + request, + GenerationEndpoint::OllamaChat); + CHECK(parsed.explicit_limit); + CHECK(parsed.value == test_case.num_predict); + + // The loop limit is unchanged: the sentinel still reaches the loop, + // which is what makes it mean "generate until stop". + CHECK( + GenerationLoopLimit(parsed, true) == + test_case.num_predict); + CHECK( + GenerationLoopLimit(parsed, false) == + test_case.num_predict); + + CHECK(RequestedMaxNewTokens(parsed) == test_case.requested); + + // And the REST reading now agrees with the console reading of the + // same number, which is where this idiom already lived. + CHECK( + RequestedMaxNewTokens(parsed) == + CliRequestedMaxNewTokens(test_case.num_predict)); + } + + // Asking explicitly for the unbounded behaviour must reach the admission + // rule exactly as omitting the field does. That equivalence is the whole + // finding: omission was allowed, the explicit sentinel was refused. + const ParsedGenerationLimit omitted = + ParseGenerationLimit( + ordered_json{{"options", ordered_json::object()}}, + GenerationEndpoint::OllamaChat); + const ParsedGenerationLimit forever = + ParseGenerationLimit( + ordered_json{{"options", {{"num_predict", -1}}}}, + GenerationEndpoint::OllamaChat); + CHECK( + RequestedMaxNewTokens(omitted) == + RequestedMaxNewTokens(forever)); + + // The same rule on every other endpoint's own field, so /api/chat is not + // special-cased into agreement. + for (const auto& [endpoint, field] : + std::vector>{ + {GenerationEndpoint::Generate, "max_tokens"}, + {GenerationEndpoint::OpenAiCompletion, "max_tokens"}, + {GenerationEndpoint::OpenAiChatCompletion, "max_tokens"}, + {GenerationEndpoint::OpenAiChatCompletion, + "max_completion_tokens"}}) { + for (const int value : {-2, -1, 0}) { + ordered_json request; + request[field] = value; + const ParsedGenerationLimit parsed = + ParseGenerationLimit(request, endpoint); + CHECK(parsed.explicit_limit); + CHECK(!RequestedMaxNewTokens(parsed).has_value()); + CHECK(GenerationLoopLimit(parsed, true) == value); + } + } +} + +// Legacy behaviour on /api/chat must not change: an omitted num_predict +// is still the 4096 soft bound, and an explicit one is still honoured. +void TestOllamaChatLegacyLimitIsUnchanged() { + const ordered_json default_request = { + {"options", ordered_json::object()}, + }; + CHECK(OllamaChatGenerationLoopLimit(default_request) == 4096); + CHECK(OllamaChatGenerationLoopLimit(ordered_json::object()) == 4096); + + const ordered_json explicit_request = { + {"options", {{"num_predict", 91}}}, + }; + CHECK(OllamaChatGenerationLoopLimit(explicit_request) == 91); + + // The legacy spelling and the shared rule are the same function. + for (const ordered_json& request : + {default_request, explicit_request, ordered_json::object()}) { + const ParsedGenerationLimit parsed = + ParseGenerationLimit( + request, + GenerationEndpoint::OllamaChat); + CHECK( + GenerationLoopLimit(parsed, false) == + OllamaChatGenerationLoopLimit(request)); + } +} + +// On AIE4 an omitted /api/chat limit must mean "until the context cap", +// not the legacy 4096, and an explicit one must reach the admission rule +// through requested_max_new_tokens. Both were bypassed before. +void TestOllamaChatAie4LimitAndAdmission() { + const ordered_json omitted = {{"options", ordered_json::object()}}; + const ParsedGenerationLimit omitted_parsed = + ParseGenerationLimit(omitted, GenerationEndpoint::OllamaChat); + CHECK(GenerationLoopLimit(omitted_parsed, true) == -1); + CHECK(!RequestedMaxNewTokens(omitted_parsed).has_value()); + + const ordered_json over_limit = { + {"options", {{"num_predict", 8192}}}, + }; + const ParsedGenerationLimit over_parsed = + ParseGenerationLimit(over_limit, GenerationEndpoint::OllamaChat); + CHECK(GenerationLoopLimit(over_parsed, true) == 8192); + CHECK( + RequestedMaxNewTokens(over_parsed) == std::optional(8192)); +} + +// The rule the /api/chat gap was missing. Three of four endpoints were +// covered by inspection; this derives the set instead of restating it. +// +// It reads the two production sources, finds every RestHandler::handle_* +// whose body reaches the causal engine's generate() or +// generate_with_prompt(), and maps those handlers to the routes server.cpp +// registers for them. Each such handler must then satisfy two conditions, +// because the defect that motivated this rule had BOTH shapes available +// and took the second: +// +// 1. its route is declared in GenerationRoutes() -- catches a fifth +// endpoint appearing with no declaration at all; and +// 2. it obtains its limit through the shared functions -- +// ParseGenerationLimit, GenerationLoopLimit and RequestedMaxNewTokens +// -- and calls no other *GenerationLoopLimit. +// +// handle_chat was declared and routed all along. What it did wrong was +// compute its limit privately, through OllamaChatGenerationLoopLimit, and +// a rule that only checked the route set would have passed it. Condition 2 +// is what makes this guard able to catch the bug it exists for. +std::string ReadSource(const char* relative) { + const std::filesystem::path path = + std::filesystem::path(FLM_TEST_SOURCE_DIR) / relative; + std::ifstream file(path, std::ios::binary); + if (!file) { + throw std::runtime_error( + "failed to read " + path.string()); + } + std::ostringstream buffer; + buffer << file.rdbuf(); + return buffer.str(); +} + +struct GeneratingHandler { + std::string name; + std::string body; +}; + +std::vector GeneratingHandlers() { + const std::string source = ReadSource("server/rest_handler.cpp"); + std::vector generating; + constexpr std::string_view kDefinition = "void RestHandler::handle_"; + for (std::size_t at = source.find(kDefinition); + at != std::string::npos; + at = source.find(kDefinition, at + 1)) { + const std::size_t name_start = + at + std::string("void RestHandler::").size(); + const std::size_t name_end = source.find('(', name_start); + if (name_end == std::string::npos) { + continue; + } + const std::string name = + source.substr(name_start, name_end - name_start); + const std::size_t body_end = + source.find(kDefinition, at + 1); + const std::string body = source.substr( + at, + body_end == std::string::npos + ? std::string::npos + : body_end - at); + if ( + body.find("auto_chat_engine->generate(") != + std::string::npos || + body.find("auto_chat_engine->generate_with_prompt(") != + std::string::npos) { + generating.push_back({name, body}); + } + } + return generating; +} + +bool IsIdentifierCharacter(char value) noexcept { + return value == '_' || + (value >= '0' && value <= '9') || + (value >= 'a' && value <= 'z') || + (value >= 'A' && value <= 'Z'); +} + +// True when `body` calls something whose name ENDS in the given suffix but +// is not exactly it -- OllamaChatGenerationLoopLimit for +// GenerationLoopLimit, say. That is the private-path shape. +std::string FindQualifiedVariantCall( + const std::string& body, + const std::string& call) { + for (std::size_t at = body.find(call); + at != std::string::npos; + at = body.find(call, at + 1)) { + if (at == 0 || !IsIdentifierCharacter(body[at - 1])) { + continue; + } + std::size_t start = at; + while (start > 0 && IsIdentifierCharacter(body[start - 1])) { + --start; + } + return body.substr(start, at + call.size() - start - 1); + } + return {}; +} + +// The (method, path) the handler declares to RequireGenerationEndpoint. +std::pair DeclaredRoute( + const std::string& body) { + constexpr std::string_view kCall = "RequireGenerationEndpoint("; + const std::size_t at = body.find(kCall); + if (at == std::string::npos) { + return {}; + } + const std::size_t method_start = body.find('"', at + kCall.size()); + if (method_start == std::string::npos) { + return {}; + } + const std::size_t method_end = body.find('"', method_start + 1); + const std::size_t path_start = body.find('"', method_end + 1); + const std::size_t path_end = body.find('"', path_start + 1); + if (path_end == std::string::npos) { + return {}; + } + return { + body.substr(method_start + 1, method_end - method_start - 1), + body.substr(path_start + 1, path_end - path_start - 1)}; +} + +std::map> +RegisteredRoutesByHandler() { + const std::string source = ReadSource("server/server.cpp"); + std::map> routes; + constexpr std::string_view kRegister = "register_handler(\""; + for (std::size_t at = source.find(kRegister); + at != std::string::npos; + at = source.find(kRegister, at + 1)) { + std::size_t cursor = at + kRegister.size(); + const std::size_t method_end = source.find('"', cursor); + if (method_end == std::string::npos) { + continue; + } + const std::string method = + source.substr(cursor, method_end - cursor); + const std::size_t path_start = source.find('"', method_end + 1); + if (path_start == std::string::npos) { + continue; + } + const std::size_t path_end = source.find('"', path_start + 1); + if (path_end == std::string::npos) { + continue; + } + const std::string path = + source.substr(path_start + 1, path_end - path_start - 1); + + // The handler this route dispatches to, before the next + // registration begins. + const std::size_t next = source.find(kRegister, at + 1); + const std::string block = source.substr( + path_end, + next == std::string::npos + ? std::string::npos + : next - path_end); + constexpr std::string_view kCall = "rest_handler->"; + const std::size_t call_at = block.find(kCall); + if (call_at == std::string::npos) { + continue; + } + const std::size_t call_start = call_at + kCall.size(); + const std::size_t call_end = block.find('(', call_start); + if (call_end == std::string::npos) { + continue; + } + routes.emplace( + block.substr(call_start, call_end - call_start), + std::pair(method, path)); + } + return routes; +} + +void TestEveryGenerationRouteIsDeclared() { + const auto generating = GeneratingHandlers(); + const auto registered = RegisteredRoutesByHandler(); + CHECK(!generating.empty()); + CHECK(!registered.empty()); + + std::set declared; + for (const GenerationRoute& route : GenerationRoutes()) { + declared.insert( + std::string(route.method) + " " + std::string(route.path)); + CHECK( + GenerationEndpointForRoute(route.method, route.path) == + std::optional(route.endpoint)); + CHECK( + requires_npu_access( + std::string(route.method), + std::string(route.path))); + } + + std::set discovered; + for (const GeneratingHandler& handler : generating) { + const auto found = registered.find(handler.name); + if (found == registered.end()) { + throw std::runtime_error( + "generation handler " + handler.name + + " is not registered on any route in server.cpp"); + } + const std::string route = + found->second.first + " " + found->second.second; + discovered.insert(route); + + // Condition 1: the route is declared. + if (!declared.contains(route)) { + throw std::runtime_error( + "route " + route + " reaches the causal engine but is " + "missing from GenerationRoutes(); it would bypass the " + "AIE4 admission rule"); + } + + // Condition 2: the handler goes through the shared rule. This is + // the one that catches the /api/chat defect, which was declared + // and routed but parsed its own limit. + for (const std::string& required : { + std::string("ParseGenerationLimit("), + std::string("GenerationLoopLimit("), + std::string("RequestedMaxNewTokens("), + std::string("RequireGenerationEndpoint(")}) { + if (handler.body.find(required) == std::string::npos) { + throw std::runtime_error( + handler.name + " serves " + route + + " but never calls " + required + + "; a generation handler must obtain its limit " + "through the shared rule, not privately"); + } + } + const std::string variant = + FindQualifiedVariantCall( + handler.body, + "GenerationLoopLimit("); + if (!variant.empty()) { + throw std::runtime_error( + handler.name + " serves " + route + " and calls " + + variant + + "; that is the private limit path /api/chat used, and it " + "bypasses the admission rule and the HTTP 400 response"); + } + + // The route it declares must be the route it is registered on, so + // a copy-pasted path cannot silently select another endpoint's + // limit field. + const auto declared_route = DeclaredRoute(handler.body); + if (declared_route != found->second) { + throw std::runtime_error( + handler.name + " is registered on " + route + + " but declares RequireGenerationEndpoint(\"" + + declared_route.first + "\", \"" + + declared_route.second + "\")"); + } + CHECK( + GenerationEndpointForRoute( + declared_route.first, + declared_route.second) + .has_value()); + } + // And no declared route is stale. + CHECK(discovered == declared); + CHECK(declared.size() == 4); + CHECK(declared.contains("POST /api/chat")); + + // A route that does not generate has no endpoint, and an unknown one + // fails loudly rather than silently defaulting. + CHECK( + !GenerationEndpointForRoute("POST", "/v1/embeddings") + .has_value()); + CHECK( + !GenerationEndpointForRoute("GET", "/api/chat").has_value()); + bool threw = false; + try { + (void)RequireGenerationEndpoint("POST", "/api/not-a-route"); + } catch (const std::logic_error&) { + threw = true; + } + CHECK(threw); + CHECK( + RequireGenerationEndpoint("POST", "/api/chat") == + GenerationEndpoint::OllamaChat); +} + +void TestNestedModelErrorAndHttpStatus() { + const ordered_json bad_request = + ModelErrorResponse("too many tokens", 400, false); + CHECK(bad_request["error"]["message"] == "too many tokens"); + CHECK(bad_request["error"]["type"] == "invalid_request_error"); + CHECK(bad_request["error"]["code"] == 400); + CHECK(bad_request["error"]["session_cleared"] == false); + CHECK(HttpStatusForResponse(bad_request) == 400); + + const ordered_json server_error = + ModelErrorResponse("submission failed", 500, true); + CHECK(server_error["error"]["message"] == "submission failed"); + CHECK(server_error["error"]["type"] == "server_error"); + CHECK(server_error["error"]["code"] == 500); + CHECK(server_error["error"]["session_cleared"] == true); + CHECK(HttpStatusForResponse(server_error) == 500); + + CHECK(HttpStatusForResponse(ordered_json{{"ok", true}}) == 200); + CHECK( + HttpStatusForResponse( + ordered_json{{"error", "legacy string error"}}) == 200); + CHECK(!UseFinalStreamingErrorChunk(false)); + CHECK(UseFinalStreamingErrorChunk(true)); +} + +void TestOpenAiStreamingErrorFramingAndParsing() { + const ordered_json error = + ModelErrorResponse("submission failed", 500, true); + std::vector> transmitted; + SendOpenAiStreamingError( + error, + [&](const ordered_json& data, bool is_final) { + CHECK(data.is_string()); + transmitted.emplace_back( + data.get(), + is_final); + }); + const std::string expected_error = + "data: " + error.dump() + "\n\n"; + + CHECK(transmitted.size() == 2); + CHECK(transmitted[0].first == expected_error); + CHECK(!transmitted[0].second); + CHECK(transmitted[1].first == "data: [DONE]\n\n"); + CHECK(transmitted[1].second); + + const auto parse_sse_data = [](const std::string& frame) { + CHECK(frame.starts_with("data: ")); + CHECK(frame.ends_with("\n\n")); + return frame.substr(6, frame.size() - 8); + }; + const ordered_json parsed = + ordered_json::parse( + parse_sse_data(transmitted[0].first)); + CHECK(parsed["error"]["session_cleared"] == true); + CHECK(parse_sse_data(transmitted[1].first) == "[DONE]"); +} + +void TestCliLimitAndRecoverableNotice() { + CHECK(!CliRequestedMaxNewTokens(-1).has_value()); + CHECK(!CliRequestedMaxNewTokens(0).has_value()); + CHECK(CliRequestedMaxNewTokens(17) == std::optional(17)); + + CHECK( + CliModelErrorNotice("capacity exceeded", false) == + "ERROR: capacity exceeded"); + // "at a recoverable point", not "before submission": the engine's + // irrevocable boundary is per submission group, so this notice is also + // shown for a failure past a COMPLETED synchronize. Pinning the old + // wording here is what would let the retraction reach the code and stop. + CHECK( + CliModelErrorNotice("ignored", true) == + "ERROR: AIE4 inference failed at a recoverable point; the " + "current conversation was cleared."); + CHECK( + CliModelErrorNotice("ignored", true).find("before submission") == + std::string::npos); +} + +void TestAie4ModelInfoDetection() { + CHECK( + IsCorelibAie4ModelInfo( + ordered_json{ + {"details", {{"execution_backend", "corelib_aie4"}}}})); + CHECK( + !IsCorelibAie4ModelInfo( + ordered_json{{"details", {{"family", "phi4"}}}})); + CHECK(!IsCorelibAie4ModelInfo(ordered_json::object())); +} + +enum class HandlerExit { + NonStreaming, + Streaming, + Exception, + CancellationFinal +}; + +void RunProductionShapedGateInterleaving(HandlerExit exit) { + CHECK(NPUAccessManager::is_npu_available()); + CHECK(NPUAccessManager::get_active_npu_requests() == 0); + CHECK(NPUAccessManager::try_acquire_npu_access()); + + std::mutex mutex; + std::condition_variable ready; + bool final_response_sent = false; + bool second_attempted_while_handler_active = false; + bool second_attempt_rejected = false; + bool response_was_streaming = false; + bool cancellation_finalized = false; + bool post_response_cleanup_completed = false; + bool handler_returned = false; + bool completion_point_reached = false; + bool second_insert_entered = false; + bool second_entered_before_cleanup = false; + std::exception_ptr second_error; + + std::thread second([&] { + try { + { + std::unique_lock lock(mutex); + ready.wait(lock, [&] { return final_response_sent; }); + } + + const bool acquired_early = + NPUAccessManager::try_acquire_npu_access(); + { + std::lock_guard lock(mutex); + second_attempted_while_handler_active = true; + second_attempt_rejected = !acquired_early; + } + ready.notify_all(); + if (acquired_early) { + NPUAccessManager::release_npu_access(); + return; + } + + { + std::unique_lock lock(mutex); + ready.wait(lock, [&] { return completion_point_reached; }); + } + if (!NPUAccessManager::try_acquire_npu_access()) { + throw std::runtime_error( + "queued request did not acquire released NPU gate"); + } + { + std::lock_guard lock(mutex); + second_insert_entered = true; + second_entered_before_cleanup = + !post_response_cleanup_completed || + !handler_returned; + } + NPUAccessManager::release_npu_access(); + } catch (...) { + second_error = std::current_exception(); + ready.notify_all(); + } + }); + + { + NPURequestCompletionGuard completion([&] { + NPUAccessManager::release_npu_access(); + { + std::lock_guard lock(mutex); + completion_point_reached = true; + } + ready.notify_all(); + }); + + try { + { + std::lock_guard lock(mutex); + response_was_streaming = + exit == HandlerExit::Streaming || + exit == HandlerExit::CancellationFinal; + cancellation_finalized = + exit == HandlerExit::CancellationFinal; + final_response_sent = true; + } + ready.notify_all(); + { + std::unique_lock lock(mutex); + ready.wait( + lock, + [&] { + return second_attempted_while_handler_active; + }); + } + + if (exit == HandlerExit::Exception) { + throw std::runtime_error("forced handler failure"); + } + } catch (const std::exception&) { + // Production process_task catches before its completion guard exits. + } + + { + std::lock_guard lock(mutex); + post_response_cleanup_completed = true; + handler_returned = true; + } + } + + second.join(); + if (second_error) { + std::rethrow_exception(second_error); + } + + CHECK(second_attempt_rejected); + CHECK( + response_was_streaming == + (exit == HandlerExit::Streaming || + exit == HandlerExit::CancellationFinal)); + CHECK( + cancellation_finalized == + (exit == HandlerExit::CancellationFinal)); + CHECK(post_response_cleanup_completed); + CHECK(handler_returned); + CHECK(second_insert_entered); + CHECK(!second_entered_before_cleanup); + CHECK(NPUAccessManager::is_npu_available()); + CHECK(NPUAccessManager::get_active_npu_requests() == 0); +} + +void TestCompleteRequestGate() { + CHECK(requires_npu_access("POST", "/api/generate")); + CHECK(requires_npu_access("POST", "/api/chat")); + CHECK(requires_npu_access("POST", "/v1/chat/completions")); + CHECK(requires_npu_access("POST", "/v1/completions")); + CHECK(!requires_npu_access("GET", "/v1/completions")); + + for (const HandlerExit exit : { + HandlerExit::NonStreaming, + HandlerExit::Streaming, + HandlerExit::Exception, + HandlerExit::CancellationFinal}) { + RunProductionShapedGateInterleaving(exit); + } + + CHECK(NPUAccessManager::try_acquire_npu_access()); + { + NPURequestCompletionGuard completion([] { + throw std::runtime_error("forced queue handoff failure"); + }); + } + CHECK(NPUAccessManager::is_npu_available()); + CHECK(NPUAccessManager::get_active_npu_requests() == 0); +} + +volatile std::sig_atomic_t g_signal_observation = 0; + +void PriorSignalHandler(int) { + g_signal_observation = 1; +} + +void ServeSignalHandler(int) { + g_signal_observation = 2; +} + +void TestServeSignalScopeAndShutdownOrder() { + const auto original = std::signal(SIGINT, PriorSignalHandler); + CHECK(original != SIG_ERR); + { + ScopedSignalHandler scope(SIGINT, ServeSignalHandler); + g_signal_observation = 0; + CHECK(std::raise(SIGINT) == 0); + CHECK(g_signal_observation == 2); + } + + g_signal_observation = 0; + CHECK(std::raise(SIGINT) == 0); + CHECK(g_signal_observation == 1); + CHECK(std::signal(SIGINT, original) != SIG_ERR); + + std::vector order; + const bool healthy = CompleteServeShutdown( + [&] { order.push_back("stop-admission-and-wait-inflight"); }, + [&] { order.push_back("destroy-server-handler-engine"); }, + [&] { + order.push_back("shutdown-corelib"); + return true; + }); + CHECK(healthy); + CHECK( + order == std::vector({ + "stop-admission-and-wait-inflight", + "destroy-server-handler-engine", + "shutdown-corelib"})); +} + +} // namespace + +int main() { + try { + TestHandlerSpecificPresence(); + TestEndpointDefaultsAndPropagation(); + TestNonPositiveLimitSentinelsRequestNoBudget(); + TestOllamaChatLegacyLimitIsUnchanged(); + TestOllamaChatAie4LimitAndAdmission(); + TestEveryGenerationRouteIsDeclared(); + TestNestedModelErrorAndHttpStatus(); + TestOpenAiStreamingErrorFramingAndParsing(); + TestCliLimitAndRecoverableNotice(); + TestAie4ModelInfoDetection(); + TestCompleteRequestGate(); + TestServeSignalScopeAndShutdownOrder(); + std::cout << "test_generation_limit: PASS\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << "test_generation_limit: FAIL: " + << error.what() << '\n'; + return 1; + } +} diff --git a/src/test/phi4_corelib_aie4/test_model_catalog.cpp b/src/test/phi4_corelib_aie4/test_model_catalog.cpp new file mode 100644 index 00000000..774eecb4 --- /dev/null +++ b/src/test/phi4_corelib_aie4/test_model_catalog.cpp @@ -0,0 +1,401 @@ +#include "test_support.hpp" + +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace { + +using nlohmann::json; + +constexpr std::string_view kTag = "phi4-mini-it-aie4:4b"; +constexpr std::string_view kCommit = + "e751fb68c2cfffe6b0d32942118f75ac0a0365bb"; +constexpr std::string_view kRepository = + "https://huggingface.co/amd/phi-4-mini-instruct-oga-dml"; + +json ReadJson(const std::filesystem::path& path) { + std::ifstream input(path, std::ios::binary); + if (!input) { + throw std::runtime_error("failed to open " + path.string()); + } + return json::parse(input); +} + +std::uint64_t FileSize(const std::filesystem::path& path) { + return static_cast( + std::filesystem::file_size(path)); +} + +std::set JsonStringSet(const json& values) { + std::set result; + for (const auto& value : values) { + result.insert(value.get()); + } + return result; +} + +class TempDirectory final { +public: + TempDirectory() { + const auto parent = std::filesystem::temp_directory_path(); + for (int attempt = 0; attempt < 100; ++attempt) { + path_ = parent / + ("flm-model-overlay-" + + std::to_string(GetCurrentProcessId()) + "-" + + std::to_string(GetTickCount64()) + "-" + + std::to_string(attempt)); + std::error_code error; + if (std::filesystem::create_directory(path_, error)) { + return; + } + } + throw std::runtime_error("failed to create temporary directory"); + } + + ~TempDirectory() { + std::error_code ignored; + std::filesystem::remove_all(path_, ignored); + } + + const std::filesystem::path& path() const noexcept { + return path_; + } + +private: + std::filesystem::path path_; +}; + +void TestCatalogContract() { + const std::filesystem::path source(FLM_TEST_SOURCE_DIR); + const json catalog = ReadJson(source / "model_list.json"); + const json metadata = ReadJson(source / "model_info.json"); + const auto& model = + catalog.at("models").at("phi4-mini-it-aie4").at("4b"); + + CHECK(model.at("url").get() == kRepository); + CHECK(model.at("revision").get() == kCommit); + const std::string expected_api = + "https://huggingface.co/api/models/amd/" + "phi-4-mini-instruct-oga-dml/tree/" + + std::string(kCommit) + + "?recursive=true&expand=false"; + CHECK(model.at("file_url") == expected_api); + CHECK(model.at("flm_min_version") == "1.0.4"); + CHECK(model.at("default_context_length") == 4096); + CHECK(model.at("max_prefill_len") == 4096); + CHECK(model.at("vlm") == false); + CHECK(model.at("modelscope_supported") == false); + CHECK(!model.contains("ms_url")); + CHECK(model.at("details").at("family") == "phi4"); + CHECK( + model.at("details").at("execution_backend") == + "corelib_aie4"); + + // Design 8.1. `.gitattributes` is a Git repository artifact and + // `genai_config.json` is excluded by MODEL-2: flm.exe runs no ORT or genai + // graph, and an unused configuration invites a future reader to believe it + // is authoritative. `chat_template.jinja` is kept deliberately, because it + // is the verbatim source of the template the overlay inlines and is what + // makes the overlay auditable on the target machine. + const std::set expected_files{ + "added_tokens.json", + "chat_template.jinja", + "config.json", + "corelib_phi4_manifest.json", + "merges.txt", + "model.onnx", + "model.onnx.data", + "provenance.json", + "special_tokens_map.json", + "tokenizer.json", + "tokenizer_config.json", + "vocab.json", + }; + CHECK(JsonStringSet(model.at("files")) == expected_files); + + const auto& overlays = model.at("bundled_overlays"); + CHECK(overlays.size() == 4); + CHECK(overlays.contains("config.json")); + CHECK(overlays.contains("corelib_phi4_manifest.json")); + CHECK(overlays.contains("provenance.json")); + CHECK(overlays.contains("tokenizer_config.json")); + + const std::filesystem::path overlay_root = + source / "model_overlays"; + std::uint64_t overlay_size = 0; + for (const auto& [target, record] : overlays.items()) { + const auto path = overlay_root / record.at("path").get(); + CHECK(std::filesystem::is_regular_file(path)); + CHECK(record.at("size").get() == FileSize(path)); + CHECK( + record.at("sha256") == + flm::pull::CalculateFileSha256(path)); + overlay_size += FileSize(path); + } + + const auto& remote = metadata.at(kTag); + CHECK(remote.size() == 11); + CHECK( + std::is_sorted( + remote.begin(), + remote.end(), + [](const json& left, const json& right) { + return left.at("path").get() < + right.at("path").get(); + })); + const auto find_record = [&](std::string_view path) -> const json& { + const auto found = std::find_if( + remote.begin(), + remote.end(), + [&](const json& record) { + return record.at("path").get() == path; + }); + CHECK(found != remote.end()); + return *found; + }; + CHECK( + find_record("model.onnx").at("oid") == + "df6d4309745d4627b82fd92c615589193ea528db"); + CHECK( + find_record("model.onnx").at("lfs").at("oid") == + "e80b9d83e018784eda6263d09fa2ab7729087722c6073c72123366f2dbec4529"); + CHECK( + find_record("model.onnx.data").at("lfs").at("size") == + UINT64_C(3248488448)); + CHECK( + find_record("tokenizer.json").at("lfs").at("oid") == + "382cc235b56c725945e149cc25f191da667c836655efd0857b004320e90e91ea"); + + // `size` and `footprint` cover the assembled on-disk directory: the + // upstream files actually downloaded plus the overlay files shipped inside + // FastFlow. Deriving it from the metadata here, rather than restating a + // literal, is what keeps the number honest when the file set changes. + std::uint64_t upstream_size = 0; + for (const auto& file : model.at("files")) { + const auto name = file.get(); + if (overlays.contains(name)) { + continue; + } + upstream_size += find_record(name).at("size").get(); + } + const std::uint64_t expected_size = upstream_size + overlay_size; + CHECK(model.at("size").get() == expected_size); + const double expected_footprint = + std::round( + static_cast(expected_size) / + static_cast(UINT64_C(1024) * 1024 * 1024) * + 100.0) / + 100.0; + CHECK(model.at("footprint").get() == expected_footprint); +} + +// Design `PACKAGE-1`. The assembled package has two provenances and they are +// checked separately. Requiring every catalog file to carry a Hugging Face +// metadata record cannot hold: the overlay files are FastFlow-authored and do +// not exist upstream by construction. The check that matters is the reverse +// one. +void TestSplitProvenance() { + const std::filesystem::path source(FLM_TEST_SOURCE_DIR); + const json catalog = ReadJson(source / "model_list.json"); + const json metadata = ReadJson(source / "model_info.json"); + const auto& model = + catalog.at("models").at("phi4-mini-it-aie4").at("4b"); + const auto& overlays = model.at("bundled_overlays"); + const auto& remote = metadata.at(kTag); + + std::set upstream_paths; + for (const auto& record : remote) { + upstream_paths.insert(record.at("path").get()); + } + + // Upstream files: every one must have a metadata record to download and + // hash-check against. + for (const auto& file : model.at("files")) { + const auto name = file.get(); + if (overlays.contains(name)) { + continue; + } + CHECK(upstream_paths.count(name) == 1); + } + + // Overlay files: each must be installed beside FastFlow. Three of them + // must have no upstream record at all; an upstream record for one would + // mean FastFlow's contract was published to the model repository, which + // makes the two provenances indistinguishable. + const std::filesystem::path overlay_root = source / "model_overlays"; + for (const auto& name : { + std::string("config.json"), + std::string("corelib_phi4_manifest.json"), + std::string("provenance.json")}) { + CHECK(overlays.contains(name)); + CHECK(upstream_paths.count(name) == 0); + CHECK( + std::filesystem::is_regular_file( + overlay_root / + overlays.at(name).at("path").get())); + } + + // tokenizer_config.json is the one overlay that also exists upstream. The + // overlay shadows it because the published file carries neither a chat + // template nor eos_token_id, so its upstream record is expected. + CHECK(overlays.contains("tokenizer_config.json")); + CHECK(upstream_paths.count("tokenizer_config.json") == 1); + + // Nothing upstream is silently dropped: every record is either packaged or + // one of the two files excluded on purpose. + const std::set excluded{".gitattributes", "genai_config.json"}; + const auto packaged = JsonStringSet(model.at("files")); + for (const auto& path : upstream_paths) { + CHECK(packaged.count(path) == 1 || excluded.count(path) == 1); + } + for (const auto& name : excluded) { + CHECK(upstream_paths.count(name) == 1); + CHECK(packaged.count(name) == 0); + } +} + +// `ModelDownloader::check_model_compatibility` compares three versions: the +// overlay config.json's flm_version, the catalog's flm_min_version, and the +// binary's own __FLM_VERSION__. A binary built as 1.0.3 reports Incompatible +// for a 1.0.4 overlay and refuses its own catalog entry, which is a failure +// that only appears at first real load. Pin the relationship here instead. +void TestVersionGateIsSelfConsistent() { + const std::filesystem::path source(FLM_TEST_SOURCE_DIR); + const json catalog = ReadJson(source / "model_list.json"); + const auto& model = + catalog.at("models").at("phi4-mini-it-aie4").at("4b"); + const json overlay_config = + ReadJson(source / "model_overlays" / "phi4-mini-it-aie4" / + "config.json"); + + const auto encode = [](const std::string& version) -> std::uint32_t { + int major = -1; + int minor = -1; + int patch = -1; + CHECK( + std::sscanf(version.c_str(), "%d.%d.%d", &major, &minor, &patch) == + 3); + CHECK(major >= 0 && minor >= 0 && patch >= 0); + return static_cast( + major * 1000000 + minor * 1000 + patch); + }; + + const auto minimum = model.at("flm_min_version").get(); + const auto packaged = overlay_config.at("flm_version").get(); + CHECK(packaged == minimum); + CHECK(encode(packaged) >= encode(minimum)); + CHECK(encode(std::string(__FLM_VERSION__)) >= encode(packaged)); +} + +void TestSourcePolicyAndPinnedUrls() { + const std::filesystem::path source(FLM_TEST_SOURCE_DIR); + const json catalog = ReadJson(source / "model_list.json"); + const auto& model = + catalog.at("models").at("phi4-mini-it-aie4").at("4b"); + + flm::pull::RequireSupportedModelSource(model, false); + CheckThrowsContains( + [&] { flm::pull::RequireSupportedModelSource(model, true); }, + "does not support --modelscope"); + CHECK( + flm::pull::BuildRemoteFileUrl(model, "model.onnx") == + std::string(kRepository) + "/resolve/" + + std::string(kCommit) + "/model.onnx?download=true"); + + const auto remote_files = flm::pull::RemoteModelFiles(model); + CHECK( + std::find( + remote_files.begin(), + remote_files.end(), + "tokenizer_config.json") == remote_files.end()); + CHECK( + std::find( + remote_files.begin(), + remote_files.end(), + "model.onnx") != remote_files.end()); +} + +void TestBundledOverlayCopyAndIntegrity() { + const std::filesystem::path source(FLM_TEST_SOURCE_DIR); + const json catalog = ReadJson(source / "model_list.json"); + const auto& model = + catalog.at("models").at("phi4-mini-it-aie4").at("4b"); + const auto overlay_root = source / "model_overlays"; + + TempDirectory temporary; + const auto model_dir = temporary.path() / "model"; + flm::pull::StageBundledOverlays(model, overlay_root, model_dir); + + for (const auto& [target, record] : + model.at("bundled_overlays").items()) { + const auto installed = model_dir / target; + CHECK(std::filesystem::is_regular_file(installed)); + CHECK( + flm::pull::CalculateFileSha256(installed) == + record.at("sha256").get()); + } + + const auto corrupt_root = temporary.path() / "corrupt"; + std::filesystem::create_directories( + corrupt_root / "phi4-mini-it-aie4"); + for (const auto& [target, record] : + model.at("bundled_overlays").items()) { + std::filesystem::copy_file( + overlay_root / record.at("path").get(), + corrupt_root / record.at("path").get()); + } + { + std::ofstream corrupt( + corrupt_root / "phi4-mini-it-aie4" / "config.json", + std::ios::binary | std::ios::trunc); + corrupt << std::string( + model.at("bundled_overlays") + .at("config.json") + .at("size") + .get(), + 'x'); + } + CheckThrowsContains( + [&] { + flm::pull::StageBundledOverlays( + model, + corrupt_root, + temporary.path() / "rejected"); + }, + "bundled overlay SHA-256 mismatch"); + CHECK( + !std::filesystem::exists( + temporary.path() / "rejected" / "config.json")); +} + +} // namespace + +int main() { + try { + TestCatalogContract(); + TestSplitProvenance(); + TestVersionGateIsSelfConsistent(); + TestSourcePolicyAndPinnedUrls(); + TestBundledOverlayCopyAndIntegrity(); + std::cout << "model catalog tests passed\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/src/test/phi4_corelib_aie4/test_packaged_runtime.ps1 b/src/test/phi4_corelib_aie4/test_packaged_runtime.ps1 new file mode 100644 index 00000000..eb38d892 --- /dev/null +++ b/src/test/phi4_corelib_aie4/test_packaged_runtime.ps1 @@ -0,0 +1,1014 @@ +# Packaging and clean-environment tests for the optional Phi-4 AIE4 feature. +# +# `-CorelibRuntimeDir` (with `-DependencyDir`) enables the checks that matter +# most, and they are the ones that cannot be faked: the closure is derived from +# the real ryzenai_corelib.dll, loaded from the staged directory with +# development paths removed, and then re-loaded with one staged DLL at a time +# hidden. Without the negative control a green result proves nothing, because +# an ambient conda or toolchain prefix supplies the missing DLL on precisely +# the machine that built the binary. + +param( + [string]$FlmExe = "", + [string]$CorelibRuntimeDir = "", + [string]$XrtRuntimeDir = "", + [string]$DependencyDir = "", + # The CMake to configure the fixture with. CMakeLists passes + # ${CMAKE_COMMAND}, so the guard runs against the same CMake that + # configured the suite. The default is only for a hand-run. + # + # Bare `cmake` is not good enough. On the AIE4 target PATH finds Cygwin's + # CMake 3.14.5, which is below the 3.24 that StageAie4Runtime.cmake + # requires for file(GET_RUNTIME_DEPENDENCIES) -- so the closure guard + # failed for a reason that had nothing to do with the closure. + [string]$CmakeExe = "cmake", + [switch]$RunAie4ModelLoad +) + +$ErrorActionPreference = "Stop" + +# Without this, a terminating error inside the script's try/finally exits 1 +# with NOTHING on either stream when the script is launched with `pwsh -File`. +# That is how the CMake version problem above stayed invisible: the packaging +# guard reported failure and said nothing about it, which is the worst thing a +# guard can do. +trap { + [Console]::Error.WriteLine("test_packaged_runtime: $($_.Exception.Message)") + [Console]::Error.WriteLine($_.ScriptStackTrace) + exit 1 +} + +# Skipped work is reported as skipped, never as passed. A script that prints a +# success line for blocks it never entered reads as coverage it does not have, +# which is the defect test_real_corelib had to fix in Task 10R. Exit code 77 is +# CTest's SKIP_RETURN_CODE, so a partial run shows as Skipped rather than +# Passed. +$ran = @() +$skipped = @() +$SKIP_EXIT = 77 + +$sourceRoot = (Resolve-Path (Join-Path $PSScriptRoot "../..")).Path +$modulePath = Join-Path $sourceRoot "cmake/ConfigureAie4Runtime.cmake" +$stageScript = Join-Path $sourceRoot "cmake/StageAie4Runtime.cmake" +$temporary = Join-Path ([System.IO.Path]::GetTempPath()) ( + "flm-aie4-package-{0}-{1}" -f $PID, [DateTime]::UtcNow.Ticks) + +Add-Type @" +using System; +using System.Runtime.InteropServices; +public static class FlmAie4Loader { + [DllImport("kernel32", CharSet = CharSet.Unicode, SetLastError = true)] + public static extern IntPtr LoadLibraryEx( + string path, IntPtr file, uint flags); + [DllImport("kernel32", SetLastError = true)] + public static extern bool FreeLibrary(IntPtr module); +} +"@ + +# Get-FileHash is not reliably resolvable in the constrained -NoProfile host +# CTest launches, so hash through .NET directly rather than depending on module +# autoloading. +function Get-Sha256Hex { + param([string]$Path) + $algorithm = [System.Security.Cryptography.SHA256]::Create() + try { + $stream = [System.IO.File]::OpenRead($Path) + try { + $bytes = $algorithm.ComputeHash($stream) + } finally { + $stream.Dispose() + } + } finally { + $algorithm.Dispose() + } + return ( + -join ($bytes | ForEach-Object { $_.ToString("x2") }) + ) +} + +function Invoke-Configure { + param( + [string]$Build, + [bool]$Enabled, + [string]$Corelib = "", + [string]$Xrt = "", + [string]$Dependency = "", + [bool]$ExpectSuccess = $true + ) + $arguments = @( + "-S", (Join-Path $temporary "fixture"), + "-B", $Build, + "-DFLM_ENABLE_CORELIB_AIE4=$( + if ($Enabled) { "ON" } else { "OFF" })" + ) + if ($Corelib) { + $arguments += "-DRYZENAI_CORELIB_RUNTIME_DIR=$Corelib" + } + if ($Xrt) { + $arguments += "-DXRT_RUNTIME_DIR=$Xrt" + } + if ($Dependency) { + $arguments += "-DFLM_AIE4_DEPENDENCY_DIRS=$Dependency" + } + $previousPreference = $ErrorActionPreference + $ErrorActionPreference = "Continue" + $output = (& $CmakeExe @arguments 2>&1 | + ForEach-Object { $_.ToString() } | + Out-String) + $exitCode = $LASTEXITCODE + $ErrorActionPreference = $previousPreference + if (($exitCode -eq 0) -ne $ExpectSuccess) { + throw "Unexpected CMake configure result.`n$output" + } + return $output +} + +function Invoke-Install { + param( + [string]$Build, + [string]$Prefix, + [bool]$ExpectSuccess = $true + ) + $previousPreference = $ErrorActionPreference + $ErrorActionPreference = "Continue" + $output = (& $CmakeExe --install $Build --prefix $Prefix 2>&1 | + ForEach-Object { $_.ToString() } | + Out-String) + $exitCode = $LASTEXITCODE + $ErrorActionPreference = $previousPreference + if (($exitCode -eq 0) -ne $ExpectSuccess) { + throw "Unexpected CMake install result.`n$output" + } + return $output +} + +# Loads the corelib DLL by absolute path from $Dir with development paths +# removed. LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR | LOAD_LIBRARY_SEARCH_DEFAULT_DIRS +# resolves dependencies from the staged directory and the approved system +# directories only; PATH is not consulted at all, which is the product's own +# search behaviour and the point of design CLOSURE-2. +function Invoke-CleanEnvironmentLoad { + param([string]$Dir) + $savedPath = $env:PATH + $savedCorelib = $env:RYZENAI_CORELIB_PATH + $savedXrt = $env:XILINX_XRT + try { + $env:PATH = "$env:SystemRoot\System32;$env:SystemRoot" + Remove-Item Env:RYZENAI_CORELIB_PATH -ErrorAction SilentlyContinue + Remove-Item Env:XILINX_XRT -ErrorAction SilentlyContinue + $module = [FlmAie4Loader]::LoadLibraryEx( + (Join-Path $Dir "ryzenai_corelib.dll"), + [IntPtr]::Zero, + 0x00000100 -bor 0x00001000) + if ($module -eq [IntPtr]::Zero) { + return [Runtime.InteropServices.Marshal]::GetLastWin32Error() + } + [FlmAie4Loader]::FreeLibrary($module) | Out-Null + return 0 + } finally { + $env:PATH = $savedPath + if ($null -eq $savedCorelib) { + Remove-Item Env:RYZENAI_CORELIB_PATH -ErrorAction SilentlyContinue + } else { + $env:RYZENAI_CORELIB_PATH = $savedCorelib + } + if ($null -eq $savedXrt) { + Remove-Item Env:XILINX_XRT -ErrorAction SilentlyContinue + } else { + $env:XILINX_XRT = $savedXrt + } + } +} + +try { + $inno = Get-Content (Join-Path $sourceRoot "inno/flm.iss") -Raw + foreach ($required in @( + 'AppVersion=1.0.4', + 'Name: "aie4runtime"', + 'corelib_phi4_manifest.json', + 'tokenizer_config.json', + 'config.json', + 'provenance.json' + )) { + if ($inno -notmatch [regex]::Escape($required)) { + throw "Inno manifest is missing: $required" + } + } + $wix = Get-Content (Join-Path $sourceRoot "wix/flm.wxs") -Raw + foreach ($required in @( + 'Version="1.0.4"', + 'Feature Id="Aie4Feature"', + 'ComponentGroup Id="Aie4OverlayComponents"', + 'corelib_phi4_manifest.json', + 'tokenizer_config.json', + 'config.json', + 'provenance.json' + )) { + if ($wix -notmatch [regex]::Escape($required)) { + throw "WiX manifest is missing: $required" + } + } + [xml]$parsedWix = $wix + $ran += "installer-manifests" + + # "The main installer must still build with no AIE4 closure present" was + # asserted here by forbidding the substring `exit /b 1` anywhere in either + # get_files.bat. That guard is retired, deliberately. + # + # It never checked the property it was named for. It checked that the + # scripts contain no error handling at all -- so the moment a DIFFERENT + # failure legitimately needed reporting (the 2026-09-01 decision to ship + # the closure report, which must fail loudly when the report is absent) + # the guard fired on correct code while still being unable to notice a + # regression that failed the no-closure path some other way, such as + # `goto :eof` after an echo, or a non-zero errorlevel left by the last + # command in the else branch. + # + # The property is now proven by RUNNING both shipped scripts with no + # closure staged and requiring exit 0 -- see the installer-staging block + # below, which reports under this same name. + + # flm.exe must never gain a link-time dependency on ryzenai_corelib. The + # DLL is resolved at run time by absolute path precisely so that a binary + # without the AIE4 runtime installed still starts. dumpbin on the packaged + # exe is the end check, but it only runs where an exe exists; this catches + # the regression at its source. + # + # The scan reads whole balanced-paren command invocations, not lines. Every + # link call in this tree spans multiple lines -- the AIE4 target's own does + # -- so a line-at-a-time match would miss the exact form the regression + # would take and report success for a check it never performed. + $cmakeFiles = @( + Get-ChildItem -Path $sourceRoot -Recurse -File -Include @( + "CMakeLists.txt", "*.cmake" + ) | Where-Object { + $_.FullName -notmatch "[\\/](build|out|third_party)[^\\/]*[\\/]" + } + ) + if ($cmakeFiles.Count -lt 3) { + throw "The link-guard scan found only $($cmakeFiles.Count) CMake files" + } + $scannedLinkCalls = 0 + foreach ($file in $cmakeFiles) { + $text = Get-Content $file.FullName -Raw + if ($null -eq $text) { continue } + # Strip line comments so a commented-out example cannot trip the guard + # and, more importantly, cannot hide a real call behind an unbalanced + # parenthesis inside a comment. + $text = [regex]::Replace($text, '(?m)#.*$', '') + foreach ($match in [regex]::Matches( + $text, '(?i)\b(target_link_libraries|link_libraries)\s*\(')) { + $depth = 1 + $index = $match.Index + $match.Length + while ($index -lt $text.Length -and $depth -gt 0) { + if ($text[$index] -eq '(') { $depth++ } + elseif ($text[$index] -eq ')') { $depth-- } + $index++ + } + if ($depth -ne 0) { + throw "Unbalanced $($match.Value) in $($file.FullName)" + } + $scannedLinkCalls++ + $body = $text.Substring( + $match.Index, $index - $match.Index) + if ($body -match '(?i)\bryzenai_corelib\b') { + $relative = $file.FullName.Substring($sourceRoot.Length + 1) + throw ( + "$relative links ryzenai_corelib: " + + ($body -replace '\s+', ' ')) + } + } + } + if ($scannedLinkCalls -lt 1) { + throw "The link-guard scan matched no link calls; it is inert" + } + $ran += "no-corelib-import-guard ($scannedLinkCalls link calls)" + + New-Item -ItemType Directory -Path $temporary | Out-Null + $fixture = Join-Path $temporary "fixture" + New-Item -ItemType Directory -Path $fixture | Out-Null + @" +cmake_minimum_required(VERSION 3.24) +project(flm_aie4_packaging NONE) +option(FLM_ENABLE_CORELIB_AIE4 "" OFF) +include("$($modulePath.Replace('\', '/'))") +flm_aie4_warn_if_unstageable() +flm_aie4_install_runtime(DESTINATION bin/aie4 COMPONENT AIE4) +"@ | Set-Content -Path (Join-Path $fixture "CMakeLists.txt") -Encoding utf8 + + # Feature OFF: no corelib runtime is required and nothing is staged. + $offBuild = Join-Path $temporary "off" + Invoke-Configure -Build $offBuild -Enabled $false | Out-Null + Invoke-Install -Build $offBuild -Prefix (Join-Path $temporary "off-stage") | + Out-Null + if (Test-Path (Join-Path $temporary "off-stage/bin/aie4")) { + throw "Feature OFF unexpectedly staged an AIE4 directory" + } + + # Feature ON without a runtime directory: configuring must SUCCEED, because + # flm.exe resolves the corelib DLL at run time by absolute path and never + # links its import library. Only the install step needs the runtime. + $devBuild = Join-Path $temporary "dev" + $devOutput = Invoke-Configure -Build $devBuild -Enabled $true + if ($devOutput -notmatch "RYZENAI_CORELIB_RUNTIME_DIR") { + throw "Feature-ON configure did not warn about the missing runtime" + } + $devInstall = Invoke-Install ` + -Build $devBuild ` + -Prefix (Join-Path $temporary "dev-stage") ` + -ExpectSuccess $false + if ($devInstall -notmatch "RYZENAI_CORELIB_RUNTIME_DIR") { + throw "Install-time failure was not actionable.`n$devInstall" + } + + # A configured but wrong runtime directory must fail at install, loudly. + $badBuild = Join-Path $temporary "bad" + Invoke-Configure ` + -Build $badBuild ` + -Enabled $true ` + -Corelib (Join-Path $temporary "does-not-exist") | Out-Null + $badInstall = Invoke-Install ` + -Build $badBuild ` + -Prefix (Join-Path $temporary "bad-stage") ` + -ExpectSuccess $false + if ($badInstall -notmatch "RYZENAI_CORELIB_RUNTIME_DIR") { + throw "Missing runtime directory was not reported.`n$badInstall" + } + + # A directory that exists but holds no ryzenai_corelib.dll is the mistake a + # packager is most likely to make, so it gets its own named failure rather + # than an unresolved-import message later. + $emptyCorelib = Join-Path $temporary "empty-corelib" + New-Item -ItemType Directory -Path $emptyCorelib | Out-Null + $emptyBuild = Join-Path $temporary "empty" + Invoke-Configure ` + -Build $emptyBuild ` + -Enabled $true ` + -Corelib $emptyCorelib | Out-Null + $emptyInstall = Invoke-Install ` + -Build $emptyBuild ` + -Prefix (Join-Path $temporary "empty-stage") ` + -ExpectSuccess $false + if ($emptyInstall -notmatch "no ryzenai_corelib\.dll") { + throw "Empty runtime directory was not reported.`n$emptyInstall" + } + $ran += "packaging-configure-and-install-contract" + + if ($CorelibRuntimeDir) { + $resolvedCorelib = (Resolve-Path $CorelibRuntimeDir).Path + # -DependencyDir is a semicolon-separated list, matching the CMake + # cache variable it feeds. Resolving each entry separately is also what + # exercises the multi-entry path end to end: a single interpolated + # string silently loses everything after the first `;`. + $resolvedDependencies = @( + $DependencyDir -split ';' | + Where-Object { $_ } | + ForEach-Object { (Resolve-Path $_).Path } + ) + $realBuild = Join-Path $temporary "real" + Invoke-Configure ` + -Build $realBuild ` + -Enabled $true ` + -Corelib $resolvedCorelib ` + -Xrt $(if ($XrtRuntimeDir) { + (Resolve-Path $XrtRuntimeDir).Path } else { "" }) ` + -Dependency ($resolvedDependencies -join ';') | Out-Null + $realStage = Join-Path $temporary "real-stage" + Invoke-Install -Build $realBuild -Prefix $realStage | Out-Null + $stagedDir = Join-Path $realStage "bin/aie4" + + # CLOSURE-1: the staged set is whatever the walker derived from this + # exact binary, so the test asserts properties of the derivation rather + # than a transcribed list that would silently disagree with the other + # DynamicDispatch linkage. + $report = Join-Path $stagedDir "aie4-closure.txt" + if (-not (Test-Path $report)) { + throw "The install did not record a derived closure report" + } + $derived = @( + Get-Content $report | + Where-Object { $_ -like "staged`t*" } | + ForEach-Object { ($_ -split "`t")[1] } + ) + if ($derived -notcontains "ryzenai_corelib.dll") { + throw "The derived closure does not contain ryzenai_corelib.dll" + } + $staged = @( + Get-ChildItem $stagedDir -File | + Where-Object { $_.Extension -eq ".dll" } | + ForEach-Object Name + ) + if (Compare-Object ($derived | Sort-Object) ($staged | Sort-Object)) { + throw "Staged AIE4 files do not match the derived closure" + } + if ($derived -contains "msvcp140.dll") { + throw "The closure staged a build-machine Visual C++ runtime" + } + # The report ships inside bin/aie4, so it must not carry absolute + # paths from the machine that built it. The audit record that does + # name them stays in the build tree. + foreach ($line in Get-Content $report) { + if ($line -match '(?i)[a-z]:[\\/]') { + throw "The shipped closure report leaks a build path: $line" + } + } + # The corelib's own identity, ahead of the file list. Human decision, + # 2026-09-01: the shipped artifact must answer "which runtime produced + # this" without reference to a build tree that travelled nowhere. + # + # Asserted by key, not by position, and asserted to be PRESENT -- + # dropping the identity silently is the failure mode that matters, + # because the report would still parse and still verify every hash. + $identity = @{} + foreach ($line in Get-Content $report) { + $fields = $line -split "`t" + if ($fields[0] -eq 'staged') { + if ( + $fields.Count -ne 3 -or + $fields[2] -notmatch '^[0-9a-f]{64}$' + ) { + throw "The shipped closure report lacks a SHA-256: $line" + } + $actual = Get-Sha256Hex (Join-Path $stagedDir $fields[1]) + if ($actual -ne $fields[2]) { + throw ( + "Staged $($fields[1]) does not match its recorded hash") + } + } elseif ($fields.Count -eq 2) { + $identity[$fields[0]] = $fields[1] + } else { + throw "Unrecognised line in the shipped closure report: $line" + } + } + foreach ($key in @('corelib_version', 'corelib_sha256')) { + if (-not $identity.ContainsKey($key)) { + throw ( + "The shipped closure report carries no $key. A field " + + "report about a wrong number cannot then say which " + + "runtime produced it.") + } + } + if ($identity['corelib_version'] -notmatch '^\d+\.\d+\.\d+$') { + throw ( + "corelib_version is not MAJOR.MINOR.PATCH: " + + $identity['corelib_version']) + } + # Two independent things, both checked: the recorded hash is the hash + # of the file that shipped, and it is the same hash the file list + # already recorded for that file. Either one alone can agree with a + # wrong value. + $rootHash = Get-Sha256Hex (Join-Path $stagedDir 'ryzenai_corelib.dll') + if ($identity['corelib_sha256'] -ne $rootHash) { + throw ( + "corelib_sha256 does not match the staged ryzenai_corelib.dll") + } + $rootLine = @( + Get-Content $report | + Where-Object { $_ -like "staged`tryzenai_corelib.dll`t*" } + ) + if ($rootLine.Count -ne 1) { + throw "The file list does not name ryzenai_corelib.dll exactly once" + } + if (($rootLine[0] -split "`t")[2] -ne $identity['corelib_sha256']) { + throw "corelib_sha256 disagrees with the file list's own entry" + } + + # CLOSURE-2, positive control. + $code = Invoke-CleanEnvironmentLoad -Dir $stagedDir + if ($code -ne 0) { + throw "Clean-environment corelib load failed: Win32 error $code" + } + + # CLOSURE-2, negative control. Every derived import must be + # load-bearing from the staged directory. If hiding one still loads, + # the environment supplied it and the positive control certified + # nothing. `dyn_bins.dll` is exempt: it is opened by name at run time + # rather than imported, so it is discovered by presence, not by the + # walker, and its absence does not break LoadLibrary. + $exempt = @("dyn_bins.dll") + $proved = 0 + foreach ($name in $derived) { + if ($exempt -contains $name) { continue } + $path = Join-Path $stagedDir $name + $hidden = "$path.hidden" + Rename-Item -Path $path -NewName "$name.hidden" + try { + $missingCode = Invoke-CleanEnvironmentLoad -Dir $stagedDir + } finally { + Rename-Item -Path $hidden -NewName $name + } + if ($missingCode -eq 0) { + throw ( + "Removing $name from the staged closure still loaded. " + + "The load resolved it from outside the staged directory, " + + "so this closure is not proven.") + } + $proved += 1 + } + if ($proved -lt 1) { + throw "No staged dependency was proven load-bearing" + } + $ran += "real-closure (CLOSURE-1/2, $proved load-bearing DLLs proven)" + } else { + $skipped += ( + "real-closure (CLOSURE-1/2): pass -CorelibRuntimeDir, and " + + "-DependencyDir where the corelib's own dependencies live") + } + + # ------------------------------------------------------------------ + # The closure report must survive the INSTALLER staging, not just the + # CMake install. + # + # Human decision, 2026-09-01: ship the provenance with the artifact. Both + # get_files.bat scripts copy `..\build\aie4\*` wholesale, so the report was + # already arriving -- incidentally. An incidental dependency is exactly the + # kind that breaks silently: narrow that copy to `*.dll` and the installed + # runtime loses the only thing in it that says which corelib produced a + # result, with nothing failing. + # + # Driven through the REAL shipped .bat files against a synthetic tree, so + # this cannot agree with a copy of the logic that has drifted from the one + # that ships. Needs nothing but cmd.exe, so it always runs. + # ------------------------------------------------------------------ + $installerRoot = Join-Path $temporary "installer" + $fakeSrc = Join-Path $installerRoot "src" + foreach ($relative in @( + "build/aie4", "lib/xrt", "wix", "inno")) { + New-Item -ItemType Directory -Force ` + -Path (Join-Path $fakeSrc $relative) | Out-Null + } + Set-Content -Path (Join-Path $fakeSrc "build/flm.exe") -Value "not an exe" + Set-Content -Path (Join-Path $fakeSrc "lib/placeholder.dll") -Value "dll" + Set-Content -Path (Join-Path $fakeSrc "lib/xrt/placeholder_xrt.dll") ` + -Value "dll" + Set-Content -Path (Join-Path $fakeSrc "model_list.json") -Value "{}" + Set-Content -Path (Join-Path $fakeSrc "model_info.json") -Value "{}" + Set-Content -Path (Join-Path $fakeSrc "inno/logo.ico") -Value "ico" + Set-Content -Path (Join-Path $fakeSrc "inno/terms.rtf") -Value "rtf" + Copy-Item (Join-Path $sourceRoot "wix/get_files.bat") ` + (Join-Path $fakeSrc "wix/get_files.bat") + Copy-Item (Join-Path $sourceRoot "inno/get_files.bat") ` + (Join-Path $fakeSrc "inno/get_files.bat") + + $fakeAie4 = Join-Path $fakeSrc "build/aie4" + $fakeCorelib = Join-Path $fakeAie4 "ryzenai_corelib.dll" + $fakeReport = Join-Path $fakeAie4 "aie4-closure.txt" + Set-Content -Path $fakeCorelib -Value "not a dll" + # Shaped like the real report, including the identity header the shipped + # artifact now carries. + $fakeReportText = ( + "corelib_version`t0.1.0`n" + + "corelib_sha256`t$('a' * 64)`n" + + "staged`tryzenai_corelib.dll`t$('a' * 64)`n") + [System.IO.File]::WriteAllText($fakeReport, $fakeReportText) + + function Invoke-GetFiles { + param([string]$Directory) + $previous = $ErrorActionPreference + $ErrorActionPreference = "Continue" + $output = (& cmd.exe /c ( + "cd /d `"$Directory`" && get_files.bat") 2>&1 | + ForEach-Object { $_.ToString() } | Out-String) + $code = $LASTEXITCODE + $ErrorActionPreference = $previous + return [pscustomobject]@{ ExitCode = $code; Output = $output } + } + + $installerCases = @( + [pscustomobject]@{ + Name = "wix" + Directory = (Join-Path $fakeSrc "wix") + Staged = (Join-Path $fakeSrc "wix/package/aie4") + }, + [pscustomobject]@{ + Name = "inno" + Directory = (Join-Path $fakeSrc "inno") + Staged = (Join-Path $fakeSrc "inno/aie4") + } + ) + + foreach ($case in $installerCases) { + # 1. With a complete closure the report must arrive, unmodified and + # still free of build-machine paths. + $result = Invoke-GetFiles -Directory $case.Directory + if ($result.ExitCode -ne 0) { + throw ( + "$($case.Name)/get_files.bat failed with a complete closure " + + "(exit $($result.ExitCode)).`n$($result.Output)") + } + $stagedReport = Join-Path $case.Staged "aie4-closure.txt" + if (-not (Test-Path -LiteralPath $stagedReport)) { + throw ( + "$($case.Name)/get_files.bat did not stage aie4-closure.txt " + + "beside the AIE4 runtime") + } + $stagedText = [System.IO.File]::ReadAllText($stagedReport) + if ($stagedText -ne $fakeReportText) { + throw "$($case.Name) staged a modified aie4-closure.txt" + } + foreach ($line in $stagedText -split "`r?`n") { + if ($line -match '(?i)[a-z]:[\\/]') { + throw ( + "The $($case.Name)-staged closure report leaks a build " + + "path: $line") + } + } + + # 2. RED proof. Remove the report and the staging must FAIL, because a + # guard that cannot fail is not a guard. This is the case the + # wholesale xcopy would pass silently. + Remove-Item -LiteralPath $fakeReport -Force + Remove-Item -LiteralPath $stagedReport -Force + $result = Invoke-GetFiles -Directory $case.Directory + if ($result.ExitCode -eq 0) { + throw ( + "$($case.Name)/get_files.bat reported success while staging " + + "an AIE4 runtime with no aie4-closure.txt.`n$($result.Output)") + } + # A non-zero exit for some unrelated reason would satisfy the check + # above while proving nothing, so the diagnostic has to name the file. + if ($result.Output -notmatch 'aie4-closure\.txt') { + throw ( + "$($case.Name)/get_files.bat failed without saying the " + + "closure report was missing.`n$($result.Output)") + } + [System.IO.File]::WriteAllText($fakeReport, $fakeReportText) + + # 3. The optional feature stays optional. With no closure staged at + # all, the ordinary NPU2 installer must still build -- the + # regression dc15d66f introduced and Task 11 fixed. + Rename-Item -LiteralPath $fakeCorelib -NewName "ryzenai_corelib.hidden" + try { + $result = Invoke-GetFiles -Directory $case.Directory + } finally { + Rename-Item ` + -LiteralPath (Join-Path $fakeAie4 "ryzenai_corelib.hidden") ` + -NewName "ryzenai_corelib.dll" + } + if ($result.ExitCode -ne 0) { + throw ( + "$($case.Name)/get_files.bat failed with no AIE4 closure " + + "staged, making an optional feature a precondition for " + + "shipping.`n$($result.Output)") + } + } + + # The wix branch additionally emits the include file WiX compiles. Assert + # both shapes, because the empty ComponentGroup is the one no build has + # ever exercised. + $wxi = Join-Path $fakeSrc "wix/package/aie4.wxi" + if (-not (Test-Path -LiteralPath $wxi)) { + throw "wix/get_files.bat emitted no package\aie4.wxi" + } + $wxiText = [System.IO.File]::ReadAllText($wxi) + if ($wxiText -notmatch ']*/>') { + throw ( + "With no closure staged, aie4.wxi must define an EMPTY " + + "Aie4RuntimeComponents group.`n$wxiText") + } + # The namespace is asserted on BOTH shapes, and this one first. + # + # get_files.bat emits the `` line twice, once per + # branch, and until 2026-09-03 only the closure-PRESENT shape was checked + # (below, after the rerun). Dropping it from the else branch would have + # sailed past this guard -- and the else branch is the one that produces + # the ordinary non-AIE4 MSI, which is the build WIX0200 actually broke. + # The empty group is still a ComponentGroup, so it lands in the empty + # namespace and is rejected in exactly the same way. + if ($wxiText -notmatch '') { + throw ( + "With no closure staged, aie4.wxi must still declare the WiX " + + "namespace or WiX rejects the ordinary non-AIE4 MSI with " + + "WIX0200.`n$wxiText") + } + $result = Invoke-GetFiles -Directory (Join-Path $fakeSrc "wix") + if ($result.ExitCode -ne 0) { + throw "wix/get_files.bat failed on the closure-present rerun" + } + $wxiText = [System.IO.File]::ReadAllText($wxi) + # Two defects a real `wix build` found on 2026-09-03, both of which had + # been asserted-not-demonstrated since Task 11: + # + # 1. The generated declared no namespace, so every element + # inside it landed in the empty namespace and WiX rejected the file + # with WIX0200. That failed BOTH MSI builds -- including the ordinary + # non-AIE4 one, which had nothing to do with this feature. + # 2. `Include="package\aie4\**"` doubled the path. The Files element + # resolves against the directory of the file CONTAINING it, and + # aie4.wxi is generated into package\, so WiX looked in + # package\package\aie4, found nothing, and emitted WIX8601 -- a + # WARNING. The build succeeded and produced an MSI with no AIE4 + # runtime in it at all, byte-for-byte the same size as the no-closure + # MSI. A silent empty feature is exactly the outcome an "optional" + # feature must not have. + foreach ($shape in @( + @{ + Pattern = '' + Why = ("aie4.wxi must declare the WiX namespace or WiX rejects " + + "ComponentGroup with WIX0200") + }, + @{ + Pattern = '' + Why = ("the Files path is relative to aie4.wxi's own directory; " + + "prefixing it with package\ makes WiX harvest nothing and " + + "only warn") + } + )) { + if ($wxiText -notmatch $shape.Pattern) { + throw "$($shape.Why).`n$wxiText" + } + } + if ($wxiText -match 'Include="package\\aie4') { + throw ( + "aie4.wxi doubles the staged path: WiX resolves it against " + + "package\, so this harvests package\package\aie4 and ships an " + + "empty AIE4 feature with a warning.`n$wxiText") + } + $ran += ( + "installer-staging (aie4-closure.txt ships, RED-proven, both " + + "get_files.bat, both aie4.wxi shapes)") + $ran += "optional-feature-packaging (no-closure staging exits 0, executed)" + + # ------------------------------------------------------------------ + # cmake/ReadCorelibVersion.ps1 must survive a build environment's LIB. + # + # Design 5.4's probe calls Add-Type; PowerShell compiles that C# with + # warnings-as-errors; the C# compiler reads LIB and INCLUDE. Under MSBuild + # those hold the MSVC toolchain's paths, one of which is RELATIVE + # (`lib\um\x64`), and the compiler refuses a relative search path: + # + # Warning as Error: Invalid search path 'lib\um\x64' specified in + # 'LIB environment variable' + # + # The staging step that runs the probe is a POST_BUILD command, so that + # took the entire flm build down -- and it passed every test beforehand + # because every test ran it from a plain shell where LIB is unset. The fix + # (clear LIB/INCLUDE around Add-Type, restore afterwards) was then verified + # in exactly one environment, once, by hand: no test referenced the script + # and no test set LIB. The installer-staging block above cannot help -- + # its `ryzenai_corelib.dll` contains the text "not a dll" and never reaches + # the probe. + # + # WHAT THIS COVERS: that with a relative entry in LIB the script gets past + # Add-Type and past LoadLibraryEx, and then fails where it should -- at the + # missing export -- with its own diagnostic rather than a compiler error. + # That is the whole of the regression, and it is able to fail: reverting + # the save/restore turns the output into the compiler error above. + # + # WHAT THIS DOES NOT COVER: reading a real version number. That needs a + # real ryzenai_corelib.dll, which a machine without the AIE4 runtime does + # not have. The success path is asserted in the real-closure block above, + # via the staged report's `corelib_version` -- which StageAie4Runtime.cmake + # obtained from this same script. + # + # Launched through powershell.exe by absolute path, and that is + # load-bearing rather than incidental: Windows PowerShell 5.1 shells out to + # the .NET Framework csc.exe, which reads LIB, while pwsh 7 compiles + # in-process with Roslyn and never looks at it. Under pwsh this check + # cannot fail, so inheriting the host would make it vacuous on exactly the + # machines that run pwsh. 5.1 is also what StageAie4Runtime.cmake invokes. + # ------------------------------------------------------------------ + $probeScript = Join-Path $sourceRoot "cmake/ReadCorelibVersion.ps1" + if (-not (Test-Path -LiteralPath $probeScript -PathType Leaf)) { + throw "the corelib version probe is missing: $probeScript" + } + # A real PE, in an approved system directory, that certainly does not + # export ryzenai_corelib_get_version. + $probeDll = Join-Path $env:SystemRoot "System32\version.dll" + $windowsPowerShell = Join-Path $env:SystemRoot ` + "System32\WindowsPowerShell\v1.0\powershell.exe" + foreach ($needed in @($probeDll, $windowsPowerShell)) { + if (-not (Test-Path -LiteralPath $needed -PathType Leaf)) { + throw ( + "the corelib-version-probe regression check needs $needed " + + "and it is absent, so the check would pass vacuously") + } + } + $savedProbeLib = $env:LIB + $savedProbeInclude = $env:INCLUDE + $previousPreference = $ErrorActionPreference + $probeOutput = "" + $probeExit = $null + try { + # The shape that broke it: a relative entry beside an absolute one, + # which is how vcvars64 leaves LIB. + $env:LIB = "lib\um\x64;$env:SystemRoot\System32" + $env:INCLUDE = "include\um" + $ErrorActionPreference = "Continue" + $probeOutput = (& $windowsPowerShell -NoProfile -NonInteractive ` + -ExecutionPolicy Bypass -File $probeScript -Dll $probeDll 2>&1 | + ForEach-Object { $_.ToString() } | Out-String) + $probeExit = $LASTEXITCODE + } finally { + $ErrorActionPreference = $previousPreference + if ($null -eq $savedProbeLib) { + Remove-Item Env:LIB -ErrorAction SilentlyContinue + } else { + $env:LIB = $savedProbeLib + } + if ($null -eq $savedProbeInclude) { + Remove-Item Env:INCLUDE -ErrorAction SilentlyContinue + } else { + $env:INCLUDE = $savedProbeInclude + } + } + if ( + $probeOutput -match 'LIB environment variable' -or + $probeOutput -match '(?i)Cannot add type' + ) { + throw ( + "ReadCorelibVersion.ps1 is environment-sensitive again: an " + + "MSBuild-shaped LIB reaches its Add-Type. This is the defect " + + "that took the flm build down, because the staging step runs " + + "from a POST_BUILD command.`n$probeOutput") + } + # Positive requirement, so that a probe which failed EARLIER for some other + # reason cannot satisfy the check above by never compiling at all. + if ($probeOutput -notmatch 'exports no ryzenai_corelib_get_version') { + throw ( + "the probe did not reach GetProcAddress (exit $probeExit), so " + + "nothing was proven about Add-Type surviving LIB.`n$probeOutput") + } + if ($probeExit -eq 0) { + throw ( + "the probe reported success for a DLL that is not corelib.`n" + + $probeOutput) + } + $ran += ( + "corelib-version-probe (5.4 LIB regression, run under Windows " + + "PowerShell 5.1 against a non-corelib PE)") + + if ($FlmExe) { + $resolvedFlm = (Resolve-Path $FlmExe).Path + # -FlmExe is the first positional parameter, so a stray argument from a + # mis-quoted argument vector lands here. Refusing a non-file keeps that + # mistake from entering this block and reporting a run it never made. + if (-not (Test-Path -LiteralPath $resolvedFlm -PathType Leaf)) { + throw "-FlmExe is not a file: $resolvedFlm" + } + + # dumpbin's exit code is load-bearing. If it runs but errors -- wrong + # architecture, missing tool, unreadable image -- its output simply + # fails to match the import pattern, and the Step 8 guard passes + # vacuously while checking nothing. + $imports = (& dumpbin /nologo /dependents $resolvedFlm 2>&1 | + Out-String) + if ($LASTEXITCODE -ne 0) { + throw "dumpbin failed with exit code ${LASTEXITCODE}.`n$imports" + } + if ($imports -notmatch "(?i)Image has the following dependencies") { + throw ( + "dumpbin produced no dependency listing, so the no-import " + + "check would pass vacuously.`n$imports") + } + if ($imports -match "(?im)^\s*ryzenai_corelib\.dll\s*$") { + throw "flm.exe has an unexpected ryzenai_corelib import" + } + + $savedPath = $env:PATH + $savedCorelib = $env:RYZENAI_CORELIB_PATH + $savedXrt = $env:XILINX_XRT + try { + Remove-Item Env:RYZENAI_CORELIB_PATH -ErrorAction SilentlyContinue + Remove-Item Env:XILINX_XRT -ErrorAction SilentlyContinue + $env:PATH = "$env:SystemRoot\System32;$env:SystemRoot" + $validation = (& $resolvedFlm validate --json 2>&1 | Out-String) + if ($LASTEXITCODE -ne 0) { + throw "Clean-environment flm validate failed.`n$validation" + } + if ($RunAie4ModelLoad) { + # Driven through drive_flm_console.ps1, NOT by piping into + # `flm run`. + # + # This block used to be `"/bye" | & $resolvedFlm run `. + # `flm run` reads keystrokes with ReadConsoleInput, a + # console-only API that cannot see a redirected pipe -- Task 16 + # established that against the real binary -- so the prompt was + # never delivered and the block would have sat in the REPL + # until the CTest timeout killed it. It had never executed: + # nothing in this tree passes -RunAie4ModelLoad, so a latent + # hang shipped for five tasks under a line that read + # "none skipped". + # + # The driver injects KEY_EVENT records into the console input + # buffer the REPL is actually reading, so this exercises the + # product's own interactive path. `/status` rather than a + # generation turn: it is the REPL command wired to + # AutoModel::show_profile, so it proves the AIE4 model loaded + # and names the corelib DLL that answered, without spending a + # decode on a packaging test. + $driver = Join-Path $PSScriptRoot "drive_flm_console.ps1" + if (-not (Test-Path -LiteralPath $driver)) { + throw "the console driver is missing: $driver" + } + $smokeJson = Join-Path $temporary "aie4-model-load.json" + $smokeScreen = Join-Path $temporary "aie4-model-load-screen.txt" + # The driver AllocConsole()s and takes its own stdout with it, + # so it has to be a separate process and its streams must be + # left alone -- piping them makes flm.exe inherit a pipe and + # spin on a read it can never satisfy. So: a separate + # PowerShell with untouched streams, and the result read back + # from the JSON file the driver writes rather than from stdout. + $psExe = (Get-Process -Id $PID).Path + & $psExe -NoProfile -ExecutionPolicy Bypass -File $driver ` + -FlmExe $resolvedFlm ` + -ModelTag 'phi4-mini-it-aie4:4b' ` + -Turns '/status' ` + -OutJson $smokeJson ` + -ScreenLog $smokeScreen + $driverExit = $LASTEXITCODE + if (-not (Test-Path -LiteralPath $smokeJson)) { + throw ( + "the console driver exited $driverExit and wrote no " + + "result, so nothing is known about the model load") + } + $smoke = Get-Content $smokeJson -Raw | ConvertFrom-Json + if ($driverExit -ne 0 -or -not $smoke.reached_prompt) { + throw ( + "Clean-environment AIE4 model load failed (driver " + + "exit $driverExit): $($smoke.error)`n" + + "$($smoke.screen_final)") + } + # The screen has to name the tag that was loaded. Reaching a + # prompt alone would also be true of a build that fell back to + # some other model. + if ("$($smoke.screen_final)" -notmatch 'phi4-mini-it-aie4') { + throw ( + "the REPL reached its prompt but never named " + + "phi4-mini-it-aie4.`n$($smoke.screen_final)") + } + $ran += "flm-exe AIE4 model load (console-driven, /status)" + } else { + # Reported, because it is the block this test is named for and + # the one that needs AIE4 hardware. It had no else at all: with + # -FlmExe supplied the run printed "none skipped" having never + # loaded a model, which is this branch's recurring defect -- + # a record that reads better than the run it describes. + $skipped += ( + "flm-exe AIE4 model load (-RunAie4ModelLoad not " + + "supplied; needs AIE4 hardware and the packaged model)") + } + } finally { + $env:PATH = $savedPath + if ($null -eq $savedCorelib) { + Remove-Item Env:RYZENAI_CORELIB_PATH ` + -ErrorAction SilentlyContinue + } else { + $env:RYZENAI_CORELIB_PATH = $savedCorelib + } + if ($null -eq $savedXrt) { + Remove-Item Env:XILINX_XRT -ErrorAction SilentlyContinue + } else { + $env:XILINX_XRT = $savedXrt + } + } + + $portable = Join-Path $temporary "without-aie4" + Copy-Item ` + -Path (Split-Path $resolvedFlm -Parent) ` + -Destination $portable ` + -Recurse + Remove-Item (Join-Path $portable "aie4") ` + -Recurse ` + -Force ` + -ErrorAction SilentlyContinue + & (Join-Path $portable (Split-Path $resolvedFlm -Leaf)) list | + Out-Null + if ($LASTEXITCODE -ne 0) { + throw "Non-AIE4 command failed without the AIE4 directory" + } + $ran += "flm-exe (Step 7 clean environment, Step 8 dumpbin)" + } else { + # Names only what this branch skips. It used to also invite + # -RunAie4ModelLoad, which is (a) irrelevant here, since that flag does + # nothing without -FlmExe, and (b) an invitation to a hang, back when + # the model-load block piped a prompt into a REPL that reads through + # ReadConsoleInput. The model-load skip is now reported where it + # belongs: on the -FlmExe branch, where the flag is actually consulted. + $skipped += ( + "flm-exe (Step 7 clean environment, Step 8 dumpbin): pass -FlmExe") + } + + foreach ($item in $ran) { + Write-Output "RAN : $item" + } + foreach ($item in $skipped) { + Write-Output "SKIPPED : $item" + } + if ($skipped.Count -gt 0) { + Write-Output ( + "packaged runtime tests INCOMPLETE: " + + "$($ran.Count) block(s) ran, $($skipped.Count) skipped") + $script:exitCode = $SKIP_EXIT + } else { + Write-Output ( + "packaged runtime tests passed: $($ran.Count) block(s), " + + "none skipped") + $script:exitCode = 0 + } +} finally { + if (Test-Path $temporary) { + Remove-Item $temporary -Recurse -Force -ErrorAction SilentlyContinue + } +} +exit $script:exitCode diff --git a/src/test/phi4_corelib_aie4/test_phi4_engine.cpp b/src/test/phi4_corelib_aie4/test_phi4_engine.cpp new file mode 100644 index 00000000..fdea7e2e --- /dev/null +++ b/src/test/phi4_corelib_aie4/test_phi4_engine.cpp @@ -0,0 +1,2894 @@ +#include "fake_corelib.hpp" +#include "test_support.hpp" + +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using flm::corelib::CorelibApi; +using flm::corelib::CorelibError; +using flm::corelib::CorelibRuntime; +using flm::corelib::FatalRecordStore; +using flm::corelib::ProcessState; +using flm::phi4::Phi4Aie4Metrics; +using flm::phi4::Phi4DebugSnapshot; +using flm::phi4::phi4_corelib_aie4; +using nlohmann::json; + +namespace constants = flm::phi4::constants; + +constexpr std::string_view kManifestName = + "corelib_phi4_manifest.json"; +constexpr std::string_view kDataFile = "weights.bin"; +constexpr std::uint64_t kDataBytes = + 200064ull * 3072ull * sizeof(std::uint16_t); +constexpr std::uint16_t kPoison = 0xDEADu; + +std::int64_t PaddedRows(std::int64_t rows) { + return rows == 1 ? 1 : ((rows + 3) / 4) * 4; +} + +using PadRowsFunction = + std::function; + +struct PaddingFunctions { + PadRowsFunction query_projection = PaddedRows; + PadRowsFunction kv_projection = PaddedRows; + PadRowsFunction lm_head = PaddedRows; + PadRowsFunction ssmlp = PaddedRows; + PadRowsFunction attention = PaddedRows; +}; + +class TempDirectory final { +public: + explicit TempDirectory(std::string_view stem) { + const auto nonce = + std::chrono::steady_clock::now().time_since_epoch().count(); + path_ = std::filesystem::temp_directory_path() / + (std::string(stem) + "-" + + std::to_string(GetCurrentProcessId()) + "-" + + std::to_string(nonce)); + std::filesystem::create_directories(path_); + } + + ~TempDirectory() noexcept { + std::error_code error; + std::filesystem::remove_all(path_, error); + } + + TempDirectory(const TempDirectory&) = delete; + TempDirectory& operator=(const TempDirectory&) = delete; + + const std::filesystem::path& path() const noexcept { + return path_; + } + +private: + std::filesystem::path path_; +}; + +void CreateSparseFile( + const std::filesystem::path& path, + std::uint64_t size) { + HANDLE file = CreateFileW( + path.c_str(), + GENERIC_READ | GENERIC_WRITE, + FILE_SHARE_READ, + nullptr, + CREATE_ALWAYS, + FILE_ATTRIBUTE_NORMAL, + nullptr); + if (file == INVALID_HANDLE_VALUE) { + throw std::runtime_error( + "failed to create sparse engine-test model data"); + } + + DWORD ignored = 0; + if ( + DeviceIoControl( + file, + FSCTL_SET_SPARSE, + nullptr, + 0, + nullptr, + 0, + &ignored, + nullptr) == FALSE) { + CloseHandle(file); + throw std::runtime_error( + "engine-test volume does not support sparse files"); + } + + LARGE_INTEGER end{}; + end.QuadPart = static_cast(size); + const bool success = + SetFilePointerEx(file, end, nullptr, FILE_BEGIN) != FALSE && + SetEndOfFile(file) != FALSE; + CloseHandle(file); + if (!success) { + throw std::runtime_error( + "failed to size sparse engine-test model data"); + } +} + +std::uint64_t ItemSize(std::string_view dtype) { + if (dtype == "uint8") { + return 1; + } + if (dtype == "float16") { + return 2; + } + if (dtype == "float32" || dtype == "int64") { + return dtype == "float32" ? 4 : 8; + } + throw std::runtime_error("unsupported synthetic dtype"); +} + +std::uint64_t ByteLength( + std::string_view dtype, + const std::vector& shape) { + std::uint64_t elements = 1; + for (const std::int64_t dimension : shape) { + elements *= static_cast(dimension); + } + return elements * ItemSize(dtype); +} + +void AddInitializer( + json& initializers, + const std::string& name, + std::string dtype, + std::vector shape, + std::string role, + std::uint64_t offset) { + CHECK(!initializers.contains(name)); + initializers[name] = { + {"file", std::string(kDataFile)}, + {"offset", offset}, + {"length", ByteLength(dtype, shape)}, + {"dtype", std::move(dtype)}, + {"shape", std::move(shape)}, + {"role", std::move(role)}}; +} + +class ManifestBuilder final { +public: + explicit ManifestBuilder(std::uint64_t model_size) + : manifest_{ + {"schema_version", 1}, + {"execution_backend", "corelib_aie4"}, + {"model", + { + {"family", "phi4"}, + {"layers", 32}, + {"hidden_size", 3072}, + {"intermediate_size", 8192}, + {"num_heads", 24}, + {"kv_heads", 8}, + {"head_size", 128}, + {"vocab_size", 200064}, + {"group_size", 128}, + {"rope_dim", 96}, + {"rms_epsilon", 0.00001}, + }}, + {"backend", {{"max_seq", 4096}}}, + {"files", + { + {"model.onnx", {{"size", model_size}}}, + {std::string(kDataFile), {{"size", kDataBytes}}}, + }}, + {"initializers", json::object()}, + {"weight_objects", json::array()}} {} + + void AddMatMul( + const std::string& name, + std::int64_t k, + std::int64_t n) { + const std::string qweight = name + ".qweight"; + const std::string scales = name + ".scales"; + const std::string qzeros = name + ".qzeros"; + manifest_["weight_objects"].push_back({ + {"name", name}, + {"kind", "matmul"}, + {"descriptor", + { + {"k", k}, + {"n", n}, + {"group_size", 128}, + {"has_bias", false}, + }}, + {"roles", + { + {"qweight", qweight}, + {"scales", scales}, + {"qzeros", qzeros}, + }}}); + + AddInitializer( + manifest_["initializers"], + qweight, + "uint8", + {n, k / 2}, + "matmul.qweight", + NextOffset()); + AddInitializer( + manifest_["initializers"], + scales, + "float16", + {n, k / 128}, + "matmul.scales", + NextOffset()); + AddInitializer( + manifest_["initializers"], + qzeros, + "uint8", + {n, ((k / 128) + 1) / 2}, + "matmul.qzeros", + NextOffset()); + } + + void AddSsMlp(int layer) { + const std::string base = + "model.layers." + std::to_string(layer); + const std::string object_name = base + ".ssmlp"; + const std::string norm0 = + base + ".post_attention_layernorm.weight"; + const std::string norm1 = + layer == 31 + ? "model.layers.32.final_norm_layernorm.weight" + : "model.layers." + std::to_string(layer + 1) + + ".input_layernorm.weight"; + json roles = { + {"norm0", norm0}, + {"norm1", norm1}, + }; + + for (const std::string projection : {"gate", "up", "down"}) { + const std::int64_t k = + projection == "down" ? 8192 : 3072; + const std::int64_t n = + projection == "down" ? 3072 : 8192; + const std::string prefix = + base + ".mlp." + projection + + "_proj.MatMulNBits"; + const std::string role_prefix = + "ssmlp." + projection; + for (const std::string component : + {"qweight", "scales", "qzeros"}) { + roles[projection + "_" + component] = + prefix + "." + component; + } + AddInitializer( + manifest_["initializers"], + prefix + ".qweight", + "uint8", + {n, k / 2}, + role_prefix + ".qweight", + NextOffset()); + AddInitializer( + manifest_["initializers"], + prefix + ".scales", + "float16", + {n, k / 128}, + role_prefix + ".scales", + NextOffset()); + AddInitializer( + manifest_["initializers"], + prefix + ".qzeros", + "uint8", + {n, ((k / 128) + 1) / 2}, + role_prefix + ".qzeros", + NextOffset()); + } + + AddInitializer( + manifest_["initializers"], + norm0, + "float16", + {3072}, + "ssmlp.norm0", + NextOffset()); + AddInitializer( + manifest_["initializers"], + norm1, + "float16", + {3072}, + "ssmlp.norm1", + NextOffset()); + manifest_["weight_objects"].push_back({ + {"name", object_name}, + {"kind", "ssmlp"}, + {"descriptor", + { + {"k", 3072}, + {"n", 8192}, + {"group_size", 128}, + }}, + {"roles", std::move(roles)}}); + } + + json Finish() { + for (int layer = 0; layer < 32; ++layer) { + const std::string base = + "model.layers." + std::to_string(layer) + + ".attn."; + AddMatMul( + base + "q_proj.MatMulNBits", + 3072, + 3072); + AddMatMul( + base + "k_proj.MatMulNBits", + 3072, + 1024); + AddMatMul( + base + "v_proj.MatMulNBits", + 3072, + 1024); + AddMatMul( + base + "o_proj.MatMulNBits", + 3072, + 3072); + AddSsMlp(layer); + } + AddMatMul("lm_head.MatMulNBits", 3072, 200064); + + AddInitializer( + manifest_["initializers"], + "model.embed_tokens.weight", + "float16", + {200064, 3072}, + "embedding", + 0); + AddInitializer( + manifest_["initializers"], + "model.layers.0.input_layernorm.weight", + "float16", + {3072}, + "input_norm", + NextOffset()); + AddInitializer( + manifest_["initializers"], + "cos_cache", + "float16", + {4096, 48}, + "cos_cache", + NextOffset()); + AddInitializer( + manifest_["initializers"], + "sin_cache", + "float16", + {4096, 48}, + "sin_cache", + NextOffset()); + + CHECK(manifest_["weight_objects"].size() == 161); + CHECK(manifest_["initializers"].size() == 743); + return std::move(manifest_); + } + +private: + std::uint64_t NextOffset() { + const std::uint64_t result = next_offset_; + next_offset_ += 16; + return result; + } + + json manifest_; + std::uint64_t next_offset_ = 4096; +}; + +class SyntheticPackage final { +public: + SyntheticPackage() + : temp_("fastflowlm-phi4-engine-model") { + const auto model_path = temp_.path() / "model.onnx"; + { + std::ofstream model(model_path, std::ios::binary); + model << "model"; + } + CreateSparseFile(temp_.path() / kDataFile, kDataBytes); + ManifestBuilder builder( + std::filesystem::file_size(model_path)); + std::ofstream manifest( + temp_.path() / kManifestName, + std::ios::binary); + manifest << builder.Finish().dump(2) << '\n'; + if (!manifest) { + throw std::runtime_error( + "failed to write synthetic engine manifest"); + } + } + + const std::filesystem::path& path() const noexcept { + return temp_.path(); + } + +private: + TempDirectory temp_; +}; + +float HalfToFloat(std::uint16_t value) { + const bool negative = (value & 0x8000u) != 0; + const unsigned exponent = (value >> 10) & 0x1fu; + const unsigned mantissa = value & 0x03ffu; + float result = 0.0f; + if (exponent == 0) { + result = std::ldexp(static_cast(mantissa), -24); + } else if (exponent == 31) { + result = mantissa == 0 + ? std::numeric_limits::infinity() + : std::numeric_limits::quiet_NaN(); + } else { + result = std::ldexp( + 1.0f + static_cast(mantissa) / 1024.0f, + static_cast(exponent) - 15); + } + return negative ? -result : result; +} + +std::uint16_t FloatToHalf(float value) { + const std::uint32_t bits = std::bit_cast(value); + const std::uint16_t sign = + static_cast((bits >> 16) & 0x8000u); + const std::uint32_t source_exponent = (bits >> 23) & 0xffu; + const std::uint32_t source_mantissa = bits & 0x007fffffu; + if (source_exponent == 0xffu) { + return static_cast( + sign | (source_mantissa == 0 ? 0x7c00u : 0x7e00u)); + } + const int exponent = static_cast(source_exponent) - 127 + 15; + if (exponent >= 31) { + return static_cast(sign | 0x7c00u); + } + if (exponent <= 0) { + if (exponent < -10) { + return sign; + } + const std::uint32_t mantissa = + source_mantissa | 0x00800000u; + const unsigned shift = static_cast(14 - exponent); + const std::uint32_t rounded = + (mantissa + (1u << (shift - 1)) - 1u + + ((mantissa >> shift) & 1u)) >> + shift; + return static_cast(sign | rounded); + } + const std::uint32_t rounded = + source_mantissa + 0x00000fffu + + ((source_mantissa >> 13) & 1u); + if ((rounded & 0x00800000u) != 0) { + if (exponent + 1 >= 31) { + return static_cast(sign | 0x7c00u); + } + return static_cast( + sign | (static_cast(exponent + 1) << 10)); + } + return static_cast( + sign | (static_cast(exponent) << 10) | + (rounded >> 13)); +} + +std::uint16_t FloatToBf16(float value) { + std::uint32_t bits = std::bit_cast(value); + bits += 0x7fffu + ((bits >> 16) & 1u); + return static_cast(bits >> 16); +} + +float ReadElement( + ryzenai_corelib_data_type type, + const void* source, + std::size_t index) { + switch (type) { + case ryzenai_corelib_data_type_fp16: + return HalfToFloat( + static_cast(source)[index]); + case ryzenai_corelib_data_type_bf16: { + const std::uint32_t bits = + static_cast( + static_cast( + source)[index]) + << 16; + return std::bit_cast(bits); + } + case ryzenai_corelib_data_type_fp32: + return static_cast(source)[index]; + default: + throw std::runtime_error( + "engine fake received unsupported conversion source"); + } +} + +void WriteElement( + ryzenai_corelib_data_type type, + void* destination, + std::size_t index, + float value) { + switch (type) { + case ryzenai_corelib_data_type_fp16: + static_cast(destination)[index] = + FloatToHalf(value); + return; + case ryzenai_corelib_data_type_bf16: + static_cast(destination)[index] = + FloatToBf16(value); + return; + case ryzenai_corelib_data_type_fp32: + static_cast(destination)[index] = value; + return; + default: + throw std::runtime_error( + "engine fake received unsupported conversion destination"); + } +} + +enum class ObjectKind { Stream, Tensor, MatMulWeight, SsMlpWeight }; + +struct FakeObject { + explicit FakeObject(ObjectKind kind_value, std::string label_value) + : kind(kind_value), + label(std::move(label_value)) {} + virtual ~FakeObject() = default; + + ObjectKind kind; + std::string label; + bool released = false; +}; + +struct FakeTensor final : FakeObject { + static constexpr std::size_t kPageBytes = 4096; + using Page = std::array; + + FakeTensor( + ryzenai_corelib_data_type type, + std::vector dimensions, + std::size_t bytes, + std::string label) + : FakeObject(ObjectKind::Tensor, std::move(label)), + data_type(type), + shape(std::move(dimensions)), + byte_size(bytes) {} + + std::byte DefaultByte(std::size_t offset) const noexcept { + return std::byte{ + static_cast( + offset % 2 == 0 ? 0xADu : 0xDEu)}; + } + + Page& MutablePage(std::size_t page_index) { + auto [found, inserted] = + pages.try_emplace(page_index, nullptr); + if (inserted) { + found->second = std::make_unique(); + const std::size_t base = page_index * kPageBytes; + for (std::size_t index = 0; index < kPageBytes; ++index) { + (*found->second)[index] = DefaultByte(base + index); + } + } + return *found->second; + } + + std::byte ReadByte(std::size_t offset) const { + const std::size_t page_index = offset / kPageBytes; + const auto found = pages.find(page_index); + if (found == pages.end()) { + return DefaultByte(offset); + } + return (*found->second)[offset % kPageBytes]; + } + + bool Write( + const void* source, + std::size_t size, + std::size_t offset) { + if ( + (source == nullptr && size != 0) || + offset > byte_size || + size > byte_size - offset) { + return false; + } + const auto* bytes = static_cast(source); + for (std::size_t index = 0; index < size; ++index) { + MutablePage((offset + index) / kPageBytes) + [(offset + index) % kPageBytes] = bytes[index]; + } + return true; + } + + bool Read( + void* destination, + std::size_t size, + std::size_t offset) const { + if ( + (destination == nullptr && size != 0) || + offset > byte_size || + size > byte_size - offset) { + return false; + } + auto* bytes = static_cast(destination); + for (std::size_t index = 0; index < size; ++index) { + bytes[index] = ReadByte(offset + index); + } + return true; + } + + std::uint16_t ReadWord(std::size_t word_index) const { + const std::size_t offset = word_index * sizeof(std::uint16_t); + const std::array bytes{ + ReadByte(offset), + ReadByte(offset + 1)}; + std::uint16_t value = 0; + std::memcpy(&value, bytes.data(), sizeof(value)); + return value; + } + + void WriteWord(std::size_t word_index, std::uint16_t value) { + const std::size_t offset = word_index * sizeof(value); + CHECK(Write(&value, sizeof(value), offset)); + } + + void FillWords( + std::size_t first, + std::size_t count, + std::uint16_t value) { + for (std::size_t index = 0; index < count; ++index) { + WriteWord(first + index, value); + } + } + + bool ContainsPoisonWords( + std::size_t first, + std::size_t count) const { + for (std::size_t index = 0; index < count; ++index) { + if (ReadWord(first + index) == kPoison) { + return true; + } + } + return false; + } + + ryzenai_corelib_data_type data_type; + std::vector shape; + std::size_t byte_size; + std::string last_producer; + std::unordered_map> pages; +}; + +struct FakeMatMulWeight final : FakeObject { + FakeMatMulWeight( + std::string label, + ryzenai_corelib_matmul_bf16_weights_desc value) + : FakeObject(ObjectKind::MatMulWeight, std::move(label)), + desc(value) {} + + ryzenai_corelib_matmul_bf16_weights_desc desc{}; +}; + +struct FakeSsMlpWeight final : FakeObject { + FakeSsMlpWeight( + std::string label, + ryzenai_corelib_ssmlp_bf16_weights_desc value) + : FakeObject(ObjectKind::SsMlpWeight, std::move(label)), + desc(value) {} + + ryzenai_corelib_ssmlp_bf16_weights_desc desc{}; +}; + +struct MatMulCall { + std::string label; + FakeTensor* input; + FakeTensor* output; + std::int64_t rows; +}; + +struct SsMlpCall { + FakeTensor* input; + FakeTensor* residual; + FakeTensor* skip_sum; + FakeTensor* normalized; + std::int64_t rows; +}; + +struct MhaCall { + int layer; + FakeTensor* query; + FakeTensor* key; + FakeTensor* key_cache; + FakeTensor* value_cache; + FakeTensor* output; + std::int64_t rows; + std::int64_t position; +}; + +struct TensorWriteCall { + FakeTensor* tensor; + // API-7: elements of the tensor's own dtype, never bytes. + std::size_t count; + std::size_t offset; +}; + +enum class FailurePoint { + None, + FirstQ, + KAfterQ, + // Layer 1's k, i.e. a dispatch rejected after layer 0's q has been + // submitted and AFTER four completed synchronizes have cleared the + // submission flag. Proves the flag re-arms. + KAfterQLayerOne, + Synchronize, + DestructionSynchronize, + StageBadAlloc, + ScatterBadAlloc, + ScatterUnknown, + // The V read of layer 3, thirteen synchronizes into the step. + ScatterBadAllocDeepLayer, + // flat_mha refused at layer 0, after the q/k/v synchronize. This is the + // shape of the 4096-token-window refusal seen on hardware. + MhaRefusedAfterSynchronize, + // The other three dispatches that follow a completed synchronize. + // + // The engine's own comment claimed all four of `flat_mha`, `o`, `ssmlp` + // and `lm_head` were recoverable when only `flat_mha` had a test -- a + // comment asserting four cases on evidence covering one. Each of these + // sits immediately after a checked_synchronize in RunRows, so each is a + // rejected dispatch with the outstanding list provably empty. + ORefusedAfterSynchronize, + SsMlpRefusedAfterSynchronize, + LmHeadRefusedAfterSynchronize +}; + +struct UnknownFailure final {}; + +struct RecordingState { + std::vector> objects; + std::unordered_map object_index; + std::vector tensors; + std::vector events; + std::vector release_labels; + std::vector matmul_calls; + std::vector ssmlp_calls; + std::vector mha_calls; + std::vector stage_writes; + PaddingFunctions padding; + std::set q_tensors; + std::set k_tensors; + std::set v_tensors; + std::set attention_tensors; + std::set skip_sum_tensors; + std::set normalized_tensors; + FakeTensor* lm_input = nullptr; + FakeTensor* staged_hidden = nullptr; + FakeTensor* staged_residual = nullptr; + FailurePoint failure = FailurePoint::None; + std::size_t matmul_weight_count = 0; + std::size_t ssmlp_weight_count = 0; + std::size_t synchronize_calls = 0; + int active_layer = -1; + std::int64_t active_rows = 0; + bool input_poison_observed = false; + bool host_read_poison_observed = false; + bool cache_publish_poison_observed = false; + bool destruction_failure_returned = false; + std::filesystem::path post_failure_marker; + bool terminator_called = false; + unsigned int termination_code = 0; + std::optional load_thread; + bool load_thread_consistent = true; + + template + Object* Create(Args&&... args) { + auto object = + std::make_unique(std::forward(args)...); + auto* result = object.get(); + object_index.emplace(result, result); + objects.push_back(std::move(object)); + return result; + } + + FakeObject* Object(void* value) const { + const auto found = object_index.find(value); + return found == object_index.end() ? nullptr : found->second; + } + + FakeTensor* Tensor(void* value) const { + auto* object = Object(value); + if (object == nullptr || object->kind != ObjectKind::Tensor) { + return nullptr; + } + return static_cast(object); + } + + void ObserveLoadThread() { + const auto current = std::this_thread::get_id(); + if (!load_thread.has_value()) { + load_thread = current; + } else { + load_thread_consistent = + load_thread_consistent && *load_thread == current; + } + } + + void ResetExecutionRecords() { + events.clear(); + matmul_calls.clear(); + ssmlp_calls.clear(); + mha_calls.clear(); + stage_writes.clear(); + q_tensors.clear(); + k_tensors.clear(); + v_tensors.clear(); + attention_tensors.clear(); + skip_sum_tensors.clear(); + normalized_tensors.clear(); + synchronize_calls = 0; + active_layer = -1; + active_rows = 0; + input_poison_observed = false; + host_read_poison_observed = false; + cache_publish_poison_observed = false; + } +}; + +RecordingState* g_state = nullptr; + +RecordingState& State() { + if (g_state == nullptr) { + throw std::runtime_error("engine recording fake is not active"); + } + return *g_state; +} + +std::size_t TypeSize(ryzenai_corelib_data_type type) { + switch (type) { + case ryzenai_corelib_data_type_fp16: + case ryzenai_corelib_data_type_bf16: + return 2; + case ryzenai_corelib_data_type_fp32: + return 4; + default: + throw std::runtime_error( + "engine fake cannot size this tensor dtype"); + } +} + +std::size_t TensorBytes( + ryzenai_corelib_data_type type, + std::span shape) { + std::size_t elements = 1; + for (const std::int64_t dimension : shape) { + if (dimension <= 0) { + throw std::runtime_error( + "engine fake received non-positive tensor shape"); + } + elements *= static_cast(dimension); + } + return elements * TypeSize(type); +} + +std::string MatMulLabel(std::size_t ordinal) { + if (ordinal == 128) { + return "lm_head"; + } + const std::size_t layer = ordinal / 4; + constexpr std::array projections{ + "q", + "k", + "v", + "o"}; + return std::string(projections[ordinal % 4]) + "_" + + std::to_string(layer); +} + +std::string Projection(const std::string& label) { + if (label == "lm_head") { + return label; + } + const auto separator = label.find('_'); + return separator == std::string::npos + ? label + : label.substr(0, separator); +} + +int WeightLayer(const std::string& label) { + if (label == "lm_head") { + return -1; + } + return std::stoi(label.substr(label.find('_') + 1)); +} + +std::int64_t MatMulPaddedRows( + const RecordingState& state, + const ryzenai_corelib_matmul_bf16_weights_desc& desc, + std::int64_t rows) { + if ( + desc.k == constants::kHiddenSize && + desc.n == constants::kVocabularySize) { + return state.padding.lm_head(rows); + } + if ( + desc.k == constants::kHiddenSize && + desc.n == constants::kKvDimension) { + return state.padding.kv_projection(rows); + } + if ( + desc.k == constants::kHiddenSize && + desc.n == constants::kQueryDimension) { + return state.padding.query_projection(rows); + } + throw std::runtime_error( + "engine fake received an unknown matmul shape"); +} + +ryzenai_corelib_status RecordingMatMulPadShape( + std::int64_t* m, + std::int64_t* k, + std::int64_t* n, + std::uint32_t) { + auto& state = State(); + state.ObserveLoadThread(); + if ( + m == nullptr || k == nullptr || n == nullptr || + *m <= 0 || *k <= 0 || *n <= 0) { + return ryzenai_corelib_status_bad_argument; + } + ryzenai_corelib_matmul_bf16_weights_desc desc{}; + desc.k = *k; + desc.n = *n; + *m = MatMulPaddedRows(state, desc, *m); + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status RecordingSsMlpPadRows( + std::int64_t* m, + std::int64_t, + std::int64_t, + std::uint32_t) { + auto& state = State(); + state.ObserveLoadThread(); + if (m == nullptr || *m <= 0) { + return ryzenai_corelib_status_bad_argument; + } + *m = state.padding.ssmlp(*m); + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status RecordingMhaPadRows( + std::int64_t* m, + const ryzenai_corelib_flat_mha_bf16_desc*) { + auto& state = State(); + state.ObserveLoadThread(); + if (m == nullptr || *m <= 0) { + return ryzenai_corelib_status_bad_argument; + } + *m = state.padding.attention(*m); + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status RecordingCreateStream( + ryzenai_corelib_stream_ptr* out) { + auto& state = State(); + state.ObserveLoadThread(); + if (out == nullptr) { + return ryzenai_corelib_status_bad_argument; + } + *out = state.Create(ObjectKind::Stream, "stream"); + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status RecordingCreateTensor( + ryzenai_corelib_data_type data_type, + const std::int64_t* shape, + std::size_t shape_len, + ryzenai_corelib_tensor_ptr* out) { + auto& state = State(); + state.ObserveLoadThread(); + if (shape == nullptr || shape_len == 0 || out == nullptr) { + return ryzenai_corelib_status_bad_argument; + } + std::vector dimensions(shape, shape + shape_len); + const auto ordinal = state.tensors.size(); + auto* tensor = state.Create( + data_type, + dimensions, + TensorBytes(data_type, dimensions), + "tensor_" + std::to_string(ordinal)); + state.tensors.push_back(tensor); + if (ordinal == 8) { + state.lm_input = tensor; + } + *out = tensor; + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status RecordingTensorGetByteSize( + ryzenai_corelib_tensor_ptr tensor, + std::size_t* out) { + auto* value = State().Tensor(tensor); + if (value == nullptr || out == nullptr) { + return ryzenai_corelib_status_bad_argument; + } + *out = value->byte_size; + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status RecordingTensorGetDataType( + ryzenai_corelib_tensor_ptr tensor, + ryzenai_corelib_data_type* out) { + auto* value = State().Tensor(tensor); + if (value == nullptr || out == nullptr) { + return ryzenai_corelib_status_bad_argument; + } + *out = value->data_type; + return ryzenai_corelib_status_success; +} + +// `count` and `offset` are ELEMENTS of the TENSOR's dtype. The caller may +// hold a different dtype -- FP32 activations written into a BF16 tensor -- +// and corelib converts at this boundary, which is the only conversion +// path e5258d2 offers. +ryzenai_corelib_status RecordingTensorWrite( + ryzenai_corelib_tensor_ptr tensor, + ryzenai_corelib_data_type source_type, + const void* source, + std::size_t count, + std::size_t offset) { + auto& state = State(); + auto* value = state.Tensor(tensor); + if (value == nullptr || source == nullptr) { + return ryzenai_corelib_status_bad_argument; + } + const std::size_t element_size = TypeSize(value->data_type); + const std::size_t tensor_elements = value->byte_size / element_size; + if (offset > tensor_elements || count > tensor_elements - offset) { + return ryzenai_corelib_status_bad_argument; + } + const std::size_t size = count * element_size; + + const bool is_cache = + value->shape == + std::vector{8, 4096, 128}; + if (is_cache) { + if ( + state.failure == FailurePoint::ScatterBadAlloc) { + // The injected allocation failure belongs to the preceding V read. + return ryzenai_corelib_status_bad_argument; + } + if (value->label.starts_with("tensor_")) { + value->label = + "v_cache_" + std::to_string(state.active_layer); + } + const auto* words = + static_cast(source); + state.cache_publish_poison_observed = + state.cache_publish_poison_observed || + std::any_of( + words, + words + count, + [](std::uint16_t word) { return word == kPoison; }); + constexpr std::size_t head_pitch_elements = 4096u * 128u; + const std::size_t head = offset / head_pitch_elements; + state.events.push_back( + "tensor_write_v_head_" + std::to_string(head)); + } else if ( + state.active_layer >= 0 && + (value->last_producer == "q" || + value->last_producer == "k" || + value->last_producer == "attention" || + value->last_producer == "o" || + value->last_producer == "normalized")) { + state.events.push_back( + "padding_write_" + value->last_producer); + } else if ( + value == state.lm_input && + value->last_producer.empty()) { + state.events.emplace_back("padding_write_lm_input"); + value->last_producer = "lm_initialized"; + } else if ( + value->data_type == ryzenai_corelib_data_type_bf16 && + value->shape == + std::vector{4096, 3072}) { + if (state.failure == FailurePoint::StageBadAlloc) { + // The staging write is the first corelib call of a pass, so + // this stands in for the host allocation that used to fail + // inside the removed converter. + throw std::bad_alloc{}; + } + // Design 10.2: the host stays in FP32 and lets corelib narrow. + if (source_type != ryzenai_corelib_data_type_fp32) { + return ryzenai_corelib_status_bad_argument; + } + state.stage_writes.push_back({value, count, offset}); + if (state.stage_writes.size() % 2 == 1) { + state.staged_hidden = value; + } else { + state.staged_residual = value; + } + } + + std::vector staged(size); + for (std::size_t index = 0; index < count; ++index) { + WriteElement( + value->data_type, + staged.data(), + index, + ReadElement(source_type, source, index)); + } + return value->Write(staged.data(), size, offset * element_size) + ? ryzenai_corelib_status_success + : ryzenai_corelib_status_bad_argument; +} + +ryzenai_corelib_status RecordingTensorRead( + ryzenai_corelib_tensor_ptr tensor, + ryzenai_corelib_data_type destination_type, + void* destination, + std::size_t count, + std::size_t offset) { + auto& state = State(); + auto* value = state.Tensor(tensor); + if (value == nullptr || destination == nullptr) { + return ryzenai_corelib_status_bad_argument; + } + const std::size_t element_size = TypeSize(value->data_type); + const std::size_t tensor_elements = value->byte_size / element_size; + if (offset > tensor_elements || count > tensor_elements - offset) { + return ryzenai_corelib_status_bad_argument; + } + const std::size_t size = count * element_size; + + if (state.v_tensors.contains(value)) { + if (state.failure == FailurePoint::ScatterBadAlloc) { + throw std::bad_alloc{}; + } + if ( + state.failure == FailurePoint::ScatterBadAllocDeepLayer && + state.active_layer == 3) { + throw std::bad_alloc{}; + } + if (state.failure == FailurePoint::ScatterUnknown) { + throw UnknownFailure{}; + } + state.events.emplace_back("tensor_read_v"); + const std::size_t expected = + static_cast(state.active_rows) * + static_cast(constants::kKvDimension); + if (offset != 0 || count != expected) { + state.host_read_poison_observed = true; + } + } else if ( + value->shape.size() == 2 && + value->shape[1] == constants::kVocabularySize) { + state.events.emplace_back("tensor_read_logits"); + state.active_layer = -1; + if (state.lm_input != nullptr) { + state.lm_input->last_producer.clear(); + } + } + + state.host_read_poison_observed = + state.host_read_poison_observed || + (element_size == sizeof(std::uint16_t) && + value->ContainsPoisonWords(offset, count)); + + std::vector staged(size); + if (!value->Read(staged.data(), size, offset * element_size)) { + return ryzenai_corelib_status_bad_argument; + } + for (std::size_t index = 0; index < count; ++index) { + WriteElement( + destination_type, + destination, + index, + ReadElement(value->data_type, staged.data(), index)); + } + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status RecordingMatMulWeightsCreate( + const ryzenai_corelib_matmul_bf16_weights_desc* desc, + const ryzenai_corelib_matmul_bf16_onnx_components* components, + uint32_t threads, + ryzenai_corelib_matmul_bf16_weights_ptr* out) { + auto& state = State(); + state.ObserveLoadThread(); + if (desc == nullptr || components == nullptr || out == nullptr) { + return ryzenai_corelib_status_bad_argument; + } + if (threads != 0) { + return ryzenai_corelib_status_bad_argument; + } + const auto ordinal = state.matmul_weight_count++; + *out = state.Create( + MatMulLabel(ordinal), + *desc); + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status RecordingSsMlpWeightsCreate( + const ryzenai_corelib_ssmlp_bf16_weights_desc* desc, + const ryzenai_corelib_ssmlp_bf16_onnx_components* components, + uint32_t threads, + ryzenai_corelib_ssmlp_bf16_weights_ptr* out) { + auto& state = State(); + state.ObserveLoadThread(); + if (desc == nullptr || components == nullptr || out == nullptr) { + return ryzenai_corelib_status_bad_argument; + } + if (threads != 0) { + return ryzenai_corelib_status_bad_argument; + } + const auto ordinal = state.ssmlp_weight_count++; + *out = state.Create( + "ssmlp_" + std::to_string(ordinal), + *desc); + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status RecordingMatMulWeightsGetData( + ryzenai_corelib_matmul_bf16_weights_ptr weights, + const void** data, + std::size_t* size) { + if ( + State().Object(weights) == nullptr || data != nullptr || + size == nullptr) { + return ryzenai_corelib_status_bad_argument; + } + *size = 17; + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status RecordingSsMlpWeightsGetData( + ryzenai_corelib_ssmlp_bf16_weights_ptr weights, + const void** data, + std::size_t* size) { + if ( + State().Object(weights) == nullptr || data != nullptr || + size == nullptr) { + return ryzenai_corelib_status_bad_argument; + } + *size = 29; + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status RecordingMatMul( + ryzenai_corelib_stream_ptr stream, + ryzenai_corelib_tensor_ptr input, + std::int64_t rows, + ryzenai_corelib_matmul_bf16_weights_ptr weights, + ryzenai_corelib_tensor_ptr output) { + auto& state = State(); + auto* stream_object = state.Object(stream); + auto* input_tensor = state.Tensor(input); + auto* output_tensor = state.Tensor(output); + auto* weight_object = state.Object(weights); + if ( + stream_object == nullptr || + stream_object->kind != ObjectKind::Stream || + input_tensor == nullptr || output_tensor == nullptr || + weight_object == nullptr || + weight_object->kind != ObjectKind::MatMulWeight || + rows <= 0) { + return ryzenai_corelib_status_bad_argument; + } + auto* weight = static_cast(weight_object); + const std::string projection = Projection(weight->label); + const int layer = WeightLayer(weight->label); + if (layer >= 0) { + state.active_layer = layer; + } + state.active_rows = rows; + state.events.push_back("matmul_" + projection); + state.matmul_calls.push_back( + {weight->label, input_tensor, output_tensor, rows}); + + if ( + state.failure == FailurePoint::FirstQ && + weight->label == "q_0") { + flm::test::SetLastErrorMessage("injected first-q failure"); + return ryzenai_corelib_status_failure; + } + if ( + state.failure == FailurePoint::KAfterQ && + weight->label == "k_0") { + flm::test::SetLastErrorMessage("injected k failure"); + return ryzenai_corelib_status_failure; + } + if ( + state.failure == FailurePoint::KAfterQLayerOne && + weight->label == "k_1") { + flm::test::SetLastErrorMessage("injected layer-1 k failure"); + return ryzenai_corelib_status_failure; + } + // Rejected AFTER a completed synchronize, so nothing is outstanding. + // `o_0` follows the flat_mha synchronize; `lm_head` follows layer 31's + // ssmlp synchronize, 128 synchronizes into the step. + if ( + state.failure == FailurePoint::ORefusedAfterSynchronize && + weight->label == "o_0") { + flm::test::SetLastErrorMessage( + "injected o refusal after a completed synchronize"); + return ryzenai_corelib_status_unsupported; + } + if ( + state.failure == FailurePoint::LmHeadRefusedAfterSynchronize && + weight->label == "lm_head") { + flm::test::SetLastErrorMessage( + "injected lm_head refusal after a completed synchronize"); + return ryzenai_corelib_status_unsupported; + } + + const std::int64_t padded_rows = + MatMulPaddedRows(state, weight->desc, rows); + const std::size_t input_words = + static_cast(padded_rows) * + static_cast(weight->desc.k); + state.input_poison_observed = + state.input_poison_observed || + input_tensor->ContainsPoisonWords(0, input_words); + const std::size_t output_words = + static_cast(padded_rows) * + static_cast(weight->desc.n); + output_tensor->FillWords( + 0, + output_words, + projection == "lm_head" + ? static_cast(0x3F80u) + : static_cast( + 0x1000u + static_cast( + std::max(layer, 0) * 8 + + static_cast( + state.matmul_calls.size() % 8)))); + output_tensor->last_producer = projection; + + if (projection == "q") { + state.q_tensors.insert(output_tensor); + } else if (projection == "k") { + state.k_tensors.insert(output_tensor); + } else if (projection == "v") { + state.v_tensors.insert(output_tensor); + } + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status RecordingSsMlp( + ryzenai_corelib_stream_ptr stream, + ryzenai_corelib_tensor_ptr input, + ryzenai_corelib_tensor_ptr residual, + std::int64_t rows, + ryzenai_corelib_ssmlp_bf16_weights_ptr weights, + ryzenai_corelib_tensor_ptr skip_sum, + ryzenai_corelib_tensor_ptr normalized) { + auto& state = State(); + auto* stream_object = state.Object(stream); + auto* input_tensor = state.Tensor(input); + auto* residual_tensor = state.Tensor(residual); + auto* skip_tensor = state.Tensor(skip_sum); + auto* normalized_tensor = state.Tensor(normalized); + auto* weight_object = state.Object(weights); + if ( + stream_object == nullptr || + stream_object->kind != ObjectKind::Stream || + input_tensor == nullptr || residual_tensor == nullptr || + skip_tensor == nullptr || normalized_tensor == nullptr || + weight_object == nullptr || + weight_object->kind != ObjectKind::SsMlpWeight || + rows <= 0) { + return ryzenai_corelib_status_bad_argument; + } + // Layer 0's ssmlp, which follows the `o` synchronize: three completed + // synchronizes into the step, with nothing outstanding. + if ( + state.failure == FailurePoint::SsMlpRefusedAfterSynchronize && + state.active_layer == 0) { + flm::test::SetLastErrorMessage( + "injected ssmlp refusal after a completed synchronize"); + return ryzenai_corelib_status_unsupported; + } + state.events.emplace_back("ssmlp"); + state.ssmlp_calls.push_back( + {input_tensor, + residual_tensor, + skip_tensor, + normalized_tensor, + rows}); + + const std::size_t words = + static_cast(state.padding.ssmlp(rows)) * + static_cast(constants::kHiddenSize); + state.input_poison_observed = + state.input_poison_observed || + input_tensor->ContainsPoisonWords(0, words) || + residual_tensor->ContainsPoisonWords(0, words); + skip_tensor->FillWords(0, words, 0x2200u); + normalized_tensor->FillWords(0, words, 0x2300u); + skip_tensor->last_producer = "skip_sum"; + normalized_tensor->last_producer = "normalized"; + state.skip_sum_tensors.insert(skip_tensor); + state.normalized_tensors.insert(normalized_tensor); + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status RecordingFlatMha( + ryzenai_corelib_stream_ptr stream, + const ryzenai_corelib_flat_mha_bf16_desc* desc, + ryzenai_corelib_tensor_ptr query, + ryzenai_corelib_tensor_ptr key, + std::int64_t rows, + std::int64_t position, + ryzenai_corelib_tensor_ptr cos, + ryzenai_corelib_tensor_ptr sin, + ryzenai_corelib_tensor_ptr key_cache, + ryzenai_corelib_tensor_ptr value_cache, + ryzenai_corelib_tensor_ptr output) { + auto& state = State(); + auto* stream_object = state.Object(stream); + auto* q = state.Tensor(query); + auto* k = state.Tensor(key); + auto* cos_tensor = state.Tensor(cos); + auto* sin_tensor = state.Tensor(sin); + auto* k_cache = state.Tensor(key_cache); + auto* v_cache = state.Tensor(value_cache); + auto* out = state.Tensor(output); + if ( + stream_object == nullptr || + stream_object->kind != ObjectKind::Stream || + desc == nullptr || q == nullptr || k == nullptr || + cos_tensor == nullptr || sin_tensor == nullptr || + k_cache == nullptr || v_cache == nullptr || out == nullptr || + rows <= 0 || position < 0 || + (rows > 1 && position != 0)) { + return ryzenai_corelib_status_bad_argument; + } + if (k_cache->label.starts_with("tensor_")) { + k_cache->label = + "k_cache_" + std::to_string(state.active_layer); + } + if (v_cache->label.starts_with("tensor_")) { + v_cache->label = + "v_cache_" + std::to_string(state.active_layer); + } + if ( + state.failure == FailurePoint::MhaRefusedAfterSynchronize && + state.active_layer == 0) { + // The refusal corelib raises for an unservable attention window. + // It is raised before flat_mha touches the device, so nothing was + // enqueued -- and the q/k/v synchronize has already completed, so + // nothing earlier is outstanding either. + flm::test::SetLastErrorMessage( + "no token attention kernel ships for a 4096-token window"); + return ryzenai_corelib_status_unsupported; + } + state.events.emplace_back("flat_mha"); + state.mha_calls.push_back( + {state.active_layer, + q, + k, + k_cache, + v_cache, + out, + rows, + position}); + + const std::size_t padded = + static_cast(state.padding.attention(rows)); + state.input_poison_observed = + state.input_poison_observed || + q->ContainsPoisonWords( + 0, + padded * + static_cast( + constants::kQueryDimension)) || + k->ContainsPoisonWords( + 0, + padded * + static_cast( + constants::kKvDimension)); + for (std::size_t head = 0; head < 8; ++head) { + for (std::size_t row = 0; + row < static_cast(rows); + ++row) { + const std::size_t cache_base = + ((head * 4096u) + + static_cast(position) + row) * + 128u; + state.input_poison_observed = + state.input_poison_observed || + v_cache->ContainsPoisonWords(cache_base, 128); + k_cache->FillWords(cache_base, 128, 0x3100u); + } + } + out->FillWords( + 0, + padded * + static_cast(constants::kQueryDimension), + 0x3200u); + out->last_producer = "attention"; + state.attention_tensors.insert(out); + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status RecordingSynchronize( + ryzenai_corelib_stream_ptr stream) { + auto& state = State(); + auto* object = state.Object(stream); + if (object == nullptr || object->kind != ObjectKind::Stream) { + return ryzenai_corelib_status_bad_argument; + } + ++state.synchronize_calls; + state.events.emplace_back("synchronize"); + if ( + state.failure == FailurePoint::Synchronize || + state.failure == FailurePoint::DestructionSynchronize) { + state.destruction_failure_returned = + state.failure == FailurePoint::DestructionSynchronize; + flm::test::SetLastErrorMessage("injected synchronize failure"); + return ryzenai_corelib_status_failure; + } + return ryzenai_corelib_status_success; +} + +void MarkPostDestructionFailure(std::string_view action) { + auto& state = State(); + if ( + !state.destruction_failure_returned || + state.post_failure_marker.empty()) { + return; + } + std::ofstream output( + state.post_failure_marker, + std::ios::binary | std::ios::app); + output << action << '\n'; +} + +void RecordingRelease(ryzenai_corelib_object_ptr object) { + auto& state = State(); + MarkPostDestructionFailure("release"); + auto* value = state.Object(object); + CHECK(value != nullptr); + CHECK(!value->released); + value->released = true; + state.release_labels.push_back(value->label); +} + +void RecordingCleanup() { + MarkPostDestructionFailure("cleanup"); +} + +template +void* FunctionAddress(Function function) { + return reinterpret_cast(function); +} + +std::shared_ptr ResolveRecordingCorelib( + RecordingState& state) { + g_state = &state; + auto resolver = flm::test::CompleteCorelibResolver(); + resolver["ryzenai_corelib_object_release"] = FunctionAddress( + static_cast( + &RecordingRelease)); + resolver["ryzenai_corelib_cleanup"] = FunctionAddress( + static_cast( + &RecordingCleanup)); + resolver["ryzenai_corelib_create_stream"] = FunctionAddress( + static_cast( + &RecordingCreateStream)); + resolver["ryzenai_corelib_stream_synchronize"] = FunctionAddress( + static_cast( + &RecordingSynchronize)); + resolver["ryzenai_corelib_create_device_tensor"] = FunctionAddress( + static_cast( + &RecordingCreateTensor)); + resolver["ryzenai_corelib_tensor_write"] = FunctionAddress( + static_cast( + &RecordingTensorWrite)); + resolver["ryzenai_corelib_tensor_read"] = FunctionAddress( + static_cast( + &RecordingTensorRead)); + resolver["ryzenai_corelib_tensor_get_byte_size"] = FunctionAddress( + static_cast( + &RecordingTensorGetByteSize)); + resolver["ryzenai_corelib_tensor_get_data_type"] = FunctionAddress( + static_cast( + &RecordingTensorGetDataType)); + resolver["ryzenai_corelib_matmul_bf16_pad_shape"] = + FunctionAddress( + static_cast< + decltype(&::ryzenai_corelib_matmul_bf16_pad_shape)>( + &RecordingMatMulPadShape)); + resolver["ryzenai_corelib_ssmlp_bf16_pad_rows"] = + FunctionAddress( + static_cast< + decltype(&::ryzenai_corelib_ssmlp_bf16_pad_rows)>( + &RecordingSsMlpPadRows)); + resolver["ryzenai_corelib_flat_mha_bf16_pad_rows"] = + FunctionAddress( + static_cast< + decltype(&::ryzenai_corelib_flat_mha_bf16_pad_rows)>( + &RecordingMhaPadRows)); + resolver["ryzenai_corelib_matmul_bf16_weights_create_onnx"] = + FunctionAddress( + static_cast( + &RecordingMatMulWeightsCreate)); + resolver["ryzenai_corelib_matmul_bf16_weights_get_data"] = + FunctionAddress( + static_cast( + &RecordingMatMulWeightsGetData)); + resolver["ryzenai_corelib_ssmlp_bf16_weights_create_onnx"] = + FunctionAddress( + static_cast( + &RecordingSsMlpWeightsCreate)); + resolver["ryzenai_corelib_ssmlp_bf16_weights_get_data"] = + FunctionAddress( + static_cast( + &RecordingSsMlpWeightsGetData)); + resolver["ryzenai_corelib_matmul_bf16"] = FunctionAddress( + static_cast( + &RecordingMatMul)); + resolver["ryzenai_corelib_ssmlp_bf16"] = FunctionAddress( + static_cast( + &RecordingSsMlp)); + resolver["ryzenai_corelib_flat_mha_bf16"] = FunctionAddress( + static_cast( + &RecordingFlatMha)); + return CorelibApi::ResolveForTest( + [resolver = std::move(resolver)](std::string_view name) mutable + -> void* { + const auto found = resolver.find(std::string(name)); + return found == resolver.end() ? nullptr : found->second; + }); +} + +std::chrono::system_clock::time_point KnownStartTime() { + using namespace std::chrono; + return sys_days{year{2026} / September / day{1}}; +} + +FatalRecordStore MakeRecords(const std::filesystem::path& root) { + return FatalRecordStore( + root, + GetCurrentProcessId(), + KnownStartTime(), + [](DWORD) + -> std::optional { + return std::nullopt; + }); +} + +json ReadFatalRecord(const std::filesystem::path& root) { + for (const auto& entry : + std::filesystem::directory_iterator(root)) { + const auto name = entry.path().filename().string(); + if ( + name.starts_with("corelib-fatal-") && + name.ends_with(".json")) { + std::ifstream input(entry.path(), std::ios::binary); + return json::parse(input); + } + } + throw std::runtime_error("expected corelib fatal record"); +} + +struct TerminationIntercept final {}; + +class EngineFixture final { +public: + explicit EngineFixture( + const SyntheticPackage& package, + std::uint32_t max_length = 4096, + PaddingFunctions padding = {}) + : fatal_root_("fastflowlm-phi4-engine-fatal") { + flm::test::ResetFakeCorelib(); + state.padding = std::move(padding); + api_ = ResolveRecordingCorelib(state); + runtime = CorelibRuntime::Create( + api_, + MakeRecords(fatal_root_.path()), + [this](unsigned int code) { + state.terminator_called = true; + state.termination_code = code; + throw TerminationIntercept{}; + }); + LM_Config config; + engine = std::make_unique( + std::move(config), + package.path(), + runtime, + max_length); + } + + ~EngineFixture() noexcept { + try { + if ( + runtime && + runtime->state() == ProcessState::Terminating) { + (void)engine.release(); + if (g_state == &state) { + g_state = nullptr; + } + return; + } + engine.reset(); + if (runtime && runtime->state() == ProcessState::Healthy) { + runtime->ShutdownHealthy(); + } + } catch (...) { + std::terminate(); + } + if (g_state == &state) { + g_state = nullptr; + } + } + + void DestroyHealthy() { + CHECK(runtime->state() == ProcessState::Healthy); + engine.reset(); + CHECK(api_->live_object_count() == 0); + runtime->ShutdownHealthy(); + } + + json FatalRecord() const { + return ReadFatalRecord(fatal_root_.path()); + } + + // Existence without parsing, so a recoverable case can assert the + // ABSENCE of a record. `FatalRecord()` throws when there is none, which + // is the wrong shape for that: the comment on + // TestFailuresPastACompletedSynchronizeAreRecoverable claimed each case + // "asserts that no fatal record was written" while nothing did. + bool AnyFatalRecord() const { + for (const auto& entry : + std::filesystem::directory_iterator(fatal_root_.path())) { + const auto name = entry.path().filename().string(); + if ( + name.starts_with("corelib-fatal-") && + name.ends_with(".json")) { + return true; + } + } + return false; + } + + RecordingState state; + std::shared_ptr runtime; + std::unique_ptr engine; + +private: + TempDirectory fatal_root_; + std::shared_ptr api_; +}; + +void CheckLayerOrder(const std::vector& events) { + const std::array expected{ + "matmul_q", + "matmul_k", + "matmul_v", + "synchronize", + "tensor_read_v", + "tensor_write_v_head_0", + "tensor_write_v_head_1", + "tensor_write_v_head_2", + "tensor_write_v_head_3", + "tensor_write_v_head_4", + "tensor_write_v_head_5", + "tensor_write_v_head_6", + "tensor_write_v_head_7", + "flat_mha", + "synchronize", + "matmul_o", + "synchronize", + "ssmlp", + "synchronize"}; + CHECK(events.size() == 32u * expected.size() + 4u); + for (std::size_t layer = 0; layer < 32; ++layer) { + CHECK(std::equal( + expected.begin(), + expected.end(), + events.begin() + layer * expected.size())); + } + const auto final = events.end() - 4; + CHECK(final[0] == "padding_write_lm_input"); + CHECK(final[1] == "matmul_lm_head"); + CHECK(final[2] == "synchronize"); + CHECK(final[3] == "tensor_read_logits"); +} + +void CheckDistinctAndPingPong(const RecordingState& state) { + CHECK(state.ssmlp_calls.size() == 32); + FakeTensor* first_normalized = + state.ssmlp_calls.front().normalized; + FakeTensor* second_normalized = + state.ssmlp_calls.at(1).normalized; + CHECK(first_normalized != second_normalized); + for (std::size_t layer = 0; + layer < state.ssmlp_calls.size(); + ++layer) { + const auto& call = state.ssmlp_calls[layer]; + CHECK(call.input != call.residual); + CHECK(call.input != call.skip_sum); + CHECK(call.input != call.normalized); + CHECK(call.residual != call.skip_sum); + CHECK(call.residual != call.normalized); + CHECK(call.skip_sum != call.normalized); + CHECK( + call.normalized == + (layer % 2 == 0 ? first_normalized : second_normalized)); + if (layer + 1 < state.ssmlp_calls.size()) { + CHECK( + state.ssmlp_calls[layer + 1].input == + call.normalized); + CHECK( + state.ssmlp_calls[layer + 1].residual == + call.skip_sum); + } + } +} + +void CheckActualPersistentTail( + const std::set& tensors, + std::size_t live_prefix_words) { + CHECK(!tensors.empty()); + for (const auto* tensor : tensors) { + CHECK(tensor->ReadWord(live_prefix_words) == kPoison); + } +} + +void PoisonActualPersistentTail( + const std::set& tensors, + std::size_t first_word, + std::size_t word_count) { + for (auto* tensor : tensors) { + tensor->FillWords(first_word, word_count, kPoison); + } +} + +void CheckMetrics( + const Phi4Aie4Metrics& metrics, + std::uint64_t passes) { + CHECK(metrics.model_load_ns > 0); + CHECK(metrics.weight_pack_ns > 0); + CHECK(metrics.packed_weight_bytes == + 129u * 17u + 32u * 29u); + CHECK(metrics.mapped_source_bytes == kDataBytes); + CHECK(metrics.kv_bytes == 536870912u); + CHECK(metrics.scratch_bytes > 0); + CHECK(metrics.device_tensor_create_count == 76); + CHECK(metrics.weight_create_count == 161); + CHECK(metrics.dispatch_count == passes * 193u); + CHECK(metrics.synchronize_count == passes * 129u); + CHECK(metrics.padding_write_calls == passes); + CHECK( + metrics.padding_bytes == + passes * + static_cast(constants::kHiddenSize) * + sizeof(std::uint16_t)); + CHECK(metrics.attention_extent_queries == passes); + CHECK(metrics.output_projection_extent_queries == passes); + CHECK(metrics.lm_head_extent_queries == passes); + CHECK(std::all_of( + metrics.helper_transition_counts.begin(), + metrics.helper_transition_counts.end(), + [](std::uint32_t count) { return count > 0; })); + CHECK(metrics.v_read_calls == passes * 32u); + CHECK(metrics.v_write_calls == passes * 32u * 8u); + CHECK(metrics.v_bytes > 0); + CHECK(metrics.v_scatter_ns > 0); +} + +void TestOrderBuffersTailsStateAndMetrics( + const SyntheticPackage& package) { + EngineFixture fixture(package); + CHECK(fixture.state.load_thread_consistent); + CHECK(fixture.state.load_thread.has_value()); + CHECK(fixture.state.matmul_weight_count == 129); + CHECK(fixture.state.ssmlp_weight_count == 32); + CHECK(fixture.state.tensors.size() == 76); + + fixture.state.ResetExecutionRecords(); + std::vector prompt{1, 2, 3}; + buffer logits = fixture.engine->prefill(prompt); + CHECK(logits.size() == 200064); + CHECK(logits.is_owner()); + CHECK(fixture.engine->get_current_context_length() == 3); + CHECK(fixture.state.synchronize_calls == 129); + CheckLayerOrder(fixture.state.events); + CheckDistinctAndPingPong(fixture.state); + CHECK(fixture.state.stage_writes.size() == 2); + const std::size_t padded_hidden_elements = + static_cast(PaddedRows(3)) * + static_cast(constants::kHiddenSize); + CHECK(fixture.state.stage_writes[0].count == + padded_hidden_elements); + CHECK(fixture.state.stage_writes[1].count == + padded_hidden_elements); + CHECK(!fixture.state.input_poison_observed); + CHECK(!fixture.state.host_read_poison_observed); + CHECK(!fixture.state.cache_publish_poison_observed); + + const std::size_t hidden_tail = + static_cast(PaddedRows(3)) * + static_cast(constants::kHiddenSize); + const std::size_t kv_tail = + static_cast(PaddedRows(3)) * + static_cast(constants::kKvDimension); + CheckActualPersistentTail( + fixture.state.q_tensors, + hidden_tail); + CheckActualPersistentTail( + fixture.state.k_tensors, + kv_tail); + CheckActualPersistentTail( + fixture.state.attention_tensors, + hidden_tail); + CheckActualPersistentTail( + fixture.state.skip_sum_tensors, + hidden_tail); + CheckActualPersistentTail( + fixture.state.normalized_tensors, + hidden_tail); + CHECK(fixture.state.staged_hidden->ReadWord(hidden_tail) == + kPoison); + CHECK(fixture.state.staged_residual->ReadWord(hidden_tail) == + kPoison); + +#if defined(DEV_BUILD) + fixture.state.events.clear(); + const Phi4DebugSnapshot snapshot = + fixture.engine->debug_snapshot(); + CHECK(snapshot.live_rows == 3); + CHECK(snapshot.position == 3); + CHECK(snapshot.layer0_k.size() == 8u * 3u * 128u); + CHECK(snapshot.layer0_v.size() == 8u * 3u * 128u); + CHECK(snapshot.layer31_k.size() == 8u * 3u * 128u); + CHECK(snapshot.layer31_v.size() == 8u * 3u * 128u); + CHECK(snapshot.last_hidden.size() == 3072); + CHECK(snapshot.logits.size() == 200064); + CHECK(std::none_of( + snapshot.layer0_k.begin(), + snapshot.layer0_k.end(), + [](std::uint16_t value) { return value == kPoison; })); + CHECK(std::none_of( + snapshot.layer31_v.begin(), + snapshot.layer31_v.end(), + [](std::uint16_t value) { return value == kPoison; })); +#endif + + CheckMetrics(fixture.engine->metrics(), 1); + + const std::size_t one_hidden_row = + static_cast(constants::kHiddenSize); + const std::size_t one_kv_row = + static_cast(constants::kKvDimension); + PoisonActualPersistentTail( + fixture.state.q_tensors, + one_hidden_row, + hidden_tail - one_hidden_row); + PoisonActualPersistentTail( + fixture.state.k_tensors, + one_kv_row, + kv_tail - one_kv_row); + PoisonActualPersistentTail( + fixture.state.attention_tensors, + one_hidden_row, + hidden_tail - one_hidden_row); + PoisonActualPersistentTail( + fixture.state.skip_sum_tensors, + one_hidden_row, + hidden_tail - one_hidden_row); + PoisonActualPersistentTail( + fixture.state.normalized_tensors, + one_hidden_row, + hidden_tail - one_hidden_row); + fixture.state.staged_hidden->FillWords( + one_hidden_row, + hidden_tail - one_hidden_row, + kPoison); + fixture.state.staged_residual->FillWords( + one_hidden_row, + hidden_tail - one_hidden_row, + kPoison); + + fixture.state.ResetExecutionRecords(); + std::vector suffix{4, 5, 6}; + buffer suffix_logits = fixture.engine->prefill(suffix); + CHECK(suffix_logits.is_owner()); + CHECK(fixture.engine->get_current_context_length() == 6); + CHECK(fixture.state.mha_calls.size() == 3u * 32u); + for (std::size_t pass = 0; pass < 3; ++pass) { + for (std::size_t layer = 0; layer < 32; ++layer) { + const auto& call = + fixture.state.mha_calls[pass * 32 + layer]; + CHECK(call.rows == 1); + CHECK(call.position == 3 + static_cast(pass)); + } + } + CHECK(fixture.state.synchronize_calls == 3u * 129u); + CHECK(!fixture.state.input_poison_observed); + CHECK(!fixture.state.host_read_poison_observed); + CHECK(!fixture.state.cache_publish_poison_observed); + CheckActualPersistentTail( + fixture.state.q_tensors, + one_hidden_row); + CheckActualPersistentTail( + fixture.state.k_tensors, + one_kv_row); + CheckActualPersistentTail( + fixture.state.attention_tensors, + one_hidden_row); + CheckActualPersistentTail( + fixture.state.skip_sum_tensors, + one_hidden_row); + CheckActualPersistentTail( + fixture.state.normalized_tensors, + one_hidden_row); + CHECK( + fixture.state.staged_hidden->ReadWord(one_hidden_row) == + kPoison); + CHECK( + fixture.state.staged_residual->ReadWord(one_hidden_row) == + kPoison); + + fixture.state.ResetExecutionRecords(); + buffer forward_logits = fixture.engine->forward(7); + CHECK(forward_logits.is_owner()); + CHECK(fixture.engine->get_current_context_length() == 7); + CHECK(fixture.state.mha_calls.size() == 32); + CHECK(std::all_of( + fixture.state.mha_calls.begin(), + fixture.state.mha_calls.end(), + [](const MhaCall& call) { + return call.rows == 1 && call.position == 6; + })); + + CHECK(fixture.engine->checkpoint() == 7); + (void)fixture.engine->forward(8); + CHECK(fixture.engine->get_current_context_length() == 8); + CHECK(fixture.engine->restore() == 7); + CHECK(fixture.engine->get_current_context_length() == 7); + fixture.engine->set_context_length(7); + CheckThrowsContains( + [&] { fixture.engine->set_context_length(6); }, + "current"); + + fixture.engine->update_max_length(7); + CheckThrowsContains( + [&] { (void)fixture.engine->forward(9); }, + "maximum"); + CheckThrowsContains( + [&] { fixture.engine->update_max_length(0); }, + "1..4096"); + CheckThrowsContains( + [&] { fixture.engine->update_max_length(4097); }, + "1..4096"); + CheckThrowsContains( + [&] { fixture.engine->update_max_length(6); }, + "current"); + fixture.engine->update_max_length(4096); + + CheckThrowsContains( + [&] { (void)fixture.engine->get_k_cache(0, 0); }, + "unsupported"); + CheckThrowsContains( + [&] { (void)fixture.engine->get_v_cache(0, 0); }, + "unsupported"); + alignas(Q4NX) std::array q4nx_storage{}; + auto& q4nx = + *reinterpret_cast(q4nx_storage.data()); + CheckThrowsContains( + [&] { fixture.engine->load_weights(q4nx); }, + "unsupported"); + + fixture.engine->clear_context(); + CHECK(fixture.engine->get_current_context_length() == 0); + CheckThrowsContains( + [&] { (void)fixture.engine->restore(); }, + "checkpoint"); + + fixture.state.ResetExecutionRecords(); + std::vector one_token{10}; + (void)fixture.engine->prefill(one_token); + CHECK(fixture.engine->get_current_context_length() == 1); + CHECK(std::all_of( + fixture.state.mha_calls.begin(), + fixture.state.mha_calls.end(), + [](const MhaCall& call) { + return call.rows == 1 && call.position == 0; + })); + CHECK(fixture.state.stage_writes.size() == 2); + CHECK( + fixture.state.stage_writes[0].count == + static_cast(constants::kHiddenSize)); + + fixture.engine->clear_context(); + fixture.state.ResetExecutionRecords(); + std::vector too_many(4097, 0); + CheckThrowsContains( + [&] { (void)fixture.engine->prefill(too_many); }, + "maximum"); + CHECK(fixture.state.matmul_calls.empty()); + CHECK(fixture.engine->get_current_context_length() == 0); + + fixture.state.ResetExecutionRecords(); + fixture.DestroyHealthy(); + CHECK(fixture.state.synchronize_calls == 1); + CHECK(!fixture.state.release_labels.empty()); + CHECK(fixture.state.release_labels.front() == "stream"); + const auto first_weight = std::find_if( + fixture.state.release_labels.begin(), + fixture.state.release_labels.end(), + [](const std::string& label) { + return label.starts_with("q_") || + (label.starts_with("k_") && + !label.starts_with("k_cache_")) || + (label.starts_with("v_") && + !label.starts_with("v_cache_")) || + label.starts_with("o_") || + label.starts_with("ssmlp_") || + label == "lm_head"; + }); + CHECK(first_weight != fixture.state.release_labels.end()); + CHECK(std::all_of( + fixture.state.release_labels.begin() + 1, + first_weight, + [](const std::string& label) { + return label.starts_with("tensor_") || + label.starts_with("k_cache_") || + label.starts_with("v_cache_"); + })); + CHECK(std::none_of( + first_weight, + fixture.state.release_labels.end(), + [](const std::string& label) { + return label.starts_with("tensor_") || + label.starts_with("k_cache_") || + label.starts_with("v_cache_"); + })); +} + +std::size_t EventCount( + const RecordingState& state, + std::string_view event) { + return static_cast(std::count( + state.events.begin(), + state.events.end(), + event)); +} + +std::int64_t AlignRows(std::int64_t rows, std::int64_t alignment) { + return ((rows + alignment - 1) / alignment) * alignment; +} + +void CheckNoStaleConsumption(const RecordingState& state) { + CHECK(!state.input_poison_observed); + CHECK(!state.host_read_poison_observed); + CHECK(!state.cache_publish_poison_observed); +} + +void TestDivergentPaddingGrids(const SyntheticPackage& package) { + { + PaddingFunctions padding; + padding.query_projection = + [](std::int64_t rows) { return AlignRows(rows, 4); }; + padding.kv_projection = padding.query_projection; + padding.attention = + [](std::int64_t rows) { return AlignRows(rows, 8); }; + padding.ssmlp = padding.attention; + padding.lm_head = + [](std::int64_t rows) { return AlignRows(rows, 8); }; + EngineFixture fixture(package, 4096, std::move(padding)); + + fixture.state.ResetExecutionRecords(); + std::vector prompt{1, 2, 3}; + (void)fixture.engine->prefill(prompt); + CheckNoStaleConsumption(fixture.state); + CHECK(EventCount(fixture.state, "padding_write_q") == 32); + CHECK(EventCount(fixture.state, "padding_write_k") == 32); + CHECK( + EventCount(fixture.state, "padding_write_attention") == + 0); + CHECK(EventCount(fixture.state, "padding_write_o") == 32); + CHECK( + EventCount(fixture.state, "padding_write_normalized") == + 0); + CHECK( + EventCount(fixture.state, "padding_write_lm_input") == + 1); + CHECK(fixture.state.stage_writes.size() == 2); + CHECK( + fixture.state.stage_writes[0].count == + 4u * static_cast(constants::kHiddenSize)); + CHECK( + fixture.state.stage_writes[1].count == + 8u * static_cast(constants::kHiddenSize)); + const auto& metrics = fixture.engine->metrics(); + CHECK(metrics.padding_write_calls == 97); + CHECK(metrics.attention_extent_queries == 1); + CHECK(metrics.output_projection_extent_queries == 1); + CHECK(metrics.lm_head_extent_queries == 1); + CHECK( + metrics.padding_bytes == + 32u * 4u * + static_cast( + 2 * constants::kHiddenSize + + constants::kKvDimension) * + sizeof(std::uint16_t) + + 8u * + static_cast( + constants::kHiddenSize) * + sizeof(std::uint16_t)); + } + + { + PaddingFunctions padding; + padding.query_projection = + [](std::int64_t rows) { return AlignRows(rows, 8); }; + padding.kv_projection = padding.query_projection; + padding.attention = + [](std::int64_t rows) { return AlignRows(rows, 4); }; + padding.ssmlp = padding.attention; + padding.lm_head = + [](std::int64_t rows) { return AlignRows(rows, 8); }; + EngineFixture fixture(package, 4096, std::move(padding)); + + fixture.state.ResetExecutionRecords(); + std::vector prompt{1, 2, 3}; + (void)fixture.engine->prefill(prompt); + CheckNoStaleConsumption(fixture.state); + CHECK(EventCount(fixture.state, "padding_write_q") == 0); + CHECK(EventCount(fixture.state, "padding_write_k") == 0); + CHECK( + EventCount(fixture.state, "padding_write_attention") == + 32); + CHECK(EventCount(fixture.state, "padding_write_o") == 0); + CHECK( + EventCount(fixture.state, "padding_write_normalized") == + 31); + CHECK( + EventCount(fixture.state, "padding_write_lm_input") == + 1); + CHECK(fixture.state.stage_writes.size() == 2); + CHECK( + fixture.state.stage_writes[0].count == + 8u * static_cast(constants::kHiddenSize)); + CHECK( + fixture.state.stage_writes[1].count == + 4u * static_cast(constants::kHiddenSize)); + const auto& metrics = fixture.engine->metrics(); + CHECK(metrics.padding_write_calls == 64); + CHECK(metrics.attention_extent_queries == 1); + CHECK(metrics.output_projection_extent_queries == 1); + CHECK(metrics.lm_head_extent_queries == 1); + CHECK( + metrics.padding_bytes == + 63u * 4u * + static_cast( + constants::kHiddenSize) * + sizeof(std::uint16_t) + + 8u * + static_cast( + constants::kHiddenSize) * + sizeof(std::uint16_t)); + } +} + +void TestRecoverablePreSubmitFailures( + const SyntheticPackage& package) { + EngineFixture fixture(package); + std::vector prompt{1, 2}; + + fixture.state.failure = FailurePoint::StageBadAlloc; + try { + (void)fixture.engine->prefill(prompt); + } catch (const std::bad_alloc&) { + } catch (...) { + throw std::runtime_error( + "pre-submit staging bad_alloc changed exception type"); + } + CHECK(fixture.engine->get_current_context_length() == 0); + CHECK(fixture.runtime->state() == ProcessState::Healthy); + CHECK(fixture.state.matmul_calls.empty()); + + fixture.state.failure = FailurePoint::FirstQ; + try { + (void)fixture.engine->prefill(prompt); + } catch (const CorelibError& error) { + CHECK(error.call == "q"); + CHECK(error.detail == "injected first-q failure"); + } + CHECK(fixture.engine->get_current_context_length() == 0); + CHECK(fixture.runtime->state() == ProcessState::Healthy); + + fixture.state.failure = FailurePoint::None; + (void)fixture.engine->prefill(prompt); + CHECK(fixture.engine->get_current_context_length() == 2); +} + +// The engine's own guard on the token attention window. +// +// This is the layer that protects ANY caller, including one that never +// consults the frontend's aie4_active_cap() -- a different frontend, a direct +// embedder, or a future endpoint. The frontend bound is the one that produces +// a nice truncation; this one is the backstop, so it needs its own test rather +// than being covered by implication. +// +// Measured on hardware: a rows=1 step at position 4095 asks for a 4096-token +// window, which the shipped token attention kernel refuses. That refusal +// arrives from flat_mha mid-step, after q, k and v have been submitted and +// this layer's V cache has been scattered. It used to take the whole process +// with it; since the irrevocable boundary became per submission group it is +// recoverable, but the caller still loses the conversation. The whole point +// of checking here is to reach the same conclusion BEFORE anything is +// submitted, where it costs nothing at all. +void TestDecodeWindowRefusedBeforeSubmit( + const SyntheticPackage& package) { + EngineFixture fixture(package); + + // Jump to the last position a decode step can legally run from. Prefill + // uses the prefill attention path, whose window does reach the full + // kMaxSequenceLength, so this itself is allowed. + const auto window = + static_cast(flm::phi4::constants::kMaxDecodeWindow); + std::vector prompt(static_cast(window - 1), 1); + (void)fixture.engine->prefill(prompt); + CHECK(fixture.engine->get_current_context_length() == window - 1); + + // Window == kMaxDecodeWindow exactly. Allowed: this is the boundary, not + // one past it, and an off-by-one in the guard would show up here as a + // spurious refusal rather than as a missing one. + const std::size_t dispatches_before = fixture.state.matmul_calls.size(); + (void)fixture.engine->forward(2); + CHECK(fixture.engine->get_current_context_length() == window); + CHECK(fixture.state.matmul_calls.size() > dispatches_before); + + // Window == kMaxDecodeWindow + 1. Refused. + const std::size_t dispatches_at_boundary = + fixture.state.matmul_calls.size(); + const std::size_t syncs_at_boundary = fixture.state.synchronize_calls; + CheckThrowsContains( + [&] { (void)fixture.engine->forward(3); }, + "attention window"); + + // PRE-SUBMIT is the whole claim. Not one dispatch, not one synchronize: + // if anything had been submitted the failure policy would be entitled to + // terminate, and this guard would have bought nothing. + CHECK(fixture.state.matmul_calls.size() == dispatches_at_boundary); + CHECK(fixture.state.synchronize_calls == syncs_at_boundary); + // And the session survives, at the position it had reached. + CHECK(fixture.runtime->state() == ProcessState::Healthy); + CHECK(!fixture.state.terminator_called); + CHECK(fixture.engine->get_current_context_length() == window); + + // The same bound applies to a single-token prefill, which is a rows=1 + // dispatch wearing a different name. + std::vector one{4}; + CheckThrowsContains( + [&] { (void)fixture.engine->prefill(one); }, + "attention window"); + CHECK(fixture.state.matmul_calls.size() == dispatches_at_boundary); + CHECK(fixture.runtime->state() == ProcessState::Healthy); +} + +template +void CheckFatalFailure( + const SyntheticPackage& package, + FailurePoint failure, + CheckRecord&& check_record) { + EngineFixture fixture(package); + fixture.state.failure = failure; + std::vector prompt{1, 2}; + try { + (void)fixture.engine->prefill(prompt); + } catch (const TerminationIntercept&) { + } + CHECK(fixture.state.terminator_called); + CHECK(fixture.state.termination_code == 0xE0040001u); + CHECK(fixture.runtime->state() == ProcessState::Terminating); + CHECK(fixture.engine->get_current_context_length() == 0); + check_record(fixture.FatalRecord()); +} + +// I14. The irrevocable boundary is per submission group, not per step. +// +// `submission` answers exactly one question: is there dispatched work whose +// completion state this process cannot know? A successful stream_synchronize +// is a full barrier -- corelib waits every outstanding command and empties +// its own outstanding list -- so past one, the answer is no. +// +// It used to be latched for the whole of RunRows. After layer 0's first +// matmul, ~100 further dispatches and every host operation between them were +// classified irrevocable, so a bad_alloc in the V scatter or an overflow in a +// padding write killed the server (exit 0xE0040001) at a point where the +// stream was provably quiescent. The seven cases below are the ones that used +// to terminate and must not: three host failures past a synchronize, and all +// four rejected dispatches that follow one. +// +// The engine's only lasting state at these points is partially-updated KV +// caches, and the recoverable contract is that the caller throws the session +// away -- which the frontend does, via clear_after_corelib_error(). So each +// case also asserts that the process survives, that no fatal record was +// written, that the logical position did NOT advance, and that a clean +// prefill still works on the same engine afterwards. +void TestFailuresPastACompletedSynchronizeAreRecoverable( + const SyntheticPackage& package) { + const auto check_recoverable = + [&](FailurePoint failure, auto&& run) { + EngineFixture fixture(package); + fixture.state.failure = failure; + std::vector prompt{1, 2}; + run(fixture, prompt); + CHECK(!fixture.state.terminator_called); + CHECK(fixture.runtime->state() == ProcessState::Healthy); + // The comment above says "no fatal record was written". Now it + // is asserted rather than described. + CHECK(!fixture.AnyFatalRecord()); + // Nothing was committed: RunRows advances `position` only on a + // clean return. + CHECK(fixture.engine->get_current_context_length() == 0); + + // And the engine is still usable. The caller is expected to + // clear the session, but a recoverable classification that left + // the engine wedged would be a lie. + fixture.state.failure = FailurePoint::None; + std::vector retry{3, 4}; + (void)fixture.engine->prefill(retry); + CHECK(fixture.engine->get_current_context_length() == 2); + }; + + // A host allocation failure in the V scatter, one synchronize into the + // step. Previously terminated with a v_scatter fatal record. + check_recoverable( + FailurePoint::ScatterBadAlloc, + [](EngineFixture& fixture, std::vector& prompt) { + bool threw = false; + try { + (void)fixture.engine->prefill(prompt); + } catch (const std::bad_alloc&) { + threw = true; + } + CHECK(threw); + }); + + // The same, thirteen synchronizes in: the flag must stay clear across + // every one of them, not just the first. + check_recoverable( + FailurePoint::ScatterBadAllocDeepLayer, + [](EngineFixture& fixture, std::vector& prompt) { + bool threw = false; + try { + (void)fixture.engine->prefill(prompt); + } catch (const std::bad_alloc&) { + threw = true; + } + CHECK(threw); + CHECK(fixture.state.synchronize_calls == 13u); + }); + + // A non-standard exception takes the same route through catch(...). + check_recoverable( + FailurePoint::ScatterUnknown, + [](EngineFixture& fixture, std::vector& prompt) { + bool threw = false; + try { + (void)fixture.engine->prefill(prompt); + } catch (const UnknownFailure&) { + threw = true; + } + CHECK(threw); + }); + + // A REJECTED DISPATCH after a completed synchronize. This is the wider + // half of the change and it is deliberate: checked_submit sets the flag + // only after api->Check succeeds, because a rejected dispatch enqueued + // nothing -- the same reason a rejected FIRST dispatch of a step has + // always been recoverable. Confirmed against the corelib sources: every + // refusal in matmul_bf16, ssmlp_bf16 and flat_mha_bf16 precedes that + // call's enqueue point, and the 4096-token-window refusal modelled here + // is the first statement of flat_mha's Execute(). + check_recoverable( + FailurePoint::MhaRefusedAfterSynchronize, + [](EngineFixture& fixture, std::vector& prompt) { + bool threw = false; + try { + (void)fixture.engine->prefill(prompt); + } catch (const CorelibError& error) { + threw = true; + CHECK(error.call == "flat_mha"); + CHECK( + error.status == + ryzenai_corelib_status_unsupported); + CHECK( + error.detail.find("4096-token window") != + std::string::npos); + } + CHECK(threw); + CHECK(fixture.state.synchronize_calls == 1u); + }); + + // The OTHER THREE dispatches that follow a completed synchronize. + // + // The engine comment beside checked_synchronize named four -- flat_mha, + // o, ssmlp and lm_head -- and only flat_mha had a test. A comment + // asserting four cases on evidence covering one is a test agreeing with + // the implementation about which case exists, so here are the other + // three. Each pins the synchronize count as well as the call, because + // "after a completed synchronize" is the whole claim and a case that + // silently moved to a different point in the step would still throw. + const auto check_refused_dispatch = + [&](FailurePoint failure, + const char* call, + const char* detail, + unsigned synchronizes) { + check_recoverable( + failure, + [&](EngineFixture& fixture, std::vector& prompt) { + bool threw = false; + try { + (void)fixture.engine->prefill(prompt); + } catch (const CorelibError& error) { + threw = true; + CHECK(error.call == call); + CHECK( + error.status == + ryzenai_corelib_status_unsupported); + CHECK(error.detail == detail); + } + CHECK(threw); + CHECK( + fixture.state.synchronize_calls == synchronizes); + }); + }; + + // `o` at layer 0: after the qkv synchronize and the flat_mha + // synchronize. + check_refused_dispatch( + FailurePoint::ORefusedAfterSynchronize, + "o", + "injected o refusal after a completed synchronize", + 2u); + + // `ssmlp` at layer 0: one more synchronize on, after `o`'s. + check_refused_dispatch( + FailurePoint::SsMlpRefusedAfterSynchronize, + "ssmlp", + "injected ssmlp refusal after a completed synchronize", + 3u); + + // `lm_head`: the last dispatch of the step, 128 synchronizes in -- + // four per layer across all 32. If the flag failed to re-arm anywhere in + // that span this case would not reach lm_head at all. + check_refused_dispatch( + FailurePoint::LmHeadRefusedAfterSynchronize, + "lm_head", + "injected lm_head refusal after a completed synchronize", + 128u); +} + +void TestIrrevocableFailurePolicies( + const SyntheticPackage& package) { + CheckFatalFailure( + package, + FailurePoint::KAfterQ, + [](const json& record) { + CHECK(record.at("status") == + ryzenai_corelib_status_failure); + CHECK(record.at("call") == "k"); + CHECK(record.at("detail") == "injected k failure"); + CHECK(record.at("phase") == "qkv"); + CHECK(record.at("layer") == 0); + CHECK(record.at("rows") == 2); + CHECK(record.at("position") == 0); + }); + + CheckFatalFailure( + package, + FailurePoint::Synchronize, + [](const json& record) { + CHECK(record.at("status") == + ryzenai_corelib_status_failure); + CHECK( + record.at("call") == + "ryzenai_corelib_stream_synchronize"); + CHECK( + record.at("detail") == + "injected synchronize failure"); + CHECK(record.at("phase") == "qkv"); + CHECK(record.at("layer") == 0); + CHECK(record.at("rows") == 2); + CHECK(record.at("position") == 0); + }); + + // The flag RE-ARMS. Clearing it on synchronize would be a much worse + // defect than the one it fixes if it disarmed the boundary for the rest + // of the step: layer 1's q is submitted after four completed + // synchronizes have cleared it, and layer 1's k is then rejected with + // that q outstanding and unsynchronized. That must still terminate. + CheckFatalFailure( + package, + FailurePoint::KAfterQLayerOne, + [](const json& record) { + CHECK(record.at("status") == + ryzenai_corelib_status_failure); + CHECK(record.at("call") == "k"); + CHECK( + record.at("detail") == "injected layer-1 k failure"); + CHECK(record.at("phase") == "qkv"); + CHECK(record.at("layer") == 1); + CHECK(record.at("rows") == 2); + CHECK(record.at("position") == 0); + }); +} + +void TestSynchronizeFailureTerminatesWithoutSubmissionFlag() { + TempDirectory fatal_root("fastflowlm-phi4-policy-fatal"); + RecordingState state; + flm::test::ResetFakeCorelib(); + auto api = ResolveRecordingCorelib(state); + auto runtime = CorelibRuntime::Create( + api, + MakeRecords(fatal_root.path()), + [&state](unsigned int code) { + state.terminator_called = true; + state.termination_code = code; + throw TerminationIntercept{}; + }); + flm::corelib::StepSubmissionState submission; + CHECK(!submission.irrevocable()); + const CorelibError error( + ryzenai_corelib_status_failure, + "ryzenai_corelib_stream_synchronize", + "injected policy synchronize failure", + "failure"); + + try { + flm::phi4::testing::ApplyCorelibFailurePolicyForTest( + runtime, + error, + true, + submission, + "qkv", + 5, + 4, + 17); + } catch (const TerminationIntercept&) { + } + CHECK(state.terminator_called); + CHECK(state.termination_code == 0xE0040001u); + CHECK(runtime->state() == ProcessState::Terminating); + g_state = nullptr; +} + +std::filesystem::path CurrentExecutablePath() { + std::vector buffer(32768); + const DWORD length = GetModuleFileNameW( + nullptr, + buffer.data(), + static_cast(buffer.size())); + if (length == 0 || length == buffer.size()) { + throw std::runtime_error( + "failed to resolve engine-test executable path"); + } + return std::filesystem::path( + std::wstring(buffer.data(), length)); +} + +std::wstring QuoteProcessArgument( + const std::filesystem::path& argument) { + const std::wstring value = argument.wstring(); + if (value.find(L'"') != std::wstring::npos) { + throw std::runtime_error( + "engine-test child argument contains a quote"); + } + return L"\"" + value + L"\""; +} + +int RunDestructorFailureChild( + const std::filesystem::path& model_path, + const std::filesystem::path& fatal_root, + const std::filesystem::path& post_failure_marker) { + RecordingState state; + state.post_failure_marker = post_failure_marker; + flm::test::ResetFakeCorelib(); + auto api = ResolveRecordingCorelib(state); + auto runtime = CorelibRuntime::Create( + api, + MakeRecords(fatal_root), + [](unsigned int code) { + (void)TerminateProcess(GetCurrentProcess(), code); + ExitProcess(code); + }); + LM_Config config; + { + phi4_corelib_aie4 engine( + std::move(config), + model_path, + runtime, + 4096); + state.failure = FailurePoint::DestructionSynchronize; + } + return 3; +} + +void TestDestructorSynchronizeFailureChild( + const SyntheticPackage& package) { + TempDirectory fatal_root( + "fastflowlm-phi4-destruction-fatal"); + const auto marker = + fatal_root.path() / "post-failure-cleanup.txt"; + const auto executable = CurrentExecutablePath(); + std::wstring command = + QuoteProcessArgument(executable) + + L" --destructor-failure-child " + + QuoteProcessArgument(package.path()) + L" " + + QuoteProcessArgument(fatal_root.path()) + L" " + + QuoteProcessArgument(marker); + std::vector mutable_command( + command.begin(), + command.end()); + mutable_command.push_back(L'\0'); + + STARTUPINFOW startup{}; + startup.cb = sizeof(startup); + PROCESS_INFORMATION process{}; + if (!CreateProcessW( + nullptr, + mutable_command.data(), + nullptr, + nullptr, + FALSE, + 0, + nullptr, + nullptr, + &startup, + &process)) { + throw std::runtime_error( + "failed to start destruction-failure child"); + } + CloseHandle(process.hThread); + const DWORD wait_result = + WaitForSingleObject(process.hProcess, 120000); + if (wait_result != WAIT_OBJECT_0) { + (void)TerminateProcess(process.hProcess, 2); + CloseHandle(process.hProcess); + throw std::runtime_error( + "destruction-failure child did not exit"); + } + DWORD exit_code = 0; + const BOOL queried = + GetExitCodeProcess(process.hProcess, &exit_code); + CloseHandle(process.hProcess); + CHECK(queried != FALSE); + CHECK(exit_code == 0xE0040001u); + CHECK(!std::filesystem::exists(marker)); + + const json record = ReadFatalRecord(fatal_root.path()); + CHECK( + record.at("status") == + ryzenai_corelib_status_failure); + CHECK( + record.at("call") == + "ryzenai_corelib_stream_synchronize"); + CHECK(record.at("phase") == "destruction"); + CHECK(record.at("layer").is_null()); + CHECK(record.at("rows") == 0); + CHECK(record.at("position") == 0); +} + +void TestConfigIdentityMismatchIsRejected( + const SyntheticPackage& package) { + RecordingState state; + flm::test::ResetFakeCorelib(); + TempDirectory fatal_root( + "fastflowlm-phi4-config-identity"); + auto api = ResolveRecordingCorelib(state); + auto runtime = CorelibRuntime::Create( + api, + MakeRecords(fatal_root.path()), + [](unsigned int) { + throw std::runtime_error( + "config validation must not terminate"); + }); + + LM_Config config; + config._json_config = { + {"hidden_size", constants::kHiddenSize + 1}, + {"num_hidden_layers", constants::kLayerCount}, + {"vocab_size", constants::kVocabularySize}, + }; + + bool rejected = false; + try { + phi4_corelib_aie4 engine( + std::move(config), + package.path(), + runtime, + 4096); + } catch (const std::invalid_argument& error) { + CHECK( + std::string_view(error.what()).find("hidden_size") != + std::string_view::npos); + rejected = true; + } + runtime->ShutdownHealthy(); + if (g_state == &state) { + g_state = nullptr; + } + CHECK(rejected); + CHECK(state.objects.empty()); + CHECK(state.tensors.empty()); + CHECK(state.matmul_weight_count == 0); + CHECK(state.ssmlp_weight_count == 0); +} + +static_assert(std::is_base_of_v); +static_assert(std::has_virtual_destructor_v); + +} // namespace + +int wmain(int argc, wchar_t* argv[]) { + try { + if ( + argc == 5 && + std::wstring_view(argv[1]) == + L"--destructor-failure-child") { + return RunDestructorFailureChild( + argv[2], + argv[3], + argv[4]); + } + SyntheticPackage package; + TestConfigIdentityMismatchIsRejected(package); + TestOrderBuffersTailsStateAndMetrics(package); + TestDivergentPaddingGrids(package); + TestRecoverablePreSubmitFailures(package); + TestDecodeWindowRefusedBeforeSubmit(package); + TestDestructorSynchronizeFailureChild(package); + TestFailuresPastACompletedSynchronizeAreRecoverable(package); + TestIrrevocableFailurePolicies(package); + TestSynchronizeFailureTerminatesWithoutSubmissionFlag(); + std::cout << "test_phi4_engine: PASS\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } catch (...) { + std::cerr << "unexpected non-standard exception\n"; + return 1; + } +} diff --git a/src/test/phi4_corelib_aie4/test_phi4_frontend.cpp b/src/test/phi4_corelib_aie4/test_phi4_frontend.cpp new file mode 100644 index 00000000..7ac9c3e4 --- /dev/null +++ b/src/test/phi4_corelib_aie4/test_phi4_frontend.cpp @@ -0,0 +1,1841 @@ +#include +#include +#include + +#if defined(FLM_ENABLE_CORELIB_AIE4) +#include +#include +#include +#endif + +#define FLM_PHI4_FRONTEND_TEST_SUPPORT +#include "test_support.hpp" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +std::vector g_encoded_tokens; +int g_sample_token = 7; +std::deque g_sample_tokens; +int g_sampler_reset_count = 0; +int g_sampler_sample_count = 0; + +std::filesystem::path CurrentExecutablePath() { + std::wstring buffer(32768, L'\0'); + const DWORD written = GetModuleFileNameW( + nullptr, + buffer.data(), + static_cast(buffer.size())); + if (written == 0 || written >= buffer.size()) { + throw std::runtime_error("GetModuleFileNameW failed"); + } + buffer.resize(written); + return std::filesystem::path(buffer); +} + +class TempModelPackage final { +public: + explicit TempModelPackage( + std::vector eos_ids = {200020, 199999}, + std::optional hidden_size = 3072, + nlohmann::json config_overrides = nlohmann::json::object()) { + const auto base = std::filesystem::temp_directory_path(); + for (int attempt = 0; attempt < 100; ++attempt) { + path_ = base / + ("flm-phi4-frontend-" + + std::to_string(GetCurrentProcessId()) + "-" + + std::to_string(GetTickCount64()) + "-" + + std::to_string(attempt)); + std::error_code error; + if (std::filesystem::create_directory(path_, error)) { + break; + } + if (attempt == 99) { + throw std::runtime_error( + "failed to create temporary Phi-4 package"); + } + } + + nlohmann::json config = { + {"model_type", "phi4"}, + {"num_hidden_layers", 32}, + {"hidden_size", hidden_size.value_or(3072)}, + {"intermediate_size", 8192}, + {"num_attention_heads", 24}, + {"num_key_value_heads", 8}, + {"head_dim", 128}, + {"vocab_size", 200064}, + {"rms_norm_eps", 1.0e-5}, + }; + for (const auto& [key, value] : config_overrides.items()) { + config[key] = value; + } + WriteJson(path_ / "config.json", config); + + nlohmann::json tokenizer_config = { + {"chat_template", + "{% for message in messages %}" + "{{ message['content'] }}" + "{% endfor %}"}, + {"eos_token_id", std::move(eos_ids)}, + }; + WriteJson(path_ / "tokenizer_config.json", tokenizer_config); + WriteText(path_ / "tokenizer.json", "{}"); + } + + ~TempModelPackage() { + std::error_code ignored; + std::filesystem::remove_all(path_, ignored); + } + + TempModelPackage(const TempModelPackage&) = delete; + TempModelPackage& operator=(const TempModelPackage&) = delete; + + const std::filesystem::path& path() const noexcept { + return path_; + } + + void OverwriteFile( + std::string_view name, + std::string_view contents) const { + WriteText(path_ / name, contents); + } + +private: + static void WriteJson( + const std::filesystem::path& path, + const nlohmann::json& value) { + WriteText(path, value.dump()); + } + + static void WriteText( + const std::filesystem::path& path, + std::string_view value) { + std::ofstream output(path, std::ios::binary); + if (!output) { + throw std::runtime_error( + "failed to create frontend test package file"); + } + output.write( + value.data(), + static_cast(value.size())); + } + + std::filesystem::path path_; +}; + +struct FactoryState { + int calls = 0; + bool last_was_corelib = false; + FakeEngine* engine = nullptr; +}; + +FactoryState g_factory; + +nlohmann::ordered_json ModelInfo( + int default_context_length, + std::optional execution_backend = std::nullopt) { + nlohmann::ordered_json details = { + {"family", "phi4"}, + }; + if (execution_backend.has_value()) { + details["execution_backend"] = *execution_backend; + } + return { + {"default_context_length", default_context_length}, + {"details", std::move(details)}, + }; +} + +chat_meta_info_t Meta() { + chat_meta_info_t meta; + meta.max_prefill_len = 64; + return meta; +} + +lm_uniform_input_t Prompt( + std::optional requested = std::nullopt) { + lm_uniform_input_t input; + input.prompt = "frontend-test"; + input.requested_max_new_tokens = requested; + return input; +} + +void SetSamples(std::initializer_list samples) { + g_sample_tokens.assign(samples.begin(), samples.end()); +} + +template +void CheckRequestError( + Function&& function, + int expected_code, + bool expected_session_cleared, + std::string_view expected_message) { + try { + function(); + } catch (const ModelRequestError& error) { + CHECK(error.http_code() == expected_code); + CHECK(error.session_cleared() == expected_session_cleared); + CHECK( + std::string_view(error.what()).find(expected_message) != + std::string_view::npos); + return; + } + throw std::runtime_error("expected ModelRequestError was not thrown"); +} + +} // namespace + +Tokenizer::Tokenizer(const std::string&) { + is_doubled_encoded = false; +} + +Tokenizer::~Tokenizer() = default; + +std::vector Tokenizer::encode(const std::string&) { + return g_encoded_tokens; +} + +std::string Tokenizer::decode(const std::vector&) { + return "decoded"; +} + +std::string Tokenizer::run_time_decoder(int token) { + return "token-" + std::to_string(token); +} + +SafeTensors::~SafeTensors() = default; + +Sampler::Sampler(int features, sampler_config& config) + : in_features(features), + rep_penalty(config.rep_penalty), + freq_penalty(config.freq_penalty), + pre_penalty(config.pre_penalty), + top_k(config.top_k), + top_p(config.top_p), + min_p(config.min_p), + temperature(config.temperature), + total_tokens(0), + freq_penalty_window(config.freq_penalty_window), + rep_penalty_window(config.rep_penalty_window), + repeat_last_n(config.repeat_last_n), + use_optimized_sampling(config.use_optimized_sampling) { + logits.resize(1); + counters.resize(1); + token_positions.resize(1, -1); +} + +void Sampler::reset_penalties() { + ++g_sampler_reset_count; + std::fill(counters.begin(), counters.end(), 0); + std::fill(token_positions.begin(), token_positions.end(), -1); + token_counts_sparse.clear(); + token_history.clear(); + total_tokens = 0; +} + +int Sampler::sample(buffer&) { + ++g_sampler_sample_count; + ++total_tokens; + const int sampled_token = g_sample_tokens.empty() + ? g_sample_token + : g_sample_tokens.front(); + if (!g_sample_tokens.empty()) { + g_sample_tokens.pop_front(); + } + token_history.push_back(sampled_token); + return sampled_token; +} + +namespace utils { + +std::string get_executable_directory() { + return CurrentExecutablePath().parent_path().string(); +} + +} // namespace utils + +namespace flm::phi4::testing { + +class Phi4FrontendTestAccess final { +public: + using Factory = std::function( + bool, + const LM_Config&, + npu_xclbin_manager*, + const std::filesystem::path&, + std::uint32_t)>; + + static void InstallFactory() { + g_factory = {}; + Phi4::engine_factory_for_testing_ = + [](bool corelib, + const LM_Config&, + npu_xclbin_manager*, + const std::filesystem::path&, + std::uint32_t max_length) { + ++g_factory.calls; + g_factory.last_was_corelib = corelib; + auto engine = + std::make_unique(max_length); + g_factory.engine = engine.get(); + return engine; + }; + } + + static void RemoveFactory() { + Phi4::engine_factory_for_testing_ = {}; + g_factory = {}; + } + + static bool HasLegacyNpu(const Phi4& model) { + return model.npu != nullptr; + } + +#if defined(FLM_ENABLE_CORELIB_AIE4) + static bool HasRuntime(const Phi4& model) { + return model.corelib_runtime_ != nullptr; + } + + static void SetMetrics( + Phi4& model, + const Phi4Aie4Metrics& metrics) { + model.metrics_for_testing_ = metrics; + } + + static void ForceRoute( + Phi4& model, + ForcedContinuationRoute route) { + model.forced_continuation_route_ = route; + } +#endif + + static const std::vector& History(const Phi4& model) { + return model.token_history; + } + + static int LastToken(const Phi4& model) { + return model.last_token; + } + + static bool SharedInsert( + Phi4& model, + chat_meta_info_t& meta, + std::vector& tokens, + void* payload) { + return model._shared_insert( + meta, + tokens, + [] { return false; }, + payload, + 0); + } +}; + +} // namespace flm::phi4::testing + +namespace { + +class NonPhiModel final : public AutoModel { +public: + NonPhiModel() : AutoModel(nullptr, "non-phi-test") {} + + std::string generate( + chat_meta_info_t&, + int, + std::ostream&, + std::function) override { + return {}; + } + + bool insert( + chat_meta_info_t&, + lm_uniform_input_t&, + std::function) override { + return true; + } + + std::string generate_with_prompt( + chat_meta_info_t&, + lm_uniform_input_t&, + int, + std::ostream&) override { + return {}; + } + + std::string apply_chat_template( + nlohmann::ordered_json&, + nlohmann::ordered_json) override { + return {}; + } +}; + +using flm::phi4::ContinuationRoute; +using flm::phi4::ForcedContinuationRoute; +using flm::phi4::kContinuationAppendThreshold; +using flm::phi4::SelectContinuationRoute; + +// The release-fixed threshold is generated by +// `tools/calibrate_phi4_corelib_continuation.py`; the tests below are written +// against the symbol, not its current value. This bound is the one thing about +// the value they do assert: a threshold at or above the physical 4096-row +// cache could never be reached by a suffix that also fits, so it would make +// the Automatic append branch dead in a way no route test would notice. +static_assert( + kContinuationAppendThreshold < 4096, + "continuation append threshold must be below the physical KV row count"); +using flm::phi4::testing::Phi4FrontendTestAccess; + +struct FactoryScope final { + FactoryScope() { + Phi4FrontendTestAccess::InstallFactory(); + } + + ~FactoryScope() { + Phi4FrontendTestAccess::RemoveFactory(); + } +}; + +std::unique_ptr Load( + const TempModelPackage& package, + nlohmann::ordered_json model_info, + int requested_context = -1, + bool preemption = false, + flm_rt::device* device = + reinterpret_cast(std::uintptr_t{1})) { + auto model = std::make_unique(device); + model->load_model( + package.path().string(), + std::move(model_info), + requested_context, + preemption); + return model; +} + +// Every branch of `SelectContinuationRoute`, expressed against the generated +// `kContinuationAppendThreshold` rather than against the number it happens to +// hold today. +// +// The earlier version of this test asserted that suffix 1 re-prefills under +// Automatic. That was true only because the threshold was still the +// placeholder zero, so the test and the header agreed on an assumption neither +// of them stated -- and the test would have gone on passing while silently +// covering nothing, because with a zero threshold the Automatic append branch +// is unreachable. Calibration set the threshold to a positive value and the +// assertion became wrong. What follows enumerates the cases the FUNCTION +// branches on: the zero-suffix short circuit under all three forced values, +// both forced overrides above and below the threshold, and both sides of the +// Automatic comparison including its exact boundary. +void TestContinuationSelector() { + constexpr std::size_t kThreshold = kContinuationAppendThreshold; + + // A zero-length suffix re-prefills whatever the caller forced: there is no + // token to append, and this precedes the forced switch. + for (const ForcedContinuationRoute forced : { + ForcedContinuationRoute::Automatic, + ForcedContinuationRoute::Append, + ForcedContinuationRoute::Reprefill}) { + CHECK( + SelectContinuationRoute(0, forced) == + ContinuationRoute::Reprefill); + } + + // A forced route ignores the threshold in both directions. + for (const std::size_t suffix : { + std::size_t{1}, + kThreshold, + kThreshold + 1, + std::size_t{999}}) { + if (suffix == 0) { + continue; // covered above; `kThreshold` may legitimately be 0 + } + CHECK( + SelectContinuationRoute( + suffix, + ForcedContinuationRoute::Append) == + ContinuationRoute::Append); + CHECK( + SelectContinuationRoute( + suffix, + ForcedContinuationRoute::Reprefill) == + ContinuationRoute::Reprefill); + } + + // Automatic: append at and below the threshold, re-prefill above it. + // + // A zero threshold is a legitimate calibration outcome -- Section 10.7 + // selects it when no sampled suffix length wins or the winners are not + // prefix-contiguous -- and it makes the append side unreachable. Asserting + // it separately keeps this test from silently becoming vacuous if a later + // recalibration lands there. + if (kThreshold == 0) { + CHECK( + SelectContinuationRoute( + 1, + ForcedContinuationRoute::Automatic) == + ContinuationRoute::Reprefill); + } else { + for (std::size_t suffix = 1; suffix <= kThreshold; ++suffix) { + CHECK( + SelectContinuationRoute( + suffix, + ForcedContinuationRoute::Automatic) == + ContinuationRoute::Append); + } + } + for (const std::size_t suffix : { + kThreshold + 1, + kThreshold + 2, + std::size_t{4096}}) { + CHECK( + SelectContinuationRoute( + suffix, + ForcedContinuationRoute::Automatic) == + ContinuationRoute::Reprefill); + } +} + +void TestLegacyRoutingAndUnknownBackend() { + TempModelPackage package({200020}); + FactoryScope factory; + + auto legacy = Load(package, ModelInfo(1024)); + CHECK(g_factory.calls == 1); + CHECK(!g_factory.last_was_corelib); + CHECK(!legacy->uses_corelib_aie4()); + CHECK(Phi4FrontendTestAccess::HasLegacyNpu(*legacy)); + + FakeEngine* legacy_engine = g_factory.engine; + g_encoded_tokens = {1, 2}; + auto meta = Meta(); + auto input = Prompt(); + CHECK(legacy->insert(meta, input)); + legacy_engine->prefill_calls.clear(); + g_encoded_tokens = {1, 2, 3, 4}; + CHECK(legacy->insert(meta, input)); + CHECK(legacy_engine->prefill_calls.size() == 1); + CHECK( + legacy_engine->prefill_calls[0] == + std::vector({3, 4})); + + legacy->set_max_length(512); + CHECK(legacy->get_max_length() == 1024); + CHECK(legacy_engine->max_length == 512); + CHECK( + legacy->show_profile() == + legacy->AutoModel::show_profile()); + + CheckThrowsContains( + [&] { + auto model = Load( + package, + ModelInfo(64, "invented_backend")); + (void)model; + }, + "invented_backend"); + CHECK(g_factory.calls == 1); +} + +void TestLoadedBackendControlsOmittedLimit() { + const ParsedGenerationLimit omitted{false, -1}; + const nlohmann::ordered_json misleading_catalog_info = { + {"details", {{"execution_backend", "corelib_aie4"}}}, + }; + CHECK(IsCorelibAie4ModelInfo(misleading_catalog_info)); + + NonPhiModel non_phi; + CHECK(!non_phi.uses_corelib_aie4()); + CHECK( + GenerationLoopLimit( + omitted, + non_phi.uses_corelib_aie4()) == 4096); + + TempModelPackage package({200020}); + FactoryScope factory; + auto legacy_phi = Load(package, ModelInfo(1024)); + CHECK(!legacy_phi->uses_corelib_aie4()); + CHECK( + GenerationLoopLimit( + omitted, + legacy_phi->uses_corelib_aie4()) == 4096); +} + +void TestLegacyExactRepeatPreservesEmptyPrefillPayload() { + TempModelPackage package({200020}); + FactoryScope factory; + auto model = Load(package, ModelInfo(64)); + FakeEngine* engine = g_factory.engine; + + int first_payload = 1; + int repeated_payload = 2; + auto meta = Meta(); + std::vector first{1, 2}; + CHECK(Phi4FrontendTestAccess::SharedInsert( + *model, + meta, + first, + &first_payload)); + const int samples_before_repeat = g_sampler_sample_count; + + std::vector repeated{1, 2}; + CHECK(Phi4FrontendTestAccess::SharedInsert( + *model, + meta, + repeated, + &repeated_payload)); + CHECK(engine->prefill_calls.size() == 2); + CHECK(engine->prefill_calls.back().empty()); + CHECK(engine->prefill_payloads.back() == &repeated_payload); + CHECK(g_sampler_sample_count == samples_before_repeat + 1); +} + +#if !defined(FLM_ENABLE_CORELIB_AIE4) + +void TestFeatureOffRejectsCorelibTag() { + TempModelPackage package; + FactoryScope factory; + + CheckThrowsContains( + [&] { + auto model = Load( + package, + ModelInfo(64, "corelib_aie4"), + -1, + false, + nullptr); + (void)model; + }, + "without Phi-4 AIE4 corelib support"); + CHECK(g_factory.calls == 0); +} + +#else + +void ConfigureFakeCorelibDll() { + const auto fake_dll = + CurrentExecutablePath().parent_path() / + "fake_ryzenai_corelib.dll"; + CHECK(std::filesystem::exists(fake_dll)); + if (_wputenv_s( + L"RYZENAI_CORELIB_PATH", + fake_dll.c_str()) != 0) { + throw std::runtime_error( + "failed to configure RYZENAI_CORELIB_PATH"); + } +} + +// Design `MODEL-2`. The overlay config.json is a restatement of the model +// contract, so any disagreement with the validated constants must fail the +// load rather than quietly reconfigure the model. Each corruption below is a +// value the AIE4 frontend and weight loader would otherwise trust. +void TestCorruptOverlayConfigFailsLoad() { + const nlohmann::json corruptions[] = { + {{"num_hidden_layers", 28}}, + {{"hidden_size", 4096}}, + {{"intermediate_size", 8960}}, + {{"num_attention_heads", 32}}, + {{"num_key_value_heads", 4}}, + {{"head_dim", 64}}, + {{"vocab_size", 200065}}, + {{"rms_norm_eps", 1.0e-6}}, + {{"model_type", "phi3"}}, + }; + for (const auto& corruption : corruptions) { + TempModelPackage package({200020, 199999}, 3072, corruption); + FactoryScope factory; + CheckThrowsContains( + [&] { + auto model = Load( + package, + ModelInfo(64, "corelib_aie4"), + -1, + false, + nullptr); + (void)model; + }, + "config.json"); + // The engine must never be constructed from a package whose declared + // contract does not match the validated one. + CHECK(g_factory.calls == 0); + } + + // A config.json that is not even parseable must fail the same way rather + // than falling through to a partially initialized model. + { + TempModelPackage package; + package.OverwriteFile("config.json", "{not json"); + FactoryScope factory; + CheckThrowsContains( + [&] { + auto model = Load( + package, + ModelInfo(64, "corelib_aie4"), + -1, + false, + nullptr); + (void)model; + }, + "config.json"); + CHECK(g_factory.calls == 0); + } +} + +void TestCorelibRoutingAndPreemption() { + TempModelPackage package; + FactoryScope factory; + + CheckThrowsContains( + [&] { + auto model = Load( + package, + ModelInfo(64, "corelib_aie4"), + -1, + true, + nullptr); + (void)model; + }, + "preemption"); + CHECK(g_factory.calls == 0); + + auto model = Load( + package, + ModelInfo(64, "corelib_aie4"), + -1, + false, + nullptr); + CHECK(g_factory.calls == 1); + CHECK(g_factory.last_was_corelib); + CHECK(model->uses_corelib_aie4()); + CHECK(!Phi4FrontendTestAccess::HasLegacyNpu(*model)); + CHECK(Phi4FrontendTestAccess::HasRuntime(*model)); +} + +void TestInitialAndAtomicCaps() { + TempModelPackage package; + FactoryScope factory; + + CheckThrowsContains( + [&] { + auto model = Load( + package, + ModelInfo(4097, "corelib_aie4"), + -1, + false, + nullptr); + (void)model; + }, + "1..4096"); + CHECK(g_factory.calls == 0); + + auto model = Load( + package, + ModelInfo(1024, "corelib_aie4"), + -1, + false, + nullptr); + FakeEngine* engine = g_factory.engine; + CHECK(model->get_max_length() == 1024); + CHECK(engine->max_length == 1024); + + CheckThrowsContains( + [&] { model->set_max_length(4097); }, + "1..4096"); + CHECK(model->get_max_length() == 1024); + CHECK(engine->max_length == 1024); + + model->clear_context(); + CHECK(model->get_current_context_length() == 0); + model->set_max_length(512); + CHECK(model->get_max_length() == 512); + CHECK(engine->max_length == 512); + + g_encoded_tokens.assign(10, 11); + auto meta = Meta(); + auto input = Prompt(); + CHECK(model->insert(meta, input)); + CHECK(model->get_current_context_length() == 10); + CHECK(engine->position == 10); + + CheckThrowsContains( + [&] { model->set_max_length(9); }, + "current"); + CHECK(model->get_max_length() == 512); + CHECK(engine->max_length == 512); + CHECK(engine->position == 10); +} + +void TestEnginePositionIsAuthoritativeForCapUpdate() { + TempModelPackage package; + FactoryScope factory; + auto model = Load( + package, + ModelInfo(64, "corelib_aie4"), + -1, + false, + nullptr); + FakeEngine* engine = g_factory.engine; + + g_encoded_tokens.assign(10, 11); + auto meta = Meta(); + auto input = Prompt(); + CHECK(model->insert(meta, input)); + engine->position = 11; + + model->set_max_length(11); + CHECK(model->get_max_length() == 11); + CHECK(engine->max_length == 11); + + CheckThrowsContains( + [&] { model->set_max_length(10); }, + "engine position"); + CHECK(model->get_max_length() == 11); + CHECK(engine->max_length == 11); +} + +void TestRenderedCapacityIsAtomic() { + TempModelPackage package; + FactoryScope factory; + auto model = Load( + package, + ModelInfo(512, "corelib_aie4"), + -1, + false, + nullptr); + FakeEngine* engine = g_factory.engine; + engine->prefill_calls.clear(); + const int clear_count = engine->clear_count; + + g_encoded_tokens.assign(500, 12); + auto meta = Meta(); + auto input = Prompt(13); + CheckRequestError( + [&] { (void)model->insert(meta, input); }, + 400, + false, + "500"); + CHECK(engine->position == 0); + CHECK(engine->clear_count == clear_count); + CHECK(engine->prefill_calls.empty()); + CHECK(Phi4FrontendTestAccess::History(*model).empty()); + + input.requested_max_new_tokens = std::nullopt; + CHECK(model->insert(meta, input)); + CHECK(engine->position == 500); + + Phi4FrontendTestAccess::ForceRoute( + *model, + ForcedContinuationRoute::Append); + g_encoded_tokens.push_back(13); + input.requested_max_new_tokens = 12; + const auto calls_before_append_rejection = + engine->prefill_calls.size(); + CheckRequestError( + [&] { (void)model->insert(meta, input); }, + 400, + false, + "501"); + CHECK(engine->position == 500); + CHECK( + engine->prefill_calls.size() == + calls_before_append_rejection); + CHECK(Phi4FrontendTestAccess::History(*model).size() == 500); +} + +void CheckAligned( + Phi4& model, + const FakeEngine& engine, + const std::vector& expected_history) { + CHECK(Phi4FrontendTestAccess::History(model) == expected_history); + CHECK( + model.get_current_context_length() == + static_cast(expected_history.size())); + CHECK( + engine.position == + static_cast(expected_history.size())); +} + +void TestDefaultChatLimitWithoutExplicitRequestIsAdmitted() { + TempModelPackage package; + FactoryScope factory; + auto model = Load( + package, + ModelInfo(4096, "corelib_aie4"), + -1, + false, + nullptr); + + g_encoded_tokens = {41}; + SetSamples({200020}); + auto meta = Meta(); + auto input = Prompt(); + std::ostringstream output; + CHECK( + model->generate_with_prompt( + meta, + input, + 4096, + output) + .empty()); + CHECK(!input.requested_max_new_tokens.has_value()); + CheckAligned(*model, *g_factory.engine, {41, 200020}); +} + +void TestEndpointLimitsAtLoweredCap() { + TempModelPackage package; + FactoryScope factory; + auto model = Load( + package, + ModelInfo(512, "corelib_aie4"), + -1, + false, + nullptr); + FakeEngine* engine = g_factory.engine; + + for (const GenerationEndpoint endpoint : { + GenerationEndpoint::Generate, + GenerationEndpoint::OpenAiChatCompletion, + GenerationEndpoint::OpenAiCompletion}) { + const ParsedGenerationLimit omitted = + ParseGenerationLimit( + nlohmann::ordered_json::object(), + endpoint); + CHECK(!omitted.explicit_limit); + CHECK(GenerationLoopLimit(omitted, true) == -1); + + g_encoded_tokens = {41}; + g_sample_tokens.clear(); + g_sample_token = 7; + auto meta = Meta(); + auto input = Prompt(RequestedMaxNewTokens(omitted)); + CHECK(model->insert(meta, input)); + std::ostringstream output; + (void)model->generate( + meta, + GenerationLoopLimit(omitted, true), + output); + CHECK(meta.generated_tokens == 511); + CHECK(model->get_current_context_length() == 512); + CHECK(engine->position == 512); + + model->clear_context(); + nlohmann::ordered_json explicit_request; + if (endpoint == + GenerationEndpoint::OpenAiChatCompletion) { + explicit_request["max_completion_tokens"] = 512; + } else { + explicit_request["max_tokens"] = 512; + } + const ParsedGenerationLimit explicit_limit = + ParseGenerationLimit(explicit_request, endpoint); + CHECK(explicit_limit.explicit_limit); + CHECK(explicit_limit.value == 512); + + meta = Meta(); + input = Prompt(RequestedMaxNewTokens(explicit_limit)); + const int clear_count = engine->clear_count; + const auto prefill_count = engine->prefill_calls.size(); + CheckRequestError( + [&] { (void)model->insert(meta, input); }, + 400, + false, + "512"); + CHECK(engine->position == 0); + CHECK(engine->clear_count == clear_count); + CHECK(engine->prefill_calls.size() == prefill_count); + CHECK(Phi4FrontendTestAccess::History(*model).empty()); + } +} + +// I13. The same /api/chat body must be admitted the same way whichever +// backend serves it. +// +// `num_predict` is Ollama's field and it carries sentinels: -1 is "generate +// forever" and -2 is "fill context". GenerationLoopLimit returns both +// verbatim, and BOTH generation loops -- generate_aie4 and the shared legacy +// one -- gate on `length_limit > 0`, so both already read them as "no bound". +// +// RequestedMaxNewTokens used to hand that same -1 to validate_aie4_capacity +// as an output token COUNT, and the count guard refuses anything negative. +// So POST /api/chat {"options":{"num_predict":-1}} answered HTTP 400 on the +// AIE4 tag and HTTP 200 on every other backend -- and asking explicitly for +// the unbounded behaviour was refused where OMITTING the field allowed it. +// +// This pins the observable outcome -- admitted, or the 400 -- for every +// sentinel, against a real AIE4-backed Phi4 and a real legacy-backed Phi4 +// built in the same process from the same package, driven from request +// bodies rather than from hand-built ParsedGenerationLimit values. The +// over-cap positive case is here so that a "fix" which simply removed the +// admission rule would fail this test rather than pass it. +void TestNumPredictSentinelsAdmitIdenticallyOnBothBackends() { + TempModelPackage package; + FactoryScope factory; + + auto aie4 = Load( + package, + ModelInfo(4096, "corelib_aie4"), + -1, + false, + nullptr); + auto legacy = Load(package, ModelInfo(4096)); + CHECK(aie4->uses_corelib_aie4()); + CHECK(!legacy->uses_corelib_aie4()); + + struct Case { + std::string_view description; + std::optional num_predict; + bool aie4_admits; + int aie4_loop_limit; + int legacy_loop_limit; + }; + + // A 10-token prompt plus 4090 requested output exceeds the 4095 active + // cap, so 4090 is the one value here that must still be refused on AIE4. + const Case cases[] = { + {"absent", std::nullopt, true, -1, 4096}, + {"-1 (Ollama: generate forever)", -1, true, -1, -1}, + {"-2 (Ollama: fill context)", -2, true, -2, -2}, + {"0", 0, true, 0, 0}, + {"64", 64, true, 64, 64}, + {"4090 (over the AIE4 active cap)", 4090, false, 4090, 4090}, + }; + + for (const Case& test_case : cases) { + nlohmann::ordered_json options = + nlohmann::ordered_json::object(); + if (test_case.num_predict.has_value()) { + options["num_predict"] = *test_case.num_predict; + } + const nlohmann::ordered_json request = { + {"options", std::move(options)}, + }; + const ParsedGenerationLimit parsed = + ParseGenerationLimit( + request, + RequireGenerationEndpoint("POST", "/api/chat")); + CHECK( + parsed.explicit_limit == + test_case.num_predict.has_value()); + CHECK( + GenerationLoopLimit(parsed, true) == + test_case.aie4_loop_limit); + CHECK( + GenerationLoopLimit(parsed, false) == + test_case.legacy_loop_limit); + + g_encoded_tokens.assign(10, 7); + g_sample_tokens.clear(); + g_sample_token = 7; + + // The legacy backend serves every one of these. That is the + // baseline the AIE4 tag has to match, so it is asserted rather + // than assumed. + auto legacy_meta = Meta(); + auto legacy_input = Prompt(RequestedMaxNewTokens(parsed)); + if (!legacy->insert(legacy_meta, legacy_input)) { + throw std::runtime_error( + "legacy backend refused num_predict " + + std::string(test_case.description)); + } + legacy->clear_context(); + + auto meta = Meta(); + auto input = Prompt(RequestedMaxNewTokens(parsed)); + if (test_case.aie4_admits) { + if (!aie4->insert(meta, input)) { + throw std::runtime_error( + "AIE4 refused num_predict " + + std::string(test_case.description) + + " that the legacy backend served"); + } + aie4->clear_context(); + } else { + CheckRequestError( + [&] { (void)aie4->insert(meta, input); }, + 400, + false, + "exceeds the active context cap"); + } + } +} + +// 0 is in the non-positive bucket deliberately, not by accident. +// +// Both generation loops test `length_limit > 0`, so an explicit 0 is read as +// "no bound" exactly like -1 and -2 and runs to the backend's cap. Whether +// an explicit 0 OUGHT to mean "emit nothing" is a question about those two +// loops and about the legacy backend, not about the AIE4 admission rule, and +// this change does not decide it. What it does do is write the behaviour +// down: if either loop's treatment of 0 moves, this fails. +void TestNonPositiveLimitsGenerateToTheCapOnBothBackends() { + TempModelPackage package; + FactoryScope factory; + + constexpr int kCap = 16; + auto aie4 = Load( + package, + ModelInfo(kCap, "corelib_aie4"), + -1, + false, + nullptr); + auto legacy = Load(package, ModelInfo(kCap)); + + for (const int num_predict : {-1, -2, 0}) { + const nlohmann::ordered_json request = { + {"options", {{"num_predict", num_predict}}}, + }; + const ParsedGenerationLimit parsed = + ParseGenerationLimit( + request, + GenerationEndpoint::OllamaChat); + CHECK(parsed.explicit_limit); + CHECK(parsed.value == num_predict); + // The admission rule sees no budget at all -- same as omission. + CHECK(!RequestedMaxNewTokens(parsed).has_value()); + + for (Phi4* model : {aie4.get(), legacy.get()}) { + model->clear_context(); + g_encoded_tokens = {41}; + // Never an end token: only the cap can stop these loops. + g_sample_tokens.clear(); + g_sample_token = 7; + + auto meta = Meta(); + auto input = Prompt(RequestedMaxNewTokens(parsed)); + CHECK(model->insert(meta, input)); + std::ostringstream output; + (void)model->generate( + meta, + GenerationLoopLimit( + parsed, + model->uses_corelib_aie4()), + output); + CHECK(model->get_current_context_length() == kCap); + } + // The AIE4 loop reports the truncation; the legacy one leaves + // stop_reason at EOT_DETECTED when it exits on MAX_L, which is a + // pre-existing legacy quirk this change does not touch. + CHECK(aie4->uses_corelib_aie4()); + } +} + +void TestLengthStopCommitsTokensBeforeForcedAppend() { + TempModelPackage package; + FactoryScope factory; + auto model = Load( + package, + ModelInfo(64, "corelib_aie4"), + -1, + false, + nullptr); + FakeEngine* engine = g_factory.engine; + + g_encoded_tokens = {1, 2}; + SetSamples({101, 102}); + auto meta = Meta(); + auto input = Prompt(); + CHECK(model->insert(meta, input)); + std::ostringstream output; + CHECK( + model->generate(meta, 2, output) == + "token-101token-102"); + CHECK(meta.generated_tokens == 2); + CHECK(meta.stop_reason == MAX_LENGTH_REACHED); + CheckAligned(*model, *engine, {1, 2, 101, 102}); + + model->set_max_length(16); + CHECK(model->get_max_length() == 16); + CHECK(engine->max_length == 16); + + Phi4FrontendTestAccess::ForceRoute( + *model, + ForcedContinuationRoute::Append); + engine->prefill_calls.clear(); + g_encoded_tokens = {1, 2, 101, 102, 3}; + SetSamples({103}); + auto append_meta = Meta(); + CHECK(model->insert(append_meta, input)); + CHECK(engine->prefill_calls == std::vector>({{3}})); + CheckAligned(*model, *engine, {1, 2, 101, 102, 3}); +} + +void TestEosStopCommitsTokenBeforeCapUpdateAndAppend() { + TempModelPackage package; + FactoryScope factory; + auto model = Load( + package, + ModelInfo(64, "corelib_aie4"), + -1, + false, + nullptr); + FakeEngine* engine = g_factory.engine; + + g_encoded_tokens = {10}; + SetSamples({101, 200020}); + auto meta = Meta(); + auto input = Prompt(); + CHECK(model->insert(meta, input)); + std::ostringstream output; + CHECK(model->generate(meta, 8, output) == "token-101"); + CHECK(meta.generated_tokens == 2); + CHECK(meta.stop_reason == EOT_DETECTED); + CheckAligned(*model, *engine, {10, 101, 200020}); + + model->set_max_length(8); + CHECK(model->get_max_length() == 8); + CHECK(engine->max_length == 8); + + Phi4FrontendTestAccess::ForceRoute( + *model, + ForcedContinuationRoute::Append); + engine->prefill_calls.clear(); + g_encoded_tokens = {10, 101, 200020, 11}; + SetSamples({102}); + auto append_meta = Meta(); + CHECK(model->insert(append_meta, input)); + CHECK(engine->prefill_calls == std::vector>({{11}})); + CheckAligned(*model, *engine, {10, 101, 200020, 11}); +} + +// The decode loop must stop one step BEFORE the declared context length. +// +// Measured on the AIE4 target 2026-09-02: a default /api/chat request with no +// limit field ran the decode loop to position 4095, where the token attention +// kernel has no 4096-wide window. The refusal arrives from flat_mha after q, k +// and v have been submitted in the same step, so under the then-current +// per-step irrevocable boundary the process was TERMINATED, taking the server +// with it. The fatal record read: +// +// {"status":3,"call":"flat_mha","phase":"flat_mha","layer":0,"rows":1, +// "position":4095, +// "detail":"no token attention kernel ships for a 4096-token window"} +// +// Design 11.2 and SEQ-4 require an unbounded request to stop before the +// user-visible total would exceed the cap. That bound existed for the +// explicit-limit path and not for the AIE4 no-limit path, where +// GenerationLoopLimit returns kNoExplicitGenerationLimit and the loop was +// bounded only by MAX_L -- one step too many -- and by EOS. +// +// This pins the boundary WITHOUT relying on EOS arriving: the sampler here +// never emits an end token, which is exactly the condition under which the +// defect fired. +// +// The boundary is now per submission group, so the same refusal would be +// recoverable rather than fatal. That does not make this bound optional: the +// refusal still arrives after the step's V scatter, so the caller still loses +// the whole conversation to a 500 where this stops one step earlier and +// returns an ordinary MAX_LENGTH_REACHED truncation. +void TestDecodeStopsBelowTheTokenAttentionWindow() { + TempModelPackage package; + FactoryScope factory; + auto model = Load( + package, + ModelInfo(4096, "corelib_aie4"), + -1, + false, + nullptr); + FakeEngine* engine = g_factory.engine; + CHECK(engine->max_length == 4096); + + // One prompt token, then an unending stream of ordinary tokens. Nothing + // here will ever stop the loop except the cap. + g_encoded_tokens = {1}; + std::vector samples; + samples.reserve(5000); + for (int index = 0; index < 5000; ++index) { + samples.push_back(500 + (index % 100)); + } + g_sample_tokens.assign(samples.begin(), samples.end()); + + auto meta = Meta(); + auto input = Prompt(); + CHECK(model->insert(meta, input)); + + std::ostringstream output; + // A limit far beyond the cap, standing in for the no-limit path: on AIE4 + // GenerationLoopLimit returns kNoExplicitGenerationLimit when no field is + // present, so the cap is the only thing that can stop this. + (void)model->generate(meta, 100000, output); + + CHECK(meta.stop_reason == MAX_LENGTH_REACHED); + // 4095, not 4096. The last position the engine is asked to run at is + // 4094, whose attention window is 4095 -- the largest the token kernel + // serves. One more would be the step that killed the server. + CHECK( + model->get_current_context_length() == + static_cast(flm::phi4::constants::kMaxDecodeWindow)); + CHECK(engine->position == flm::phi4::constants::kMaxDecodeWindow); + CHECK(engine->position < 4096); +} + +// Admission must refuse a request it cannot finish. +// +// A rendered prompt plus an explicit output that totals exactly the declared +// 4096 was previously admitted, because the cap compared against MAX_L. The +// final decode step of such a request is the unservable one, so admitting it +// meant accepting a request whose only possible outcome was a dead process. +void TestAdmissionRefusesTheUnservableFinalStep() { + TempModelPackage package; + FactoryScope factory; + auto model = Load( + package, + ModelInfo(4096, "corelib_aie4"), + -1, + false, + nullptr); + FakeEngine* engine = g_factory.engine; + + g_encoded_tokens.assign(10, 7); + + // 10 + 4086 == 4096 == MAX_L. Refused: the last step cannot be served. + auto meta = Meta(); + auto rejected = Prompt(4086); + CheckRequestError( + [&] { (void)model->insert(meta, rejected); }, + 400, + false, + "exceeds the active context cap"); + CHECK(engine->position == 0); + + // 10 + 4085 == 4095. Admitted, because every step of it can run. + SetSamples({101}); + auto accepted = Prompt(4085); + CHECK(model->insert(meta, accepted)); + CHECK(engine->position == 10); +} + +void TestActiveCapNeverEmitsUncommittedToken() { + TempModelPackage package; + FactoryScope factory; + auto model = Load( + package, + ModelInfo(3, "corelib_aie4"), + -1, + false, + nullptr); + FakeEngine* engine = g_factory.engine; + + g_encoded_tokens = {1, 2}; + SetSamples({101, 102}); + auto meta = Meta(); + auto input = Prompt(); + CHECK(model->insert(meta, input)); + std::ostringstream output; + CHECK(model->generate(meta, 8, output) == "token-101"); + CHECK(meta.generated_tokens == 1); + CHECK(meta.stop_reason == MAX_LENGTH_REACHED); + CHECK(engine->forward_tokens == std::vector({101})); + CHECK(g_sample_tokens.size() == 1); + CheckAligned(*model, *engine, {1, 2, 101}); +} + +void TestCancellationLeavesOnlyCommittedTokensVisible() { + TempModelPackage package; + FactoryScope factory; + auto model = Load( + package, + ModelInfo(64, "corelib_aie4"), + -1, + false, + nullptr); + FakeEngine* engine = g_factory.engine; + + g_encoded_tokens = {1, 2}; + SetSamples({101, 102}); + auto meta = Meta(); + auto input = Prompt(); + CHECK(model->insert(meta, input)); + int cancellation_checks = 0; + std::ostringstream output; + CHECK( + model->generate( + meta, + 8, + output, + [&] { return cancellation_checks++ == 1; }) == + "token-101"); + CHECK(meta.generated_tokens == 1); + CHECK(meta.stop_reason == CANCEL_DETECTED); + CHECK(engine->forward_tokens == std::vector({101})); + CheckAligned(*model, *engine, {1, 2, 101}); +} + +void TestExactRepeatReprefillsAndSamplesFreshToken() { + TempModelPackage package; + FactoryScope factory; + auto model = Load( + package, + ModelInfo(64, "corelib_aie4"), + -1, + false, + nullptr); + FakeEngine* engine = g_factory.engine; + + g_encoded_tokens = {1}; + SetSamples({101}); + auto first_meta = Meta(); + auto input = Prompt(); + CHECK(model->insert(first_meta, input)); + std::ostringstream first_output; + CHECK(model->generate(first_meta, 1, first_output) == "token-101"); + CheckAligned(*model, *engine, {1, 101}); + + const int clear_before = engine->clear_count; + engine->prefill_calls.clear(); + Phi4FrontendTestAccess::ForceRoute( + *model, + ForcedContinuationRoute::Append); + g_encoded_tokens = {1, 101}; + SetSamples({202}); + auto repeated_meta = Meta(); + CHECK(model->insert(repeated_meta, input)); + CHECK(engine->clear_count == clear_before + 1); + CHECK( + engine->prefill_calls == + std::vector>({{1, 101}})); + std::ostringstream repeated_output; + CHECK( + model->generate(repeated_meta, 1, repeated_output) == + "token-202"); + CHECK( + engine->forward_tokens == + std::vector({101, 202})); + CheckAligned(*model, *engine, {1, 101, 202}); +} + +void TestForcedAppendAndCancellationAlignment() { + TempModelPackage package; + FactoryScope factory; + auto model = Load( + package, + ModelInfo(64, "corelib_aie4"), + -1, + false, + nullptr); + FakeEngine* engine = g_factory.engine; + + g_encoded_tokens = {1, 2}; + auto first_meta = Meta(); + auto first = Prompt(); + CHECK(model->insert(first_meta, first)); + engine->prefill_calls.clear(); + + Phi4FrontendTestAccess::ForceRoute( + *model, + ForcedContinuationRoute::Append); + g_encoded_tokens = {1, 2, 3, 4, 5}; + auto second_meta = Meta(); + auto second = Prompt(); + int cancellation_checks = 0; + const bool inserted = model->insert( + second_meta, + second, + [&] { return cancellation_checks++ == 1; }); + CHECK(!inserted); + CHECK(second_meta.stop_reason == CANCEL_DETECTED); + CHECK(engine->prefill_calls.size() == 1); + CHECK(engine->prefill_calls[0] == std::vector({3})); + CHECK(engine->position == 3); + CHECK(model->get_current_context_length() == 3); + CHECK( + Phi4FrontendTestAccess::History(*model) == + std::vector({1, 2, 3})); +} + +// The frontend's Automatic route on BOTH sides of the generated threshold, +// plus the forced re-prefill it has to keep honouring. +// +// The earlier version of this test drove Automatic with a one-token suffix and +// asserted a re-prefill. That only held because the threshold was the +// placeholder zero: the test never exercised the Automatic append path at all, +// and its expectation inverted the moment calibration produced a positive +// constant. The suffix lengths below are derived from +// `kContinuationAppendThreshold` so that one case lands at the boundary the +// selector compares against and the other lands one token past it, whatever +// that constant is. +void TestForcedAndAutomaticReprefill() { + constexpr int kThreshold = + static_cast(kContinuationAppendThreshold); + + TempModelPackage package; + FactoryScope factory; + // Enough context for the history plus the widest suffix this test builds. + auto model = Load( + package, + ModelInfo(4 * (kThreshold + 4) + 64, "corelib_aie4"), + -1, + false, + nullptr); + FakeEngine* engine = g_factory.engine; + + auto meta = Meta(); + auto input = Prompt(); + std::vector history = {1, 2}; + g_encoded_tokens = history; + CHECK(model->insert(meta, input)); + + // Forced re-prefill: the whole rendered history goes through as one call + // after a clear, regardless of how short the suffix is. + engine->prefill_calls.clear(); + const int clear_before_forced = engine->clear_count; + Phi4FrontendTestAccess::ForceRoute( + *model, + ForcedContinuationRoute::Reprefill); + history = {1, 2, 3, 4}; + g_encoded_tokens = history; + CHECK(model->insert(meta, input)); + CHECK(engine->clear_count == clear_before_forced + 1); + CHECK(engine->prefill_calls.size() == 1); + CHECK(engine->prefill_calls[0] == history); + CHECK(engine->position == static_cast(history.size())); + + Phi4FrontendTestAccess::ForceRoute( + *model, + ForcedContinuationRoute::Automatic); + + // Automatic at the boundary: a suffix of exactly `kThreshold` tokens is + // "at most the threshold", so it appends one row at a time and never + // clears. A zero threshold makes that unreachable, and Section 10.7 allows + // a zero threshold, so that case asserts the other behaviour rather than + // skipping. + if (kThreshold > 0) { + engine->prefill_calls.clear(); + const int clear_before_append = engine->clear_count; + const int appended_from = static_cast(history.size()); + std::vector appended; + for (int index = 0; index < kThreshold; ++index) { + history.push_back(1000 + index); + appended.push_back(1000 + index); + } + g_encoded_tokens = history; + CHECK(model->insert(meta, input)); + CHECK(engine->clear_count == clear_before_append); + CHECK( + engine->prefill_calls.size() == + static_cast(kThreshold)); + for (int index = 0; index < kThreshold; ++index) { + CHECK( + engine->prefill_calls[static_cast(index)] == + std::vector({appended[static_cast(index)]})); + } + CHECK(engine->position == appended_from + kThreshold); + CHECK(Phi4FrontendTestAccess::History(*model) == history); + } else { + engine->prefill_calls.clear(); + const int clear_before_zero = engine->clear_count; + history.push_back(1000); + g_encoded_tokens = history; + CHECK(model->insert(meta, input)); + CHECK(engine->clear_count == clear_before_zero + 1); + CHECK(engine->prefill_calls.size() == 1); + CHECK(engine->prefill_calls[0] == history); + } + + // Automatic one token past the boundary: a suffix of `kThreshold + 1` + // clears and re-prefills the complete rendered history as one call. + engine->prefill_calls.clear(); + const int clear_before_automatic = engine->clear_count; + for (int index = 0; index <= kThreshold; ++index) { + history.push_back(2000 + index); + } + g_encoded_tokens = history; + CHECK(model->insert(meta, input)); + CHECK(engine->clear_count == clear_before_automatic + 1); + CHECK(engine->prefill_calls.size() == 1); + CHECK(engine->prefill_calls[0] == history); + CHECK(engine->position == static_cast(history.size())); +} + +void TestEosValidationAndFrontendStop() { + FactoryScope factory; + TempModelPackage invalid_package({200020}); + CheckThrowsContains( + [&] { + auto model = Load( + invalid_package, + ModelInfo(64, "corelib_aie4"), + -1, + false, + nullptr); + (void)model; + }, + "199999"); + + TempModelPackage package; + auto model = Load( + package, + ModelInfo(64, "corelib_aie4"), + -1, + false, + nullptr); + FakeEngine* engine = g_factory.engine; + g_encoded_tokens = {10, 11}; + g_sample_token = 200020; + auto meta = Meta(); + auto input = Prompt(); + CHECK(model->insert(meta, input)); + std::ostringstream output; + CHECK(model->generate(meta, 8, output).empty()); + CHECK(engine->forward_tokens == std::vector({200020})); + CheckAligned(*model, *engine, {10, 11, 200020}); + g_sample_token = 7; +} + +void TestRecoverableFailuresClearSession() { + TempModelPackage package; + FactoryScope factory; + auto model = Load( + package, + ModelInfo(64, "corelib_aie4"), + -1, + false, + nullptr); + FakeEngine* engine = g_factory.engine; + + g_encoded_tokens = {20}; + auto meta = Meta(); + auto input = Prompt(); + CHECK(model->insert(meta, input)); + Phi4FrontendTestAccess::ForceRoute( + *model, + ForcedContinuationRoute::Append); + engine->checkpoint(); + engine->fail_next_prefill = true; + const int resets_before_insert = g_sampler_reset_count; + g_encoded_tokens = {20, 21}; + CheckRequestError( + [&] { (void)model->insert(meta, input); }, + 500, + true, + "current conversation was cleared"); + CHECK(engine->position == 0); + CHECK(!engine->checkpoint_position.has_value()); + CHECK(Phi4FrontendTestAccess::History(*model).empty()); + CHECK(g_sampler_reset_count > resets_before_insert); + + g_encoded_tokens = {31, 32}; + CHECK(model->insert(meta, input)); + engine->checkpoint(); + engine->fail_next_forward = true; + const int resets_before_generate = g_sampler_reset_count; + std::ostringstream output; + CheckRequestError( + [&] { (void)model->generate(meta, 8, output); }, + 500, + true, + "current conversation was cleared"); + CHECK(engine->position == 0); + CHECK(!engine->checkpoint_position.has_value()); + CHECK(Phi4FrontendTestAccess::History(*model).empty()); + CHECK(g_sampler_reset_count > resets_before_generate); +} + +void CheckPartialAppendFailureClearsSession( + FakeEngine::FailureKind failure) { + TempModelPackage package; + FactoryScope factory; + auto model = Load( + package, + ModelInfo(64, "corelib_aie4"), + -1, + false, + nullptr); + FakeEngine* engine = g_factory.engine; + + g_encoded_tokens = {1}; + SetSamples({101}); + auto meta = Meta(); + auto input = Prompt(); + CHECK(model->insert(meta, input)); + engine->checkpoint(); + Phi4FrontendTestAccess::ForceRoute( + *model, + ForcedContinuationRoute::Append); + engine->prefill_failure = failure; + engine->successful_prefills_before_failure = 1; + const int resets_before = g_sampler_reset_count; + g_encoded_tokens = {1, 2, 3, 4}; + + CheckRequestError( + [&] { (void)model->insert(meta, input); }, + 500, + true, + "current conversation was cleared"); + CHECK(engine->prefill_calls.back() == std::vector({2})); + CHECK(engine->position == 0); + CHECK(!engine->checkpoint_position.has_value()); + CHECK(model->get_current_context_length() == 0); + CHECK(Phi4FrontendTestAccess::History(*model).empty()); + CHECK(Phi4FrontendTestAccess::LastToken(*model) == -1); + CHECK(g_sampler_reset_count > resets_before); +} + +void TestPartialAppendStandardFailureClearsCommittedPrefix() { + CheckPartialAppendFailureClearsSession( + FakeEngine::FailureKind::Standard); +} + +void TestPartialAppendUnknownFailureClearsCommittedPrefix() { + CheckPartialAppendFailureClearsSession( + FakeEngine::FailureKind::Unknown); +} + +void TestStandardGenerateFailureClearsSession() { + TempModelPackage package; + FactoryScope factory; + auto model = Load( + package, + ModelInfo(64, "corelib_aie4"), + -1, + false, + nullptr); + FakeEngine* engine = g_factory.engine; + + g_encoded_tokens = {31, 32}; + SetSamples({101}); + auto meta = Meta(); + auto input = Prompt(); + CHECK(model->insert(meta, input)); + engine->checkpoint(); + engine->forward_failure = FakeEngine::FailureKind::Standard; + const int resets_before = g_sampler_reset_count; + std::ostringstream output; + + CheckRequestError( + [&] { (void)model->generate(meta, 8, output); }, + 500, + true, + "current conversation was cleared"); + CHECK(engine->position == 0); + CHECK(!engine->checkpoint_position.has_value()); + CHECK(model->get_current_context_length() == 0); + CHECK(Phi4FrontendTestAccess::History(*model).empty()); + CHECK(Phi4FrontendTestAccess::LastToken(*model) == -1); + CHECK(g_sampler_reset_count > resets_before); +} + +void TestProfileSplit() { + TempModelPackage package; + FactoryScope factory; + auto model = Load( + package, + ModelInfo(64, "corelib_aie4"), + -1, + false, + nullptr); + + flm::phi4::Phi4Aie4Metrics metrics; + metrics.model_load_ns = 101; + metrics.weight_pack_ns = 202; + metrics.dispatch_count = 303; + metrics.synchronize_count = 404; + metrics.helper_transition_counts = {1, 2, 3, 4, 5, 6}; + Phi4FrontendTestAccess::SetMetrics(*model, metrics); + + g_encoded_tokens = {1}; + auto meta = Meta(); + auto input = Prompt(); + CHECK(model->insert(meta, input)); + Phi4FrontendTestAccess::ForceRoute( + *model, + ForcedContinuationRoute::Append); + g_encoded_tokens = {1, 2}; + CHECK(model->insert(meta, input)); + + const std::string profile = model->show_profile(); + CHECK(profile.find("corelib_aie4") != std::string::npos); + CHECK(profile.find("Continuation route: append") != std::string::npos); + // Telemetry must report the generated constant, not a number this test + // pinned when the header still held the placeholder zero. + CHECK( + profile.find( + "Append threshold: " + + std::to_string(kContinuationAppendThreshold)) != + std::string::npos); + CHECK( + profile.find("fake_ryzenai_corelib.dll") != + std::string::npos); + CHECK(profile.find("Dispatches: 303") != std::string::npos); + CHECK(profile.find("Synchronizations: 404") != std::string::npos); + CHECK(profile.find("Helper transitions: 1/2/3/4/5/6") != + std::string::npos); + CHECK(profile.find("Cold model load: 101 ns") != + std::string::npos); + CHECK(profile.find("Cold weight pack: 202 ns") != + std::string::npos); + CHECK(profile.find("Continuation time:") != std::string::npos); +} + +#endif + +} // namespace + +int main() { + try { + TestContinuationSelector(); + TestLegacyRoutingAndUnknownBackend(); + TestLoadedBackendControlsOmittedLimit(); + TestLegacyExactRepeatPreservesEmptyPrefillPayload(); +#if defined(FLM_ENABLE_CORELIB_AIE4) + ConfigureFakeCorelibDll(); + TestCorruptOverlayConfigFailsLoad(); + TestCorelibRoutingAndPreemption(); + TestInitialAndAtomicCaps(); + TestEnginePositionIsAuthoritativeForCapUpdate(); + TestRenderedCapacityIsAtomic(); + TestDefaultChatLimitWithoutExplicitRequestIsAdmitted(); + TestEndpointLimitsAtLoweredCap(); + TestNumPredictSentinelsAdmitIdenticallyOnBothBackends(); + TestNonPositiveLimitsGenerateToTheCapOnBothBackends(); + TestLengthStopCommitsTokensBeforeForcedAppend(); + TestEosStopCommitsTokenBeforeCapUpdateAndAppend(); + TestDecodeStopsBelowTheTokenAttentionWindow(); + TestAdmissionRefusesTheUnservableFinalStep(); + TestActiveCapNeverEmitsUncommittedToken(); + TestCancellationLeavesOnlyCommittedTokensVisible(); + TestExactRepeatReprefillsAndSamplesFreshToken(); + TestForcedAppendAndCancellationAlignment(); + TestForcedAndAutomaticReprefill(); + TestEosValidationAndFrontendStop(); + TestRecoverableFailuresClearSession(); + TestPartialAppendStandardFailureClearsCommittedPrefix(); + TestPartialAppendUnknownFailureClearsCommittedPrefix(); + TestStandardGenerateFailureClearsSession(); + TestProfileSplit(); + Phi4FrontendTestAccess::RemoveFactory(); + flm::corelib::CorelibRuntime::ShutdownProcess(); + std::cout << "test_phi4_frontend_on: PASS\n"; +#else + TestFeatureOffRejectsCorelibTag(); + std::cout << "test_phi4_frontend_off: PASS\n"; +#endif + return 0; + } catch (const std::exception& error) { +#if defined(FLM_ENABLE_CORELIB_AIE4) + Phi4FrontendTestAccess::RemoveFactory(); + try { + flm::corelib::CorelibRuntime::ShutdownProcess(); + } catch (...) { + } +#endif + std::cerr << "test_phi4_frontend: FAIL: " + << error.what() << '\n'; + return 1; + } +} diff --git a/src/test/phi4_corelib_aie4/test_phi4_hardware.cpp b/src/test/phi4_corelib_aie4/test_phi4_hardware.cpp new file mode 100644 index 00000000..07f7f533 --- /dev/null +++ b/src/test/phi4_corelib_aie4/test_phi4_hardware.cpp @@ -0,0 +1,568 @@ +// Task 12 Steps 1 and 5: the real corelib, on a real AIE4 device. +// +// test_real_corelib covers the entry points the header documents as needing no +// NPU. Everything here needs one. `has_device_context()` is ASSERTED rather +// than recorded, a Stream and a DeviceTensor are really created on the device, +// and the RoPE upload is a real `tensor_write` rather than a fake that records +// its arguments. +// +// The RoPE case is the one that covers FastFlow rather than corelib. Corelib +// `e5258d2` removed `convert_strided`, so the [4096, 48] slice out of a wider +// table is now FastFlow's own bounds-checked loop over a read-only file +// mapping. The fixture ends the last source row immediately before an +// inaccessible page, so an off-by-one in that loop faults instead of silently +// reading a neighbouring page -- and the gathered rows are then pushed through +// one real FP16-source write into a real FP32 device tensor, which is the pair +// of steps the product actually performs at load. + +#include "phi4_package_fixture.hpp" +#include "test_support.hpp" + +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +namespace constants = flm::phi4::constants; +namespace fixture = flm::test::phi4fixture; + +using flm::corelib::CorelibApi; +using flm::phi4::Phi4Package; +using flm::phi4::Phi4ShapePlan; +using flm::phi4::RowUse; + +#if !defined(FLM_REAL_CORELIB_RUNTIME_DIR) +#define FLM_REAL_CORELIB_RUNTIME_DIR "" +#endif + +#if !defined(FLM_REAL_CORELIB_EXTRA_DLL_DIRS) +#define FLM_REAL_CORELIB_EXTRA_DLL_DIRS "" +#endif + +// CorelibApi::Load uses LOAD_LIBRARY_SEARCH_DEFAULT_DIRS, which honours +// directories added here and deliberately ignores PATH. Design CLOSURE-2: a +// pass that depended on PATH would certify the build machine, not the closure. +void AddExtraDllDirectories(std::string_view directories) { + std::size_t start = 0; + while (start <= directories.size()) { + const std::size_t end = directories.find(';', start); + const std::string_view entry = directories.substr( + start, + end == std::string_view::npos ? std::string_view::npos + : end - start); + if (!entry.empty()) { + const std::filesystem::path directory(entry); + if (!std::filesystem::exists(directory)) { + throw std::runtime_error( + "extra corelib DLL directory does not exist: " + + directory.string()); + } + if (AddDllDirectory(directory.c_str()) == nullptr) { + throw std::runtime_error( + "AddDllDirectory failed for " + directory.string()); + } + std::cout << "added DLL directory " << directory.string() + << '\n'; + } + if (end == std::string_view::npos) { + break; + } + start = end + 1; + } +} + +struct NamedRowUse { + RowUse use; + std::string_view name; +}; + +constexpr std::array kMultiRowUses{{ + {RowUse::QueryProjection, "query_projection"}, + {RowUse::KvProjection, "kv_projection"}, + {RowUse::Attention, "attention"}, + {RowUse::OutputProjection, "output_projection"}, + {RowUse::SsMlp, "ssmlp"}, +}}; + +// 1a. Version identity. API-5 requires an exact major.minor.patch match while +// the compiled-against major is 0, and `CorelibApi::Load` enforces it before +// resolving any other symbol -- so loading at all means the gate passed. +// Restating it here keeps this file from going vacuous if the gate is ever +// relaxed, and prints both sides so a mismatch is diagnosable from the log. +void CheckVersionIdentity(const std::shared_ptr& api) { + const auto compiled = flm::corelib::CompiledCorelibVersion(); + const auto& runtime = api->runtime_version(); + std::cout << "runtime corelib version " + << flm::corelib::FormatCorelibVersion(runtime) + << ", compiled against " + << flm::corelib::FormatCorelibVersion(compiled) << '\n'; + CHECK(runtime.major == compiled.major); + CHECK(runtime.minor == compiled.minor); + CHECK(runtime.patch == compiled.patch); + CHECK(flm::corelib::IsCorelibVersionCompatible(compiled, runtime)); +} + +// 1b. Dependency self-test and device context. On this machine the device +// context is REQUIRED, not recorded: the whole point of the target box is that +// there is an AIE4 device, and a run that quietly proceeded without one would +// report every later check as passed while testing nothing. +void CheckDependenciesAndDevice( + const std::shared_ptr& api) { + api->Check( + api->functions().selftest_dependencies(), + "ryzenai_corelib_selftest_dependencies"); + std::cout << "selftest_dependencies: ok\n"; + + const bool has_context = api->functions().has_device_context(); + std::cout << "has_device_context: " + << (has_context ? "true" : "false") << '\n'; + if (!has_context) { + throw std::runtime_error( + "ryzenai_corelib_has_device_context reported no AIE4 device " + "context. This suite must run on the AIE4 target; a pass " + "without a device would certify nothing. Check that no other " + "process is holding a device context."); + } +} + +// 1c. A real Stream and a real DeviceTensor, with a bounded element round +// trip. `count` and `offset` are ELEMENTS of the tensor's own dtype (API-7), +// so a BF16 tensor written from an FP32 source consumes half as many source +// bytes as destination bytes -- the asymmetry that makes a byte-taking +// overload dangerous enough that FastFlow does not offer one. +void CheckStreamAndTensorRoundTrip( + const std::shared_ptr& api) { + ryzenai_corelib_stream_ptr raw_stream = nullptr; + api->Check( + api->functions().create_stream(&raw_stream), + "ryzenai_corelib_create_stream"); + flm::corelib::UniqueStream stream(api, raw_stream); + CHECK(static_cast(stream)); + + constexpr std::int64_t kRows = 4; + constexpr std::int64_t kWidth = 128; + constexpr std::size_t kElements = + static_cast(kRows * kWidth); + const std::array shape{kRows, kWidth}; + + ryzenai_corelib_tensor_ptr raw_tensor = nullptr; + api->Check( + api->functions().create_device_tensor( + ryzenai_corelib_data_type_bf16, + shape.data(), + shape.size(), + &raw_tensor), + "ryzenai_corelib_create_device_tensor"); + flm::corelib::UniqueTensor tensor(api, raw_tensor); + CHECK(static_cast(tensor)); + + std::size_t byte_size = 0; + api->Check( + api->functions().tensor_get_byte_size( + tensor.get(), + &byte_size), + "ryzenai_corelib_tensor_get_byte_size"); + CHECK(byte_size == kElements * sizeof(std::uint16_t)); + + ryzenai_corelib_data_type data_type{}; + api->Check( + api->functions().tensor_get_data_type( + tensor.get(), + &data_type), + "ryzenai_corelib_tensor_get_data_type"); + CHECK(data_type == ryzenai_corelib_data_type_bf16); + + // FP32 in, BF16 stored, FP32 back out. The values are chosen to survive + // BF16 exactly -- powers of two and small sums of them -- so a mismatch + // means the transfer moved the wrong elements, not that rounding lost a + // bit. That distinction is why this does not compare with a tolerance. + std::vector source(kElements); + for (std::size_t index = 0; index < kElements; ++index) { + source[index] = + static_cast((index % 64) + 1) * 0.5f; + } + api->WriteElements( + tensor.get(), + ryzenai_corelib_data_type_fp32, + source.data(), + kElements, + 0); + + std::vector destination(kElements, -1.0f); + api->ReadElements( + tensor.get(), + ryzenai_corelib_data_type_fp32, + destination.data(), + kElements, + 0); + CHECK(destination == source); + + // A bounded interior window, to show that a non-zero element offset lands + // where it is asked to and leaves its neighbours untouched. Reading the + // whole tensor back afterwards is what proves the "leaves neighbours + // untouched" half; checking only the window would pass for a write that + // clobbered the rest. + constexpr std::size_t kWindowOffset = kWidth; + constexpr std::size_t kWindowCount = 16; + std::vector window(kWindowCount, 8.0f); + api->WriteElements( + tensor.get(), + ryzenai_corelib_data_type_fp32, + window.data(), + kWindowCount, + kWindowOffset); + + std::vector expected = source; + std::copy( + window.begin(), + window.end(), + expected.begin() + kWindowOffset); + std::fill(destination.begin(), destination.end(), -1.0f); + api->ReadElements( + tensor.get(), + ryzenai_corelib_data_type_fp32, + destination.data(), + kElements, + 0); + CHECK(destination == expected); + + // Released before the healthy cleanup below. CorelibRuntime refuses to + // call cleanup() while live objects remain, so a leak here would surface + // as a shutdown failure rather than as a quiet leak. + tensor.reset(); + stream.reset(); + CHECK(api->live_object_count() == 0); + std::cout << "stream and device tensor round trip: ok\n"; +} + +// 1d. The RoPE gather at a guard page, followed by ONE real tensor_write. +// +// The gather is FastFlow's code and the write is corelib's, and this is the +// only place the two meet on real hardware. The fixture's last source row ends +// exactly at the mapped extent, with PAGE_NOACCESS immediately after, so the +// process dies on an over-read rather than passing with garbage. +void CheckRopeGatherAndRealUpload( + const std::shared_ptr& api) { + fixture::SyntheticPackage synthetic; + auto package = Phi4Package::Load(synthetic.path(), api, false); + + const auto& source = package.Require("cos_cache"); + CHECK(source.size == fixture::kRopeBytes); + CHECK(source.owner->size() == fixture::kRopeMappedBytes); + + constexpr std::size_t kRopeElements = + static_cast(constants::kMaxSequenceLength) * + static_cast(constants::kRopeDimension / 2); + static_assert(kRopeElements == 4096u * 48u); + + const std::array shape{ + constants::kMaxSequenceLength, + constants::kRopeDimension / 2}; + ryzenai_corelib_tensor_ptr raw_tensor = nullptr; + api->Check( + api->functions().create_device_tensor( + ryzenai_corelib_data_type_fp32, + shape.data(), + shape.size(), + &raw_tensor), + "ryzenai_corelib_create_device_tensor"); + flm::corelib::UniqueTensor cos_tensor(api, raw_tensor); + + flm::phi4::RopeSourceView rope{}; + { + auto* one_past = + const_cast(source.data + source.size); + fixture::NoAccessGuard guard(one_past); + rope = package.MaterializeRopeGather("cos_cache"); + } + CHECK(rope.dtype == ryzenai_corelib_data_type_fp16); + CHECK(rope.count == kRopeElements); + + // One write, in the SOURCE dtype, with `count` in elements of the + // DESTINATION tensor. tensor_write is the only widening boundary corelib + // e5258d2 offers, and this is the call the engine makes at load. + api->WriteElements( + cos_tensor.get(), + rope.dtype, + rope.data, + kRopeElements, + 0); + + // Read back the three rows the fixture seeded, so the write is shown to + // have landed rather than merely to have returned success. Row 4095's + // column 47 is the last element before the guard page, which is the + // element an over- or under-reading gather would get wrong. + const auto read_element = [&](std::size_t index) { + float value = 0.0f; + api->ReadElements( + cos_tensor.get(), + ryzenai_corelib_data_type_fp32, + &value, + 1, + index); + return value; + }; + CHECK(read_element(0) == 1.0f); + CHECK(read_element(48) == 2.0f); + CHECK(read_element(kRopeElements - 1) == 3.0f); + + cos_tensor.reset(); + CHECK(api->live_object_count() == 0); + std::cout << "RoPE gather at guard page plus real tensor_write: ok\n"; +} + +// 5. Helper boundaries, discovered rather than transcribed. +// +// The row grid is a property of the SHIPPED kernel set, so the transitions are +// read back out of the running helper table and the boundary rows are derived +// from them. A hard-coded {1, 64, 128, ...} would keep passing against a +// library that had changed its grid, which is exactly the failure this is here +// to catch. +void CheckHelperBoundaries(const std::shared_ptr& api) { + const Phi4ShapePlan plan = Phi4ShapePlan::Build(api); + + for (const auto& row_use : kMultiRowUses) { + const auto& transitions = plan.Transitions(row_use.use); + CHECK(!transitions.empty()); + + std::vector probes{ + 1, + constants::kMaxSequenceLength}; + for (const auto& [live_rows, padded_rows] : transitions) { + for (const std::int64_t offset : {-1, 0, 1}) { + const std::int64_t probe = live_rows + offset; + if ( + probe >= 1 && + probe <= constants::kMaxSequenceLength) { + probes.push_back(probe); + } + } + } + std::sort(probes.begin(), probes.end()); + probes.erase( + std::unique(probes.begin(), probes.end()), + probes.end()); + + std::cout << " " << row_use.name << ": " + << transitions.size() << " transitions, " + << probes.size() << " boundary probes"; + + std::int64_t previous_padded = 0; + for (const std::int64_t rows : probes) { + const std::int64_t padded = + plan.RowsFor(row_use.use, rows); + // Never below the live rows, never above the single peak + // allocation, and never decreasing: those three together are + // what make one capacity-sized tensor safe for every row count. + CHECK(padded >= rows); + CHECK(padded <= plan.capacities().layer_rows); + CHECK(padded >= previous_padded); + previous_padded = padded; + + // And the plan must still agree with the library it was built + // from, asked directly at this row count. A plan that had + // memoised a stale answer would pass every check above. + std::int64_t m = rows; + std::int64_t k = constants::kHiddenSize; + std::int64_t n = row_use.use == RowUse::KvProjection + ? constants::kKvDimension + : constants::kQueryDimension; + if (row_use.use == RowUse::SsMlp) { + m = rows; + api->Check( + api->functions().ssmlp_pad_rows( + &m, + constants::kHiddenSize, + constants::kIntermediateSize, + constants::kGroupSize), + "ryzenai_corelib_ssmlp_bf16_pad_rows"); + } else if (row_use.use == RowUse::Attention) { + api->Check( + api->functions().flat_mha_pad_rows( + &m, + &plan.attention_desc()), + "ryzenai_corelib_flat_mha_bf16_pad_rows"); + } else { + api->Check( + api->functions().matmul_pad_shape( + &m, + &k, + &n, + constants::kGroupSize), + "ryzenai_corelib_matmul_bf16_pad_shape"); + // MEM-5: the capacity model assumes padding touches M only. + CHECK(k == constants::kHiddenSize); + } + CHECK(m == padded); + } + std::cout << ", padded 1 -> " << plan.RowsFor(row_use.use, 1) + << ", " << constants::kMaxSequenceLength << " -> " + << plan.RowsFor( + row_use.use, + constants::kMaxSequenceLength) + << '\n'; + + // Fresh row 1 must stay 1 on every multi-row helper: decode + // allocates a single row and flat MHA pads its KV window instead. + CHECK(plan.RowsFor(row_use.use, 1) == 1); + } + + // The LM head is not a multi-row helper. It ships M in {1, 128} and + // ERRORS above 128 rather than rounding up, which is the property that + // makes Phi4ShapePlan's single-row query correct. Assert the refusal + // rather than inferring it from the code that avoids it. + CHECK(plan.capacities().lm_head_rows == 1); + CHECK(plan.RowsFor(RowUse::LmHead, 1) == 1); + for (const std::int64_t rows : + {std::int64_t{1}, std::int64_t{2}, std::int64_t{128}}) { + std::int64_t m = rows; + std::int64_t k = constants::kHiddenSize; + std::int64_t n = constants::kVocabularySize; + api->Check( + api->functions().matmul_pad_shape( + &m, + &k, + &n, + constants::kGroupSize), + "ryzenai_corelib_matmul_bf16_pad_shape"); + CHECK(m == (rows == 1 ? 1 : 128)); + } + for (const std::int64_t rows : + {std::int64_t{129}, + std::int64_t{256}, + std::int64_t{4096}}) { + std::int64_t m = rows; + std::int64_t k = constants::kHiddenSize; + std::int64_t n = constants::kVocabularySize; + const auto status = api->functions().matmul_pad_shape( + &m, + &k, + &n, + constants::kGroupSize); + if (status == ryzenai_corelib_status_success) { + throw std::runtime_error( + "the LM-head shape accepted M=" + std::to_string(rows) + + " and padded to " + std::to_string(m) + + ". Phi4ShapePlan queries the LM head at m=1 only because " + "the shipped kernel set refuses everything above 128; if " + "that changed, the single-row assumption needs revisiting " + "rather than this assertion needs relaxing."); + } + } + std::cout << "helper boundaries agree with the running kernel set\n"; + + // The peak allocation must really be creatable on the device. The + // capacity model is only sound if a tensor of that extent exists, and a + // pad-shape query alone would not show that. + const std::array peak_shape{ + plan.capacities().layer_rows, + constants::kQueryDimension}; + ryzenai_corelib_tensor_ptr raw_tensor = nullptr; + api->Check( + api->functions().create_device_tensor( + ryzenai_corelib_data_type_bf16, + peak_shape.data(), + peak_shape.size(), + &raw_tensor), + "ryzenai_corelib_create_device_tensor"); + flm::corelib::UniqueTensor peak(api, raw_tensor); + CHECK(static_cast(peak)); + peak.reset(); + CHECK(api->live_object_count() == 0); + std::cout << "peak layer allocation (" + << plan.capacities().layer_rows << " x " + << constants::kQueryDimension + << " BF16) is creatable on the device\n"; +} + +} // namespace + +// CTest's SKIP_RETURN_CODE. Returning 0 without a runtime directory would +// report Passed, and a green-and-inert hardware check reads as coverage it +// does not have. +constexpr int kCTestSkipReturnCode = 77; + +// Opt-in, and deliberately separate from RYZENAI_CORELIB_RUNTIME_DIR. +// +// A configured runtime directory says which DLL to load; it does not say that +// this machine is the AIE4 target. The development box has an NPU and a real +// corelib but is not AIE4, so keying the hardware run off the directory alone +// would turn every dev-box `ctest` red for a reason that is not a defect. +// Keying it off nothing would be worse: an absent device on the target would +// then read as a skip, and the one machine where this must run would be the +// one machine where a silent skip goes unnoticed. With the flag set, a missing +// device context is a hard failure. +bool HardwareRunRequested() { + char value[8] = {}; + const DWORD length = GetEnvironmentVariableA( + "FLM_AIE4_HARDWARE", + value, + sizeof(value)); + return length != 0 && length < sizeof(value) && + std::string_view(value) == "1"; +} + +int main() { + const std::string runtime_dir(FLM_REAL_CORELIB_RUNTIME_DIR); + if (runtime_dir.empty()) { + std::cout + << "test_phi4_hardware: SKIPPED -- configure with " + "-DRYZENAI_CORELIB_RUNTIME_DIR= and run on the AIE4 target.\n"; + return kCTestSkipReturnCode; + } + if (!HardwareRunRequested()) { + std::cout + << "test_phi4_hardware: SKIPPED -- set FLM_AIE4_HARDWARE=1 to " + "run this on the AIE4 target. Set it only there: on a machine " + "with no device it turns a skip into a failure.\n"; + return kCTestSkipReturnCode; + } + + try { + const std::filesystem::path library = + std::filesystem::absolute( + std::filesystem::path(runtime_dir) / + "ryzenai_corelib.dll") + .lexically_normal(); + if (!std::filesystem::exists(library)) { + throw std::runtime_error( + "RYZENAI_CORELIB_RUNTIME_DIR is set but " + + library.string() + " does not exist"); + } + AddExtraDllDirectories(FLM_REAL_CORELIB_EXTRA_DLL_DIRS); + std::cout << "loading " << library.string() << '\n'; + + auto api = CorelibApi::Load(library); + CheckVersionIdentity(api); + CheckDependenciesAndDevice(api); + CheckStreamAndTensorRoundTrip(api); + CheckRopeGatherAndRealUpload(api); + CheckHelperBoundaries(api); + + api->functions().cleanup(); + std::cout << "test_phi4_hardware: PASS\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << "test_phi4_hardware: FAIL: " << error.what() << '\n'; + return 1; + } +} diff --git a/src/test/phi4_corelib_aie4/test_phi4_host.cpp b/src/test/phi4_corelib_aie4/test_phi4_host.cpp new file mode 100644 index 00000000..84964106 --- /dev/null +++ b/src/test/phi4_corelib_aie4/test_phi4_host.cpp @@ -0,0 +1,738 @@ +#include "fake_corelib.hpp" +#include "test_support.hpp" + +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using flm::corelib::CorelibApi; +using flm::phi4::ArgmaxLowest; +using flm::phi4::GatherEmbedding; +using flm::phi4::RmsNorm; +using flm::phi4::ScatterV; +using flm::phi4::StageFp32; +using flm::phi4::VScatterMetrics; +using bf16 = biovault::bfloat16_t; + +constexpr std::uint16_t kPoison = 0xDEADu; + +struct ReadCall { + ryzenai_corelib_tensor_ptr tensor; + ryzenai_corelib_data_type destination_type; + void* destination; + std::size_t count; + std::size_t offset; +}; + +struct WriteCall { + ryzenai_corelib_tensor_ptr tensor; + ryzenai_corelib_data_type source_type; + const void* source; + std::size_t count; + std::size_t offset; + std::vector values; +}; + +struct RecordingState { + std::vector reads; + std::vector writes; + std::size_t synchronize_calls = 0; + std::vector v_source; + int v_tensor_storage = 0; + int v_cache_storage = 0; + + ryzenai_corelib_tensor_ptr v_tensor() noexcept { + return &v_tensor_storage; + } + + ryzenai_corelib_tensor_ptr v_cache() noexcept { + return &v_cache_storage; + } + + void ResetCalls() { + reads.clear(); + writes.clear(); + synchronize_calls = 0; + } +}; + +RecordingState g_recording; + +// `count` and `offset` are BF16 ELEMENTS of the V tensor, never bytes. A +// caller that still passes bytes asks for twice the elements and is +// rejected here rather than reading past the source. +ryzenai_corelib_status RecordingTensorRead( + ryzenai_corelib_tensor_ptr tensor, + ryzenai_corelib_data_type destination_type, + void* destination, + std::size_t count, + std::size_t offset) { + if (tensor != g_recording.v_tensor() || destination == nullptr || + destination_type != ryzenai_corelib_data_type_bf16 || + offset > g_recording.v_source.size() || + count > g_recording.v_source.size() - offset) { + return ryzenai_corelib_status_bad_argument; + } + g_recording.reads.push_back( + ReadCall{tensor, destination_type, destination, count, offset}); + std::memcpy( + destination, + g_recording.v_source.data() + offset, + count * sizeof(std::uint16_t)); + + const auto tick = std::chrono::steady_clock::now(); + while (std::chrono::steady_clock::now() == tick) { + } + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status RecordingTensorWrite( + ryzenai_corelib_tensor_ptr tensor, + ryzenai_corelib_data_type source_type, + const void* source, + std::size_t count, + std::size_t offset) { + constexpr std::size_t cache_elements = + static_cast(flm::phi4::constants::kKvHeadCount) * + static_cast( + flm::phi4::constants::kMaxSequenceLength) * + static_cast(flm::phi4::constants::kHeadSize); + if (tensor != g_recording.v_cache() || source == nullptr || + source_type != ryzenai_corelib_data_type_bf16 || + offset > cache_elements || count > cache_elements - offset) { + return ryzenai_corelib_status_bad_argument; + } + const auto* values = static_cast(source); + g_recording.writes.push_back(WriteCall{ + tensor, + source_type, + source, + count, + offset, + std::vector(values, values + count)}); + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status RecordingStreamSynchronize( + ryzenai_corelib_stream_ptr) { + ++g_recording.synchronize_calls; + return ryzenai_corelib_status_success; +} + +template +void* FunctionAddress(Function function) { + return reinterpret_cast(function); +} + +std::shared_ptr ResolveRecordingCorelib() { + auto resolver = flm::test::CompleteCorelibResolver(); + resolver["ryzenai_corelib_tensor_read"] = + FunctionAddress( + static_cast( + &RecordingTensorRead)); + resolver["ryzenai_corelib_tensor_write"] = + FunctionAddress( + static_cast( + &RecordingTensorWrite)); + resolver["ryzenai_corelib_stream_synchronize"] = + FunctionAddress( + static_cast< + decltype(&::ryzenai_corelib_stream_synchronize)>( + &RecordingStreamSynchronize)); + return CorelibApi::ResolveForTest( + [resolver = std::move(resolver)](std::string_view name) mutable + -> void* { + const auto found = resolver.find(std::string(name)); + return found == resolver.end() ? nullptr : found->second; + }); +} + +// The two host conversions design API-6 permits, and only those two. +// This one is lossless, so it is checked against exact values. +void TestWidenFp16IsExactAcrossTheFp16Range() { + struct Case { + std::uint16_t bits; + float value; + }; + constexpr Case kCases[]{ + {0x0000u, 0.0f}, + {0x8000u, -0.0f}, + {0x3C00u, 1.0f}, + {0xBC00u, -1.0f}, + {0x4000u, 2.0f}, + {0xC000u, -2.0f}, + {0x4200u, 3.0f}, + {0x3800u, 0.5f}, + {0x0001u, 5.9604645e-8f}, // smallest subnormal + {0x03FFu, 6.0975552e-5f}, // largest subnormal + {0x0400u, 6.1035156e-5f}, // smallest normal + {0x7BFFu, 65504.0f}, // largest finite + }; + for (const auto& item : kCases) { + CHECK(flm::corelib::WidenFp16(item.bits) == item.value); + } + CHECK(std::isinf(flm::corelib::WidenFp16(0x7C00u))); + CHECK(flm::corelib::WidenFp16(0x7C00u) > 0.0f); + CHECK(std::isinf(flm::corelib::WidenFp16(0xFC00u))); + CHECK(std::isnan(flm::corelib::WidenFp16(0x7E00u))); + + // Every representable FP16 round-trips through float, which is what + // "lossless, so it has no rounding policy" means in practice. + for (std::uint32_t bits = 0; bits <= 0xFFFFu; ++bits) { + const auto narrow = static_cast(bits); + const float widened = flm::corelib::WidenFp16(narrow); + if (std::isnan(widened)) { + continue; + } + const auto exponent = (narrow >> 10) & 0x1Fu; + const auto mantissa = narrow & 0x3FFu; + const bool negative = (narrow & 0x8000u) != 0; + double expected = 0.0; + if (exponent == 0u) { + expected = std::ldexp(static_cast(mantissa), -24); + } else if (exponent == 0x1Fu) { + continue; + } else { + expected = std::ldexp( + 1.0 + static_cast(mantissa) / 1024.0, + static_cast(exponent) - 15); + } + if (negative) { + expected = -expected; + } + CHECK(static_cast(widened) == expected); + } +} + +// The reference driver's to_bf16, transcribed. FastFlow's helper must +// agree with it bit for bit; there is no second BF16 rounding policy. +std::uint16_t ReferenceToBf16(std::uint32_t bits) { + const std::uint64_t rounded = + static_cast(bits) + 0x7FFFull + + ((static_cast(bits) >> 16) & 1ull); + return static_cast(rounded >> 16); +} + +void TestNarrowFp32ToBf16MatchesTheReferenceDriverBitForBit() { + constexpr std::uint32_t kExact[]{ + 0x00000000u, + 0x80000000u, + 0x3F800000u, + 0x3F808000u, // exact tie, rounds to even + 0x3F818000u, // exact tie, rounds up to even + 0xBF800000u, + 0x40000000u, + 0xC0000000u, + 0x3F000000u, + 0x322BCC77u, // 1e-8f + 0x3727C5ACu, // 1e-5f, the Phi-4 RMS epsilon + 0x7F800000u, + 0xFF800000u, + 0x7FC00000u, + 0xFFFFFFFFu, // the wrap the driver hides in uint64 + 0xFFFF8000u, + }; + for (const std::uint32_t bits : kExact) { + CHECK( + flm::corelib::NarrowFp32ToBf16(std::bit_cast(bits)) == + ReferenceToBf16(bits)); + } + + // A dense sweep of the low mantissa bits, where the tie-breaking + // actually differs between truncation and round-to-nearest-even. + for (std::uint32_t low = 0; low < 0x20000u; ++low) { + const std::uint32_t bits = 0x3F800000u + low; + CHECK( + flm::corelib::NarrowFp32ToBf16(std::bit_cast(bits)) == + ReferenceToBf16(bits)); + } + // And a stride across the whole exponent range. + for (std::uint64_t bits = 0; bits <= 0xFFFFFFFFull; bits += 65413ull) { + const auto value = static_cast(bits); + CHECK( + flm::corelib::NarrowFp32ToBf16(std::bit_cast(value)) == + ReferenceToBf16(value)); + } + + CHECK( + flm::corelib::NarrowFp32ToBf16( + static_cast(flm::phi4::constants::kRmsEpsilon)) == + ReferenceToBf16(std::bit_cast( + static_cast(flm::phi4::constants::kRmsEpsilon)))); +} + +void TestGatherEmbeddingWidensWithoutCorelib() { + constexpr std::size_t width = static_cast( + flm::phi4::constants::kHiddenSize); + std::vector embedding(3u * width); + std::fill_n(embedding.begin(), width, 0x3C00u); + std::fill_n(embedding.begin() + width, width, 0xC000u); + std::fill_n(embedding.begin() + 2u * width, width, 0x4200u); + + const std::array ids{2, 0}; + std::vector output(ids.size() * width); + g_recording.ResetCalls(); + GatherEmbedding(embedding, ids, output); + + CHECK(std::all_of( + output.begin(), + output.begin() + width, + [](float value) { return value == 3.0f; })); + CHECK(std::all_of( + output.begin() + width, + output.end(), + [](float value) { return value == 1.0f; })); + + const std::array second_ids{1}; + output.assign(width, 0.0f); + GatherEmbedding(embedding, second_ids, output); + CHECK(std::all_of( + output.begin(), + output.end(), + [](float value) { return value == -2.0f; })); + + // No corelib call is made at all: the widening is FastFlow's own. + CHECK(g_recording.reads.empty()); + CHECK(g_recording.writes.empty()); +} + +// The embedding table is a read-only file mapping. A vectorized widening +// reads up to 14 bytes past its source, which faults instead of returning +// garbage -- so the gather must touch nothing beyond the last row. +void TestGatherEmbeddingStopsAtAGuardPage() { + constexpr std::size_t width = static_cast( + flm::phi4::constants::kHiddenSize); + constexpr std::size_t rows = 4; + constexpr std::size_t table_bytes = rows * width * sizeof(std::uint16_t); + static_assert(table_bytes % 4096u == 0u); + + auto* base = static_cast(VirtualAlloc( + nullptr, + table_bytes + 4096u, + MEM_RESERVE, + PAGE_NOACCESS)); + if (base == nullptr) { + throw std::runtime_error("failed to reserve guard-page range"); + } + if (VirtualAlloc(base, table_bytes, MEM_COMMIT, PAGE_READWRITE) == + nullptr) { + VirtualFree(base, 0, MEM_RELEASE); + throw std::runtime_error("failed to commit guard-page table"); + } + + auto* table = reinterpret_cast(base); + for (std::size_t index = 0; index < rows * width; ++index) { + table[index] = 0x3C00u; + } + const std::span embedding(table, rows * width); + + // The last row ends exactly at the boundary of the reserved, + // never-committed page that follows. + const std::array ids{static_cast(rows) - 1}; + std::vector output(width); + GatherEmbedding(embedding, ids, output); + const bool all_one = std::all_of( + output.begin(), + output.end(), + [](float value) { return value == 1.0f; }); + + VirtualFree(base, 0, MEM_RELEASE); + CHECK(all_one); +} + +// The reduction is accumulated in DOUBLE and the result is FP32. +// +// This golden previously pinned a serial FP32 accumulator, and on the AIE4 +// target that turned out to be the source of a measured divergence from the +// corelib reference driver: up to 4.8e-6 relative error in the layer-0 norm, +// which moved 25 of 58368 values onto a different BF16 number and, through 32 +// layers, produced a logit correlation of 0.9991 against a required 0.9999. A +// double accumulator brings the differing count to zero. +// +// The four-element case below is regenerated but barely discriminating -- two +// of its four words move by one ULP. TestRmsNormReductionSurvivesALongRow is +// the one that actually fails if the accumulator goes back to FP32. +void TestRmsNormUsesDoubleAccumulationAndSharedEpsilon() { + const std::array input{ + std::bit_cast(0xBE8BBBACu), + std::bit_cast(0xBCCC9DE0u), + std::bit_cast(0xBFED682Fu), + std::bit_cast(0xC2CD01EDu)}; + const std::array scale{ + 1.0f, + 1.0f, + 1.0f, + 1.0f}; + std::array output{}; + + RmsNorm( + input, + scale, + 1, + 4, + static_cast( + flm::phi4::constants::kRmsEpsilon), + output); + + constexpr std::array expected{ + 0xBBAE75DBu, + 0xB9FF7820u, + 0xBD143451u, + 0xBFFFF50Bu}; + for (std::size_t index = 0; index < output.size(); ++index) { + CHECK(std::bit_cast(output[index]) == + expected[index]); + } +} + +// A full 3072-wide row, which is the width the model actually uses. +// +// One large square followed by 3071 small ones is the case a serial FP32 sum +// loses outright: each small addend is far below the running total's ULP, so +// almost all of them vanish. Every one of the 3072 outputs differs between the +// two accumulators here, so this fails loudly if the reduction is ever +// narrowed back to FP32 -- unlike the four-element case above, where the two +// disagree in only two words and a reviewer could plausibly wave it through. +void TestRmsNormReductionSurvivesALongRow() { + constexpr std::size_t kWidth = 3072; + std::vector input(kWidth, 0.03125f); + input[0] = 1024.0f; + const std::vector scale(kWidth, 1.0f); + std::vector output(kWidth); + + RmsNorm( + input, + scale, + 1, + static_cast(kWidth), + static_cast(flm::phi4::constants::kRmsEpsilon), + output); + + // Produced by a double accumulator; 0x425DB3D8 and 0x3ADDB3D8 are what a + // serial FP32 accumulator produces for the same two elements. + CHECK(std::bit_cast(output[0]) == 0x425DB3C3u); + CHECK(std::bit_cast(output[1]) == 0x3ADDB3C3u); + CHECK(std::bit_cast(output[0]) != 0x425DB3D8u); + CHECK(std::bit_cast(output[1]) != 0x3ADDB3D8u); +} + +void TestStageFp32ZerosOnlyInitialInputPrefixes() { + const std::array normalized{ + std::bit_cast(0x3F800000u), + std::bit_cast(0x3F808000u), + std::bit_cast(0x3F818000u), + std::bit_cast(0xBF800000u)}; + const std::array residual{ + std::bit_cast(0x40000000u), + std::bit_cast(0xC0000000u), + std::bit_cast(0x00000000u), + std::bit_cast(0x3F000000u)}; + constexpr float kPoisonFloat = -12345.0f; + std::vector hidden(8, kPoisonFloat); + std::vector residual_device(8, kPoisonFloat); + + // Task 7 has no production consumer for q/k/attention/skip-sum/ + // next-hidden padded tails. Task 8's dispatch test must poison those + // actual persistent buffers and prove its live/helper-authorized + // regions exclude stale values. + + g_recording.ResetCalls(); + StageFp32(normalized, 2, 3, 2, hidden); + StageFp32(residual, 2, 3, 2, residual_device); + + // Design 10.2: the host stays in FP32 and never rounds to BF16, so + // the staged bits are the input bits, unchanged. + const std::array expected_hidden{ + normalized[0], + normalized[1], + normalized[2], + normalized[3], + 0.0f, + 0.0f, + kPoisonFloat, + kPoisonFloat}; + const std::array expected_residual{ + residual[0], + residual[1], + residual[2], + residual[3], + 0.0f, + 0.0f, + kPoisonFloat, + kPoisonFloat}; + for (std::size_t index = 0; index < hidden.size(); ++index) { + CHECK( + std::bit_cast(hidden[index]) == + std::bit_cast(expected_hidden[index])); + CHECK( + std::bit_cast(residual_device[index]) == + std::bit_cast(expected_residual[index])); + } + + // Nothing crossed the corelib boundary: the FP32-to-BF16 narrowing on + // this path is corelib's, inside tensor_write. + CHECK(g_recording.reads.empty()); + CHECK(g_recording.writes.empty()); +} + +std::uint16_t VValue( + std::size_t row, + std::size_t head, + std::size_t column) { + return static_cast( + row * 4096u + head * 256u + column); +} + +void PopulateVSource(std::size_t capacity_rows) { + constexpr std::size_t heads = 8; + constexpr std::size_t width = 128; + g_recording.v_source.assign( + capacity_rows * heads * width, + kPoison); + for (std::size_t row = 0; row < 3; ++row) { + for (std::size_t head = 0; head < heads; ++head) { + for (std::size_t column = 0; column < width; ++column) { + const std::size_t index = + (row * heads + head) * width + column; + g_recording.v_source[index] = + VValue(row, head, column); + } + } + } +} + +void CheckScatterWrites( + std::int64_t rows, + std::int64_t position) { + constexpr std::size_t heads = 8; + constexpr std::size_t width = 128; + CHECK(g_recording.writes.size() == heads); + for (std::size_t head = 0; head < heads; ++head) { + const auto& write = g_recording.writes[head]; + // API-7: elements of the cache's BF16 dtype, not bytes. The + // previous spelling multiplied both by sizeof(uint16_t) and was + // wrong by exactly 2x. + const std::size_t expected_offset = + ((head * 4096u) + + static_cast(position)) * + width; + CHECK(write.tensor == g_recording.v_cache()); + CHECK(write.source_type == ryzenai_corelib_data_type_bf16); + CHECK(write.offset == expected_offset); + CHECK( + write.count == static_cast(rows) * width); + CHECK( + write.values.size() == + static_cast(rows) * width); + for (std::size_t row = 0; + row < static_cast(rows); + ++row) { + for (std::size_t column = 0; column < width; ++column) { + CHECK( + write.values[row * width + column] == + VValue(row, head, column)); + } + } + CHECK(std::none_of( + write.values.begin(), + write.values.end(), + [](std::uint16_t value) { return value == kPoison; })); + } +} + +void TestScatterVReadsOnlyLiveRowsWithoutHiddenSynchronize( + const std::shared_ptr& api) { + constexpr std::size_t heads = 8; + constexpr std::size_t width = 128; + PopulateVSource(5); + std::vector staging; + VScatterMetrics metrics{}; + + g_recording.ResetCalls(); + ScatterV( + *api, + g_recording.v_tensor(), + g_recording.v_cache(), + 3, + 17, + staging, + metrics); + + const std::size_t read_elements = 3u * heads * width; + CHECK(g_recording.reads.size() == 1); + CHECK(g_recording.reads.front().tensor == + g_recording.v_tensor()); + CHECK( + g_recording.reads.front().destination_type == + ryzenai_corelib_data_type_bf16); + CHECK(g_recording.reads.front().count == read_elements); + CHECK(g_recording.reads.front().offset == 0); + CHECK(g_recording.reads.front().destination == staging.data()); + CHECK(g_recording.synchronize_calls == 0); + CheckScatterWrites(3, 17); + + const auto* staging_begin = staging.data(); + const auto* staging_end = staging.data() + staging.size(); + for (const auto& write : g_recording.writes) { + const auto* source = + static_cast(write.source); + CHECK(source >= staging_begin); + CHECK(source + write.count <= staging_end); + } + CHECK(metrics.read_calls == 1); + CHECK(metrics.write_calls == 8); + // Metrics stay in bytes; only the transfer arguments are elements. + CHECK( + metrics.bytes == + 2u * read_elements * sizeof(std::uint16_t)); + CHECK(metrics.nanoseconds > 0); + CHECK(std::all_of( + g_recording.v_source.begin() + 3u * heads * width, + g_recording.v_source.end(), + [](std::uint16_t value) { return value == kPoison; })); + + const void* first_data = staging.data(); + const std::size_t first_capacity = staging.capacity(); + const std::uint64_t first_ns = metrics.nanoseconds; + g_recording.ResetCalls(); + ScatterV( + *api, + g_recording.v_tensor(), + g_recording.v_cache(), + 1, + 20, + staging, + metrics); + CHECK(staging.data() == first_data); + CHECK(staging.capacity() == first_capacity); + CHECK(g_recording.reads.size() == 1); + CHECK(g_recording.reads.front().count == heads * width); + CHECK(g_recording.synchronize_calls == 0); + CheckScatterWrites(1, 20); + CHECK(metrics.read_calls == 2); + CHECK(metrics.write_calls == 16); + CHECK( + metrics.bytes == + (2u * read_elements + 2u * heads * width) * + sizeof(std::uint16_t)); + CHECK(metrics.nanoseconds > first_ns); +} + +void TestArgmaxLowestChoosesLowestTokenOnTie() { + std::vector logits(12, bf16{0xBF80u, true}); + logits[2] = bf16{0x4000u, true}; + logits[7] = bf16{0x4040u, true}; + logits[9] = bf16{0x4040u, true}; + CHECK(ArgmaxLowest(logits) == 7); + + const std::array negative{ + bf16{0xC040u, true}, + bf16{0xC000u, true}, + bf16{0xC000u, true}}; + CHECK(ArgmaxLowest(negative) == 1); +} + +void TestInvalidHostArgumentsFailBeforeCorelib( + const std::shared_ptr& api) { + constexpr std::size_t width = static_cast( + flm::phi4::constants::kHiddenSize); + std::vector embedding(width); + const std::array invalid_id{1}; + std::vector embedding_output(width); + std::vector staged(4); + std::vector scatter_staging; + VScatterMetrics metrics{}; + + g_recording.ResetCalls(); + CheckThrowsContains( + [&] { + GatherEmbedding(embedding, invalid_id, embedding_output); + }, + "token ID"); + CheckThrowsContains( + [&] { + StageFp32( + std::span{embedding_output}.first(2), + 2, + 1, + 1, + staged); + }, + "padded rows"); + CheckThrowsContains( + [&] { + ScatterV( + *api, + g_recording.v_tensor(), + g_recording.v_cache(), + 2, + 4095, + scatter_staging, + metrics); + }, + "cache"); + CheckThrowsContains( + [&] { + std::array rms_output{}; + RmsNorm( + std::span{embedding_output}.first(1), + std::span{embedding_output}.first(1), + 1, + 2, + static_cast( + flm::phi4::constants::kRmsEpsilon), + rms_output); + }, + "shape"); + CheckThrowsContains( + [] { + const std::span empty; + (void)ArgmaxLowest(empty); + }, + "empty"); + CHECK(g_recording.reads.empty()); + CHECK(g_recording.writes.empty()); +} + +} // namespace + +int main() { + try { + const auto api = ResolveRecordingCorelib(); + TestWidenFp16IsExactAcrossTheFp16Range(); + TestNarrowFp32ToBf16MatchesTheReferenceDriverBitForBit(); + TestGatherEmbeddingWidensWithoutCorelib(); + TestGatherEmbeddingStopsAtAGuardPage(); + TestRmsNormUsesDoubleAccumulationAndSharedEpsilon(); + TestRmsNormReductionSurvivesALongRow(); + TestStageFp32ZerosOnlyInitialInputPrefixes(); + TestScatterVReadsOnlyLiveRowsWithoutHiddenSynchronize(api); + TestArgmaxLowestChoosesLowestTokenOnTie(); + TestInvalidHostArgumentsFailBeforeCorelib(api); + std::cout << "phi4 host operation tests passed\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/src/test/phi4_corelib_aie4/test_phi4_manifest.cpp b/src/test/phi4_corelib_aie4/test_phi4_manifest.cpp new file mode 100644 index 00000000..e8c26952 --- /dev/null +++ b/src/test/phi4_corelib_aie4/test_phi4_manifest.cpp @@ -0,0 +1,661 @@ +#include "fake_corelib.hpp" +#include "phi4_package_fixture.hpp" +#include "test_support.hpp" + +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using flm::corelib::CorelibApi; +using flm::phi4::InitializerView; +using flm::phi4::MappedFile; +using flm::phi4::Phi4Package; +using flm::phi4::SourceDType; +using flm::phi4::WeightObjectKind; +using nlohmann::json; + +using flm::test::phi4fixture::kDataBytes; +using flm::test::phi4fixture::kDataFile; +using flm::test::phi4fixture::kFp16Scale; +using flm::test::phi4fixture::kFp32Norm; +using flm::test::phi4fixture::kFp32Scale; +using flm::test::phi4fixture::kRopeBytes; +using flm::test::phi4fixture::kRopeMappedBytes; +using flm::test::phi4fixture::NoAccessGuard; +using flm::test::phi4fixture::SyntheticPackage; + +template +void* FunctionAddress(Function function) { + return reinterpret_cast(function); +} + +std::shared_ptr ResolveRecordingCorelib() { + auto resolver = flm::test::CompleteCorelibResolver(); + return CorelibApi::ResolveForTest( + [resolver = std::move(resolver)](std::string_view name) mutable + -> void* { + const auto found = resolver.find(std::string(name)); + return found == resolver.end() ? nullptr : found->second; + }); +} + +void RenameFile( + json& manifest, + std::string_view old_name, + std::string new_name) { + auto record = manifest["files"].at(std::string(old_name)); + manifest["files"].erase(std::string(old_name)); + manifest["files"][new_name] = std::move(record); + for (auto& [_, initializer] : + manifest["initializers"].items()) { + if ( + initializer["file"].get() == + std::string(old_name)) { + initializer["file"] = new_name; + } + } +} + +template +void ExpectLoadFailure( + const SyntheticPackage& fixture, + const std::shared_ptr& api, + Mutation&& mutation, + std::string_view expected, + bool verify_full_hash = false) { + json manifest = fixture.manifest(); + std::invoke( + std::forward(mutation), + manifest); + fixture.Write(manifest); + try { + (void)Phi4Package::Load( + fixture.path(), + api, + verify_full_hash); + } catch (const std::exception& error) { + if ( + std::string_view(error.what()).find(expected) == + std::string_view::npos) { + throw std::runtime_error( + "expected load failure containing '" + + std::string(expected) + "', got: " + error.what()); + } + return; + } + throw std::runtime_error( + "expected package load failure containing '" + + std::string(expected) + "'"); +} + +void TestValidMappingAndExplicitRoles( + const SyntheticPackage& fixture, + const std::shared_ptr& api) { + fixture.Write(fixture.manifest()); + + auto package = Phi4Package::Load(fixture.path(), api, false); + CHECK(package.weight_objects().size() == 161); + const auto& first = package.weight_objects().front(); + CHECK(first.name == + "model.layers.0.attn.q_proj.MatMulNBits"); + CHECK(first.kind == WeightObjectKind::MatMul); + CHECK(first.k == 3072); + CHECK(first.n == 3072); + CHECK(first.group_size == 128); + CHECK(!first.has_bias); + CHECK( + first.components.at("qweight") == + "model.layers.0.attn.q_proj.MatMulNBits.qweight"); + + const auto& last = package.weight_objects().back(); + CHECK(last.name == "lm_head.MatMulNBits"); + CHECK(last.n == 200064); + + const auto& embedding = + package.Require("model.embed_tokens.weight"); + CHECK(embedding.dtype == SourceDType::Float16); + CHECK(embedding.shape == + std::vector({200064, 3072})); + CHECK(embedding.size == kDataBytes); + CHECK(embedding.data != nullptr); + CHECK(embedding.owner != nullptr); + CHECK(embedding.owner->path().filename() == kDataFile); + CHECK(embedding.owner->size() == kDataBytes); + CheckThrowsContains( + [&] { + (void)package.Require("missing.initializer"); + }, + "missing initializer"); +} + +void TestOgaQuantizedLayoutsAccepted( + const SyntheticPackage& fixture, + const std::shared_ptr& api) { + json manifest = fixture.manifest(); + for (auto& [name, initializer] : + manifest["initializers"].items()) { + if (name.ends_with(".qweight")) { + const auto logical = + initializer["shape"].get>(); + CHECK(logical.size() == 2); + CHECK(logical[1] % 64 == 0); + initializer["shape"] = { + logical[0], + logical[1] / 64, + 64}; + } else if ( + name.ends_with(".scales") || + name.ends_with(".qzeros")) { + const auto logical = + initializer["shape"].get>(); + CHECK(logical.size() == 2); + initializer["shape"] = {logical[0] * logical[1]}; + } + } + fixture.Write(manifest); + + auto package = Phi4Package::Load(fixture.path(), api, false); + CHECK(package.weight_objects().size() == 161); + CHECK( + package.Require( + "model.layers.0.attn.q_proj.MatMulNBits.qweight") + .shape == + std::vector({3072, 24, 64})); +} + +void TestMappedOwnerOutlivesPackage( + const SyntheticPackage& fixture, + const std::shared_ptr& api) { + fixture.Write(fixture.manifest()); + std::optional retained; + { + auto package = + Phi4Package::Load(fixture.path(), api, false); + retained = package.Require(kFp16Scale); + CHECK(retained->owner.use_count() > 1); + } + CHECK(retained->owner.use_count() == 1); + CHECK(retained->data != nullptr); + CHECK( + *reinterpret_cast(retained->data) == + 0x3c00u); + + auto direct = + MappedFile::OpenReadOnly(fixture.path() / "model.onnx"); + CHECK(direct->size() == 5); + CHECK(direct->path() == fixture.path() / "model.onnx"); + CHECK(direct->data()[0] == std::byte{'m'}); +} + +void TestPathRangeAndHashRejections( + const SyntheticPackage& fixture, + const std::shared_ptr& api) { + ExpectLoadFailure( + fixture, + api, + [](json& manifest) { + RenameFile(manifest, kDataFile, "C:/outside.bin"); + }, + "relative"); + ExpectLoadFailure( + fixture, + api, + [](json& manifest) { + RenameFile(manifest, kDataFile, "../outside.bin"); + }, + "traversal"); + ExpectLoadFailure( + fixture, + api, + [](json& manifest) { + auto& record = manifest["initializers"].at( + "model.embed_tokens.weight"); + record["offset"] = + std::numeric_limits::max() - 1; + }, + "range overflow"); + ExpectLoadFailure( + fixture, + api, + [](json& manifest) { + auto& record = + manifest["initializers"].at(std::string(kFp16Scale)); + record["length"] = + record["length"].get() - 1; + }, + "byte count"); + ExpectLoadFailure( + fixture, + api, + [](json& manifest) { + manifest["initializers"] + .at(std::string(kFp16Scale))["offset"] = 1; + }, + "dtype-aligned"); + ExpectLoadFailure( + fixture, + api, + [](json& manifest) { + manifest["files"].at(std::string(kDataFile))["size"] = + kDataBytes - 1; + }, + "file size"); + ExpectLoadFailure( + fixture, + api, + [](json& manifest) { + for (auto& [_, record] : manifest["files"].items()) { + record["sha256"] = std::string(64, '0'); + } + }, + "SHA-256", + true); +} + +void TestWeightObjectRejections( + const SyntheticPackage& fixture, + const std::shared_ptr& api) { + ExpectLoadFailure( + fixture, + api, + [](json& manifest) { + manifest.erase("weight_objects"); + }, + "weight_objects"); + ExpectLoadFailure( + fixture, + api, + [](json& manifest) { + manifest["weight_objects"] = json::array(); + }, + "weight_objects"); + ExpectLoadFailure( + fixture, + api, + [](json& manifest) { + manifest["weight_objects"].erase( + manifest["weight_objects"].end() - 1); + }, + "161"); + ExpectLoadFailure( + fixture, + api, + [](json& manifest) { + manifest["weight_objects"][1]["name"] = + manifest["weight_objects"][0]["name"]; + }, + "duplicate weight object"); + ExpectLoadFailure( + fixture, + api, + [](json& manifest) { + manifest["weight_objects"][0]["kind"] = "convolution"; + }, + "kind"); + ExpectLoadFailure( + fixture, + api, + [](json& manifest) { + manifest["weight_objects"][0]["descriptor"].erase("n"); + }, + "descriptor"); + ExpectLoadFailure( + fixture, + api, + [](json& manifest) { + manifest["weight_objects"][0]["descriptor"]["n"] = 1024; + }, + "descriptor"); + ExpectLoadFailure( + fixture, + api, + [](json& manifest) { + manifest["weight_objects"][0]["roles"]["mystery"] = + "cos_cache"; + }, + "role"); + ExpectLoadFailure( + fixture, + api, + [](json& manifest) { + const std::string target = + manifest["weight_objects"][0]["roles"]["qweight"]; + manifest["initializers"]["unreferenced-placeholder"] = + manifest["initializers"].at(target); + manifest["initializers"].erase(target); + }, + "unresolved initializer"); + ExpectLoadFailure( + fixture, + api, + [](json& manifest) { + manifest["weight_objects"][0]["roles"]["scales"] = + manifest["weight_objects"][0]["roles"]["qweight"]; + }, + "duplicate initializer"); +} + +void TestRoleInitializerIdentityRejections( + const SyntheticPackage& fixture, + const std::shared_ptr& api) { + ExpectLoadFailure( + fixture, + api, + [](json& manifest) { + auto& q_role = + manifest["weight_objects"][0]["roles"]["qweight"]; + auto& o_role = + manifest["weight_objects"][3]["roles"]["qweight"]; + const std::string q_initializer = + q_role.get(); + q_role = o_role.get(); + o_role = q_initializer; + }, + "model.layers.0.attn.q_proj.MatMulNBits.qweight " + "(model.layers.0.attn.o_proj.MatMulNBits.qweight)"); + ExpectLoadFailure( + fixture, + api, + [](json& manifest) { + constexpr std::size_t layer5 = 5u * 5u + 4u; + constexpr std::size_t layer31 = 31u * 5u + 4u; + auto& layer5_norm = + manifest["weight_objects"][layer5]["roles"]["norm1"]; + auto& layer31_norm = + manifest["weight_objects"][layer31]["roles"]["norm1"]; + const std::string initializer = + layer5_norm.get(); + layer5_norm = layer31_norm.get(); + layer31_norm = initializer; + }, + "model.layers.5.ssmlp.norm1 " + "(model.layers.32.final_norm_layernorm.weight)"); +} + +void TestExactSourceValidation( + const SyntheticPackage& fixture, + const std::shared_ptr& api) { + ExpectLoadFailure( + fixture, + api, + [](json& manifest) { + auto& record = manifest["initializers"].at( + "model.embed_tokens.weight"); + record["dtype"] = "uint8"; + record["length"] = + 200064ull * 3072ull; + }, + "embedding"); + ExpectLoadFailure( + fixture, + api, + [](json& manifest) { + auto& record = manifest["initializers"].at( + "model.layers.0.input_layernorm.weight"); + record["dtype"] = "uint8"; + record["length"] = 3072; + }, + "input_norm"); + ExpectLoadFailure( + fixture, + api, + [](json& manifest) { + auto& record = + manifest["initializers"].at("cos_cache"); + record["shape"] = {4095, 64}; + record["length"] = + 4095ull * 64ull * sizeof(std::uint16_t); + }, + "cos_cache"); + ExpectLoadFailure( + fixture, + api, + [](json& manifest) { + const std::string name = + manifest["weight_objects"][0]["roles"]["qweight"]; + auto& record = manifest["initializers"].at(name); + record["shape"] = {3071, 1536}; + record["length"] = 3071ull * 1536ull; + }, + "qweight"); + ExpectLoadFailure( + fixture, + api, + [](json& manifest) { + const std::string name = + manifest["weight_objects"][0]["roles"]["qweight"]; + manifest["initializers"][name]["role"] = "unknown.role"; + }, + "semantic role"); +} + +void TestComponentDiagnosticsIdentifyWeightObjects( + const SyntheticPackage& fixture, + const std::shared_ptr& api) { + std::string failures; + const auto verify = [&]( + std::string_view scenario, + auto mutation, + std::string_view expected) { + try { + ExpectLoadFailure( + fixture, + api, + std::move(mutation), + expected); + } catch (const std::exception& error) { + failures += "\n" + std::string(scenario) + ": " + error.what(); + } + }; + + verify( + "MatMul dtype", + [](json& manifest) { + constexpr std::size_t object_index = 7u * 5u + 1u; + const std::string initializer = + manifest["weight_objects"][object_index]["roles"]["scales"]; + auto& record = manifest["initializers"].at(initializer); + record["dtype"] = "uint8"; + record["length"] = 1024u * 24u; + }, + "model.layers.7.attn.k_proj.MatMulNBits.scales " + "(model.layers.7.attn.k_proj.MatMulNBits.scales)"); + verify( + "SSMLP projection shape", + [](json& manifest) { + constexpr std::size_t object_index = 19u * 5u + 4u; + const std::string initializer = + manifest["weight_objects"][object_index]["roles"] + ["up_scales"]; + auto& record = manifest["initializers"].at(initializer); + record["shape"] = {8191, 24}; + record["length"] = + 8191u * 24u * sizeof(std::uint16_t); + }, + "model.layers.19.ssmlp.up_scales " + "(model.layers.19.mlp.up_proj.MatMulNBits.scales)"); + verify( + "missing MatMul role", + [](json& manifest) { + constexpr std::size_t object_index = 12u * 5u + 2u; + manifest["weight_objects"][object_index]["roles"].erase( + "qzeros"); + }, + "model.layers.12.attn.v_proj.MatMulNBits.qzeros"); + verify( + "SSMLP norm shape", + [](json& manifest) { + constexpr std::size_t object_index = 27u * 5u + 4u; + const std::string initializer = + manifest["weight_objects"][object_index]["roles"]["norm0"]; + auto& record = manifest["initializers"].at(initializer); + record["shape"] = {3071}; + record["length"] = + 3071u * sizeof(std::uint16_t); + }, + "model.layers.27.ssmlp.norm0 " + "(model.layers.27.post_attention_layernorm.weight)"); + + if (!failures.empty()) { + throw std::runtime_error( + "component diagnostics did not identify their objects:" + + failures); + } +} + +void TestOwnedScaleAndNormConversions( + const SyntheticPackage& fixture, + const std::shared_ptr& api) { + fixture.Write(fixture.manifest()); + auto package = Phi4Package::Load(fixture.path(), api, false); + + // FP16 scales are an element-wise copy into a contiguous model-owned + // buffer, not a conversion. WEIGHT-2 still holds. + const auto fp16 = package.MaterializeFp16(kFp16Scale); + CHECK(fp16.size() == 1024u * 24u); + CHECK(fp16[0] == 0x3c00u); + CHECK(fp16[1] == 0xc000u); + CHECK( + reinterpret_cast(fp16.data()) != + reinterpret_cast( + package.Require(kFp16Scale).data)); + const auto* fp16_address = fp16.data(); + + const auto same_fp16 = package.MaterializeFp16(kFp16Scale); + CHECK(same_fp16.data() == fp16_address); + + // Design Section 9.3: an FP32 scales array is REJECTED, not narrowed. + // Narrowing it would need a host FP32-to-FP16 converter, which API-6 + // does not permit, so admitting one is a spec change. + CheckThrowsContains( + [&] { (void)package.MaterializeFp16(kFp32Scale); }, + "FP16"); + CheckThrowsContains( + [&] { (void)package.MaterializeFp16(kFp32Scale); }, + "rejected"); + CHECK(fp16.data() == fp16_address); + + // Norms are raw BF16 packer blobs, so they keep the API-6 host + // round-to-nearest-even helper. + const auto bf16 = package.MaterializeBf16(kFp32Norm); + CHECK(bf16.size() == 3072u); + CHECK(bf16[0] == 0x3f80u); + CHECK(bf16[1] == 0xc000u); + + CheckThrowsContains( + [&] { + (void)package.MaterializeFp16( + "model.layers.0.attn.q_proj.MatMulNBits.qweight"); + }, + "FP16"); + CheckThrowsContains( + [&] { + (void)package.MaterializeBf16( + "model.layers.0.attn.q_proj.MatMulNBits.qweight"); + }, + "floating"); +} + +// Corelib e5258d2 removed convert_strided, so this slice is now +// FastFlow's own code -- which makes the guard page more important, not +// less. The last source row sits immediately before an inaccessible page, +// and the gather must take its 48 columns without touching the tail. +void TestRopeGatherStaysInSourceDtypeAtGuardPage( + const SyntheticPackage& fixture, + const std::shared_ptr& api) { + fixture.Write(fixture.manifest()); + auto package = Phi4Package::Load(fixture.path(), api, false); + const auto& source = package.Require("cos_cache"); + CHECK(source.size == kRopeBytes); + CHECK(source.owner->size() == kRopeMappedBytes); + CHECK(source.data == source.owner->data()); + + auto* one_past = + const_cast(source.data + source.size); + NoAccessGuard guard(one_past); + + const auto rope = package.MaterializeRopeGather("cos_cache"); + // The gather preserves the SOURCE dtype: tensor_write does the + // widening to the FP32 device tensor, and this path performs no + // conversion of its own. + CHECK(rope.dtype == ryzenai_corelib_data_type_fp16); + CHECK(rope.count == 4096u * 48u); + const auto* elements = + static_cast(rope.data); + CHECK(elements[0] == 0x3c00u); + CHECK(elements[48] == 0x4000u); + CHECK(elements[4096u * 48u - 1u] == 0x4200u); + + const auto again = package.MaterializeRopeGather("cos_cache"); + CHECK(again.data == rope.data); + CHECK(again.dtype == rope.dtype); + CHECK(again.count == rope.count); +} + +void TestFp32RopeSource( + const SyntheticPackage& fixture, + const std::shared_ptr& api) { + fixture.Write(fixture.manifest()); + auto package = Phi4Package::Load(fixture.path(), api, false); + + const auto rope = package.MaterializeRopeGather("sin_cache"); + CHECK(rope.dtype == ryzenai_corelib_data_type_fp32); + CHECK(rope.count == 196608u); + CHECK(static_cast(rope.data)[0] == 4.0f); +} + +static_assert(!std::is_copy_constructible_v); +static_assert(!std::is_copy_assignable_v); +static_assert(std::is_nothrow_move_constructible_v); +static_assert(std::is_nothrow_move_assignable_v); + +} // namespace + +int main() { + try { + SyntheticPackage fixture; + auto api = ResolveRecordingCorelib(); + TestValidMappingAndExplicitRoles(fixture, api); + TestOgaQuantizedLayoutsAccepted(fixture, api); + TestMappedOwnerOutlivesPackage(fixture, api); + TestPathRangeAndHashRejections(fixture, api); + TestWeightObjectRejections(fixture, api); + TestRoleInitializerIdentityRejections(fixture, api); + TestExactSourceValidation(fixture, api); + TestComponentDiagnosticsIdentifyWeightObjects(fixture, api); + TestOwnedScaleAndNormConversions(fixture, api); + TestRopeGatherStaysInSourceDtypeAtGuardPage(fixture, api); + TestFp32RopeSource(fixture, api); + std::cout << "test_phi4_manifest: PASS\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/src/test/phi4_corelib_aie4/test_phi4_shape_plan.cpp b/src/test/phi4_corelib_aie4/test_phi4_shape_plan.cpp new file mode 100644 index 00000000..8027c4fa --- /dev/null +++ b/src/test/phi4_corelib_aie4/test_phi4_shape_plan.cpp @@ -0,0 +1,689 @@ +#include "fake_corelib.hpp" +#include "test_support.hpp" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using flm::corelib::CorelibApi; +using flm::phi4::Phi4ShapePlan; +using flm::phi4::RowUse; + +struct MatMulCall { + std::int64_t m; + std::int64_t k; + std::int64_t n; + std::uint32_t group_size; +}; + +struct SsMlpCall { + std::int64_t m; + std::int64_t k; + std::int64_t n; + std::uint32_t group_size; +}; + +struct AttentionCall { + std::int64_t m; + ryzenai_corelib_flat_mha_bf16_desc desc; +}; + +enum class HelperKind { + None, + MatMul, + SsMlp, + Attention +}; + +enum class FaultKind { + None, + Unsupported, + InvalidRows, + MutateK, + MutateN +}; + +struct Fault { + HelperKind helper = HelperKind::None; + FaultKind kind = FaultKind::None; + std::int64_t live_rows = 0; + std::int64_t matmul_n = 0; +}; + +struct HelperState { + std::vector matmul_calls; + std::vector ssmlp_calls; + std::vector attention_calls; + Fault fault; + + void Reset() { + matmul_calls.clear(); + ssmlp_calls.clear(); + attention_calls.clear(); + fault = {}; + } + + std::size_t TotalCalls() const noexcept { + return matmul_calls.size() + ssmlp_calls.size() + + attention_calls.size(); + } +}; + +HelperState g_helpers; + +bool Matches( + HelperKind helper, + std::int64_t live_rows, + std::int64_t matmul_n = 0) { + return g_helpers.fault.helper == helper && + g_helpers.fault.live_rows == live_rows && + (helper != HelperKind::MatMul || + g_helpers.fault.matmul_n == matmul_n); +} + +std::int64_t QueryProjectionRows(std::int64_t live_rows) { + if (live_rows == 1) { + return 1; + } + if (live_rows <= 32) { + return 32; + } + if (live_rows <= 512) { + return 512; + } + return 4096; +} + +std::int64_t KvProjectionRows(std::int64_t live_rows) { + if (live_rows == 1) { + return 1; + } + if (live_rows <= 64) { + return 64; + } + if (live_rows <= 1024) { + return 1024; + } + return 4160; +} + +std::int64_t SsMlpRows(std::int64_t live_rows) { + if (live_rows == 1) { + return 1; + } + if (live_rows <= 128) { + return 128; + } + if (live_rows <= 2048) { + return 2048; + } + return 4608; +} + +std::int64_t AttentionRows(std::int64_t live_rows) { + if (live_rows == 1) { + return 1; + } + if (live_rows <= 256) { + return 256; + } + if (live_rows <= 3072) { + return 3072; + } + return 4352; +} + +ryzenai_corelib_status RecordingMatMulPadShape( + std::int64_t* m, + std::int64_t* k, + std::int64_t* n, + std::uint32_t group_size) { + if (m == nullptr || k == nullptr || n == nullptr) { + return ryzenai_corelib_status_bad_argument; + } + + const MatMulCall call{*m, *k, *n, group_size}; + g_helpers.matmul_calls.push_back(call); + if (Matches(HelperKind::MatMul, call.m, call.n)) { + switch (g_helpers.fault.kind) { + case FaultKind::Unsupported: + return ryzenai_corelib_status_unsupported; + case FaultKind::InvalidRows: + *m = call.m - 1; + return ryzenai_corelib_status_success; + case FaultKind::MutateK: + ++*k; + return ryzenai_corelib_status_success; + case FaultKind::MutateN: + ++*n; + return ryzenai_corelib_status_success; + case FaultKind::None: + break; + } + } + + if (call.k != 3072 || call.group_size != 128) { + return ryzenai_corelib_status_bad_argument; + } + if (call.n == 3072) { + *m = QueryProjectionRows(call.m); + } else if (call.n == 1024) { + *m = KvProjectionRows(call.m); + } else if (call.n == 200064 && call.m == 1) { + *m = 1; + } else { + return ryzenai_corelib_status_bad_argument; + } + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status RecordingSsMlpPadRows( + std::int64_t* m, + std::int64_t k, + std::int64_t n, + std::uint32_t group_size) { + if (m == nullptr) { + return ryzenai_corelib_status_bad_argument; + } + + const SsMlpCall call{*m, k, n, group_size}; + g_helpers.ssmlp_calls.push_back(call); + if (Matches(HelperKind::SsMlp, call.m)) { + if (g_helpers.fault.kind == FaultKind::Unsupported) { + return ryzenai_corelib_status_unsupported; + } + if (g_helpers.fault.kind == FaultKind::InvalidRows) { + *m = call.m - 1; + return ryzenai_corelib_status_success; + } + } + + if (call.k != 3072 || call.n != 8192 || + call.group_size != 128) { + return ryzenai_corelib_status_bad_argument; + } + *m = SsMlpRows(call.m); + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status RecordingAttentionPadRows( + std::int64_t* m, + const ryzenai_corelib_flat_mha_bf16_desc* desc) { + if (m == nullptr || desc == nullptr) { + return ryzenai_corelib_status_bad_argument; + } + + const AttentionCall call{*m, *desc}; + g_helpers.attention_calls.push_back(call); + if (Matches(HelperKind::Attention, call.m)) { + if (g_helpers.fault.kind == FaultKind::Unsupported) { + return ryzenai_corelib_status_unsupported; + } + if (g_helpers.fault.kind == FaultKind::InvalidRows) { + *m = call.m - 1; + return ryzenai_corelib_status_success; + } + } + + if (desc->num_heads != 24 || desc->kv_num_heads != 8 || + desc->head_size != 128 || desc->max_seq != 4096 || + desc->rope_dim != 96) { + return ryzenai_corelib_status_bad_argument; + } + *m = AttentionRows(call.m); + return ryzenai_corelib_status_success; +} + +template +void* FunctionAddress(Function function) { + return reinterpret_cast(function); +} + +std::shared_ptr ResolveRecordingCorelib() { + auto resolver = flm::test::CompleteCorelibResolver(); + resolver["ryzenai_corelib_matmul_bf16_pad_shape"] = + FunctionAddress( + static_cast< + decltype(&::ryzenai_corelib_matmul_bf16_pad_shape)>( + &RecordingMatMulPadShape)); + resolver["ryzenai_corelib_ssmlp_bf16_pad_rows"] = + FunctionAddress( + static_cast< + decltype(&::ryzenai_corelib_ssmlp_bf16_pad_rows)>( + &RecordingSsMlpPadRows)); + resolver["ryzenai_corelib_flat_mha_bf16_pad_rows"] = + FunctionAddress( + static_cast< + decltype(&::ryzenai_corelib_flat_mha_bf16_pad_rows)>( + &RecordingAttentionPadRows)); + return CorelibApi::ResolveForTest( + [resolver = std::move(resolver)](std::string_view name) mutable + -> void* { + const auto found = resolver.find(std::string(name)); + return found == resolver.end() ? nullptr : found->second; + }); +} + +void CheckTransitions( + const std::vector>& actual, + std::initializer_list< + std::pair> expected) { + const std::vector> + expected_transitions(expected); + CHECK(actual == expected_transitions); +} + +template +void CheckThrowsEquals(Function&& function, std::string_view expected) { + try { + function(); + } catch (const std::exception& error) { + CHECK(std::string_view(error.what()) == expected); + return; + } + throw std::runtime_error("expected exception was not thrown"); +} + +void TestCompleteQueriesAndCachedTransitions( + const std::shared_ptr& api) { + g_helpers.Reset(); + const auto plan = Phi4ShapePlan::Build(api); + + CHECK(g_helpers.matmul_calls.size() == 8193); + std::array query_rows{}; + std::array kv_rows{}; + int lm_head_calls = 0; + for (const auto& call : g_helpers.matmul_calls) { + CHECK(call.k == 3072); + CHECK(call.group_size == 128); + if (call.n == 3072) { + CHECK(call.m >= 1 && call.m <= 4096); + ++query_rows[static_cast(call.m)]; + } else if (call.n == 1024) { + CHECK(call.m >= 1 && call.m <= 4096); + ++kv_rows[static_cast(call.m)]; + } else { + CHECK(call.n == 200064); + CHECK(call.m == 1); + ++lm_head_calls; + } + } + for (std::int64_t row = 1; row <= 4096; ++row) { + CHECK(query_rows[static_cast(row)] == 1); + CHECK(kv_rows[static_cast(row)] == 1); + } + CHECK(lm_head_calls == 1); + + CHECK(g_helpers.ssmlp_calls.size() == 4096); + CHECK(g_helpers.attention_calls.size() == 4096); + for (std::int64_t row = 1; row <= 4096; ++row) { + const auto index = static_cast(row - 1); + const auto& ssmlp = g_helpers.ssmlp_calls[index]; + CHECK(ssmlp.m == row); + CHECK(ssmlp.k == 3072); + CHECK(ssmlp.n == 8192); + CHECK(ssmlp.group_size == 128); + + const auto& attention = g_helpers.attention_calls[index]; + CHECK(attention.m == row); + CHECK(attention.desc.num_heads == 24); + CHECK(attention.desc.kv_num_heads == 8); + CHECK(attention.desc.head_size == 128); + CHECK(attention.desc.max_seq == 4096); + CHECK(attention.desc.rope_dim == 96); + } + + const auto& desc = plan.attention_desc(); + CHECK(desc.num_heads == 24); + CHECK(desc.kv_num_heads == 8); + CHECK(desc.head_size == 128); + CHECK(desc.max_seq == 4096); + CHECK(desc.rope_dim == 96); + + CHECK(plan.capacities().layer_rows == 4608); + CHECK(plan.capacities().lm_head_rows == 1); + + CheckTransitions( + plan.Transitions(RowUse::QueryProjection), + {{1, 1}, {2, 32}, {33, 512}, {513, 4096}}); + CheckTransitions( + plan.Transitions(RowUse::OutputProjection), + {{1, 1}, {2, 32}, {33, 512}, {513, 4096}}); + CheckTransitions( + plan.Transitions(RowUse::KvProjection), + {{1, 1}, {2, 64}, {65, 1024}, {1025, 4160}}); + CheckTransitions( + plan.Transitions(RowUse::SsMlp), + {{1, 1}, {2, 128}, {129, 2048}, {2049, 4608}}); + CheckTransitions( + plan.Transitions(RowUse::Attention), + {{1, 1}, {2, 256}, {257, 3072}, {3073, 4352}}); + CheckTransitions( + plan.Transitions(RowUse::LmHead), + {{1, 1}}); + + constexpr std::array uses{ + RowUse::QueryProjection, + RowUse::KvProjection, + RowUse::Attention, + RowUse::OutputProjection, + RowUse::SsMlp, + RowUse::LmHead}; + for (const auto use : uses) { + CHECK(plan.RowsFor(use, 1) == 1); + } + + const std::size_t calls_after_build = g_helpers.TotalCalls(); + CHECK(plan.RowsFor(RowUse::QueryProjection, 2) == 32); + CHECK(plan.RowsFor(RowUse::QueryProjection, 33) == 512); + CHECK(plan.RowsFor(RowUse::KvProjection, 4096) == 4160); + CHECK(plan.RowsFor(RowUse::Attention, 3072) == 3072); + CHECK(plan.RowsFor(RowUse::SsMlp, 2049) == 4608); + CHECK(plan.RowsFor(RowUse::LmHead, 1) == 1); + CHECK(g_helpers.TotalCalls() == calls_after_build); +} + +void TestMatMulDimensionMutationsRejectBuild( + const std::shared_ptr& api) { + struct Case { + FaultKind kind; + std::int64_t row; + std::int64_t n; + std::string_view message; + }; + constexpr std::array cases{{ + {FaultKind::MutateK, + 37, + 3072, + "query/output projection MatMul K/N mismatch at live row 37: " + "requested K=3072, N=3072; returned K=3073, N=3072"}, + {FaultKind::MutateN, + 93, + 1024, + "key/value projection MatMul K/N mismatch at live row 93: " + "requested K=3072, N=1024; returned K=3072, N=1025"}, + {FaultKind::MutateN, + 1, + 200064, + "LM head MatMul K/N mismatch at live row 1: " + "requested K=3072, N=200064; returned K=3072, N=200065"}, + }}; + + for (const auto& test_case : cases) { + g_helpers.Reset(); + g_helpers.fault = { + HelperKind::MatMul, + test_case.kind, + test_case.row, + test_case.n}; + CheckThrowsEquals( + [&] { + (void)Phi4ShapePlan::Build(api); + }, + test_case.message); + } +} + +void TestUnsupportedRowsRejectBuild( + const std::shared_ptr& api) { + struct Case { + HelperKind helper; + std::int64_t row; + std::int64_t n; + std::string_view call; + }; + constexpr std::array cases{{ + {HelperKind::MatMul, + 41, + 3072, + "ryzenai_corelib_matmul_bf16_pad_shape"}, + {HelperKind::SsMlp, + 43, + 0, + "ryzenai_corelib_ssmlp_bf16_pad_rows"}, + {HelperKind::Attention, + 47, + 0, + "ryzenai_corelib_flat_mha_bf16_pad_rows"}, + }}; + + for (const auto& test_case : cases) { + g_helpers.Reset(); + g_helpers.fault = { + test_case.helper, + FaultKind::Unsupported, + test_case.row, + test_case.n}; + CheckThrowsContains( + [&] { + (void)Phi4ShapePlan::Build(api); + }, + test_case.call); + } +} + +void TestInvalidPaddedRowsRejectBuild( + const std::shared_ptr& api) { + struct Case { + HelperKind helper; + std::int64_t row; + std::int64_t n; + std::string_view context; + }; + constexpr std::array cases{{ + {HelperKind::MatMul, 53, 3072, "query/output projection"}, + {HelperKind::SsMlp, 59, 0, "SSMLP"}, + {HelperKind::Attention, 61, 0, "attention"}, + {HelperKind::MatMul, 1, 200064, "LM head"}, + }}; + + for (const auto& test_case : cases) { + g_helpers.Reset(); + g_helpers.fault = { + test_case.helper, + FaultKind::InvalidRows, + test_case.row, + test_case.n}; + CheckThrowsContains( + [&] { + (void)Phi4ShapePlan::Build(api); + }, + test_case.context); + } +} + +void TestInvalidInputsRejectWithoutHelperCalls( + const std::shared_ptr& api) { + g_helpers.Reset(); + CheckThrowsContains( + [] { + (void)Phi4ShapePlan::Build(nullptr); + }, + "CorelibApi"); + CHECK(g_helpers.TotalCalls() == 0); + + const auto plan = Phi4ShapePlan::Build(api); + const auto calls_after_build = g_helpers.TotalCalls(); + CheckThrowsContains( + [&] { + (void)plan.RowsFor(RowUse::QueryProjection, 0); + }, + "live rows"); + CheckThrowsContains( + [&] { + (void)plan.RowsFor(RowUse::Attention, 4097); + }, + "live rows"); + CheckThrowsContains( + [&] { + (void)plan.RowsFor(RowUse::LmHead, 2); + }, + "live rows"); + CheckThrowsContains( + [&] { + (void)plan.Transitions(static_cast(99)); + }, + "RowUse"); + CHECK(g_helpers.TotalCalls() == calls_after_build); +} + +// Everything above drives Phi4ShapePlan through a synthetic padding grid, +// which is what lets it test the plan's own logic. This one does the +// opposite: it builds against the UNMODIFIED fake, whose pad helpers encode +// the grid measured from the real e5258d2 library. +// +// It exists because that grid was otherwise consumed only by +// test_real_corelib, which skips without a runtime directory. The two +// together form a chain -- test_real_corelib asserts fake == library, and +// this asserts fake == the numbers written down here -- so a regression in +// either end fails something that runs by default. +std::shared_ptr ResolveUnmodifiedFake() { + auto resolver = flm::test::CompleteCorelibResolver(); + return CorelibApi::ResolveForTest( + [resolver = std::move(resolver)](std::string_view name) mutable + -> void* { + const auto found = resolver.find(std::string(name)); + return found == resolver.end() ? nullptr : found->second; + }); +} + +void TestFakeReproducesTheShippedKernelGrid() { + auto api = ResolveUnmodifiedFake(); + const Phi4ShapePlan plan = Phi4ShapePlan::Build(api); + + const std::vector> expected{ + {1, 1}, + {2, 64}, + {65, 128}, + {129, 256}, + {257, 512}, + {513, 1024}, + {1025, 2048}, + {2049, 3072}, + {3073, 4096}}; + for (const RowUse use : { + RowUse::QueryProjection, + RowUse::KvProjection, + RowUse::Attention, + RowUse::OutputProjection, + RowUse::SsMlp}) { + CHECK(plan.Transitions(use) == expected); + } + CHECK( + plan.Transitions(RowUse::LmHead) == + (std::vector>{{1, 1}})); + CHECK(plan.capacities().layer_rows == 4096); + CHECK(plan.capacities().lm_head_rows == 1); + + // Decode stays unpadded on every helper. + for (const RowUse use : { + RowUse::QueryProjection, + RowUse::KvProjection, + RowUse::Attention, + RowUse::OutputProjection, + RowUse::SsMlp, + RowUse::LmHead}) { + CHECK(plan.RowsFor(use, 1) == 1); + } + CHECK(plan.RowsFor(RowUse::SsMlp, 2) == 64); + CHECK(plan.RowsFor(RowUse::Attention, 64) == 64); + CHECK(plan.RowsFor(RowUse::QueryProjection, 65) == 128); + CHECK(plan.RowsFor(RowUse::QueryProjection, 3072) == 3072); + CHECK(plan.RowsFor(RowUse::QueryProjection, 4096) == 4096); +} + +// The LM head ships M = 1 and M = 128 and refuses anything larger rather +// than rounding up. Phi4ShapePlan never asks for more, so this branch of +// the fake had no other caller -- and an unexercised branch is not a model +// of the library, it is dead code that happens to be written down. +void TestFakeRefusesOutOfGridLmHeadRows() { + auto api = ResolveUnmodifiedFake(); + const auto& functions = api->functions(); + + for (const auto& expectation : std::vector< + std::pair>{ + {1, 1}, + {2, 128}, + {128, 128}}) { + std::int64_t m = expectation.first; + std::int64_t k = 3072; + std::int64_t n = 200064; + CHECK( + functions.matmul_pad_shape(&m, &k, &n, 128) == + ryzenai_corelib_status_success); + CHECK(m == expectation.second); + // K and N are never padded; MEM-5 rests on that. + CHECK(k == 3072); + CHECK(n == 200064); + } + + for (const std::int64_t rows : + {std::int64_t{129}, std::int64_t{256}, std::int64_t{4096}}) { + std::int64_t m = rows; + std::int64_t k = 3072; + std::int64_t n = 200064; + CHECK( + functions.matmul_pad_shape(&m, &k, &n, 128) == + ryzenai_corelib_status_unsupported); + } + + // The layer shapes have no such ceiling: 4096 is on their grid. + std::int64_t m = 4096; + std::int64_t k = 3072; + std::int64_t n = 3072; + CHECK( + functions.matmul_pad_shape(&m, &k, &n, 128) == + ryzenai_corelib_status_success); + CHECK(m == 4096); + + // And nothing beyond the grid is silently accepted anywhere. + m = 4097; + CHECK( + functions.matmul_pad_shape(&m, &k, &n, 128) == + ryzenai_corelib_status_unsupported); + m = 4097; + CHECK( + functions.ssmlp_pad_rows(&m, 3072, 8192, 128) == + ryzenai_corelib_status_unsupported); + const ryzenai_corelib_flat_mha_bf16_desc desc{ + 24, + 8, + 128, + 4096, + 96}; + m = 4097; + CHECK( + functions.flat_mha_pad_rows(&m, &desc) == + ryzenai_corelib_status_unsupported); +} + +} // namespace + +int main() { + try { + auto api = ResolveRecordingCorelib(); + TestCompleteQueriesAndCachedTransitions(api); + TestMatMulDimensionMutationsRejectBuild(api); + TestUnsupportedRowsRejectBuild(api); + TestInvalidPaddedRowsRejectBuild(api); + TestInvalidInputsRejectWithoutHelperCalls(api); + TestFakeReproducesTheShippedKernelGrid(); + TestFakeRefusesOutOfGridLmHeadRows(); + std::cout << "test_phi4_shape_plan: PASS\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/src/test/phi4_corelib_aie4/test_phi4_weights.cpp b/src/test/phi4_corelib_aie4/test_phi4_weights.cpp new file mode 100644 index 00000000..506cda57 --- /dev/null +++ b/src/test/phi4_corelib_aie4/test_phi4_weights.cpp @@ -0,0 +1,1245 @@ +#include "fake_corelib.hpp" +#include "test_support.hpp" + +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using flm::corelib::CorelibApi; +using flm::corelib::CorelibError; +using flm::phi4::Phi4Package; +using flm::phi4::Phi4Weights; +using flm::phi4::WeightObjectKind; +using nlohmann::json; + +namespace constants = flm::phi4::constants; + +constexpr std::string_view kManifestName = + "corelib_phi4_manifest.json"; +constexpr std::string_view kDataFile = "weights.bin"; +constexpr std::uint64_t kDataBytes = + 200064ull * 3072ull * sizeof(std::uint16_t); +constexpr std::size_t kMatMulPackedBytes = 17; +constexpr std::size_t kSsMlpPackedBytes = 29; +constexpr std::uint16_t kBf16Epsilon = 0x3728u; + +enum class FailurePoint { + None, + MatMulCreate, + MatMulGetData, + SsMlpCreate, + SsMlpGetData +}; + +struct FakeWeightHandle { + std::weak_ptr package; +}; + +struct MatMulCreateRecord { + ryzenai_corelib_matmul_bf16_weights_desc desc{}; + ryzenai_corelib_matmul_bf16_onnx_components components{}; + std::uint32_t threads = 0; + void* object = nullptr; + std::thread::id thread; +}; + +struct SsMlpCreateRecord { + ryzenai_corelib_ssmlp_bf16_weights_desc desc{}; + ryzenai_corelib_ssmlp_bf16_onnx_components components{}; + std::uint32_t threads = 0; + void* object = nullptr; + std::thread::id thread; +}; + +struct RecordingState { + std::mutex mutex; + std::weak_ptr current_package; + FailurePoint failure = FailurePoint::None; + std::size_t failure_ordinal = 1; + std::size_t matmul_create_attempts = 0; + std::size_t matmul_get_attempts = 0; + std::size_t ssmlp_create_attempts = 0; + std::size_t ssmlp_get_attempts = 0; + std::vector matmul_creates; + std::vector ssmlp_creates; + std::vector creation_order; + std::vector release_order; + std::vector package_alive_at_release; + std::vector events; + std::size_t get_data_calls = 0; + bool every_get_data_pointer_argument_was_null = true; + bool every_get_data_size_argument_was_nonnull = true; + std::size_t matmul_packed_bytes = kMatMulPackedBytes; + std::size_t ssmlp_packed_bytes = kSsMlpPackedBytes; +}; + +RecordingState* g_recording = nullptr; + +RecordingState& State() { + if (g_recording == nullptr) { + throw std::runtime_error("recording corelib is not active"); + } + return *g_recording; +} + +bool ShouldFail( + FailurePoint configured, + FailurePoint current, + std::size_t ordinal) { + return configured == current && + ordinal == State().failure_ordinal; +} + +ryzenai_corelib_status RecordingMatMulCreate( + const ryzenai_corelib_matmul_bf16_weights_desc* desc, + const ryzenai_corelib_matmul_bf16_onnx_components* components, + uint32_t threads, + ryzenai_corelib_matmul_bf16_weights_ptr* out) { + if (desc == nullptr || components == nullptr || out == nullptr) { + return ryzenai_corelib_status_bad_argument; + } + *out = nullptr; + + auto& state = State(); + std::lock_guard lock(state.mutex); + const std::size_t ordinal = ++state.matmul_create_attempts; + state.events.emplace_back("matmul_create"); + if (ShouldFail( + state.failure, + FailurePoint::MatMulCreate, + ordinal)) { + flm::test::SetLastErrorMessage( + "intentional Task 6 MatMul create failure"); + return ryzenai_corelib_status_unsupported; + } + + auto* handle = new FakeWeightHandle{state.current_package}; + *out = handle; + state.matmul_creates.push_back( + {*desc, + *components, + threads, + handle, + std::this_thread::get_id()}); + state.creation_order.push_back(handle); + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status RecordingMatMulGetData( + ryzenai_corelib_matmul_bf16_weights_ptr weights, + const void** data, + std::size_t* size) { + auto& state = State(); + std::lock_guard lock(state.mutex); + const std::size_t ordinal = ++state.matmul_get_attempts; + state.events.emplace_back("matmul_get_data"); + ++state.get_data_calls; + state.every_get_data_pointer_argument_was_null = + state.every_get_data_pointer_argument_was_null && + data == nullptr; + state.every_get_data_size_argument_was_nonnull = + state.every_get_data_size_argument_was_nonnull && + size != nullptr; + if ( + weights == nullptr || data != nullptr || size == nullptr) { + return ryzenai_corelib_status_bad_argument; + } + if (ShouldFail( + state.failure, + FailurePoint::MatMulGetData, + ordinal)) { + flm::test::SetLastErrorMessage( + "intentional Task 6 MatMul get-data failure"); + return ryzenai_corelib_status_unsupported; + } + *size = state.matmul_packed_bytes; + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status RecordingSsMlpCreate( + const ryzenai_corelib_ssmlp_bf16_weights_desc* desc, + const ryzenai_corelib_ssmlp_bf16_onnx_components* components, + uint32_t threads, + ryzenai_corelib_ssmlp_bf16_weights_ptr* out) { + if (desc == nullptr || components == nullptr || out == nullptr) { + return ryzenai_corelib_status_bad_argument; + } + *out = nullptr; + + auto& state = State(); + std::lock_guard lock(state.mutex); + const std::size_t ordinal = ++state.ssmlp_create_attempts; + state.events.emplace_back("ssmlp_create"); + if (ShouldFail( + state.failure, + FailurePoint::SsMlpCreate, + ordinal)) { + flm::test::SetLastErrorMessage( + "intentional Task 6 SSMLP create failure"); + return ryzenai_corelib_status_unsupported; + } + + auto* handle = new FakeWeightHandle{state.current_package}; + *out = handle; + state.ssmlp_creates.push_back( + {*desc, + *components, + threads, + handle, + std::this_thread::get_id()}); + state.creation_order.push_back(handle); + return ryzenai_corelib_status_success; +} + +ryzenai_corelib_status RecordingSsMlpGetData( + ryzenai_corelib_ssmlp_bf16_weights_ptr weights, + const void** data, + std::size_t* size) { + auto& state = State(); + std::lock_guard lock(state.mutex); + const std::size_t ordinal = ++state.ssmlp_get_attempts; + state.events.emplace_back("ssmlp_get_data"); + ++state.get_data_calls; + state.every_get_data_pointer_argument_was_null = + state.every_get_data_pointer_argument_was_null && + data == nullptr; + state.every_get_data_size_argument_was_nonnull = + state.every_get_data_size_argument_was_nonnull && + size != nullptr; + if ( + weights == nullptr || data != nullptr || size == nullptr) { + return ryzenai_corelib_status_bad_argument; + } + if (ShouldFail( + state.failure, + FailurePoint::SsMlpGetData, + ordinal)) { + flm::test::SetLastErrorMessage( + "intentional Task 6 SSMLP get-data failure"); + return ryzenai_corelib_status_unsupported; + } + *size = state.ssmlp_packed_bytes; + return ryzenai_corelib_status_success; +} + +void RecordingRelease(ryzenai_corelib_object_ptr object) { + auto* handle = static_cast(object); + auto& state = State(); + { + std::lock_guard lock(state.mutex); + state.release_order.push_back(object); + state.package_alive_at_release.push_back( + !handle->package.expired()); + state.events.emplace_back("release"); + } + delete handle; +} + +template +void* FunctionAddress(Function function) { + return reinterpret_cast(function); +} + +std::shared_ptr ResolveRecordingCorelib( + RecordingState& state) { + g_recording = &state; + auto resolver = flm::test::CompleteCorelibResolver(); + resolver["ryzenai_corelib_object_release"] = FunctionAddress( + static_cast( + &RecordingRelease)); + resolver["ryzenai_corelib_matmul_bf16_weights_create_onnx"] = + FunctionAddress( + static_cast( + &RecordingMatMulCreate)); + resolver["ryzenai_corelib_matmul_bf16_weights_get_data"] = + FunctionAddress( + static_cast( + &RecordingMatMulGetData)); + resolver["ryzenai_corelib_ssmlp_bf16_weights_create_onnx"] = + FunctionAddress( + static_cast( + &RecordingSsMlpCreate)); + resolver["ryzenai_corelib_ssmlp_bf16_weights_get_data"] = + FunctionAddress( + static_cast( + &RecordingSsMlpGetData)); + return CorelibApi::ResolveForTest( + [resolver = std::move(resolver)](std::string_view name) mutable + -> void* { + const auto found = resolver.find(std::string(name)); + return found == resolver.end() ? nullptr : found->second; + }); +} + +class TempDirectory final { +public: + TempDirectory() { + const auto nonce = + std::chrono::steady_clock::now().time_since_epoch().count(); + path_ = std::filesystem::temp_directory_path() / + ("fastflowlm-phi4-weights-" + + std::to_string(GetCurrentProcessId()) + "-" + + std::to_string(nonce)); + std::filesystem::create_directories(path_); + } + + ~TempDirectory() noexcept { + std::error_code error; + std::filesystem::remove_all(path_, error); + } + + TempDirectory(const TempDirectory&) = delete; + TempDirectory& operator=(const TempDirectory&) = delete; + + const std::filesystem::path& path() const noexcept { + return path_; + } + +private: + std::filesystem::path path_; +}; + +void CreateSparseFile( + const std::filesystem::path& path, + std::uint64_t size) { + HANDLE file = CreateFileW( + path.c_str(), + GENERIC_READ | GENERIC_WRITE, + FILE_SHARE_READ, + nullptr, + CREATE_ALWAYS, + FILE_ATTRIBUTE_NORMAL, + nullptr); + if (file == INVALID_HANDLE_VALUE) { + throw std::runtime_error("failed to create sparse weight file"); + } + + DWORD ignored = 0; + if ( + DeviceIoControl( + file, + FSCTL_SET_SPARSE, + nullptr, + 0, + nullptr, + 0, + &ignored, + nullptr) == FALSE) { + CloseHandle(file); + throw std::runtime_error( + "test volume does not support sparse files"); + } + + LARGE_INTEGER end{}; + end.QuadPart = static_cast(size); + const bool success = + SetFilePointerEx(file, end, nullptr, FILE_BEGIN) != FALSE && + SetEndOfFile(file) != FALSE; + CloseHandle(file); + if (!success) { + throw std::runtime_error("failed to size sparse weight file"); + } +} + +std::uint64_t ItemSize(std::string_view dtype) { + if (dtype == "uint8") { + return 1; + } + if (dtype == "float16") { + return 2; + } + if (dtype == "float32") { + return 4; + } + throw std::runtime_error("unsupported synthetic dtype"); +} + +std::uint64_t ByteLength( + std::string_view dtype, + const std::vector& shape) { + std::uint64_t elements = 1; + for (const std::int64_t dimension : shape) { + elements *= static_cast(dimension); + } + return elements * ItemSize(dtype); +} + +void AddInitializer( + json& initializers, + const std::string& name, + std::string dtype, + std::vector shape, + std::string role, + std::uint64_t offset) { + CHECK(!initializers.contains(name)); + initializers[name] = { + {"file", std::string(kDataFile)}, + {"offset", offset}, + {"length", ByteLength(dtype, shape)}, + {"dtype", std::move(dtype)}, + {"shape", std::move(shape)}, + {"role", std::move(role)}}; +} + +class ManifestBuilder final { +public: + ManifestBuilder(std::uint64_t model_size, bool fp32_q_proj_scales) + : fp32_q_proj_scales_(fp32_q_proj_scales), + manifest_{ + {"schema_version", 1}, + {"execution_backend", "corelib_aie4"}, + {"model", + { + {"family", "phi4"}, + {"layers", 32}, + {"hidden_size", 3072}, + {"intermediate_size", 8192}, + {"num_heads", 24}, + {"kv_heads", 8}, + {"head_size", 128}, + {"vocab_size", 200064}, + {"group_size", 128}, + {"rope_dim", 96}, + {"rms_epsilon", 0.00001}, + }}, + {"backend", {{"max_seq", 4096}}}, + {"files", + { + {"model.onnx", {{"size", model_size}}}, + {std::string(kDataFile), {{"size", kDataBytes}}}, + }}, + {"initializers", json::object()}, + {"weight_objects", json::array()}} {} + + void AddMatMul( + const std::string& name, + std::int64_t k, + std::int64_t n) { + const std::string qweight = name + ".qweight"; + const std::string scales = name + ".scales"; + const std::string qzeros = name + ".qzeros"; + manifest_["weight_objects"].push_back({ + {"name", name}, + {"kind", "matmul"}, + {"descriptor", + { + {"k", k}, + {"n", n}, + {"group_size", 128}, + {"has_bias", false}, + }}, + {"roles", + { + {"qweight", qweight}, + {"scales", scales}, + {"qzeros", qzeros}, + }}}); + + AddInitializer( + manifest_["initializers"], + qweight, + "uint8", + {n, k / 2}, + "matmul.qweight", + NextOffset()); + AddInitializer( + manifest_["initializers"], + scales, + (fp32_q_proj_scales_ && + name == "model.layers.0.attn.q_proj.MatMulNBits") + ? "float32" + : "float16", + {n, k / 128}, + "matmul.scales", + NextOffset()); + AddInitializer( + manifest_["initializers"], + qzeros, + "uint8", + {n, ((k / 128) + 1) / 2}, + "matmul.qzeros", + NextOffset()); + } + + void AddSsMlp(int layer) { + const std::string base = + "model.layers." + std::to_string(layer); + const std::string object_name = base + ".ssmlp"; + const std::string norm0 = + base + ".post_attention_layernorm.weight"; + const std::string norm1 = + layer == 31 + ? "model.layers.32.final_norm_layernorm.weight" + : "model.layers." + std::to_string(layer + 1) + + ".input_layernorm.weight"; + json roles = { + {"norm0", norm0}, + {"norm1", norm1}, + }; + + for (const std::string projection : {"gate", "up", "down"}) { + const std::int64_t k = + projection == "down" ? 8192 : 3072; + const std::int64_t n = + projection == "down" ? 3072 : 8192; + const std::string prefix = + base + ".mlp." + projection + + "_proj.MatMulNBits"; + const std::string role_prefix = + "ssmlp." + projection; + for (const std::string component : + {"qweight", "scales", "qzeros"}) { + roles[projection + "_" + component] = + prefix + "." + component; + } + AddInitializer( + manifest_["initializers"], + prefix + ".qweight", + "uint8", + {n, k / 2}, + role_prefix + ".qweight", + NextOffset()); + AddInitializer( + manifest_["initializers"], + prefix + ".scales", + "float16", + {n, k / 128}, + role_prefix + ".scales", + NextOffset()); + AddInitializer( + manifest_["initializers"], + prefix + ".qzeros", + "uint8", + {n, ((k / 128) + 1) / 2}, + role_prefix + ".qzeros", + NextOffset()); + } + + AddInitializer( + manifest_["initializers"], + norm0, + layer == 0 ? "float32" : "float16", + {3072}, + "ssmlp.norm0", + NextOffset()); + AddInitializer( + manifest_["initializers"], + norm1, + "float16", + {3072}, + "ssmlp.norm1", + NextOffset()); + manifest_["weight_objects"].push_back({ + {"name", object_name}, + {"kind", "ssmlp"}, + {"descriptor", + { + {"k", 3072}, + {"n", 8192}, + {"group_size", 128}, + }}, + {"roles", std::move(roles)}}); + } + + json Finish() { + for (int layer = 0; layer < 32; ++layer) { + const std::string base = + "model.layers." + std::to_string(layer) + + ".attn."; + AddMatMul( + base + "q_proj.MatMulNBits", + 3072, + 3072); + AddMatMul( + base + "k_proj.MatMulNBits", + 3072, + 1024); + AddMatMul( + base + "v_proj.MatMulNBits", + 3072, + 1024); + AddMatMul( + base + "o_proj.MatMulNBits", + 3072, + 3072); + AddSsMlp(layer); + } + AddMatMul( + "lm_head.MatMulNBits", + 3072, + 200064); + + AddInitializer( + manifest_["initializers"], + "model.embed_tokens.weight", + "float16", + {200064, 3072}, + "embedding", + 0); + AddInitializer( + manifest_["initializers"], + "model.layers.0.input_layernorm.weight", + "float16", + {3072}, + "input_norm", + NextOffset()); + AddInitializer( + manifest_["initializers"], + "cos_cache", + "float16", + {4096, 48}, + "cos_cache", + NextOffset()); + AddInitializer( + manifest_["initializers"], + "sin_cache", + "float32", + {4096, 48}, + "sin_cache", + NextOffset()); + + CHECK(manifest_["weight_objects"].size() == 161); + CHECK(manifest_["initializers"].size() == 743); + return std::move(manifest_); + } + +private: + std::uint64_t NextOffset() { + const std::uint64_t result = next_offset_; + next_offset_ += 16; + return result; + } + + bool fp32_q_proj_scales_ = false; + json manifest_; + std::uint64_t next_offset_ = 4096; +}; + +class SyntheticPackage final { +public: + explicit SyntheticPackage(bool fp32_q_proj_scales = false) { + const auto model_path = temp_.path() / "model.onnx"; + { + std::ofstream model(model_path, std::ios::binary); + model << "model"; + } + CreateSparseFile(temp_.path() / kDataFile, kDataBytes); + ManifestBuilder builder( + std::filesystem::file_size(model_path), + fp32_q_proj_scales); + std::ofstream manifest( + temp_.path() / kManifestName, + std::ios::binary); + manifest << builder.Finish().dump(2) << '\n'; + if (!manifest) { + throw std::runtime_error( + "failed to write synthetic weight manifest"); + } + } + + std::shared_ptr Load( + const std::shared_ptr& api) const { + return std::make_shared( + Phi4Package::Load(temp_.path(), api, false)); + } + +private: + TempDirectory temp_; +}; + +const std::string& Role( + const flm::phi4::WeightObjectView& object, + std::string_view role) { + return object.components.at(std::string(role)); +} + +void CheckMatMulComponents( + const MatMulCreateRecord& actual, + const flm::phi4::WeightObjectView& expected, + Phi4Package& package) { + CHECK(expected.kind == WeightObjectKind::MatMul); + CHECK(actual.desc.k == expected.k); + CHECK(actual.desc.n == expected.n); + CHECK(actual.desc.group_size == 128); + CHECK(actual.desc.has_bias == false); + + const auto& qweight = package.Require(Role(expected, "qweight")); + const auto scales = + package.MaterializeFp16(Role(expected, "scales")); + const auto& qzeros = package.Require(Role(expected, "qzeros")); + CHECK(actual.components.qweight == qweight.data); + CHECK(actual.components.scales == scales.data()); + CHECK(actual.components.qzeros == qzeros.data); + CHECK(actual.components.scales != + package.Require(Role(expected, "scales")).data); + CHECK( + *static_cast(actual.components.qweight) == + std::byte{0}); + CHECK( + *static_cast(actual.components.qzeros) == + std::byte{0}); + // The synthetic data file is sparse, so a faithful element-wise copy + // reproduces its zeros. The old expectation was an artifact of the + // recording converter writing a constant. + CHECK( + *static_cast(actual.components.scales) == + 0x0000u); + CHECK(actual.threads == 0u); +} + +void CheckSsMlpComponents( + const SsMlpCreateRecord& actual, + const flm::phi4::WeightObjectView& expected, + Phi4Package& package) { + CHECK(expected.kind == WeightObjectKind::SsMlp); + CHECK(actual.desc.k == 3072); + CHECK(actual.desc.n == 8192); + CHECK(actual.desc.group_size == 128); + CHECK(actual.threads == 0u); + CHECK(actual.components.epsilon != nullptr); + // The API-6 host round-to-nearest-even helper, on 1e-5f. + CHECK( + *static_cast( + actual.components.epsilon) == kBf16Epsilon); + + const auto check_norm = [&]( + const void* actual_pointer, + std::string_view role) { + const std::string& name = Role(expected, role); + const auto value = package.MaterializeBf16(name); + CHECK(actual_pointer == value.data()); + CHECK(actual_pointer != package.Require(name).data); + CHECK( + *static_cast(actual_pointer) == + 0x0000u); + }; + const auto check_projection = [&]( + const void* qweight, + const void* scales, + const void* qzeros, + std::string_view prefix) { + const std::string qweight_role = + std::string(prefix) + "_qweight"; + const std::string scales_role = + std::string(prefix) + "_scales"; + const std::string qzeros_role = + std::string(prefix) + "_qzeros"; + CHECK(qweight == + package.Require(Role(expected, qweight_role)).data); + CHECK(scales == + package.MaterializeFp16( + Role(expected, scales_role)) + .data()); + CHECK(qzeros == + package.Require(Role(expected, qzeros_role)).data); + CHECK(scales != + package.Require(Role(expected, scales_role)).data); + }; + + check_norm(actual.components.norm0, "norm0"); + check_norm(actual.components.norm1, "norm1"); + check_projection( + actual.components.gate_qweight, + actual.components.gate_scales, + actual.components.gate_qzeros, + "gate"); + check_projection( + actual.components.up_qweight, + actual.components.up_scales, + actual.components.up_qzeros, + "up"); + check_projection( + actual.components.down_qweight, + actual.components.down_scales, + actual.components.down_qzeros, + "down"); +} + +void CheckImmediateSizeQueries(const RecordingState& state) { + CHECK(state.events.size() == 322); + for (std::size_t index = 0; index < state.events.size(); index += 2) { + const bool matmul = + state.events[index] == "matmul_create"; + CHECK(matmul || state.events[index] == "ssmlp_create"); + CHECK( + state.events[index + 1] == + (matmul ? "matmul_get_data" : "ssmlp_get_data")); + } +} + +void CheckReleaseOrder( + const std::vector& creation_order, + const std::vector& release_order) { + CHECK(creation_order.size() == release_order.size()); + CHECK(std::equal( + release_order.begin(), + release_order.end(), + creation_order.rbegin(), + creation_order.rend())); +} + +void TestExactConstructionAndLifetime( + const SyntheticPackage& fixture) { + RecordingState state; + flm::test::ResetFakeCorelib(); + auto api = ResolveRecordingCorelib(state); + auto package = fixture.Load(api); + state.current_package = package; + const std::weak_ptr package_lifetime = package; + const std::thread::id load_thread = std::this_thread::get_id(); + + { + Phi4Weights weights = Phi4Weights::Load(api, package); + CHECK(weights.layers().size() == 32); + CHECK(state.matmul_creates.size() == 32u * 4u + 1u); + CHECK(state.ssmlp_creates.size() == 32u); + CHECK(state.get_data_calls == 161u); + CHECK(state.every_get_data_pointer_argument_was_null); + CHECK(state.every_get_data_size_argument_was_nonnull); + CHECK(weights.packed_bytes() == + 129u * kMatMulPackedBytes + + 32u * kSsMlpPackedBytes); + CHECK(api->live_object_count() == 161); + CheckImmediateSizeQueries(state); + + const auto& objects = package->weight_objects(); + CHECK(objects.size() == 161); + for (std::size_t layer = 0; layer < 32; ++layer) { + const std::size_t object_base = layer * 5; + const std::size_t matmul_base = layer * 4; + for (std::size_t projection = 0; + projection < 4; + ++projection) { + CheckMatMulComponents( + state.matmul_creates.at( + matmul_base + projection), + objects.at(object_base + projection), + *package); + } + CheckSsMlpComponents( + state.ssmlp_creates.at(layer), + objects.at(object_base + 4), + *package); + CHECK(weights.layers()[layer].q); + CHECK(weights.layers()[layer].k); + CHECK(weights.layers()[layer].v); + CHECK(weights.layers()[layer].o); + CHECK(weights.layers()[layer].mlp); + } + CheckMatMulComponents( + state.matmul_creates.back(), + objects.back(), + *package); + CHECK(weights.lm_head()); + CHECK(state.matmul_creates.front().desc.has_bias == false); + CHECK(state.matmul_creates.back().desc.n == 200064); + CHECK( + Role(objects.at(31u * 5u + 4u), "norm1") == + "model.layers.32.final_norm_layernorm.weight"); + CHECK( + state.ssmlp_creates.back().components.norm1 == + package + ->MaterializeBf16( + "model.layers.32.final_norm_layernorm.weight") + .data()); + + // Every packing call takes the header's "one thread" hint: design + // Section 19 defers concurrent packing to the caller. + CHECK(std::all_of( + state.matmul_creates.begin(), + state.matmul_creates.end(), + [](const MatMulCreateRecord& call) { + return call.threads == 0u; + })); + CHECK(std::all_of( + state.ssmlp_creates.begin(), + state.ssmlp_creates.end(), + [](const SsMlpCreateRecord& call) { + return call.threads == 0u; + })); + CHECK(std::all_of( + state.matmul_creates.begin(), + state.matmul_creates.end(), + [load_thread](const MatMulCreateRecord& call) { + return call.thread == load_thread; + })); + CHECK(std::all_of( + state.ssmlp_creates.begin(), + state.ssmlp_creates.end(), + [load_thread](const SsMlpCreateRecord& call) { + return call.thread == load_thread; + })); + + const std::vector creation_order = + state.creation_order; + Phi4Weights moved(std::move(weights)); + CHECK(state.release_order.empty()); + CHECK(api->live_object_count() == 161); + package.reset(); + CHECK(!package_lifetime.expired()); + + auto retained = package_lifetime.lock(); + CHECK(retained != nullptr); + const auto& retained_objects = retained->weight_objects(); + for (std::size_t layer = 0; layer < 32; ++layer) { + for (std::size_t projection = 0; + projection < 4; + ++projection) { + CheckMatMulComponents( + state.matmul_creates.at( + layer * 4 + projection), + retained_objects.at( + layer * 5 + projection), + *retained); + } + CheckSsMlpComponents( + state.ssmlp_creates.at(layer), + retained_objects.at(layer * 5 + 4), + *retained); + } + CheckMatMulComponents( + state.matmul_creates.back(), + retained_objects.back(), + *retained); + retained.reset(); + CHECK(!package_lifetime.expired()); + (void)moved; + CHECK(creation_order.size() == 161); + } + + CHECK(state.release_order.size() == 161); + CHECK(std::all_of( + state.package_alive_at_release.begin(), + state.package_alive_at_release.end(), + [](bool value) { return value; })); + CheckReleaseOrder(state.creation_order, state.release_order); + CHECK(package_lifetime.expired()); + CHECK(api->live_object_count() == 0); + g_recording = nullptr; +} + +void TestMoveAssignmentReleasesBeforeOwners( + const SyntheticPackage& fixture) { + RecordingState state; + flm::test::ResetFakeCorelib(); + auto api = ResolveRecordingCorelib(state); + auto source_package = fixture.Load(api); + auto destination_package = fixture.Load(api); + const std::weak_ptr source_lifetime = source_package; + const std::weak_ptr destination_lifetime = + destination_package; + + { + state.current_package = source_package; + Phi4Weights source = + Phi4Weights::Load(api, source_package); + const std::vector source_objects = + state.creation_order; + + state.current_package = destination_package; + Phi4Weights destination = + Phi4Weights::Load(api, destination_package); + const std::vector destination_objects( + state.creation_order.begin() + 161, + state.creation_order.end()); + CHECK(source_objects.size() == 161); + CHECK(destination_objects.size() == 161); + CHECK(api->live_object_count() == 322); + + source_package.reset(); + destination_package.reset(); + destination = std::move(source); + + CHECK(state.release_order.size() == 161); + CheckReleaseOrder( + destination_objects, + state.release_order); + CHECK(destination_lifetime.expired()); + CHECK(!source_lifetime.expired()); + CHECK(api->live_object_count() == 161); + } + + CHECK(state.release_order.size() == 322); + const std::vector source_releases( + state.release_order.begin() + 161, + state.release_order.end()); + const std::vector source_objects( + state.creation_order.begin(), + state.creation_order.begin() + 161); + CheckReleaseOrder(source_objects, source_releases); + CHECK(std::all_of( + state.package_alive_at_release.begin(), + state.package_alive_at_release.end(), + [](bool value) { return value; })); + CHECK(source_lifetime.expired()); + CHECK(api->live_object_count() == 0); + g_recording = nullptr; +} + +void CheckLoadFailure( + RecordingState& state, + const std::shared_ptr& api, + std::shared_ptr package, + FailurePoint failure, + std::size_t failure_ordinal, + std::size_t expected_created, + std::string_view object_name, + std::string_view call, + std::string_view detail) { + state.failure = failure; + state.failure_ordinal = failure_ordinal; + state.matmul_create_attempts = 0; + state.matmul_get_attempts = 0; + state.ssmlp_create_attempts = 0; + state.ssmlp_get_attempts = 0; + state.matmul_creates.clear(); + state.ssmlp_creates.clear(); + state.creation_order.clear(); + state.release_order.clear(); + state.package_alive_at_release.clear(); + state.events.clear(); + state.get_data_calls = 0; + state.every_get_data_pointer_argument_was_null = true; + state.every_get_data_size_argument_was_nonnull = true; + state.current_package = package; + flm::test::SetLastErrorMessage({}); + + try { + (void)Phi4Weights::Load(api, std::move(package)); + } catch (const CorelibError& error) { + const std::string expected_context = + "Phi-4 weight object '" + std::string(object_name) + "'"; + const std::string expected_detail = + expected_context + ": " + std::string(detail); + const std::string expected_what = + std::string(call) + " failed: unsupported: " + + expected_detail; + CHECK(error.status == ryzenai_corelib_status_unsupported); + CHECK(error.call == call); + CHECK(error.detail == expected_detail); + CHECK(std::string_view(error.what()) == expected_what); + CHECK(state.creation_order.size() == expected_created); + CHECK(state.release_order.size() == expected_created); + CheckReleaseOrder( + state.creation_order, + state.release_order); + CHECK(api->live_object_count() == 0); + CHECK(std::all_of( + state.package_alive_at_release.begin(), + state.package_alive_at_release.end(), + [](bool value) { return value; })); + CHECK(state.current_package.expired()); + return; + } catch (const std::exception& error) { + throw std::runtime_error( + "expected typed CorelibError for " + + std::string(object_name) + ", got: " + error.what()); + } + throw std::runtime_error( + "expected Phi4Weights::Load failure was not thrown"); +} + +void TestActionableFailures(const SyntheticPackage& fixture) { + RecordingState state; + flm::test::ResetFakeCorelib(); + auto api = ResolveRecordingCorelib(state); + auto package = fixture.Load(api); + + const auto check_validation_error = []( + auto&& action, + std::string_view expected) { + try { + action(); + } catch (const CorelibError&) { + throw std::runtime_error( + "non-corelib validation became CorelibError"); + } catch (const std::invalid_argument& error) { + CHECK(std::string_view(error.what()).find(expected) != + std::string_view::npos); + return; + } + throw std::runtime_error( + "expected non-corelib validation error"); + }; + check_validation_error( + [&] { + (void)Phi4Weights::Load(nullptr, package); + }, + "CorelibApi"); + check_validation_error( + [&] { + (void)Phi4Weights::Load(api, nullptr); + }, + "Phi4Package"); + package.reset(); + + CheckLoadFailure( + state, + api, + fixture.Load(api), + FailurePoint::MatMulCreate, + 70, + 86, + "model.layers.17.attn.k_proj.MatMulNBits", + "ryzenai_corelib_matmul_bf16_weights_create_onnx", + "intentional Task 6 MatMul create failure"); + CheckLoadFailure( + state, + api, + fixture.Load(api), + FailurePoint::MatMulGetData, + 75, + 93, + "model.layers.18.attn.v_proj.MatMulNBits", + "ryzenai_corelib_matmul_bf16_weights_get_data", + "intentional Task 6 MatMul get-data failure"); + CheckLoadFailure( + state, + api, + fixture.Load(api), + FailurePoint::SsMlpCreate, + 24, + 119, + "model.layers.23.ssmlp", + "ryzenai_corelib_ssmlp_bf16_weights_create_onnx", + "intentional Task 6 SSMLP create failure"); + CheckLoadFailure( + state, + api, + fixture.Load(api), + FailurePoint::SsMlpGetData, + 29, + 145, + "model.layers.28.ssmlp", + "ryzenai_corelib_ssmlp_bf16_weights_get_data", + "intentional Task 6 SSMLP get-data failure"); + g_recording = nullptr; +} + +void TestNonCorelibObjectFailureRemainsDistinct( + const SyntheticPackage& fixture) { + RecordingState state; + state.matmul_packed_bytes = + std::numeric_limits::max(); + flm::test::ResetFakeCorelib(); + auto api = ResolveRecordingCorelib(state); + auto package = fixture.Load(api); + const std::weak_ptr package_lifetime = package; + state.current_package = package; + + try { + (void)Phi4Weights::Load(api, std::move(package)); + } catch (const CorelibError&) { + throw std::runtime_error( + "packed-byte validation became CorelibError"); + } catch (const std::runtime_error& error) { + const std::string_view message(error.what()); + CHECK( + message.find( + "model.layers.0.attn.k_proj.MatMulNBits") != + std::string_view::npos); + CHECK(message.find("overflows size_t") != + std::string_view::npos); + CHECK(state.creation_order.size() == 2); + CHECK(state.release_order.size() == 2); + CheckReleaseOrder( + state.creation_order, + state.release_order); + CHECK(std::all_of( + state.package_alive_at_release.begin(), + state.package_alive_at_release.end(), + [](bool value) { return value; })); + CHECK(package_lifetime.expired()); + CHECK(api->live_object_count() == 0); + g_recording = nullptr; + return; + } + throw std::runtime_error( + "expected packed-byte validation failure"); +} + +// Design Section 9.3: scales must be FP16 in an accepted package. An FP32 +// scales array is rejected at load with an actionable error rather than +// narrowed, because narrowing needs a host FP32-to-FP16 converter that +// API-6 does not permit. Admitting FP32 scales is a spec change. +void TestFp32ScalesAreRejectedWithAnActionableError() { + SyntheticPackage fp32_scales(true); + RecordingState state; + auto api = ResolveRecordingCorelib(state); + auto package = fp32_scales.Load(api); + state.current_package = package; + + try { + (void)Phi4Weights::Load(api, package); + } catch (const std::exception& error) { + const std::string_view message(error.what()); + CHECK( + message.find("model.layers.0.attn.q_proj.MatMulNBits") != + std::string_view::npos); + CHECK(message.find("FP16") != std::string_view::npos); + CHECK(message.find("rejected") != std::string_view::npos); + CHECK(message.find("repackage") != std::string_view::npos); + // The first object fails, so nothing is left behind. + CHECK(api->live_object_count() == 0); + package.reset(); + g_recording = nullptr; + return; + } + g_recording = nullptr; + throw std::runtime_error( + "an FP32 scales array must fail the Phi-4 weight load"); +} + +static_assert(!std::is_copy_constructible_v); +static_assert(!std::is_copy_assignable_v); +static_assert(std::is_nothrow_move_constructible_v); +static_assert(std::is_nothrow_move_assignable_v); + +} // namespace + +int main() { + try { + SyntheticPackage fixture; + TestExactConstructionAndLifetime(fixture); + TestMoveAssignmentReleasesBeforeOwners(fixture); + TestActionableFailures(fixture); + TestNonCorelibObjectFailureRemainsDistinct(fixture); + TestFp32ScalesAreRejectedWithAnActionableError(); + std::cout << "test_phi4_weights: PASS\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/src/test/phi4_corelib_aie4/test_real_corelib.cpp b/src/test/phi4_corelib_aie4/test_real_corelib.cpp new file mode 100644 index 00000000..58d31245 --- /dev/null +++ b/src/test/phi4_corelib_aie4/test_real_corelib.cpp @@ -0,0 +1,439 @@ +// Validates FastFlow against the REAL ryzenai_corelib, off-hardware. +// +// Everything else in this suite runs against fake_ryzenai_corelib, which is +// FastFlow's own code -- so it validates FastFlow against FastFlow's model +// of corelib, not against corelib. That model stayed green through a full +// ABI break, which is exactly the gap this file closes. +// +// The header documents three things as needing no NPU, and those are what +// runs here: +// +// 1. selftest_dependencies -- "Allocates host memory only, no NPU." +// 2. matmul_bf16_pad_shape / ssmlp_bf16_pad_rows / flat_mha_bf16_pad_rows +// -- "Needs no NPU: it is a lookup over the shipped kernel set." +// 3. has_device_context -- recorded, not asserted: the development box +// has an NPU but is not the AIE4 target. +// +// The shape-plan check is the load-bearing one. The whole allocation +// strategy rests on MatMul padded K/N equalling logical K/N, and until now +// that was only "confirmed" against a fake that returns whatever FastFlow +// expects. + +#include "fake_corelib.hpp" +#include "test_support.hpp" + +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +namespace constants = flm::phi4::constants; + +using flm::corelib::CorelibApi; +using flm::phi4::Phi4ShapePlan; +using flm::phi4::RowUse; + +#if !defined(FLM_REAL_CORELIB_RUNTIME_DIR) +#define FLM_REAL_CORELIB_RUNTIME_DIR "" +#endif + +// Semicolon-separated directories holding parts of corelib's dependency +// closure that do not sit beside the DLL. On the dev box that is the conda +// prefix the build linked protobuf from; Task 12 is where the closure +// becomes self-contained. +#if !defined(FLM_REAL_CORELIB_EXTRA_DLL_DIRS) +#define FLM_REAL_CORELIB_EXTRA_DLL_DIRS "" +#endif + +// CorelibApi::Load uses LOAD_LIBRARY_SEARCH_DEFAULT_DIRS, which honours +// directories added here and deliberately ignores PATH. +void AddExtraDllDirectories(std::string_view directories) { + std::size_t start = 0; + while (start <= directories.size()) { + const std::size_t end = directories.find(';', start); + const std::string_view entry = directories.substr( + start, + end == std::string_view::npos ? std::string_view::npos + : end - start); + if (!entry.empty()) { + const std::filesystem::path directory(entry); + if (!std::filesystem::exists(directory)) { + throw std::runtime_error( + "extra corelib DLL directory does not exist: " + + directory.string()); + } + if (AddDllDirectory(directory.c_str()) == nullptr) { + throw std::runtime_error( + "AddDllDirectory failed for " + directory.string()); + } + std::cout << "added DLL directory " << directory.string() + << '\n'; + } + if (end == std::string_view::npos) { + break; + } + start = end + 1; + } +} + +struct NamedRowUse { + RowUse use; + std::string_view name; +}; + +constexpr std::array kRowUses{{ + {RowUse::QueryProjection, "query_projection"}, + {RowUse::KvProjection, "kv_projection"}, + {RowUse::Attention, "attention"}, + {RowUse::OutputProjection, "output_projection"}, + {RowUse::SsMlp, "ssmlp"}, + {RowUse::LmHead, "lm_head"}, +}}; + +// The three MatMul shapes a Phi-4 forward pass dispatches. +struct MatMulDescriptor { + std::int64_t k; + std::int64_t n; + std::string_view name; + // The LM head is only ever dispatched at one row (design 10.5), and + // the real kernel set has no padded M for it beyond that -- asking is + // an error, not a larger answer. Phi4ShapePlan queries it at 1 only. + bool multi_row; +}; + +constexpr std::array kMatMulDescriptors{{ + {constants::kHiddenSize, constants::kQueryDimension, "q/o_proj", true}, + {constants::kHiddenSize, constants::kKvDimension, "k/v_proj", true}, + {constants::kHiddenSize, + constants::kVocabularySize, + "lm_head", + false}, +}}; + +std::shared_ptr ResolveFakeCorelib() { + auto resolver = flm::test::CompleteCorelibResolver(); + return CorelibApi::ResolveForTest( + [resolver = std::move(resolver)](std::string_view name) mutable + -> void* { + const auto found = resolver.find(std::string(name)); + return found == resolver.end() ? nullptr : found->second; + }); +} + +void PrintTransitions( + std::string_view label, + const std::vector>& + transitions) { + std::cout << " " << label << ": " << transitions.size() + << " transitions"; + const std::size_t shown = + transitions.size() < 12u ? transitions.size() : 12u; + for (std::size_t index = 0; index < shown; ++index) { + std::cout << (index == 0 ? " [" : " ") << transitions[index].first + << "->" << transitions[index].second; + } + if (shown != 0) { + std::cout << (shown < transitions.size() ? " ...]" : "]"); + } + std::cout << " last=" << transitions.back().first << "->" + << transitions.back().second << '\n'; +} + +// 1. Dependency self-test, against the real DynamicDispatch and RyzenMM. +void CheckVersionAndSelftest(const std::shared_ptr& api) { + const auto compiled = flm::corelib::CompiledCorelibVersion(); + const auto& runtime = api->runtime_version(); + std::cout << "real corelib version: " + << flm::corelib::FormatCorelibVersion(runtime) + << " (compiled against " + << flm::corelib::FormatCorelibVersion(compiled) << ")\n"; + // Loading at all means the API-5 gate passed; restate it so a future + // relaxation of the gate cannot make this file vacuous. + CHECK(flm::corelib::IsCorelibVersionCompatible(compiled, runtime)); + + api->Check( + api->functions().selftest_dependencies(), + "ryzenai_corelib_selftest_dependencies"); + std::cout << "selftest_dependencies: ok\n"; +} + +// 3. Missing-device path, recorded rather than asserted. +void RecordDeviceContext(const std::shared_ptr& api) { + const bool has_context = api->functions().has_device_context(); + std::cout << "has_device_context: " + << (has_context ? "true" : "false") + << " (recorded, not asserted: this box is not the AIE4 " + "target)\n"; + if (!has_context) { + std::cout + << " note: tensors, weights and dispatch will fail with " + "unsupported here; padding and packing still work.\n"; + } +} + +// 2. Shape plan against the real kernel set. +void CheckRealShapePlan(const std::shared_ptr& api) { + // MEM-5 and the Task 5 capacity model rest on this: padded K/N must + // equal logical K/N for every Phi-4 MatMul, at every live row class. + // Phi4ShapePlan::Build enforces it over 1..4096 and throws by name if + // it ever fails, so building it IS the assertion. Restate it directly + // at the boundaries so the check does not depend on Build's internals. + for (const auto& descriptor : kMatMulDescriptors) { + for (const std::int64_t rows : + {std::int64_t{1}, + std::int64_t{2}, + std::int64_t{63}, + std::int64_t{64}, + std::int64_t{65}, + std::int64_t{2048}, + std::int64_t{4095}, + std::int64_t{4096}}) { + if (rows != 1 && !descriptor.multi_row) { + continue; + } + std::int64_t m = rows; + std::int64_t k = descriptor.k; + std::int64_t n = descriptor.n; + api->Check( + api->functions().matmul_pad_shape( + &m, + &k, + &n, + constants::kGroupSize), + "ryzenai_corelib_matmul_bf16_pad_shape"); + if (k != descriptor.k || n != descriptor.n) { + throw std::runtime_error( + "real corelib padded " + std::string(descriptor.name) + + " K/N at rows " + std::to_string(rows) + ": K " + + std::to_string(descriptor.k) + "->" + + std::to_string(k) + ", N " + + std::to_string(descriptor.n) + "->" + + std::to_string(n) + + "; MEM-5 and the capacity model assume K/N are " + "unchanged"); + } + CHECK(m >= rows); + } + } + std::cout << "matmul padded K/N equals logical K/N for all three " + "Phi-4 descriptors\n"; + + // The LM head ships M = 1 and M = 128 and NOTHING ABOVE, and asking for + // more is an error rather than a larger answer. This is the constraint + // that makes Phi4ShapePlan correct: it queries the LM head at m = 1 + // only, and RowsFor(LmHead, n>1) throws. If the real library ever + // started rounding an out-of-grid M up instead of refusing, that design + // would be resting on a property the library no longer has -- so assert + // the refusal here rather than inferring it from the code that avoids + // it. + for (const std::int64_t rows : + {std::int64_t{1}, std::int64_t{2}, std::int64_t{128}}) { + std::int64_t m = rows; + std::int64_t k = constants::kHiddenSize; + std::int64_t n = constants::kVocabularySize; + api->Check( + api->functions().matmul_pad_shape( + &m, + &k, + &n, + constants::kGroupSize), + "ryzenai_corelib_matmul_bf16_pad_shape"); + CHECK(m == (rows == 1 ? 1 : 128)); + } + for (const std::int64_t rows : + {std::int64_t{129}, + std::int64_t{256}, + std::int64_t{2048}, + std::int64_t{4096}}) { + std::int64_t m = rows; + std::int64_t k = constants::kHiddenSize; + std::int64_t n = constants::kVocabularySize; + const auto status = api->functions().matmul_pad_shape( + &m, + &k, + &n, + constants::kGroupSize); + if (status == ryzenai_corelib_status_success) { + throw std::runtime_error( + "real corelib accepted LM-head rows " + + std::to_string(rows) + " and padded to " + + std::to_string(m) + + "; Phi4ShapePlan assumes the LM head is single-row and " + "RowsFor(LmHead) throws above 1, which is only safe while " + "the library refuses"); + } + } + std::cout << "LM head pads 1->1 and 2..128->128, and refuses every M " + "above 128\n"; + + // rows = 1 must stay 1 on every helper: decode allocates a single row + // and the header says flat MHA pads its KV window instead. + for (const auto& descriptor : kMatMulDescriptors) { + std::int64_t m = 1; + std::int64_t k = descriptor.k; + std::int64_t n = descriptor.n; + api->Check( + api->functions().matmul_pad_shape( + &m, + &k, + &n, + constants::kGroupSize), + "ryzenai_corelib_matmul_bf16_pad_shape"); + std::cout << " matmul " << descriptor.name << " rows 1 -> " << m + << '\n'; + CHECK(m == 1); + } + { + std::int64_t m = 1; + api->Check( + api->functions().ssmlp_pad_rows( + &m, + constants::kHiddenSize, + constants::kIntermediateSize, + constants::kGroupSize), + "ryzenai_corelib_ssmlp_bf16_pad_rows"); + std::cout << " ssmlp rows 1 -> " << m << '\n'; + CHECK(m == 1); + } + { + const ryzenai_corelib_flat_mha_bf16_desc desc{ + constants::kQueryHeadCount, + constants::kKvHeadCount, + constants::kHeadSize, + constants::kMaxSequenceLength, + constants::kRopeDimension}; + std::int64_t m = 1; + api->Check( + api->functions().flat_mha_pad_rows(&m, &desc), + "ryzenai_corelib_flat_mha_bf16_pad_rows"); + std::cout << " flat_mha rows 1 -> " << m << '\n'; + CHECK(m == 1); + } + + const Phi4ShapePlan real_plan = Phi4ShapePlan::Build(api); + std::cout << "real Phi4ShapePlan over rows 1..4096:\n"; + for (const auto& row_use : kRowUses) { + PrintTransitions( + row_use.name, + real_plan.Transitions(row_use.use)); + } + std::cout << " capacities: layer_rows=" + << real_plan.capacities().layer_rows + << " lm_head_rows=" << real_plan.capacities().lm_head_rows + << '\n'; + CHECK(real_plan.capacities().layer_rows >= constants::kMaxSequenceLength); + CHECK(real_plan.capacities().lm_head_rows == 1); + CHECK(real_plan.RowsFor(RowUse::LmHead, 1) == 1); + + // The fake must reproduce the real library's answers exactly. A fake + // that cannot is not a test double, it is a second implementation of + // the same guess -- and every other file in this suite trusts it. + const Phi4ShapePlan fake_plan = + Phi4ShapePlan::Build(ResolveFakeCorelib()); + for (const auto& row_use : kRowUses) { + const auto& real_transitions = real_plan.Transitions(row_use.use); + const auto& fake_transitions = fake_plan.Transitions(row_use.use); + if (real_transitions != fake_transitions) { + std::cout << " DIVERGENCE " << row_use.name + << ": real has " << real_transitions.size() + << " transitions, fake has " + << fake_transitions.size() << '\n'; + PrintTransitions(" real", real_transitions); + PrintTransitions(" fake", fake_transitions); + throw std::runtime_error( + "fake_ryzenai_corelib disagrees with the real shipped " + "kernel set for " + std::string(row_use.name) + + "; fix the fake to match, then re-run the host suite"); + } + } + CHECK( + fake_plan.capacities().layer_rows == + real_plan.capacities().layer_rows); + CHECK( + fake_plan.capacities().lm_head_rows == + real_plan.capacities().lm_head_rows); + std::cout << "fake transition lists match the real ones\n"; + + // The engine only ever reads RowsFor(), so the property it depends on + // is that padding is monotonic and never below the live rows -- which + // is what makes a single peak allocation safe. + for (const auto& row_use : kRowUses) { + if (row_use.use == RowUse::LmHead) { + continue; + } + std::int64_t previous = 0; + for (std::int64_t rows = 1; + rows <= constants::kMaxSequenceLength; + ++rows) { + const std::int64_t padded = + real_plan.RowsFor(row_use.use, rows); + CHECK(padded >= rows); + CHECK(padded >= previous); + CHECK(padded <= real_plan.capacities().layer_rows); + previous = padded; + } + } + std::cout << "real padding is monotonic, never below live rows, and " + "never above the planned capacity\n"; +} + +} // namespace + +// CTest's SKIP_RETURN_CODE. Returning 0 here would report Passed, and a +// green-and-inert check is worse than an absent one because it reads as +// coverage. +constexpr int kCTestSkipReturnCode = 77; + +int main() { + const std::string runtime_dir(FLM_REAL_CORELIB_RUNTIME_DIR); + if (runtime_dir.empty()) { + std::cout + << "test_real_corelib: SKIPPED -- configure with " + "-DRYZENAI_CORELIB_RUNTIME_DIR= to run it.\n"; + return kCTestSkipReturnCode; + } + + try { + const std::filesystem::path library = + std::filesystem::absolute( + std::filesystem::path(runtime_dir) / + "ryzenai_corelib.dll") + .lexically_normal(); + if (!std::filesystem::exists(library)) { + // Configured but absent is a failure, not a skip: a silently + // skipped ABI check is how a stale DLL stays hidden. + throw std::runtime_error( + "RYZENAI_CORELIB_RUNTIME_DIR is set but " + + library.string() + " does not exist"); + } + AddExtraDllDirectories(FLM_REAL_CORELIB_EXTRA_DLL_DIRS); + std::cout << "loading " << library.string() << '\n'; + + auto api = CorelibApi::Load(library); + CheckVersionAndSelftest(api); + RecordDeviceContext(api); + CheckRealShapePlan(api); + + api->functions().cleanup(); + std::cout << "test_real_corelib: PASS\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << "test_real_corelib: FAIL: " << error.what() << '\n'; + return 1; + } +} diff --git a/src/test/phi4_corelib_aie4/test_support.hpp b/src/test/phi4_corelib_aie4/test_support.hpp new file mode 100644 index 00000000..b472fd19 --- /dev/null +++ b/src/test/phi4_corelib_aie4/test_support.hpp @@ -0,0 +1,198 @@ +#pragma once + +#include +#include +#include +#include + +inline void Check( + bool condition, + std::string_view expression, + const char* file, + int line) { + if (!condition) { + throw std::runtime_error( + std::string(file) + ":" + std::to_string(line) + + " CHECK failed: " + std::string(expression)); + } +} + +#define CHECK(expression) \ + Check(static_cast(expression), #expression, __FILE__, __LINE__) + +template +void CheckThrowsContains(Function&& function, std::string_view expected) { + try { + function(); + } catch (const std::exception& error) { + CHECK(std::string_view(error.what()).find(expected) != + std::string_view::npos); + return; + } + throw std::runtime_error("expected exception was not thrown"); +} + +#if defined(FLM_PHI4_FRONTEND_TEST_SUPPORT) + +#include +#include +#include +#include +#include + +class FakeEngine final : public causal_lm { +public: + enum class FailureKind { + None, + Corelib, + Standard, + Unknown, + }; + + explicit FakeEngine(std::uint32_t max_length) + : max_length(max_length) {} + + buffer forward(int id) override { +#if defined(FLM_ENABLE_CORELIB_AIE4) + if (fail_next_forward) { + fail_next_forward = false; + throw flm::corelib::CorelibError( + ryzenai_corelib_status_failure, + "fake_forward", + "injected pre-submit forward failure", + "failure"); + } + MaybeFail( + forward_failure, + successful_forwards_before_failure, + "fake_forward"); +#endif + forward_tokens.push_back(id); + ++position; + return MakeLogits(); + } + + buffer prefill( + std::vector& ids, + void* payload) override { +#if defined(FLM_ENABLE_CORELIB_AIE4) + if (fail_next_prefill) { + fail_next_prefill = false; + throw flm::corelib::CorelibError( + ryzenai_corelib_status_failure, + "fake_prefill", + "injected pre-submit prefill failure", + "failure"); + } + MaybeFail( + prefill_failure, + successful_prefills_before_failure, + "fake_prefill"); +#endif + prefill_calls.push_back(ids); + prefill_payloads.push_back(payload); + position += static_cast(ids.size()); + return MakeLogits(); + } + + void set_context_length(int length) override { + position = length; + } + + void load_weights(Q4NX&) override {} + + void update_max_length(std::uint32_t requested) override { + if (requested == 0 || requested > 4096) { + throw std::out_of_range( + "fake AIE4 maximum length must be in 1..4096"); + } + if (requested < static_cast(position)) { + throw std::out_of_range( + "fake AIE4 maximum length cannot be below current"); + } + max_length = requested; + } + + void clear_context() override { + ++clear_count; + position = 0; + checkpoint_position.reset(); + } + + buffer get_k_cache(int, int) override { + return MakeLogits(); + } + + buffer get_v_cache(int, int) override { + return MakeLogits(); + } + + int get_current_context_length() override { + return position; + } + + int checkpoint() override { + checkpoint_position = position; + return position; + } + + int restore() override { + if (!checkpoint_position.has_value()) { + throw std::logic_error("fake engine has no checkpoint"); + } + position = *checkpoint_position; + return position; + } + + static buffer MakeLogits() { + return buffer(1); + } + +#if defined(FLM_ENABLE_CORELIB_AIE4) + static void MaybeFail( + FailureKind& failure, + int& successful_calls_before_failure, + const char* operation) { + if (failure == FailureKind::None) { + return; + } + if (successful_calls_before_failure > 0) { + --successful_calls_before_failure; + return; + } + + const FailureKind injected = std::exchange( + failure, + FailureKind::None); + if (injected == FailureKind::Corelib) { + throw flm::corelib::CorelibError( + ryzenai_corelib_status_failure, + operation, + "injected pre-submit corelib failure", + "failure"); + } + if (injected == FailureKind::Standard) { + throw std::bad_alloc(); + } + throw 17; + } +#endif + + std::uint32_t max_length; + int position = 0; + int clear_count = 0; + std::optional checkpoint_position; + std::vector> prefill_calls; + std::vector prefill_payloads; + std::vector forward_tokens; +#if defined(FLM_ENABLE_CORELIB_AIE4) + bool fail_next_prefill = false; + bool fail_next_forward = false; + FailureKind prefill_failure = FailureKind::None; + FailureKind forward_failure = FailureKind::None; + int successful_prefills_before_failure = 0; + int successful_forwards_before_failure = 0; +#endif +}; + +#endif diff --git a/src/tools/compare_phi4_corelib_e2e.py b/src/tools/compare_phi4_corelib_e2e.py new file mode 100644 index 00000000..e9440502 --- /dev/null +++ b/src/tools/compare_phi4_corelib_e2e.py @@ -0,0 +1,1207 @@ +#!/usr/bin/env python3 +"""Task 12 Step 3: compare FastFlow's Phi-4 AIE4 engine against the corelib +reference driver, on the same explicit token IDs and the same forced route. + +Two modes, and they are separate PROCESSES on purpose: + + emit-reference loads the reference driver, runs it, writes JSON, and + calls corelib.cleanup() before exiting. + compare reads two JSON files and holds no device context at all. + +Nothing here ever runs while the C++ harness is running. Two processes holding +AIE4 device contexts at once fail in ways that look like defects, so the suite +script serialises them and this file makes that easy to honour rather than +something to remember. + +Three things about the `e5258d2` reference are worth stating, because each one +would otherwise look like a bug in this comparator: + + * `Phi4.forward` and `Phi4.logits_for` return FP32 arrays directly. Corelib + widens on `tensor_read`, so there is no `from_bf16` unpacking left to + mirror. It is FASTFLOW's BF16 logits that get widened here. + * The reference driver still uses the collapsed two-synchronize-per-layer + schedule. Design Section 10.4 no longer considers that sound and FastFlow + deliberately uses four. Its VALUES are the reference; its SCHEDULE is not, + so synchronize counts are never compared. + * The driver is read-only. No `--continuation-route` option is added to it + and that repository is not modified; the routes are composed here out of + `Phi4.forward` calls. +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from pathlib import Path + +import numpy as np + +# Thresholds from design Section 12.4. They are named rather than inlined so a +# report can quote the number that actually ran, and they are SEALED below -- +# see `_DETERM2_SEALED`. Do not change either number here alone. +MIN_CORRELATION = 0.9999 +MIN_DECODE_STEPS = 16 +TOP_K = 32 +TOP_5 = 5 + +# DETERM-2. Two bounds, deliberately in DIFFERENT UNITS. +# +# CROSS-IMPLEMENTATION (design 12.4): absolute 0.25 over the union top-32. +# FastFlow and the reference driver can differ for reasons other than +# accumulation order, so an absolute ceiling is the right shape there. +# +# RUN-TO-RUN (design 15.3, DETERM-1): 2 BF16 ULP of the larger of the two +# compared values. The phenomenon is ULP-relative, not absolute. The one +# observed event sat at exactly 0.25, which is 2 ULP for logits in [16, 32) -- +# an absolute 0.25 bound would therefore have hard-failed the identical 2-ULP +# wobble on a logit in [32, 64), where 2 ULP is 0.5, for no reason but +# magnitude. This corrects the UNITS; it does not widen the tolerance. +# +# Both are OBSERVED CEILINGS WITH NO MARGIN, not tolerances. A run that exceeds +# either has done something not previously seen, and the answer is to measure +# and amend the spec -- never to widen the number so a red run goes green. +# +# TASK 13 MEASURED IT, AND THE 2-ULP FIGURE IS NOT AN UPPER BOUND. Two events +# exceeded it during that baseline: a reprefill pair differing by up to 0.3125 +# on a logit of 11.125 -- about 2.5 ULP, with 17512 logits over the bound -- +# and an append pair differing by up to 48.34 after the two runs emitted +# DIFFERENT TOKENS from decode step 7. So this constant no longer describes a +# ceiling. It describes the largest run-to-run difference that has ever been +# benign. +# +# THAT IS DELIBERATELY LEFT AS IS. DETERM-2 says a logit difference above the +# bound is a failure and not a wider tolerance, and both events were caught by +# this gate doing exactly its job. Raising the number to accommodate them is +# the move DETERM-2 exists to forbid, and it would convert a measured device +# property into an unbounded one. The right response is the design decision +# Task 13 report asks for, not an edit here. +MAX_TOP32_ABS_DIFF = 0.25 +RUN_TO_RUN_MAX_ULPS = 2 + +# The seal, and it covers EVERY name a failure message can hand you. +# +# The previous version sealed one private constant and then exposed unsealed +# public aliases -- including the very name its own error message advertised. +# Grepping the name from the message landed on an alias that could be edited +# freely, and the tool started normally and went green. The bump was off to the +# side of the road. +# +# So: every constant below is cross-checked against a literal that exists for +# no other purpose, and the runtime failure messages point at this block by +# name so that grepping what the message says leads HERE. +# +# IT NOW COVERS THE DESIGN 12.4 ACCEPTANCE THRESHOLDS TOO, and not because +# 12.4 asked for it. `MIN_CORRELATION` and `MIN_DECODE_STEPS` sat outside the +# seal while their values were printed verbatim in four failure messages -- +# the exact grep-the-message-and-edit path the seal exists to block, forty +# lines above the constants it did block. Sealing the instance and leaving the +# class open is the mistake this file has already paid for once, in the alias +# incident described above; two lines close the class. +# +# Each entry carries the reason its own number cannot be moved, because the +# reasons are NOT the same -- two are measured ceilings and two are the size +# of the experiment -- and one message that fitted both would be true of +# neither. +_DETERM2_SEALED = { + "MAX_TOP32_ABS_DIFF": ( + 0.25, + "the largest cross-implementation top-32 logit difference ever " + "measured on this hardware, recorded with no margin (design 12.4)", + ), + "RUN_TO_RUN_MAX_ULPS": ( + 2, + "in BF16 ULP, the largest run-to-run difference that has ever been " + "benign; Task 13 measured two events above it and they are defects, " + "not headroom (design 15.3, DETERM-1)", + ), + "MIN_CORRELATION": ( + 0.9999, + "the correlation FastFlow must reach against the corelib reference " + "before the two are called the same computation; lowering it does " + "not make them agree, it stops the comparison asking (design 12.4)", + ), + "MIN_DECODE_STEPS": ( + 16, + "the shortest decode run design 12.4 accepts as evidence; fewer " + "steps is a smaller experiment, not a passing one", + ), + # TOP_K was the third one left outside, and it is not a threshold: it is + # the SIZE OF THE WINDOW the MAX_TOP32_ABS_DIFF threshold is measured + # over. Shrinking it trips no bound, it just stops the comparison looking + # at the logits where the two implementations disagree -- so it is the + # cheapest way to make a red run green, and its value is interpolated + # verbatim into the very message that reports the failure. + "TOP_K": ( + 32, + "the width of the union top-k window MAX_TOP32_ABS_DIFF is measured " + "over; design 12.4's '0.25 over the union top-32' is one claim, not " + "two independent numbers, so narrowing k narrows the experiment while " + "every bound still reads as met", + ), +} +for _sealed_name, (_sealed_value, _sealed_reason) in _DETERM2_SEALED.items(): + _actual = globals()[_sealed_name] + if _actual != _sealed_value: + raise SystemExit( + f"DETERM-2: {_sealed_name} has been changed from its sealed " + f"value {_sealed_value} to {_actual}.\n" + "\n" + f"{_sealed_name} is {_sealed_reason}. Moving it does not make a " + "failing run acceptable; it makes the suite stop reporting a " + "change in behaviour that nobody has looked at.\n" + "\n" + "If a run genuinely misses one of these: measure it, characterise " + "it the way report section 14.2 characterises the run-to-run " + "case, and amend the design section the entry names. Only then " + "update the constant AND its entry in _DETERM2_SEALED together." + ) + + +def _two_bf16_ulp(magnitude: np.ndarray) -> np.ndarray: + """`RUN_TO_RUN_MAX_ULPS` ULP at each magnitude, exactly. + + BF16 keeps 8 significand bits, so ULP at a value with exponent e is + 2**(e - 7). The exponent is read out of the FP32 bit pattern rather than + via log2: these values were widened from BF16 so the field is exact, and + log2 of a value a hair under a power of two is exactly the case that would + round the wrong way and silently shift the bound by a factor of two. + + Zero gets a bound of zero -- two runs that both produced zero are already + equal, and anything else there is a real difference. + """ + magnitude = np.asarray(magnitude, dtype=np.float32) + bits = np.ascontiguousarray(magnitude).view(np.uint32) + exponent = ((bits >> 23) & 0xFF).astype(np.int32) - 127 + # N * 2**(e - 7), written out rather than folded into the exponent: the + # folded form is only correct while N is a power of two, and a later change + # to RUN_TO_RUN_MAX_ULPS would silently compute the wrong bound. + ulp = np.exp2((exponent - 7).astype(np.float64)) + return np.where(magnitude == 0.0, 0.0, RUN_TO_RUN_MAX_ULPS * ulp) + + +# The ULP bound is checked at import, at the boundaries that matter. +# +# The whole correction in DETERM-2 is that the bound scales with magnitude, so +# a bound that silently failed to step at a power of two would reintroduce the +# absolute-ceiling bug while looking relative. 20 and 31.5 must give 0.25 (the +# observed event), 32 and 40 must give 0.5, and the step must land exactly on +# the binade edge -- which is why the exponent comes from the bit pattern and +# not from log2. +_ULP_SELF_CHECK = ((16.0, 0.25), (20.0, 0.25), (31.5, 0.25), + (32.0, 0.5), (40.0, 0.5), (64.0, 1.0), (0.0, 0.0)) +if RUN_TO_RUN_MAX_ULPS == 2: + _probe = _two_bf16_ulp( + np.array([v for v, _ in _ULP_SELF_CHECK], dtype=np.float32)) + for _index, (_value, _expected) in enumerate(_ULP_SELF_CHECK): + if float(_probe[_index]) != _expected: + raise SystemExit( + f"DETERM-2: the ULP bound is miscomputed. 2 ULP of {_value} " + f"should be {_expected}, got {float(_probe[_index])}. The " + "run-to-run gate would be applying the wrong tolerance." + ) + + +def _import_reference(): + """Import the corelib reference driver from RYZENAI_CORELIB_SOURCE. + + Imported by path rather than vendored: a copy would drift, and the whole + value of this comparison is that the reference is the corelib repository's + own driver rather than a second transcription of the same guess. + """ + source = os.environ.get("RYZENAI_CORELIB_SOURCE") + if not source: + raise SystemExit( + "RYZENAI_CORELIB_SOURCE is not set. Point it at the " + "ryzenai-corelib checkout whose python/ holds phi4_driver.py." + ) + python_dir = Path(source) / "python" + if not (python_dir / "phi4_driver.py").is_file(): + raise SystemExit(f"no phi4_driver.py under {python_dir}") + sys.path.insert(0, str(python_dir)) + import phi4_driver # noqa: E402 + import ryzenai_corelib as corelib # noqa: E402 + + return phi4_driver, corelib + + +def _argmax_lowest(values) -> int: + """The lowest ID among ties. + + `np.argmax` already returns the first maximal index, which for a dense + logit vector IS the lowest ID. It is spelled out because FastFlow's + `ArgmaxLowest` promises the same thing and the comparison below is only + meaningful if both sides use one convention. + """ + return int(np.argmax(np.asarray(values))) + + +def _narrow_bf16(values) -> list: + """FP32 -> raw BF16 bits, round-to-nearest-even. + + Mirrors `phi4_driver.to_bf16`, reimplemented here so `compare` stays a pure + JSON-to-JSON operation that imports neither the driver nor the corelib + bindings and therefore cannot open a device context. + """ + bits = np.ascontiguousarray(values, dtype=np.float32).view(np.uint32) + rounded = bits.astype(np.uint64) + 0x7FFF + ((bits >> 16) & 1) + return (rounded >> 16).astype(np.uint16).tolist() + + +def _sha256_file(path) -> str: + import hashlib + + digest = hashlib.sha256() + with open(path, "rb") as handle: + for chunk in iter(lambda: handle.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _widen_bf16(bits) -> np.ndarray: + """FastFlow emits raw BF16 bit patterns; the reference is already FP32. + + BF16 to FP32 is a 16-bit left shift and nothing else, so this is exact and + introduces no error of its own into the comparison below. + """ + raw = np.asarray(bits, dtype=np.uint16).astype(np.uint32) << np.uint32(16) + return np.ascontiguousarray(raw).view(np.float32) + + +# --------------------------------------------------------------------------- +# Reference +# --------------------------------------------------------------------------- + + +def _read_live_cache(tensor, position: int, driver) -> list[int]: + """Live rows only, in FastFlow's [head][row][head_size] order. + + The cache is allocated at the full 4096-row window but only [0, position) + has ever been written. Comparing the tail would be comparing uninitialised + device memory on both sides and calling the agreement a result. + """ + if position <= 0: + return [] + head_size = driver.HEAD_SIZE + max_seq = driver.MAX_SEQ + out = [] + for head in range(driver.KV_HEADS): + offset = (head * max_seq) * head_size + raw = tensor.read(position * head_size, offset, driver.DataType.BF16) + out.extend(np.frombuffer(raw, dtype=np.uint16).tolist()) + return out + + +def emit_reference(args) -> int: + driver, corelib = _import_reference() + + if not corelib.load_library().ryzenai_corelib_has_device_context(): + raise SystemExit( + "no AIE4 device context for the reference driver. Check that " + "nothing else is holding one; the C++ harness must not be " + "running concurrently." + ) + + plan = json.loads(Path(args.token_ids_json).read_text(encoding="utf-8")) + if isinstance(plan, list): + prefix, suffix = list(plan), [] + else: + prefix = list(plan["prefix"]) + suffix = list(plan.get("suffix", [])) + + model_onnx = Path(args.model_dir) / "model.onnx" + if not model_onnx.is_file(): + raise SystemExit(f"no model.onnx under {args.model_dir}") + + # Which DLL the PYTHON side resolved. + # + # The C++ harness and this script find the library by different fallbacks, + # so "both used the same corelib" was an assumption rather than a recorded + # fact. Recording it on both sides makes the comparison state which two + # binaries it actually compared. + library = corelib.load_library() + library_path = str(getattr(library, "_dll", None)._name) + record = { + "continuation_route": args.continuation_route, + "prefix_ids": prefix, + "suffix_ids": suffix, + "corelib_loaded_path": library_path, + "corelib_sha256": _sha256_file(library_path), + } + print(f"reference corelib: {library_path}") + print(f"sha256 : {record['corelib_sha256']}") + try: + model = driver.Phi4(model_onnx) + + # Routes composed here, out of Phi4.forward calls, exactly as design + # Section 12.4 describes them. + if args.continuation_route == "force_append": + hidden = model.forward( + model.embed_rows(prefix), len(prefix), 0 + ) + position = len(prefix) + for token in suffix: + hidden = model.forward( + model.embed_rows([token]), 1, position + ) + position += 1 + elif args.continuation_route == "force_reprefill": + # A fresh process is a cleared reference state, and the full + # rendered history is recomputed from position zero in one call. + full = prefix + suffix + hidden = model.forward(model.embed_rows(full), len(full), 0) + position = len(full) + else: + raise SystemExit( + "--continuation-route must be force_append or " + "force_reprefill" + ) + + logits = model.logits_for(hidden) + record["continuation"] = { + "logits": logits.tolist(), + "top1_id": _argmax_lowest(logits), + } + + decode = [] + token = _argmax_lowest(logits) + for _ in range(args.decode_steps): + hidden = model.forward(model.embed_rows([token]), 1, position) + position += 1 + logits = model.logits_for(hidden) + step_top1 = _argmax_lowest(logits) + decode.append( + { + "input_id": token, + "logits": logits.tolist(), + "top1_id": step_top1, + } + ) + token = step_top1 + record["decode"] = decode + record["final_position"] = position + record["final_snapshot"] = { + "position": position, + # Design 15.3 lists the final hidden state as a checkpoint. It is + # narrowed to BF16 here because that is what the device stores: + # FastFlow reads `lm_input_tensor` back as raw BF16, and the + # reference's FP32 array is the value corelib narrowed on the way + # in. Comparing the FP32 against the BF16 would be comparing two + # different things and would need a tolerance to hide it. + "last_hidden": _narrow_bf16(hidden), + "layer0_k": _read_live_cache(model.k_cache[0], position, driver), + "layer0_v": _read_live_cache(model.v_cache[0], position, driver), + "layer31_k": _read_live_cache(model.k_cache[-1], position, driver), + "layer31_v": _read_live_cache(model.v_cache[-1], position, driver), + } + finally: + # Released before the C++ harness starts. The suite runs this process + # to completion for exactly this reason. + corelib.cleanup() + + Path(args.output_json).write_text(json.dumps(record), encoding="utf-8") + print(f"reference written to {args.output_json}") + return 0 + + +# --------------------------------------------------------------------------- +# Comparison +# --------------------------------------------------------------------------- + + +class Failures: + def __init__(self) -> None: + self.messages: list[str] = [] + + def check(self, condition: bool, message: str) -> None: + if not condition: + self.messages.append(message) + + def report(self, label: str) -> int: + if not self.messages: + print(f"{label}: PASS") + return 0 + print(f"{label}: FAIL") + for message in self.messages: + print(f" - {message}") + return 1 + + +def _correlation(left: np.ndarray, right: np.ndarray) -> float: + left = left.astype(np.float64) + right = right.astype(np.float64) + left -= left.mean() + right -= right.mean() + denominator = np.linalg.norm(left) * np.linalg.norm(right) + if denominator == 0.0: + return 0.0 + return float(np.dot(left, right) / denominator) + + +def _compare_step( + failures: Failures, + label: str, + mine_bits, + theirs, +) -> None: + mine = _widen_bf16(mine_bits) + reference = np.asarray(theirs, dtype=np.float32) + if mine.shape != reference.shape: + failures.check( + False, + f"{label}: logit vector length {mine.shape} vs " + f"{reference.shape}", + ) + return + + if not np.all(np.isfinite(mine)) or not np.all(np.isfinite(reference)): + failures.check(False, f"{label}: non-finite logits") + return + + correlation = _correlation(mine, reference) + failures.check( + correlation >= MIN_CORRELATION, + f"{label}: correlation {correlation:.8f} < {MIN_CORRELATION}", + ) + + mine_top1 = _argmax_lowest(mine) + reference_top1 = _argmax_lowest(reference) + failures.check( + mine_top1 == reference_top1, + f"{label}: top-1 {mine_top1} vs {reference_top1}", + ) + + # Ties resolve to the lowest ID on both sides. Asserting it directly + # matters because a tie is exactly where two argmax conventions diverge, + # and BF16 logits tie far more often than FP32 ones. + for name, values, chosen in ( + ("fastflow", mine, mine_top1), + ("reference", reference, reference_top1), + ): + tied = np.flatnonzero(values == values[chosen]) + failures.check( + int(tied[0]) == chosen, + f"{label}: {name} argmax {chosen} is not the lowest tied ID " + f"{int(tied[0])}", + ) + + mine_top5 = set(np.argsort(-mine, kind="stable")[:TOP_5].tolist()) + reference_top5 = set( + np.argsort(-reference, kind="stable")[:TOP_5].tolist() + ) + failures.check( + mine_top5 == reference_top5, + f"{label}: top-5 {sorted(mine_top5)} vs {sorted(reference_top5)}", + ) + + union = sorted( + set(np.argsort(-mine, kind="stable")[:TOP_K].tolist()) + | set(np.argsort(-reference, kind="stable")[:TOP_K].tolist()) + ) + index = np.asarray(union, dtype=np.int64) + max_abs = float(np.max(np.abs(mine[index] - reference[index]))) + failures.check( + max_abs <= MAX_TOP32_ABS_DIFF, + f"{label}: max |diff| over the union top-{TOP_K} is {max_abs:.4f} " + f"> {MAX_TOP32_ABS_DIFF}", + ) + + +def compare(args) -> int: + mine = json.loads(Path(args.fastflow_json).read_text(encoding="utf-8")) + theirs = json.loads(Path(args.reference_json).read_text(encoding="utf-8")) + failures = Failures() + + # Which two binaries are actually being compared. Reported always; if both + # sides recorded a hash they must agree, because a comparison between two + # different corelib builds is not the comparison anyone thinks it is. + mine_sha = mine.get("corelib_sha256") + theirs_sha = theirs.get("corelib_sha256") + if mine_sha and theirs_sha: + print(f"fastflow corelib {mine_sha} {mine.get('corelib_loaded_path')}") + print(f"reference corelib {theirs_sha} {theirs.get('corelib_loaded_path')}") + failures.check( + mine_sha == theirs_sha, + "the two sides loaded DIFFERENT corelib builds:\n" + f" fastflow {mine_sha} {mine.get('corelib_loaded_path')}\n" + f" reference {theirs_sha} {theirs.get('corelib_loaded_path')}", + ) + else: + failures.check( + False, + "one side recorded no corelib SHA-256, so this comparison cannot " + "say which two binaries it compared", + ) + + failures.check( + mine["continuation_route"].replace("force_", "") + == theirs["continuation_route"].replace("force_", ""), + f"route {mine['continuation_route']} vs " + f"{theirs['continuation_route']}", + ) + failures.check( + mine["prefix_ids"] == theirs["prefix_ids"] + and mine["suffix_ids"] == theirs["suffix_ids"], + "the two runs did not use the same explicit token IDs", + ) + + # Bit-exactness is tracked as its own property, separate from the design + # 12.4 thresholds. + # + # The thresholds are the release gate. Bit-exactness is the claim the + # report makes, and until now no check produced it -- it was computed by + # hand from the artifacts and asserted in prose, which is exactly the kind + # of claim that quietly stops being true. `--require-bit-exact` makes it + # fail the run instead. + exact = {"logits": [], "kv": [], "last_hidden": None} + + _compare_step( + failures, + "continuation", + mine["continuation"]["logits_bf16"], + theirs["continuation"]["logits"], + ) + exact["logits"].append( + mine["continuation"]["logits_bf16"] + == _narrow_bf16(theirs["continuation"]["logits"]) + ) + + steps = min(len(mine["decode"]), len(theirs["decode"])) + failures.check( + steps >= MIN_DECODE_STEPS, + f"only {steps} decode steps compared; design Section 12.4 requires " + f"at least {MIN_DECODE_STEPS}", + ) + for index in range(steps): + mine_step = mine["decode"][index] + reference_step = theirs["decode"][index] + failures.check( + mine_step["input_id"] == reference_step["input_id"], + f"decode[{index}]: fed {mine_step['input_id']} vs " + f"{reference_step['input_id']}", + ) + _compare_step( + failures, + f"decode[{index}]", + mine_step["logits_bf16"], + reference_step["logits"], + ) + exact["logits"].append( + mine_step["logits_bf16"] == _narrow_bf16(reference_step["logits"]) + ) + + # Live K/V only, and only where both sides reached the same position. + # Comparing beyond `position` would compare uninitialised device memory. + mine_snapshot = mine["final_snapshot"] + reference_snapshot = theirs["final_snapshot"] + failures.check( + mine_snapshot["position"] == reference_snapshot["position"], + f"final position {mine_snapshot['position']} vs " + f"{reference_snapshot['position']}", + ) + for name in ("layer0_k", "layer0_v", "layer31_k", "layer31_v"): + left = _widen_bf16(mine_snapshot[name]) + right = _widen_bf16(reference_snapshot[name]) + if left.shape != right.shape: + failures.check( + False, + f"{name}: live extent {left.shape} vs {right.shape}", + ) + exact["kv"].append(False) + continue + if left.size == 0: + failures.check(False, f"{name}: no live rows to compare") + exact["kv"].append(False) + continue + correlation = _correlation(left, right) + failures.check( + correlation >= MIN_CORRELATION, + f"{name}: live-cache correlation {correlation:.8f} < " + f"{MIN_CORRELATION}", + ) + identical = int(np.sum(left == right)) + exact["kv"].append(identical == left.size) + print( + f"{name}: correlation {correlation:.8f}, " + f"{identical}/{left.size} elements identical" + ) + + # Design 15.3 lists the final hidden state as a checkpoint. It was emitted + # by the harness from the first version of this suite and never compared, + # which made it a payload rather than a check. + if "last_hidden" in reference_snapshot: + mine_hidden = mine_snapshot["last_hidden"] + reference_hidden = reference_snapshot["last_hidden"] + if len(mine_hidden) != len(reference_hidden): + failures.check( + False, + f"last_hidden: length {len(mine_hidden)} vs " + f"{len(reference_hidden)}", + ) + exact["last_hidden"] = False + else: + widened_mine = _widen_bf16(mine_hidden) + widened_reference = _widen_bf16(reference_hidden) + correlation = _correlation(widened_mine, widened_reference) + failures.check( + correlation >= MIN_CORRELATION, + f"last_hidden: correlation {correlation:.8f} < " + f"{MIN_CORRELATION}", + ) + identical = int(np.sum(widened_mine == widened_reference)) + exact["last_hidden"] = identical == len(mine_hidden) + print( + f"last_hidden: correlation {correlation:.8f}, " + f"{identical}/{len(mine_hidden)} elements identical" + ) + else: + failures.check( + False, + "the reference emitted no last_hidden, so the design 15.3 final " + "hidden checkpoint was not compared", + ) + + # The route's expected token sequence, from a file committed to the + # repository. Without this the golden is re-derived from the reference on + # every run, so the pair could drift together -- a corelib change that + # moved both sides identically would pass every check above. + if args.expected_tokens: + expected_document = json.loads( + Path(args.expected_tokens).read_text(encoding="utf-8") + ) + # The artifact records the route as ContinuationRouteName() spells it + # ("append"/"reprefill"); the suite and the golden file use the forced + # spelling ("force_append"/"force_reprefill"). Accept either, so a + # naming difference cannot be mistaken for a missing golden. + route_key = mine["continuation_route"] + if route_key not in expected_document: + alternate = ( + route_key[len("force_") :] + if route_key.startswith("force_") + else "force_" + route_key + ) + if alternate in expected_document: + route_key = alternate + if route_key not in expected_document: + failures.check( + False, + f"no expected token sequence recorded for route " + f"'{route_key}' in {args.expected_tokens}", + ) + else: + expected = expected_document[route_key] + observed = [mine["continuation"]["top1_id"]] + [ + step["top1_id"] for step in mine["decode"] + ] + failures.check( + observed == expected, + f"route '{route_key}' emitted a different token sequence " + f"from the one recorded in {args.expected_tokens}:\n" + f" expected {expected}\n" + f" observed {observed}", + ) + + logits_exact = all(exact["logits"]) + kv_exact = bool(exact["kv"]) and all(exact["kv"]) + hidden_exact = exact["last_hidden"] is True + all_exact = logits_exact and kv_exact and hidden_exact + print( + "bit-exact vs the reference: logits " + f"{sum(1 for v in exact['logits'] if v)}/{len(exact['logits'])} steps, " + f"K/V {sum(1 for v in exact['kv'] if v)}/{len(exact['kv'])} tensors, " + f"last_hidden {hidden_exact}" + ) + + # MODEL STATE must be bit-identical; the LM head's output need not be. + # + # This split is measured, not a convenience. Across every CROSS- + # IMPLEMENTATION run recorded to date the live K/V for layers 0 and 31 and + # the final hidden state have been bit-identical to the reference -- the + # whole 32-layer computation agreeing exactly -- while some runs had a + # single logit vector out of 17 differ, with every design 12.4 threshold + # still met and the same top-1. + # + # THE RATE HERE READ "one run in three" AND WAS ALREADY STALE WHEN WRITTEN. + # Task 13 counted the surviving artifacts: the append route differed from + # the reference on 2 of the 4 comparisons whose compare-summary records + # exist, the reprefill route on 0 of 4. Stated with its n rather than as a + # fraction, because n=4 does not support a rate anyone should rely on. It + # is enough to say the effect is route-dependent and real. Task 12 reported + # two further append divergences whose artifacts were overwritten before + # run-scoped directories existed; they are not counted here. + # + # NOTE, and it matters: this comment is about the CROSS-IMPLEMENTATION + # comparison, where the model state HAS always matched. It is not evidence + # about the RUN-TO-RUN case, where Task 13 measured model-state divergence + # at layer 31 with layer 0 identical. See the self_consistency docstring. + # + # So state bit-identity is enforced: a regression there would mean the + # model computed something different. Logit bit-identity is reported and + # not enforced, because enforcing a property that is not reliably true + # produces a red suite for something that is not a defect -- and a gate + # nobody trusts is worse than a number everybody reads. + failures.check( + kv_exact, + "live K/V is NOT bit-identical to the reference. The 32-layer " + "computation itself has diverged, which is a different and more " + "serious thing than a logit difference within tolerance.", + ) + failures.check( + hidden_exact, + "the final hidden state is NOT bit-identical to the reference " + "(design 15.3 checkpoint)", + ) + if args.require_bit_exact: + failures.check( + all_exact, + "--require-bit-exact was given and the LOGITS are not " + "bit-identical. State and last_hidden are checked " + "unconditionally; this flag additionally demands the LM-head " + "output match, which is a diagnostic rather than a release gate.", + ) + + # A reported property has to outlive stdout. + # + # These counts were prints and nothing else: absent from every artifact and + # from the suite summary, and never aggregated, so a rising rate of + # LM-head divergence would have been invisible -- the suite's last line + # reads the same at 17/17 and at 16/17. Writing them where the runner can + # pick them up is what makes "reported rather than gated" an actual + # position rather than a way of not looking. + if args.summary_json: + Path(args.summary_json).write_text( + json.dumps( + { + "route": mine["continuation_route"], + "corelib_sha256": mine_sha, + "logits_bit_exact_steps": sum( + 1 for value in exact["logits"] if value + ), + "logits_total_steps": len(exact["logits"]), + "kv_bit_exact_tensors": sum( + 1 for value in exact["kv"] if value + ), + "kv_total_tensors": len(exact["kv"]), + "last_hidden_bit_exact": hidden_exact, + "all_bit_exact": all_exact, + "failures": failures.messages, + }, + indent=2, + ), + encoding="utf-8", + ) + + # Deliberately NOT compared: synchronize counts. FastFlow uses four + # synchronizes per layer by design and the reference still uses two, so + # equality there would mean FastFlow had regressed to a schedule design + # Section 10.4 rejected. The 129-per-step count is asserted inside the + # C++ harness against FastFlow's own contract instead. + if "final_metrics" in mine: + print( + "fastflow synchronize_count=" + f"{mine['final_metrics']['synchronize_count']} " + "(not compared against the reference: different schedules)" + ) + + return failures.report(f"compare[{mine['continuation_route']}]") + + +def self_consistency(args) -> int: + """Two runs of the same binary on the same input, per design 15.3. + + DETERM-1. The product does NOT claim run-to-run logit bit-identity, and + this check is written to that claim rather than to a stronger one nobody + can honour. What must hold exactly: + + * live K/V, last_hidden and every non-timing metric, bit-identical; + * the emitted token-ID sequence, identical; + * every logit within RUN_TO_RUN_MAX_ULPS BF16 ULP of the larger of + the two compared values -- a RELATIVE bound, because the + phenomenon is. + + What is recorded but not gated: the logit bit-identity rate and the + observed maximum difference. + + The reason for the split is measured, not conceded: run-to-run logit + bit-identity is not reliably true on this hardware, so gating on it would + make the suite intermittently red for a property the product does not + promise. + + CORRECTED BY TASK 13 BASELINE MEASUREMENT, 2026-09-02. This docstring used + to say the non-determinism was "confined to" the 3072 x 200064 LM-head + dispatch and "changes no decision", on the strength of ONE observed event: + 100389 of 200064 logits differing at decode[13] by at most 0.25, with + state, metrics and tokens identical. Two of those three claims are now + known false. + + IT IS NOT CONFINED TO THE LM HEAD. A reprefill event measured on + 2026-09-02 left layer 0 K and V bit-identical while layer 31 K and V and + last_hidden all differed, with the emitted tokens unchanged. The LM head + cannot write a layer-31 K cache, so the divergence entered the model body + at some layer above 0. Separately, an FP64 host reference computed from the + same ONNX components corelib packs from found the LM head to be a correctly + rounded function of its own input in every run measured: 200059 and 200060 + of 200064 logits within half a BF16 ULP of truth, mean 0.2497 ULP, signs + balanced to four parts in 200064. A correctly rounded function of identical + input cannot produce different output, so the earlier localisation -- an + INFERENCE from end-of-run state identity, never an observation of the + LM-head input at the diverging step -- does not hold. + + IT CAN CHANGE A DECISION. An append event in the same campaign produced + DIFFERENT EMITTED TOKEN SEQUENCES from two runs of the same binary on the + same input, diverging at decode step 7. + + What survives: both runs provably load the same DLL, and the phenomenon is + intermittent. For the rate, the spread and the n they rest on, see the + DETERM-3 baseline in docs/docs/benchmarks/phi4_results.md. + + The gates below are unchanged and they are what caught both events. Do not + weaken them. + """ + left = json.loads(Path(args.a).read_text(encoding="utf-8")) + right = json.loads(Path(args.b).read_text(encoding="utf-8")) + failures = Failures() + + # Same library, first. Without this the check cannot tell "the same binary + # twice" from "two runs that loaded different DLLs", and the second is a + # live hypothesis for the divergence in report section 5.1 -- so a pass + # here would have been evidence for a claim it never tested. + for field, label in ( + ("corelib_sha256", "corelib SHA-256"), + ("corelib_loaded_path", "loaded corelib path"), + # I-7. The corelib DLL was pinned and the FastFlow binary that drove + # it was not, so two records could describe different FastFlow builds + # with nothing able to tell. A determinism record pooled across a + # build tree needs both halves of "the same binary twice". + ("harness_sha256", "harness SHA-256"), + ): + a_value = left.get(field) + b_value = right.get(field) + failures.check( + a_value is not None and b_value is not None, + f"a run recorded no {label}, so this check cannot show the two " + f"runs used the same library", + ) + if a_value is not None and b_value is not None: + failures.check( + a_value == b_value, + f"the two runs used a different {label}:\n" + f" A {a_value}\n B {b_value}", + ) + if left.get("corelib_sha256"): + print(f"both runs loaded {left['corelib_sha256']}") + + failures.check( + left.get("prefix_ids") == right.get("prefix_ids") + and left.get("suffix_ids") == right.get("suffix_ids") + and left.get("continuation_route") == right.get("continuation_route"), + "the two runs were not given the same input", + ) + + def logit_steps(document): + steps = [("continuation", document["continuation"]["logits_bf16"])] + for index, step in enumerate(document["decode"]): + steps.append((f"decode[{index}]", step["logits_bf16"])) + return steps + + a_steps = logit_steps(left) + b_steps = logit_steps(right) + failures.check( + len(a_steps) == len(b_steps), + f"step counts differ: {len(a_steps)} vs {len(b_steps)}", + ) + + # THE LOCALISATION, MEASURED. Same step index as logit_steps, so the two + # lists line up. + def lm_head_inputs(document): + rows = [document["continuation"].get("lm_head_input_bf16")] + for step in document["decode"]: + rows.append(step.get("lm_head_input_bf16")) + return rows + + a_inputs = lm_head_inputs(left) + b_inputs = lm_head_inputs(right) + + first_divergence = None + first_divergence_index = None + bit_exact_steps = 0 + observed_max_diff = 0.0 + for index, ((label, a_bits), (_, b_bits)) in enumerate( + zip(a_steps, b_steps) + ): + if a_bits == b_bits: + bit_exact_steps += 1 + continue + differing = sum(1 for x, y in zip(a_bits, b_bits) if x != y) + mine = _widen_bf16(a_bits) + theirs = _widen_bf16(b_bits) + if mine.shape != theirs.shape: + failures.check( + False, + f"{label}: logit vector lengths differ, " + f"{mine.shape} vs {theirs.shape}", + ) + continue + difference = np.abs(mine - theirs) + step_max = float(np.max(difference)) + observed_max_diff = max(observed_max_diff, step_max) + if first_divergence is None: + first_divergence = label + first_divergence_index = index + + # GATED in ULP, RECORDED for the fact of differing. + # + # DETERM-1 does not promise bit-identity here, but it does bound the + # size of the difference -- and the bound is relative, because the + # phenomenon is. Comparing each element against 2 ULP of the larger of + # the two values treats a 2-ULP wobble the same whether the logit is 20 + # or 40; an absolute ceiling would have failed the second for nothing + # but magnitude. + allowed = _two_bf16_ulp(np.maximum(np.abs(mine), np.abs(theirs))) + # NaN compares false against everything, so a non-finite result on + # either side lands here rather than slipping through the <= . + over = ~(difference <= allowed) + over_count = int(np.count_nonzero(over)) + if over_count: + worst = int(np.argmax(np.where(over, difference, 0.0))) + failures.check( + False, + f"{label}: {over_count}/{len(a_bits)} logits differ by more " + f"than {RUN_TO_RUN_MAX_ULPS} BF16 ULP between two runs of the " + f"same binary. Worst at index {worst}: {mine[worst]:.9g} vs " + f"{theirs[worst]:.9g}, difference {difference[worst]:.6g}, " + f"allowed {allowed[worst]:.6g}. Do NOT widen the bound -- see " + f"_DETERM2_SEALED in this file; measure what changed.", + ) + print( + f" {label}: {differing}/{len(a_bits)} logits differ, " + f"max |diff| {step_max:.6g}, " + f"{'ALL' if not over_count else f'{len(a_bits) - over_count}/{len(a_bits)}'} " + f"within {RUN_TO_RUN_MAX_ULPS} ULP" + ) + + a_tokens = [left["continuation"]["top1_id"]] + [ + step["top1_id"] for step in left["decode"] + ] + b_tokens = [right["continuation"]["top1_id"]] + [ + step["top1_id"] for step in right["decode"] + ] + failures.check( + a_tokens == b_tokens, + f"emitted token sequences differ:\n {a_tokens}\n {b_tokens}", + ) + if first_divergence is not None: + print(f"first divergence at {first_divergence}") + + # The snapshot, not just the logits. The logits are one row out of the last + # LM-head dispatch; the K/V caches are the accumulated state of all 32 + # layers over every step, so they are where an intermittent divergence + # shows up earliest and most visibly. + a_snapshot = left.get("final_snapshot", {}) + b_snapshot = right.get("final_snapshot", {}) + for name in ( + "position", + "live_rows", + "last_hidden", + "layer0_k", + "layer0_v", + "layer31_k", + "layer31_v", + ): + a_value = a_snapshot.get(name) + b_value = b_snapshot.get(name) + if a_value is None or b_value is None: + failures.check(False, f"snapshot field {name} is missing") + continue + if a_value != b_value: + if isinstance(a_value, list) and len(a_value) == len(b_value): + differing = sum(1 for x, y in zip(a_value, b_value) if x != y) + failures.check( + False, + f"snapshot {name}: {differing}/{len(a_value)} elements " + f"differ between two runs of the same binary", + ) + else: + failures.check( + False, + f"snapshot {name}: {a_value!r} vs {b_value!r}", + ) + + # The counts too. These are the design 5.2 / 10.4 contract, and two runs + # that dispatched a different number of times are not the same run even if + # the numbers happened to land in the same place. + a_metrics = left.get("final_metrics", {}) + b_metrics = right.get("final_metrics", {}) + failures.check( + bool(a_metrics) and bool(b_metrics), + "a run recorded no final_metrics", + ) + for key in sorted(set(a_metrics) | set(b_metrics)): + # Wall-clock fields are not reproducible and are not part of the + # contract; everything else is a count or an extent and must match. + if key.endswith("_ns"): + continue + failures.check( + a_metrics.get(key) == b_metrics.get(key), + f"metric {key}: {a_metrics.get(key)} vs {b_metrics.get(key)}", + ) + + # DETERM-1 requires the rate and the observed maximum to be recorded every + # run, in the artifact and in the suite summary -- not merely printed. The + # gate above answers "is this within what we have measured"; this record is + # what makes "is the rate degrading from baseline" answerable at all, and + # that question cannot be asked of a number that only ever existed on + # stdout. + print( + f"run-to-run: logits bit-identical " + f"{bit_exact_steps}/{len(a_steps)} steps, " + f"max |diff| {observed_max_diff:.6g} " + f"(bound {RUN_TO_RUN_MAX_ULPS} BF16 ULP, relative)" + ) + if first_divergence is not None: + print(f"first divergence at {first_divergence}") + # WHERE THE DIVERGENCE ENTERED, asked of the recorded LM-head input at the + # step that actually diverged rather than inferred from end-of-run state. + # + # DETERM-1's argument was "identical LM-head input with non-identical + # LM-head output localises this to the 3072 x 200064 dispatch". The input + # half of that was never observed. This computes it: if the two runs fed + # the LM head the same 3072-element row and still produced different + # logits, the LM head is the source; if the rows differ, the divergence + # entered the model body and the LM head is faithfully transforming + # different inputs. + localisation = { + "measured": False, + "reason": "the runs recorded no per-step LM-head input", + } + if first_divergence_index is not None: + a_row = a_inputs[first_divergence_index] + b_row = b_inputs[first_divergence_index] + if a_row is None or b_row is None: + localisation["reason"] = ( + "a run predates the per-step LM-head input capture; rebuild " + "the end-to-end harness with DEV_BUILD and re-run" + ) + elif len(a_row) != len(b_row): + localisation["reason"] = ( + f"LM-head input lengths differ, {len(a_row)} vs {len(b_row)}" + ) + else: + differing = sum(1 for x, y in zip(a_row, b_row) if x != y) + localisation = { + "measured": True, + "step": first_divergence, + "lm_head_input_elements": len(a_row), + "lm_head_input_differing_elements": differing, + "source": "lm_head" if differing == 0 else "model_body", + # THE INSTRUMENT, recorded with the result. + # + # Capturing the LM-head input after every step adds a host + # tensor read and a stream acquisition between model steps, + # which changes the timing of exactly the window a race would + # live in. That is a caveat on every localisation measured + # this way, and a reader of a downstream document should not + # have to find it in a task report. It travels with the + # record so the document can state it from the data. + "measured_by": "per_step_lm_head_input_capture", + "instrumentation_effect": ( + "the capture adds a host tensor read and a stream " + "acquisition between every model step, which perturbs " + "the timing of the window a race would occupy" + ), + "reason": ( + "the two runs fed the LM head an identical row and it " + "produced different logits" + if differing == 0 + else ( + "the two runs fed the LM head DIFFERENT rows, so the " + "divergence entered before the LM head; DETERM-1's " + "localisation to the 3072 x 200064 dispatch does not " + "hold for this event" + ) + ), + } + print( + "localisation at " + f"{first_divergence}: {localisation.get('source', 'unmeasured')}" + f" -- {localisation['reason']}" + ) + if args.summary_json: + Path(args.summary_json).write_text( + json.dumps( + { + "route": left.get("continuation_route"), + "corelib_sha256": left.get("corelib_sha256"), + "harness_sha256": left.get("harness_sha256"), + "a": str(args.a), + "b": str(args.b), + "logits_bit_exact_steps": bit_exact_steps, + "logits_total_steps": len(a_steps), + "observed_max_abs_diff": observed_max_diff, + "determ2_bound_ulps": RUN_TO_RUN_MAX_ULPS, + "determ2_bound_kind": "relative_bf16_ulp", + "first_divergence": first_divergence, + "localisation": localisation, + "failures": failures.messages, + }, + indent=2, + ), + encoding="utf-8", + ) + + return failures.report( + f"self-consistency[{left['continuation_route']}]" + ) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="mode", required=True) + + emit = subparsers.add_parser("emit-reference") + emit.add_argument("--model-dir", required=True) + emit.add_argument("--token-ids-json", required=True) + emit.add_argument("--decode-steps", type=int, default=MIN_DECODE_STEPS) + emit.add_argument("--continuation-route", required=True) + emit.add_argument("--output-json", required=True) + emit.set_defaults(handler=emit_reference) + + check = subparsers.add_parser("compare") + check.add_argument("--fastflow-json", required=True) + check.add_argument("--reference-json", required=True) + check.add_argument( + "--expected-tokens", + help="JSON file of route-keyed expected top-1 token sequences, " + "committed to the repository so the golden is not re-derived from " + "the reference on every run", + ) + check.add_argument( + "--summary-json", + help="write the bit-exactness counts here, so the reported (ungated) " + "half of the comparison survives the run and can be aggregated", + ) + check.add_argument( + "--require-bit-exact", + action="store_true", + help="fail unless every logit vector, every live K/V tensor and the " + "final hidden state are bit-identical to the reference", + ) + check.set_defaults(handler=compare) + + repeat = subparsers.add_parser("self-consistency") + repeat.add_argument("--a", required=True) + repeat.add_argument("--b", required=True) + repeat.add_argument( + "--summary-json", + help="write the DETERM-1 record here: the run-to-run logit " + "bit-identity rate and the observed maximum difference", + ) + repeat.set_defaults(handler=self_consistency) + + args = parser.parse_args() + return args.handler(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/wix/flm.wxs b/src/wix/flm.wxs index bbd49a8d..4648a536 100644 --- a/src/wix/flm.wxs +++ b/src/wix/flm.wxs @@ -18,12 +18,13 @@ - + + + + + + + + + + @@ -69,6 +78,11 @@ + + + + + @@ -219,6 +233,26 @@ + + + + + + + + + + + + + + + + + + diff --git a/src/wix/get_files.bat b/src/wix/get_files.bat index a04b986f..10a526bc 100644 --- a/src/wix/get_files.bat +++ b/src/wix/get_files.bat @@ -22,4 +22,45 @@ echo Copying static assets... copy "..\inno\logo.ico" "package\logo.ico" copy "..\inno\terms.rtf" "package\terms.rtf" +REM Copy the optional, derived AIE4 runtime closure when one has been staged. +REM The AIE4 feature is optional, so a missing closure is a skip and not an +REM error: requiring it here would make the AIE4 build a precondition of +REM shipping the ordinary NPU2 product. +if exist "..\build\aie4\ryzenai_corelib.dll" ( + echo Copying optional AIE4 runtime closure... + if not exist "package\aie4" mkdir "package\aie4" + xcopy "..\build\aie4\*" "package\aie4\" /E /I /Y + REM The closure report ships beside the runtime. It is the only thing in + REM an installed tree that says which corelib produced a given result, so + REM it is copied deliberately and its arrival is checked -- the wholesale + REM xcopy above already carries it, and that is exactly why: an incidental + REM dependency breaks silently the day somebody narrows the copy. + copy /Y "..\build\aie4\aie4-closure.txt" "package\aie4\aie4-closure.txt" >nul + if not exist "package\aie4\aie4-closure.txt" goto :aie4_missing_closure_report + > "package\aie4.wxi" echo ^ + >> "package\aie4.wxi" echo ^ + >> "package\aie4.wxi" echo ^ + >> "package\aie4.wxi" echo ^ + >> "package\aie4.wxi" echo ^ +) else ( + echo No AIE4 runtime closure found; building without the AIE4 feature. + if not exist "package\aie4" mkdir "package\aie4" + > "package\aie4.wxi" echo ^ + >> "package\aie4.wxi" echo ^ + >> "package\aie4.wxi" echo ^ +) + echo Done! +exit /b 0 + +REM Reported by jumping out of the block rather than by exiting inside it. +REM cmd.exe discards the exit code of an `exit /b` that runs inside a +REM parenthesised block containing redirections -- the diagnostic prints, +REM the batch stops, and the caller still sees 0. Verified on the target: +REM the same nested `exit /b 1` returns 1 without the redirected lines and +REM 0 with them. +:aie4_missing_closure_report +echo ERROR: the staged AIE4 closure has no aie4-closure.txt, so the +echo installed runtime would carry no provenance. Re-run the AIE4 +echo staging step: cmake -P src/cmake/StageAie4Runtime.cmake +exit /b 1 diff --git a/tools/__init__.py b/tools/__init__.py new file mode 100644 index 00000000..1ddde4c7 --- /dev/null +++ b/tools/__init__.py @@ -0,0 +1 @@ +"""Offline FastFlowLM tooling.""" diff --git a/tools/calibrate_phi4_corelib_continuation.py b/tools/calibrate_phi4_corelib_continuation.py new file mode 100644 index 00000000..0f9e6f7c --- /dev/null +++ b/tools/calibrate_phi4_corelib_continuation.py @@ -0,0 +1,1349 @@ +"""Select the release-fixed Phi-4 continuation append threshold. Task 14. + +Design Section 10.7 fixes ONE integer, `kContinuationAppendThreshold`, applied +regardless of history length: + + append when `suffix_tokens <= continuation_append_threshold`; and + clear plus fresh re-prefill when the suffix is larger. + +and it fixes how that integer is chosen: + + the largest suffix length whose append p95 is lower at both history + lengths, but only after asserting that the set of winning sampled lengths + is a prefix-contiguous set. If the winners are non-monotonic or none wins, + the threshold is zero and every prefix hit re-prefills. + +This script implements exactly that sentence against the committed Task 13 +measurement. Three things about it are worth stating up front, because each is +a place where an obvious implementation would be wrong. + +**The conjunction is over histories, and it is not a worst-case reduction.** +"Append p95 is lower at both history lengths" means `append[h] < reprefill[h]` +for every `h`. Collapsing the histories first -- comparing the largest append +p95 against the smallest re-prefill p95 -- is strictly stronger and would +reject suffixes that genuinely win at both. + +**The suffix grid is a floor, not a whitelist.** Section 10.7 names suffix +lengths 1, 2, 32, 128 and 256. Task 13 measured that grid and reported the +threshold 2; that answer was retracted, because on `{1, 2, 32, ...}` the last +winning sampled length is 2 whether the true crossover is at 3 or at 31. The +denser 11-point grid Task 13 went on to measure resolves it. So the five named +lengths are REQUIRED to be present and every additional measured length is +USED. Dropping the extra points would faithfully reproduce a retracted answer. +The two history lengths are treated the other way round -- required and +exclusive -- because the conjunction ranges over them, so adding a third would +silently change the rule rather than sharpen it. + +**A non-interleaved record is refused outright.** In a non-interleaved run the +two routes are measured in separate blocks, so a machine regime shift lands on +one route and not the other. On the shared lab box that is not hypothetical: +Task 13 measured a 1.8x swing in decode throughput between runs of the same +binary within two hours. The first round of the crossover sweep was +non-interleaved and its answer was retracted. Refusing is not the same as +selecting zero: zero is a measured outcome, and a record that cannot support +any outcome is an input error. + +What this script does NOT do, deliberately: it does not emit a per-history +table, a cost model, or anything the runtime consults. Section 10.7 makes the +threshold a release-fixed backend constant, not model-package data and not a +runtime calibration, so the only artefact is one integer in a header. + +Usage:: + + python -m tools.calibrate_phi4_corelib_continuation \\ + --baseline docs/docs/benchmarks/phi4_aie4_baseline.json \\ + --crossover-history docs/docs/benchmarks/phi4_aie4_crossover_history.json \\ + --header src/include/models/phi4/phi4_corelib_aie4_tuning.hpp \\ + --document docs/docs/benchmarks/phi4_results.md [--check] +""" + +from __future__ import annotations + +import argparse +import dataclasses +import json +import math +import pathlib +import re +import sys +from typing import Iterable, Mapping, Sequence + +# Section 10.7 pins both history lengths. The winner rule is a conjunction over +# this set, so it is required AND exclusive: see the module docstring. +REQUIRED_HISTORIES = (512, 2048) + +# Section 10.7 pins these suffix lengths as the measurement grid. They are a +# floor: any additional measured length is used. See the module docstring for +# why -- the five-point grid produced the retracted answer "2". +REQUIRED_SUFFIXES = (1, 2, 32, 128, 256) + +MIN_WARM_SAMPLES = 5 + +ROUTES = ("append", "reprefill") + +_HEADER_BEGIN = "// BEGIN generated by tools/calibrate_phi4_corelib_continuation.py" +_HEADER_END = "// END generated by tools/calibrate_phi4_corelib_continuation.py" +_HEADER_DECL = re.compile( + r"^[ \t]*inline constexpr std::uint32_t kContinuationAppendThreshold" + r"[ \t]*=[ \t]*\d+[ \t]*;[ \t]*$", + re.MULTILINE, +) + +_DOCUMENT_BEGIN = "" +_DOCUMENT_END = "" + + +class CalibrationError(Exception): + """A record, header or document this script refuses to act on.""" + + +# -------------------------------------------------------------------------- +# Statistics +# -------------------------------------------------------------------------- + + +def percentile_ns(samples: Iterable[float], quantile: float) -> float: + """Nearest-rank percentile. + + Nearest-rank rather than an interpolating definition because that is what + the Task 13 recorder used, and this script cross-checks every recorded + p50/p95 against a recomputation from the raw samples. A different + definition here would turn that cross-check into noise. + """ + ordered = sorted(samples) + if not ordered: + raise CalibrationError("cannot take a percentile of zero samples") + if not 0.0 < quantile <= 1.0: + raise CalibrationError(f"quantile {quantile!r} is outside (0, 1]") + rank = math.ceil(quantile * len(ordered)) + return ordered[min(len(ordered) - 1, max(0, rank - 1))] + + +# -------------------------------------------------------------------------- +# The Section 10.7 selection rule +# -------------------------------------------------------------------------- + + +@dataclasses.dataclass(frozen=True) +class Selection: + threshold: int + verdict: str + winners: tuple + ordered: tuple + diagnostics: tuple + + +def _is_int(value) -> bool: + return isinstance(value, int) and not isinstance(value, bool) + + +def _number(entry: Mapping, key: str, where: str) -> float: + if key not in entry: + raise CalibrationError(f"{where}: missing {key!r}") + value = entry[key] + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise CalibrationError(f"{where}: {key!r} is not a number: {value!r}") + return float(value) + + +def _entry_wins(suffix, entry: Mapping) -> bool: + """Does append beat re-prefill at EVERY history recorded for `suffix`? + + Two entry shapes are accepted and they must not be mixed: + + * `{"append_p95": x, "reprefill_p95": y}` -- one history; and + * `{"per_history": {h: {"append_p95": x, "reprefill_p95": y}, ...}}`. + + The comparison is strict. Section 10.7 says append p95 must be LOWER, so a + tie is not a win and routes to re-prefill. + """ + where = f"suffix {suffix}" + if not isinstance(entry, Mapping): + raise CalibrationError(f"{where}: sample entry is not a mapping") + flat = "append_p95" in entry or "reprefill_p95" in entry + nested = "per_history" in entry + if flat and nested: + raise CalibrationError( + f"{where}: entry mixes the flat and per-history shapes; pick one" + ) + if not flat and not nested: + raise CalibrationError( + f"{where}: entry has neither 'append_p95'/'reprefill_p95' nor " + f"'per_history'" + ) + if flat: + return _number(entry, "append_p95", where) < _number( + entry, "reprefill_p95", where + ) + + per_history = entry["per_history"] + if not isinstance(per_history, Mapping) or not per_history: + raise CalibrationError(f"{where}: 'per_history' is empty") + return all( + _number(measured, "append_p95", f"{where} history {history}") + < _number(measured, "reprefill_p95", f"{where} history {history}") + for history, measured in sorted(per_history.items()) + ) + + +def select_threshold_detailed(samples: Mapping) -> Selection: + """Apply the Section 10.7 rule and say which branch it took. + + Verdicts: `empty`, `no_winner`, `non_monotonic`, `selected`. The first + three all yield threshold zero -- every prefix hit re-prefills -- and only + `non_monotonic` is a signal that something is wrong. With an unstable + upper bracket edge, a non-prefix winner set says the machine moved during + the sweep, not that append got faster at a longer suffix than a shorter + one. + """ + for suffix in samples: + if not _is_int(suffix): + raise CalibrationError(f"suffix length {suffix!r} is not an integer") + if suffix <= 0: + raise CalibrationError(f"suffix length {suffix!r} is not positive") + + ordered = tuple(sorted(samples)) + if not ordered: + return Selection( + 0, + "empty", + (), + (), + ("no sampled suffix lengths; every prefix hit re-prefills",), + ) + + winners = tuple(suffix for suffix in ordered if _entry_wins(suffix, samples[suffix])) + if not winners: + return Selection( + 0, + "no_winner", + (), + ordered, + ( + "append p95 beat re-prefill p95 at no sampled suffix length; " + "every prefix hit re-prefills", + ), + ) + + prefix = ordered[: len(winners)] + if winners != prefix: + losses = [suffix for suffix in prefix if suffix not in winners] + late = [suffix for suffix in winners if suffix not in prefix] + return Selection( + 0, + "non_monotonic", + winners, + ordered, + ( + "the winning suffix lengths are not a prefix of the sampled " + f"set: append lost at {losses} and then won again at {late}. " + "Append cost grows with suffix length, so this is the machine " + "moving during the sweep, not a real re-crossing. Threshold " + "is zero and every prefix hit re-prefills.", + ), + ) + + return Selection(int(max(winners)), "selected", winners, ordered, ()) + + +def select_threshold(samples: Mapping) -> int: + """The Section 10.7 rule, as one integer.""" + return select_threshold_detailed(samples).threshold + + +# -------------------------------------------------------------------------- +# Ingestion +# -------------------------------------------------------------------------- + + +@dataclasses.dataclass(frozen=True) +class RoutePoint: + history_rows: int + suffix: int + route: str + count: int + p50_ns: float + p95_ns: float + + +@dataclasses.dataclass(frozen=True) +class Measurements: + histories: tuple + suffixes: tuple + points: Mapping + identity: Mapping + recorded_crossover: Mapping + decision_rule: str + + def point(self, history: int, suffix: int, route: str) -> RoutePoint: + return self.points[(history, suffix)][route] + + def samples(self) -> dict: + """The `select_threshold` input, in the per-history shape.""" + return { + suffix: { + "per_history": { + history: { + "append_p95": self.point(history, suffix, "append").p95_ns, + "reprefill_p95": self.point( + history, suffix, "reprefill" + ).p95_ns, + } + for history in self.histories + } + } + for suffix in self.suffixes + } + + def winners_by_history(self) -> dict: + """Per history, the suffix lengths at which append p95 wins.""" + return { + history: tuple( + suffix + for suffix in self.suffixes + if self.point(history, suffix, "append").p95_ns + < self.point(history, suffix, "reprefill").p95_ns + ) + for history in self.histories + } + + def unclosed_histories(self) -> tuple: + """Histories whose own winner set is NOT downward-closed. + + Section 10.7 asserts monotonicity of the CONJUNCTION, and that is the + gate. This is the same property one level down, per history, and it is + worth checking for two reasons. + + It carries the same meaning: append cost grows with suffix length, so a + loss at a short suffix followed by a win at a longer one at the SAME + history is the machine moving during the sweep. + + And it is the exact condition under which `min` over per-history + ceilings stops equalling the Section 10.7 answer -- the identity the + cross-run bound in `history_agreement` would need in order to be an + equality rather than an upper bound. Here the winner sets are known, so + unlike in a stored crossover record the property can actually be + checked. + """ + found = [] + for history, won in self.winners_by_history().items(): + if won != self.suffixes[: len(won)]: + missing = [s for s in self.suffixes[: len(won)] if s not in won] + found.append( + f"history {history}: append lost at {missing} and won again " + f"at a longer suffix; winners {list(won)} are not a prefix " + f"of the sampled set {list(self.suffixes)}" + ) + return tuple(found) + + def crossover_disagreements(self, selection: Selection) -> tuple: + """Compare the p95 rule against the recorder's own p50-based verdict. + + Task 13's `append_wins_up_to` was decided by a different statistic -- a + p50 gap that had to clear both the within-point spread and the drift + across the point. Section 10.7's rule is p95. They should agree; where + they do not, the constant rests on a statistic the published bracket + contradicts, and that has to be said rather than resolved silently. + """ + if not self.recorded_crossover: + return () + found = [] + winners = self.winners_by_history() + for history in self.histories: + recorded = self.recorded_crossover.get(str(history)) + if not isinstance(recorded, Mapping): + continue + if "append_wins_up_to" not in recorded: + continue + declared = recorded["append_wins_up_to"] + ours = max(winners[history]) if winners[history] else 0 + if declared is None: + declared = 0 + if declared != ours: + found.append( + f"history {history}: the recorded bracket says append wins " + f"up to {declared}, the Section 10.7 p95 rule says {ours}" + ) + return tuple(found) + + +def _require_point_fields(raw, index: int) -> RoutePoint: + where = f"continuation point {index}" + if not isinstance(raw, Mapping): + raise CalibrationError(f"{where}: not an object") + for field in ("history_rows", "suffix", "route", "count", "samples_ns"): + if field not in raw: + raise CalibrationError(f"{where}: missing {field!r}") + history = raw["history_rows"] + suffix = raw["suffix"] + if not _is_int(history) or not _is_int(suffix): + raise CalibrationError(f"{where}: history_rows/suffix must be integers") + if suffix <= 0 or history <= 0: + raise CalibrationError(f"{where}: history_rows/suffix must be positive") + route = raw["route"] + if route not in ROUTES: + raise CalibrationError( + f"{where}: unknown route {route!r}; expected one of {list(ROUTES)}" + ) + + where = f"history {history} suffix {suffix} route {route}" + samples = raw["samples_ns"] + if not isinstance(samples, Sequence) or isinstance(samples, (str, bytes)): + raise CalibrationError(f"{where}: 'samples_ns' is not a list") + for sample in samples: + if isinstance(sample, bool) or not isinstance(sample, (int, float)): + raise CalibrationError(f"{where}: non-numeric sample {sample!r}") + if sample <= 0: + raise CalibrationError(f"{where}: non-positive sample {sample!r}") + if raw["count"] != len(samples): + raise CalibrationError( + f"{where}: recorded count {raw['count']} but " + f"{len(samples)} samples" + ) + if len(samples) < MIN_WARM_SAMPLES: + raise CalibrationError( + f"{where}: {len(samples)} warm samples, Section 10.7 requires at " + f"least {MIN_WARM_SAMPLES}" + ) + + p50 = percentile_ns(samples, 0.5) + p95 = percentile_ns(samples, 0.95) + for field, recomputed in (("p50_ns", p50), ("p95_ns", p95)): + if field in raw and raw[field] != recomputed: + raise CalibrationError( + f"{where}: recorded {field} {raw[field]} does not match the " + f"nearest-rank value {recomputed} recomputed from samples_ns" + ) + return RoutePoint(history, suffix, route, len(samples), p50, p95) + + +def load_continuation_samples(document: Mapping) -> Measurements: + """Validate a Task 13 baseline document and reduce it to p50/p95 points.""" + if not isinstance(document, Mapping): + raise CalibrationError("baseline document is not an object") + block = document.get("continuation") + if not isinstance(block, Mapping): + raise CalibrationError("baseline document has no 'continuation' object") + + interleaved = block.get("samples_interleaved") + if interleaved is not True: + raise CalibrationError( + "this record is not marked samples_interleaved=true. Append and " + "re-prefill measured in separate blocks cannot be compared on a " + "machine whose throughput has been measured to swing 1.8x between " + "runs; the first, non-interleaved crossover round was retracted " + f"for exactly this. Got samples_interleaved={interleaved!r}." + ) + + declared_floor = block.get("warm_samples_per_point") + if not _is_int(declared_floor) or declared_floor < MIN_WARM_SAMPLES: + raise CalibrationError( + f"warm_samples_per_point is {declared_floor!r}; Section 10.7 " + f"requires at least {MIN_WARM_SAMPLES}" + ) + + raw_points = block.get("points") + if not isinstance(raw_points, Sequence) or not raw_points: + raise CalibrationError("'continuation.points' is missing or empty") + + points: dict = {} + for index, raw in enumerate(raw_points): + point = _require_point_fields(raw, index) + bucket = points.setdefault((point.history_rows, point.suffix), {}) + if point.route in bucket: + raise CalibrationError( + f"duplicate point: history {point.history_rows} suffix " + f"{point.suffix} route {point.route} appears more than once" + ) + bucket[point.route] = point + + histories = sorted({key[0] for key in points}) + unexpected = [h for h in histories if h not in REQUIRED_HISTORIES] + if unexpected: + raise CalibrationError( + f"unexpected history length(s) {unexpected}: Section 10.7's rule is " + f"a conjunction over exactly {list(REQUIRED_HISTORIES)}, so an " + f"extra history would change the rule rather than sharpen it" + ) + missing = [h for h in REQUIRED_HISTORIES if h not in histories] + if missing: + raise CalibrationError( + f"missing required history length(s) {missing}; Section 10.7 " + f"requires {list(REQUIRED_HISTORIES)}" + ) + + suffixes = sorted({key[1] for key in points}) + missing = [s for s in REQUIRED_SUFFIXES if s not in suffixes] + if missing: + raise CalibrationError( + f"missing required suffix length(s) {missing}; Section 10.7 " + f"requires at least {list(REQUIRED_SUFFIXES)}" + ) + + for history in REQUIRED_HISTORIES: + for suffix in suffixes: + bucket = points.get((history, suffix), {}) + for route in ROUTES: + if route not in bucket: + raise CalibrationError( + f"history {history} suffix {suffix}: no {route} point; " + f"both routes are needed to compare them" + ) + + crossover = block.get("crossover") + return Measurements( + histories=tuple(sorted(REQUIRED_HISTORIES)), + suffixes=tuple(suffixes), + points=points, + identity=document.get("identity") or {}, + recorded_crossover=crossover if isinstance(crossover, Mapping) else {}, + decision_rule=str(block.get("decision_rule") or ""), + ) + + +# -------------------------------------------------------------------------- +# Cross-run agreement +# -------------------------------------------------------------------------- + + +@dataclasses.dataclass(frozen=True) +class HistoryRow: + utc: str + interleaved: bool + comparable: bool + # `min` over that run's per-history winner ceilings. An UPPER BOUND on what + # Section 10.7 would have selected from it, not the selection itself -- see + # `history_agreement` for why the two differ. + upper_bound: int + permits: bool + tight: bool + note: str + # The per-history lower edges this run recorded, as `(history, lower)` + # pairs in `REQUIRED_HISTORIES` order, kept rather than reduced away. + # + # `upper_bound` is their minimum, and the minimum cannot answer the + # question the document was asking of it: whether the lower edge is the + # same value from run to run. The renderer asserted an answer instead -- + # "Task 13 measured the lower edges to be stable" -- which was true when it + # was written and false once a third run recorded 24 where the first two + # recorded 12, and the generator had thrown away the only data that could + # have told it so. + lower_edges: tuple = () + + +@dataclasses.dataclass(frozen=True) +class HistoryReport: + rows: tuple + comparable: bool + # Every interleaved run bounds the threshold at or above its value. + permits: bool + # The interleaved runs whose bound the threshold exactly meets. + tight: tuple + diagnostics: tuple + + +def history_agreement(records: Sequence[Mapping], threshold: int) -> HistoryReport: + """Do the earlier runs PERMIT this constant? Not: would they select it. + + Each crossover record carries, per history, the bracket `[lower, upper]`. + The lower edge is the largest suffix at which append decisively won, so it + is that run's per-history winner CEILING. The minimum of those ceilings is + an **upper bound** on what Section 10.7 would have selected from that run, + and in general it is only an upper bound. + + The reason matters, because the obvious reading is wrong. Section 10.7 + selects the largest suffix in the INTERSECTION of the per-history winner + sets, and `min` of the ceilings equals that only when each winner set is + downward-closed. It need not be. Take sampled `{1, 2, 4, 8}` with + `W512 = {1, 8}` and `W2048 = {1, 2, 4}`: the intersection is `{1}`, which + is a prefix, so the monotonicity assertion passes and the rule returns 1 -- + while `min(8, 4)` is 4. A crossover record stores only the edges, never the + winner sets, so downward-closure CANNOT be verified from it. + + What is sound without that assumption is the inequality: any suffix that + wins at every history is at most every ceiling, hence at most their + minimum. So this function checks + + threshold <= min(ceilings) -- sound, enforced + threshold == min(ceilings) -- "tight", reported + + A run whose bound the threshold EXCEEDS contradicts it outright and is a + hard diagnostic. A run whose bound it merely meets is consistent, and is + reported as tight only under the stated assumption. A run whose bound it + falls below permits the constant without confirming it. + + This bound touches only the lower edges, so an unstable UPPER edge cannot + move it. That is a property of the construction and is all this docstring + ever meant to claim. It used to add that Task 13 had found the lower edges + stable, which was a claim about data rather than about the bound, and it + went false when a later run recorded a different lower edge -- so the + lower edges are now carried through on `HistoryRow.lower_edges` and + whether they held still is derived where it is published, never asserted + here. Non-interleaved records are shown but excluded from the verdict. + """ + if not _is_int(threshold) or threshold < 0: + raise CalibrationError(f"threshold {threshold!r} is not a non-negative int") + + rows = [] + for index, record in enumerate(records): + if not isinstance(record, Mapping): + raise CalibrationError(f"crossover history record {index} is not an object") + edges = record.get("edges") + if not isinstance(edges, Mapping): + raise CalibrationError( + f"crossover history record {index} has no 'edges' object" + ) + interleaved = record.get("samples_interleaved") is True + lowers = [] + for history in REQUIRED_HISTORIES: + pair = edges.get(str(history), edges.get(history)) + if not isinstance(pair, Sequence) or len(pair) != 2: + raise CalibrationError( + f"crossover history record {index}: no [lower, upper] edge " + f"pair for history {history}" + ) + lower = pair[0] + if lower is None: + lowers.append(0) + elif _is_int(lower) and lower >= 0: + lowers.append(lower) + else: + raise CalibrationError( + f"crossover history record {index}: history {history} lower " + f"edge {lower!r} is not a non-negative integer or null" + ) + upper_bound = min(lowers) + rows.append( + HistoryRow( + utc=str(record.get("utc", "")), + interleaved=interleaved, + comparable=interleaved, + upper_bound=upper_bound, + permits=interleaved and threshold <= upper_bound, + tight=interleaved and threshold == upper_bound, + note=str(record.get("note", "")), + lower_edges=tuple(zip(REQUIRED_HISTORIES, lowers)), + ) + ) + + comparable_rows = [row for row in rows if row.comparable] + diagnostics = [] + if not comparable_rows: + diagnostics.append( + "no interleaved run is on record, so nothing cross-checks this " + "constant; a non-interleaved run cannot, because its two routes " + "were not measured against the same machine state" + ) + contradicting = [row for row in comparable_rows if not row.permits] + if contradicting: + diagnostics.append( + "the constant exceeds what an interleaved run can support: " + + ", ".join( + f"{row.utc} bounds it at {row.upper_bound}" for row in contradicting + ) + + f", and the selected threshold is {threshold}. At that run, some " + "history had already stopped favouring append below this suffix " + "length. A release-fixed constant that one recorded run " + "contradicts is a decision for a human, not for this script." + ) + return HistoryReport( + rows=tuple(rows), + comparable=bool(comparable_rows), + permits=bool(comparable_rows) and not contradicting, + tight=tuple(row for row in comparable_rows if row.tight), + diagnostics=tuple(diagnostics), + ) + + +def recorded_lower_edges(agreement: HistoryReport) -> dict: + """Per history, the DISTINCT lower edges the interleaved runs recorded. + + `{512: (4,), 2048: (12, 24)}` reads: at history 512 every comparable run + put the lower edge at 4; at history 2048 the runs disagree, so the edge is + not a measured constant and no sentence may call it one. Non-interleaved + runs are excluded, exactly as they are from the verdict. + """ + observed: dict = {} + for row in agreement.rows: + if not row.comparable: + continue + for history, lower in row.lower_edges: + observed.setdefault(history, set()).add(lower) + return { + history: tuple(sorted(values)) + for history, values in sorted(observed.items()) + } + + +def _edge_list(observed: Mapping) -> str: + return "; ".join( + f"{', '.join(str(value) for value in values)} at history {history}" + for history, values in observed.items() + ) + + +def lower_edge_confidence(selection: Selection, agreement: HistoryReport) -> str: + """The confidence argument for the selected constant, DERIVED. + + This paragraph used to be a constant string asserting that Task 13 had + found the lower edge stable and the upper edge not. Two things were wrong + with that even before it was falsified: it said "three times" while the + record it sat above showed two comparable runs, and it could not change + when the data did. It is now computed from the same rows the table below + it prints, so the two cannot disagree. + """ + comparable = [row for row in agreement.rows if row.comparable] + if not comparable: + return ( + "There is a second consideration, and on this record it cannot be " + "settled: no interleaved run is on record, so nothing cross-checks " + "where the crossover sat on any other run." + ) + observed = recorded_lower_edges(agreement) + moved = { + history: values for history, values in observed.items() if len(values) > 1 + } + lead = ( + f"There is a second reason to prefer the lower candidate, and it is " + f"about confidence rather than cost. Across the {len(comparable)} " + f"interleaved run(s) on record the bracket's LOWER edge has been " + f"{_edge_list(observed)}." + ) + if moved: + lead += ( + f" It is NOT a measured constant: at " + f"{', '.join(f'history {history}' for history in moved)} it took " + f"more than one value across runs of the same binary, so a " + f"threshold cannot be justified by pointing at one run's edge." + ) + else: + lead += ( + " Every history reports one value, so on this record the lower " + "edge has not moved between runs — which is a property of these " + "runs and not a guarantee about the next one." + ) + if agreement.permits: + return lead + ( + f" What carries the choice is the inequality rather than the " + f"stability: {selection.threshold} is at or below EVERY recorded " + f"lower edge, so it is inside the append-wins region on every " + f"interleaved run on record. A larger threshold would be supported " + f"by some of them and contradicted by others." + ) + return lead + ( + f" And {selection.threshold} is NOT at or below every recorded lower " + f"edge — see the refusal below. That is the condition this argument " + f"needs and this run does not meet." + ) + + +# -------------------------------------------------------------------------- +# The generated header +# -------------------------------------------------------------------------- + + +def apply_threshold_to_header( + source: str, + threshold: int, + provenance: Sequence[str], +) -> str: + """Replace the generated threshold block in an existing header. + + The whole file is NOT overwritten. `phi4_corelib_aie4_tuning.hpp` also + holds `SelectContinuationRoute`, `ContinuationRoute` and + `ForcedContinuationRoute`, all hand-written and all depended on by the + frontend; regenerating the file wholesale would delete them. Only the + declaration, and the marker block around it, are generated. + """ + if not _is_int(threshold) or threshold < 0: + raise CalibrationError(f"threshold {threshold!r} is not a non-negative int") + + lines = [_HEADER_BEGIN, "//"] + # `f"// {line}"` on a blank provenance line would leave trailing + # whitespace, which some editors and hooks strip -- and a stripped file + # would then fail `--check` for a reason that has nothing to do with the + # measurement. + lines += [f"// {line}".rstrip() for line in provenance] + lines.append( + f"inline constexpr std::uint32_t kContinuationAppendThreshold " + f"= {threshold};" + ) + lines.append(_HEADER_END) + block = "\n".join(lines) + + begin = source.find(_HEADER_BEGIN) + end = source.find(_HEADER_END) + if begin != -1 and end != -1: + if source.count(_HEADER_BEGIN) != 1 or source.count(_HEADER_END) != 1: + raise CalibrationError("header has more than one generated block") + return source[:begin] + block + source[end + len(_HEADER_END) :] + if begin != -1 or end != -1: + raise CalibrationError("header has an unterminated generated block") + + matches = _HEADER_DECL.findall(source) + if not matches: + raise CalibrationError( + "header has no 'inline constexpr std::uint32_t " + "kContinuationAppendThreshold = ;' declaration to replace" + ) + if len(matches) > 1: + raise CalibrationError( + f"header declares kContinuationAppendThreshold {len(matches)} times" + ) + return _HEADER_DECL.sub(lambda _: block, source, count=1) + + +# -------------------------------------------------------------------------- +# The published evidence +# -------------------------------------------------------------------------- + + +def _ms(nanoseconds: float) -> str: + return f"{nanoseconds / 1e6:,.1f} ms" + + +def conceded_points(measurements: Measurements, threshold: int) -> tuple: + """What the single constant gives up, worst cost first. + + A conceded point is a `(history, suffix)` where append p95 beat re-prefill + p95 but the suffix exceeds the threshold, so the route goes to re-prefill + anyway. Ordered by SLOWDOWN, descending -- not by suffix. + + That ordering is the point. The penalty is `reprefill - append`; append + grows with suffix length and re-prefill is roughly flat in it, so within a + conceded band the NARROWEST suffix is the most expensive one. Quoting the + widest conceded suffix understates the cost, which is the mistake this + function exists to prevent. + + Returns `(history, suffix, penalty_ns, slowdown)` tuples. + """ + winners = measurements.winners_by_history() + points = [] + for history in measurements.histories: + for suffix in winners[history]: + if suffix <= threshold: + continue + append = measurements.point(history, suffix, "append").p95_ns + reprefill = measurements.point(history, suffix, "reprefill").p95_ns + points.append( + ( + history, + suffix, + reprefill - append, + reprefill / append if append else float("inf"), + ) + ) + return tuple(sorted(points, key=lambda row: (-row[3], row[0], row[1]))) + + +def _identity_rows(identity: Mapping) -> list: + wanted = ( + ("machine", "machine"), + ("CPU", "cpu_sku"), + ("NPU", "npu_sku"), + ("NPU driver", "npu_driver_version"), + ("corelib SHA-256", "corelib_dll_sha256"), + ("corelib source revision", "corelib_source_revision"), + ("model SHA-256", "model_sha256"), + ("FastFlow revision", "fastflow_revision"), + ("measured (UTC)", "utc"), + ) + return [ + (label, str(identity[key])) + for label, key in wanted + if identity.get(key) is not None + ] + + +def build_document_section( + measurements: Measurements, + selection: Selection, + agreement: HistoryReport, +) -> str: + """Render the Section 15.6 telemetry for the selected constant.""" + winners = measurements.winners_by_history() + unclosed = measurements.unclosed_histories() + out = [] + add = out.append + + add(_DOCUMENT_BEGIN) + add("") + add("## Phi-4 continuation routing — the release-fixed threshold") + add("") + add( + "Design Section 10.7 fixes ONE integer for the release and applies it " + "at every history length: append when the suffix is at most the " + "threshold, clear and re-prefill when it is larger. It is a FastFlow " + "backend constant — not model-package data, not a runtime " + "calibration — so route choice does not drift with thermal state or " + "load." + ) + add("") + add(f"```cpp") + add(f"inline constexpr std::uint32_t kContinuationAppendThreshold = " + f"{selection.threshold};") + add("```") + add("") + + if selection.verdict == "selected": + add( + f"**Selected: {selection.threshold}.** Suffix lengths " + f"{list(selection.winners)} are the sampled lengths whose append " + f"p95 is lower than re-prefill p95 at BOTH measured history " + f"lengths, and they form a prefix of the sampled set " + f"{list(selection.ordered)}, so the largest of them is the " + f"threshold." + ) + else: + add( + f"**Selected: 0 — every prefix hit re-prefills.** " + + " ".join(selection.diagnostics) + ) + add("") + + add("### Which sampled lengths append wins, per history") + add("") + add("| history rows | append p95 wins at | last winning length |") + add("| ---: | :--- | ---: |") + for history in measurements.histories: + won = winners[history] + add( + f"| {history} | {list(won) if won else 'none'} | " + f"{max(won) if won else 0} |" + ) + add("") + # NOT "so the constant is the smaller of those ceilings". The rule takes + # the largest suffix in the INTERSECTION of the winner sets, which coincides + # with the smaller ceiling only when the sets are downward-closed -- a + # property of this run's data, checked below, and not a consequence of the + # rule. Stating it causally here is the inference the calibrator spent a + # review round removing, and this is the first prose a reader meets. + add( + f"The rule is a conjunction: the constant is the largest sampled length " + f"in the INTERSECTION of those winner sets — here " + f"{list(selection.winners) if selection.winners else 'empty'}, giving " + f"{selection.threshold}. It is not in general the smaller of the two " + f"ceilings; that holds only when each winner set is downward-closed, " + f"which is a property of the measurement rather than of the rule and is " + f"checked further down. It is also NOT a per-history policy: Section " + f"10.7 specifies one integer, and one integer cannot be optimal at two " + f"history lengths whose crossovers differ." + ) + add("") + + add("### What the single constant gives up") + add("") + concessions = conceded_points(measurements, selection.threshold) + if concessions: + # EVERY conceded point is listed, worst cost first. Naming a + # representative one would understate the cost: the penalty is + # `reprefill - append`, append grows with suffix length and re-prefill + # does not, so the most expensive point in a conceded band is its + # NARROWEST suffix, not its widest. + worst = concessions[0] + add( + f"**Worst case: history {worst[0]}, suffix {worst[1]} — " + f"{worst[3]:,.1f}x slower, +{_ms(worst[2])}.** Every conceded point " + f"follows, ordered by slowdown. The widest conceded suffix is not " + f"the most expensive one, because append cost grows with suffix " + f"length while re-prefill cost does not — so the band's cost must " + f"be read off its narrow end." + ) + add("") + add( + "| history rows | suffix | append p95 | re-prefill p95 | route " + "taken | penalty | slowdown |" + ) + add("| ---: | ---: | ---: | ---: | :--- | ---: | ---: |") + for history, suffix, penalty, slowdown in concessions: + add( + f"| {history} | {suffix} | " + f"{_ms(measurements.point(history, suffix, 'append').p95_ns)} | " + f"{_ms(measurements.point(history, suffix, 'reprefill').p95_ns)} " + f"| re-prefill | +{_ms(penalty)} | {slowdown:,.1f}x |" + ) + for history in measurements.histories: + if not any(row[0] == history for row in concessions): + add(f"| {history} | none | — | — | — | — | — |") + add("") + add( + "Those are measured losses taken deliberately. The alternative — a " + "threshold above the selected one — is optimal for the longer " + "history and WRONG for the shorter one, where it would append past " + "the point at which append has already lost. Conceding measured " + "throughput at one history is the cheaper error than routing " + "against the measurement at the other." + ) + add("") + add(lower_edge_confidence(selection, agreement)) + else: + add( + "Nothing. At every measured history length, append wins at no " + "sampled suffix length beyond the threshold, so the single " + "constant costs nothing that the measurement can see." + ) + add("") + + add("### The measurement behind it") + add("") + add("| history rows | suffix | append p50 | append p95 | re-prefill p50 | " + "re-prefill p95 | append p95 wins |") + add("| ---: | ---: | ---: | ---: | ---: | ---: | :---: |") + for history in measurements.histories: + for suffix in measurements.suffixes: + append = measurements.point(history, suffix, "append") + reprefill = measurements.point(history, suffix, "reprefill") + add( + f"| {history} | {suffix} | {_ms(append.p50_ns)} | " + f"{_ms(append.p95_ns)} | {_ms(reprefill.p50_ns)} | " + f"{_ms(reprefill.p95_ns)} | " + f"{'yes' if append.p95_ns < reprefill.p95_ns else 'no'} |" + ) + add("") + add( + f"At least {MIN_WARM_SAMPLES} warm samples per route per point, " + f"append and re-prefill interleaved WITHIN each point. p50 and p95 are " + f"nearest-rank and were recomputed from the raw samples by the " + f"calibrator, not copied." + ) + add("") + + add("### Do the earlier runs permit this constant?") + add("") + add( + "Not: would they have selected it. Each recorded crossover run carries, " + "per history, the bracket `[lower, upper]`, and the lower edge is that " + "run's per-history winner CEILING. The minimum of those ceilings is an " + "**upper bound** on what Section 10.7 would have selected from that " + "run, and in general only an upper bound: the rule takes the largest " + "suffix in the INTERSECTION of the winner sets, which equals the " + "minimum of the ceilings only when each winner set is downward-closed. " + "A crossover record stores the edges and never the winner sets, so that " + "property cannot be checked from it. What holds with no assumption is " + "the inequality — any suffix winning at every history is at most every " + "ceiling — so that is what is enforced here." + ) + add("") + add( + "This bound touches only the lower edges and never the upper ones, so " + "however far an upper edge moves between runs it cannot move the " + "bound. The lower edges are printed per run below rather than reduced " + "away, because their MINIMUM cannot answer whether the edge itself " + "held still: where a history shows different values on different runs " + "the edge moved, the bound is still their minimum, and the inequality " + "is still sound — but nothing here may then call that edge stable." + ) + add("") + add("| measured (UTC) | interleaved | lower edge per history | bounds the " + f"threshold at | permits {selection.threshold} | meets the bound " + f"exactly |") + add("| --- | :---: | :--- | ---: | :---: | :---: |") + for row in agreement.rows: + edges = ", ".join( + f"{history}: {lower}" for history, lower in row.lower_edges + ) + if not row.comparable: + add( + f"| {row.utc} | no | {edges} | ({row.upper_bound}) | excluded — " + f"routes not measured against the same machine state | — |" + ) + else: + add( + f"| {row.utc} | yes | {edges} | {row.upper_bound} | " + f"{'yes' if row.permits else 'NO'} | " + f"{'yes' if row.tight else 'no'} |" + ) + add("") + if agreement.permits: + tight = len(agreement.tight) + total = sum(1 for row in agreement.rows if row.comparable) + add( + f"Every interleaved run on record permits {selection.threshold}, " + f"and {tight} of {total} bound it there exactly. An exactly-met " + f"bound is what " + f"\"that run would have selected the same constant\" would need, " + f"but only under the downward-closure assumption above, which its " + f"record does not attest." + ) + else: + for line in agreement.diagnostics: + add(f"**{line}**") + add("") + if unclosed: + add( + "**This run's own winner sets are not all downward-closed**, which " + "is the condition under which the bound above stops being an " + "equality — and, more importantly, means append lost at a short " + "suffix and won again at a longer one:" + ) + for line in unclosed: + add(f"- {line}") + else: + add( + "This run's own per-history winner sets ARE downward-closed at " + "every history, checked directly from the sampled points rather " + "than assumed. That is only a property of this run; it says nothing " + "about the earlier ones, whose winner sets were never recorded." + ) + add("") + add( + "The non-interleaved rows are shown for provenance and excluded from " + "the verdict, because their two routes were measured in separate " + "blocks and a machine regime shift there lands on one route and not " + "the other. The first of them bounds the constant lower for a reason " + "that has nothing to do with the machine: it swept only the five " + "suffix lengths Section 10.7 names, and on `{1, 2, 32, 128, 256}` the " + "last winning sampled length is 2 whether the true crossover is at 3 " + "or at 31. **Two earlier answers are withdrawn and must not be " + "reused: the threshold `2` from that sparse grid, and the " + "extrapolated figures `≈9 at history 512, ≈26 at history 2048` from a " + "contended non-interleaved run.**" + ) + add("") + add( + "**The selection RULE is Section 10.7's, verbatim and unqualified; the " + "measurement GRID is wider than Section 10.7 specifies, deliberately.** " + "Those are two different things and only one of them changed. The rule " + "— largest suffix whose append p95 is lower at both history lengths, " + "after asserting the winners are prefix-contiguous — was applied as " + "written. The grid was not: Section 10.7 names five suffix lengths, and " + "on those five alone this same measurement yields 2, the answer since " + "retracted as a grid artifact. Locating a crossover needs sample points " + "near it, so the five named lengths are treated as a floor and every " + "additional measured length is used. The measurement plan was " + "corrected; the decision rule was not touched." + ) + add("") + + add("### Identity of the run this constant came from") + add("") + add("| | |") + add("| --- | --- |") + for label, value in _identity_rows(measurements.identity): + add(f"| {label} | `{value}` |") + add("") + add( + "**\"Release-fixed\" does not mean hardware-independent.** Every " + "measurement behind this constant comes from the single machine, " + "corelib build and model named above. Nothing here establishes where " + "the crossover sits on different silicon, on a corelib whose append or " + "prefill path changed, or on a different model. Section 10.7 fixes the " + "constant for the release rather than calibrating at run time, so that " + "scope is intended — but a port to other hardware needs this " + "re-measured, not inherited." + ) + add("") + add( + "The timing table above is published here and is NOT shipped in the " + "model package: Section 10.7 carries the threshold and no cost table." + ) + add("") + add(_DOCUMENT_END) + return "\n".join(out) + "\n" + + +def apply_section_to_document(source: str, section: str) -> str: + begin = source.find(_DOCUMENT_BEGIN) + end = source.find(_DOCUMENT_END) + if begin != -1 and end != -1: + return source[:begin] + section.rstrip("\n") + source[end + len(_DOCUMENT_END) :] + if begin != -1 or end != -1: + raise CalibrationError("document has an unterminated threshold section") + prefix = source if source.endswith("\n") else source + "\n" + return prefix + "\n" + section + + +# -------------------------------------------------------------------------- +# Entry point +# -------------------------------------------------------------------------- + + +def _read_json(path: pathlib.Path, what: str): + try: + text = path.read_text(encoding="utf-8") + except OSError as error: + raise CalibrationError(f"cannot read {what} {path}: {error}") from error + try: + return json.loads(text) + except json.JSONDecodeError as error: + raise CalibrationError(f"{what} {path} is not valid JSON: {error}") from error + + +def _provenance(measurements: Measurements, selection: Selection) -> list: + identity = measurements.identity + concessions = conceded_points(measurements, selection.threshold) + return [ + "Generated. Do not edit this block by hand; re-run the calibrator.", + "", + "Design Section 10.7: the largest sampled suffix length whose append", + "p95 is lower than re-prefill p95 at BOTH history lengths " + f"{list(measurements.histories)},", + "after asserting the winning lengths are a prefix of the sampled set.", + "", + f"verdict: {selection.verdict}", + f"winners: {list(selection.winners)} of sampled {list(selection.ordered)}", + f"measured: {identity.get('utc', 'unknown')} on " + f"{identity.get('machine', 'unknown')}", + f"corelib: {identity.get('corelib_dll_sha256', 'unknown')}", + f"model: {identity.get('model_sha256', 'unknown')}", + "", + "RELEASE-FIXED IS NOT HARDWARE-INDEPENDENT. Every measurement behind", + "this value comes from the single machine, corelib build and model", + "above. Nothing establishes where the crossover sits on different", + "silicon, on a corelib whose append or prefill path changed, or on a", + "different model. A port needs this re-measured, not inherited.", + "", + "What this value gives up, measured: " + + ( + "; ".join( + f"history {history} suffix {suffix} " + f"({slowdown:,.1f}x slower, +{delta / 1e6:,.1f} ms)" + for history, suffix, delta, slowdown in concessions + ) + if concessions + else "nothing the measurement can see" + ) + + ".", + ] + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description="Select the release-fixed Phi-4 continuation threshold.", + ) + parser.add_argument("--baseline", required=True, type=pathlib.Path) + parser.add_argument("--crossover-history", required=True, type=pathlib.Path) + parser.add_argument("--header", required=True, type=pathlib.Path) + parser.add_argument("--document", required=True, type=pathlib.Path) + parser.add_argument("--json", type=pathlib.Path, default=None) + parser.add_argument( + "--check", + action="store_true", + help="verify the checked-in header and document already match; write " + "nothing and exit non-zero if they do not", + ) + args = parser.parse_args(list(argv) if argv is not None else None) + + try: + measurements = load_continuation_samples( + _read_json(args.baseline, "baseline") + ) + selection = select_threshold_detailed(measurements.samples()) + crossover_notes = measurements.crossover_disagreements(selection) + unclosed = measurements.unclosed_histories() + records = _read_json(args.crossover_history, "crossover history") + if not isinstance(records, Sequence) or isinstance(records, (str, bytes)): + raise CalibrationError("crossover history is not a list of records") + agreement = history_agreement(records, selection.threshold) + + section = build_document_section(measurements, selection, agreement) + provenance = _provenance(measurements, selection) + + try: + header_source = args.header.read_text(encoding="utf-8") + except OSError as error: + raise CalibrationError( + f"cannot read header {args.header}: {error}" + ) from error + header_wanted = apply_threshold_to_header( + header_source, selection.threshold, provenance + ) + try: + document_source = args.document.read_text(encoding="utf-8") + except OSError as error: + raise CalibrationError( + f"cannot read document {args.document}: {error}" + ) from error + document_wanted = apply_section_to_document(document_source, section) + except CalibrationError as error: + print(f"calibration refused: {error}", file=sys.stderr) + return 2 + + for line in selection.diagnostics: + print(f"diagnostic: {line}", file=sys.stderr) + for line in crossover_notes: + print(f"diagnostic: {line}", file=sys.stderr) + for line in unclosed: + print(f"diagnostic: {line}", file=sys.stderr) + for line in agreement.diagnostics: + print(f"diagnostic: {line}", file=sys.stderr) + + failed = [] + if crossover_notes: + failed.append( + "the Section 10.7 p95 rule contradicts the recorded bracket" + ) + if unclosed: + failed.append( + "a per-history winner set is not downward-closed, so append lost " + "at a short suffix and won again at a longer one" + ) + if not agreement.permits: + failed.append("the constant is contradicted by an interleaved run") + + stale = [] + if args.check: + if header_source != header_wanted: + stale.append(f"{args.header} is stale") + if document_source != document_wanted: + stale.append(f"{args.document} is stale") + for line in stale: + print(f"stale: {line}", file=sys.stderr) + else: + if not failed: + args.header.write_text(header_wanted, encoding="utf-8", newline="\n") + args.document.write_text( + document_wanted, encoding="utf-8", newline="\n" + ) + else: + print( + "refusing to write the header or the document; see the " + "diagnostics above", + file=sys.stderr, + ) + + if args.json is not None: + args.json.write_text( + json.dumps( + { + "threshold": selection.threshold, + "verdict": selection.verdict, + "winners": list(selection.winners), + "sampled_suffixes": list(selection.ordered), + "winners_by_history": { + str(history): list(won) + for history, won in measurements.winners_by_history().items() + }, + "diagnostics": list(selection.diagnostics) + + list(crossover_notes) + + list(unclosed) + + list(agreement.diagnostics), + "winners_downward_closed": not unclosed, + "history_agreement": { + "comparable": agreement.comparable, + "permits": agreement.permits, + "tight": [row.utc for row in agreement.tight], + "rows": [ + dataclasses.asdict(row) for row in agreement.rows + ], + }, + "identity": dict(measurements.identity), + "checked": bool(args.check), + "stale": stale, + }, + indent=1, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + newline="\n", + ) + + print(f"threshold = {selection.threshold} ({selection.verdict})") + if failed or stale: + return 1 + return 0 + + +if __name__ == "__main__": # pragma: no cover + sys.exit(main()) diff --git a/tools/generate_phi4_corelib_manifest.py b/tools/generate_phi4_corelib_manifest.py new file mode 100644 index 00000000..6b2bbcc4 --- /dev/null +++ b/tools/generate_phi4_corelib_manifest.py @@ -0,0 +1,908 @@ +from __future__ import annotations + +import argparse +import hashlib +import json +import re +from pathlib import Path, PurePosixPath, PureWindowsPath + +import onnx +from onnx import TensorProto + + +SCHEMA_VERSION = 1 +MAX_U64 = (1 << 64) - 1 +EMBEDDED_INITIALIZERS_FILE = "corelib_embedded_initializers.bin" + +MODEL_IDENTITY: dict[str, object] = { + "family": "phi4", + "layers": 32, + "hidden_size": 3072, + "intermediate_size": 8192, + "num_heads": 24, + "kv_heads": 8, + "head_size": 128, + "vocab_size": 200064, + "group_size": 128, + "rope_dim": 96, + "rms_epsilon": 0.00001, +} + +_ATTENTION_PROJECTIONS = ( + ("q_proj", 3072, 3072), + ("k_proj", 3072, 1024), + ("v_proj", 3072, 1024), + ("o_proj", 3072, 3072), +) + +_DTYPE_INFO = { + TensorProto.UINT8: ("uint8", 1), + TensorProto.FLOAT16: ("float16", 2), + TensorProto.FLOAT: ("float32", 4), + TensorProto.INT64: ("int64", 8), +} + + +def _positive_integer(value: int, field: str) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value <= 0: + raise ValueError(f"{field} must be a positive integer") + return value + + +def expected_matmul_components( + k: int, + n: int, + group: int, +) -> dict[str, tuple[object, list[int]]]: + """Return the exact ONNX MatMulNBits component contracts.""" + k = _positive_integer(k, "k") + n = _positive_integer(n, "n") + group = _positive_integer(group, "group") + if k % 2 != 0: + raise ValueError("k must be even for nibble-packed qweight") + if k % group != 0: + raise ValueError("k must be divisible by group") + + groups = k // group + return { + "qweight": ("uint8", [n, k // 2]), + "scales": ({"float16", "float32"}, [n, groups]), + "qzeros": ("uint8", [n, (groups + 1) // 2]), + } + + +def _component_role( + role: str, + weight_object: str, + dtype: object, + shape: list[int], +) -> dict[str, object]: + dtypes = {dtype} if isinstance(dtype, str) else set(dtype) + return { + "role": role, + "weight_object": weight_object, + "dtypes": dtypes, + "shape": list(shape), + } + + +def matmul_roles( + prefix: str, + k: int, + n: int, + group: int, +) -> dict[str, dict[str, object]]: + """Return fully qualified initializer contracts for one MatMul object.""" + if not prefix: + raise ValueError("MatMul initializer prefix must not be empty") + expected = expected_matmul_components(k, n, group) + return { + f"{prefix}.{component}": _component_role( + f"matmul.{component}", + prefix, + dtype, + shape, + ) + for component, (dtype, shape) in expected.items() + } + + +def ssmlp_roles(layer: int) -> dict[str, dict[str, object]]: + """Return exact projection and norm contracts for one fused SSMLP.""" + if isinstance(layer, bool) or not isinstance(layer, int): + raise ValueError("layer must be an integer") + if layer < 0 or layer >= 32: + raise ValueError("layer must be in [0, 31]") + + base = f"model.layers.{layer}" + weight_object = f"{base}.ssmlp" + roles: dict[str, dict[str, object]] = {} + for projection, k, n in ( + ("gate", 3072, 8192), + ("up", 3072, 8192), + ("down", 8192, 3072), + ): + prefix = f"{base}.mlp.{projection}_proj.MatMulNBits" + for component, (dtype, shape) in expected_matmul_components( + k, + n, + 128, + ).items(): + roles[f"{prefix}.{component}"] = _component_role( + f"ssmlp.{projection}.{component}", + weight_object, + dtype, + shape, + ) + + norm_dtypes = {"float16", "float32"} + roles[f"{base}.post_attention_layernorm.weight"] = { + "role": "ssmlp.norm0", + "weight_object": weight_object, + "dtypes": set(norm_dtypes), + "shape": [3072], + } + next_norm = ( + "model.layers.32.final_norm_layernorm.weight" + if layer == 31 + else f"model.layers.{layer + 1}.input_layernorm.weight" + ) + roles[next_norm] = { + "role": "ssmlp.norm1", + "weight_object": weight_object, + "dtypes": set(norm_dtypes), + "shape": [3072], + } + return roles + + +def host_role(role: str) -> dict[str, object]: + """Return the accepted source contract for one host tensor.""" + if role == "embedding": + return { + "role": role, + "dtypes": {"float16"}, + "shape": [200064, 3072], + } + if role == "input_norm": + return { + "role": role, + "dtypes": {"float16", "float32"}, + "shape": [3072], + } + if role in {"cos_cache", "sin_cache"}: + return { + "role": role, + "dtypes": {"float16", "float32"}, + "rank": 2, + "minimum_shape": [4096, 48], + } + raise ValueError(f"unknown host tensor role: {role}") + + +def _merge_roles( + destination: dict[str, dict[str, object]], + additions: dict[str, dict[str, object]], +) -> None: + duplicates = sorted(destination.keys() & additions.keys()) + if duplicates: + raise RuntimeError( + "initializer role generated more than once: " + duplicates[0] + ) + destination.update(additions) + + +def required_initializer_roles() -> dict[str, dict[str, object]]: + """Return all 743 required source initializers in the driver name map.""" + roles: dict[str, dict[str, object]] = {} + for layer in range(32): + base = f"model.layers.{layer}" + # The driver maps o_proj from Q_DIM to HIDDEN. Both are 3072 for + # Phi-4-mini, but keeping both logical dimensions explicit prevents a + # future name-only inference from changing the source contract. + for projection, k, n in _ATTENTION_PROJECTIONS: + prefix = f"{base}.attn.{projection}.MatMulNBits" + _merge_roles(roles, matmul_roles(prefix, k, n, 128)) + _merge_roles(roles, ssmlp_roles(layer)) + + _merge_roles( + roles, + matmul_roles("lm_head.MatMulNBits", 3072, 200064, 128), + ) + _merge_roles( + roles, + { + "model.embed_tokens.weight": host_role("embedding"), + "model.layers.0.input_layernorm.weight": host_role("input_norm"), + "cos_cache": host_role("cos_cache"), + "sin_cache": host_role("sin_cache"), + }, + ) + + weight_objects = { + record["weight_object"] + for record in roles.values() + if "weight_object" in record + } + if len(roles) != 743 or len(weight_objects) != 161: + raise RuntimeError( + "internal Phi-4 role map mismatch: " + f"{len(roles)} initializers, {len(weight_objects)} weight objects" + ) + return roles + + +def _matmul_weight_object( + prefix: str, + k: int, + n: int, + group_size: int, +) -> dict[str, object]: + return { + "name": prefix, + "kind": "matmul", + "descriptor": { + "k": k, + "n": n, + "group_size": group_size, + "has_bias": False, + }, + "roles": { + component: f"{prefix}.{component}" + for component in ("qweight", "scales", "qzeros") + }, + } + + +def _ssmlp_weight_object(layer: int) -> dict[str, object]: + base = f"model.layers.{layer}" + next_norm = ( + "model.layers.32.final_norm_layernorm.weight" + if layer == 31 + else f"model.layers.{layer + 1}.input_layernorm.weight" + ) + roles = { + "norm0": f"{base}.post_attention_layernorm.weight", + "norm1": next_norm, + } + for projection in ("gate", "up", "down"): + prefix = f"{base}.mlp.{projection}_proj.MatMulNBits" + for component in ("qweight", "scales", "qzeros"): + roles[f"{projection}_{component}"] = f"{prefix}.{component}" + return { + "name": f"{base}.ssmlp", + "kind": "ssmlp", + "descriptor": { + "k": 3072, + "n": 8192, + "group_size": 128, + }, + "roles": roles, + } + + +def required_weight_objects() -> list[dict[str, object]]: + """Return the deterministic 161-object corelib construction plan.""" + objects: list[dict[str, object]] = [] + for layer in range(32): + base = f"model.layers.{layer}.attn" + for projection, k, n in _ATTENTION_PROJECTIONS: + objects.append( + _matmul_weight_object( + f"{base}.{projection}.MatMulNBits", + k, + n, + 128, + ) + ) + objects.append(_ssmlp_weight_object(layer)) + objects.append( + _matmul_weight_object( + "lm_head.MatMulNBits", + 3072, + 200064, + 128, + ) + ) + + names = [record["name"] for record in objects] + initializer_names = set(required_initializer_roles()) + references = { + initializer_name + for record in objects + for initializer_name in record["roles"].values() + } + if ( + len(objects) != 161 + or len(names) != len(set(names)) + or not references.issubset(initializer_names) + ): + raise RuntimeError("internal Phi-4 weight-object map mismatch") + return objects + + +def _parse_u64(value: str, field: str, initializer: str) -> int: + if not value or not value.isdecimal(): + raise ValueError( + f"{initializer}: external {field} must be an unsigned decimal" + ) + parsed = int(value) + if parsed > MAX_U64: + raise ValueError(f"{initializer}: external {field} exceeds uint64") + return parsed + + +def _safe_location(location: str, initializer: str) -> str: + if not location or "\x00" in location: + raise ValueError(f"{initializer}: invalid external location") + + windows = PureWindowsPath(location) + posix = PurePosixPath(location.replace("\\", "/")) + if ( + windows.is_absolute() + or bool(windows.drive) + or bool(windows.root) + or posix.is_absolute() + or bool(posix.root) + ): + raise ValueError( + f"{initializer}: external location must be a relative path" + ) + + parts = tuple( + part + for part in posix.parts + if part not in {"", "."} + ) + if not parts or ".." in parts: + raise ValueError( + f"{initializer}: external location contains path traversal" + ) + return PurePosixPath(*parts).as_posix() + + +def _external_source( + tensor: TensorProto, + initializer: str, + model_dir: Path, +) -> tuple[str, Path, int, int]: + metadata: dict[str, str] = {} + for item in tensor.external_data: + if item.key in metadata: + raise ValueError( + f"{initializer}: duplicate external metadata key {item.key}" + ) + metadata[item.key] = item.value + + if "location" not in metadata: + raise ValueError(f"{initializer}: missing external location") + if "length" not in metadata: + raise ValueError(f"{initializer}: missing external length") + + location = _safe_location(metadata["location"], initializer) + offset = _parse_u64(metadata.get("offset", "0"), "offset", initializer) + length = _parse_u64(metadata["length"], "length", initializer) + if length == 0: + raise ValueError(f"{initializer}: external length must be positive") + if offset > MAX_U64 - length: + raise ValueError(f"{initializer}: external range overflow") + + source = model_dir.joinpath(*PurePosixPath(location).parts) + try: + resolved = source.resolve(strict=True) + except FileNotFoundError as error: + raise ValueError( + f"{initializer}: external file does not exist: {location}" + ) from error + if not resolved.is_relative_to(model_dir): + raise ValueError( + f"{initializer}: external location escapes the model directory" + ) + if not resolved.is_file(): + raise ValueError( + f"{initializer}: external location is not a file: {location}" + ) + return location, resolved, offset, length + + +def _dtype_and_item_size( + tensor: TensorProto, + initializer: str, +) -> tuple[str, int]: + try: + return _DTYPE_INFO[tensor.data_type] + except KeyError as error: + type_name = TensorProto.DataType.Name(tensor.data_type) + raise ValueError( + f"{initializer}: unsupported ONNX dtype {type_name}" + ) from error + + +def _checked_byte_count( + shape: list[int], + item_size: int, + initializer: str, +) -> int: + elements = 1 + for dimension in shape: + if dimension <= 0: + raise ValueError( + f"{initializer}: shape dimensions must be positive" + ) + if elements > MAX_U64 // dimension: + raise ValueError(f"{initializer}: shape element count overflow") + elements *= dimension + if elements > MAX_U64 // item_size: + raise ValueError(f"{initializer}: tensor byte count overflow") + return elements * item_size + + +def _validate_contract( + initializer: str, + tensor: TensorProto, + contract: dict[str, object], +) -> tuple[str, int, list[int], int]: + dtype, item_size = _dtype_and_item_size(tensor, initializer) + accepted_dtypes = contract.get("dtypes") + if not isinstance(accepted_dtypes, set) or dtype not in accepted_dtypes: + expected = ", ".join(sorted(accepted_dtypes or ())) + raise ValueError( + f"{initializer}: dtype {dtype} does not match {expected}" + ) + + shape = [int(dimension) for dimension in tensor.dims] + if "shape" in contract: + expected_shape = contract["shape"] + accepted_shapes = [expected_shape] + role = contract["role"] + if ( + role.endswith(".qweight") + and len(expected_shape) == 2 + and expected_shape[1] % 64 == 0 + ): + accepted_shapes.append( + [expected_shape[0], expected_shape[1] // 64, 64] + ) + elif ( + len(expected_shape) == 2 + and ( + role.endswith(".scales") + or role.endswith(".qzeros") + ) + ): + accepted_shapes.append( + [expected_shape[0] * expected_shape[1]] + ) + if shape not in accepted_shapes: + raise ValueError( + f"{initializer}: shape {shape} does not match any " + f"accepted ONNX layout {accepted_shapes}" + ) + else: + rank = contract.get("rank") + minimum_shape = contract.get("minimum_shape") + if len(shape) != rank: + raise ValueError( + f"{initializer}: shape rank {len(shape)} does not match {rank}" + ) + if ( + not isinstance(minimum_shape, list) + or len(minimum_shape) != len(shape) + or any( + actual < minimum + for actual, minimum in zip(shape, minimum_shape) + ) + ): + raise ValueError( + f"{initializer}: shape {shape} is smaller than " + f"{minimum_shape}" + ) + + byte_count = _checked_byte_count(shape, item_size, initializer) + return dtype, item_size, shape, byte_count + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + while chunk := stream.read(1024 * 1024): + digest.update(chunk) + return digest.hexdigest() + + +def _verified_file_metadata( + path: Path, + metadata: dict[str, object], +) -> tuple[int, str]: + if set(metadata) != {"size", "sha256"}: + raise ValueError( + f"verified file metadata must contain size and sha256: {path}" + ) + size = metadata["size"] + sha256 = metadata["sha256"] + if ( + isinstance(size, bool) + or not isinstance(size, int) + or size < 0 + or size > MAX_U64 + ): + raise ValueError(f"verified file size exceeds uint64: {path}") + if ( + not isinstance(sha256, str) + or re.fullmatch(r"[0-9a-fA-F]{64}", sha256) is None + ): + raise ValueError(f"verified file SHA-256 is invalid: {path}") + return size, sha256.lower() + + +def _lfs_pointer_metadata(path: Path) -> tuple[int, str] | None: + if path.stat().st_size > 1024: + return None + try: + text = path.read_text(encoding="ascii") + except (OSError, UnicodeDecodeError): + return None + match = re.fullmatch( + r"version https://git-lfs\.github\.com/spec/v1\r?\n" + r"oid sha256:([0-9a-fA-F]{64})\r?\n" + r"size ([0-9]+)\r?\n?", + text, + ) + if match is None: + return None + size = int(match.group(2)) + if size > MAX_U64: + raise ValueError(f"Git LFS pointer size exceeds uint64: {path}") + return size, match.group(1).lower() + + +def _file_record( + path: Path, + full_hash: bool, + verified_metadata: dict[str, object] | None = None, +) -> dict[str, object]: + physical_size = path.stat().st_size + if verified_metadata is not None: + size, sha256 = _verified_file_metadata(path, verified_metadata) + pointer = _lfs_pointer_metadata(path) + if physical_size == size: + if _sha256(path) != sha256: + raise ValueError( + f"verified file SHA-256 does not match: {path}" + ) + elif pointer != (size, sha256): + raise ValueError( + f"file is neither the verified payload nor its Git LFS pointer: " + f"{path}" + ) + else: + size = physical_size + sha256 = _sha256(path) if full_hash else "" + if size < 0 or size > MAX_U64: + raise ValueError(f"file size exceeds uint64: {path}") + record: dict[str, object] = {"size": size} + if full_hash: + record["sha256"] = sha256 + return record + + +def _load_initializers(model_path: Path) -> dict[str, TensorProto]: + try: + model = onnx.load(str(model_path), load_external_data=False) + except Exception as error: + raise ValueError(f"failed to parse ONNX model {model_path}") from error + + initializers: dict[str, TensorProto] = {} + for tensor in model.graph.initializer: + if tensor.name in initializers: + raise ValueError(f"duplicate initializer: {tensor.name}") + initializers[tensor.name] = tensor + return initializers + + +def _initializer_record( + contract: dict[str, object], + *, + dtype: str, + shape: list[int], + file: str, + offset: int, + length: int, +) -> dict[str, object]: + record: dict[str, object] = { + "file": file, + "offset": offset, + "length": length, + "dtype": dtype, + "shape": shape, + "role": contract["role"], + } + return record + + +def _validate_weight_objects( + weight_objects: list[dict[str, object]], + initializer_names: set[str], +) -> None: + names: set[str] = set() + expected_roles = { + "matmul": {"qweight", "scales", "qzeros"}, + "ssmlp": { + "norm0", + "norm1", + "gate_qweight", + "gate_scales", + "gate_qzeros", + "up_qweight", + "up_scales", + "up_qzeros", + "down_qweight", + "down_scales", + "down_qzeros", + }, + } + expected_descriptor_keys = { + "matmul": {"k", "n", "group_size", "has_bias"}, + "ssmlp": {"k", "n", "group_size"}, + } + + for record in weight_objects: + name = record.get("name") + kind = record.get("kind") + descriptor = record.get("descriptor") + roles = record.get("roles") + if not isinstance(name, str) or not name: + raise ValueError("weight object has an invalid name") + if name in names: + raise ValueError(f"duplicate weight object: {name}") + names.add(name) + if kind not in expected_roles: + raise ValueError(f"{name}: invalid weight object kind") + if ( + not isinstance(descriptor, dict) + or set(descriptor) != expected_descriptor_keys[kind] + ): + raise ValueError(f"{name}: invalid weight object descriptor") + for field in ("k", "n", "group_size"): + value = descriptor[field] + if ( + isinstance(value, bool) + or not isinstance(value, int) + or value <= 0 + ): + raise ValueError(f"{name}: invalid descriptor {field}") + if kind == "matmul" and descriptor["has_bias"] is not False: + raise ValueError(f"{name}: MatMul has_bias must be false") + if not isinstance(roles, dict) or set(roles) != expected_roles[kind]: + raise ValueError(f"{name}: invalid weight object role map") + if len(set(roles.values())) != len(roles): + raise ValueError(f"{name}: duplicate initializer role reference") + for initializer_name in roles.values(): + if ( + not isinstance(initializer_name, str) + or initializer_name not in initializer_names + ): + raise ValueError( + f"{name}: unresolved initializer {initializer_name}" + ) + + +def _generate_manifest( + model_dir: Path, + output: Path, + full_hash: bool, + roles: dict[str, dict[str, object]], + weight_objects: list[dict[str, object]] | None = None, + *, + file_metadata: dict[str, dict[str, object]] | None = None, +) -> dict[str, object]: + """Generate a manifest using an explicit role map. + + The public generator always supplies ``required_initializer_roles()``. + The explicit map keeps small synthetic unit models practical without + allocating the accepted model's multi-gigabyte tensors. + """ + model_dir = Path(model_dir).resolve(strict=True) + output = Path(output) + if not model_dir.is_dir(): + raise ValueError(f"model directory is not a directory: {model_dir}") + if not isinstance(full_hash, bool): + raise ValueError("full_hash must be a boolean") + if file_metadata is None: + file_metadata = {} + if not isinstance(file_metadata, dict): + raise ValueError("file_metadata must be an object") + + model_path = model_dir / "model.onnx" + if not model_path.is_file(): + raise ValueError(f"model.onnx does not exist in {model_dir}") + if not output.parent.exists() or not output.parent.is_dir(): + raise ValueError(f"output parent directory does not exist: {output.parent}") + + initializers = _load_initializers(model_path) + missing = sorted(set(roles) - set(initializers)) + if missing: + preview = ", ".join(missing[:8]) + if len(missing) > 8: + preview += f", ... ({len(missing)} total)" + raise ValueError(f"missing initializer(s): {preview}") + + records: dict[str, dict[str, object]] = {} + external_files: dict[str, Path] = {} + external_file_records: dict[str, dict[str, object]] = {} + embedded: list[tuple[str, bytes]] = [] + embedded_offset = 0 + + for name in sorted(roles): + tensor = initializers[name] + contract = roles[name] + dtype, item_size, shape, byte_count = _validate_contract( + name, + tensor, + contract, + ) + is_external = ( + tensor.data_location == TensorProto.EXTERNAL + or bool(tensor.external_data) + ) + if is_external: + location, path, offset, length = _external_source( + tensor, + name, + model_dir, + ) + if length != byte_count: + raise ValueError( + f"{name}: external byte count {length} does not match " + f"dtype/shape byte count {byte_count}" + ) + if offset % item_size != 0: + raise ValueError( + f"{name}: external offset is not dtype-aligned" + ) + file_record = _file_record( + path, + full_hash, + file_metadata.get(location), + ) + size = file_record["size"] + if offset > size or length > size - offset: + raise ValueError( + f"{name}: external range exceeds file size" + ) + previous = external_files.setdefault(location, path) + if previous != path: + raise ValueError( + f"{name}: external location resolves inconsistently" + ) + external_file_records[location] = file_record + records[name] = _initializer_record( + contract, + dtype=dtype, + shape=shape, + file=location, + offset=offset, + length=length, + ) + continue + + raw_data = bytes(tensor.raw_data) + if not raw_data: + raise ValueError( + f"{name}: embedded initializer must use raw_data" + ) + if len(raw_data) != byte_count: + raise ValueError( + f"{name}: embedded byte count {len(raw_data)} does not match " + f"dtype/shape byte count {byte_count}" + ) + if embedded_offset > MAX_U64 - byte_count: + raise ValueError("embedded initializer sidecar offset overflow") + records[name] = _initializer_record( + contract, + dtype=dtype, + shape=shape, + file=EMBEDDED_INITIALIZERS_FILE, + offset=embedded_offset, + length=byte_count, + ) + embedded.append((name, raw_data)) + embedded_offset += byte_count + + sidecar_path = model_dir / EMBEDDED_INITIALIZERS_FILE + if embedded and EMBEDDED_INITIALIZERS_FILE in external_files: + raise ValueError( + "embedded initializer sidecar conflicts with an external data file" + ) + + protected_paths = {model_path.resolve()} + protected_paths.update(external_files.values()) + if embedded: + protected_paths.add(sidecar_path.resolve(strict=False)) + output_resolved = output.resolve(strict=False) + if output_resolved in protected_paths: + raise ValueError("output path would overwrite ONNX initializer data") + + if embedded: + with sidecar_path.open("wb") as stream: + for _, raw_data in embedded: + stream.write(raw_data) + + files: dict[str, dict[str, object]] = { + "model.onnx": _file_record( + model_path, + full_hash, + file_metadata.get("model.onnx"), + ) + } + for location in sorted(external_files): + files[location] = external_file_records[location] + if embedded: + files[EMBEDDED_INITIALIZERS_FILE] = _file_record( + sidecar_path, + full_hash, + ) + unused_metadata = sorted(set(file_metadata) - set(files)) + if unused_metadata: + raise ValueError( + "verified metadata does not describe a manifest file: " + + unused_metadata[0] + ) + + emitted_weight_objects = ( + [] if weight_objects is None else weight_objects + ) + _validate_weight_objects(emitted_weight_objects, set(records)) + manifest: dict[str, object] = { + "schema_version": SCHEMA_VERSION, + "execution_backend": "corelib_aie4", + "model": dict(MODEL_IDENTITY), + "backend": {"max_seq": 4096}, + "files": files, + "initializers": records, + "weight_objects": emitted_weight_objects, + } + serialized = json.dumps( + manifest, + ensure_ascii=False, + indent=2, + sort_keys=True, + ) + output.write_text(serialized + "\n", encoding="utf-8", newline="\n") + return manifest + + +def generate_manifest( + model_dir: Path, + output: Path, + full_hash: bool, + *, + file_metadata: dict[str, dict[str, object]] | None = None, +) -> dict[str, object]: + arguments = ( + model_dir, + output, + full_hash, + required_initializer_roles(), + required_weight_objects(), + ) + if file_metadata is None: + return _generate_manifest(*arguments) + return _generate_manifest( + *arguments, + file_metadata=file_metadata, + ) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--model-dir", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--full-hash", action="store_true") + args = parser.parse_args() + generate_manifest(args.model_dir, args.output, args.full_hash) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/package_phi4_corelib_aie4.py b/tools/package_phi4_corelib_aie4.py new file mode 100644 index 00000000..a57ed83f --- /dev/null +++ b/tools/package_phi4_corelib_aie4.py @@ -0,0 +1,1096 @@ +from __future__ import annotations + +import argparse +import copy +import hashlib +import json +import re +import subprocess +import urllib.error +import urllib.request +from pathlib import Path + +from tools import generate_phi4_corelib_manifest as manifest_tool + + +UPSTREAM_COMMIT = "e751fb68c2cfffe6b0d32942118f75ac0a0365bb" +FLM_MIN_VERSION = "1.0.4" +EXPECTED_EOS_IDS = [200020, 199999] +UPSTREAM_REPOSITORY = ( + "https://huggingface.co/amd/phi-4-mini-instruct-oga-dml" +) +UPSTREAM_API_URL = ( + "https://huggingface.co/api/models/amd/" + f"phi-4-mini-instruct-oga-dml/tree/{UPSTREAM_COMMIT}" + "?recursive=true&expand=false" +) + +# Design 8.1 / `PACKAGE-1`. The assembled model directory has two provenances +# and they are validated separately, because a rule requiring every catalog +# file to carry Hugging Face metadata cannot succeed: these files are +# FastFlow-authored and do not exist upstream by construction. +# +# `tokenizer_config.json` is the one overlay that also exists upstream. The +# overlay shadows it because the published file carries neither a chat template +# nor eos_token_id, so it is downloaded from nowhere and its upstream size is +# not counted, but it legitimately has an upstream record. The other three must +# never acquire one: an upstream record for `config.json`, +# `corelib_phi4_manifest.json` or `provenance.json` would mean FastFlow's +# contract was published to the model repository, which is a provenance error +# and not a convenience. +OVERLAY_FILES = ( + "config.json", + "corelib_phi4_manifest.json", + "provenance.json", + "tokenizer_config.json", +) +OVERLAY_FILES_WITHOUT_UPSTREAM = ( + "config.json", + "corelib_phi4_manifest.json", + "provenance.json", +) + +# Upstream files deliberately not carried into the assembled package. +# +# `genai_config.json` is excluded by `MODEL-2`: flm.exe runs no ORT or genai +# graph, and shipping an unused configuration invites a future reader to treat +# it as authoritative. `.gitattributes` is a Git repository artifact rather +# than a model file. `chat_template.jinja` is deliberately NOT excluded: it is +# the verbatim source of the template the overlay inlines, and keeping it is +# what makes the overlay auditable on the target machine. +EXCLUDED_UPSTREAM_FILES = ( + ".gitattributes", + "genai_config.json", +) + +_EXPECTED_DECODER = { + "head_size": 128, + "hidden_size": 3072, + "num_attention_heads": 24, + "num_hidden_layers": 32, + "num_key_value_heads": 8, +} +_EXPECTED_MODEL = { + "vocab_size": 200064, +} + + +def _require_mapping(value: object, field: str) -> dict[str, object]: + if not isinstance(value, dict): + raise ValueError(f"{field} must be an object") + return value + + +def _require_exact_value( + mapping: dict[str, object], + field: str, + expected: object, + context: str, +) -> object: + value = mapping.get(field) + if value != expected: + raise ValueError( + f"{context}.{field} must be {expected!r}, got {value!r}" + ) + return value + + +def normalize_config(genai_config: dict[str, object]) -> dict[str, object]: + model = _require_mapping(genai_config.get("model"), "model") + decoder = _require_mapping(model.get("decoder"), "model.decoder") + for field, expected in _EXPECTED_DECODER.items(): + _require_exact_value(decoder, field, expected, "model.decoder") + for field, expected in _EXPECTED_MODEL.items(): + _require_exact_value(model, field, expected, "model") + + return { + "flm_version": FLM_MIN_VERSION, + "head_dim": decoder["head_size"], + "hidden_size": decoder["hidden_size"], + "intermediate_size": 8192, + "model_type": "phi4", + "num_attention_heads": decoder["num_attention_heads"], + "num_hidden_layers": decoder["num_hidden_layers"], + "num_key_value_heads": decoder["num_key_value_heads"], + "rms_norm_eps": 1.0e-5, + "vocab_size": model["vocab_size"], + } + + +def normalize_tokenizer_config( + tokenizer_config: dict[str, object], + chat_template: str, + genai_config: dict[str, object], +) -> dict[str, object]: + if not isinstance(chat_template, str) or not chat_template: + raise ValueError("chat template must be a non-empty string") + model = _require_mapping(genai_config.get("model"), "model") + _require_exact_value( + model, + "eos_token_id", + EXPECTED_EOS_IDS, + "model", + ) + normalized = dict(tokenizer_config) + normalized["chat_template"] = chat_template + normalized["eos_token_id"] = list(EXPECTED_EOS_IDS) + return normalized + + +def catalog_measurements( + model_dir: Path, + logical_sizes: dict[str, int] | None = None, +) -> tuple[int, float]: + model_dir = Path(model_dir) + logical_sizes = {} if logical_sizes is None else dict(logical_sizes) + files = [path for path in model_dir.rglob("*") if path.is_file()] + relative_paths = { + path.relative_to(model_dir).as_posix(): path for path in files + } + unknown = sorted(set(logical_sizes) - set(relative_paths)) + if unknown: + raise ValueError( + f"logical size has no matching package file: {unknown[0]}" + ) + size = 0 + for relative, path in relative_paths.items(): + logical_size = logical_sizes.get(relative, path.stat().st_size) + if ( + isinstance(logical_size, bool) + or not isinstance(logical_size, int) + or logical_size < 0 + ): + raise ValueError(f"invalid logical size for {relative}") + size += logical_size + footprint_gib = round(size / (1024**3), 2) + return size, footprint_gib + + +def _sha256_record(path: Path) -> dict[str, object]: + data = path.read_bytes() + return { + "size": len(data), + "sha256": hashlib.sha256(data).hexdigest(), + } + + +def build_provenance( + upstream_dir: Path, + overlay_dir: Path, + upstream_commit: str, + generated_files: list[str], + git_files: list[dict[str, object]] | None = None, +) -> dict[str, object]: + if upstream_commit != UPSTREAM_COMMIT: + raise ValueError( + f"upstream commit must be pinned to {UPSTREAM_COMMIT}" + ) + upstream_dir = Path(upstream_dir) + overlay_dir = Path(overlay_dir) + input_names = ( + "chat_template.jinja", + "genai_config.json", + "tokenizer_config.json", + ) + provenance: dict[str, object] = { + "upstream": { + "repository": ( + UPSTREAM_REPOSITORY + ), + "commit": upstream_commit, + "inputs": { + name: _sha256_record(upstream_dir / name) + for name in input_names + }, + }, + "generated": { + name: _sha256_record(overlay_dir / name) + for name in sorted(generated_files) + }, + } + if git_files is not None: + provenance["upstream"]["git_files"] = sorted( + git_files, + key=lambda record: record["path"], + ) + return provenance + + +def _git_blob_oid(data: bytes) -> str: + header = f"blob {len(data)}\0".encode("ascii") + return hashlib.sha1(header + data).hexdigest() + + +def _index_git_records( + records: list[dict[str, object]], +) -> dict[str, dict[str, object]]: + indexed: dict[str, dict[str, object]] = {} + for record in records: + if not isinstance(record, dict): + raise ValueError("Git metadata record must be an object") + path = record.get("path") + if not isinstance(path, str) or not path: + raise ValueError("Git metadata record has an invalid path") + if path in indexed: + raise ValueError(f"duplicate Git metadata path: {path}") + if record.get("type") != "file": + raise ValueError(f"Git metadata path is not a file: {path}") + indexed[path] = record + return indexed + + +def _validate_input_git_record( + upstream_dir: Path, + record: dict[str, object], +) -> None: + path = upstream_dir / str(record["path"]) + data = path.read_bytes() + if record.get("size") != len(data): + raise ValueError(f"Git metadata size does not match {record['path']}") + if "lfs" in record: + lfs = _require_mapping(record["lfs"], f"{record['path']}.lfs") + if lfs.get("size") != len(data): + raise ValueError( + f"Git LFS size does not match {record['path']}" + ) + if lfs.get("oid") != hashlib.sha256(data).hexdigest(): + raise ValueError( + f"Git LFS SHA-256 does not match {record['path']}" + ) + elif record.get("oid") != _git_blob_oid(data): + raise ValueError(f"Git blob OID does not match {record['path']}") + + +def _manifest_file_metadata( + indexed: dict[str, dict[str, object]], +) -> dict[str, dict[str, object]]: + metadata: dict[str, dict[str, object]] = {} + for name in ("model.onnx", "model.onnx.data"): + record = indexed.get(name) + if record is None: + raise ValueError(f"Git metadata is missing {name}") + lfs = _require_mapping(record.get("lfs"), f"{name}.lfs") + if record.get("size") != lfs.get("size"): + raise ValueError(f"Git LFS logical size does not match {name}") + metadata[name] = { + "size": lfs.get("size"), + "sha256": lfs.get("oid"), + } + return metadata + + +def _write_json(path: Path, value: object) -> None: + path.write_text( + json.dumps( + value, + ensure_ascii=False, + indent=2, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + newline="\n", + ) + + +# The two catalog documents are NOT written with `_write_json`, and the reason +# is worth stating because the obvious thing was tried and it was wrong. +# +# `model_list.json` is the product's most actively edited file: 2,234 lines, 27 +# model families, and an upstream that adds to it constantly. A `json.dump` of +# the whole document for a single added entry changed indentation (4 -> 2), +# line endings (LF -> CRLF) and key order (alphabetised) across every one of +# those lines. It was semantics-preserving and it was still the wrong output: +# it conflicts with any concurrent model addition, and the one real change is +# invisible in review. +# +# `model_info.json` never shipped that churn only because the entry that was +# committed was not written by this tool. Run `refresh-catalog` against the +# committed file and `_write_json` reformats all 3,387 of its lines the same +# way. Fixing one file and leaving the other would have moved the defect +# rather than closed it, so both go through the writer below. +# +# Matching the style instead of preserving it does not work either. Neither +# committed file is the output of any `json.dumps` call -- `model_list.json` +# carries hand-authored details a serialiser normalises away (`"label":[` with +# no space, `"9b":{`, a trailing-whitespace line, a stray blank line) and +# `model_info.json` has no trailing newline. A re-dump of the catalog at +# indent 4 with the original key order still rewrites 66 lines. +# +# So the entry is spliced in as text and every other byte is left exactly as it +# was found. Textual editing of JSON is easy to get subtly wrong, so the writer +# proves both halves of its own contract before it writes: the produced text +# must parse to the object graph the caller asked for, and it must reduce to +# the input byte-for-byte once the entry is removed again. +CATALOG_MODEL_NAME = "phi4-mini-it-aie4" +CATALOG_MODEL_ANCHOR = "phi4-mini-it" +CATALOG_INFO_KEY = f"{CATALOG_MODEL_NAME}:4b" +_CATALOG_MODELS_KEY = "models" +_CATALOG_MODEL_DEPTH = 2 +_CATALOG_INFO_DEPTH = 1 + + +def _json_text_style(text: str) -> tuple[str, str]: + """The document's newline and its one indent step, read out of itself.""" + newline = "\r\n" if "\r\n" in text else "\n" + for line in text.split(newline): + stripped = line.lstrip(" \t") + if stripped and stripped != line: + return newline, line[: len(line) - len(stripped)] + raise ValueError("cannot determine the catalog's indent step") + + +_JSON_CLOSERS = {"{": "}", "[": "]"} + + +def _json_value_span(text: str, search_from: int) -> tuple[int, int]: + """Half-open span of the object or array at or after `search_from`. + + Bracket counting with string awareness rather than a regex: model names + and URLs contain braces and escaped quotes, and a regex that gets that + wrong fails by silently truncating somebody else's entry. Arrays count + because `model_info.json` maps each model to a list of file records. + """ + candidates = [ + position + for position in ( + text.find("{", search_from), + text.find("[", search_from), + ) + if position >= 0 + ] + if not candidates: + raise ValueError("no JSON object or array in the catalog") + open_at = min(candidates) + closer = _JSON_CLOSERS[text[open_at]] + depth = 0 + in_string = False + escaped = False + for index in range(open_at, len(text)): + character = text[index] + if in_string: + if escaped: + escaped = False + elif character == "\\": + escaped = True + elif character == '"': + in_string = False + continue + if character == '"': + in_string = True + elif character == text[open_at]: + depth += 1 + elif character == closer: + depth -= 1 + if depth == 0: + return open_at, index + 1 + raise ValueError("unterminated JSON value in the catalog") + + +def _member_span( + text: str, + name: str, + depth: int, + newline: str, + indent_unit: str, +) -> tuple[int, int] | None: + """Span of `"name": ` written at `depth` indent steps, or None. + + The indent is part of the match on purpose. `"phi4-mini-it"` also occurs + inside URLs and inside the nested `4b` object of the entry being written; + only the occurrence at the members' own level is the member. + """ + marker = f"{newline}{indent_unit * depth}{json.dumps(name)}:" + first = text.find(marker) + if first < 0: + return None + if text.find(marker, first + 1) >= 0: + raise ValueError(f"catalog has more than one {name} member") + start = first + len(newline) + _, end = _json_value_span(text, first + len(marker)) + return start, end + + +def _rendered_member( + name: str, + value: object, + depth: int, + newline: str, + indent_unit: str, +) -> str: + """`"": ` in the document's own style.""" + rendered = json.dumps( + {name: value}, + ensure_ascii=False, + indent=indent_unit, + ) + # Strip the wrapper object `json.dumps` needed in order to emit the key, + # then push the remainder out to the depth it will live at. + body = rendered.split("\n")[1:-1] + extra = indent_unit * (depth - 1) + return newline.join(extra + line for line in body) + + +def _splice_member( + text: str, + name: str, + value: object, + depth: int, + anchor_name: str | None, + container_name: str | None, +) -> str: + """Insert or replace one member, touching nothing else. + + `anchor_name` is the member the new one is placed after when it is not + already present; `container_name` is the object it belongs to, or None for + the document root. Both mirror `updated_catalog_documents`, so the text and + the object graph agree about where the entry goes. + """ + newline, indent_unit = _json_text_style(text) + member = _rendered_member(name, value, depth, newline, indent_unit) + + existing = _member_span(text, name, depth, newline, indent_unit) + if existing is not None: + # Re-running the tool must be idempotent. Replacing in place also keeps + # the entry where a previous run put it rather than moving it. + start, end = existing + return text[:start] + member + text[end:] + + if anchor_name is not None: + anchor = _member_span(text, anchor_name, depth, newline, indent_unit) + if anchor is not None: + # The insert lands between the anchor's closing brace and the + # comma that already follows it, so the anchor's own line is + # untouched and the comma ends up closing the new member instead. + insert_at = anchor[1] + return text[:insert_at] + "," + newline + member + text[insert_at:] + + if container_name is None: + container = _json_value_span(text, 0) + else: + found = _member_span( + text, container_name, depth - 1, newline, indent_unit + ) + if found is None: + raise ValueError(f"catalog has no {container_name} member") + container = found + # No anchor: append as the last member, which is what + # `updated_catalog_documents` does with the same input. + insert_at = len(text[: container[1] - 1].rstrip()) + return text[:insert_at] + "," + newline + member + text[insert_at:] + + +def _strip_member(text: str, name: str, depth: int) -> str: + """`text` with that member and its separator comma removed.""" + newline, indent_unit = _json_text_style(text) + span = _member_span(text, name, depth, newline, indent_unit) + if span is None: + return text + start, end = span + head = text[:start] + suffix = indent_unit * depth + if head.endswith(suffix): + head = head[: -len(suffix)] + if head.endswith(newline): + head = head[: -len(newline)] + tail = text[end:] + if head.endswith(","): + head = head[:-1] + elif tail.startswith(","): + tail = tail[1:] + return head + tail + + +def _write_spliced( + path: Path, + original: str, + updated: object, + value: object, + name: str, + depth: int, + anchor_name: str | None, + container_name: str | None, +) -> None: + """Splice one member in and write, but only if the result checks out. + + Two checks, both before anything reaches disk, because a textual edit that + is subtly wrong is worse than a reformat: a reformat is merely noisy. + """ + produced = _splice_member( + original, name, value, depth, anchor_name, container_name + ) + if json.loads(produced) != updated: + raise ValueError( + f"the spliced {name} member does not parse to the requested " + "document" + ) + if _strip_member(produced, name, depth) != _strip_member( + original, name, depth + ): + raise ValueError( + f"the spliced document changed bytes outside the {name} entry" + ) + Path(path).write_bytes(produced.encode("utf-8")) + + +def write_model_list(path: Path, original: str, updated_list: object) -> None: + """Write `model_list.json` with only the AIE4 entry changed.""" + value = _require_mapping( + _require_mapping(updated_list, "catalog").get(_CATALOG_MODELS_KEY), + _CATALOG_MODELS_KEY, + ).get(CATALOG_MODEL_NAME) + if value is None: + raise ValueError(f"updated catalog has no {CATALOG_MODEL_NAME} entry") + _write_spliced( + path, + original, + updated_list, + value, + CATALOG_MODEL_NAME, + _CATALOG_MODEL_DEPTH, + CATALOG_MODEL_ANCHOR, + _CATALOG_MODELS_KEY, + ) + + +def write_model_info(path: Path, original: str, updated_info: object) -> None: + """Write `model_info.json` with only the AIE4 record list changed. + + No anchor: `updated_catalog_documents` appends this key, and the committed + file already has it last. Preserving that keeps the file a fixed point. + """ + value = _require_mapping(updated_info, "metadata").get(CATALOG_INFO_KEY) + if value is None: + raise ValueError(f"updated metadata has no {CATALOG_INFO_KEY} entry") + _write_spliced( + path, + original, + updated_info, + value, + CATALOG_INFO_KEY, + _CATALOG_INFO_DEPTH, + None, + None, + ) + + +def generate_overlay( + upstream_dir: Path, + overlay_dir: Path, + upstream_commit: str, + git_files: list[dict[str, object]], +) -> dict[str, object]: + if upstream_commit != UPSTREAM_COMMIT: + raise ValueError( + f"upstream commit must be pinned to {UPSTREAM_COMMIT}" + ) + upstream_dir = Path(upstream_dir).resolve(strict=True) + overlay_dir = Path(overlay_dir) + overlay_dir.mkdir(parents=True, exist_ok=True) + indexed = _index_git_records(git_files) + + input_names = ( + "chat_template.jinja", + "genai_config.json", + "tokenizer_config.json", + ) + for name in input_names: + record = indexed.get(name) + if record is None: + raise ValueError(f"Git metadata is missing {name}") + _validate_input_git_record(upstream_dir, record) + + genai_config = json.loads( + (upstream_dir / "genai_config.json").read_text(encoding="utf-8") + ) + tokenizer_config = json.loads( + (upstream_dir / "tokenizer_config.json").read_text(encoding="utf-8") + ) + # Read bytes and decode explicitly. `read_text` applies universal newline + # translation, which would turn a CRLF template into an LF string and break + # the byte-equality that `_require_inlined_template_matches_upstream` + # depends on. + chat_template_bytes = (upstream_dir / "chat_template.jinja").read_bytes() + chat_template = chat_template_bytes.decode("utf-8") + + _write_json( + overlay_dir / "config.json", + normalize_config(genai_config), + ) + _write_json( + overlay_dir / "tokenizer_config.json", + normalize_tokenizer_config( + tokenizer_config, + chat_template, + genai_config, + ), + ) + manifest_tool.generate_manifest( + upstream_dir, + overlay_dir / "corelib_phi4_manifest.json", + True, + file_metadata=_manifest_file_metadata(indexed), + ) + + generated_files = [ + "config.json", + "corelib_phi4_manifest.json", + "tokenizer_config.json", + ] + provenance = build_provenance( + upstream_dir, + overlay_dir, + upstream_commit, + generated_files, + git_files, + ) + _write_json(overlay_dir / "provenance.json", provenance) + _require_inlined_template_matches_upstream(overlay_dir) + return provenance + + +def build_catalog_entry( + overlay_dir: Path, + git_files: list[dict[str, object]], +) -> dict[str, object]: + overlay_dir = Path(overlay_dir) + overlays = { + name: { + "path": f"{overlay_dir.name}/{name}", + **_sha256_record(overlay_dir / name), + } + for name in OVERLAY_FILES + } + + indexed = _index_git_records(git_files) + upstream_files = { + path + for path in indexed + if path not in overlays and path not in EXCLUDED_UPSTREAM_FILES + } + final_files = sorted(upstream_files | set(overlays)) + # `size` and `footprint` cover the assembled on-disk directory: the + # upstream files actually downloaded plus the overlay files shipped inside + # FastFlow. They are not the upstream repository's size, which is larger, + # and not the overlay's, which is negligible. + remote_size = sum(int(indexed[path]["size"]) for path in upstream_files) + overlay_size = sum(int(record["size"]) for record in overlays.values()) + size = remote_size + overlay_size + footprint = round(size / (1024**3), 2) + return { + "name": "Phi-4-mini-instruct-oga-dml-AIE4", + "url": UPSTREAM_REPOSITORY, + "revision": UPSTREAM_COMMIT, + "file_url": UPSTREAM_API_URL, + "size": size, + "default_context_length": 4096, + "max_prefill_len": 4096, + "details": { + "family": "phi4", + "think": False, + "think_toggleable": False, + "parameter_size": "4B", + "quantization_level": "MatMulNBits Q4", + "execution_backend": "corelib_aie4", + }, + "flm_min_version": FLM_MIN_VERSION, + "vlm": False, + "modelscope_supported": False, + "files": final_files, + "bundled_overlays": overlays, + "footprint": footprint, + } + + +def _require_inlined_template_matches_upstream(overlay_dir: Path) -> None: + """Require the inlined chat template to equal the upstream jinja file. + + `AutoModel::setup_tokenizer` prefers a standalone `chat_template.jinja` + over the `chat_template` key in `tokenizer_config.json`: when the file is + present it *overwrites* the key. Both are in the package, so the upstream + file wins at run time and the overlay's inlined copy is dead code unless + the two are byte-identical. + + That makes any drift silent and one-directional -- the overlay would look + edited while the model kept using the upstream template -- so the equality + is asserted rather than assumed. The check is offline: `provenance.json` + records the upstream file's SHA-256, so no download is needed. + """ + overlay_dir = Path(overlay_dir) + tokenizer_config = json.loads( + (overlay_dir / "tokenizer_config.json").read_text(encoding="utf-8") + ) + provenance = json.loads( + (overlay_dir / "provenance.json").read_text(encoding="utf-8") + ) + inlined = tokenizer_config.get("chat_template") + if not isinstance(inlined, str) or not inlined: + raise ValueError( + "overlay tokenizer_config.json has no string chat_template" + ) + record = _require_mapping( + _require_mapping( + _require_mapping(provenance.get("upstream"), "upstream").get( + "inputs" + ), + "upstream.inputs", + ).get("chat_template.jinja"), + "upstream.inputs['chat_template.jinja']", + ) + encoded = inlined.encode("utf-8") + actual = hashlib.sha256(encoded).hexdigest() + if actual != record.get("sha256") or len(encoded) != record.get("size"): + raise ValueError( + "the overlay's inlined chat_template does not match the upstream " + "chat_template.jinja it was generated from. AutoModel prefers the " + "standalone .jinja file, so the inlined copy would be silently " + "ignored: regenerate the overlay instead of editing it." + ) + + +def validate_catalog_provenance( + entry: dict[str, object], + upstream_records: list[dict[str, object]], + overlay_dir: Path, +) -> None: + """Check the two provenances of the assembled package separately. + + Design `PACKAGE-1`. Upstream files must each carry a Hugging Face metadata + record at the pinned revision and are what the downloader fetches and + hash-checks. Overlay files must exist in the shipped overlay directory and, + unless they shadow a published file, must have no upstream record at all. + """ + overlay_dir = Path(overlay_dir) + indexed = _index_git_records(upstream_records) + + if entry.get("revision") != UPSTREAM_COMMIT: + raise ValueError( + f"catalog revision must be pinned to {UPSTREAM_COMMIT}" + ) + if UPSTREAM_COMMIT not in str(entry.get("file_url", "")): + raise ValueError("catalog file_url must reference the pinned revision") + if entry.get("flm_min_version") != FLM_MIN_VERSION: + raise ValueError( + f"catalog flm_min_version must be {FLM_MIN_VERSION}" + ) + if entry.get("modelscope_supported") is not False: + raise ValueError("this tag has no ModelScope publication") + if "ms_url" in entry: + raise ValueError("a tag without a ModelScope publication has no ms_url") + + _require_inlined_template_matches_upstream(overlay_dir) + + overlays = _require_mapping(entry.get("bundled_overlays"), "bundled_overlays") + if set(overlays) != set(OVERLAY_FILES): + raise ValueError( + "bundled_overlays must be exactly " + f"{sorted(OVERLAY_FILES)}, got {sorted(overlays)}" + ) + + catalog_files = entry.get("files") + if not isinstance(catalog_files, list): + raise ValueError("catalog files must be a list") + if sorted(catalog_files) != list(catalog_files): + raise ValueError("catalog files must be sorted") + if len(set(catalog_files)) != len(catalog_files): + raise ValueError("catalog files contains duplicates") + + for name, record in overlays.items(): + if name not in catalog_files: + raise ValueError(f"overlay {name} is missing from catalog files") + path = overlay_dir.parent / str(record["path"]) + if not path.is_file(): + raise ValueError(f"overlay file is not installed: {path}") + actual = _sha256_record(path) + if actual != {"size": record["size"], "sha256": record["sha256"]}: + raise ValueError(f"overlay {name} does not match its catalog record") + if name in OVERLAY_FILES_WITHOUT_UPSTREAM and name in indexed: + raise ValueError( + f"overlay {name} has an upstream metadata record. FastFlow's " + "own package contract appears to have been published to " + f"{UPSTREAM_REPOSITORY}, which makes the two provenances " + "indistinguishable." + ) + + for name in catalog_files: + if name in overlays: + continue + if name not in indexed: + raise ValueError( + f"upstream file {name} has no Hugging Face metadata record" + ) + if name in EXCLUDED_UPSTREAM_FILES: + raise ValueError( + f"{name} is excluded from the package but is still listed" + ) + + for name in indexed: + if name in catalog_files or name in EXCLUDED_UPSTREAM_FILES: + continue + raise ValueError( + f"upstream file {name} is neither packaged nor explicitly excluded" + ) + + expected_size = sum( + int(indexed[name]["size"]) + for name in catalog_files + if name not in overlays + ) + sum(int(record["size"]) for record in overlays.values()) + if entry.get("size") != expected_size: + raise ValueError( + f"catalog size must be {expected_size}, got {entry.get('size')}" + ) + expected_footprint = round(expected_size / (1024**3), 2) + if entry.get("footprint") != expected_footprint: + raise ValueError( + f"catalog footprint must be {expected_footprint}, " + f"got {entry.get('footprint')}" + ) + + +def updated_catalog_documents( + model_list: dict[str, object], + model_info: dict[str, object], + entry: dict[str, object], + git_files: list[dict[str, object]], +) -> tuple[dict[str, object], dict[str, object]]: + updated_list = copy.deepcopy(model_list) + models = _require_mapping(updated_list.get("models"), "models") + reordered: dict[str, object] = {} + inserted = False + for name, value in models.items(): + if name == CATALOG_MODEL_NAME: + # A re-run reads back its own output. Skipping the stale copy here + # is what makes the freshly built entry survive to the end of the + # loop instead of being overwritten by the one on disk. + continue + reordered[name] = value + if name == CATALOG_MODEL_ANCHOR: + reordered[CATALOG_MODEL_NAME] = {"4b": copy.deepcopy(entry)} + inserted = True + if not inserted: + reordered[CATALOG_MODEL_NAME] = {"4b": copy.deepcopy(entry)} + updated_list["models"] = reordered + + updated_info = copy.deepcopy(model_info) + updated_info["phi4-mini-it-aie4:4b"] = sorted( + copy.deepcopy(git_files), + key=lambda record: record["path"], + ) + return updated_list, updated_info + + +_LFS_POINTER = re.compile( + rb"version https://git-lfs\.github\.com/spec/v1\r?\n" + rb"oid sha256:([0-9a-fA-F]{64})\r?\n" + rb"size ([0-9]+)\r?\n?" +) + + +def git_metadata_records( + git_dir: Path, + commit: str, +) -> list[dict[str, object]]: + if commit != UPSTREAM_COMMIT: + raise ValueError(f"upstream commit must be pinned to {UPSTREAM_COMMIT}") + repository = str(Path(git_dir)) + resolved = subprocess.check_output( + ["git", "-C", repository, "rev-parse", commit], + text=True, + ).strip() + if resolved != commit: + raise ValueError( + f"metadata checkout resolved {resolved}, expected {commit}" + ) + tree = subprocess.check_output( + ["git", "-C", repository, "ls-tree", "-r", "--long", commit], + text=True, + ) + records: list[dict[str, object]] = [] + line_pattern = re.compile( + r"^[0-9]+ blob ([0-9a-f]{40})\s+([0-9]+)\t(.+)$" + ) + for line in tree.splitlines(): + match = line_pattern.fullmatch(line) + if match is None: + raise ValueError(f"unexpected git ls-tree record: {line}") + oid, pointer_size_text, path = match.groups() + pointer_size = int(pointer_size_text) + content = subprocess.check_output( + ["git", "-C", repository, "show", f"{commit}:{path}"] + ) + lfs_match = _LFS_POINTER.fullmatch(content) + record: dict[str, object] = { + "type": "file", + "oid": oid, + "size": pointer_size, + "path": path, + } + if lfs_match is not None: + logical_size = int(lfs_match.group(2)) + record["size"] = logical_size + record["lfs"] = { + "oid": lfs_match.group(1).decode("ascii").lower(), + "size": logical_size, + "pointerSize": pointer_size, + } + records.append(record) + return sorted(records, key=lambda record: record["path"]) + + +def huggingface_metadata_records( + git_dir: Path, + commit: str, +) -> list[dict[str, object]]: + request = urllib.request.Request( + UPSTREAM_API_URL, + headers={"User-Agent": "FastFlowLM-model-packager/1"}, + ) + try: + with urllib.request.urlopen(request) as response: + payload = json.load(response) + except urllib.error.HTTPError as error: + if error.code not in {401, 403}: + raise + return git_metadata_records(git_dir, commit) + + if not isinstance(payload, list): + raise ValueError("Hugging Face tree response must be an array") + records = [ + record + for record in payload + if isinstance(record, dict) and record.get("type") == "file" + ] + if len(records) != len(payload): + raise ValueError("Hugging Face recursive tree contains non-file records") + normalized: list[dict[str, object]] = [] + for record in records: + value: dict[str, object] = { + "type": "file", + "oid": record["oid"], + "size": record["size"], + "path": record["path"], + } + if "lfs" in record: + lfs = _require_mapping(record["lfs"], f"{record['path']}.lfs") + value["lfs"] = { + "oid": lfs["oid"], + "size": lfs["size"], + "pointerSize": lfs["pointerSize"], + } + normalized.append(value) + return sorted(normalized, key=lambda record: record["path"]) + + +def update_catalog_files( + model_list_path: Path, + model_info_path: Path, + overlay_dir: Path, + git_files: list[dict[str, object]], +) -> None: + # Read bytes and decode. `read_text` applies universal newline translation, + # which would hide the file's real line endings from the style-preserving + # writer and make it rewrite every line of a CRLF catalog. + model_list_text = Path(model_list_path).read_bytes().decode("utf-8") + model_list = json.loads(model_list_text) + model_info_text = Path(model_info_path).read_bytes().decode("utf-8") + model_info = json.loads(model_info_text) + entry = build_catalog_entry(overlay_dir, git_files) + validate_catalog_provenance(entry, git_files, Path(overlay_dir)) + updated_list, updated_info = updated_catalog_documents( + model_list, + model_info, + entry, + git_files, + ) + write_model_list(Path(model_list_path), model_list_text, updated_list) + write_model_info(Path(model_info_path), model_info_text, updated_info) + + +def validate_catalog_files( + model_list_path: Path, + model_info_path: Path, + overlay_dir: Path, +) -> None: + """Re-check the committed catalog without contacting the network. + + Regeneration is not always possible on a machine without a metadata + checkout, but the committed documents can still be held to the same + contract, which is what keeps a hand edit from slipping through. + """ + model_list = json.loads( + Path(model_list_path).read_text(encoding="utf-8") + ) + model_info = json.loads( + Path(model_info_path).read_text(encoding="utf-8") + ) + entry = model_list["models"]["phi4-mini-it-aie4"]["4b"] + validate_catalog_provenance( + entry, + model_info["phi4-mini-it-aie4:4b"], + Path(overlay_dir), + ) + + +def main() -> int: + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers(dest="command", required=True) + + overlay = subparsers.add_parser("generate-overlay") + overlay.add_argument("--upstream-dir", type=Path, required=True) + overlay.add_argument("--git-dir", type=Path, required=True) + overlay.add_argument("--output-dir", type=Path, required=True) + overlay.add_argument( + "--upstream-commit", + default=UPSTREAM_COMMIT, + choices=[UPSTREAM_COMMIT], + ) + + catalog = subparsers.add_parser("refresh-catalog") + catalog.add_argument("--git-dir", type=Path, required=True) + catalog.add_argument("--overlay-dir", type=Path, required=True) + catalog.add_argument("--model-list", type=Path, required=True) + catalog.add_argument("--model-info", type=Path, required=True) + catalog.add_argument( + "--upstream-commit", + default=UPSTREAM_COMMIT, + choices=[UPSTREAM_COMMIT], + ) + + validate = subparsers.add_parser("validate-catalog") + validate.add_argument("--overlay-dir", type=Path, required=True) + validate.add_argument("--model-list", type=Path, required=True) + validate.add_argument("--model-info", type=Path, required=True) + + args = parser.parse_args() + if args.command == "validate-catalog": + validate_catalog_files( + args.model_list, + args.model_info, + args.overlay_dir, + ) + return 0 + + records = huggingface_metadata_records( + args.git_dir, + args.upstream_commit, + ) + if args.command == "generate-overlay": + generate_overlay( + args.upstream_dir, + args.output_dir, + args.upstream_commit, + records, + ) + else: + update_catalog_files( + args.model_list, + args.model_info, + args.overlay_dir, + records, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/phi4_host_lm_head_reference.py b/tools/phi4_host_lm_head_reference.py new file mode 100644 index 00000000..b69f0d4b --- /dev/null +++ b/tools/phi4_host_lm_head_reference.py @@ -0,0 +1,695 @@ +"""Host LM-head reference for the `DETERM-1` diagnostic. Task 13 Step 9b. + +THIS IS A DIAGNOSTIC, NOT A PRODUCT PATH. Nothing here is imported by +`phi4_corelib_aie4`, no host island is added, and no production decode is +routed through it. Design section 10.5 keeps the LM head on AIE4; moving it to +the host would cost 614M MACs per token and roughly 1.2 GB of dequantised +weights to fix something that changes no emitted token. Human decision +2026-09-02: build the reference, keep the product as it is. + +WHAT IT ANSWERS. `DETERM-1` records that the AIE4 LM-head MatMul is not +bit-deterministic across runs, and it localises that **by inference**: +identical LM-head input, identical model state, identical emitted tokens, +non-identical LM-head output, therefore the difference is inside the +`3072 x 200064` dispatch. That inference is sound but it compares two NPU runs +against each other, with no third point. This script supplies the third point +-- ground truth -- computed in FP64 from the SAME ONNX components corelib packs +from: `lm_head.MatMulNBits` qweight, scales and qzeros, at the byte offsets the +package manifest records, dequantised. + +The question worth answering is not "do they differ" but which of three: + + 1. both NPU runs sit within rounding distance of the true value, straddling + it -- the wobble is benign accumulation-order nondeterminism; + 2. one run is systematically further from truth than the other -- something + is wrong beyond rounding; or + 3. both are offset from truth in the same direction -- a bias, not noise. + +(2) or (3) is a DIFFERENT finding from the one `DETERM-1` accepts and would +reopen the product decision, so the classification is emitted explicitly and +the exit code carries it. + +THE UNIT THAT MATTERS IS THE ULP, NOT THE ABSOLUTE. The NPU logits are BF16, +so the very best any implementation can do is the correctly rounded BF16 +neighbour of the true value -- half a BF16 ULP away, which at a logit of 20 is +0.0625 and at a logit of 40 is 0.125. Judging in absolute units would call the +larger logit worse for nothing but magnitude, which is the units error design +section 15.3 already had to correct once. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +import numpy as np + +# ONNX MatMulNBits with 4-bit weights and a 128-element group. +BITS = 4 +GROUP_SIZE = 128 +HIDDEN = 3072 +VOCAB = 200064 +LM_HEAD_OBJECT = "lm_head.MatMulNBits" + +# Rows of the [200064, 3072] weight matrix dequantised at a time. 8192 rows is +# 100 MB in FP32 and 200 MB promoted to FP64; the full matrix would be 4.9 GB +# in FP64, which is the reason this is chunked at all. +CHUNK_ROWS = 8192 + +# Classification thresholds, in BF16 ULP of the value being compared, stated +# here rather than buried in the code that applies them. +# +# A correctly rounded BF16 result is at most 0.5 ULP from truth. A mean signed +# deviation is a BIAS if it is a substantial fraction of that: an unbiased +# rounding error averages to roughly zero over 200,064 values, so anything +# above a quarter ULP of consistent offset is not rounding. +BIAS_MEAN_SIGNED_ULP = 0.25 +# One run is "systematically further" if its mean absolute deviation exceeds +# the other's by more than a quarter of the larger. Two runs of the same +# kernel differing only in accumulation order should be indistinguishable on +# this measure. +FURTHER_RELATIVE = 0.25 +# Above this mean absolute deviation the two sides are not computing the same +# thing at all, and the classification below would be meaningless -- worse +# than meaningless, since "one run further" or "a bias" read as findings about +# the device when the likeliest cause is a mistake in the dequantisation here. +# A BF16 dot product over K=3072 accumulates well inside a few ULP of the +# exact result; tens of ULP means a layout or zero-point convention is wrong. +SANITY_MAX_MEAN_ULP = 8.0 + + +def widen_bf16(bits) -> np.ndarray: + """BF16 bit patterns to FP32. Exact and lossless; BF16 is the FP32 top + half, so this reconstructs the value the device actually holds.""" + raw = (np.asarray(bits, dtype=np.uint16).astype(np.uint32) << 16) + return raw.view(np.float32) + + +def bf16_ulp(magnitude: np.ndarray) -> np.ndarray: + """One BF16 ULP at each magnitude. + + The exponent is read from the FP32 BIT PATTERN, not via log2. These values + were widened from BF16 so the exponent field is exact, and log2 of a value + a hair under a power of two rounds the wrong way and shifts the answer by + a factor of two -- the same trap design section 15.3's implementation note + calls out. + """ + magnitude = np.abs(np.asarray(magnitude, dtype=np.float32)) + raw = magnitude.view(np.uint32) + exponent = ((raw >> 23) & 0xFF).astype(np.int32) - 127 + ulp = np.ldexp(np.ones_like(exponent, dtype=np.float64), exponent - 7) + # Subnormal or zero: the smallest normal BF16 ULP is the right floor, and + # a zero exponent field would otherwise produce a meaningless bound. + ulp[(raw & 0x7F800000) == 0] = np.ldexp(1.0, -126 - 7) + return ulp + + +def read_initializer(model_dir: Path, manifest: dict, name: str) -> np.ndarray: + record = manifest["initializers"][name] + path = model_dir / record["file"] + dtype = { + "uint8": np.uint8, + "float16": np.float16, + "float32": np.float32, + "int64": np.int64, + }[record["dtype"]] + count = record["length"] // np.dtype(dtype).itemsize + with path.open("rb") as handle: + handle.seek(record["offset"]) + raw = handle.read(record["length"]) + if len(raw) != record["length"]: + raise RuntimeError( + f"{name}: read {len(raw)} bytes at offset {record['offset']} of " + f"{path}, expected {record['length']}" + ) + return np.frombuffer(raw, dtype=dtype, count=count) + + +def dequantise_rows( + qweight: np.ndarray, + scales: np.ndarray, + qzeros: np.ndarray, + first: int, + count: int, + k: int = HIDDEN, + n: int = VOCAB, + group_size: int = GROUP_SIZE, +) -> np.ndarray: + """Dequantise output rows [first, first + count) of the LM head. + + ONNX MatMulNBits packs two 4-bit weights per byte, low nibble first, in + groups of `GROUP_SIZE` along K. The dequantised value is + `(q - zero) * scale`, which is exact in FP32: `q - zero` is a small + integer and `scale` is an FP16 value, so the product has at most 4 + 11 + significant bits. + """ + groups = k // group_size + bytes_per_group = group_size // 2 + zero_bytes_per_row = -(-groups // 2) + + block = qweight.reshape(n, groups, bytes_per_group)[first : first + count] + low = (block & 0x0F).astype(np.int16) + high = (block >> 4).astype(np.int16) + # Interleave so element j of the group comes from nibble j: even j in the + # low nibble, odd j in the high nibble. + quantised = np.empty((count, groups, group_size), dtype=np.int16) + quantised[:, :, 0::2] = low + quantised[:, :, 1::2] = high + + zero_block = qzeros.reshape(n, zero_bytes_per_row)[first : first + count] + zero_low = (zero_block & 0x0F).astype(np.int16) + zero_high = (zero_block >> 4).astype(np.int16) + zeros = np.empty((count, zero_bytes_per_row * 2), dtype=np.int16) + zeros[:, 0::2] = zero_low + zeros[:, 1::2] = zero_high + zeros = zeros[:, :groups] + + scale_block = ( + scales.reshape(n, groups)[first : first + count] + ).astype(np.float32) + + centred = (quantised - zeros[:, :, None]).astype(np.float32) + weights = centred * scale_block[:, :, None] + return weights.reshape(count, k) + + +def host_lm_head(model_dir: Path, hidden: np.ndarray) -> np.ndarray: + """FP64 logits for one LM-head row, from the packed ONNX components.""" + manifest = json.loads( + (model_dir / "corelib_phi4_manifest.json").read_text(encoding="utf-8") + ) + descriptor = next( + entry + for entry in manifest["weight_objects"] + if entry["name"] == LM_HEAD_OBJECT + ) + if ( + descriptor["descriptor"]["k"] != HIDDEN + or descriptor["descriptor"]["n"] != VOCAB + or descriptor["descriptor"]["group_size"] != GROUP_SIZE + ): + raise RuntimeError( + f"the manifest's LM head is {descriptor['descriptor']}, not the " + f"K={HIDDEN} N={VOCAB} group={GROUP_SIZE} this reference " + f"implements" + ) + roles = descriptor["roles"] + qweight = read_initializer(model_dir, manifest, roles["qweight"]) + scales = read_initializer(model_dir, manifest, roles["scales"]) + qzeros = read_initializer(model_dir, manifest, roles["qzeros"]) + + x = np.asarray(hidden, dtype=np.float64) + logits = np.empty(VOCAB, dtype=np.float64) + for first in range(0, VOCAB, CHUNK_ROWS): + count = min(CHUNK_ROWS, VOCAB - first) + weights = dequantise_rows(qweight, scales, qzeros, first, count) + logits[first : first + count] = weights.astype(np.float64) @ x + return logits + + +def step_labels(document: dict) -> list[str]: + return ["continuation"] + [ + f"decode[{index}]" for index in range(len(document["decode"])) + ] + + +def step_logits(document: dict, index: int) -> list[int]: + if index == 0: + return document["continuation"]["logits_bf16"] + return document["decode"][index - 1]["logits_bf16"] + + +def lm_head_input(document: dict, index: int): + """The LM-head input row recorded at step `index`, or None. + + Runs made before the per-step capture existed carry the same row in + `final_snapshot.last_hidden` -- it is the same `lm_input_tensor` read at + the same moment -- but ONLY for the last step, because the snapshot is + taken once at the end. The fallback is therefore restricted to that step + rather than applied wherever the field is missing, which would silently + answer a different question at every other index. + """ + if index == 0: + row = document["continuation"].get("lm_head_input_bf16") + else: + row = document["decode"][index - 1].get("lm_head_input_bf16") + if row is not None: + return row + if index == len(step_labels(document)) - 1: + return document.get("final_snapshot", {}).get("last_hidden") + return None + + +def lm_head_input_source(document: dict, index: int) -> str: + if index == 0: + recorded = document["continuation"].get("lm_head_input_bf16") + else: + recorded = document["decode"][index - 1].get("lm_head_input_bf16") + if recorded is not None: + return "per_step_capture" + return "final_snapshot_fallback" + + +def first_diverging_step(documents: list[dict]): + """Index of the first step whose logits are not identical across runs.""" + if len(documents) < 2: + return None + for index in range(len(step_labels(documents[0]))): + rows = [step_logits(document, index) for document in documents] + if any(row != rows[0] for row in rows): + return index + return None + + +def bin_counts(indices: np.ndarray, bins: int = 32) -> list[int]: + edges = np.linspace(0, VOCAB, bins + 1) + counts, _ = np.histogram(indices, bins=edges) + return [int(value) for value in counts] + + +def describe_run( + label: str, + npu: np.ndarray, + reference: np.ndarray, + top32: np.ndarray, +) -> dict: + deviation = npu.astype(np.float64) - reference + absolute = np.abs(deviation) + ulp = bf16_ulp(np.maximum(np.abs(npu), np.abs(reference).astype(np.float32))) + in_ulp = absolute / ulp + + # "Within half a BF16 ULP of truth" rather than "equal to the correctly + # rounded truth". The two mean the same thing away from tie points, and + # this form avoids a double-rounding artefact: reducing the FP64 reference + # to FP32 and then to BF16 disagrees with a direct FP64-to-BF16 rounding + # for values within 2^-24 of a BF16 midpoint, which over 200,064 logits + # would silently misclassify a handful of them. + within_half_ulp = absolute <= 0.5 * ulp + positive = int(np.count_nonzero(deviation > 0)) + negative = int(np.count_nonzero(deviation < 0)) + zero = int(np.count_nonzero(deviation == 0)) + + top_dev = absolute[top32] + top_ulp = in_ulp[top32] + + # Uniform across the vocabulary, or concentrated? The differing logits are + # binned into 32 equal ID ranges; a uniform mechanism puts roughly 1/32 of + # them in each. The max/mean ratio is the single number that says which, + # and the counts are emitted so nobody has to take the ratio on trust. + differing = np.flatnonzero(~within_half_ulp) + bins = bin_counts(differing) + mean_bin = float(np.mean(bins)) if bins else 0.0 + + return { + "label": label, + "max_abs_deviation": float(np.max(absolute)), + "mean_abs_deviation": float(np.mean(absolute)), + "mean_signed_deviation": float(np.mean(deviation)), + "max_deviation_ulp": float(np.max(in_ulp)), + "mean_abs_deviation_ulp": float(np.mean(in_ulp)), + "mean_signed_deviation_ulp": float(np.mean(deviation / ulp)), + "within_half_ulp": int(np.count_nonzero(within_half_ulp)), + "total": int(npu.size), + "within_half_ulp_fraction": float( + np.count_nonzero(within_half_ulp) / npu.size + ), + "sign_distribution": { + "npu_above_reference": positive, + "npu_below_reference": negative, + "exactly_equal": zero, + }, + "top32": { + "count": int(top32.size), + "max_abs_deviation": float(np.max(top_dev)), + "mean_abs_deviation": float(np.mean(top_dev)), + "max_deviation_ulp": float(np.max(top_ulp)), + "within_half_ulp": int( + np.count_nonzero(within_half_ulp[top32]) + ), + }, + "differing_index_distribution": { + "bins": 32, + "counts": bins, + "max_over_mean": float(max(bins) / mean_bin) if mean_bin else 0.0, + }, + } + + +def classify(runs: list[dict], straddle: dict | None) -> tuple[str, list[str]]: + """Which of `DETERM-1`'s three possibilities the evidence supports.""" + notes: list[str] = [] + + # Before any of the three: is the reference even describing the same + # computation? A wrong nibble order or zero-point convention here would + # produce a confident "common bias" about a device that is fine. + worst = max(run["mean_abs_deviation_ulp"] for run in runs) if runs else 0.0 + if worst > SANITY_MAX_MEAN_ULP: + notes.append( + f"mean absolute deviation is {worst:.1f} BF16 ULP, far beyond the " + f"{SANITY_MAX_MEAN_ULP} ULP a BF16 dot product over K=3072 can " + f"accumulate. Either this script's dequantisation of " + f"lm_head.MatMulNBits is wrong -- the likelier explanation, and " + f"the first thing to check -- or the device is not computing the " + f"LM head this package describes. Either way, the three-way " + f"classification below would be meaningless and is not offered." + ) + return "gross_disagreement", notes + + biased = [ + run + for run in runs + if abs(run["mean_signed_deviation_ulp"]) > BIAS_MEAN_SIGNED_ULP + ] + if len(biased) == len(runs) and runs: + signs = { + run["mean_signed_deviation_ulp"] > 0 for run in runs + } + if len(signs) == 1: + notes.append( + "every run's mean signed deviation exceeds " + f"{BIAS_MEAN_SIGNED_ULP} BF16 ULP in the SAME direction: " + + ", ".join( + f"{run['label']} {run['mean_signed_deviation_ulp']:+.4f} ULP" + for run in runs + ) + ) + return "common_bias", notes + + if len(runs) == 2: + left, right = runs + larger = max( + left["mean_abs_deviation_ulp"], right["mean_abs_deviation_ulp"] + ) + gap = abs( + left["mean_abs_deviation_ulp"] - right["mean_abs_deviation_ulp"] + ) + if larger > 0 and gap / larger > FURTHER_RELATIVE: + notes.append( + f"the two runs are not equidistant from truth: " + f"{left['label']} {left['mean_abs_deviation_ulp']:.4f} ULP " + f"vs {right['label']} " + f"{right['mean_abs_deviation_ulp']:.4f} ULP, a relative gap " + f"of {gap / larger:.2%} against a {FURTHER_RELATIVE:.0%} " + f"threshold" + ) + return "one_run_further", notes + + if straddle is not None and straddle.get("applicable") is False: + notes.append(straddle["reason"]) + return "model_body_divergence", notes + + if straddle is not None and straddle["differing_logits"] == 0: + notes.append( + "the two runs produced bit-identical logits for this step, so " + "there is no run-to-run divergence in this sample to attribute. " + "The deviation-from-truth figures below still stand, and they " + "describe the kernel rather than the divergence." + ) + return "benign_no_divergence_in_sample", notes + + if straddle is not None and straddle.get("applicable"): + notes.append( + f"{straddle['reference_between']}/{straddle['differing_logits']} " + f"of the logits where the two runs disagree have the true value " + f"between them, and " + f"{straddle['both_within_half_ulp']}/" + f"{straddle['differing_logits']} have both runs within half a " + f"BF16 ULP of truth" + ) + return "benign_accumulation_order", notes + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description=( + "Compare NPU LM-head logits against an FP64 host reference " + "computed from the same ONNX components corelib packs from. " + "Diagnostic only; see the module docstring." + ) + ) + parser.add_argument("--model-dir", required=True) + parser.add_argument( + "--run-json", + action="append", + required=True, + help=( + "an AIE4 end-to-end harness output document. Pass twice to " + "analyse a run-to-run pair." + ), + ) + parser.add_argument("--output-json", required=True) + args = parser.parse_args(argv) + + documents = [ + json.loads(Path(path).read_text(encoding="utf-8")) + for path in args.run_json + ] + if not documents: + print("no run documents supplied", file=sys.stderr) + return 2 + + for field, label in ( + ("corelib_sha256", "corelib DLL"), + ("harness_sha256", "FastFlow harness"), + ): + values = {document.get(field) for document in documents} + if len(values) != 1 or None in values: + print( + f"the run documents do not agree on one {label} ({values}); " + f"the result would not be about one binary", + file=sys.stderr, + ) + return 2 + + # THE STEP THAT ACTUALLY DIVERGED, not the last one. + # + # The first version of this tool always analysed the final decode step. + # For a divergent pair that is the WRONG step: once the two runs have + # emitted different tokens they are computing different continuations, so + # their final hidden states legitimately differ and comparing them answers + # nothing. The question DETERM-1 poses lives at the FIRST step whose + # logits differ, and it is answerable only because the harness now records + # the LM-head input at every step. + steps = step_labels(documents[0]) + index = first_diverging_step(documents) + if index is None: + index = len(steps) - 1 + label = steps[index] + + inputs = [lm_head_input(document, index) for document in documents] + for path, row in zip(args.run_json, inputs): + if row is None: + print( + f"{path} carries no lm_head_input_bf16 at {label}. Rebuild " + f"the end-to-end harness with DEV_BUILD and re-run: without the " + f"recorded input this tool would have to infer it, which is " + f"the inference DETERM-1 got wrong.", + file=sys.stderr, + ) + return 2 + if len(row) != HIDDEN: + raise RuntimeError( + f"lm_head_input_bf16 holds {len(row)} values, expected " + f"{HIDDEN}" + ) + + inputs_identical = all(row == inputs[0] for row in inputs) + model_dir = Path(args.model_dir) + + npu_runs = [ + widen_bf16(step_logits(document, index)) for document in documents + ] + for values in npu_runs: + if values.size != VOCAB: + raise RuntimeError( + f"logits hold {values.size} values, expected {VOCAB}" + ) + + # One reference per DISTINCT input. When the two runs fed the LM head the + # same row there is one truth and the straddle question is meaningful; + # when they did not, each run is judged against ITS OWN truth and the + # straddle question does not arise -- which is itself the answer. + references = [ + host_lm_head(model_dir, widen_bf16(row)) for row in inputs + ] + reference = references[0] + hidden = widen_bf16(inputs[0]) + + reference_order = np.argsort(-reference, kind="stable")[:32] + union_top32 = set(int(index) for index in reference_order) + for values in npu_runs: + union_top32.update( + int(index) for index in np.argsort(-values, kind="stable")[:32] + ) + top32 = np.array(sorted(union_top32), dtype=np.int64) + + runs = [ + describe_run(f"run{position}", values, references[position], top32) + for position, values in enumerate(npu_runs) + ] + for position, run in enumerate(runs): + run["reference"] = ( + "shared" if inputs_identical else f"own input at {label}" + ) + run["source"] = args.run_json[position] + + straddle = None + if len(npu_runs) == 2 and not inputs_identical: + differing_input = sum( + 1 for x, y in zip(inputs[0], inputs[1]) if x != y + ) + straddle = { + "applicable": False, + "lm_head_input_differing_elements": differing_input, + "reason": ( + f"the two runs fed the LM head DIFFERENT rows at {label} " + f"({differing_input}/{HIDDEN} elements differ), so their " + f"logits are supposed to differ and there is no straddle " + f"question. The divergence entered before the LM head." + ), + } + elif len(npu_runs) == 2: + left, right = npu_runs + differing = np.flatnonzero(left != right) + if differing.size: + low = np.minimum(left[differing], right[differing]).astype( + np.float64 + ) + high = np.maximum(left[differing], right[differing]).astype( + np.float64 + ) + truth = reference[differing] + between = int( + np.count_nonzero((truth >= low) & (truth <= high)) + ) + half_ulp = 0.5 * bf16_ulp( + np.maximum( + np.abs(left[differing]), np.abs(right[differing]) + ) + ) + both_close = int( + np.count_nonzero( + (np.abs(left[differing] - truth) <= half_ulp) + & (np.abs(right[differing] - truth) <= half_ulp) + ) + ) + else: + between = 0 + both_close = 0 + straddle = { + "applicable": True, + "differing_logits": int(differing.size), + "reference_between": between, + "both_within_half_ulp": both_close, + "differing_index_distribution": { + "bins": 32, + "counts": bin_counts(differing), + }, + } + + verdict, notes = classify(runs, straddle) + + result = { + "diagnostic": "phi4 host LM-head reference (Task 13 Step 9b)", + "product_path": False, + "model_dir": str(model_dir), + "corelib_sha256": documents[0].get("corelib_sha256"), + "corelib_loaded_path": documents[0].get("corelib_loaded_path"), + "continuation_route": documents[0].get("continuation_route"), + "harness_sha256": documents[0].get("harness_sha256"), + "analysed_step": label, + "analysed_step_index": index, + "step_selected_because": ( + "first step whose logits differ between the two runs" + if first_diverging_step(documents) is not None + else "no logit divergence; the last step was analysed" + ), + "lm_head_inputs_identical": inputs_identical, + "lm_head_input_source": lm_head_input_source(documents[0], index), + "prefix_ids": documents[0].get("prefix_ids"), + "suffix_ids": documents[0].get("suffix_ids"), + + "reference": { + "implementation": ( + "FP64 dense matmul over lm_head.MatMulNBits dequantised as " + "(q - zero) * scale, 4-bit, group 128, read at the byte " + "offsets the package manifest records" + ), + "k": HIDDEN, + "n": VOCAB, + "group_size": GROUP_SIZE, + "accumulation": "float64", + "top1_id": int(reference_order[0]), + "top5_ids": [int(index) for index in reference_order[:5]], + }, + "npu_top1_ids": [ + int(np.argsort(-values, kind="stable")[0]) for values in npu_runs + ], + "runs": runs, + "run_to_run": straddle, + "thresholds": { + "bias_mean_signed_ulp": BIAS_MEAN_SIGNED_ULP, + "one_run_further_relative": FURTHER_RELATIVE, + }, + "verdict": verdict, + "notes": notes, + } + Path(args.output_json).write_text( + json.dumps(result, indent=2), encoding="utf-8" + ) + + for run in runs: + print( + f"{run['label']}: max |dev| {run['max_abs_deviation']:.6g} " + f"({run['max_deviation_ulp']:.3f} ULP), mean |dev| " + f"{run['mean_abs_deviation']:.6g} " + f"({run['mean_abs_deviation_ulp']:.4f} ULP), within half a ULP " + f"{run['within_half_ulp']}/{run['total']}, signed mean " + f"{run['mean_signed_deviation_ulp']:+.4f} ULP" + ) + if straddle is not None and straddle.get("applicable"): + print( + f"run-to-run: {straddle['differing_logits']} logits differ; " + f"truth between the two in {straddle['reference_between']}, " + f"both within half a ULP in {straddle['both_within_half_ulp']}" + ) + elif straddle is not None: + print(f"run-to-run: {straddle['reason']}") + for note in notes: + print(f"note: {note}") + print(f"verdict: {verdict}") + + # (2) and (3) are a different finding from the one DETERM-1 accepts and + # they reopen the product decision, so they reach the exit code rather + # than sitting in a JSON file nobody reads. + if verdict in ("one_run_further", "common_bias"): + print( + "STOP AND REPORT: this is not the benign accumulation-order " + "nondeterminism DETERM-1 accepts.", + file=sys.stderr, + ) + return 1 + if verdict == "model_body_divergence": + print( + "STOP AND REPORT: the two runs fed the LM head different inputs, " + "so the divergence entered the model body. DETERM-1's " + "localisation to the LM-head dispatch does not hold for this " + "event.", + file=sys.stderr, + ) + return 1 + if verdict == "gross_disagreement": + print( + "STOP: the reference and the device disagree by far more than " + "rounding. Check this script's dequantisation before reading " + "anything into the numbers.", + file=sys.stderr, + ) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/report_phi4_corelib_baseline.py b/tools/report_phi4_corelib_baseline.py new file mode 100644 index 00000000..1d46b19f --- /dev/null +++ b/tools/report_phi4_corelib_baseline.py @@ -0,0 +1,1747 @@ +"""Validate and render the Phi-4 AIE4 performance and memory baseline. + +Task 13 Step 10. + +WHAT THIS TOOL IS FOR, because it is easy to build the wrong thing. + +Design section 4 says performance is explicitly not a release blocker for this +release, and design section 15.6 says to record the baseline "without pass/fail +thresholds". So no latency or throughput number in here is ever a gate. What +this tool produces is an honest, reproducible RECORD with enough identity +attached that a later change can be compared against it, and what it gates on +is only the three things whose absence would make the record worthless or +misleading: + + 1. every required section and identity field is present. A baseline that + does not say which machine, which corelib DLL and which model produced it + is not a baseline, it is a set of numbers; + 2. the 128-token memory window is stable. That is not a performance figure -- + design 18.7 and design 15.4 make bounded post-warm allocation a + correctness property, and an unbounded one is a leak; and + 3. the `DETERM-3` bit-identity baseline states its rate, the spread of the + observed maximum absolute difference, and the n those rest on, over at + least the 20 runs per route `DETERM-3` requires. + +Gate 3 deserves its own paragraph. `DETERM-2` names "a bit-identity rate that +degrades from the recorded baseline" as a failure condition. As of 2026-09-02 +no such baseline existed, so that clause enforced nothing, and `DETERM-3` says +so explicitly rather than letting it look enforced. This tool is what closes +that -- and the failure mode it must avoid is not "no baseline" but "a +confident number derived from too few runs", which is harder to notice than the +absence it would replace. So a short campaign fails the exit code and renders +nothing, rather than publishing a floor. +""" + +from __future__ import annotations + +import argparse +import glob as globlib +import json +import math +import statistics +import sys +from pathlib import Path + +# Step 1 of the task brief fixes this set. +REQUIRED_SECTIONS = { + "identity", + "model_load", + "ttft", + "prefill", + "continuation", + "decode", + "memory", + "v_scatter", +} + +# Design section 14's telemetry list, reduced to the fields without which two +# baselines cannot be compared at all. `corelib_source_revision` rather than +# only the version: `ryzenai_corelib_get_version` is a hard-coded 0.1.0 in +# corelib's own source and spans the whole 0.x history, so it cannot identify a +# revision (Task 12 report section 1). The DLL SHA-256 identifies the exact +# binary; the source revision identifies what it was built from. Both, because +# MSVC RelWithDebInfo is not byte-reproducible and the hash alone changes on a +# rebuild of identical source. +REQUIRED_IDENTITY_FIELDS = { + "machine", + "cpu_sku", + "npu_sku", + "npu_driver_version", + "corelib_dll_path", + "corelib_dll_sha256", + "corelib_version", + "corelib_source_revision", + "dynamic_dispatch_version", + "ryzen_mm_version", + "xrt_version", + "model_dir", + "model_sha256", + "fastflow_revision", +} + +# Step 2's bounds, and the design contracts that are contracts rather than +# measurements. +MAX_PRIVATE_BYTES_GROWTH = 8 * 1024 * 1024 +MAX_PRIVATE_BYTES_SLOPE_PER_TOKEN = 64 * 1024 +SYNCHRONIZES_PER_MODEL_STEP = 129 +V_READS_PER_MODEL_STEP = 32 +V_WRITES_PER_MODEL_STEP = 256 + +# Design `DETERM-3`: "it requires at least 20 runs per route". +MIN_DETERMINISM_RUNS_PER_ROUTE = 20 + +# I-6. The significance level for the route-dependence claim. 0.05 is the +# conventional value and is named here rather than inlined, because the claim +# it gates -- "the rate is route-dependent" -- was previously published in +# bold from a float inequality on 1-vs-2 divergent runs of 33. +ROUTE_DEPENDENCE_ALPHA = 0.05 + +# Design 5.1. The LM-head input row is one hidden state, and the localisation +# counts are reported against it. +HIDDEN_SIZE = 3072 + +# The routes `DETERM-3` requires separately. Append and re-prefill drive +# different row extents through the same LM-head shape, so a difference +# between them is informative about the mechanism and must not be averaged +# away. +DETERMINISM_ROUTES = ("append", "reprefill") + +_MARKDOWN_BEGIN = "" +_MARKDOWN_END = "" + + +def _fisher_exact_two_sided( + left: tuple[int, int], + right: tuple[int, int], +) -> float: + """Two-sided Fisher exact p for a 2x2 table of counts. + + Exact and dependency-free: `math.comb` is all this needs, and pulling in + SciPy for one hypothesis test would add a dependency to a validator that + otherwise runs anywhere Python does. + + Sums the hypergeometric probability of every table at least as extreme as + the observed one, which is the standard two-sided construction. + """ + a, b = left + c, d = right + row1, row2 = a + b, c + d + col1 = a + c + total = row1 + row2 + if total == 0 or row1 == 0 or row2 == 0 or col1 == 0 or col1 == total: + return 1.0 + + def probability(value: int) -> float: + return ( + math.comb(row1, value) + * math.comb(row2, col1 - value) + / math.comb(total, col1) + ) + + observed = probability(a) + low = max(0, col1 - row2) + high = min(col1, row1) + # 1e-9 relative slack: the equal-probability tables belong in the tail, and + # floating-point evaluation of equal binomial ratios does not reproduce + # bit-identical values. + return min( + 1.0, + sum( + probability(value) + for value in range(low, high + 1) + if probability(value) <= observed * (1 + 1e-9) + ), + ) + + +def memory_is_stable(record: dict[str, object]) -> bool: + """Step 2's gate, verbatim from the task brief. + + Indexing rather than `.get`: an absent counter must raise, not read as + zero. A validator that treats a missing measurement as a passing one is + the defect this project has now found seven times. + """ + return ( + record["device_tensor_creates_after_warmup"] == 0 + and record["weight_creates_after_warmup"] == 0 + and record["live_corelib_object_delta"] == 0 + and record["private_bytes_growth"] <= 8 * 1024 * 1024 + and record["private_bytes_slope_per_token"] <= 64 * 1024 + ) + + +def _missing_identity(identity: object) -> list[str]: + problems: list[str] = [] + if not isinstance(identity, dict): + return ["identity is not an object"] + for field in sorted(REQUIRED_IDENTITY_FIELDS): + if field not in identity: + problems.append(f"identity.{field} is absent") + continue + value = identity[field] + # "" and None are what a collector writes when it could not find the + # thing, and recording them would name a machine nobody can identify. + if value is None or (isinstance(value, str) and not value.strip()): + problems.append(f"identity.{field} is empty") + return problems + + +def validate(document: dict[str, object]) -> list[str]: + """Return every problem found, rather than the first. + + A caller re-running a two-hour hardware campaign for one missing field at + a time is a worse outcome than a long list. + """ + problems: list[str] = [] + for section in sorted(REQUIRED_SECTIONS): + if section not in document: + problems.append(f"section {section!r} is absent") + problems.extend(_missing_identity(document.get("identity"))) + + memory = document.get("memory") + if isinstance(memory, dict): + try: + stable = memory_is_stable(memory) + except KeyError as error: + problems.append( + f"memory is missing the counter {error.args[0]!r}, so the " + f"128-token stability window cannot be judged" + ) + else: + if not stable: + problems.append( + "memory: the 128-token stability window is NOT stable " + f"(device tensors created after warmup " + f"{memory.get('device_tensor_creates_after_warmup')}, " + f"weight objects " + f"{memory.get('weight_creates_after_warmup')}, " + f"live-object delta " + f"{memory.get('live_corelib_object_delta')}, " + f"private-byte growth " + f"{memory.get('private_bytes_growth')} > " + f"{MAX_PRIVATE_BYTES_GROWTH}, slope " + f"{memory.get('private_bytes_slope_per_token')} > " + f"{MAX_PRIVATE_BYTES_SLOPE_PER_TOKEN})" + ) + + scatter = document.get("v_scatter") + if isinstance(scatter, dict): + # Design 10.3 and 18.5. These two are a CONTRACT, not a measurement: + # one complete model pass copies 32 V caches out and 256 per-head + # slices back. A number other than those means the schedule changed. + for field, expected in ( + ("reads_per_model_step", V_READS_PER_MODEL_STEP), + ("writes_per_model_step", V_WRITES_PER_MODEL_STEP), + ): + if field not in scatter: + problems.append(f"v_scatter.{field} is absent") + elif scatter[field] != expected: + problems.append( + f"v_scatter.{field} is {scatter[field]}, design 18.5 " + f"requires {expected}" + ) + for field in ("bytes", "nanoseconds"): + if field not in scatter: + problems.append(f"v_scatter.{field} is absent") + # I-4. The two counts above are only worth reading if they were + # DIVIDED OUT of the observed totals rather than copied from the same + # constants they are checked against. A benchmark that restates the + # constants produces a row that cannot fail and reads as measured, so + # the document refuses to render a record that does not assert the + # provenance. + if scatter.get("counts_are_measured") is not True: + problems.append( + "v_scatter.counts_are_measured is not true: the per-step " + "read and write counts must be the observed totals divided " + "by the observed model-step count, not the design constants " + "restated. A gate that compares a constant against itself " + "cannot fail and must not be rendered as a measurement." + ) + + decode = document.get("decode") + if isinstance(decode, dict): + runs = decode.get("runs") + if not isinstance(runs, list) or not runs: + problems.append("decode.runs is absent or empty") + else: + for run in runs: + count = run.get("synchronizes_per_pass") + if count != SYNCHRONIZES_PER_MODEL_STEP: + problems.append( + f"decode run at context {run.get('start_context')}: " + f"synchronizes_per_pass is {count}, design 10.4 " + f"requires {SYNCHRONIZES_PER_MODEL_STEP}" + ) + + for section in ("prefill", "continuation"): + body = document.get(section) + if isinstance(body, dict) and not body.get("points"): + problems.append(f"{section}.points is absent or empty") + + problems.extend(_continuation_problems(document.get("continuation"))) + return problems + + +def _is_number(value: object) -> bool: + """An integer that is not a bool. + + `isinstance(True, int)` is True, so a JSON `true` satisfied every + `isinstance(..., int)` check in this file -- including the ones added to + stop a present-but-unusable field from validating clean. + """ + return isinstance(value, int) and not isinstance(value, bool) + + +def _continuation_problems(continuation: object) -> list[str]: + """N-3. The interleaving claim has to be attested by the data. + + The rendered document states, in prose, that the samples are interleaved + and that each point beat its own drift. Neither was required of the + record, so re-rendering an older baseline would publish the claim over + non-interleaved data -- the same defect `counts_are_measured` was added to + close, in the section that decides the crossover Task 14 consumes. + + So the flag is required AND CHECKED. Each point carries the start offset + of every measured sample, and interleaving is verified by pairing the two + routes at each (history, suffix) and requiring their sample starts to + strictly alternate. A boolean can be wrong; the timeline cannot. + """ + problems: list[str] = [] + if not isinstance(continuation, dict): + return problems + points = continuation.get("points") + if not isinstance(points, list) or not points: + return problems + + if continuation.get("samples_interleaved") is not True: + problems.append( + "continuation.samples_interleaved is not true: the rendered " + "document states the append and re-prefill samples are " + "interleaved, and the decision rule that sets the crossover " + "bracket is only defensible if they are." + ) + + by_point: dict[tuple, dict[str, dict]] = {} + for point in points: + if not isinstance(point, dict): + continue + # PRESENT IS NOT USABLE. `field not in point` accepts a null, and a + # record with every `sample_starts_ns` set to null validated clean -- + # the same skip-instead-of-fail shape this round closed twice + # elsewhere, reintroduced in the check added to close it. + # `isinstance(True, int)` is True in Python, so a JSON `true` passed + # a check written to require a number. Excluded explicitly. + if not _is_number(point.get("drift_ns")): + problems.append( + f"continuation point " + f"{point.get('history_rows')}/{point.get('suffix')}" + f"/{point.get('route')}: drift_ns is " + f"{point.get('drift_ns')!r}, not an integer. The drift term " + f"is half the decision rule." + ) + starts = point.get("sample_starts_ns") + if not isinstance(starts, list) or not starts: + problems.append( + f"continuation point " + f"{point.get('history_rows')}/{point.get('suffix')}" + f"/{point.get('route')}: sample_starts_ns is {starts!r}, not " + f"a non-empty list. Without it the interleaving cannot be " + f"checked and the published claim is unattested." + ) + elif not all(_is_number(value) for value in starts): + problems.append( + f"continuation point " + f"{point.get('history_rows')}/{point.get('suffix')}" + f"/{point.get('route')}: sample_starts_ns holds a non-integer" + ) + key = (point.get("history_rows"), point.get("suffix")) + route = str(point.get("route")) + by_point.setdefault(key, {})[route] = point + + for key, routes in sorted( + by_point.items(), key=lambda item: (str(item[0][0]), str(item[0][1])) + ): + # A point with only one route was skipped silently. Both routes are + # measured at every point by construction, so one missing means the + # record is not what the document says it is. + missing = {"append", "reprefill"} - set(routes) + if missing: + problems.append( + f"continuation point {key[0]}/{key[1]} has no " + f"{', '.join(sorted(missing))} entry, so the two routes " + f"cannot have been interleaved there" + ) + continue + append = routes.get("append", {}).get("sample_starts_ns") + reprefill = routes.get("reprefill", {}).get("sample_starts_ns") + if not isinstance(append, list) or not isinstance(reprefill, list): + continue + # Already reported above as a bad field. Sorting a mixed list of + # strings and ints raises, which would turn a reportable defect in the + # record into a crash in the validator. + if not all(_is_number(value) for value in append + reprefill): + continue + if len(append) != len(reprefill): + problems.append( + f"continuation point {key[0]}/{key[1]}: the routes have " + f"{len(append)} and {len(reprefill)} samples, so they cannot " + f"have been interleaved" + ) + continue + merged = [(value, "a") for value in append] + merged += [(value, "r") for value in reprefill] + merged.sort() + order = "".join(tag for _, tag in merged) + if order != "ar" * len(append): + problems.append( + f"continuation point {key[0]}/{key[1]}: the sample start " + f"times do not alternate between the routes (observed " + f"{order!r}), so the samples were NOT interleaved and the " + f"decision rule's paired-in-time assumption does not hold" + ) + return problems + + +def _format_number(value: float) -> str: + # Histogram keys have to survive a JSON round trip and stay comparable + # between runs, so they are formatted rather than repr'd. + if value == int(value): + return str(int(value)) if abs(value) < 1e16 else repr(value) + return f"{value:.6g}" + + +def determinism_baseline(records: list[dict]) -> dict: + """Aggregate `determ1-.json` records into the `DETERM-3` baseline. + + Reports per route, never pooled. Design `DETERM-3`: "If the rate turns out + to be route-dependent, say so rather than averaging: append and re-prefill + drive different row extents through the same LM-head shape, and a + difference between them would itself be informative about the mechanism." + + Every figure is emitted with the n it rests on, and a route that has not + reached `MIN_DETERMINISM_RUNS_PER_ROUTE` is marked `is_baseline: false` + and produces a problem. That combination is the point: the numbers are + still visible, so the campaign's progress is legible, but nothing + downstream can cite them as a baseline. + + Two kinds of problem, kept apart on purpose. + + `blocking_problems` are the ones that mean there is NO baseline: too few + runs, no records at all, or records from more than one binary. Those stop + the report being rendered, because a document that implies a baseline + exists when it does not is the exact defect `DETERM-3` names. + + `problems` also includes hard-gate failures, and those must NOT block + rendering. A run that breached `DETERM-2` is a finding, and suppressing + the whole record because the campaign found something is the opposite of + what this task is for. It is counted in the rate, called out in the + document, and it reaches the exit code. + """ + problems: list[str] = [] + blocking: list[str] = [] + routes: dict[str, dict] = {} + # I-7. Which files this baseline was actually built from, so a pooled glob + # is auditable rather than trusted. + # A record that cannot be shown to come from the same FastFlow binary is + # dropped here, before anything is counted, rather than quietly averaged + # in with the rest. + excluded = [ + record for record in records if not record.get("harness_sha256") + ] + excluded_sources = sorted( + str(record.get("_source", "")) for record in excluded + ) + records = [record for record in records if record.get("harness_sha256")] + sources = sorted( + str(record.get("_source", "")) for record in records + ) + + by_route: dict[str, list[dict]] = {} + for record in records: + route = record.get("route") + if route not in DETERMINISM_ROUTES: + blocking.append( + f"a DETERM-1 record names an unknown route {route!r}; it is " + f"not counted toward any baseline" + ) + continue + by_route.setdefault(route, []).append(record) + + # I-7. BOTH halves of "the same binary". The corelib DLL was pinned and + # the FastFlow harness that drove it was not, so a glob spanning a build + # tree could pool records from two different FastFlow builds with nothing + # able to notice. A record that predates the harness-hash field is itself + # a reason to refuse: it cannot be shown to belong. + for field, label in ( + ("corelib_sha256", "corelib SHA-256"), + ("harness_sha256", "FastFlow harness SHA-256"), + ): + hashes = { + record.get(field) for record in records if record.get(field) + } + if len(hashes) > 1: + blocking.append( + f"the records do not all share one {label} " + f"({len(hashes)} distinct values), so they do not describe " + f"one binary and cannot be pooled into a baseline: " + + ", ".join(sorted(value[:16] for value in hashes)) + ) + if excluded: + # EXCLUDED, NOT BLOCKED, and counted. + # + # A record predating the harness-hash field cannot be shown to belong + # to the pool, so it must not be counted. But refusing to produce any + # baseline while such a record exists anywhere under the glob would + # make the committed DETERM-4 evidence permanently unusable, which is + # the opposite of why it is committed. It is dropped, the count and + # the reason are stated, and `sources` lists exactly what WAS used. + problems.append( + f"{len(excluded)} record(s) carry no FastFlow harness SHA-256 " + f"and were EXCLUDED from the pooled figures: they cannot be shown " + f"to describe the same binary as the rest. They remain committed " + f"as evidence and are listed under `excluded_sources`." + ) + if not records: + blocking.append("no DETERM-1 records were supplied") + + for route in DETERMINISM_ROUTES: + entries = by_route.get(route, []) + if not entries: + blocking.append( + f"route {route!r}: no DETERM-1 records at all, so DETERM-3's " + f"minimum of {MIN_DETERMINISM_RUNS_PER_ROUTE} runs is not met" + ) + routes[route] = { + "runs": 0, + "steps": 0, + "bit_identical_runs": 0, + "step_bit_identity_rate": None, + "run_bit_identity_rate": None, + "observed_max_abs_diff": { + "max": None, + "min": None, + "median": None, + "nonzero_runs": 0, + "histogram": {}, + }, + "runs_with_gate_failures": 0, + "first_divergences": {}, + "localisation": { + "measured_runs": 0, + "by_source": {}, + "elements_by_source": {}, + "steps_by_source": {}, + "lm_head_input_elements": HIDDEN_SIZE, + "instruments": {}, + }, + "is_baseline": False, + } + continue + + runs = len(entries) + total_steps = sum(int(entry["logits_total_steps"]) for entry in entries) + exact_steps = sum( + int(entry["logits_bit_exact_steps"]) for entry in entries + ) + bit_identical_runs = sum( + 1 + for entry in entries + if int(entry["logits_bit_exact_steps"]) + == int(entry["logits_total_steps"]) + ) + maxima = [ + float(entry.get("observed_max_abs_diff") or 0.0) for entry in entries + ] + histogram: dict[str, int] = {} + for value in maxima: + if value == 0.0: + continue + key = _format_number(value) + histogram[key] = histogram.get(key, 0) + 1 + nonzero = [value for value in maxima if value != 0.0] + failing = [entry for entry in entries if entry.get("failures")] + divergences: dict[str, int] = {} + for entry in entries: + label = entry.get("first_divergence") + if label: + divergences[label] = divergences.get(label, 0) + 1 + + # WHERE THE DIVERGENCE ENTERED, aggregated so the published document + # carries the finding and not only the rate. + # + # A reader of the benchmark document was learning that 2 runs in 41 + # were not bit-identical, and nothing about the thing that actually + # matters: that the two runs fed the LM head DIFFERENT rows, so the + # divergence is upstream of it. That was in the task report, the + # records README and the design spec, and in none of the places a + # reader of the benchmarks would look. + # PER SOURCE, never pooled. + # + # The first version pooled the differing-element counts and the steps + # across every source and then stated "in every measured event the two + # runs fed the LM head different rows". An `lm_head` event has + # differing == 0 by construction -- it is the DETERM-1-supporting case + # -- so as soon as one appeared the document would list a zero among + # the differing counts and assert a conclusion the same window + # contradicts. A generator that states a false finding in the section + # carrying the headline claim is the worst thing in this tree, so the + # counts are kept apart at the point they are collected. + localisations: dict[str, int] = {} + by_source_elements: dict[str, list[int]] = {} + by_source_steps: dict[str, list[str]] = {} + instruments: dict[str, int] = {} + for entry in entries: + record = entry.get("localisation") or {} + if not record.get("measured"): + continue + source = str(record.get("source", "unknown")) + localisations[source] = localisations.get(source, 0) + 1 + if record.get("lm_head_input_differing_elements") is not None: + by_source_elements.setdefault(source, []).append( + int(record["lm_head_input_differing_elements"]) + ) + if record.get("step"): + by_source_steps.setdefault(source, []).append( + str(record["step"]) + ) + # Only a per-step capture can measure a localisation at all, so a + # measured record implies the instrument even when it predates the + # field that names it. + instrument = str( + record.get("measured_by", "per_step_lm_head_input_capture") + ) + instruments[instrument] = instruments.get(instrument, 0) + 1 + + routes[route] = { + "runs": runs, + "steps": total_steps, + "bit_identical_steps": exact_steps, + "bit_identical_runs": bit_identical_runs, + # Two rates, because they answer different questions and reporting + # only one invites the reader to assume the other. The STEP rate is + # the per-logit-vector property DETERM-1 records; the RUN rate is + # the probability that a whole 17-step comparison comes back clean, + # which is what a suite operator actually observes. + "step_bit_identity_rate": exact_steps / total_steps + if total_steps + else None, + "run_bit_identity_rate": bit_identical_runs / runs, + "observed_max_abs_diff": { + "max": max(maxima), + "min": min(nonzero) if nonzero else 0.0, + "median": statistics.median(maxima), + "nonzero_runs": len(nonzero), + "histogram": dict( + sorted(histogram.items(), key=lambda item: float(item[0])) + ), + }, + "runs_with_gate_failures": len(failing), + "first_divergences": dict(sorted(divergences.items())), + "localisation": { + "measured_runs": sum(localisations.values()), + "by_source": dict(sorted(localisations.items())), + "elements_by_source": { + source: sorted(values) + for source, values in sorted(by_source_elements.items()) + }, + "steps_by_source": { + source: sorted(values) + for source, values in sorted(by_source_steps.items()) + }, + "lm_head_input_elements": HIDDEN_SIZE, + "instruments": dict(sorted(instruments.items())), + }, + "is_baseline": runs >= MIN_DETERMINISM_RUNS_PER_ROUTE, + } + if runs < MIN_DETERMINISM_RUNS_PER_ROUTE: + blocking.append( + f"route {route!r}: {runs} run(s), below DETERM-3's minimum of " + f"{MIN_DETERMINISM_RUNS_PER_ROUTE}. No baseline is established " + f"for this route; the figures above are what has been measured " + f"so far and must not be cited as a floor." + ) + if failing: + problems.append( + f"route {route!r}: {len(failing)} run(s) recorded a DETERM-2 " + f"gate failure. They ARE counted in the rate above -- dropping " + f"them would bias it upward -- but a baseline should not be " + f"declared over a window containing a hard-gate failure." + ) + + # I-6. ROUTE DEPENDENCE IS A CLAIM, AND IT NEEDS A TEST. + # + # This used to be `len(set(rates)) > 1` -- a float inequality on two + # ratios. With 1 divergent run of 33 against 2 of 33 that is true, and the + # renderer published "the rate is route-dependent" in bold. It is the same + # n-limited overclaim DETERM-3 exists to prevent, made about DETERM-3's own + # output. + # + # A two-sided Fisher exact test on the 2x2 table of (divergent, clean) per + # route is the standard answer and introduces no tunable of its own. 1-of-33 + # against 2-of-33 gives p = 1.0; the observed counts differ and the + # difference is not supported. Both facts are reported, separately, because + # they are different statements and collapsing them is what went wrong. + counts = [ + (entry["runs"] - entry["bit_identical_runs"], entry["bit_identical_runs"]) + for entry in routes.values() + if entry["runs"] > 0 + ] + observed_rates_differ = ( + len( + { + entry["run_bit_identity_rate"] + for entry in routes.values() + if entry["run_bit_identity_rate"] is not None + } + ) + > 1 + ) + route_dependence_p = ( + _fisher_exact_two_sided(counts[0], counts[1]) + if len(counts) == 2 + else None + ) + route_dependent = ( + route_dependence_p is not None + and route_dependence_p < ROUTE_DEPENDENCE_ALPHA + ) + + gate_failures = sum( + entry["runs_with_gate_failures"] for entry in routes.values() + ) + return { + "min_runs_per_route": MIN_DETERMINISM_RUNS_PER_ROUTE, + "routes": routes, + "observed_rates_differ": observed_rates_differ, + "route_dependence_p": route_dependence_p, + "route_dependence_alpha": ROUTE_DEPENDENCE_ALPHA, + "route_dependent": route_dependent, + "gate_failures": gate_failures, + # I-3. A window containing a DETERM-2 hard-gate failure is NOT a + # baseline, and this flag used to say it was -- four lines below the + # problem text saying it must not. The flag is the machine-readable + # form of that sentence, so it has to agree with it. + "is_baseline": not blocking + and bool(routes) + and gate_failures == 0 + and all(entry["is_baseline"] for entry in routes.values()), + "sources": sources, + "excluded_sources": excluded_sources, + "blocking_problems": blocking, + "problems": blocking + problems, + } + + +def crossover_entry(document: dict, source: str = "") -> dict: + """One run's crossover edges, reduced to what a history needs. + + Task 14 reads the benchmark document, not the task report. The document + published `(4, 12]` and `(12, 64]` with margins and one sentence saying + four points were undecided -- and nowhere that the upper edge had been + `8` and `16` one run earlier. A reader takes an edge with a margin beside + it as measured. One of these edges is; the other moved by a factor of + four between two runs of the same binary. + """ + identity = document.get("identity", {}) + continuation = document.get("continuation", {}) + crossover = continuation.get("crossover") or {} + edges: dict[str, list] = {} + undecided: dict[str, list] = {} + decided = 0 + total = 0 + for history, entry in crossover.items(): + edges[str(history)] = [ + entry.get("append_wins_up_to"), + entry.get("reprefill_wins_from"), + ] + for decision in entry.get("decisions", []): + total += 1 + if decision.get("decided"): + decided += 1 + else: + undecided.setdefault(str(history), []).append( + decision.get("suffix") + ) + return { + "utc": identity.get("utc"), + "fastflow_revision": identity.get("fastflow_revision"), + "corelib_dll_sha256": identity.get("corelib_dll_sha256"), + "samples_interleaved": continuation.get("samples_interleaved"), + "suffix_grid": continuation.get("suffixes"), + "edges": edges, + "points_decided": decided, + "points_total": total, + "undecided_suffixes": { + key: sorted(value) for key, value in sorted(undecided.items()) + }, + "source": source, + } + + +def merge_crossover_history(history: list[dict], entry: dict) -> list[dict]: + """Append `entry` unless a run with the same UTC stamp is already there. + + Keyed on the measurement time rather than on position, so re-rendering the + same baseline twice does not invent a second observation. + """ + stamps = {item.get("utc") for item in history} + if entry.get("utc") in stamps: + return history + return sorted( + history + [entry], key=lambda item: str(item.get("utc") or "") + ) + + +def crossover_edge_stability(history: list[dict]) -> dict: + """Which edges have held across runs and which have not. + + Derived from the history rather than asserted, because the conclusion -- + read the lower edge as measured and the upper as an upper bound -- is + exactly the kind of guidance that was living in a task report where the + reader who needs it will not look. + """ + lower: dict[str, set] = {} + upper: dict[str, set] = {} + for item in history: + if not item.get("samples_interleaved"): + # A non-interleaved run measured a different thing; it is kept in + # the table for provenance but must not vote on stability. + continue + for key, edge in (item.get("edges") or {}).items(): + if not isinstance(edge, list) or len(edge) != 2: + continue + lower.setdefault(key, set()).add(edge[0]) + upper.setdefault(key, set()).add(edge[1]) + return { + "runs_considered": sum( + 1 for item in history if item.get("samples_interleaved") + ), + "lower_stable": { + key: sorted(values) for key, values in sorted(lower.items()) + }, + "upper_observed": { + key: sorted(values) for key, values in sorted(upper.items()) + }, + "lower_is_stable": all(len(values) == 1 for values in lower.values()) + and bool(lower), + "upper_is_stable": all(len(values) == 1 for values in upper.values()) + and bool(upper), + } + + +def crossover_stability_narrative(stability: dict) -> tuple[str, str]: + """The lead-in and the guidance for one stability result. + + ONE function returning BOTH, because the defect this replaces was that + they were emitted separately: the lead-in "the bracket's upper edge is not + stable and its lower edge is" was unconditional, printed whenever more + than one run existed, while only the guidance beneath it was conditional. + A future run whose LOWER edge moved would have printed that sentence in + bold directly above a table showing it moved -- and lost the guidance + entirely, because neither branch fired. + + Returning a pair makes that structurally impossible: every case has a + lead-in that matches its own data and a guidance paragraph, and adding a + case without both is a change to this function rather than an omission + somewhere in a render. + """ + runs = stability.get("runs_considered", 0) + lower = stability.get("lower_stable") or {} + upper = stability.get("upper_observed") or {} + + def edges(mapping: dict) -> str: + return "; ".join( + f"{', '.join(str(value) for value in values)} at history {key}" + for key, values in mapping.items() + ) + + if runs < 2 or not lower: + return ( + "This table is every render of this document that recorded a " + "crossover, from the committed baseline artifacts.", + "> **Not enough interleaved runs to say whether either edge is " + "stable.** Treat the bracket as a single observation until a " + "second interleaved run extends this table.", + ) + + if stability.get("lower_is_stable") and not stability.get( + "upper_is_stable" + ): + return ( + "**The bracket's upper edge is not stable and its lower edge " + "is.** This table is every render of this document that recorded " + "a crossover, from the committed baseline artifacts.", + "> **Read the lower edge as measured and the upper edge as an " + f"upper bound.** Across the {runs} interleaved runs the lower " + "edge has been " + edges(lower) + " every time, while the upper " + "edge has taken " + edges(upper) + " on the same binary and the " + "same model. The upper edge moves with how quiet the machine " + "was, because that is what decides how many points near the " + "crossover can be called at all.", + ) + + if stability.get("lower_is_stable") and stability.get("upper_is_stable"): + return ( + f"**Both edges have held across {runs} interleaved runs.** This " + "table is every render of this document that recorded a " + "crossover, from the committed baseline artifacts.", + f"> Both edges have held across the {runs} interleaved runs: " + + edges(lower) + " below, " + edges(upper) + " above. That is a " + "small number of runs on a machine measured moving by a factor " + "of 1.8, so it is agreement rather than proof.", + ) + + if not stability.get("lower_is_stable") and stability.get( + "upper_is_stable" + ): + return ( + "**The bracket's LOWER edge has moved between runs.** This table " + "is every render of this document that recorded a crossover, " + "from the committed baseline artifacts.", + "> **Do not read the lower edge as measured.** It has taken " + + edges(lower) + f" across {runs} interleaved runs of the same " + "binary on the same model. The upper edge has held at " + + edges(upper) + ", but with the lower edge moving the bracket " + "is not a stable interval and should be treated as a single " + "observation.", + ) + + return ( + "**Neither edge of the bracket is stable between runs.** This table " + "is every render of this document that recorded a crossover, from " + "the committed baseline artifacts.", + "> **Neither edge can be read as measured.** Across " + f"{runs} interleaved runs of the same binary the lower edge has " + "taken " + edges(lower) + " and the upper edge " + edges(upper) + + ". Whatever a single run reports is one observation on an unstable " + "machine, and the crossover is not resolved by this data.", + ) + + +def load_determinism_records(pattern: str) -> list[dict]: + records = [] + for path in sorted(globlib.glob(pattern)): + record = json.loads(Path(path).read_text(encoding="utf-8")) + # I-7. Where each record came from, so a pooled glob can be audited + # rather than trusted. Prefixed so it cannot collide with a field the + # comparator writes. + record["_source"] = path + records.append(record) + return records + + +def _ns(value: object) -> str: + if not isinstance(value, (int, float)): + return "n/a" + return f"{float(value) / 1e6:.1f} ms" + + +def _bytes(value: object) -> str: + if not isinstance(value, (int, float)): + return "n/a" + value = float(value) + for unit in ("B", "KiB", "MiB", "GiB"): + if abs(value) < 1024 or unit == "GiB": + return f"{value:,.0f} {unit}" if unit == "B" else f"{value:,.2f} {unit}" + value /= 1024 + return f"{value} B" + + +def _rate(value: object, total: object) -> str: + if value is None: + return "not measured" + return f"{float(value) * 100:.2f}% (n = {total})" + + +def render_markdown(document: dict) -> str: + identity = document.get("identity", {}) + load = document.get("model_load", {}) + ttft = document.get("ttft", {}) + memory = document.get("memory", {}) + scatter = document.get("v_scatter", {}) + determinism = document.get("determinism", {}) + + lines: list[str] = [] + add = lines.append + add(_MARKDOWN_BEGIN) + add("") + add("## Phi-4 mini instruct on AIE4 (corelib backend) — recorded baseline") + add("") + add( + "Recorded per design section 15.6, **without pass/fail thresholds**. " + "Design section 4 makes performance explicitly not a release blocker " + "for this release; these figures exist so a later change can be " + "compared against a measured starting point, not so a number can be " + "defended." + ) + add("") + # A caveat that is always present, because it applies to every latency + # figure below and a reader who does not know it will read ordinary + # machine noise as a regression. + add( + "> **Every latency and throughput figure here comes from ONE run on a " + "shared lab machine.** Task 13 ran this same benchmark three times " + "against the same binary and the same model within two hours and " + "measured decode throughput of 22.5, 22.4 and 12.4 tokens/s at " + "context 128 — a factor of 1.8, with no code change. Within a single " + "run, per-token append latency stepped from 76 ms to 46 ms partway " + "through the continuation sweep and stayed there. The machine runs " + "corporate endpoint agents whose scans are not under this project's " + "control, and the host share of a decode token is large enough for " + "CPU contention to show." + ) + add("") + # TWO COMPARISONS, TWO RULES, AND WHICH IS WHICH. + # + # This caveat used to end "treat a difference below roughly 2x as + # unresolved", stated unconditionally -- and thirty lines below it the + # crossover table published a bracket decided on differences far smaller + # than 2x. Two live rules, one contradicting the other, is the same defect + # as publishing a grid artifact as a measurement: the reader cannot tell + # which number to act on. + add( + "> **Two different comparisons, and they do not share a rule.** " + "Comparing a figure here against one from a DIFFERENT run — a later " + "revision, another machine, this document a month from now — is " + "subject to that 1.8x instability, so treat a difference below " + "roughly 2x as unresolved unless it is reproduced across runs. The " + "append-versus-re-prefill decision AT EACH POINT is not that " + "comparison: its samples are **interleaved within a single point**, " + "so a regime shift moves both routes together, and each point " + "additionally has to beat the drift measured across it. That is why " + "a single point can resolve differences the 2x rule could not." + ) + add("") + # SCOPED, because the exemption above was being read too widely. + # + # This blockquote used to end "the instability it would be guarding + # against has been measured and subtracted rather than assumed away", + # full stop -- which tells a reader that the crossover bracket is exempt + # from run-to-run doubt. Its upper edge then moved from 16 to 64 between + # two runs of the same binary. The per-point decision is paired in time; + # WHICH points can be decided at all is not, and the bracket is built out + # of that. + add( + "> **The exemption is for a single point, not for the bracket.** " + "Whether a given suffix can be decided depends on how quiet the " + "machine was during that point, so the SET of decided points — and " + "therefore the bracket's width — is subject to the same run-to-run " + "instability as everything else here. The run-to-run table under " + "\"Where append stops winning\" shows how far it has actually " + "moved." + ) + add("") + add("### Identity") + add("") + add("| | |") + add("| --- | --- |") + for label, key in ( + ("machine", "machine"), + ("CPU", "cpu_sku"), + ("NPU", "npu_sku"), + ("NPU driver", "npu_driver_version"), + ("corelib DLL", "corelib_dll_path"), + ("corelib SHA-256", "corelib_dll_sha256"), + ("corelib version", "corelib_version"), + ("corelib source revision", "corelib_source_revision"), + ("DynamicDispatch", "dynamic_dispatch_version"), + ("RyzenMM", "ryzen_mm_version"), + ("XRT", "xrt_version"), + ("model directory", "model_dir"), + ("model SHA-256", "model_sha256"), + ("FastFlow revision", "fastflow_revision"), + ("measured (UTC)", "utc"), + ): + add(f"| {label} | `{identity.get(key, 'n/a')}` |") + add("") + + add("### Model load and TTFT") + add("") + add("| | |") + add("| --- | --- |") + add(f"| manifest parse and file mapping | {_ns(load.get('manifest_map_ns'))} |") + # I-5. The shape plan is 86% of model load, and the first rendered version + # of this table omitted it entirely -- leaving 11.2 s of a 12.8 s load + # unexplained in the document while the finding sat only in a task report. + # Every phase is listed, and the remainder is stated rather than left for + # the reader to subtract. + share = load.get("shape_plan_share") + share_text = f" — **{share * 100:.0f}% of load**" if isinstance( + share, (int, float) + ) else "" + add( + f"| **1..4096 helper interrogation (`Phi4ShapePlan::Build`)** | " + f"**{_ns(load.get('shape_plan_ns'))}**{share_text} |" + ) + add( + f"| weight pack/upload ({load.get('weight_objects', 'n/a')} objects) " + f"| {_ns(load.get('weight_pack_ns'))} |" + ) + add( + f"| stream, {load.get('device_tensors', 'n/a')} device tensors, RoPE " + f"upload | {_ns(load.get('device_setup_ns'))} |" + ) + add(f"| unaccounted | {_ns(load.get('unaccounted_ns'))} |") + add(f"| total model load | {_ns(load.get('total_ns'))} |") + add( + f"| cold TTFT ({ttft.get('prompt_token_count', 'n/a')} prompt tokens, " + f"row extent {ttft.get('row_extent', 'n/a')}) " + f"| {_ns(ttft.get('cold_ns'))} |" + ) + add( + f"| warm TTFT, same Stream after `clear_context()` " + f"| {_ns(ttft.get('warm_ns'))} |" + ) + add("") + + points = document.get("prefill", {}).get("points", []) + if points: + add("### Fresh prefill") + add("") + add("| rows | padded rows | wall time | tokens/s |") + add("| ---: | ---: | ---: | ---: |") + for point in points: + add( + f"| {point.get('rows')} | {point.get('padded_rows', 'n/a')} | " + f"{_ns(point.get('ns'))} | " + f"{point.get('tokens_per_second', float('nan')):,.1f} |" + ) + add("") + + runs = document.get("decode", {}).get("runs", []) + if runs: + add("### Decode") + add("") + add( + "| starting context | tokens | tokens/s | p50 | p95 | " + "synchronizes per pass |" + ) + add("| ---: | ---: | ---: | ---: | ---: | ---: |") + for run in runs: + add( + f"| {run.get('start_context')} | {run.get('tokens')} | " + f"{run.get('tokens_per_second', float('nan')):,.2f} | " + f"{_ns(run.get('p50_ns'))} | {_ns(run.get('p95_ns'))} | " + f"{run.get('synchronizes_per_pass')} |" + ) + add("") + + continuation = document.get("continuation", {}) + if continuation.get("points"): + add("### Continuation routes") + add("") + add("| history rows | suffix | route | samples | p50 | p95 |") + add("| ---: | ---: | --- | ---: | ---: | ---: |") + for point in continuation["points"]: + add( + f"| {point.get('history_rows')} | {point.get('suffix')} | " + f"{point.get('route')} | " + f"{len(point.get('samples_ns', []))} | " + f"{_ns(point.get('p50_ns'))} | {_ns(point.get('p95_ns'))} |" + ) + add("") + crossover = continuation.get("crossover", {}) + if crossover: + add("#### Where append stops winning") + add("") + add( + "**This is a BRACKET, not a threshold.** Append and " + "re-prefill samples are interleaved within each point, so a " + "machine regime shift moves both together rather than one. A " + "point counts as decided only when the gap between the routes " + "exceeds **both** the larger within-point p50-to-p95 spread " + "**and** the larger drift between a route's first and last " + "sample there; anything else widens the bracket. Task 14 has " + "to choose this constant, so what it needs to see is how much " + "room the measurement leaves — not a number picked because it " + "was the last grid point where append happened to win." + ) + add("") + add( + "| rendered history | append decisively wins up to | " + "re-prefill decisively wins from | crossover lies in | " + "margin at each bracket edge |" + ) + add("| ---: | ---: | ---: | :--- | :--- |") + for history, entry in crossover.items(): + lower = entry.get("append_wins_up_to") + upper = entry.get("reprefill_wins_from") + if not upper: + span = "**not bracketed by this grid**" + elif entry.get("bracket_is_tight"): + span = f"exactly {upper}" + else: + span = f"`({lower}, {upper}]` — not resolved further" + # HOW FIRM EACH EDGE IS, as the gap over the uncertainty that + # had to be beaten. Without it "decisively" is a word rather + # than a number, and the bracket's edges are exactly where a + # reader should be allowed to check the strength of the call. + margins = [] + for decision in entry.get("decisions", []): + if decision.get("suffix") in (lower, upper): + ratio = decision.get("gap_over_uncertainty") + if isinstance(ratio, (int, float)): + margins.append( + f"{decision['suffix']}: {ratio:.1f}x" + ) + add( + f"| {history} | {lower} | {upper} | {span} | " + f"{', '.join(margins) if margins else 'n/a'} |" + ) + add("") + undecided_by_history: dict[str, list] = {} + for history, entry in crossover.items(): + for decision in entry.get("decisions", []): + if not decision.get("decided"): + undecided_by_history.setdefault( + str(history), [] + ).append(decision.get("suffix")) + undecided = sum( + len(value) for value in undecided_by_history.values() + ) + total = sum( + len(entry.get("decisions", [])) for entry in crossover.values() + ) + add( + f"{total - undecided} of {total} sweep points were decided; " + f"{undecided} were not and widen the brackets above. The " + f"margin column is the gap between the routes divided by the " + f"uncertainty it had to beat at that suffix." + ) + add("") + # WHICH points, not just how many. "Four were undecided" tells a + # reader nothing about where the measurement ran out. + if undecided_by_history: + add( + "Undecided points, which is where the bracket's width " + "comes from: " + + "; ".join( + f"history {key} at suffix " + + ", ".join(str(value) for value in sorted(values)) + for key, values in sorted( + undecided_by_history.items() + ) + ) + + "." + ) + add("") + + history = document.get("crossover_history") or [] + if len(history) > 1: + stability = crossover_edge_stability(history) + lead_in, guidance = crossover_stability_narrative(stability) + add("##### The same measurement, run to run") + add("") + add(lead_in) + add("") + add( + "| measured (UTC) | interleaved | grid | " + + " | ".join( + f"history {key}" + for key in sorted( + { + key + for item in history + for key in (item.get("edges") or {}) + } + ) + ) + + " | decided | note |" + ) + keys = sorted( + { + key + for item in history + for key in (item.get("edges") or {}) + } + ) + add( + "| --- | :---: | ---: | " + + " | ".join(["---:"] * len(keys)) + + " | ---: | :--- |" + ) + for item in history: + cells = [] + for key in keys: + edge = (item.get("edges") or {}).get(key) + if not isinstance(edge, list) or len(edge) != 2: + cells.append("n/a") + elif edge[1] is None: + # A run that reported a single threshold rather + # than a bracket. Shown as what it was, not as a + # bracket with a hole in it. + cells.append(f"threshold `{edge[0]}`") + else: + cells.append(f"`({edge[0]}, {edge[1]}]`") + decided = ( + f"{item.get('points_decided')}" + f"/{item.get('points_total')}" + if item.get("points_total") + else "n/a" + ) + note = item.get("note", "") + add( + f"| {item.get('utc', 'n/a')} " + f"| {'yes' if item.get('samples_interleaved') else 'no'} " + f"| {len(item.get('suffix_grid') or [])} points | " + + " | ".join(cells) + + f" | {decided} | {note} |" + ) + add("") + # ALWAYS emitted, whatever the stability result. The + # guidance used to live in two `if` branches with no `else`, + # so the one case nobody had rendered -- a lower edge that + # moved -- printed a false lead-in and no guidance at all. + add(guidance) + add("") + if True: + # Every non-interleaved ROW, because that is what the + # table above shows. The filter used to also require a + # bracket, which silently dropped the sparse-grid row and + # made the sentence say "1" under a table with two `no`s. + older = [ + item + for item in history + if not item.get("samples_interleaved") + ] + if older: + add( + f"{len(older)} of the {len(history)} rows are " + "non-interleaved and are excluded from that " + "comparison, because they measured the routes in " + "blocks rather than paired in time. They are " + "shown for provenance." + ) + add("") + add( + f"Prefix-monotonic: " + f"`{continuation.get('prefix_monotonic')}`. Decision rule: " + f"{continuation.get('decision_rule', 'n/a')}." + ) + add("") + + add("### V scatter, memory and synchronization") + add("") + add("| | |") + add("| --- | --- |") + add( + f"| V cache reads per model step | " + f"{scatter.get('reads_per_model_step')} |" + ) + add( + f"| per-head V writes per model step | " + f"{scatter.get('writes_per_model_step')} |" + ) + add(f"| V bytes transferred | {_bytes(scatter.get('bytes'))} |") + add(f"| V scatter wall time | {_ns(scatter.get('nanoseconds'))} |") + add(f"| FP16 embedding | {_bytes(memory.get('embedding_bytes'))} |") + add(f"| KV cache | {_bytes(memory.get('kv_bytes'))} |") + add(f"| corelib packed weights | {_bytes(memory.get('packed_weight_bytes'))} |") + add(f"| scratch and device tensors | {_bytes(memory.get('scratch_bytes'))} |") + add(f"| mapped ONNX source | {_bytes(memory.get('mapped_source_bytes'))} |") + add(f"| peak host private bytes | {_bytes(memory.get('peak_private_bytes'))} |") + add( + f"| peak host working set | " + f"{_bytes(memory.get('peak_working_set_bytes'))} |" + ) + add("") + add( + "128-token stability window: device tensors created after warmup " + f"**{memory.get('device_tensor_creates_after_warmup')}**, weight " + f"objects **{memory.get('weight_creates_after_warmup')}**, net live " + f"corelib objects **{memory.get('live_corelib_object_delta')}**, " + f"private-byte growth **{_bytes(memory.get('private_bytes_growth'))}**, " + "least-squares private-byte slope over tokens 9..128 " + f"**{_bytes(memory.get('private_bytes_slope_per_token'))}/token**." + ) + add("") + + add("### DETERM-3 — run-to-run logit bit-identity baseline") + add("") + if determinism: + add( + "Two runs of the same binary, same device, same explicit token " + "IDs, proven by recorded SHA-256 to have loaded the same corelib " + "DLL. Per `DETERM-3` the routes are reported separately and never " + "averaged: append and re-prefill drive different row extents " + "through the same LM-head shape." + ) + add("") + add( + "| route | runs (n) | step bit-identity | run bit-identity | " + "max abs diff: max / median | nonzero runs | " + f"≥ {MIN_DETERMINISM_RUNS_PER_ROUTE} runs? |" + ) + # The last column is the run-count precondition, NOT "is this a + # settled baseline". Labelling it "baseline?" put a `yes` beside a + # route whose window contained a hard-gate failure, two lines above + # the blockquote saying exactly that must not be read as a baseline. + add("| --- | ---: | ---: | ---: | ---: | ---: | --- |") + for route, entry in determinism.get("routes", {}).items(): + spread = entry.get("observed_max_abs_diff", {}) + add( + f"| {route} | {entry.get('runs')} | " + f"{_rate(entry.get('step_bit_identity_rate'), entry.get('steps'))}" + f" | " + f"{_rate(entry.get('run_bit_identity_rate'), entry.get('runs'))}" + f" | {spread.get('max')} / {spread.get('median')} | " + f"{spread.get('nonzero_runs')} | " + f"{'yes' if entry.get('is_baseline') else '**no — too few runs**'} |" + ) + add("") + if not determinism.get("is_baseline"): + add( + "**This window is not a settled baseline.** The figures above " + "are what has been measured; the problems listed below say " + "why they cannot yet be cited as the floor `DETERM-2`'s " + '"degrades from the recorded baseline" clause needs.' + ) + add("") + for route, entry in determinism.get("routes", {}).items(): + histogram = entry.get("observed_max_abs_diff", {}).get("histogram") + if histogram: + add( + f"Observed maximum absolute difference, `{route}`: " + + ", ".join( + f"`{key}` × {count}" for key, count in histogram.items() + ) + + "." + ) + if any( + entry.get("observed_max_abs_diff", {}).get("histogram") + for entry in determinism.get("routes", {}).values() + ): + add("") + by_source: dict[str, int] = {} + elements_by_source: dict[str, list[int]] = {} + steps_by_source: dict[str, list[str]] = {} + instruments: dict[str, int] = {} + width = HIDDEN_SIZE + for entry in determinism.get("routes", {}).values(): + record = entry.get("localisation") or {} + for source, count in (record.get("by_source") or {}).items(): + by_source[source] = by_source.get(source, 0) + count + for source, values in ( + record.get("elements_by_source") or {} + ).items(): + elements_by_source.setdefault(source, []).extend(values) + for source, values in (record.get("steps_by_source") or {}).items(): + steps_by_source.setdefault(source, []).extend(values) + for name, count in (record.get("instruments") or {}).items(): + instruments[name] = instruments.get(name, 0) + count + width = record.get("lm_head_input_elements", width) + if by_source: + add("#### Where the divergence enters — measured") + add("") + add( + "At the step whose logits first differ, the harness records " + "the exact row that was fed to the LM head in each run, so " + "this is an observation and not an inference. " + + ", ".join( + f"**{count} event(s): `{source}`**" + for source, count in sorted(by_source.items()) + ) + + "." + ) + add("") + # N-2. THE INSTRUMENT, in the section that carries the claim. + # + # This paragraph asserts "an observation and not an inference" and + # said nothing about the instrument. The caveat existed only in a + # task report, and a reader of this document is exactly who needs + # it. Conditional on the data: only a per-step capture can measure + # a localisation at all. + if instruments: + # NO RATE CLAIM HERE. + # + # This blockquote used to say the phenomenon "still + # reproduces at a rate consistent with the uninstrumented + # campaigns". That is the sentence report section 26 + # withdrew -- per route the distribution flipped, and the + # comparison would have needed a sevenfold change to register + # -- and it was republished as a hardcoded literal into the + # reader-facing document, where it carried no correction and + # no supporting data. The document contains no uninstrumented + # figures at all, so it could not have supported it. + add( + "> **The instrument perturbs what it measures.** These " + f"{sum(instruments.values())} localisation(s) were " + "measured by capturing the LM-head input after every " + "model step, which adds a host tensor read and a stream " + "acquisition between steps — changing the timing of " + "exactly the window a race would occupy. The phenomenon " + "survives the instrumentation: it still reproduces, and " + "with the same coarse signature. What this data cannot " + "rule out is that the capture shifts the rate, or which " + "step is reached first — the campaigns are far too small " + "to detect either." + ) + add("") + if by_source.get("model_body"): + count = by_source["model_body"] + values = sorted(elements_by_source.get("model_body", [])) + steps = sorted(set(steps_by_source.get("model_body", []))) + add( + f"In the **{count}** `model_body` event(s) the two runs " + f"fed the LM head **different rows** — " + + ", ".join(f"{value:,}" for value in values) + + f" of {width:,} elements differing, at " + + ", ".join(f"`{step}`" for step in steps) + + ". **In those events the divergence enters the model " + "body, not the LM-head dispatch.** Each run's LM head was " + "separately measured to be correctly rounded against its " + "own input, so it is faithfully transforming inputs that " + "already differ." + ) + add("") + if by_source.get("lm_head"): + count = by_source["lm_head"] + steps = sorted(set(steps_by_source.get("lm_head", []))) + # Built in pieces, deliberately. + # + # This was one expression with a trailing conditional, and + # Python bound the `if` to the WHOLE concatenation rather than + # the last clause. In a window with no `model_body` event the + # leading sentence -- the count and the steps -- was discarded + # and the document published a paragraph beginning with a bare + # period, silently losing the evidence. The test passed over + # it because it asserted only the trailing substring. + where = ( + ", at " + ", ".join(f"`{step}`" for step in steps) + if steps + else "" + ) + lead = ( + f"In the **{count}** `lm_head` event(s) the two runs fed " + f"the LM head an **identical** row and it produced " + f"different logits{where}." + ) + if by_source.get("model_body"): + tail = ( + " That is the case `DETERM-1` describes, and it is a " + "different mechanism from the one above. **Both are " + "present in this window**, so neither explanation " + "covers it alone." + ) + else: + tail = ( + " That is the case `DETERM-1` describes: the " + "divergence is inside the LM-head dispatch." + ) + add(lead + tail) + add("") + if by_source.get("model_body"): + add( + "**The layer at which it enters is not known.** Layer 0's " + "K and V have been bit-identical in the events where the " + "emitted tokens matched, and layer 31's have not, which " + "bounds it to somewhere above layer 0. Narrowing it " + "further needs per-layer capture, and until then this is " + "an open question rather than a characterised one." + ) + add("") + gate_failures = sum( + entry.get("runs_with_gate_failures", 0) + for entry in determinism.get("routes", {}).values() + ) + if gate_failures: + add( + f"> **{gate_failures} run(s) in this window breached a " + f"`DETERM-2` HARD GATE.** They are counted in the rates above " + f"-- dropping them would bias the figures upward, and " + f"silently -- and they are not a wobble within the recorded " + f"tolerance. Read the run records before citing anything here " + f"as a settled baseline." + ) + add("") + p_value = determinism.get("route_dependence_p") + if determinism.get("route_dependent"): + add( + "**The rate is route-dependent** (Fisher exact " + f"p = {p_value:.3g} < " + f"{determinism.get('route_dependence_alpha')}). The two " + "routes are reported separately above rather than pooled." + ) + add("") + elif determinism.get("observed_rates_differ"): + add( + "The two routes' observed rates differ, but **the n does not " + "support calling the rate route-dependent** (Fisher exact " + f"p = {p_value:.3g} against " + f"{determinism.get('route_dependence_alpha')}). They are " + "still reported separately, per `DETERM-3`, because pooling " + "them would hide a difference that a larger campaign might " + "resolve — not because this one resolved it." + ) + add("") + for problem in determinism.get("problems", []): + add(f"- {problem}") + if determinism.get("problems"): + add("") + else: + add( + "No `DETERM-1` records were supplied, so **no baseline** exists. " + "`DETERM-2`'s \"degrades from the recorded baseline\" clause " + "continues to enforce nothing, per `DETERM-3`." + ) + add("") + add(_MARKDOWN_END) + return "\n".join(lines) + "\n" + + +def _splice(existing: str, section: str) -> str: + begin = existing.find(_MARKDOWN_BEGIN) + end = existing.find(_MARKDOWN_END) + if begin != -1 and end != -1: + return existing[:begin] + section + existing[end + len(_MARKDOWN_END) + 1 :] + separator = "" if existing.endswith("\n\n") else "\n" + return existing + separator + "\n---\n\n" + section + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description=( + "Validate the Phi-4 AIE4 baseline and render it into the " + "benchmark document." + ) + ) + parser.add_argument( + "--input", + required=True, + help=( + "phi4_aie4_baseline.json, as written by the AIE4 benchmark " + "instrument on a machine with a device (not in this branch)" + ), + ) + parser.add_argument( + "--determinism-glob", + required=True, + help=( + "glob matching the run-scoped determ1-.json artifacts, for " + "example src/build/phi4-hardware/artifacts/*/determ1-*.json. " + "Required, not optional: DETERM-3's baseline is the point of this " + "task, and a tool that silently rendered a report without it " + "would reproduce the absence it exists to close." + ), + ) + parser.add_argument( + "--markdown", + required=True, + help="benchmark document to splice the rendered section into", + ) + parser.add_argument( + "--output-json", + help="write the validated document, with the determinism baseline " + "merged in, to this path", + ) + parser.add_argument( + "--crossover-history", + help=( + "a committed JSON list of previous runs' crossover edges. This " + "run's edges are appended (keyed on the measurement UTC, so " + "re-rendering does not invent an observation) and the history is " + "rendered into the document. Without it the document publishes a " + "bracket with no indication that its upper edge has moved by a " + "factor of four between runs of the same binary." + ), + ) + parser.add_argument( + "--allow-incomplete-determinism", + action="store_true", + help=( + "render even when a route has fewer than " + f"{MIN_DETERMINISM_RUNS_PER_ROUTE} runs. The rendered section then " + "says NO BASELINE for that route and the exit code is still " + "non-zero; this only exists so a partial campaign can be inspected." + ), + ) + args = parser.parse_args(argv) + + document = json.loads(Path(args.input).read_text(encoding="utf-8")) + problems = validate(document) + + # Merged in memory now, persisted only if the document is actually + # rendered. + # + # It used to be written straight away, so a baseline that FAILED + # validation still appended a row to the committed history -- a run whose + # numbers were never published becoming a permanent observation that + # later runs are compared against. + history: list[dict] = [] + history_path = ( + Path(args.crossover_history) if args.crossover_history else None + ) + if history_path is not None: + if history_path.exists(): + history = json.loads(history_path.read_text(encoding="utf-8")) + history = merge_crossover_history( + history, crossover_entry(document, source=str(args.input)) + ) + document["crossover_history"] = history + + records = load_determinism_records(args.determinism_glob) + if not records: + print( + f"no DETERM-1 records matched {args.determinism_glob!r}. " + f"DETERM-3's baseline cannot be established, and this tool will " + f"not render a report that implies one exists.", + file=sys.stderr, + ) + problems.append("DETERM-3: no DETERM-1 records were found") + determinism = determinism_baseline(records) + document["determinism"] = determinism + for problem in determinism["problems"]: + problems.append(f"DETERM-3: {problem}") + + if args.output_json: + Path(args.output_json).write_text( + json.dumps(document, indent=2), encoding="utf-8" + ) + + # Rendering is blocked by a structural problem with the baseline itself, + # never by a finding IN it. A DETERM-2 hard-gate failure among the samples + # is a result this document exists to carry; refusing to write the + # document because the campaign found something would discard the + # measurement and leave only an exit code. + determinism_incomplete = bool(determinism["blocking_problems"]) or not records + render_blocked = bool( + [ + problem + for problem in problems + if not problem.startswith("DETERM-3:") + ] + ) or (determinism_incomplete and not args.allow_incomplete_determinism) + + if not render_blocked: + if history_path is not None: + history_path.write_text( + json.dumps(history, indent=2), encoding="utf-8" + ) + target = Path(args.markdown) + existing = target.read_text(encoding="utf-8") if target.exists() else "" + target.write_text( + _splice(existing, render_markdown(document)), encoding="utf-8" + ) + print(f"rendered the baseline into {target}") + + if problems: + print( + "report_phi4_corelib_baseline: FAIL — " + f"{len(problems)} problem(s):", + file=sys.stderr, + ) + for problem in problems: + print(f" - {problem}", file=sys.stderr) + if render_blocked: + print( + " the benchmark document was NOT modified.", + file=sys.stderr, + ) + return 1 + + print("report_phi4_corelib_baseline: PASS") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/requirements-phi4-corelib.txt b/tools/requirements-phi4-corelib.txt new file mode 100644 index 00000000..05e2a42b --- /dev/null +++ b/tools/requirements-phi4-corelib.txt @@ -0,0 +1,3 @@ +numpy +onnx +tokenizers diff --git a/tools/tests/__init__.py b/tools/tests/__init__.py new file mode 100644 index 00000000..73c719f2 --- /dev/null +++ b/tools/tests/__init__.py @@ -0,0 +1 @@ +"""Tests for offline FastFlowLM tooling.""" diff --git a/tools/tests/test_calibrate_phi4_corelib_continuation.py b/tools/tests/test_calibrate_phi4_corelib_continuation.py new file mode 100644 index 00000000..613ce737 --- /dev/null +++ b/tools/tests/test_calibrate_phi4_corelib_continuation.py @@ -0,0 +1,1823 @@ +"""Tests for the Phi-4 continuation threshold calibrator. Task 14 Steps 1-5. + +Design Section 10.7 fixes ONE integer for the release, chosen as "the largest +suffix length whose append p95 is lower at BOTH history lengths, but only after +asserting that the set of winning sampled lengths is a prefix-contiguous set". +Everything below tests that sentence and the ways it can go wrong -- not the +one dataset that happens to be checked in today. + +Three defects in Task 13 survived because a test and its implementation shared +an assumption, so the test could not see the implementation being wrong about +it. The rule followed here is therefore to enumerate the cases the CODE +BRANCHES ON rather than the case that exists: + + * `select_threshold` branches on empty / no-winner / prefix-winner / + non-monotonic, on strict-versus-tied comparison, and on the two accepted + entry shapes -- all four verdicts and both shapes are exercised; + * ingestion branches on every validation it performs, so every rejection has + a test that constructs the specific malformed record; + * the grid width is a branch that MATTERS: the retracted answer "2" came from + honouring only the five spec-named suffixes. There is a test that the same + routes measured on a denser grid select a different, larger threshold, so + an implementation that silently drops the extra points fails; + * the header writer branches on bootstrap-versus-regenerate and must be + idempotent, so it is applied twice; + * the cross-run check branches on permits / contradicts / permits-without- + confirming / nothing comparable, and the contradicting case must reach a + non-zero exit; and + * the published concession is ordered by COST, and the cheapest member of a + conceded band is its widest suffix -- so there is a test that the reported + worst case is not the widest one. +""" + +from __future__ import annotations + +import copy +import json +import pathlib +import re +import tempfile +import unittest + +from tools.report_phi4_corelib_baseline import crossover_entry +from tools.calibrate_phi4_corelib_continuation import ( + CalibrationError, + MIN_WARM_SAMPLES, + REQUIRED_HISTORIES, + REQUIRED_SUFFIXES, + Selection, + apply_section_to_document, + apply_threshold_to_header, + build_document_section, + conceded_points, + history_agreement, + load_continuation_samples, + lower_edge_confidence, + main, + percentile_ns, + recorded_lower_edges, + select_threshold, + select_threshold_detailed, +) + +_REPO_ROOT = pathlib.Path(__file__).resolve().parents[2] +_BASELINE = _REPO_ROOT / "docs" / "docs" / "benchmarks" / "phi4_aie4_baseline.json" +_HISTORY = ( + _REPO_ROOT / "docs" / "docs" / "benchmarks" / "phi4_aie4_crossover_history.json" +) +_HEADER = ( + _REPO_ROOT + / "src" + / "include" + / "models" + / "phi4" + / "phi4_corelib_aie4_tuning.hpp" +) +_DOCUMENT = _REPO_ROOT / "docs" / "docs" / "benchmarks" / "phi4_results.md" + +# The task report is a plan artifact rather than a shipped file, so it sits +# in the SDD working area beside the repository and is not committed here. +# The guard reads it when it is present: a retracted claim survived two +# review rounds in that file precisely because nothing read it back. +_REPORT = ( + _REPO_ROOT + / ".superpowers" + / "sdd" + / "2026-08-31-phi4-aie4-corelib-fastflow" + / "task-14-report.md" +) + + +def _flat(append: float, reprefill: float) -> dict: + return {"append_p95": append, "reprefill_p95": reprefill} + + +def _per_history(**by_history: tuple) -> dict: + return { + "per_history": { + int(name.lstrip("h")): _flat(*pair) + for name, pair in by_history.items() + } + } + + +class SelectThresholdTest(unittest.TestCase): + """The Section 10.7 selection rule, one branch at a time.""" + + def test_non_monotonic_winners_select_zero(self): + samples = { + 1: {"append_p95": 4.0, "reprefill_p95": 5.0}, + 2: {"append_p95": 7.0, "reprefill_p95": 5.0}, + 32: {"append_p95": 4.0, "reprefill_p95": 5.0}, + } + self.assertEqual(select_threshold(samples), 0) + + def test_non_monotonic_emits_a_diagnostic_naming_the_offender(self): + samples = { + 1: _flat(4.0, 5.0), + 2: _flat(7.0, 5.0), + 32: _flat(4.0, 5.0), + } + selection = select_threshold_detailed(samples) + self.assertIsInstance(selection, Selection) + self.assertEqual(selection.threshold, 0) + self.assertEqual(selection.verdict, "non_monotonic") + self.assertEqual(selection.winners, (1, 32)) + self.assertTrue(selection.diagnostics) + joined = " ".join(selection.diagnostics) + self.assertIn("32", joined) + self.assertIn("2", joined) + + def test_no_winner_selects_zero(self): + samples = { + 1: _flat(9.0, 5.0), + 2: _flat(9.0, 5.0), + 32: _flat(9.0, 5.0), + } + selection = select_threshold_detailed(samples) + self.assertEqual(selection.threshold, 0) + self.assertEqual(selection.verdict, "no_winner") + self.assertEqual(selection.winners, ()) + + def test_prefix_contiguous_winners_select_the_largest_winner(self): + samples = { + 1: _flat(4.0, 5.0), + 2: _flat(4.0, 5.0), + 32: _flat(9.0, 5.0), + 128: _flat(9.0, 5.0), + } + selection = select_threshold_detailed(samples) + self.assertEqual(selection.threshold, 2) + self.assertEqual(selection.verdict, "selected") + self.assertEqual(selection.winners, (1, 2)) + self.assertEqual(selection.diagnostics, ()) + + def test_every_sampled_length_winning_selects_the_largest_sampled_length(self): + samples = {1: _flat(1.0, 5.0), 2: _flat(1.0, 5.0), 32: _flat(1.0, 5.0)} + selection = select_threshold_detailed(samples) + self.assertEqual(selection.threshold, 32) + self.assertEqual(selection.verdict, "selected") + + def test_empty_sample_set_selects_zero_rather_than_raising(self): + selection = select_threshold_detailed({}) + self.assertEqual(selection.threshold, 0) + self.assertEqual(selection.verdict, "empty") + + def test_a_tie_is_not_a_win(self): + """Section 10.7 says append p95 must be LOWER, not "not higher".""" + samples = {1: _flat(5.0, 5.0), 2: _flat(9.0, 5.0)} + selection = select_threshold_detailed(samples) + self.assertEqual(selection.threshold, 0) + self.assertEqual(selection.verdict, "no_winner") + + def test_insertion_order_does_not_change_the_answer(self): + forwards = {1: _flat(1.0, 5.0), 2: _flat(1.0, 5.0), 32: _flat(9.0, 5.0)} + backwards = {32: _flat(9.0, 5.0), 2: _flat(1.0, 5.0), 1: _flat(1.0, 5.0)} + self.assertEqual(select_threshold(forwards), 2) + self.assertEqual(select_threshold(backwards), 2) + + def test_the_return_value_is_a_plain_int(self): + value = select_threshold({1: _flat(1.0, 5.0)}) + self.assertIs(type(value), int) + + def test_a_win_at_one_history_and_a_loss_at_the_other_is_not_a_win(self): + samples = { + 1: _per_history(h512=(1.0, 5.0), h2048=(1.0, 5.0)), + 2: _per_history(h512=(9.0, 5.0), h2048=(1.0, 5.0)), + 4: _per_history(h512=(9.0, 5.0), h2048=(9.0, 5.0)), + } + selection = select_threshold_detailed(samples) + self.assertEqual(selection.threshold, 1) + self.assertEqual(selection.winners, (1,)) + + def test_a_win_at_every_history_is_a_win(self): + samples = { + 1: _per_history(h512=(1.0, 5.0), h2048=(1.0, 5.0)), + 2: _per_history(h512=(1.0, 5.0), h2048=(1.0, 5.0)), + } + self.assertEqual(select_threshold(samples), 2) + + def test_the_conjunction_is_not_a_worst_case_reduction(self): + """max(append) < min(reprefill) is STRICTER than a win at each history. + + A reduction that collapses the histories to one worst-case pair before + comparing would reject suffix 1 here, because max append 10 is not + below min reprefill 2 -- even though append wins at BOTH histories. + """ + samples = {1: _per_history(h512=(10.0, 20.0), h2048=(1.0, 2.0))} + self.assertEqual(select_threshold(samples), 1) + + def test_an_empty_per_history_map_is_rejected(self): + with self.assertRaises(CalibrationError): + select_threshold({1: {"per_history": {}}}) + + def test_mixing_the_two_entry_shapes_in_one_entry_is_rejected(self): + entry = _flat(1.0, 5.0) + entry["per_history"] = {512: _flat(1.0, 5.0)} + with self.assertRaises(CalibrationError): + select_threshold({1: entry}) + + def test_an_entry_with_neither_shape_is_rejected(self): + with self.assertRaises(CalibrationError): + select_threshold({1: {"append_p50": 1.0}}) + + def test_a_non_numeric_measurement_is_rejected(self): + with self.assertRaises(CalibrationError): + select_threshold({1: _flat("fast", 5.0)}) + + def test_a_non_positive_suffix_length_is_rejected(self): + with self.assertRaises(CalibrationError): + select_threshold({0: _flat(1.0, 5.0)}) + with self.assertRaises(CalibrationError): + select_threshold({-4: _flat(1.0, 5.0)}) + + def test_a_non_integer_suffix_length_is_rejected(self): + with self.assertRaises(CalibrationError): + select_threshold({1.5: _flat(1.0, 5.0)}) + + def test_a_boolean_suffix_length_is_rejected(self): + """`True == 1` in Python; a bool key must not pass as suffix 1.""" + with self.assertRaises(CalibrationError): + select_threshold({True: _flat(1.0, 5.0)}) + + +class PercentileTest(unittest.TestCase): + """Nearest-rank, because that is what the recorded p50/p95 used.""" + + def test_median_of_an_odd_sample_count(self): + self.assertEqual(percentile_ns([5, 1, 3, 2, 4], 0.5), 3) + + def test_median_of_an_even_sample_count_does_not_interpolate(self): + self.assertEqual(percentile_ns([1, 2, 3, 4], 0.5), 2) + + def test_p95_of_five_samples_is_the_largest(self): + self.assertEqual(percentile_ns([5, 1, 3, 2, 4], 0.95), 5) + + def test_p95_of_twenty_samples_is_the_nineteenth(self): + self.assertEqual(percentile_ns(list(range(1, 21)), 0.95), 19) + + def test_a_single_sample_is_both_percentiles(self): + self.assertEqual(percentile_ns([7], 0.5), 7) + self.assertEqual(percentile_ns([7], 0.95), 7) + + def test_an_empty_sample_list_is_rejected(self): + with self.assertRaises(CalibrationError): + percentile_ns([], 0.5) + + +def _point(history: int, suffix: int, route: str, base_ns: int) -> dict: + samples = [base_ns + step * 1000 for step in range(MIN_WARM_SAMPLES)] + ordered = sorted(samples) + return { + "history_rows": history, + "suffix": suffix, + "route": route, + "count": len(samples), + "samples_ns": samples, + "p50_ns": ordered[2], + "p95_ns": ordered[4], + "interleaved_with_reprefill": True, + } + + +def _synthetic_document( + suffixes=(1, 2, 32, 128, 256), + append_ns=None, + reprefill_ns=500_000_000, +) -> dict: + """A minimal but VALID document; each test breaks exactly one thing. + + Append costs 50 ms per suffix token, so at the default re-prefill cost of + 500 ms append wins at suffix 1, 2, 4 and 8 and loses from 32 up. + `reprefill_ns` may be one number or a per-history mapping, which is how a + test makes the crossover sit at a DIFFERENT suffix for each history. + """ + append_ns = append_ns or {} + if not isinstance(reprefill_ns, dict): + reprefill_ns = {history: reprefill_ns for history in REQUIRED_HISTORIES} + points = [] + for history in REQUIRED_HISTORIES: + for suffix in suffixes: + points.append( + _point( + history, + suffix, + "append", + append_ns.get((history, suffix), 50_000_000 * suffix), + ) + ) + points.append( + _point(history, suffix, "reprefill", reprefill_ns[history]) + ) + return { + "identity": { + "machine": "xcomedusad-43", + "utc": "2026-09-02T16:32:18Z", + "fastflow_revision": "deadbeef", + "corelib_dll_sha256": "0" * 64, + "model_sha256": "1" * 64, + "cpu_sku": "AMD Eng Sample", + "npu_sku": "AMD XDNA(TM) NPU", + "npu_driver_version": "32.0.20214.4161", + }, + "continuation": { + "histories": list(REQUIRED_HISTORIES), + "suffixes": list(suffixes), + "warm_samples_per_point": MIN_WARM_SAMPLES, + "samples_interleaved": True, + "prefix_monotonic": True, + "points": points, + }, + } + + +def _unclosed_document() -> dict: + """A record whose history-512 winner set is NOT downward-closed. + + `W512 = {1, 8}` and `W2048 = {1, 2, 4}`: append loses at 512/2 and 512/4 and + wins again at 512/8. The intersection is `{1}`, which IS a prefix, so + Section 10.7's monotonicity assertion passes and the rule returns 1 -- while + `min` of the ceilings (8 and 4) would say 4. This is the review's I-1 + counter-example as a loadable document. + + It is also the only shape in which concessions can appear at BOTH histories. + With downward-closed winner sets the ceilings are nested, so everything + above the threshold belongs to whichever history has the larger ceiling and + the conceded band lives at exactly one history. Two-history ordering is + therefore only reachable through a record like this one. + """ + append = { + (512, 1): 10_000_000, + (512, 2): 2_000_000_000, + (512, 4): 2_000_000_000, + (512, 8): 20_000_000, + (512, 32): 5_000_000_000, + (512, 128): 5_000_000_000, + (512, 256): 5_000_000_000, + (2048, 1): 10_000_000, + (2048, 2): 500_000_000, + (2048, 4): 800_000_000, + (2048, 8): 2_000_000_000, + (2048, 32): 5_000_000_000, + (2048, 128): 5_000_000_000, + (2048, 256): 5_000_000_000, + } + return _synthetic_document( + suffixes=(1, 2, 4, 8, 32, 128, 256), + append_ns=append, + reprefill_ns={512: 1_000_000_000, 2048: 1_000_000_000}, + ) + + +class IngestionTest(unittest.TestCase): + """Every rejection ingestion performs gets the record that triggers it.""" + + def test_a_valid_synthetic_document_loads(self): + loaded = load_continuation_samples(_synthetic_document()) + self.assertEqual(loaded.histories, tuple(sorted(REQUIRED_HISTORIES))) + self.assertEqual(loaded.suffixes, (1, 2, 32, 128, 256)) + self.assertEqual(len(loaded.points), 2 * 5) + + def test_the_spec_grid_alone_selects_two_and_a_denser_grid_selects_more(self): + """The retracted "2" was a property of the grid, not of the machine. + + Same routes, same crossover: on `{1, 2, 32, 128, 256}` the last winning + sampled length is 2, and on a grid that also contains 4 and 8 it is 8. + An implementation that honoured only the five spec-named suffixes would + pass every other test in this file and still reproduce the retracted + answer, so this is the test that separates them. + """ + sparse = load_continuation_samples(_synthetic_document()) + self.assertEqual(select_threshold(sparse.samples()), 2) + + dense = load_continuation_samples( + _synthetic_document(suffixes=(1, 2, 4, 8, 32, 128, 256)) + ) + self.assertEqual(dense.suffixes, (1, 2, 4, 8, 32, 128, 256)) + self.assertEqual(select_threshold(dense.samples()), 8) + + def test_a_missing_required_history_is_rejected(self): + for dropped in REQUIRED_HISTORIES: + with self.subTest(history=dropped): + document = _synthetic_document() + block = document["continuation"] + block["points"] = [ + point + for point in block["points"] + if point["history_rows"] != dropped + ] + block["histories"] = [ + h for h in block["histories"] if h != dropped + ] + with self.assertRaises(CalibrationError) as caught: + load_continuation_samples(document) + self.assertIn(str(dropped), str(caught.exception)) + + def test_an_unexpected_history_is_rejected(self): + """The conjunction is OVER the histories, so its domain is fixed.""" + document = _synthetic_document() + block = document["continuation"] + block["points"].append(_point(1024, 1, "append", 1_000_000)) + block["points"].append(_point(1024, 1, "reprefill", 2_000_000)) + block["histories"].append(1024) + with self.assertRaises(CalibrationError) as caught: + load_continuation_samples(document) + self.assertIn("1024", str(caught.exception)) + + def test_a_missing_required_suffix_is_rejected(self): + for dropped in REQUIRED_SUFFIXES: + with self.subTest(suffix=dropped): + document = _synthetic_document() + block = document["continuation"] + block["points"] = [ + point for point in block["points"] if point["suffix"] != dropped + ] + block["suffixes"] = [ + s for s in block["suffixes"] if s != dropped + ] + with self.assertRaises(CalibrationError) as caught: + load_continuation_samples(document) + self.assertIn(str(dropped), str(caught.exception)) + + def test_too_few_warm_samples_at_one_point_is_rejected(self): + document = _synthetic_document() + victim = document["continuation"]["points"][3] + victim["samples_ns"] = victim["samples_ns"][: MIN_WARM_SAMPLES - 1] + victim["count"] = len(victim["samples_ns"]) + ordered = sorted(victim["samples_ns"]) + victim["p50_ns"] = ordered[len(ordered) // 2] + victim["p95_ns"] = ordered[-1] + with self.assertRaises(CalibrationError) as caught: + load_continuation_samples(document) + self.assertIn("warm", str(caught.exception).lower()) + + def test_a_declared_warm_sample_floor_below_five_is_rejected(self): + document = _synthetic_document() + document["continuation"]["warm_samples_per_point"] = MIN_WARM_SAMPLES - 1 + with self.assertRaises(CalibrationError): + load_continuation_samples(document) + + def test_a_point_missing_one_route_is_rejected(self): + for route in ("append", "reprefill"): + with self.subTest(route=route): + document = _synthetic_document() + block = document["continuation"] + index = next( + i + for i, point in enumerate(block["points"]) + if point["route"] == route and point["suffix"] == 32 + ) + del block["points"][index] + with self.assertRaises(CalibrationError) as caught: + load_continuation_samples(document) + self.assertIn(route, str(caught.exception)) + + def test_a_duplicate_point_is_rejected(self): + document = _synthetic_document() + block = document["continuation"] + block["points"].append(copy.deepcopy(block["points"][0])) + with self.assertRaises(CalibrationError) as caught: + load_continuation_samples(document) + self.assertIn("duplicate", str(caught.exception).lower()) + + def test_an_unknown_route_name_is_rejected(self): + document = _synthetic_document() + document["continuation"]["points"][0]["route"] = "appendix" + with self.assertRaises(CalibrationError): + load_continuation_samples(document) + + def test_a_non_interleaved_record_is_refused(self): + """The first, non-interleaved round is the one that was retracted.""" + for flag in (False, None): + with self.subTest(flag=flag): + document = _synthetic_document() + document["continuation"]["samples_interleaved"] = flag + with self.assertRaises(CalibrationError) as caught: + load_continuation_samples(document) + self.assertIn("interleav", str(caught.exception).lower()) + + def test_a_missing_interleaving_flag_is_refused(self): + document = _synthetic_document() + del document["continuation"]["samples_interleaved"] + with self.assertRaises(CalibrationError): + load_continuation_samples(document) + + def test_a_recorded_percentile_that_disagrees_with_the_samples_is_rejected(self): + for field in ("p50_ns", "p95_ns"): + with self.subTest(field=field): + document = _synthetic_document() + document["continuation"]["points"][0][field] += 1 + with self.assertRaises(CalibrationError) as caught: + load_continuation_samples(document) + self.assertIn(field, str(caught.exception)) + + def test_a_count_that_disagrees_with_the_samples_is_rejected(self): + document = _synthetic_document() + document["continuation"]["points"][0]["count"] += 1 + with self.assertRaises(CalibrationError) as caught: + load_continuation_samples(document) + self.assertIn("count", str(caught.exception).lower()) + + def test_a_missing_continuation_block_is_rejected(self): + with self.assertRaises(CalibrationError): + load_continuation_samples({"identity": {}}) + + def test_a_non_positive_sample_is_rejected(self): + document = _synthetic_document() + document["continuation"]["points"][0]["samples_ns"][0] = 0 + with self.assertRaises(CalibrationError): + load_continuation_samples(document) + + def test_the_recorded_crossover_is_cross_checked_when_present(self): + """A recorded `append_wins_up_to` that the p95 rule contradicts.""" + document = _synthetic_document() + document["continuation"]["crossover"] = { + "512": {"append_wins_up_to": 128}, + "2048": {"append_wins_up_to": 2}, + } + loaded = load_continuation_samples(document) + selection = select_threshold_detailed(loaded.samples()) + diagnostics = loaded.crossover_disagreements(selection) + self.assertTrue(diagnostics) + self.assertIn("512", " ".join(diagnostics)) + + def test_an_agreeing_recorded_crossover_produces_no_diagnostic(self): + document = _synthetic_document() + document["continuation"]["crossover"] = { + "512": {"append_wins_up_to": 2}, + "2048": {"append_wins_up_to": 2}, + } + loaded = load_continuation_samples(document) + selection = select_threshold_detailed(loaded.samples()) + self.assertEqual(loaded.crossover_disagreements(selection), ()) + + +class MinIdentityTest(unittest.TestCase): + """`min` over per-history ceilings is an UPPER BOUND, not the rule. + + Section 10.7 selects the largest suffix in the INTERSECTION of the + per-history winner sets. `min` of the ceilings equals that only when each + winner set is downward-closed, and a crossover record stores edges rather + than winner sets, so it cannot attest that. These tests pin the gap so a + later simplification back to `min` cannot pass silently. + """ + + def test_min_of_ceilings_can_exceed_what_the_rule_selects(self): + """W512 = {1, 8}, W2048 = {1, 2, 4}: rule gives 1, `min` gives 4.""" + samples = { + 1: _per_history(h512=(1.0, 5.0), h2048=(1.0, 5.0)), + 2: _per_history(h512=(9.0, 5.0), h2048=(1.0, 5.0)), + 4: _per_history(h512=(9.0, 5.0), h2048=(1.0, 5.0)), + 8: _per_history(h512=(1.0, 5.0), h2048=(9.0, 5.0)), + } + selection = select_threshold_detailed(samples) + # The intersection is {1}, which IS a prefix, so the Section 10.7 + # monotonicity assertion passes and the rule returns a threshold. + self.assertEqual(selection.verdict, "selected") + self.assertEqual(selection.winners, (1,)) + self.assertEqual(selection.threshold, 1) + # Ceilings are 8 and 4; `min` would have said 4. + ceilings = {512: 8, 2048: 4} + self.assertEqual(min(ceilings.values()), 4) + self.assertNotEqual(selection.threshold, min(ceilings.values())) + + def test_the_loader_reports_a_history_whose_winners_are_not_closed(self): + document = _synthetic_document( + suffixes=(1, 2, 4, 8, 32, 128, 256), + reprefill_ns={512: 500_000_000, 2048: 500_000_000}, + ) + # Make append LOSE at 512/2 and keep winning at 512/4, so history 512's + # winner set is {1, 4, 8} -- not downward-closed. + for point in document["continuation"]["points"]: + if ( + point["history_rows"] == 512 + and point["suffix"] == 2 + and point["route"] == "append" + ): + point["samples_ns"] = [900_000_000 + i for i in range(5)] + point["count"] = 5 + point["p50_ns"] = 900_000_002 + point["p95_ns"] = 900_000_004 + loaded = load_continuation_samples(document) + self.assertEqual(loaded.winners_by_history()[512], (1, 4, 8)) + unclosed = loaded.unclosed_histories() + self.assertTrue(unclosed) + self.assertIn("512", " ".join(unclosed)) + self.assertNotIn("2048", " ".join(unclosed)) + + def test_closed_winner_sets_report_nothing(self): + loaded = load_continuation_samples(_synthetic_document()) + self.assertEqual(loaded.unclosed_histories(), ()) + + def test_the_committed_measurement_has_closed_winner_sets(self): + loaded = load_continuation_samples( + json.loads(_BASELINE.read_text(encoding="utf-8")) + ) + self.assertEqual(loaded.unclosed_histories(), ()) + + +class UnclosedWinnersConsumersTest(unittest.TestCase): + """The three things that CONSUME `unclosed_histories()`. + + The function itself was tested when it was written; its consumers were not, + and a review pointed out that the write-up claimed otherwise. Each of these + is a separate place the finding has to survive to: the exit code, the + rendered document, and the machine-readable summary. A finding that reaches + only one of them is the failure mode this whole branch keeps hitting. + """ + + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.addCleanup(self._tmp.cleanup) + self.directory = pathlib.Path(self._tmp.name) + self.baseline = self.directory / "baseline.json" + self.baseline.write_text( + json.dumps(_unclosed_document()), encoding="utf-8" + ) + self.header = self.directory / "tuning.hpp" + self.header.write_text(_BARE_HEADER, encoding="utf-8") + self.document = self.directory / "results.md" + self.document.write_text("# heading\n", encoding="utf-8") + self.summary = self.directory / "summary.json" + + def _run(self, extra=()): + return main( + [ + "--baseline", + str(self.baseline), + "--crossover-history", + str(_HISTORY), + "--header", + str(self.header), + "--document", + str(self.document), + "--json", + str(self.summary), + *extra, + ] + ) + + def test_it_reaches_the_exit_code(self): + self.assertNotEqual(self._run(), 0) + + def test_it_refuses_to_write_the_header_or_the_document(self): + self._run() + self.assertEqual(self.header.read_text(encoding="utf-8"), _BARE_HEADER) + self.assertEqual(self.document.read_text(encoding="utf-8"), "# heading\n") + + def test_it_reaches_the_json_summary(self): + self._run() + payload = json.loads(self.summary.read_text(encoding="utf-8")) + self.assertFalse(payload["winners_downward_closed"]) + self.assertTrue( + any("not a prefix" in line for line in payload["diagnostics"]), + payload["diagnostics"], + ) + + def test_it_reaches_the_rendered_document(self): + loaded = load_continuation_samples(_unclosed_document()) + selection = select_threshold_detailed(loaded.samples()) + agreement = history_agreement( + json.loads(_HISTORY.read_text(encoding="utf-8")), selection.threshold + ) + section = build_document_section(loaded, selection, agreement) + self.assertIn("not all downward-closed", section) + self.assertIn("won again at a longer suffix", section) + self.assertNotIn("ARE downward-closed", section) + + def test_a_closed_record_takes_the_other_branch_everywhere(self): + """Otherwise the four assertions above could pass for every input.""" + self.baseline.write_text( + json.dumps(_synthetic_document()), encoding="utf-8" + ) + self.assertEqual(self._run(), 0) + self.assertNotEqual(self.header.read_text(encoding="utf-8"), _BARE_HEADER) + payload = json.loads(self.summary.read_text(encoding="utf-8")) + self.assertTrue(payload["winners_downward_closed"]) + self.assertIn( + "ARE downward-closed", self.document.read_text(encoding="utf-8") + ) + + +class ConcessionTest(unittest.TestCase): + """What the single constant gives up, and which member of it is worst. + + The penalty at a conceded point is `reprefill - append`. Append cost grows + with suffix length and re-prefill cost does not, so within a conceded band + the NARROWEST suffix is the most expensive and the widest is the cheapest. + Naming the widest -- the obvious thing to do, and the thing the first + version of this did -- understates the cost. + """ + + def _conceded_document(self): + # Append 50 ms/token; re-prefill 500 ms at 512 and 1,700 ms at 2048. + # Winners: {1,2,4,8} at 512, {1,2,4,8,32} at 2048. Threshold 8; the + # conceded band is 2048 at suffix 32 only, so widen it by also making + # 512 lose earlier. + return _synthetic_document( + suffixes=(1, 2, 4, 8, 32, 128, 256), + reprefill_ns={512: 300_000_000, 2048: 1_700_000_000}, + ) + + def test_the_worst_conceded_point_is_the_narrowest_not_the_widest(self): + loaded = load_continuation_samples(self._conceded_document()) + selection = select_threshold_detailed(loaded.samples()) + # 512 re-prefill is 300 ms, so append wins only at 1, 2, 4. + self.assertEqual(loaded.winners_by_history()[512], (1, 2, 4)) + self.assertEqual(loaded.winners_by_history()[2048], (1, 2, 4, 8, 32)) + self.assertEqual(selection.threshold, 4) + + rows = conceded_points(loaded, selection.threshold) + self.assertEqual([(row[0], row[1]) for row in rows], [(2048, 8), (2048, 32)]) + # Ordered worst-first, and the worst is the NARROWER suffix. + self.assertGreater(rows[0][3], rows[1][3]) + self.assertLess(rows[0][1], rows[1][1]) + # The widest conceded suffix would have been the cheaper quote. + widest = max(rows, key=lambda row: row[1]) + self.assertLess(widest[3], rows[0][3]) + + def test_the_section_quotes_the_worst_case_not_the_widest_suffix(self): + loaded = load_continuation_samples(self._conceded_document()) + selection = select_threshold_detailed(loaded.samples()) + rows = conceded_points(loaded, selection.threshold) + agreement = history_agreement( + [ + { + "utc": "2026-09-02T15:34:01Z", + "samples_interleaved": True, + "edges": {"512": [4, 8], "2048": [32, 128]}, + } + ], + selection.threshold, + ) + section = build_document_section(loaded, selection, agreement) + self.assertIn(f"suffix {rows[0][1]} — {rows[0][3]:,.1f}x slower", section) + # Every conceded point appears, not just the worst. + for _, suffix, _, _ in rows: + self.assertIn(f"| 2048 | {suffix} |", section) + + def test_no_concession_when_nothing_is_given_up(self): + loaded = load_continuation_samples(_synthetic_document()) + selection = select_threshold_detailed(loaded.samples()) + self.assertEqual(conceded_points(loaded, selection.threshold), ()) + + def test_the_ordering_is_by_cost_across_histories_not_by_suffix(self): + """The one case a single-history fixture cannot see. + + Within one history, slowdown-descending and suffix-ascending coincide, + so every other test here would still pass if the sort key reverted to + the suffix. With concessions at TWO histories they diverge: ordered by + cost the band is [(512, 8), (2048, 2), (2048, 4)] and ordered by suffix + it is [(2048, 2), (2048, 4), (512, 8)], which disagree on the very + first element -- the one the document quotes as the worst case. + """ + loaded = load_continuation_samples(_unclosed_document()) + selection = select_threshold_detailed(loaded.samples()) + self.assertEqual(selection.threshold, 1) + + rows = conceded_points(loaded, selection.threshold) + self.assertEqual( + [(row[0], row[1]) for row in rows], + [(512, 8), (2048, 2), (2048, 4)], + ) + # Both histories are represented, which is what makes the orderings + # distinguishable at all. + self.assertEqual({row[0] for row in rows}, {512, 2048}) + + by_suffix = sorted(rows, key=lambda row: (row[1], row[0])) + self.assertNotEqual(rows, by_suffix) + self.assertNotEqual(rows[0], by_suffix[0]) + # And not the narrowest suffix overall either: a naive "min suffix" + # fix would pick (2048, 2), which is 2.0x rather than 50.0x. + self.assertEqual(min(rows, key=lambda row: row[1])[:2], (2048, 2)) + self.assertGreater(rows[0][3], 40.0) + + def test_the_committed_measurement_concedes_2048_at_8_worst(self): + loaded = load_continuation_samples( + json.loads(_BASELINE.read_text(encoding="utf-8")) + ) + rows = conceded_points(loaded, 4) + self.assertEqual([(row[0], row[1]) for row in rows], [(2048, 8), (2048, 12)]) + self.assertAlmostEqual(rows[0][3], 2.995, places=2) + self.assertAlmostEqual(rows[1][3], 1.961, places=2) + + +class HistoryAgreementTest(unittest.TestCase): + """Whether earlier interleaved runs PERMIT the constant. + + Not whether they would have selected it: see `MinIdentityTest` for why the + stronger claim is not available from a crossover record. + """ + + @staticmethod + def _record(interleaved, edges, utc="2026-09-02T15:34:01Z", note=None): + record = { + "utc": utc, + "samples_interleaved": interleaved, + "edges": {str(k): list(v) for k, v in edges.items()}, + } + if note: + record["note"] = note + return record + + def test_interleaved_runs_that_permit_it_produce_no_diagnostic(self): + records = [ + self._record(True, {512: (4, 8), 2048: (12, 16)}), + self._record(True, {512: (4, 12), 2048: (12, 64)}), + ] + report = history_agreement(records, 4) + self.assertTrue(report.comparable) + self.assertTrue(report.permits) + self.assertEqual(report.diagnostics, ()) + self.assertEqual( + [row.upper_bound for row in report.rows if row.comparable], [4, 4] + ) + self.assertEqual(len(report.tight), 2) + + def test_a_run_that_contradicts_the_threshold_is_a_diagnostic(self): + records = [ + self._record(True, {512: (4, 8), 2048: (12, 16)}), + self._record(True, {512: (2, 12), 2048: (12, 64)}), + ] + report = history_agreement(records, 4) + self.assertTrue(report.comparable) + self.assertFalse(report.permits) + self.assertTrue(report.diagnostics) + self.assertIn("bounds it at 2", " ".join(report.diagnostics)) + + def test_a_run_whose_bound_is_higher_permits_without_confirming(self): + """8 > 4: that run cannot contradict 4, and cannot vouch for it.""" + records = [self._record(True, {512: (8, 12), 2048: (12, 64)})] + report = history_agreement(records, 4) + self.assertTrue(report.permits) + self.assertEqual(report.diagnostics, ()) + self.assertEqual(report.tight, ()) + self.assertEqual(report.rows[0].upper_bound, 8) + self.assertFalse(report.rows[0].tight) + + def test_non_interleaved_runs_are_excluded_from_the_verdict(self): + records = [ + self._record(None, {512: (2, None), 2048: (2, None)}, note="retracted"), + self._record(True, {512: (4, 8), 2048: (12, 16)}), + ] + report = history_agreement(records, 4) + self.assertTrue(report.permits) + self.assertEqual(sum(1 for row in report.rows if not row.comparable), 1) + + def test_a_non_interleaved_run_cannot_contradict_the_threshold(self): + """Its bound is 2, below 4 -- and it must still not fail the check.""" + records = [ + self._record(None, {512: (2, None), 2048: (2, None)}), + self._record(True, {512: (4, 8), 2048: (12, 16)}), + ] + report = history_agreement(records, 4) + self.assertEqual(report.rows[0].upper_bound, 2) + self.assertFalse(report.rows[0].permits) + self.assertTrue(report.permits) + self.assertEqual(report.diagnostics, ()) + + def test_only_non_interleaved_runs_means_nothing_is_comparable(self): + records = [self._record(False, {512: (2, None), 2048: (2, None)})] + report = history_agreement(records, 4) + self.assertFalse(report.comparable) + self.assertFalse(report.permits) + self.assertTrue(report.diagnostics) + + def test_a_null_lower_edge_bounds_at_zero(self): + records = [self._record(True, {512: (None, None), 2048: (12, 16)})] + report = history_agreement(records, 0) + self.assertTrue(report.permits) + self.assertEqual(report.rows[0].upper_bound, 0) + self.assertTrue(report.rows[0].tight) + + def test_a_null_lower_edge_contradicts_a_positive_threshold(self): + records = [self._record(True, {512: (None, None), 2048: (12, 16)})] + report = history_agreement(records, 4) + self.assertFalse(report.permits) + self.assertTrue(report.diagnostics) + + def test_an_empty_history_file_means_nothing_is_comparable(self): + report = history_agreement([], 4) + self.assertFalse(report.comparable) + self.assertTrue(report.diagnostics) + + def test_a_record_missing_a_required_history_edge_is_rejected(self): + records = [self._record(True, {512: (4, 8)})] + with self.assertRaises(CalibrationError): + history_agreement(records, 4) + + +_BARE_HEADER = """#pragma once + +#include + +namespace flm::phi4 { + +inline constexpr std::uint32_t kContinuationAppendThreshold = 0; + +} // namespace flm::phi4 +""" + + +class HeaderWriterTest(unittest.TestCase): + """Bootstrap, regenerate, and refuse -- and stay byte-identical.""" + + _PROVENANCE = ("source: phi4_aie4_baseline.json", "measured: 2026-09-02T16:32:18Z") + + def test_bootstrapping_a_bare_declaration_wraps_it_in_markers(self): + written = apply_threshold_to_header(_BARE_HEADER, 4, self._PROVENANCE) + self.assertIn( + "inline constexpr std::uint32_t kContinuationAppendThreshold = 4;", + written, + ) + self.assertIn("BEGIN generated", written) + self.assertIn("END generated", written) + self.assertIn("source: phi4_aie4_baseline.json", written) + + def test_regenerating_is_idempotent(self): + once = apply_threshold_to_header(_BARE_HEADER, 4, self._PROVENANCE) + twice = apply_threshold_to_header(once, 4, self._PROVENANCE) + self.assertEqual(once, twice) + + def test_regenerating_replaces_the_previous_block_rather_than_nesting(self): + four = apply_threshold_to_header(_BARE_HEADER, 4, self._PROVENANCE) + twelve = apply_threshold_to_header(four, 12, self._PROVENANCE) + self.assertEqual(twelve.count("BEGIN generated"), 1) + self.assertNotIn("= 4;", twelve) + self.assertIn( + "inline constexpr std::uint32_t kContinuationAppendThreshold = 12;", + twelve, + ) + + def test_the_surrounding_hand_written_code_survives(self): + source = _HEADER.read_text(encoding="utf-8") + written = apply_threshold_to_header(source, 4, self._PROVENANCE) + for kept in ( + "SelectContinuationRoute", + "ContinuationRouteName", + "enum class ForcedContinuationRoute", + ): + self.assertIn(kept, written) + + def test_a_header_without_the_declaration_is_refused(self): + with self.assertRaises(CalibrationError): + apply_threshold_to_header("#pragma once\n", 4, self._PROVENANCE) + + def test_two_declarations_are_refused(self): + doubled = _BARE_HEADER + ( + "inline constexpr std::uint32_t kContinuationAppendThreshold = 1;\n" + ) + with self.assertRaises(CalibrationError): + apply_threshold_to_header(doubled, 4, self._PROVENANCE) + + def test_a_negative_threshold_is_refused(self): + with self.assertRaises(CalibrationError): + apply_threshold_to_header(_BARE_HEADER, -1, self._PROVENANCE) + + +class DocumentSectionTest(unittest.TestCase): + """The published evidence, and its idempotence.""" + + def _section_inputs(self): + loaded = load_continuation_samples(_synthetic_document()) + selection = select_threshold_detailed(loaded.samples()) + records = [ + { + "utc": "2026-09-02T15:34:01Z", + "samples_interleaved": True, + "edges": {"512": [2, 32], "2048": [2, 32]}, + } + ] + return loaded, selection, history_agreement(records, selection.threshold) + + def test_the_section_carries_the_threshold_and_the_identity(self): + loaded, selection, agreement = self._section_inputs() + section = build_document_section(loaded, selection, agreement) + self.assertIn("xcomedusad-43", section) + self.assertIn("kContinuationAppendThreshold", section) + self.assertIn(str(selection.threshold), section) + self.assertIn("2026-09-02T16:32:18Z", section) + + def test_the_section_names_what_the_single_constant_gives_up(self): + """A longer history whose winners run past the chosen constant. + + Re-prefill costs 500 ms at history 512 and 1,700 ms at history 2048, so + append wins up to 8 at 512 and up to 32 at 2048. The conjunction gives + 8, and suffix 32 at history 2048 is what that constant gives up. + """ + document = _synthetic_document( + suffixes=(1, 2, 4, 8, 32, 128, 256), + reprefill_ns={512: 500_000_000, 2048: 1_700_000_000}, + ) + loaded = load_continuation_samples(document) + selection = select_threshold_detailed(loaded.samples()) + self.assertEqual(selection.threshold, 8) + self.assertEqual( + loaded.winners_by_history(), + {512: (1, 2, 4, 8), 2048: (1, 2, 4, 8, 32)}, + ) + agreement = history_agreement( + [ + { + "utc": "2026-09-02T15:34:01Z", + "samples_interleaved": True, + "edges": {"512": [8, 32], "2048": [16, 32]}, + } + ], + selection.threshold, + ) + section = build_document_section(loaded, selection, agreement) + self.assertIn("gives up", section.lower()) + self.assertIn("2048", section) + + def test_the_baseline_block_is_left_alone(self): + """Two generators write into `phi4_results.md`; neither owns the file. + + `report_phi4_corelib_baseline.py` replaces everything between its + `phi4-aie4-baseline` markers and this script replaces everything + between its own. If the sections ever nested or overlapped, one tool + would silently eat the other's evidence the next time it ran. + """ + loaded, selection, agreement = self._section_inputs() + section = build_document_section(loaded, selection, agreement) + original = _DOCUMENT.read_text(encoding="utf-8") + baseline_begin = original.index("") + baseline_end = original.index("") + baseline_block = original[ + baseline_begin : baseline_end + len("") + ] + + rewritten = apply_section_to_document(original, section) + self.assertIn(baseline_block, rewritten) + self.assertEqual(rewritten.count(""), 1) + self.assertEqual( + rewritten.count(""), 1 + ) + self.assertGreater( + rewritten.index(""), + rewritten.index(""), + ) + self.assertEqual(rewritten, apply_section_to_document(rewritten, section)) + + def test_appending_to_a_document_without_the_section_is_idempotent(self): + loaded, selection, agreement = self._section_inputs() + section = build_document_section(loaded, selection, agreement) + once = apply_section_to_document("# heading\n", section) + self.assertEqual(once, apply_section_to_document(once, section)) + + def test_an_unterminated_section_is_refused(self): + loaded, selection, agreement = self._section_inputs() + section = build_document_section(loaded, selection, agreement) + with self.assertRaises(CalibrationError): + apply_section_to_document( + "# heading\n\n", section + ) + + def test_the_section_is_deterministic(self): + loaded, selection, agreement = self._section_inputs() + self.assertEqual( + build_document_section(loaded, selection, agreement), + build_document_section(loaded, selection, agreement), + ) + + @staticmethod + def _records(*edges_by_run, interleaved=True): + return [ + { + "utc": f"2026-09-0{index + 2}T00:00:00Z", + "samples_interleaved": interleaved, + "edges": {str(h): [low, None] for h, low in edges.items()}, + } + for index, edges in enumerate(edges_by_run) + ] + + def test_a_moved_lower_edge_is_never_called_stable(self): + """The paragraph that went false, at the branch that made it false. + + It was a constant string asserting the lower edge was stable, so it + could not respond to a third run recording 24 where the first two + recorded 12. This is that exact shape. + """ + agreement = history_agreement( + self._records( + {512: 4, 2048: 12}, {512: 4, 2048: 12}, {512: 4, 2048: 24} + ), + 4, + ) + self.assertEqual(recorded_lower_edges(agreement), {512: (4,), 2048: (12, 24)}) + text = lower_edge_confidence(Selection(4, "selected", (1, 2, 4), (), ()), agreement) + self.assertIn("12, 24 at history 2048", text) + self.assertIn("NOT a measured constant", text) + self.assertNotIn("stable across runs", text) + # And the inequality, which does still hold, is what carries it. + self.assertIn("at or below EVERY recorded lower edge", text) + + def test_an_unmoved_lower_edge_is_reported_without_overclaiming(self): + agreement = history_agreement( + self._records({512: 4, 2048: 12}, {512: 4, 2048: 12}), 4 + ) + self.assertEqual(recorded_lower_edges(agreement), {512: (4,), 2048: (12,)}) + text = lower_edge_confidence(Selection(4, "selected", (1, 2, 4), (), ()), agreement) + self.assertIn("has not moved between runs", text) + # Two runs agreeing is not a guarantee, and the sentence has to say so + # -- this is the wording whose absence let one run falsify the claim. + self.assertIn("not a guarantee about the next one", text) + + def test_a_non_interleaved_run_does_not_vote_on_the_edge(self): + agreement = history_agreement( + self._records({512: 4, 2048: 24}, interleaved=False), 4 + ) + self.assertEqual(recorded_lower_edges(agreement), {}) + text = lower_edge_confidence(Selection(4, "selected", (1, 2, 4), (), ()), agreement) + self.assertIn("no interleaved run is on record", text) + + def test_a_contradicted_threshold_does_not_claim_the_inequality(self): + agreement = history_agreement(self._records({512: 4, 2048: 12}), 8) + self.assertFalse(agreement.permits) + text = lower_edge_confidence(Selection(8, "selected", (), (), ()), agreement) + self.assertIn("is NOT at or below every recorded lower edge", text) + + def test_the_rendered_section_prints_the_lower_edges_it_reasons_from(self): + """The claim and the table it sits beside read the same field. + + They used to be independent: the table printed `min(lowers)` and the + prose asserted a stability result from nowhere, so nothing made them + agree. Against the committed history the section must show the run + that moved the edge, in the table AND in the sentence. + """ + loaded = load_continuation_samples( + json.loads(_BASELINE.read_text(encoding="utf-8")) + ) + selection = select_threshold_detailed(loaded.samples()) + agreement = history_agreement( + json.loads(_HISTORY.read_text(encoding="utf-8")), selection.threshold + ) + section = build_document_section(loaded, selection, agreement) + self.assertIn("| 2026-09-03T08:20:08Z | yes | 512: 4, 2048: 24 |", section) + self.assertIn("12, 24 at history 2048", section) + self.assertIn("NOT a measured constant", section) + + def test_the_section_says_so_when_no_length_wins(self): + document = _synthetic_document(reprefill_ns=1_000_000) + loaded = load_continuation_samples(document) + selection = select_threshold_detailed(loaded.samples()) + self.assertEqual(selection.threshold, 0) + agreement = history_agreement( + [ + { + "utc": "2026-09-02T15:34:01Z", + "samples_interleaved": True, + "edges": {"512": [None, 1], "2048": [None, 1]}, + } + ], + 0, + ) + section = build_document_section(loaded, selection, agreement) + self.assertIn("every prefix hit re-prefills", section) + + +class CommandLineTest(unittest.TestCase): + """Exit codes, `--check`, and the skipped-work rule.""" + + def _run(self, extra, header_text=None, document_text=None): + directory = pathlib.Path(self._tmp.name) + header = directory / "tuning.hpp" + header.write_text( + header_text + if header_text is not None + else _HEADER.read_text(encoding="utf-8"), + encoding="utf-8", + ) + document = directory / "results.md" + document.write_text( + document_text + if document_text is not None + else _DOCUMENT.read_text(encoding="utf-8"), + encoding="utf-8", + ) + argv = [ + "--baseline", + str(_BASELINE), + "--crossover-history", + str(_HISTORY), + "--header", + str(header), + "--document", + str(document), + *extra, + ] + return main(argv), header, document + + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.addCleanup(self._tmp.cleanup) + + def test_writing_then_checking_succeeds_and_is_stable(self): + code, header, document = self._run([]) + self.assertEqual(code, 0) + first_header = header.read_text(encoding="utf-8") + first_document = document.read_text(encoding="utf-8") + + self.assertEqual( + main( + [ + "--baseline", + str(_BASELINE), + "--crossover-history", + str(_HISTORY), + "--header", + str(header), + "--document", + str(document), + "--check", + ] + ), + 0, + ) + code, _, _ = self._run([], first_header, first_document) + self.assertEqual(code, 0) + self.assertEqual(header.read_text(encoding="utf-8"), first_header) + self.assertEqual(document.read_text(encoding="utf-8"), first_document) + + def test_check_fails_when_the_header_is_stale(self): + stale = _BARE_HEADER.replace("= 0;", "= 7;") + code, _, _ = self._run(["--check"], header_text=stale) + self.assertNotEqual(code, 0) + + def test_check_fails_when_the_document_section_is_missing(self): + code, _, _ = self._run(["--check"], document_text="# nothing here\n") + self.assertNotEqual(code, 0) + + def test_a_malformed_baseline_exits_non_zero(self): + directory = pathlib.Path(self._tmp.name) + broken = directory / "broken.json" + document = json.loads(_BASELINE.read_text(encoding="utf-8")) + document["continuation"]["samples_interleaved"] = False + broken.write_text(json.dumps(document), encoding="utf-8") + header = directory / "tuning.hpp" + header.write_text(_BARE_HEADER, encoding="utf-8") + code = main( + [ + "--baseline", + str(broken), + "--crossover-history", + str(_HISTORY), + "--header", + str(header), + "--document", + str(directory / "out.md"), + ] + ) + self.assertNotEqual(code, 0) + self.assertEqual(header.read_text(encoding="utf-8"), _BARE_HEADER) + + def test_a_missing_crossover_history_exits_non_zero(self): + directory = pathlib.Path(self._tmp.name) + code = main( + [ + "--baseline", + str(_BASELINE), + "--crossover-history", + str(directory / "absent.json"), + "--header", + str(directory / "tuning.hpp"), + "--document", + str(directory / "out.md"), + ] + ) + self.assertNotEqual(code, 0) + + def test_the_json_summary_records_the_threshold_and_the_verdict(self): + directory = pathlib.Path(self._tmp.name) + summary = directory / "summary.json" + code, _, _ = self._run(["--json", str(summary)]) + self.assertEqual(code, 0) + payload = json.loads(summary.read_text(encoding="utf-8")) + self.assertEqual(payload["verdict"], "selected") + self.assertEqual(payload["threshold"], 4) + self.assertTrue(payload["history_agreement"]["permits"]) + self.assertTrue(payload["winners_downward_closed"]) + + +def _markdown_blocks(text): + """Split markdown into `(collapsed_text, is_quoted)` blocks. + + A block is a run of consecutive non-blank lines, whitespace-collapsed so a + claim broken across a line wrap is still found -- checking line by line + would miss exactly the occurrences a text reflow creates. + + `is_quoted` is true when every line of the block is a blockquote, which is + how a retracted claim is legitimately cited: inside a marked correction + that quotes the old wording in order to say it was wrong. + """ + for raw in re.split(r"\n\s*\n", text): + lines = [line for line in raw.splitlines() if line.strip()] + if not lines: + continue + quoted = all(line.lstrip().startswith(">") for line in lines) + yield " ".join(raw.split()), quoted + + +class RetractedClaimsTest(unittest.TestCase): + """Retractions must reach the RENDERED artifacts, not just the code. + + Four times on this branch a claim was retracted in the source while the + rendered document went on asserting it, because nothing read the rendered + output back. A fifth time it survived in the task REPORT -- the very file + documenting the retraction -- because the guard read the document and not + the report. So both are read back here. + + The claims are matched by TEXT, not by line number: a line number stops + meaning anything the moment the prose moves, and would then pass vacuously. + """ + + # Each entry is (substring, why it is wrong). The substrings are the exact + # retracted wording, not a paraphrase and not a keyword: "would have + # selected", for instance, appears LEGITIMATELY in the corrected prose, + # which says the bound is not what that run would have selected. A matcher + # loose enough to catch the retracted claim by keyword would forbid its own + # correction. Likewise "at the widest conceded suffix" rather than "widest + # conceded suffix", because the correction has to be able to say that the + # widest conceded suffix is the cheapest one. + _RETRACTED = ( + ( + "the constant is the smaller of those ceilings", + "Section 10.7 takes the largest suffix in the INTERSECTION of the " + "per-history winner sets. That coincides with the smaller ceiling " + "only when the sets are downward-closed, which is a property of " + "the data and not of the rule.", + ), + ( + "recomputable from every recorded run", + "a crossover record stores edges and never winner sets, so it " + "BOUNDS what Section 10.7 would have selected and cannot " + "reproduce it.", + ), + ( + "at the widest conceded suffix", + "the widest suffix in a conceded band is its CHEAPEST member. The " + "penalty is reprefill - append, append grows with suffix length " + "and re-prefill does not, so the cost is read off the narrow end.", + ), + # Task 15's re-measurement (2026-09-03T08:20:08Z, committed at + # `phi4_aie4_baseline_task15_rerun.json` and now the fifth record in + # `phi4_aie4_crossover_history.json`) measured the lower edge at + # history 2048 to be 24. Two claims in the baseline block asserted the + # opposite in the present tense, and the correction reached only the + # hand-written section 200 lines below them. + ( + "the lower edge has been 12 at history 2048", + "the third interleaved run on record measured it at 24. Across " + "the three interleaved runs in the committed crossover history " + "the lower edge at history 2048 has taken 12 and 24, so it is not " + "a measured constant.", + ), + ( + "upper edge is not stable and its lower edge is", + "neither edge is stable. `crossover_edge_stability` over the " + "committed crossover history reports lower_is_stable false once " + "the 2026-09-03 run is included, and its own narrative for that " + "case reads \"Neither edge of the bracket is stable between " + "runs.\"", + ), + # The same falsification, in the two paragraphs the CALIBRATOR emitted. + # Both were constant strings: they could not change when the data did, + # and one of them said "three times" above a table of two comparable + # runs. `build_document_section` now derives both from + # `HistoryRow.lower_edges`, so the claim and the table it sits beside + # cannot disagree. Attributed past-tense wording about what Task 13 + # measured is NOT retracted -- only the unattributed present-tense form + # that says the edges are stable. + ( + "found the bracket's LOWER edge stable across runs", + "the lower edge at history 2048 took 12 on the first two " + "interleaved runs and 24 on the third. The renderer now prints " + "the per-run lower edges and derives whether they held still.", + ), + ( + "measured the lower edges to be stable", + "they are not stable: history 2048 recorded 12 and 24 across the " + "three interleaved runs. What the bound actually relies on is " + "that it touches only the lower edges, which is a property of the " + "construction and is what the paragraph says now.", + ), + ) + + @staticmethod + def _collapse(text): + """Whitespace-collapse, so a line wrap cannot hide a claim. + + The report matcher collapsed and the DOCUMENT matcher did not, which + made the strict check score zero on any wrapped occurrence. It was + harmless only because `build_document_section` emits one-line + paragraphs -- and it went vacuous the moment the check was widened to + the whole committed document, which contains hand-wrapped prose. + """ + return " ".join(text.split()) + + @classmethod + def _asserts(cls, claim, text): + """Does `text` contain `claim` as a phrase, not inside a longer word? + + Plain `in` was wrong in a way that took a failing run to see: "at the + widest conceded suffix" is a substring of "th-AT THE widest conceded + suffix", so the matcher flagged the very sentence explaining that the + widest conceded suffix is the cheapest. Third variant of the same trap + -- a matcher loose enough to catch the claim also catches its + correction -- so the left edge is anchored to a word boundary. + + Both sides are whitespace-collapsed. Without that, a claim broken over + a line wrap scored zero here while the report matcher (which reads + collapsed blocks) found it -- see + `test_the_strict_matcher_also_finds_a_claim_split_across_a_line_wrap`, + which fails against the uncollapsed version. + """ + return ( + re.search( + r"(?`. + """ + violations = [] + for block, quoted in _markdown_blocks(text): + if quoted and blockquote_is_citation: + continue + cited = [ + match.span() for match in self._INLINE_QUOTE.finditer(block) + ] + for claim, why in self._RETRACTED: + for match in re.finditer( + r"(?") + end = document.find("") + self.assertNotEqual(begin, -1, "the Task 14 section is missing") + self.assertNotEqual(end, -1, "the Task 14 section is unterminated") + self.assertLess(begin, end) + return document[begin:end] + + def test_the_committed_document_asserts_no_retracted_claim(self): + """The WHOLE committed document, not one block. + + This used to read `self._task14_section()`, on the reasoning that + Task 14 may not edit prose it does not own. The cost of that reasoning + was I4/I9: Task 15 retracted a stability claim, the retraction reached + a hand-written section 200 lines below the claim, and the guard built + over three fix rounds to stop exactly that could not see it -- the + claim lives in the Task 13 baseline block, which was out of scope. + + Scope is now the file. A retracted claim is not allowed to survive + anywhere in the published document, whoever generated the paragraph + it sits in. The citation rule is the report's, not the strict one, + because the document now carries retraction prose of its own and + prose that retracts a claim has to be able to quote it. + """ + self._assert_clean_unless_quoted( + _DOCUMENT.read_text(encoding="utf-8"), + str(_DOCUMENT), + blockquote_is_citation=False, + ) + + def test_the_committed_report_asserts_no_retracted_claim(self): + """The file that documents the retractions is not exempt from them. + + Two retracted claims sat unmarked in it for two review rounds, in the + sections describing the very fixes that retracted them, because the + guard read the rendered document and stopped there. + """ + if not _REPORT.exists(): # pragma: no cover - report lives beside the plan + self.skipTest(f"{_REPORT} is not present in this checkout") + self._assert_clean_unless_quoted( + _REPORT.read_text(encoding="utf-8"), str(_REPORT) + ) + + def test_a_freshly_rendered_section_asserts_no_retracted_claim(self): + loaded = load_continuation_samples( + json.loads(_BASELINE.read_text(encoding="utf-8")) + ) + selection = select_threshold_detailed(loaded.samples()) + agreement = history_agreement( + json.loads(_HISTORY.read_text(encoding="utf-8")), + selection.threshold, + ) + self._assert_clean( + build_document_section(loaded, selection, agreement), + "a freshly rendered section", + ) + + def test_the_guard_would_notice_each_claim_coming_back(self): + """The matcher must actually match; otherwise the two tests above + pass for every possible document, including one that reinstates the + claim verbatim.""" + for claim, _ in self._RETRACTED: + with self.subTest(claim=claim): + self.assertEqual( + len(self._violations(f"prose prose {claim} prose")), 1 + ) + with self.assertRaises(AssertionError): + self._assert_clean( + f"prose prose {claim} prose", "a synthetic document" + ) + + def test_a_claim_inside_a_longer_word_is_not_a_claim(self): + """"that the widest conceded suffix is the cheapest" is the + CORRECTION, and it contains "at the widest conceded suffix" as a bare + substring. The guard found this by failing on it.""" + correction = ( + "carries an inline note explaining that the widest conceded " + "suffix is the cheapest" + ) + self.assertEqual(self._violations(correction), []) + # The same words with a real word boundary in front ARE the claim. + self.assertEqual( + len(self._violations("the p95 at the widest conceded suffix")), 1 + ) + + def test_the_quotation_exemption_is_not_a_blanket_pass(self): + """Otherwise the report check passes for any report at all.""" + claim = self._RETRACTED[0][0] + # Asserted as the author's own prose: caught. + self.assertEqual( + len(self._collect_unquoted(f"Some prose. {claim}. More prose.\n")), 1 + ) + # Quoted inside a marked correction: allowed. + self.assertEqual( + self._collect_unquoted( + f"> **Corrected.** It originally read {claim}.\n" + ), + [], + ) + # A blockquote glued to unquoted prose does NOT launder it: the block + # is mixed, so it is treated as the author speaking. + self.assertEqual( + len(self._collect_unquoted(f"Prose {claim}.\n> quoted tail\n")), 1 + ) + # Cited inline, in the author's own paragraph: allowed. Every citation + # in fix round 3's prose takes this form, and a blockquote-only rule + # went red on the round that introduced it. + self.assertEqual( + self._collect_unquoted(f'It originally read "{claim}", which is wrong.\n'), + [], + ) + self.assertEqual( + self._collect_unquoted(f"It originally read `{claim}`.\n"), [] + ) + # A claim in the middle of a longer quoted span is still cited. + self.assertEqual( + self._collect_unquoted(f'It read "so {claim} without re-measuring".\n'), + [], + ) + + def test_an_unbalanced_quote_cannot_exempt_the_rest_of_a_block(self): + """A runaway quoted span would silently disable the guard downstream.""" + claim = self._RETRACTED[0][0] + block = 'He said "' + ("filler " * 90) + f". Then {claim}.\n" + self.assertGreater(len(block.split('"')[1]), 400) + self.assertEqual(len(self._collect_unquoted(block)), 1) + + def test_a_claim_split_across_a_line_wrap_is_still_found(self): + """Line-by-line matching would miss every reflowed occurrence.""" + claim = self._RETRACTED[1][0] + head, tail = claim.split(" ", 1) + self.assertEqual(len(self._collect_unquoted(f"Prose {head}\n{tail}.\n")), 1) + + def test_the_strict_matcher_also_finds_a_claim_split_across_a_line_wrap(self): + """The parked defect: only the REPORT matcher collapsed whitespace. + + `_collect_unquoted` reads whitespace-collapsed blocks; `_asserts` read + the raw text, so a claim broken over a line wrap scored ZERO under the + strict check that guards the rendered section. It was harmless only + because `build_document_section` emits one-line paragraphs, and it goes + vacuous the day that generator wraps prose -- or, as here, the day the + check is pointed at a hand-wrapped file. This case failed before the + collapse was added to `_asserts`. + """ + for claim, _ in self._RETRACTED: + head, tail = claim.split(" ", 1) + wrapped = f"prose prose {head}\n{tail} prose" + with self.subTest(claim=claim): + self.assertEqual(len(self._violations(wrapped)), 1) + with self.assertRaises(AssertionError): + self._assert_clean(wrapped, "a wrapped synthetic document") + + def test_a_blockquote_does_not_launder_a_claim_in_the_document(self): + """`crossover_stability_narrative` returns its guidance AS a + blockquote, so in the published document a leading `>` is the + renderer's voice and not a human marking a citation. Measured: with + blockquotes exempt, the falsified "the lower edge has been 12 at + history 2048" sat unflagged in the committed document.""" + claim = "the lower edge has been 12 at history 2048" + self.assertIn(claim, [entry[0] for entry in self._RETRACTED]) + quoted_block = f"> **Read the lower edge as measured.** Across runs {claim}.\n" + # Report rules: a blockquote is a citation. + self.assertEqual(self._collect_unquoted(quoted_block), []) + # Document rules: it is not. + self.assertEqual( + len(self._collect_unquoted(quoted_block, blockquote_is_citation=False)), + 1, + ) + # An inline citation is still allowed under document rules, because + # the document now carries its own retraction prose. + self.assertEqual( + self._collect_unquoted( + f'The table concluded "{claim}", which the 2026-09-03 run ' + f"falsified.\n", + blockquote_is_citation=False, + ), + [], + ) + + def test_the_corrected_wording_is_not_itself_flagged(self): + """"would have selected" appears in the correction. A matcher that + forbade it would make the fix unwritable.""" + corrected = ( + "The minimum of those ceilings is an upper bound on what Section " + "10.7 would have selected from that run, and in general only an " + "upper bound." + ) + self.assertEqual(self._violations(corrected), []) + + def test_the_intersection_is_stated_where_the_reader_meets_it(self): + """Removing the wrong claim is not enough; the right one must be + there, and above the paragraph that explains it.""" + section = self._task14_section() + # assertIn first: `str.index` on a missing phrase raises ValueError, + # which reports as an ERROR and hides which phrase went missing. + self.assertIn("INTERSECTION of those winner sets", section) + self.assertIn("only when each winner set is downward-closed", section) + self.assertLess( + section.index("INTERSECTION of those winner sets"), + section.index("only when each winner set is downward-closed"), + ) + + +class CheckedInArtefactsTest(unittest.TestCase): + """The committed constant and document must match the committed data. + + This is the check that would catch the header, the document and the + measurement drifting apart -- the failure mode that no unit test over + synthetic records can see. + """ + + def test_the_committed_measurement_selects_the_committed_constant(self): + loaded = load_continuation_samples( + json.loads(_BASELINE.read_text(encoding="utf-8")) + ) + selection = select_threshold_detailed(loaded.samples()) + self.assertEqual(selection.verdict, "selected") + self.assertEqual( + loaded.winners_by_history(), + {512: (1, 2, 4), 2048: (1, 2, 4, 8, 12)}, + ) + self.assertEqual(selection.winners, (1, 2, 4)) + self.assertEqual(selection.threshold, 4) + self.assertEqual(loaded.crossover_disagreements(selection), ()) + self.assertIn( + f"kContinuationAppendThreshold = {selection.threshold};", + _HEADER.read_text(encoding="utf-8"), + ) + + def test_the_committed_artefacts_are_already_up_to_date(self): + self.assertEqual( + main( + [ + "--baseline", + str(_BASELINE), + "--crossover-history", + str(_HISTORY), + "--header", + str(_HEADER), + "--document", + str(_DOCUMENT), + "--check", + ] + ), + 0, + ) + + def test_every_interleaved_run_on_record_supports_the_committed_constant(self): + records = json.loads(_HISTORY.read_text(encoding="utf-8")) + report = history_agreement(records, 4) + self.assertTrue(report.comparable) + self.assertTrue(report.permits) + + def test_the_run_that_moved_the_lower_edge_reached_the_history_file(self): + """The falsification has to reach the file the TOOLING reads. + + Task 15 measured a third interleaved run whose lower edge at history + 2048 is 24, wrote it up in prose, and never appended it to + `phi4_aie4_crossover_history.json` -- so `--crossover-history` kept + feeding `history_agreement` the two runs that agree, and the calibrator + kept rendering "2 of 2 bound it there exactly" from data that had been + falsified. Prose is not an input to anything. + + Every figure below is derived, not typed: it is what + `report_phi4_corelib_baseline.crossover_entry()` produces from the + committed `phi4_aie4_baseline_task15_rerun.json`. + """ + rerun = ( + _REPO_ROOT + / "docs" + / "docs" + / "benchmarks" + / "phi4_aie4_baseline_task15_rerun.json" + ) + derived = crossover_entry( + json.loads(rerun.read_text(encoding="utf-8")), source="" + ) + records = json.loads(_HISTORY.read_text(encoding="utf-8")) + by_utc = {record.get("utc"): record for record in records} + self.assertIn( + derived["utc"], + by_utc, + f"{rerun.name} is committed but its run is absent from " + f"{_HISTORY.name}, so nothing that reads the history can see it", + ) + committed = by_utc[derived["utc"]] + for field in ( + "edges", + "samples_interleaved", + "suffix_grid", + "points_decided", + "points_total", + "undecided_suffixes", + "fastflow_revision", + "corelib_dll_sha256", + ): + self.assertEqual(committed[field], derived[field], field) + + # And the property that makes it matter: across the interleaved runs + # the lower edge at history 2048 is NOT a single value. + lowers = sorted( + { + record["edges"]["2048"][0] + for record in records + if record.get("samples_interleaved") + } + ) + self.assertEqual(lowers, [12, 24]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/tests/test_compare_phi4_corelib_e2e_seal.py b/tools/tests/test_compare_phi4_corelib_e2e_seal.py new file mode 100644 index 00000000..52035b47 --- /dev/null +++ b/tools/tests/test_compare_phi4_corelib_e2e_seal.py @@ -0,0 +1,183 @@ +"""The `_DETERM2_SEALED` import-time seal, exercised without hardware. + +Nothing else in the repository does. `src/tools/compare_phi4_corelib_e2e.py` +runs against documents produced on an AIE4 device with the real model, so +nothing in this branch invokes it and the seal was verified by hand and by +review and by nothing that runs. A seal whose failure path is never executed +is a comment. These tests are what make it a seal. + +Each test edits the constant the way somebody would after grepping it out of a +failure message -- in a copy, then executing the real file -- and requires the +import to die. The seal has to hold for the design 12.4 acceptance thresholds +as well as the DETERM-2 bounds: those two were outside it while their values +were printed verbatim in four failure messages, which is the entire path the +seal exists to block. +""" + +from __future__ import annotations + +import re +import unittest +from pathlib import Path + + +COMPARATOR = ( + Path(__file__).resolve().parents[2] + / "src" + / "tools" + / "compare_phi4_corelib_e2e.py" +) + +# name -> (source line as committed, the same line with the number moved) +# +# The "after" values are the plausible edits, not absurd ones: loosening the +# correlation, shortening the decode run, doubling either bound. +TAMPERS = { + "MAX_TOP32_ABS_DIFF": ( + "MAX_TOP32_ABS_DIFF = 0.25", + "MAX_TOP32_ABS_DIFF = 0.5", + ), + "RUN_TO_RUN_MAX_ULPS": ( + "RUN_TO_RUN_MAX_ULPS = 2", + "RUN_TO_RUN_MAX_ULPS = 4", + ), + "MIN_CORRELATION": ( + "MIN_CORRELATION = 0.9999", + "MIN_CORRELATION = 0.99", + ), + "MIN_DECODE_STEPS": ( + "MIN_DECODE_STEPS = 16", + "MIN_DECODE_STEPS = 4", + ), + "TOP_K": ( + "TOP_K = 32", + "TOP_K = 2", + ), +} + +# Every `{NAME}` an f-string in the comparator interpolates, where NAME looks +# like a module constant. DERIVED, never typed. +# +# The previous version of the test below compared the seal against `TAMPERS`, +# a literal in this file, and then claimed to be "the assertion that would have +# caught this omission in the first place". It was not: the universe and the +# expectation were the same hand-maintained list, so an unsealed constant was +# invisible to it by construction -- and there was one in the tree, `TOP_K`, +# whose value is interpolated into the union-top-k failure message and which +# could be dropped from 32 to 2 with the whole module green. Deriving the +# universe from the source is the only shape in which this assertion means +# what it says. +_INTERPOLATED_CONSTANT = re.compile(r"\{([A-Z][A-Z0-9_]*)\}") + + +def _execute(source: str) -> dict[str, object]: + """Run the comparator's module body, as import would. + + `__name__` is not `__main__`, so the `main()` entry point at the bottom + does not fire and nothing touches a device: only the import-time checks + run, which is exactly what is under test. + """ + namespace: dict[str, object] = {"__name__": "_determ2_seal_probe"} + exec(compile(source, str(COMPARATOR), "exec"), namespace) + return namespace + + +class Determ2SealTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + # Newlines normalised, because the tampers below are `\n`-joined + # literals. `core.autocrlf` is true by default on Windows and + # `.gitattributes` exempts only src/model_overlays, so a fresh clone + # gets this file with CRLF -- and every tamper silently stops matching, + # turning the whole seal suite red for a reason that has nothing to do + # with the seal. Observed on this tree. + cls.source = ( + COMPARATOR.read_bytes() + .decode("utf-8") + .replace("\r\n", "\n") + ) + + def test_the_committed_comparator_imports(self): + namespace = _execute(self.source) + sealed = namespace["_DETERM2_SEALED"] + for name, (value, reason) in sealed.items(): + self.assertEqual(namespace[name], value) + self.assertTrue(reason, f"{name} has no sealed reason") + + def test_every_threshold_quoted_in_a_failure_message_is_sealed(self): + # The alias incident, generalised: a constant whose value reaches a + # failure message is a constant somebody will grep and edit, so it + # must be in the dict. + # + # The universe is ENUMERATED OUT OF THE SOURCE, not compared against a + # list in this file. A literal expectation can only ever agree with + # itself, which is how `TOP_K` sat unsealed underneath a test that + # claimed to forbid exactly that. + namespace = _execute(self.source) + sealed = set(namespace["_DETERM2_SEALED"]) + quoted = { + name + for name in _INTERPOLATED_CONSTANT.findall(self.source) + if isinstance(namespace.get(name), (int, float)) + and not isinstance(namespace.get(name), bool) + } + # Non-vacuity: a regex that stopped matching would make the next + # assertion pass by finding nothing to require. + self.assertGreaterEqual( + len(quoted), + len(TAMPERS), + f"only {sorted(quoted)} were found interpolated into the " + "comparator's messages; the scan is not reading the file", + ) + self.assertEqual( + quoted - sealed, + set(), + f"{sorted(quoted - sealed)} reach a failure message but are not " + "in _DETERM2_SEALED. That is the grep-the-message-and-edit path " + "the seal exists to block: add each to _DETERM2_SEALED with the " + "reason its number cannot move, and to TAMPERS here.", + ) + # And every sealed constant must have a tamper case below, or the + # seal's failure path is unexercised for it. + self.assertEqual(sealed, set(TAMPERS)) + for name in TAMPERS: + self.assertIn( + f"{{{name}}}", + self.source, + f"{name} is sealed but no message quotes it; if it stopped " + "being user-visible, say so rather than leaving this stale", + ) + + def test_moving_a_sealed_constant_stops_the_tool(self): + for name, (before, after) in TAMPERS.items(): + with self.subTest(constant=name): + self.assertEqual( + self.source.count(before), + 1, + f"{before!r} is not a unique line; the tamper would " + "not be testing what it claims to", + ) + with self.assertRaises(SystemExit) as caught: + _execute(self.source.replace(before, after)) + message = str(caught.exception) + self.assertIn(f"DETERM-2: {name} has been changed", message) + # The message must lead back to the seal, not to the constant. + self.assertIn("_DETERM2_SEALED", message) + + def test_editing_the_seal_to_match_is_still_caught_at_the_other_end(self): + # Editing BOTH halves is a deliberate act and the seal lets it + # through by design -- but only when they agree. Moving the entry + # without the constant is the half-finished version of that act and + # it must not start. + source = self.source.replace( + ' 0.25,\n "the largest cross-implementation', + ' 0.5,\n "the largest cross-implementation', + ) + self.assertNotEqual(source, self.source) + with self.assertRaises(SystemExit) as caught: + _execute(source) + self.assertIn("MAX_TOP32_ABS_DIFF", str(caught.exception)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/tests/test_generate_phi4_corelib_manifest.py b/tools/tests/test_generate_phi4_corelib_manifest.py new file mode 100644 index 00000000..da8eb9dc --- /dev/null +++ b/tools/tests/test_generate_phi4_corelib_manifest.py @@ -0,0 +1,1158 @@ +from __future__ import annotations + +from collections import Counter +import hashlib +import json +import os +import sys +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +import onnx +from onnx import TensorProto, helper + +from tools import generate_phi4_corelib_manifest as manifest_tool +from tools.generate_phi4_corelib_manifest import ( + MAX_U64, + _generate_manifest, + expected_matmul_components, + host_role, + matmul_roles, + required_initializer_roles, + ssmlp_roles, +) + + +class ManifestGeneratorTests(unittest.TestCase): + def _write_model( + self, + root: Path, + initializers: list[TensorProto], + ) -> Path: + graph = helper.make_graph( + nodes=[], + name="synthetic-phi4", + inputs=[], + outputs=[], + initializer=initializers, + ) + model = helper.make_model(graph) + path = root / "model.onnx" + path.write_bytes(model.SerializeToString()) + return path + + def _embedded( + self, + name: str, + data_type: int, + shape: list[int], + raw_data: bytes, + ) -> TensorProto: + tensor = TensorProto() + tensor.name = name + tensor.data_type = data_type + tensor.dims.extend(shape) + tensor.raw_data = raw_data + return tensor + + def _external( + self, + name: str, + data_type: int, + shape: list[int], + metadata: list[tuple[str, str]], + ) -> TensorProto: + tensor = TensorProto() + tensor.name = name + tensor.data_type = data_type + tensor.dims.extend(shape) + tensor.data_location = TensorProto.EXTERNAL + for key, value in metadata: + item = tensor.external_data.add() + item.key = key + item.value = value + return tensor + + def _exact_roles( + self, + name: str, + *, + dtypes: set[str] | None = None, + shape: list[int] | None = None, + ) -> dict[str, dict[str, object]]: + return { + name: { + "role": "test.tensor", + "dtypes": dtypes or {"uint8"}, + "shape": shape or [2, 2], + } + } + + def test_matmul_component_shapes(self): + expected = expected_matmul_components(k=3072, n=1024, group=128) + self.assertEqual(expected["qweight"], ("uint8", [1024, 1536])) + self.assertEqual( + expected["scales"], + ({"float16", "float32"}, [1024, 24]), + ) + self.assertEqual(expected["qzeros"], ("uint8", [1024, 12])) + + odd_group_count = expected_matmul_components( + k=384, + n=5, + group=128, + ) + self.assertEqual(odd_group_count["qzeros"], ("uint8", [5, 2])) + + def test_matmul_roles_use_exact_names_and_constraints(self): + prefix = "model.layers.7.attn.k_proj.MatMulNBits" + self.assertEqual( + matmul_roles(prefix, 3072, 1024, 128), + { + f"{prefix}.qweight": { + "role": "matmul.qweight", + "weight_object": prefix, + "dtypes": {"uint8"}, + "shape": [1024, 1536], + }, + f"{prefix}.scales": { + "role": "matmul.scales", + "weight_object": prefix, + "dtypes": {"float16", "float32"}, + "shape": [1024, 24], + }, + f"{prefix}.qzeros": { + "role": "matmul.qzeros", + "weight_object": prefix, + "dtypes": {"uint8"}, + "shape": [1024, 12], + }, + }, + ) + + def test_ssmlp_gate_up_down_and_norm_shapes(self): + roles = ssmlp_roles(0) + base = "model.layers.0" + group = f"{base}.ssmlp" + + self.assertEqual( + roles[f"{base}.mlp.gate_proj.MatMulNBits.qweight"], + { + "role": "ssmlp.gate.qweight", + "weight_object": group, + "dtypes": {"uint8"}, + "shape": [8192, 1536], + }, + ) + self.assertEqual( + roles[f"{base}.mlp.up_proj.MatMulNBits.scales"]["shape"], + [8192, 24], + ) + self.assertEqual( + roles[f"{base}.mlp.up_proj.MatMulNBits.qzeros"]["shape"], + [8192, 12], + ) + self.assertEqual( + roles[f"{base}.mlp.down_proj.MatMulNBits.qweight"]["shape"], + [3072, 4096], + ) + self.assertEqual( + roles[f"{base}.mlp.down_proj.MatMulNBits.scales"]["shape"], + [3072, 64], + ) + self.assertEqual( + roles[f"{base}.mlp.down_proj.MatMulNBits.qzeros"]["shape"], + [3072, 32], + ) + self.assertEqual( + roles[f"{base}.post_attention_layernorm.weight"], + { + "role": "ssmlp.norm0", + "weight_object": group, + "dtypes": {"float16", "float32"}, + "shape": [3072], + }, + ) + self.assertEqual( + roles["model.layers.1.input_layernorm.weight"]["role"], + "ssmlp.norm1", + ) + + def test_required_roles_cover_exact_32_layer_driver_map(self): + roles = required_initializer_roles() + self.assertEqual(len(roles), 743) + + groups = { + record["weight_object"] + for record in roles.values() + if "weight_object" in record + } + self.assertEqual(len(groups), 161) + expected_attention_groups = { + f"model.layers.{layer}.attn.{projection}.MatMulNBits" + for layer in range(32) + for projection in ("q_proj", "k_proj", "v_proj", "o_proj") + } + self.assertTrue(expected_attention_groups.issubset(groups)) + self.assertEqual( + { + group + for group in groups + if str(group).endswith(".ssmlp") + }, + {f"model.layers.{layer}.ssmlp" for layer in range(32)}, + ) + self.assertIn("lm_head.MatMulNBits", groups) + + o_prefix = "model.layers.31.attn.o_proj.MatMulNBits" + self.assertEqual( + roles[f"{o_prefix}.qweight"]["shape"], + [3072, 1536], + ) + self.assertNotIn( + "model.layers.32.attn.q_proj.MatMulNBits.qweight", + roles, + ) + + def test_layer_31_uses_phantom_layer_32_final_norm(self): + roles = ssmlp_roles(31) + final_norm = "model.layers.32.final_norm_layernorm.weight" + self.assertIn(final_norm, roles) + self.assertEqual(roles[final_norm]["role"], "ssmlp.norm1") + self.assertNotIn("model.layers.32.input_layernorm.weight", roles) + self.assertNotIn("model.norm.weight", required_initializer_roles()) + + def test_host_tensor_constraints(self): + self.assertEqual( + host_role("embedding"), + { + "role": "embedding", + "dtypes": {"float16"}, + "shape": [200064, 3072], + }, + ) + self.assertEqual( + host_role("input_norm"), + { + "role": "input_norm", + "dtypes": {"float16", "float32"}, + "shape": [3072], + }, + ) + for role in ("cos_cache", "sin_cache"): + self.assertEqual( + host_role(role), + { + "role": role, + "dtypes": {"float16", "float32"}, + "rank": 2, + "minimum_shape": [4096, 48], + }, + ) + + def test_external_initializer_metadata_and_hash(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + data_path = root / "weights" / "data.bin" + data_path.parent.mkdir() + data_path.write_bytes(b"HEADER" + bytes([1, 2, 3, 4]) + b"TAIL") + tensor = self._external( + "test.weight", + TensorProto.UINT8, + [2, 2], + [ + ("location", "weights/data.bin"), + ("offset", "6"), + ("length", "4"), + ], + ) + model_path = self._write_model(root, [tensor]) + output = root / "manifest.json" + + manifest = _generate_manifest( + root, + output, + True, + self._exact_roles("test.weight"), + ) + + self.assertEqual( + manifest["initializers"]["test.weight"], + { + "dtype": "uint8", + "file": "weights/data.bin", + "length": 4, + "offset": 6, + "role": "test.tensor", + "shape": [2, 2], + }, + ) + self.assertEqual( + set(manifest["files"]), + {"model.onnx", "weights/data.bin"}, + ) + self.assertEqual( + manifest["files"]["weights/data.bin"], + { + "size": data_path.stat().st_size, + "sha256": hashlib.sha256(data_path.read_bytes()).hexdigest(), + }, + ) + self.assertEqual( + manifest["files"]["model.onnx"]["sha256"], + hashlib.sha256(model_path.read_bytes()).hexdigest(), + ) + + def test_full_hash_flag_controls_sha256_metadata(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + tensor = self._embedded( + "test.weight", + TensorProto.UINT8, + [2, 2], + b"data", + ) + self._write_model(root, [tensor]) + + manifest = _generate_manifest( + root, + root / "manifest.json", + False, + self._exact_roles("test.weight"), + ) + + self.assertTrue(manifest["files"]) + for record in manifest["files"].values(): + self.assertNotIn("sha256", record) + + def test_initializer_schema_omits_validation_only_group_metadata(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + tensor = self._embedded( + "test.weight", + TensorProto.UINT8, + [2, 2], + b"data", + ) + self._write_model(root, [tensor]) + roles = self._exact_roles("test.weight") + roles["test.weight"]["weight_object"] = "test.matmul" + + manifest = _generate_manifest( + root, + root / "manifest.json", + False, + roles, + ) + + self.assertEqual( + manifest["initializers"]["test.weight"], + { + "dtype": "uint8", + "file": "corelib_embedded_initializers.bin", + "length": 4, + "offset": 0, + "role": "test.tensor", + "shape": [2, 2], + }, + ) + + def test_embedded_initializers_are_sorted_into_one_sidecar(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + z_tensor = self._embedded( + "z.tensor", + TensorProto.UINT8, + [3], + b"XYZ", + ) + a_tensor = self._embedded( + "a.tensor", + TensorProto.UINT8, + [2], + b"ab", + ) + self._write_model(root, [z_tensor, a_tensor]) + roles = { + "z.tensor": { + "role": "test.z", + "dtypes": {"uint8"}, + "shape": [3], + }, + "a.tensor": { + "role": "test.a", + "dtypes": {"uint8"}, + "shape": [2], + }, + } + + first = root / "first.json" + second = root / "second.json" + manifest = _generate_manifest(root, first, True, roles) + _generate_manifest(root, second, True, roles) + + sidecar = root / "corelib_embedded_initializers.bin" + self.assertEqual(sidecar.read_bytes(), b"abXYZ") + self.assertEqual( + manifest["initializers"]["a.tensor"]["offset"], + 0, + ) + self.assertEqual( + manifest["initializers"]["z.tensor"]["offset"], + 2, + ) + self.assertEqual( + manifest["files"]["corelib_embedded_initializers.bin"], + { + "size": 5, + "sha256": hashlib.sha256(b"abXYZ").hexdigest(), + }, + ) + self.assertEqual(first.read_bytes(), second.read_bytes()) + decoded = json.loads(first.read_text(encoding="utf-8")) + self.assertEqual( + list(decoded["initializers"]), + ["a.tensor", "z.tensor"], + ) + + def test_wider_and_longer_rope_source_is_recorded(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + rows = 4097 + columns = 64 + tensor = self._embedded( + "cos_cache", + TensorProto.FLOAT16, + [rows, columns], + bytes(rows * columns * 2), + ) + self._write_model(root, [tensor]) + + manifest = _generate_manifest( + root, + root / "manifest.json", + False, + {"cos_cache": host_role("cos_cache")}, + ) + + self.assertEqual( + manifest["initializers"]["cos_cache"]["shape"], + [4097, 64], + ) + self.assertEqual( + manifest["initializers"]["cos_cache"]["dtype"], + "float16", + ) + + def test_path_traversal_and_absolute_locations_are_rejected(self): + bad_locations = ( + "../escape.bin", + "safe/../../escape.bin", + r"safe\..\escape.bin", + "/absolute.bin", + "C:/absolute.bin", + r"\\server\share\absolute.bin", + ) + for location in bad_locations: + with self.subTest(location=location): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + tensor = self._external( + "test.weight", + TensorProto.UINT8, + [2, 2], + [ + ("location", location), + ("offset", "0"), + ("length", "4"), + ], + ) + self._write_model(root, [tensor]) + + with self.assertRaisesRegex(ValueError, "path|location"): + _generate_manifest( + root, + root / "manifest.json", + False, + self._exact_roles("test.weight"), + ) + + def test_symlink_escape_is_rejected(self): + with tempfile.TemporaryDirectory() as directory: + parent = Path(directory) + root = parent / "model" + root.mkdir() + outside = parent / "outside.bin" + outside.write_bytes(b"data") + link = root / "weights.bin" + try: + link.symlink_to(outside) + except (NotImplementedError, OSError) as error: + if os.name == "nt": + self.skipTest( + f"Windows cannot create the required symlink: {error}" + ) + raise + + tensor = self._external( + "test.weight", + TensorProto.UINT8, + [2, 2], + [ + ("location", "weights.bin"), + ("offset", "0"), + ("length", "4"), + ], + ) + self._write_model(root, [tensor]) + + with self.assertRaisesRegex(ValueError, "escapes"): + _generate_manifest( + root, + root / "manifest.json", + False, + self._exact_roles("test.weight"), + ) + + def test_out_of_file_range_reaches_range_guard(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "weights.bin").write_bytes(b"ab") + tensor = self._external( + "test.weight", + TensorProto.UINT8, + [2], + [ + ("location", "weights.bin"), + ("offset", "1"), + ("length", "2"), + ], + ) + self._write_model(root, [tensor]) + + with self.assertRaisesRegex(ValueError, "range exceeds file size"): + _generate_manifest( + root, + root / "manifest.json", + False, + self._exact_roles("test.weight", shape=[2]), + ) + + def test_multibyte_dtype_rejects_odd_external_offset(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "weights.bin").write_bytes(b"x" + bytes(4)) + tensor = self._external( + "test.scale", + TensorProto.FLOAT16, + [2], + [ + ("location", "weights.bin"), + ("offset", "1"), + ("length", "4"), + ], + ) + self._write_model(root, [tensor]) + + with self.assertRaisesRegex(ValueError, "dtype-aligned"): + _generate_manifest( + root, + root / "manifest.json", + False, + self._exact_roles( + "test.scale", + dtypes={"float16"}, + shape=[2], + ), + ) + + def test_duplicate_external_metadata_key_is_rejected(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "weights.bin").write_bytes(b"data") + tensor = self._external( + "test.weight", + TensorProto.UINT8, + [2, 2], + [ + ("location", "weights.bin"), + ("location", "weights.bin"), + ("offset", "0"), + ("length", "4"), + ], + ) + self._write_model(root, [tensor]) + + with self.assertRaisesRegex( + ValueError, + "duplicate external metadata key location", + ): + _generate_manifest( + root, + root / "manifest.json", + False, + self._exact_roles("test.weight"), + ) + + def test_generated_sidecar_cannot_replace_required_external_data(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + sidecar = root / "corelib_embedded_initializers.bin" + sidecar.write_bytes(b"source") + external = self._external( + "external.weight", + TensorProto.UINT8, + [6], + [ + ("location", "corelib_embedded_initializers.bin"), + ("offset", "0"), + ("length", "6"), + ], + ) + embedded = self._embedded( + "embedded.weight", + TensorProto.UINT8, + [4], + b"data", + ) + self._write_model(root, [external, embedded]) + roles = { + "external.weight": { + "role": "test.external", + "dtypes": {"uint8"}, + "shape": [6], + }, + "embedded.weight": { + "role": "test.embedded", + "dtypes": {"uint8"}, + "shape": [4], + }, + } + + with self.assertRaisesRegex(ValueError, "sidecar conflicts"): + _generate_manifest( + root, + root / "manifest.json", + False, + roles, + ) + self.assertEqual(sidecar.read_bytes(), b"source") + + def test_output_cannot_overwrite_required_external_data(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + data_path = root / "weights.bin" + data_path.write_bytes(b"data") + tensor = self._external( + "test.weight", + TensorProto.UINT8, + [2, 2], + [ + ("location", "weights.bin"), + ("offset", "0"), + ("length", "4"), + ], + ) + self._write_model(root, [tensor]) + + with self.assertRaisesRegex( + ValueError, + "output path would overwrite", + ): + _generate_manifest( + root, + data_path, + False, + self._exact_roles("test.weight"), + ) + self.assertEqual(data_path.read_bytes(), b"data") + + def test_missing_length_and_range_overflow_are_rejected(self): + cases = ( + ( + [ + ("location", "weights.bin"), + ("offset", "0"), + ], + "length", + ), + ( + [ + ("location", "weights.bin"), + ("offset", str(MAX_U64)), + ("length", "2"), + ], + "overflow", + ), + ) + for metadata, message in cases: + with self.subTest(message=message): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "weights.bin").write_bytes(b"data") + tensor = self._external( + "test.weight", + TensorProto.UINT8, + [2, 2], + metadata, + ) + self._write_model(root, [tensor]) + + with self.assertRaisesRegex(ValueError, message): + _generate_manifest( + root, + root / "manifest.json", + False, + self._exact_roles("test.weight"), + ) + + def test_wrong_dtype_rank_and_shape_are_rejected(self): + cases = ( + ( + TensorProto.FLOAT16, + [2, 2], + bytes(8), + "dtype", + ), + ( + TensorProto.UINT8, + [4], + bytes(4), + "shape", + ), + ( + TensorProto.UINT8, + [2, 3], + bytes(6), + "shape", + ), + ) + for data_type, shape, raw_data, message in cases: + with self.subTest(data_type=data_type, shape=shape): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + tensor = self._embedded( + "test.weight", + data_type, + shape, + raw_data, + ) + self._write_model(root, [tensor]) + + with self.assertRaisesRegex(ValueError, message): + _generate_manifest( + root, + root / "manifest.json", + False, + self._exact_roles("test.weight"), + ) + + def test_byte_count_mismatches_are_rejected(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + embedded = self._embedded( + "test.weight", + TensorProto.UINT8, + [2, 2], + b"abc", + ) + self._write_model(root, [embedded]) + with self.assertRaisesRegex(ValueError, "byte count"): + _generate_manifest( + root, + root / "embedded.json", + False, + self._exact_roles("test.weight"), + ) + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "weights.bin").write_bytes(b"abc") + external = self._external( + "test.weight", + TensorProto.UINT8, + [2, 2], + [ + ("location", "weights.bin"), + ("offset", "0"), + ("length", "3"), + ], + ) + self._write_model(root, [external]) + with self.assertRaisesRegex(ValueError, "byte count"): + _generate_manifest( + root, + root / "external.json", + False, + self._exact_roles("test.weight"), + ) + + def test_missing_and_duplicate_required_initializers_are_rejected(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + self._write_model(root, []) + with self.assertRaisesRegex(ValueError, "missing initializer"): + _generate_manifest( + root, + root / "missing.json", + False, + self._exact_roles("test.weight"), + ) + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + first = self._embedded( + "test.weight", + TensorProto.UINT8, + [2, 2], + b"abcd", + ) + second = self._embedded( + "test.weight", + TensorProto.UINT8, + [2, 2], + b"efgh", + ) + self._write_model(root, [first, second]) + with self.assertRaisesRegex(ValueError, "duplicate initializer"): + _generate_manifest( + root, + root / "duplicate.json", + False, + self._exact_roles("test.weight"), + ) + + def test_manifest_has_locked_schema_and_model_identity(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + tensor = self._embedded( + "test.weight", + TensorProto.UINT8, + [2, 2], + b"abcd", + ) + self._write_model(root, [tensor]) + + manifest = _generate_manifest( + root, + root / "manifest.json", + False, + self._exact_roles("test.weight"), + ) + + self.assertEqual(manifest["schema_version"], 1) + self.assertEqual(manifest["execution_backend"], "corelib_aie4") + self.assertEqual( + manifest["model"], + { + "family": "phi4", + "group_size": 128, + "head_size": 128, + "hidden_size": 3072, + "intermediate_size": 8192, + "kv_heads": 8, + "layers": 32, + "num_heads": 24, + "rms_epsilon": 0.00001, + "rope_dim": 96, + "vocab_size": 200064, + }, + ) + self.assertEqual(manifest["backend"], {"max_seq": 4096}) + + def test_required_weight_objects_have_exact_descriptors_and_roles(self): + objects = manifest_tool.required_weight_objects() + self.assertEqual(len(objects), 161) + self.assertEqual(objects, manifest_tool.required_weight_objects()) + + names = [record["name"] for record in objects] + self.assertEqual(len(names), len(set(names))) + self.assertEqual( + Counter(record["kind"] for record in objects), + {"matmul": 129, "ssmlp": 32}, + ) + self.assertEqual( + sum( + record["kind"] == "matmul" + and ".attn." in record["name"] + for record in objects + ), + 128, + ) + expected_names = [ + name + for layer in range(32) + for name in ( + f"model.layers.{layer}.attn.q_proj.MatMulNBits", + f"model.layers.{layer}.attn.k_proj.MatMulNBits", + f"model.layers.{layer}.attn.v_proj.MatMulNBits", + f"model.layers.{layer}.attn.o_proj.MatMulNBits", + f"model.layers.{layer}.ssmlp", + ) + ] + expected_names.append("lm_head.MatMulNBits") + self.assertEqual(names, expected_names) + self.assertEqual( + names[:5], + [ + "model.layers.0.attn.q_proj.MatMulNBits", + "model.layers.0.attn.k_proj.MatMulNBits", + "model.layers.0.attn.v_proj.MatMulNBits", + "model.layers.0.attn.o_proj.MatMulNBits", + "model.layers.0.ssmlp", + ], + ) + self.assertEqual(names[-1], "lm_head.MatMulNBits") + + by_name = {record["name"]: record for record in objects} + projection_descriptors = ( + ("q_proj", 3072, 3072), + ("k_proj", 3072, 1024), + ("v_proj", 3072, 1024), + ("o_proj", 3072, 3072), + ) + for layer in range(32): + for projection, k, n in projection_descriptors: + prefix = ( + f"model.layers.{layer}.attn.{projection}.MatMulNBits" + ) + self.assertEqual( + by_name[prefix]["descriptor"], + { + "k": k, + "n": n, + "group_size": 128, + "has_bias": False, + }, + ) + self.assertEqual( + by_name[prefix]["roles"], + { + "qweight": f"{prefix}.qweight", + "scales": f"{prefix}.scales", + "qzeros": f"{prefix}.qzeros", + }, + ) + + base = f"model.layers.{layer}" + next_norm = ( + "model.layers.32.final_norm_layernorm.weight" + if layer == 31 + else f"model.layers.{layer + 1}.input_layernorm.weight" + ) + expected_ssmlp_roles = { + "norm0": f"{base}.post_attention_layernorm.weight", + "norm1": next_norm, + } + for projection in ("gate", "up", "down"): + prefix = f"{base}.mlp.{projection}_proj.MatMulNBits" + for component in ("qweight", "scales", "qzeros"): + expected_ssmlp_roles[f"{projection}_{component}"] = ( + f"{prefix}.{component}" + ) + self.assertEqual( + by_name[f"{base}.ssmlp"], + { + "name": f"{base}.ssmlp", + "kind": "ssmlp", + "descriptor": { + "k": 3072, + "n": 8192, + "group_size": 128, + }, + "roles": expected_ssmlp_roles, + }, + ) + + self.assertEqual( + by_name["model.layers.0.attn.q_proj.MatMulNBits"], + { + "name": "model.layers.0.attn.q_proj.MatMulNBits", + "kind": "matmul", + "descriptor": { + "k": 3072, + "n": 3072, + "group_size": 128, + "has_bias": False, + }, + "roles": { + "qweight": ( + "model.layers.0.attn.q_proj." + "MatMulNBits.qweight" + ), + "scales": ( + "model.layers.0.attn.q_proj." + "MatMulNBits.scales" + ), + "qzeros": ( + "model.layers.0.attn.q_proj." + "MatMulNBits.qzeros" + ), + }, + }, + ) + self.assertEqual( + by_name["model.layers.0.attn.k_proj.MatMulNBits"]["descriptor"], + { + "k": 3072, + "n": 1024, + "group_size": 128, + "has_bias": False, + }, + ) + self.assertEqual( + by_name["model.layers.0.attn.v_proj.MatMulNBits"]["descriptor"], + { + "k": 3072, + "n": 1024, + "group_size": 128, + "has_bias": False, + }, + ) + self.assertEqual( + by_name["model.layers.0.attn.o_proj.MatMulNBits"]["descriptor"], + { + "k": 3072, + "n": 3072, + "group_size": 128, + "has_bias": False, + }, + ) + self.assertEqual( + by_name["lm_head.MatMulNBits"]["descriptor"], + { + "k": 3072, + "n": 200064, + "group_size": 128, + "has_bias": False, + }, + ) + + final_ssmlp = by_name["model.layers.31.ssmlp"] + self.assertEqual( + final_ssmlp["descriptor"], + {"k": 3072, "n": 8192, "group_size": 128}, + ) + self.assertEqual( + final_ssmlp["roles"]["norm1"], + "model.layers.32.final_norm_layernorm.weight", + ) + self.assertNotIn("epsilon", final_ssmlp["roles"]) + self.assertEqual(manifest_tool.MODEL_IDENTITY["rms_epsilon"], 0.00001) + + initializers = required_initializer_roles() + for weight_object in objects: + for initializer_name in weight_object["roles"].values(): + self.assertIn(initializer_name, initializers) + + def test_manifest_emits_top_level_weight_objects_only(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + names = ("test.qweight", "test.scales", "test.qzeros") + tensors = [ + self._embedded(name, TensorProto.UINT8, [1], bytes([index])) + for index, name in enumerate(names, start=1) + ] + self._write_model(root, tensors) + roles = { + name: { + "role": f"test.{name.rsplit('.', 1)[-1]}", + "weight_object": "test.matmul", + "dtypes": {"uint8"}, + "shape": [1], + } + for name in names + } + weight_objects = [ + { + "name": "test.matmul", + "kind": "matmul", + "descriptor": { + "k": 2, + "n": 1, + "group_size": 1, + "has_bias": False, + }, + "roles": { + "qweight": "test.qweight", + "scales": "test.scales", + "qzeros": "test.qzeros", + }, + } + ] + + manifest = _generate_manifest( + root, + root / "manifest.json", + False, + roles, + weight_objects, + ) + + self.assertEqual(manifest["weight_objects"], weight_objects) + for initializer in manifest["initializers"].values(): + self.assertNotIn("weight_object", initializer) + + def test_generate_manifest_binds_real_role_and_weight_maps(self): + model_dir = Path("model package") + output = Path("output manifest.json") + sentinel = {"result": "sentinel"} + + with patch.object( + manifest_tool, + "_generate_manifest", + return_value=sentinel, + ) as lower: + result = manifest_tool.generate_manifest( + model_dir, + output, + True, + ) + + self.assertIs(result, sentinel) + self.assertEqual(len(lower.call_args.args), 5) + self.assertEqual(lower.call_args.args[:3], (model_dir, output, True)) + self.assertEqual( + lower.call_args.args[3], + required_initializer_roles(), + ) + self.assertEqual( + lower.call_args.args[4], + manifest_tool.required_weight_objects(), + ) + + def test_main_wires_exact_paths_and_full_hash_flag(self): + cases = ( + ([], False), + (["--full-hash"], True), + ) + for extra, expected_full_hash in cases: + with self.subTest(full_hash=expected_full_hash): + argv = [ + "generate_phi4_corelib_manifest.py", + "--model-dir", + "model package", + "--output", + "output manifest.json", + *extra, + ] + with ( + patch.object(sys, "argv", argv), + patch.object( + manifest_tool, + "generate_manifest", + ) as generate, + ): + self.assertEqual(manifest_tool.main(), 0) + + generate.assert_called_once_with( + Path("model package"), + Path("output manifest.json"), + expected_full_hash, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/tests/test_package_phi4_corelib_aie4.py b/tools/tests/test_package_phi4_corelib_aie4.py new file mode 100644 index 00000000..c9ea1d62 --- /dev/null +++ b/tools/tests/test_package_phi4_corelib_aie4.py @@ -0,0 +1,998 @@ +from __future__ import annotations + +import difflib +import hashlib +import io +import json +import tempfile +import unittest +from pathlib import Path +from urllib.error import HTTPError +from unittest.mock import patch + +import onnx +from onnx import TensorProto, helper + +from tools import generate_phi4_corelib_manifest as manifest_tool +from tools import package_phi4_corelib_aie4 as package_tool + + +class Phi4CorelibOverlayTests(unittest.TestCase): + def _genai_config(self) -> dict[str, object]: + return { + "model": { + "bos_token_id": 199999, + "context_length": 131072, + "decoder": { + "head_size": 128, + "hidden_size": 3072, + "num_attention_heads": 24, + "num_hidden_layers": 32, + "num_key_value_heads": 8, + }, + "eos_token_id": [200020, 199999], + "type": "phi3", + "vocab_size": 200064, + }, + "search": {"max_length": 131072}, + } + + def _git_blob_oid(self, data: bytes) -> str: + header = f"blob {len(data)}\0".encode("ascii") + return hashlib.sha1(header + data).hexdigest() + + def _write_accepted_upstream( + self, + root: Path, + ) -> list[dict[str, object]]: + genai_data = ( + json.dumps(self._genai_config(), indent=2, sort_keys=True) + "\n" + ).encode() + tokenizer_data = b'{"model_max_length":131072}\n' + template_data = b"{{ messages | length }}\n" + for name, data in ( + ("genai_config.json", genai_data), + ("tokenizer_config.json", tokenizer_data), + ("chat_template.jinja", template_data), + ): + (root / name).write_bytes(data) + + item_sizes = { + "uint8": (TensorProto.UINT8, 1), + "float16": (TensorProto.FLOAT16, 2), + "float32": (TensorProto.FLOAT, 4), + "int64": (TensorProto.INT64, 8), + } + tensors: list[TensorProto] = [] + offset = 0 + for name, contract in sorted( + manifest_tool.required_initializer_roles().items() + ): + accepted = sorted(contract["dtypes"]) + dtype = "float16" if "float16" in accepted else accepted[0] + data_type, item_size = item_sizes[dtype] + shape = list( + contract.get("shape", contract.get("minimum_shape")) + ) + if name.endswith(".qweight"): + shape = [shape[0], shape[1] // 64, 64] + elif ( + name.endswith(".scales") or + name.endswith(".qzeros") + ): + shape = [shape[0] * shape[1]] + length = item_size + for dimension in shape: + length *= dimension + tensor = TensorProto() + tensor.name = name + tensor.data_type = data_type + tensor.dims.extend(shape) + tensor.data_location = TensorProto.EXTERNAL + for key, value in ( + ("location", "model.onnx.data"), + ("offset", str(offset)), + ("length", str(length)), + ): + item = tensor.external_data.add() + item.key = key + item.value = value + tensors.append(tensor) + offset += length + + logical_sha = "b" * 64 + (root / "model.onnx.data").write_text( + "version https://git-lfs.github.com/spec/v1\n" + f"oid sha256:{logical_sha}\n" + f"size {offset}\n", + encoding="ascii", + newline="\n", + ) + model = helper.make_model( + helper.make_graph([], "accepted-phi4", [], [], tensors) + ) + model_data = model.SerializeToString() + (root / "model.onnx").write_bytes(model_data) + + records: list[dict[str, object]] = [] + for name, data in ( + ("chat_template.jinja", template_data), + ("genai_config.json", genai_data), + ("tokenizer_config.json", tokenizer_data), + ): + records.append( + { + "type": "file", + "oid": self._git_blob_oid(data), + "size": len(data), + "path": name, + } + ) + records.extend( + [ + { + "type": "file", + "oid": "1" * 40, + "size": len(model_data), + "lfs": { + "oid": hashlib.sha256(model_data).hexdigest(), + "size": len(model_data), + "pointerSize": 131, + }, + "path": "model.onnx", + }, + { + "type": "file", + "oid": "2" * 40, + "size": offset, + "lfs": { + "oid": logical_sha, + "size": offset, + "pointerSize": 135, + }, + "path": "model.onnx.data", + }, + ] + ) + return records + + def test_normalized_config_is_derived_from_oga_and_backend_contract(self): + self.assertEqual( + package_tool.normalize_config(self._genai_config()), + { + "flm_version": "1.0.4", + "head_dim": 128, + "hidden_size": 3072, + "intermediate_size": 8192, + "model_type": "phi4", + "num_attention_heads": 24, + "num_hidden_layers": 32, + "num_key_value_heads": 8, + "rms_norm_eps": 1.0e-5, + "vocab_size": 200064, + }, + ) + + def test_normalized_tokenizer_preserves_upstream_and_adds_exact_sources(self): + upstream = { + "add_bos_token": False, + "model_max_length": 131072, + "tokenizer_class": "GPT2Tokenizer", + } + template = "{{ messages | length }}" + + normalized = package_tool.normalize_tokenizer_config( + upstream, + template, + self._genai_config(), + ) + + self.assertEqual(normalized["add_bos_token"], False) + self.assertEqual(normalized["model_max_length"], 131072) + self.assertEqual(normalized["tokenizer_class"], "GPT2Tokenizer") + self.assertEqual(normalized["chat_template"], template) + self.assertEqual(normalized["eos_token_id"], [200020, 199999]) + + def test_normalization_rejects_unapproved_oga_identity(self): + bad = self._genai_config() + bad["model"]["decoder"]["hidden_size"] = 4096 + with self.assertRaisesRegex(ValueError, "hidden_size"): + package_tool.normalize_config(bad) + + bad = self._genai_config() + bad["model"]["eos_token_id"] = [199999] + with self.assertRaisesRegex(ValueError, "eos_token_id"): + package_tool.normalize_tokenizer_config({}, "template", bad) + + def test_manifest_accepts_verified_lfs_pointer_as_logical_external_file(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + logical_sha = "a" * 64 + logical_size = 4 + (root / "weights.bin").write_text( + "version https://git-lfs.github.com/spec/v1\n" + f"oid sha256:{logical_sha}\n" + f"size {logical_size}\n", + encoding="ascii", + newline="\n", + ) + + tensor = TensorProto() + tensor.name = "test.weight" + tensor.data_type = TensorProto.UINT8 + tensor.dims.extend([2, 2]) + tensor.data_location = TensorProto.EXTERNAL + for key, value in ( + ("location", "weights.bin"), + ("offset", "0"), + ("length", "4"), + ): + item = tensor.external_data.add() + item.key = key + item.value = value + model = helper.make_model( + helper.make_graph([], "logical-lfs", [], [], [tensor]) + ) + model_path = root / "model.onnx" + model_path.write_bytes(model.SerializeToString()) + + manifest = manifest_tool._generate_manifest( + root, + root / "manifest.json", + True, + { + "test.weight": { + "role": "test.tensor", + "dtypes": {"uint8"}, + "shape": [2, 2], + } + }, + file_metadata={ + "weights.bin": { + "size": logical_size, + "sha256": logical_sha, + } + }, + ) + + self.assertEqual( + manifest["files"]["weights.bin"], + {"size": logical_size, "sha256": logical_sha}, + ) + self.assertEqual( + manifest["files"]["model.onnx"], + { + "size": model_path.stat().st_size, + "sha256": hashlib.sha256(model_path.read_bytes()).hexdigest(), + }, + ) + + def test_catalog_measurements_use_remote_logical_and_overlay_sizes(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "remote-lfs.bin").write_bytes(b"pointer") + (root / "config.json").write_bytes(b"{}") + size, footprint = package_tool.catalog_measurements( + root, + {"remote-lfs.bin": 1024**3}, + ) + self.assertEqual(size, 1024**3 + 2) + self.assertEqual(footprint, 1.0) + + def test_provenance_records_pinned_inputs_and_generated_outputs(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + upstream = root / "upstream" + overlay = root / "overlay" + upstream.mkdir() + overlay.mkdir() + (upstream / "genai_config.json").write_bytes(b'{"source":1}\n') + (upstream / "tokenizer_config.json").write_bytes(b'{"source":2}\n') + (upstream / "chat_template.jinja").write_bytes(b"template\n") + (overlay / "config.json").write_bytes(b'{"output":1}\n') + + provenance = package_tool.build_provenance( + upstream, + overlay, + "e751fb68c2cfffe6b0d32942118f75ac0a0365bb", + ["config.json"], + ) + + self.assertEqual( + provenance["upstream"]["commit"], + "e751fb68c2cfffe6b0d32942118f75ac0a0365bb", + ) + self.assertEqual( + set(provenance["upstream"]["inputs"]), + { + "chat_template.jinja", + "genai_config.json", + "tokenizer_config.json", + }, + ) + self.assertEqual( + provenance["generated"]["config.json"]["size"], + len(b'{"output":1}\n'), + ) + self.assertEqual( + len(provenance["generated"]["config.json"]["sha256"]), + 64, + ) + + def test_overlay_generation_is_complete_and_deterministic(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + upstream = root / "upstream" + first = root / "first" + second = root / "second" + upstream.mkdir() + records = self._write_accepted_upstream(upstream) + + package_tool.generate_overlay( + upstream, + first, + package_tool.UPSTREAM_COMMIT, + records, + ) + package_tool.generate_overlay( + upstream, + second, + package_tool.UPSTREAM_COMMIT, + records, + ) + + expected = { + "config.json", + "corelib_phi4_manifest.json", + "provenance.json", + "tokenizer_config.json", + } + self.assertEqual( + {path.name for path in first.iterdir()}, + expected, + ) + self.assertEqual( + { + path.name: path.read_bytes() + for path in first.iterdir() + }, + { + path.name: path.read_bytes() + for path in second.iterdir() + }, + ) + manifest = json.loads( + (first / "corelib_phi4_manifest.json").read_text() + ) + self.assertEqual(len(manifest["initializers"]), 743) + self.assertEqual(len(manifest["weight_objects"]), 161) + self.assertEqual( + set(manifest["files"]), + {"model.onnx", "model.onnx.data"}, + ) + self.assertNotIn( + "corelib_embedded_initializers.bin", + manifest["files"], + ) + provenance = json.loads( + (first / "provenance.json").read_text() + ) + self.assertEqual( + provenance["upstream"]["commit"], + package_tool.UPSTREAM_COMMIT, + ) + self.assertEqual( + provenance["upstream"]["git_files"], + sorted(records, key=lambda record: record["path"]), + ) + + def test_catalog_entry_uses_exact_remote_and_overlay_sizes(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + overlay = root / "phi4-mini-it-aie4" + overlay.mkdir() + for name, data in ( + ("config.json", b"config"), + ("corelib_phi4_manifest.json", b"manifest"), + ("provenance.json", b"provenance"), + ("tokenizer_config.json", b"normalized-tokenizer"), + ): + (overlay / name).write_bytes(data) + records = [ + { + "type": "file", + "oid": "1" * 40, + "size": 100, + "path": "model.onnx", + }, + { + "type": "file", + "oid": "2" * 40, + "size": 200, + "path": "tokenizer_config.json", + }, + # Present upstream, deliberately not carried into the package. + { + "type": "file", + "oid": "3" * 40, + "size": 400, + "path": "genai_config.json", + }, + { + "type": "file", + "oid": "4" * 40, + "size": 800, + "path": ".gitattributes", + }, + ] + + entry = package_tool.build_catalog_entry(overlay, records) + + # The shadowed upstream tokenizer_config.json and the excluded + # files contribute nothing: size covers the assembled directory, + # not the upstream repository. + expected_size = ( + 100 + len(b"config") + len(b"manifest") + + len(b"provenance") + len(b"normalized-tokenizer") + ) + self.assertEqual(entry["size"], expected_size) + self.assertEqual( + entry["footprint"], + round(expected_size / (1024**3), 2), + ) + self.assertEqual( + set(entry["files"]), + { + "config.json", + "corelib_phi4_manifest.json", + "model.onnx", + "provenance.json", + "tokenizer_config.json", + }, + ) + self.assertEqual( + set(entry["bundled_overlays"]), + set(package_tool.OVERLAY_FILES), + ) + self.assertNotIn("genai_config.json", entry["files"]) + self.assertNotIn(".gitattributes", entry["files"]) + + def test_metadata_refresh_preserves_unrelated_entries(self): + model_list = { + "models": { + "existing": {"1b": {"name": "keep"}}, + } + } + model_info = {"existing:1b": [{"path": "keep"}]} + entry = {"name": "generated"} + records = [{"path": "model.onnx", "type": "file"}] + + updated_list, updated_info = package_tool.updated_catalog_documents( + model_list, + model_info, + entry, + records, + ) + + self.assertEqual( + updated_list["models"]["existing"], + {"1b": {"name": "keep"}}, + ) + self.assertEqual( + updated_info["existing:1b"], + [{"path": "keep"}], + ) + self.assertEqual( + updated_list["models"]["phi4-mini-it-aie4"]["4b"], + entry, + ) + self.assertEqual( + updated_info["phi4-mini-it-aie4:4b"], + records, + ) + + def test_http_unauthorized_falls_back_to_pinned_git_metadata(self): + unauthorized = HTTPError( + package_tool.UPSTREAM_API_URL, + 401, + "Unauthorized", + {}, + None, + ) + expected = [{"path": "model.onnx", "type": "file"}] + with ( + patch.object( + package_tool.urllib.request, + "urlopen", + side_effect=unauthorized, + ), + patch.object( + package_tool, + "git_metadata_records", + return_value=expected, + ) as git_records, + ): + actual = package_tool.huggingface_metadata_records( + Path("metadata checkout"), + package_tool.UPSTREAM_COMMIT, + ) + + self.assertEqual(actual, expected) + git_records.assert_called_once_with( + Path("metadata checkout"), + package_tool.UPSTREAM_COMMIT, + ) + + def test_http_metadata_is_normalized_to_git_fallback_schema(self): + response = io.BytesIO( + json.dumps( + [ + { + "type": "file", + "oid": "1" * 40, + "size": 4, + "lfs": { + "oid": "a" * 64, + "size": 4, + "pointerSize": 127, + }, + "xetHash": "environment-specific", + "path": "model.onnx", + } + ] + ).encode() + ) + with patch.object( + package_tool.urllib.request, + "urlopen", + return_value=response, + ): + records = package_tool.huggingface_metadata_records( + Path("unused"), + package_tool.UPSTREAM_COMMIT, + ) + + self.assertEqual( + records, + [ + { + "type": "file", + "oid": "1" * 40, + "size": 4, + "lfs": { + "oid": "a" * 64, + "size": 4, + "pointerSize": 127, + }, + "path": "model.onnx", + } + ], + ) + + +class Phi4CatalogProvenanceTests(unittest.TestCase): + """Design `PACKAGE-1`: the two provenances are validated separately. + + A rule requiring every catalog file to carry Hugging Face metadata cannot + hold, because the overlay files do not exist upstream by construction. The + interesting failure is the opposite one: an overlay file that acquires an + upstream record means FastFlow's package contract was published to the + model repository, and the two provenances stop being distinguishable. + """ + + SOURCE = Path(__file__).resolve().parents[2] / "src" + + def _committed(self): + model_list = json.loads( + (self.SOURCE / "model_list.json").read_text(encoding="utf-8") + ) + model_info = json.loads( + (self.SOURCE / "model_info.json").read_text(encoding="utf-8") + ) + return ( + model_list["models"]["phi4-mini-it-aie4"]["4b"], + model_info["phi4-mini-it-aie4:4b"], + self.SOURCE / "model_overlays" / "phi4-mini-it-aie4", + ) + + def test_committed_catalog_satisfies_both_provenances(self): + entry, records, overlay = self._committed() + package_tool.validate_catalog_provenance(entry, records, overlay) + + def test_every_upstream_file_has_a_metadata_record(self): + entry, records, overlay = self._committed() + overlays = set(entry["bundled_overlays"]) + indexed = {record["path"] for record in records} + for name in entry["files"]: + if name in overlays: + continue + self.assertIn(name, indexed, f"{name} has no upstream record") + + def test_authored_overlays_have_no_upstream_record(self): + entry, records, overlay = self._committed() + indexed = {record["path"] for record in records} + for name in package_tool.OVERLAY_FILES_WITHOUT_UPSTREAM: + self.assertIn(name, entry["bundled_overlays"]) + self.assertNotIn(name, indexed) + self.assertTrue((overlay / name).is_file()) + + def test_shadowed_overlay_keeps_its_upstream_record(self): + # tokenizer_config.json is the one overlay that also exists upstream. + # The overlay replaces it because the published file carries neither a + # chat template nor eos_token_id, so its upstream record is expected + # and must not be mistaken for a published FastFlow contract. + entry, records, _ = self._committed() + indexed = {record["path"] for record in records} + self.assertIn("tokenizer_config.json", entry["bundled_overlays"]) + self.assertIn("tokenizer_config.json", indexed) + + def test_published_overlay_contract_is_rejected(self): + entry, records, overlay = self._committed() + for name in package_tool.OVERLAY_FILES_WITHOUT_UPSTREAM: + polluted = list(records) + [ + {"type": "file", "oid": "0" * 40, "size": 1, "path": name} + ] + with self.assertRaises(ValueError) as caught: + package_tool.validate_catalog_provenance( + entry, polluted, overlay + ) + self.assertIn("upstream metadata record", str(caught.exception)) + + def test_unaccounted_upstream_file_is_rejected(self): + entry, records, overlay = self._committed() + extra = list(records) + [ + {"type": "file", "oid": "0" * 40, "size": 7, "path": "surprise.bin"} + ] + with self.assertRaises(ValueError) as caught: + package_tool.validate_catalog_provenance(entry, extra, overlay) + self.assertIn("explicitly excluded", str(caught.exception)) + + def test_excluded_upstream_files_are_not_packaged(self): + entry, records, _ = self._committed() + indexed = {record["path"] for record in records} + for name in package_tool.EXCLUDED_UPSTREAM_FILES: + self.assertIn(name, indexed, f"{name} should exist upstream") + self.assertNotIn(name, entry["files"]) + + def test_overlay_hash_drift_is_rejected(self): + entry, records, overlay = self._committed() + drifted = json.loads(json.dumps(entry)) + drifted["bundled_overlays"]["config.json"]["sha256"] = "0" * 64 + with self.assertRaises(ValueError) as caught: + package_tool.validate_catalog_provenance(drifted, records, overlay) + self.assertIn("does not match its catalog record", str(caught.exception)) + + def test_size_and_footprint_cover_the_assembled_directory(self): + entry, records, overlay = self._committed() + indexed = {record["path"]: record for record in records} + overlays = entry["bundled_overlays"] + expected = sum( + indexed[name]["size"] + for name in entry["files"] + if name not in overlays + ) + sum(record["size"] for record in overlays.values()) + self.assertEqual(entry["size"], expected) + self.assertEqual(entry["footprint"], round(expected / (1024**3), 2)) + + def test_catalog_pins_version_and_rejects_modelscope(self): + entry, _, _ = self._committed() + self.assertEqual(entry["flm_min_version"], "1.0.4") + self.assertEqual(entry["revision"], package_tool.UPSTREAM_COMMIT) + self.assertIs(entry["modelscope_supported"], False) + self.assertNotIn("ms_url", entry) + + def test_catalog_size_matches_a_measured_assembled_directory(self): + # Step 2 forbids hand-estimating the catalog numbers. The entry is + # computed from metadata because nobody assembles a 3 GiB directory to + # write a catalog line, so this pins the two derivations together: if + # they ever disagree, the published number stops describing the + # directory a user actually gets. + entry, records, overlay = self._committed() + indexed = {record["path"]: record for record in records} + overlays = entry["bundled_overlays"] + with tempfile.TemporaryDirectory() as directory: + assembled = Path(directory) + logical_sizes = {} + for name in entry["files"]: + target = assembled / name + if name in overlays: + target.write_bytes((overlay / name).read_bytes()) + else: + # Stand in for the payload; the logical size comes from the + # upstream record, exactly as it does for an LFS pointer. + target.write_bytes(b"") + logical_sizes[name] = indexed[name]["size"] + size, footprint = package_tool.catalog_measurements( + assembled, logical_sizes + ) + self.assertEqual(size, entry["size"]) + self.assertEqual(footprint, entry["footprint"]) + + def test_inlined_chat_template_matches_upstream_jinja(self): + # AutoModel prefers the standalone chat_template.jinja and overwrites + # the tokenizer_config key with it, so the inlined copy is dead unless + # the two are byte-identical. Drift would be silent. + _, _, overlay = self._committed() + package_tool._require_inlined_template_matches_upstream(overlay) + + tokenizer_config = json.loads( + (overlay / "tokenizer_config.json").read_text(encoding="utf-8") + ) + provenance = json.loads( + (overlay / "provenance.json").read_text(encoding="utf-8") + ) + record = provenance["upstream"]["inputs"]["chat_template.jinja"] + encoded = tokenizer_config["chat_template"].encode("utf-8") + self.assertEqual(len(encoded), record["size"]) + self.assertEqual(hashlib.sha256(encoded).hexdigest(), record["sha256"]) + + def test_drifted_inlined_chat_template_is_rejected(self): + _, _, overlay = self._committed() + with tempfile.TemporaryDirectory() as directory: + drifted = Path(directory) / "phi4-mini-it-aie4" + drifted.mkdir() + for name in package_tool.OVERLAY_FILES: + (drifted / name).write_bytes((overlay / name).read_bytes()) + tokenizer_config = json.loads( + (drifted / "tokenizer_config.json").read_text(encoding="utf-8") + ) + # A single trailing newline is enough: it is the kind of edit that + # looks harmless and would be silently discarded at run time. + tokenizer_config["chat_template"] += "\n" + (drifted / "tokenizer_config.json").write_text( + json.dumps(tokenizer_config), encoding="utf-8" + ) + with self.assertRaises(ValueError) as caught: + package_tool._require_inlined_template_matches_upstream(drifted) + self.assertIn("silently ignored", str(caught.exception)) + + def test_genai_config_is_not_carried_through(self): + # MODEL-2: flm.exe runs no ORT or genai graph, and an unused config + # invites a future reader to believe it is authoritative. + entry, _, overlay = self._committed() + self.assertNotIn("genai_config.json", entry["files"]) + self.assertFalse((overlay / "genai_config.json").exists()) + + +class Phi4CatalogWriterTests(unittest.TestCase): + """The catalog is edited in place, not regenerated. + + An earlier version wrote `model_list.json` with + `json.dump(indent=2, sort_keys=True)`. It was semantics-preserving and it + still reformatted all 2,234 lines of the product's most actively edited + file for one added entry -- conflicting with every concurrent model + addition and hiding the one real change from review. These tests fail if + that behaviour comes back, and they check it the way a reviewer would: + by diffing the bytes, not by trusting the writer's own helpers. + """ + + # Deliberately not the output of any `json.dumps` call. The real catalog + # is hand-maintained and carries `"label":[` with no space, `"9b":{`, and + # a line with trailing whitespace; a serialiser normalises all three away. + # A fixture that round-trips cleanly would let a re-dump pass. + FIXTURE = ( + "{\n" + ' "model_path": "models",\n' + ' "models": {\n' + ' "alpha": {\n' + ' "1b": {\n' + ' "name": "Alpha-1B",\n' + ' "label":[\n' + ' "reasoning"\n' + " ],\n" + ' "footprint": 1.0\n' + " }\n" + " }, \n" + ' "phi4-mini-it": {\n' + ' "4b": {\n' + ' "name": "Phi4-mini-Instruct-NPU2",\n' + # Braces and an escaped quote inside a string: a brace-counting + # splice that is not string-aware truncates somebody else's entry + # here, and the damage looks like a hand edit. + ' "url": "https://e.invalid/{a}\\"b\\"",\n' + ' "footprint": 3.4\n' + " }\n" + " },\n" + ' "omega": {\n' + ' "9b":{\n' + ' "name": "Omega-9B"\n' + " }\n" + " }\n" + " }\n" + "}\n" + ) + + ENTRY = {"name": "generated", "footprint": 3.05} + + # `model_info.json` has its own hazards: the value is an ARRAY rather than + # an object, the key sits at the document root, and the committed file has + # NO TRAILING NEWLINE -- which a writer that re-serialises silently adds. + INFO_FIXTURE = ( + "{\n" + ' "alpha:1b": [\n' + " {\n" + ' "type": "file",\n' + ' "path": "config.json"\n' + " }\n" + " ],\n" + ' "phi4-mini-it:4b": [\n' + " {\n" + ' "type": "file",\n' + ' "path": "model.q4nx"\n' + " }\n" + " ]\n" + "}" + ) + + RECORDS = [{"type": "file", "path": "model.onnx"}] + + def _write(self, original: str, entry: dict[str, object]) -> str: + updated, _ = package_tool.updated_catalog_documents( + json.loads(original), + {}, + entry, + [], + ) + with tempfile.TemporaryDirectory() as workspace: + path = Path(workspace) / "model_list.json" + package_tool.write_model_list(path, original, updated) + produced = path.read_bytes().decode("utf-8") + self.assertEqual(json.loads(produced), updated) + return produced + + def _write_info( + self, + original: str, + records: list[dict[str, object]], + ) -> str: + _, updated = package_tool.updated_catalog_documents( + {"models": {}}, + json.loads(original), + {}, + records, + ) + with tempfile.TemporaryDirectory() as workspace: + path = Path(workspace) / "model_info.json" + package_tool.write_model_info(path, original, updated) + produced = path.read_bytes().decode("utf-8") + self.assertEqual(json.loads(produced), updated) + return produced + + def _sole_edit(self, before: str, after: str): + """The one changed line range, asserted to be the only one. + + Computed here from `difflib` rather than from the writer's own + `_strip_catalog_member`, so a bug shared by both cannot hide. + """ + matcher = difflib.SequenceMatcher( + None, + before.splitlines(keepends=True), + after.splitlines(keepends=True), + autojunk=False, + ) + edits = [op for op in matcher.get_opcodes() if op[0] != "equal"] + self.assertEqual(len(edits), 1, f"expected one edit, got {edits}") + return edits[0] + + def test_new_entry_is_the_only_change_to_the_document(self): + produced = self._write(self.FIXTURE, self.ENTRY) + tag, i1, i2, j1, j2 = self._sole_edit(self.FIXTURE, produced) + self.assertEqual(tag, "insert") + inserted = produced.splitlines(keepends=True)[j1:j2] + self.assertIn(' "phi4-mini-it-aie4": {\n', inserted) + # The insert lands after the anchor, not at the end of the catalog. + self.assertLess(i1, len(self.FIXTURE.splitlines())) + self.assertIn('"omega"', "".join(produced.splitlines(True)[j2:])) + # Style, not just structure: 4-space steps and no CRLF. + for line in inserted: + self.assertTrue(line.endswith("\n")) + self.assertNotIn("\r", line) + body = line.rstrip("\n") + self.assertEqual( + (len(body) - len(body.lstrip(" "))) % 4, + 0, + f"indent is not a 4-space step: {line!r}", + ) + + def test_crlf_catalog_is_not_converted_to_lf(self): + original = self.FIXTURE.replace("\n", "\r\n") + produced = self._write(original, self.ENTRY) + self.assertEqual(produced.count("\n"), produced.count("\r\n")) + tag, _, _, j1, j2 = self._sole_edit(original, produced) + self.assertEqual(tag, "insert") + self.assertIn( + ' "phi4-mini-it-aie4": {\r\n', + produced.splitlines(keepends=True)[j1:j2], + ) + + def test_rerun_replaces_in_place_and_leaves_the_rest_alone(self): + # The churn the reviewer flagged returns on the *second* run, when the + # tool reads back its own output, so that is the run under test. + first = self._write(self.FIXTURE, self.ENTRY) + again = self._write(first, self.ENTRY) + self.assertEqual(again, first) + + moved = self._write(first, {"name": "generated", "footprint": 3.06}) + tag, _, _, j1, j2 = self._sole_edit(first, moved) + self.assertEqual(tag, "replace") + self.assertIn( + ' "footprint": 3.06\n', + moved.splitlines(keepends=True)[j1:j2], + ) + + def test_committed_catalog_files_are_fixed_points_of_the_writers(self): + # The product files themselves: re-writing them with the entries they + # already carry must not move a single byte. `model_info.json` is in + # here because it is written by the same code path and would have + # taken the same 3,387-line reformat on the next `refresh-catalog`. + source = Phi4CatalogProvenanceTests.SOURCE + cases = ( + ("model_list.json", package_tool.write_model_list), + ("model_info.json", package_tool.write_model_info), + ) + for name, writer in cases: + with self.subTest(document=name): + original = (source / name).read_bytes().decode("utf-8") + with tempfile.TemporaryDirectory() as workspace: + target = Path(workspace) / name + writer(target, original, json.loads(original)) + self.assertEqual( + target.read_bytes().decode("utf-8"), + original, + ) + + def test_metadata_records_are_appended_without_a_trailing_newline(self): + produced = self._write_info(self.INFO_FIXTURE, self.RECORDS) + tag, _, _, j1, j2 = self._sole_edit(self.INFO_FIXTURE, produced) + self.assertEqual(tag, "insert") + inserted = produced.splitlines(keepends=True)[j1:j2] + self.assertIn(' "phi4-mini-it-aie4:4b": [\n', inserted) + # `_write_json` appends one. The committed file does not have one, and + # a writer that adds it churns the last line of a 3,387-line diff. + self.assertFalse(produced.endswith("\n")) + + def test_metadata_rerun_replaces_the_record_list_in_place(self): + first = self._write_info(self.INFO_FIXTURE, self.RECORDS) + self.assertEqual(self._write_info(first, self.RECORDS), first) + moved = self._write_info( + first, [{"type": "file", "path": "vocab.json"}] + ) + tag, _, _, j1, j2 = self._sole_edit(first, moved) + self.assertEqual(tag, "replace") + self.assertIn( + ' "path": "vocab.json"\n', + moved.splitlines(keepends=True)[j1:j2], + ) + + def test_a_catalog_without_the_anchor_still_gets_one_insert(self): + original = self.FIXTURE.replace("phi4-mini-it", "beta-mini-it") + produced = self._write(original, self.ENTRY) + tag, _, _, j1, j2 = self._sole_edit(original, produced) + self.assertEqual(tag, "insert") + self.assertIn( + ' "phi4-mini-it-aie4": {\n', + produced.splitlines(keepends=True)[j1:j2], + ) + + def test_a_damaged_splice_is_refused_rather_than_written(self): + # The writer's two self-checks are the only thing standing between a + # textual JSON edit and a corrupted catalog, so they are tested by + # breaking the splice rather than by reading them. + updated, _ = package_tool.updated_catalog_documents( + json.loads(self.FIXTURE), {}, self.ENTRY, [] + ) + with tempfile.TemporaryDirectory() as workspace: + path = Path(workspace) / "model_list.json" + with patch.object( + package_tool, + "_splice_member", + lambda text, *rest: text, + ): + with self.assertRaises(ValueError) as caught: + package_tool.write_model_list(path, self.FIXTURE, updated) + self.assertIn("does not parse", str(caught.exception)) + self.assertFalse(path.exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/tests/test_phi4_host_lm_head_reference.py b/tools/tests/test_phi4_host_lm_head_reference.py new file mode 100644 index 00000000..504bb4fe --- /dev/null +++ b/tools/tests/test_phi4_host_lm_head_reference.py @@ -0,0 +1,314 @@ +"""Tests for the `DETERM-1` host LM-head reference. Task 13 Step 9b. + +The script itself can only run against a 3.2 GB model on the AIE4 target, so +what is testable here is everything that decides what the numbers MEAN: + + * the ONNX `MatMulNBits` unpacking, against a hand-built block whose answer + is worked out by hand rather than by a second copy of the same code; + * the BF16 ULP function at the binade edges, which is where a `log2`-based + implementation would silently be off by a factor of two; + * the step selection, which must land on the FIRST diverging step and not + the last one — analysing the last step of a pair that has already emitted + different tokens answers a different question; and + * the three-way classification, including the refusal to classify at all + when the reference and the device disagree by more than rounding. + +A review found the earlier version of this script wired into nothing and +covered by nothing, which is why this file exists. +""" + +from __future__ import annotations + +import unittest + +import numpy as np + +from tools.phi4_host_lm_head_reference import ( + BIAS_MEAN_SIGNED_ULP, + SANITY_MAX_MEAN_ULP, + bf16_ulp, + classify, + dequantise_rows, + first_diverging_step, + lm_head_input, + lm_head_input_source, + step_labels, + step_logits, + widen_bf16, +) + + +def _bf16_bits(value: float) -> int: + """The BF16 pattern of an exactly representable float.""" + raw = np.float32(value).view(np.uint32) + assert raw & 0xFFFF == 0, f"{value} is not exactly representable in BF16" + return int(raw >> 16) + + +class DequantisationTests(unittest.TestCase): + """corelib.h: `qweight` is UINT4 codes [N, K/2], two per byte, LOW NIBBLE + FIRST; `qzeros` is UINT4 codes, two per byte; the value is + `(q - zero) * scale`.""" + + def test_nibble_order_and_zero_point(self): + # One output row, K = 8, two groups of 4. Codes chosen so every nibble + # is distinguishable and the expected answer is arithmetic anyone can + # check: group 0 codes 1..4 with zero 8 and scale 2, group 1 codes + # 9..12 with zero 3 and scale 0.5. + k, n, group = 8, 1, 4 + qweight = np.array( + [ + # group 0: elements 0,1 then 2,3 -> low nibble first + (2 << 4) | 1, + (4 << 4) | 3, + # group 1 + (10 << 4) | 9, + (12 << 4) | 11, + ], + dtype=np.uint8, + ) + # Two groups, so one byte of zero points: group 0 low, group 1 high. + qzeros = np.array([(3 << 4) | 8], dtype=np.uint8) + scales = np.array([2.0, 0.5], dtype=np.float16) + + weights = dequantise_rows( + qweight, scales, qzeros, 0, 1, k=k, n=n, group_size=group + ) + expected = np.array( + [ + (1 - 8) * 2.0, + (2 - 8) * 2.0, + (3 - 8) * 2.0, + (4 - 8) * 2.0, + (9 - 3) * 0.5, + (10 - 3) * 0.5, + (11 - 3) * 0.5, + (12 - 3) * 0.5, + ], + dtype=np.float32, + ) + np.testing.assert_array_equal(weights[0], expected) + + def test_a_swapped_nibble_order_would_not_pass(self): + # The guard the previous test is really providing: if the unpacking + # took the HIGH nibble first, element 0 would be 2 rather than 1. + k, n, group = 4, 1, 4 + qweight = np.array([(2 << 4) | 1, (4 << 4) | 3], dtype=np.uint8) + qzeros = np.array([0], dtype=np.uint8) + scales = np.array([1.0], dtype=np.float16) + weights = dequantise_rows( + qweight, scales, qzeros, 0, 1, k=k, n=n, group_size=group + ) + self.assertEqual(weights[0][0], 1.0) + self.assertEqual(weights[0][1], 2.0) + + def test_multiple_output_rows_are_sliced_independently(self): + k, n, group = 4, 3, 4 + qweight = np.array( + [(2 << 4) | 1, (4 << 4) | 3] * 3, dtype=np.uint8 + ) + qweight[2] = (6 << 4) | 5 + qzeros = np.array([0, 0, 0], dtype=np.uint8) + scales = np.array([1.0, 1.0, 1.0], dtype=np.float16) + rows = dequantise_rows( + qweight, scales, qzeros, 1, 2, k=k, n=n, group_size=group + ) + self.assertEqual(rows.shape, (2, 4)) + self.assertEqual(rows[0][0], 5.0) + + +class Bf16UlpTests(unittest.TestCase): + def test_binade_edges(self): + # BF16 has 8 significand bits, so one ULP at magnitude m in [2^e, + # 2^(e+1)) is 2^(e-7). + for magnitude, expected in ( + (1.0, 2.0**-7), + (1.9921875, 2.0**-7), + (2.0, 2.0**-6), + (16.0, 2.0**-3), + (31.75, 2.0**-3), + (32.0, 2.0**-2), + (40.0, 2.0**-2), + ): + self.assertEqual( + float(bf16_ulp(np.float32([magnitude]))[0]), + expected, + f"magnitude {magnitude}", + ) + + def test_the_units_error_the_design_had_to_correct(self): + # Design 15.3: an absolute 0.25 bound is 2 ULP at magnitude 20 and + # only 1 ULP at 40, which is the whole reason the bound is stated in + # ULP. If this function were wrong that correction would be undone. + self.assertEqual(float(bf16_ulp(np.float32([20.0]))[0]) * 2, 0.25) + self.assertEqual(float(bf16_ulp(np.float32([40.0]))[0]) * 2, 0.5) + + def test_zero_does_not_produce_a_meaningless_bound(self): + self.assertGreater(float(bf16_ulp(np.float32([0.0]))[0]), 0.0) + + def test_widen_bf16_round_trips(self): + bits = [_bf16_bits(1.5), _bf16_bits(-20.0), _bf16_bits(0.0)] + np.testing.assert_array_equal( + widen_bf16(bits), np.float32([1.5, -20.0, 0.0]) + ) + + +def _document(logits_per_step, inputs_per_step=None): + steps = len(logits_per_step) + inputs_per_step = inputs_per_step or [[0] * 4] * steps + document = { + "continuation": { + "logits_bf16": logits_per_step[0], + "lm_head_input_bf16": inputs_per_step[0], + }, + "decode": [ + { + "logits_bf16": logits_per_step[index], + "lm_head_input_bf16": inputs_per_step[index], + } + for index in range(1, steps) + ], + } + return document + + +class StepSelectionTests(unittest.TestCase): + """The bug this replaces: the tool always analysed the LAST decode step. + + For a pair that has already emitted different tokens, the last step's + hidden states legitimately differ and the comparison answers nothing. The + question lives at the FIRST diverging step. + """ + + def test_labels_and_accessors_line_up(self): + document = _document([[1], [2], [3]]) + self.assertEqual( + step_labels(document), ["continuation", "decode[0]", "decode[1]"] + ) + self.assertEqual(step_logits(document, 0), [1]) + self.assertEqual(step_logits(document, 2), [3]) + self.assertEqual(lm_head_input(document, 2), [0, 0, 0, 0]) + + def test_the_first_diverging_step_is_found_not_the_last(self): + left = _document([[1], [2], [3], [4]]) + right = _document([[1], [9], [9], [9]]) + self.assertEqual(first_diverging_step([left, right]), 1) + + def test_divergence_in_the_continuation_step_is_index_zero(self): + left = _document([[1], [2]]) + right = _document([[7], [2]]) + self.assertEqual(first_diverging_step([left, right]), 0) + + def test_identical_runs_have_no_diverging_step(self): + left = _document([[1], [2]]) + self.assertIsNone(first_diverging_step([left, _document([[1], [2]])])) + + def test_a_single_document_has_no_diverging_step(self): + self.assertIsNone(first_diverging_step([_document([[1]])])) + + def test_a_missing_input_at_the_last_step_falls_back_to_the_snapshot(self): + # Runs predating the per-step capture carry the same row in + # final_snapshot.last_hidden -- the same tensor read at the same + # moment -- but only for the LAST step, because the snapshot is taken + # once at the end. + document = _document([[1], [2]]) + del document["decode"][0]["lm_head_input_bf16"] + document["final_snapshot"] = {"last_hidden": [7, 7, 7, 7]} + self.assertEqual(lm_head_input(document, 1), [7, 7, 7, 7]) + self.assertEqual( + lm_head_input_source(document, 1), "final_snapshot_fallback" + ) + + def test_the_fallback_does_not_apply_to_an_earlier_step(self): + # Applying it anywhere the field is missing would answer a different + # question at every index but the last. + document = _document([[1], [2], [3]]) + del document["decode"][0]["lm_head_input_bf16"] + document["final_snapshot"] = {"last_hidden": [7, 7, 7, 7]} + self.assertIsNone(lm_head_input(document, 1)) + + def test_a_recorded_input_is_preferred_over_the_snapshot(self): + document = _document([[1], [2]], [[1, 1, 1, 1], [5, 5, 5, 5]]) + document["final_snapshot"] = {"last_hidden": [9, 9, 9, 9]} + self.assertEqual(lm_head_input(document, 1), [5, 5, 5, 5]) + self.assertEqual( + lm_head_input_source(document, 1), "per_step_capture" + ) + + +def _run(label, mean_signed_ulp=0.0, mean_abs_ulp=0.25): + return { + "label": label, + "mean_signed_deviation_ulp": mean_signed_ulp, + "mean_abs_deviation_ulp": mean_abs_ulp, + } + + +class ClassificationTests(unittest.TestCase): + def test_benign_accumulation_order(self): + straddle = { + "applicable": True, + "differing_logits": 100, + "reference_between": 100, + "both_within_half_ulp": 100, + } + verdict, _ = classify([_run("a"), _run("b")], straddle) + self.assertEqual(verdict, "benign_accumulation_order") + + def test_one_run_systematically_further(self): + verdict, notes = classify( + [_run("a", 0.0, 0.25), _run("b", 0.0, 0.9)], None + ) + self.assertEqual(verdict, "one_run_further") + self.assertTrue(notes) + + def test_a_common_bias(self): + offset = BIAS_MEAN_SIGNED_ULP + 0.1 + verdict, notes = classify( + [_run("a", offset, 0.3), _run("b", offset, 0.3)], None + ) + self.assertEqual(verdict, "common_bias") + self.assertTrue(notes) + + def test_opposite_offsets_are_not_a_common_bias(self): + offset = BIAS_MEAN_SIGNED_ULP + 0.1 + verdict, _ = classify( + [_run("a", offset, 0.3), _run("b", -offset, 0.3)], None + ) + self.assertNotEqual(verdict, "common_bias") + + def test_gross_disagreement_refuses_to_classify(self): + # A wrong nibble order in this script would land here, and reporting + # "common bias" about the device instead would be a false alarm about + # hardware caused by a bug in the analysis. + worst = SANITY_MAX_MEAN_ULP + 1 + verdict, notes = classify( + [_run("a", 5.0, worst), _run("b", 5.0, worst)], None + ) + self.assertEqual(verdict, "gross_disagreement") + self.assertIn("dequantisation", notes[0]) + + def test_different_lm_head_inputs_are_a_model_body_divergence(self): + straddle = { + "applicable": False, + "lm_head_input_differing_elements": 2587, + "reason": "the two runs fed the LM head DIFFERENT rows", + } + verdict, notes = classify([_run("a"), _run("b")], straddle) + self.assertEqual(verdict, "model_body_divergence") + self.assertTrue(notes) + + def test_a_bit_identical_pair_says_there_is_nothing_to_attribute(self): + straddle = { + "applicable": True, + "differing_logits": 0, + "reference_between": 0, + "both_within_half_ulp": 0, + } + verdict, notes = classify([_run("a"), _run("b")], straddle) + self.assertEqual(verdict, "benign_no_divergence_in_sample") + self.assertIn("bit-identical", notes[0]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/tests/test_report_phi4_corelib_baseline.py b/tools/tests/test_report_phi4_corelib_baseline.py new file mode 100644 index 00000000..46a43e96 --- /dev/null +++ b/tools/tests/test_report_phi4_corelib_baseline.py @@ -0,0 +1,1689 @@ +"""Tests for the Phi-4 AIE4 performance/memory baseline validator. + +Task 13 Steps 1, 2 and 10. + +The validator's job is narrow and worth stating, because it is easy to build +the wrong thing here. Design section 4 makes performance explicitly NOT a +release blocker, and design section 15.6 says to record the numbers "without +pass/fail thresholds". So this module must never gate on a latency or a +throughput figure. What it DOES gate on: + + * every required section and identity field is present -- a baseline whose + machine, corelib DLL hash or model hash is missing cannot be compared + against anything later, which is the only reason the baseline exists; + * the 128-token memory window is stable, which IS a correctness property + (design 18.7, and design 15.4's "at least 128 decode tokens with stable + post-warm allocation"); and + * the `DETERM-3` bit-identity baseline states its rate, its spread and the n + it rests on, with at least the 20 runs per route design `DETERM-3` + requires. + +That last one is the reason this file is careful about a shape that looks +harmless: a "baseline" that reports a floor without the sample size behind it. +`DETERM-2` names "a bit-identity rate that degrades from the recorded +baseline" as a failure condition, and `DETERM-3` says in as many words that a +floor derived from a handful of runs is the unmeasured number `DETERM-2` +forbids. A validator that accepted such a record would make the absence +invisible, which is worse than the absence. +""" + +from __future__ import annotations + +import io +import json +import tempfile +import unittest +from contextlib import redirect_stderr, redirect_stdout +from pathlib import Path + +from tools import report_phi4_corelib_baseline as report +from tools.report_phi4_corelib_baseline import ( + MIN_DETERMINISM_RUNS_PER_ROUTE, + crossover_edge_stability, + crossover_entry, + crossover_stability_narrative, + merge_crossover_history, + ROUTE_DEPENDENCE_ALPHA, + _fisher_exact_two_sided, + REQUIRED_IDENTITY_FIELDS, + REQUIRED_SECTIONS, + determinism_baseline, + memory_is_stable, + render_markdown, + validate, +) + + +_MARKDOWN_STUB = "---\ntitle: Phi4\n---\n\n## Existing content\n" + + +def _identity() -> dict: + return { + "machine": "xcomedusad-43", + "cpu_sku": "AMD Eng Sample: 100-000001713-33_N", + "npu_sku": "AMD XDNA(TM) NPU", + "npu_driver_version": "32.0.20214.4161", + "corelib_dll_path": "C:/Users/chiz/work/aie4-runtime-mirrored/" + "ryzenai_corelib.dll", + "corelib_dll_sha256": "a523b238" + "0" * 56, + "corelib_version": "0.1.0", + "corelib_source_revision": "e5258d29b5cb979d4a538994409b90ceff6e6e7a", + "dynamic_dispatch_version": "1.2.3.4", + "ryzen_mm_version": "1.0.0.0", + "xrt_version": "2.19.0", + "model_dir": "C:/models/phi-4", + "model_sha256": "c48fd647" + "0" * 56, + "fastflow_revision": "084e2c4d", + "utc": "2026-09-02T12:00:00Z", + } + + +def _memory(**overrides) -> dict: + record = { + "device_tensor_creates_after_warmup": 0, + "weight_creates_after_warmup": 0, + "live_corelib_object_delta": 0, + "private_bytes_growth": 2 * 1024 * 1024, + "private_bytes_slope_per_token": 1024, + "peak_private_bytes": 6 * 1024**3, + "peak_working_set_bytes": 5 * 1024**3, + "embedding_bytes": 1229193216, + "kv_bytes": 536870912, + "packed_weight_bytes": 2 * 1024**3, + "scratch_bytes": 1024**3, + "mapped_source_bytes": 3248488448, + "samples": [], + } + record.update(overrides) + return record + + +def _baseline(**overrides) -> dict: + document = { + "identity": _identity(), + "model_load": { + "manifest_map_ns": 1, + "shape_plan_ns": 90, + "weight_pack_ns": 2, + "device_setup_ns": 5, + "unaccounted_ns": 2, + "shape_plan_share": 0.9, + "total_ns": 100, + "weight_objects": 161, + "device_tensors": 76, + }, + "ttft": { + "cold_ns": 10, + "warm_ns": 5, + "prompt_token_count": 12, + "prompt_id": "phi4_tokens.prefix", + "row_extent": 12, + "stream_rebuilt_for_warm": False, + }, + "prefill": {"points": [{"rows": 1, "ns": 100, "tokens_per_second": 10.0}]}, + "continuation": { + "samples_interleaved": True, + "points": [ + { + "history_rows": 512, + "suffix": 1, + "route": "append", + "samples_ns": [1, 2, 3, 4, 5], + "sample_starts_ns": [0, 200, 400, 600, 800], + "drift_ns": 4, + "p50_ns": 3, + "p95_ns": 5, + }, + { + "history_rows": 512, + "suffix": 1, + "route": "reprefill", + "samples_ns": [9, 9, 9, 9, 9], + "sample_starts_ns": [100, 300, 500, 700, 900], + "drift_ns": 0, + "p50_ns": 9, + "p95_ns": 9, + }, + ], + "crossover": { + "512": { + "append_wins_up_to": 8, + "reprefill_wins_from": 12, + "crossover_bracket": [8, 12], + "bracket_is_tight": False, + "decisions": [ + { + "suffix": 8, + "decided": True, + "gap_over_uncertainty": 3.9, + }, + { + "suffix": 12, + "decided": True, + "gap_over_uncertainty": 19.3, + }, + ], + }, + "2048": { + "append_wins_up_to": 24, + "reprefill_wins_from": 32, + "crossover_bracket": [24, 32], + "bracket_is_tight": False, + "decisions": [ + { + "suffix": 24, + "decided": True, + "gap_over_uncertainty": 5.2, + }, + { + "suffix": 32, + "decided": False, + "gap_over_uncertainty": 0.4, + }, + ], + }, + }, + "decision_rule": "gap must exceed the within-route spread", + "prefix_monotonic": True, + }, + "decode": { + "runs": [ + { + "start_context": 128, + "tokens": 128, + "tokens_per_second": 20.0, + "synchronizes_per_pass": 129, + "per_token_ns": [1] * 128, + "p50_ns": 1, + "p95_ns": 1, + } + ] + }, + "memory": _memory(), + "v_scatter": { + "reads_per_model_step": 32, + "writes_per_model_step": 256, + "counts_are_measured": True, + "bytes": 1024, + "nanoseconds": 2048, + }, + } + document.update(overrides) + return document + + +def _determ(route: str, bit_exact: int, total: int, max_diff: float) -> dict: + return { + "route": route, + "corelib_sha256": "a523b238" + "0" * 56, + "harness_sha256": "beefcafe" + "0" * 56, + "logits_bit_exact_steps": bit_exact, + "logits_total_steps": total, + "observed_max_abs_diff": max_diff, + "determ2_bound_ulps": 2, + "determ2_bound_kind": "relative_bf16_ulp", + "first_divergence": None if bit_exact == total else "decode[13]", + "failures": [], + } + + +def _determ_runs(route: str, count: int) -> list[dict]: + return [_determ(route, 17, 17, 0.0) for _ in range(count)] + + +class MemoryStabilityTests(unittest.TestCase): + """Step 2. The one gate in this file that is about correctness.""" + + def test_a_bounded_allocator_fluctuation_is_accepted(self): + self.assertTrue(memory_is_stable(_memory())) + + def test_a_400_kib_per_token_leak_is_rejected(self): + # 400 KiB/token over the 120-token measurement window is ~47 MiB, so + # both the slope and the total growth are out of bounds. A validator + # that only checked one of them would accept a leak that stayed under + # the other, so the test drives both from the same leak rather than + # constructing two independent violations. + leak_per_token = 400 * 1024 + self.assertFalse( + memory_is_stable( + _memory( + private_bytes_slope_per_token=leak_per_token, + private_bytes_growth=leak_per_token * 120, + ) + ) + ) + + def test_the_slope_alone_rejects(self): + self.assertFalse( + memory_is_stable(_memory(private_bytes_slope_per_token=400 * 1024)) + ) + + def test_the_growth_alone_rejects(self): + self.assertFalse( + memory_is_stable(_memory(private_bytes_growth=9 * 1024 * 1024)) + ) + + def test_a_new_device_tensor_after_warmup_rejects(self): + self.assertFalse( + memory_is_stable(_memory(device_tensor_creates_after_warmup=1)) + ) + + def test_a_new_weight_object_after_warmup_rejects(self): + self.assertFalse(memory_is_stable(_memory(weight_creates_after_warmup=1))) + + def test_a_live_object_increase_rejects(self): + self.assertFalse(memory_is_stable(_memory(live_corelib_object_delta=1))) + + def test_the_boundary_values_are_accepted(self): + # Design 15.4 wants "stable", and the bounds are the ones Step 2 of the + # task brief fixes. Exactly at the bound is stable; one byte over is + # not. Pinning this stops a later ">=" edit from silently loosening it. + self.assertTrue( + memory_is_stable( + _memory( + private_bytes_growth=8 * 1024 * 1024, + private_bytes_slope_per_token=64 * 1024, + ) + ) + ) + self.assertFalse( + memory_is_stable(_memory(private_bytes_growth=8 * 1024 * 1024 + 1)) + ) + self.assertFalse( + memory_is_stable(_memory(private_bytes_slope_per_token=64 * 1024 + 1)) + ) + + def test_a_missing_field_is_not_silently_stable(self): + record = _memory() + del record["live_corelib_object_delta"] + # An absent counter must not read as zero. That is the shape this + # project has now found seven times: a check that reports success for + # work it did not do. + with self.assertRaises(KeyError): + memory_is_stable(record) + + +class SchemaTests(unittest.TestCase): + """Step 1.""" + + def test_a_complete_document_validates(self): + self.assertEqual(validate(_baseline()), []) + + def test_every_required_section_is_required(self): + for section in sorted(REQUIRED_SECTIONS): + document = _baseline() + del document[section] + problems = validate(document) + self.assertTrue( + any(section in problem for problem in problems), + f"removing {section!r} produced {problems!r}", + ) + + def test_the_required_section_set_is_the_one_the_brief_fixes(self): + self.assertEqual( + REQUIRED_SECTIONS, + { + "identity", + "model_load", + "ttft", + "prefill", + "continuation", + "decode", + "memory", + "v_scatter", + }, + ) + + def test_every_required_identity_field_is_required(self): + for field in sorted(REQUIRED_IDENTITY_FIELDS): + document = _baseline() + del document["identity"][field] + problems = validate(document) + self.assertTrue( + any(field in problem for problem in problems), + f"removing identity.{field} produced {problems!r}", + ) + + def test_an_empty_identity_value_is_not_a_value(self): + # "" and None are what a collector writes when it could not find the + # thing. Accepting them would record a baseline that names a machine + # nobody can identify. + for empty in ("", None): + document = _baseline() + document["identity"]["corelib_dll_sha256"] = empty + self.assertTrue( + any( + "corelib_dll_sha256" in problem + for problem in validate(document) + ), + f"an identity value of {empty!r} was accepted", + ) + + def test_the_identity_set_covers_what_the_brief_names(self): + for field in ( + "machine", + "cpu_sku", + "npu_sku", + "corelib_dll_sha256", + "dynamic_dispatch_version", + "ryzen_mm_version", + "xrt_version", + "model_sha256", + "fastflow_revision", + ): + self.assertIn(field, REQUIRED_IDENTITY_FIELDS) + + def test_an_unstable_memory_window_fails_validation(self): + document = _baseline() + document["memory"]["private_bytes_slope_per_token"] = 400 * 1024 + problems = validate(document) + self.assertTrue(any("memory" in problem for problem in problems)) + + def test_latency_and_throughput_are_recorded_not_gated(self): + # Design section 4: performance is not a release blocker. A validator + # that failed on a slow number would convert a recorded figure into a + # release threshold, which is precisely what 15.6 says not to do. + document = _baseline() + document["decode"]["runs"][0]["tokens_per_second"] = 0.001 + document["prefill"]["points"][0]["tokens_per_second"] = 0.001 + document["ttft"]["cold_ns"] = 10**12 + self.assertEqual(validate(document), []) + + def test_the_v_scatter_counts_are_the_design_contract(self): + # 32 reads and 256 writes per model step is design 10.3/18.5, and it + # IS a contract rather than a measurement. + document = _baseline() + document["v_scatter"]["writes_per_model_step"] = 255 + self.assertTrue(any("v_scatter" in p for p in validate(document))) + + def test_the_synchronize_count_is_the_design_contract(self): + document = _baseline() + document["decode"]["runs"][0]["synchronizes_per_pass"] = 2 + self.assertTrue(any("synchronize" in p for p in validate(document))) + + +class DeterminismBaselineTests(unittest.TestCase): + """`DETERM-3`. The highest-value part of this task, and the easiest to + get wrong by stating a confident number.""" + + def test_twenty_clean_runs_per_route_produce_a_baseline(self): + records = _determ_runs("append", 20) + _determ_runs("reprefill", 20) + baseline = determinism_baseline(records) + self.assertEqual(baseline["problems"], []) + for route in ("append", "reprefill"): + entry = baseline["routes"][route] + self.assertEqual(entry["runs"], 20) + self.assertEqual(entry["bit_identical_runs"], 20) + self.assertEqual(entry["step_bit_identity_rate"], 1.0) + self.assertEqual(entry["observed_max_abs_diff"]["max"], 0.0) + + def test_fewer_than_twenty_runs_is_reported_as_no_baseline(self): + records = _determ_runs("append", 3) + _determ_runs("reprefill", 20) + baseline = determinism_baseline(records) + self.assertTrue( + any("append" in problem for problem in baseline["problems"]), + baseline["problems"], + ) + # And it must not quietly publish a rate for the short route as if it + # were a baseline. + self.assertFalse(baseline["routes"]["append"]["is_baseline"]) + self.assertTrue(baseline["routes"]["reprefill"]["is_baseline"]) + + def test_a_route_with_no_runs_at_all_is_a_problem(self): + baseline = determinism_baseline(_determ_runs("append", 20)) + self.assertTrue( + any("reprefill" in problem for problem in baseline["problems"]), + baseline["problems"], + ) + + def test_the_minimum_is_the_one_determ3_states(self): + self.assertEqual(MIN_DETERMINISM_RUNS_PER_ROUTE, 20) + + def test_every_route_figure_carries_its_n(self): + records = _determ_runs("append", 20) + _determ_runs("reprefill", 21) + baseline = determinism_baseline(records) + for route, entry in baseline["routes"].items(): + self.assertIn("runs", entry, route) + self.assertIn("steps", entry, route) + self.assertIn("step_bit_identity_rate", entry, route) + self.assertIn("observed_max_abs_diff", entry, route) + + def test_routes_are_reported_separately_and_never_averaged(self): + # DETERM-3: "If the rate turns out to be route-dependent, say so + # rather than averaging". A pooled figure would hide exactly the + # mechanism difference the two routes exist to expose. + records = [_determ("append", 16, 17, 0.25) for _ in range(20)] + records += _determ_runs("reprefill", 20) + baseline = determinism_baseline(records) + self.assertNotIn("step_bit_identity_rate", baseline) + self.assertLess( + baseline["routes"]["append"]["step_bit_identity_rate"], + baseline["routes"]["reprefill"]["step_bit_identity_rate"], + ) + self.assertTrue(baseline["route_dependent"]) + + def test_identical_rates_are_not_reported_as_route_dependent(self): + records = _determ_runs("append", 20) + _determ_runs("reprefill", 20) + self.assertFalse(determinism_baseline(records)["route_dependent"]) + + def test_the_max_abs_diff_distribution_is_reported_not_just_its_peak(self): + records = [_determ("append", 16, 17, 0.25) for _ in range(10)] + records += [_determ("append", 16, 17, 0.5) for _ in range(10)] + records += _determ_runs("reprefill", 20) + entry = determinism_baseline(records)["routes"]["append"] + spread = entry["observed_max_abs_diff"] + self.assertEqual(spread["max"], 0.5) + self.assertEqual(spread["min"], 0.25) + self.assertEqual(spread["histogram"], {"0.25": 10, "0.5": 10}) + self.assertEqual(spread["nonzero_runs"], 20) + + def test_a_run_that_recorded_failures_is_counted_and_flagged(self): + # A run whose self-consistency check FAILED is exactly the run a rate + # baseline most needs to count. Dropping it would bias the figure + # upward, and silently. + failed = _determ("append", 16, 17, 4.0) + failed["failures"] = ["decode[3]: 12 logits differ by more than 2 ULP"] + records = [failed] + _determ_runs("append", 19) + _determ_runs( + "reprefill", 20 + ) + baseline = determinism_baseline(records) + entry = baseline["routes"]["append"] + self.assertEqual(entry["runs"], 20) + self.assertEqual(entry["runs_with_gate_failures"], 1) + self.assertTrue( + any("gate" in problem.lower() for problem in baseline["problems"]), + baseline["problems"], + ) + # A gate failure is a FINDING, not a structural defect in the + # baseline. It must not block the record from being written; see the + # blocking/non-blocking split in determinism_baseline. + self.assertEqual(baseline["blocking_problems"], []) + + def test_runs_that_loaded_different_libraries_are_flagged(self): + odd = _determ("append", 17, 17, 0.0) + odd["corelib_sha256"] = "deadbeef" + "0" * 56 + records = [odd] + _determ_runs("append", 19) + _determ_runs( + "reprefill", 20 + ) + baseline = determinism_baseline(records) + self.assertTrue( + any("sha-256" in problem.lower() for problem in baseline["problems"]), + baseline["problems"], + ) + + def test_no_records_at_all_is_a_problem_not_an_empty_success(self): + baseline = determinism_baseline([]) + self.assertTrue(baseline["problems"]) + self.assertFalse(baseline["is_baseline"]) + # A zero-run route is still listed, so the report shows what is + # missing by name rather than by omission -- but nothing about it may + # read as a measured figure. + for route, entry in baseline["routes"].items(): + self.assertFalse(entry["is_baseline"], route) + self.assertEqual(entry["runs"], 0, route) + self.assertIsNone(entry["step_bit_identity_rate"], route) + + +class RenderingTests(unittest.TestCase): + def test_the_markdown_states_the_rate_the_spread_and_the_n(self): + document = _baseline() + document["determinism"] = determinism_baseline( + _determ_runs("append", 20) + _determ_runs("reprefill", 20) + ) + text = render_markdown(document) + self.assertIn("DETERM-3", text) + self.assertIn("n = 20", text) + self.assertIn("xcomedusad-43", text) + self.assertIn("a523b238", text) + + def test_the_markdown_never_states_a_floor_without_its_n(self): + document = _baseline() + document["determinism"] = determinism_baseline( + _determ_runs("append", 3) + _determ_runs("reprefill", 3) + ) + text = render_markdown(document) + self.assertIn("no baseline", text.lower()) + self.assertIn("3", text) + + +class MainTests(unittest.TestCase): + def _run(self, argv): + out, err = io.StringIO(), io.StringIO() + with redirect_stdout(out), redirect_stderr(err): + code = report.main(argv) + return code, out.getvalue() + err.getvalue() + + def _fixture(self, directory: Path, runs: int) -> tuple[Path, str, Path]: + baseline = directory / "phi4_aie4_baseline.json" + baseline.write_text(json.dumps(_baseline()), encoding="utf-8") + artifacts = directory / "artifacts" + for index in range(runs): + run_dir = artifacts / f"20260902T00{index:04d}Z-1" + run_dir.mkdir(parents=True) + for route in ("append", "reprefill"): + name = "force_append" if route == "append" else "force_reprefill" + (run_dir / f"determ1-{name}.json").write_text( + json.dumps(_determ(route, 17, 17, 0.0)), encoding="utf-8" + ) + markdown = directory / "phi4_results.md" + markdown.write_text( + "---\ntitle: Phi4\n---\n\n## Existing content\n", encoding="utf-8" + ) + return baseline, str(artifacts / "*" / "determ1-*.json"), markdown + + def test_a_complete_run_writes_the_markdown_and_exits_zero(self): + with tempfile.TemporaryDirectory() as raw: + directory = Path(raw) + baseline, glob, markdown = self._fixture(directory, 20) + code, output = self._run( + [ + "--input", + str(baseline), + "--determinism-glob", + glob, + "--markdown", + str(markdown), + ] + ) + self.assertEqual(code, 0, output) + text = markdown.read_text(encoding="utf-8") + self.assertIn("## Existing content", text) + self.assertIn("DETERM-3", text) + self.assertIn("n = 20", text) + + def test_rendering_twice_does_not_duplicate_the_section(self): + with tempfile.TemporaryDirectory() as raw: + directory = Path(raw) + baseline, glob, markdown = self._fixture(directory, 20) + argv = [ + "--input", + str(baseline), + "--determinism-glob", + glob, + "--markdown", + str(markdown), + ] + self.assertEqual(self._run(argv)[0], 0) + first = markdown.read_text(encoding="utf-8") + self.assertEqual(self._run(argv)[0], 0) + second = markdown.read_text(encoding="utf-8") + self.assertEqual(first, second) + + def test_an_incomplete_determinism_campaign_fails_the_exit_code(self): + # Skipped work must reach the exit code. Three runs per route is the + # state DETERM-3 describes as "no baseline exists", and a tool that + # rendered a document anyway would publish the thing DETERM-3 forbids. + with tempfile.TemporaryDirectory() as raw: + directory = Path(raw) + baseline, glob, markdown = self._fixture(directory, 3) + before = markdown.read_text(encoding="utf-8") + code, output = self._run( + [ + "--input", + str(baseline), + "--determinism-glob", + glob, + "--markdown", + str(markdown), + ] + ) + self.assertNotEqual(code, 0) + self.assertIn("DETERM-3", output) + self.assertEqual(markdown.read_text(encoding="utf-8"), before) + + def test_a_gate_failure_is_published_and_still_fails_the_exit_code(self): + # The distinction this test pins: "there is no baseline" blocks the + # render, "the baseline contains a hard-gate failure" does not. + # Suppressing the whole record because the campaign found something + # would discard the measurement and leave only an exit code, which is + # the opposite of what this baseline is for. + with tempfile.TemporaryDirectory() as raw: + directory = Path(raw) + baseline, glob, markdown = self._fixture(directory, 20) + broken = _determ("append", 0, 17, 48.34375) + broken["failures"] = ["decode[7]: emitted token sequences differ"] + first = sorted((directory / "artifacts").glob("*"))[0] + (first / "determ1-force_append.json").write_text( + json.dumps(broken), encoding="utf-8" + ) + code, output = self._run( + [ + "--input", + str(baseline), + "--determinism-glob", + glob, + "--markdown", + str(markdown), + ] + ) + self.assertNotEqual(code, 0) + self.assertIn("gate", output.lower()) + text = markdown.read_text(encoding="utf-8") + self.assertIn("DETERM-3", text) + self.assertIn("HARD GATE", text) + self.assertIn("n = 20", text) + + def test_an_unstable_memory_window_fails_the_exit_code(self): + with tempfile.TemporaryDirectory() as raw: + directory = Path(raw) + baseline, glob, markdown = self._fixture(directory, 20) + document = json.loads(baseline.read_text(encoding="utf-8")) + document["memory"]["private_bytes_slope_per_token"] = 400 * 1024 + baseline.write_text(json.dumps(document), encoding="utf-8") + code, output = self._run( + [ + "--input", + str(baseline), + "--determinism-glob", + glob, + "--markdown", + str(markdown), + ] + ) + self.assertNotEqual(code, 0) + self.assertIn("memory", output) + + def test_a_determinism_glob_matching_nothing_fails(self): + with tempfile.TemporaryDirectory() as raw: + directory = Path(raw) + baseline, _, markdown = self._fixture(directory, 20) + code, output = self._run( + [ + "--input", + str(baseline), + "--determinism-glob", + str(directory / "nowhere" / "*.json"), + "--markdown", + str(markdown), + ] + ) + self.assertNotEqual(code, 0) + self.assertIn("no DETERM-1 records", output) + + def test_the_merged_json_carries_the_determinism_section(self): + with tempfile.TemporaryDirectory() as raw: + directory = Path(raw) + baseline, glob, markdown = self._fixture(directory, 20) + merged = directory / "merged.json" + code, output = self._run( + [ + "--input", + str(baseline), + "--determinism-glob", + glob, + "--markdown", + str(markdown), + "--output-json", + str(merged), + ] + ) + self.assertEqual(code, 0, output) + document = json.loads(merged.read_text(encoding="utf-8")) + self.assertEqual( + document["determinism"]["routes"]["append"]["runs"], 20 + ) + + +class RouteDependenceTests(unittest.TestCase): + """I-6. "The rate is route-dependent" is a claim, and it needs a test. + + It was previously a float inequality on two ratios, published in bold from + 1 divergent run of 33 against 2 of 33 -- the same n-limited overclaim + DETERM-3 exists to prevent, made about DETERM-3's own output. + """ + + def test_fisher_matches_known_values(self): + # Symmetric table: nothing to distinguish, p is exactly 1. + self.assertEqual(_fisher_exact_two_sided((2, 31), (2, 31)), 1.0) + # A table with an empty margin cannot be extreme. + self.assertEqual(_fisher_exact_two_sided((0, 5), (0, 5)), 1.0) + # Textbook 2x2: ((1,9),(8,2)) has two-sided p = 0.0055 to 2 s.f. + self.assertAlmostEqual( + _fisher_exact_two_sided((1, 9), (8, 2)), 0.005477, places=5 + ) + self.assertLessEqual(_fisher_exact_two_sided((3, 30), (0, 33)), 1.0) + + def test_one_versus_two_divergent_runs_of_33_is_not_route_dependence(self): + records = _determ_runs("append", 32) + [_determ("append", 0, 17, 48.3)] + records += _determ_runs("reprefill", 31) + [ + _determ("reprefill", 15, 17, 0.3125), + _determ("reprefill", 9, 17, 0.25), + ] + baseline = determinism_baseline(records) + self.assertTrue(baseline["observed_rates_differ"]) + self.assertFalse(baseline["route_dependent"]) + self.assertGreaterEqual( + baseline["route_dependence_p"], ROUTE_DEPENDENCE_ALPHA + ) + + def test_a_large_separation_is_route_dependence(self): + records = _determ_runs("append", 33) + records += _determ_runs("reprefill", 21) + [ + _determ("reprefill", 9, 17, 0.25) for _ in range(12) + ] + baseline = determinism_baseline(records) + self.assertTrue(baseline["route_dependent"]) + self.assertLess( + baseline["route_dependence_p"], ROUTE_DEPENDENCE_ALPHA + ) + + def test_the_document_says_the_n_does_not_support_the_claim(self): + document = _baseline() + records = _determ_runs("append", 32) + [_determ("append", 0, 17, 48.3)] + records += _determ_runs("reprefill", 33) + document["determinism"] = determinism_baseline(records) + text = render_markdown(document) + self.assertIn("does not support", text) + self.assertNotIn("**The rate is route-dependent**", text) + + +class BaselineFlagTests(unittest.TestCase): + """I-3. `is_baseline` is the machine-readable form of the sentence four + lines below it, and it used to contradict it.""" + + def test_a_gate_failure_makes_is_baseline_false(self): + broken = _determ("append", 0, 17, 48.34375) + broken["failures"] = ["decode[7]: emitted token sequences differ"] + records = [broken] + _determ_runs("append", 32) + _determ_runs( + "reprefill", 33 + ) + baseline = determinism_baseline(records) + self.assertEqual(baseline["gate_failures"], 1) + self.assertFalse(baseline["is_baseline"]) + # Per-route run counts are still met, so the ROUTE flag stays true; + # it answers "does this route have enough runs", not "is this a + # settled baseline". + self.assertTrue(baseline["routes"]["append"]["is_baseline"]) + + def test_a_clean_window_is_a_baseline(self): + records = _determ_runs("append", 33) + _determ_runs("reprefill", 33) + baseline = determinism_baseline(records) + self.assertEqual(baseline["gate_failures"], 0) + self.assertTrue(baseline["is_baseline"]) + + +class BinaryIdentityTests(unittest.TestCase): + """I-7. Both halves of "the same binary", and an auditable source list.""" + + def test_a_differing_harness_hash_blocks_the_baseline(self): + odd = _determ("append", 17, 17, 0.0) + odd["harness_sha256"] = "deadbeef" + "0" * 56 + records = [odd] + _determ_runs("append", 32) + _determ_runs( + "reprefill", 33 + ) + baseline = determinism_baseline(records) + self.assertTrue( + any( + "harness" in problem.lower() + for problem in baseline["blocking_problems"] + ), + baseline["blocking_problems"], + ) + + def test_a_record_with_no_harness_hash_is_excluded_and_counted(self): + # A record predating the field cannot be shown to belong to the pool, + # and silently including it is how a glob over a build tree turns into + # a baseline nobody can reproduce. It is dropped, not blocked: + # refusing to produce a baseline at all while such a record exists + # anywhere under the glob would make the committed DETERM-4 evidence + # permanently unusable. + stale = _determ("append", 0, 17, 48.0) + stale["_source"] = "old/determ1-force_append-010.json" + del stale["harness_sha256"] + records = [stale] + _determ_runs("append", 33) + _determ_runs( + "reprefill", 33 + ) + baseline = determinism_baseline(records) + self.assertEqual(baseline["blocking_problems"], []) + self.assertTrue( + any( + "EXCLUDED" in problem for problem in baseline["problems"] + ), + baseline["problems"], + ) + # And it really is excluded: 33 append runs counted, not 34, and the + # divergent one is not dragged into the rate. + self.assertEqual(baseline["routes"]["append"]["runs"], 33) + self.assertEqual(baseline["routes"]["append"]["run_bit_identity_rate"], 1.0) + self.assertEqual( + baseline["excluded_sources"], + ["old/determ1-force_append-010.json"], + ) + + def test_the_sources_are_recorded(self): + records = _determ_runs("append", 33) + _determ_runs("reprefill", 33) + for index, record in enumerate(records): + record["_source"] = f"artifacts/run/determ1-{index}.json" + baseline = determinism_baseline(records) + self.assertEqual(len(baseline["sources"]), 66) + self.assertIn("artifacts/run/determ1-0.json", baseline["sources"]) + + +class CrossoverRenderingTests(unittest.TestCase): + """C-1. The document published a grid artifact as a measured threshold.""" + + def test_the_document_publishes_a_bracket_and_says_so(self): + text = render_markdown(_baseline()) + self.assertIn("BRACKET, not a threshold", text) + self.assertIn("(8, 12]", text) + self.assertIn("(24, 32]", text) + # And never the old phrasing, which read as a measured answer. + self.assertNotIn("Longest suffix at which append beats", text) + + def test_the_bracket_edges_carry_their_margin(self): + # "Decisively" is a word until it is a number, and the edges are + # exactly where a reader should be able to check the call. + text = render_markdown(_baseline()) + self.assertIn("margin at each bracket edge", text) + self.assertIn("8: 3.9x", text) + self.assertIn("24: 5.2x", text) + + def test_undecided_points_are_counted_in_the_document(self): + text = render_markdown(_baseline()) + self.assertIn("3 of 4 sweep points were decided; 1 were not", text) + + def test_an_unbracketed_crossover_says_so_rather_than_naming_a_number(self): + document = _baseline() + document["continuation"]["crossover"]["512"][ + "reprefill_wins_from" + ] = 0 + text = render_markdown(document) + self.assertIn("not bracketed by this grid", text) + + def test_the_load_breakdown_names_the_shape_plan(self): + text = render_markdown(_baseline()) + self.assertIn("Phi4ShapePlan::Build", text) + self.assertIn("90% of load", text) + self.assertIn("unaccounted", text) + + +class VScatterProvenanceTests(unittest.TestCase): + """I-4. The per-step counts must be measured, not restated constants.""" + + def test_a_record_that_does_not_claim_measurement_is_rejected(self): + document = _baseline() + del document["v_scatter"]["counts_are_measured"] + self.assertTrue( + any("counts_are_measured" in p for p in validate(document)), + validate(document), + ) + + def test_a_record_that_claims_measurement_falsely_is_rejected(self): + document = _baseline() + document["v_scatter"]["counts_are_measured"] = False + self.assertTrue( + any("counts_are_measured" in p for p in validate(document)) + ) + + +def _determ_with_localisation(route, step, differing): + """A record whose localisation was measured. + + `differing == 0` is the `lm_head` case -- the two runs fed the LM head an + identical row and it produced different logits, which is what `DETERM-1` + describes. Anything else is `model_body`. The source is DERIVED here, not + hardcoded, because a fixture that could only produce one of them is how + the pooled-sentence defect survived: every test agreed with the generator + about which case existed. + """ + record = _determ(route, 9, 17, 49.25) + record["failures"] = [f"{step}: logits differ by more than 2 BF16 ULP"] + record["first_divergence"] = step + record["localisation"] = { + "measured": True, + "step": step, + "lm_head_input_elements": 3072, + "lm_head_input_differing_elements": differing, + "source": "lm_head" if differing == 0 else "model_body", + "measured_by": "per_step_lm_head_input_capture", + "reason": "measured at the first diverging step", + } + return record + + +class LocalisationPublicationTests(unittest.TestCase): + """The finding, not just the rate. + + A reader of the benchmark document was learning that 2 runs in 41 were not + bit-identical, and nothing about the thing that matters: the two runs fed + the LM head different rows, so the divergence is upstream of it. That was + in the task report, the records README and the design spec, and in none of + the places a reader of the benchmarks would look. + """ + + def _document(self): + records = _determ_runs("append", 39) + [ + _determ_with_localisation("append", "decode[8]", 2571), + _determ_with_localisation("append", "decode[6]", 2754), + ] + records += _determ_runs("reprefill", 41) + document = _baseline() + document["determinism"] = determinism_baseline(records) + return document + + def test_the_localisation_is_aggregated_per_route(self): + entry = self._document()["determinism"]["routes"]["append"][ + "localisation" + ] + self.assertEqual(entry["measured_runs"], 2) + self.assertEqual(entry["by_source"], {"model_body": 2}) + self.assertEqual( + entry["elements_by_source"], {"model_body": [2571, 2754]} + ) + self.assertEqual( + entry["steps_by_source"], + {"model_body": ["decode[6]", "decode[8]"]}, + ) + + def test_an_unmeasured_localisation_is_not_counted(self): + # Records predating the per-step capture carry no localisation. They + # must not be counted as evidence of anything. + records = _determ_runs("append", 41) + _determ_runs("reprefill", 41) + entry = determinism_baseline(records)["routes"]["append"][ + "localisation" + ] + self.assertEqual(entry["measured_runs"], 0) + self.assertEqual(entry["by_source"], {}) + + def test_the_document_states_the_finding(self): + text = render_markdown(self._document()) + self.assertIn("Where the divergence enters", text) + self.assertIn("model_body", text) + self.assertIn("2,571", text) + self.assertIn("2,754", text) + self.assertIn("the divergence enters the model body", text) + + def test_the_instrumentation_caveat_is_in_the_document(self): + # N-2. The section asserts "an observation and not an inference" and + # said nothing about the instrument that adds a host read and a stream + # acquisition between every step. A reader of this document is exactly + # who needs that. + text = render_markdown(self._document()) + self.assertIn("instrument perturbs what it measures", text) + self.assertIn("host tensor read", text) + + def test_no_caveat_when_nothing_was_measured(self): + document = _baseline() + document["determinism"] = determinism_baseline( + _determ_runs("append", 41) + _determ_runs("reprefill", 41) + ) + self.assertNotIn( + "instrument perturbs", render_markdown(document) + ) + + def test_the_document_says_the_layer_is_unknown(self): + text = render_markdown(self._document()) + self.assertIn("layer at which it enters is not known", text) + + def test_a_document_with_no_measured_localisation_says_nothing(self): + document = _baseline() + document["determinism"] = determinism_baseline( + _determ_runs("append", 41) + _determ_runs("reprefill", 41) + ) + text = render_markdown(document) + self.assertNotIn("Where the divergence enters", text) + + +class DecisionRuleReconciliationTests(unittest.TestCase): + """Two live rules, one contradicting the other, is C-1's failure mode in a + subtler form: the reader cannot tell which number to act on.""" + + def test_the_2x_rule_is_scoped_to_cross_run_comparisons(self): + text = render_markdown(_baseline()) + self.assertIn("Two different comparisons", text) + self.assertIn("roughly 2x as unresolved", text) + # And it must be visibly scoped, not stated unconditionally. + self.assertIn("DIFFERENT run", text) + + def test_the_crossover_blurb_names_the_rule_it_actually_uses(self): + text = render_markdown(_baseline()) + self.assertIn("interleaved within each point", text) + self.assertIn("drift between a route's first and last", text) + + +class MixedLocalisationWindowTests(unittest.TestCase): + """N-1. The generator stated a FALSE finding the moment an `lm_head` event + appeared. + + It gated the detail paragraph on the presence of a `model_body` event and + then pooled the differing-element counts and steps across ALL sources, + asserting "in every measured event the two runs fed the LM head different + rows". An `lm_head` event has `differing == 0` by construction -- it is + precisely the `DETERM-1`-supporting case -- so the document would list a + zero among the differing counts and then assert a conclusion the same + window contradicts. + """ + + def _mixed(self): + records = _determ_runs("append", 39) + [ + _determ_with_localisation("append", "decode[8]", 2571), + _determ_with_localisation("append", "decode[3]", 0), + ] + records += _determ_runs("reprefill", 41) + document = _baseline() + document["determinism"] = determinism_baseline(records) + return document + + def test_the_two_sources_are_counted_apart(self): + entry = self._mixed()["determinism"]["routes"]["append"][ + "localisation" + ] + self.assertEqual(entry["by_source"], {"lm_head": 1, "model_body": 1}) + self.assertEqual( + entry["elements_by_source"], {"lm_head": [0], "model_body": [2571]} + ) + self.assertEqual( + entry["steps_by_source"], + {"lm_head": ["decode[3]"], "model_body": ["decode[8]"]}, + ) + + def test_the_zero_is_never_listed_among_the_differing_counts(self): + text = self._render() + # The sentence naming the differing counts must contain 2,571 and + # must NOT have picked up the lm_head event's zero. + line = next( + line for line in text.splitlines() if "different rows" in line + ) + self.assertIn("2,571", line) + self.assertNotIn("0, 2,571", line) + self.assertNotIn("2,571, 0", line) + + def _render(self): + return render_markdown(self._mixed()) + + def test_the_model_body_claim_is_scoped_to_its_own_events(self): + text = self._render() + self.assertIn("In the **1** `model_body` event(s)", text) + self.assertIn("In those events the divergence enters", text) + # And never the unscoped universal that was false. + self.assertNotIn("In every measured event", text) + + def test_the_lm_head_events_are_reported_and_not_absorbed(self): + text = self._render() + self.assertIn("`lm_head` event(s)", text) + self.assertIn("identical", text) + self.assertIn("Both are", text) + self.assertIn("neither explanation covers", text) + + def test_a_pure_lm_head_window_supports_determ1_instead(self): + records = _determ_runs("append", 40) + [ + _determ_with_localisation("append", "decode[3]", 0) + ] + records += _determ_runs("reprefill", 41) + document = _baseline() + document["determinism"] = determinism_baseline(records) + text = render_markdown(document) + self.assertIn("divergence is inside the LM-head dispatch", text) + self.assertNotIn("enters the model body", text) + # And the layer paragraph, which only makes sense for a body + # divergence, must not appear. + self.assertNotIn("layer at which it enters", text) + + # THE EVIDENCE ITSELF, which the first version silently dropped. + # + # The trailing conditional bound to the whole concatenation rather + # than the last clause, so with no model_body event the count and the + # steps were discarded and the paragraph began with a bare period. + # This test used to pass over that because it asserted only the + # trailing substring. + self.assertIn("In the **1** `lm_head` event(s)", text) + self.assertIn("`decode[3]`", text) + paragraph = next( + line + for line in text.splitlines() + if "LM-head dispatch" in line and "DETERM-1" in line + ) + self.assertFalse( + paragraph.startswith("."), + f"the paragraph lost its leading sentence: {paragraph!r}", + ) + self.assertTrue(paragraph.startswith("In the "), paragraph) + + +class InterleavingAttestationTests(unittest.TestCase): + """N-3. The interleaving claim is checked against the timeline, not taken + from a boolean. + + The rendered document states the samples are interleaved and that each + point beat its own drift. Neither was required of the record, so + re-rendering an older baseline would publish the claim over + non-interleaved data — in the section that decides the crossover Task 14 + consumes. + """ + + def test_a_complete_continuation_validates(self): + self.assertEqual(validate(_baseline()), []) + + def test_the_flag_is_required(self): + document = _baseline() + del document["continuation"]["samples_interleaved"] + self.assertTrue( + any("samples_interleaved" in p for p in validate(document)), + validate(document), + ) + + def test_a_false_flag_is_rejected(self): + document = _baseline() + document["continuation"]["samples_interleaved"] = False + self.assertTrue( + any("samples_interleaved" in p for p in validate(document)) + ) + + def test_the_drift_term_is_required(self): + document = _baseline() + del document["continuation"]["points"][0]["drift_ns"] + self.assertTrue( + any("drift_ns" in p for p in validate(document)), + validate(document), + ) + + def test_the_sample_starts_are_required(self): + document = _baseline() + del document["continuation"]["points"][1]["sample_starts_ns"] + self.assertTrue( + any("sample_starts_ns" in p for p in validate(document)) + ) + + def test_a_true_flag_over_block_sampled_data_is_caught(self): + # THE CASE THE FLAG ALONE CANNOT CATCH: all five append samples, then + # all five re-prefill samples, with the boolean still saying true. + # This is the shape the round-1 benchmark actually produced. + document = _baseline() + document["continuation"]["points"][0]["sample_starts_ns"] = [ + 0, + 100, + 200, + 300, + 400, + ] + document["continuation"]["points"][1]["sample_starts_ns"] = [ + 500, + 600, + 700, + 800, + 900, + ] + problems = validate(document) + self.assertTrue( + any("do not alternate" in p for p in problems), problems + ) + self.assertTrue(any("aaaaarrrrr" in p for p in problems), problems) + + def test_mismatched_sample_counts_are_caught(self): + document = _baseline() + document["continuation"]["points"][1]["sample_starts_ns"] = [100, 300] + self.assertTrue( + any("cannot have been interleaved" in p for p in validate(document)) + ) + + def test_one_swapped_pair_is_caught(self): + # A single inversion is enough to break the paired-in-time assumption + # the decision rule rests on, and it must not be rounded away. + document = _baseline() + document["continuation"]["points"][0]["sample_starts_ns"] = [ + 0, + 350, + 400, + 600, + 800, + ] + self.assertTrue( + any("do not alternate" in p for p in validate(document)) + ) + + +class ValidatorUsabilityTests(unittest.TestCase): + """A present-but-unusable field must fail, not skip. + + `field not in point` accepts a null, so a record with every + `sample_starts_ns` set to null validated clean — the skip-instead-of-fail + shape this task closed twice elsewhere, reintroduced in the check added to + close it. + """ + + def test_null_sample_starts_are_rejected(self): + document = _baseline() + for point in document["continuation"]["points"]: + point["sample_starts_ns"] = None + problems = validate(document) + self.assertTrue( + any("sample_starts_ns" in p for p in problems), problems + ) + + def test_empty_sample_starts_are_rejected(self): + document = _baseline() + document["continuation"]["points"][0]["sample_starts_ns"] = [] + self.assertTrue( + any("sample_starts_ns" in p for p in validate(document)) + ) + + def test_non_integer_sample_starts_are_rejected(self): + document = _baseline() + document["continuation"]["points"][0]["sample_starts_ns"] = [ + 0, + "200", + 400, + 600, + 800, + ] + self.assertTrue( + any("non-integer" in p for p in validate(document)) + ) + + def test_null_drift_is_rejected(self): + document = _baseline() + document["continuation"]["points"][0]["drift_ns"] = None + self.assertTrue(any("drift_ns" in p for p in validate(document))) + + def test_a_point_with_only_one_route_is_rejected(self): + document = _baseline() + document["continuation"]["points"] = [ + document["continuation"]["points"][0] + ] + problems = validate(document) + self.assertTrue( + any("has no reprefill entry" in p for p in problems), problems + ) + + +class CrossoverHistoryTests(unittest.TestCase): + """Task 14 reads the document, not the task report. + + The document published `(4, 12]` and `(12, 64]` with margins beside them + and never said the upper edge had been 8 and 16 one run earlier. A reader + takes an edge with a margin as measured; one of these is, and the other + moved by a factor of four between runs of the same binary. + """ + + def _entry(self, utc, lower512, upper512, lower2048, upper2048, + interleaved=True, undecided=()): + document = _baseline() + document["identity"]["utc"] = utc + document["continuation"]["samples_interleaved"] = interleaved + document["continuation"]["crossover"] = { + "512": { + "append_wins_up_to": lower512, + "reprefill_wins_from": upper512, + "decisions": [ + {"suffix": 4, "decided": True}, + {"suffix": 8, "decided": 8 not in undecided}, + ], + }, + "2048": { + "append_wins_up_to": lower2048, + "reprefill_wins_from": upper2048, + "decisions": [{"suffix": 12, "decided": True}], + }, + } + return crossover_entry(document, source=f"test:{utc}") + + def test_an_entry_reduces_a_document_to_its_edges(self): + entry = self._entry("2026-09-02T15:34:01Z", 4, 8, 12, 16) + self.assertEqual(entry["edges"], {"512": [4, 8], "2048": [12, 16]}) + self.assertEqual(entry["points_decided"], 3) + self.assertEqual(entry["points_total"], 3) + self.assertTrue(entry["samples_interleaved"]) + + def test_undecided_points_are_recorded_by_history(self): + entry = self._entry( + "2026-09-02T16:32:18Z", 4, 12, 12, 64, undecided=(8,) + ) + self.assertEqual(entry["undecided_suffixes"], {"512": [8]}) + self.assertEqual(entry["points_decided"], 2) + + def test_merging_the_same_run_twice_adds_one_observation(self): + entry = self._entry("2026-09-02T15:34:01Z", 4, 8, 12, 16) + history = merge_crossover_history([], entry) + history = merge_crossover_history(history, entry) + self.assertEqual(len(history), 1) + + def test_history_is_ordered_by_measurement_time(self): + late = self._entry("2026-09-02T16:32:18Z", 4, 12, 12, 64) + early = self._entry("2026-09-02T15:34:01Z", 4, 8, 12, 16) + history = merge_crossover_history( + merge_crossover_history([], late), early + ) + self.assertEqual( + [item["utc"] for item in history], + ["2026-09-02T15:34:01Z", "2026-09-02T16:32:18Z"], + ) + + def test_stability_sees_a_moving_upper_edge(self): + history = [ + self._entry("2026-09-02T15:34:01Z", 4, 8, 12, 16), + self._entry("2026-09-02T16:32:18Z", 4, 12, 12, 64), + ] + stability = crossover_edge_stability(history) + self.assertTrue(stability["lower_is_stable"]) + self.assertFalse(stability["upper_is_stable"]) + self.assertEqual(stability["upper_observed"]["2048"], [16, 64]) + self.assertEqual(stability["lower_stable"]["512"], [4]) + + def test_non_interleaved_runs_do_not_vote(self): + history = [ + self._entry( + "2026-09-02T14:55:44Z", 9, 9, 9, 9, interleaved=False + ), + self._entry("2026-09-02T15:34:01Z", 4, 8, 12, 16), + self._entry("2026-09-02T16:32:18Z", 4, 12, 12, 64), + ] + stability = crossover_edge_stability(history) + self.assertEqual(stability["runs_considered"], 2) + self.assertEqual(stability["lower_stable"]["512"], [4]) + + def test_the_document_publishes_the_history_and_the_guidance(self): + document = _baseline() + document["crossover_history"] = [ + self._entry("2026-09-02T15:34:01Z", 4, 8, 12, 16), + self._entry( + "2026-09-02T16:32:18Z", 4, 12, 12, 64, undecided=(8,) + ), + ] + text = render_markdown(document) + self.assertIn("The same measurement, run to run", text) + self.assertIn("2026-09-02T15:34:01Z", text) + self.assertIn("Read the lower edge as measured", text) + self.assertIn("upper edge as an upper bound", text) + # Prose, not a Python list repr. The document used to print + # "has taken the values [16, 64] at history 2048". + self.assertIn("16, 64 at history 2048", text) + self.assertNotIn("[16, 64]", text) + + def test_a_single_run_publishes_no_stability_claim(self): + document = _baseline() + document["crossover_history"] = [ + self._entry("2026-09-02T15:34:01Z", 4, 8, 12, 16) + ] + self.assertNotIn( + "The same measurement, run to run", render_markdown(document) + ) + + def test_the_undecided_points_are_named_in_the_document(self): + # "Four were undecided" tells a reader nothing about where the + # measurement ran out. + document = _baseline() + document["continuation"]["crossover"]["512"]["decisions"] = [ + {"suffix": 8, "decided": False, "gap_over_uncertainty": 0.6}, + {"suffix": 12, "decided": True, "gap_over_uncertainty": 19.3}, + ] + text = render_markdown(document) + self.assertIn("Undecided points", text) + self.assertIn("history 512 at suffix 8", text) + + def test_the_2x_exemption_is_scoped_to_a_single_point(self): + text = render_markdown(_baseline()) + self.assertIn("exemption is for a single point, not for the bracket", text) + # And the sentence that granted it unconditionally is gone. + self.assertNotIn("measured and subtracted rather than assumed away", text) + + +class InstrumentCaveatWordingTests(unittest.TestCase): + def test_the_withdrawn_rate_claim_is_not_republished(self): + # Report section 26 withdrew "a rate consistent with the + # uninstrumented campaigns"; this fix round had republished it as a + # hardcoded literal into the reader-facing document, where it carried + # no correction and no supporting data. + records = _determ_runs("append", 40) + [ + _determ_with_localisation("append", "decode[8]", 2571) + ] + records += _determ_runs("reprefill", 41) + document = _baseline() + document["determinism"] = determinism_baseline(records) + text = render_markdown(document) + self.assertIn("instrument perturbs what it measures", text) + self.assertNotIn("rate consistent with the uninstrumented", text) + self.assertIn("with the same coarse signature", text) + + +class CrossoverNarrativeTests(unittest.TestCase): + """EVERY stability case, driven by the test rather than assumed. + + The defect this covers: the bold lead-in "the bracket's upper edge is not + stable and its lower edge is" was emitted whenever more than one run + existed, independent of the stability computation, while only the guidance + beneath it was conditional. A future run whose LOWER edge moved would have + printed that sentence above a table showing it moved, AND lost the + guidance entirely because neither branch fired. + + The reviewer's note is the one that matters: the tests kept agreeing with + the generator about which case exists. That is the third time in this + task. So this class enumerates all four cases and asserts both halves -- + the lead-in matching the data, and a guidance paragraph being present -- + for each of them. + """ + + def _stability(self, lower, upper, runs=2): + return { + "runs_considered": runs, + "lower_stable": lower, + "upper_observed": upper, + "lower_is_stable": all(len(v) == 1 for v in lower.values()) + and bool(lower), + "upper_is_stable": all(len(v) == 1 for v in upper.values()) + and bool(upper), + } + + def test_every_case_produces_a_lead_in_and_a_guidance(self): + cases = [ + ("lower stable, upper not", {"512": [4]}, {"512": [8, 12]}, 2), + ("both stable", {"512": [4]}, {"512": [8]}, 2), + ("lower moved, upper stable", {"512": [4, 8]}, {"512": [12]}, 2), + ("neither stable", {"512": [4, 8]}, {"512": [12, 64]}, 2), + ("too few runs", {}, {}, 1), + ] + for label, lower, upper, runs in cases: + lead, guidance = crossover_stability_narrative( + self._stability(lower, upper, runs) + ) + self.assertTrue(lead.strip(), label) + self.assertTrue(guidance.strip(), label) + self.assertTrue( + guidance.lstrip().startswith(">"), + f"{label}: guidance is not a blockquote: {guidance!r}", + ) + + def test_a_moving_lower_edge_is_never_called_stable(self): + lead, guidance = crossover_stability_narrative( + self._stability({"512": [4, 8]}, {"512": [12]}) + ) + self.assertIn("LOWER edge has moved", lead) + self.assertNotIn("lower edge is.**", lead) + self.assertIn("Do not read the lower edge as measured", guidance) + + def test_both_stable_does_not_say_the_upper_is_unstable(self): + lead, guidance = crossover_stability_narrative( + self._stability({"512": [4]}, {"512": [8]}) + ) + self.assertIn("Both edges have held", lead) + self.assertNotIn("not stable", lead) + self.assertIn("agreement rather than proof", guidance) + + def test_neither_stable_refuses_the_bracket(self): + lead, guidance = crossover_stability_narrative( + self._stability({"512": [4, 8]}, {"512": [12, 64]}) + ) + self.assertIn("Neither edge", lead) + self.assertIn("not resolved by this data", guidance) + + def test_one_run_says_so_rather_than_claiming_stability(self): + lead, guidance = crossover_stability_narrative( + self._stability({}, {}, runs=1) + ) + self.assertNotIn("stable", lead) + self.assertIn("Not enough interleaved runs", guidance) + + +class CrossoverNarrativeRenderTests(unittest.TestCase): + """The same four cases, through the real renderer. + + Testing the pure function is not enough: the defect was that the renderer + emitted its own unconditional sentence beside it. + """ + + def _document(self, edges_per_run): + document = _baseline() + history = [] + for index, edges in enumerate(edges_per_run): + history.append( + { + "utc": f"2026-09-02T1{index}:00:00Z", + "samples_interleaved": True, + "suffix_grid": [1, 2, 4], + "edges": edges, + "points_decided": 3, + "points_total": 3, + "undecided_suffixes": {}, + } + ) + document["crossover_history"] = history + return document + + def test_a_moving_lower_edge_renders_honestly(self): + # THE CASE THAT WOULD HAVE SHIPPED A FALSE CLAIM: the lower edge moves + # on a future hardware run appended to this history. + # The lower edge moves and the upper HOLDS: 4 -> 8 with 12 both + # times. Writing this fixture the obvious way -- (4, 8] then (8, 12] + # -- moves both edges and lands in the "neither is stable" case, which + # is a different sentence. The first draft of this test did exactly + # that and failed, which is the test doing its job rather than + # agreeing with the generator. + text = render_markdown( + self._document([{"512": [4, 12]}, {"512": [8, 12]}]) + ) + self.assertIn("LOWER edge has moved", text) + self.assertNotIn( + "upper edge is not stable and its lower edge is", text + ) + # And the guidance must NOT vanish, which is what used to happen. + self.assertIn("Do not read the lower edge as measured", text) + + def test_both_stable_renders_without_contradiction(self): + text = render_markdown( + self._document([{"512": [4, 8]}, {"512": [4, 8]}]) + ) + self.assertIn("Both edges have held", text) + self.assertNotIn( + "upper edge is not stable and its lower edge is", text + ) + + def test_neither_stable_renders_honestly(self): + text = render_markdown( + self._document([{"512": [4, 8]}, {"512": [8, 64]}]) + ) + self.assertIn("Neither edge of the bracket is stable", text) + self.assertIn("not resolved by this data", text) + + def test_the_todays_case_still_renders_the_guidance(self): + text = render_markdown( + self._document([{"512": [4, 8]}, {"512": [4, 12]}]) + ) + self.assertIn( + "upper edge is not stable and its lower edge is", text + ) + self.assertIn("Read the lower edge as measured", text) + + +class BooleanIsNotANumberTests(unittest.TestCase): + """`isinstance(True, int)` is True, so a JSON `true` satisfied every + numeric check added to stop unusable fields validating clean.""" + + def test_a_true_drift_is_rejected(self): + document = _baseline() + document["continuation"]["points"][0]["drift_ns"] = True + self.assertTrue( + any("drift_ns" in p for p in validate(document)), + validate(document), + ) + + def test_a_true_sample_start_is_rejected(self): + document = _baseline() + document["continuation"]["points"][0]["sample_starts_ns"] = [ + 0, + True, + 400, + 600, + 800, + ] + self.assertTrue( + any("non-integer" in p for p in validate(document)) + ) + + +class HistoryPersistenceTests(unittest.TestCase): + """A run whose document was never published must not become a permanent + observation that later runs are compared against.""" + + def _fixture(self, directory: Path, valid: bool): + baseline = directory / "phi4_aie4_baseline.json" + document = _baseline() + document["continuation"]["crossover"] = { + "512": { + "append_wins_up_to": 4, + "reprefill_wins_from": 8, + "decisions": [{"suffix": 4, "decided": True}], + } + } + if not valid: + document["memory"]["private_bytes_slope_per_token"] = 400 * 1024 + baseline.write_text(json.dumps(document), encoding="utf-8") + artifacts = directory / "artifacts" + for index in range(20): + run_dir = artifacts / f"20260902T00{index:04d}Z-1" + run_dir.mkdir(parents=True) + for route in ("append", "reprefill"): + name = "force_append" if route == "append" else "force_reprefill" + (run_dir / f"determ1-{name}.json").write_text( + json.dumps(_determ(route, 17, 17, 0.0)), encoding="utf-8" + ) + markdown = directory / "phi4_results.md" + markdown.write_text(_MARKDOWN_STUB, encoding="utf-8") + return baseline, str(artifacts / "*" / "determ1-*.json"), markdown + + def _run(self, argv): + out, err = io.StringIO(), io.StringIO() + with redirect_stdout(out), redirect_stderr(err): + code = report.main(argv) + return code, out.getvalue() + err.getvalue() + + def test_a_failing_document_does_not_append_to_the_history(self): + with tempfile.TemporaryDirectory() as raw: + directory = Path(raw) + baseline, glob, markdown = self._fixture(directory, valid=False) + history = directory / "history.json" + code, _ = self._run( + [ + "--input", str(baseline), + "--determinism-glob", glob, + "--markdown", str(markdown), + "--crossover-history", str(history), + ] + ) + self.assertNotEqual(code, 0) + self.assertFalse( + history.exists(), + "a document that failed validation appended to the history", + ) + + def test_a_passing_document_appends_once(self): + with tempfile.TemporaryDirectory() as raw: + directory = Path(raw) + baseline, glob, markdown = self._fixture(directory, valid=True) + history = directory / "history.json" + argv = [ + "--input", str(baseline), + "--determinism-glob", glob, + "--markdown", str(markdown), + "--crossover-history", str(history), + ] + self.assertEqual(self._run(argv)[0], 0) + self.assertEqual( + len(json.loads(history.read_text(encoding="utf-8"))), 1 + ) + self.assertEqual(self._run(argv)[0], 0) + self.assertEqual( + len(json.loads(history.read_text(encoding="utf-8"))), 1 + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/tests/test_windows_dependency_linkage.py b/tools/tests/test_windows_dependency_linkage.py new file mode 100644 index 00000000..136bbdae --- /dev/null +++ b/tools/tests/test_windows_dependency_linkage.py @@ -0,0 +1,581 @@ +"""The Windows non-vcpkg Boost linkage decision, read back out of src/CMakeLists.txt. + +Why this file exists rather than a comment. + +Before the AIE4 branch, `src/CMakeLists.txt` made ONE decision about Boost on +the `WIN32 AND NOT VCPKG_TOOLCHAIN` path and made it in one place: it defined +`CURL_STATICLIB` / `BOOST_ALL_NO_LIB` / `BOOST_ALL_STATIC_LINK` unconditionally +and linked `libboost_program_options-vc143-mt-x64-1_88` by name. b2 encodes +linkage in the file name -- a `lib` prefix is the static archive, a bare +`boost_` name is the import library for a DLL -- so the macros and the name +said the same thing, and the shipped flm.exe had no Boost DLL dependency at +all. That is why `src/lib` vendors no `boost_program_options.dll` and +`src/wix/get_files.bat` stages none. + +Commit 0db2c3ac split that one decision into two independent ones -- an +`option(FLM_WIN_STATIC_DEPS ... OFF)` choosing the macros, and a search list +choosing the name -- and set them to disagree with the shipping configuration: +the option is set ON nowhere in the tree, no preset sets a `toolchainFile` (so +`VCPKG_TOOLCHAIN` is false on every Windows preset), and CI runs +`cmake --preset windows-vs18` bare. The shipping binary's Boost linkage flipped +from static to shared with no diff anyone read as saying so, and the resulting +MSI was missing a DLL nothing in the repository produces. + +Nothing offline could have caught that, because the defect is not in any single +line: every line was defensible, and the pair was wrong. So the assertions here +are about the RELATIONSHIP between the two halves -- + + * the option's default is the configuration CI and the MSI actually ship; + * for EACH value of the option, the macro set and the first-searched library + name describe the same linkage; and + * a configure-time `FATAL_ERROR` exists to catch the residual case the source + cannot rule out, where the search lands on the other variant anyway. + +-- plus a non-vacuity test, because a parser that quietly finds nothing would +make every one of those pass. + +Set FLM_TEST_WIN_LINKAGE_CMAKELISTS to point the parse at a scratch copy; that +is how these assertions are themselves verified against mutated inputs. +FLM_TEST_WIN_LINKAGE_PRESETS does the same for src/CMakePresets.json. + +WHY THE PRESET FILE IS READ HERE TOO +------------------------------------ +The assertions above are about `option(FLM_WIN_STATIC_DEPS ... ON)`, and a +default is only the shipping value while nothing overrides it. CI runs +`cmake --preset windows-vs18`, so a single `"FLM_WIN_STATIC_DEPS": "OFF"` in a +`cacheVariables` block -- on that preset or on anything it inherits from -- +re-breaks CRIT-3 exactly, with the option line untouched and every assertion +above still green. That vector was demonstrated and this file did not see it, +because it never opened CMakePresets.json. It does now. The same goes for a +later `set(FLM_WIN_STATIC_DEPS OFF ...)` in CMakeLists.txt, which reaches the +same end without touching the `option()` call either. +""" + +from __future__ import annotations + +import json +import os +import pathlib +import re +import unittest + +_REPO_ROOT = pathlib.Path(__file__).resolve().parents[2] +_DEFAULT_CMAKELISTS = _REPO_ROOT / "src" / "CMakeLists.txt" +_DEFAULT_PRESETS = _REPO_ROOT / "src" / "CMakePresets.json" + +# CMake's false constants, which is what a re-break would spell. +_CMAKE_FALSE = {"OFF", "FALSE", "N", "NO", "0", "", "IGNORE", "NOTFOUND"} + +# b2's spellings. A `lib` prefix is the static archive; the bare name is the +# import library for boost_program_options.dll. +_STATIC_NAME = re.compile(r"^libboost_program_options") +_SHARED_NAME = re.compile(r"^boost_program_options$") + +_STATIC_MACROS = {"CURL_STATICLIB", "BOOST_ALL_NO_LIB", "BOOST_ALL_STATIC_LINK"} +_SHARED_MACROS = {"BOOST_ALL_NO_LIB", "BOOST_ALL_DYN_LINK"} + + +class CMakeParseError(AssertionError): + """Raised when the file no longer has the shape this test reads. + + Deliberately an AssertionError: a restructure that defeats the parse is a + test failure, not a silent pass. + """ + + +def _strip_comments(text: str) -> str: + """Drop whole-line CMake comments. + + Needed because the shared branch explains itself by NAMING the macros and + library spellings it is not using; a token scan over the raw text would + read those explanations as code. + """ + out = [] + for line in text.splitlines(): + out.append("" if line.lstrip().startswith("#") else line) + return "\n".join(out) + + +def _if_else_bodies(code: str, header: str, start: int = 0) -> tuple[str, str, int]: + """Split the `if(
) ... else() ... endif()` starting at/after `start`. + + Returns (if_body, else_body, index_of_endif). Counts nesting so an inner + `if()` cannot terminate the outer block. + """ + lines = code.splitlines(keepends=True) + offsets = [] + pos = 0 + for line in lines: + offsets.append(pos) + pos += len(line) + + begin = None + for idx, line in enumerate(lines): + if offsets[idx] < start: + continue + if line.strip() == header: + begin = idx + break + if begin is None: + raise CMakeParseError(f"no line reading exactly {header!r} found") + + depth = 0 + in_else = False + if_body: list[str] = [] + else_body: list[str] = [] + for idx in range(begin, len(lines)): + stripped = lines[idx].strip() + if idx == begin: + depth = 1 + continue + if stripped == "endif()": + depth -= 1 + if depth == 0: + return "".join(if_body), "".join(else_body), offsets[idx] + (else_body if in_else else if_body).append(lines[idx]) + continue + if stripped.startswith("if("): + depth += 1 + if depth == 1 and stripped == "else()": + in_else = True + continue + (else_body if in_else else if_body).append(lines[idx]) + raise CMakeParseError(f"{header} is never closed by endif()") + + +def _matching_endforeach(code: str, start: int) -> int: + depth = 0 + pos = start + for match in re.finditer(r"^\s*(foreach\(|endforeach\(\))", code[start:], re.M): + if match.group(1) == "foreach(": + depth += 1 + else: + depth -= 1 + if depth == 0: + return start + match.end() + raise CMakeParseError("the Windows dependency foreach is never closed") + + +class WindowsBoostLinkage: + """What src/CMakeLists.txt decides about Boost, per FLM_WIN_STATIC_DEPS.""" + + def __init__(self, text: str) -> None: + self.raw = text + code = _strip_comments(text) + self.code = code + + option = re.search( + r"option\(\s*FLM_WIN_STATIC_DEPS\s+" + r'"(?P[^"]*)"\s+(?P[A-Za-z0-9_]+)\s*\)', + code, + ) + if option is None: + raise CMakeParseError( + "option(FLM_WIN_STATIC_DEPS \"...\" ) not found" + ) + self.option_default = option.group("default") + self.option_doc = option.group("doc") + + # Half one: the linkage macros. The `if(WIN32 AND NOT VCPKG_TOOLCHAIN)` + # wrapper is required, because these macros are only correct there. + guard = code.find("if(WIN32 AND NOT VCPKG_TOOLCHAIN)") + if guard == -1: + raise CMakeParseError("no if(WIN32 AND NOT VCPKG_TOOLCHAIN) block") + static_body, shared_body, _ = _if_else_bodies( + code, "if(FLM_WIN_STATIC_DEPS)", start=guard + ) + self.static_macros = self._definitions(static_body, "static") + self.shared_macros = self._definitions(shared_body, "shared") + + # Half two: the searched library names, per branch. + entry = re.search( + r'"required\|boost_program_options\|(?P[^"]*)"', code + ) + if entry is None: + raise CMakeParseError( + "no 'required|boost_program_options|...' dependency entry" + ) + self.dependency_entry_names = entry.group("names") + self.static_names = self._names_for(True) + self.shared_names = self._names_for(False) + + self._locate_guard() + + @staticmethod + def _definitions(body: str, which: str) -> frozenset[str]: + call = re.search( + r"target_compile_definitions\(\s*flm\s+PUBLIC(?P[^)]*)\)", body + ) + if call is None: + raise CMakeParseError( + f"the {which} branch defines no macros on target flm" + ) + return frozenset(call.group("args").split()) + + def _variables(self) -> dict[str, str]: + found = {} + for name in ("_flm_boost_po_static", "_flm_boost_po_shared"): + match = re.search( + rf'set\(\s*{name}\s+"(?P[^"]*)"\s*\)', self.code + ) + if match is not None: + found[name] = match.group("value") + return found + + def _names_for(self, static: bool) -> list[str]: + """The ordered spellings searched when FLM_WIN_STATIC_DEPS is/isn't ON. + + Handles both shapes a regression could take: the list derived from the + option (today), and a hard-coded literal list (the pre-fix shape, which + must then read the same for both branches and so fail the agreement + test rather than the parse). + """ + raw = self.dependency_entry_names + if "${_flm_boost_po_names}" not in raw: + return [part for part in raw.split(";") if part] + + static_body, shared_body, _ = _if_else_bodies( + self.code, "if(FLM_WIN_STATIC_DEPS)", start=self.code.find( + "_flm_boost_po_static" + ) + ) + body = static_body if static else shared_body + assign = re.search( + r'set\(\s*_flm_boost_po_names\s+"(?P[^"]*)"\s*\)', body + ) + if assign is None: + raise CMakeParseError( + "_flm_boost_po_names is not assigned in the " + f"{'static' if static else 'shared'} branch" + ) + value = assign.group("value") + for name, expansion in self._variables().items(): + value = value.replace("${" + name + "}", expansion) + if "${" in value: + raise CMakeParseError( + f"unresolved variable in the boost name list: {value!r}" + ) + return [part for part in value.split(";") if part] + + def _locate_guard(self) -> None: + """Find the configure-time consistency guard and record what it says.""" + loop = self.code.find("foreach(_flm_dep IN ITEMS") + if loop == -1: + raise CMakeParseError("the Windows dependency foreach is missing") + after_loop = _matching_endforeach(self.code, loop) + link = self.code.find( + "target_link_libraries(flm PUBLIC ${_flm_win_libs})", after_loop + ) + if link == -1: + raise CMakeParseError( + "the resolved Windows libraries are never linked" + ) + region = self.code[after_loop:link] + self.guard_region = region + self.guard_reads_resolved_library = ( + "FLM_LIB_boost_program_options" in region + ) + self.guard_fatal_messages = [ + match.group("body") + for match in re.finditer( + r"message\(\s*FATAL_ERROR(?P.*?)\)\s*$", + region, + re.S | re.M, + ) + ] + self.guard_warning_count = len( + re.findall(r"message\(\s*WARNING", region) + ) + + +def _preset_overrides() -> list[tuple[str, str]]: + """Every (preset, value) that sets FLM_WIN_STATIC_DEPS in CMakePresets.json. + + Both spellings of a cache variable are handled: the plain string form and + the `{"type": ..., "value": ...}` form. + """ + path = pathlib.Path( + os.environ.get("FLM_TEST_WIN_LINKAGE_PRESETS", _DEFAULT_PRESETS) + ) + if not path.is_file(): + raise CMakeParseError(f"{path} does not exist") + document = json.loads(path.read_text(encoding="utf-8")) + found = [] + for preset in document.get("configurePresets", []): + variables = preset.get("cacheVariables") or {} + if "FLM_WIN_STATIC_DEPS" not in variables: + continue + entry = variables["FLM_WIN_STATIC_DEPS"] + value = entry.get("value") if isinstance(entry, dict) else entry + found.append((str(preset.get("name", "")), str(value))) + return found + + +def _preset_names() -> list[str]: + path = pathlib.Path( + os.environ.get("FLM_TEST_WIN_LINKAGE_PRESETS", _DEFAULT_PRESETS) + ) + if not path.is_file(): + raise CMakeParseError(f"{path} does not exist") + document = json.loads(path.read_text(encoding="utf-8")) + return [ + str(preset.get("name", "")) + for preset in document.get("configurePresets", []) + ] + + +def _load() -> WindowsBoostLinkage: + path = pathlib.Path( + os.environ.get("FLM_TEST_WIN_LINKAGE_CMAKELISTS", _DEFAULT_CMAKELISTS) + ) + if not path.is_file(): + raise CMakeParseError(f"{path} does not exist") + return WindowsBoostLinkage(path.read_text(encoding="utf-8")) + + +class WindowsDependencyLinkageParseTest(unittest.TestCase): + """Non-vacuity. Every other test here is a claim about parsed content.""" + + def test_every_part_of_the_decision_was_actually_found(self) -> None: + parsed = _load() + self.assertIn( + parsed.option_default, + {"ON", "OFF"}, + "FLM_WIN_STATIC_DEPS has no boolean default; the parse below is " + "reading something else", + ) + self.assertTrue(parsed.static_macros, "static branch macros not parsed") + self.assertTrue(parsed.shared_macros, "shared branch macros not parsed") + self.assertTrue(parsed.static_names, "static boost name list not parsed") + self.assertTrue(parsed.shared_names, "shared boost name list not parsed") + self.assertTrue( + parsed.guard_region.strip(), + "nothing at all sits between the dependency loop and the link " + "call; the consistency guard cannot be there", + ) + + def test_the_two_branches_are_not_the_same_branch(self) -> None: + # A "fix" that makes both arms identical would satisfy several of the + # assertions below while restoring the defect. + parsed = _load() + self.assertNotEqual( + parsed.static_macros, + parsed.shared_macros, + "the static and shared arms define the same macros", + ) + self.assertNotEqual( + parsed.static_names, + parsed.shared_names, + "the static and shared arms search the same name order, so the " + "option no longer selects a linkage", + ) + + +class WindowsDependencyLinkageDefaultTest(unittest.TestCase): + def test_static_is_the_default(self) -> None: + parsed = _load() + self.assertEqual( + parsed.option_default, + "ON", + "FLM_WIN_STATIC_DEPS must default ON. It is set ON nowhere in the " + "tree, no CMake preset sets a toolchainFile (so VCPKG_TOOLCHAIN is " + "false on every Windows preset), and CI runs " + "`cmake --preset windows-vs18` bare -- so this default IS the " + "linkage of the shipped binary and the MSI. OFF makes flm.exe " + "depend on boost_program_options.dll, which src/lib does not " + "vendor and src/wix/get_files.bat does not stage.", + ) + + def test_the_default_configuration_links_boost_statically(self) -> None: + # Stated as the consequence rather than the setting: this is what the + # packaging in src/lib and src/wix/get_files.bat assumes. + parsed = _load() + default_static = parsed.option_default == "ON" + macros = parsed.static_macros if default_static else parsed.shared_macros + names = parsed.static_names if default_static else parsed.shared_names + self.assertIn( + "BOOST_ALL_STATIC_LINK", + macros, + "the default configure does not declare static Boost linkage", + ) + self.assertRegex( + names[0], + _STATIC_NAME, + "the default configure prefers a Boost import library over the " + "static archive, so the shipped flm.exe would import " + "boost_program_options.dll", + ) + + +class WindowsDependencyLinkageAgreementTest(unittest.TestCase): + """The macros and the name preference must describe the same linkage.""" + + def test_static_branch(self) -> None: + parsed = _load() + self.assertEqual( + parsed.static_macros, + _STATIC_MACROS, + "FLM_WIN_STATIC_DEPS=ON must define exactly CURL_STATICLIB, " + "BOOST_ALL_NO_LIB and BOOST_ALL_STATIC_LINK", + ) + self.assertRegex( + parsed.static_names[0], + _STATIC_NAME, + "FLM_WIN_STATIC_DEPS=ON declares static Boost linkage but searches " + f"{parsed.static_names[0]!r} first -- b2's bare name is the import " + "library for boost_program_options.dll, not an archive", + ) + + def test_shared_branch(self) -> None: + parsed = _load() + self.assertEqual( + parsed.shared_macros, + _SHARED_MACROS, + "FLM_WIN_STATIC_DEPS=OFF must define exactly BOOST_ALL_NO_LIB and " + "BOOST_ALL_DYN_LINK (NO_LIB stays: MSVC auto-linking would " + "otherwise ask for the decorated static name)", + ) + self.assertNotIn( + "CURL_STATICLIB", + parsed.shared_macros, + "CURL_STATICLIB against an import library removes the " + "__declspec(dllimport) that library expects", + ) + self.assertRegex( + parsed.shared_names[0], + _SHARED_NAME, + "FLM_WIN_STATIC_DEPS=OFF declares BOOST_ALL_DYN_LINK but searches " + f"{parsed.shared_names[0]!r} first -- a 'lib'-prefixed archive, " + "which exports no data symbols and fails the link on " + "boost::program_options::arg", + ) + + def test_both_spellings_remain_reachable_as_fallbacks(self) -> None: + # Order carries the preference; dropping the other spelling would turn + # a prefix that ships only one variant into a configure failure, and + # would make the FATAL_ERROR guard unreachable. + parsed = _load() + for label, names in ( + ("static", parsed.static_names), + ("shared", parsed.shared_names), + ): + with self.subTest(branch=label): + self.assertTrue( + any(_STATIC_NAME.match(name) for name in names), + f"{label} branch never searches the static archive name", + ) + self.assertTrue( + any(_SHARED_NAME.match(name) for name in names), + f"{label} branch never searches the import library name", + ) + + +class WindowsDependencyLinkageGuardTest(unittest.TestCase): + """The configure-time check that the resolved library matches the macros.""" + + def test_guard_is_present_and_fatal(self) -> None: + parsed = _load() + self.assertTrue( + parsed.guard_reads_resolved_library, + "nothing between the dependency loop and the link call inspects " + "FLM_LIB_boost_program_options, so a search that resolves to the " + "wrong variant -- or a stale cache entry, which find_library never " + "re-searches -- is not detected", + ) + self.assertGreaterEqual( + len(parsed.guard_fatal_messages), + 2, + "the guard must FATAL_ERROR on both contradictions: a " + "'lib'-prefixed archive under BOOST_ALL_DYN_LINK, and a bare " + f"'boost_' import library under BOOST_ALL_STATIC_LINK. Found " + f"{len(parsed.guard_fatal_messages)} FATAL_ERROR message(s) and " + f"{parsed.guard_warning_count} WARNING(s) -- a warning in a " + "thousand-line configure log is what was missed the first time.", + ) + + def test_guard_names_both_halves_and_the_way_out(self) -> None: + parsed = _load() + joined = " ".join(parsed.guard_fatal_messages) + for token in ( + "FLM_WIN_STATIC_DEPS", + "BOOST_ALL_STATIC_LINK", + "BOOST_ALL_DYN_LINK", + ): + with self.subTest(token=token): + self.assertIn( + token, + joined, + f"the guard's diagnostics never mention {token}; the " + "reader cannot tell which of the two halves to change", + ) + self.assertIn( + "${FLM_LIB_boost_program_options}", + joined, + "the guard does not report which library it actually resolved", + ) + + +class WindowsDependencyLinkageEffectiveValueTest(unittest.TestCase): + """Nothing between the `option()` line and the configure may flip it OFF. + + `test_static_is_the_default` reads the option's default and stops there. A + default is only the shipping value while nothing overrides it, and the two + cheapest overrides -- a preset cache variable and a later `set()` -- leave + the option line byte-identical. Both were applied to this tree and the + suite stayed green; these are the assertions that turn them red. + """ + + def test_no_preset_turns_the_option_off(self) -> None: + offenders = [ + f"{name}={value}" + for name, value in _preset_overrides() + if value.strip().upper() in _CMAKE_FALSE + ] + self.assertEqual( + offenders, + [], + "a CMake preset sets FLM_WIN_STATIC_DEPS to a false value " + f"({', '.join(offenders)}). CI runs `cmake --preset windows-vs18` " + "bare, so a preset cache variable IS the shipping linkage: this " + "makes flm.exe import boost_program_options.dll, which src/lib " + "does not vendor and src/wix/get_files.bat does not stage. " + "cacheVariables are inherited, so a false value on ANY preset in " + "this file is reported here.", + ) + + def test_the_preset_file_was_actually_read(self) -> None: + # Non-vacuity. A missing, empty or restructured preset file would make + # the assertion above pass by finding nothing -- which is precisely + # how this file failed to see the preset re-break before. + names = _preset_names() + self.assertIn( + "windows-vs18", + names, + "src/CMakePresets.json has no `windows-vs18` configure preset, so " + "the preset scan is not reading the file CI invokes. Found: " + f"{names}", + ) + + def test_cmakelists_does_not_reassign_the_option(self) -> None: + # `set(FLM_WIN_STATIC_DEPS OFF CACHE BOOL "" FORCE)` after the option, + # or a plain `set(FLM_WIN_STATIC_DEPS OFF)` before it (CMP0077 makes + # `option()` honour an existing normal variable), both reach the same + # place without editing the `option()` call. + parsed = _load() + assignments = re.findall( + r"^\s*set\(\s*FLM_WIN_STATIC_DEPS\b[^)]*\)", + parsed.code, + re.M, + ) + self.assertEqual( + assignments, + [], + "src/CMakeLists.txt assigns FLM_WIN_STATIC_DEPS outside its " + f"option() declaration: {assignments}. Under CMP0077 that " + "overrides the default the tests above read, so the option line " + "no longer tells you the shipping linkage.", + ) + + +if __name__ == "__main__": + unittest.main()