diff --git a/.claude/settings.local.cleaned.json b/.claude/settings.local.cleaned.json deleted file mode 100644 index 069fd140..00000000 --- a/.claude/settings.local.cleaned.json +++ /dev/null @@ -1,79 +0,0 @@ -{ - "permissions": { - "allow": [ - "Bash(./build/projectMM:*)", - "Bash(./build/test/mm_scenarios:*)", - "Bash(./build/test/mm_tests:*)", - "Bash(arp -a)", - "Bash(awk:*)", - "Bash(cat:*)", - "Bash(clang++:*)", - "Bash(clang:*)", - "Bash(cmake:*)", - "Bash(cp .claude/settings.local.json .claude/settings.local.cleaned.json)", - "Bash(ctest:*)", - "Bash(curl -s*github.com/MoonModules*)", - "Bash(curl -s*raw.githubusercontent.com*)", - "Bash(echo:*)", - "Bash(gh api *)", - "Bash(gh auth *)", - "Bash(gh pr *)", - "Bash(gh run *)", - "Bash(gh search *)", - "Bash(git check-ignore *)", - "Bash(git fetch *)", - "Bash(git ls-remote *)", - "Bash(git mv:*)", - "Bash(grep:*)", - "Bash(gunzip:*)", - "Bash(head:*)", - "Bash(ipconfig getsummary *)", - "Bash(ls:*)", - "Bash(lsof:*)", - "Bash(networksetup -listallhardwareports)", - "Bash(node --check:*)", - "Bash(node --input-type=module -e ' *)", - "Bash(node --test:*)", - "Bash(node -e ' *)", - "Bash(npm install *)", - "Bash(npm test *)", - "Bash(objcopy:*)", - "Bash(objdump:*)", - "Bash(otool:*)", - "Bash(pkill:*)", - "Bash(ps:*)", - "Bash(sed:*)", - "Bash(sort:*)", - "Bash(system_profiler SPUSBDataType)", - "Bash(tail:*)", - "Bash(timeout 5 dns-sd -B _http._tcp local.)", - "Bash(uv run *)", - "Edit", - "Read(//Users/ewoud/**)", - "Read(//Users/ewoud/esp/esp-idf/**)", - "Read(//dev/**)", - "WebFetch(domain:components.espressif.com)", - "WebFetch(domain:components101.com)", - "WebFetch(domain:developer.espressif.com)", - "WebFetch(domain:devices.esphome.io)", - "WebFetch(domain:docs.espressif.com)", - "WebFetch(domain:easyelecmodule.com)", - "WebFetch(domain:files.waveshare.com)", - "WebFetch(domain:github.com)", - "WebFetch(domain:kno.wled.ge)", - "WebFetch(domain:mm.kno.wled.ge)", - "WebFetch(domain:moonmodules.org)", - "WebFetch(domain:raw.githubusercontent.com)", - "WebFetch(domain:www.digikey.com)", - "WebFetch(domain:www.farnell.com)", - "WebFetch(domain:www.waveshare.com)", - "WebSearch", - "Write" - ], - "additionalDirectories": [ - "/tmp", - "/Users/ewoud/esp/esp-idf/components/esp_mm/include", - "/Users/ewoud/Developer/GitHub/MoonModules/projectMM/.claude" - ] - } -} diff --git a/CLAUDE.md b/CLAUDE.md index d55f4dff..01bb9c5d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -80,7 +80,7 @@ Commit message: title ≤ 72 characters, imperative. Then a 1–3 sentence end-u The PO pushes the branch; external review runs on the PR; findings are processed on the branch. On "run pre-merge": `uv run moondeck/event/premerge.py`, which re-runs the mechanical checks over the whole branch diff and lists the judgment gates it cannot decide. -Those judgment gates: review feedback addressed; the Reviewer agent over the whole branch diff (start it first, it runs in parallel; scope: boundaries, bespoke conventions, unnecessary abstractions, duplication, hot path, spec conformance, bloat); lessons carried forward only when VERY important — most learning lives in the commit/PR record; a truly important gotcha → `lessons.md`, a major architectural decision → a new ADR, a hardened rule → CLAUDE.md or coding-standards; docs sync; the PR title and description matching the actual diff; the performance snapshot when tick-path code changed; the permission review (prune the accumulated local list and snapshot the approved result — never broaden destructive or network-mutating permissions without explicit approval, err toward tight); a README refresh when build, flash, or first-run changed. +Those judgment gates: review feedback addressed; the Reviewer agent over the whole branch diff (start it first, it runs in parallel; scope: boundaries, bespoke conventions, unnecessary abstractions, duplication, hot path, spec conformance, bloat); lessons carried forward only when VERY important — most learning lives in the commit/PR record; a truly important gotcha → `lessons.md`, a major architectural decision → a new ADR, a hardened rule → CLAUDE.md or coding-standards; docs sync; the PR title and description matching the actual diff; the performance snapshot when tick-path code changed; a README refresh when build, flash, or first-run changed. ### Release diff --git a/CMakeLists.txt b/CMakeLists.txt index c7cda635..3f625991 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -71,6 +71,17 @@ else() endif() endif() +# MM_MOONLIVE_NO_HOST_JIT builds this machine the way a desktop with NO MoonLive backend sees the +# code. Every x86-64 desktop is one (Windows, Linux, Intel macOS ship no backend), so a test that +# wrongly presumes a compile succeeds passes on the arm64 bench and fails only once CI runs it. +# A compile DEFINITION rather than a CMAKE_CXX_FLAGS entry: this file configures warnings through +# add_compile_options, which does not consume that variable, so a flag passed there would reach the +# cache and never the compiler. +option(MM_MOONLIVE_NO_HOST_JIT "Build as a desktop with no MoonLive backend" OFF) +if(MM_MOONLIVE_NO_HOST_JIT) + add_compile_definitions(MM_MOONLIVE_FORCE_NO_HOST_JIT=1) +endif() + # `uv` is the project's Python launcher (see CLAUDE.md / moondeck/MoonDeck.md). # The build invokes Python helpers for build_info.h generation and UI embedding; # resolving them through `uv run python …` keeps Windows (where `python3` isn't @@ -94,6 +105,7 @@ add_library(mm_core STATIC src/core/Scheduler.cpp src/core/moonlive/MoonLive.cpp src/core/moonlive/MoonLiveCompiler.cpp + src/core/moonlive/MoonLiveSpill.cpp ) target_include_directories(mm_core PUBLIC src/) target_link_libraries(mm_core PUBLIC mm_platform) diff --git a/docs/MIGRATING.md b/docs/MIGRATING.md index 624ae916..9e34049e 100644 --- a/docs/MIGRATING.md +++ b/docs/MIGRATING.md @@ -20,6 +20,23 @@ projectMM ships **no migration code**: the persistence layer is robust by defaul ## Unreleased (`next-iteration`) +### MoonLive scripts move to the filesystem (2026-08-11) + +A scripted module used to carry its script as a `source` textarea — a fixed 1 KB array per module, plus a second 1 KB copy to notice edits, **resident whether or not a script was loaded**. Six modules cost 13 KB of a classic ESP32's 320 KB for text that was mostly empty. The script now lives in a file under `/moonlive/`, and the module holds only its **name** (~32 bytes): it is read into a right-sized buffer to compile and freed immediately, so nothing script-sized stays in RAM. A script is bounded by the filesystem instead of by a 1 KB array. + +**Action: *re-add a module* — or, to keep your scripts, *update a file* first.** + +The `source` control no longer exists, so a persisted `"source"` value is an unknown key and is ignored (the robust-reader rule). A MoonLive module therefore boots with **no script**, reporting `no script — set the script name`, and renders nothing until one is named. + +| What | Why | What to do | +|---|---|---| +| Your script text | It was persisted under `source`, a control that is gone | **Copy it out before updating** — it is in `/.config/Layouts.json` (or `Effects.json`) as `"N.source"`. Save it as `/moonlive/.mlv` via the File Manager, then set the module's `script` control to `.mlv` | +| The module's own controls | A script's `@control` sliders exist only once it has compiled, so they are absent until a script is named | Nothing — they reappear with the script, keeping their persisted values | + +`/moonlive/` is created on demand: naming a script is enough to make the folder appear, so a fresh device needs no setup. + +**Editing today** goes through the File Manager rather than the module's own card. Wiring the card's editor to the same file is a separate change. + ### MoonLive: a script can no longer declare a name the engine supplies (2026-08-10) `t` (elapsed milliseconds), `width`/`height`/`depth` (the logical grid) and `x`/`y`/`z` (the light a modifier is transforming) are now **system variables** the engine supplies, so a script cannot declare one. Previously each binding faked them by prepending hidden declarations to the script, which meant an effect could declare its own `width` and quietly disagree with the layer it was drawing into. @@ -32,7 +49,7 @@ A **layout** is the one script that legitimately used those names for its own co | What | Why | What to do | |---|---|---| -| A scripted layout declaring `width`/`height` | The name is what the layout is defining, so the declaration is a compile error and no lights are placed | Edit the script's `source` control, renaming its own controls (the shipped `grid.mlv` uses `cols`/`rows`) | +| A scripted layout declaring `width`/`height` | The name is what the layout is defining, so the declaration is a compile error and no lights are placed | Edit the `.mlv` file in the File Manager, renaming its own controls (the shipped `grid.mlv` uses `cols`/`rows`), then set the module's `script` control to that file | | A scripted **modifier** using `x`, `y` or `z` as a loop variable | A modifier IS handed a coordinate under those names, so they cannot also be counters there | Rename the loop variable to something the modifier is not handed (`i`, `n`) | Effects and modifiers need no change: they were already being handed these values, just through a preamble instead of by name. The error names the clash, and the module shows it on its card, so a broken script says why rather than failing silently. diff --git a/docs/backlog/backlog-light.md b/docs/backlog/backlog-light.md index 8ca1324e..e98a6cea 100644 --- a/docs/backlog/backlog-light.md +++ b/docs/backlog/backlog-light.md @@ -292,6 +292,31 @@ The LED-driver increments **shipped**: increment 1 (RMT/WS2812B single-strand on - **A scripted modifier that reshapes the grid** (2026-08-10). `ModifierBase::modifyLogicalSize` lets a modifier change the logical `width`/`height`/`depth` — a Multiply kaleidoscope grows the grid, a crop shrinks it — and a compiled modifier uses it. A SCRIPTED one cannot: system variables are read-only, so `MoonLiveModifier` writes the box in and never reads it back. Needs a writable system variable — the binding reads the slots after the script returns and reports the result through `modifyLogicalSize` — which is a new `SysVarKind` (or a mutable flag on `SysVar`) plus the read-back, not a new builtin. Until then a scripted modifier can fold coordinates but not resize the grid they live in. +- **Editing a script's CONTENTS through /api/file does not recompile it** (2026-08-14). A binding + caches `compiledHash_` and skips the compile while it is non-zero; the hash is cleared when the + script NAME changes (`onControlChanged`, `setScript`), but a write to `/moonlive/` via + the File Manager leaves it set, so the layout keeps running the previous code until the name is + touched or the device reboots. `MoonLiveModifier` does not have this: it re-hashes the source on + every prepare and compares, which is the shape to copy. + + The fix belongs at the filesystem seam rather than in the binding — a write under `/moonlive/` + invalidates whatever compiled from that path — so it is a small core/HTTP change, not a MoonLive + one. Pre-existing, not introduced by the stack-machine work. + +- **MoonLive has no x86-64 backend — scripts do not run on Windows** (2026-08-14). The desktop + assembler (`moonlive_asm_host.cpp`) is arm64-only, so `MM_MOONLIVE_HAS_HOST_JIT` is 0 on x86-64 + Windows, x86-64 Linux and Intel macOS. `compileSource` fails cleanly there and scripted modules + render dark — no crash, but no MoonLive either, on the desktop platform most users run. Apple + Silicon macOS is the only desktop where scripts work today, which is why this stayed invisible: + the bench is arm64 and CI's x86-64 runners gate their MoonLive tests on the macro. + + Closing it is one more backend behind the unchanged IR (the seam's whole promise): an + `x86_64` branch alongside the three that exist. It is the widest ISA of the four — variable-length + encoding, and a different calling convention per OS (System V on Linux/macOS, Microsoft x64 on + Windows), so the `call()` save-set and argument registers differ from everything written so far. + `disasm.py --isa x86_64` should land with it, since no test executes emitted bytes for any backend + but the host's. + - **Drain MoonLive's `print()` through a queue** (2026-08-09). `print(v)` writes to serial directly, and an EFFECT script runs on the render tick — so a print inside one blocks the frame for as long as the UART takes. The burst cap bounds it (a handful of writes per compile, then a compare and a return), but bounded is not free, and `tick()` is annotated `MM_NONBLOCKING`. **What it costs when it comes:** a small preallocated record queue the built-in writes into, drained from a housekeeping path through the existing platform output seam. The budget and the burst-spent message stay as they are; only where the bytes are written moves. Worth doing when a script is left with a print in it on a real fixture, which is the case the cap exists for. diff --git a/docs/history/lessons.md b/docs/history/lessons.md index b10937cf..286d7e8f 100644 --- a/docs/history/lessons.md +++ b/docs/history/lessons.md @@ -452,3 +452,52 @@ share a shape: a green build, plausible-looking output, and a defect only counti no amount of reading the new code reveals. Diffing old against new across every span and 2.8M inputs found it in seconds. **For a hot-path primitive whose replacement is meant to be behaviour-identical, prove it by exhaustive comparison against the original.** + +## Lessons from the MoonLive-on-Xtensa branch (the register-window frame bug) + +MoonLive's JIT ran on desktop arm64 and on RISC-V, and reset every Xtensa board the moment a +script called a host function. Three days, because every check we had was looking at the +instructions and the defect was in the stack layout around them. + +- **On Xtensa, the top 32 bytes of a call8 frame belong to the hardware, not to the routine.** + The window-overflow handler (`_WindowOverflow8`, esp-idf `components/xtensa/xtensa_vectors.S`) + writes two 16-byte bands into a spilled frame: the top 16 take an *older* frame's a0-a3 + (`s32e aN, a9, -16..-4`), and the next 16 take the routine's *own* a4-a7 + (`s32e aN, a0, -32..-20`, where a0 is that frame's top). GCC obeys the same rule — every + call8-making function it compiles reserves locals + exactly 32, at every size. Our emitter + reserved 16, so the parked control-arena pointer sat inside the second band and a window + spill overwrote it with an expression temp. **A frame-layout question is answered by the + platform's own spill handler and a compiler probe, not by the ISA prose.** + +- **Window rotation is deterministic; the SPILL is not, so the contract is spatial.** Rotation + happens only at `call8`/`retw`, in our own instruction stream. The spill is lazy: it fires on + any interrupt that lands while a call is in flight, at an instruction we do not choose. No + ordering, no code sequence and no "quiet moment" can dodge it — only the layout can be right + or wrong. This is also the whole symptom explanation: deep call chains (plasma's `sin`/`beat` + into libm) gave every tick interrupt a wide window and died in under a second, brief leaf + calls almost never coincided, and call-free scripts never faulted at all. **A crash whose + frequency tracks how LONG a call runs, rather than what it computes, is an interrupt-timing + bug — look at what the hardware writes asynchronously, not at the code path.** + +- **Byte-perfect encodings prove the instructions, and say nothing about the frame.** Round-trip + checks against `xtensa-esp32-elf-as` passed for every instruction we emit, including the + `entry` whose immediate was the bug. Two other backends running the identical IR pipeline were + flawless, which read as "the compiler is fine, the target is haunted". **When a defect is + target-specific and every static check is green, suspect the contract with the platform + (frame, ABI, alignment) rather than the code generator.** + +- **The guardrail had to see the bug before the fix went in.** `MM_ISA_RESERVED_TOP` was raised + to 32 in the structural checker *first*, and it failed on the shipped emitter, naming the + offending frame offset. Only then did the emitter constant change. A test written after a fix + proves the fix compiles; a test that fails first proves it *detects*. The checker decodes the + real `entry` immediate out of the emitted bytes, so it pins the invariant rather than the + constant. + +- **A reserve constant is coupled to the widest call the emitter can produce.** 32 is exact for + `call8`; `call12` would need 48 (its extra save area holds a8-a11 as well). We only ever emit + `call8`, so the checker's 32 is an equality — but nothing in the code says so, and a future + wider call would leave the test green while the frames went wrong again. hpwit, who wrote the + new-parser Xtensa compiler, independently reached the same floor from experience ("I always + start at at least 32") and phrases it as a minimum for exactly this reason. The reserve is + therefore derived from the emitted call opcode rather than written down, so widening the call + moves the reserve with it or fails the build. diff --git "a/docs/history/plans/Plan-20260809 - MoonLive scales \342\200\224 right-sized IR, and the stack as the register overflow.md" "b/docs/history/plans/Plan-20260809 - MoonLive scales \342\200\224 right-sized IR, and the stack as the register overflow.md" new file mode 100644 index 00000000..edd42650 --- /dev/null +++ "b/docs/history/plans/Plan-20260809 - MoonLive scales \342\200\224 right-sized IR, and the stack as the register overflow.md" @@ -0,0 +1,326 @@ +# Plan: MoonLive scales — right-sized IR, and the stack as the register overflow + +> **Steps 1–3 shipped. Steps 4–5 (the register allocator) are SUPERSEDED by +> [Plan-20260813 — MoonLive on a stack machine](Plan-20260813%20-%20MoonLive%20on%20a%20stack%20machine%20%E2%80%94%20the%20frame%20is%20where%20values%20live.md).** +> The allocator was built and works on the host at every budget, but on Xtensa it leaves ZERO +> allocatable registers (10 − 1 scratch − 5 ABI vregs − 4 reload temps), so every looped script is +> refused there. Bench-measured on an S3. The successor plan puts every variable in the frame +> instead and keeps registers for expression temporaries only. + +## Context + +MoonLive scripts hit hard walls far below what a user would call a complex script. Two separate +ceilings, both measured on the host, neither obvious from the error text (every one reports +`codegen failed (unsupported on this target, or too large)`): + +- **Seven `addLight` statements in a row fail.** `kMaxIrOps = 64` ([MoonLiveIr.h:33](src/core/moonlive/MoonLiveIr.h#L33)) + and a call costs ~9 IR ops. No nesting, no register pressure — this is the wall a user meets first. +- **Nested `for` loops are refused on Xtensa.** Measured: `LOWER BAIL: vregsUsed=11 +2 > kRegCount=12` + ([moonlive_lower_xtensa.cpp:24](src/platform/esp32/moonlive_lower_xtensa.cpp#L24)). They work on + desktop and RISC-V, which have larger register maps. The shipped default layout script `grid.mlv` + is a nested loop, so the module's own default cannot compile on the smallest target. + +The goal is that a script's complexity is bounded by memory the device actually has, not by +constants chosen when a script was one statement. That means two changes, and they are +independent: **right-size the IR** (removes the statement wall) and **spill to the stack** +(removes the register wall). Neither alone is enough — shipping only the spiller leaves the +7-statement wall, which is the one users hit first. + +Spilling is also the industry-standard answer: values that outlive the register file live in the +frame. It is what every real compiler does, and it is the mechanism that makes "how complex can a +script be" a memory question instead of a register-count question. + +## Every ceiling, and what happens to it + +Seven fixed constants bound a script. They are **not** one problem: what each costs, and where its +storage lives, decides the treatment. Measured sizes: + +| Ceiling | Value | Limits | Where it lives | Treatment | +|---|---|---|---|---| +| `kMaxIrOps` | 64 | total instructions | `IrProgram` = **2056 B stack local** ([MoonLiveCompiler.cpp:512](src/core/moonlive/MoonLiveCompiler.cpp#L512)) | **right-size on the heap** — the wall users hit first | +| `kCodeCap` | 768 B | emitted machine code | `buf_[kCap]` inside the assembler, itself a **1368 B stack local** ([moonlive_lower_xtensa.cpp:28](src/platform/esp32/moonlive_lower_xtensa.cpp#L28)) | **right-size on the heap**, same mechanism | +| `kMaxVRegs` | 16 | values a program can name | index width in `IrInst` | **raise to 32** once spilling makes >16 usable; `IrProgram::push` keeps validating | +| `kMaxFixups` | 32 | branches | assembler member | **right-size** with the code buffer (same owner, same lifetime) | +| `kIrLabels`/`kMaxLabels` | 16 | ~8 loops | IR + assembler members | **right-size** with the op array; the estimator counts `for` tokens | +| `locals[4]` | 4 | loop nesting depth | Parser, stack | **raise to 8**, 12 B — a fixed bump, not worth an allocation | +| `kMaxCtrls` | 8 | script-declared controls | 128 B in Parser + the binding's name pool | **raise to 16**; bounded by UI sanity, not by memory. Note the *binding* mirrors this in a fixed name pool, so both move together | + +**The constraint that drives this:** `CONFIG_ESP_MAIN_TASK_STACK_SIZE = 12288` +([sdkconfig.defaults:8](esp32/sdkconfig.defaults#L8)), and the compile path already burns **~3.4 KB** +of it (`IrProgram` 2056 + assembler 1368, both live at once). Naively raising `kMaxIrOps` to 256 +makes `IrProgram` alone 8 KB of *stack* — a bootloop, not a fix. This project has already lost a P4 +to a large stack frame. So the two big arrays move to the heap and are sized to the script; the small +ones are simply raised, because 12 B or 128 B does not need an allocator. + +## The end state (PO, 2026-08-11) + +MoonLive is **bounded by memory, not by registers or fixed arrays** — a fairly complete language, +large scripts, nice effects. Three goals, and every step towards them is judged against the +**classic ESP32**: 320 KB internal, no PSRAM, so anything assuming plentiful RAM fails there first. +(The classic is not structurally blocked — its Xtensa backend compiles and its exec heap is enabled; +what stopped it was a crash, parked separately.) + +1. **Spilling** — register allocation with spilling to the stack, so a script is never refused for + naming more live values than the ISA has registers. Linear-scan (Poletto & Sarkar), designed + below. +2. **Scripts on the filesystem** — the source lives on LittleFS, not in a fixed per-module array. +3. **Classic ESP32 runnability** — the yardstick for all of the above. + +### Decided: a heap buffer, not a streaming lexer + +Goal 2 could load the script into a right-sized heap buffer for the compile and free it after, or +stream it from the file so no buffer exists at all. **Heap buffer**, for three reasons: + +- **The waste is the fixed array, not the transient buffer.** `source_` + `compiled_` + names is + **2240 B per module, always resident — 13.1 KB across six modules, 4.1% of a classic's internal + RAM, held whether or not a script is loaded.** A compile-time buffer is proportional to the script + and freed immediately; the fixed arrays are permanent and mostly empty. Removing them is the win. +- **Seeking a file is not simpler than seeking RAM.** LittleFS does wear-levelling and block + caching, so `parseFor`'s backward re-lex of the step clause could hit flash mid-compile — + unpredictable latency in place of a pointer decrement. +- **Streaming needs compiler surgery first.** `parseFor` re-lexes the step from a saved source + pointer after emitting the body, and `DeclaredControl::name` points INTO the source and outlives + the compile (13 sites hold such pointers). Both are fixable, neither is a lexer swap. + +Streaming stays possible later; it is an optimisation of a transient allocation, not the thing that +makes large scripts fit. + +**Two consequences:** `kMaxScriptBytes` stops being a ceiling (a script is bounded by heap), and +`compiled_` — which only answers "did the source change" — becomes a hash rather than a second full +copy, removing another 1 KB per binding. FNV-1a is already the project's idiom for that. + +### Sequence + +Each step makes the next cheaper: + +1. ✅ **Right-size `IrProgram`** (2026-08-11). The op array is heap-allocated and sized from a + token count before parsing; `IrProgram` owns it RAII (destructor frees, copy deleted), so there + is no manual free path to miss — unlike the reverted `32026eb5`, whose four independently- + nullable tables produced the heap corruption its own comment records. `kMaxIrOps` 64 → 4096 is + now a sanity bound, not the working limit: **7 sequential statements used to fail, 40 compile**, + and ~2 KB moved off the 12 KB main-task stack. + + **Found while verifying:** widening `count` to `uint16_t` left four `uint8_t` loop counters + iterating over it — three lowerers plus `IrProgram::hasInline` — which wrapped at 256 ops and + spun forever. On a device that is a watchdog reset from a script that merely got long. Bisected + (60 statements fine, 80 hung), fixed, and pinned by a test that HANGS when the fix is reverted. + The first version of that test passed either way: repeated statements hit the code-buffer + ceiling before reaching the wrap, so it needed a long arithmetic chain instead — many cheap ops, + little emitted code. + + **Still standing:** `kCodeCap` is a separate ceiling and now the binding one (40 statements + exceed it on Xtensa), as are `kIrLabels`, `locals[4]` and `kMaxVRegs`. +2. **Scripts on the filesystem.** Independent of the compiler work — different files, different + risk — and the step that most helps the classic. +3. **Spilling.** By then it has stack headroom and no 1 KB source ceiling to fight. Landing it first + would put the hardest algorithm on the tightest stack budget, where an overrun reads as a + bootloop rather than a compiler bug. + +## Decisions taken + +- **Allocate to fit the script, not to `kMax`.** The op array and the code buffer move to a + right-sized `platform::alloc` ([platform.h:55](src/platform/platform.h#L55)), sized from a cheap + pre-pass over the source and freed when compilation ends. A one-statement script pays for one + statement instead of 3.4 KB, so this *reduces* peak memory for the common case while removing the + ceiling for the rare one. Compilation is cold-path, so an allocation there costs nothing that + matters. +- **Raise the cheap ceilings rather than allocating them.** `locals`, `kMaxCtrls` and `kMaxVRegs` are + tens of bytes. Subjecting them to an allocator would add machinery that buys nothing — the standard + construct is only worth it where the size actually varies. +- **The spill algorithm lives in core, once.** Correct spilling across a loop back-edge is the + hardest logic here, and only the arm64 backend is ever executed by tests — three copies would + leave two permanently under-tested. Backends supply their register count and consume two new IR + ops. (CLAUDE.md Principle 3: core owns the hard constructs, written once.) +- **Spill everywhere, not interval splitting.** Once a value is chosen for spilling it is spilled + for its whole lifetime: every def stores, every use reloads. Splitting halves the reload traffic + but the correctness argument across a back-edge is exactly the part that goes wrong. The simple + form is provably safe and these loops are cold-path. +- **Fix the RISC-V scratch aliasing first, on its own.** Verified bug, latent today: `kScratchFn = 16` + ([moonlive_asm_riscv.cpp:20](src/platform/esp32/moonlive_asm_riscv.cpp#L20)) is x16/a6, but + `kRvReg[12] == 16` — so it *is* vreg R12. In `call()`, `mv a6, a0` stashes the result, the restore + loop reloads x16 from frame offset 48 and destroys it, then `mv dst, a6` returns R12's stale value. + It only bites when `vregsUsed > 12`, which is precisely what this work causes. Landing it inside + the feature would make the first hardware symptom look like "the new spiller broke calls". +- **Not in scope: narrowing `call()`'s save-sets.** The spill pass computes exactly the + live-across-call mask that would shrink RISC-V's 18-register and arm64's 14-register unconditional + saves. Deliberately deferred: an over-long interval only costs an unnecessary spill (fail-safe), + whereas a register wrongly omitted from a save-set corrupts a value (fail-dangerous) from the same + analysis. Ship the safe consumer first, backlog the other by name. + +## Design + +### 1. Right-sized IR and code buffer (removes the 7-statement wall) + +`IrProgram` gains a heap op array instead of `IrInst ops[kMaxIrOps]`: + +```cpp +struct IrProgram { + IrInst* ops = nullptr; // platform::alloc'd to fit; freed in the destructor + uint16_t cap = 0; // what was allocated + uint16_t count = 0; + VReg vregsUsed = kFirstTemp; + bool reserve(uint16_t ops); // false on alloc failure — degrade, never crash +}; +``` + +The **assembler gets the same treatment**: `buf_[kCap]`, `labelPos_[]` and `fixups_[]` become one +right-sized allocation with the same lifetime. This is the other 1368 B of stack, and `kCodeCap` is +a ceiling in its own right — a long script overflows the code buffer even when its IR fits. + +**Sizing.** One cheap pre-pass over the token stream counts statements, call arguments and `for` +keywords, then multiplies by the known worst-case ops (and bytes) per construct. Over-estimating is +free — a few unused entries; under-estimating must be impossible, so the estimator is deliberately +conservative and `push()`/`emit()` still fail cleanly if it is ever wrong. The existing +`overflow_` path stays as the backstop it already is. + +`count`/`cap` widen to `uint16_t`, so the ceiling stops being a `uint8_t`. `kMaxIrOps` and `kCodeCap` +survive as upper *sanity* bounds — a runaway script fails with a diagnostic rather than exhausting +the heap — not as the working limit. + +Both allocations are freed when compilation ends: they are compile-time scratch, not part of the +running program. The only thing that outlives a compile is the exec block, which is unchanged. + +### 2. Spill to the frame (removes the register wall) + +**The frame is a call frame, not a slot file.** Script-local functions are the next feature: callable +from the script, taking arguments, containing loops and `if`, calling other functions, and +recursive. A recursive function's spill slots cannot be one fixed region — each activation needs its +own — so slots are addressed as offsets from a frame pointer that a prologue establishes, which is +the layout a nested call reuses by pushing another frame. Spilling one top-level program is what +ships here; the frame discipline is chosen so functions add a call sequence rather than a redesign. + +**New IR ops** ([MoonLiveIr.h](src/core/moonlive/MoonLiveIr.h)): + +```cpp +Spill, // slot[imm] = a +Reload, // dst = slot[imm] +``` + +**New core pass** `src/core/moonlive/MoonLiveSpill.{h,cpp}`: + +```cpp +/// Rewrite `ir` so no op names a vreg the target does not have, inserting Spill/Reload against a +/// fixed slot file. False when even the spilled form does not fit (fail, never miscompile). +bool spillToBudget(IrProgram& ir, const RegBudget& budget); +``` + +**Algorithm: linear-scan register allocation** (Poletto & Sarkar) over the op array, with +**loop-extended live intervals**. Three passes, no heap beyond the interval array: + +- **Find loops.** The grammar has no `break`, `continue` or `goto`, so a loop is exactly a + `BranchNe` whose target label is bound earlier in the array. The op array is therefore already in + reverse-postorder and no CFG needs building — that is the one bespoke simplification, and it + carries a guard: a branch pattern that is *not* properly nested makes the pass refuse rather than + allocate against a wrong interval, so a future `break` fails loudly instead of miscompiling. +- **Naive intervals**, then **loop extension**: any value live at a loop header is live to the end of + that loop, applied innermost-first. This is the step naive "first def to last use" gets wrong, and + it is conservative — it can only lengthen an interval, so it may cost a needless spill but never + produces a wrong one. +- **Scan**, spilling the active interval with the furthest end when no register is free. + +**Backend surface** — identical on all three, and the algorithm appears nowhere in the platform layer: + +```cpp +void prologue(uint8_t slots); // slots == 0 emits nothing: a non-spilling script pays zero +void spillStore(Reg r, uint8_t slot); +void spillLoad(Reg r, uint8_t slot); +``` + +Each lowerer gains two switch arms and *loses* its hand-rolled budget bail (the three duplicated +`vregsUsed + N > kRegCount` checks collapse into the one core pass). + +Where the slots live differs per target and is the real per-backend work: + +| backend | frame today | spill slots | +|---|---|---| +| Xtensa | whole-routine 48 B from `entry a1, 48`; `call()` uses 16/20/24/28 | bytes 32–47 are free — 4 slots at zero cost; `entry` immediate grows for more | +| RISC-V | **none outside `call()`** (`prologue()` is empty) | needs a real 2-instruction prologue/epilogue; `encSw`/`encLw` already exist at file scope | +| arm64 | **none outside `call()`**; the `call()` frame is 100% full | needs a prologue *and* two new `str`/`ldr` encoders — the only backend with no general store/load | + +### 3. Conditional inline scratch (the cheap part of the register fix) + +All three lowerers reserve scratch vregs for `FillElems` unconditionally — `+3` on host, `+2` on +Xtensa and RISC-V — even when the program contains no such op. A layout script never emits one. This +single unconditional reservation is what makes `grid.mlv` (11 vregs, budget 12) fail on Xtensa. +Core reports which inline ops a program actually contains; each backend maps that to its own scratch +count. Nested loops compile on Xtensa from this alone, and it is the `RegBudget.reserved` field the +spill pass consumes — not throwaway. + +## Files + +- `src/core/moonlive/MoonLiveIr.h` — heap op array, `uint16_t` counts, `Spill`/`Reload`, + `inlineScratch()`, the raised `kMaxVRegs`/`kIrLabels` +- `src/core/moonlive/MoonLiveSpill.{h,cpp}` — **new**, the linear scan +- `src/core/moonlive/MoonLiveCompiler.cpp` — the sizing pre-pass, the pass call site (~line 516), the + temp allocator's failure path (lines 141-160), `locals[]` and `kMaxCtrls` +- `src/light/moonlive/MoonLive{Layout,Effect,Modifier}.h` — the bindings' `ctrlNames_` name pool + mirrors `kMaxCtrls` and must grow with it, or the extra controls compile but never appear in the UI +- `src/core/moonlive/moonlive_emit.h` — the `RegBudget` seam +- `src/platform/esp32/moonlive_asm_riscv.cpp` — the `kScratchFn` fix (line 20), prologue + spill surface +- `src/platform/desktop/moonlive_asm_host.{h,cpp}` — new `str`/`ldr` encoders, prologue; the only + backend tests execute +- `src/platform/esp32/moonlive_asm_xtensa.{h,cpp}` — promote the private `s32i`/`l32i` lambdas to the + spill surface +- the three `moonlive_lower_*.cpp` — two switch arms each, minus their budget bails +- `moondeck/moonlive/` — generalise `emit_xtensa.cpp` to an ISA flag so `disasm.py --isa riscv` works +- `docs/moonmodules/light/MoonLive*.md`, `moonlive/README.md` — the new limits +- `docs/backlog/backlog-light.md` — narrow `call()` save-sets, by name + +Add a `static_assert` per backend that no scratch register appears in its vreg map — the invariant +the RISC-V bug broke, made unbreakable rather than commented. + +## Verification + +The governing risk: **only arm64 is executed by tests**; Xtensa and RISC-V are compile-time-excluded +and validated on hardware. So arm64 carries the correctness proof, and the device backends carry +only encoding risk, which `disasm.py` retires without a flash. + +1. **The key test — a squeezed budget on the host.** A test-only budget override runs + `spillToBudget` with a register count *smaller* than the host's, forcing the spiller to run on the + one backend that executes. Same script compiled at full and squeezed budgets must produce + identical pixels. This makes the hard algorithm testable rather than hardware-only. +2. **The back-edge case specifically**: a nested loop at a squeezed budget where the counters are + guaranteed spilled — every expected light placed exactly once. +3. **Spill across a call**: value spilled, `random16()` called, value used. Proves slots survive + `call()`'s own frame — the RISC-V case to check hardest, since prologue and `call()` both move sp. +4. **One test per ceiling**, each a script that fails today and must pass after — this is what proves + the overview table was actually delivered rather than partly delivered: + - 30+ straight-line statements (`kMaxIrOps`, fails at 7 today) + - a script whose emitted code exceeds 768 bytes (`kCodeCap`) + - 6-deep loop nesting (`locals`) and 10+ loops in one script (`kIrLabels`/`kMaxLabels`/`kMaxFixups`) + - 12 declared controls (`kMaxCtrls`) — and the *binding* surfaces all 12, since it mirrors the cap + in its own name pool + - a script needing more than 16 live values (`kMaxVRegs`, only reachable once spilling works) +5. **Stack, not just heap**: assert the compile path's stack frame *shrank*. `IrProgram` and the + assembler stop being 3.4 KB of stack locals; a one-statement script must allocate proportionally + less than a hundred-statement one. Without this the change could pass every functional test while + quietly moving the bootloop somewhere else. +6. **Degrade**: a deliberately absurd script fails with a clear diagnostic and no crash; an alloc + failure in `reserve()` fails the compile cleanly rather than writing through a null pointer. +6. **Unchanged behaviour**: `unit_moonlive_ir` / `unit_moonlive_fill` (the `fill` behavioural golden, + and kArg4 surviving a call) stay green — they pin that a `FillElems` program still gets its + scratch. `unit_MoonLiveScripts.cpp:118` (bare vs commented produce equal length) is the canary for + the pass accidentally becoming source-dependent. +7. **Encodings on device backends without flashing**: `uv run moondeck/moonlive/disasm.py` on a + spilling script, reading the actual `s32i`/`l32i` offsets against the frame layout. This is the + tool that found the `Mov`→`addi 0` bug. Extend it to RISC-V, which has no equivalent today. +8. **Memory + hot path**: `collect_kpi.py --commit`. The IR allocation is cold-path, but a modifier + script runs once per light, so measure a mapping rebuild on a large grid. Confirm a non-spilling + script emits no prologue and costs nothing. +9. **Hardware, the final gate (PO)**: flash `grid.mlv` on an S3 (Xtensa — the target that fails + today) and a P4 (RISC-V — the target with the scratch bug), and look at the wall. + +## Suggested commit boundaries + +The PO decides commits and branches; this is the order that keeps each step independently +verifiable, riskiest-last: + +1. RISC-V `kScratchFn` aliasing fix + the `static_assert`s (independent bug) +2. Conditional inline scratch — nested loops compile on Xtensa +3. Right-sized IR + code buffer, and the cheap ceilings raised (`locals`, `kMaxCtrls`, `kIrLabels`, + `kMaxFixups`) — the 7-statement wall goes and the compile stack shrinks +4. Spill surface on the three assemblers (encodings verifiable in isolation, dead code until 5) +5. The core spill pass + `kMaxVRegs` raised — the register ceiling goes + +Steps 1–3 deliver the ceiling a user meets first and are independently shippable; 4–5 are the +spiller. If the branch needs splitting for review size, that is the seam. diff --git "a/docs/history/plans/Plan-20260813 - MoonLive on a stack machine \342\200\224 the frame is where values live.md" "b/docs/history/plans/Plan-20260813 - MoonLive on a stack machine \342\200\224 the frame is where values live.md" new file mode 100644 index 00000000..aa7e9619 --- /dev/null +++ "b/docs/history/plans/Plan-20260813 - MoonLive on a stack machine \342\200\224 the frame is where values live.md" @@ -0,0 +1,402 @@ +# Plan: MoonLive on a stack machine — the frame is where values live + +Supersedes [Plan-20260809 — MoonLive scales](Plan-20260809%20-%20MoonLive%20scales%20%E2%80%94%20right-sized%20IR,%20and%20the%20stack%20as%20the%20register%20overflow.md), +whose steps 1–3 shipped and stand. This replaces its steps 4–5 (the register allocator) with a +different answer to the same goal. + +## Why the change + +The 20260809 plan set the right goal — a script bounded by memory, not by register count — and +reached for register allocation with spilling. That shipped, works on the host at every budget from +14 registers down to 10, and is textbook-correct. + +It still leaves MoonLive broken on Xtensa, and the arithmetic says it always will: + +```text +10 registers (a2..a11) − 1 inline scratch = 9 + − 5 fixed ABI vregs = 4 + − 4 reload temps = 0 keepable +``` + +Zero keepable registers means the allocator has nothing to allocate with, so **every looped script +is refused on Xtensa** — `codegen failed (unsupported on this target, or too large)`. Bench-measured +on an S3: straight-line scripts run, every `for` is refused. The ten is not negotiable: a12/a13 are +call scratch and the address register for `store8`, and a14/a15 carry the routine's own `retw.n` +return linkage (using them as vregs is what corrupted the return path and produced +`Guru Meditation (IllegalInstruction)` on every scripted layout). + +So on the smallest supported target the allocator's working budget is zero. A design whose margin is +zero on the platform it exists to serve is the wrong design, however correct its algorithm. + +## The decision: a stack machine + +**Every script variable gets a home in the call frame.** A read is a load from a frame offset, a +write is a store to it. Registers hold expression temporaries only, for the span of one expression. + +It is also the shape prior art on this chip settles on: every variable carries a stack position, +values move by `l32i`/`s32i` against the frame pointer, and registers come from a small rotating +pool popped for expression temporaries. + +### Why this is subtraction, not a sideways move + +The register model makes every language feature interact with the allocator. The stack model makes +each one an application of the same recursive mechanism: + +| feature | register machine | stack machine | +|---|---|---| +| nested `for` | loop-extended live intervals, innermost-first | nesting is nesting | +| `if` | live-range merge at the join | a branch over a region; storage is untouched | +| function call | live-across-call analysis, a save-set per call site | push args, `call`; callee addresses its own frame | +| function arguments | bounded by spare vregs (ours is fixed at 3) | as many slots as you push | +| recursion | a fixed slot file cannot hold two activations | each activation gets its own frame | +| local functions calling each other | compounding of all of the above | nesting again | + +In a stack machine a function's frame size falls out of the same stack position that placed the +variables, and an argument is one more slot in that frame. One mechanism, three jobs — locals, +arguments and the frame itself. + +What this deletes: `MoonLiveSpill.{h,cpp}` (375 lines), `RegBudget` and its 34 plumbing sites across +three lowerers, the reload-temp accounting, loop-extended interval analysis, and `kMaxVRegs` as a +ceiling. CLAUDE.md Principle 3: *the first question on any change is what it can remove.* + +### What it costs + +A variable access becomes a load or a store rather than a register reference — roughly two extra +instructions per touch. On arithmetic-heavy inner loops expect 20–40% slower; on scripts dominated +by host calls (`setRGB`, `beat`, `sin`, and every layout's `addLight`, which is a `call8`) the +difference is close to nothing, because the call already dwarfs a pair of loads. + +Two things make that the right trade today. Architecture.md's claim is "near-hand-written speed **in +the hot path**", and the hot path is the inline buffer write and the host calls, not counter +arithmetic. And the honest comparison is not slower-versus-faster: on Xtensa the current alternative +is *refused*. A script that runs slightly slower beats a script that does not compile. + +### One implementation per construct, not one per target + +A stack machine makes most of the lowering ISA-independent, and that has to be spent on removing +duplication rather than replicating a simpler design three times. Measured today: the three lowerers +are 142/138/150 lines and roughly **65% identical** (48 differing lines between Xtensa and RISC-V +after normalising names) — the same walk over the same IR, three times. + +What is genuinely per-target is small and nameable: instruction encodings, the register/frame ABI, +and the branch forms. Everything above that — how a `for` becomes an entry guard and a back edge, +how a call passes arguments, how an expression evaluates, where a variable's frame slot is — is one +algorithm that belongs in core, written once. + +**The rule for this rework:** a new target should be a new *assembler* (encodings + ABI constants), +not a new lowerer. Adding an 8086, an ARM32, or anything else must not mean copying the IR walk +again. Where a construct genuinely differs per ISA, the difference is a named hook on the assembler +surface — not a forked copy of the surrounding logic. This is CLAUDE.md Principle 3: core owns the +hard construct, written once; the platform layer holds only what is truly platform-specific. + +### One binding, three roles + +The same duplication exists in the light domain. `MoonLiveLayout`, `MoonLiveEffect` and +`MoonLiveModifier` are 196/140/182 lines and each carries its own copy of the same machinery: +`char script_[32]`, `uint32_t compiledHash_`, a `MoonLive engine_`, and the load-compile-cache-report +sequence around them. A fourth role (a scripted DRIVER) is coming, and it must not mean a fourth copy. + +Factor the shared part into one place — the script name, the compiled-content hash, the engine, the +compile-on-demand-and-report path, and the "a failed load leaves a decided state" rule that a bug +this session had to fix in the layout alone. What stays per-role is only what genuinely differs: +what the compiled program is called with, and what it does with the result. + +Architecture.md already states the standard this meets: *"When a scripted binding needs a mechanism +its compiled sibling does not, that is a finding: either the mechanism belongs in the base for +everyone, or the divergence needs its reason stated where it is introduced."* Three copies of the +same mechanism is the same finding, one level down. + +### One system-variable vocabulary, not three + +The three roles are handed three different `SysVarTable`s today — `layoutSysVars` (the clock only), +`effectSysVars` (+ `width`/`height`/`depth`) and `modifierSysVars` (+ `x`/`y`/`z`). **Collapse them +into one table every script gets**, and let the script author use what makes sense for the job. + +The current split buys less than it costs. It does not prevent a mistake — a layout that reads +`width` gets a compile error rather than a wrong answer, which is the same outcome as reading a +variable that is always zero — and it creates a trap: the tables are *different vocabularies, not +nested ones*, so a name means one thing in one role and is reserved in another. That trap is not +theoretical. A layout may use `x`/`y` as ordinary loop counters precisely because it is NOT handed +them, which the shipped `grid.mlv` does — and `disasm.py`, which compiled everything against the +widest table, therefore refused the one script most worth inspecting with "name is a system +variable". The tool was blind to the default layout for as long as it existed, and that cost more +debugging time this session than any single bug. + +Two changes make one table work: + +- **`width`/`height`/`depth` mean the same thing in every script**: the dimensions of the grid. A + layout DEFINES them by the coordinates it places; an effect and a modifier READ them. Same name, + same meaning, no per-role reservation. +- **A modifier's per-light coordinate is renamed `xPos`/`yPos`/`zPos`.** `x`/`y`/`z` are the names a + script author naturally reaches for as loop counters, so reserving them globally would break the + most ordinary code there is. The renamed form is unambiguous, and it frees `x`/`y`/`z` for their + obvious use everywhere. + +The arena offsets are already fixed constants (`kSysWidth`, `kSysX`, …) shared by every role, so +this is removing a distinction the storage layer never made — not introducing one. What a binding +still decides is which slots it WRITES each frame; reading is uniform. + +This is a breaking change for any script using a modifier's `x`/`y`/`z`, so it needs its +[MIGRATING.md](../../MIGRATING.md) entry and a sweep of the shipped `moonlive/` scripts. + +### Clean first, with speed decisions made deliberately + +Build the industry-standard construct first and keep it whole. Where a compromise is genuinely +needed for speed, make it **at that moment, explicitly, with its reason recorded next to it** — not +by pre-emptively complicating the design against a cost nobody has measured. + +That means: no speculative fast paths, no "we might need this in a register" hedges, and no +special-casing a construct because it *might* be hot. Write the clean form, measure it +(`collect_kpi.py`, and the bench), and when a number says a specific thing is too slow, fix that +specific thing and say why in a comment where it lives. A compromise with its rationale attached is +maintainable; a design pre-bent around an unmeasured fear is not. + +### The limits: which are physics, which are choices + +"Bounded by memory, not by constants" needs to know which constants can actually move. Measured: + +**Hardware-fixed — these cannot be raised at all.** + +| limit | ceiling | why | +|---|---|---| +| `kRegCount` | Xtensa **10**, RISC-V 14, host 14 | the machine's register file — the wall this whole plan routes around | +| `kArenaBytes` | 255 | `LoadCtrl` lowers to `l8ui`/`lbu`, whose offset immediate is one byte | + +**Chosen numbers, far below what the hardware allows — raisable when there is a reason.** + +| limit | now | hardware allows | what raising costs | +|---|---|---|---| +| `kMaxSpillSlots` / `kMaxLocals` | 16 | **243** on Xtensa (`s32i`'s offset byte counts 4-byte words; `entry` reaches a 32 KB frame), **512** on RISC-V | 16 B per local in `locals[]`, plus frame bytes per running script | +| `kMaxVRegs` | 32 | 255 (the index is a `uint8_t`) | `Interval iv[]` in the spiller, ~16 B per vreg of compile stack | +| `kIrLabels` | 16 | 255 | 32 B per lowerer | +| `kMaxCtrls` | 8 | the arena's 255, minus the system variables | 16 B each — and the UI has to render them | + +**Already unbounded in practice.** `kMaxIrOps` and `kCodeCap` size HEAP allocations that are already +right-sized per script, and `platform::alloc` prefers PSRAM where a device has it. They are sanity +bounds so a runaway source fails with a diagnostic rather than exhausting the heap — not working +limits. The remaining fixed arrays total roughly 600 bytes per compile (`locals[16]` at 256 B is the +largest); moving those to the heap would add allocation, failure paths and lifetimes to save half a +kilobyte on a cold path, which is the opposite of subtraction. + +**`kMaxLocals` and `kMaxSpillSlots` must move together.** They index the ONE frame: the front end +numbers a script's variables from zero and the register allocator numbers its spills above them, so +raising one alone silently shrinks the other's room. `spillToBudget` refuses a compile when the front +end asks for more slots than the backend can address, which turns a mismatch into a diagnostic rather +than a truncated offset writing over a live value. + +**Do not raise any of these speculatively.** Once variables live in the frame, frame slots — not +registers — are what a complex script consumes, so `kMaxLocals` becomes the real ceiling on script +size. Set it from a measurement once the mechanism exists, not from a guess before it does. + +### Register optimization is explicitly out of scope + +Deliberately deferred, in full, including the tempting parts: + +- No promotion of hot values into registers. +- No pinning of the fixed ABI vregs (`buf`, `nLights`, `cpl`, `ctrls`, `t`) — **whether any of them + stay in registers is itself a register decision, and it is reopened, not assumed.** They may all + live in the frame in the baseline. +- No caching of the innermost loop counter. + +The reason is that register optimization intertwines with everything it touches; adding it to a +design that already has it half-present makes both harder to read. Get a clean stack machine first, +measure it, and then decide what — if anything — to promote, as an addon over a correct baseline. +That inverts the failure mode from "refuse to compile" to "compile, possibly slower", which is the +Robustness principle. + +## Verification comes first + +Today's session produced five wrong theories about emitted Xtensa code, each plausible from reading +the source, each falsified by the device. The lesson is that **no test executes Xtensa code** — only +arm64 runs in tests, so a codegen defect ships silently and is debugged by flashing. + +That gap is fixed **before** the rework, not after: + +1. **Golden-bytes tests over the shipped scripts.** `disasm.py` emits Xtensa for a script on the + host; assert the byte stream for `grid.mlv` and friends. Any codegen change that alters emission + fails loudly and visibly, with the diff readable. +2. **A structural checker on emitted code**: every branch and jump target lands on an instruction + boundary inside the program; frame offsets stay inside the frame `entry` allocated; no + instruction names a register outside the backend's map. These are the three defect classes found + today, each made unrepeatable. +3. **Keep the squeezed-budget behavioural tests.** They pin that the same script computes the same + pixels regardless of storage decisions — which is exactly the invariant this rework must preserve. + +`disasm.py` itself needed two fixes to be usable at all (it did not link `MoonLiveSpill.cpp`, and it +compiled every script against `modifierSysVars()` — so `x`/`y` were reserved and it had never once +successfully read the shipped `grid.mlv`). Tool blindness cost more time this session than any bug. + +## Sequence + +Each step is independently verifiable, and the branch stays green throughout: + +1. ✅ **The emitted-code tests above**, against current behaviour. Two per-ISA test TUs share one + body, so the checks are written once: the device backends now compile and run on the development + machine through a `lower` seam on `compileSource`. 11 tests. +2. ✅ **A frame slot per script variable.** A loop's counter and limit each get a slot; reading a + variable emits a Reload into a temp that dies immediately. The guard that protected locals' + registers is GONE — every vreg reaching freeTemp is now a temp. Measured on Xtensa: grid.mlv + 212 → 186 bytes, and three-deep nesting compiled for the first time. +3. ✅ **Registers become expression temporaries.** Call arguments are staged through the frame — + each is parked as soon as it is computed and all are reloaded for the one instruction that reads + them, so only ONE argument holds a register at a time. This is what let a looped effect, a + four-deep nested layout and plasma compile on Xtensa at all. +3b. ✅ **The HOST ARGUMENTS go to the frame too** (`buf`, `nLights`, `cpl`, `t`, `ctrls`). They are + read by the inline ops and LoadCtrl and never written, yet they permanently occupied FIVE + registers — on Xtensa, five of the six the windowed ABI leaves a routine that calls. Parked at + entry and reloaded where read, they cost a load at the point of use and free the register file + for temporaries. This belongs in CORE, not in a lowerer: it is the same "the frame is where + values live" rule the script's own variables follow, and doing it per backend would be three + copies of one policy — the duplication step 4 exists to remove. +3c. ✅ **Unlimited call arguments.** Shipped as designed: `HostCallFn` takes `(const uintptr_t* args, + uint32_t argc, const uint8_t* arena)`, the three `call()` sequences are untouched, and arity is + bounded by frame slots (`kMaxCallArgs = kMaxLocals`, 16). The seven-argument `line()` the section + below argues for is now an ordinary builtin. The plan's own table promises "as many slots as you push", and + the machinery for it already exists — `parseCall` parks every argument in a CONSECUTIVE frame + slot as it is evaluated. Three arbitrary constants cap it anyway: `VReg args[4]`, `n >= 4`, and + "a call takes at most three arguments" (`HostCallFn` is a 3-parameter C function pointer). + + That cap is a real design limit, not a detail. A `line(x0, y0, x1, y1)` does not fit, and the + workaround — splitting it into `lineH`/`lineV`, or setting colour through ambient state — is + exactly the bespoke special case a stack machine exists to avoid. Every builtin added after it + would inherit the same distortion. + + **Do NOT widen `HostCallFn` and the three assemblers' `call()`.** That spends the change on the + most fragile code in the project (the Xtensa call sequence has produced three separate defects) + and still leaves a fixed maximum, just a larger one. + + **Pass a POINTER to the argument slots.** The arguments are already in consecutive frame slots; + the call just has to say where. Each assembler already knows its own frame layout — `spillStore` + and `spillLoad` compute exactly this address — so each can materialise `framePtr + argBase*4` + into the first argument register. The host signature becomes: + + ```cpp + using HostCallFn = uint32_t (*)(const uint32_t* args, uint32_t argc, const uint8_t* arena); + ``` + + Three parameters, so `call()` and its 3-argument sequence are UNCHANGED in all three assemblers — + the fragile Xtensa windowed-call code is not touched. Arity is bounded by frame slots, which is a + memory question: the goal of this plan. + + NOT the arena: its bytes are `uint8` (the offset is an 8-bit immediate in `l8ui`/`lbu`), so it + cannot carry a `uint32_t` argument. Frame slots are 4-byte words and already hold full values. + + Each existing builtin becomes `args[0]`, `args[1]`, `args[2]` instead of named parameters — a + mechanical change, and one that finally lets `line(x0, y0, x1, y1)` and the seven-argument + `draw::line` be ordinary calls rather than a special case. Script-local functions will pass their + arguments the same way when they arrive. + +4. ⬜ **Collapse the three lowerers into one.** With storage no longer per-target, the IR walk is one + algorithm; what remains per-backend is encodings, the frame/ABI constants and the branch forms. + Verify by adding nothing to the platform layer that is not genuinely platform-specific. +5. ⬜ **Delete the allocator** (`MoonLiveSpill.{h,cpp}`, `RegBudget`, the per-backend budget plumbing) + once nothing calls it. The `prologue(slots)` / `spillStore` / `spillLoad` surface on the three + assemblers is KEPT — it is already frame-addressed through a frame pointer for exactly this + reason, and the stack machine uses it directly. +6. ⬜ **One system-variable table for every role**, with a modifier's coordinate renamed to + `xPos`/`yPos`/`zPos`. Independent of the codegen work — different files, no shared risk — so it + can land whenever, but it comes before step 7 because it is what makes the bindings differ by + almost nothing. Needs a MIGRATING entry and a sweep of the shipped scripts. +7. ⬜ **Factor the three bindings onto one shared base**, so the script name, hash, engine and the + compile-and-report path exist once. The scripted driver that follows should then be a small + subclass, and that is the test of whether this step actually worked. +8. ✅ **Bench: S3 and P4**, a scripted layout and a scripted effect, both with nested loops. Done on + FOUR boards (S3, classic ESP32, P4, S31), scripted layout + effect, plasma and the heavier + ripples, after the Xtensa frame fix below. +9. ✅ **Measure** with `collect_kpi.py` and record the cost honestly in performance.md, so the later + decision about register promotion is made against numbers rather than intuition. + +Steps 4, 6 and 7 are the deduplication, and they come AFTER the mechanism works rather than during it — +collapsing three copies while the design underneath is still moving would mean doing it twice. + +## Xtensa: the one target still failing, and why + +**Status after steps 1-3: two of three targets run the whole stack.** Desktop (arm64) renders a +scripted grid layout with a plasma effect at 76,923 fps. An ESP32-S31 (RISC-V) has held layout + +effect + modifier for over an hour. An ESP32-S3 (Xtensa) crashes on any script that stores a pixel, +and did so before this rework too — this is a pre-existing backend defect the stack machine exposed, +not one it introduced. + +### What a windowed register ABI is + +Most CPUs have a **flat** register file: one set of registers, and a function that wants to keep a +value across a call must save it to the stack itself. Every save is an instruction you can read in +the disassembly. + +Xtensa instead has a large physical register file (64 registers) of which a function sees a **window** +of 16 at a time. `call8` does not jump — it *rotates the window by eight* before jumping. The callee's +`a0..a7` are physically the caller's `a8..a15`, and the callee's `entry` instruction slides the window +further. Nothing is saved by an instruction; the renaming IS the save. When the physical file wraps +around, hardware exception handlers spill the oldest window to the stack automatically. + +### Why Xtensa has it and the others do not + +It was a 1990s answer to "calls are expensive": rotating a window makes a call cheaper than pushing +registers, which mattered when memory was slow relative to the core. SPARC made the same choice. +Modern designs went the other way — RISC-V and ARM64 both use a flat file with an explicit +caller/callee-saved split, because compilers got good at register allocation and a predictable, +visible ABI is worth more than saved store instructions. Xtensa is a configurable core and ESP32 +ships the windowed option, so the ESP32 classic and S3 have it; the P4 and S31 are RISC-V and do not. + +### Why this breaks a JIT specifically + +A compiler emitting Xtensa knows the rule and never puts a live value in `a8..a15` around a call. Our +backend chose its vreg map by counting free registers rather than by asking which survive a call: +`kXtReg` is `a2..a11`, and ESP-IDF's own `coreasm.h` states plainly that `a8..a15` are **clobbered** +by `call8`. So four of ten vregs sat inside the rotation window. The measured crash had `A0 = 0x100` +— the value 256, which is `nLights`, a script value that had landed in the return-address register. + +Three separate defects of this family have already been found and fixed here: `a14`/`a15` used as +vregs (they carry the `retw.n` linkage), the call RESULT stashed in `a12` (the callee's `a4` after +rotation, so the callee overwrote it), and branch displacements truncated past ±127. None of them can +exist on a flat-file backend, which is exactly why arm64 and RISC-V never showed a symptom. + +### How to deal with it: make it look flat + +**Yes — the windowed ABI can be treated as flat, by simply not using the window.** Two options: + +1. **Restrict the vreg map to `a2..a7`** — the six registers that survive `call8` — and treat + `a8..a15` as if they did not exist. The rotation still happens on a call, but no value we care + about lives in the rotated range, so it becomes invisible. This is the smaller change and it makes + the Xtensa backend behave exactly like the flat ones. + +2. **Use `call4` instead of `call8`** — rotating by four leaves `a4..a15` intact for the caller. It + widens the usable range, but the callee then sees a smaller window, and every host builtin is + ordinary compiled C we do not control. Rejected: it constrains code we do not own. + +Option 1 is the plan. The blocker is arithmetic, and it is precisely what step 3b removes: six +registers minus the five permanently held by the host arguments leaves one, which fits nothing. Park +the host arguments in the frame — the same rule every other value now follows — and six registers is +ample, because after steps 2 and 3 registers only ever hold one expression's temporaries. + +**So the Xtensa fix is not a special case; it is step 3b plus a one-line map change.** That is the +argument for doing 3b next rather than treating Xtensa as its own problem: the same subtraction that +simplifies all three backends is what makes the smallest one correct. + +An attempt at 3b during this session got every shipped script compiling on Xtensa at eight registers, +then hit a SIGSEGV in the HOST backend and was reverted. The cause is known: register numbering is +derived independently in four places — core's compaction, the spill pass's reservations, each +lowerer's scratch arithmetic, and each assembler's map — and moving the host arguments perturbs all +four. `src/core/moonlive/register-and-slot-contract.md` now writes that ownership down, and 3b should +be re-attempted against it rather than by iteration. + +**RESOLVED, and this section's diagnosis was only half right.** Step 3b landed and the vreg map +stayed at ten, not six: no value was ever lost to the ROTATION, because `call()` saves and restores +a8..a11 around the call itself. The real defect was one the analysis above does not reach. The +window-overflow handler spills a frame's a4..a7 into the frame's OWN top 32 bytes, and the emitter +reserved 16, so the parked arena pointer of step 3b sat in hardware-owned memory and any interrupt +during a host call destroyed it. Frame LAYOUT, not register choice; spatial, not temporal; and +invisible to every encoding check because each instruction was correct. See +[lessons § the register-window frame bug](../lessons.md#lessons-from-the-moonlive-on-xtensa-branch-the-register-window-frame-bug). +All four boards (S3, classic, P4, S31) now run scripted layouts and effects. + +## Then, separately + +Only after the above is on main and measured: + +- Whether to promote anything into registers, and which — including whether any of the fixed ABI + vregs earn a register at all. +- Script-local functions with arguments, `if`, and recursion — the features this design exists to + make cheap. +- A scripted DRIVER as the fourth role, which is the honest test of step 6: if it needs more than a + small subclass, the factoring did not go far enough. diff --git a/docs/metrics/repo-health.json b/docs/metrics/repo-health.json index 69cf5d69..88b746db 100644 --- a/docs/metrics/repo-health.json +++ b/docs/metrics/repo-health.json @@ -1,21 +1,22 @@ { - "commit": "9d77ade4", + "commit": "e9a7cca8", "flash": { - "esp32": 1762368, - "esp32p4-eth": 1604272, + "esp32": 1722656, + "esp32p4-eth": 1611632, "esp32p4-eth-wifi": 1793760, - "esp32s3-n16r8": 1753232, + "esp32s3-n16r8": 1761424, "esp32s3-n8r8": 1753232, - "esp32s31": 2025600, - "desktop": 1137928, + "esp32s31": 2033152, + "desktop": 1156376, "esp32-16mb": 1714608, - "esp32-eth": 1324928, - "esp32-wrover": 1765504 + "esp32-eth": 1324816, + "esp32-wrover": 1765504, + "qemu": 1318160 }, "perf": { "desktop": { - "tick_us": 129, - "fps": 7751 + "tick_us": 179, + "fps": 5586 }, "esp32": { "tick_us": 2151, @@ -23,54 +24,54 @@ } }, "loc": { - "core": 16980, - "light": 24317, - "platform": 12860, + "core": 17763, + "light": 24625, + "platform": 13583, "ui": 6468, - "test": 41246, - "moondeck": 20323 + "test": 42195, + "moondeck": 20830 }, "comments": { "core": { - "lines": 6402, - "ratio": 0.41 + "lines": 6766, + "ratio": 0.414 }, "light": { - "lines": 9427, - "ratio": 0.429 + "lines": 9581, + "ratio": 0.43 }, "platform": { - "lines": 4365, - "ratio": 0.375 + "lines": 4768, + "ratio": 0.387 }, "ui": { "lines": 1670, "ratio": 0.274 }, "test": { - "lines": 7169, - "ratio": 0.201 + "lines": 7451, + "ratio": 0.204 }, "moondeck": { - "lines": 3246, - "ratio": 0.183 + "lines": 3353, + "ratio": 0.184 } }, "tests": { - "cases": 1325, + "cases": 1345, "scenarios": 23 }, "docs": { "md_files": 178, - "md_lines": 24401, + "md_lines": 24507, "plans_files": 91, - "backlog_lines": 3629, - "lessons_lines": 454, + "backlog_lines": 3654, + "lessons_lines": 503, "claude_md_lines": 135 }, "complexity": { - "functions": 2473, - "over_threshold": 151, - "worst_ccn": 93 + "functions": 2524, + "over_threshold": 158, + "worst_ccn": 105 } } diff --git a/docs/metrics/repo-health.md b/docs/metrics/repo-health.md index 374c9616..610d36be 100644 --- a/docs/metrics/repo-health.md +++ b/docs/metrics/repo-health.md @@ -1,6 +1,6 @@ # Repo health -Measured at `9d77ade4`. Generated by [`moondeck/check/repo_health.py`](../../moondeck/check/repo_health.py) on every KPI-gate run. **Do not edit by hand.** +Measured at `e9a7cca8`. Generated by [`moondeck/check/repo_health.py`](../../moondeck/check/repo_health.py) on every KPI-gate run. **Do not edit by hand.** Current state only; the trend is this file's git history (`git log -p docs/metrics/repo-health.md`). Nothing here fails a build: the numbers make growth visible, the judgment stays human. @@ -8,58 +8,59 @@ Current state only; the trend is this file's git history (`git log -p docs/metri | Target | Flash | |---|---:| -| desktop | 1,111 KB (+0 KB) ⚠ | -| esp32 | 1,721 KB | +| desktop | 1,129 KB | +| esp32 | 1,682 KB | | esp32-16mb | 1,674 KB | | esp32-eth | 1,294 KB | | esp32-wrover | 1,724 KB | -| esp32p4-eth | 1,567 KB (+0 KB) ⚠ | +| esp32p4-eth | 1,574 KB | | esp32p4-eth-wifi | 1,752 KB | -| esp32s3-n16r8 | 1,712 KB | -| esp32s3-n8r8 | 1,712 KB (+84 KB) ⚠ | -| esp32s31 | 1,978 KB | +| esp32s3-n16r8 | 1,720 KB (+0 KB) ⚠ | +| esp32s3-n8r8 | 1,712 KB | +| esp32s31 | 1,986 KB | +| qemu | 1,287 KB | ## Render performance | Target | Tick | FPS | |---|---:|---:| -| desktop | 129 µs (+1 µs) ⚠ | 7,751 (−61) ⚠ | +| desktop | 179 µs (+1 µs) ⚠ | 5,586 (−31) ⚠ | | esp32 | 2,151 µs | 464 | ## Code | Area | Lines | Comments | Comment share | |---|---:|---:|---:| -| core | 16,980 | 6,402 | 41.0 % | -| light | 24,317 (+5) ⚠ | 9,427 | 42.9 % | -| platform | 12,860 | 4,365 | 37.5 % | +| core | 17,763 (+6) ⚠ | 6,766 | 41.4 % | +| light | 24,625 (+11) ⚠ | 9,581 | 43.0 % | +| platform | 13,583 | 4,768 | 38.7 % | | ui | 6,468 | 1,670 | 27.4 % | -| test | 41,246 | 7,169 | 20.1 % | -| moondeck | 20,323 | 3,246 | 18.3 % | +| test | 42,195 (+56) ⚠ | 7,451 | 20.4 % | +| moondeck | 20,830 (+22) ⚠ | 3,353 | 18.4 % | ## Tests | Kind | Count | |---|---:| -| unit cases | 1,325 | +| unit cases | 1,345 (+2) ✓ | | scenarios | 23 | ## Complexity | Metric | Value | |---|---:| -| functions | 2,473 (+1) ✓ | -| over threshold | 151 | -| worst CCN | 93 | +| functions | 2,524 | +| over threshold | 158 | +| worst CCN | 105 | ## Documentation | Metric | Value | |---|---:| | markdown files | 178 | -| markdown lines | 24,401 | +| markdown lines | 24,507 (+10) ⚠ | | plan files | 91 | -| backlog lines | 3,629 | -| lessons lines | 454 | +| backlog lines | 3,654 | +| lessons lines | 503 | | CLAUDE.md lines | 135 | diff --git a/docs/moonmodules/light/MoonLiveEffect.md b/docs/moonmodules/light/MoonLiveEffect.md index e22431d2..d4d624a9 100644 --- a/docs/moonmodules/light/MoonLiveEffect.md +++ b/docs/moonmodules/light/MoonLiveEffect.md @@ -4,7 +4,7 @@ MoonLive is projectMM's **live-script engine** — author an effect as text and Scripts call the same [power functions](power-functions.md) compiled effects use, reached through the builtin table — so the vocabulary is shared, in its flat scalar form. -A scripted effect carries its **script source** as an editable, persisted multi-line text control (a resizable `textarea` in the UI), and a front-end (lexer → parser → IR → per-ISA assembler) compiles it to native code on the next tick. The grammar is a sequence of **statements** — a function call, or a `for` loop over them — with **expression arguments**, so any argument may be a literal or a nested call: +A scripted effect names a **script file** under `/moonlive/`; the UI loads, edits and saves that file, and the module holds only the name (~32 bytes) — the text is read into a right-sized buffer to compile and freed immediately, so nothing script-sized stays resident. A front-end (lexer → parser → IR → per-ISA assembler) compiles it to native code on the next tick. The grammar is a sequence of **statements** — a function call, or a `for` loop over them — with **expression arguments**, so any argument may be a literal or a nested call: ``` setRGB(random16(256), 0, 0, 255); // a random pixel, blue @@ -16,7 +16,7 @@ The functions are **not built into the compiler** — `setRGB`, `fill`, `random1 ## Controls -- `source` — the script text (default: random pixels — `setRGB(random16(256), random16(256), random16(256), random16(256));`, one random light in a random color each tick). Editing it recompiles live: a valid script swaps in on the next tick; a failed compile frees the old code, shows the diagnostic in the module status, and renders dark until fixed (the script-editor loop, robust + no reboot). +- `script` — the file name under `/moonlive/`, e.g. `lines.mlv`. A fresh module has none: it reports `no script — set the script name` and renders nothing, rather than every new module compiling the same default. Naming one (or re-naming it after an edit) recompiles live: a valid script swaps in on the next tick; a failed compile frees the old code, shows the diagnostic in the module status, and renders dark until fixed (the script-editor loop, robust + no reboot). The directory is created on demand. - **Scripted controls** — a script declares a tunable variable with a range annotation, and the engine surfaces it as a real `uint8` MoonModule control (slider + UI + persistence), bound to a live value the running native code reads each tick: ```c @@ -25,7 +25,7 @@ The functions are **not built into the compiler** — `setRGB`, `fill`, `random1 setRGB(speed, hue, 0, 255); ``` - Declaring the variable is what **creates** the control: `uint8_t = ;` becomes a `` slider (default ``, range `0..255`). The trailing `// @control ..` only **adjusts that control's range**; it's optional. A declared name used in a statement reads the control's **current** value. Editing a control's slider does **not** recompile — the value lands in the engine's control-values arena and the next render tick reads it (the live-edit guarantee, the *no-reboot* principle). Editing the `source` recompiles and re-derives the control set; a control kept across the edit keeps its slider value, a removed control's saved value drops. Stage 1 is `uint8` only. + Declaring the variable is what **creates** the control: `uint8_t = ;` becomes a `` slider (default ``, range `0..255`). The trailing `// @control ..` only **adjusts that control's range**; it's optional. A declared name used in a statement reads the control's **current** value. Editing a control's slider does **not** recompile — the value lands in the engine's control-values arena and the next render tick reads it (the live-edit guarantee, the *no-reboot* principle). Saving the script file and re-naming it recompiles and re-derives the control set; a control kept across the edit keeps its slider value, a removed control's saved value drops. Stage 1 is `uint8` only. ### System variables — what the engine hands a script @@ -55,6 +55,7 @@ Registered by the light domain, not built into the compiler (the core owns only | `setXYZ(index, x, y, z)` | write one position (a [modifier](MoonLiveModifier.md)) | | `fill(r, g, b)` | write every light | | `addLight(x, y, z)` | place the next light (a [layout](MoonLiveLayout.md)) | +| `line(x1, y1, x2, y2, r, g, b)` | a straight segment on the grid, via the shared `draw::line` | | `random16(n)` | a value in `[0, n)` | | `mod(a, b)` | `a % b` — the wrap a cyclic animation needs | | `beat(bpm, t)` | a `0..65535` sawtooth at `bpm` | @@ -70,7 +71,7 @@ Registered by the light domain, not built into the compiler (the core owns only ### Wire contract — control declaration -The controls are **derived from `source`** (one per declared `uint8` control; the optional `@control` annotation only refines a control's range), then **surfaced in `/api/state`** — the device JSON view the integrator consumes — as regular `uint8` controls alongside `source`. So an integrator sees and writes them exactly like any other control — e.g. `POST /api/control` with `{"module": "ML", "control": "speed", "value": 80}`; they're fully present in the device JSON, just authored in the script rather than fixed in the module. The script's `\n` line breaks are standard JSON string escapes the device decodes, so a multi-line `source` round-trips. +The controls are **derived from the script** (one per declared `uint8` control; the optional `@control` annotation only refines a control's range), then **surfaced in `/api/state`** — the device JSON view the integrator consumes — as regular `uint8` controls alongside `script`. So an integrator sees and writes them exactly like any other control — e.g. `POST /api/control` with `{"module": "ML", "control": "speed", "value": 80}`; they're fully present in the device JSON, just authored in the script rather than fixed in the module. The script's `\n` line breaks are standard JSON string escapes the device decodes, so a multi-line script round-trips through `/api/file`. ## Pieces @@ -78,8 +79,8 @@ The controls are **derived from `source`** (one per declared `uint8` control; th - **`MoonLiveBuiltins`** (`src/core/moonlive/MoonLiveBuiltins.h`) — the **neutral host-binding seam**: a `BuiltinTable` of `{name → descriptor}`, where a descriptor is either `Call` (a host C function pointer — a pure helper like `random16`) or `Inline` (a neutral opcode tag the backend emits inline — the hot-path buffer writers, no per-pixel call). The core owns no function names; it resolves a call against whatever the host registered. - **`MoonLiveCompiler`** (`src/core/moonlive/MoonLiveCompiler.h/.cpp`) — the **platform-independent front-end**: a recursive-descent lexer + expression parser that lowers each statement to the typed IR (`MoonLiveIr.h`). Pure (source + table in, IR out, deterministic). Knows the *language*, never an ISA and never a domain. - **`MoonLiveBuiltins_light`** (`src/light/moonlive/MoonLiveBuiltins_light.h`) — the **light-domain registration**: the only place the LED vocabulary lives. Registers the whole vocabulary above — Inline ops lowering to stores, and Calls into host helpers — plus the system variables each binding supplies. A different host (display, sensor) writes its own table; the core is unchanged. -- **per-ISA assembler + lowering** (`src/platform//moonlive_asm_*` + `moonlive_lower_*`) — a tiny named-instruction MacroAssembler with label back-patching, and the IR→bytes lowering that drives it. Xtensa for the classic/S3 (`__XTENSA__`), the host ISA on desktop (arm64/x86-64). Adding an ISA is a new assembler + lowering; the front-end and IR are unchanged. (`emitFill`/`emitAnimatedFill` remain as the hand-encoded `fill` references the assembler's output is checked against.) -- **`MoonLiveEffect`** (`src/light/moonlive/MoonLiveEffect.h`) — the **thin binding**: a first-class `EffectBase` carrying the `source` control, whose `tick()` delegates to the engine over its own `buffer()`. `compile(source, table, sysvars)` takes both host tables: the shared `lightBuiltins()`, and the system variables THIS binding supplies — `effectSysVars()` here, `modifierSysVars()` for a modifier, `layoutSysVars()` for a layout, which is what decides the names each kind of script can read and cannot declare. The engine is projectMM-agnostic; the binding is the only coupled layer. +- **per-ISA assembler + lowering** (`src/platform//moonlive_asm_*` + `moonlive_lower_*`): a tiny named-instruction MacroAssembler with label back-patching, and the IR→bytes lowering that drives it. Xtensa for the classic/S3 (`__XTENSA__`), the host ISA on desktop (arm64/x86-64). Adding an ISA is a new assembler + lowering; the front-end and IR are unchanged. (`emitFill`/`emitAnimatedFill` remain as the hand-encoded `fill` references the assembler's output is checked against.) An ISA also brings its own **frame contract**, which the emitter honors before a single instruction matters: on Xtensa the top 32 bytes of every frame belong to the register-window spill hardware, enforced by a `static_assert` tied to the widest call emitted plus the structural codegen test ([why, and how it was found](../../history/lessons.md#lessons-from-the-moonlive-on-xtensa-branch-the-register-window-frame-bug)). +- **`MoonLiveEffect`** (`src/light/moonlive/MoonLiveEffect.h`) — the **thin binding**: a first-class `EffectBase` carrying the `script` control, whose `tick()` delegates to the engine over its own `buffer()`. `compile(source, table, sysvars)` takes both host tables: the shared `lightBuiltins()`, and the system variables THIS binding supplies — `effectSysVars()` here, `modifierSysVars()` for a modifier, `layoutSysVars()` for a layout, which is what decides the names each kind of script can read and cannot declare. The engine is projectMM-agnostic; the binding is the only coupled layer. ## Cross-domain wiring @@ -95,9 +96,9 @@ MoonLive's native-codegen approach — compile a small C-like language straight [unit_moonlive_fill](../../../test/unit/core/unit_moonlive_fill.cpp) runs the engine path in-process on the desktop host backend (`compile`/`run`, the animated routine, zero-lights, recompile, `free`, the `allocExec`/`writeExec`/`freeExec` round-trip, the buffer-shape guards). [unit_moonlive_ir](../../../test/unit/core/unit_moonlive_ir.cpp) pins the **behavioral golden** — a compiled `fill` and the hand-encoded reference render an identical buffer — plus setRGB's single-pixel write and the runtime bounds guard. [unit_moonlive_compiler](../../../test/unit/core/unit_moonlive_compiler.cpp) pins the expression grammar (`random16` in any/every argument slot, uint16 bounds), the parser diagnostics (no crash on malformed input), live recompile, and the **domain-neutral** property: with an empty builtin table the core knows *no* functions, and a host can register an arbitrary name against the same machinery. -The grammar + bounds guard are verified live on the S3/Olimex (Xtensa) by editing the `source` control — the device compiles the expression on-chip and renders it. +The grammar + bounds guard are verified live on the S3/Olimex (Xtensa) by saving a script file and naming it — the device compiles the expression on-chip and renders it. -[scenario_MoonLiveEffect_livescript](../../../test/scenarios/light/scenario_MoonLiveEffect_livescript.json) exercises the effect **as a wired MoonModule** — what the unit tests can't reach: add it, live-edit the `source` to recolor (recompile), push a broken script (`MoonLive::compile` fails, frees the previous code, `MoonLiveEffect` reports the parse error in the status and renders dark — no crash), recover, resize the grid to 1×1 and back while rendering (the every-grid-size hard rule), then remove and re-add (exec memory re-acquired clean). It runs in-process on the desktop backend each commit, and the same JSON runs live over REST against the device backends. The Xtensa/RISC-V backends are validated by the live S3/P4 runs (a `MoonLiveEffect` on a Layer lights the grid from its `source`), which the desktop tests can't reach. +[scenario_MoonLiveEffect_livescript](../../../test/scenarios/light/scenario_MoonLiveEffect_livescript.json) exercises the effect **as a wired MoonModule** — what the unit tests can't reach: add it, live-edit the script file to recolor (recompile), push a broken script (`MoonLive::compile` fails, frees the previous code, `MoonLiveEffect` reports the parse error in the status and renders dark — no crash), recover, resize the grid to 1×1 and back while rendering (the every-grid-size hard rule), then remove and re-add (exec memory re-acquired clean). It runs in-process on the desktop backend each commit, and the same JSON runs live over REST against the device backends. The Xtensa/RISC-V backends are validated by the live S3/P4 runs (a `MoonLiveEffect` on a Layer lights the grid from its script file), which the desktop tests can't reach. ## Source diff --git a/docs/moonmodules/light/MoonLiveLayout.md b/docs/moonmodules/light/MoonLiveLayout.md index d46d77b3..9b85dd90 100644 --- a/docs/moonmodules/light/MoonLiveLayout.md +++ b/docs/moonmodules/light/MoonLiveLayout.md @@ -77,7 +77,7 @@ So it runs twice. On the first pass `addLight` counts; on the second it emits ea | control | what it does | |---|---| -| `source` | the script; editing it recompiles and re-places the lights live | +| `script` | the file name under `/moonlive/`; naming it (or re-naming it after an edit) recompiles and re-places the lights live | Plus one control per `@control` the script declares. diff --git a/docs/moonmodules/light/MoonLiveModifier.md b/docs/moonmodules/light/MoonLiveModifier.md index 43ee9ff6..014a5384 100644 --- a/docs/moonmodules/light/MoonLiveModifier.md +++ b/docs/moonmodules/light/MoonLiveModifier.md @@ -42,7 +42,7 @@ It is for debugging and comes back out again — [what print costs](../../../moo | control | what it does | |---|---| -| `source` | the script; editing it recompiles and re-maps live | +| `script` | the file name under `/moonlive/`; naming it (or re-naming it after an edit) recompiles and re-maps live | Plus one control per `@control` the script declares — `uint8_t amount = 4; // @control 0..64` becomes a slider, and moving it rebuilds the mapping just as editing the script does. diff --git a/docs/performance.md b/docs/performance.md index 151852a5..953230d0 100644 --- a/docs/performance.md +++ b/docs/performance.md @@ -222,7 +222,7 @@ The cheapest (Lines, Checkerboard, PlasmaPalette) clear ~100 FPS even at 16K; th ### MoonLive (scripted effect) — tick + memory -A `MoonLiveEffect` compiles its `source` text to native Xtensa once on the cold path (`prepare`), then `run()` is a single function-pointer call each tick. Measured on the S3 at 16×16 (the bench grid the engine is exercised on; the per-tick cost is the native loop, not interpretation): +A `MoonLiveEffect` compiles its script to native code for whichever ISA the board runs, once on the cold path (`prepare`), then `run()` is a single function-pointer call each tick. Measured on the S3 at 16×16 (the bench grid the engine is exercised on; the per-tick cost is the native loop, not interpretation): | Script | Tick (µs) | Exec block (heap) | |--------|----:|----:| @@ -232,7 +232,17 @@ A `MoonLiveEffect` compiles its `source` text to native Xtensa once on the cold The rows above are a dated S3 bench record; the numbers below them are what a desktop run measures today. The tick cost is native-code speed — a `setRGB` is a bounds-guard + three byte stores (~26 µs including the per-tick module overhead), `fill` adds the per-light loop. The **exec block scales with the program**, not a fixed cap: a one-liner is tens of bytes of machine code (`place()` allocates the emitted length, word-rounded), reported as the module's dynamic memory (`setDynamicBytes(engine_.heapBytes())` — the exec block plus the control arena) so it shows on the UI card. At rest the engine itself is ~48 B of members + that exec block; the compile path's transient buffers (staging, IR, assembler ≈ 4 KB) live on the cold-path stack and are freed on return — see [docs/backlog/livescripts-analysis-top-down.md § 3.7](backlog/livescripts-analysis-top-down.md) for how this scales as the language grows. -**System variables cost a byte store each, per binding.** They are arena slots the binding refreshes before `run()` — a null check and a byte store apiece, replacing nothing, so the per-tick figure above is unchanged by them. An **effect** writes three (`width`/`height`/`depth`) once per tick; a **modifier** writes six (those plus the `x`/`y`/`z` it is handed) on the mapping-build cold path, not per frame; a **layout** writes none, since it is given no dimensions. `t` adds no arena byte: it is an argument register the host already passes. Not quite free, though — a callee may clobber an argument register under the ABI, so a backend saves it across calls (the arm64 one stacks x3 with the vreg pool; `unit_moonlive_fill` pins that a script reading `t` after a call still sees the host's value). `t` costs nothing at all: it is an argument register the host already passes. The compile path grew (a system-variable table, resolved before locals and controls) but that is cold-path, once per `source` edit. +**System variables cost a byte store each, per binding.** They are arena slots the binding refreshes before `run()` — a null check and a byte store apiece, replacing nothing, so the per-tick figure above is unchanged by them. An **effect** writes three (`width`/`height`/`depth`) once per tick; a **modifier** writes six (those plus the `x`/`y`/`z` it is handed) on the mapping-build cold path, not per frame; a **layout** writes none, since it is given no dimensions. `t` adds no arena byte: it is an argument register the host already passes. Not quite free, though — a callee may clobber an argument register under the ABI, so a backend saves it across calls (the arm64 one stacks x3 with the vreg pool; `unit_moonlive_fill` pins that a script reading `t` after a call still sees the host's value). The compile path grew (a system-variable table, resolved before locals and controls) but that is cold-path, once per `source` edit. + +**Across the four board classes** (2026-08-17, 16×16, after the Xtensa frame fix): the same shipped scripts on every ISA, so the numbers compare directly. + +| Script | S3 (Xtensa) | classic (Xtensa) | P4 (RISC-V) | S31 (RISC-V) | Exec block | +|---|---:|---:|---:|---:|---:| +| `lines.mlv` (two `line()` calls) | 76 µs | 128 µs | 30 µs | 92 µs | 2292 B | +| `plasma.mlv` (9 host calls per cell) | 780 µs | 1038 µs | 577 µs | 725 µs | ~1124 B | +| `ripples.mlv` (~15 host calls per cell) | 1695 µs | 2265 µs | 1098 µs | 1377 µs | 2372 B | + +The exec block is the emitted machine code, so it varies by ISA (the RISC-V rows above are the larger encoding); the tick is the native loop. `lines.mlv` is the cheapest of the three because `line()` moves the per-cell loop out of emitted code and into the shared `draw::line`, which is the argument for adding power functions as builtins rather than writing them in script. **Desktop tick across this cycle:** 150 → 122 µs (6666 → 8196 fps), measured by `collect_kpi.py --commit` at each commit. The gain is not from MoonLive — it tracks the two heap-overrun fixes and the register-reuse work landing earlier in the branch. No scenario `contract` was renegotiated on this branch: all 20 scenarios pass inside their existing budgets, which is the assertion surface this page defers to. @@ -248,7 +258,7 @@ Each parallel LED driver run on real hardware at a 128×128 = 16384-light grid, | Peripheral | Board | Pins used (8 lanes) | Result | Ceiling / bound | |---|---|---|---|---| -| **Parlio** | ESP32-P4 (Waveshare P4-NANO) | `20,21,22,23,24,25,26,27` | `Drivers` tick ~30100 µs, fps 30 at 16384 lights (8 lanes, SWAR transpose) | Parlio's single-shot transfer caps at 65535 bytes TOTAL (not per lane), and a light costs `channels × 24 × slotBytes` — so the ceiling is **898 lights/lane at 8 lanes RGB**, 673 RGBW, and halves to 443/332 at 16 lanes (a 16-bit bus doubles `slotBytes`). Over that, the driver reports `too many lights per pin` and keeps running; lifting the ceiling is the [chunked-DMA work](backlog/backlog-light.md) (tier 1 → ~16-21K). | +| **Parlio** | ESP32-P4 (Waveshare P4-NANO) | `20,21,22,23,24,25,26,27` | `Drivers` tick ~30100 µs, fps 30 at 16384 lights (8 lanes, SWAR transpose) | Parlio's single-shot transfer caps at 65535 bytes TOTAL (not per lane), and a light costs `channels × 24 × slotBytes` — so the ceiling is **897 lights/lane at 8 lanes RGB**, 673 RGBW, and halves to 442/332 at 16 lanes (a 16-bit bus doubles `slotBytes`). Over that, the driver reports `too many lights per pin` and keeps running; lifting the ceiling is the [chunked-DMA work](backlog/backlog-light.md) (tier 1 → ~16-21K). | | **LCD_CAM i80** (MultiPinLedDriver) | ESP32-S3 N16R8 Dev | data `18,5,6,7,8,9,10,11` · WR(clock) `12` · DC `13` | Same encoder, healthy on real i80; encode scales ~6 µs/light (8×512 = 4096 → 23 ms; 8×1024 = 8192 → 50 ms) | **single-DMA init ceiling 8192–12288 lights** (8×1024 inits; 8×1536 → "LCD init failed — check pins/memory"). A data lane on WR/DC only corrupts *that* lane (it carries the bus-control waveform, not pixels), so the driver **warns and keeps running** — a board that wires all lanes but drives fewer strands can legitimately park WR/DC on an unused data pin. WR and DC on the *same* GPIO is rejected up front (the bus needs two distinct control lines). | | **RMT** | classic ESP32 (LOLIN D32 / WROOM) | `2,4,13,14,16,17,18,19` (pin 2 = a real 24-LED strand) | 8-pin RMT drives **8×256 = 2048 lights** (tick ~12.6 ms), scales to ~8192 before the tick plateaus; all lanes healthy, pin-2 strand verified lit | **silent alloc-fail:** the RMT symbol buffer sizes for the driver's `count` window, so `count=0` on a 16384-grid needs ~1.5 MB, fails on the ~90 KB heap, and `tick()` bails with **no status** (LEDs dark). Bound the driver with the start/count window; a status for this is [backlogged](backlog/backlog-light.md). | | **I2S i80** | classic ESP32 (ESP32-WROVER) | data `2,4,13,14,18,19,21,22` · WR(clock) `32` · DC `33` (pin 2 = a real strand, verified lit) | The classic ESP32 runs the **same** `MultiPinLedDriver` over the **I2S peripheral in i80 mode** (IDF routes the i80 API to I2S here, to LCD_CAM on the S3/P4 — one driver, chip-picked backend). 8-lane doubling sweep (128×128 grid, 2026-07-13): 64/pin (512) → 4877 µs, 128/pin (1024) → 8575 µs, 256/pin (2048) → 15638 µs. Scales linearly at **~7.6 µs/light** (heavier than the S3's LCD_CAM ~6 µs — the classic I2S clock path). `frameTime` reports the WS2812 wire floor (512 → 243 fps, 2048 → 67 fps). The `MultiPinLed` status reports the live count. **16 lanes work on classic too** (the I2S peripheral does the 16-bit i80 bus, 16×256 = 4096 verified), but the WROVER exposes only ~13 non-strap pins, so 8-lane is the practical set. | **Internal-RAM ceiling: 2048 lights at 8 lanes (4096 at 16).** The classic I2S backend **cannot DMA from PSRAM** (`esp_lcd_i80_alloc_draw_buffer` rejects `MALLOC_CAP_SPIRAM` — "external memory is not supported"), so its frame buffer is internal-DMA-RAM only (`maxBlock` ≈ 76 KB). Swept at 8 lanes on a 128×128 grid (2026-07-13): 64/pin (512) ✅, 128/pin (1024) ✅, **256/pin (2048) ✅ — then 512/pin (4096) and above → `i80 bus init failed — check pins / memory`**, a **clean degrade, not a crash** (uptime kept climbing through every rung). That lands exactly on the parallel-I2S acceptance floor (8×256 = 2048), so the classic chip meets its floor and no more. The opposite of the LCD_CAM row below, which reaches 16384 via PSRAM — the classic chip's DMA simply can't get there. **The render is decoupled from this ceiling:** the same sweep kept rendering the full 128×128 = 16384-light grid at every rung (`Layer` ≈ 511 ms/frame, from PSRAM) while the *output* was capped — so a big grid still renders, it just can't all reach the LEDs. At 16K lights the effect render (511 ms) dwarfs the output (24 ms), so multicore cannot help: the render is the wall on this chip. Two classic-only quirks the driver handles: the I2S i80 tx has an unconditional command phase whose busy-wait hangs to a watchdog reset unless given a real 8-bit command (`lcd_cmd_bits=8` / `kI80Cmd=0`), and the draw buffer + a done-ISR marked `IRAM_ATTR`. | diff --git a/esp32/main/CMakeLists.txt b/esp32/main/CMakeLists.txt index 8a76048c..2faad75c 100644 --- a/esp32/main/CMakeLists.txt +++ b/esp32/main/CMakeLists.txt @@ -12,6 +12,7 @@ idf_component_register( "../../src/core/Scheduler.cpp" "../../src/core/moonlive/MoonLive.cpp" "../../src/core/moonlive/MoonLiveCompiler.cpp" + "../../src/core/moonlive/MoonLiveSpill.cpp" "../../src/platform/esp32/platform_esp32.cpp" "../../src/platform/esp32/moonlive_emit.cpp" "../../src/platform/esp32/moonlive_asm_xtensa.cpp" diff --git a/esp32/sdkconfig.defaults.qemu b/esp32/sdkconfig.defaults.qemu new file mode 100644 index 00000000..d0007b8f --- /dev/null +++ b/esp32/sdkconfig.defaults.qemu @@ -0,0 +1,28 @@ +# QEMU, the emulated-board firmware. Layered onto sdkconfig.defaults by +# moondeck/build/build_esp32.py. +# +# WHY A VARIANT AT ALL: QEMU emulates the CPU, memory and several peripherals, but it has no radio +# and no Ethernet PHY. A stock firmware therefore asserts in esp_phy_enable the moment WiFi starts, +# and reboot-loops before reaching the light pipeline. This fragment turns WiFi off and swaps in the +# one network interface an emulator DOES provide. +# +# OPENETH is what makes the run useful rather than merely alive: QEMU's `open_eth` device is an +# emulated OpenCores MAC, and IDF ships the matching driver. With it the emulated board gets a real +# IP stack, so the REST API and the web UI work exactly as on hardware, the same scripts, the same +# tests, the same browser. Without it a QEMU run can only be watched on the serial console. +# +# Pair with, on the host: +# -nic user,model=open_eth,hostfwd=tcp::8080-:80 +# which gives the guest a DHCP address and forwards host port 8080 to its HTTP server. + +# The emulated MAC, and the driver for it. +CONFIG_ETH_USE_OPENETH=y +CONFIG_ETH_OPENETH_DMA_RX_BUFFER_NUM=4 +CONFIG_ETH_OPENETH_DMA_TX_BUFFER_NUM=1 + +# No radio in the emulator. Leaving WiFi enabled is not merely useless, esp_phy_enable asserts on +# the missing PHY clock bits and takes the whole board down at boot. +CONFIG_ESP_WIFI_ENABLED=n + +# The internal EMAC has no emulated silicon either; openeth replaces it entirely. +CONFIG_ETH_USE_ESP32_EMAC=n diff --git a/moondeck/MoonDeck.md b/moondeck/MoonDeck.md index 015f8a66..4222c024 100644 --- a/moondeck/MoonDeck.md +++ b/moondeck/MoonDeck.md @@ -965,6 +965,41 @@ Backtrace: 0x4210b93b:0x3fcc8fa0 0x4200fbf8:0x3fcc8fc0 So a panic names its source line in the monitor rather than starting a separate addr2line session. Same purpose as PlatformIO's `esp32_exception_decoder` monitor filter; here it is the toolchain's own `addr2line` against `build/esp32-/projectMM.elf`, picking the Xtensa or RISC-V tool from the firmware name. Without `--firmware`, or when that build has no ELF, addresses print raw and the monitor runs as before — decoding must never cost you the serial output. +### check_encodings + +Verify every instruction MoonLive emits against the toolchain's own assembler. + +```bash +uv run moondeck/moonlive/check_encodings.py # every ISA that has a toolchain +uv run moondeck/moonlive/check_encodings.py --isa xtensa +``` + +MoonLive hand-encodes machine instructions, which is the right call for a JIT (an assembler cannot ship to a device) but means a wrong offset field, a truncated displacement or a misplaced register nibble is a bug nothing else notices: the golden-bytes tests compare our emission to our *previous* emission, and the structural checks compare it to a model we also wrote. Both agree with a consistent mistake. + +This compares it to something nobody here wrote: `xtensa-esp32-elf-as` and `riscv32-esp-elf-as`, which *are* the definition of a valid encoding for these ISAs. For each instruction we emit, it assembles the same mnemonic and requires identical bytes. It caught a hand-computed `add.n` whose register nibbles were transposed, before that instruction ever ran. + +What it covers, and what it does not: it proves each instruction is **encoded** correctly. It cannot prove the **sequence** is right, since a correctly encoded instruction can still save the wrong register. Execution-level checks ([run_qemu](#run_qemu), the bench) are what cover that. + +### run_qemu + +Run the firmware on an **emulated ESP32** on this machine, no board attached. Long-running: shows a Stop button. + +```bash +uv run moondeck/qemu/run_qemu.py # boot it, web UI on :8410 +uv run moondeck/qemu/run_qemu.py --erase # wipe the emulated flash first +uv run moondeck/qemu/run_qemu.py --gdb # freeze at reset, wait for a debugger on :1234 +``` + +Uses [Espressif's QEMU fork](https://github.com/espressif/qemu) ([docs](https://docs.espressif.com/projects/esp-idf/en/stable/esp32/api-guides/tools/qemu.html)), which emulates the ESP32's CPU, memory and enough peripherals to boot a real firmware image. Install it with `python3 $IDF_PATH/tools/idf_tools.py install qemu-xtensa`. + +**Why it earns its place: it EXECUTES the code.** Every other check compares emitted bytes against a model of what they should be, so none can catch a mistake the model shares. The emulator runs the instructions the way silicon does, including Xtensa's register window, `entry`/`retw` and the exception path, so a JIT defect faults here, on this machine, in seconds, instead of on a bench with only a crash dump to read. That is what it was built for ([the register-window frame bug](../docs/history/lessons.md#lessons-from-the-moonlive-on-xtensa-branch-the-register-window-frame-bug)). + +The emulated board is a full device, not a console toy: the `qemu` firmware variant swaps WiFi (no radio exists) for QEMU's emulated OpenCores MAC, so the guest gets a DHCP address and the REST API and web UI work exactly as on hardware. The same scripts, tests and browser drive it. Host port 8410 forwards to the guest's HTTP server, deliberately not 8080 so a desktop build can run alongside. + +**Erase flash** (the checkbox) deletes the merged flash image so every data partition, settings and scripts, comes back blank. On a real board that is `erase_flash_esp32`; here the whole chip is one file, so removing it is the same operation. Without it, everything persists across restarts. + +Two things to know when reading a QEMU run: the guest clock is emulated, so **timings are never KPI material**, and after any crash the emulator can boot-loop until the QEMU *process* itself is restarted (`--stop`, then start again). + ### improv_provision Push WiFi credentials to a running projectMM device over USB-serial. Uses the [Improv-WiFi](https://www.improv-wifi.com/serial/) protocol — the same wire format the browser flow at improv-wifi.com uses. Device must be running a firmware that includes the Improv listener. diff --git a/moondeck/build/build_desktop.py b/moondeck/build/build_desktop.py index 4755d842..c9afb475 100644 --- a/moondeck/build/build_desktop.py +++ b/moondeck/build/build_desktop.py @@ -57,6 +57,12 @@ def main(): ap.add_argument("--gcc", action="store_true", help="build with GCC instead of the default compiler — the toolchain CI uses. " "Catches the warnings clang does not emit.") + ap.add_argument("--no-jit", action="store_true", + help="build as a desktop WITHOUT a MoonLive backend (forces " + "MM_MOONLIVE_HAS_HOST_JIT=0). Every x86-64 desktop is one: Windows, Linux " + "and Intel macOS ship no backend, so a test that presumes a compile " + "succeeds passes on an arm64 bench and fails only on CI. This runs the " + "suite the way those hosts see it.") ap.add_argument("--tests", action="store_true", help="compile the test binaries (mm_tests + mm_scenarios) instead of the firmware. " "The default build makes only projectMM; the ~130 test units are a separate, " @@ -76,6 +82,12 @@ def main(): build_type = "Debug" extra = [f"-DCMAKE_C_COMPILER={cc}", f"-DCMAKE_CXX_COMPILER={cxx}"] print(f"Using GCC ({cxx}) in Debug — the exact toolchain + build type CI uses.") + if args.no_jit: + # Its own build dir, like --gcc: the macro changes which TEST_CASEs compile at all, so + # sharing a cache with the normal build would mean rebuilding the world on every switch. + bdir = "build/nojit" + extra += ["-DMM_MOONLIVE_NO_HOST_JIT=ON"] + print("Forcing MM_MOONLIVE_HAS_HOST_JIT=0 — the x86-64 desktop's view.") what = "test binaries" if args.tests else "desktop target" print(f"Building {what} into {bdir}/ ...") # CMAKE_BUILD_TYPE is honoured by single-config generators (Ninja, Make). diff --git a/moondeck/build/build_esp32.py b/moondeck/build/build_esp32.py index e886cf3f..d3462714 100644 --- a/moondeck/build/build_esp32.py +++ b/moondeck/build/build_esp32.py @@ -141,6 +141,25 @@ def check_idf_pin(idf_path: Path) -> None: "from deviceModels.json, default LAN8720 pins).", "ships": True, }, + # The EMULATED board. Not silicon: QEMU has no radio and no Ethernet PHY, so this variant swaps + # WiFi for the emulated OpenCores MAC (see sdkconfig.defaults.qemu). `eth_only` is True for the + # same reason an Ethernet-only board sets it, there is no WiFi to cascade to. + # + # ships=False: nobody flashes this to a device. It exists so the whole firmware, MoonLive's + # emitted machine code included, can be RUN and debugged on a development machine, with the REST + # API and web UI reachable through a forwarded host port. + "qemu": { + "chip": "esp32", + "fragments": ["sdkconfig.defaults", "sdkconfig.defaults.qemu"], + "eth_only": True, + "description": "ESP32 classic under QEMU, emulated Ethernet (openeth), no WiFi. " + "Run with moondeck/qemu/run_qemu.py, not flashed to hardware.", + "ships": False, + # Not silicon: keep it out of web-installer/firmwares.json entirely. `ships` already stops + # the release pipeline building it; this stops it reaching the installer's list, whose + # entries are things a user can flash to a board. + "installable": False, + }, "esp32-16mb": { "chip": "esp32", "fragments": ["sdkconfig.defaults", "sdkconfig.defaults.16mb", @@ -458,7 +477,11 @@ def firmware_cmake_args(firmware: str, release: str = "", version: str = "", # and check for the actual enabling line rather than pattern-matching the # filename: the S31 enables EMAC in `sdkconfig.defaults.esp32s31` (no ".eth" in # the name), which a filename heuristic would miss and silently stub eth out. - eth_symbols = {"CONFIG_ETH_USE_ESP32_EMAC=y", "CONFIG_ETH_USE_SPI_ETHERNET=y"} + # openeth is the third PHY driver: QEMU's emulated MAC. Without it here the qemu variant would + # be treated as having no Ethernet at all, ethInit() would be stubbed to `return false`, and the + # emulated board would come up with no IP stack, no REST API, no web UI. + eth_symbols = {"CONFIG_ETH_USE_ESP32_EMAC=y", "CONFIG_ETH_USE_SPI_ETHERNET=y", + "CONFIG_ETH_USE_OPENETH=y"} def fragment_enables_eth(frag: str) -> bool: path = ESP32_DIR / frag diff --git a/moondeck/build/generate_firmwares.py b/moondeck/build/generate_firmwares.py index a09cb4cd..6826b86e 100644 --- a/moondeck/build/generate_firmwares.py +++ b/moondeck/build/generate_firmwares.py @@ -53,7 +53,12 @@ def build_doc() -> dict: "ships": spec["ships"], "description": spec["description"], } - for name, spec in FIRMWARES.items() + # FLASHABLE variants only. This file feeds the WEB INSTALLER, the list of things a + # user puts on a board, so a variant that is not silicon has no place in it. The QEMU + # variant runs an emulator on a development machine and is driven from MoonDeck, never + # from the installer; listing it made the installer's naming rules have to describe + # something that will never appear in its UI. + for name, spec in FIRMWARES.items() if spec.get("installable", True) ] } diff --git a/moondeck/event/_gates.py b/moondeck/event/_gates.py index 24bb82f3..870606bd 100644 --- a/moondeck/event/_gates.py +++ b/moondeck/event/_gates.py @@ -144,6 +144,16 @@ def when(*prefixes, exclude=()): Gate("GCC build (CI's toolchain)", UV + ["moondeck/build/build_desktop.py", "--gcc", "--tests"], when(*COMPILES_DESKTOP)), + # The other half of "green here, red on CI": this bench is arm64 and has a MoonLive + # backend, while every x86-64 desktop (Windows, Linux, Intel macOS, and CI's runners) + # has none. A test that presumes a script compiles therefore passes locally and fails + # only after a push. Building with the backend gated out runs the suite the way those + # hosts see it. Triggered by MoonLive sources and by the tests that exercise them. + Gate("no-backend build (the x86-64 desktop's view)", + UV + ["moondeck/build/build_desktop.py", "--no-jit", "--tests"], + lambda f: touches(f, "src/core/moonlive/", "src/light/moonlive/", + "src/platform/desktop/moonlive", "test/unit/core/unit_moonlive", + "test/unit/light/unit_MoonLive")), # Reports what the compiler proved about THIS change: -Wfunction-effects checks the # render path transitively, and `--incremental` restricts the rebuild to what the commit # touched, so the gate answers "did this add a blocking call" in ~1s rather than diff --git a/moondeck/event/premerge.py b/moondeck/event/premerge.py index 83853b44..0915bda4 100644 --- a/moondeck/event/premerge.py +++ b/moondeck/event/premerge.py @@ -7,7 +7,7 @@ Scope is the whole branch diff (merge-base to HEAD), because architectural drift is visible across N commits in a way one commit hides. The judgment gates (Reviewer agent, -external review, lessons, PR description) are MANUAL by construction — an agent reports, +external review, lessons, PR description) are MANUAL by construction: an agent reports, the product owner decides. Product-owner initiated; never runs itself, never merges. """ @@ -62,14 +62,9 @@ def build_gates(firmware): # cannot have changed. Gate("performance snapshot in performance.md", None, lambda f: touches(f, *_TICK_PATH), - manual_hint="tick-path code changed on this branch — compare tick/FPS to the " + manual_hint="tick-path code changed on this branch: compare tick/FPS to the " "previous committed values and explain significant changes"), - Gate("permission review", None, - manual_hint="prune .claude/settings.local.json, then snapshot the approved " - "list to .claude/settings.local.cleaned.json and commit it; never " - "broaden destructive or network-mutating permissions"), - Gate("README / quick-start refresh", None, manual_hint="only if build, flash, or first-run UX changed"), ] @@ -85,7 +80,7 @@ def main(): sys.exit(run_gates(build_gates(args.firmware), changed_files(base=args.base), f"Merge gates (branch diff vs {args.base})", - next_step="The manual gates above are the product owner's call — " + next_step="The manual gates above are the product owner's call: " "this script never merges.")) diff --git a/moondeck/moondeck.py b/moondeck/moondeck.py index 1a54ca26..6538278a 100644 --- a/moondeck/moondeck.py +++ b/moondeck/moondeck.py @@ -1870,6 +1870,12 @@ def _handle_stream(self, script_id: str): line = stream.readline() except OSError: break # pty EIO on child exit → EOF + except ValueError: + # The stop button: kill_script closes the pty master fd while this thread is + # blocked in readline, and reading a CLOSED Python file raises ValueError, not + # OSError. Uncaught it takes down the whole request handler with a traceback, + # so stopping a long-running card looked like a MoonDeck crash. + break if not line: break # pipe EOF text = line.decode("utf-8", errors="replace").rstrip("\r\n") diff --git a/moondeck/moondeck_config.json b/moondeck/moondeck_config.json index cf353c5d..29684ad8 100644 --- a/moondeck/moondeck_config.json +++ b/moondeck/moondeck_config.json @@ -391,6 +391,34 @@ "needs_firmware": true, "pass_port_device_model": true }, + { + "id": "check_encodings", + "tab": "esp32", + "group": "qemu", + "label": "Check encodings", + "speed": "fast", + "help": "check_encodings", + "script": "moonlive/check_encodings.py" + }, + { + "id": "qemu_run", + "tab": "esp32", + "group": "qemu", + "label": "Run emulator", + "speed": "slow", + "help": "run_qemu", + "script": "qemu/run_qemu.py", + "flags": [ + { + "id": "erase", + "label": "Erase flash", + "arg": "--erase", + "default": false + } + ], + "long_running": true, + "process_name": "qemu-system-xtensa" + }, { "id": "erase_flash_esp32", "tab": "esp32", diff --git a/moondeck/moonlive/check_encodings.py b/moondeck/moonlive/check_encodings.py new file mode 100644 index 00000000..c0cc09ed --- /dev/null +++ b/moondeck/moonlive/check_encodings.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +"""Verify every instruction MoonLive emits against the toolchain's own assembler. + +We hand-encode machine instructions. That is the right call for a JIT, an assembler cannot be +shipped to a device, but it means a wrong offset field, a truncated displacement or a mis-placed +register nibble is a bug nothing else notices: the golden-bytes tests compare our emission to our +PREVIOUS emission, and the structural checks compare it to a model we also wrote. Both agree with a +consistent mistake. + +This compares it to something nobody here wrote: `xtensa-esp32-elf-as` / `riscv32-esp-elf-as`, which +ARE the definition of a valid encoding for these ISAs. For each instruction we emit, assemble the +same mnemonic and require identical bytes. + +What it covers, and what it does not: it proves each instruction is ENCODED correctly. It cannot +prove the SEQUENCE is right, a correctly encoded instruction can still save the wrong register. +Execution-level checks (QEMU, the bench) are what cover that. + + uv run moondeck/moonlive/check_encodings.py # every ISA that has a toolchain + uv run moondeck/moonlive/check_encodings.py --isa xtensa +""" + +import argparse +import glob +import os +import subprocess +import sys +import tempfile + +# Per ISA: where its assembler lives, and the instructions our emitter produces. Each case is +# (mnemonic-as-the-toolchain-spells-it, the bytes our assembler emits for it, why it is here). +# +# The expected bytes are OURS, written down from the emitter's own encoders. The test's value is +# that `as` must independently agree; when it does not, one of the two is wrong and the diff says +# which field. +ISAS = { + "xtensa": { + "as": "~/.espressif/tools/xtensa-esp-elf/*/xtensa-esp-elf/bin/xtensa-esp32-elf-as", + "objcopy": "~/.espressif/tools/xtensa-esp-elf/*/xtensa-esp-elf/bin/xtensa-esp32-elf-objcopy", + # --no-transform stops `as` "helpfully" relaxing a wide instruction into its narrow + # equivalent (s32i -> s32i.n). Both encode the same operation, but our emitter always picks + # the wide form, one encoder that covers every offset, so without this the comparison + # reports a difference that is not a defect. + "asflags": ["--no-transform"], + "cases": [ + # The frame. `entry`'s immediate counts EIGHT-byte units and sits at bits 12..23, the + # field that was silently wrong when the base-save area was missing. + ("entry a1, 160", "364101", "prologue: the whole-routine frame"), + ("entry a1, 48", "366100", "prologue: the minimum frame"), + # Frame slots. The offset field counts FOUR-byte words, so a slot index maps onto it + # directly, and a slot past 255 words would silently truncate. + ("s32i a2, a1, 112", "22611c", "spillStore: park a host argument"), + ("l32i a11, a1, 128", "b22120", "spillLoad: reload the arena pointer"), + ("s32i a10, a1, 32", "a26108", "call: park the result"), + ("addi a9, a1, 60", "92c13c", "slotAddr: the address of an argument block"), + # Arithmetic, narrow and wide forms. + ("add.n a2, a2, a13", "da22", "add: the narrow two-byte form"), + ("mull a9, a2, a11", "b09282", "mul"), + ("movi a2, 255", "22a0ff", "movi: an eight-bit immediate"), + ("mov.n a13, a9", "dd09", "mov: the narrow form"), + # Control flow. `bltu`'s displacement is a single SIGNED byte (+/-127), the field that + # truncated silently on a long loop body before branch relaxation. + ("callx8 a8", "e00800", "call: the windowed call"), + ("retw.n", "1df0", "epilogue"), + ("l8ui a2, a11, 8", "220b08", "LoadCtrl: read a control byte from the arena"), + ("s8i a3, a12, 0", "324c00", "StoreElem: write one channel"), + ], + }, + "riscv": { + "as": "~/.espressif/tools/riscv32-esp-elf/*/riscv32-esp-elf/bin/riscv32-esp-elf-as", + "objcopy": "~/.espressif/tools/riscv32-esp-elf/*/riscv32-esp-elf/bin/riscv32-esp-elf-objcopy", + # rv32im WITHOUT the compressed extension: our emitter only ever produces the 4-byte + # forms, and letting `as` pick a 2-byte one would compare two valid encodings of the + # same mnemonic and report a difference that is not a defect. + "asflags": ["-march=rv32im", "-mno-relax"], + "cases": [ + ("addi sp, sp, -96", "130101fa", "prologue: allocate the frame"), + ("sw s0, 92(sp)", "232e8104", "prologue: save the frame pointer"), + ("addi s0, sp, 96", "13040106", "prologue: s0 = the caller's sp"), + ("sw a0, -32(s0)", "2320a4fe", "spillStore: park a host argument"), + ("lw a1, -28(s0)", "832544fe", "spillLoad"), + ("addi t1, s0, -96", "130304fa", "slotAddr"), + ("mul a0, a1, a2", "3385c502", "mul"), + ("jalr ra, t6, 0", "e7800f00", "call"), + ("ret", "67800000", "epilogue"), + ], + }, +} + + +def tool(pattern: str, isa: str, what: str) -> str: + hits = glob.glob(os.path.expanduser(pattern)) + if not hits: + sys.exit(f"{what} for {isa} not found, install the matching ESP-IDF toolchain") + return sorted(hits)[-1] + + +def check(isa: str) -> int: + spec = ISAS[isa] + as_bin = tool(spec["as"], isa, "assembler") + objcopy = tool(spec["objcopy"], isa, "objcopy") + failures = 0 + + print(f"\n== {isa} ==") + with tempfile.TemporaryDirectory() as tmp: + for mnemonic, ours, why in spec["cases"]: + src = os.path.join(tmp, "one.S") + with open(src, "w") as f: + f.write(f" {mnemonic}\n") + obj = os.path.join(tmp, "one.o") + r = subprocess.run([as_bin, *spec.get("asflags", []), "-o", obj, src], + capture_output=True, text=True) + if r.returncode != 0: + print(f" FAIL {mnemonic:24s} assembler rejected it: {r.stderr.strip()[:80]}") + failures += 1 + continue + binf = os.path.join(tmp, "one.bin") + oc = subprocess.run([objcopy, "-O", "binary", obj, binf], capture_output=True, text=True) + if oc.returncode != 0: + print(f" FAIL {mnemonic:24s} objcopy failed: {oc.stderr.strip()[:80]}") + failures += 1 + continue + with open(binf, "rb") as f: + theirs = f.read().hex() + if theirs == ours: + print(f" ok {mnemonic:24s} {ours:<10s} {why}") + else: + print(f" FAIL {mnemonic:24s} we emit {ours}, the assembler says {theirs} ({why})") + failures += 1 + return failures + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--isa", choices=[*ISAS, "all"], default="all") + args = ap.parse_args() + + isas = list(ISAS) if args.isa == "all" else [args.isa] + total = sum(check(i) for i in isas) + print() + if total: + print(f"{total} encoding(s) disagree with the toolchain assembler") + return 1 + print("every encoding matches the toolchain assembler") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/moondeck/moonlive/disasm.py b/moondeck/moonlive/disasm.py index 49f3a655..8ca20ae9 100644 --- a/moondeck/moonlive/disasm.py +++ b/moondeck/moonlive/disasm.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Disassemble the machine code MoonLive emits for a script, without a device. +"""Disassemble the machine code MoonLive emits for a script, on any backend, without a device. Why this exists: an emitted-code bug on a device shows up as "the script did nothing" — it compiles, reports no error, and places no lights. Reasoning about hand-written encoders from that @@ -8,11 +8,18 @@ (the ISA reuses that slot for -1), so a loop counter started at -1 and every loop exited immediately. The per-ISA assemblers are ordinary C++ behind a target guard, so they build and run on the host: -this compiles a script through the real Xtensa backend and pipes the bytes to the ESP-IDF objdump. +this compiles a script through a REAL backend and pipes the bytes to that ISA's objdump. uv run moondeck/moonlive/disasm.py "for (i = 0; i < 3; i = i + 1) { addLight(i, 0, 0); }" + uv run moondeck/moonlive/disasm.py --isa riscv moonlive/effects/plasma.mlv effect + uv run moondeck/moonlive/disasm.py --isa all moonlive/layouts/grid.mlv layout + +Note there is no x86_64 backend to disassemble: the desktop assembler is arm64-only +(`moonlive_asm_host.cpp`), and on an x86 host a compile fails cleanly and the module renders dark +(backlogged). The three backends below are therefore every backend that exists. """ +import argparse import binascii import glob import os @@ -21,35 +28,75 @@ import tempfile ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -TOOL_SRC = os.path.join(ROOT, "moondeck", "moonlive", "emit_xtensa.cpp") - - -def objdump() -> str: - """The ESP-IDF Xtensa objdump, wherever the toolchain was installed.""" - hits = glob.glob(os.path.expanduser( - "~/.espressif/tools/xtensa-esp-elf/*/xtensa-esp-elf/bin/xtensa-esp32s3-elf-objdump")) +TOOL_SRC = os.path.join(ROOT, "moondeck", "moonlive", "emit_isa.cpp") + +# Per ISA: the emitter's -D flag, the objdump to disassemble with, and objdump's -m architecture. +# `toolchain` is a glob because the ESP-IDF toolchains are installed per version; `None` means the +# host's own objdump already understands this ISA (arm64 on an Apple Silicon machine). +ISAS = { + "xtensa": { + "define": "MM_EMIT_XTENSA", + "toolchain": "~/.espressif/tools/xtensa-esp-elf/*/xtensa-esp-elf/bin/xtensa-esp32s3-elf-objdump", + "machine": "xtensa", + }, + "riscv": { + "define": "MM_EMIT_RISCV", + "toolchain": "~/.espressif/tools/riscv32-esp-elf/*/riscv32-esp-elf/bin/riscv32-esp-elf-objdump", + "machine": "riscv:rv32", + }, + "arm64": { + "define": "MM_EMIT_ARM64", + "toolchain": None, + "machine": "aarch64", + }, +} + + +def objdump(isa: str) -> str: + """The objdump that understands `isa`, wherever its toolchain was installed.""" + pattern = ISAS[isa]["toolchain"] + if pattern is None: + # llvm-objdump ships with the Xcode command line tools and knows aarch64. It is REQUIRED + # rather than merely preferred on macOS: the `objdump` on PATH there is Apple's, which + # rejects the raw-binary flags (`-b binary`) this tool depends on. An ESP-IDF toolchain + # brings its own llvm-objdump, so check those too before giving up. + found = subprocess.run(["which", "llvm-objdump"], capture_output=True, text=True) + if found.returncode == 0: + return found.stdout.strip() + # On macOS llvm-objdump ships inside the Xcode command line tools but is NOT on PATH; + # xcrun is how the platform expects you to find it. + found = subprocess.run(["xcrun", "-f", "llvm-objdump"], capture_output=True, text=True) + if found.returncode == 0 and found.stdout.strip(): + return found.stdout.strip() + sys.exit("llvm-objdump not found — needed to disassemble arm64") + hits = glob.glob(os.path.expanduser(pattern)) if not hits: - sys.exit("xtensa-esp32s3-elf-objdump not found — install the ESP-IDF Xtensa toolchain") + sys.exit(f"objdump for {isa} not found — install the matching ESP-IDF toolchain") return sorted(hits)[-1] -def main() -> int: - script = sys.argv[1] if len(sys.argv) > 1 else \ - "for (i = 0; i < 3; i = i + 1) { addLight(i, 0, 0); }" - +def disassemble(isa: str, script: str, binding: str) -> int: + """Compile `script` through the `isa` backend and print its disassembly.""" with tempfile.TemporaryDirectory() as tmp: emitter = os.path.join(tmp, "emit") build = subprocess.run( ["c++", "-std=c++20", "-O0", "-I", os.path.join(ROOT, "src"), "-I", os.path.join(ROOT, "src", "platform", "desktop"), + "-D", ISAS[isa]["define"], TOOL_SRC, os.path.join(ROOT, "src", "core", "moonlive", "MoonLiveCompiler.cpp"), + # Every backend runs the register allocator before lowering, so the pass comes along + # too — without it the tool fails to link on spillToBudget. + os.path.join(ROOT, "src", "core", "moonlive", "MoonLiveSpill.cpp"), + # The IR sizes its op array with platform::alloc, so the platform implementation has + # to come along — the compiler is no longer self-contained. + os.path.join(ROOT, "src", "platform", "desktop", "platform_desktop.cpp"), "-o", emitter], capture_output=True, text=True) if build.returncode != 0: print(build.stderr[:2000]) return 1 - run = subprocess.run([emitter, script], capture_output=True, text=True) + run = subprocess.run([emitter, script, binding], capture_output=True, text=True) print(run.stdout.split("\n")[0]) # the script if run.returncode != 0: print(run.stdout.strip() or run.stderr.strip()) @@ -69,8 +116,26 @@ def is_hex(line: str) -> bool: binpath = os.path.join(tmp, "code.bin") with open(binpath, "wb") as f: f.write(raw) - dis = subprocess.run([objdump(), "-D", "-b", "binary", "-m", "xtensa", binpath], - capture_output=True, text=True) + tool = objdump(isa) + if os.path.basename(tool).startswith("llvm-"): + # llvm-objdump has NO raw-binary mode (no `-b binary`), so the bytes are assembled into + # a real object file first — `.incbin` is the one-line way to put a blob in a .text + # section the disassembler will then walk. + asm = os.path.join(tmp, "blob.s") + with open(asm, "w") as f: + f.write(f'.text\n.incbin "{binpath}"\n') + obj = os.path.join(tmp, "blob.o") + wrap = subprocess.run(["cc", "-c", asm, "-o", obj], capture_output=True, text=True) + if wrap.returncode != 0: + print(wrap.stderr.strip()[:600]) + return 1 + argv = [tool, "-d", obj] + else: + argv = [tool, "-D", "-b", "binary", "-m", ISAS[isa]["machine"], binpath] + dis = subprocess.run(argv, capture_output=True, text=True) + if dis.returncode != 0: + print(dis.stderr.strip()[:600]) + return 1 # Skip objdump's file header; the instructions start after the section line. lines = dis.stdout.splitlines() start = next((i for i, line in enumerate(lines) if line.startswith("00000000")), 0) @@ -78,5 +143,36 @@ def is_hex(line: str) -> bool: return 0 +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--isa", default="xtensa", choices=[*ISAS, "all"], + help="which backend to emit through (default: xtensa; 'all' runs every one)") + ap.add_argument("script", nargs="?", + default="for (i = 0; i < 3; i = i + 1) { addLight(i, 0, 0); }", + help="script text, or a path to a .mlv file") + # Which binding's system variables to compile against: layout (default), effect or modifier. + # They are different vocabularies, not nested ones — a layout may use `x`/`y` as loop counters + # precisely because it is NOT handed them, so the binding has to be stated. + ap.add_argument("binding", nargs="?", default="layout", + choices=["layout", "effect", "modifier"], + help="which system-variable vocabulary to compile against (default: layout)") + args = ap.parse_args() + + script = args.script + # A path is read as a file; anything else is the script text itself. + if os.path.isfile(script): + with open(script) as f: + script = f.read() + + isas = list(ISAS) if args.isa == "all" else [args.isa] + rc = 0 + for isa in isas: + if len(isas) > 1: + print(f"\n{'=' * 60}\n== {isa}\n{'=' * 60}") + rc |= disassemble(isa, script, args.binding) + return rc + + if __name__ == "__main__": sys.exit(main()) diff --git a/moondeck/moonlive/emit_isa.cpp b/moondeck/moonlive/emit_isa.cpp new file mode 100644 index 00000000..d9cadece --- /dev/null +++ b/moondeck/moonlive/emit_isa.cpp @@ -0,0 +1,72 @@ +#include +#include +#include +#include + +// Compile a MoonLive script through ONE REAL BACKEND on the development machine, and print the +// bytes it emits. Each backend is guarded by its target macro (`__XTENSA__`, `__riscv`, +// `__aarch64__`) so on this host it normally compiles to nothing; defining the macro and including +// the sources is what lets the host run a device emitter. The firmware build never sees this file, +// so there is still exactly one definition of each backend in the shipped binary. +// +// Which backend is selected by -DMM_EMIT_ from disasm.py. One file rather than one per ISA: +// the whole body below is identical for every backend, and only the includes differ. + +#if defined(MM_EMIT_XTENSA) +#define __XTENSA__ 1 +#include "platform/esp32/moonlive_asm_xtensa.h" +#include "platform/esp32/moonlive_asm_xtensa.cpp" +#elif defined(MM_EMIT_RISCV) +// The RISC-V backend tests `__riscv` only, so defining it is enough to expose the whole file. +#define __riscv 1 +#include "platform/esp32/moonlive_asm_riscv.h" +#include "platform/esp32/moonlive_asm_riscv.cpp" +#elif defined(MM_EMIT_ARM64) +#define __aarch64__ 1 +#include "platform/desktop/moonlive_asm_host.h" +#include "platform/desktop/moonlive_asm_host.cpp" +#else +#error "define MM_EMIT_XTENSA, MM_EMIT_RISCV or MM_EMIT_ARM64" +#endif + +// The lowerer body, with the emit seam it expects. +#include "core/moonlive/MoonLiveIr.h" +#include "core/moonlive/MoonLiveBuiltins.h" +// The CANONICAL declaration, rather than a copy: this tool once redeclared a three-argument +// lowerToBytes that linked only because it never passes `squeeze`, so the tool's view of the +// backend could drift from the backend's own. +#include "core/moonlive/moonlive_emit.h" + +#if defined(MM_EMIT_XTENSA) +#include "platform/esp32/moonlive_lower_xtensa.cpp" +#undef __XTENSA__ +#elif defined(MM_EMIT_RISCV) +#include "platform/esp32/moonlive_lower_riscv.cpp" +#undef __riscv +#elif defined(MM_EMIT_ARM64) +#include "platform/desktop/moonlive_lower_host.cpp" +#undef __aarch64__ +#endif + +#include "core/moonlive/MoonLiveCompiler.h" +#include "light/moonlive/MoonLiveBuiltins_light.h" +using namespace mm; +int main(int argc, char** argv) { + const char* src = argc > 1 ? argv[1] : "for (i = 0; i < 3; i = i + 1) { addLight(i, 0, 0); }"; + uint8_t buf[4096]; + // Which BINDING to compile as, because the system-variable tables are different vocabularies and + // not nested supersets: a modifier is handed `x`/`y`/`z`, and a LAYOUT deliberately is not, so it + // may use those names as ordinary loop counters — which the shipped grid.mlv does. Compiling + // every script against the widest table therefore refuses exactly the scripts most worth + // inspecting ("name is a system variable"), which is how this tool came to never see grid.mlv. + const char* binding = argc > 2 ? argv[2] : "layout"; + const auto sysvars = std::strcmp(binding, "modifier") == 0 ? moonlive::modifierSysVars() + : std::strcmp(binding, "effect") == 0 ? moonlive::effectSysVars() + : moonlive::layoutSysVars(); + auto r = moonlive::compileSource(src, moonlive::lightBuiltins(), sysvars, buf, sizeof(buf)); + if (!r.ok) { printf("compile failed: %s\n", r.error); return 1; } + printf("# %s\n# %zu bytes\n", src, r.len); + for (size_t i = 0; i < r.len; i++) printf("%02x%s", buf[i], (i % 16 == 15) ? "\n" : " "); + printf("\n"); + return 0; +} diff --git a/moondeck/moonlive/emit_xtensa.cpp b/moondeck/moonlive/emit_xtensa.cpp deleted file mode 100644 index 9649aa45..00000000 --- a/moondeck/moonlive/emit_xtensa.cpp +++ /dev/null @@ -1,38 +0,0 @@ -#include -#include -#include -#include -// The Xtensa backend is `#if defined(__XTENSA__)`, so on this host it compiles to nothing — there -// is no library to link against, which is the whole point: this tool runs the REAL Xtensa emitter on -// the development machine so an encoding can be read without flashing a board. Defining the macro -// and including the sources is what makes that possible; the target build never sees this file, so -// there is still exactly one backend definition in the firmware. -#define __XTENSA__ 1 -#include "platform/esp32/moonlive_asm_xtensa.h" -#include "platform/esp32/moonlive_asm_xtensa.cpp" -// The lowerer body, with the emit seam it expects. -#include "core/moonlive/MoonLiveIr.h" -#include "core/moonlive/MoonLiveBuiltins.h" -namespace mm::moonlive { -size_t lowerToBytes(const IrProgram& ir, uint8_t* out, size_t cap); -} -#include "platform/esp32/moonlive_lower_xtensa.cpp" -#undef __XTENSA__ - -#include "core/moonlive/MoonLiveCompiler.h" -#include "light/moonlive/MoonLiveBuiltins_light.h" -using namespace mm; -int main(int argc, char** argv) { - const char* src = argc > 1 ? argv[1] : "for (i = 0; i < 3; i = i + 1) { addLight(i, 0, 0); }"; - uint8_t buf[4096]; - // The WIDEST system-variable list on purpose: this tool disassembles whatever script is passed - // on the command line — layout, effect or modifier — so it must accept every name any binding - // supplies. A narrower list would refuse the scripts it exists to inspect. - auto r = moonlive::compileSource(src, moonlive::lightBuiltins(), - moonlive::modifierSysVars(), buf, sizeof(buf)); - if (!r.ok) { printf("compile failed: %s\n", r.error); return 1; } - printf("# %s\n# %zu bytes\n", src, r.len); - for (size_t i = 0; i < r.len; i++) printf("%02x%s", buf[i], (i % 16 == 15) ? "\n" : " "); - printf("\n"); - return 0; -} diff --git a/moondeck/qemu/run_qemu.py b/moondeck/qemu/run_qemu.py new file mode 100644 index 00000000..d764c35a --- /dev/null +++ b/moondeck/qemu/run_qemu.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +"""Run the firmware under QEMU, a real ESP32, emulated, on the development machine. + +Why this exists: MoonLive emits machine code, and a defect in it shows up as a board that resets. +Every other check we have compares emitted bytes to a model of what they should be, which cannot +catch a mistake the model shares. QEMU EXECUTES the code the way silicon does, including Xtensa's +register window, `entry`/`retw` and the exception path, so a fault happens here, on the host, in a +debugger, instead of on a bench with only a crash dump to read. + +The emulated board is a full device: the REST API and the web UI work, so the same scripts, tests +and browser drive it exactly as they drive hardware. + + uv run moondeck/qemu/run_qemu.py # boot it, forward the UI to :8410 + uv run moondeck/qemu/run_qemu.py --gdb # freeze at reset, wait for a debugger + uv run moondeck/qemu/run_qemu.py --seconds 20 # run a while, print the log, exit + +With --gdb, attach from another terminal: + xtensa-esp32-elf-gdb build/esp32-qemu/projectMM.elf -ex 'target remote :1234' +""" + +import argparse +import glob +import os +import subprocess +import sys +import time + +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +BUILD = os.path.join(ROOT, "build", "esp32-qemu") + +# esptool lives in the ESP-IDF venv, not in the interpreter `uv run` starts. build_esp32.py already +# knows how to find the right one (there can be several IDF versions installed), so reuse that rather +# than re-deriving the path here. +sys.path.insert(0, os.path.join(ROOT, "moondeck", "build")) +from build_esp32 import find_idf_python # noqa: E402 + + +def qemu_binary() -> str: + hits = glob.glob(os.path.expanduser( + "~/.espressif/tools/qemu-xtensa/*/qemu/bin/qemu-system-xtensa")) + if not hits: + sys.exit("qemu-system-xtensa not found, install it with:\n" + " python3 $IDF_PATH/tools/idf_tools.py install qemu-xtensa") + return sorted(hits)[-1] + + +def merged_flash(force: bool) -> str: + """The 4 MB flash image QEMU boots from: bootloader + partition table + app, in one file.""" + out = os.path.join(BUILD, "qemu-flash.bin") + args = os.path.join(BUILD, "flash_args") + if not os.path.exists(args): + sys.exit(f"no build at {BUILD}, build it first:\n" + " uv run moondeck/build/build_esp32.py --firmware qemu") + # Freshness is judged against the APP BINARY, not `flash_args`. flash_args is written once when + # CMake configures the build dir and never touched again, so it is always older than the image , + # which made this cache never expire: after a rebuild the emulator kept booting the PREVIOUS + # app, and code that was plainly in the .bin appeared not to run at all. + app = os.path.join(BUILD, "projectMM.bin") + newest_input = max((os.path.getmtime(p) for p in (args, app) if os.path.exists(p)), default=0) + if os.path.exists(out) and not force and os.path.getmtime(out) > newest_input: + return out + # find_idf_python returns the venv DIRECTORY; the interpreter is bin/python inside it. + venv = find_idf_python() + idf_py = os.path.join(str(venv), "bin", "python") if venv else "" + if not idf_py or not os.path.exists(idf_py): + sys.exit("no ESP-IDF Python env found, source export.sh, or install the IDF tools") + r = subprocess.run([idf_py, "-m", "esptool", "--chip", "esp32", "merge_bin", + "-o", out, "--fill-flash-size", "4MB", f"@{args}"], + cwd=BUILD, capture_output=True, text=True) + if r.returncode != 0: + sys.exit(f"merge_bin failed:\n{r.stderr[:800]}") + return out + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--gdb", action="store_true", + help="freeze the CPU at reset and wait for a debugger on :1234") + ap.add_argument("--seconds", type=int, default=0, + help="run for N seconds, then exit (0 = run until interrupted)") + ap.add_argument("--http-port", type=int, default=8410, + help="host port forwarded to the guest's HTTP server (default 8410). " + "Deliberately not 8080: a desktop projectMM build listens there, and " + "QEMU fails to start rather than sharing the port.") + ap.add_argument("--rebuild-image", action="store_true", + help="re-merge the flash image even if it looks current") + ap.add_argument("--erase", action="store_true", + help="wipe the emulated flash (settings + scripts) before starting, the " + "equivalent of erase_flash_esp32.py on a real board. Use when a saved " + "config boot-loops the guest: a merged image keeps the DATA partitions, " + "so re-merging alone does NOT clear it.") + ap.add_argument("--stop", action="store_true", + help="stop a running emulator instead of starting one") + ap.add_argument("--background", action="store_true", + help="start it and return, instead of holding the terminal (MoonDeck uses this)") + args = ap.parse_args() + + if args.stop: + # pkill by the binary path, not by our own PID: MoonDeck starts the emulator in one task and + # stops it from another, so there is no shared handle to keep. + r = subprocess.run(["pkill", "-f", "qemu-system-xtensa"], capture_output=True) + print("emulator stopped" if r.returncode == 0 else "no emulator was running") + return 0 + + if args.erase: + # Deleting the image is the erase: merged_flash rebuilds it from the bootloader, partition + # table and app alone, so every DATA partition (nvs, otadata, spiffs, where the module tree + # and the .mlv scripts live) comes back blank. On a real board this is erase_flash_esp32.py; + # here the whole "flash chip" is one file, so removing it is the same operation. + img = os.path.join(BUILD, "qemu-flash.bin") + if os.path.exists(img): + os.remove(img) + print(" erased → emulated flash wiped (settings + scripts gone)") + else: + print(" erased → no image to wipe") + + # Free the forward port FIRST. QEMU treats a failed hostfwd as fatal but still exits 0, so a + # second emulator (or a leftover one) makes the new run die in under a second while the caller + # reports success, and the UI simply never appears, with nothing in the log to say why. There is + # only ever one emulator, so taking the port from a previous instance is always what was meant. + holder = subprocess.run(["lsof", "-nP", f"-iTCP:{args.http_port}", "-sTCP:LISTEN", "-t"], + capture_output=True, text=True) + for pid in holder.stdout.split(): + print(f" freeing → port {args.http_port} was held by pid {pid}, stopping it") + subprocess.run(["kill", pid], capture_output=True) + if holder.stdout.strip(): + time.sleep(1) + + flash = merged_flash(args.rebuild_image or args.erase) + cmd = [ + qemu_binary(), "-nographic", "-machine", "esp32", + "-drive", f"file={flash},if=mtd,format=raw", + # The emulated MAC, plus a host-port forward so the device's web UI opens in a browser. + # `user` networking needs no privileges and no host interface, the guest gets a DHCP + # address on QEMU's internal network and reaches the outside through the host. + "-nic", f"user,model=open_eth,hostfwd=tcp::{args.http_port}-:80", + ] + if args.gdb: + cmd += ["-s", "-S"] + + print(f"QEMU: {os.path.basename(flash)}") + print(f" web UI → http://localhost:{args.http_port}/ (once the guest has a DHCP lease)") + if args.gdb: + print(" debugger→ frozen at reset, waiting on :1234") + print(f" xtensa-esp32-elf-gdb {BUILD}/projectMM.elf -ex 'target remote :1234'") + print() + + if args.background: + # Detached, with the serial log on disk: the caller gets its prompt back and can read the + # boot output from build/esp32-qemu/qemu-serial.log while the guest keeps running. + logf = os.path.join(BUILD, "qemu-serial.log") + with open(logf, "wb") as f: + subprocess.Popen(cmd, stdout=f, stderr=subprocess.STDOUT, + stdin=subprocess.DEVNULL, start_new_session=True) + print(f" serial → {logf}") + print(" stop it → uv run moondeck/qemu/run_qemu.py --stop") + return 0 + + try: + if args.seconds: + subprocess.run(cmd, timeout=args.seconds) + else: + subprocess.run(cmd) + except subprocess.TimeoutExpired: + pass + except KeyboardInterrupt: + pass + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/moondeck/run/monitor_esp32.py b/moondeck/run/monitor_esp32.py index 8b83ecb4..34e6d81e 100644 --- a/moondeck/run/monitor_esp32.py +++ b/moondeck/run/monitor_esp32.py @@ -100,13 +100,17 @@ def main(): print("Press Ctrl+C (or Stop in MoonDeck) to stop.\n") sys.stdout.flush() - with raised_log_level(active_device_ips(), LOG_INFO): - try: - ser = serial.Serial(args.port, args.baud, timeout=1) - except serial.SerialException as e: - print(f"Cannot open {args.port}: {e}") - sys.exit(1) + # OPEN THE PORT FIRST. raised_log_level contacts every device in moondeck.json over HTTP at a + # 3 s timeout each — with a dozen registered and most powered off, that is half a minute of + # blocking before a single byte is read, and the boot output you were monitoring FOR is already + # gone. The log level is a nicety; the serial stream is the point. + try: + ser = serial.Serial(args.port, args.baud, timeout=1) + except serial.SerialException as e: + print(f"Cannot open {args.port}: {e}") + sys.exit(1) + with raised_log_level(active_device_ips(), LOG_INFO): with open(LOG_FILE, "w") as log: try: while True: diff --git a/moonlive/README.md b/moonlive/README.md index 9f133ea4..5f608f23 100644 --- a/moonlive/README.md +++ b/moonlive/README.md @@ -7,7 +7,7 @@ and it compiles to native code on the next tick. | folder | run by | a script writes | |---|---|---| | `layouts/` | [MoonLiveLayout](../docs/moonmodules/light/MoonLiveLayout.md) | where the lights physically are — `addLight(x, y, z)` | -| `effects/` | [MoonLiveEffect](../docs/moonmodules/light/MoonLiveEffect.md) | a colour per light — `setRGB(index, r, g, b)` | +| `effects/` | [MoonLiveEffect](../docs/moonmodules/light/MoonLiveEffect.md) | a colour per light: `setRGB(index, r, g, b)`, or a whole shape at once with `line(x1, y1, x2, y2, r, g, b)` | | `modifiers/` | [MoonLiveModifier](../docs/moonmodules/light/MoonLiveModifier.md) | where one light lands — `setXYZ(0, x, y, z)` | Each module ships one of these as its default, so the folder doubles as the reference for what a diff --git a/moonlive/effects/lines.mlv b/moonlive/effects/lines.mlv index 88e4d988..e2193370 100644 --- a/moonlive/effects/lines.mlv +++ b/moonlive/effects/lines.mlv @@ -1,12 +1,17 @@ -// A red column and a green row sweeping the grid. +// A red column and a green row sweeping the grid, each drawn as ONE line() call. // `width`/`height` come from the LAYER. The fill clears last frame. +// +// line(x1, y1, x2, y2, r, g, b) is the seven-argument draw builtin: the script names the +// endpoints and the shared draw::line walks the pixels, replacing the per-cell loop this +// script used to spell out. uint8_t bpm = 30; // @control 1..240 fill(0, 0, 0); -for (y = 0; y < height; y = y + 1) { - setRGB(y * width + scale(beat(bpm, t), width), 255, 0, 0); -} -for (x = 0; x < width; x = x + 1) { - setRGB(scale(beat(bpm, t), height) * width + x, 0, 255, 0); -} +line(scale(beat(bpm, t), width), 0, + scale(beat(bpm, t), width), height - 1, + 255, 0, 0); + +line(0, scale(beat(bpm, t), height), + width - 1, scale(beat(bpm, t), height), + 0, 255, 0); diff --git a/moonlive/effects/plasma.mlv b/moonlive/effects/plasma.mlv new file mode 100644 index 00000000..10be8a2e --- /dev/null +++ b/moonlive/effects/plasma.mlv @@ -0,0 +1,19 @@ +// Plasma: two travelling sine waves summed per cell, in the demoscene shape. +// The colour of a cell comes from sin(x-ish) + sin(y-ish), both scrolling on the clock, so +// the field drifts and interferes without anything being stored between frames. +// +// The heaviest script that ships: a nested loop over the whole grid with nine host calls per +// cell (3 beat, 2 sin, 1 cos, 3 scale) feeding one setRGB. `beat(bpm, t)` reads the clock +// through the same path an effect always does. +uint8_t bpm = 12; // @control 1..120 +uint8_t zoom = 24; // @control 1..64 + +for (y = 0; y < height; y = y + 1) { + for (x = 0; x < width; x = x + 1) { + // Each axis gets its own wave, offset by the beat so the pattern travels diagonally. + setRGB(y * width + x, + scale(sin(x * zoom * 8 + beat(bpm, t)), 256), + scale(sin(y * zoom * 8 + beat(bpm, t)), 256), + scale(cos((x + y) * zoom * 4 + beat(bpm, t)), 256)); + } +} diff --git a/moonlive/effects/ripples.mlv b/moonlive/effects/ripples.mlv new file mode 100644 index 00000000..14a1f97d --- /dev/null +++ b/moonlive/effects/ripples.mlv @@ -0,0 +1,23 @@ +// Ripples: two wave sources gliding around the grid, their expanding rings interfering +// where they cross. Nothing is stored between frames: the picture IS the distance field, +// and it moves because the sources do. Squared distance feeds sin directly (one turn is +// 0..65535 and wraps), so no square root is needed, and unsigned wrap-around makes +// (x - cx) * (x - cx) correct even when the source is to the right of the cell. +// +// The heaviest shipped script: ~15 host calls per cell against plasma's 9, so it is also +// the working stress test for the call path. +uint8_t bpm = 10; // @control 1..120 +uint8_t rings = 8; // @control 1..32 + +for (y = 0; y < height; y = y + 1) { + for (x = 0; x < width; x = x + 1) { + setRGB(y * width + x, + scale(sin(((x - beatsin(bpm, t, width - 1)) * (x - beatsin(bpm, t, width - 1)) + + (y - beatsin(bpm + 4, t, height - 1)) * (y - beatsin(bpm + 4, t, height - 1))) + * rings * 64), 256), + scale(sin(((x - beatsin(bpm + 7, t, width - 1)) * (x - beatsin(bpm + 7, t, width - 1)) + + (y - beatsin(bpm + 3, t, height - 1)) * (y - beatsin(bpm + 3, t, height - 1))) + * rings * 64), 256), + scale(sin((x + y) * rings * 32 + beat(bpm, t)), 256)); + } +} diff --git a/moonlive/layouts/rose.mlv b/moonlive/layouts/rose.mlv new file mode 100644 index 00000000..5b1710bd --- /dev/null +++ b/moonlive/layouts/rose.mlv @@ -0,0 +1,20 @@ +// Rose: the strand traces a rhodonea curve, a circle whose radius swells and collapses +// `petals` times per revolution, drawing a flower. The classic polar curve r = sin(k * a), +// built from the layout vocabulary alone: turn(n) steps the angle, sin(a * petals) is the +// petal envelope, and the biased-unsigned trick from the effect docs +// (scale(cos(a), 2 * r + 1) sweeps the whole diameter) centers each axis. +// +// The envelope is recomputed where it is used: the grammar has no locals, and a layout +// walk runs once per edit, so clarity beats the repeated call. +uint8_t petals = 2; // @control 1..8 +uint8_t radius = 15; // @control 4..30 + +for (i = 0; i < 256; i = i + 1) { + addLight(radius - scale(sin(i * turn(256) * petals), radius + 1) + + scale(cos(i * turn(256)), + 2 * scale(sin(i * turn(256) * petals), radius + 1) + 1), + radius - scale(sin(i * turn(256) * petals), radius + 1) + + scale(sin(i * turn(256)), + 2 * scale(sin(i * turn(256) * petals), radius + 1) + 1), + 0); +} diff --git a/src/core/HttpServerModule.cpp b/src/core/HttpServerModule.cpp index b037b35a..8ab213e8 100644 --- a/src/core/HttpServerModule.cpp +++ b/src/core/HttpServerModule.cpp @@ -1679,7 +1679,7 @@ HttpServerModule::OpResult HttpServerModule::applyAddModule( mod->defineControls(); mod->setup(); mod->applyState(); - if (scheduler_) scheduler_->prepareTree(); + if (scheduler_) scheduler_->requestPrepareTree(); requestFullResync(); // structural change (see requestFullResync) // Persist the new tree shape (debounced save via noteDirty). @@ -1751,7 +1751,7 @@ HttpServerModule::OpResult HttpServerModule::applyClearChildren(const char* pare removedAny = true; } if (removedAny) { - if (scheduler_) scheduler_->prepareTree(); + if (scheduler_) scheduler_->requestPrepareTree(); requestFullResync(); // structural change (see requestFullResync) parent->markDirty(); FilesystemModule::noteDirty(); @@ -1834,7 +1834,7 @@ void HttpServerModule::handleDeleteModule(platform::TcpConnection& conn, const c mod->release(); Scheduler::deleteTree(mod); - if (scheduler_) scheduler_->prepareTree(); + if (scheduler_) scheduler_->requestPrepareTree(); requestFullResync(); // structural change (see requestFullResync) // Persist the new tree shape — marking the parent dirty rewrites its file @@ -1928,7 +1928,7 @@ void HttpServerModule::handleReplaceModule(platform::TcpConnection& conn, const // Re-run prepare across the tree so Layer LUT / Drivers buffer // wiring re-forms — a replaced effect/driver re-wires like a freshly added one. - if (scheduler_) scheduler_->prepareTree(); + if (scheduler_) scheduler_->requestPrepareTree(); requestFullResync(); // structural change (see requestFullResync) // Persist: children are encoded positionally, so marking the parent dirty @@ -2027,7 +2027,7 @@ void HttpServerModule::handleMoveModule(platform::TcpConnection& conn, const cha // file is rewritten with the new order (same as add/delete handlers). parent->markDirty(); FilesystemModule::noteDirty(); - if (scheduler_) scheduler_->prepareTree(); + if (scheduler_) scheduler_->requestPrepareTree(); requestFullResync(); // structural change (see requestFullResync) sendResponse(conn, 200, "application/json", "{\"ok\":true}"); } diff --git a/src/core/NetworkModule.h b/src/core/NetworkModule.h index 3c8f364f..53fad16d 100644 --- a/src/core/NetworkModule.h +++ b/src/core/NetworkModule.h @@ -282,11 +282,13 @@ class NetworkModule : public MoonModule { controls_.setHidden(controls_.count() - 1, state_ != State::ConnectedSta); // TX power applies whenever the WiFi radio is active (STA or AP). // Hide on Ethernet / Idle where the radio is off. + // Expert-only: a radio-tuning readout, not something a normal install reads. controls_.addReadOnlyInt("txPower", txPower_, "dBm"); const bool radioOn = (state_ == State::ConnectedSta || state_ == State::WaitingSta || state_ == State::AP); controls_.setHidden(controls_.count() - 1, !radioOn); + controls_.setAdvanced(controls_.count() - 1); // Writable TX-power cap (the weak-power / brown-out WiFi cap). Range 0..21 dBm. // 0 = "no override" (sentinel — syncTxPower then writes the // ESP-IDF ceiling, ~20 dBm, to actively lift any prior cap; @@ -299,8 +301,11 @@ class NetworkModule : public MoonModule { // TX-power cap is meaningless on Ethernet / Idle where the radio is off. controls_.addInt16("txPowerSetting", txPowerSetting_, 0, 21); controls_.setHidden(controls_.count() - 1, !radioOn); + controls_.setAdvanced(controls_.count() - 1); } + // Expert-only: discovery works without it, and the projectMM UI finds devices over UDP. controls_.addBool("mDNS", mdnsEnabled_); + controls_.setAdvanced(controls_.count() - 1); // addressing goes immediately before the static-IP fields it conditions, so // the dropdown and the fields it reveals stay adjacent (mDNS, unrelated, @@ -747,7 +752,11 @@ class NetworkModule : public MoonModule { // ethConfigDefault so a board that DOES opt in gets its chip's historical pins without // re-listing them; only the PHY *selection* defaults off. Matches the installer UI, // whose Ethernet pill is "active" (green) only when ethType is set (ethConfigured()). - uint8_t ethType_ = 0; // 0 = None; a board opts in via its catalog eth block + // 0 = None; a board opts in via its catalog eth block. Left at None even where the platform + // FIXES the interface (ethPhyIsFixed): syncEthConfig overrides the value on those targets, so + // seeding it here would buy nothing and would put a value outside this Select's own option list + // into a persisted control, which the settings loader then clamps to a DIFFERENT, real PHY. + uint8_t ethType_ = static_cast(platform::ethNone); // GPIO/address members are int8_t (one byte; -1 = unused). A GPIO never exceeds // ~54 on any ESP32-family chip, so int8 is ample — bound via addPin (Pin control // → number input). ethConfigDefault's fields are plain int; the values are all @@ -795,7 +804,11 @@ class NetworkModule : public MoonModule { void syncEthConfig() { if constexpr (platform::hasEthernet) { platform::EthPinConfig cfg{}; - cfg.phyType = ethType_; + // Where the platform FIXES the interface (ethPhyIsFixed), its type wins over the stored + // control: a persisted value would otherwise select hardware that does not exist there. + // On real silicon the flag is false and the board's own catalog value wins, as before. + cfg.phyType = platform::ethPhyIsFixed + ? static_cast(platform::ethConfigDefault.phyType) : ethType_; cfg.phyAddr = ethPhyAddr_; cfg.mdcGpio = ethMdcGpio_; cfg.mdioGpio = ethMdioGpio_; diff --git a/src/core/Scheduler.cpp b/src/core/Scheduler.cpp index 4e88fcb8..089fdc46 100644 --- a/src/core/Scheduler.cpp +++ b/src/core/Scheduler.cpp @@ -67,6 +67,25 @@ void Scheduler::tick() MM_NONBLOCKING { uint32_t now = platform::millis(); uint32_t tickStart = platform::micros(); + // A rebuild asked for from another task happens HERE, at a frame boundary on the render thread. + // The walk runs a scripted layout's compiled code, and that code's frame is ordinary stack on + // whichever task calls it — so doing it inline in an HTTP handler put a script on the web + // server's small stack instead of the render task's, which is the budget every other effect is + // measured against. Deferring also means the pipeline is never rebuilt underneath a half-rendered + // frame. + // exchange, not test-then-clear: a request arriving between the two would be dropped. + // + // This gate is what keeps tick() honest about its MM_NONBLOCKING annotation. prepareTree + // allocates, reads the filesystem and runs the JIT, none of which belongs in a frame. It runs + // only when another task asked for a rebuild, which is a user action (a control edit, a module + // added or removed, a script renamed), never a frame; a steady-state tick pays one relaxed + // exchange. The static analyser cannot see that, so it reports the path transitively, and the + // finding stays in the report on purpose: if prepareTree ever becomes reachable WITHOUT this + // gate, that report is the only thing that will say so. + if (prepareRequested_.exchange(false, std::memory_order_relaxed)) { + prepareTree(); + } + // Scheduler gates loop callbacks by `enabled()` — disabled modules don't tick. // System modules that need to keep running regardless (HttpServer, Network, // Filesystem — so users can re-enable other modules through them) override @@ -220,7 +239,7 @@ Scheduler::SetControlResult Scheduler::setControl(const char* moduleName, target->setEnabled(mm::json::parseBool(valueJson, "value")); target->markDirty(); if (noteDirtyHook_) noteDirtyHook_(); - prepareTree(); + requestPrepareTree(); // `enabled` rides the FULL state, not the per-leaf value patch — so the client only learns the new // value from a full resync. Request one (the same signal a schema change sends); without it the // client's cached state keeps the old `enabled` and reverts the toggle a second later. @@ -248,7 +267,7 @@ Scheduler::SetControlResult Scheduler::setControl(const char* moduleName, target->onControlChanged(controlName); target->markDirty(); if (noteDirtyHook_) noteDirtyHook_(); - if (target->affectsPrepare(controlName)) prepareTree(); + if (target->affectsPrepare(controlName)) requestPrepareTree(); return SetControlResult::Ok; } return SetControlResult::ControlNotFound; diff --git a/src/core/Scheduler.h b/src/core/Scheduler.h index 98917d44..d6d0ced8 100644 --- a/src/core/Scheduler.h +++ b/src/core/Scheduler.h @@ -3,6 +3,7 @@ #include "core/MoonModule.h" #include +#include #include namespace mm { @@ -66,8 +67,20 @@ class Scheduler { void release(); uint32_t elapsed() const; + + /// Rebuild derived state across the whole tree — buffers, mappings, and any scripted module's + /// compiled program. Runs the work IMMEDIATELY on the calling thread. + /// + /// Prefer requestPrepareTree() from anything but the render loop: this walk runs a scripted + /// layout's JIT'd code, whose frame lives on the CALLING TASK's stack like any other function's. + /// Called from an HTTP handler it therefore executes on the small web-server task rather than + /// the render task the rest of the pipeline is budgeted against. void prepareTree(); + /// Ask for a rebuild at the next frame boundary, on the render thread. Cheap and safe to call + /// from any task — it sets a flag; tick() does the work. + void requestPrepareTree() { prepareRequested_.store(true, std::memory_order_relaxed); } + uint32_t tickTimeUs() const { return tickTimeUs_; } uint32_t fps() const { return tickTimeUs_ > 0 ? 1000000 / tickTimeUs_ : 0; } uint8_t moduleCount() const { return moduleCount_; } @@ -124,6 +137,9 @@ class Scheduler { static inline Scheduler* instance_ = nullptr; std::array modules_{}; uint8_t moduleCount_ = 0; + // ATOMIC: written from HTTP handlers (any task) and consumed on the render thread, so a + // plain bool is a data race — and a lost request means a script edit silently never applies. + std::atomic prepareRequested_{false}; // asked for off-thread; tick() honours it LoadAllFn loadAllHook_ = nullptr; NoteDirtyFn noteDirtyHook_ = nullptr; uint32_t startTime_ = 0; diff --git a/src/core/moonlive/MoonLive.cpp b/src/core/moonlive/MoonLive.cpp index 85872b1c..574181c6 100644 --- a/src/core/moonlive/MoonLive.cpp +++ b/src/core/moonlive/MoonLive.cpp @@ -25,9 +25,9 @@ void MoonLive::freeCode() { void* MoonLive::place(const uint8_t* staged, size_t len) { freeCode(); // drop any prior compilation's code — (re)compile is a clean re-emit (arena kept) if (len == 0) { error_ = "emit failed"; return nullptr; } - // Allocate only what was emitted, word-rounded (writeExec stores 32-bit words on IRAM), not - // the worst-case kCodeCap — a fill is ~50 bytes, a four-call setRGB ~600. The staging buffer - // is sized for the worst case; the live exec block is sized for THIS program. + // Allocate only what was emitted, word-rounded (writeExec stores 32-bit words on IRAM) — a fill + // is ~50 bytes, a four-call setRGB ~600. The staging buffer is sized from the script's token + // count (codeCapFor); the live exec block is sized to what the program actually emitted. size_t cap = (len + 3) & ~size_t(3); void* block = platform::allocExec(cap); if (!block) { error_ = "no executable memory"; return nullptr; } @@ -39,25 +39,52 @@ void* MoonLive::place(const uint8_t* staged, size_t len) { return block; } +// The emitted-code staging buffer, on the HEAP. +// +// It was `uint8_t staging[kCodeCap]` — 2 KB of stack, in a call chain that also holds the +// assembler's own 2 KB buffer and its tables: ~4.7 KB in one go. On a classic ESP32 that overflowed +// the task the compile runs on, and the fault surfaced as `Double exception` inside +// _xt_context_save (the handler faulting while saving context) with LBEG pointing back into +// MoonLive::compile — a crash on the HTTP task from naming a script. Compilation is cold path, so +// the allocation costs nothing that matters, and this is the same reasoning that moved IrProgram's +// op array off the stack. +// Sized per compile rather than to kCodeCap: that constant is now the SANITY bound (16 KB), and +// allocating it for every script would trade one wall for a heap cost on the smallest script. +namespace { +struct Staging { + explicit Staging(size_t bytes) : p(static_cast(platform::alloc(bytes))), n(bytes) {} + ~Staging() { platform::free(p); } + Staging(const Staging&) = delete; // owns a buffer; a copy would double-free + Staging& operator=(const Staging&) = delete; + explicit operator bool() const { return p != nullptr; } + uint8_t* p; + size_t n; +}; +} // namespace + bool MoonLive::compile(uint8_t r, uint8_t g, uint8_t b) { - uint8_t staging[kCodeCap]; - size_t len = emitFill(staging, kCodeCap, r, g, b); - void* block = place(staging, len); + // A fixed blob with no source to measure, so codeCapFor(0) gives its 256-byte floor — emitFill + // emits a few dozen bytes and the exec block is allocated to the real length. + Staging staging(codeCapFor(0)); + if (!staging) { error_ = "no memory to compile"; return false; } + size_t len = emitFill(staging.p, staging.n, r, g, b); + void* block = place(staging.p, len); if (!block) return false; fn_ = reinterpret_cast(block); return true; } bool MoonLive::compile(const char* source, const BuiltinTable& table, const SysVarTable& sysvars) { - uint8_t staging[kCodeCap]; - CompileResult cr = compileSource(source, table, sysvars, staging, kCodeCap); + Staging staging(codeCapFor(countTokens(source))); + if (!staging) { freeCode(); error_ = "no memory to compile"; return false; } + CompileResult cr = compileSource(source, table, sysvars, staging.p, staging.n); if (!cr.ok) { freeCode(); error_ = cr.error; return false; } // surface the parse diagnostic // Allocate the control arena (fixed address) and seed new slots, BEFORE publishing the control // set — ensureArena reads the previous controlCount_ to know which slots are new. if (!ensureArena(cr.controls, cr.controlCount)) { freeCode(); error_ = "no control memory"; return false; } // Place the code. Only after it succeeds do we publish the new control set — a failed place() // must not leave declaredControls() advertising controls for code that isn't running. - void* block = place(staging, cr.len); + void* block = place(staging.p, cr.len); if (!block) return false; // controlCount_/controls_ unchanged // Clamp any kept slot whose range shrank (e.g. @control 0..99 edited to 0..10) so a stale live // value can't fall outside the new bounds before the native code reads it. @@ -67,7 +94,16 @@ bool MoonLive::compile(const char* source, const BuiltinTable& table, const SysV else if (ctrlArena_[i] > hi) ctrlArena_[i] = hi; } controlCount_ = cr.controlCount; - for (uint8_t i = 0; i < cr.controlCount; i++) controls_[i] = cr.controls[i]; + for (uint8_t i = 0; i < cr.controlCount; i++) { + controls_[i] = cr.controls[i]; + // Re-point `name` at our own copy: the parser's pointer is into the source text, which the + // caller may free as soon as this returns. + const uint8_t len = cr.controls[i].nameLen < kMaxControlName - 1 + ? cr.controls[i].nameLen : static_cast(kMaxControlName - 1); + for (uint8_t j = 0; j < len; j++) ctrlNames_[i][j] = cr.controls[i].name[j]; + ctrlNames_[i][len] = '\0'; + controls_[i].name = ctrlNames_[i]; + } ctrl_ = reinterpret_cast(block); return true; } @@ -91,9 +127,10 @@ bool MoonLive::ensureArena(const DeclaredControl* decls, uint8_t count) { } bool MoonLive::compileAnimated() { - uint8_t staging[kCodeCap]; - size_t len = emitAnimatedFill(staging, kCodeCap); - void* block = place(staging, len); + Staging staging(codeCapFor(0)); // a fixed blob, like compile(r,g,b) — no source to measure + if (!staging) { error_ = "no memory to compile"; return false; } + size_t len = emitAnimatedFill(staging.p, staging.n); + void* block = place(staging.p, len); if (!block) return false; anim_ = reinterpret_cast(block); return true; diff --git a/src/core/moonlive/MoonLive.h b/src/core/moonlive/MoonLive.h index bac22e45..45fb5b36 100644 --- a/src/core/moonlive/MoonLive.h +++ b/src/core/moonlive/MoonLive.h @@ -53,7 +53,11 @@ class MoonLive { // grid size / layout, the hard rule). void run(uint8_t* buf, uint32_t nLights, uint8_t cpl, uint32_t t) const { if (!buf || nLights == 0 || cpl < 3) return; - if (ctrl_) ctrl_(buf, nLights, cpl, t, ctrlArena_); // front-end-compiled (reads controls) + // The arena is the fifth argument, and a front-end-compiled program reads its controls and + // system variables straight through it — a null there is dereferenced by the EMITTED code, + // which faults as a LoadProhibited at a nonsense address with no C++ frame to blame. Checked + // with the other preconditions rather than trusted: every other operand of the call is. + if (ctrl_ && ctrlArena_) ctrl_(buf, nLights, cpl, t, ctrlArena_); // front-end-compiled else if (fn_) fn_(buf, nLights, cpl); // hand-encoded fixed fill else if (anim_) anim_(buf, nLights, cpl, t); // hand-encoded animated fill } @@ -62,6 +66,13 @@ class MoonLive { // memory). void free(); + // Drop the compiled code, keeping the control arena. What a caller wants when a script STOPS + // being usable (renamed, deleted, emptied) but the module lives on: ok() goes false so a tick + // renders nothing, while a bound control pointer stays valid and a scripted control keeps the + // live value the user set. free() would take the arena too, and the next compile would re-seed + // every control from its declared default. + void freeCode(); + // The emitted code length, for the golden-bytes test (0 until compiled). size_t codeLen() const { return codeLen_; } // The allocated exec-block size (word-rounded codeLen) — the actual heap held, for memory @@ -95,10 +106,6 @@ class MoonLive { // cast by the caller. void* place(const uint8_t* staged, size_t len); - // Drop the prior compilation's code (exec block + fn pointers) but keep the control arena, so a - // recompile re-emits cleanly without moving the arena a bound control pointer references. - void freeCode(); - // Ensure the control arena holds `count` bytes, seeding new slots from `decls[i].def`. Grows // capacity (never shrinks/moves) so a control pointer the binding holds stays valid across a // recompile; preserves an existing slot's live value when the script is edited but the control @@ -116,6 +123,12 @@ class MoonLive { uint8_t* ctrlArena_ = nullptr; // live control + system-variable bytes (platform::alloc, kArenaBytes, fixed) uint8_t controlCount_ = 0; // controls the current program declared DeclaredControl controls_[kMaxCtrls] = {}; // the declared-control metadata for the binding + // The declared NAMES, owned. A DeclaredControl's `name` points into the SOURCE TEXT, which the + // caller is free to release the moment compile() returns — and does, now that a script is read + // from a file into a transient buffer. Copying the bytes here is what lets the engine outlive + // the text it was built from; without it a binding reads freed memory when it publishes its + // controls, which showed up as a control literally named "\x05". + char ctrlNames_[kMaxCtrls][kMaxControlName] = {}; }; } // namespace mm::moonlive diff --git a/src/core/moonlive/MoonLiveBuiltins.h b/src/core/moonlive/MoonLiveBuiltins.h index f7fd8cf3..03aa770b 100644 --- a/src/core/moonlive/MoonLiveBuiltins.h +++ b/src/core/moonlive/MoonLiveBuiltins.h @@ -37,14 +37,25 @@ enum class BuiltinKind : uint8_t { Call, Inline }; // A host callable. THREE unsigned args in, one unsigned result out. // // One argument covered a unary helper like random16, but a binding that hands the host a POSITION -// needs three at once — `addLight(x, y, z)` is the shape MoonLight's own script binding uses -// (`void addLight(uint16_t,uint16_t,uint16_t)`), and it is what lets a scripted layout emit a light -// instead of writing into an array the host has to size in advance. A unary helper simply ignores -// the arguments it was not given; the caller passes 0 for them. +// A host function receives a POINTER to its arguments, not the arguments themselves. // -// A typed alias keeps the table and IR type-safe across desktop and ESP32 instead of threading a -// bare void*. -using HostCallFn = uint32_t (*)(uint32_t, uint32_t, uint32_t); +// The compiler already evaluates every argument into a CONSECUTIVE frame slot (the stack machine's +// argument staging), so the call only has to say where they start. Each backend materialises that +// address from its own frame pointer — the arithmetic spillStore/spillLoad already do — which means +// the number of arguments is bounded by frame slots rather than by how many the calling convention +// can carry. `draw::line` takes seven; a fixed three would have forced it to be split into bespoke +// halves, and every power function added after it would inherit the same distortion. +// +// Deliberately still THREE C parameters, so each assembler's call sequence is untouched. On Xtensa +// that sequence is a windowed `call8`, the most defect-prone code in this project; widening it would +// have spent the change there and still left a fixed maximum, just a larger one. +// +// `arena` is the control/system-variable block, as before. +// `args` points at `argc` frame slots. The element type is uintptr_t because a frame slot IS one +// machine word — 8 bytes on arm64, 4 on Xtensa and RISC-V — and the backends store a whole word per +// slot. Reading them as uint32_t made args[1] land on the upper half of slot 0 on a 64-bit host, +// which is a value of 0 rather than the argument: correct on both devices, wrong on the desktop. +using HostCallFn = uint32_t (*)(const uintptr_t* args, uint32_t argc, const uint8_t* arena); struct Builtin { const char* name = nullptr; // the script-visible name (host-owned) @@ -87,26 +98,33 @@ static constexpr uint8_t kMaxCtrls = 8; // a script declares a handful // System variables sit ABOVE the script's range so that adding or removing a control — which // renumbers every control offset — cannot move them. The binding caches their slot pointers, so a // moving offset would silently write the wrong byte. -// Fixed cap for an emitted routine, shared by the engine's staging buffer and EVERY backend's code -// buffer — they must agree, or a script that fits the caller's buffer still overflows the -// assembler's. One constant is what makes that structural instead of a comment in four files. -// -// It was once sized for the heaviest single STATEMENT, but a real effect is several statements, and -// the shipped `lines.mlv` emits 908 bytes on RISC-V against 461 on Xtensa for the same script: RISC-V -// is fixed-4-byte and saves the whole register pool around every call, so it needs roughly twice the -// room for identical work. Sizing to the DENSEST backend silently made a script that runs on an S3 -// fail on an S31. +// The emitted-code buffer is sized to THE SCRIPT (codeCapFor below), not to a constant, for the +// same reason the IR op array is: the backends differ by up to 1.9x on identical source — RISC-V is +// fixed-4-byte and saves the whole register pool around every call where Xtensa has 3-byte narrow +// forms — so any single number is either too small for the sparsest backend or wasteful for the +// densest. A fixed 2 KB let `plasma.mlv` run on an S3 and desktop and REFUSED it on an S31 by 96 +// bytes, which is the second time one constant made a script's portability depend on its ISA. // -// 2 KB covers the measured worst case with headroom. The emitter returns the real length and the -// live exec block is allocated to THAT, so the tail costs nothing beyond one staging buffer during -// the compile. Word-aligned so allocExec/writeExec's word-rounding never exceeds it. -static constexpr size_t kCodeCap = 2048; - -// The script text a binding holds. 1 KB, not 512 B: 512 could not hold a DOCUMENTED script — the -// shipped lines.mlv is ~490 characters with its comments — and a script that overruns is silently -// truncated mid-token, so it fails to compile with no hint that length was the reason. A binding is -// ~2 KB at this size, which the smallest board still carries. -static constexpr size_t kMaxScriptBytes = 1024; +// kCodeCap survives as the SANITY bound only: a runaway script fails with a diagnostic instead of +// exhausting the heap. It is not the working limit, so it is sized well above any real script. +static constexpr size_t kCodeCap = 16384; + +/// Bytes to reserve for a script of `tokens` tokens. Over-estimating costs one cold-path allocation +/// that is freed when the compile ends; under-estimating fails a script that would have fit, so the +/// direction of the error is deliberate — the same rule the IR's op estimate follows. +/// +/// 64 bytes/token, measured across every shipped script on all three backends with `countTokens` +/// (which skips comments, so a long header does not inflate the count). The densest is +/// `random-pixel.mlv` at 39.3 — one statement, four nested `random16()` calls, and on RISC-V each +/// call saves and restores the whole register pool — so this is a ~1.6x margin over the worst real +/// case. A SHORT call-dense script sets the bound, not a long one: a call lowers to a save/restore +/// while declarations and operators lower to a few instructions each, so bytes-per-token FALLS as a +/// script grows. The floor covers a tiny script's fixed prologue and epilogue, which no per-token +/// figure expresses. +constexpr size_t codeCapFor(uint32_t tokens) { + const size_t want = size_t(tokens) * 64 + 256; + return want > kCodeCap ? kCodeCap : want; +} static constexpr uint8_t kMaxSysVars = 8; static constexpr uint8_t kArenaBytes = kMaxCtrls + kMaxSysVars; diff --git a/src/core/moonlive/MoonLiveCompiler.cpp b/src/core/moonlive/MoonLiveCompiler.cpp index d6d0622e..4cb43298 100644 --- a/src/core/moonlive/MoonLiveCompiler.cpp +++ b/src/core/moonlive/MoonLiveCompiler.cpp @@ -115,12 +115,22 @@ struct Parser { VReg nextTemp = kFirstTemp; // high-water mark — also IrProgram.vregsUsed VReg freeStack[kMaxVRegs] = {}; // recycled temps (LIFO), so a dead vreg is reused uint8_t freeCount = 0; - // Script-local variables — today only a `for` loop's counter. Distinct from a declared control - // (a control is a UI value the script READS; a local is one the script WRITES) and from a temp - // (a temp is write-once and recycled). Held in a vreg for the loop's lifetime. - struct Local { const char* name; size_t nameLen; VReg reg; }; - Local locals[4] = {}; + // Script-local variables — a `for` loop's counter and its limit. Distinct from a declared + // control (a control is a UI value the script READS; a local is one the script WRITES) and from + // a temp (a temp is write-once and recycled). + // + // A local lives in a FRAME SLOT, not a register. Holding it in a register for the whole of its + // scope is what made a nested loop unfittable on the smallest target: Xtensa's windowed ABI + // leaves ten registers, five carry the host arguments, and the reload temps take the rest — so a + // loop counter that never yields its register left nothing to compute with and every looped + // script was refused. In the frame, the number of live variables is bounded by memory rather + // than by the register file, which is the whole point of the stack machine, and it is also what + // makes function arguments and recursion fall out later rather than needing new machinery. + struct Local { const char* name; size_t nameLen; uint8_t slot; }; + Local locals[kMaxLocals] = {}; uint8_t localCount = 0; + uint8_t slotHighWater = 0; // slots currently in scope + uint8_t slotsUsed = 0; // PEAK slots — what the prologue reserves uint8_t nextLabel = 0; // IR label ids, handed out in source order DeclaredControl controls[kMaxCtrls] = {}; // controls the script declared (decl lines) @@ -154,10 +164,8 @@ struct Parser { return kFirstTemp; } void freeTemp(VReg v) { - // A loop variable's register is live for the whole loop, and parseExpr hands it back - // directly (see parsePrimary), so a consumer freeing "its" operand would recycle a vreg the - // loop still reads and the counter would be overwritten mid-iteration. - for (uint8_t i = 0; i < localCount; i++) if (locals[i].reg == v) return; + // No local-register guard is needed: a variable lives in a frame slot, and reading one + // hands back an ordinary temp that the consumer owns. Every vreg reaching here is a temp. if (v >= kFirstTemp && freeCount < kMaxVRegs) freeStack[freeCount++] = v; } @@ -252,7 +260,11 @@ struct Parser { // declaration paths reject the name, so nothing can shadow it. if (const SysVar* sv = sysvars.find(lex.identBeg, lex.identLen)) { lex.advance(); - if (sv->kind == SysVarKind::Arg) return sv->where; // free: already in a register + if (sv->kind == SysVarKind::Arg) { + VReg v = alloc(); // parked at entry — bring it back for this one read + emit({IrOp::Reload, v, 0,0,0,0, hostArgSlot(sv->where), nullptr, {}}); + return v; + } VReg v = alloc(); emit({IrOp::LoadCtrl, v, 0,0,0,0, sv->where, nullptr, {}}); return v; @@ -260,13 +272,14 @@ struct Parser { for (uint8_t li = 0; li < localCount; li++) { if (locals[li].nameLen == lex.identLen && std::strncmp(locals[li].name, lex.identBeg, lex.identLen) == 0) { - // Return the variable's OWN register rather than copying it into a temp. A copy - // per read burns a vreg each time, and the budget is small — the lowerer needs - // three scratch registers above the program's high-water mark, so a script has - // about six temps in total. Callers must therefore not freeTemp() a local; the - // free-list only ever holds values alloc() handed out. + // Load the variable from its frame slot into a fresh temp. The temp is ordinary: + // the caller consumes it and frees it like any other expression value, so a + // variable occupies a register only for the instruction that reads it rather + // than for the whole of its scope. lex.advance(); - return locals[li].reg; + VReg v = alloc(); + emit({IrOp::Reload, v, 0,0,0,0, locals[li].slot, nullptr, {}}); + return v; } } int ci = findControl(lex.identBeg, lex.identLen); @@ -294,51 +307,81 @@ struct Parser { lex.advance(); if (!expect(Tok::LParen, "expected '(' after the function name")) return; - // Evaluate each argument expression into a vreg. - VReg args[4] = {0,0,0,0}; + // Evaluate each argument, PARKING it in a frame slot as soon as it is finished. + // + // The op that consumes them reads every argument at once, so evaluating straight into vregs + // keeps all four live simultaneously — and each argument's own sub-expression needs + // registers on top of that. On Xtensa's ten that is what refused every looped effect: a + // four-operand setRGB left nothing to compute the operands WITH. Staging through the frame + // means only ONE argument occupies a register at a time; they come back together in the + // reload just before the op, where the peak is exactly the operand count and nothing more. + // Stage every argument into a CONSECUTIVE frame slot as it is evaluated. The call then + // carries the slot BASE and the COUNT rather than the values, so how many arguments a + // builtin takes is bounded by frame slots — `draw::line` wants seven — and only one + // argument occupies a register at a time. + const uint8_t argBase = slotHighWater; uint8_t n = 0; if (lex.kind != Tok::RParen) { while (true) { - if (n >= fn->argc || n >= 4) { fail("too many arguments"); return; } - args[n++] = parseExpr(); + if (n >= fn->argc) { fail("too many arguments"); return; } + const VReg v = parseExpr(); if (failed) return; + if (slotHighWater >= kMaxLocals) { fail("too many arguments to hold"); return; } + emit({IrOp::Spill, 0, v, 0,0,0, slotHighWater++, nullptr, {}}); + freeTemp(v); // its register is free again immediately + n++; if (lex.kind == Tok::Comma) { lex.advance(); continue; } break; } } + if (slotHighWater > slotsUsed) slotsUsed = slotHighWater; if (n != fn->argc) { fail("wrong number of arguments"); return; } if (!expect(Tok::RParen, "expected ')'")) return; // The IR Call op carries a single argument vreg, so a Call-kind builtin must be unary. // (Today random16 is the only one.) Reject a multi-arg Call up front rather than silently // dropping args[1..]; a future N-ary helper needs the IR Call contract widened first. - if (fn->kind == BuiltinKind::Call && fn->argc > 3) { fail("a call takes at most three arguments"); return; } - if (resultOut) { if (fn->kind != BuiltinKind::Call || !fn->returns) { fail("this function does not return a value"); return; } - // The argument temps are consumed by the call; free them, then allocate the result - // (which may reuse one of them) — this is what bounds the register count across a - // chain of calls. The IR Call reads its arg before the result is written, so reuse is - // safe even when result == arg. - for (uint8_t i = 0; i < n; i++) freeTemp(args[i]); + // The arguments live in the frame, so nothing is held in a register across the call: + // `imm` carries the slot they start at and `b` how many there are. VReg r = alloc(); - emit({IrOp::Call, r, args[0], args[1], args[2], 0, 0, fn->fn, {}}); + emit({IrOp::Call, r, 0, n, 0, 0, argBase, fn->fn, {}}); *resultOut = r; } else { // A statement call. Call kinds with a result are also allowed as statements (result // discarded); Inline kinds emit the inline op with their operands. if (fn->kind == BuiltinKind::Call) { - for (uint8_t i = 0; i < n; i++) freeTemp(args[i]); VReg r = alloc(); - emit({IrOp::Call, r, args[0], args[1], args[2], 0, 0, fn->fn, {}}); + emit({IrOp::Call, r, 0, n, 0, 0, argBase, fn->fn, {}}); freeTemp(r); } else { - // Inline op: hand the operand vregs to the backend via an Inline IR op. The - // operand mapping per inline op is the backend's contract (documented there). - emit({IrOp::Inline, 0, args[0], args[1], args[2], args[3], 0, nullptr, fn->inlineOp}); - for (uint8_t i = 0; i < n; i++) freeTemp(args[i]); + // An inline op reads its operands from REGISTERS (it is emitted as instructions, not + // a call), so reload the staged arguments for the one op that consumes them. + // + // FOUR is the IrInst operand ceiling for an inline op. A builtin declaring more used + // to be truncated here — the extra arguments evaluated, then silently dropped — which + // emits a working-looking op that computes the wrong thing. Refuse instead: only a + // CALL is unbounded (its arguments go through the frame), so a wider inline builtin + // needs the IR widened first, not its arguments quietly discarded. + if (n > 4) { fail("this function takes too many arguments to inline"); return; } + VReg a0 = 0, a1 = 0, a2 = 0, a3 = 0; + VReg* slot[4] = {&a0, &a1, &a2, &a3}; + for (uint8_t i = 0; i < n && i < 4; i++) { + *slot[i] = alloc(); + emit({IrOp::Reload, *slot[i], 0,0,0,0, static_cast(argBase + i), nullptr, {}}); + } + emit({IrOp::Inline, 0, a0, a1, a2, a3, 0, nullptr, fn->inlineOp}); + for (uint8_t i = 0; i < n && i < 4; i++) freeTemp(*slot[i]); } } + // Give the staging slots back — but only down to where THIS call started, and only once its + // result has been produced. A nested call (`setRGB(1, mod(t, 200), …)`) stages inside its + // parent's argument block, so resetting to a fixed base would hand the inner call the slots + // the outer one is still filling: measured as mod()'s arguments landing on setRGB's, and its + // result then overwriting them. `slotsUsed` keeps the peak, which is what the prologue + // reserves, so releasing here costs nothing and lets sequential calls reuse the space. + slotHighWater = argBase; } // A control declaration: `uint8_t ident = number ;` optionally followed by `// @control min..max`. @@ -414,7 +457,9 @@ struct Parser { // --- init: ident = expr --- if (lex.kind != Tok::Ident) { fail("expected a loop variable"); return false; } - if (localCount >= 4) { fail("too many nested loops"); return false; } + // Two slots per loop: the counter and the limit. Both must outlive the body, and both live + // in the frame — nesting depth is now bounded by frame slots, not by the register file. + if (localCount + 2 > kMaxLocals) { fail("too many nested loops"); return false; } const char* varName = lex.identBeg; const size_t varLen = lex.identLen; if (sysvars.find(varName, varLen)) { fail("name is a system variable"); return false; } @@ -431,11 +476,16 @@ struct Parser { if (!expect(Tok::Assign, "expected '=' in the for's first clause")) return false; VReg init = parseExpr(); if (failed) return false; - VReg counter = alloc(); - emit({IrOp::Mov, counter, init, 0,0,0, 0, nullptr, {}}); + // The counter starts life in its slot; the temp that computed it is released immediately. + // Bounded on slotHighWater, not just localCount: a call RELEASES its argument staging slots + // (slotHighWater = argBase) without ever having counted them as locals, so the two can + // diverge and the localCount check above is not sufficient on its own. + if (slotHighWater >= kMaxLocals) { fail("too many loop variables"); return false; } + const uint8_t counterSlot = slotHighWater++; + emit({IrOp::Spill, 0, init, 0,0,0, counterSlot, nullptr, {}}); freeTemp(init); const uint8_t myLocal = localCount; - locals[localCount++] = {varName, varLen, counter}; + locals[localCount++] = {varName, varLen, counterSlot}; if (!expect(Tok::Semicolon, "expected ';' after the for's first clause")) return false; // --- condition: ident < expr (the only comparison the language has) --- @@ -449,12 +499,15 @@ struct Parser { } lex.advance(); if (!expect(Tok::Less, "expected '<' — it is the only comparison a for condition takes")) return false; - // Hold the bound in the vreg parseExpr produced rather than copying it into a fresh one. - // The copy cost a register for the whole body, and the budget is small: Xtensa maps twelve - // registers, five of which are argument slots, so a NESTED loop plus a three-argument call - // ran out and the compile was refused on that target while succeeding on the host. - VReg limit = parseExpr(); + // The bound goes to its own slot: it is read at the entry guard and again at the back edge, + // so it has to survive the body — and a body containing a call would otherwise have to keep + // it in a register across that call. + VReg limitTmp = parseExpr(); if (failed) return false; + if (slotHighWater >= kMaxLocals) { fail("too many loop variables"); return false; } + const uint8_t limitSlot = slotHighWater++; + emit({IrOp::Spill, 0, limitTmp, 0,0,0, limitSlot, nullptr, {}}); + freeTemp(limitTmp); if (!expect(Tok::Semicolon, "expected ';' after the for's condition")) return false; // --- step: ident = expr (parsed now, emitted after the body) --- @@ -483,7 +536,16 @@ struct Parser { const uint8_t lDone = nextLabel++; const uint8_t lTop = nextLabel++; - emit({IrOp::BranchGe, 0, counter, limit, 0,0, lDone, nullptr, {}}); // empty range + // Each test reloads both operands: they live in the frame, so a comparison is + // load-load-branch. The reload temps die immediately, which is what keeps the body's + // register demand independent of how deeply loops nest. + { + VReg c = alloc(), l = alloc(); + emit({IrOp::Reload, c, 0,0,0,0, counterSlot, nullptr, {}}); + emit({IrOp::Reload, l, 0,0,0,0, limitSlot, nullptr, {}}); + emit({IrOp::BranchGe, 0, c, l, 0,0, lDone, nullptr, {}}); // empty range + freeTemp(l); freeTemp(c); + } emit({IrOp::Label, 0, 0,0,0,0, lTop, nullptr, {}}); while (!failed && lex.kind != Tok::RBrace && lex.kind != Tok::End) { @@ -504,25 +566,29 @@ struct Parser { if (lex.kind != Tok::RParen) { lex = save; fail("unexpected token in the for's step"); return false; } - emit({IrOp::Mov, counter, s, 0,0,0, 0, nullptr, {}}); + emit({IrOp::Spill, 0, s, 0,0,0, counterSlot, nullptr, {}}); freeTemp(s); lex = save; } // Re-test the limit at the top rather than relying on equality alone: a step that jumps // PAST the limit would never make counter == limit, and the loop would run away. - emit({IrOp::BranchGe, 0, counter, limit, 0,0, lDone, nullptr, {}}); - emit({IrOp::BranchNe, 0, counter, limit, 0,0, lTop, nullptr, {}}); + { + VReg c = alloc(), l = alloc(); + emit({IrOp::Reload, c, 0,0,0,0, counterSlot, nullptr, {}}); + emit({IrOp::Reload, l, 0,0,0,0, limitSlot, nullptr, {}}); + emit({IrOp::BranchGe, 0, c, l, 0,0, lDone, nullptr, {}}); + emit({IrOp::BranchNe, 0, c, l, 0,0, lTop, nullptr, {}}); + freeTemp(l); freeTemp(c); + } emit({IrOp::Label, 0, 0,0,0,0, lDone, nullptr, {}}); - freeTemp(limit); localCount = myLocal; // the loop variable leaves scope - // ...and its REGISTER goes back to the pool. Dropping only the name left the vreg allocated - // for the rest of the compile, so every `for` a script wrote cost one permanently — two - // sequential loops held two counters even though the first was dead. That is what put a - // two-loop effect one register over the smallest file (Xtensa has twelve) while each loop - // compiled fine alone. Freed only AFTER localCount drops, since freeTemp refuses to recycle - // a register any live local still names. - freeTemp(counter); + // ...and its two SLOTS are returned, so sequential loops reuse the same frame space instead + // of each costing its own for the rest of the program. Nesting still stacks, which is what + // makes a script bounded by frame size rather than by register count. `slotsUsed` keeps the + // PEAK, because that is what the prologue has to reserve. + if (slotHighWater > slotsUsed) slotsUsed = slotHighWater; + slotHighWater = counterSlot; return true; } @@ -539,6 +605,12 @@ struct Parser { } bool parseProgram() { + // Park the host arguments in their fixed slots at the top of the frame, once, before any + // script code runs. They are read-only, so one store each is all it takes — and every later + // read becomes a Reload, which frees five registers for the whole program. + for (VReg v = 0; v < kFirstTemp; v++) + emit({IrOp::Spill, 0, v, 0,0,0, hostArgSlot(v), nullptr, {}}); + while (!failed && atTypeKeyword()) { lex.advance(); parseDecl(); } if (failed) return false; if (lex.kind == Tok::End) { fail("empty program (no statement)"); return false; } @@ -554,18 +626,49 @@ struct Parser { } // namespace +uint32_t countTokens(const char* source) { + if (!source) return 0; + uint32_t tokens = 0; + for (Lexer scan(source); scan.kind != Tok::End && scan.kind != Tok::Error; scan.advance()) { + if (++tokens > kMaxIrOps) break; // runaway source — reserve() rejects past the bound + } + return tokens; +} + CompileResult compileSource(const char* source, const BuiltinTable& table, - const SysVarTable& sysvars, uint8_t* out, size_t cap) { + const SysVarTable& sysvars, uint8_t* out, size_t cap, + const RegBudget* squeeze, LowerFn lower) { CompileResult r; if (!source) { r.error = "no source"; return r; } if (!out || cap == 0) { r.error = "no code buffer"; return r; } - Lexer lex(source); + // Size the op array to THIS script before parsing. The bound is per-TOKEN rather than + // per-construct: no token the lexer can produce lowers to more than a handful of ops (the + // densest is a call argument — evaluate, then the Call itself), so counting tokens and + // multiplying is an over-estimate that cannot undershoot. Over-estimating costs a few unused + // entries on a cold path; undershooting would fail a script that fits, so the direction of the + // error is the whole point. push() still refuses past `cap`, so a wrong estimate degrades with + // a diagnostic rather than corrupting memory. + const uint32_t tokens = countTokens(source); IrProgram ir; + // +8 covers a program's fixed overhead (the prologue/epilogue ops a tiny script still needs) + // so a one-token source cannot round down to nothing. + if (!ir.reserve(static_cast(tokens * kIrOpsPerToken + 8 > kMaxIrOps + ? kMaxIrOps : tokens * kIrOpsPerToken + 8))) { + r.error = "script too large"; + return r; + } + Lexer lex(source); Parser parser{lex, table, sysvars, ir}; if (!parser.parseProgram()) { r.error = parser.error; r.errorCol = parser.errorCol; return r; } - - size_t len = lowerToBytes(ir, out, cap); + // Hand the backend the frame the script's variables need. The register allocator numbers any + // further slot from here up, so the two never overlap in the one frame they share. + ir.localSlots = parser.slotsUsed; + + // `lower` is normally this build's own backend; a test passes a DIFFERENT ISA's lowerer to read + // what a device would execute without flashing one. The front end is identical either way, which + // is the point — the seam is one function pointer, not a second copy of the compiler. + size_t len = lower ? lower(ir, out, cap, squeeze) : lowerToBytes(ir, out, cap, squeeze); if (len == 0) { r.error = kCodegenFailed; return r; } r.ok = true; r.len = len; diff --git a/src/core/moonlive/MoonLiveCompiler.h b/src/core/moonlive/MoonLiveCompiler.h index de6b4f98..b31e72cf 100644 --- a/src/core/moonlive/MoonLiveCompiler.h +++ b/src/core/moonlive/MoonLiveCompiler.h @@ -4,6 +4,7 @@ #include #include "core/moonlive/MoonLiveBuiltins.h" #include "core/moonlive/MoonLiveIr.h" // DeclaredControl, kMaxCtrls (surfaced on CompileResult) +#include "core/moonlive/moonlive_emit.h" // RegBudget — the test-only register-budget override // MoonLive front-end (§3.2) — the platform-independent compiler: source text → tokens → AST → // IR → native code (via the per-ISA assembler). The grammar is a single statement that is a @@ -43,7 +44,22 @@ struct CompileResult { // Pure: no I/O, no allocation beyond the caller's buffer. /// `sysvars` are names the HOST defines and a script may only read (`t`, `width`, …). They are /// reserved: a declaration that reuses one fails to compile, so a name means one thing everywhere. +/// `squeeze` overrides the register budget the backend would pick for itself. It is the seam that +/// makes the register allocator testable: on a host with fourteen registers, compiling the same +/// script at the full budget and at a deliberately smaller one must render IDENTICAL pixels — the +/// spilled program and the unspilled one are the same program. Production callers omit it. +/// The IR→bytes step, as a function pointer. Normally null, meaning "this build's own backend". +/// A test passes ANOTHER ISA's lowerer to read what a device would execute without flashing one — +/// the front end is identical either way, so the seam is one pointer rather than a second compiler. +using LowerFn = size_t (*)(IrProgram&, uint8_t*, size_t, const RegBudget*); + CompileResult compileSource(const char* source, const BuiltinTable& table, - const SysVarTable& sysvars, uint8_t* out, size_t cap); + const SysVarTable& sysvars, uint8_t* out, size_t cap, + const RegBudget* squeeze = nullptr, LowerFn lower = nullptr); + +/// Tokens in `source`, the one measure both right-sized buffers derive from: the caller sizes its +/// code buffer with `codeCapFor`, and compileSource sizes the IR op array from the same count. One +/// function so a caller cannot measure the script differently from the compiler. +uint32_t countTokens(const char* source); } // namespace mm::moonlive diff --git a/src/core/moonlive/MoonLiveIr.h b/src/core/moonlive/MoonLiveIr.h index 598e0b83..b3f25359 100644 --- a/src/core/moonlive/MoonLiveIr.h +++ b/src/core/moonlive/MoonLiveIr.h @@ -3,6 +3,7 @@ #include #include #include "core/moonlive/MoonLiveBuiltins.h" // InlineOp (a neutral opcode tag) +#include "platform/platform.h" // alloc/free — the op array is sized to the script // MoonLive IR — the typed intermediate representation between the front-end and the per-ISA // assembler (§3.2 of livescripts-analysis-top-down.md). The front-end lowers an AST to a flat @@ -29,8 +30,20 @@ using VReg = uint8_t; // updates live, without a recompile (the kArg3/t pattern, one slot over). enum : VReg { kArg0 = 0, kArg1 = 1, kArg2 = 2, kArg3 = 3, kArg4 = 4, kFirstTemp = 5 }; -static constexpr uint8_t kMaxVRegs = 16; // a statement uses a handful; no allocator yet -static constexpr uint8_t kMaxIrOps = 64; // a statement is a handful of ops; fixed, no heap +// How many values one program may NAME. Deliberately larger than any target's register file: the +// register allocator (MoonLiveSpill.h) parks whatever does not fit in frame slots, so this bounds +// the compiler's own tables — the interval array is one entry per vreg, on the stack of a 12 KB task +// — rather than the script. 32 keeps that array at a few hundred bytes. +static constexpr uint8_t kMaxVRegs = 32; +// An upper SANITY bound, not the working limit: the op array is sized to the script (see IrProgram), +// so a one-statement script pays for one statement. This exists only so a runaway source fails with +// a diagnostic instead of asking for an allocation that would exhaust a small device's heap. +static constexpr uint16_t kMaxIrOps = 4096; + +// Ops a single source token can lower to, worst case. The compiler sizes its op array by counting +// tokens and multiplying — an over-estimate by construction, which is the safe direction: a few +// unused entries on a cold path, versus refusing a script that would have fit. +static constexpr uint16_t kIrOpsPerToken = 4; // The op set — neutral. Three-address form: dst plus up to three source operands. (Counted // Control flow arrived with the script-level `for`, which is what the note here anticipated: the @@ -44,9 +57,11 @@ enum class IrOp : uint8_t { Add, // dst = a + b AddImm, // dst = a + imm Mul, // dst = a * b - Call, // dst = (*callFn)(a, b, c) — call a host-registered function. Three operands - // because a binding that hands the host a POSITION needs them at once; a - // unary helper ignores b and c, and the compiler passes a zero vreg. + Call, // dst = (*callFn)(&frame[imm], b, arena) — call a host-registered function. + // `imm` is the frame slot where the arguments start and `b` is how many there are: + // the parser stages every argument into consecutive slots, so a call carries a + // POSITION and a COUNT rather than the values, and arity is bounded by frame slots + // instead of by operand fields. Backends materialise the address themselves. Inline, // a host-registered inline op (inlineOp tag); operands a/b/c/d (op-specific) LoadCtrl, // dst = ((const uint8_t*)kArg4)[imm] — read a control value byte at offset imm Mov, // dst = a — the assignment a loop variable needs (vregs are otherwise write-once) @@ -54,8 +69,16 @@ enum class IrOp : uint8_t { BranchGe, // if (a >= b) goto label `imm` — UNSIGNED. The loop's ENTRY guard: skip a loop // whose range is empty, which is also what makes `for (i = 0; i < 0; …)` correct. BranchNe, // if (a != b) goto label `imm` — the BACKWARD edge that closes the loop. + Spill, // frame slot `imm` = a — a value the register file could not hold, parked + Reload, // dst = frame slot `imm` — the same value brought back for one use }; +// Spill/Reload are the register allocator's output, never the parser's: MoonLiveSpill rewrites a +// program that names more live values than the target has registers into one that fits, parking the +// overflow in the CALL FRAME (the textbook answer — a value that outlives the register file lives in +// memory). `imm` is a slot INDEX; each backend turns it into an offset from its own frame pointer, +// so the core pass never knows a stack layout and the backends never know the algorithm. + // Why these two branches and no unconditional jump: a bottom-tested loop needs exactly an entry // guard and a back edge, and every backend here already has both (bgeu / bne on Xtensa and RISC-V, // cmp + b.hs / b.ne on arm64). It is the shape FillElems has always lowered by hand, so the @@ -91,28 +114,111 @@ struct DeclaredControl { /// rather than silently miscompiling past it. Nesting depth is bounded separately, by `locals`. static constexpr uint8_t kIrLabels = 16; +/// Script variables live in FRAME SLOTS, and this bounds how many one program may hold at once. +/// Sixteen matches what every backend's frame can address (`kMaxSpillSlots`), so a program that +/// parses is a program the assembler can encode. Raising it means widening the frame on all three +/// backends together — the slot index is an instruction field, not just a table size. +static constexpr uint8_t kMaxLocals = 16; + +/// The most arguments one call can carry. Bounded by the FRAME, not by the register file: the parser +/// stages arguments into consecutive local slots, so a call can take as many as the locals range +/// holds — which is what makes a seven-argument `draw::line` expressible at all. `push` validates a +/// Call's count against this instead of against kMaxVRegs, which would cap arity at the register +/// count the frame staging exists to escape. +static constexpr uint8_t kMaxCallArgs = kMaxLocals; + +/// Where the HOST ARGUMENTS are parked. buf/nLights/cpl/t/ctrls arrive in registers and are never +/// written, but holding them there costs FIVE registers for the whole program — on Xtensa, five of +/// the ten the windowed ABI leaves, which is the difference between a script compiling and not. They +/// are stored to these slots once at entry and reloaded where read, exactly like a script variable. +/// +/// The slots sit at the TOP of the addressable range, above everything the parser and the register +/// allocator hand out (both number upward from zero), so neither can collide with them. +static constexpr uint8_t kHostArgSlots = kFirstTemp; // 5 +constexpr uint8_t hostArgSlot(VReg v) { + return static_cast(kMaxLocals + kHostArgSlots - kFirstTemp + v); +} +/// Total frame slots a backend must be able to address: the parser/allocator range plus the parked +/// host arguments above it. +static constexpr uint8_t kTotalSlots = kMaxLocals + kHostArgSlots; + static constexpr uint8_t kMaxControlName = 24; // max control-name length (incl. NUL); the compiler // rejects longer names so the binding's name pool // can't truncate distinct names into a collision -// A lowered program: a fixed list of ops plus the vreg high-water mark. +// A lowered program: the ops, sized to the script, plus the vreg high-water mark. +// +// The op array is HEAP-ALLOCATED rather than an `IrInst ops[kMaxIrOps]` member. As a member it cost +// the same ~2 KB of STACK for a one-statement script as for a full one, and this object is a local +// on the compile path of a 12 KB main task — so raising the ceiling by growing the array would have +// traded a compile limit for a stack overflow (this project has already lost a P4 to a large stack +// frame). Sizing to the script makes the small case cheaper AND the large case possible. +// Compilation is cold path, so the allocation costs nothing that matters. +// +// Ownership is RAII: one allocation, freed in the destructor, copying deleted. There is no manual +// free path to miss — the reverted 32026eb5 turned four tables into independently-nullable pointers +// and its own comment records the heap corruption that followed from missing one guard. struct IrProgram { - IrInst ops[kMaxIrOps]; - uint8_t count = 0; - VReg vregsUsed = kFirstTemp; + IrInst* ops = nullptr; + uint16_t cap = 0; // entries allocated + uint16_t count = 0; + VReg vregsUsed = kFirstTemp; + /// Frame slots the FRONT END allocated for script variables. Slot indices 0..localSlots-1 are + /// already spoken for when the backend sizes its prologue, and the register allocator numbers + /// any spill it still needs from here up — the two share one frame, so they cannot both start + /// at zero without a variable and a spilled temp landing on the same bytes. + uint8_t localSlots = 0; + + IrProgram() = default; + ~IrProgram() { platform::free(ops); } + IrProgram(const IrProgram&) = delete; // owns a buffer; a copy would double-free + IrProgram& operator=(const IrProgram&) = delete; + + /// Size the op array to `n` entries. False when the allocation fails or `n` exceeds the sanity + /// bound, so the caller reports a diagnostic instead of writing through a null pointer. + bool reserve(uint16_t n) { + if (n == 0 || n > kMaxIrOps) return false; + platform::free(ops); + ops = static_cast(platform::alloc(sizeof(IrInst) * n)); + cap = ops ? n : 0; + count = 0; + return ops != nullptr; + } bool push(const IrInst& i) { - if (count >= kMaxIrOps) return false; + if (!ops || count >= cap) return false; // Reject any op that names a vreg outside the fixed register budget — an invalid program // is dropped at the seam rather than reaching a backend that would index past its map. - if (i.dst >= kMaxVRegs || i.a >= kMaxVRegs || i.b >= kMaxVRegs || - i.c >= kMaxVRegs || i.d >= kMaxVRegs) return false; + // + // OPCODE-SPECIFIC, because not every operand field holds a vreg. For a Call, `b` is the + // ARGUMENT COUNT and `imm` the frame-slot base — its arguments live in the frame, not in + // registers. Validating `b` as a vreg capped calls at kMaxVRegs arguments, which is the very + // limit staging through the frame exists to remove. + if (i.dst >= kMaxVRegs) return false; + if (i.op == IrOp::Call) { + if (i.b > kMaxCallArgs) return false; // a count, not a register + } else if (i.a >= kMaxVRegs || i.b >= kMaxVRegs || + i.c >= kMaxVRegs || i.d >= kMaxVRegs) { + return false; + } ops[count++] = i; if (i.dst + 1 > vregsUsed) vregsUsed = static_cast(i.dst + 1); return true; } + /// Exchange contents with `o`. The register allocator builds the rewritten program in a second + /// IrProgram (inserting a Reload before a use and a Spill after a def cannot be done in place in + /// a right-sized array) and swaps it in; the old buffer then dies with the local. Copy is deleted + /// precisely because two owners would double-free, so a swap is how ownership moves here. + void swap(IrProgram& o) { + IrInst* p = ops; ops = o.ops; o.ops = p; + uint16_t t = cap; cap = o.cap; o.cap = t; + t = count; count = o.count; o.count = t; + VReg v = vregsUsed; vregsUsed = o.vregsUsed; o.vregsUsed = v; + uint8_t ls = localSlots; localSlots = o.localSlots; o.localSlots = ls; + } + /// Which inline ops this program contains, so a backend reserves scratch only for what is there. /// /// The backends reserved their maximum unconditionally, which cost a register no matter what the @@ -122,7 +228,7 @@ struct IrProgram { /// fold into the index vreg) and stays with each backend; WHICH ops are present is a property of /// the program, so it is answered once here. bool hasInline(InlineOp which) const { - for (uint8_t i = 0; i < count; i++) + for (uint16_t i = 0; i < count; i++) // uint16_t: `count` is, so a uint8_t never terminates if (ops[i].op == IrOp::Inline && ops[i].inlineOp == which) return true; return false; } diff --git a/src/core/moonlive/MoonLiveSpill.cpp b/src/core/moonlive/MoonLiveSpill.cpp new file mode 100644 index 00000000..9c37117c --- /dev/null +++ b/src/core/moonlive/MoonLiveSpill.cpp @@ -0,0 +1,380 @@ +#include "core/moonlive/MoonLiveSpill.h" + +// Linear-scan register allocation with spilling to the call frame. See MoonLiveSpill.h for why this +// lives in core rather than in each backend. +// +// The shape of the frame is the part that has to survive what comes next. MoonLive is gaining +// user-defined functions — with arguments, loops, and recursion — so the overflow storage is +// deliberately a CALL FRAME addressed through a frame pointer, not a global slot file: slot `n` is +// an offset from the frame the currently-executing routine owns, and a nested or recursive call +// pushes its own. A global file would work for one top-level program and would then have to be +// thrown away, because two activations of the same function would share the same slot and the inner +// one would clobber the outer's values. + +namespace mm::moonlive { + +namespace { + +// Reload temps: registers held back from allocation so a spilled operand has somewhere to land for +// the one instruction that reads it. Four is the worst case, set by the widest op — the StoreElem +// inline reads a, b, c AND d. A dst reuses a source temp rather than claiming a fifth: every op here +// is one machine instruction that reads its sources and then writes its destination, so `mul d,a,b` +// with d aliasing a is well-defined on all three ISAs, and on a target with twelve registers the +// fifth would be a real cost. +constexpr uint8_t kMaxReloadTemps = 4; + +// One value's live range, in op indices. `end` is the last index that mentions the vreg, AFTER loop +// extension. Sixteen bytes per vreg with kMaxVRegs = 32: a few hundred bytes of stack, small enough +// to stay a local on the 12 KB task the compile shares (a classic ESP32 has already been watchdog +// reset by this path; a heap allocation here would be a second failure mode for no gain). +struct Interval { + uint16_t start = 0; + uint16_t end = 0; + VReg vreg = 0; + bool live = false; // does this vreg appear at all? + bool spilled = false; + uint8_t slot = 0; // frame slot, when spilled + VReg assigned = 0; // compacted register number, when not +}; + +// A loop, as op indices: the header (where the back edge lands) and the back edge itself. The +// grammar has no break, continue or goto, so a loop is EXACTLY a BranchNe whose label was bound +// earlier in the array — the op array is already in reverse postorder and no CFG has to be built. +// That is the one bespoke simplification here, and it carries the guard below: a branch structure +// that is not properly nested makes the pass refuse rather than allocate against a wrong interval, +// so adding `break` later fails loudly instead of miscompiling in silence. +struct Loop { uint16_t header; uint16_t back; }; + +// Every vreg an op READS. Written as one function so the interval builder and the rewriter cannot +// disagree about which operand fields an op uses — a mismatch there is exactly the bug that produces +// a value spilled but never reloaded. +uint8_t sourcesOf(const IrInst& in, VReg* out) { + switch (in.op) { + case IrOp::Const: return 0; + case IrOp::Reload: return 0; + case IrOp::Label: return 0; + case IrOp::Mov: + case IrOp::AddImm: + case IrOp::Spill: out[0] = in.a; return 1; + case IrOp::LoadCtrl: out[0] = kArg4; return 1; // reads the arena pointer + case IrOp::Add: + case IrOp::Mul: + case IrOp::BranchGe: + case IrOp::BranchNe: out[0] = in.a; out[1] = in.b; return 2; + // A Call reads NO registers. Its arguments were staged into consecutive frame slots by the + // parser, so `imm` is their base and `b` is how MANY there are — a literal count, not a + // vreg. Reporting a/b/c as sources gave the count a live interval and let the rewrite below + // remap it into a register number; it survived only because a fixed ABI vreg maps to itself. + case IrOp::Call: return 0; + case IrOp::Inline: + // The inline ops read every operand field the host filled in. Both of today's ops also + // read kArg0..kArg2 (buf, nLights, cpl), but those are fixed ABI vregs this pass never + // reassigns, so they need no interval. + out[0] = in.a; out[1] = in.b; out[2] = in.c; out[3] = in.d; + return 4; + } + return 0; +} + +// Does this op WRITE its dst? Branches, Label, Spill and the inline ops do not — their dst field is +// a zero the front-end never fills in, and reading it as a definition would give vreg 0 (kArg0, the +// buffer pointer) a spurious live range that the allocator would then try to manage. +bool writesDst(IrOp op) { + switch (op) { + case IrOp::Label: case IrOp::BranchGe: case IrOp::BranchNe: + case IrOp::Spill: case IrOp::Inline: return false; + default: return true; + } +} + +} // namespace + +bool spillToBudget(IrProgram& ir, const RegBudget& budget, uint8_t& slotsUsed) { + // The front end already owns slots 0..localSlots-1 for the script's variables, and both live in + // the ONE frame — so a spill numbered from zero would land on a loop counter. Start above them, + // and report the total the prologue must reserve. + slotsUsed = ir.localSlots; + if (!ir.ops) return false; + + const uint8_t avail = budget.allocatable(); + // The front end's own variables have to fit the frame whether or not anything spills — it hands + // out slot indices without knowing the target, and a slot the backend cannot address would be + // encoded as a truncated offset writing over something else. Checked BEFORE the early return + // below, or a program that needs no spilling skips the check entirely. + if (ir.localSlots > kMaxLocals || ir.localSlots > budget.slots) return false; + + // Already fits: leave the program byte-identical. A script that never needed the allocator must + // not pay a renumbering for its existence — and this is the path every shipped script takes. + if (ir.vregsUsed <= avail) return true; + if (ir.vregsUsed > kMaxVRegs) return false; + + // The fixed ABI vregs (buf, nLights, cpl, t, ctrls) arrive in machine registers the host chose + // and every backend indexes them directly, so they can be neither renumbered nor spilled. They + // plus the reload temps are the floor: below it there is nothing left to allocate WITH, and the + // honest answer is to refuse rather than emit code that names a register the target lacks. + // Reserve temps for the widest op this program ACTUALLY contains, counting DISTINCT sources — + // not for the widest op the IR can express. + // + // The reservation is pure overhead for a program that never uses it, and it is subtracted from a + // register file that on Xtensa is ten deep. Reserving the theoretical maximum of four left + // 10 - 1 scratch - 5 fixed ABI vregs - 4 = ZERO keepable, so every looped script was refused + // outright ("codegen failed") — the allocator had nothing to allocate with. Counting what the + // program needs is both correct and what makes a loop fit at all on the smallest target. + // Only a FIXED ABI vreg is exempt: those arrive in registers and are never spilled, so an op + // reading `buf` or `t` needs no temp for it. Everything else may end up in a slot and therefore + // may need somewhere to land, so count the distinct non-ABI sources of the widest op present. + // + // Note the shape this leaves on Xtensa: 10 registers - 1 inline scratch - 5 fixed ABI vregs + // leaves 4, and a setRGB reads exactly 4 distinct operands — so a looped effect lands on + // keepable == 0 and is refused. The pass is right to refuse (it has nothing to allocate with); + // what is wrong is that FIVE registers are reserved for host arguments that a script reads + // rarely. Freeing those is register-promotion work, deliberately out of scope for this step. + uint8_t reloadTemps = 0; + for (uint16_t i = 0; i < ir.count; i++) { + VReg s[4]; + const uint8_t n = sourcesOf(ir.ops[i], s); + uint8_t distinct = 0; + for (uint8_t a = 0; a < n; a++) { + if (s[a] < kFirstTemp) continue; // a fixed ABI vreg: always a register + bool seen = false; + for (uint8_t b = 0; b < a; b++) if (s[b] == s[a]) { seen = true; break; } + if (!seen) distinct++; + } + if (distinct > reloadTemps) reloadTemps = distinct; + } + if (reloadTemps > kMaxReloadTemps) reloadTemps = kMaxReloadTemps; + // The fixed ABI vregs are NOT subtracted any more: core parks them in frame slots at entry, so + // they hold a register only for the parking store itself — which runs before any temp exists, so + // sharing those registers afterwards is not a conflict. Reserving five here was holding space for + // values that had already moved out, and on a ten-register target that was the entire budget. + if (avail <= reloadTemps) return false; + const uint8_t keepable = static_cast(avail - reloadTemps); + + // --- 1. Find the loops, innermost first ---------------------------------------------------- + // Bounded by kIrLabels because a loop needs a label, so this array cannot overflow a program the + // front-end could build. + Loop loops[kIrLabels]; + uint8_t loopCount = 0; + { + int32_t labelAt[kIrLabels]; + for (auto& p : labelAt) p = -1; + for (uint16_t i = 0; i < ir.count; i++) + if (ir.ops[i].op == IrOp::Label && ir.ops[i].imm >= 0 && ir.ops[i].imm < kIrLabels) + labelAt[ir.ops[i].imm] = i; + for (uint16_t i = 0; i < ir.count; i++) { + const IrInst& in = ir.ops[i]; + if (in.op != IrOp::BranchNe) continue; + if (in.imm < 0 || in.imm >= kIrLabels) return false; // an unbindable label: refuse + const int32_t tgt = labelAt[in.imm]; + if (tgt < 0 || static_cast(tgt) > i) continue; // forward branch — not a loop + if (loopCount >= kIrLabels) return false; + loops[loopCount++] = {static_cast(tgt), i}; + } + // Proper nesting is what makes "innermost first" meaningful and what the extension rule below + // assumes. Two loops must be disjoint or one must contain the other; anything else (which is + // what a `break` or a `goto` would produce) is refused here rather than allocated against an + // interval that does not describe the real control flow. + for (uint8_t x = 0; x < loopCount; x++) + for (uint8_t y = static_cast(x + 1); y < loopCount; y++) { + const bool disjoint = loops[x].back < loops[y].header || loops[y].back < loops[x].header; + const bool xInY = loops[y].header <= loops[x].header && loops[x].back <= loops[y].back; + const bool yInX = loops[x].header <= loops[y].header && loops[y].back <= loops[x].back; + if (!disjoint && !xInY && !yInX) return false; + } + } + + // --- 2. Live intervals, then loop extension ------------------------------------------------- + Interval iv[kMaxVRegs]; + for (uint8_t v = 0; v < kMaxVRegs; v++) iv[v].vreg = v; + + auto mention = [&](VReg v, uint16_t at) { + if (v >= kMaxVRegs) return; + if (!iv[v].live) { iv[v].live = true; iv[v].start = at; iv[v].end = at; return; } + if (at < iv[v].start) iv[v].start = at; + if (at > iv[v].end) iv[v].end = at; + }; + for (uint16_t i = 0; i < ir.count; i++) { + const IrInst& in = ir.ops[i]; + VReg src[4]; + const uint8_t n = sourcesOf(in, src); + for (uint8_t s = 0; s < n; s++) mention(src[s], i); + if (writesDst(in.op)) mention(in.dst, i); + } + + // A value whose live range TOUCHES a loop is live to that loop's end. Naive first-def-to-last-use + // is wrong across a back edge: a value defined before the loop and last read early in the body + // looks dead from the second instruction onward, so the scan would hand its register to something + // else and the next iteration would read that other value. Extension can only LENGTHEN a range, + // so its error direction is a needless spill, never a wrong one. Innermost-first, because an + // extension to an inner loop's end may then have to reach the enclosing loop's end as well. + for (uint8_t pass = 0; pass < loopCount; pass++) { + // pick the innermost unprocessed loop = the one containing no other unprocessed loop + uint8_t pickIdx = 0xff; + for (uint8_t x = 0; x < loopCount; x++) { + if (loops[x].header == 0xffff) continue; // already processed + bool containsAnother = false; + for (uint8_t y = 0; y < loopCount; y++) { + if (y == x || loops[y].header == 0xffff) continue; + if (loops[x].header <= loops[y].header && loops[y].back <= loops[x].back) + containsAnother = true; + } + if (!containsAnother) { pickIdx = x; break; } + } + if (pickIdx == 0xff) break; + const Loop lp = loops[pickIdx]; + loops[pickIdx].header = 0xffff; // mark processed + for (uint8_t v = 0; v < kMaxVRegs; v++) + if (iv[v].live && iv[v].start <= lp.back && iv[v].end >= lp.header && iv[v].end < lp.back) + iv[v].end = lp.back; + } + + // --- 3. Linear scan (Poletto & Sarkar) ------------------------------------------------------ + // Sweep the temps in order of increasing interval start, keeping an `active` set ordered by + // increasing end. When the set is full, the interval with the FURTHEST end is spilled — it is the + // one whose register would otherwise be tied up longest, so freeing it buys the most. + VReg order[kMaxVRegs]; + uint8_t nOrder = 0; + for (uint8_t v = kFirstTemp; v < ir.vregsUsed; v++) if (iv[v].live) order[nOrder++] = v; + for (uint8_t i = 1; i < nOrder; i++) { // insertion sort: nOrder <= 32 + const VReg k = order[i]; + uint8_t j = i; + while (j > 0 && iv[order[j - 1]].start > iv[k].start) { order[j] = order[j - 1]; j--; } + order[j] = k; + } + + VReg active[kMaxVRegs]; + uint8_t nActive = 0; + uint8_t nSpilled = ir.localSlots; + for (uint8_t i = 0; i < nOrder; i++) { + const VReg cur = order[i]; + // expire: everything whose interval ended before this one starts is free again + uint8_t w = 0; + for (uint8_t j = 0; j < nActive; j++) + if (iv[active[j]].end >= iv[cur].start) active[w++] = active[j]; + nActive = w; + + if (nActive < keepable) { + active[nActive++] = cur; + // keep `active` sorted by end so the furthest is always the last element + for (uint8_t j = nActive - 1; j > 0 && iv[active[j - 1]].end > iv[active[j]].end; j--) { + const VReg t = active[j]; active[j] = active[j - 1]; active[j - 1] = t; + } + continue; + } + // nActive >= keepable >= 1 here: the `avail <= reloadTemps` guard above makes keepable at + // least one, and this branch is only reached when nActive is not below it. Stated because + // the index below would read active[-1] if that invariant ever moved. + if (nActive == 0) return false; + const VReg furthest = active[nActive - 1]; + if (iv[furthest].end > iv[cur].end) { + iv[furthest].spilled = true; + iv[furthest].slot = nSpilled++; + active[nActive - 1] = cur; + for (uint8_t j = nActive - 1; j > 0 && iv[active[j - 1]].end > iv[active[j]].end; j--) { + const VReg t = active[j]; active[j] = active[j - 1]; active[j - 1] = t; + } + } else { + iv[cur].spilled = true; + iv[cur].slot = nSpilled++; + } + } + // Bounded by kMaxLocals, NOT budget.slots: slots kMaxLocals..kTotalSlots-1 hold the parked host + // arguments (hostArgSlot), which are stored once at entry and reloaded wherever a script reads + // buf/nLights/cpl/t/ctrls. Allowing a spill into that range would overwrite them — budget.slots + // is the frame's whole capacity, of which only the bottom kMaxLocals are assignable. + if (nSpilled > kMaxLocals || nSpilled > budget.slots) return false; + + // --- 4. Compact the survivors --------------------------------------------------------------- + // The kept temps take the register numbers directly above the fixed ABI vregs, so the rewritten + // program's high-water mark drops to something the target actually has. The reload temps sit + // above them, and the backend's own inline scratch above that — a single ascending layout, which + // is what lets each backend keep computing its scratch from vregsUsed as it already does. + // Number the kept temps from the BOTTOM of the register file. The ABI vregs keep their own + // identity for the entry parking store, and that store runs before any temp exists — so an + // overlap afterwards is not a conflict. Starting at kFirstTemp reserved five registers for + // values that had already moved to the frame, which pushed the top temps past the budget. + VReg next = 0; + for (uint8_t v = kFirstTemp; v < ir.vregsUsed; v++) + if (iv[v].live && !iv[v].spilled) iv[v].assigned = next++; + for (uint8_t v = 0; v < kFirstTemp; v++) { iv[v].assigned = v; iv[v].spilled = false; } + const VReg firstTemp = next; // the reload temps start here + const VReg newHighWater = static_cast(firstTemp + reloadTemps); + + // --- 5. Rewrite ----------------------------------------------------------------------------- + // Into a SECOND program: a Reload has to be inserted before the op that reads a spilled value and + // a Spill after the op that defines one, and a right-sized array has no room to shift into. + IrProgram out; + // Worst case per op: four Reloads, the op, one Spill. Over-estimating costs a cold-path + // allocation; under-estimating would fail a script that fits, so the direction is deliberate. + const uint32_t want = static_cast(ir.count) * (kMaxReloadTemps + 2); + if (want > kMaxIrOps) return false; + if (!out.reserve(static_cast(want))) return false; + + auto emit = [&](const IrInst& in) { + // push() also re-validates every vreg against kMaxVRegs, so a rewrite that named a register + // outside the budget fails the compile here instead of reaching a backend's register map. + if (!out.push(in)) return false; + return true; + }; + + for (uint16_t i = 0; i < ir.count; i++) { + IrInst in = ir.ops[i]; + VReg src[4]; + const uint8_t n = sourcesOf(in, src); + + // Reload each DISTINCT spilled source into its own temp; a repeated operand reuses the temp + // already holding it, which is both cheaper and necessary — two Reloads of the same slot into + // different temps would be pure waste on the tightest register file here. + VReg tempOf[4] = {0, 0, 0, 0}; + uint8_t nTemp = 0; + for (uint8_t s = 0; s < n; s++) { + const VReg v = src[s]; + if (v >= kMaxVRegs || !iv[v].spilled) continue; + bool already = false; + for (uint8_t p = 0; p < s; p++) if (src[p] == v) { tempOf[s] = tempOf[p]; already = true; break; } + if (already) continue; + tempOf[s] = static_cast(firstTemp + nTemp); + nTemp++; + IrInst rl{}; + rl.op = IrOp::Reload; + rl.dst = tempOf[s]; + rl.imm = iv[v].slot; + if (!emit(rl)) return false; + } + + // Rewrite the operands in place: a spilled one now names its temp, a kept one its compacted + // number. LoadCtrl's source is kArg4, which is fixed, so it needs no case of its own. + auto mapped = [&](VReg v, uint8_t slotIdx) -> VReg { + if (v >= kMaxVRegs) return v; + return iv[v].spilled ? tempOf[slotIdx] : iv[v].assigned; + }; + if (n > 0) in.a = mapped(src[0], 0); + if (n > 1) in.b = mapped(src[1], 1); + if (n > 2) in.c = mapped(src[2], 2); + if (n > 3) in.d = mapped(src[3], 3); + + const bool dstSpilled = writesDst(in.op) && in.dst < kMaxVRegs && iv[in.dst].spilled; + const uint8_t dstSlot = dstSpilled ? iv[in.dst].slot : 0; + if (writesDst(in.op)) { + // A spilled destination is computed into a reload temp and then stored. It may reuse a + // temp that carried a source: the op reads its sources and writes its destination as one + // instruction, so the aliasing is the ordinary `add d, d, b` every ISA here defines. + in.dst = dstSpilled ? firstTemp : (in.dst < kMaxVRegs ? iv[in.dst].assigned : in.dst); + } + if (!emit(in)) return false; + if (dstSpilled) { + IrInst sp{}; + sp.op = IrOp::Spill; + sp.a = firstTemp; + sp.imm = dstSlot; + if (!emit(sp)) return false; + } + } + + out.vregsUsed = newHighWater; + ir.swap(out); + slotsUsed = nSpilled; + return true; +} + +} // namespace mm::moonlive diff --git a/src/core/moonlive/MoonLiveSpill.h b/src/core/moonlive/MoonLiveSpill.h new file mode 100644 index 00000000..5bbbe1b2 --- /dev/null +++ b/src/core/moonlive/MoonLiveSpill.h @@ -0,0 +1,35 @@ +#pragma once + +#include + +#include "core/moonlive/MoonLiveIr.h" +#include "core/moonlive/moonlive_emit.h" // RegBudget — the one thing a backend tells the allocator + +// MoonLive register allocation — linear scan with spilling (Poletto & Sarkar, "Linear Scan Register +// Allocation", ACM TOPLAS 1999). The pass that makes a script's complexity a MEMORY question rather +// than a register-count one: a program naming more live values than the target has registers is +// rewritten so the overflow lives in the call frame, instead of being refused. +// +// It lives in CORE, once. Correct spilling across a loop back edge is the hardest logic in this +// compiler, and only the arm64 backend is ever executed by tests — three copies would leave two +// permanently untested (CLAUDE.md Principle 3). Every backend supplies a RegBudget and consumes two +// new IR ops; the algorithm appears nowhere in the platform layer. + +namespace mm::moonlive { + +/// Rewrite `ir` so no op names a vreg the target does not have, parking the overflow in frame slots +/// (Spill/Reload). A program that already fits is left byte-identical — a non-spilling script pays +/// nothing, not even a renumbering. +/// +/// False when even the spilled form does not fit: fewer registers than the fixed ABI vregs plus the +/// reload temps need, more slots than the frame can address, or a branch structure the interval +/// analysis cannot prove properly nested. FAIL, NEVER MISCOMPILE — the caller reports a diagnostic +/// and the script runs dark, which is recoverable; a wrong interval silently computes with a stale +/// value, which is not. +/// +/// `slotsUsed` receives how many frame slots the rewritten program needs, so the backend can size +/// its prologue: the program's own locals (`ir.localSlots`) plus anything this pass spilled. It is +/// therefore NOT zero when nothing spills — the locals still need their prologue capacity. +bool spillToBudget(IrProgram& ir, const RegBudget& budget, uint8_t& slotsUsed); + +} // namespace mm::moonlive diff --git a/src/core/moonlive/moonlive_emit.h b/src/core/moonlive/moonlive_emit.h index 4993b380..a362a2d7 100644 --- a/src/core/moonlive/moonlive_emit.h +++ b/src/core/moonlive/moonlive_emit.h @@ -52,10 +52,37 @@ size_t emitAnimatedFill(uint8_t* out, size_t cap); struct IrProgram; // src/core/moonlive/MoonLiveIr.h +// What one target's register file offers the allocator — the ONLY thing core's spill pass needs to +// know about an ISA, and the reason the allocator is written once instead of three times. Each +// backend fills this in from its own map and hands it to spillToBudget (MoonLiveSpill.h); nothing +// ISA-specific crosses in the other direction. +struct RegBudget { + uint8_t regs = 0; // machine registers the vreg map exposes (kRegCount) + uint8_t reserved = 0; // registers the backend keeps for the inline ops this program contains + uint8_t slots = 0; // spill slots the backend's frame can address (0 = cannot spill at all) + + /// Registers left for the allocator once the backend's inline scratch is taken out. Saturating, + /// because a program whose scratch demand exceeds the whole file must report "no registers" + /// rather than wrap to a huge count and allocate against a register that does not exist. + uint8_t allocatable() const { return regs > reserved ? uint8_t(regs - reserved) : uint8_t(0); } +}; + // Lower a typed IR program to machine code for this TU's ISA, via the per-ISA assembler. // This is the general codegen path the front-end uses; emitFill/emitAnimatedFill are the // hand-encoded references the assembler-built output is behaviorally checked against. Returns // the byte count, or 0 on overflow / cap too small (the caller degrades). -size_t lowerToBytes(const IrProgram& ir, uint8_t* out, size_t cap); +// +// `ir` is taken by NON-CONST reference because the backend runs core's register allocator over it +// first (MoonLiveSpill.h), which rewrites a program that names more live values than this target has +// registers. Taking a copy instead would double the compile's peak memory on the smallest device for +// no benefit — the IR is compile-time scratch the caller drops immediately afterwards. +/// +/// `squeeze`, when non-null, REPLACES the register budget this backend would compute for itself. +/// It exists because the spiller is the hardest logic in the compiler and only one backend is ever +/// executed by tests: with a budget deliberately smaller than the host's, the allocator runs on the +/// arm64 path a test can actually call and RUN, and the same script at the full and the squeezed +/// budget must render identical pixels. Without this seam the pass would be verifiable only on +/// hardware. Production callers pass nullptr and get the target's real budget. +size_t lowerToBytes(IrProgram& ir, uint8_t* out, size_t cap, const RegBudget* squeeze = nullptr); } // namespace mm::moonlive diff --git a/src/core/moonlive/register-and-slot-contract.md b/src/core/moonlive/register-and-slot-contract.md new file mode 100644 index 00000000..bbc45177 --- /dev/null +++ b/src/core/moonlive/register-and-slot-contract.md @@ -0,0 +1,75 @@ +# The MoonLive register and frame-slot contract + +One machine, three roles. This governs the compiler and the three backends, not a binding: a scripted +layout, effect and modifier all compile through the same front end and the same lowering, so the +ownership rules below hold identically whichever role a script plays. ("Layout" in this file means +the memory layout of the register file and the frame — never the light-placement role.) + +Who owns which register index, who owns which frame slot, and who is allowed to compute what from +`vregsUsed`. Written down because four places derive numbers from each other — core's compaction, +the spill pass, each backend's inline scratch, and each assembler's map — and changing where a value +lives perturbs all four, so a change made against intuition rather than against this table +silently miscompiles on one target while the others stay green. + +## Registers: one ascending layout, three owners + +A vreg is an index. Each backend maps it to a machine register through its own table (`kXtReg`, +`kRvReg`, `kArm64Reg`). The index space is carved as follows, **in this order, no gaps**: + +| range | owner | meaning | +|---|---|---| +| `0 .. kFirstTemp-1` | **core, fixed** | the host arguments: buf, nLights, cpl, t, ctrls. They arrive in registers at entry and every backend indexes them directly. | +| `kFirstTemp .. vregsUsed-1` | **core (parser, then spill pass)** | the program's values. The parser hands them out; the spill pass renumbers them down to what fits. | +| `vregsUsed .. vregsUsed+scratch-1` | **the backend** | the inline ops' scratch, computed per lowerer from `vregsUsed`. | + +**The invariant:** `vregsUsed + scratch <= kRegCount`, and the spill pass is what guarantees it by +rewriting the program until `vregsUsed` is small enough. `RegBudget.reserved` is how a backend tells +the pass how much scratch it will add on top. + +**Consequences that have each been a bug:** + +- A backend must not name a register outside its own map. `ar()`/`xr()`/`mr()` index the table with + `vregsUsed + n`, so an off-by-one reads past the array and emits a register chosen by accident. + All three are bounds-checked for that reason. +- A register that is scratch for the assembler (an address temp, a call staging register) must not + also appear in the vreg map. `static_assert` in each backend enforces it. +- On Xtensa the map may only contain registers that survive `call8`. The windowed ABI rotates by + eight, so a call clobbers `a8..a15` (ESP-IDF's `coreasm.h` states it). A vreg mapped there holds a + script value that a host call destroys. + +## Frame slots: two owners, growing towards each other + +One frame, indexed `0 .. budget.slots-1`, addressed by `spillStore`/`spillLoad`. + +| range | owner | meaning | +|---|---|---| +| `0 .. ir.localSlots-1` | **core (parser)** | script variables (a `for`'s counter and limit) and call-argument staging. Handed out by `slotHighWater`, peak recorded in `slotsUsed`, published as `ir.localSlots`. | +| `ir.localSlots .. slotsUsed-1` | **the spill pass** | values the register file could not hold. Numbered from `ir.localSlots` UPWARD, never from zero. | + +**The invariant:** the spill pass starts numbering at `ir.localSlots` so the two ranges cannot +overlap, and refuses the compile when either exceeds `budget.slots` — a slot the backend cannot +address would encode a truncated offset and write over a live value. + +**`ir.localSlots` is what the FRONT END claimed, not the frame size.** Setting it to the whole +addressable range hands the allocator a full frame and leaves it nowhere to spill; that refuses every +script that needs one spill. + +## What `prologue(slots)` must reserve + +`slotsUsed` — the total from both owners, which `spillToBudget` returns. The backend turns a slot +index into a frame offset its own way (Xtensa counts 4-byte words from `kFrameBase`; RISC-V and arm64 +build a frame in their prologue). Slot indices are the shared currency; offsets are private. + +## Where a proposed change has to be checked + +Any change to where a value lives must be checked against **all five** of: + +1. `Parser` — does it still hand out slots from zero upward, and is `slotsUsed` the peak? +2. `compileSource` — is `ir.localSlots` the parser's claim only? +3. `spillToBudget` — does its reservation arithmetic still match what actually occupies registers? +4. The compaction in `spillToBudget` — does the renumbering leave the ranges above intact? +5. Each of the three lowerers — is `scratch` still the number of extra registers it adds, and does + every register it names come from the map? + +Missing any one of these produces a program that compiles and then behaves wrongly, which is far +worse than one that is refused. diff --git a/src/light/drivers/ParallelLedDriver.h b/src/light/drivers/ParallelLedDriver.h index 241dda59..642bdab6 100644 --- a/src/light/drivers/ParallelLedDriver.h +++ b/src/light/drivers/ParallelLedDriver.h @@ -580,7 +580,7 @@ class ParallelLedDriver : public DriverBase { // it instead, with the number the user has to act on: the ceiling in LIGHTS PER LANE, since // that is the control they set. (github.com/MoonModules/projectMM/issues/44) if (frameBytes_ > peripheral_->busCapacity()) { - reportOverCapacity(outCh); + reportOverCapacity(outCh, peripheral_->busCapacity()); return; } @@ -785,15 +785,25 @@ class ParallelLedDriver : public DriverBase { /// The frame does not fit one transfer. Report the ceiling the way the user sets it — lights per /// lane — rather than the byte figure they would have to derive it from. Cleared by reinit(), so /// lowering the count restores normal reporting. - void reportOverCapacity(uint8_t outCh) { + /// `cap` is the byte ceiling to measure against — the peripheral's live buffer capacity on the + /// tick path, or its declared DMA budget at reinit(), where no bus exists yet and busCapacity() + /// would read 0. Passing it in keeps one message for both, instead of a KB figure on one path + /// and the actionable light count on the other. + void reportOverCapacity(uint8_t outCh, size_t cap) { if (overCapReported_) return; overCapReported_ = true; const uint8_t opp = outputsPerPin(); - const size_t pad = padBytesFor(slotBytes(), opp); const size_t rowBytes = rowBytesFor(outCh, slotBytes(), opp); - const size_t cap = peripheral_->busCapacity(); - const size_t usable = cap > pad ? cap - pad : 0; - const unsigned fits = rowBytes ? static_cast(usable / rowBytes) : 0; + const size_t pad = padBytesFor(slotBytes(), opp); + // Count DOWN through frameBytesFor, not up through a division: the frame is 64-byte ROUNDED, + // so `(cap - pad) / rowBytes` overshoots by one — it reported 898 lights, whose frame rounds + // to 65536 against a 65535 cap. A limit that still fails is worse than no limit. + unsigned fits = 0; + if (rowBytes && cap > pad) { + fits = static_cast((cap - pad) / rowBytes); + while (fits > 0 && frameBytesFor(static_cast(fits), outCh, + slotBytes(), opp) > cap) fits--; + } std::snprintf(overCapBuf_, sizeof(overCapBuf_), "too many lights per pin: %u exceeds this peripheral's %u — lower ledsPerPin", static_cast(maxLaneLights_), fits); @@ -1862,14 +1872,11 @@ class ParallelLedDriver : public DriverBase { if (const size_t budget = peripheral_->dmaBudgetBytes(); !frameFitsDmaBudget(frameBytes_, budget)) { // deinit() above already cleared the bus and inited_ — just report and bail. - if (char* b = failBufEnsure()) { - std::snprintf(b, kFailBufLen, "frame %uKB over the bus %uKB: fewer lights/pin", - static_cast(frameBytes_ / 1024), - static_cast(budget / 1024)); - setStatus(b, Severity::Error); - } else { - setStatus(peripheral_->initFailMsg(), Severity::Error); - } + // Same message the tick path gives, measured against the DECLARED budget (no bus is up + // yet, so busCapacity() would read 0): the light count the user has to lower, not a KB + // figure they would have to convert. reinit() cleared overCapReported_ above, so this + // reports once per geometry rather than once per attempt. + reportOverCapacity(correction_.outChannels, budget); return; } // Allocate the second buffer only when wanted (see wantSecond above — gated on the toggle AND the diff --git a/src/light/drivers/ParlioLedDriver.h b/src/light/drivers/ParlioLedDriver.h index 3963f05b..16159a85 100644 --- a/src/light/drivers/ParlioLedDriver.h +++ b/src/light/drivers/ParlioLedDriver.h @@ -76,11 +76,12 @@ class ParlioPeripheral : public LedPeripheral { uint8_t* busBuffer(uint8_t i) override { return platform::parlioWs2812Buffer(parlio_, i); } /// The per-buffer byte capacity (fixed at bus creation; both buffers equal). size_t busCapacity() const override { return platform::parlioWs2812BufferCapacity(parlio_); } - // Parlio sends a frame in ONE transfer, and the peripheral caps that at 65535 bytes — a hard - // limit, unlike a memory budget that varies with the heap. Declaring it here lets reinit() - // refuse an oversized frame with an actionable status BEFORE busInit tries (and fails) to - // allocate it, which is what left the LEDs frozen with a healthy UI (issue #44). - size_t dmaBudgetBytes() const override { return 65535; } + // Parlio sends a frame in ONE transfer, and the peripheral caps that. The cap is a HARDWARE + // fact (derived from PARLIO_LL_TX_MAX_BITS_PER_FRAME), so the platform owns the number and this + // asks for it — reinit() then refuses an oversized frame with an actionable status BEFORE + // busInit tries (and fails) to allocate it, which is what left the LEDs frozen with a healthy + // UI (issue #44). + size_t dmaBudgetBytes() const override { return platform::parlioMaxTransferBytes(); } /// Kick off the autonomous transfer of the first `bytes` of DMA buffer `i`; /// returns whether it started. bool busTransmit(uint8_t i, size_t bytes) override { return platform::parlioWs2812Transmit(parlio_, i, bytes); } diff --git a/src/light/moonlive/MoonLiveBuiltins_light.h b/src/light/moonlive/MoonLiveBuiltins_light.h index 161aa712..f9ed89c5 100644 --- a/src/light/moonlive/MoonLiveBuiltins_light.h +++ b/src/light/moonlive/MoonLiveBuiltins_light.h @@ -5,10 +5,12 @@ #include "core/moonlive/MoonLiveBuiltins.h" #include "core/moonlive/MoonLiveIr.h" // kArg3 — the register `t` is passed in +#include #include #include "core/math8.h" // beatsin16 — the shared time vocabulary #include "core/math16.h" // beat16 / triwave16 — full-range waveforms +#include "light/draw.h" // draw::line, the shared 3D Bresenham a script draws with // MoonLive — the LIGHT-DOMAIN built-in registration. This is the only place the LED vocabulary // lives: the function NAMES (`setRGB`, `fill`, `random16`), their arg counts, and the meaning @@ -22,10 +24,20 @@ namespace mm::moonlive { // random16(n) → a pseudo-random value in [0, n). A simple LCG, deterministic enough that the // runtime Bounds guard always sees an in-range index; the same implementation on every target // so a script behaves identically. The one host helper exposed as a Call so far. -extern "C" inline uint32_t mm_light_random16(uint32_t n, uint32_t, uint32_t) { - static uint32_t s = 0x2545F491u; - s = s * 1664525u + 1013904223u; - return n ? (s >> 16) % n : 0u; +extern "C" inline uint32_t mm_light_random16(const uintptr_t* args, uint32_t, const uint8_t*) { + const uint32_t n = uint32_t(args[0]); + // ATOMIC, because two threads run scripts at once: the render task walks a layout for the frame + // while the HTTP task asks the same layout for its light count after a control edit (the reason + // the addLight sink is a per-thread table below). A plain `static` here is a data race, and a + // lost update would additionally let two draws return the SAME value, which for a "random" + // helper is a correctness bug rather than a tolerable one. compare_exchange keeps the sequence + // exactly the LCG's, just serialized. + static std::atomic seed{0x2545F491u}; + uint32_t prev = seed.load(std::memory_order_relaxed), next; + do { + next = prev * 1664525u + 1013904223u; + } while (!seed.compare_exchange_weak(prev, next, std::memory_order_relaxed)); + return n ? (next >> 16) % n : 0u; } // mod(a, b) → a % b, the wrap a cyclic animation needs. `t` grows without bound, so every effect @@ -36,7 +48,8 @@ extern "C" inline uint32_t mm_light_random16(uint32_t n, uint32_t, uint32_t) { // all, and emitting a division routine inline would cost more code than the whole script. One host // function, called like any other builtin, keeps the emitted code small and the three backends // identical. b == 0 returns 0 rather than trapping: a script must degrade, never fault. -extern "C" inline uint32_t mm_light_mod(uint32_t a, uint32_t b, uint32_t) { +extern "C" inline uint32_t mm_light_mod(const uintptr_t* args, uint32_t, const uint8_t*) { + const uint32_t a = uint32_t(args[0]), b = uint32_t(args[1]); return b ? a % b : 0u; } @@ -57,10 +70,12 @@ extern "C" inline uint32_t mm_light_mod(uint32_t a, uint32_t b, uint32_t) { // arrives as zero and the animation silently stands still. Explicit also matches the C++ signature // (beat16(bpm, ms)), so a script and an effect read the same. The modulo and divide these need live // in the host function, which is why they are Calls — no ISA here has a cheap integer divide. -extern "C" inline uint32_t mm_light_beat(uint32_t bpm, uint32_t ms, uint32_t) { +extern "C" inline uint32_t mm_light_beat(const uintptr_t* args, uint32_t, const uint8_t*) { + const uint32_t bpm = uint32_t(args[0]), ms = uint32_t(args[1]); return beat16(static_cast(bpm), ms); } -extern "C" inline uint32_t mm_light_beatsin(uint32_t bpm, uint32_t ms, uint32_t high) { +extern "C" inline uint32_t mm_light_beatsin(const uintptr_t* args, uint32_t, const uint8_t*) { + const uint32_t bpm = uint32_t(args[0]), ms = uint32_t(args[1]), high = uint32_t(args[2]); // low is 0 and high is the caller's: a Call carries three arguments and bpm + ms take two, so // the common "oscillate from 0 up to N" form is the one exposed rather than a packed pair. return beatsin16(static_cast(bpm), ms, 0, static_cast(high)); @@ -76,21 +91,25 @@ extern "C" inline uint32_t mm_light_beatsin(uint32_t bpm, uint32_t ms, uint32_t // math16's sin16/cos16 return SIGNED -32768..32767; a script's values are unsigned, so the result // is biased into 0..65535 with the zero line at 32768. A script that wants a coordinate scales the // result: `scale(sin(a), width)` sweeps the whole axis, which is the same `scale` a beat uses. -extern "C" inline uint32_t mm_light_sin(uint32_t angle, uint32_t, uint32_t) { +extern "C" inline uint32_t mm_light_sin(const uintptr_t* args, uint32_t, const uint8_t*) { + const uint32_t angle = uint32_t(args[0]); return static_cast(sin16(static_cast(angle)) + 32768); } -extern "C" inline uint32_t mm_light_cos(uint32_t angle, uint32_t, uint32_t) { +extern "C" inline uint32_t mm_light_cos(const uintptr_t* args, uint32_t, const uint8_t*) { + const uint32_t angle = uint32_t(args[0]); return static_cast(cos16(static_cast(angle)) + 32768); } // turn(n) → the angle step that divides one full revolution into n parts. A full turn is 65536 — // one past the largest number a script can write — so even with a divide operator the expression // could not be spelled. A circle therefore needs this as a builtin rather than as arithmetic. -extern "C" inline uint32_t mm_light_turn(uint32_t n, uint32_t, uint32_t) { +extern "C" inline uint32_t mm_light_turn(const uintptr_t* args, uint32_t, const uint8_t*) { + const uint32_t n = uint32_t(args[0]); return n ? 65536u / n : 0u; } -extern "C" inline uint32_t mm_light_scale(uint32_t value, uint32_t n, uint32_t) { +extern "C" inline uint32_t mm_light_scale(const uintptr_t* args, uint32_t, const uint8_t*) { + const uint32_t value = uint32_t(args[0]), n = uint32_t(args[1]); return (value * n) >> 16; } @@ -109,20 +128,29 @@ extern "C" inline uint32_t mm_light_scale(uint32_t value, uint32_t n, uint32_t) /// The remaining print budget. A binding resets it when it compiles, so every edit of a script gets /// a fresh window — without that, one burst silences the debugging tool for the life of the process, /// which is exactly when a second look at a misbehaving script is most needed. -inline uint32_t& printBudget() { static uint32_t n = 0; return n; } +/// ATOMIC for the same reason random16's seed is: two threads run scripts concurrently. The decrement +/// below is the one that matters: read-modify-write on a plain uint32_t lets two threads both see 1, +/// both decrement, and the budget WRAP to ~4 billion, turning the bound that keeps `print` off the +/// render tick's critical path into no bound at all. +inline std::atomic& printBudget() { static std::atomic n{0}; return n; } /// Grant a fresh burst. Call from the binding's prepare(), alongside the compile. /// /// print() writes to serial, which blocks, and an effect script runs on the render tick — so the /// burst is what bounds the cost: a handful of writes per compile, after which the call is a compare /// and a return. Draining through a queue would take the last of it off the tick; backlogged. -inline void resetPrintBudget() { printBudget() = 32; } +inline void resetPrintBudget() { printBudget().store(32, std::memory_order_relaxed); } -extern "C" inline uint32_t mm_light_print(uint32_t v, uint32_t, uint32_t) { - uint32_t& left = printBudget(); - if (left > 0) { +extern "C" inline uint32_t mm_light_print(const uintptr_t* args, uint32_t, const uint8_t*) { + const uint32_t v = uint32_t(args[0]); + // Claim one unit before printing, and only if there is one: a plain `if (left > 0) --left` + // can underflow past zero when two threads pass the test together. + auto& left = printBudget(); + uint32_t have = left.load(std::memory_order_relaxed); + while (have > 0 && !left.compare_exchange_weak(have, have - 1, std::memory_order_relaxed)) {} + if (have > 0) { std::printf("[script] %u\n", static_cast(v)); - if (--left == 0) std::printf("[script] (burst spent; edit the script for a fresh one)\n"); + if (have == 1) std::printf("[script] (burst spent; edit the script for a fresh one)\n"); } return v; } @@ -140,22 +168,106 @@ extern "C" inline uint32_t mm_light_print(uint32_t v, uint32_t, uint32_t) { // something. using AddLightFn = void (*)(void* ctx, uint16_t x, uint16_t y, uint16_t z); -/// THREAD_LOCAL, not one global: the sink belongs to whichever thread is running a script, and more +/// PER-THREAD, not one global: the sink belongs to whichever thread is running a script, and more /// than one does. A layout is asked for its light count and its coordinates from the HTTP task when a /// control is edited, while the render task walks the same layout for the frame — as one global, one /// thread cleared the sink while the other was mid-run and the built-in called through a live /// function pointer with a null context. That is a null dereference on the render core, seen as an /// intermittent crash while resizing a scripted layout. /// -/// The function and the context are ONE struct so they cannot be observed half-updated. Same shape -/// as the WDT subscription flag in the ESP32 worker, which had the same bug for the same reason. +/// Keyed on platform::currentThreadId() rather than C++ `thread_local`, which is UNUSABLE on the +/// ESP32: the compiler reaches TLS through the THREADPTR special register, and a FreeRTOS task +/// created without TLS has THREADPTR = 0 — so the access dereferences a small offset from null and +/// dies inside the exception handler. Measured: EXCVADDR 0xfffffff0, `Double exception` in +/// _xt_context_save, on every scripted LAYOUT (the only binding whose script calls a host function). +/// Reading the task handle costs one load and needs no per-task setup. +/// +/// The function and the context are ONE struct so they cannot be observed half-updated. Two slots: +/// the render task and whichever task edits a control are the two that ever run a script at once, +/// and a third would mean a genuinely new concurrency story rather than a bigger table. struct AddLightSink { AddLightFn fn = nullptr; void* ctx = nullptr; }; -inline AddLightSink& addLightSink() { static thread_local AddLightSink s; return s; } + +namespace detail { +// `owner` is ATOMIC and claimed with compare_exchange: the claim used to be a load then a store, +// so two threads could both see the same slot free and both take it — leaving them sharing one +// sink, which is the very aliasing this table exists to prevent. +// +// The slot is the ONE per-thread home for everything a running script's built-ins reach: the +// addLight sink (a layout run installs it) and the draw canvas (an effect run installs it). A +// second table would repeat the claim/release machinery for the same lifetime. +struct SinkSlot { std::atomic owner{0}; AddLightSink sink; draw::Canvas canvas; }; +/// Two slots: the render task and whichever task edits a control are the two that ever run a script +/// at once. A third concurrent runner gets the overflow slot, which holds no sink — so its addLight +/// calls no-op instead of writing through someone else's context. +// constinit at namespace scope, not a function-local static: a local static carries a thread-safe +// initialisation guard, which is a lock, and this is read from the render tick. Constant +// initialisation happens before main, so the accessor is a plain address. +inline constinit SinkSlot gSinkSlots[2]{}; +inline SinkSlot* sinkSlots() MM_NONBLOCKING { return gSinkSlots; } +/// PERMANENTLY EMPTY. A third concurrent runner reads this and finds no sink, so its addLight calls +/// no-op — setAddLightSink deliberately never installs here, because a shared sink would let two +/// overflow threads write through each other's context. +inline const AddLightSink& sinkOverflow() { static const AddLightSink s; return s; } +/// The canvas twin of sinkOverflow: data stays null, so a third runner's draw calls no-op. +inline const draw::Canvas& canvasOverflow() { static const draw::Canvas c{}; return c; } + +/// The slot this thread owns; with `claim`, take a free one when none is owned yet. Null when +/// unowned and not claiming, or when both slots belong to other threads (the overflow case). +inline SinkSlot* ownedSlot(bool claim) MM_NONBLOCKING { + const uintptr_t me = platform::currentThreadId(); + SinkSlot* slots = sinkSlots(); + for (uint8_t i = 0; i < 2; i++) + if (slots[i].owner.load(std::memory_order_acquire) == me) return &slots[i]; + if (!claim) return nullptr; + for (uint8_t i = 0; i < 2; i++) { + uintptr_t free = 0; + if (slots[i].owner.compare_exchange_strong(free, me, std::memory_order_acq_rel, + std::memory_order_relaxed)) + return &slots[i]; + } + return nullptr; +} +/// Release only a fully empty slot: the sink and the canvas detach independently, and a release +/// while the other half is live would hand this thread's context to the next claimer. +inline void releaseIfEmpty(SinkSlot* s) MM_NONBLOCKING { + if (s && !s->sink.fn && !s->sink.ctx && !s->canvas.data) + s->owner.store(0, std::memory_order_release); +} +} // namespace detail + +/// This thread's sink. A READ never claims: a freshly claimed slot is empty by construction, so +/// claiming here could only ever return the empty sink it just made, while pinning a slot that +/// nothing will release (only the detach paths release, and a thread that installed nothing never +/// takes one). A script calling addLight from a binding that installs no sink (a modifier) would +/// hold a slot for the life of its task, and two such tasks would exhaust the table and silently +/// stop every later install. Installing claims; reading does not. +inline const AddLightSink& addLightSink() { + detail::SinkSlot* s = detail::ownedSlot(false); + return s ? s->sink : detail::sinkOverflow(); +} /// Point addLight at a consumer for the duration of one run; pass nullptr to detach. -inline void setAddLightSink(AddLightFn fn, void* ctx) { addLightSink() = {fn, ctx}; } +/// +/// Detaching RELEASES this thread's slot (unless the canvas half is still live), so two slots are +/// not exhausted by tasks that come and go: an HTTP request lands on whichever worker is free. +inline void setAddLightSink(AddLightFn fn, void* ctx) { + if (!fn && !ctx) { + // Clear FIRST, release last: the slot must not look free until the sink is cleared. + detail::SinkSlot* s = detail::ownedSlot(false); + if (s) { s->sink = {}; detail::releaseIfEmpty(s); } + return; // unowned: the overflow holds no sink to clear (see below) + } + // Install ONLY into an owned slot. Writing through addLightSink() would install into the shared + // overflow sink when both slots are taken, and a second overflow thread would then run through + // the first one's context — the exact aliasing the two-slot table exists to prevent. A third + // concurrent runner instead gets no sink at all, so its addLight calls no-op: visibly nothing + // placed, rather than lights written through another thread's layout. + detail::SinkSlot* s = detail::ownedSlot(true); + if (s) s->sink = {fn, ctx}; +} -extern "C" inline uint32_t mm_light_addLight(uint32_t x, uint32_t y, uint32_t z) { +extern "C" inline uint32_t mm_light_addLight(const uintptr_t* args, uint32_t, const uint8_t*) { + const uint32_t x = uint32_t(args[0]), y = uint32_t(args[1]), z = uint32_t(args[2]); // Both halves checked: a sink is only ever installed as a pair, but a context of null with a live // function is exactly what the crash was, so the guard states the whole precondition. const AddLightSink s = addLightSink(); @@ -164,6 +276,48 @@ extern "C" inline uint32_t mm_light_addLight(uint32_t x, uint32_t y, uint32_t z) return 0; } +/// The canvas the DRAW builtins render into, valid for the duration of one run(). The binding +/// installs it just before engine.run and detaches (a default Canvas) right after. It lives in +/// the same per-thread slot as the addLight sink, the one home for what a running script's +/// built-ins may reach, because C++ `thread_local` is unusable on the ESP32 (see the sink's +/// comment: THREADPTR is 0 on a task without TLS). A binding that installs nothing (a layout, a +/// modifier) leaves the data pointer null and every draw call no-ops: visibly nothing drawn, +/// never a write through another binding's buffer. +inline const draw::Canvas& drawCanvas() { + detail::SinkSlot* s = detail::ownedSlot(false); // a read never claims, see addLightSink + return s ? s->canvas : detail::canvasOverflow(); +} +inline void setDrawCanvas(const draw::Canvas& cv) MM_NONBLOCKING { + if (!cv.data) { + // Clear FIRST, release last: the same order the sink detach keeps. + detail::SinkSlot* s = detail::ownedSlot(false); + if (s) { s->canvas = {}; detail::releaseIfEmpty(s); } + return; + } + detail::SinkSlot* s = detail::ownedSlot(true); + if (s) s->canvas = cv; +} + +/// line(x1, y1, x2, y2, r, g, b) → a straight segment on the effect's canvas, z = 0. +/// +/// The first seven-argument builtin, riding the args-array call ABI (every Call builtin receives +/// its script arguments as a memory array, so arity is a data question, not a register one). +/// Endpoints are CLAMPED to the canvas extents before drawing: script arithmetic is unsigned and +/// wraps, so a "negative" coordinate arrives as a huge value, and an unclamped pair would send the +/// Bresenham walker on a billions-of-steps march. The clamp turns that into a segment pinned to +/// the edge, visible and instant. +extern "C" inline uint32_t mm_light_line(const uintptr_t* args, uint32_t, const uint8_t*) { + const draw::Canvas& cv = drawCanvas(); + if (!cv.data) return 0; + const auto clampAxis = [](uintptr_t v, lengthType n) -> lengthType { + return v >= uintptr_t(n) ? lengthType(n > 0 ? n - 1 : 0) : lengthType(v); + }; + const Coord3D a{clampAxis(args[0], cv.dims.x), clampAxis(args[1], cv.dims.y), 0}; + const Coord3D b{clampAxis(args[2], cv.dims.x), clampAxis(args[3], cv.dims.y), 0}; + draw::line(cv, a, b, RGB{uint8_t(args[4]), uint8_t(args[5]), uint8_t(args[6])}); + return 0; +} + // The light-domain SYSTEM VARIABLES: names the host defines and a script may only read. Reserved, // so a script cannot declare one and a name means the same thing in every script. // @@ -262,6 +416,8 @@ inline BuiltinTable lightBuiltins() { t.add({"print", 1, /*returns*/ true, BuiltinKind::Call, &mm_light_print, {}}); // addLight(x, y, z) → place a light. A scripted layout's whole vocabulary. t.add({"addLight", 3, /*returns*/ false, BuiltinKind::Call, &mm_light_addLight, {}}); + // line(x1, y1, x2, y2, r, g, b) → a segment on the canvas, via the shared draw::line. + t.add({"line", 7, /*returns*/ false, BuiltinKind::Call, &mm_light_line, {}}); return t; } diff --git a/src/light/moonlive/MoonLiveEffect.h b/src/light/moonlive/MoonLiveEffect.h index 837b755d..af471562 100644 --- a/src/light/moonlive/MoonLiveEffect.h +++ b/src/light/moonlive/MoonLiveEffect.h @@ -2,6 +2,7 @@ #include "light/effects/EffectBase.h" #include "core/moonlive/MoonLive.h" +#include "light/moonlive/MoonLiveScriptFile.h" #include "light/moonlive/MoonLiveBuiltins_light.h" #include #include @@ -24,14 +25,17 @@ class MoonLiveEffect : public EffectBase { const char* tags() const override { return "📝"; } // scripted Dim dimensions() const override { return Dim::D2; } - // The effect carries its SCRIPT SOURCE as an editable, persisted text control, plus a control + // The effect carries its script's NAME as an editable, persisted text control, plus a control // for every variable the script DECLARED (`uint8_t speed = 50; // @control 0..99`). The // engine exposes the declared list after a compile; each becomes a real uint8 control bound by // reference to the engine's live control-arena slot, so a slider write lands in the slot the - // next render tick reads — no recompile (the live-edit guarantee). Editing the source - // recompiles (the script-editor loop), which re-derives the control set. + // next render tick reads, with no recompile (the live-edit guarantee). Naming a different + // script recompiles (the script-editor loop), which re-derives the control set. void defineControls() override { - controls_.addTextArea("source", source_, sizeof(source_)); + // The script NAME, not the script. The text lives in a file the UI loads, edits and + // saves through /api/file — so a module costs ~32 bytes here instead of a resident + // kilobyte, and a script is bounded by the filesystem rather than by this array. + controls_.addText("script", script_, sizeof(script_)); // Every control the script declared. System variables (`width`, `height`, `depth`, `t`) // are not controls and never appear here, so there is nothing to filter out. uint8_t n = 0; @@ -39,22 +43,19 @@ class MoonLiveEffect : public EffectBase { for (uint8_t i = 0; i < n; i++) { uint8_t* slot = engine_.controlSlot(decls[i].offset); if (!slot) continue; // engine not compiled yet (first sweep) — controls appear after prepare - // The declared name is a span into source_ (not NUL-terminated); copy it into a stable - // member pool so the control descriptor's borrowed name pointer stays valid. The compiler - // rejects names ≥ kMaxControlName, so the full name always fits — no truncation, no - // distinct-names-collapsing-to-the-same-prefix collision. - std::memcpy(ctrlNames_[i], decls[i].name, decls[i].nameLen); - ctrlNames_[i][decls[i].nameLen] = '\0'; - controls_.addUint8(ctrlNames_[i], *slot, decls[i].min, decls[i].max); + // The engine owns its declared names (MoonLive::compile copies them out of the + // source before the text is freed), so the descriptor can borrow that pointer + // directly — a second per-binding pool would be the same fact in two places. + controls_.addUint8(decls[i].name, *slot, decls[i].min, decls[i].max); } } - // A `source` edit must recompile — route it through the prepare rebuild sweep so a new - // script swaps in live (the script-editor loop). A SCRIPTED CONTROL's value change must NOT + // Naming a different script must recompile: route it through the prepare rebuild sweep so the + // new one swaps in live (the script-editor loop). A SCRIPTED CONTROL's value change must NOT // recompile: it just updates an arena byte the running native code reads next tick. So only - // "source" triggers a rebuild; every scripted control returns false (the live-edit path). + // "script" triggers a rebuild; every scripted control returns false (the live-edit path). bool affectsPrepare(const char* controlName) const override { - return std::strcmp(controlName, "source") == 0; + return std::strcmp(controlName, "script") == 0; } // Compile the source on the cold rebuild path. A failed compile (parse error or no exec @@ -68,10 +69,12 @@ class MoonLiveEffect : public EffectBase { // would be a second, disagreeing answer: set it to 16 on an 8x8 panel and the effect draws // off the edge. The compiler reserves the name, so that cannot happen. moonlive::resetPrintBudget(); - if (engine_.compile(source_, moonlive::lightBuiltins(), moonlive::effectSysVars())) { + const char* err = nullptr; + if (moonlive::compileScriptFile(engine_, script_, moonlive::lightBuiltins(), + moonlive::effectSysVars(), err)) { clearStatus(); } else { - setStatus(engine_.error(), Severity::Error); + setStatus(err, Severity::Error); } // The compile re-derives the declared-control set, so rebuild the control list to surface // it (the same rebuildControls() pattern NetworkModule uses when a state change reshapes @@ -94,7 +97,12 @@ class MoonLiveEffect : public EffectBase { writeSysVar(moonlive::kSysWidth, width()); writeSysVar(moonlive::kSysHeight, height()); writeSysVar(moonlive::kSysDepth, depth()); + // The draw builtins (line) render through the same canvas every native effect uses, + // installed for exactly one run and detached after, so a script can only ever draw into + // the layer it is ticking in. + moonlive::setDrawCanvas(canvas()); engine_.run(buffer(), nrOfLights(), cpl, elapsed()); + moonlive::setDrawCanvas({}); } void release() override { @@ -104,9 +112,11 @@ class MoonLiveEffect : public EffectBase { /// Replace the script. The next prepare() compiles it — the same path a UI edit takes, so a /// test and a user exercise identical code. - void setSource(const char* s) { - if (!s) return; - std::snprintf(source_, sizeof(source_), "%s", s); + /// Point the module at a script in the shared script directory. The file itself is written by + /// the UI (or the File Manager); this only says WHICH one, and the next prepare() compiles it. + void setScript(const char* name) { + if (!name) return; + std::snprintf(script_, sizeof(script_), "%s", name); } private: @@ -123,15 +133,9 @@ class MoonLiveEffect : public EffectBase { } - char source_[moonlive::kMaxScriptBytes] = "setRGB(random16(256), random16(256), random16(256), random16(256));"; - // 512 fits a multi-line - // multi-control script (a decl per control + the - // statement); grow-on-demand is backlogged for the - // bigger Ripples-class scripts of later stages. - // Stable NUL-terminated copies of the script-declared control names (the control descriptor - // borrows the pointer; the decl span into source_ is not NUL-terminated). Sized to the - // compiler's name limit so a name always fits without truncation. - char ctrlNames_[moonlive::kMaxCtrls][moonlive::kMaxControlName] = {}; + // A fresh card starts with NO script: it reports "no script" and renders nothing until one + // is named. Naming a default here would make every new module compile the same effect. + char script_[moonlive::kMaxScriptName + 1] = ""; }; } // namespace mm diff --git a/src/light/moonlive/MoonLiveLayout.h b/src/light/moonlive/MoonLiveLayout.h index 14cb80d3..fbc50d2b 100644 --- a/src/light/moonlive/MoonLiveLayout.h +++ b/src/light/moonlive/MoonLiveLayout.h @@ -1,6 +1,7 @@ #pragma once #include "core/moonlive/MoonLive.h" +#include "light/moonlive/MoonLiveScriptFile.h" #include "light/layouts/LayoutBase.h" #include "light/moonlive/MoonLiveBuiltins_light.h" #include @@ -39,7 +40,9 @@ class MoonLiveLayout : public LayoutBase { const char* tags() const override { return "📝"; } // scripted void defineControls() override { - controls_.addTextArea("source", source_, sizeof(source_)); + // The script NAME, not the script — the text lives in a file the UI loads and saves + // through /api/file. A module costs ~32 bytes here instead of a resident kilobyte. + controls_.addText("script", script_, sizeof(script_)); // Every control the SCRIPT declared — including any extents it loops over. A layout does not // RECEIVE a width: the pipeline derives its bounding box from the coordinates the layouts // actually place (Layouts::prepare, "max coordinate + 1 per axis"), so a width handed in @@ -50,9 +53,7 @@ class MoonLiveLayout : public LayoutBase { for (uint8_t i = 0; i < n; i++) { uint8_t* slot = engine_.controlSlot(decls[i].offset); if (!slot) continue; - std::memcpy(ctrlNames_[i], decls[i].name, decls[i].nameLen); - ctrlNames_[i][decls[i].nameLen] = '\0'; - controls_.addUint8(ctrlNames_[i], *slot, decls[i].min, decls[i].max); + controls_.addUint8(decls[i].name, *slot, decls[i].min, decls[i].max); } } @@ -89,9 +90,19 @@ class MoonLiveLayout : public LayoutBase { } /// Replace the script. The next prepare() compiles it — the path a UI edit takes. - void setSource(const char* s) { - if (!s) return; - std::snprintf(source_, sizeof(source_), "%s", s); + /// A control write lands DIRECTLY in script_ (addText binds the buffer), so setScript() is not + /// called and nothing would clear the compiled-hash — compile() would early-return and keep + /// running the previous script under a new name. Clearing it here covers both paths. + void onControlChanged(const char* name) override { + if (name && std::strcmp(name, "script") == 0) { compiledHash_ = 0; compileFailed_ = false; } + } + + /// Point the layout at a script in the shared script directory; the next prepare() compiles it. + void setScript(const char* name) { + if (!name) return; + std::snprintf(script_, sizeof(script_), "%s", name); + compiledHash_ = 0; // a different file: whatever was compiled is not it + compileFailed_ = false; // and it has not been tried yet } private: @@ -112,15 +123,31 @@ class MoonLiveLayout : public LayoutBase { /// Moving layout work to a worker would change that — the engine would then need a published /// immutable program rather than one mutated in place. void compile() const { - if (engine_.ok() && std::strcmp(source_, compiled_) == 0) return; // already current + if (engine_.ok() && compiledHash_ != 0) return; // already current for this script + // Give up only on the name that ACTUALLY failed. As a bare flag this latched on the empty + // script every device boots with, and then skipped the compile forever — the card sat at + // "no script" however many times a real one was named, because the render loop asks for the + // light count long before any control write clears a flag. + if (compileFailed_ && std::strcmp(failedScript_, script_) == 0) return; auto* self = const_cast(this); moonlive::resetPrintBudget(); // A layout is the one script with no layer to ask, so it gets the clock and nothing else: // it names its own size controls, and `x`/`y` stay free as ordinary loop counters. - if (self->engine_.compile(source_, moonlive::lightBuiltins(), moonlive::layoutSysVars())) + const char* err = nullptr; + uint32_t hash = 0; + if (moonlive::compileScriptFile(self->engine_, script_, moonlive::lightBuiltins(), + moonlive::layoutSysVars(), err, &hash)) { self->clearStatus(); - else self->setStatus(self->engine_.error(), Severity::Error); - std::snprintf(self->compiled_, sizeof(compiled_), "%s", source_); + self->compileFailed_ = false; + } else { + self->setStatus(err, Severity::Error); + // Remember WHICH name failed, so a different one is still tried. + self->compileFailed_ = true; + std::snprintf(self->failedScript_, sizeof(failedScript_), "%s", script_); + } + // The CONTENT hash, not a copy of the text: 4 bytes to answer "is what I compiled still what + // the file says", which is all the rebuild check ever needed. 0 means "nothing compiled". + self->compiledHash_ = hash; self->setDynamicBytes(engine_.heapBytes()); } @@ -151,24 +178,26 @@ class MoonLiveLayout : public LayoutBase { mutable moonlive::MoonLive engine_; - // Default script — a grid, the layout almost every panel is. The nested loop and the index - // arithmetic are the whole definition, which is the case for scripting a layout at all. - char source_[moonlive::kMaxScriptBytes] = - "uint8_t cols = 16; // @control 1..64\n" - "uint8_t rows = 16; // @control 1..64\n" - "for (y = 0; y < rows; y = y + 1) {\n" - " for (x = 0; x < cols; x = x + 1) {\n" - " addLight(x, y, 0);\n" - " }\n" - "}"; - - // The source the loaded program was built from, so compile() is a no-op when current. - // sizeof(source_), never a literal: a copy too small to hold source_ truncates, never - // compares equal, and the mapping rebuilds every frame — the blank-screen loop this - // comparison exists to prevent. - mutable char compiled_[sizeof(source_)] = {}; - - char ctrlNames_[moonlive::kMaxCtrls][moonlive::kMaxControlName] = {}; + // The script's FILE NAME, inside the shared script directory. Empty on a fresh card: it reports + // "no script" and places no lights until one is named, rather than every new layout compiling + // the same default grid. + char script_[moonlive::kMaxScriptName + 1] = ""; + + // FNV-1a of the text the loaded program was built from — 4 bytes in place of a second copy of + // the source. Non-zero means "this engine holds a compiled program for that content"; 0 means + // nothing is compiled, which is what setScript() restores when the file changes. + mutable uint32_t compiledHash_ = 0; + + // Has this script name already been tried and failed? A FAILED compile leaves compiledHash_ at 0 + // and the engine not ok(), which is indistinguishable from "not compiled yet" — so without this + // flag every lightCount()/forEachCoord() re-reads and re-compiles the file. Each attempt is two + // LittleFS operations (~5 ms on an S3), the pipeline asks repeatedly while sizing and walking the + // fixture, and the retries starve the task until the 12 s watchdog resets the device. One attempt + // per script name is all that can ever help: nothing about the file changes between two calls in + // the same rebuild. Cleared wherever compiledHash_ is, because both mean "this is a new script". + mutable bool compileFailed_ = false; + mutable char failedScript_[moonlive::kMaxScriptName + 1] = ""; // the name compileFailed_ refers to + }; } // namespace mm diff --git a/src/light/moonlive/MoonLiveModifier.h b/src/light/moonlive/MoonLiveModifier.h index a7e9628e..dd35ad22 100644 --- a/src/light/moonlive/MoonLiveModifier.h +++ b/src/light/moonlive/MoonLiveModifier.h @@ -1,6 +1,7 @@ #pragma once #include "core/moonlive/MoonLive.h" +#include "light/moonlive/MoonLiveScriptFile.h" #include "light/modifiers/ModifierBase.h" #include "light/moonlive/MoonLiveBuiltins_light.h" #include @@ -43,7 +44,9 @@ class MoonLiveModifier : public ModifierBase { const char* tags() const override { return "📝"; } // scripted void defineControls() override { - controls_.addTextArea("source", source_, sizeof(source_)); + // The script NAME, not the script — the text lives in a file the UI loads and saves + // through /api/file. A module costs ~32 bytes here instead of a resident kilobyte. + controls_.addText("script", script_, sizeof(script_)); // Every control the script declared. System variables (`x`/`y`/`z`, `width`/`height`/ // `depth`, `t`) are not controls and never appear here, so there is nothing to filter out. uint8_t n = 0; @@ -51,9 +54,7 @@ class MoonLiveModifier : public ModifierBase { for (uint8_t i = 0; i < n; i++) { uint8_t* slot = engine_.controlSlot(decls[i].offset); if (!slot) continue; - std::memcpy(ctrlNames_[i], decls[i].name, decls[i].nameLen); - ctrlNames_[i][decls[i].nameLen] = '\0'; - controls_.addUint8(ctrlNames_[i], *slot, decls[i].min, decls[i].max); + controls_.addUint8(decls[i].name, *slot, decls[i].min, decls[i].max); } } @@ -68,11 +69,13 @@ class MoonLiveModifier : public ModifierBase { // defines — the binding writes their slots per call, and the compiler reserves the names so // a script cannot declare one and shadow the value it is being handed. moonlive::resetPrintBudget(); - if (engine_.compile(source_, moonlive::lightBuiltins(), - moonlive::modifierSysVars())) { + const char* err = nullptr; + uint32_t hash = 0; + if (moonlive::compileScriptFile(engine_, script_, moonlive::lightBuiltins(), + moonlive::modifierSysVars(), err, &hash)) { clearStatus(); } else { - setStatus(engine_.error(), Severity::Error); + setStatus(err, Severity::Error); } rebuildControls(); setDynamicBytes(engine_.heapBytes()); @@ -82,8 +85,8 @@ class MoonLiveModifier : public ModifierBase { // again: setting the flag unconditionally makes the two call each other forever, the // mapping is rebuilt every frame, and the fixture renders nothing at all. Comparing the // compiled source is what breaks that cycle. - if (std::strcmp(source_, compiled_) != 0) { - std::snprintf(compiled_, sizeof(compiled_), "%s", source_); + if (hash != compiledHash_) { + compiledHash_ = hash; needsRebuild_ = true; } } @@ -143,15 +146,16 @@ class MoonLiveModifier : public ModifierBase { // treated as a first compile. Keeping it made a disabled-then-re-enabled modifier inert — // the Layer folds while the engine is empty, then prepare() recompiles, sees the same // source, and never asks for the rebuild that would apply it. - compiled_[0] = '\0'; + compiledHash_ = 0; ModifierBase::release(); } /// Replace the script. The next prepare() compiles it — the same path a UI edit takes, so a /// test and a user exercise identical code. - void setSource(const char* s) { - if (!s) return; - std::snprintf(source_, sizeof(source_), "%s", s); + void setScript(const char* name) { + if (!name) return; + std::snprintf(script_, sizeof(script_), "%s", name); + compiledHash_ = 0; // a different file: whatever was compiled is not it } private: @@ -160,15 +164,16 @@ class MoonLiveModifier : public ModifierBase { // Default script — a mirror on x. Chosen because it is instantly readable on a bench strand // (the pattern runs the other way) and is a modifier people actually reach for, so a working // binding looks like something rather than like nothing. - char source_[moonlive::kMaxScriptBytes] = "setXYZ(0, width - 1 - x, y, z);"; + // The script's FILE NAME, inside the shared script directory. Empty on a fresh card: it reports + // "no script" and passes coordinates through untouched until one is named. + char script_[moonlive::kMaxScriptName + 1] = ""; - // The source the CURRENT mapping was built from; a rebuild is needed only when it changes. - // sizeof(source_), never a literal: this is the string compared to decide whether to rebuild, - // so a copy too small to hold source_ truncates, never matches, and the mapping rebuilds every - // frame — the blank-screen loop the comparison exists to prevent. - char compiled_[sizeof(source_)] = {}; + // FNV-1a of the text the loaded program was built from — 4 bytes in place of a second copy of + // the source. It answers the one question the rebuild check ever asked ("did this change"), and + // an unconditional rebuild here would make prepare() and the Layer's rebuild call each other + // forever, which is the blank-fixture loop the comparison exists to prevent. + uint32_t compiledHash_ = 0; - char ctrlNames_[moonlive::kMaxCtrls][moonlive::kMaxControlName] = {}; bool needsRebuild_ = false; // a recompile happened; the Layer's mapping is stale Coord3D box_{0, 0, 0}; // the logical box, from modifyLogicalSize diff --git a/src/light/moonlive/MoonLiveScriptFile.h b/src/light/moonlive/MoonLiveScriptFile.h new file mode 100644 index 00000000..469a27b4 --- /dev/null +++ b/src/light/moonlive/MoonLiveScriptFile.h @@ -0,0 +1,106 @@ +#pragma once + +#include "core/moonlive/MoonLive.h" +#include "platform/platform.h" + +#include +#include + +namespace mm::moonlive { + +/// Where a scripted module's `.mlv` file lives. One fixed directory, the way `/.config` holds +/// persisted state: a module stores a NAME, not a path, so it cannot reach outside this folder and +/// the File Manager has one obvious place to look. +inline constexpr const char* kScriptDir = "/moonlive"; + +/// The largest script the loader will read into RAM at once. Not a language limit — the buffer is +/// sized to the FILE and freed the moment the compile ends — but a bound so a stray large file +/// cannot ask a 320 KB device for an allocation it will not survive. +inline constexpr long kScriptFileMax = 16384; + +/// Longest script name accepted, and the bound on the path buffer below. The bindings size their +/// `script` control buffer from this (`kMaxScriptName + 1`), so a name this loader would accept can +/// always be held: a shorter control would truncate silently, and truncation can strip the `.mlv` +/// that makes a name valid at all. +inline constexpr size_t kMaxScriptName = 40; + +/// Read `/` and compile it. The source lives in a right-sized heap buffer for the +/// duration of the compile and is freed before returning, so a module holds a filename (~32 B) and +/// the emitted code — never the script text. That is the whole point: the fixed per-module arrays +/// this replaces cost ~2 KB EACH, resident whether or not a script was loaded. +/// +/// Returns true when the script compiled. On any failure `err` names it, in the words a user needs: +/// which file, and what was wrong with it. +/// FNV-1a over the script text. A caller that must know "did this change" keeps 4 bytes rather than +/// a second copy of the source — which is the whole reason the text is not resident any more. +inline uint32_t scriptHash(const char* s, size_t len) { + uint32_t h = 2166136261u; + for (size_t i = 0; i < len; i++) { h ^= static_cast(s[i]); h *= 16777619u; } + return h; +} + +/// As compileScriptFile, and additionally reports the source's hash so a caller can tell a changed +/// script from an unchanged one without holding the text. +inline bool compileScriptFile(MoonLive& engine, const char* name, + const BuiltinTable& builtins, const SysVarTable& sysvars, + const char*& err, uint32_t* hashOut = nullptr) { + // FIRST, before any validation can return: drop whatever is already compiled. Every check + // below leaves through `return false`, and only engine.compile() releases the previous + // program, so without this a rejected script (renamed, deleted, emptied) leaves the OLD one + // executing while the module reports an error. The card says "script not found" and the + // fixture keeps rendering the script that is gone. + // + // freeCode, not free: the control ARENA must survive, or a scripted control loses the live + // value the user set whenever a compile fails. + engine.freeCode(); + + // The script directory must exist before anything can be SAVED into it, and on a fresh device + // nothing has created it yet — the write endpoint does not make parent directories, so a first + // save would fail with nowhere obvious to look. Creating it here (mkdir -p, a no-op when it is + // already there) means naming a script is enough to make the folder appear. + platform::fsMkdir(kScriptDir); + + if (!name || !name[0]) { err = "no script — set the script name"; return false; } + + // A BASENAME only. The fixed directory is the point — a module names a script, it does not + // address the filesystem — so a separator or a `..` would let a control value reach outside + // kScriptDir (`../.config/NetworkModule.json` reads the device's saved credentials). Rejected + // rather than sanitised: a name that needs rewriting to be safe is a name a user mistyped. + for (const char* c = name; *c; c++) + if (*c == '/' || *c == '\\') { err = "script name is a file in the script folder, not a path"; return false; } + if (std::strcmp(name, "..") == 0 || std::strncmp(name, "../", 3) == 0) { + err = "script name is a file in the script folder, not a path"; return false; + } + // .mlv, so a stray name cannot pull in an unrelated file that happens to sit alongside. The + // upper bound also lets the compiler see that the snprintf below cannot truncate. + const size_t len = std::strlen(name); + if (len < 5 || len > kMaxScriptName || std::strcmp(name + len - 4, ".mlv") != 0) { + err = "script name must end in .mlv"; return false; + } + + char path[96]; + std::snprintf(path, sizeof(path), "%s/%s", kScriptDir, name); + + const long size = platform::fsSize(path); + if (size < 0) { err = "script not found"; return false; } + if (size == 0) { err = "script is empty"; return false; } + if (size > kScriptFileMax) { err = "script too large"; return false; } + + // +1 for the NUL the lexer reads as End. fsRead null-terminates on success, but the buffer has + // to have room for it. + char* text = static_cast(platform::alloc(static_cast(size) + 1)); + if (!text) { err = "no memory for the script"; return false; } + + const int read = platform::fsRead(path, text, static_cast(size) + 1); + if (read <= 0) { platform::free(text); err = "script could not be read"; return false; } + + if (hashOut) *hashOut = scriptHash(text, static_cast(read)); + const bool ok = engine.compile(text, builtins, sysvars); + if (!ok) err = engine.error(); + // Freed on BOTH paths, before returning: the text has done its job either way, and a failed + // compile is exactly when a device can least afford to leak. + platform::free(text); + return ok; +} + +} // namespace mm::moonlive diff --git a/src/main.cpp b/src/main.cpp index 1287207c..a6a1ce54 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -558,7 +558,12 @@ void mm_main(volatile bool& keepRunning, uint16_t httpPort) { scheduler.addModule(layouts); scheduler.addModule(effectsContainer); scheduler.addModule(drivers); - scheduler.addModule(httpServer); + // Only where an IP stack exists. setup() binds a listening socket, and with neither WiFi nor + // Ethernet compiled in nothing calls esp_netif_init(), so the TCP/IP thread never exists and + // lwIP asserts on its null mutex, taking the board down before the light pipeline runs. Same + // gate MqttModule already uses. No shipping firmware is in that state today; the gate is what + // makes a network-less build a supported configuration rather than a boot loop. + if constexpr (mm::platform::hasNetwork) scheduler.addModule(httpServer); scheduler.setup(); diff --git a/src/platform/desktop/moonlive_asm_host.cpp b/src/platform/desktop/moonlive_asm_host.cpp index 08d6d2e9..0181ced1 100644 --- a/src/platform/desktop/moonlive_asm_host.cpp +++ b/src/platform/desktop/moonlive_asm_host.cpp @@ -10,14 +10,19 @@ namespace mm::moonlive { -#if defined(__aarch64__) +#if defined(__aarch64__) && !defined(MM_MOONLIVE_FORCE_NO_HOST_JIT) // arm64 register map: R0..R4 = the host-ABI arg registers x0..x4 (buf, nLights, cpl, t, ctrls — // the control-values arena pointer, kArg4). R5..R13 = caller-saved scratch x9..x14 then x5..x7. // Index math uses the 64-bit views (xN) for addresses, 32-bit (wN) for counters/colors — same // register number, so one map suffices. x15 is the call() address/immediate scratch (not a vreg). static constexpr uint8_t kArm64Reg[kRegCount] = {0, 1, 2, 3, 4, 9, 10, 11, 12, 13, 14, 5, 6, 7}; -static uint8_t mr(Reg r) { return kArm64Reg[r]; } +// BOUNDS-CHECKED. The inline ops address their scratch as `vregsUsed + n`, so an index one past the +// map is reachable whenever the reservation and the map disagree — and an out-of-bounds read returns +// whatever byte follows the array, making the emitted instruction name a register chosen by +// accident. Clamping turns that into a wrong-but-safe register instead of undefined behaviour; the +// static_assert below and the lowerer's reservation are what stop it happening at all. +static uint8_t mr(Reg r) { return kArm64Reg[r < kRegCount ? r : kRegCount - 1]; } // A scratch register that is ALSO a vreg silently corrupts values — see the RISC-V backend, where // kScratchFn aliased vreg R12 and every call returned a stale value. Checked here so the map can @@ -47,15 +52,69 @@ void HostAssembler::addFixup(size_t at, Label label, uint8_t kind) { } void HostAssembler::emit32(uint32_t w) { - if (len_ + 4 > kCap) { overflow_ = true; return; } + if (!buf_ || len_ + 4 > kCap) { overflow_ = true; return; } buf_[len_++] = uint8_t(w); buf_[len_++] = uint8_t(w >> 8); buf_[len_++] = uint8_t(w >> 16); buf_[len_++] = uint8_t(w >> 24); } void HostAssembler::emitBytes(const uint8_t* p, size_t n) { - if (len_ + n > kCap) { overflow_ = true; return; } + if (!buf_ || len_ + n > kCap) { overflow_ = true; return; } std::memcpy(buf_ + len_, p, n); len_ += n; } +// --- the call frame: the register allocator's overflow storage --------------------------------- +// +// x29 is the AAPCS frame pointer, callee-saved and outside both the vreg map and the {x15,x16,x17} +// scratch set, so nothing this backend emits can disturb it. Slots are addressed from x29 rather +// than sp precisely because call() moves sp by 128 bytes around every host call: sp-relative offsets +// would be wrong for the duration of the call, and a script whose spilled value is read after a +// random16() is the ordinary case, not an exotic one. It is also the layout a nested call needs — +// each activation gets its own x29 — which is why the frame pointer is here now rather than added +// later when script-defined functions arrive. +// +// Layout: [x29+0] = saved x29, [x29+8] = saved x30, slot n at [x29 + 16 + n*8]. +static constexpr uint16_t kSlotBase = 16; + +void HostAssembler::prologue(uint8_t slots) { + if (slots == 0) return; // no spilling: no frame, no cost + if (slots > kMaxSpillSlots) { overflow_ = true; return; } + // 16-byte aligned, as the AAPCS requires of sp at every instruction boundary — an unaligned sp + // faults on the first stp a callee executes, which would surface as a crash inside random16. + const uint16_t bytes = static_cast((kSlotBase + slots * 8 + 15) & ~15); + frameBytes_ = bytes; + emit32(0xa9800000u | ((uint32_t((-int32_t(bytes)) / 8) & 0x7f) << 15) | (30u << 10) | (31u << 5) | 29u); + emit32(0x910003fdu); // mov x29, sp +} +void HostAssembler::epilogue() { + if (frameBytes_ != 0) { + emit32(0x910003bfu); // mov sp, x29 (drop anything call() left behind) + emit32(0xa8c00000u | ((uint32_t(frameBytes_) / 8) << 15) | (30u << 10) | (31u << 5) | 29u); + } + ret(); +} +// str/ldr with a 12-bit SCALED unsigned offset (imm12 counts 8-byte units for the 64-bit form). +// 64-bit, not 32: a vreg can hold a pointer — kArg0 is the buffer — and truncating one to 32 bits on +// the way to a slot would produce a wild store the moment a spilled pointer came back. +void HostAssembler::spillStore(Reg r, uint8_t slot) { + // No frame means prologue() bailed; emitting would address the CALLER's stack. + if (slot >= kMaxSpillSlots || frameBytes_ == 0) { overflow_ = true; return; } + emit32(0xf9000000u | ((uint32_t(kSlotBase + slot * 8) / 8) << 10) | (29u << 5) | mr(r)); +} +void HostAssembler::spillLoad(Reg r, uint8_t slot) { + // No frame means prologue() bailed; emitting would address the CALLER's stack. + if (slot >= kMaxSpillSlots || frameBytes_ == 0) { overflow_ = true; return; } + emit32(0xf9400000u | ((uint32_t(kSlotBase + slot * 8) / 8) << 10) | (29u << 5) | mr(r)); +} + +// The ADDRESS of a frame slot, for a host call that reads its arguments from the frame. The slots +// are already where the arguments live; a call passes where they start rather than the values, which +// is what makes the number of arguments a memory question instead of a register one. +void HostAssembler::slotAddr(Reg d, uint8_t slot) { + // No frame means prologue() bailed; emitting would address the CALLER's stack. + if (slot >= kMaxSpillSlots || frameBytes_ == 0) { overflow_ = true; return; } + const uint32_t off = kSlotBase + uint32_t(slot) * 8; + emit32(0x91000000u | (off << 10) | (29u << 5) | mr(d)); // add xD, x29, #off +} + void HostAssembler::movImm(Reg d, int32_t imm) { // movz builds a ZERO-extended 16-bit constant, so a negative immediate would land as its // unsigned counterpart (-1 as 65535). The compiler emits Const(-1) to express subtraction — @@ -160,6 +219,9 @@ void HostAssembler::call(Reg d, Reg a, Reg b, Reg c, const void* fn) { void HostAssembler::ret() { emit32(0xd65f03c0u); } void HostAssembler::patchBranches() { + // Nothing was emitted if the buffer never allocated, so there is nothing to patch — + // stated rather than left to the reader to derive from fixupCount_ being 0. + if (!buf_) return; for (uint8_t i = 0; i < fixupCount_; i++) { const Fixup& f = fixups_[i]; int32_t target = labelPos_[f.label]; diff --git a/src/platform/desktop/moonlive_asm_host.h b/src/platform/desktop/moonlive_asm_host.h index 31dafe21..670f6257 100644 --- a/src/platform/desktop/moonlive_asm_host.h +++ b/src/platform/desktop/moonlive_asm_host.h @@ -1,5 +1,7 @@ #pragma once +#include "platform/platform.h" // alloc/free — the emit buffer is heap, not stack + #include "core/moonlive/MoonLiveIr.h" // kCodeCap — one cap for the staging buffer and every backend #include @@ -33,6 +35,18 @@ enum class Cond : uint8_t { Lo /* unsigned < */, Hs /* unsigned >= */, Ne /* != class HostAssembler { public: + // Owns buf_ (see below). Freed here, copying deleted — an emitter that was copied + // would double-free the buffer it emits into. + ~HostAssembler() { platform::free(buf_); } + /// `cap` is the code buffer's size, chosen per SCRIPT by the caller (codeCapFor) rather + /// than a shared constant — the backends differ by up to 1.9x on identical source, so one + /// number cannot fit them all. Defaults to the sanity bound for callers that emit a fixed + /// blob (emitFill) and have no token count to size from. + explicit HostAssembler(size_t cap = kCodeCap) + : kCap(cap), buf_(static_cast(platform::alloc(cap))) {} + HostAssembler(const HostAssembler&) = delete; + HostAssembler& operator=(const HostAssembler&) = delete; + // --- buffer --- // Resolve all branch fixups against bound labels, then expose the finished bytes. Call // once after the last instruction; bytes()/size() are valid only after finalize(). @@ -45,6 +59,19 @@ class HostAssembler { Label newLabel(); void bind(Label l); // mark l's position = current offset + // --- the call frame --- + // The register allocator's overflow storage (MoonLiveSpill.h). prologue() opens a frame with + // room for `slots` spilled values and parks a frame pointer at its base; spillStore/spillLoad + // address a slot as an offset from THAT pointer, never from sp — so a call() that moves sp + // underneath them, and the nested/recursive calls MoonLive is gaining next, leave slot + // addressing untouched. slots == 0 emits nothing at all: a script that never spilled pays zero. + void prologue(uint8_t slots); + void epilogue(); // tear the frame down, then ret + void spillStore(Reg r, uint8_t slot); + void spillLoad(Reg r, uint8_t slot); + void slotAddr(Reg d, uint8_t slot); // d = &frame[slot] — a call's argument block + static constexpr uint8_t kMaxSpillSlots = kTotalSlots; // parser/allocator range + the parked host args // what the frame below can address + // --- instructions (named, register/immediate operands) --- void movImm(Reg d, int32_t imm); // d = imm void addImm(Reg d, Reg a, int32_t imm); // d = a + imm @@ -66,8 +93,8 @@ class HostAssembler { void ret(); private: - // The emitted-code buffer, sized by the engine's shared cap (kCodeCap). - static constexpr size_t kCap = kCodeCap; + // The emitted-code buffer's size, fixed for this object's life but chosen per script. + const size_t kCap; static constexpr uint8_t kMaxLabels = 16; static constexpr uint8_t kMaxFixups = 32; @@ -75,9 +102,18 @@ class HostAssembler { void emitBytes(const uint8_t* p, size_t n); void addFixup(size_t at, Label label, uint8_t kind); // enqueue a branch fixup (bounds-checked) - uint8_t buf_[kCap] = {}; + // HEAP, not a member array: the assembler is a stack local in lowerToBytes, so a kCap-sized + // member put 2 KB on the compile chain's stack — on top of the staging buffer and the parser + // frames. On a classic ESP32 that overflowed the task and faulted inside _xt_context_save + // (the plan named this: "buf_[kCap] inside the assembler, itself a stack local"). The buffer is + // scratch that ends in a memcpy to the caller's output, so nothing outlives the object. + uint8_t* buf_; size_t len_ = 0; bool overflow_ = false; + // Frame size in bytes, 0 when no prologue was emitted. epilogue() reads it, so the teardown can + // never disagree with the setup about how far sp moved — the class of bug that returns to a + // corrupted stack and is indistinguishable from a miscompile. + uint16_t frameBytes_ = 0; // Label positions (-1 = unbound) and pending branch fixups. int32_t labelPos_[kMaxLabels]; diff --git a/src/platform/desktop/moonlive_emit.cpp b/src/platform/desktop/moonlive_emit.cpp index e4598246..fd8d99e6 100644 --- a/src/platform/desktop/moonlive_emit.cpp +++ b/src/platform/desktop/moonlive_emit.cpp @@ -15,7 +15,7 @@ namespace mm::moonlive { -#if defined(__aarch64__) +#if defined(__aarch64__) && !defined(MM_MOONLIVE_FORCE_NO_HOST_JIT) // arm64 template (assembled from fill_arm64.s, verified with clang/objdump). 18 words. // buf=x0, nLights=w1, cpl=w2. R/G/B live in `mov w4/w5/w6, #imm` at word indices 4,5,6. diff --git a/src/platform/desktop/moonlive_lower_host.cpp b/src/platform/desktop/moonlive_lower_host.cpp index b3264516..58adb051 100644 --- a/src/platform/desktop/moonlive_lower_host.cpp +++ b/src/platform/desktop/moonlive_lower_host.cpp @@ -1,5 +1,6 @@ #include "core/moonlive/moonlive_emit.h" #include "core/moonlive/MoonLiveIr.h" +#include "core/moonlive/MoonLiveSpill.h" // the register allocator — core's, run before lowering #include "moonlive_asm_host.h" #include @@ -16,13 +17,13 @@ namespace mm::moonlive { -#if defined(__aarch64__) // the host assembler is implemented for arm64 only (see moonlive_asm_host.cpp) +#if defined(__aarch64__) && !defined(MM_MOONLIVE_FORCE_NO_HOST_JIT) // arm64 only (see moonlive_asm_host.cpp) namespace { Reg reg(VReg v) { return static_cast(v); } } -size_t lowerToBytes(const IrProgram& ir, uint8_t* out, size_t cap) { +size_t lowerToBytes(IrProgram& ir, uint8_t* out, size_t cap, const RegBudget* squeeze) { // Reserve three scratch regs above the program's vregs for the inline ops' temps. // Reserve scratch only for the inline ops this program actually contains. Unlike Xtensa and // RISC-V, THIS backend's StoreElem needs one scratch too (sAddr — it does not fold the address @@ -30,7 +31,30 @@ size_t lowerToBytes(const IrProgram& ir, uint8_t* out, size_t cap) { // needs any. Reserving the maximum unconditionally cost a register every script paid for. const uint8_t scratch = ir.hasInline(InlineOp::FillElems) ? 3 : ir.hasInline(InlineOp::StoreElem) ? 1 : 0; - if (!out || cap == 0 || ir.vregsUsed + scratch > kRegCount) return 0; + // A host CALL also needs two scratch registers — the address of its argument block and the + // count — but it SHARES them with the inline ops rather than reserving its own. A Call and an + // Inline are different IR instructions, so their scratch is never live at the same time, and + // both die at the end of the one instruction that uses them. Reserving separately cost two + // registers permanently, which on Xtensa's ten is the difference between compiling and not. + // +1 for the host-argument reload. The host arguments live in frame slots now (core parks them + // at entry), so an op that reads buf/nLights/cpl/ctrls brings one back for the instruction that + // needs it. A call's two scratch registers still share with the inline ops' — different IR + // instructions, never live at once. + const uint8_t scratchTotal = static_cast((scratch < 2 ? uint8_t(2) : scratch) + 1); + if (!out || cap == 0) return 0; + + // Run the register allocator before lowering. It leaves a program that already fits untouched, + // and rewrites one that does not into Spill/Reload against this backend's frame — replacing the + // hand-rolled `vregsUsed + scratch > kRegCount` bail that used to REFUSE such a script outright. + // False here means even the spilled form does not fit, which is a diagnostic, never a miscompile. + uint8_t slots = 0; + // `squeeze` overrides the REGISTER COUNT and slot count a test wants to constrain, but never + // `reserved`: the scratch is what THIS lowerer is about to use for its inline ops and call + // argument block, so a test-supplied value would let the allocator hand out a register the + // lowerer then overwrites — miscompiling exactly the squeezed programs the seam exists to prove. + const RegBudget budget = squeeze ? RegBudget{squeeze->regs, scratchTotal, squeeze->slots} + : RegBudget{kRegCount, scratchTotal, HostAssembler::kMaxSpillSlots}; + if (!spillToBudget(ir, budget, slots)) return 0; // sAddr FIRST, because it is the one StoreElem also uses: a store-only program reserves a single // scratch, so the shared one has to be the lowest index or it would name a register outside the // reservation. sOff/sCtr are FillElems-only and sit above it. @@ -38,7 +62,21 @@ size_t lowerToBytes(const IrProgram& ir, uint8_t* out, size_t cap) { const Reg sOff = static_cast(ir.vregsUsed + 1); // base byte offset of the current light const Reg sCtr = static_cast(ir.vregsUsed + 2); // loop counter - HostAssembler a; + // Size the assembler's buffer to the CALLER's — `cap` is what the staging buffer holds, so + // the two can never disagree about how much a script may emit (they were separately + // constant, and a script that fit one overflowed the other). + HostAssembler a(cap); + // Bring a parked host argument back for the one instruction that reads it. + // The LAST reserved scratch index, derived from scratchTotal rather than hard-coded: the `+1` + // in scratchTotal above IS this register, so the reservation and the use cannot drift apart. + // A fixed offset sat OUTSIDE the reservation and only worked because the register maps happen + // to have spare entries above the high-water mark. + const Reg sHost = static_cast(ir.vregsUsed + scratchTotal - 1); + auto host = [&](VReg v) -> Reg { a.spillLoad(sHost, hostArgSlot(v)); return sHost; }; + // The frame must cover the parked HOST ARGUMENTS at the top as well as whatever the parser and + // the allocator claimed at the bottom — they are stored before any script code runs, so a frame + // sized only from `slots` would put them past its end. + a.prologue(slots > kTotalSlots ? slots : kTotalSlots); // An IR label id becomes an assembler label ON FIRST USE. Allocating the whole range up front // exhausts the assembler's fixed label table, and the inline ops (StoreElem's bounds guard, @@ -52,7 +90,9 @@ size_t lowerToBytes(const IrProgram& ir, uint8_t* out, size_t cap) { return labels[id]; }; - for (uint8_t i = 0; i < ir.count; i++) { + // uint16_t, matching IrProgram::count: the op array is sized to the script now, so a + // uint8_t counter wrapped at 256 ops and looped forever instead of emitting. + for (uint16_t i = 0; i < ir.count; i++) { const IrInst& op = ir.ops[i]; switch (op.op) { case IrOp::Const: a.movImm(reg(op.dst), op.imm); break; @@ -75,22 +115,34 @@ size_t lowerToBytes(const IrProgram& ir, uint8_t* out, size_t cap) { a.branchIf(Cond::Ne, labelFor(op.imm)); } break; - case IrOp::LoadCtrl: a.load8(reg(op.dst), reg(kArg4), op.imm); break; // dst = ctrls[imm] - case IrOp::Call: + case IrOp::LoadCtrl: a.load8(reg(op.dst), host(kArg4), op.imm); break; // dst = ctrls[imm] + // The allocator's two ops. `imm` is a slot INDEX; the assembler owns the frame layout. + case IrOp::Spill: a.spillStore(reg(op.a), static_cast(op.imm)); break; + case IrOp::Reload: a.spillLoad(reg(op.dst), static_cast(op.imm)); break; + case IrOp::Call: { + // The arguments are in consecutive frame slots starting at `imm`; hand the host + // their ADDRESS and their COUNT. Nothing is held in a register across the call, and + // how many arguments a builtin takes stops being a property of this instruction. if (!op.callFn) return 0; - a.call(reg(op.dst), reg(op.a), reg(op.b), reg(op.c), reinterpret_cast(op.callFn)); + const Reg argPtr = static_cast(ir.vregsUsed); + a.slotAddr(argPtr, static_cast(op.imm)); + const Reg argN = static_cast(ir.vregsUsed + 1); + a.movImm(argN, static_cast(op.b)); + a.call(reg(op.dst), argPtr, argN, + host(kArg4), reinterpret_cast(op.callFn)); break; + } case IrOp::Inline: switch (op.inlineOp) { case InlineOp::StoreElem: { // setRGB(index=a, r=b, g=c, b=d): bounds-guard, addr = index*cpl, store 3. Label skip = a.newLabel(); - a.cmp(reg(op.a), reg(kArg1)); // index vs nLights + a.cmp(reg(op.a), host(kArg1)); // index vs nLights a.branchIf(Cond::Hs, skip); // index >= nLights → skip - a.mulReg(sAddr, reg(op.a), reg(kArg2)); // addr = index * cpl - a.store8(reg(kArg0), sAddr, reg(op.b)); // buf[addr+0] = r - a.addImm(sAddr, sAddr, 1); a.store8(reg(kArg0), sAddr, reg(op.c)); // +1 = g - a.addImm(sAddr, sAddr, 1); a.store8(reg(kArg0), sAddr, reg(op.d)); // +2 = b + a.mulReg(sAddr, reg(op.a), host(kArg2)); // addr = index * cpl + a.store8(host(kArg0), sAddr, reg(op.b)); // buf[addr+0] = r + a.addImm(sAddr, sAddr, 1); a.store8(host(kArg0), sAddr, reg(op.c)); // +1 = g + a.addImm(sAddr, sAddr, 1); a.store8(host(kArg0), sAddr, reg(op.d)); // +2 = b a.bind(skip); break; } @@ -101,14 +153,14 @@ size_t lowerToBytes(const IrProgram& ir, uint8_t* out, size_t cap) { Label done = a.newLabel(), top = a.newLabel(); a.movImm(sOff, 0); // off = 0 a.movImm(sCtr, 0); // i = 0 - a.branchIfZero(reg(kArg1), done); // nLights==0 → skip + a.branchIfZero(host(kArg1), done); // nLights==0 → skip a.bind(top); - a.addImm(sAddr, sOff, 0); a.store8(reg(kArg0), sAddr, reg(op.a)); // buf[off+0]=r - a.addImm(sAddr, sOff, 1); a.store8(reg(kArg0), sAddr, reg(op.b)); // buf[off+1]=g - a.addImm(sAddr, sOff, 2); a.store8(reg(kArg0), sAddr, reg(op.c)); // buf[off+2]=b - a.addReg(sOff, sOff, reg(kArg2)); // off += cpl (general stride) + a.addImm(sAddr, sOff, 0); a.store8(host(kArg0), sAddr, reg(op.a)); // buf[off+0]=r + a.addImm(sAddr, sOff, 1); a.store8(host(kArg0), sAddr, reg(op.b)); // buf[off+1]=g + a.addImm(sAddr, sOff, 2); a.store8(host(kArg0), sAddr, reg(op.c)); // buf[off+2]=b + a.addReg(sOff, sOff, host(kArg2)); // off += cpl (general stride) a.addImm(sCtr, sCtr, 1); // i++ - a.cmp(sCtr, reg(kArg1)); + a.cmp(sCtr, host(kArg1)); a.branchIf(Cond::Lo, top); a.bind(done); break; @@ -118,7 +170,7 @@ size_t lowerToBytes(const IrProgram& ir, uint8_t* out, size_t cap) { default: break; // Loop/LoopEnd/Bounds/BoundsEnd are not emitted by the parser now } } - a.ret(); + a.epilogue(); // tears the spill frame down (if any), then ret a.finalize(); if (a.overflowed() || a.size() > cap) return 0; std::memcpy(out, a.bytes(), a.size()); @@ -127,7 +179,7 @@ size_t lowerToBytes(const IrProgram& ir, uint8_t* out, size_t cap) { #else // unsupported host ISA (e.g. Windows x64) — degrade: no codegen, compile fails cleanly. -size_t lowerToBytes(const IrProgram&, uint8_t*, size_t) { return 0; } +size_t lowerToBytes(IrProgram&, uint8_t*, size_t, const RegBudget*) { return 0; } #endif diff --git a/src/platform/desktop/platform_config.h b/src/platform/desktop/platform_config.h index e18de685..0ec77db8 100644 --- a/src/platform/desktop/platform_config.h +++ b/src/platform/desktop/platform_config.h @@ -69,7 +69,11 @@ constexpr bool hasWiFi = true; // Ethernet PHY config type — desktop has no Ethernet (ethInit() stubs to false), // but platform.h declares setEthConfig(const EthPinConfig&) for every platform, so // the type must exist here too. Mirror the esp32 struct; the desktop stub ignores it. -enum EthPhyType { ethNone = 0, ethLan8720 = 1, ethIp101 = 2, ethW5500 = 3 }; +// Kept in step with the ESP32 list (platform/esp32/platform_config.h) so core code can name a +// PHY type without knowing which platform it compiles for. `ethYt8531`/`ethOpeneth` never occur +// on desktop; they exist here so the NAMES resolve everywhere. +enum EthPhyType { ethNone = 0, ethLan8720 = 1, ethIp101 = 2, ethW5500 = 3, + ethYt8531 = 4, ethOpeneth = 5 }; struct EthPinConfig { int phyType; int phyAddr; int mdcGpio; int mdioGpio; int rstGpio; int rmiiClockGpio; bool rmiiClockExtIn; @@ -89,6 +93,11 @@ constexpr bool hasNamedNetInterfaces = true; // via the WiFi stubs (UdpSocket has a desktop implementation). constexpr bool hasNetwork = hasWiFi || hasEthernet; +// ethPhyIsFixed, true where the interface is a property of the PLATFORM rather than of the board, +// so a persisted or catalog-supplied PHY type must not override it. The ESP32 side is where that +// case is real (see its platform_config.h); desktop has no Ethernet to fix. +constexpr bool ethPhyIsFixed = false; + // Enough compute headroom for a per-pixel FLOAT algorithm — a raymarcher, a fractal, a feedback // loop that iterates per light. This is the ONE exception to the integer-only render-path rule in // coding-standards, and it is gated rather than assumed: an effect behind this constant is not @@ -132,7 +141,11 @@ constexpr bool hasImprov = false; // no `#if defined(__aarch64__)` outside src/platform/. A #define (not constexpr) so #include- // side test files can use it in `#if` — CLAUDE.md's `if constexpr` preference is for runtime // branches inside code, not preprocessor gating around whole TEST_CASEs. -#if defined(__aarch64__) +// MM_MOONLIVE_FORCE_NO_HOST_JIT makes an arm64 machine build as a backend-less one +// (build_desktop.py --no-jit). Every x86-64 desktop already is one, so a test that wrongly +// presumes a compile succeeds passes on an arm64 bench and fails only once CI runs it. The +// override lets that be caught before a push instead of after. +#if defined(__aarch64__) && !defined(MM_MOONLIVE_FORCE_NO_HOST_JIT) #define MM_MOONLIVE_HAS_HOST_JIT 1 #else #define MM_MOONLIVE_HAS_HOST_JIT 0 diff --git a/src/platform/desktop/platform_desktop.cpp b/src/platform/desktop/platform_desktop.cpp index 5d3fa4ed..eef662e4 100644 --- a/src/platform/desktop/platform_desktop.cpp +++ b/src/platform/desktop/platform_desktop.cpp @@ -140,6 +140,12 @@ uint32_t millis() MM_NONBLOCKING { ); } +// pthread_self() is the identity here; std::this_thread::get_id() is not convertible to an integer +// portably. The +1 guarantees a non-zero result so callers can treat 0 as "none". +uintptr_t currentThreadId() MM_NONBLOCKING { + return reinterpret_cast(pthread_self()) + 1; +} + uint32_t micros() MM_NONBLOCKING { auto now = std::chrono::steady_clock::now(); return static_cast( @@ -1603,6 +1609,10 @@ uint8_t* parlioWs2812Buffer(const ParlioWs2812Handle& h, uint8_t buffer) { size_t parlioWs2812BufferCapacity(const ParlioWs2812Handle& h) { return h.impl ? static_cast(h.impl)->capacity : 0; } +// The desktop host emulates the bus in ordinary memory, so there is no single-transfer ceiling to +// declare — 0 is the "no bound" contract dmaBudgetBytes() reads, matching every other host-side +// Parlio stub here. +size_t parlioMaxTransferBytes() { return 0; } bool parlioWs2812Transmit(ParlioWs2812Handle& h, uint8_t buffer, size_t bytes) { return h.impl && static_cast(h.impl)->transmit(buffer, bytes); } diff --git a/src/platform/esp32/moonlive_asm_riscv.cpp b/src/platform/esp32/moonlive_asm_riscv.cpp index 37ec23d9..910dfe6c 100644 --- a/src/platform/esp32/moonlive_asm_riscv.cpp +++ b/src/platform/esp32/moonlive_asm_riscv.cpp @@ -15,7 +15,12 @@ namespace mm::moonlive { // scratch (store8 address, call address build + result stash), not a vreg. static constexpr uint8_t kRvReg[kRegCount] = {10, 11, 12, 13, 14, 5, 6, 7, 28, 29, 30, 15, 16, 17}; -static uint8_t xr(Reg r) { return kRvReg[r]; } +// BOUNDS-CHECKED. The inline ops address their scratch as `vregsUsed + n`, so an index one past the +// map is reachable whenever the reservation and the map disagree — and an out-of-bounds read returns +// whatever byte follows the array, making the emitted instruction name a register chosen by +// accident. Clamping turns that into a wrong-but-safe register instead of undefined behaviour; the +// static_assert below and the lowerer's reservation are what stop it happening at all. +static uint8_t xr(Reg r) { return kRvReg[r < kRegCount ? r : kRegCount - 1]; } // t6 is the ONLY caller-saved register outside kRvReg, so it is the only safe scratch: every other // free register is callee-saved (s0/s1, s6..s11) and would have to be preserved. Both uses below are // transient — store8 consumes it within two instructions, and call() finishes with it before any @@ -35,7 +40,7 @@ constexpr bool rvScratchOutsideMap() { static_assert(rvScratchOutsideMap(), "a scratch register is also a vreg — calls will corrupt it"); void RiscvAssembler::emit32(uint32_t w) { - if (len_ + 4 > kCap) { overflow_ = true; return; } + if (!buf_ || len_ + 4 > kCap) { overflow_ = true; return; } buf_[len_++] = uint8_t(w); buf_[len_++] = uint8_t(w >> 8); buf_[len_++] = uint8_t(w >> 16); buf_[len_++] = uint8_t(w >> 24); } @@ -83,6 +88,58 @@ static uint32_t encBranch(uint8_t rs1, uint8_t rs2, uint8_t f3, int32_t off) { (f3 << 12) | (((o >> 1) & 0xf) << 8) | (((o >> 11) & 1) << 7) | 0x63; } +// --- the call frame: the register allocator's overflow storage --------------------------------- +// +// s0/fp (x8) is the standard RISC-V frame pointer: callee-saved, outside kRvReg and outside the t6 +// scratch, so nothing this backend emits disturbs it — and the routine saves and restores it, which +// is what makes using a callee-saved register legal here (the reason the map itself stops at the +// caller-saved set, see moonlive_asm_riscv.h). +// +// Layout, sp growing down: sp = s0 - frameBytes, saved s0 at [s0-4], slot n at [s0 - frameBytes + n*4]. +static constexpr uint8_t kFramePtr = 8; // s0/fp + +void RiscvAssembler::prologue(uint8_t slots) { + if (slots == 0) return; // no spilling: no frame, no cost + if (slots > kMaxSpillSlots) { overflow_ = true; return; } + // 16-byte aligned, as the RISC-V calling convention requires of sp at a call boundary — an + // unaligned sp is the kind of thing that survives every test and faults inside a callee. + const uint16_t bytes = static_cast((8 + slots * 4 + 15) & ~15); + frameBytes_ = bytes; + emit32(encAddi(2, 2, -int32_t(bytes))); // addi sp, sp, -bytes + emit32(encSw(kFramePtr, 2, bytes - 4)); // sw s0, bytes-4(sp) + emit32(encAddi(kFramePtr, 2, bytes)); // addi s0, sp, bytes (s0 = the caller's sp) +} +void RiscvAssembler::epilogue() { + if (frameBytes_ != 0) { + emit32(encLw(kFramePtr, 2, frameBytes_ - 4)); // lw s0, bytes-4(sp) + emit32(encAddi(2, 2, frameBytes_)); // addi sp, sp, bytes + } + ret(); +} +// sw/lw against s0. The offset is NEGATIVE (slots live below the frame pointer), which encAddi's +// 12-bit signed immediate and the S/I-type immediates handle directly: the whole slot file is a few +// hundred bytes, far inside the +/-2048 the field reaches. +void RiscvAssembler::spillStore(Reg r, uint8_t slot) { + // No frame means prologue() bailed (slots == 0, or past kMaxSpillSlots). Emitting anyway would + // address s0 + slot*4 — ABOVE the frame pointer, i.e. the CALLER's stack. Refuse instead. + if (slot >= kMaxSpillSlots || frameBytes_ == 0) { overflow_ = true; return; } + emit32(encSw(xr(r), kFramePtr, -int32_t(frameBytes_) + slot * 4)); +} +void RiscvAssembler::spillLoad(Reg r, uint8_t slot) { + if (slot >= kMaxSpillSlots || frameBytes_ == 0) { overflow_ = true; return; } + emit32(encLw(xr(r), kFramePtr, -int32_t(frameBytes_) + slot * 4)); +} + +// The ADDRESS of a frame slot, for a host call that reads its arguments from the frame. The slots +// already hold the arguments; the call passes where they start rather than the values, which is what +// makes the number of arguments a memory question instead of a register one. +void RiscvAssembler::slotAddr(Reg d, uint8_t slot) { + if (slot >= kMaxSpillSlots || frameBytes_ == 0) { overflow_ = true; return; } + // Against s0, the SAME base spillStore/spillLoad use. Computing it from sp instead lands + // frameBytes_ below the frame (s0 == sp + frameBytes_), so the callee writes off the end of it. + emit32(encAddi(xr(d), kFramePtr, -int32_t(frameBytes_) + slot * 4)); // addi xD, s0, off +} + void RiscvAssembler::movImm(Reg d, int32_t imm) { // addi sign-extends a 12-bit immediate, so it alone covers only -2048..2047. For wider // constants (a uint16 like 65535) materialise the full value with lui (high 20 bits) + addi @@ -169,6 +226,9 @@ void RiscvAssembler::call(Reg d, Reg a, Reg b, Reg c, const void* fn) { void RiscvAssembler::ret() { emit32(0x00008067u); } // ret = jalr x0, ra, 0 void RiscvAssembler::patchBranches() { + // Nothing was emitted if the buffer never allocated, so there is nothing to patch — + // stated rather than left to the reader to derive from fixupCount_ being 0. + if (!buf_) return; for (uint8_t i = 0; i < fixupCount_; i++) { const Fixup& f = fixups_[i]; if (labelPos_[f.label] < 0) continue; // unbound label — leave as-is (overflow_ already failed the compile) diff --git a/src/platform/esp32/moonlive_asm_riscv.h b/src/platform/esp32/moonlive_asm_riscv.h index fd340fcf..6245b646 100644 --- a/src/platform/esp32/moonlive_asm_riscv.h +++ b/src/platform/esp32/moonlive_asm_riscv.h @@ -1,5 +1,7 @@ #pragma once +#include "platform/platform.h" // alloc/free — the emit buffer is heap, not stack + #include "core/moonlive/MoonLiveIr.h" // kCodeCap — one cap for the staging buffer and every backend #include @@ -32,12 +34,38 @@ enum class Cond : uint8_t { Lo /* unsigned < */, Hs /* unsigned >= */ }; class RiscvAssembler { public: + // Owns buf_ (see below). Freed here, copying deleted — an emitter that was copied + // would double-free the buffer it emits into. + ~RiscvAssembler() { platform::free(buf_); } + /// `cap` is the code buffer's size, chosen per SCRIPT by the caller (codeCapFor) rather + /// than a shared constant — the backends differ by up to 1.9x on identical source, so one + /// number cannot fit them all. Defaults to the sanity bound for callers that emit a fixed + /// blob (emitFill) and have no token count to size from. + explicit RiscvAssembler(size_t cap = kCodeCap) + : kCap(cap), buf_(static_cast(platform::alloc(cap))) {} + RiscvAssembler(const RiscvAssembler&) = delete; + RiscvAssembler& operator=(const RiscvAssembler&) = delete; + void finalize() { patchBranches(); } const uint8_t* bytes() const { return buf_; } size_t size() const { return len_; } bool overflowed() const { return overflow_; } - void prologue() {} // RV needs no fixed prologue (sp managed in call()) + // --- the call frame --- + // The register allocator's overflow storage (core/moonlive/MoonLiveSpill.h). RV32 had no frame + // at all outside call(), so a spilling program is the first thing here that needs one: prologue + // opens it and parks s0 (the standard frame pointer) at its top, epilogue tears it down. Slots + // are addressed from s0, NOT sp, because call() moves sp by 80 bytes around every host call — + // sp-relative offsets would be wrong for its duration, and reading a spilled value after a + // random16() is the ordinary case. It is also the layout a nested or recursive script function + // needs: one s0 per activation. + // slots == 0 emits nothing, so a non-spilling script keeps today's zero-instruction entry. + void prologue(uint8_t slots = 0); + void spillStore(Reg r, uint8_t slot); + void spillLoad(Reg r, uint8_t slot); + void slotAddr(Reg d, uint8_t slot); // d = &frame[slot] — a call's argument block + static constexpr uint8_t kMaxSpillSlots = kTotalSlots; // parser/allocator range + the parked host args + Label newLabel(); void bind(Label l); @@ -52,21 +80,30 @@ class RiscvAssembler { void branchGeU(Reg a, Reg b, Label l); // bgeu a, b, l void branchNe(Reg a, Reg b, Label l); // bne a, b, l void call(Reg d, Reg a, Reg b, Reg c, const void* fn); // standard call to a host built-in - void epilogue() { ret(); } + void epilogue(); // undo prologue's frame (if any), then ret void ret(); private: - // The emitted-code buffer, sized by the engine's shared cap (kCodeCap). - static constexpr size_t kCap = kCodeCap; + // The emitted-code buffer's size, fixed for this object's life but chosen per script. + const size_t kCap; static constexpr uint8_t kMaxLabels = 16; static constexpr uint8_t kMaxFixups = 32; void emit32(uint32_t w); void addFixup(size_t at, Label label); // enqueue a branch fixup (bounds-checked) - uint8_t buf_[kCap] = {}; + // HEAP, not a member array: the assembler is a stack local in lowerToBytes, so a kCap-sized + // member put 2 KB on the compile chain's stack — on top of the staging buffer and the parser + // frames. On a classic ESP32 that overflowed the task and faulted inside _xt_context_save + // (the plan named this: "buf_[kCap] inside the assembler, itself a stack local"). The buffer is + // scratch that ends in a memcpy to the caller's output, so nothing outlives the object. + uint8_t* buf_; size_t len_ = 0; bool overflow_ = false; + // Frame size in bytes, 0 when no prologue was emitted. epilogue() reads it, so the teardown can + // never disagree with the setup about how far sp moved — a mismatch there returns to a corrupted + // stack, which looks like anything except the compiler bug it is. + uint16_t frameBytes_ = 0; int32_t labelPos_[kMaxLabels]; uint8_t labelCount_ = 0; diff --git a/src/platform/esp32/moonlive_asm_xtensa.cpp b/src/platform/esp32/moonlive_asm_xtensa.cpp index d8871edb..e6f507a8 100644 --- a/src/platform/esp32/moonlive_asm_xtensa.cpp +++ b/src/platform/esp32/moonlive_asm_xtensa.cpp @@ -17,22 +17,38 @@ namespace mm::moonlive { // R0..R3 → a2..a5 (the windowed-ABI args); R4..R11 → a6..a11, a14, a15. a12/a13 are internal // scratch (store8 address, branchIfZero zero-reg, call result stash), so not in the pool. -static constexpr uint8_t kXtReg[kRegCount] = {2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 14, 15}; -static uint8_t ar(Reg r) { return kXtReg[r]; } +static constexpr uint8_t kXtReg[kRegCount] = {2, 3, 4, 5, 6, 7, 8, 9, 10, 11}; +// Map a vreg to its machine register. BOUNDS-CHECKED: the inline ops address their scratch as +// `vregsUsed + n`, which walks off the end of the map when a program uses the whole file — and an +// out-of-bounds read here returns whatever byte follows the array, so the emitted instruction names +// a register chosen by accident. That produced a12 (a call8 window register AND this assembler's own +// scratch) in a program that had no business touching it. Clamped to the last real entry so a +// mistake is a wrong-but-safe register rather than undefined behaviour; the static_assert below and +// the reservation in the lowerer are what keep it from happening at all. +static uint8_t ar(Reg r) { return kXtReg[r < kRegCount ? r : kRegCount - 1]; } + +// The map, for the codegen test to assert against. Exposed rather than copied into the test, so the +// property being checked cannot drift away from the map it is about. +const uint8_t* xtRegMap(uint8_t& count) { count = kRegCount; return kXtReg; } // A scratch register that is ALSO a vreg silently corrupts values — see the RISC-V backend, where // kScratchFn aliased vreg R12 and every call returned a stale value. Checked here so the map can // never grow over a scratch. constexpr bool xtScratchOutsideMap() { - constexpr uint8_t scratch[] = {12, 13}; + // a12/a13 are call() scratch; a14/a15 carry the retw.n return linkage of our own `entry` frame. + // Both classes are fatal as vregs, and the second one only faults once a script makes a CALL — + // which is why it survived every effect and killed every layout. + constexpr uint8_t scratch[] = {12, 13, 14, 15}; for (uint8_t r : kXtReg) for (uint8_t s : scratch) if (r == s) return false; return true; } -static_assert(xtScratchOutsideMap(), "a scratch register is also a vreg — calls will corrupt it"); +static_assert(xtScratchOutsideMap(), "a scratch or window register is also a vreg — calls corrupt it"); void XtensaAssembler::emit(const uint8_t* p, size_t n) { - if (len_ + n > kCap) { overflow_ = true; return; } + // !buf_ covers a failed allocation: the compile then fails cleanly at overflowed() instead + // of writing through a null pointer. + if (!buf_ || len_ + n > kCap) { overflow_ = true; return; } std::memcpy(buf_ + len_, p, n); len_ += n; } void XtensaAssembler::emit2(uint16_t w) { @@ -42,11 +58,87 @@ void XtensaAssembler::emit3(uint32_t w) { const uint8_t b[3] = {uint8_t(w), uint8_t(w >> 8), uint8_t(w >> 16)}; emit(b, 3); } -// entry a1, 48 — a 48-byte frame leaves room for the call8 window rotation (a routine with no -// call would be fine with 32, but 48 is harmless and lets any program call a built-in). -void XtensaAssembler::prologue() { emit3(0x006136u); } // entry a1, 48 +// entry a1, N — a 48-byte frame leaves room for the call8 window rotation (a routine with no +// call would be fine with 32, but 48 is harmless and lets any program call a built-in). call() uses +// bytes 16..39 of it, so the register allocator's spill slots start at 48 and the frame simply grows +// to hold them: on this ISA the whole-routine frame already exists, so spilling costs a bigger +// immediate on ONE instruction and nothing else. a1 is the frame pointer, and the windowed ABI +// preserves it across callx8 — which is why a slot read after a host call still finds its value, and +// why this addressing carries over unchanged when script functions start nesting frames. +// One word inside call()'s own save area for the return value: offsets 16..28 hold the saved +// vregs, 32 is free, and a register cannot hold a result across a window rotation. +static constexpr uint8_t kResultSlot = 8; // byte offset 32 +static constexpr uint16_t kFrameBase = 48; // first byte past the bytes call() reserves +static constexpr uint16_t kSlotStride = 4; + +// The SAVE AREAS the windowed ABI reserves at the TOP of every call8-making routine's frame. +// The window overflow handler (_WindowOverflow8, esp-idf components/xtensa/xtensa_vectors.S) +// writes two 16-byte bands there, addressed from two different stack pointers: +// top 16 bytes an OLDER frame's a0..a3, incl. its return address (s32e aN, a9, -16..-4) +// next 16 bytes this routine's OWN a4..a7 (s32e aN, a0, -32..-20) +// So the top 32 bytes are the hardware's, never ours. GCC obeys the same rule: every +// call8-making function it compiles gets a frame of locals plus exactly 32. +// +// Both bands were bench-found separately. Reserving nothing put the parked host arguments under +// the a0..a3 band: any script at all reset the board with IllegalInstruction and a DATA address +// in A0, the return address overwritten. Reserving only 16 left the highest slot (the parked +// arena pointer) under the a4..a7 band, which is written on ANY interrupt that lands while a +// host call is in flight. Fast leaf calls (random16) rarely coincided with one and worked; the +// deep libm chains of plasma (sin, beat) gave every tick a wide window to hit, and the arena +// came back as an expression temp: LoadProhibited, A11=0, EXCVADDR=1. The corruption is spatial, +// not timed, so no code sequence can dodge it; only the layout can. Never seen on RISC-V or +// arm64: no register window, no hardware-owned frame bytes. +// +// The size follows the WIDEST call this assembler emits, and is derived from that instruction +// below rather than written down twice: a wider call rotates the window further and its extra +// save area grows to match (call4 spills nothing extra, call8 spills a4..a7, call12 also +// a8..a11), so the reserve is 16 + 16 * (windows rotated - 1). A future callx12 with a hand- +// held 32 here would put the top slot back under the spill band and resurrect this bug with +// every static check still green. +static constexpr uint32_t kCallxOpcode = 0x0000e0u; // callx8 a8, the one call we emit + +/// Bytes the window-overflow handler may write at the top of a frame whose widest call is the +/// given CALLX. The window increment is bits 4..5 of the opcode (1 = call4, 2 = call8, +/// 3 = call12), and each step widens the rotation by four registers, so each adds another +/// 16-byte quad above the base save area every `entry` frame already owns. call0 (increment 0) +/// is not a case here: it makes no window rotation and this assembler cannot emit it. +static constexpr uint32_t windowSaveReserveFor(uint32_t callxOpcode) { + return 16u * ((callxOpcode >> 4) & 0x3u); +} +static constexpr uint32_t kWindowSaveReserve = windowSaveReserveFor(kCallxOpcode); +static_assert(kWindowSaveReserve == 32, + "callx width changed: the frame reserve moved with it, so re-check the frame " + "layout and MM_ISA_RESERVED_TOP in the codegen test before accepting this"); + +// ENTRY is a BRI12-format instruction: op0=6, n=3, s=the base register, and the 12-bit immediate at +// bits 12..23 counts EIGHT-byte units. `entry a1, 48` is therefore 0x006136. +void XtensaAssembler::prologue(uint8_t slots) { + if (slots > kMaxSpillSlots) { overflow_ = true; return; } + // Rounded up to 8 because the immediate counts 8-byte units; the ABI additionally wants the + // frame 16-byte aligned, and 48 + a multiple of 16 keeps that. kWindowSaveReserve is added ON + // TOP of the slots so the highest slot still ends below the hardware's two save bands. + const uint32_t bytes = + (kFrameBase + uint32_t(slots) * kSlotStride + kWindowSaveReserve + 15u) & ~15u; + emit3(0x000136u | ((bytes / 8u) << 12)); +} void XtensaAssembler::epilogue() { emit2(0xf01du); } // retw.n +// s32i/l32i aR, a1, #off — the offset field counts 4-byte words, so a slot index maps straight onto +// it. No teardown counterpart: `entry`'s frame is released by retw.n, so unlike the RISC-V and arm64 +// backends there is nothing for an epilogue to undo. +void XtensaAssembler::spillStore(Reg r, uint8_t slot) { + if (slot >= kMaxSpillSlots) { overflow_ = true; return; } + const uint8_t off4 = static_cast((kFrameBase + slot * kSlotStride) / 4); + const uint8_t b[3] = {uint8_t((ar(r) << 4) | 0x2), 0x61, off4}; + emit(b, 3); +} +void XtensaAssembler::spillLoad(Reg r, uint8_t slot) { + if (slot >= kMaxSpillSlots) { overflow_ = true; return; } + const uint8_t off4 = static_cast((kFrameBase + slot * kSlotStride) / 4); + const uint8_t b[3] = {uint8_t((ar(r) << 4) | 0x2), 0x21, off4}; + emit(b, 3); +} + Label XtensaAssembler::newLabel() { if (labelCount_ == 0) for (auto& p : labelPos_) p = -1; if (labelCount_ >= kMaxLabels) { overflow_ = true; return 0; } // same overflow signal as emit @@ -64,6 +156,16 @@ void XtensaAssembler::addFixup(size_t at, Label label) { // 65535) is built as hi8<<8 | lo8: movi aD,hi8 ; slli aD,aD,8 ; movi a13,lo8 ; add.n aD,aD,a13. // a13 is the assembler's reserved scratch (also kZero in branchIfZero); it holds no live vreg. // Single movi for the common 0..255 case. Without this, Const values >255 truncate to 8 bits. +// The ADDRESS of a frame slot, for a host call that reads its arguments from the frame. The slots +// already hold the arguments; the call passes where they start rather than the values, which is what +// makes the number of arguments a memory question instead of a register one. +void XtensaAssembler::slotAddr(Reg d, uint8_t slot) { + if (slot >= kMaxSpillSlots) { overflow_ = true; return; } + const uint32_t off = kFrameBase + uint32_t(slot) * kSlotStride; + const uint8_t b[3] = {uint8_t((ar(d) << 4) | 0x2), uint8_t(0xc0 | 1), uint8_t(off)}; + emit(b, 3); // addi aD, a1, #off +} + void XtensaAssembler::movImm(Reg d, int32_t imm) { const uint8_t dr = ar(d); // The wide `movi` field is 12-bit SIGNED (-2048..2047), which is the only encoding here that can @@ -138,22 +240,41 @@ void XtensaAssembler::branchIfZero(Reg a, Label l) { static constexpr uint8_t kZero = 13; // a13 const uint8_t mv[3] = {uint8_t((kZero << 4) | 0x2), 0xa0, 0x00}; // movi a13, 0 emit(mv, 3); - addFixup(len_, l); - const uint8_t br[3] = {uint8_t((ar(a) << 4) | 0x7), uint8_t((0xb << 4) | kZero), 0x00}; // bgeu a13, a + // Same relaxed form as the other branches: bltu a13, a, +3 (the inverse of bgeu) over a `j`. + const uint8_t br[3] = {uint8_t((ar(a) << 4) | 0x7), uint8_t((0x3 << 4) | kZero), 0x02}; emit(br, 3); -} -// bgeu aA, aB, l (skip if a >= b, unsigned) -void XtensaAssembler::branchGeU(Reg a, Reg b, Label l) { addFixup(len_, l); - const uint8_t bytes[3] = {uint8_t((ar(b) << 4) | 0x7), uint8_t((0xb << 4) | ar(a)), 0x00}; - emit(bytes, 3); + const uint8_t j[3] = {0x06, 0x00, 0x00}; + emit(j, 3); } -// bne aA, aB, l -void XtensaAssembler::branchNe(Reg a, Reg b, Label l) { +// A conditional branch to `l`, emitted as the INVERTED condition over an unconditional jump: +// +// b aA, aB, +3 ; skip the jump when the branch is NOT taken +// j l ; 18-bit displacement — reaches anywhere in a script +// +// Xtensa's conditional branches carry a single SIGNED BYTE of displacement (±127), which a loop body +// outgrows easily once spill traffic is in it — grid.mlv needed 177. Truncating that silently +// retargets the branch into the middle of the program, so the choice is relax or refuse. This is the +// textbook relaxation every compiler does (GCC and LLVM emit the short form and rewrite the ones that +// do not fit); the fixed six-byte form skips the iterate-to-convergence step, which is worth a few +// bytes per branch on a cold path in exchange for not having to reason about shifting offsets. +// The 3-byte `j` keeps its own fixup, and the inverted branch's +3 is already correct as emitted. +void XtensaAssembler::branchRelaxed(uint8_t condNibble, Reg a, Reg b, Label l) { + // The inverted condition, skipping the 3-byte `j` that follows. Xtensa branch displacements are + // relative to PC+4 (the same rule patchBranches uses), so clearing a 3-byte instruction is +2. + // bne(0x9) <-> beq(0x1); bgeu(0xb) <-> bltu(0x3). + const uint8_t inv = condNibble == 0x9 ? 0x1 : condNibble == 0x1 ? 0x9 + : condNibble == 0xb ? 0x3 : 0xb; + const uint8_t br[3] = {uint8_t((ar(b) << 4) | 0x7), uint8_t((inv << 4) | ar(a)), 0x02}; + emit(br, 3); addFixup(len_, l); - const uint8_t bytes[3] = {uint8_t((ar(b) << 4) | 0x7), uint8_t((0x9 << 4) | ar(a)), 0x00}; - emit(bytes, 3); + const uint8_t j[3] = {0x06, 0x00, 0x00}; // j — the 18-bit offset is patched in + emit(j, 3); } +// bgeu aA, aB, l (skip if a >= b, unsigned) +void XtensaAssembler::branchGeU(Reg a, Reg b, Label l) { branchRelaxed(0xb, a, b, l); } +// bne aA, aB, l +void XtensaAssembler::branchNe(Reg a, Reg b, Label l) { branchRelaxed(0x9, a, b, l); } // Windowed call to a host built-in: d = fn(a). CALL8 rotates the window by 8, so the arg goes // in a10 and the result returns in a10. The caller's a2..a7 are preserved by the window for @@ -165,15 +286,12 @@ void XtensaAssembler::branchNe(Reg a, Reg b, Label l) { // byte-by-byte (movi/slli/add) — no l32r literal pool. void XtensaAssembler::call(Reg d, Reg a, Reg b, Reg c, const void* fn) { // Save the rotate-out scratch a8, a9, a11 (a10 will carry arg→result). - auto s32i = [&](uint8_t r, uint8_t off4){ const uint8_t b[3]={uint8_t((r<<4)|2),0x61,off4}; emit(b,3); }; - auto l32i = [&](uint8_t r, uint8_t off4){ const uint8_t b[3]={uint8_t((r<<4)|2),0x21,off4}; emit(b,3); }; + auto s32i = [&](uint8_t r, uint8_t off4){ const uint8_t enc[3]={uint8_t((r<<4)|2),0x61,off4}; emit(enc,3); }; + auto l32i = [&](uint8_t r, uint8_t off4){ const uint8_t enc[3]={uint8_t((r<<4)|2),0x21,off4}; emit(enc,3); }; s32i(8, 4); s32i(9, 5); s32i(11, 7); // [a1+16]=a8, [a1+20]=a9, [a1+28]=a11 - // a14/a15 are vregs R10/R11 (kXtReg), and CALL8 rotates the window out from under them — so a - // value live across a call in either was destroyed. Reachable on the SHIPPED default: grid.mlv - // is a nested loop (11 vregs, so R10 is in use) whose body calls addLight. The entry frame is 48 - // bytes and call() uses 16/20/24/28, so 32/36 are free. - s32i(14, 8); s32i(15, 9); // [a1+32]=a14, [a1+36]=a15 - + // a14/a15 are deliberately NOT saved here, because they are no longer vregs (kXtReg): they carry + // this routine's own return linkage for retw.n, so writing saved copies back into them after the + // call is what broke every scripted layout. See the Reg enum for the failure that produced. s32i(10, 6); // [a1+24]=a10 — a vreg (R8) call8 rotates out // The three args into a10/a11/a12 — call8 shifts the window by 8, so the callee reads them as @@ -188,29 +306,56 @@ void XtensaAssembler::call(Reg d, Reg a, Reg b, Reg c, const void* fn) { emit2(uint16_t((uint32_t(ar(b)) << 8) | (11 << 4) | 0xd)); // mov a11, argB emit2(uint16_t((13u << 8) | (10 << 4) | 0xd)); // mov a10, a13 + // The fn address is assembled a byte at a time, which needs a TEMPORARY alongside a8, and the + // choice of temporary is load-bearing, because call8 rotates the window: caller a8..a15 become + // callee a0..a7. So caller a9 IS THE CALLEE'S STACK POINTER (a1). Building the address through a9 + // handed the callee a fragment of a function pointer as its sp; its own `entry a1, N` then sized a + // frame from garbage and stored through it, landing anywhere in memory, including over this + // routine's own parked host arguments. That is the null arena pointer a script later read + // (LoadProhibited at EXCVADDR 0x1, arena register zero), and it only bit scripts whose loop makes + // a call AND reads a system variable, because both halves have to be present to notice. + // + // a13 is the safe temporary: it maps to callee a5, the fourth argument slot, which a three-argument + // host function never reads. It is already this assembler's kTmp for exactly this reason. uint32_t addr = static_cast(reinterpret_cast(fn)); - auto moviA8 = [&](uint8_t v){ const uint8_t b[3]={0x82,0xa0,v}; emit(b,3); }; - auto moviA9 = [&](uint8_t v){ const uint8_t b[3]={0x92,0xa0,v}; emit(b,3); }; - auto slliA8 = [&]{ const uint8_t b[3]={0x80,0x88,0x11}; emit(b,3); }; - auto addA8A9 = [&]{ emit2(0x889au); }; + auto moviA8 = [&](uint8_t v){ const uint8_t enc[3]={0x82,0xa0,v}; emit(enc,3); }; + auto moviA13 = [&](uint8_t v){ const uint8_t enc[3]={0xd2,0xa0,v}; emit(enc,3); }; + auto slliA8 = [&]{ const uint8_t enc[3]={0x80,0x88,0x11}; emit(enc,3); }; + auto addA8A13 = [&]{ emit2(0x88dau); }; // add.n a8, a8, a13 = (8<<12)|(8<<8)|(13<<4)|0xa moviA8(uint8_t(addr >> 24)); - slliA8(); moviA9(uint8_t(addr >> 16)); addA8A9(); - slliA8(); moviA9(uint8_t(addr >> 8)); addA8A9(); - slliA8(); moviA9(uint8_t(addr)); addA8A9(); - emit3(0x0000e0u | (8u << 8)); // callx8 a8 → result in a10 - // stash result (a10) in a12 (not in the saved set), restore a8/a9/a11, then dst = a12. - emit2(uint16_t((10u << 8) | (12u << 4) | 0xd)); // mov a12, a10 + slliA8(); moviA13(uint8_t(addr >> 16)); addA8A13(); + slliA8(); moviA13(uint8_t(addr >> 8)); addA8A13(); + slliA8(); moviA13(uint8_t(addr)); addA8A13(); + emit3(kCallxOpcode | (8u << 8)); // callx8 a8 → result in a10 + // Park the result in the FRAME, not in a register. + // + // It used to be stashed in a12 — but call8 rotates the window by eight, so the callee's a4 IS + // our a12: the callee overwrites the stash with its own second argument while it runs, and the + // "result" moved to the destination afterwards is whatever the callee happened to leave there. + // a12/a13 are safe as scratch only BEFORE the call, never across it. Measured: A0 = 0x100 in the + // crash dump — nLights, a script value that reached the return-address register this way. + // Slot kResultSlot sits in the bytes call() already owns, so this costs no extra frame. + s32i(10, kResultSlot); // [a1+kResultSlot*4] = result l32i(8, 4); l32i(9, 5); l32i(10, 6); l32i(11, 7); - l32i(14, 8); l32i(15, 9); - emit2(uint16_t((12u << 8) | (uint32_t(ar(d)) << 4) | 0xd)); // mov aDst, a12 + l32i(ar(d), kResultSlot); // dst = the parked result } void XtensaAssembler::patchBranches() { + // Nothing was emitted if the buffer never allocated, so there is nothing to patch — + // stated rather than left to the reader to derive from fixupCount_ being 0. + if (!buf_) return; for (uint8_t i = 0; i < fixupCount_; i++) { const Fixup& f = fixups_[i]; if (labelPos_[f.label] < 0) continue; // unbound label — leave as-is (overflow_ already failed the compile) - int32_t off = labelPos_[f.label] - (static_cast(f.at) + 4); // verified rule - buf_[f.at + 2] = static_cast(off & 0xff); // offset byte at +2 + // Every fixup now points at a `j`, whose displacement is relative to the byte AFTER the + // instruction and occupies bits 6..23 — eighteen signed bits, so it reaches any script the + // code buffer can hold. Still range-checked: refusing beats silently retargeting a jump. + const int32_t off = labelPos_[f.label] - (static_cast(f.at) + 4); + if (off < -131072 || off > 131071) { overflow_ = true; return; } + const uint32_t enc = 0x06u | ((static_cast(off) & 0x3ffffu) << 6); + buf_[f.at + 0] = static_cast(enc); + buf_[f.at + 1] = static_cast(enc >> 8); + buf_[f.at + 2] = static_cast(enc >> 16); } } diff --git a/src/platform/esp32/moonlive_asm_xtensa.h b/src/platform/esp32/moonlive_asm_xtensa.h index 5b79f125..b7ee1b0f 100644 --- a/src/platform/esp32/moonlive_asm_xtensa.h +++ b/src/platform/esp32/moonlive_asm_xtensa.h @@ -1,5 +1,7 @@ #pragma once +#include "platform/platform.h" // alloc/free — the emit buffer is heap, not stack + #include "core/moonlive/MoonLiveIr.h" // kCodeCap — one cap for the staging buffer and every backend #include @@ -16,18 +18,52 @@ namespace mm::moonlive { -enum Reg : uint8_t { R0 = 0, R1, R2, R3, R4, R5, R6, R7, R8, R9, R10, R11, kRegCount }; +// Ten vregs, mapping to a2..a11. NOT a14/a15: with the windowed ABI a routine that opened its frame +// with `entry` returns through `retw.n`, which reads the caller's linkage out of the TOP of the +// window — so a12..a15 are not general registers here, they are the return path. Using a14/a15 as +// vregs (and restoring saved copies into them after a callx8) corrupted that linkage, and `retw.n` +// then returned to a garbage address: `Guru Meditation (IllegalInstruction)` the moment a scripted +// LAYOUT ran, because addLight is the call that made the window rotate. a12/a13 stay scratch. +enum Reg : uint8_t { R0 = 0, R1, R2, R3, R4, R5, R6, R7, R8, R9, kRegCount }; using Label = uint8_t; enum class Cond : uint8_t { Lo /* unsigned < */, Hs /* unsigned >= */ }; +/// The vreg → machine-register map, for the device-codegen test. a2..a11 only: a12/a13 are call +/// scratch and the store8 address register, and a14/a15 carry the routine's own retw.n linkage. +const uint8_t* xtRegMap(uint8_t& count); + class XtensaAssembler { public: + // Owns buf_ (see below). Freed here, copying deleted — an emitter that was copied + // would double-free the buffer it emits into. + ~XtensaAssembler() { platform::free(buf_); } + /// `cap` is the code buffer's size, chosen per SCRIPT by the caller (codeCapFor) rather + /// than a shared constant — the backends differ by up to 1.9x on identical source, so one + /// number cannot fit them all. Defaults to the sanity bound for callers that emit a fixed + /// blob (emitFill) and have no token count to size from. + explicit XtensaAssembler(size_t cap = kCodeCap) + : kCap(cap), buf_(static_cast(platform::alloc(cap))) {} + XtensaAssembler(const XtensaAssembler&) = delete; + XtensaAssembler& operator=(const XtensaAssembler&) = delete; + void finalize() { patchBranches(); } const uint8_t* bytes() const { return buf_; } size_t size() const { return len_; } bool overflowed() const { return overflow_; } - void prologue(); // entry a1, 48 (must be the first instruction) + // --- the call frame --- + // The register allocator's overflow storage (core/moonlive/MoonLiveSpill.h). Xtensa already has + // a whole-routine frame from `entry a1, N`; prologue(slots) simply widens N to carry the spill + // slots above the bytes call() uses, so a spilling script costs one larger immediate and no + // extra instruction. slots == 0 keeps the frame exactly as it was, so nothing changes for a + // script that did not spill. Slots are addressed from a1, which the windowed ABI preserves + // across callx8 — the same property a nested or recursive script function will rely on. + void prologue(uint8_t slots = 0); // entry a1, N (must be the first instruction) + void spillStore(Reg r, uint8_t slot); + void spillLoad(Reg r, uint8_t slot); + void slotAddr(Reg d, uint8_t slot); // d = &frame[slot] — a call's argument block + static constexpr uint8_t kMaxSpillSlots = kTotalSlots; // parser/allocator range + the parked host args + Label newLabel(); void bind(Label l); @@ -45,8 +81,8 @@ class XtensaAssembler { void epilogue(); // retw.n private: - // The emitted-code buffer, sized by the engine's shared cap (kCodeCap). - static constexpr size_t kCap = kCodeCap; + // The emitted-code buffer's size, fixed for this object's life but chosen per script. + const size_t kCap; static constexpr uint8_t kMaxLabels = 16; static constexpr uint8_t kMaxFixups = 32; @@ -55,7 +91,12 @@ class XtensaAssembler { void emit3(uint32_t w); // wide (24-bit) instruction void addFixup(size_t at, Label label); // enqueue a branch fixup (bounds-checked) - uint8_t buf_[kCap] = {}; + // HEAP, not a member array: the assembler is a stack local in lowerToBytes, so a kCap-sized + // member put 2 KB on the compile chain's stack — on top of the staging buffer and the parser + // frames. On a classic ESP32 that overflowed the task and faulted inside _xt_context_save + // (the plan named this: "buf_[kCap] inside the assembler, itself a stack local"). The buffer is + // scratch that ends in a memcpy to the caller's output, so nothing outlives the object. + uint8_t* buf_; size_t len_ = 0; bool overflow_ = false; @@ -65,6 +106,10 @@ class XtensaAssembler { Fixup fixups_[kMaxFixups]; uint8_t fixupCount_ = 0; + // A conditional branch emitted as inverted-condition-over-`j`, so its reach is the jump's + // 18 bits rather than the branch's signed byte. See the .cpp for why that is not optional. + void branchRelaxed(uint8_t condNibble, Reg a, Reg b, Label l); + void patchBranches(); }; diff --git a/src/platform/esp32/moonlive_lower_riscv.cpp b/src/platform/esp32/moonlive_lower_riscv.cpp index 00c8cac9..8f06c65c 100644 --- a/src/platform/esp32/moonlive_lower_riscv.cpp +++ b/src/platform/esp32/moonlive_lower_riscv.cpp @@ -1,5 +1,6 @@ #include "core/moonlive/moonlive_emit.h" #include "core/moonlive/MoonLiveIr.h" +#include "core/moonlive/MoonLiveSpill.h" // the register allocator — core's, run before lowering #include "moonlive_asm_riscv.h" #include @@ -17,7 +18,7 @@ namespace { Reg reg(VReg v) { return static_cast(v); } } -size_t lowerToBytes(const IrProgram& ir, uint8_t* out, size_t cap) { +size_t lowerToBytes(IrProgram& ir, uint8_t* out, size_t cap, const RegBudget* squeeze) { // StoreElem folds the address into the index vreg (no scratch); FillElems needs two. // Reserve scratch only for the inline ops this program actually contains: FillElems needs two // (loop counter + per-channel address), StoreElem one (the address — it must NOT be folded into @@ -26,13 +27,50 @@ size_t lowerToBytes(const IrProgram& ir, uint8_t* out, size_t cap) { // the smallest file. const uint8_t scratch = ir.hasInline(InlineOp::FillElems) ? 2 : ir.hasInline(InlineOp::StoreElem) ? 1 : 0; - if (!out || cap == 0 || ir.vregsUsed + scratch > kRegCount) return 0; + // A host CALL also needs two scratch registers — the address of its argument block and the + // count — but it SHARES them with the inline ops rather than reserving its own. A Call and an + // Inline are different IR instructions, so their scratch is never live at the same time, and + // both die at the end of the one instruction that uses them. Reserving separately cost two + // registers permanently, which on Xtensa's ten is the difference between compiling and not. + // +1 for the host-argument reload. The host arguments live in frame slots now (core parks them + // at entry), so an op that reads buf/nLights/cpl/ctrls brings one back for the instruction that + // needs it. A call's two scratch registers still share with the inline ops' — different IR + // instructions, never live at once. + const uint8_t scratchTotal = static_cast((scratch < 2 ? uint8_t(2) : scratch) + 1); + if (!out || cap == 0) return 0; + + // Run the register allocator before lowering. It leaves a program that already fits untouched, + // and rewrites one that does not into Spill/Reload against this backend's frame — replacing the + // hand-rolled `vregsUsed + scratch > kRegCount` bail that used to REFUSE such a script outright. + // False here means even the spilled form does not fit, which is a diagnostic, never a miscompile. + uint8_t slots = 0; + // `squeeze` overrides the REGISTER COUNT and slot count a test wants to constrain, but never + // `reserved`: the scratch is what THIS lowerer is about to use for its inline ops and call + // argument block, so a test-supplied value would let the allocator hand out a register the + // lowerer then overwrites — miscompiling exactly the squeezed programs the seam exists to prove. + const RegBudget budget = squeeze ? RegBudget{squeeze->regs, scratchTotal, squeeze->slots} + : RegBudget{kRegCount, scratchTotal, RiscvAssembler::kMaxSpillSlots}; + if (!spillToBudget(ir, budget, slots)) return 0; // sAddr FIRST: it is the one StoreElem also uses, and a store-only program reserves a single // scratch — so the shared one has to be the lowest index or it would name an unreserved register. const Reg sAddr = static_cast(ir.vregsUsed); // per-channel address (both ops) const Reg sCtr = static_cast(ir.vregsUsed + 1); // FillElems loop counter - RiscvAssembler a; + // Size the assembler's buffer to the CALLER's — `cap` is what the staging buffer holds, so + // the two can never disagree about how much a script may emit (they were separately + // constant, and a script that fit one overflowed the other). + RiscvAssembler a(cap); + // Bring a parked host argument back for the one instruction that reads it. + // The LAST reserved scratch index, derived from scratchTotal rather than hard-coded: the `+1` + // in scratchTotal above IS this register, so the reservation and the use cannot drift apart. + // A fixed offset sat OUTSIDE the reservation and only worked because the register maps happen + // to have spare entries above the high-water mark. + const Reg sHost = static_cast(ir.vregsUsed + scratchTotal - 1); + auto host = [&](VReg v) -> Reg { a.spillLoad(sHost, hostArgSlot(v)); return sHost; }; + // The frame must cover the parked HOST ARGUMENTS at the top as well as whatever the parser and + // the allocator claimed at the bottom — they are stored before any script code runs, so a frame + // sized only from `slots` would put them past its end. + a.prologue(slots > kTotalSlots ? slots : kTotalSlots); // An IR label id becomes an assembler label ON FIRST USE. Allocating the whole range up front // exhausts the assembler's fixed label table, and the inline ops (StoreElem's bounds guard, @@ -46,7 +84,9 @@ size_t lowerToBytes(const IrProgram& ir, uint8_t* out, size_t cap) { return labels[id]; }; - for (uint8_t i = 0; i < ir.count; i++) { + // uint16_t, matching IrProgram::count: the op array is sized to the script now, so a + // uint8_t counter wrapped at 256 ops and looped forever instead of emitting. + for (uint16_t i = 0; i < ir.count; i++) { const IrInst& op = ir.ops[i]; switch (op.op) { case IrOp::Const: a.movImm(reg(op.dst), op.imm); break; @@ -70,38 +110,49 @@ size_t lowerToBytes(const IrProgram& ir, uint8_t* out, size_t cap) { if (op.imm >= 0 && op.imm < kIrLabels) a.branchNe(reg(op.a), reg(op.b), labelFor(op.imm)); break; - case IrOp::LoadCtrl: a.load8(reg(op.dst), reg(kArg4), op.imm); break; // dst = ctrls[imm] (a4 = kArg4) - case IrOp::Call: - // The IR carries the host's function pointer (the light TU's random16), valid in - // the single flashed image — call it directly, same as the other backends. + case IrOp::LoadCtrl: a.load8(reg(op.dst), host(kArg4), op.imm); break; // dst = ctrls[imm] (a4 = kArg4) + // The allocator's two ops. `imm` is a slot INDEX; the assembler owns the frame layout. + case IrOp::Spill: a.spillStore(reg(op.a), static_cast(op.imm)); break; + case IrOp::Reload: a.spillLoad(reg(op.dst), static_cast(op.imm)); break; + case IrOp::Call: { + // The IR carries the host's function pointer (valid in the single flashed image) and + // the FRAME SLOT its arguments start at. Hand the host their address and their + // count: nothing is held in a register across the call, and how many arguments a + // builtin takes stops being a property of this instruction. if (!op.callFn) return 0; - a.call(reg(op.dst), reg(op.a), reg(op.b), reg(op.c), reinterpret_cast(op.callFn)); + const Reg argPtr = static_cast(ir.vregsUsed); + a.slotAddr(argPtr, static_cast(op.imm)); + const Reg argN = static_cast(ir.vregsUsed + 1); + a.movImm(argN, static_cast(op.b)); + a.call(reg(op.dst), argPtr, argN, + host(kArg4), reinterpret_cast(op.callFn)); break; + } case IrOp::Inline: switch (op.inlineOp) { case InlineOp::StoreElem: { Label skip = a.newLabel(); // The address goes in SCRATCH, not the index vreg: folding it in destroyed // a `for` counter, which the loop's step and test read again after the store. - a.branchGeU(reg(op.a), reg(kArg1), skip); - a.mulReg(sAddr, reg(op.a), reg(kArg2)); // addr = index * cpl - a.store8(reg(kArg0), sAddr, reg(op.b)); - a.addImm(sAddr, sAddr, 1); a.store8(reg(kArg0), sAddr, reg(op.c)); - a.addImm(sAddr, sAddr, 1); a.store8(reg(kArg0), sAddr, reg(op.d)); + a.branchGeU(reg(op.a), host(kArg1), skip); + a.mulReg(sAddr, reg(op.a), host(kArg2)); // addr = index * cpl + a.store8(host(kArg0), sAddr, reg(op.b)); + a.addImm(sAddr, sAddr, 1); a.store8(host(kArg0), sAddr, reg(op.c)); + a.addImm(sAddr, sAddr, 1); a.store8(host(kArg0), sAddr, reg(op.d)); a.bind(skip); break; } case InlineOp::FillElems: { Label done = a.newLabel(), top = a.newLabel(); a.movImm(sCtr, 0); - a.branchIfZero(reg(kArg1), done); + a.branchIfZero(host(kArg1), done); a.bind(top); - a.mulReg(sAddr, sCtr, reg(kArg2)); - a.store8(reg(kArg0), sAddr, reg(op.a)); - a.addImm(sAddr, sAddr, 1); a.store8(reg(kArg0), sAddr, reg(op.b)); - a.addImm(sAddr, sAddr, 1); a.store8(reg(kArg0), sAddr, reg(op.c)); + a.mulReg(sAddr, sCtr, host(kArg2)); + a.store8(host(kArg0), sAddr, reg(op.a)); + a.addImm(sAddr, sAddr, 1); a.store8(host(kArg0), sAddr, reg(op.b)); + a.addImm(sAddr, sAddr, 1); a.store8(host(kArg0), sAddr, reg(op.c)); a.addImm(sCtr, sCtr, 1); - a.branchNe(sCtr, reg(kArg1), top); + a.branchNe(sCtr, host(kArg1), top); a.bind(done); break; } diff --git a/src/platform/esp32/moonlive_lower_xtensa.cpp b/src/platform/esp32/moonlive_lower_xtensa.cpp index 9c4224c4..ddd04f8e 100644 --- a/src/platform/esp32/moonlive_lower_xtensa.cpp +++ b/src/platform/esp32/moonlive_lower_xtensa.cpp @@ -1,5 +1,6 @@ #include "core/moonlive/moonlive_emit.h" #include "core/moonlive/MoonLiveIr.h" +#include "core/moonlive/MoonLiveSpill.h" // the register allocator — core's, run before lowering #include "moonlive_asm_xtensa.h" #include @@ -18,7 +19,7 @@ namespace { Reg reg(VReg v) { return static_cast(v); } } -size_t lowerToBytes(const IrProgram& ir, uint8_t* out, size_t cap) { +size_t lowerToBytes(IrProgram& ir, uint8_t* out, size_t cap, const RegBudget* squeeze) { // Reserve scratch only for the inline ops this program actually contains: FillElems needs two // (loop counter + per-channel address), StoreElem one (the address — it must NOT be folded into // the caller's index vreg, which destroys a `for` counter). Reserving the maximum unconditionally @@ -26,14 +27,50 @@ size_t lowerToBytes(const IrProgram& ir, uint8_t* out, size_t cap) { // the smallest file. const uint8_t scratch = ir.hasInline(InlineOp::FillElems) ? 2 : ir.hasInline(InlineOp::StoreElem) ? 1 : 0; - if (!out || cap == 0 || ir.vregsUsed + scratch > kRegCount) return 0; + // A host CALL also needs two scratch registers — the address of its argument block and the + // count — but it SHARES them with the inline ops rather than reserving its own. A Call and an + // Inline are different IR instructions, so their scratch is never live at the same time, and + // both die at the end of the one instruction that uses them. Reserving separately cost two + // registers permanently, which on Xtensa's ten is the difference between compiling and not. + // +1 for the host-argument reload. The host arguments live in frame slots now (core parks them + // at entry), so an op that reads buf/nLights/cpl/ctrls brings one back for the instruction that + // needs it. A call's two scratch registers still share with the inline ops' — different IR + // instructions, never live at once. + const uint8_t scratchTotal = static_cast((scratch < 2 ? uint8_t(2) : scratch) + 1); + if (!out || cap == 0) return 0; + + // Run the register allocator before lowering. It leaves a program that already fits untouched, + // and rewrites one that does not into Spill/Reload against this backend's frame — replacing the + // hand-rolled `vregsUsed + scratch > kRegCount` bail that used to REFUSE such a script outright. + // False here means even the spilled form does not fit, which is a diagnostic, never a miscompile. + uint8_t slots = 0; + // `squeeze` overrides the REGISTER COUNT and slot count a test wants to constrain, but never + // `reserved`: the scratch is what THIS lowerer is about to use for its inline ops and call + // argument block, so a test-supplied value would let the allocator hand out a register the + // lowerer then overwrites — miscompiling exactly the squeezed programs the seam exists to prove. + const RegBudget budget = squeeze ? RegBudget{squeeze->regs, scratchTotal, squeeze->slots} + : RegBudget{kRegCount, scratchTotal, XtensaAssembler::kMaxSpillSlots}; + if (!spillToBudget(ir, budget, slots)) return 0; // sAddr FIRST: it is the one StoreElem also uses, and a store-only program reserves a single // scratch — so the shared one has to be the lowest index or it would name an unreserved register. const Reg sAddr = static_cast(ir.vregsUsed); // per-channel address (both ops) const Reg sCtr = static_cast(ir.vregsUsed + 1); // FillElems loop counter - XtensaAssembler a; - a.prologue(); + // Size the assembler's buffer to the CALLER's — `cap` is what the staging buffer holds, so + // the two can never disagree about how much a script may emit (they were separately + // constant, and a script that fit one overflowed the other). + XtensaAssembler a(cap); + // Bring a parked host argument back for the one instruction that reads it. + // The LAST reserved scratch index, derived from scratchTotal rather than hard-coded: the `+1` + // in scratchTotal above IS this register, so the reservation and the use cannot drift apart. + // A fixed offset sat OUTSIDE the reservation and only worked because the register maps happen + // to have spare entries above the high-water mark. + const Reg sHost = static_cast(ir.vregsUsed + scratchTotal - 1); + auto host = [&](VReg v) -> Reg { a.spillLoad(sHost, hostArgSlot(v)); return sHost; }; + // The frame must cover the parked HOST ARGUMENTS at the top as well as whatever the parser and + // the allocator claimed at the bottom — they are stored before any script code runs, so a frame + // sized only from `slots` would put them past its end. + a.prologue(slots > kTotalSlots ? slots : kTotalSlots); // An IR label id becomes an assembler label ON FIRST USE. Allocating the whole range up front // exhausts the assembler's fixed label table, and the inline ops (StoreElem's bounds guard, @@ -47,7 +84,9 @@ size_t lowerToBytes(const IrProgram& ir, uint8_t* out, size_t cap) { return labels[id]; }; - for (uint8_t i = 0; i < ir.count; i++) { + // uint16_t, matching IrProgram::count: the op array is sized to the script now, so a + // uint8_t counter wrapped at 256 ops and looped forever instead of emitting. + for (uint16_t i = 0; i < ir.count; i++) { const IrInst& op = ir.ops[i]; switch (op.op) { case IrOp::Const: a.movImm(reg(op.dst), op.imm); break; @@ -71,11 +110,23 @@ size_t lowerToBytes(const IrProgram& ir, uint8_t* out, size_t cap) { if (op.imm >= 0 && op.imm < kIrLabels) a.branchNe(reg(op.a), reg(op.b), labelFor(op.imm)); break; - case IrOp::LoadCtrl: a.load8(reg(op.dst), reg(kArg4), op.imm); break; // dst = ctrls[imm] (a6 = kArg4) - case IrOp::Call: + case IrOp::LoadCtrl: a.load8(reg(op.dst), host(kArg4), op.imm); break; // dst = ctrls[imm] (a6 = kArg4) + // The allocator's two ops. `imm` is a slot INDEX; the assembler owns the frame layout. + case IrOp::Spill: a.spillStore(reg(op.a), static_cast(op.imm)); break; + case IrOp::Reload: a.spillLoad(reg(op.dst), static_cast(op.imm)); break; + case IrOp::Call: { + // The arguments are in consecutive frame slots starting at `imm`; hand the host + // their ADDRESS and their COUNT. Nothing is held in a register across the call, and + // how many arguments a builtin takes stops being a property of this instruction. if (!op.callFn) return 0; - a.call(reg(op.dst), reg(op.a), reg(op.b), reg(op.c), reinterpret_cast(op.callFn)); + const Reg argPtr = static_cast(ir.vregsUsed); + a.slotAddr(argPtr, static_cast(op.imm)); + const Reg argN = static_cast(ir.vregsUsed + 1); + a.movImm(argN, static_cast(op.b)); + a.call(reg(op.dst), argPtr, argN, + host(kArg4), reinterpret_cast(op.callFn)); break; + } case IrOp::Inline: switch (op.inlineOp) { case InlineOp::StoreElem: { @@ -86,11 +137,11 @@ size_t lowerToBytes(const IrProgram& ir, uint8_t* out, size_t cap) { // inside a loop therefore left the counter holding i*cpl+2 and the loop ran // the wrong number of times. Label skip = a.newLabel(); - a.branchGeU(reg(op.a), reg(kArg1), skip); // index >= nLights → skip - a.mulReg(sAddr, reg(op.a), reg(kArg2)); // addr = index * cpl - a.store8(reg(kArg0), sAddr, reg(op.b)); // store r - a.addImm(sAddr, sAddr, 1); a.store8(reg(kArg0), sAddr, reg(op.c)); - a.addImm(sAddr, sAddr, 1); a.store8(reg(kArg0), sAddr, reg(op.d)); + a.branchGeU(reg(op.a), host(kArg1), skip); // index >= nLights → skip + a.mulReg(sAddr, reg(op.a), host(kArg2)); // addr = index * cpl + a.store8(host(kArg0), sAddr, reg(op.b)); // store r + a.addImm(sAddr, sAddr, 1); a.store8(host(kArg0), sAddr, reg(op.c)); + a.addImm(sAddr, sAddr, 1); a.store8(host(kArg0), sAddr, reg(op.d)); a.bind(skip); break; } @@ -99,14 +150,14 @@ size_t lowerToBytes(const IrProgram& ir, uint8_t* out, size_t cap) { // Two scratch: sCtr (i), sAddr (the per-light address). Label done = a.newLabel(), top = a.newLabel(); a.movImm(sCtr, 0); - a.branchIfZero(reg(kArg1), done); + a.branchIfZero(host(kArg1), done); a.bind(top); - a.mulReg(sAddr, sCtr, reg(kArg2)); // addr = i * cpl - a.store8(reg(kArg0), sAddr, reg(op.a)); - a.addImm(sAddr, sAddr, 1); a.store8(reg(kArg0), sAddr, reg(op.b)); - a.addImm(sAddr, sAddr, 1); a.store8(reg(kArg0), sAddr, reg(op.c)); + a.mulReg(sAddr, sCtr, host(kArg2)); // addr = i * cpl + a.store8(host(kArg0), sAddr, reg(op.a)); + a.addImm(sAddr, sAddr, 1); a.store8(host(kArg0), sAddr, reg(op.b)); + a.addImm(sAddr, sAddr, 1); a.store8(host(kArg0), sAddr, reg(op.c)); a.addImm(sCtr, sCtr, 1); // i++ - a.branchNe(sCtr, reg(kArg1), top); + a.branchNe(sCtr, host(kArg1), top); a.bind(done); break; } diff --git a/src/platform/esp32/platform_config.h b/src/platform/esp32/platform_config.h index 32a37301..f3ac72c4 100644 --- a/src/platform/esp32/platform_config.h +++ b/src/platform/esp32/platform_config.h @@ -206,6 +206,19 @@ constexpr bool hasEthernet = true; // hasWiFi — an Ethernet-only board (the MHC-WLED P4 shield) still has UDP. constexpr bool hasNetwork = hasWiFi || hasEthernet; +// ethPhyIsFixed, true where the interface is a property of the PLATFORM rather than of the board, +// so a persisted or catalog-supplied PHY type must not override it. False on real silicon, where +// which PHY a board carries is exactly what deviceModels.json exists to say. +// +// Set under emulation: the emulator presents one MAC and no other kind exists to select, so a saved +// ethType from an earlier session would otherwise pick hardware that is not there and leave the +// device with no network at all. +#ifdef CONFIG_ETH_USE_OPENETH +constexpr bool ethPhyIsFixed = true; +#else +constexpr bool ethPhyIsFixed = false; +#endif + // Enough compute headroom for a per-pixel FLOAT algorithm — a raymarcher, a fractal, a feedback // loop that iterates per light. This is the ONE exception to the integer-only render-path rule in // coding-standards, and it is gated rather than assumed: an effect behind this constant is not @@ -254,6 +267,9 @@ enum EthPhyType { ethIp101 = 2, // RMII, IP101 PHY (Waveshare P4-NANO; managed component, P4-only) ethW5500 = 3, // SPI, external W5500 module (ESP32-S3 boards — SE16, LightCrafter) ethYt8531 = 4, // RGMII, YT8531 PHY (ESP32-S31 CoreBoard; on-chip 1 Gb EMAC, S31-only) + ethOpeneth = 5, // QEMU's emulated OpenCores MAC, no silicon, only ever selected under emulation. + // It is what gives an emulated device a real IP stack, and therefore the REST API + // and the web UI: without it a QEMU run can only be watched on the serial console. }; // Per-board Ethernet pin/PHY map — runtime-configurable (no longer a fixed @@ -296,6 +312,15 @@ struct EthPinConfig { // see above); rmiiClock* are unused for RGMII (clocks are set there too). See // docs/reference/esp32-s31-coreboard.md for the schematic pin map. constexpr EthPinConfig ethConfigDefault = +#ifdef CONFIG_ETH_USE_OPENETH + // Under emulation the interface is QEMU's MAC, whatever chip is being emulated. It has no pins + // and no PHY to address, so every field below is -1: ethInitOpeneth ignores them. This is a + // COMPILE-time branch because CONFIG_ETH_USE_OPENETH is only ever set by the qemu firmware , + // there is no silicon that carries it, so no real board can take this path. + EthPinConfig{ /*phyType*/ ethOpeneth, /*addr*/ 1, /*mdc*/ -1, /*mdio*/ -1, + /*rst*/ -1, /*rmiiClk*/ -1, /*extIn*/ false, + /*miso*/ -1, /*mosi*/ -1, /*sck*/ -1, /*cs*/ -1, /*irq*/ -1 }; +#else isEsp32P4 ? EthPinConfig{ /*phyType*/ ethIp101, /*addr*/ 1, /*mdc*/ 31, /*mdio*/ 52, /*rst*/ 51, /*rmiiClk*/ 50, /*extIn*/ true, /*miso*/ -1, /*mosi*/ -1, /*sck*/ -1, /*cs*/ -1, /*irq*/ -1 } @@ -308,6 +333,7 @@ constexpr EthPinConfig ethConfigDefault = : EthPinConfig{ /*phyType*/ ethLan8720, /*addr*/ 0, /*mdc*/ -1, /*mdio*/ -1, /*rst*/ 5, /*rmiiClk*/ 17, /*extIn*/ false, /*miso*/ -1, /*mosi*/ -1, /*sck*/ -1, /*cs*/ -1, /*irq*/ -1 }; +#endif // CONFIG_ETH_USE_OPENETH // OTA (esp_https_ota) is available on every ESP32 build — the OTA partition // layout in partitions/*.csv reserves app0/app1 unconditionally, and esp_https_ota diff --git a/src/platform/esp32/platform_esp32.cpp b/src/platform/esp32/platform_esp32.cpp index 3572603f..87c59a75 100644 --- a/src/platform/esp32/platform_esp32.cpp +++ b/src/platform/esp32/platform_esp32.cpp @@ -44,7 +44,11 @@ // per-PHY SPI drivers from esp_eth core into managed components — these headers // come from espressif/eth_w5500 (idf_component.yml, gated to the S3). The marker // MM_ETH_W5500 keeps the rest of the file from repeating this compound condition. -#if defined(CONFIG_ETH_USE_SPI_ETHERNET) && !defined(CONFIG_ETH_USE_ESP32_EMAC) +// ...and NOT under emulation: the QEMU variant turns the internal EMAC off (there is no +// emulated silicon for it), which would otherwise satisfy this condition and pull in a SPI +// W5500 component that variant does not carry. +#if defined(CONFIG_ETH_USE_SPI_ETHERNET) && !defined(CONFIG_ETH_USE_ESP32_EMAC) && \ + !defined(CONFIG_ETH_USE_OPENETH) #define MM_ETH_W5500 1 #include "driver/spi_master.h" // W5500 SPI Ethernet (S3 boards) — bus + device config #include "esp_eth_mac_w5500.h" // espressif/eth_w5500 managed component @@ -96,6 +100,13 @@ uint32_t millis() MM_NONBLOCKING { return static_cast(esp_timer_get_time() / 1000); } +// The task handle IS the identity, and reading it costs one load — no TLS, so it works on a task +// however it was created. That matters: THREADPTR is 0 on a task without TLS set up, which made +// C++ thread_local fault at 0xfffffff0 here. +uintptr_t currentThreadId() MM_NONBLOCKING { + return reinterpret_cast(xTaskGetCurrentTaskHandle()); +} + uint32_t micros() MM_NONBLOCKING { return static_cast(esp_timer_get_time()); } @@ -873,6 +884,102 @@ void ethStop() { ethConnected_.store(false, std::memory_order_relaxed); } +#ifdef CONFIG_ETH_USE_OPENETH +// QEMU's emulated OpenCores MAC. No pins, no clock, no PHY register access, the emulator presents a +// ready MAC and IDF ships the driver for it, so this path is a fraction of ethInitEmac's setup. +// +// Why it exists: an emulated device with no IP stack can only be observed on the serial console. With +// it, the REST API and the web UI work exactly as on hardware, so the same tests, scripts and UI drive +// an emulated board, which is the whole point of emulating one. +static bool ethInitOpeneth() { + // STEP-BY-STEP LOGGED, deliberately. Bringing this up is a chain of six calls where any one can + // fail quietly, and a silent failure looks identical to a working stack with no cable: the device + // simply never gets an address. Logging each step means the serial log ALONE says which link of + // the chain broke, instead of the failure having to be re-derived from a missing IP. + std::printf("mm_net: openeth 1/6: creating netif\n"); + esp_netif_config_t netif_cfg = ESP_NETIF_DEFAULT_ETH(); + ethNetif_ = esp_netif_new(&netif_cfg); + if (!ethNetif_) { std::printf("mm_net: openeth 1/6 FAILED: esp_netif_new returned null\n"); return false; } + + std::printf("mm_net: openeth 2/6: creating MAC + PHY\n"); + eth_mac_config_t mac_config = ETH_MAC_DEFAULT_CONFIG(); + eth_phy_config_t phy_config = ETH_PHY_DEFAULT_CONFIG(); + phy_config.phy_addr = 1; // the address QEMU's model answers on + phy_config.reset_gpio_num = -1; // nothing to reset in an emulator + + esp_eth_mac_t* mac = esp_eth_mac_new_openeth(&mac_config); + // The generic ctor: QEMU's model answers the standard IEEE-802.3 registers, and the + // per-PHY ctors (dp83848 and friends) left esp_eth core in IDF v6 anyway. + esp_eth_phy_t* phy = esp_eth_phy_new_generic(&phy_config); + if (!mac || !phy) { + std::printf("mm_net: openeth 2/6 FAILED: mac=%p phy=%p\n", (void*)mac, (void*)phy); + if (phy) phy->del(phy); + if (mac) mac->del(mac); + if (ethNetif_) { esp_netif_destroy(ethNetif_); ethNetif_ = nullptr; } + return false; + } + + std::printf("mm_net: openeth 3/6: installing driver\n"); + esp_eth_config_t eth_config = ETH_DEFAULT_CONFIG(mac, phy); + esp_eth_handle_t eth_handle = nullptr; + esp_err_t err = esp_eth_driver_install(ð_config, ð_handle); + if (err != ESP_OK) { + std::printf("mm_net: openeth 3/6 FAILED: %s\n", esp_err_to_name(err)); + phy->del(phy); mac->del(mac); + if (ethNetif_) { esp_netif_destroy(ethNetif_); ethNetif_ = nullptr; } + return false; + } + + // PROMISCUOUS: QEMU's MAC implements no multicast filter, so esp_netif's attach logs an error + // registering one for IPv4. Accepting every frame is the emulator's stand-in for the filter it + // does not model, and costs nothing here, there is no real wire to be flooded from. + std::printf("mm_net: openeth 4/6: promiscuous mode\n"); + bool promiscuous = true; + err = esp_eth_ioctl(eth_handle, ETH_CMD_S_PROMISCUOUS, &promiscuous); + if (err != ESP_OK) ESP_LOGW(NET_TAG, "openeth 4/6: promiscuous not set (%s), continuing", + esp_err_to_name(err)); + + // From here the driver owns mac+phy, so every failure unwinds through driver_uninstall rather + // than del-ing them, exactly as ethInitEmac and ethInitSpi do. Written once as a lambda because + // three exits share it, and a half-cleaned failure leaks a netif and a driver on a device that + // then has no network to report the problem over. + auto fail = [&](const char* what) -> bool { + std::printf("mm_net: openeth FAILED: %s\n", what); + esp_event_handler_unregister(ETH_EVENT, ESP_EVENT_ANY_ID, ðEventHandler); + esp_event_handler_unregister(IP_EVENT, IP_EVENT_ETH_GOT_IP, ðEventHandler); + esp_eth_driver_uninstall(eth_handle); // frees mac + phy + if (ethNetif_) { esp_netif_destroy(ethNetif_); ethNetif_ = nullptr; } + return false; + }; + + std::printf("mm_net: openeth 5/6: attaching netif glue\n"); + esp_eth_netif_glue_handle_t glue = esp_eth_new_netif_glue(eth_handle); + if (!glue) return fail("netif glue is null"); + err = esp_netif_attach(ethNetif_, glue); + if (err != ESP_OK) return fail(esp_err_to_name(err)); + + // The EVENT HANDLERS, before start: link-up is what kicks IDF's DHCP client (via applyHostname), + // and GOT_IP is what flips ethConnected_ so NetworkModule leaves WaitingEth. Registering them + // after esp_eth_start would race the very first link-up event the emulator raises immediately. + // Omitting them entirely, which this function did, leaves a driver that starts, links up, and + // never asks for an address: the interface is up and has no IP, forever. + std::printf("mm_net: openeth 6/6: registering event handlers + starting driver\n"); + err = esp_event_handler_register(ETH_EVENT, ESP_EVENT_ANY_ID, ðEventHandler, nullptr); + if (err != ESP_OK) return fail(esp_err_to_name(err)); + err = esp_event_handler_register(IP_EVENT, IP_EVENT_ETH_GOT_IP, ðEventHandler, nullptr); + if (err != ESP_OK) return fail(esp_err_to_name(err)); + + err = esp_eth_start(eth_handle); + if (err != ESP_OK) return fail(esp_err_to_name(err)); + + // Retained like the other two init paths, so ethStop() can tear this interface down instead of + // silently no-opping on a null handle. + ethHandle_ = eth_handle; + std::printf("mm_net: openeth up, waiting for link + DHCP\n"); + return true; +} +#endif // CONFIG_ETH_USE_OPENETH + bool ethInit() { ensureNetifInit(); // Dispatch on the board's PHY type (runtime, from deviceModels.json via setEthConfig). @@ -885,6 +992,9 @@ bool ethInit() { #ifdef MM_ETH_W5500 case ethW5500: return ethInitSpi(); #endif +#ifdef CONFIG_ETH_USE_OPENETH + case ethOpeneth: return ethInitOpeneth(); +#endif #ifdef CONFIG_ETH_USE_ESP32_EMAC case ethLan8720: case ethIp101: // RMII PHYs (classic ESP32, P4) @@ -1548,6 +1658,9 @@ bool mdnsInit(const char* deviceName) { // interface. Idempotent on the targets where the predef ETH already covers it: // mdns_register_netif returns ESP_ERR_INVALID_STATE ("already registered"), which we // treat as success, so this one path fixes the P4 without regressing S31/classic/S3. + // Guarded because `ethNetif_` itself only exists in the Ethernet build: an MM_NO_ETH firmware + // gets the stubs above, which give it ethConnected() but no netif handle to register. +#ifndef MM_NO_ETH if (ethNetif_ && ethConnected()) { esp_err_t regErr = mdns_register_netif(ethNetif_); if (regErr == ESP_OK || regErr == ESP_ERR_INVALID_STATE) { @@ -1558,6 +1671,7 @@ bool mdnsInit(const char* deviceName) { ESP_LOGW(NET_TAG, "mDNS eth netif register failed: %s", esp_err_to_name(regErr)); } } +#endif // FORCE A FRESH RE-ADVERTISE: remove any existing service record, then add it back. // A reconnect / interface switch / live rename re-runs this; just renaming the diff --git a/src/platform/esp32/platform_esp32_parlio.cpp b/src/platform/esp32/platform_esp32_parlio.cpp index ba79d44c..cb53673d 100644 --- a/src/platform/esp32/platform_esp32_parlio.cpp +++ b/src/platform/esp32/platform_esp32_parlio.cpp @@ -265,6 +265,8 @@ uint8_t* parlioWs2812Buffer(const ParlioWs2812Handle& h, uint8_t buffer) { return (st && buffer < 2) ? st->buf[buffer] : nullptr; } +size_t parlioMaxTransferBytes() { return kParlioMaxTransferBytes; } + size_t parlioWs2812BufferCapacity(const ParlioWs2812Handle& h) { auto* st = static_cast(h.impl); return st ? st->cap : 0; @@ -410,6 +412,7 @@ bool parlioWs2812Init(ParlioWs2812Handle&, const uint16_t*, uint8_t, uint32_t, s } uint8_t* parlioWs2812Buffer(const ParlioWs2812Handle&, uint8_t) { return nullptr; } size_t parlioWs2812BufferCapacity(const ParlioWs2812Handle&) { return 0; } +size_t parlioMaxTransferBytes() { return 0; } // no Parlio here → no bound (the budget-0 contract) bool parlioWs2812Transmit(ParlioWs2812Handle&, uint8_t, size_t) { return false; } bool parlioWs2812Wait(ParlioWs2812Handle&, uint8_t, uint32_t) { return true; } uint32_t parlioWs2812LastTransmitUs(const ParlioWs2812Handle&) { return 0; } diff --git a/src/platform/platform.h b/src/platform/platform.h index a8d2914c..7c32cd0d 100644 --- a/src/platform/platform.h +++ b/src/platform/platform.h @@ -32,6 +32,16 @@ namespace mm::platform { uint32_t millis() MM_NONBLOCKING; + +/// An opaque identity for the calling thread/task, stable for its lifetime and distinct between +/// concurrent ones. Zero is never returned, so a caller can use it as "no thread recorded". +/// +/// Exists because C++ `thread_local` is NOT usable on the ESP32: the compiler reaches TLS through +/// the THREADPTR special register, and a FreeRTOS task that was not created with TLS initialised +/// has THREADPTR = 0 — so the access dereferences a small offset from null (0xfffffff0 was the +/// measured faulting address) and dies inside the exception handler as a Double exception. This is +/// the portable seam for "which thread am I", used where per-thread state is genuinely needed. +uintptr_t currentThreadId() MM_NONBLOCKING; uint32_t micros() MM_NONBLOCKING; // Test-only override: when set to non-zero, millis() returns this value instead @@ -1165,6 +1175,14 @@ bool parlioWs2812Init(ParlioWs2812Handle& h, const uint16_t* dataPins, uint8_t* parlioWs2812Buffer(const ParlioWs2812Handle& h, uint8_t buffer); size_t parlioWs2812BufferCapacity(const ParlioWs2812Handle& h); +// The most bytes Parlio can send in ONE transfer — a HARDWARE ceiling, not a heap budget, so it +// needs no handle and holds before anything is allocated. A caller sizes a frame against it to +// refuse an impossible configuration up front instead of failing the bus init. +// +// 0 means NO BOUND (the dmaBudgetBytes contract), not "zero bytes usable" — it is what a host +// without Parlio returns, and what a caller reads as "nothing to check against". +size_t parlioMaxTransferBytes(); + // Start the autonomous DMA transfer of buffer `buffer`'s first `bytes`; pair // with parlioWs2812Wait on the SAME buffer. No refill deadline once started // (single-shot, not the loop-transmission mode Parlio also offers). diff --git a/src/ui/style.css b/src/ui/style.css index 6bd140b1..be8536ac 100644 --- a/src/ui/style.css +++ b/src/ui/style.css @@ -339,14 +339,23 @@ body.cards-resizing { /* The drag bar is docked-mode-hidden (only PiP needs a grab handle); the dock / close buttons stay available so a wide-screen user can pop the preview out. */ +/* The bar must fit ANY pane width (the PiP is 200px): the informational items (the status + text and the dot slider) shrink and ellipsize first, and the buttons never shrink, so the + dock/close controls stay reachable. Without this the nowrap status plus the slider's native + width pushed the buttons past the pane's overflow:hidden edge, and the control that restores + the preview was the very thing clipped out of view. */ .preview-bar { display: flex; align-items: center; gap: 4px; padding: 2px 4px; + min-width: 0; +} +.preview-grip { display: none; color: var(--fg-muted); cursor: grab; user-select: none; font-size: 14px; flex-shrink: 0; } +.preview-status { + color: var(--fg-muted); font-size: 11px; opacity: 0.8; white-space: nowrap; + flex: 0 1 auto; min-width: 0; overflow: hidden; text-overflow: ellipsis; } -.preview-grip { display: none; color: var(--fg-muted); cursor: grab; user-select: none; font-size: 14px; } -.preview-status { color: var(--fg-muted); font-size: 11px; opacity: 0.8; white-space: nowrap; } /* Standard screen-reader-only pattern: present in the DOM (labels a control) but not shown. */ .visually-hidden { position: absolute; width: 1px; height: 1px; padding: 0; margin: -1px; @@ -363,6 +372,7 @@ body.cards-resizing { padding: 2px 4px; border-radius: 4px; opacity: 0.6; + flex-shrink: 0; /* a control that shrinks away cannot be clicked */ } .preview-bar-btn:hover { opacity: 1; color: var(--accent); } .preview-bar-btn.active { opacity: 1; color: var(--accent); } @@ -371,6 +381,8 @@ body.cards-resizing { match the theme without a full custom track/thumb (recognisable + minimal). */ .preview-dot { width: 70px; + flex: 0 1 70px; /* shrinks (to min-width) before the buttons ever would */ + min-width: 32px; height: 14px; margin: 0 2px; cursor: pointer; @@ -404,8 +416,6 @@ body.cards-resizing { .mode-pip .preview-bar { cursor: grab; background: var(--bg-2); } .mode-pip #preview { max-height: none; } .mode-pip #preview-reset { top: 28px; width: 22px; height: 22px; font-size: 13px; } -/* Expanded PiP: a larger floating view when the user taps ⤢ in PiP. */ -.mode-pip .preview-pane.expanded { width: min(80vw, 360px); } /* When the PiP is dismissed, hide the pane and show the re-show pill. */ .workspace.preview-hidden .preview-pane { display: none; } diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 46bbbf60..d7a54fec 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -42,6 +42,9 @@ add_executable(mm_tests unit/core/unit_moonlive_fill.cpp unit/core/unit_moonlive_compiler.cpp unit/core/unit_moonlive_ir.cpp + unit/core/unit_moonlive_spill.cpp + unit/core/unit_moonlive_codegen_xtensa.cpp + unit/core/unit_moonlive_codegen_riscv.cpp unit/core/unit_MoonModule.cpp unit/core/unit_MoonModule_control_change_gate.cpp unit/core/unit_MoonModule_lifecycle.cpp diff --git a/test/scenarios/light/scenario_MoonLiveEffect_livescript.json b/test/scenarios/light/scenario_MoonLiveEffect_livescript.json index 9749f763..e454d725 100644 --- a/test/scenarios/light/scenario_MoonLiveEffect_livescript.json +++ b/test/scenarios/light/scenario_MoonLiveEffect_livescript.json @@ -661,7 +661,7 @@ "desktop-macos": { "tick_us": [ 0, - 36 + 40 ], "free_heap": [ 0, @@ -673,7 +673,7 @@ ], "at": [ "2026-06-26", - "2026-07-27" + "2026-08-14" ] }, "esp32s3-n16r8": { diff --git a/test/scenarios/light/scenario_MoonLive_pipeline.json b/test/scenarios/light/scenario_MoonLive_pipeline.json index efe47569..ce095d4e 100644 --- a/test/scenarios/light/scenario_MoonLive_pipeline.json +++ b/test/scenarios/light/scenario_MoonLive_pipeline.json @@ -58,8 +58,8 @@ "observed": { "desktop-macos": { "tick_us": [ - 5, - 16 + 4, + 33 ], "free_heap": [ 0, @@ -71,7 +71,7 @@ ], "at": [ "2026-08-09", - "2026-08-10" + "2026-08-14" ] } } @@ -96,8 +96,8 @@ "observed": { "desktop-macos": { "tick_us": [ - 5, - 16 + 4, + 21 ], "free_heap": [ 0, @@ -109,7 +109,7 @@ ], "at": [ "2026-08-09", - "2026-08-10" + "2026-08-16" ] } } @@ -128,7 +128,7 @@ "desktop-macos": { "tick_us": [ 5, - 18 + 19 ], "free_heap": [ 0, @@ -140,7 +140,7 @@ ], "at": [ "2026-08-09", - "2026-08-10" + "2026-08-16" ] } } @@ -166,7 +166,7 @@ "desktop-macos": { "tick_us": [ 5, - 13 + 14 ], "free_heap": [ 0, @@ -178,7 +178,7 @@ ], "at": [ "2026-08-09", - "2026-08-10" + "2026-08-16" ] } } @@ -197,7 +197,7 @@ "desktop-macos": { "tick_us": [ 5, - 20 + 23 ], "free_heap": [ 0, @@ -209,7 +209,7 @@ ], "at": [ "2026-08-09", - "2026-08-10" + "2026-08-16" ] } } @@ -258,8 +258,8 @@ "observed": { "desktop-macos": { "tick_us": [ - 5, - 20 + 4, + 22 ], "free_heap": [ 0, @@ -271,7 +271,7 @@ ], "at": [ "2026-08-09", - "2026-08-10" + "2026-08-14" ] } } @@ -288,7 +288,7 @@ "observed": { "desktop-macos": { "tick_us": [ - 5, + 4, 35 ], "free_heap": [ @@ -301,7 +301,7 @@ ], "at": [ "2026-08-09", - "2026-08-10" + "2026-08-13" ] } } diff --git a/test/scenarios/light/scenario_peripheral_grid_sweep.json b/test/scenarios/light/scenario_peripheral_grid_sweep.json index d31cfb64..c7119e46 100644 --- a/test/scenarios/light/scenario_peripheral_grid_sweep.json +++ b/test/scenarios/light/scenario_peripheral_grid_sweep.json @@ -389,7 +389,7 @@ }, "desktop-macos": { "tick_us": [ - 273, + 271, 1427 ], "free_heap": [ @@ -402,7 +402,7 @@ ], "at": [ "2026-07-26", - "2026-07-31" + "2026-08-13" ] } } @@ -489,7 +489,7 @@ "desktop-macos": { "tick_us": [ 4, - 21 + 30 ], "free_heap": [ 0, @@ -501,7 +501,7 @@ ], "at": [ "2026-07-26", - "2026-07-31" + "2026-08-12" ] } } @@ -567,7 +567,7 @@ "desktop-macos": { "tick_us": [ 17, - 90 + 121 ], "free_heap": [ 0, @@ -579,7 +579,7 @@ ], "at": [ "2026-07-26", - "2026-07-31" + "2026-08-12" ] } } @@ -644,7 +644,7 @@ }, "desktop-macos": { "tick_us": [ - 68, + 67, 350 ], "free_heap": [ @@ -657,7 +657,7 @@ ], "at": [ "2026-07-26", - "2026-07-31" + "2026-08-13" ] } } @@ -722,7 +722,7 @@ }, "desktop-macos": { "tick_us": [ - 271, + 270, 1357 ], "free_heap": [ @@ -735,7 +735,7 @@ ], "at": [ "2026-07-26", - "2026-08-06" + "2026-08-13" ] } } @@ -977,7 +977,7 @@ }, "desktop-macos": { "tick_us": [ - 68, + 67, 382 ], "free_heap": [ @@ -990,7 +990,7 @@ ], "at": [ "2026-07-26", - "2026-07-31" + "2026-08-13" ] } } diff --git a/test/unit/core/moonlive_device_codegen.inc b/test/unit/core/moonlive_device_codegen.inc new file mode 100644 index 00000000..8ebc01bd --- /dev/null +++ b/test/unit/core/moonlive_device_codegen.inc @@ -0,0 +1,201 @@ +// The checks a DEVICE backend's emitted code must pass, written once and run per ISA. +// +// Why this exists: only the host (arm64) backend is ever EXECUTED by tests, so an Xtensa or RISC-V +// codegen defect used to ship silently and be debugged by flashing a board. That is what happened — +// a scripted layout reset every S3, and five plausible theories read off the source were each +// falsified by the device before the disassembler found the real cause. The backends are ordinary +// C++ behind a target guard, so defining the guard and compiling the sources runs the REAL emitter +// on the development machine, and a defect becomes a failing test instead of a bench session. +// +// ONE TRANSLATION UNIT PER ISA, and this file is the body they share. Two backends cannot live in +// one TU: they both define `Reg`, `Label` and an assembler class, and the include guards would give +// the second one the first one's types. Each ISA's .cpp defines MM_ISA_NAME + the target macro, +// includes its backend and the front end above it, then includes this — so the unqualified +// `lowerToBytes` inside compileSource resolves to that backend. The firmware build never sees these +// files, so a device image still holds exactly one backend definition. Same mechanism as +// moondeck/moonlive/emit_xtensa.cpp, which reads encodings without a flash. +// +// Expects, from the including TU: MM_ISA_NAME (a string literal) and the backend + compiler already +// included in the enclosing namespace. + +#include "light/moonlive/MoonLiveBuiltins_light.h" + +#include +#include +#include +#include + +#include +#include + +namespace { + +// The shipped default layout: a nested loop whose body calls addLight. The script that crashed +// every S3, so the one worth pinning hardest. +const char* kGridLayout = + "uint8_t cols = 16; // @control 1..64\n" + "uint8_t rows = 16; // @control 1..64\n" + "for (y = 0; y < rows; y = y + 1) {\n" + " for (x = 0; x < cols; x = x + 1) {\n" + " addLight(x, y, 0);\n" + " }\n" + "}\n"; + +// The two shapes an effect takes: straight-line, and a loop over the grid. +const char* kEffectSimple = "setRGB(0, 255, 0, 0);\n"; +const char* kEffectLoop = "for (i = 0; i < 64; i = i + 1) { setRGB(i, i, 255 - i, 60); }\n"; + +/// Compile for THIS TU's backend. `sysvars` picks the binding's vocabulary — they are different +/// sets, not nested ones: a LAYOUT may use `x`/`y` as loop counters precisely because it is not +/// handed them, which is what the shipped grid.mlv does. +std::vector emitBytes(const char* src, const mm::moonlive::SysVarTable& sysvars, bool& ok) { + // Sized the way production sizes it, from the script's own token count — a test that always + // allocated the sanity bound would pass while a right-sized caller overflowed. + std::vector out(mm::moonlive::codeCapFor(mm::moonlive::countTokens(src))); + auto r = mm::moonlive::compileSource(src, mm::moonlive::lightBuiltins(), sysvars, + out.data(), out.size(), nullptr, MM_ISA_LOWER); + ok = r.ok; + out.resize(r.ok ? r.len : 0); + return out; +} + +size_t emitLen(const char* src, const mm::moonlive::SysVarTable& sysvars, bool& ok) { + return emitBytes(src, sysvars, ok).size(); +} + +/// FNV-1a over the emitted bytes — the project's idiom for "did this change". +/// +/// Only meaningful for a script with NO host call. A `Call` bakes the builtin's ADDRESS into the +/// instruction stream and ASLR moves that every process, so the bytes of a calling script genuinely +/// differ run to run and no golden value can pin them. Those scripts are covered by the structural +/// checks and by their emitted LENGTH instead, which is stable. +uint32_t emitHash(const char* src, const mm::moonlive::SysVarTable& sysvars, bool& ok) { + const auto code = emitBytes(src, sysvars, ok); + uint32_t h = 2166136261u; + for (uint8_t byte : code) { h ^= byte; h *= 16777619u; } + return h; +} + +} // namespace + +// The shipped default layout has to COMPILE for every target. It stopped compiling on Xtensa when +// the register file shrank to the ten registers the windowed ABI actually leaves free, and a script +// that is refused reads to a user exactly like one that is broken. +TEST_CASE("the shipped grid layout compiles for " MM_ISA_NAME) { + bool ok = false; + CHECK(emitLen(kGridLayout, mm::moonlive::layoutSysVars(), ok) > 0); + CHECK(ok); +} + +TEST_CASE("a scripted effect compiles for " MM_ISA_NAME ", straight-line and looped") { + bool ok = false; + CHECK(emitLen(kEffectSimple, mm::moonlive::effectSysVars(), ok) > 0); + CHECK(ok); + // A LOOPED effect is the case that did not fit Xtensa's ten registers until call arguments were + // staged through the frame: a four-operand setRGB used to need four registers live at once, plus + // whatever its operands took to compute, and the allocator ran out. Now only one argument holds + // a register at a time. + CHECK(emitLen(kEffectLoop, mm::moonlive::effectSysVars(), ok) > 0); + CHECK(ok); +} + +// `fill` plus a loop was the shape that did NOT fit the smallest register file: FillElems needs two +// scratch registers above the program's, and on Xtensa that left eight where the five fixed ABI +// vregs plus four reload temps needed nine — one short, so the golden here was 0 (REFUSED). Moving +// the host arguments into frame slots freed those five, and it now emits on every backend. +// Still pinned by length: it is the shape most sensitive to the register budget, so a regression +// that pushes any backend back over the edge shows up here first. +TEST_CASE("fill plus a loop on " MM_ISA_NAME " emits what it currently can") { + bool ok = false; + CHECK(emitLen("fill(0,0,0);\nfor (y = 0; y < height; y = y + 1) { setRGB(y, 255, 0, 0); }", + mm::moonlive::effectSysVars(), ok) == MM_GOLD_FILLLOOP_LEN); +} + +// A scripted MODIFIER is the third binding, and the one handed x/y/z. +TEST_CASE("a scripted modifier compiles for " MM_ISA_NAME) { + bool ok = false; + CHECK(emitLen("setXYZ(0, x, y, z);\n", mm::moonlive::modifierSysVars(), ok) > 0); + CHECK(ok); +} + +// The GOLDEN check: the exact machine code for the shipped scripts. +// +// A change DETECTOR, not a correctness claim. It fails on any emission change so the diff gets read +// deliberately, instead of the change being discovered by flashing a board and watching it reset. +// When a change is intended, update the constants and say why in the commit. +// +// Hashed rather than length-checked: altering a frame offset or a register field changes what +// EXECUTES without changing how many bytes it takes — which is precisely the defect class this file +// exists to catch, and a length alone sails straight past it. +TEST_CASE("the " MM_ISA_NAME " machine code for the shipped scripts is what it was") { + bool ok = false; + // The grid CALLS addLight, so its bytes carry a host address that moves each process — length + // only. The effect is pure inline code, so it is pinned byte for byte. + CHECK(emitLen(kGridLayout, mm::moonlive::layoutSysVars(), ok) == MM_GOLD_GRID_LEN); + CHECK(emitLen(kEffectSimple, mm::moonlive::effectSysVars(), ok) == MM_GOLD_FX_LEN); + CHECK(emitHash(kEffectSimple, mm::moonlive::effectSysVars(), ok) == MM_GOLD_FX_HASH); + CHECK(emitLen(kEffectLoop, mm::moonlive::effectSysVars(), ok) == MM_GOLD_FXLOOP_LEN); + CHECK(emitHash(kEffectLoop, mm::moonlive::effectSysVars(), ok) == MM_GOLD_FXLOOP_HASH); + + // KNOWN GAP: no golden pins a CALL SEQUENCE. The goldens above are inline-only code, so the + // call path (the code the windowed ABI makes subtle) is covered by the structural checks and + // the frame-offset test, not by a byte hash. A raw byte-mask golden was tried and removed: a + // masked scan cannot tell a movi immediate from a coincidental byte, so its hash flapped + // between builds. A real one needs the decoder, not a mask. +} + +// Degrade, never miscompile: a script the backend cannot encode must fail cleanly. The emitters +// signal that by returning zero bytes, which compileSource turns into a diagnostic — the device +// then shows the error and keeps rendering, instead of executing whatever the truncation produced. +TEST_CASE("an unencodable script is refused, not truncated, on " MM_ISA_NAME) { + bool ok = true; + // A loop nest far past what any label/fixup table holds. + std::string deep; + for (int i = 0; i < 40; i++) deep += "for (i" + std::to_string(i) + " = 0; i" + + std::to_string(i) + " < 2; i" + std::to_string(i) + + " = i" + std::to_string(i) + " + 1) {"; + deep += "addLight(0, 0, 0);"; + for (int i = 0; i < 40; i++) deep += "}"; + const size_t n = emitLen(deep.c_str(), mm::moonlive::layoutSysVars(), ok); + CHECK(n == 0); // refused + CHECK_FALSE(ok); // and said so +} + +// Every script we SHIP has to compile on every backend, or a user's board choice silently decides +// which effects exist. `plasma.mlv` is why this test is here: it ran on an S3 and on desktop and was +// refused on an S31, because one `kCodeCap` constant was sized to the densest ISA while RISC-V emits +// up to 1.9x more for identical source. The buffer is now sized per script, and this pins that every +// shipped script still fits on every backend. +// +// Read from moonlive/ rather than copied inline: a copy is a second source of truth that drifts the +// first time a script is edited, and it is exactly the widest scripts — the ones worth checking — +// that would silently stop being the ones under test. +TEST_CASE("every shipped script compiles for " MM_ISA_NAME) { + int checked = 0; + for (const char* sub : {"layouts", "effects", "modifiers", "drivers"}) { + const std::filesystem::path dir = + std::filesystem::path(__FILE__).parent_path().parent_path().parent_path().parent_path() + / "moonlive" / sub; + if (!std::filesystem::exists(dir)) continue; + for (const auto& entry : std::filesystem::directory_iterator(dir)) { + if (!entry.is_regular_file() || entry.path().extension() != ".mlv") continue; + std::ifstream f(entry.path()); + std::ostringstream ss; + ss << f.rdbuf(); + const std::string src = ss.str(); + // Each binding supplies a DIFFERENT system-variable vocabulary, not a nested superset — + // a layout may use x/y as loop counters precisely because it is not handed them. + const std::string kind = sub; + const auto& sysvars = kind == "layouts" ? mm::moonlive::layoutSysVars() + : kind == "modifiers" ? mm::moonlive::modifierSysVars() + : mm::moonlive::effectSysVars(); + bool ok = false; + const size_t n = emitLen(src.c_str(), sysvars, ok); + INFO("script: ", kind, "/", entry.path().filename().string(), " on " MM_ISA_NAME); + CHECK(ok); + CHECK(n > 0); + checked++; + } + } + CHECK(checked > 0); // a silently empty folder would pass without this +} diff --git a/test/unit/core/moonlive_structural.h b/test/unit/core/moonlive_structural.h new file mode 100644 index 00000000..3b48d793 --- /dev/null +++ b/test/unit/core/moonlive_structural.h @@ -0,0 +1,35 @@ +// One decoded instruction, as the structural checker sees it (moonlive_structural.inc). +// +// Split from the .inc because each ISA's decoder must be DECLARED before the checks that call it +// and DEFINED after — both need this type, so it cannot live in either. +#pragma once + +#include +#include + +namespace mm_structural { + +// A field is "absent" when its `has` flag is false; an instruction that names no frame offset +// (most of them) simply reports none. +struct Decoded { + uint8_t len = 0; // 0 means "could not decode" — the walk stops and the test fails + bool hasFrameOff = false; + uint32_t frameOff = 0; // byte offset from the frame pointer + bool hasTarget = false; + int32_t target = 0; // absolute byte offset of the branch/jump destination + // The frame size this instruction ALLOCATES, when it is the prologue. Read from the emitted + // instruction rather than recomputed by the test: a checker that models the frame with its own + // copy of the formula agrees with the backend even when the backend is wrong, which is exactly + // how the first version of this check passed against the bug it was written for. + bool hasFrameAlloc = false; + uint32_t frameAlloc = 0; + + // For the call-clobber check. A windowed call rotates the register file, so every register the + // rotation covers holds the callee's leftovers afterwards unless the caller saved and restored + // it. `readsMask`/`writesMask` are bit N = register N. + bool isCall = false; + uint32_t readsMask = 0; + uint32_t writesMask = 0; +}; + +} // namespace mm_structural diff --git a/test/unit/core/moonlive_structural.inc b/test/unit/core/moonlive_structural.inc new file mode 100644 index 00000000..ec599408 --- /dev/null +++ b/test/unit/core/moonlive_structural.inc @@ -0,0 +1,209 @@ +// The STRUCTURAL CHECKER over emitted machine code (Plan-20260813, verification item 2). +// +// A golden-bytes test says "the emission changed"; it never says "the emission is WRONG". These +// three checks say the second thing, and each one is a defect class that reached a bench: +// +// 1. Frame offsets stay inside the frame the prologue allocated, and clear of the region the ABI +// reserves at its top. The parked host arguments landed inside Xtensa's 16-byte base-save +// area — where the hardware spills the caller's a0..a3 on a window overflow — so a script's +// own data and its RETURN ADDRESS shared memory. Any script at all reset an ESP32 classic and +// an S3 with `IllegalInstruction` and a data address in A0. Found on a bench, not by a test. +// 2. Every branch and jump target lands on an instruction boundary inside the program. Xtensa's +// conditional branches reach +/-127 bytes and were silently truncated past that, so a long +// loop jumped into the middle of an instruction. +// 3. No instruction names a register outside the backend's map. a14/a15 carry the routine's +// retw.n return linkage; using them as general registers produced IllegalInstruction the +// moment a scripted layout ran. +// +// Each ISA's TU defines MM_ISA_DECODE — a decoder that walks the byte stream and reports, per +// instruction, its length plus any frame offset / branch target / register it names. The checks +// themselves are written once, here. +// +// Expects, from the including TU: MM_ISA_NAME, MM_ISA_LOWER, MM_ISA_RESERVED_TOP, +// MM_ISA_DECODE, and the compiler already included. + +#include "moonlive_structural.h" + + +// Frame offsets must land inside the frame the prologue allocated, and BELOW the region the ABI +// reserves at the top of it. This is the check that would have caught the base-save-area bug on the +// host, in CI, instead of on two boards. +TEST_CASE("emitted " MM_ISA_NAME " code keeps every frame offset inside the frame") { + struct Case { const char* name; const char* src; int binding; }; + const Case cases[] = { + {"a bare write", "setRGB(0, 255, 0, 0);\n", 1}, + {"a loop", "for (i = 0; i < 3; i = i + 1) { }\n", 0}, + {"a call in a loop", "for (i = 0; i < 4; i = i + 1) { addLight(i, 0, 0); }\n", 0}, + {"the shipped grid", kGridLayout, 0}, + }; + for (const auto& c : cases) { + bool ok = false; + const auto& sysvars = c.binding == 0 ? mm::moonlive::layoutSysVars() + : mm::moonlive::effectSysVars(); + const auto code = emitBytes(c.src, sysvars, ok); + INFO("script: ", c.name, " on " MM_ISA_NAME); + REQUIRE(ok); + REQUIRE(code.size() > 0); + + // The frame the program ACTUALLY allocated, read from its own prologue. + uint32_t frame = 0; + { + const mm_structural::Decoded p0 = MM_ISA_DECODE(code.data(), code.size(), 0); + REQUIRE(p0.hasFrameAlloc); // every emitted routine must open with a prologue + frame = p0.frameAlloc; + } + // Guarded rather than subtracted blind: `frame` is decoded from the emitted prologue, so a + // defect that emits a frame smaller than the reserved area would wrap this unsigned + // subtraction to ~4 billion and let EVERY offset below pass, the checker going blind + // exactly when the code is worst. Refuse instead. + REQUIRE(frame >= (MM_ISA_RESERVED_TOP)); + const uint32_t usable = frame - (MM_ISA_RESERVED_TOP); + + size_t pc = 0; + while (pc < code.size()) { + const mm_structural::Decoded d = MM_ISA_DECODE(code.data(), code.size(), pc); + INFO("at byte ", pc); + REQUIRE(d.len > 0); // an undecodable byte is itself a defect + if (d.hasFrameOff) { + INFO("frame offset ", d.frameOff, " vs frame ", frame, + " (usable below ", usable, ")"); + CHECK(d.frameOff + 4 <= usable); // the whole 4-byte word must fit + } + pc += d.len; + } + CHECK(pc == code.size()); // the stream decoded exactly, no trailing byte + } +} + +// A branch that lands mid-instruction executes whatever the following bytes happen to encode. +TEST_CASE("emitted " MM_ISA_NAME " code branches only to instruction boundaries") { + struct Case { const char* name; const char* src; int binding; }; + const Case cases[] = { + {"one loop", "for (i = 0; i < 3; i = i + 1) { }\n", 0}, + {"nested loops", kGridLayout, 0}, + // Long enough that a short-displacement branch would have to be relaxed to reach. + {"a long loop body", + "for (i = 0; i < 8; i = i + 1) {\n" + " addLight(i, 0, 0); addLight(i, 1, 0); addLight(i, 2, 0); addLight(i, 3, 0);\n" + " addLight(i, 4, 0); addLight(i, 5, 0); addLight(i, 6, 0); addLight(i, 7, 0);\n" + "}\n", 0}, + }; + for (const auto& c : cases) { + bool ok = false; + const auto code = emitBytes(c.src, mm::moonlive::layoutSysVars(), ok); + INFO("script: ", c.name, " on " MM_ISA_NAME); + REQUIRE(ok); + REQUIRE(code.size() > 0); + + // Pass 1: every byte offset at which an instruction starts. + std::vector boundary(code.size() + 1, false); + size_t pc = 0; + while (pc < code.size()) { + boundary[pc] = true; + const mm_structural::Decoded d = MM_ISA_DECODE(code.data(), code.size(), pc); + REQUIRE(d.len > 0); + pc += d.len; + } + // NOT one-past-the-end: the epilogue is always the LAST instruction emitted, so a real + // branch targets the `retw.n`/`ret` itself and never the byte after it. Marking that byte a + // boundary would let a jump straight off the block, into whatever heap follows the exec + // allocation, pass as legal. + + // Pass 2: every target must be one of them, and inside the program. + pc = 0; + while (pc < code.size()) { + const mm_structural::Decoded d = MM_ISA_DECODE(code.data(), code.size(), pc); + if (d.hasTarget) { + INFO("branch at ", pc, " targets ", d.target, " of ", code.size(), " bytes"); + CHECK(d.target >= 0); + CHECK(static_cast(d.target) < code.size()); + if (d.target >= 0 && static_cast(d.target) < code.size()) + CHECK(boundary[static_cast(d.target)]); + } + pc += d.len; + } + } +} + +// The windowed-call rule, as an assertion: a CALL8 rotates the register file by eight, so every +// register the rotation covers holds the callee's leftovers afterwards. Reading one after a call, +// before writing it, is reading a destroyed value. +// +// Our `call()` saves and restores a8..a11 around the call, which is what makes those four legal to +// hold values in, so the property is not "never use a8..a15" but "never READ one the save-set does +// not cover". Written as the general rule (no register is read after a call without an intervening +// write) rather than a list of register numbers, so it stays true as the save-set changes. +// +// The case list covers both shapes that matter: plain calls, and the ARENA POINTER (kArg4) that a +// script reads a control or system variable through. The arena now lives in a frame slot and is +// reloaded into a scratch register per use, and a bench bisect found that a system-variable read +// inside a loop whose body makes a host call, and only that combination, reset both Xtensa boards. +// +// COVERAGE LIMIT, stated rather than implied: the decoder reports reads for the instructions the +// emitter uses to move values (branches, the call target, spill stores, the narrow add/mov forms). +// It does NOT decode every operand of every opcode, so this clears the register-window question for +// the tracked instructions, not for the whole ISA. +TEST_CASE("emitted " MM_ISA_NAME " code reads no register a call destroyed") { + struct Case { const char* name; const char* src; int binding; }; + const Case cases[] = { + {"one call", "setRGB(0, random16(256), 0, 0);\n", 1}, + {"a call in a loop", "for (i = 0; i < 4; i = i + 1) { setRGB(i, random16(256), 0, 0); }\n", 1}, + {"nested calls", "setRGB(random16(256), random16(256), random16(256), random16(256));\n", 1}, + {"a call in a layout", "for (i = 0; i < 4; i = i + 1) { addLight(i, 0, 0); }\n", 0}, + {"the shipped grid", kGridLayout, 0}, + // The shape that crashed on hardware, and the nearest ones that did not. + {"sysvar bound + call in body", + "for (x = 0; x < width; x = x + 1) { setRGB(x, random16(256), 0, 0); }\n", 1}, + {"control bound + call in body", + "uint8_t n = 8; // @control 1..64\n" + "for (x = 0; x < n; x = x + 1) { setRGB(x, random16(256), 0, 0); }\n", 1}, + {"sysvar read inside the body, with a call", + "for (x = 0; x < 4; x = x + 1) { setRGB(x, width, random16(256), 0); }\n", 1}, + }; + for (const auto& c : cases) { + bool ok = false; + const auto& sysvars = c.binding == 0 ? mm::moonlive::layoutSysVars() + : mm::moonlive::effectSysVars(); + const auto code = emitBytes(c.src, sysvars, ok); + INFO("script: ", c.name, " on " MM_ISA_NAME); + REQUIRE(ok); + REQUIRE(code.size() > 0); + + // Branch targets first: control can reach one from somewhere with no call in between, and + // this walk is linear rather than a full dataflow analysis, so assuming nothing at a join + // point is the conservative choice that keeps it sound. + std::vector isTarget(code.size() + 1, false); + { + size_t scan = 0; + while (scan < code.size()) { + const mm_structural::Decoded d = MM_ISA_DECODE(code.data(), code.size(), scan); + REQUIRE(d.len > 0); + if (d.hasTarget && d.target >= 0 && static_cast(d.target) < code.size()) + isTarget[static_cast(d.target)] = true; + scan += d.len; + } + } + + uint32_t dirty = 0; // destroyed by the most recent call, not yet rewritten + uint32_t callsSeen = 0; + size_t pc = 0; + while (pc < code.size()) { + if (isTarget[pc]) dirty = 0; + const mm_structural::Decoded d = MM_ISA_DECODE(code.data(), code.size(), pc); + REQUIRE(d.len > 0); + for (uint8_t r = 0; r < 16; r++) + if ((d.readsMask & dirty) & (1u << r)) { + INFO("at byte ", pc, ": reads a", int(r), + " which a call destroyed and nothing restored"); + CHECK(false); + } + dirty &= ~d.writesMask; // a write makes it live again + if (d.isCall) { dirty = (MM_ISA_CALL_CLOBBER); callsSeen++; } + pc += d.len; + } + // A call MUST have been decoded, or the walk proved nothing: `dirty` stays zero and every + // read passes. Not hypothetical, the first version of this decoder tested the wrong byte + // order for callx8, matched no call at all, and went green on everything. + CHECK(callsSeen > 0); + } +} diff --git a/test/unit/core/unit_moonlive_codegen_riscv.cpp b/test/unit/core/unit_moonlive_codegen_riscv.cpp new file mode 100644 index 00000000..dba05bdf --- /dev/null +++ b/test/unit/core/unit_moonlive_codegen_riscv.cpp @@ -0,0 +1,55 @@ +// @module MoonLive + +// The RISC-V (ESP32-P4) backend's emitted code, checked on the development machine. +// See moonlive_device_codegen.inc for why this exists and unit_moonlive_codegen_xtensa.cpp for how +// the target guard is defined so the real emitter runs on this host. + +#include "doctest.h" + +// System and standard headers FIRST, at global scope. The backend below is wrapped in a namespace, +// and anything it includes for the first time would otherwise be declared INSIDE that namespace — +// which under GCC breaks both `std::memcpy` (not found where the backend calls it) and the system +// `ssize_t` typedef that drags in. Including them here means the namespace only ever wraps +// OUR code, which is all it is meant to wrap. +#include +#include +#include +#include +#include +#include + + +// The SHARED front end first, at real `mm::moonlive` scope: the backend below drags in the IR +// headers, and once their include guards are set the compiler header would otherwise resolve its +// types inside the wrapper namespace instead. +#include "core/moonlive/MoonLiveCompiler.h" +#include "core/moonlive/MoonLiveIr.h" +#include "core/moonlive/moonlive_emit.h" +#include "core/moonlive/MoonLiveSpill.h" + +namespace mm_riscv_backend { +// The IR, the spill pass and the platform seam are SHARED — one definition each in the binary. +// Pull them into scope so the backend's unqualified references resolve to those, and only the +// ISA-specific classes below end up local to this namespace. +namespace mm { using namespace ::mm; using namespace ::mm::moonlive; + namespace moonlive { using namespace ::mm::moonlive; + using ::mm::moonlive::spillToBudget; } } +#define __riscv 1 +#include "platform/esp32/moonlive_asm_riscv.h" +#include "platform/esp32/moonlive_asm_riscv.cpp" +#include "platform/esp32/moonlive_lower_riscv.cpp" +#undef __riscv +} // namespace mm_riscv_backend + +#define MM_ISA_NAME "RISC-V" +// Golden values, recorded from this backend. See the .inc for what they are and are not. +#define MM_GOLD_GRID_LEN 380u +#define MM_GOLD_FX_LEN 152u +#define MM_GOLD_FILLLOOP_LEN 328u // fits on every backend since the host args moved to the frame +#define MM_GOLD_FXLOOP_LEN 228u +#define MM_GOLD_FXLOOP_HASH 1359103091u +#define MM_GOLD_FX_HASH 2560785008u +#define MM_ISA_LOWER mm_riscv_backend::mm::moonlive::lowerToBytes +#include "moonlive_device_codegen.inc" + + diff --git a/test/unit/core/unit_moonlive_codegen_xtensa.cpp b/test/unit/core/unit_moonlive_codegen_xtensa.cpp new file mode 100644 index 00000000..6bbc7c8d --- /dev/null +++ b/test/unit/core/unit_moonlive_codegen_xtensa.cpp @@ -0,0 +1,181 @@ +// @module MoonLive + +// The Xtensa (ESP32 classic / S3) backend's emitted code, checked on the development machine. +// See moonlive_device_codegen.inc for why this exists. +// +// The backend is `#if defined(__XTENSA__)`, so on this host it normally compiles to nothing. +// Defining the macro and compiling the sources here runs the REAL emitter, and compileSource's +// `lower` seam points the shared front end at it — one function pointer rather than a second copy +// of the compiler. The firmware build never sees this file, so a device image still holds exactly +// one backend definition. + +#include "doctest.h" + +// System and standard headers FIRST, at global scope. The backend below is wrapped in a namespace, +// and anything it includes for the first time would otherwise be declared INSIDE that namespace — +// which under GCC breaks both `std::memcpy` (not found where the backend calls it) and the system +// `ssize_t` typedef that drags in. Including them here means the namespace only ever wraps +// OUR code, which is all it is meant to wrap. +#include +#include +#include +#include +#include +#include + + +// The SHARED front end first, at real `mm::moonlive` scope: the backend below drags in the IR +// headers, and once their include guards are set the compiler header would otherwise resolve its +// types inside the wrapper namespace instead. +#include "core/moonlive/MoonLiveCompiler.h" +#include "core/moonlive/MoonLiveIr.h" +#include "core/moonlive/moonlive_emit.h" +#include "core/moonlive/MoonLiveSpill.h" + +namespace mm_xtensa_backend { +// The IR, the spill pass and the platform seam are SHARED — one definition each in the binary. +// Pull them into scope so the backend's unqualified references resolve to those, and only the +// ISA-specific classes below end up local to this namespace. +namespace mm { using namespace ::mm; using namespace ::mm::moonlive; + namespace moonlive { using namespace ::mm::moonlive; + using ::mm::moonlive::spillToBudget; } } +#define __XTENSA__ 1 +#include "platform/esp32/moonlive_asm_xtensa.h" +#include "platform/esp32/moonlive_asm_xtensa.cpp" +#include "platform/esp32/moonlive_lower_xtensa.cpp" +#undef __XTENSA__ +} // namespace mm_xtensa_backend + +#define MM_ISA_NAME "Xtensa" +// Golden values, recorded from this backend. See the .inc for what they are and are not. +#define MM_GOLD_GRID_LEN 227u +#define MM_GOLD_FX_LEN 105u +#define MM_GOLD_FILLLOOP_LEN 254u // fits now: the host arguments left the register file +#define MM_GOLD_FXLOOP_LEN 190u +#define MM_GOLD_FXLOOP_HASH 307181036u +#define MM_GOLD_FX_HASH 2796457628u +#define MM_ISA_LOWER mm_xtensa_backend::mm::moonlive::lowerToBytes +#include "moonlive_device_codegen.inc" + +// --- the structural checker's Xtensa decoder ------------------------------------------------- +// +// Xtensa is variable-length: an instruction is TWO bytes when its op0 field (the low nibble of +// byte 0) is >= 8, otherwise THREE. That one rule is enough to walk the stream, and walking it is +// what makes "does this branch land on a boundary" answerable at all. +// The bytes the register-window spill hardware owns at the top of a frame, which no emitted +// slot may reach into. Stated independently of the emitter ON PURPOSE: kWindowSaveReserve in +// moonlive_asm_xtensa.cpp derives its value from the call it emits and carries the two bands +// and their `s32e` offsets, so a checker that imported it could never catch that derivation +// going wrong. This number is the expected answer, held separately. +#define MM_ISA_RESERVED_TOP 32u +// call8 rotates the window by 8, so a8..a15 are the callee's. call() saves and restores +// a8/a9/a10/a11 around it, so only a12..a15 are actually destroyed on the far side. +#define MM_ISA_CALL_CLOBBER 0xf000u // a12..a15 +#define MM_ISA_DECODE mm_xtensa_decode + +#include "moonlive_structural.h" +namespace { +mm_structural::Decoded mm_xtensa_decode(const uint8_t* p, size_t n, size_t pc); +} +#include "moonlive_structural.inc" +namespace { +mm_structural::Decoded mm_xtensa_decode(const uint8_t* p, size_t n, size_t pc) { + mm_structural::Decoded d; + if (pc >= n) return d; + const uint8_t b0 = p[pc]; + const uint8_t op0 = b0 & 0x0f; + d.len = (op0 >= 8) ? 2 : 3; + if (pc + d.len > n) { d.len = 0; return d; } + + // s32i / l32i aR, a1, #off4 — byte1 is 0x61 (store) or 0x21 (load) when the base is a1, and + // byte2 counts 4-byte words. These are the frame accesses the first check exists for. + if (d.len == 3 && (p[pc + 1] == 0x61 || p[pc + 1] == 0x21)) { + d.hasFrameOff = true; + d.frameOff = static_cast(p[pc + 2]) * 4u; + } + // addi aD, a1, #off — byte1 has the 0xc0 marker and the base register in its low nibble. + if (d.len == 3 && (p[pc + 1] & 0xf0) == 0xc0 && (p[pc + 1] & 0x0f) == 1) { + d.hasFrameOff = true; + d.frameOff = p[pc + 2]; + } + // entry a1, N — BRI12: byte0 op0 == 6 with the 0x30 marker in byte1's high nibble. The 12-bit + // immediate at bits 12..23 counts EIGHT-byte units, so this is the frame the routine allocated. + if (d.len == 3 && op0 == 0x6 && (b0 & 0xf0) == 0x30) { + const uint32_t w = uint32_t(b0) | (uint32_t(p[pc + 1]) << 8) | (uint32_t(p[pc + 2]) << 16); + d.hasFrameAlloc = true; + d.frameAlloc = ((w >> 12) & 0xfff) * 8u; + } + // j — op0 == 6 with bits 4..5 of byte0 zero. The 18-bit signed displacement sits at bits 6..23 + // of the 24-bit word and is relative to the byte AFTER the instruction... on Xtensa, PC + 4. + if (d.len == 3 && (b0 & 0x3f) == 0x06) { + const uint32_t w = uint32_t(b0) | (uint32_t(p[pc + 1]) << 8) | (uint32_t(p[pc + 2]) << 16); + int32_t off = static_cast(w >> 6) & 0x3ffff; + if (off & 0x20000) off -= 0x40000; // sign-extend 18 bits + d.hasTarget = true; + d.target = static_cast(pc) + 4 + off; + } + // Conditional branches (BRI8): op0 == 7, byte2 is a signed 8-bit displacement, PC + 4 relative. + if (d.len == 3 && op0 == 0x7) { + d.hasTarget = true; + d.target = static_cast(pc) + 4 + static_cast(p[pc + 2]); + d.readsMask |= (1u << ((p[pc] >> 4) & 0xf)) | (1u << (p[pc + 1] & 0xf)); // s, t + } + // callx8 aN, emitted as the 24-bit word 0x0000e0 | (reg << 8), which is LITTLE-ENDIAN in the + // stream: bytes e0, reg, 00. Testing p[0]==0x00 instead of p[0]==0xe0 matched nothing, so the + // clobber check silently never saw a call, it passed for the same reason a broken analyser + // reports zero findings. + if (d.len == 3 && p[pc] == 0xe0 && p[pc + 2] == 0x00) { + d.isCall = true; + d.readsMask |= 1u << (p[pc + 1] & 0xf); + } + // s32i/l32i: byte0 high nibble is the value register (read on store, written on load). + if (d.len == 3 && p[pc + 1] == 0x61) d.readsMask |= 1u << ((p[pc] >> 4) & 0xf); // store + if (d.len == 3 && p[pc + 1] == 0x21) d.writesMask |= 1u << ((p[pc] >> 4) & 0xf); // load + // movi aD, #imm8, byte1 == 0xa0, byte0 high nibble is the destination. + if (d.len == 3 && p[pc + 1] == 0xa0) d.writesMask |= 1u << ((p[pc] >> 4) & 0xf); + // add.n / mov.n (narrow): (d<<12)|(a<<8)|(b<<4)|op. + if (d.len == 2) { + const uint16_t w = uint16_t(p[pc]) | (uint16_t(p[pc + 1]) << 8); + const uint8_t op = w & 0xf; + if (op == 0xa || op == 0xd) { // add.n, mov.n + d.writesMask |= 1u << ((w >> 12) & 0xf); + d.readsMask |= 1u << ((w >> 8) & 0xf); + if (op == 0xa) d.readsMask |= 1u << ((w >> 4) & 0xf); + } + } + return d; +} +} // namespace + + +// A register outside the backend's map is a value written somewhere the program does not own. The +// map is a2..a11: a12/a13 are call scratch and the store8 address register, and a14/a15 carry this +// routine's own retw.n return linkage — using those two as general registers corrupted the return +// path and produced `Guru Meditation (IllegalInstruction)` the moment a scripted layout ran. +// Asserted against the map itself rather than a copy, so the check cannot drift from its subject. +// a8..a11 ARE in the map and that is deliberate: call8 rotates them out, so XtensaAssembler::call +// saves and restores exactly a8/a9/a10/a11 around every call. The map is legal because of that +// save-set, so the two have to agree — the count below is what fails if a register is added to the +// map without being added to the save-set. +TEST_CASE("the Xtensa vreg map names only registers the windowed ABI leaves free") { + uint8_t n = 0; + const uint8_t* map = mm_xtensa_backend::mm::moonlive::xtRegMap(n); + REQUIRE(n == 10); // a2..a11 — widening this needs call()'s save-set widened too + for (uint8_t i = 0; i < n; i++) { + INFO("vreg R" << int(i) << " -> a" << int(map[i])); + CHECK(map[i] >= 2); + CHECK(map[i] <= 11); + } + // Two vregs sharing a machine register silently alias: one value overwrites the other and the + // program computes with whichever was written last. + for (uint8_t i = 0; i < n; i++) + for (uint8_t j = static_cast(i + 1); j < n; j++) { + INFO("R" << int(i) << " and R" << int(j) << " both name a" << int(map[i])); + CHECK(map[i] != map[j]); + } +} + +// A change DETECTOR, not a correctness claim: emitted length is a cheap fingerprint that moves +// whenever codegen does. A failure here means "read the diff and decide", not "you broke it" — +// update the number and say why in the commit. It exists because the alternative way to notice an +// emission change was to flash a board and watch it reset. diff --git a/test/unit/core/unit_moonlive_compiler.cpp b/test/unit/core/unit_moonlive_compiler.cpp index 1ce3692b..62ad5c4c 100644 --- a/test/unit/core/unit_moonlive_compiler.cpp +++ b/test/unit/core/unit_moonlive_compiler.cpp @@ -310,6 +310,29 @@ TEST_CASE("a for loop's condition and step must name the loop variable") { #endif } +// The op array is sized to the script, so `count` is a uint16_t — and every loop over it has to be +// one too. A uint8_t counter wrapped at 256 ops and spun forever, which on a device is a watchdog +// reset from a script that merely got long. Found by bisecting: 60 statements fine, 80 hung. +TEST_CASE("a long script compiles or refuses, but never spins") { + uint8_t out[16384]; + // A long ARITHMETIC chain, not many statements: each `+ 1` is one cheap op, so this passes 256 + // IR ops while staying inside the code buffer. Repeated statements hit the code ceiling first + // and return before the wrap, which is why they do not pin this. + std::string many = "addLight(1"; + for (int i = 0; i < 200; i++) many += " + 1"; + many += ", 0, 0);"; + auto r = moonlive::compileSource(many.c_str(), kTable, kSys, out, sizeof(out)); + // Either answer is fine — what is NOT fine is never returning, which is what this pins. + CHECK((r.ok || std::strlen(r.error) > 0)); + + // And the sanity bound still refuses a runaway rather than trying to allocate for it. + std::string absurd; + for (int i = 0; i < 3000; i++) absurd += "addLight(1, 0, 0);"; + auto big = moonlive::compileSource(absurd.c_str(), kTable, kSys, out, sizeof(out)); + CHECK_FALSE(big.ok); + CHECK(std::string(big.error) == "script too large"); +} + TEST_CASE("compileSource: malformed control declarations fail with a diagnostic, never crash") { uint8_t out[768]; const char* bad[] = { diff --git a/test/unit/core/unit_moonlive_fill.cpp b/test/unit/core/unit_moonlive_fill.cpp index bdc34bf6..221fc682 100644 --- a/test/unit/core/unit_moonlive_fill.cpp +++ b/test/unit/core/unit_moonlive_fill.cpp @@ -7,6 +7,7 @@ #include "platform/platform.h" #include +#include #include // MoonLive Stage 1a: the load-bearing slice — emit a fixed-color fill as native code, @@ -232,4 +233,84 @@ TEST_CASE("MoonLive controls: free() releases the arena (no stale slot after rel REQUIRE(eng.controlSlot(0) != nullptr); CHECK(*eng.controlSlot(0) == 5); // re-seeded from default } + +// line(x1, y1, x2, y2, r, g, b), the SEVEN-argument builtin and the widest call a script makes. +// This is the behavioral pin for the args-array call ABI at an arity past the register file: +// all seven values must arrive intact, in order, through the same staging every backend uses. +TEST_CASE("MoonLive line() draws a horizontal segment through the installed canvas") { + moonlive::MoonLive eng; + REQUIRE(eng.compile("line(1, 0, 3, 0, 10, 20, 200);", kCtrlTable, kSys)); + REQUIRE(eng.ok()); + + // A 5-wide single-row canvas, sentinel-filled so an errant write shows. + std::vector buf(5 * 3, 0xAB); + moonlive::setDrawCanvas(draw::Canvas{buf.data(), buf.size(), Coord3D{5, 1, 1}, 3}); + eng.run(buf.data(), 5, 3, /*t*/ 0); + moonlive::setDrawCanvas({}); + + CHECK(buf[0*3] == 0xAB); // before the segment: untouched + for (int i = 1; i <= 3; i++) { + CHECK(buf[i*3 + 0] == 10); + CHECK(buf[i*3 + 1] == 20); + CHECK(buf[i*3 + 2] == 200); + } + CHECK(buf[4*3] == 0xAB); // after the segment: untouched +} + +// No canvas installed (a layout or modifier host) → every draw call must no-op, not write +// through a stale or foreign pointer. The detach after each run is what this pins. +TEST_CASE("MoonLive line() without a canvas draws nothing") { + moonlive::MoonLive eng; + REQUIRE(eng.compile("line(0, 0, 4, 0, 255, 255, 255);", kCtrlTable, kSys)); + std::vector buf(5 * 3, 0xAB); + eng.run(buf.data(), 5, 3, 0); // canvas never installed + for (uint8_t v : buf) CHECK(v == 0xAB); +} + +// A binding that installs nothing (a modifier) must not consume one of the two per-thread slots +// just by RUNNING a script that draws. Slots are claimed on install and released on detach, so a +// claim taken on the READ path is never given back. Two such threads exhaust the table, and from +// then on every install silently fails: the render thread's line() goes dark with nothing to see +// but a stopped drawing. Two threads is the smallest case that shows it, because a single thread +// re-uses the slot it already leaked. +TEST_CASE("a script that draws with nothing installed does not consume a per-thread slot") { + const char* kDraws = "line(0, 0, 4, 0, 255, 255, 255);"; + auto runWithNothingInstalled = [&] { + moonlive::MoonLive eng; + REQUIRE(eng.compile(kDraws, kCtrlTable, kSys)); + std::vector buf(5 * 3, 0xAB); + eng.run(buf.data(), 5, 3, 0); // no canvas, no sink: the modifier's shape + for (uint8_t v : buf) CHECK(v == 0xAB); // and it draws nothing + }; + // CONCURRENT, not sequential: joined threads can be handed the same identity by the OS, and + // one identity claims one slot. Two live at once is what fills the table. + std::thread a(runWithNothingInstalled), b(runWithNothingInstalled); + a.join(); + b.join(); + + // Both slots must still be claimable. A leaked claim from either thread above would make this + // install a no-op and the canvas would stay at its sentinel. + moonlive::MoonLive eng; + REQUIRE(eng.compile(kDraws, kCtrlTable, kSys)); + std::vector canvasBuf(5 * 3, 0xAB); + moonlive::setDrawCanvas(draw::Canvas{canvasBuf.data(), canvasBuf.size(), Coord3D{5, 1, 1}, 3}); + eng.run(canvasBuf.data(), 5, 3, 0); + moonlive::setDrawCanvas({}); + for (int i = 0; i < 5; i++) CHECK(canvasBuf[i*3] == 255); // the install took effect +} + +// Script arithmetic is unsigned, so a coordinate that went "negative" arrives as a huge value. +// The builtin clamps endpoints to the canvas, which keeps the draw instant and on-grid instead +// of sending the line walker on a billions-of-steps march (robustness: any input, degrade +// visibly, never stall the render thread). +TEST_CASE("MoonLive line() clamps out-of-range endpoints to the canvas edge") { + moonlive::MoonLive eng; + // x2 = 0 - 1 wraps to the full unsigned word; the clamp pins it to the last column. + REQUIRE(eng.compile("line(0, 0, 0 - 1, 0, 7, 8, 9);", kCtrlTable, kSys)); + std::vector buf(4 * 3, 0xAB); + moonlive::setDrawCanvas(draw::Canvas{buf.data(), buf.size(), Coord3D{4, 1, 1}, 3}); + eng.run(buf.data(), 4, 3, 0); + moonlive::setDrawCanvas({}); + for (int i = 0; i < 4; i++) CHECK(buf[i*3 + 0] == 7); // the whole row drawn, promptly +} #endif // MM_MOONLIVE_HAS_HOST_JIT diff --git a/test/unit/core/unit_moonlive_spill.cpp b/test/unit/core/unit_moonlive_spill.cpp new file mode 100644 index 00000000..8c244914 --- /dev/null +++ b/test/unit/core/unit_moonlive_spill.cpp @@ -0,0 +1,334 @@ +// @module MoonLive + +#include "doctest.h" +#include "core/moonlive/MoonLiveCompiler.h" +#include "core/moonlive/MoonLiveSpill.h" +#include "core/moonlive/moonlive_emit.h" +#include "light/moonlive/MoonLiveBuiltins_light.h" +#include "platform/platform.h" + +#include +#include + +// MoonLive register allocation — spilling to the call frame (core/moonlive/MoonLiveSpill.cpp). +// +// The governing risk is that only the arm64 backend is ever EXECUTED by tests, while the register +// walls these tests are about are tightest on Xtensa. The answer is the squeezed budget: compiling +// the same script at the host's real budget and at a deliberately smaller one must render identical +// pixels — spilling is by definition a change of storage, not of meaning. That turns the hardest +// algorithm in the compiler from a hardware-only proposition into something a unit test settles. + +using namespace mm; + +static moonlive::BuiltinTable kT = moonlive::lightBuiltins(); +static moonlive::SysVarTable kSys = moonlive::modifierSysVars(); + +#if MM_MOONLIVE_HAS_HOST_JIT + +namespace { + +using CtrlFn = void (*)(uint8_t*, uint32_t, uint8_t, uint32_t, const uint8_t*); + +// Compile `src` at `budget` (null = the backend's own) and render it over `nLights` lights. +// `ok` reports whether the compile succeeded, so a test can assert BOTH that a squeezed budget still +// compiles and what it produced. +std::vector renderAt(const char* src, int nLights, const moonlive::RegBudget* budget, + bool& ok, uint32_t t = 0) { + uint8_t code[moonlive::kCodeCap]; + auto r = moonlive::compileSource(src, kT, kSys, code, sizeof(code), budget); + ok = r.ok; + std::vector buf(static_cast(nLights) * 3, 0); + if (!r.ok) return buf; + void* blk = platform::allocExec(r.len); + REQUIRE(blk != nullptr); + platform::writeExec(blk, code, r.len); + uint8_t arena[moonlive::kArenaBytes] = {}; + for (uint8_t i = 0; i < r.controlCount; i++) arena[r.controls[i].offset] = r.controls[i].def; + reinterpret_cast(blk)(buf.data(), static_cast(nLights), 3, t, arena); + platform::freeExec(blk, r.len); + return buf; +} + +// A budget with FEWER registers than arm64's fourteen, so the allocator is forced to spill on the +// only backend a test can run. Nine leaves four values allocatable once the five fixed ABI vregs and +// the four reload temps are taken out — enough for a real script, tight enough that anything +// interesting overflows. `slots` matches what the host frame can address. +constexpr moonlive::RegBudget squeezed(uint8_t regs, uint8_t reserved) { + return moonlive::RegBudget{regs, reserved, 16}; +} + +int litCount(const std::vector& b) { + int n = 0; + for (size_t i = 0; i + 2 < b.size(); i += 3) if (b[i] || b[i + 1] || b[i + 2]) n++; + return n; +} + +} // namespace + +// THE key test: spilling changes where a value lives, never what the program computes. +TEST_CASE("a script renders identical pixels at a squeezed register budget as at the full one") { + // Enough live values that a nine-register budget cannot hold them all: three independent + // colour components plus two loop-carried values. + const char* src = "for (i = 0; i < 6; i = i + 1) { setRGB(i, i + 1, i + 2, i + 3); }"; + + bool fullOk = false, tightOk = false; + auto full = renderAt(src, 8, nullptr, fullOk); + const auto tightBudget = squeezed(11, 1); // 11 regs, StoreElem's one scratch + auto tight = renderAt(src, 8, &tightBudget, tightOk); + + CHECK(fullOk); + CHECK(tightOk); + CHECK(full == tight); + CHECK(litCount(full) == 6); // and it actually did something +} + +// The back-edge case. Naive first-def-to-last-use intervals look dead early in a loop body, so the +// allocator would hand a counter's register away and the next iteration would read someone else's +// value — placing lights twice, or not at all. Nested, so the extension has to apply innermost-first. +TEST_CASE("a nested loop at a squeezed budget places every light exactly once") { + const char* src = + "for (i = 0; i < 4; i = i + 1) {\n" + " for (j = 0; j < 4; j = j + 1) {\n" + " setRGB(i * 4 + j, 200, 100, 50);\n" + " }\n" + "}\n"; + + bool fullOk = false, tightOk = false; + auto full = renderAt(src, 16, nullptr, fullOk); + const auto tightBudget = squeezed(11, 1); + auto tight = renderAt(src, 16, &tightBudget, tightOk); + + CHECK(fullOk); + CHECK(tightOk); + CHECK(full == tight); + // every one of the 16 cells written, each exactly once — a counter that lost its register + // across the back edge writes the wrong cells, and the count or the colours would differ + CHECK(litCount(tight) == 16); + for (int i = 0; i < 16; i++) { + CHECK(tight[i * 3 + 0] == 200); + CHECK(tight[i * 3 + 1] == 100); + CHECK(tight[i * 3 + 2] == 50); + } +} + +// Slots live in the routine's OWN frame, and call() moves sp underneath it. Addressing a slot from +// the frame pointer rather than sp is what makes this hold; from sp it would read into the callee's +// saved registers instead. +TEST_CASE("a spilled value survives a host call and is still correct afterwards") { + // `keep` is defined before the call and used after it, so it must be live ACROSS random16 — + // and at a squeezed budget it is one of the values that has nowhere to live but a slot. + const char* src = + "uint8_t idx = 5; // @control 0..15\n" + "for (i = 0; i < 3; i = i + 1) {\n" + " setRGB(idx + i, random16(1) + 111, i + 1, 222);\n" + "}\n"; + + bool fullOk = false, tightOk = false; + auto full = renderAt(src, 16, nullptr, fullOk); + const auto tightBudget = squeezed(11, 1); + auto tight = renderAt(src, 16, &tightBudget, tightOk); + + CHECK(fullOk); + CHECK(tightOk); + CHECK(full == tight); + CHECK(litCount(tight) == 3); + // random16(1) is always 0, so red is exactly 111 — a slot clobbered by the call shows up here + for (int i = 0; i < 3; i++) { + CHECK(tight[(5 + i) * 3 + 0] == 111); + CHECK(tight[(5 + i) * 3 + 1] == i + 1); + CHECK(tight[(5 + i) * 3 + 2] == 222); + } +} + +// The control arena pointer (kArg4) is a fixed ABI vreg the allocator must never reassign or spill. +// If it did, a control read after a spill would load from a register holding something else. +TEST_CASE("a declared control still reads live at a squeezed budget") { + const char* src = + "uint8_t pos = 0; // @control 0..15\n" + "for (i = 0; i < 2; i = i + 1) {\n" + " setRGB(pos + i, 10, 20, 30);\n" + "}\n"; + uint8_t code[moonlive::kCodeCap]; + const auto tightBudget = squeezed(11, 1); + auto r = moonlive::compileSource(src, kT, kSys, code, sizeof(code), &tightBudget); + REQUIRE(r.ok); + REQUIRE(r.controlCount == 1); + void* blk = platform::allocExec(r.len); + REQUIRE(blk != nullptr); + platform::writeExec(blk, code, r.len); + auto fn = reinterpret_cast(blk); + + uint8_t arena[moonlive::kArenaBytes] = {}; + for (uint8_t v : {uint8_t(0), uint8_t(4), uint8_t(9)}) { + std::vector buf(16 * 3, 0); + arena[0] = v; + fn(buf.data(), 16, 3, 0, arena); + CHECK(buf[v * 3 + 0] == 10); // the control still selects the pixel, live + CHECK(buf[(v + 1) * 3 + 0] == 10); + CHECK(litCount(buf) == 2); + } + platform::freeExec(blk, r.len); +} + +#endif // MM_MOONLIVE_HAS_HOST_JIT + +// FAIL, NEVER MISCOMPILE. A budget with no room for the fixed ABI vregs plus the reload temps, and +// one with no frame slots to spill INTO, must both refuse — silently emitting code that names a +// register the target does not have is the failure this whole pass exists to make impossible. +TEST_CASE("an impossible register budget refuses the compile instead of emitting wrong code") { + // A script whose live values genuinely exceed the budgets below, so each really does have to + // spill and really does have nowhere to put the result. + const char* src = + "for (i = 0; i < 4; i = i + 1) {\n" + " for (j = 0; j < 4; j = j + 1) {\n" + " setRGB(i * 4 + j, 200, 100, 50);\n" + " }\n" + "}\n"; + uint8_t code[2048]; +#if MM_MOONLIVE_HAS_HOST_JIT + // Only where a backend exists: on x86-64 there is none, so the default lowerer emits nothing and + // a normal compile legitimately fails. The refusals below still mean what they say everywhere. + REQUIRE(moonlive::compileSource(src, kT, kSys, code, sizeof(code)).ok); // it does compile normally +#endif + + // Fewer registers than the reload temps need. The host arguments no longer count against this + // — they live in frame slots and hold a register only for the parking store at entry — so the + // floor is the temps alone, and a budget at or below it has nothing to compute with. + const moonlive::RegBudget noRoom{4, 1, moonlive::kTotalSlots}; + CHECK_FALSE(moonlive::compileSource(src, kT, kSys, code, sizeof(code), &noRoom).ok); + + const moonlive::RegBudget noSlots{11, 1, 0}; // room to allocate, nowhere to spill INTO + CHECK_FALSE(moonlive::compileSource(src, kT, kSys, code, sizeof(code), &noSlots).ok); +} + +// A program that fits is left byte-identical: the allocator must not exist as far as a non-spilling +// script is concerned, or every shipped script pays for a feature it never uses. +TEST_CASE("a program that already fits is untouched by the register allocator") { + moonlive::IrProgram ir; + REQUIRE(ir.reserve(8)); + REQUIRE(ir.push({moonlive::IrOp::Const, moonlive::kFirstTemp, 0, 0, 0, 0, 42, nullptr, {}})); + REQUIRE(ir.push({moonlive::IrOp::Mov, moonlive::VReg(moonlive::kFirstTemp + 1), + moonlive::kFirstTemp, 0, 0, 0, 0, nullptr, {}})); + const uint16_t before = ir.count; + + uint8_t slots = 0xff; + CHECK(moonlive::spillToBudget(ir, moonlive::RegBudget{14, 0, 16}, slots)); + CHECK(slots == 0); // nothing spilled + CHECK(ir.count == before); // and not one op inserted + CHECK(ir.ops[0].dst == moonlive::kFirstTemp); // nor renumbered +} + +// Loop extension, pinned on its own — the one part of the allocator no script in this grammar can +// currently exercise, and the part whose failure is a silent miscompile rather than a refusal. +// +// A value defined before a loop and read only EARLY in the body has a naive live range that ends at +// that read. Every later value then looks free to take its register, and the next iteration reads +// whatever took it. The IR is built by hand because the wall a script hits first is its own variable +// count: by the time a source program names enough live values to force allocation, the values are +// short-lived ones the naive analysis already gets right. Extension is therefore untestable through +// the front end today, and this is what stands in for it until script-local functions make longer +// live ranges expressible. +TEST_CASE("a value live across a loop keeps its storage for the whole loop") { + using namespace moonlive; + IrProgram ir; + REQUIRE(ir.reserve(64)); + const VReg carried = kFirstTemp; // defined before the loop, read early inside + const VReg ctr = VReg(kFirstTemp + 1), lim = VReg(kFirstTemp + 2); + REQUIRE(ir.push({IrOp::Const, carried, 0, 0, 0, 0, 7, nullptr, {}})); + REQUIRE(ir.push({IrOp::Const, ctr, 0, 0, 0, 0, 0, nullptr, {}})); + REQUIRE(ir.push({IrOp::Const, lim, 0, 0, 0, 0, 4, nullptr, {}})); + REQUIRE(ir.push({IrOp::Label, 0, 0, 0, 0, 0, 0, nullptr, {}})); + REQUIRE(ir.push({IrOp::Add, VReg(kFirstTemp + 3), carried, ctr, 0, 0, 0, nullptr, {}})); + // A tail of further values, enough that the register file cannot hold them all at once. + for (int i = 4; i < 14; i++) + REQUIRE(ir.push({IrOp::AddImm, VReg(kFirstTemp + i), VReg(kFirstTemp + 3), + 0, 0, 0, i, nullptr, {}})); + REQUIRE(ir.push({IrOp::AddImm, ctr, ctr, 0, 0, 0, 1, nullptr, {}})); + REQUIRE(ir.push({IrOp::BranchNe, 0, ctr, lim, 0, 0, 0, nullptr, {}})); // back edge to label 0 + + const moonlive::RegBudget budget{14, 1, 16}; + REQUIRE(ir.vregsUsed > budget.allocatable()); // the program really does not fit + uint8_t slots = 0; + REQUIRE(spillToBudget(ir, budget, slots)); + + // Nineteen values cannot occupy fourteen registers, so SOMETHING has to reach the frame. Without + // extension every interval ends at its early read, the allocator concludes the program fits, and + // it spills nothing at all — the exact wrong answer this asserts against. + CHECK(slots > 0); + + // And the decisive property, whatever the allocator chose: no op may name a register the target + // does not have. This is the invariant the whole pass exists to guarantee. + for (uint16_t i = 0; i < ir.count; i++) { + const auto& in = ir.ops[i]; + const bool slotRef = in.op == IrOp::Spill || in.op == IrOp::Reload; + CHECK(in.dst < budget.allocatable()); + if (!slotRef) { + CHECK(in.a < budget.allocatable()); + CHECK(in.b < budget.allocatable()); + } + if (slotRef) CHECK(in.imm < slots); // and every slot is one the prologue reserves + } +} + +#if MM_MOONLIVE_HAS_HOST_JIT + +// The shape that resets both Xtensa boards: a SYSTEM VARIABLE read as a loop bound, with a HOST CALL +// in the body. Bench-bisected, each ingredient alone is fine, and only the three together fail: +// +// loop, constant bound, call in body -> runs +// loop, @control bound, call in body -> runs +// `width` read, no loop -> runs +// `width` loop, no call in body -> runs +// `width` loop WITH a call in body -> LoadProhibited inside the emitted code +// +// A system variable lives in the controls arena, reached through kArg4, a host argument that now +// lives in a frame slot and is reloaded at each use. Run here on the one backend a test can execute, +// so the failure is debuggable in a process rather than from a crash dump. +TEST_CASE("a system variable read in a loop survives a host call in that loop") { + // `width` is the EFFECT vocabulary, not the modifier one this file's kSys uses. + static moonlive::SysVarTable fxSys = moonlive::effectSysVars(); + // Green is a NONZERO CONSTANT, and red keeps the host call. Two reasons, both learned the hard + // way: random16 is one LCG shared by the whole process, so a red channel that can draw 0 makes + // "is this light lit" depend on how many draws earlier tests made, and a green of 0 everywhere + // means a loop that runs PAST width still writes 0 there, so the past-width check could not + // detect the runaway it is named for. A constant 7 fixes both while keeping the call, which is + // the ingredient this test exists for. + const char* src = "for (x = 0; x < width; x = x + 1) { setRGB(x, random16(256), 7, 0); }\n"; + + // At the host's full budget AND at a squeezed one: Xtensa has ten registers where arm64 has + // fourteen, so the squeezed run is the closest a host test gets to the pressure the device is + // under, and pressure is what decides whether kArg4 stays in a register or goes to a slot. + uint8_t code[2048]; + const auto tight = squeezed(11, 1); + auto r = moonlive::compileSource(src, kT, fxSys, code, sizeof(code), &tight); + REQUIRE(r.ok); + + void* blk = platform::allocExec(r.len); + REQUIRE(blk != nullptr); + platform::writeExec(blk, code, r.len); + + // The arena the binding would hand over: controls at their defaults, and `width` where + // MoonLiveEffect::tick writes it. + uint8_t arena[moonlive::kArenaBytes] = {}; + for (uint8_t i = 0; i < r.controlCount; i++) arena[r.controls[i].offset] = r.controls[i].def; + const uint8_t kWidth = 8; + arena[moonlive::kSysWidth] = kWidth; + + std::vector buf(16 * 3, 0); + reinterpret_cast(blk)(buf.data(), 16, 3, 0, arena); + platform::freeExec(blk, r.len); + + // Exactly `width` lights written, and only the first `width`: a loop that read a corrupted bound + // either stops early, runs away, or writes through a wrong pointer. Asserted on the CONSTANT + // channel, so the result does not depend on what the shared random16 sequence happens to return. + for (int i = 0; i < kWidth; i++) { + INFO("light ", i, " is inside width and must carry the constant"); + CHECK(buf[i * 3 + 1] == 7); + } + for (int i = kWidth; i < 16; i++) { + INFO("light ", i, " is past width and must be untouched"); + CHECK(buf[i * 3 + 1] == 0); + } +} + +#endif // MM_MOONLIVE_HAS_HOST_JIT diff --git a/test/unit/light/MoonLiveScriptFixture.h b/test/unit/light/MoonLiveScriptFixture.h new file mode 100644 index 00000000..e618c402 --- /dev/null +++ b/test/unit/light/MoonLiveScriptFixture.h @@ -0,0 +1,62 @@ +#pragma once + +#include "doctest.h" +#include "light/moonlive/MoonLiveScriptFile.h" +#include "platform/platform.h" + +#include +#include +#include +#include +#include +#include +#include + +/// Put a script on the filesystem and return its NAME. +/// +/// A scripted module holds a file name now, not the text — so a test that wants to compile +/// something has to write the file first, which is the same path the UI takes when it saves an +/// edit. Testing through the file is the point: it exercises what actually ships, rather than a +/// text-only path production never uses. +/// +/// Each call gets a fresh name, so tests that compile several scripts do not collide. +/// Returns a name valid for the caller's thread until its next call. THREAD-LOCAL, not static: the +/// concurrency test compiles scripts from two threads at once, and a shared counter and buffer would +/// hand both threads the same name — each would then compile the other's script. +/// Every script this fixture wrote, removed when the test process exits. +/// +/// The scripts go in the SAME directory a real install keeps its scripts in — that is what makes the +/// test meaningful — so leaving them behind drops a hundred `t*.mlv` files among the user's own, and +/// the next run adds another hundred. Deleting each file at the end of its test would be wrong: a +/// test compiles the script and then re-reads it through the module, so the file has to outlive the +/// call. Process exit is the first moment they are all certainly finished with. +inline std::vector& mmScriptRegistry() { + static std::vector paths; + static const bool once = [] { + std::atexit([] { + for (const auto& p : mmScriptRegistry()) mm::platform::fsRemove(p.c_str()); + }); + return true; + }(); + (void)once; + return paths; +} + +inline const char* mmWriteScript(const char* text) { + static std::atomic counter{0}; + thread_local char name[32]; + std::snprintf(name, sizeof(name), "t%d.mlv", ++counter); + + char path[96]; + std::snprintf(path, sizeof(path), "%s/%s", mm::moonlive::kScriptDir, name); + mm::platform::fsMkdir(mm::moonlive::kScriptDir); + REQUIRE(mm::platform::fsWriteAtomic(path, text, std::strlen(text))); + { + // The concurrency test writes from two threads, so the shared registry needs a lock even + // though each thread's NAME is its own. + static std::mutex m; + std::lock_guard lock(m); + mmScriptRegistry().emplace_back(path); + } + return name; +} diff --git a/test/unit/light/unit_MoonLiveLayout.cpp b/test/unit/light/unit_MoonLiveLayout.cpp index 907f8350..a575ef19 100644 --- a/test/unit/light/unit_MoonLiveLayout.cpp +++ b/test/unit/light/unit_MoonLiveLayout.cpp @@ -11,6 +11,7 @@ // broken script leaves an empty fixture rather than taking the pipeline down. #include "doctest.h" +#include "MoonLiveScriptFixture.h" #include "light/moonlive/MoonLiveLayout.h" #include "light/moonlive/MoonLiveBuiltins_light.h" #include "platform/platform.h" @@ -38,7 +39,7 @@ namespace { std::vector place(const char* script) { MoonLiveLayout l; l.defineControls(); - if (script) l.setSource(script); + if (script) l.setScript(mmWriteScript(script)); l.prepare(); std::vector out; @@ -71,10 +72,10 @@ TEST_CASE("the light count is known before any coordinate is asked for") { // forEachCoord. A count that came from the walk would arrive too late to be useful. MoonLiveLayout l; l.defineControls(); - l.setSource("uint8_t cols = 5; // @control 1..64\n" + l.setScript(mmWriteScript("uint8_t cols = 5; // @control 1..64\n" "uint8_t rows = 3; // @control 1..64\n" "for (yy = 0; yy < rows; yy = yy + 1) {" - " for (xx = 0; xx < cols; xx = xx + 1) { addLight(xx, yy, 0); } }"); + " for (xx = 0; xx < cols; xx = xx + 1) { addLight(xx, yy, 0); } }")); l.prepare(); CHECK(l.lightCount() == 15); // answered without anyone calling forEachCoord } @@ -83,7 +84,7 @@ TEST_CASE("the count and the coordinates always agree, because one script produc // The property SphereLayout names: count and emit run the same code, so they cannot drift. MoonLiveLayout l; l.defineControls(); - l.setSource("for (i = 0; i < 7; i = i + 1) { addLight(i, 0, 0); }"); + l.setScript(mmWriteScript("for (i = 0; i < 7; i = i + 1) { addLight(i, 0, 0); }")); l.prepare(); std::vector seen; @@ -99,7 +100,7 @@ TEST_CASE("a scripted layout allocates nothing, like every other layout") { // have. The script calls out per light instead, so the only heap here is the compiled program. MoonLiveLayout l; l.defineControls(); - l.setSource("for (i = 0; i < 4096; i = i + 1) { addLight(i, 0, 0); }"); + l.setScript(mmWriteScript("for (i = 0; i < 4096; i = i + 1) { addLight(i, 0, 0); }")); l.prepare(); CHECK(l.lightCount() == 4096); // dynamicBytes is the JIT'd program only — no coordinate storage grows with the light count. @@ -129,7 +130,7 @@ TEST_CASE("a broken script leaves an empty fixture rather than taking the pipeli // fixture reports no lights, the module carries the diagnostic, and the device keeps running. MoonLiveLayout l; l.defineControls(); - l.setSource("for (i = 0; i < 4; i = i + 1) { addLight(i, i"); // unclosed + l.setScript(mmWriteScript("for (i = 0; i < 4; i = i + 1) { addLight(i, i")); // unclosed l.prepare(); CHECK(l.lightCount() == 0); CHECK(l.severity() == MoonModule::Severity::Error); @@ -139,11 +140,11 @@ TEST_CASE("editing the script changes the fixture") { // The live-edit loop: the same module, a new script, a different physical shape. MoonLiveLayout l; l.defineControls(); - l.setSource("for (i = 0; i < 4; i = i + 1) { addLight(i, 0, 0); }"); + l.setScript(mmWriteScript("for (i = 0; i < 4; i = i + 1) { addLight(i, 0, 0); }")); l.prepare(); CHECK(l.lightCount() == 4); - l.setSource("for (i = 0; i < 2; i = i + 1) { addLight(i, 0, 0); }"); + l.setScript(mmWriteScript("for (i = 0; i < 2; i = i + 1) { addLight(i, 0, 0); }")); l.prepare(); CHECK(l.lightCount() == 2); } @@ -173,7 +174,7 @@ TEST_CASE("the scripts the documentation shows all compile") { for (const char* s : fromDocs) { MoonLiveLayout l; l.defineControls(); - l.setSource(s); + l.setScript(mmWriteScript(s)); l.prepare(); INFO("script: " << s); CHECK(l.severity() != MoonModule::Severity::Error); @@ -188,7 +189,7 @@ TEST_CASE("the scripts the documentation shows all compile") { TEST_CASE("a layout answers count and coordinates every time it is asked") { MoonLiveLayout l; l.defineControls(); - l.setSource("for (i = 0; i < 6; i = i + 1) { addLight(i, 0, 0); }"); + l.setScript(mmWriteScript("for (i = 0; i < 6; i = i + 1) { addLight(i, 0, 0); }")); l.prepare(); CHECK(l.lightCount() == 6); @@ -219,7 +220,7 @@ TEST_CASE("a subtraction feeding a loop bound produces the whole value") { MoonLiveLayout l; l.defineControls(); // 10 - 4 must be 6 lights. A widened -1 makes the bound enormous and the count is not 6. - l.setSource("for (i = 0; i < 10 - 4; i = i + 1) { addLight(i, 0, 0); }"); + l.setScript(mmWriteScript("for (i = 0; i < 10 - 4; i = i + 1) { addLight(i, 0, 0); }")); l.prepare(); CHECK(l.lightCount() == 6); @@ -243,22 +244,22 @@ TEST_CASE("a subtraction feeding a loop bound produces the whole value") { TEST_CASE("a scripted control keeps its live value when the script is edited") { MoonLiveLayout l; l.defineControls(); - l.setSource("uint8_t cols = 16; // @control 1..64\n" - "for (i = 0; i < cols; i = i + 1) { addLight(i, 0, 0); }"); + l.setScript(mmWriteScript("uint8_t cols = 16; // @control 1..64\n" + "for (i = 0; i < cols; i = i + 1) { addLight(i, 0, 0); }")); l.prepare(); CHECK(l.lightCount() == 16); // A second script declaring cols at the same offset inherits the live 16, not its own 8. - l.setSource("uint8_t cols = 8; // @control 1..64\n" - "for (i = 0; i < cols; i = i + 1) { addLight(i, 1, 0); }"); + l.setScript(mmWriteScript("uint8_t cols = 8; // @control 1..64\n" + "for (i = 0; i < cols; i = i + 1) { addLight(i, 1, 0); }")); l.prepare(); CHECK(l.lightCount() == 16); // A script whose first control is a NEW slot gets its own initialiser: nothing to inherit. - l.setSource("uint8_t cols = 16; // @control 1..64\n" + l.setScript(mmWriteScript("uint8_t cols = 16; // @control 1..64\n" "uint8_t rows = 3; // @control 1..64\n" "for (yy = 0; yy < rows; yy = yy + 1) {" - " for (xx = 0; xx < cols; xx = xx + 1) { addLight(xx, yy, 0); } }"); + " for (xx = 0; xx < cols; xx = xx + 1) { addLight(xx, yy, 0); } }")); l.prepare(); CHECK(l.lightCount() == 48); // 16 inherited, rows 3 its own } @@ -293,7 +294,7 @@ TEST_CASE("a loop counter survives the body that uses it") { SUBCASE("through a call — addLight") { MoonLiveLayout l; l.defineControls(); - l.setSource("for (i = 0; i < 6; i = i + 1) { addLight(i, i, 0); }"); + l.setScript(mmWriteScript("for (i = 0; i < 6; i = i + 1) { addLight(i, i, 0); }")); l.prepare(); CHECK(l.lightCount() == 6); // a clobbered counter gives some other number } @@ -327,7 +328,7 @@ TEST_CASE("a loop counter survives the body that uses it") { TEST_CASE("a stray character in a for header is rejected, not spun on") { MoonLiveLayout l; l.defineControls(); - l.setSource("for (i = 0; i < 4; i = i @ 1) { addLight(i, 0, 0); }"); + l.setScript(mmWriteScript("for (i = 0; i < 4; i = i @ 1) { addLight(i, 0, 0); }")); l.prepare(); // must return — a hang fails by timeout CHECK(l.severity() == MoonModule::Severity::Error); CHECK(l.lightCount() == 0); @@ -346,7 +347,7 @@ TEST_CASE("two threads can run scripts at once without stealing each other's sin l.defineControls(); char src[128]; std::snprintf(src, sizeof(src), "for (i = 0; i < %d; i = i + 1) { addLight(i, 0, 0); }", cols); - l.setSource(src); + l.setScript(mmWriteScript(src)); l.prepare(); for (int r = 0; r < reps; r++) if (l.lightCount() != static_cast(cols)) return false; @@ -369,15 +370,15 @@ TEST_CASE("two threads can run scripts at once without stealing each other's sin TEST_CASE("a scripted layout reports every heap byte it holds, compiled or not") { MoonLiveLayout l; l.defineControls(); - l.setSource("uint8_t cols = 4; // @control 1..64\n" - "for (i = 0; i < cols; i = i + 1) { addLight(i, 0, 0); }"); + l.setScript(mmWriteScript("uint8_t cols = 4; // @control 1..64\n" + "for (i = 0; i < cols; i = i + 1) { addLight(i, 0, 0); }")); l.prepare(); const size_t compiled = l.dynamicBytes(); CHECK(compiled > 0); CHECK(l.lightCount() == 4); // A broken script frees the code but keeps the arena, so the figure drops without reaching zero. - l.setSource("for (i = 0; i < 4; i = i + 1) { addLight(i, i"); // unclosed + l.setScript(mmWriteScript("for (i = 0; i < 4; i = i + 1) { addLight(i, i")); // unclosed l.prepare(); CHECK(l.severity() == MoonModule::Severity::Error); CHECK(l.dynamicBytes() < compiled); // the code block is gone @@ -395,9 +396,12 @@ TEST_CASE("a scripted layout reports every heap byte it holds, compiled or not") TEST_CASE("a layout that changes size mid-build cannot overrun the mapping") { MoonLiveLayout layout; layout.defineControls(); - layout.setSource("uint8_t cols = 4; // @control 1..64\n" - "for (i = 0; i < cols; i = i + 1) { addLight(i, 0, 0); }"); + layout.setScript(mmWriteScript("uint8_t cols = 4; // @control 1..64\n" + "for (i = 0; i < cols; i = i + 1) { addLight(i, 0, 0); }")); layout.prepare(); + // The script's own controls (`cols`) exist only once it has COMPILED, and a module starts with + // no script now — so the control list has to be rebuilt after prepare() for setWidth to find it. + layout.rebuildControls(); mm::Layouts group; group.addChild(&layout); @@ -428,3 +432,171 @@ TEST_CASE("a layout that changes size mid-build cannot overrun the mapping") { } #endif // MM_MOONLIVE_HAS_HOST_JIT + +// A control write lands directly in the module's buffer — addText binds it — so setScript() is NOT +// called. Nothing then cleared the compiled-hash, and compile()'s early-return kept the OLD program +// running under the new name. Found by review; the same class of bug hardware found in the effect. +// Needs a backend: without one BOTH counts are zero and the test passes without proving the swap. +#if MM_MOONLIVE_HAS_HOST_JIT +TEST_CASE("naming a different script through the control actually swaps the program") { + MoonLiveLayout l; + l.defineControls(); + const char* four = mmWriteScript("for (i = 0; i < 4; i = i + 1) { addLight(i, 0, 0); }"); + l.setScript(four); + l.prepare(); + REQUIRE(l.lightCount() == 4); + + // Write the OTHER script the way the API does: straight into the bound control buffer. + const char* nine = mmWriteScript("for (i = 0; i < 9; i = i + 1) { addLight(i, 0, 0); }"); + const auto& cs = l.controls(); + for (uint8_t i = 0; i < cs.count(); i++) + if (cs[i].name && std::strcmp(cs[i].name, "script") == 0) + std::snprintf(static_cast(cs[i].ptr), 32, "%s", nine); + l.onControlChanged("script"); + l.prepare(); + CHECK(l.lightCount() == 9); // the new file, not the cached program +} +#endif // MM_MOONLIVE_HAS_HOST_JIT + +// A layout that cannot compile must stay quiet, not keep trying. +// +// The pipeline asks a layout for its size and then walks it, and BOTH ask it to compile first — so a +// failure that leaves "nothing is compiled" looks exactly like "not compiled yet" and every ask +// re-reads the file. On an ESP32 one attempt is two LittleFS operations (~5 ms), and the repeated +// asks during a single rebuild starved the task until the 12-second watchdog reset the board: a +// missing script took the whole device down rather than showing an error. The behaviour to pin is +// that a failed layout still places no lights however many times it is asked, and says so. +TEST_CASE("a layout whose script is missing reports it without retrying forever") { + MoonLiveLayout l; + l.defineControls(); + l.setScript("definitely-not-there.mlv"); + l.prepare(); + CHECK(l.severity() == MoonModule::Severity::Error); + // Every ask the pipeline could make, several times over. Each one used to re-read the file. + int placed = 0; + CoordSink sink{[](void* ctx, nrOfLightsType, lengthType, lengthType, lengthType) { + (*static_cast(ctx))++; + }, nullptr, &placed}; + for (int i = 0; i < 50; i++) { + CHECK(l.lightCount() == 0); + l.forEachCoord(sink); + } + CHECK(placed == 0); + CHECK(l.severity() == MoonModule::Severity::Error); // and it still says what is wrong + + // A working script after a failed one must still compile — the give-up is per script name, not + // permanent, or fixing a typo would need a reboot. + const char* good = "for (i = 0; i < 5; i = i + 1) { addLight(i, 0, 0); }"; + l.setScript(mmWriteScript(good)); + l.prepare(); + // The COUNT needs an emitting backend; the give-up-is-per-name behaviour above does not, so + // only this line is gated and the rest of the case still runs on x86_64 (where CI runs). +#if MM_MOONLIVE_HAS_HOST_JIT + CHECK(l.lightCount() == 5); +#endif +} + +// A layout that starts with NO script must still compile the first real one it is given. +// +// Every device boots a fresh layout card with an empty script control, so the very first compile +// always fails with "no script — set the script name". When the give-up flag was a bare bool that +// failure latched, and the card then reported "no script" forever however many valid names were set +// afterwards: the render loop asks for the light count long before a control write can clear a flag, +// so the guard re-armed itself on every tick. Bench-caught on an S3 — the host never saw it because +// a test constructs a fresh layout per case and never boots one empty. +TEST_CASE("a layout that starts empty still compiles the first script it is given") { + MoonLiveLayout l; + l.defineControls(); + l.prepare(); // the empty-script boot: fails, as it should + CHECK(l.severity() == MoonModule::Severity::Error); + CHECK(l.lightCount() == 0); + + // The RENDER LOOP keeps asking while no script is set — this is the step that re-armed the + // flag on device and that a straight prepare/setScript sequence never reproduces. + for (int i = 0; i < 5; i++) CHECK(l.lightCount() == 0); + + // Write the control the way the UI does — straight into the bound buffer, then + // onControlChanged — because addText binds `script_` directly and setScript() is NOT called on + // that path. That is exactly how a device sets a script, and where the latch survived. + const char* name = mmWriteScript("for (i = 0; i < 6; i = i + 1) { addLight(i, 0, 0); }"); + auto& cs = l.controls(); + for (uint8_t i = 0; i < cs.count(); i++) + if (cs[i].name && std::strcmp(cs[i].name, "script") == 0) + std::snprintf(static_cast(cs[i].ptr), 32, "%s", name); + l.onControlChanged("script"); + l.prepare(); +#if MM_MOONLIVE_HAS_HOST_JIT + CHECK(l.lightCount() == 6); // the give-up must not have latched +#endif +} + +// The fixed script directory is a boundary: a module names a file inside it, and cannot address the +// filesystem. Without this, a control value of "../.config/NetworkModule.json" reads the device's +// saved WiFi credentials as if they were a script. +TEST_CASE("a script name cannot escape the script folder") { + MoonLiveLayout l; + l.defineControls(); + for (const char* bad : {"../.config/NetworkModule.json", "..", "sub/dir.mlv", "grid.txt"}) { + INFO(bad); + l.setScript(bad); + l.prepare(); + CHECK(l.severity() == MoonModule::Severity::Error); + CHECK(l.lightCount() == 0); + } +} + +// A script that STOPS being valid must take its lights with it. Every check in the loader returns +// before the compile, and the compile is what releases the previous program, so a rename, a delete +// or an emptied file used to leave the old code executing while the card reported the error: the +// fixture kept rendering a script the user had removed. The one state a user can never debug is a +// device that disagrees with its own status line. +TEST_CASE("a script that disappears takes its lights with it") { + MoonLiveLayout l; + l.defineControls(); + l.setScript(mmWriteScript("addLight(1, 1, 0); addLight(2, 2, 0);")); + l.prepare(); +#if MM_MOONLIVE_HAS_HOST_JIT + REQUIRE(l.lightCount() == 2); // a working script first + CHECK(l.severity() != MoonModule::Severity::Error); +#endif + + l.setScript("gone.mlv"); // never written, so the loader rejects it + l.prepare(); + CHECK(l.severity() == MoonModule::Severity::Error); + CHECK(l.lightCount() == 0); // the old program is gone, not just unreported +} + +// The name the LOADER accepts and the name the CONTROL can hold must be the same length. They were +// not: the control held 31 characters while the loader accepted 40, so a longer valid name was +// silently truncated on its way in — and truncation can cut the `.mlv` off, turning a real script +// into a name the loader then rejects. The user sees "script must end in .mlv" for a file that does. +TEST_CASE("a script name at the accepted length survives the control it is stored in") { + // A name exactly at the limit: filler + ".mlv", written so the file really exists. + std::string longName(mm::moonlive::kMaxScriptName - 4, 'a'); + longName += ".mlv"; + REQUIRE(longName.size() == mm::moonlive::kMaxScriptName); + + // Write a real script under that name, then name it. If the control clipped it, the loader + // would see a truncated name (possibly without .mlv) and report an error instead of rendering. + char path[128]; + std::snprintf(path, sizeof(path), "%s/%s", mm::moonlive::kScriptDir, longName.c_str()); + mm::platform::fsMkdir(mm::moonlive::kScriptDir); + const char* body = "addLight(3, 3, 0);"; + mm::platform::fsWriteAtomic(path, body, std::strlen(body)); + mmScriptRegistry().push_back(path); + + MoonLiveLayout l; + l.defineControls(); + l.setScript(longName.c_str()); + l.prepare(); + // The name reached the loader intact: a clipped one is rejected for its missing extension, so + // the status would name the NAME rather than anything about the script's contents. Asserted + // this way because a host without a MoonLive backend (x86-64) fails every compile by design, + // and this test is about the control buffer, not about codegen. + if (l.severity() == MoonModule::Severity::Error) + CHECK(std::string(l.status()).find(".mlv") == std::string::npos); +#if MM_MOONLIVE_HAS_HOST_JIT + CHECK(l.severity() != MoonModule::Severity::Error); + CHECK(l.lightCount() == 1); +#endif +} diff --git a/test/unit/light/unit_MoonLiveModifier.cpp b/test/unit/light/unit_MoonLiveModifier.cpp index 263a2761..b9eb3297 100644 --- a/test/unit/light/unit_MoonLiveModifier.cpp +++ b/test/unit/light/unit_MoonLiveModifier.cpp @@ -8,6 +8,7 @@ // broken script degrades to a pass-through instead of taking the pipeline down. #include "doctest.h" +#include "MoonLiveScriptFixture.h" #include "light/moonlive/MoonLiveModifier.h" #include "light/moonlive/MoonLiveEffect.h" #include "platform/platform.h" @@ -34,7 +35,7 @@ Coord3D transform(const char* script, lengthType x, lengthType y, lengthType z, lengthType w = 255, lengthType h = 255, lengthType d = 1) { MoonLiveModifier m; m.defineControls(); - if (script) m.setSource(script); + if (script) m.setScript(mmWriteScript(script)); m.prepare(); Coord3D box{w, h, d}; m.modifyLogicalSize(box); // the Layer always does this before folding @@ -47,7 +48,7 @@ Coord3D transform(const char* script, lengthType x, lengthType y, lengthType z, TEST_CASE("a scripted modifier mirrors the pattern, the way a hand-written one would") { // The default script. A mirror is the shape that makes a working binding obvious on a bench // strand — the pattern simply runs the other way. - const Coord3D p = transform(nullptr, 10, 20, 0); + const Coord3D p = transform("setXYZ(0, width - 1 - x, y, z);", 10, 20, 0); CHECK(p.x == 244); // width(255) - 1 - 10 CHECK(p.y == 20); // untouched axes stay put CHECK(p.z == 0); @@ -84,7 +85,7 @@ TEST_CASE("a broken script leaves the pattern alone rather than taking the layer // the pipeline keeps rendering. MoonLiveModifier m; m.defineControls(); - m.setSource("setXYZ(0, x, y"); // no closing paren, no semicolon + m.setScript(mmWriteScript("setXYZ(0, x, y")); // no closing paren, no semicolon m.prepare(); Coord3D box{255, 255, 1}; m.modifyLogicalSize(box); @@ -109,15 +110,16 @@ TEST_CASE("editing the script changes the transform without a rebuild of the fir // The live-edit loop: the same module, a new script, a different mapping. MoonLiveModifier m; m.defineControls(); + m.setScript(mmWriteScript("setXYZ(0, width - 1 - x, y, z);")); m.prepare(); Coord3D box{255, 255, 1}; m.modifyLogicalSize(box); // the Layer hands every modifier its box before folding Coord3D a{10, 20, 0}; m.modifyLogical(a); - CHECK(a.x == 244); // the default mirror + CHECK(a.x == 244); // the mirror - m.setSource("setXYZ(0, x, y, z);"); + m.setScript(mmWriteScript("setXYZ(0, x, y, z);")); m.prepare(); Coord3D b{10, 20, 0}; @@ -163,7 +165,7 @@ TEST_CASE("folding a wall's worth of lights compiles the script once, not once p // an unchanged value across the whole fold proves no compile happened inside it. MoonLiveModifier m; m.defineControls(); - m.setSource("setXYZ(0, width - 1 - x, y, z);"); + m.setScript(mmWriteScript("setXYZ(0, width - 1 - x, y, z);")); m.prepare(); Coord3D box{255, 255, 1}; m.modifyLogicalSize(box); @@ -186,12 +188,12 @@ TEST_CASE("folding a wall's worth of lights compiles the script once, not once p TEST_CASE("editing a script asks the layer to rebuild its mapping") { MoonLiveModifier m; m.defineControls(); - m.setSource("setXYZ(0, x, y, z);"); + m.setScript(mmWriteScript("setXYZ(0, x, y, z);")); m.prepare(); CHECK(m.consumeNeedsRebuild() == true); // the first compile needs one too CHECK(m.consumeNeedsRebuild() == false); // and it is consumed, not sticky - m.setSource("setXYZ(0, 7 - x, y, z);"); + m.setScript(mmWriteScript("setXYZ(0, 7 - x, y, z);")); m.prepare(); CHECK(m.consumeNeedsRebuild() == true); // an edit asks again @@ -207,13 +209,13 @@ TEST_CASE("editing a script asks the layer to rebuild its mapping") { // be able to read the EXTENT it is folding within, and the default has to use it. TEST_CASE("the default script mirrors within the grid it is given, not a fixed 255") { // A 16-wide grid: x=0 must land on the far end of THAT grid, 15 — not 245. - const Coord3D p = transform(nullptr, 0, 0, 0, /*w=*/16, /*h=*/16, /*d=*/1); + const Coord3D p = transform("setXYZ(0, width - 1 - x, y, z);", 0, 0, 0, /*w=*/16, /*h=*/16, /*d=*/1); CHECK(p.x == 15); CHECK(p.y == 0); // Every coordinate has to stay inside the box, or the Layer discards it. for (lengthType i = 0; i < 16; i++) { - const Coord3D q = transform(nullptr, i, 0, 0, 16, 16, 1); + const Coord3D q = transform("setXYZ(0, width - 1 - x, y, z);", i, 0, 0, 16, 16, 1); CAPTURE(i); CHECK(q.x >= 0); CHECK(q.x < 16); @@ -240,7 +242,7 @@ TEST_CASE("a script that computes a position outside the grid leaves lights mapp // bytes cannot fail: draw::fill writes every byte itself, whatever the fold decided. MoonLiveModifier m; m.defineControls(); - m.setSource("setXYZ(0, x + 200, y, z);"); // deliberately off the end of a 16-wide grid + m.setScript(mmWriteScript("setXYZ(0, x + 200, y, z);")); // deliberately off the end of a 16-wide grid m.prepare(); Coord3D box{16, 16, 1}; m.modifyLogicalSize(box); @@ -273,6 +275,10 @@ TEST_CASE("a script that computes a position outside the grid leaves lights mapp TEST_CASE("re-preparing with an unchanged script does not ask for another rebuild") { MoonLiveModifier m; m.defineControls(); + // A module with no script compiles nothing and therefore asks for nothing — the rebuild request + // exists to APPLY a new transform, and there is none. Name one, so the first prepare has + // something to compile and the "unchanged" case below is the real question. + m.setScript(mmWriteScript("setXYZ(0, width - 1 - x, y, z);")); m.prepare(); CHECK(m.consumeNeedsRebuild() == true); // the first compile needs one @@ -284,7 +290,7 @@ TEST_CASE("re-preparing with an unchanged script does not ask for another rebuil CHECK(m.consumeNeedsRebuild() == false); // A real edit still asks. - m.setSource("setXYZ(0, y, x, z);"); + m.setScript(mmWriteScript("setXYZ(0, y, x, z);")); m.prepare(); CHECK(m.consumeNeedsRebuild() == true); } @@ -329,7 +335,7 @@ TEST_CASE("a subtraction produces the whole value, not just its low byte") { TEST_CASE("a for loop runs its body once per step") { MoonLiveModifier m; m.defineControls(); - m.setSource("for (i = 0; i < 4; i = i + 1) { print(i); } setXYZ(0, x, y, z);"); + m.setScript(mmWriteScript("for (i = 0; i < 4; i = i + 1) { print(i); } setXYZ(0, x, y, z);")); m.prepare(); CHECK(m.severity() != MoonModule::Severity::Error); // it compiles at all Coord3D box{16, 16, 1}; m.modifyLogicalSize(box); @@ -342,7 +348,7 @@ TEST_CASE("a loop over an empty range runs its body no times") { // The entry guard: `i < 0` must skip the body entirely rather than wrap and run forever. MoonLiveModifier m; m.defineControls(); - m.setSource("for (i = 0; i < 0; i = i + 1) { print(99); } setXYZ(0, x, y, z);"); + m.setScript(mmWriteScript("for (i = 0; i < 0; i = i + 1) { print(99); } setXYZ(0, x, y, z);")); m.prepare(); CHECK(m.severity() != MoonModule::Severity::Error); Coord3D box{16, 16, 1}; m.modifyLogicalSize(box); @@ -354,8 +360,8 @@ TEST_CASE("a loop over an empty range runs its body no times") { TEST_CASE("loops nest, which is what placing a grid of lights needs") { MoonLiveModifier m; m.defineControls(); - m.setSource("for (a = 0; a < 2; a = a + 1) { for (b = 0; b < 2; b = b + 1) { print(a); } }" - " setXYZ(0, x, y, z);"); + m.setScript(mmWriteScript("for (a = 0; a < 2; a = a + 1) { for (b = 0; b < 2; b = b + 1) { print(a); } }" + " setXYZ(0, x, y, z);")); m.prepare(); CHECK(m.severity() != MoonModule::Severity::Error); Coord3D box{16, 16, 1}; m.modifyLogicalSize(box); @@ -379,7 +385,7 @@ TEST_CASE("a loop in an effect script paints every light it walks") { layer.setChannelsPerLight(3); auto* fx = new MoonLiveEffect(); fx->defineControls(); - fx->setSource("for (i = 0; i < 8; i = i + 1) { setRGB(i, i, 0, 0); }"); + fx->setScript(mmWriteScript("for (i = 0; i < 8; i = i + 1) { setRGB(i, i, 0, 0); }")); layer.addChild(fx); layouts.applyState(); layer.applyState();