From a8430df235836bd343f17883dbac3967c362675d Mon Sep 17 00:00:00 2001 From: Jan Hubener Date: Sat, 5 Sep 2026 14:37:08 +0000 Subject: [PATCH 1/4] =?UTF-8?q?planner:=20NestedBands=20=E2=80=94=20the=20?= =?UTF-8?q?exposure=20meter=20as=20a=20sealed=20nested=20mask=20set=20(D-N?= =?UTF-8?q?XG-1,=20D-NXG-5=20merge=20arm,=20room=203=20moments)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit crates/lance-graph-planner/src/nested_bands.rs. NestedBandsBuilder seals a version-keyed NestedBands (boundaries, band masks, bucket masks via mask_ternlog::, popcounts). Every method is &self; split and merge return new values; the top band is the universe by construction (E-NXG-18 becomes a type invariant). overflow() is the budget test, collapsed()/merge() is the merge-on-collapse arm, best_achievable_floor is E-NXG-20's floor by bisection over gt_i32_to_mask + popcount, and moments()/sigma_exact() close plan room 3: sigma is exact from the seal iff it stores two accumulators per bucket. 12 tests on the three real recordings in data/tts-cascade. Two of the probes' pinned numbers corrected in the process: the strict floor on speech is 15077 (the probe's 15072 was one row over the target), and the midpoint-sigma estimator is not sign-stable (over-reads 12% or under-reads 7% depending on the arbitrary top-bucket midpoint). Board: E-NXG-21, AGENT_LOG, STATUS_BOARD D-NXG-1/3/5, plan §6 rows; supersession index regenerated last. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016WkNBjHc2e3zuyz9i8qJEv --- .claude/board/AGENT_LOG.md | 7 + .claude/board/EPIPHANIES.md | 39 + .claude/board/STATUS_BOARD.md | 6 +- .../nexgen-mask-histogram-thresholds-v1.md | 6 +- crates/lance-graph-planner/src/lib.rs | 3 + .../lance-graph-planner/src/nested_bands.rs | 834 ++++++++++++++++++ 6 files changed, 889 insertions(+), 6 deletions(-) create mode 100644 crates/lance-graph-planner/src/nested_bands.rs diff --git a/.claude/board/AGENT_LOG.md b/.claude/board/AGENT_LOG.md index 59381cd8f..07890ba50 100644 --- a/.claude/board/AGENT_LOG.md +++ b/.claude/board/AGENT_LOG.md @@ -1,3 +1,10 @@ +## 2026-09-05 — D-NXG-1 `NestedBands`: one Sonnet worker from spec, orchestrator-gated + +- **Why:** operator "Go ahead autoattended" + "Sonnet agents for grindwork" after the three probes merged. D-NXG-1 is a write-this-file-from-spec unit: bounded input (a 134-line spec naming every type, method and test), known output shape. +- **Split:** orchestrator wrote the spec (types, signatures verified against `ndarray/src/simd_int_ops.rs` + `bitwise.rs` + `contract/src/thought_atoms.rs`, twelve named tests on the three real recordings); one Sonnet `general-purpose` worker wrote the 711-line file, edit-only, no cargo, and corrected the spec's `include_bytes!` path (three levels up, not four — verified with `ls`, reported). Orchestrator registered the module in `lib.rs` and ran every gate in the shared `target/`. +- **Gates:** first run 10/12 — both failures were SPEC errors the worker faithfully reproduced (a pinned floor value one row over the target; a sign-assumed σ assertion). Both fixed in the tests to the strict definitions, recorded in E-NXG-21. One clippy `-D warnings` (test-only import) moved into the tests module. Final: `nested_bands` 12/12, planner lib suite green, clippy clean, fmt clean, append-only + citation-decay + plan-dids gates local, supersession index regenerated last. +- **Board:** E-NXG-21; STATUS_BOARD D-NXG-1/3/5 rows; plan §6 rows. + ## 2026-09-05 — PROBE-NXG-ROLL-1 + PROBE-NXG-FLOOR-1: orchestrator-only, three falsifications kept - **Why:** operator "fast forward" after #1178 merged. Plan §5 steps 3 and 4, the last two gates before rooms 4–8 may leave PROPOSAL. diff --git a/.claude/board/EPIPHANIES.md b/.claude/board/EPIPHANIES.md index fb07bf6a2..3f65e8535 100644 --- a/.claude/board/EPIPHANIES.md +++ b/.claude/board/EPIPHANIES.md @@ -1,3 +1,42 @@ +## 2026-09-05 — E-NXG-21 — `NestedBands` sealed: the three probes became one type and twelve tests, and two of the probes' numbers were one row and one sign off + +**Status:** SHIPPED (`crates/lance-graph-planner/src/nested_bands.rs`, D-NXG-1; merge arm of D-NXG-5; room 3 closed by moments). +**Confidence:** High — 12/12 tests on the same three real recordings the probes used, planner lib suite green. + +`NestedBandsBuilder::{new, budget_factor, calibrate, with_boundaries}` seals a +`NestedBands` (version, boundaries, band masks, bucket masks, popcounts). +Every method is `&self`; `split` and `merge` return NEW values under a new +version and the original is asserted unchanged (data-flow rule, one writer at +build). The top band is the universe by construction (E-NXG-18 is now a type +invariant, not a probe caveat). `overflow()` is the budget test (E-NXG-19), +`collapsed()` is the merge-on-collapse arm — it fires when speech boundaries +meet the quiet recording and stays silent on the calibration epoch; `merge` +removes one boundary and the merged popcount equals the pair's sum. +`best_achievable_floor` is E-NXG-20's floor by bisection over +`gt_i32_to_mask` + popcount, no sort. + +**Two corrections the type forced on the probes' numbers.** (1) The strict +floor — the SMALLEST value whose exceedance is ≤ the rate — lands at 15 077 on +speech, not the probe's 15 072: the probe's rank floor sat one row OVER the +target (473/94 572 = 0.0050015) inside its ±1/n tolerance. A floor with a row of +slack is not "never exceeds"; the type has no slack and the test pins both the +literal and a sorted-copy reference. (2) The midpoint-σ estimator is not even +sign-stable: with the stale top boundary it over-read by 12 % (E-NXG-17); with +the last boundary as the top bucket's midpoint it under-reads by 7 % +(3 525.8 vs 3 785.8). An open-ended top bucket has no midpoint, so the +estimator has no principled value at all. The regression guard is now +direction-agnostic (|Δ|/σ > 5 %), and room 3 closes the other way: σ is exact +from the seal iff the seal stores two accumulators per bucket (`BucketMoment +{count, sum, sumsq}`, 16 B/bucket) — `sigma_exact` matches the direct σ to +1e-9. "Recoverable for free" was wrong; "recoverable for 256 bytes" is right. + +**Not done, on purpose.** D-NXG-3 (`bisect_column_by_mask`) lives inside +`NestedBands::split`, not as a named `ndarray::simd` primitive: it is a +memory-bound gather-and-popcount loop, not a lane op, and the W1a contract's +all-backends parity requirement would be ceremony for a scalar. Promotion, if +ever, is its own deliberate PR. No consumer is wired yet — the type exists, the +first caller (D-NXG-4's `shape × rank` payload) is the next unit. + ## 2026-09-05 — E-NXG-18 — the ladder's top band must BE the universe, or the histogram loses rows silently **Status:** FINDING (PROBE-NXG-ROLL-1 falsified its own first run). diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index 151625c10..973eeacce 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -30,11 +30,11 @@ every row is Queued until PROBE-NXG-HIST-1 / ROLL-1 / FLOOR-1 (plan §5) run. | D-id | deliverable | status | |---|---|---| -| D-NXG-1 | `NestedBands` sealed shape (T2), version-keyed, one owner | **In progress** — structure measured by PROBE-NXG-HIST-1 (E-NXG-17, 2026-09-05); seal/ownership shape not yet built | +| D-NXG-1 | `NestedBands` sealed shape (T2), version-keyed, one owner | **Shipped 2026-09-05** — `crates/lance-graph-planner/src/nested_bands.rs`, 12 tests on real recordings (E-NXG-21); no consumer wired yet | | D-NXG-2 | T1 name audit: `popcount_words` present or minted; bucket = `mask_ternlog::<0x10>` by name | **Half-closed 2026-09-05** — `ndarray::simd::popcount_batch_u64` exists (scalar); bucket-by-name proven bit-identical to `mask_andnot` (C3) | -| D-NXG-3 | T1 `bisect_column_by_mask` partial-popcount bisection | **Measured 2026-09-05** — implemented probe-locally in PROBE-NXG-ROLL-1 and shown to split inside the bucket's range; not yet a named T1 primitive in `ndarray::simd` | +| D-NXG-3 | T1 `bisect_column_by_mask` partial-popcount bisection | **Lives in `NestedBands::split` 2026-09-05** — deliberately NOT promoted to `ndarray::simd` (gather+popcount loop, not a lane op; E-NXG-21). Promotion is its own PR if a second caller appears | | D-NXG-4 | Prozentrang = bucket index; `shape × rank` computed from the slab | **Measured 2026-09-05** — rank == partition point on 94 572/94 572 real rows (C2); not yet wired to any consumer | -| D-NXG-5 | rollover: split on budget or entropy, merge on collapse, never rewrite | **Split arm GREEN 2026-09-05** (PROBE-NXG-ROLL-1: never-rewrite asserted, max bucket halved). Merge-on-collapse arm still Queued. Trigger order corrected — budget leads, E-NXG-19 | +| D-NXG-5 | rollover: split on budget or entropy, merge on collapse, never rewrite | **Both arms shipped 2026-09-05** — `NestedBands::{overflow, split, collapsed, merge}`, never-rewrite asserted in tests (E-NXG-21). Trigger order: budget leads (E-NXG-19) | | D-NXG-6 | rank-derived reject floor; σ demoted to diagnostic | **Measured 2026-09-05** (PROBE-NXG-FLOOR-1, E-NXG-20): `k` does not name a rate and at k=3 one real column's floor is unreachable; the rank floor is the best achievable boundary, exact only absent ties | | D-NXG-7 | preheat by mask inheritance | Queued | | D-NXG-8 | one early-exit rule for search and alarm | Queued | diff --git a/.claude/nexgen/plans/nexgen-mask-histogram-thresholds-v1.md b/.claude/nexgen/plans/nexgen-mask-histogram-thresholds-v1.md index 987269085..ce76e7050 100644 --- a/.claude/nexgen/plans/nexgen-mask-histogram-thresholds-v1.md +++ b/.claude/nexgen/plans/nexgen-mask-histogram-thresholds-v1.md @@ -178,11 +178,11 @@ falsifier that would kill it. Ordered by how many rooms ahead it sits. | D-id | deliverable | status | |---|---|---| -| D-NXG-1 | `NestedBands` sealed shape (T2), version-keyed, one owner | In progress (structure measured, E-NXG-17) | +| D-NXG-1 | `NestedBands` sealed shape (T2), version-keyed, one owner | Shipped 2026-09-05 (`planner/src/nested_bands.rs`, E-NXG-21) | | D-NXG-2 | T1 name audit: `popcount_words` present or minted; bucket = `mask_ternlog::<0x10>` by name | Queued | -| D-NXG-3 | T1 `bisect_column_by_mask` partial-popcount bisection | Queued | +| D-NXG-3 | T1 `bisect_column_by_mask` partial-popcount bisection | Lives in `NestedBands::split`; not promoted to T1 (not a lane op, E-NXG-21) | | D-NXG-4 | Prozentrang = bucket index; `shape × rank` computed from the slab | Queued | -| D-NXG-5 | rollover: split on budget or entropy, merge on collapse, never rewrite | Queued | +| D-NXG-5 | rollover: split on budget or entropy, merge on collapse, never rewrite | Shipped 2026-09-05 — both arms, budget leads (E-NXG-19/21) | | D-NXG-6 | rank-derived reject floor; σ demoted to diagnostic | Queued | | D-NXG-7 | preheat by mask inheritance | Queued | | D-NXG-8 | one early-exit rule for search and alarm | Queued | diff --git a/crates/lance-graph-planner/src/lib.rs b/crates/lance-graph-planner/src/lib.rs index 3fdd23e96..313d2c8bf 100644 --- a/crates/lance-graph-planner/src/lib.rs +++ b/crates/lance-graph-planner/src/lib.rs @@ -54,6 +54,9 @@ pub mod physical; pub mod plan; pub mod thinking; +// === NestedBands (D-NXG-1: the exposure meter as a sealed nested mask set) === +pub mod nested_bands; + // === Dynamic Elevation (cost model that smells resistance) === pub mod elevation; diff --git a/crates/lance-graph-planner/src/nested_bands.rs b/crates/lance-graph-planner/src/nested_bands.rs new file mode 100644 index 000000000..1d9f33065 --- /dev/null +++ b/crates/lance-graph-planner/src/nested_bands.rs @@ -0,0 +1,834 @@ +//! `NestedBands` — the T2 sealed shape from plan +//! `.claude/nexgen/plans/nexgen-mask-histogram-thresholds-v1.md`. +//! +//! A Belichtungsmesser reading over an `i32` value column is a chain of +//! nested row masks `M_0 ⊆ M_1 ⊆ … ⊆ M_{B-1}`. Bucket `i` is +//! `M_i ∧ ¬M_{i-1}` by the NAMED immediate `mask_ternlog::`. A +//! row's Prozentrang is its bucket index — a partition point over nested +//! masks, never a sort. `NestedBands` is built once (`NestedBandsBuilder`), +//! sealed, version-keyed, and NEVER mutated: `split`/`merge` return NEW +//! values under a new version (data-flow rule: no `&mut self` during +//! computation). +//! +//! Cites: E-NXG-1 (a Belichtungsmesser reading IS a nested mask set), +//! E-NXG-17 (PROBE-NXG-HIST-1 GREEN: nested/disjoint/rank-by-mask-walk), +//! E-NXG-18 (**the top band is the universe** — rows above the last +//! boundary must never be lost silently; a ladder whose last mask is not +//! the universe drops rows), E-NXG-19 (the budget/overflow test fires +//! strictly before the entropy/collapse signal on a real epoch shift — +//! the two are not interchangeable), E-NXG-20 (`mu + k·sigma` does not +//! name a rate; a floor read off the rank ladder is the best achievable +//! boundary for a target exceedance rate, and midpoint-estimated sigma +//! misreads the exact sigma, and not even in a stable direction). +//! +//! The three probes this module re-derives as library code (their mask +//! helpers' LOGIC was read and carried over, not their prose): +//! `examples/probe_nxg_hist_1.rs`, `examples/probe_nxg_roll_1.rs`, +//! `examples/probe_nxg_floor_1.rs`. + +use lance_graph_contract::thought_atoms::normalized_entropy; +use ndarray::simd::ternlog::{AND2, AND_ANDNOT2}; +use ndarray::simd::{gt_i32_to_mask, mask_ternlog, popcount_batch_u64}; + +/// A sealed `NestedBands` value's version. Every `split`/`merge` returns a +/// new `NestedBands` under a caller-supplied new version; the version is +/// never incremented internally. +pub type Version = u64; + +/// Per-bucket moment accumulator (`count`, `sum`, `sumsq` of the raw +/// column values that fell in the bucket). Two accumulators per bucket +/// are exactly what is needed to recover an exact sigma from the seal +/// (room-3 correction: histogram-midpoint sigma misreads the true +/// value — see [`NestedBands::sigma_exact`]). +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct BucketMoment { + /// Number of rows in the bucket. + pub count: u64, + /// Sum of `value as f64` over the bucket's rows. + pub sum: f64, + /// Sum of `(value as f64)^2` over the bucket's rows. + pub sumsq: f64, +} + +/// Builder for a [`NestedBands`]. Configures band count and overflow +/// budget factor, then calibrates (quantile boundaries) or accepts an +/// explicit boundary ladder. +#[derive(Debug, Clone)] +pub struct NestedBandsBuilder { + bands: usize, + budget_factor: f64, +} + +/// A sealed, nested-mask Belichtungsmesser reading over one `i32` column. +/// +/// Invariants (asserted at construction, re-verified by the test suite): +/// bands are nested (`M_i` implies `M_{i+1}`), buckets are pairwise +/// disjoint, bucket popcounts sum to `rows`, and the top band is always +/// the universe (all-ones, tail cleared) so no row is ever silently +/// dropped (E-NXG-18). +#[derive(Debug, Clone)] +pub struct NestedBands { + version: Version, + /// Strictly ascending, length `bands - 1`. The top band has NO + /// boundary entry — it is the universe by construction. + boundaries: Vec, + /// `bands[i]` = rows with value `<= boundaries[i]`; the last entry is + /// all-ones (tail cleared beyond `rows`), never `le(top_boundary)`. + bands: Vec>, + /// `buckets[0] = bands[0]`; `buckets[i] = mask_ternlog::(bands[i], bands[i-1], bands[i-1])`. + buckets: Vec>, + /// Popcount of each bucket, cached at build time. + popcounts: Vec, + rows: usize, + budget_factor: f64, +} + +fn words_for(n: usize) -> usize { + n.div_ceil(64) +} + +/// `M = rows with value <= boundary`, as `!gt` with the tail beyond `n` +/// cleared. Shared mask-walk logic, carried over from the probes. +fn le_mask(values: &[i32], boundary: i32) -> Vec { + let n = values.len(); + let mut m = vec![0u64; words_for(n)]; + gt_i32_to_mask(values, boundary, &mut m); + for w in m.iter_mut() { + *w = !*w; + } + if !n.is_multiple_of(64) { + let last = m.len() - 1; + m[last] &= (1u64 << (n % 64)) - 1; + } + m +} + +/// The universe mask: all-ones, tail beyond `n` cleared. +fn universe_mask(n: usize) -> Vec { + let mut m = vec![u64::MAX; words_for(n)]; + if !n.is_multiple_of(64) { + let last = m.len() - 1; + m[last] &= (1u64 << (n % 64)) - 1; + } + m +} + +fn bit(m: &[u64], row: usize) -> bool { + (m[row / 64] >> (row % 64)) & 1 == 1 +} + +/// Build the nested band masks for a boundary ladder (E-NXG-18: the last +/// band is always the universe, never `le(top_boundary)`). +fn build_bands(values: &[i32], boundaries: &[i32]) -> Vec> { + let n = values.len(); + let last = boundaries.len(); + (0..=last) + .map(|i| { + if i == last { + universe_mask(n) + } else { + le_mask(values, boundaries[i]) + } + }) + .collect() +} + +/// Bucket `i` = `M_i ∧ ¬M_{i-1}`, by the NAMED immediate (E-NXG-1 / C3 of +/// PROBE-NXG-HIST-1). +fn build_buckets(bands: &[Vec]) -> Vec> { + let words = bands[0].len(); + (0..bands.len()) + .map(|i| { + let mut b = vec![0u64; words]; + if i == 0 { + b.copy_from_slice(&bands[0]); + } else { + mask_ternlog::(&bands[i], &bands[i - 1], &bands[i - 1], &mut b); + } + b + }) + .collect() +} + +impl NestedBandsBuilder { + /// Start a builder targeting `bands` bands (before dedup in + /// [`calibrate`](Self::calibrate)). `bands` must be at least 2 — a + /// single band carries no rank information. + pub fn new(bands: usize) -> Self { + assert!(bands >= 2, "NestedBands needs at least 2 bands"); + Self { + bands, + budget_factor: 2.0, + } + } + + /// Set the overflow budget factor (a bucket may hold this many times + /// its equal-mass share before [`NestedBands::overflow`] fires). + /// Default 2.0. + pub fn budget_factor(mut self, f: f64) -> Self { + self.budget_factor = f; + self + } + + /// Calibrate quantile boundaries from `column` (the one sort in this + /// module — a sorted COPY, never mutating the caller's data), dedup + /// them to strictly ascending, and seal a [`NestedBands`] at + /// `version`. + pub fn calibrate(self, column: &[i32], version: Version) -> NestedBands { + let n = column.len(); + let mut sorted = column.to_vec(); + sorted.sort_unstable(); + let mut boundaries: Vec = (0..self.bands - 1) + .map(|i| sorted[((i + 1) * n / self.bands).saturating_sub(1)]) + .collect(); + boundaries.dedup(); + assert!( + !boundaries.is_empty(), + "NestedBands::calibrate: column is degenerate (all quantile boundaries collapsed to one value): {column:?}" + ); + self.with_boundaries(boundaries, column, version) + } + + /// Seal a [`NestedBands`] from an explicit, strictly ascending + /// boundary ladder. `boundaries` must be non-empty; the builder's own + /// `bands` field is only consulted by [`calibrate`](Self::calibrate) — + /// here `band_count` is `boundaries.len() + 1`. + pub fn with_boundaries( + self, + boundaries: Vec, + column: &[i32], + version: Version, + ) -> NestedBands { + assert!( + !boundaries.is_empty(), + "NestedBands::with_boundaries: empty boundary ladder" + ); + assert!( + boundaries.windows(2).all(|w| w[0] < w[1]), + "NestedBands::with_boundaries: boundaries not strictly ascending: {boundaries:?}" + ); + let rows = column.len(); + let bands = build_bands(column, &boundaries); + let buckets = build_buckets(&bands); + let popcounts = buckets.iter().map(|m| popcount_batch_u64(m)).collect(); + NestedBands { + version, + boundaries, + bands, + buckets, + popcounts, + rows, + budget_factor: self.budget_factor, + } + } +} + +impl NestedBands { + /// This sealed value's version. + pub fn version(&self) -> Version { + self.version + } + + /// Number of rows this reading was built over. + pub fn rows(&self) -> usize { + self.rows + } + + /// Number of bands (= `boundaries().len() + 1`). + pub fn band_count(&self) -> usize { + self.boundaries.len() + 1 + } + + /// The strictly ascending boundary ladder. Length `band_count() - 1` + /// — the top band has no boundary entry (it is the universe). + pub fn boundaries(&self) -> &[i32] { + &self.boundaries + } + + /// The `i`-th cumulative band mask (`M_i`). + pub fn band(&self, i: usize) -> &[u64] { + &self.bands[i] + } + + /// The `i`-th bucket mask (`M_i ∧ ¬M_{i-1}`). + pub fn bucket(&self, i: usize) -> &[u64] { + &self.buckets[i] + } + + /// Per-bucket popcounts, cached at build time. + pub fn popcounts(&self) -> &[u64] { + &self.popcounts + } + + /// Alias of [`popcounts`](Self::popcounts) — the payload-law "shape" + /// of this reading. + pub fn shape(&self) -> &[u64] { + &self.popcounts + } + + /// The bucket index containing `row`, read directly off the bucket + /// masks (never the value column). `row` must be `< rows()`. + pub fn rank(&self, row: usize) -> usize { + debug_assert!( + row < self.rows, + "NestedBands::rank: row {row} out of bounds ({})", + self.rows + ); + (0..self.buckets.len()) + .find(|&i| bit(&self.buckets[i], row)) + .expect("NestedBands::rank: row is in no bucket — the top band is not the universe") + } + + /// The bucket index a raw value `v` would fall into, computed + /// directly from the boundary ladder (never exceeds `band_count() - 1` + /// by construction, since `partition_point` on a length-`(band_count + /// - 1)` slice returns at most that length). + pub fn rank_of_value(&self, v: i32) -> usize { + self.boundaries.partition_point(|&b| b < v) + } + + /// Normalized Shannon entropy of the bucket-popcount histogram (1.0 = + /// perfectly flat, falling toward 0 as mass concentrates in fewer + /// buckets). `0.0` if the histogram is degenerate. + pub fn entropy(&self) -> f32 { + let weights: Vec = self.popcounts.iter().map(|&p| p as f32).collect(); + normalized_entropy(&weights).unwrap_or(0.0) + } + + /// The overflow budget: `budget_factor * rows / band_count`, rounded + /// down. A bucket past this many rows is holding more than its + /// allotted multiple of an equal-mass share. + pub fn budget(&self) -> u64 { + (self.budget_factor * self.rows as f64 / self.band_count() as f64) as u64 + } + + /// Index of the largest bucket, iff its popcount exceeds + /// [`budget`](Self::budget) — `None` if every bucket is within + /// budget (E-NXG-19: this test fires strictly before + /// [`entropy`](Self::entropy) collapses on a real epoch shift, and + /// stays silent on the ladder's own calibration epoch). + pub fn overflow(&self) -> Option { + let budget = self.budget(); + let (idx, &max) = self + .popcounts + .iter() + .enumerate() + .max_by_key(|&(_, &p)| p) + .expect("NestedBands: at least one bucket"); + (max > budget).then_some(idx) + } + + /// The merge-on-collapse arm: the smallest adjacent bucket pair + /// `(i, i+1)` whose combined popcount is under half an equal share of + /// the rows, iff one exists. `None` on a well-spread ladder (e.g. its + /// own calibration epoch). + pub fn collapsed(&self) -> Option { + let threshold = self.rows / (2 * self.band_count()); + (0..self.popcounts.len().saturating_sub(1)) + .filter(|&i| self.popcounts[i] + self.popcounts[i + 1] < threshold as u64) + .min_by_key(|&i| self.popcounts[i] + self.popcounts[i + 1]) + } + + /// Split `bucket` by partial-popcount bisection restricted to that + /// bucket's own mask, inserting a new boundary strictly inside its + /// value range and rebuilding under `version`. Returns a NEW value — + /// `self` is never mutated (data-flow rule: no `&mut self` during + /// computation). `None` if every candidate value in range gives an + /// all-ties split (`below == 0` or `below == pop` for every `v`), or + /// if the chosen value already sits in the boundary ladder. + pub fn split(&self, bucket: usize, column: &[i32], version: Version) -> Option { + assert_eq!( + column.len(), + self.rows, + "NestedBands::split: column length mismatch" + ); + let band_count = self.band_count(); + let lo = if bucket == 0 { + 0 + } else { + self.boundaries[bucket - 1] + 1 + }; + let hi = if bucket == band_count - 1 { + *column + .iter() + .max() + .expect("NestedBands::split: empty column") + } else { + self.boundaries[bucket] + }; + let bucket_mask = &self.buckets[bucket]; + let pop = popcount_batch_u64(bucket_mask); + let target = pop / 2; + let words = bucket_mask.len(); + let mut scratch = vec![0u64; words]; + let (mut a, mut b) = (lo, hi); + let mut best: Option<(i32, u64, u64)> = None; + while a < b { + let mid = a + (b - a) / 2; + let m = le_mask(column, mid); + mask_ternlog::(bucket_mask, &m, bucket_mask, &mut scratch); + let below = popcount_batch_u64(&scratch); + let err = below.abs_diff(target); + if best.is_none_or(|(_, e, _)| err < e) { + best = Some((mid, err, below)); + } + if below < target { + a = mid + 1; + } else { + b = mid; + } + } + let (v, _, below) = best?; + if below == 0 || below == pop { + return None; + } + if self.boundaries.contains(&v) { + return None; + } + let mut new_boundaries = self.boundaries.clone(); + let pos = new_boundaries.partition_point(|&x| x < v); + new_boundaries.insert(pos, v); + Some( + NestedBandsBuilder::new(2) + .budget_factor(self.budget_factor) + .with_boundaries(new_boundaries, column, version), + ) + } + + /// Remove boundary `lower` (merging buckets `lower` and `lower + 1` + /// into one) and rebuild under `version`. Returns a NEW value — `self` + /// is never mutated. `None` if `band_count() <= 2` (nothing left to + /// merge into) or `lower >= band_count() - 1` (out of range). + pub fn merge(&self, lower: usize, column: &[i32], version: Version) -> Option { + assert_eq!( + column.len(), + self.rows, + "NestedBands::merge: column length mismatch" + ); + let band_count = self.band_count(); + if band_count <= 2 || lower >= band_count - 1 { + return None; + } + let mut new_boundaries = self.boundaries.clone(); + new_boundaries.remove(lower); + Some( + NestedBandsBuilder::new(2) + .budget_factor(self.budget_factor) + .with_boundaries(new_boundaries, column, version), + ) + } + + /// E-NXG-20: bisection over `[min(column), max(column))` using only + /// `gt_i32_to_mask` + popcount (no sort) — the smallest value `v` + /// whose exceedance rate (`popcount(gt v) / rows`) is at most `rate`. + /// Exceedance is non-increasing in `v`, so this is the best achievable + /// floor for the requested rate: a lower floor's rate can only be + /// worse (higher), and no strictly smaller value in range beats it. + /// Returns `(v, exceedance(v))`. + pub fn best_achievable_floor(&self, column: &[i32], rate: f64) -> (i32, f64) { + let n = column.len(); + let exceedance = |v: i32| -> f64 { + let mut m = vec![0u64; words_for(n)]; + gt_i32_to_mask(column, v, &mut m); + if !n.is_multiple_of(64) { + let last = m.len() - 1; + m[last] &= (1u64 << (n % 64)) - 1; + } + popcount_batch_u64(&m) as f64 / n as f64 + }; + let mut lo = *column.iter().min().expect("NestedBands: empty column"); + let mut hi = *column.iter().max().expect("NestedBands: empty column"); + while lo < hi { + let mid = lo + (hi - lo) / 2; + if exceedance(mid) <= rate { + hi = mid; + } else { + lo = mid + 1; + } + } + (lo, exceedance(lo)) + } + + /// Per-bucket `(count, sum, sumsq)` of `column`'s values, walked bit + /// by bit off each bucket mask (word scan + `trailing_zeros` + + /// `w &= w - 1`). + pub fn moments(&self, column: &[i32]) -> Vec { + self.buckets + .iter() + .map(|bucket| { + let mut count = 0u64; + let mut sum = 0.0f64; + let mut sumsq = 0.0f64; + for (word_idx, &word) in bucket.iter().enumerate() { + let mut w = word; + while w != 0 { + let bit_idx = w.trailing_zeros() as usize; + let row = word_idx * 64 + bit_idx; + let v = column[row] as f64; + count += 1; + sum += v; + sumsq += v * v; + w &= w - 1; + } + } + BucketMoment { count, sum, sumsq } + }) + .collect() + } + + /// Exact sigma recovered from [`moments`](Self::moments) — the room-3 + /// correction: sigma is recoverable from the seal iff the seal stores + /// two accumulators per bucket (16 bytes/bucket), not from bucket + /// midpoints, which misread the true value in a direction that depends + /// on the arbitrary top-bucket midpoint (E-NXG-17). + pub fn sigma_exact(&self, column: &[i32]) -> f64 { + let moments = self.moments(column); + let n: u64 = moments.iter().map(|m| m.count).sum(); + let total: f64 = moments.iter().map(|m| m.sum).sum(); + let totsq: f64 = moments.iter().map(|m| m.sumsq).sum(); + let n = n as f64; + (totsq / n - (total / n).powi(2)).max(0.0).sqrt() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use ndarray::simd::mask_andnot; + use ndarray::simd::ternlog::AND2 as TEST_AND2; + + const WAV_HEADER: usize = 44; + const SPEECH: &[u8] = include_bytes!("../../../data/tts-cascade/tts_real_output.wav"); + const SATURATED: &[u8] = + include_bytes!("../../../data/tts-cascade/cascade_speech_128frames.wav"); + const QUIET: &[u8] = include_bytes!("../../../data/tts-cascade/cascade_output.wav"); + + /// Decode a 16-bit mono PCM WAV's samples as `|sample|` (copied from + /// the probes' `load_abs_samples`, minus the file read). + fn abs_samples(bytes: &[u8]) -> Vec { + assert!( + bytes.len() > WAV_HEADER && &bytes[0..4] == b"RIFF", + "not a RIFF file" + ); + assert_eq!(&bytes[8..12], b"WAVE", "not WAVE"); + let bits = u16::from_le_bytes([bytes[34], bytes[35]]); + let ch = u16::from_le_bytes([bytes[22], bytes[23]]); + assert_eq!((bits, ch), (16, 1), "expected 16-bit mono PCM"); + bytes[WAV_HEADER..] + .chunks_exact(2) + .map(|c| (i16::from_le_bytes([c[0], c[1]]) as i32).abs()) + .collect() + } + + fn nested_ok(nb: &NestedBands) -> bool { + let words = nb.band(0).len(); + let mut scratch = vec![0u64; words]; + for i in 0..nb.band_count() - 1 { + mask_andnot(nb.band(i), nb.band(i + 1), &mut scratch); + if popcount_batch_u64(&scratch) != 0 { + return false; + } + } + true + } + + fn disjoint_ok(nb: &NestedBands) -> bool { + let words = nb.bucket(0).len(); + let mut scratch = vec![0u64; words]; + for i in 0..nb.band_count() { + for j in i + 1..nb.band_count() { + mask_ternlog::(nb.bucket(i), nb.bucket(j), nb.bucket(i), &mut scratch); + if popcount_batch_u64(&scratch) != 0 { + return false; + } + } + } + true + } + + #[test] + fn hist_partition_and_rank_on_speech() { + let speech = abs_samples(SPEECH); + let n = speech.len(); + let nb = NestedBandsBuilder::new(16).calibrate(&speech, 1); + assert_eq!(nb.popcounts().iter().sum::(), n as u64); + assert!(disjoint_ok(&nb), "buckets must be pairwise disjoint"); + assert!(nested_ok(&nb), "bands must be nested"); + for (row, &v) in speech.iter().enumerate() { + assert_eq!( + nb.rank(row), + nb.rank_of_value(v), + "rank mismatch at row {row}" + ); + } + let nonempty = nb.popcounts().iter().filter(|&&p| p > 0).count(); + assert_eq!( + nonempty, 16, + "all 16 buckets must be non-empty on real speech" + ); + let max_bucket = *nb.popcounts().iter().max().unwrap(); + assert!( + (max_bucket as usize) * 2 < n, + "largest bucket must hold less than half the rows" + ); + } + + #[test] + fn top_band_is_the_universe_on_a_shifted_epoch() { + let speech = abs_samples(SPEECH); + let saturated = abs_samples(SATURATED); + let nb = NestedBandsBuilder::new(16).calibrate(&speech, 1); + let stream: Vec = speech.iter().chain(saturated.iter()).copied().collect(); + let shifted = + NestedBandsBuilder::new(16).with_boundaries(nb.boundaries().to_vec(), &stream, 2); + assert_eq!( + shifted.popcounts().iter().sum::(), + stream.len() as u64, + "E-NXG-18: top band must be the universe" + ); + assert_eq!(shifted.overflow(), Some(shifted.band_count() - 1)); + } + + #[test] + fn overflow_stays_silent_on_its_calibration_epoch() { + let speech = abs_samples(SPEECH); + let nb = NestedBandsBuilder::new(16).calibrate(&speech, 1); + assert!( + nb.overflow().is_none(), + "the budget rule must not fire on its own calibration epoch" + ); + } + + #[test] + fn split_halves_the_worst_bucket_and_rewrites_nothing() { + let speech = abs_samples(SPEECH); + let saturated = abs_samples(SATURATED); + let nb = NestedBandsBuilder::new(16).calibrate(&speech, 1); + let stream: Vec = speech.iter().chain(saturated.iter()).copied().collect(); + let shifted = + NestedBandsBuilder::new(16).with_boundaries(nb.boundaries().to_vec(), &stream, 2); + let before = shifted.popcounts().to_vec(); + let w = shifted.overflow().unwrap(); + let s = shifted.split(w, &stream, 3).unwrap(); + assert_eq!(s.band_count(), shifted.band_count() + 1); + assert_eq!(s.version(), 3); + assert_eq!(s.popcounts().iter().sum::(), stream.len() as u64); + assert!(nested_ok(&s)); + assert!(*s.popcounts().iter().max().unwrap() < *before.iter().max().unwrap()); + for b in shifted.boundaries() { + assert!( + s.boundaries().contains(b), + "old boundary {b} lost after split" + ); + } + assert_eq!( + shifted.popcounts(), + before.as_slice(), + "split must not mutate the original" + ); + } + + #[test] + fn collapse_fires_on_a_quiet_epoch_and_stays_silent_on_calibration() { + let speech = abs_samples(SPEECH); + let quiet = abs_samples(QUIET); + let nb = NestedBandsBuilder::new(16).calibrate(&speech, 1); + assert!( + nb.collapsed().is_none(), + "collapse must not fire on its own calibration epoch" + ); + let on_quiet = + NestedBandsBuilder::new(16).with_boundaries(nb.boundaries().to_vec(), &quiet, 2); + assert!( + on_quiet.collapsed().is_some(), + "collapse must fire when the ladder meets a quiet epoch" + ); + } + + #[test] + fn merge_removes_one_boundary_and_sums_the_pair() { + let speech = abs_samples(SPEECH); + let quiet = abs_samples(QUIET); + let nb = NestedBandsBuilder::new(16).calibrate(&speech, 1); + let on_quiet = + NestedBandsBuilder::new(16).with_boundaries(nb.boundaries().to_vec(), &quiet, 2); + let i = on_quiet.collapsed().unwrap(); + let p = on_quiet.popcounts().to_vec(); + let m = on_quiet.merge(i, &quiet, 5).unwrap(); + assert_eq!(m.band_count(), on_quiet.band_count() - 1); + assert_eq!(m.popcounts()[i], p[i] + p[i + 1]); + assert_eq!(m.popcounts().iter().sum::(), quiet.len() as u64); + assert!(nested_ok(&m)); + + // merge on a 2-band ladder returns None. + let two_band = NestedBandsBuilder::new(2).with_boundaries(vec![100], &quiet, 9); + assert!(two_band.merge(0, &quiet, 10).is_none()); + } + + #[test] + fn split_returns_none_when_a_bucket_is_all_ties() { + // Synthetic is fine HERE: this is a degenerate-input guard, not a + // distributional claim, so a real recording adds nothing. + let column = vec![7i32; 640]; + let nb = NestedBandsBuilder::new(2).with_boundaries(vec![7], &column, 1); + assert!(nb.split(0, &column, 2).is_none()); + } + + #[test] + fn best_achievable_floor_pins_the_measured_values() { + // The floor is the SMALLEST value whose exceedance is <= rate — strictly, + // no row of slack. PROBE-NXG-FLOOR-1 pinned speech at 15 072, but that + // rank floor sat ONE ROW over the target (473/94 572 = 0.0050015 > 0.005) + // inside the probe's ±1/n tolerance; the strict definition lands at + // 15 077 (472 rows above, 0.004991). Saturated and quiet are unchanged. + let rate = 0.005; + for (bytes, expect_floor) in [(SPEECH, 15077), (SATURATED, 32765), (QUIET, 5346)] { + let column = abs_samples(bytes); + let nb = NestedBandsBuilder::new(2).with_boundaries( + vec![*column.iter().max().unwrap() - 1], + &column, + 1, + ); + let (floor, achieved) = nb.best_achievable_floor(&column, rate); + assert_eq!(floor, expect_floor, "floor mismatch"); + assert!( + achieved <= rate, + "achieved rate {achieved} exceeds the target {rate}" + ); + // Reference: the strict definition computed from a sorted copy. + let mut sorted = column.clone(); + sorted.sort_unstable(); + let n = column.len(); + let max_over = (rate * n as f64).floor() as usize; + let expect_ref = sorted[n - 1 - max_over]; + assert_eq!( + floor, expect_ref, + "floor disagrees with the sorted-copy reference" + ); + + // The next distinct value below the floor overshoots the rate — + // E-NXG-20's "best achievable". + let mut sorted = column.clone(); + sorted.sort_unstable(); + if let Some(&next_below) = sorted.iter().rev().find(|&&x| x < floor) { + let mut m = vec![0u64; column.len().div_ceil(64)]; + gt_i32_to_mask(&column, next_below, &mut m); + let n = column.len(); + if !n.is_multiple_of(64) { + let last = m.len() - 1; + m[last] &= (1u64 << (n % 64)) - 1; + } + let over = popcount_batch_u64(&m) as f64 / n as f64; + assert!( + over > rate, + "value below the floor did not overshoot the rate" + ); + } + } + } + + #[test] + fn sigma_exact_matches_direct_and_midpoints_do_not() { + let speech = abs_samples(SPEECH); + let n = speech.len() as f64; + let nb = NestedBandsBuilder::new(16).calibrate(&speech, 1); + + let mean_d = speech.iter().map(|&v| v as f64).sum::() / n; + let var_d = speech + .iter() + .map(|&v| (v as f64 - mean_d).powi(2)) + .sum::() + / n; + let sigma_direct = var_d.sqrt(); + let sigma_exact = nb.sigma_exact(&speech); + let rel_err = (sigma_exact - sigma_direct).abs() / sigma_direct; + assert!( + rel_err < 1e-9, + "sigma_exact relative error {rel_err} too large" + ); + + // Test-local midpoint estimate: bucket midpoints weighted by popcount. + let boundaries = nb.boundaries(); + let mids: Vec = (0..nb.band_count()) + .map(|i| { + let lo = if i == 0 { + 0.0 + } else { + boundaries[i - 1] as f64 + }; + // The top bucket is open-ended (E-NXG-18) and has no boundary, + // so it has no midpoint either. Use the column maximum, as the + // probe effectively did; this arbitrariness is the finding. + let hi = if i == boundaries.len() { + *speech.iter().max().unwrap() as f64 + } else { + boundaries[i] as f64 + }; + (lo + hi) / 2.0 + }) + .collect(); + let mean_h = mids + .iter() + .zip(nb.popcounts()) + .map(|(m, &p)| m * p as f64) + .sum::() + / n; + let var_h = mids + .iter() + .zip(nb.popcounts()) + .map(|(m, &p)| (m - mean_h).powi(2) * p as f64) + .sum::() + / n; + let sigma_h = var_h.sqrt(); + // Direction is NOT asserted: with the stale top boundary the probe + // over-read by 12 %; with the top bucket's midpoint at the last + // boundary it under-reads by 7 %. The estimator is not even + // sign-stable, which is why the seal stores moments instead. + let rel = (sigma_h - sigma_direct).abs() / sigma_direct; + assert!( + rel > 0.05, + "midpoint sigma {sigma_h} is within 5% of direct sigma {sigma_direct} — E-NXG-17's correction would be unnecessary" + ); + } + + #[test] + fn entropy_reads_flat_on_calibration_and_collapsed_after_shift() { + let speech = abs_samples(SPEECH); + let saturated = abs_samples(SATURATED); + let nb = NestedBandsBuilder::new(16).calibrate(&speech, 1); + assert!( + nb.entropy() > 0.999, + "entropy on the calibration epoch should read flat: {}", + nb.entropy() + ); + let stream: Vec = speech.iter().chain(saturated.iter()).copied().collect(); + let shifted = + NestedBandsBuilder::new(16).with_boundaries(nb.boundaries().to_vec(), &stream, 2); + assert!( + shifted.entropy() < 0.9, + "entropy after the shift should have collapsed: {}", + shifted.entropy() + ); + } + + #[test] + #[should_panic] + fn builder_rejects_fewer_than_two_bands() { + let _ = NestedBandsBuilder::new(1); + } + + #[test] + fn rank_walk_matches_partition_point_on_all_three_columns() { + for bytes in [SPEECH, SATURATED, QUIET] { + let column = abs_samples(bytes); + let nb = NestedBandsBuilder::new(8).calibrate(&column, 1); + for (row, &v) in column.iter().enumerate() { + assert_eq!( + nb.rank(row), + nb.rank_of_value(v), + "rank mismatch at row {row}" + ); + } + } + } +} From 75f8764608f8464e0be1178d4c9c0542dc2a9e99 Mon Sep 17 00:00:00 2001 From: Jan Hubener Date: Sat, 5 Sep 2026 14:47:55 +0000 Subject: [PATCH 2/4] =?UTF-8?q?contract:=20shape=5Frank=20=E2=80=94=20the?= =?UTF-8?q?=20D-BLW-5=20payload-law=20DTO=20(shape=20x=20Prozentrang,=20ne?= =?UTF-8?q?ver=20the=20raw=20statistic)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit lance_graph_contract::shape_rank, zero-dep. ShapeRankPayload has no f64 field by construction — the Goodhart guard enforced by shape. RemeasureLedger is write-once: a second seal at a sealed key errors and never overwrites. Producer (NestedBands::shape_rank in the planner) follows in the next commit. D-BLW-5 stays PAUSED as a loop; only the payload half is relaunched. Gates: contract shape_rank 6/6, clippy -D warnings clean, fmt clean. LATEST_STATE inventory delta in-commit. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016WkNBjHc2e3zuyz9i8qJEv --- .claude/board/LATEST_STATE.md | 7 + crates/lance-graph-contract/src/lib.rs | 3 + crates/lance-graph-contract/src/shape_rank.rs | 286 ++++++++++++++++++ 3 files changed, 296 insertions(+) create mode 100644 crates/lance-graph-contract/src/shape_rank.rs diff --git a/.claude/board/LATEST_STATE.md b/.claude/board/LATEST_STATE.md index 6f955dfd5..4f32049e8 100644 --- a/.claude/board/LATEST_STATE.md +++ b/.claude/board/LATEST_STATE.md @@ -1,3 +1,10 @@ +## 2026-09-05 — branch (D-NXG-4 → D-BLW-5, contract half): the payload-law DTO exists — INVENTORY DELTA + +- ADDED `lance_graph_contract::shape_rank` (zero-dep, std only): `SHAPE_BUCKETS = 16`; `ShapeRankPayload { shape: [u64; 16], rank: u8, version: u64 }` with `new` (asserts `rank < 16`), `mass`, `mass_below`, `prozentrang`, `rank_fraction`, `is_frozen_at`, and a `const _` guard `size_of <= 144`; `RemeasureKey { stat_id, arm, cohort, metric, dataset_version }` (`Ord`); `RemeasureError::AlreadySealed { key, sealed_version }`; `RemeasureLedger { seal (write-once, second write ERRORS, never overwrites), get, len, is_empty }`. +- WHY here and not in the planner: the doctrine's `shape₀ × rank₀` is what crosses into awareness; the type has NO `f64` field by construction, which is the Goodhart guard the payload law asks for, enforced by shape rather than discipline. `ndarray` stays unreachable supervisor-side (D-BLW-5 design §c), so the DTO is plain numbers and the producer (`NestedBands::shape_rank`, planner, next commit) does the mask work. +- STATUS: D-BLW-5 remains **PAUSED** as a loop (operator 2026-08-05); the operator relaunched ONLY the payload half on 2026-09-05 ("go ahead with D-NXG-4 into D-BLW-5"). The four arms, bloom criterion and supervisor test file are untouched. +- Gates: contract `shape_rank` 6/6, clippy `--lib --tests -D warnings` clean, fmt clean. Written by one Sonnet worker from a 62-line spec, edit-only; orchestrator-gated. + ## 2026-09-04 — two mechanical board gates land, and they were shipped WITHOUT their own hygiene — INVENTORY DELTA - ADDED `.claude/tools/append_only_gate.py` + `.github/workflows/append-only-gate.yml` (#1167) — 8 protected board files, compared against `git merge-base` (not the raw base ref, which reports a shrink whenever the base moved ahead). Absent-at-base = new, silent; deleted = FAIL; unresolvable ref = fail closed with a `fetch-depth: 0` hint. Verified at four levels: 7 self-test cases (3 fire / 4 silent), the real git path against `b67f1958`, an END-TO-END fire test (`ISSUES.md` shortened 2591 → 2541 lines → exit 1), and the fail-closed arm (`origin/main~200` → exit 1). diff --git a/crates/lance-graph-contract/src/lib.rs b/crates/lance-graph-contract/src/lib.rs index a5bc64a86..0dc887859 100644 --- a/crates/lance-graph-contract/src/lib.rs +++ b/crates/lance-graph-contract/src/lib.rs @@ -172,6 +172,9 @@ pub mod scheduler; pub mod selection; pub mod sensorium; pub mod settlement; +/// D-BLW-5 payload-law DTO (`ShapeRankPayload`, `RemeasureKey`, `RemeasureLedger`) — +/// distribution shape × Prozentrang, never the raw statistic. See module docs. +pub mod shape_rank; pub mod sigma_propagation; pub mod sla; pub mod soa_envelope; diff --git a/crates/lance-graph-contract/src/shape_rank.rs b/crates/lance-graph-contract/src/shape_rank.rs new file mode 100644 index 000000000..26ecc753a --- /dev/null +++ b/crates/lance-graph-contract/src/shape_rank.rs @@ -0,0 +1,286 @@ +//! Payload-law DTO for D-BLW-5 (`.claude/knowledge/observer-effect-tfpn-doctrine.md` +//! §2: "distribution SHAPE × Prozentrang, NEVER the raw statistic"; design +//! `.claude/board/exec-runs/d-blw-5-design-main-thread.md` §c: 16-bucket +//! histogram, `rank₀`, frozen at V₀, remeasure guard keyed +//! `(StatId, Arm, Cohort, Metric, DatasetVersion)`). +//! +//! [`ShapeRankPayload`] cannot carry a raw scalar: it has no `f64` field by +//! construction — that is the Goodhart guard enforced by shape, not by +//! discipline. The producer is `lance_graph_planner::nested_bands::NestedBands::shape_rank` +//! (D-NXG-4); this crate only holds the data that crosses into awareness. +//! Cite E-NXG-21 (NestedBands shipped) and E-NXG-2 (Prozentrang existed only +//! in doctrine before D-NXG-4). + +use std::collections::BTreeMap; +use std::fmt; + +/// Number of histogram buckets a [`ShapeRankPayload`] carries. +pub const SHAPE_BUCKETS: usize = 16; + +/// The shape-of-a-distribution payload: a 16-bucket histogram and a coarse +/// rank into it, frozen at a dataset version. +/// +/// There is no `f64` field here — not an oversight, the guard itself. A raw +/// statistic invites Goodhart's law the moment it crosses into awareness +/// (optimize the number, not the distribution it summarized); a shape +/// (histogram + bucket rank) cannot be collapsed back into a single number +/// to chase, so the DTO makes the misuse structurally unrepresentable. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct ShapeRankPayload { + /// 16-bucket histogram counts. + pub shape: [u64; SHAPE_BUCKETS], + /// Coarse ladder position, `0..SHAPE_BUCKETS`. + pub rank: u8, + /// Dataset version this payload was frozen at (V₀). + pub version: u64, +} + +impl ShapeRankPayload { + /// Construct a payload. Panics if `rank >= SHAPE_BUCKETS`. + #[must_use] + pub fn new(shape: [u64; SHAPE_BUCKETS], rank: u8, version: u64) -> Self { + assert!((rank as usize) < SHAPE_BUCKETS, "rank out of range"); + Self { + shape, + rank, + version, + } + } + + /// Total mass across all buckets, `Σ shape`. + #[must_use] + pub fn mass(&self) -> u64 { + self.shape.iter().sum() + } + + /// Mass strictly below `rank`, `Σ shape[..rank]`. + #[must_use] + pub fn mass_below(&self) -> u64 { + self.shape[..self.rank as usize].iter().sum() + } + + /// The percentile proper: `mass_below / mass`, `0.0` when `mass == 0` + /// (no distribution to be a percentile of). + #[must_use] + pub fn prozentrang(&self) -> f32 { + let mass = self.mass(); + if mass == 0 { + 0.0 + } else { + self.mass_below() as f32 / mass as f32 + } + } + + /// The coarse ladder position: `rank / SHAPE_BUCKETS`. + #[must_use] + pub fn rank_fraction(&self) -> f32 { + self.rank as f32 / SHAPE_BUCKETS as f32 + } + + /// Whether this payload was frozen at exactly `version`. + #[must_use] + pub fn is_frozen_at(&self, version: u64) -> bool { + self.version == version + } +} + +// A compile-time guard that the DTO stays scalar-free and small. +const _: () = assert!(core::mem::size_of::() <= 144); + +/// Remeasure guard key: `(StatId, Arm, Cohort, Metric, DatasetVersion)`. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct RemeasureKey { + /// Statistic identity. + pub stat_id: u32, + /// Experiment arm. + pub arm: u8, + /// Cohort identity. + pub cohort: u32, + /// Metric identity. + pub metric: u32, + /// Dataset version. + pub dataset_version: u64, +} + +/// Error returned when a [`RemeasureLedger`] key is sealed twice. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum RemeasureError { + /// The key was already sealed at `sealed_version`; the new write was + /// rejected rather than overwriting it. + AlreadySealed { + /// The key that was already sealed. + key: RemeasureKey, + /// The dataset version the existing payload was sealed at. + sealed_version: u64, + }, +} + +impl fmt::Display for RemeasureError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + RemeasureError::AlreadySealed { + key, + sealed_version, + } => write!( + f, + "remeasure rejected: stat_id={} already sealed at version {sealed_version}", + key.stat_id + ), + } + } +} + +impl std::error::Error for RemeasureError {} + +/// Write-once ledger of sealed [`ShapeRankPayload`]s, keyed by [`RemeasureKey`]. +#[derive(Debug, Default, Clone)] +pub struct RemeasureLedger { + sealed: BTreeMap, +} + +impl RemeasureLedger { + /// An empty ledger. + #[must_use] + pub fn new() -> Self { + Self { + sealed: BTreeMap::new(), + } + } + + /// Write-once. `&mut self` here is the ONE sanctioned write path (a + /// registry/builder, not a compute path — data-flow rule). A second + /// write to a sealed key ERRORS; it never overwrites. + pub fn seal( + &mut self, + key: RemeasureKey, + payload: ShapeRankPayload, + ) -> Result<(), RemeasureError> { + if let Some(existing) = self.sealed.get(&key) { + return Err(RemeasureError::AlreadySealed { + key, + sealed_version: existing.version, + }); + } + self.sealed.insert(key, payload); + Ok(()) + } + + /// Look up a sealed payload by key. + #[must_use] + pub fn get(&self, key: &RemeasureKey) -> Option<&ShapeRankPayload> { + self.sealed.get(key) + } + + /// Number of sealed entries. + #[must_use] + pub fn len(&self) -> usize { + self.sealed.len() + } + + /// Whether the ledger has no sealed entries. + #[must_use] + pub fn is_empty(&self) -> bool { + self.sealed.is_empty() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn payload_has_no_scalar_and_fits_in_144_bytes() { + assert!(core::mem::size_of::() <= 144); + let payload = ShapeRankPayload::new([1; SHAPE_BUCKETS], 4, 1); + assert_eq!(payload.mass(), 16); + assert_eq!(payload.mass_below(), 4); + assert_eq!(payload.prozentrang(), 0.25); + assert_eq!(payload.rank_fraction(), 0.25); + } + + #[test] + fn prozentrang_is_zero_on_empty_mass() { + let payload = ShapeRankPayload::new([0; SHAPE_BUCKETS], 4, 1); + assert_eq!(payload.prozentrang(), 0.0); + assert!(!payload.prozentrang().is_nan()); + } + + #[test] + #[should_panic] + fn new_rejects_rank_out_of_range() { + let _ = ShapeRankPayload::new([0; SHAPE_BUCKETS], 16, 1); + } + + #[test] + fn ledger_seals_once_and_errors_on_remeasure() { + let mut ledger = RemeasureLedger::new(); + let key = RemeasureKey { + stat_id: 1, + arm: 0, + cohort: 1, + metric: 1, + dataset_version: 1, + }; + let v1 = ShapeRankPayload::new([1; SHAPE_BUCKETS], 4, 1); + let v2 = ShapeRankPayload::new([2; SHAPE_BUCKETS], 4, 2); + + assert!(ledger.seal(key, v1).is_ok()); + let err = ledger.seal(key, v2).unwrap_err(); + assert_eq!( + err, + RemeasureError::AlreadySealed { + key, + sealed_version: 1 + } + ); + assert_eq!(ledger.get(&key), Some(&v1)); + assert_eq!(ledger.len(), 1); + } + + #[test] + fn ledger_distinguishes_every_key_field() { + let mut ledger = RemeasureLedger::new(); + let base = RemeasureKey { + stat_id: 1, + arm: 0, + cohort: 1, + metric: 1, + dataset_version: 1, + }; + let payload = ShapeRankPayload::new([1; SHAPE_BUCKETS], 4, 1); + assert!(ledger.seal(base, payload).is_ok()); + + let variants = [ + RemeasureKey { stat_id: 2, ..base }, + RemeasureKey { arm: 1, ..base }, + RemeasureKey { cohort: 2, ..base }, + RemeasureKey { metric: 2, ..base }, + RemeasureKey { + dataset_version: 2, + ..base + }, + ]; + for variant in variants { + assert!(ledger.seal(variant, payload).is_ok()); + } + assert_eq!(ledger.len(), 6); + } + + #[test] + fn error_display_names_the_key() { + let key = RemeasureKey { + stat_id: 42, + arm: 0, + cohort: 1, + metric: 1, + dataset_version: 1, + }; + let err = RemeasureError::AlreadySealed { + key, + sealed_version: 7, + }; + let msg = format!("{err}"); + assert!(msg.contains("stat_id")); + assert!(msg.contains('7')); + } +} From 4f3c1ad602cf1efbdcb9b9ac46d63036ba4553d9 Mon Sep 17 00:00:00 2001 From: Jan Hubener Date: Sat, 5 Sep 2026 14:53:56 +0000 Subject: [PATCH 3/4] D-NXG-4 into D-BLW-5: NestedBands::shape_rank produces the payload; jc::stats::fisher_2z is the axis MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit planner: Z2_SCALE, quantize_2z, NestedBandsBuilder::calibrate_equal_width (the D-BLW-5 design's equal-width-in-2z ladder), NestedBands::shape_rank(observed, V0) -> contract::shape_rank::ShapeRankPayload. Five tests; the real one builds the pooled prior from 92 speech frames' lag-1 autocorrelation. jc: FISHER_CLAMP_EPS, fisher_2z, fisher_2z_inv (forced copy of helix's hyperbolic_depth with a transcribed-formula parity test; live cross-crate parity blocked by helix's git-pinned ndarray). Finding kept (E-NXG-22): rank saturates at the edge bucket for statistics outside the prior's support — both non-speech recordings rank 0 and are indistinguishable from the prior's minimum. The loop-level fix is a design decision for D-BLW-5, which stays PAUSED; only the payload half is relaunched. Gates: planner 428/428, contract 1315/1315, jc 141/141; planner + contract clippy -D warnings clean; fmt clean. jc clippy is red on base in files this arc does not touch (TD-JC-CLIPPY-RED-ON-BASE-1). Board: E-NXG-22, AGENT_LOG, STATUS_BOARD D-NXG-4 + D-BLW-5, TECH_DEBT, LATEST_STATE delta, plan §6; supersession index regenerated last. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016WkNBjHc2e3zuyz9i8qJEv --- .claude/board/AGENT_LOG.md | 9 + .claude/board/EPIPHANIES.md | 36 ++++ .claude/board/LATEST_STATE.md | 7 + .claude/board/STATUS_BOARD.md | 4 +- .claude/board/TECH_DEBT.md | 4 + .../nexgen-mask-histogram-thresholds-v1.md | 2 +- crates/jc/src/stats.rs | 93 ++++++++ .../lance-graph-planner/src/nested_bands.rs | 200 ++++++++++++++++++ 8 files changed, 352 insertions(+), 3 deletions(-) diff --git a/.claude/board/AGENT_LOG.md b/.claude/board/AGENT_LOG.md index 07890ba50..bbfea978d 100644 --- a/.claude/board/AGENT_LOG.md +++ b/.claude/board/AGENT_LOG.md @@ -1,3 +1,12 @@ +## 2026-09-05 — D-NXG-4 → D-BLW-5 payload: two Sonnet workers on disjoint files, orchestrator-gated + +- **Why:** operator: "go ahead with D-NXG-4 into D-BLW-5". A Sonnet census first (`exec-runs/d-blw-5-design-main-thread.md`, doctrine §2, STATUS_BOARD): D-BLW-5 has NO shipped code and is PAUSED (operator 2026-08-05); its designed home is a `cycle-driver`-gated supervisor test where the planner is reachable and `ndarray` is not. That fixed the membrane split: DTO in the zero-dep contract, producer in the planner, transform in jc. +- **W1 (Sonnet, contract):** `shape_rank.rs` + one `lib.rs` line from a 62-line spec; verified the 144-byte layout by field arithmetic. 6/6, clippy clean — committed first as `75f87646` with its LATEST_STATE delta in-commit. +- **W2 (Sonnet, jc + planner):** `fisher_2z`/`fisher_2z_inv`/`FISHER_CLAMP_EPS` in `jc::stats` with a transcribed-formula parity test against helix (live cross-crate parity blocked: helix pins a git `ndarray`, jc a path one); `Z2_SCALE`, `quantize_2z`, `calibrate_equal_width`, `shape_rank` + 5 tests in the planner. W2 flagged its one guess (`RemeasureKey::new`) — correct call, it did not exist; fixed to a struct literal at the gate. +- **Falsification kept:** the can-it-fire arm ("saturated vs quiet first frame land in different buckets") could not fire — both recordings open with silence and both whole-file statistics sit below the prior's support. Restated inside the support (prior min → 0, prior max → 15) and the out-of-support saturation pinned as E-NXG-22. +- **Gates:** planner 428/428 + clippy clean, contract 1315/1315 + clippy clean, jc 141/141 (via `--manifest-path`, workspace-excluded), fmt clean on all three; jc clippy is red on `origin/main` in eight files none of which this arc touched, CI runs no jc clippy — TD row, not a fix here. Append-only + citation-decay gates local; supersession index last. +- **Board:** E-NXG-22, STATUS_BOARD D-NXG-4 + D-BLW-5 rows, TECH_DEBT TD-JC-CLIPPY-RED-ON-BASE-1, LATEST_STATE delta (planner + jc half), plan §6. + ## 2026-09-05 — D-NXG-1 `NestedBands`: one Sonnet worker from spec, orchestrator-gated - **Why:** operator "Go ahead autoattended" + "Sonnet agents for grindwork" after the three probes merged. D-NXG-1 is a write-this-file-from-spec unit: bounded input (a 134-line spec naming every type, method and test), known output shape. diff --git a/.claude/board/EPIPHANIES.md b/.claude/board/EPIPHANIES.md index 3f65e8535..b69fd4d4d 100644 --- a/.claude/board/EPIPHANIES.md +++ b/.claude/board/EPIPHANIES.md @@ -1,3 +1,39 @@ +## 2026-09-05 — E-NXG-22 — `shape × rank` is bounded by the prior's support: an out-of-support statistic saturates at the edge bucket + +**Status:** FINDING (pinned by `nested_bands::tests::shape_rank_from_real_lag1_autocorrelations`); D-NXG-4 SHIPPED into the D-BLW-5 payload. +**Confidence:** High on the mechanism; the consequence for the D-BLW-5 loop is a design question left open on purpose. + +D-NXG-4 now has its first consumer: `NestedBands::shape_rank(observed, V₀)` +produces `lance_graph_contract::shape_rank::ShapeRankPayload` — the doctrine's +`shape₀ × rank₀`, frozen at V₀, with no `f64` field by construction. The +transform lives in `jc::stats::fisher_2z` (ruled home of calibrated math), the +ladder is `calibrate_equal_width` (the D-BLW-5 design's "equal-width in 2z"), +and the remeasure guard is `RemeasureLedger::seal` (second write ERRORS). + +**The finding.** The pooled prior in the test is real: the lag-1 Pearson +autocorrelation of 92 frames of speech, `r ∈ [0.9205, 0.9917]`, i.e. +`2z ∈ [3.18, 5.48]`. The two other recordings' whole-file statistics are +`2z ≈ 1.46` (saturated) and `1.67` (quiet) — far BELOW the prior's support. +Both rank 0, and both are therefore indistinguishable from the prior's own +minimum, which also ranks 0. The first version of the test asked the two +out-of-support values to land in DIFFERENT buckets; they cannot, and the +test was restated to what is true: inside the support the ladder +discriminates (prior min → rank 0, prior max → rank 15), and outside it +saturates. **`rank` carries no "how far out".** That is partly by design — +the payload law forbids the raw scalar precisely so awareness cannot parrot +a number — but the D-BLW-5 loop's "T-silence is a reportable null" and bloom +criterion may need to know that an observation left the prior's support +entirely. The candidate fix is one bit (`out_of_support`), not a scalar; it is +a decision for the loop, which stays PAUSED, and is not made here. + +**Also measured, not claimed:** first frames of both recordings are silence +(that is why the spec's "first frame" observed statistic was wrong — it +measured leading silence, not the recording); `quantize_2z` at ×1024 keeps +three decimals of 2z and the 16 equal-width boundaries over the speech prior +differ in width by at most one unit. Gates: planner 428/428, contract +1315/1315, jc 141/141, planner + contract clippy clean; jc clippy red on +base and untouched by this arc (TD-JC-CLIPPY-RED-ON-BASE-1). + ## 2026-09-05 — E-NXG-21 — `NestedBands` sealed: the three probes became one type and twelve tests, and two of the probes' numbers were one row and one sign off **Status:** SHIPPED (`crates/lance-graph-planner/src/nested_bands.rs`, D-NXG-1; merge arm of D-NXG-5; room 3 closed by moments). diff --git a/.claude/board/LATEST_STATE.md b/.claude/board/LATEST_STATE.md index 4f32049e8..8719bf006 100644 --- a/.claude/board/LATEST_STATE.md +++ b/.claude/board/LATEST_STATE.md @@ -1,3 +1,10 @@ +## 2026-09-05 — branch (D-NXG-4 → D-BLW-5, producer half): the payload has a producer and a transform — INVENTORY DELTA + +- ADDED `lance_graph_planner::nested_bands::{Z2_SCALE, quantize_2z}`, `NestedBandsBuilder::calibrate_equal_width` (the D-BLW-5 design's equal-width-in-2z ladder, beside the equal-mass `calibrate`), `NestedBands::shape_rank(observed, V₀) -> ShapeRankPayload` (asserts 16 bands). Five tests, one on a REAL pooled prior (92 speech frames' lag-1 autocorrelation). +- ADDED `jc::stats::{FISHER_CLAMP_EPS, fisher_2z, fisher_2z_inv}` — the "2z" of plan §Stage A, forced copy of `helix::Similarity::hyperbolic_depth` with a transcribed-formula parity test (live cross-crate parity blocked by helix's git-pinned `ndarray`; recorded in the doc comment). +- FINDING E-NXG-22: `rank` saturates at the edge bucket for out-of-support statistics; the loop-level fix (one bit) is left to D-BLW-5, which stays PAUSED. +- TD-JC-CLIPPY-RED-ON-BASE-1 opened (pre-existing, untouched files, no CI clippy on jc). + ## 2026-09-05 — branch (D-NXG-4 → D-BLW-5, contract half): the payload-law DTO exists — INVENTORY DELTA - ADDED `lance_graph_contract::shape_rank` (zero-dep, std only): `SHAPE_BUCKETS = 16`; `ShapeRankPayload { shape: [u64; 16], rank: u8, version: u64 }` with `new` (asserts `rank < 16`), `mass`, `mass_below`, `prozentrang`, `rank_fraction`, `is_frozen_at`, and a `const _` guard `size_of <= 144`; `RemeasureKey { stat_id, arm, cohort, metric, dataset_version }` (`Ord`); `RemeasureError::AlreadySealed { key, sealed_version }`; `RemeasureLedger { seal (write-once, second write ERRORS, never overwrites), get, len, is_empty }`. diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index 973eeacce..c7d05f2c2 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -33,7 +33,7 @@ every row is Queued until PROBE-NXG-HIST-1 / ROLL-1 / FLOOR-1 (plan §5) run. | D-NXG-1 | `NestedBands` sealed shape (T2), version-keyed, one owner | **Shipped 2026-09-05** — `crates/lance-graph-planner/src/nested_bands.rs`, 12 tests on real recordings (E-NXG-21); no consumer wired yet | | D-NXG-2 | T1 name audit: `popcount_words` present or minted; bucket = `mask_ternlog::<0x10>` by name | **Half-closed 2026-09-05** — `ndarray::simd::popcount_batch_u64` exists (scalar); bucket-by-name proven bit-identical to `mask_andnot` (C3) | | D-NXG-3 | T1 `bisect_column_by_mask` partial-popcount bisection | **Lives in `NestedBands::split` 2026-09-05** — deliberately NOT promoted to `ndarray::simd` (gather+popcount loop, not a lane op; E-NXG-21). Promotion is its own PR if a second caller appears | -| D-NXG-4 | Prozentrang = bucket index; `shape × rank` computed from the slab | **Measured 2026-09-05** — rank == partition point on 94 572/94 572 real rows (C2); not yet wired to any consumer | +| D-NXG-4 | Prozentrang = bucket index; `shape × rank` computed from the slab | **Shipped 2026-09-05** — first consumer is the D-BLW-5 payload: `NestedBands::shape_rank` → `contract::shape_rank::ShapeRankPayload`, transform `jc::stats::fisher_2z`, guard `RemeasureLedger` (E-NXG-22) | | D-NXG-5 | rollover: split on budget or entropy, merge on collapse, never rewrite | **Both arms shipped 2026-09-05** — `NestedBands::{overflow, split, collapsed, merge}`, never-rewrite asserted in tests (E-NXG-21). Trigger order: budget leads (E-NXG-19) | | D-NXG-6 | rank-derived reject floor; σ demoted to diagnostic | **Measured 2026-09-05** (PROBE-NXG-FLOOR-1, E-NXG-20): `k` does not name a rate and at k=3 one real column's floor is unreachable; the rank floor is the best achievable boundary, exact only absent ties | | D-NXG-7 | preheat by mask inheritance | Queued | @@ -633,7 +633,7 @@ Plan: `.claude/plans/kanban-64k-inverted-awareness-v1.md` (operator anchors a/b; | PROBE-IGNITION | The write path DRIVEN: arm by MetaWord write -> discover by board scan -> cast write-on-behalf -> seal -> apply. 64 real MailboxSoA owners, real KJV corpus, 6 cycles, 7 cohorts, 11 gates both halves | lance-graph | **GREEN 2026-08-05** — 2/2 tests; c1 = 24 casts (20 Flow + 4 Block), c5/c6 rest with zero casts and no seal; G9/G10 pin the two OPEN #879 caveats | `tests/probe_ignition.rs`; AGENT_LOG 2026-08-05 | | MEASURE-64K-AXES | Operator-specified five-axis benchmark: B0 dummy baseline, B1a/B1b ownership+representation split, W0/W1 WAL segment curve (one fdatasync/one version per cycle), T0-T2 temporal phases over 1,048,576 rows, L1a/L1b chunked-layout control, EXP-KIA-A2-64K exploratory concurrency (non-claiming, A2 untouched). One release binary `measure_wal_curve`; four answers, axes never blended | lance-graph | Stage A0 MEASURED (3 of 4 answers; WAL knee NOT REPRODUCIBLE and unclaimed). v3's M-arm and O-arm both **MEASURED 2026-08-05, both NEGATIVE** (pre-registered two-sided, so both are findings): M-arm — digests MATCHED (`68128e3662df105c`), reorder 9.4 ms, downstream −25.8 ms ⇒ **Δtotal +35.2 ms, Morton LOSES**; ordered-chunk fast path 350.9 ms was slower than the generic 339.7 ms. O-arm — **DIVERGED** (`64565f362db2e4a5` ≠ `3e71c2aa7be8e325`) ⇒ the seal's ordering is load-bearing FOR THIS O-B CONSTRUCTION; follow-on question queued as PROBE-SEAL-VS-TEMPORAL. Open measurement defect: `ISS-MARM-T1-4X-A0-GAP` (M-arm T1 320-340 ms vs A0's 78-86 ms — blocks that one cross-run comparison only). A-arm (allocator-vs-architecture decomposition) deferred. v2 rolling-epoch-closure model supersedes v1's execution model (v1 = Stage A0 baseline + instrumentation, lane in build); then A1/A2 rolling+Morton lane; crypto REMOVED from the seal benchmark per operator sanity-check (seal path verified crypto-free in source; encryption = separate later layer, AEADs dep no longer blocking); EXP-KIA-A2-ROLLING-CLOSURE recorded (A2 frozen) | plans measure-64k-axes-v1 + v2 | | D-IGN-B | Ignition starts the REAL lenses: arming z ∈ {0 unarmed, 1-4 = the four stances, 5 = Fusion (Strict/Aware gap read)} — six ordinals in MetaWord's 6-bit field, no 36-style bridge (Q1 sidestepped); thought bodies = the shipped nars stance machinery via cycle_driver's pluggable seam (D-BLW-1 precedent) + blw_fusion's two-projection read. Can-fire: different lenses over byte-identical rows ⇒ non-identical readouts; silent twin: same lens ⇒ bit-identical; unarmed ⇒ none | lance-graph | **GREEN 2026-08-05** — 1/1 test, L0-L7 + z5-BLOCKED; L1 Kant≠Wittgenstein over byte-identical rows, same-lens bit-identical; Hegel/Nietzsche NON-empty on the text path; z=5 reserved with the printed blocker | plan cycle-driver 12.11; `tests/d_ign_b_lenses.rs` | -| D-BLW-5 | Observer-effect loop: a jc statistic about the cohort fed back into awareness; four pre-registered arms (true/false±/placebo) + the §12.8 bloom criterion as the frozen null instrument. KILL: placebo movement invalidates; T-silence is a reportable null. Payload refined §12.9a: distribution shape × Prozentrang (never the raw scalar); single-measurement law + remeasure guard; doctrine doc `observer-effect-tfpn-doctrine.md` | lance-graph | **PAUSED by operator 2026-08-05** — the Opus design lane was stopped mid-run (controlling signal; not relaunched). Banked and committed: the Sonnet API inventory (`exec-runs/d-blw-5-api-inventory-sonnet.md` — BeliefArena admits hand-built statements; jc+run_cycle live in disjoint crates, supervisor+jc dev-dep pre-ruled acceptable; ndarray unreachable supervisor-side). Gate to resume: operator direction | plan cycle-driver 12.9 | +| D-BLW-5 | Observer-effect loop: a jc statistic about the cohort fed back into awareness; four pre-registered arms (true/false±/placebo) + the §12.8 bloom criterion as the frozen null instrument. KILL: placebo movement invalidates; T-silence is a reportable null. Payload refined §12.9a: distribution shape × Prozentrang (never the raw scalar); single-measurement law + remeasure guard; doctrine doc `observer-effect-tfpn-doctrine.md` | lance-graph | **PAUSED by operator 2026-08-05** (loop) / **payload half RELAUNCHED by operator 2026-09-05** — `shape₀ × rank₀` DTO + remeasure guard shipped in `contract::shape_rank`, producer `NestedBands::shape_rank` (E-NXG-22); the four arms, bloom criterion and supervisor test file remain untouched — the Opus design lane was stopped mid-run (controlling signal; not relaunched). Banked and committed: the Sonnet API inventory (`exec-runs/d-blw-5-api-inventory-sonnet.md` — BeliefArena admits hand-built statements; jc+run_cycle live in disjoint crates, supervisor+jc dev-dep pre-ruled acceptable; ndarray unreachable supervisor-side). Gate to resume: operator direction | plan cycle-driver 12.9 | | PROBE-ARC-TORQUE | Torque of an arc = 2× Heron triangle area from 3 HHTL O(1) distances (magnitude metric-only; chirality via helix_orient Fisher-2z frame codes); Fisher 2z = logit((1+r)/2) as the additive equal-information embedding, hydratable via tanh. Stage B: translator stray/mindset vs source (floor = intra-language variance; Romans 5:12 in-quo/eph-hō as known-answer falsifier). Stage C: author-bias fields on the redactional layer + attribution of non-canonical books (G1-G5 in-canon ground-truth gates first). KILL per stage: F1 radial/tangential non-separation; clamp-rate ceiling; G1-G5 failures | lance-graph | Queued — proposed §12.10, behind PROBE-IGNITION + D-BLW-5 | plan cycle-driver 12.10 | | D-ACK-CLEANUP | Delete the ack/pump/tick theater entirely + add the visibility surface (operator-directed, context-hot). `kanban_actor.rs` → message-free module: `PhaseCensus` (`&self` fleet census; absorbing DAG-derived; empty ≠ at-rest) + pure `mul_target`/`parse_kanban_step`; `KanbanMsg`/`KanbanActor`/5 RPC drivers/`run_to_absorbing`/`KanbanRouteError` DELETED. Lane E migrated to direct `&mut` owner (supervisor+ractor out of its feature); W2b probe rewritten direct + census-over-real-SoA. Zombie verdict: half yes (lib.rs re-exports + lane E kept it alive; `ack_and_propose` already absent — the ack half lived only in docs). OGAR boundary verified: zero consumers; its ActionHandler ack surface is legitimate membrane protocol. Kanbanstep (`VersionScheduler::on_version`) NOT theater — stays canonical; naming question flagged only | lance-graph | **SHIPPED 2026-08-05** — all gates green (supervisor 9 lib + w2b 3/3 + cycle-driver 4/4; onebrc lane-e 20/20; clippy --no-deps clean; fmt clean) | `E-ACK-THEATER-DELETED-1`; TD-MESSAGE-RESIDUE resolution; `E-PROGRESSION-IS-EXISTENCE-NOT-COMMAND-1` | | D-HWV-1 / EXP-HOT-WINDOW | The hot version window (operator-directed): publication clock decoupled from persistence clock — every sealed cycle publishes to RAM immediately (`published_head`), durability is a batched background **sync barrier** (`durable_head`, barrier-flush fork: K unsynced Lance commits + ONE fdatasync, so 1 cycle = 1 real DatasetVersion survives everywhere). Vertical batching (time, not owners); Nagle-shaped flush policy (bytes/16-dirty/200ms/pressure/shutdown/durable-only-reader). Five panel-bought invariants H-1..H-5 (checkpoint fencing, torn-tail cleanup, no-veto-after-publish, zero-copy conditions, rung-decided visibility — H-5's original "ack rebase" clause RETRACTED per `E-PROGRESSION-IS-EXISTENCE-NOT-COMMAND-1`: no pump/ack/scheduler; seal → publish → immediately queryable → durability trails; the window is a resident horizon of immutable versions, not a message queue). Version-multiplexing fork REJECTED (temporal.rs has no cycle-within-version coordinate ⇒ intra-version hindsight). P1-P5 pre-registered with named KILLs | lance-graph | **Design banked 2026-08-05, panel-hardened (1 sweep + 1 adversarial refuter; fork inverted by evidence), NOT built** — build lane gated on operator word; P1/P2 comparisons additionally gated on ISS-MARM-T1-4X-A0-GAP + TD-LANCE9 remeasure | plan measure-64k-axes-v4; `E-HOT-WINDOW-DECOUPLES-THE-CLOCKS-1`; v2 cross-note; seal-vs-temporal caveats | diff --git a/.claude/board/TECH_DEBT.md b/.claude/board/TECH_DEBT.md index a1bcec0cd..ca788504d 100644 --- a/.claude/board/TECH_DEBT.md +++ b/.claude/board/TECH_DEBT.md @@ -1,3 +1,7 @@ +## TD-JC-CLIPPY-RED-ON-BASE-1 (2026-09-05) — OPEN + +**`crates/jc` does not pass `cargo clippy --manifest-path crates/jc/Cargo.toml --all-targets -- -D warnings`, and did not before this arc.** Measured by stashing the working tree: the same `unusual_byte_groupings` errors reproduce on `origin/main` in `dueker_zoubouloglou.rs`, `ewa_sandwich.rs`, `ewa_sandwich_3d.rs`, `koestenberger.rs`, `lib.rs`, `pearl.rs`, `pflug.rs`, `probe_p1_gamma_phase.rs`, `sigma_codebook_probe.rs`, `weyl.rs` — none touched by the D-NXG-4 arc; `stats.rs` (the file this arc edits) has zero findings. CI runs tests and `fmt --check` on jc (`jc-proof.yml`, `style.yml:226`) but no clippy step, which is how it went unseen. Same shape as `TD-SIGKER-CLIPPY-RED-ON-BASE-1` (resolved 2026-09-04 by fixing + arming the lint in CI); the same two-step fix applies. Not done here: it is not this arc's code and would widen a payload PR into a lint sweep. + ## TD-SIGMA-PROBE-MASK-ALIGNMENT-AND-WORK-METRIC-1 (2026-09-05) — OPEN **`crates/lance-graph/examples/sigma_probe_masked_traverse.rs` (shipped in diff --git a/.claude/nexgen/plans/nexgen-mask-histogram-thresholds-v1.md b/.claude/nexgen/plans/nexgen-mask-histogram-thresholds-v1.md index ce76e7050..00a2e5867 100644 --- a/.claude/nexgen/plans/nexgen-mask-histogram-thresholds-v1.md +++ b/.claude/nexgen/plans/nexgen-mask-histogram-thresholds-v1.md @@ -181,7 +181,7 @@ falsifier that would kill it. Ordered by how many rooms ahead it sits. | D-NXG-1 | `NestedBands` sealed shape (T2), version-keyed, one owner | Shipped 2026-09-05 (`planner/src/nested_bands.rs`, E-NXG-21) | | D-NXG-2 | T1 name audit: `popcount_words` present or minted; bucket = `mask_ternlog::<0x10>` by name | Queued | | D-NXG-3 | T1 `bisect_column_by_mask` partial-popcount bisection | Lives in `NestedBands::split`; not promoted to T1 (not a lane op, E-NXG-21) | -| D-NXG-4 | Prozentrang = bucket index; `shape × rank` computed from the slab | Queued | +| D-NXG-4 | Prozentrang = bucket index; `shape × rank` computed from the slab | Shipped 2026-09-05 into the D-BLW-5 payload (E-NXG-22) | | D-NXG-5 | rollover: split on budget or entropy, merge on collapse, never rewrite | Shipped 2026-09-05 — both arms, budget leads (E-NXG-19/21) | | D-NXG-6 | rank-derived reject floor; σ demoted to diagnostic | Queued | | D-NXG-7 | preheat by mask inheritance | Queued | diff --git a/crates/jc/src/stats.rs b/crates/jc/src/stats.rs index c6c6fef5c..d72775926 100644 --- a/crates/jc/src/stats.rs +++ b/crates/jc/src/stats.rs @@ -1166,6 +1166,43 @@ pub fn anova_one_way(groups: &[Vec]) -> Option { }) } +// ── Fisher 2z (D-BLW-5 payload space) ── +// +// This is the "2z" of plan `cycle-loop-closure-driver-v1.md` §Stage A and the +// D-BLW-5 histogram axis — NOT `reliability`'s or any other module's +// variance-stabilising `z = ½·ln(...)`. It is a FORCED COPY of +// `helix::fisher_z::Similarity::hyperbolic_depth`: `helix` pins a git +// `ndarray` and `jc` a path one, so a live cross-crate dependency between the +// two would double-build `ndarray`. The parity gate is the test +// `fisher_2z_matches_the_helix_formula_on_a_fixed_vector` below, which +// transcribes helix's formula locally and asserts equality — that test IS the +// contract keeping this copy honest. + +/// Rim clamp for the Fisher transform, identical to +/// `helix::fisher_z::Similarity::CLAMP_EPS` (1e-9). +pub const FISHER_CLAMP_EPS: f64 = 1e-9; + +/// Fisher **2z**: `ln((1+r)/(1−r)) = 2·atanh(r)`, `r` clamped to +/// `[-1+FISHER_CLAMP_EPS, 1-FISHER_CLAMP_EPS]` so the result is always finite +/// for a finite input. NaN input produces NaN output (propagated, not +/// clamped away) — every other `f64` input produces a finite result. +/// +/// This is `2 ×` `helix::fisher_z::Similarity::fisher_z`'s value (equivalently +/// `helix`'s `hyperbolic_depth`), NOT the variance-stabilising `z` on its own +/// — see the module-level note above. +pub fn fisher_2z(r: f64) -> f64 { + if r.is_nan() { + return f64::NAN; + } + let s = r.clamp(-1.0 + FISHER_CLAMP_EPS, 1.0 - FISHER_CLAMP_EPS); + (1.0 + s).ln() - (1.0 - s).ln() +} + +/// Inverse of [`fisher_2z`]: `tanh(z / 2)`. +pub fn fisher_2z_inv(z: f64) -> f64 { + (z / 2.0).tanh() +} + #[cfg(test)] mod tests { use super::*; @@ -2153,4 +2190,60 @@ mod tests { .unwrap_or(true) ); } + + // ─────────────────────────── Fisher 2z ─────────────────────────── + + #[test] + fn fisher_2z_is_zero_at_zero_and_odd() { + assert_eq!(fisher_2z(0.0), 0.0); + for r in [0.1, 0.3, 0.7, 0.9] { + assert!( + approx(fisher_2z(-r), -fisher_2z(r), 1e-12), + "fisher_2z is not odd at r={r}" + ); + } + } + + #[test] + fn fisher_2z_is_finite_at_the_rim() { + for r in [-1.0, 1.0, -2.0, 2.0] { + assert!(fisher_2z(r).is_finite(), "fisher_2z({r}) was not finite"); + } + } + + #[test] + fn fisher_2z_round_trips_through_inv() { + for r in [-0.9, -0.5, 0.0, 0.3, 0.7, 0.99] { + let z = fisher_2z(r); + let back = fisher_2z_inv(z); + assert!( + (back - r).abs() < 1e-12, + "round trip failed for r={r}: got back {back}" + ); + } + } + + #[test] + fn fisher_2z_matches_the_helix_formula_on_a_fixed_vector() { + // FORCED-COPY GATE (see the module-level note above `fisher_2z`): + // `helix` pins a git `ndarray`, `jc` a path one, so a live + // cross-crate dependency here would double-build `ndarray`. This + // test transcribes `helix::fisher_z::Similarity::fisher_z`'s formula + // locally — `0.5*((1+s).ln()-(1-s).ln())` with the same + // `CLAMP_EPS = 1e-9` — and asserts `fisher_2z(r) == 2 * that` to + // 1e-15 for a fixed vector. If `helix`'s formula ever changes, this + // is the test that must be updated to keep the two in parity. + fn helix_fisher_z(s: f64) -> f64 { + let s = s.clamp(-1.0 + 1e-9, 1.0 - 1e-9); + 0.5 * ((1.0 + s).ln() - (1.0 - s).ln()) + } + for r in [-0.99, -0.7, -0.3, -0.01, 0.01, 0.3, 0.7, 0.99] { + let expect = 2.0 * helix_fisher_z(r); + let got = fisher_2z(r); + assert!( + (got - expect).abs() < 1e-15, + "fisher_2z({r})={got} vs 2*helix_fisher_z={expect}" + ); + } + } } diff --git a/crates/lance-graph-planner/src/nested_bands.rs b/crates/lance-graph-planner/src/nested_bands.rs index 1d9f33065..2c897fff8 100644 --- a/crates/lance-graph-planner/src/nested_bands.rs +++ b/crates/lance-graph-planner/src/nested_bands.rs @@ -26,10 +26,31 @@ //! `examples/probe_nxg_hist_1.rs`, `examples/probe_nxg_roll_1.rs`, //! `examples/probe_nxg_floor_1.rs`. +use lance_graph_contract::shape_rank::{ShapeRankPayload, SHAPE_BUCKETS}; use lance_graph_contract::thought_atoms::normalized_entropy; use ndarray::simd::ternlog::{AND2, AND_ANDNOT2}; use ndarray::simd::{gt_i32_to_mask, mask_ternlog, popcount_batch_u64}; +/// Fixed-point scale for Fisher-2z values on the i32 mask column: 2z ∈ +/// roughly [−21, 21] at EPS=1e-9, so ×1024 keeps 3 decimals and stays far +/// inside i32. +pub const Z2_SCALE: f64 = 1024.0; + +/// `round(z * Z2_SCALE)` saturated to i32; NaN → 0 (documented). +pub fn quantize_2z(z: f64) -> i32 { + if z.is_nan() { + return 0; + } + let scaled = (z * Z2_SCALE).round(); + if scaled >= i32::MAX as f64 { + i32::MAX + } else if scaled <= i32::MIN as f64 { + i32::MIN + } else { + scaled as i32 + } +} + /// A sealed `NestedBands` value's version. Every `split`/`merge` returns a /// new `NestedBands` under a caller-supplied new version; the version is /// never incremented internally. @@ -189,6 +210,34 @@ impl NestedBandsBuilder { self.with_boundaries(boundaries, column, version) } + /// Equal-WIDTH boundaries over `[min, max]` of `column` (the D-BLW-5 + /// design's "equal-width in 2z ≈ equal-information"), as opposed to + /// [`calibrate`](Self::calibrate)'s equal-mass quantiles. `bands-1` + /// boundaries at `min + k*(max-min)/bands`, `k = 1..bands-1`, deduped + /// strictly ascending; panics if fewer than one boundary survives (a + /// degenerate column). Then [`with_boundaries`](Self::with_boundaries). + pub fn calibrate_equal_width(self, column: &[i32], version: Version) -> NestedBands { + let min = *column + .iter() + .min() + .expect("NestedBandsBuilder::calibrate_equal_width: empty column"); + let max = *column + .iter() + .max() + .expect("NestedBandsBuilder::calibrate_equal_width: empty column"); + let span = (max - min) as f64; + let bands = self.bands as f64; + let mut boundaries: Vec = (1..self.bands) + .map(|k| min + ((k as f64) * span / bands).round() as i32) + .collect(); + boundaries.dedup(); + assert!( + !boundaries.is_empty(), + "NestedBandsBuilder::calibrate_equal_width: column is degenerate (all equal-width boundaries collapsed to one value): {column:?}" + ); + self.with_boundaries(boundaries, column, version) + } + /// Seal a [`NestedBands`] from an explicit, strictly ascending /// boundary ladder. `boundaries` must be non-empty; the builder's own /// `bands` field is only consulted by [`calibrate`](Self::calibrate) — @@ -287,6 +336,23 @@ impl NestedBands { self.boundaries.partition_point(|&b| b < v) } + /// D-NXG-4 → D-BLW-5: the payload-law object. `shape` = this ladder's + /// popcounts (the pooled prior's census), `rank` = `rank_of_value(observed)` + /// (the observed statistic's Prozentrang bucket), `version` = the V₀ the + /// caller freezes. Panics unless `band_count() == SHAPE_BUCKETS`. + pub fn shape_rank(&self, observed: i32, version: Version) -> ShapeRankPayload { + assert_eq!( + self.band_count(), + SHAPE_BUCKETS, + "NestedBands::shape_rank: band_count() must equal SHAPE_BUCKETS ({SHAPE_BUCKETS}), got {}", + self.band_count() + ); + let mut shape = [0u64; SHAPE_BUCKETS]; + shape.copy_from_slice(&self.popcounts); + let rank = self.rank_of_value(observed) as u8; + ShapeRankPayload::new(shape, rank, version) + } + /// Normalized Shannon entropy of the bucket-popcount histogram (1.0 = /// perfectly flat, falling toward 0 as mass concentrates in fewer /// buckets). `0.0` if the histogram is degenerate. @@ -831,4 +897,138 @@ mod tests { } } } + + // ─────────────────────── D-BLW-5 / shape_rank ─────────────────────── + + #[test] + fn equal_width_boundaries_are_equal_width() { + let speech = abs_samples(SPEECH); + let nb = NestedBandsBuilder::new(16).calibrate_equal_width(&speech, 1); + let boundaries = nb.boundaries(); + let diffs: Vec = boundaries.windows(2).map(|w| w[1] - w[0]).collect(); + let min_d = *diffs.iter().min().expect("at least one diff"); + let max_d = *diffs.iter().max().expect("at least one diff"); + assert!( + max_d - min_d <= 1, + "equal-width boundary diffs vary by more than 1: min={min_d} max={max_d} ({diffs:?})" + ); + } + + #[test] + fn quantize_2z_saturates_and_maps_nan_to_zero() { + assert_eq!(quantize_2z(f64::NAN), 0); + assert_eq!(quantize_2z(0.0), 0); + assert_eq!(quantize_2z(1.0), 1024); + assert_eq!(quantize_2z(1e30), i32::MAX); + assert_eq!(quantize_2z(-1e30), i32::MIN); + assert_eq!(quantize_2z(f64::INFINITY), i32::MAX); + assert_eq!(quantize_2z(f64::NEG_INFINITY), i32::MIN); + } + + /// Lag-1 Pearson autocorrelation of `frame`, `0.0` if either half has + /// zero variance. Test-local: no generator, real data only. + fn lag1_r(frame: &[i32]) -> f64 { + let n = frame.len(); + if n < 2 { + return 0.0; + } + let xs: Vec = frame[..n - 1].iter().map(|&v| v as f64).collect(); + let ys: Vec = frame[1..].iter().map(|&v| v as f64).collect(); + let mx = xs.iter().sum::() / xs.len() as f64; + let my = ys.iter().sum::() / ys.len() as f64; + let mut cov = 0.0; + let mut vx = 0.0; + let mut vy = 0.0; + for (a, b) in xs.iter().zip(ys.iter()) { + cov += (a - mx) * (b - my); + vx += (a - mx) * (a - mx); + vy += (b - my) * (b - my); + } + if vx == 0.0 || vy == 0.0 { + return 0.0; + } + cov / (vx.sqrt() * vy.sqrt()) + } + + #[test] + fn shape_rank_from_real_lag1_autocorrelations() { + const FRAME: usize = 1024; + let speech = abs_samples(SPEECH); + let saturated = abs_samples(SATURATED); + let quiet = abs_samples(QUIET); + + // Cut speech into 1024-sample frames (drop the tail), lag-1 r per + // frame, through jc::stats::fisher_2z, quantized — the pooled prior. + let frames: Vec = speech + .chunks_exact(FRAME) + .map(|f| quantize_2z(jc::stats::fisher_2z(lag1_r(f)))) + .collect(); + let frame_count = frames.len(); + + let nb = NestedBandsBuilder::new(16).calibrate_equal_width(&frames, 7); + + // Can-it-fire, INSIDE the prior's support: the speech frame with the + // lowest lag-1 r and the one with the highest are real statistics + // from the same population the ladder was calibrated on, so they + // must land in different buckets — the ladder discriminates. + let (lo_q, hi_q) = (*frames.iter().min().unwrap(), *frames.iter().max().unwrap()); + let lo_payload = nb.shape_rank(lo_q, 7); + let hi_payload = nb.shape_rank(hi_q, 7); + assert_eq!( + lo_payload.shape.iter().sum::(), + frame_count as u64, + "shape must sum to the frame count" + ); + assert_eq!(lo_payload.version, 7); + assert_eq!(lo_payload.rank, 0, "the minimum of the prior is bucket 0"); + assert_eq!( + hi_payload.rank, 15, + "the maximum of the prior is the open-ended top bucket" + ); + assert!(hi_payload.prozentrang() > lo_payload.prozentrang()); + + // FINDING (pinned, not weakened): the two OTHER recordings' whole-file + // lag-1 r sit far BELOW the speech prior's support (2z ≈ 1.46 and 1.67 + // against a prior spanning ≈ 3.18..5.48). Both therefore rank 0 — and + // are indistinguishable from the prior's own minimum. shape × rank + // saturates at the edge bucket: an out-of-support statistic carries + // no "how far out" beyond rank 0. Recorded as E-NXG-22; the fix, if + // wanted, is a design decision for the D-BLW-5 loop, not a test edit. + let whole = |samples: &[i32]| quantize_2z(jc::stats::fisher_2z(lag1_r(samples))); + let (sat_q, quiet_q) = (whole(&saturated), whole(&quiet)); + assert!( + sat_q < nb.boundaries()[0] && quiet_q < nb.boundaries()[0], + "both must be below the ladder" + ); + assert_eq!(nb.shape_rank(sat_q, 7).rank, 0); + assert_eq!(nb.shape_rank(quiet_q, 7).rank, 0); + } + + #[test] + #[should_panic] + fn shape_rank_panics_off_16_bands() { + let column: Vec = (0..8).collect(); + let nb = NestedBandsBuilder::new(8).with_boundaries(vec![1, 2, 3, 4, 5, 6, 7], &column, 1); + let _ = nb.shape_rank(0, 1); + } + + #[test] + fn shape_rank_round_trips_through_the_remeasure_ledger() { + use lance_graph_contract::shape_rank::{RemeasureKey, RemeasureLedger}; + let speech = abs_samples(SPEECH); + let nb = NestedBandsBuilder::new(16).calibrate_equal_width(&speech, 1); + let payload = nb.shape_rank(speech[0], 1); + + let mut ledger = RemeasureLedger::new(); + let key = RemeasureKey { + stat_id: 1, + arm: 0, + cohort: 1, + metric: 1, + dataset_version: 7, + }; + assert!(ledger.seal(key, payload).is_ok()); + assert_eq!(ledger.get(&key), Some(&payload)); + assert!(ledger.seal(key, payload).is_err()); + } } From 7d9e03a37101b64f13a4b88a1f59e3a7839c9986 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 5 Sep 2026 15:00:03 +0000 Subject: [PATCH 4/4] =?UTF-8?q?review:=20CodeRabbit=20round=201=20on=20#11?= =?UTF-8?q?81=20=E2=80=94=20seal=20version=20guard,=20negative=20split,=20?= =?UTF-8?q?length=20asserts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - RemeasureLedger::seal rejects a payload whose frozen version differs from the key's dataset_version (new RemeasureError::VersionMismatch) before the AlreadySealed check; test fixtures aligned so payload.version == key.dataset_version - NestedBands::split: bucket-0 lower bound is the column minimum, not 0 (signed columns); regression test with signed samples - best_achievable_floor / moments / sigma_exact assert column length == rows - doc wording: producer (D-NXG-4) now exists; E-NXG-21 title, plan Status and STATUS_BOARD nexgen line synced to what shipped Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_016WkNBjHc2e3zuyz9i8qJEv --- .claude/board/EPIPHANIES.md | 2 +- .claude/board/STATUS_BOARD.md | 2 + .../nexgen-mask-histogram-thresholds-v1.md | 5 +- crates/lance-graph-contract/src/shape_rank.rs | 54 +++++++++++++++- .../lance-graph-planner/src/nested_bands.rs | 64 ++++++++++++++++++- 5 files changed, 121 insertions(+), 6 deletions(-) diff --git a/.claude/board/EPIPHANIES.md b/.claude/board/EPIPHANIES.md index b69fd4d4d..0b9fb0a41 100644 --- a/.claude/board/EPIPHANIES.md +++ b/.claude/board/EPIPHANIES.md @@ -34,7 +34,7 @@ differ in width by at most one unit. Gates: planner 428/428, contract 1315/1315, jc 141/141, planner + contract clippy clean; jc clippy red on base and untouched by this arc (TD-JC-CLIPPY-RED-ON-BASE-1). -## 2026-09-05 — E-NXG-21 — `NestedBands` sealed: the three probes became one type and twelve tests, and two of the probes' numbers were one row and one sign off +## 2026-09-05 — E-NXG-21 — `NestedBands` sealed: the three probes became one type and twelve tests, and two of the probes' numbers were off by one row and by one sign **Status:** SHIPPED (`crates/lance-graph-planner/src/nested_bands.rs`, D-NXG-1; merge arm of D-NXG-5; room 3 closed by moments). **Confidence:** High — 12/12 tests on the same three real recordings the probes used, planner lib suite green. diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index c7d05f2c2..2290cec52 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -28,6 +28,8 @@ records against itself. `.claude/nexgen/plans/nexgen-mask-histogram-thresholds-v1.md`. Nothing built; every row is Queued until PROBE-NXG-HIST-1 / ROLL-1 / FLOOR-1 (plan §5) run. +> **Update 2026-09-05 (#1181):** the sentence above is superseded — HIST-1, ROLL-1 and FLOOR-1 all ran GREEN (E-NXG-17/18/19/20) and D-NXG-1/4/5 are shipped; rows below carry the live status. + | D-id | deliverable | status | |---|---|---| | D-NXG-1 | `NestedBands` sealed shape (T2), version-keyed, one owner | **Shipped 2026-09-05** — `crates/lance-graph-planner/src/nested_bands.rs`, 12 tests on real recordings (E-NXG-21); no consumer wired yet | diff --git a/.claude/nexgen/plans/nexgen-mask-histogram-thresholds-v1.md b/.claude/nexgen/plans/nexgen-mask-histogram-thresholds-v1.md index 00a2e5867..b95ffec77 100644 --- a/.claude/nexgen/plans/nexgen-mask-histogram-thresholds-v1.md +++ b/.claude/nexgen/plans/nexgen-mask-histogram-thresholds-v1.md @@ -1,6 +1,6 @@ # nexgen-mask-histogram-thresholds-v1 — the exposure meter is a nested mask set -**Status:** PROPOSAL (2026-09-05). Nothing below is built or measured. +**Status:** IN PROGRESS (2026-09-05). §5 steps 1–4 GREEN; D-NXG-1/4/5 shipped in #1181 (`planner::nested_bands`, `contract::shape_rank`, `jc::stats::fisher_2z`); D-NXG-3 lives inside `NestedBands::split` (not promoted to T1); rooms 9–27 of §3 remain PROPOSAL and unmeasured. **D-ids:** D-NXG-1 … D-NXG-12 (rows on `.claude/board/STATUS_BOARD.md`). **Evidence:** `.claude/nexgen/harvest/` (11 verbatim agent reports, 2026-09-05). **Board block:** `.claude/board/EPIPHANIES.md` 2026-09-05 `E-NXG-*` (16 entries). @@ -64,6 +64,9 @@ at every membrane without losing a reading. - Adds (D-NXG-3): a `bisect_column_by_mask(col: &[u16], within: &[u64], target_popcount) -> boundary` partial-popcount bisection. This is the one primitive that reads a value column, and it must stay a T1 NAME so T2 never spells it out of a sort. + *(2026-09-05: shipped as the private bisection inside `NestedBands::split`, + NOT as a T1 name — it is a gather-and-popcount loop, not a lane op; see §6 + and E-NXG-21. The T1 claim above is the original proposal, kept for the record.)* - Entropy: +1 or +2 names. Every T2 concept below is expressible in the existing eight immediates plus these. diff --git a/crates/lance-graph-contract/src/shape_rank.rs b/crates/lance-graph-contract/src/shape_rank.rs index 26ecc753a..d43e18f61 100644 --- a/crates/lance-graph-contract/src/shape_rank.rs +++ b/crates/lance-graph-contract/src/shape_rank.rs @@ -7,7 +7,8 @@ //! [`ShapeRankPayload`] cannot carry a raw scalar: it has no `f64` field by //! construction — that is the Goodhart guard enforced by shape, not by //! discipline. The producer is `lance_graph_planner::nested_bands::NestedBands::shape_rank` -//! (D-NXG-4); this crate only holds the data that crosses into awareness. +//! (D-NXG-4, shipped in the same arc as this module); this crate only holds the +//! data that crosses into awareness. //! Cite E-NXG-21 (NestedBands shipped) and E-NXG-2 (Prozentrang existed only //! in doctrine before D-NXG-4). @@ -113,6 +114,15 @@ pub enum RemeasureError { /// The dataset version the existing payload was sealed at. sealed_version: u64, }, + /// The payload's frozen `version` does not equal the key's + /// `dataset_version`: a lookup by the key would return a payload frozen + /// at a different V₀ than the key names. Rejected before insertion. + VersionMismatch { + /// The key the caller tried to seal under. + key: RemeasureKey, + /// The version the payload is actually frozen at. + payload_version: u64, + }, } impl fmt::Display for RemeasureError { @@ -126,6 +136,14 @@ impl fmt::Display for RemeasureError { "remeasure rejected: stat_id={} already sealed at version {sealed_version}", key.stat_id ), + RemeasureError::VersionMismatch { + key, + payload_version, + } => write!( + f, + "remeasure rejected: stat_id={} payload frozen at version {payload_version} does not match key dataset_version {}", + key.stat_id, key.dataset_version + ), } } } @@ -155,6 +173,12 @@ impl RemeasureLedger { key: RemeasureKey, payload: ShapeRankPayload, ) -> Result<(), RemeasureError> { + if payload.version != key.dataset_version { + return Err(RemeasureError::VersionMismatch { + key, + payload_version: payload.version, + }); + } if let Some(existing) = self.sealed.get(&key) { return Err(RemeasureError::AlreadySealed { key, @@ -222,7 +246,7 @@ mod tests { dataset_version: 1, }; let v1 = ShapeRankPayload::new([1; SHAPE_BUCKETS], 4, 1); - let v2 = ShapeRankPayload::new([2; SHAPE_BUCKETS], 4, 2); + let v2 = ShapeRankPayload::new([2; SHAPE_BUCKETS], 4, 1); assert!(ledger.seal(key, v1).is_ok()); let err = ledger.seal(key, v2).unwrap_err(); @@ -261,6 +285,7 @@ mod tests { }, ]; for variant in variants { + let payload = ShapeRankPayload::new([1; SHAPE_BUCKETS], 4, variant.dataset_version); assert!(ledger.seal(variant, payload).is_ok()); } assert_eq!(ledger.len(), 6); @@ -283,4 +308,29 @@ mod tests { assert!(msg.contains("stat_id")); assert!(msg.contains('7')); } + + #[test] + fn ledger_rejects_payload_version_mismatch() { + let mut ledger = RemeasureLedger::new(); + let key = RemeasureKey { + stat_id: 1, + arm: 0, + cohort: 1, + metric: 1, + dataset_version: 1, + }; + let payload = ShapeRankPayload::new([1; SHAPE_BUCKETS], 3, 2); + let err = ledger.seal(key, payload).unwrap_err(); + assert_eq!( + err, + RemeasureError::VersionMismatch { + key, + payload_version: 2 + } + ); + assert!(ledger.is_empty(), "a rejected payload must not be inserted"); + assert!(format!("{err}").contains("dataset_version 1")); + let ok = ShapeRankPayload::new([1; SHAPE_BUCKETS], 3, 1); + assert!(ledger.seal(key, ok).is_ok()); + } } diff --git a/crates/lance-graph-planner/src/nested_bands.rs b/crates/lance-graph-planner/src/nested_bands.rs index 2c897fff8..a94115b5b 100644 --- a/crates/lance-graph-planner/src/nested_bands.rs +++ b/crates/lance-graph-planner/src/nested_bands.rs @@ -409,8 +409,14 @@ impl NestedBands { "NestedBands::split: column length mismatch" ); let band_count = self.band_count(); + // Bucket 0 covers everything <= boundaries[0], which on an i32 column + // includes negative values (Fisher-2z of a negative r is negative), so + // the search starts at the column's minimum, never at 0. let lo = if bucket == 0 { - 0 + *column + .iter() + .min() + .expect("NestedBands::split: empty column") } else { self.boundaries[bucket - 1] + 1 }; @@ -492,6 +498,11 @@ impl NestedBands { /// worse (higher), and no strictly smaller value in range beats it. /// Returns `(v, exceedance(v))`. pub fn best_achievable_floor(&self, column: &[i32], rate: f64) -> (i32, f64) { + assert_eq!( + column.len(), + self.rows, + "NestedBands::best_achievable_floor: column length mismatch" + ); let n = column.len(); let exceedance = |v: i32| -> f64 { let mut m = vec![0u64; words_for(n)]; @@ -519,6 +530,11 @@ impl NestedBands { /// by bit off each bucket mask (word scan + `trailing_zeros` + /// `w &= w - 1`). pub fn moments(&self, column: &[i32]) -> Vec { + assert_eq!( + column.len(), + self.rows, + "NestedBands::moments: column length mismatch" + ); self.buckets .iter() .map(|bucket| { @@ -548,6 +564,11 @@ impl NestedBands { /// midpoints, which misread the true value in a direction that depends /// on the arbitrary top-bucket midpoint (E-NXG-17). pub fn sigma_exact(&self, column: &[i32]) -> f64 { + assert_eq!( + column.len(), + self.rows, + "NestedBands::sigma_exact: column length mismatch" + ); let moments = self.moments(column); let n: u64 = moments.iter().map(|m| m.count).sum(); let total: f64 = moments.iter().map(|m| m.sum).sum(); @@ -1004,6 +1025,45 @@ mod tests { assert_eq!(nb.shape_rank(quiet_q, 7).rank, 0); } + /// Signed decode of the same recordings: a REAL column with negative values. + fn signed_samples(bytes: &[u8]) -> Vec { + bytes[WAV_HEADER..] + .chunks_exact(2) + .map(|c| i16::from_le_bytes([c[0], c[1]]) as i32) + .collect() + } + + #[test] + fn split_handles_a_negative_bucket_zero() { + // CodeRabbit finding on #1181: bucket 0's bisection started at 0, so a + // column whose bucket 0 lies below zero could never be split. + let speech = signed_samples(SPEECH); + assert!( + *speech.iter().min().unwrap() < 0, + "the signed column must go negative" + ); + let nb = NestedBandsBuilder::new(16).calibrate(&speech, 1); + assert!( + nb.boundaries()[0] < 0, + "bucket 0 must lie entirely below zero for this test to bite" + ); + let before = nb.popcounts()[0]; + let s = nb + .split(0, &speech, 2) + .expect("a negative bucket 0 must be splittable"); + assert_eq!(s.band_count(), nb.band_count() + 1); + assert!( + s.boundaries()[0] < nb.boundaries()[0], + "the new boundary sits inside bucket 0" + ); + assert!(s.popcounts()[0] < before && s.popcounts()[1] < before); + assert_eq!( + s.popcounts()[0] + s.popcounts()[1], + before, + "the split partitions bucket 0 exactly" + ); + } + #[test] #[should_panic] fn shape_rank_panics_off_16_bands() { @@ -1025,7 +1085,7 @@ mod tests { arm: 0, cohort: 1, metric: 1, - dataset_version: 7, + dataset_version: 1, }; assert!(ledger.seal(key, payload).is_ok()); assert_eq!(ledger.get(&key), Some(&payload));