From ae47bf3130dd723a447323bea14469eb560d95d5 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 2 Sep 2026 21:44:29 +0000 Subject: [PATCH 1/5] D-TEH-3: jc::drift + jc::quorum, and the lift-gate comparisons in jc::reliability MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The calibration MATH of the thinking-engine battery moves to jc per the ruling that jc is the single home of calibrated math (lift if correct, perfect in jc if not). - jc::drift: reencode_drift / reencode_batch (the re-encode convergence statistic, codec-agnostic — the round trip is a closure the caller supplies) and delta_summary (mean / mean|d| / max|d| / population sigma / fractions above two caller-named cut-offs). 8 tests. - jc::quorum: pairwise_agreement_u8 (per-pair 1 - sigma/sigma_max over k u8 lens tables), QuorumLevel with its 230/179/128 floors, and cronbach_report (alpha by delegation to reliability::cronbach_alpha + per-subject variances + the mean+sigma disagreement count). 6 tests. - jc::reliability: two lift-gate tests carrying the retired lab forms verbatim. Cronbach: same estimator, agrees to 1e-5 on the known-value fixture, the f32 copy loses the 1e7-shifted fixture the f64 form holds to 1e-9 (LIFT). Spearman: the retired copy ranked ties by position; tie-free fixtures cannot separate them, one tie does (1.000 vs 0.948683) (PERFECT-IN-JC, already there). No estimator changed. jc lib 135/135; the new modules are clippy -D warnings clean; fmt clean. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01PFnYKqw6d7TTiB9cT8eFdK --- crates/jc/src/drift.rs | 403 +++++++++++++++++++++++++ crates/jc/src/lib.rs | 11 + crates/jc/src/quorum.rs | 289 ++++++++++++++++++ crates/jc/src/reliability.rs | 166 ++++++++++ crates/thinking-engine/src/cronbach.rs | 358 ---------------------- 5 files changed, 869 insertions(+), 358 deletions(-) create mode 100644 crates/jc/src/drift.rs create mode 100644 crates/jc/src/quorum.rs delete mode 100644 crates/thinking-engine/src/cronbach.rs diff --git a/crates/jc/src/drift.rs b/crates/jc/src/drift.rs new file mode 100644 index 000000000..92c99f8cd --- /dev/null +++ b/crates/jc/src/drift.rs @@ -0,0 +1,403 @@ +//! Drift statistics — **re-encode convergence** and **correction-delta +//! summaries**. +//! +//! NOT a pillar. A calibration toolkit lifted out of the thinking-engine lab +//! crate under D-TEH-3 (`thinking-engine-harvest-closure-v1` §1d), per the +//! ruling that scientifically calibrated math lives in `jc` +//! (`E-JC-IS-THE-HOME-OF-ALL-CALIBRATED-MATH-1`). The lab crate keeps the +//! codec-specific GLUE (which round trip: BF16, γ+φ, the full chain) and calls +//! this module for the statistic; it carries no private copy. +//! +//! # Re-encode drift +//! +//! A codec is *re-encode safe* if iterating `decode(encode(x))` stabilises: +//! the error against the ORIGINAL value stops changing after a bounded number +//! of round trips instead of accumulating. The test is the same one an ICC +//! colour profile has to pass — `encode(decode(encode(x))) == encode(x)`. +//! [`reencode_drift`] runs the iteration for one value against any round-trip +//! closure and reports where it converged; [`reencode_batch`] aggregates a +//! sweep. +//! +//! The convergence criterion is preserved from the lifted source: the run is +//! converged at iteration `i > 0` when `|e_i − e_{i−1}| < ` +//! [`CONVERGENCE_EPS`], where `e_i` is the absolute error against the original +//! `f64` value (NOT against the previous iterate — a codec that collapses to a +//! wrong fixed point still counts as converged, and its `max_error` says how +//! wrong). The remaining history is filled with the converged error so +//! `error_history.len() == max_iterations` always holds. +//! +//! # Correction-delta summary +//! +//! [`delta_summary`] is the descriptive battery for a set of correction deltas +//! (the lifted use: `cos(activated) − cos(raw)` per centroid pair): mean, +//! mean absolute, max absolute, population standard deviation, and the +//! fraction of deltas above two caller-supplied thresholds ("material" and +//! "large"). The thresholds are parameters, not constants, so a caller cannot +//! inherit a cut-off that was tuned for a different table. + +use std::fmt; + +/// Convergence tolerance on consecutive absolute errors, in the units of the +/// original `f64` value. Preserved from the lifted source. +pub const CONVERGENCE_EPS: f64 = 1e-15; + +/// Result of one re-encode drift run. +#[derive(Clone, Debug, PartialEq)] +pub struct ReencodeDrift { + /// Iteration at which the error stopped changing; `max_iterations` if it + /// never did. + pub converged_at: usize, + /// Largest absolute error against the original value over the run. + pub max_error: f64, + /// Absolute error at the last iteration. + pub final_error: f64, + /// Absolute error per iteration; always `max_iterations` long (the tail + /// after convergence is filled with the converged error). + pub error_history: Vec, + /// `converged_at < max_iterations`. + pub safe: bool, + /// Caller-supplied codec label, for reports. + pub codec: String, +} + +impl fmt::Display for ReencodeDrift { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "{}: {} after {} iterations (max_err={:.2e}, final_err={:.2e})", + self.codec, + if self.safe { "SAFE" } else { "UNSAFE" }, + self.converged_at, + self.max_error, + self.final_error + ) + } +} + +/// Iterate `round_trip` on `value` up to `max_iterations` times and report +/// where the error against `value` stopped changing. +/// +/// `round_trip` is one `decode(encode(x))` pass in the codec's own working +/// precision (`f32`, as every lifted codec operates); the error is measured in +/// `f64` against the ORIGINAL `value`, per the module doc. +/// +/// `max_iterations == 0` yields an empty history, `converged_at == 0` and +/// `safe == false` — nothing was run, so nothing was proven. +/// +/// ``` +/// use jc::drift::reencode_drift; +/// // The identity codec is trivially safe: the error is 0 from the first +/// // pass and stops changing at the second. +/// let r = reencode_drift(0.123_456_789, 16, "identity", |x| x); +/// assert!(r.safe); +/// assert_eq!(r.converged_at, 1); +/// assert!(r.max_error < 1e-7); // only the f64 → f32 cast of the input +/// ``` +pub fn reencode_drift( + value: f64, + max_iterations: usize, + codec: impl Into, + mut round_trip: impl FnMut(f32) -> f32, +) -> ReencodeDrift { + let mut current = value as f32; + let mut errors: Vec = Vec::with_capacity(max_iterations); + let mut converged_at = max_iterations; + + for i in 0..max_iterations { + let decoded = round_trip(current); + let error = (f64::from(decoded) - value).abs(); + errors.push(error); + if i > 0 && (errors[i] - errors[i - 1]).abs() < CONVERGENCE_EPS { + converged_at = i; + errors.resize(max_iterations, error); + break; + } + current = decoded; + } + + let max_error = errors.iter().copied().fold(0.0_f64, f64::max); + let final_error = errors.last().copied().unwrap_or(0.0); + + ReencodeDrift { + converged_at, + max_error, + final_error, + error_history: errors, + safe: converged_at < max_iterations, + codec: codec.into(), + } +} + +/// Aggregate of a sweep of [`reencode_drift`] runs. +#[derive(Clone, Debug, PartialEq)] +pub struct DriftBatch { + /// Every run converged. + pub all_safe: bool, + /// The run with the largest `max_error`; `None` for an empty sweep. + pub worst: Option, + /// Largest `converged_at` over the sweep. + pub max_converged_at: usize, + /// Number of safe runs. + pub safe_count: usize, + /// Number of runs. + pub total: usize, +} + +/// Run `drift` on every value and aggregate. +pub fn reencode_batch(values: &[f64], mut drift: impl FnMut(f64) -> ReencodeDrift) -> DriftBatch { + let mut all_safe = true; + let mut worst: Option = None; + let mut max_converged_at = 0; + let mut safe_count = 0; + for &v in values { + let r = drift(v); + if r.safe { + safe_count += 1; + } else { + all_safe = false; + } + max_converged_at = max_converged_at.max(r.converged_at); + if worst.as_ref().is_none_or(|w| r.max_error > w.max_error) { + worst = Some(r); + } + } + DriftBatch { + all_safe, + worst, + max_converged_at, + safe_count, + total: values.len(), + } +} + +/// Descriptive summary of a set of correction deltas. +#[derive(Clone, Debug, PartialEq)] +pub struct DeltaSummary { + /// Number of deltas. + pub count: usize, + /// Arithmetic mean (signed). + pub mean: f64, + /// Mean absolute delta. + pub mean_abs: f64, + /// Largest absolute delta. + pub max_abs: f64, + /// Population standard deviation (divisor `n`). + pub std_dev: f64, + /// Fraction of deltas with `|δ| > material_threshold`. + pub material_fraction: f64, + /// Fraction of deltas with `|δ| > large_threshold`. + pub large_fraction: f64, + /// The "material" cut-off the fractions were computed against. + pub material_threshold: f64, + /// The "large" cut-off the fractions were computed against. + pub large_threshold: f64, +} + +impl DeltaSummary { + /// The summary of no deltas at all: every statistic is zero and `count` + /// is `0`. For callers that must render something for an empty sample; + /// [`delta_summary`] itself returns `None` there so the two cases are + /// distinguishable at the call site. + pub fn empty(material_threshold: f64, large_threshold: f64) -> Self { + Self { + count: 0, + mean: 0.0, + mean_abs: 0.0, + max_abs: 0.0, + std_dev: 0.0, + material_fraction: 0.0, + large_fraction: 0.0, + material_threshold, + large_threshold, + } + } +} + +/// Summarise `deltas` against two magnitude cut-offs. +/// +/// Returns `None` for an empty slice or any non-finite delta (the no-`NaN` +/// contract shared with [`crate::reliability`]). +/// +/// ``` +/// use jc::drift::delta_summary; +/// let s = delta_summary(&[0.02, -0.005, 0.15, 0.0], 0.01, 0.1).unwrap(); +/// assert_eq!(s.count, 4); +/// assert!((s.material_fraction - 0.5).abs() < 1e-12); // 0.02 and 0.15 +/// assert!((s.large_fraction - 0.25).abs() < 1e-12); // 0.15 only +/// ``` +pub fn delta_summary( + deltas: &[f64], + material_threshold: f64, + large_threshold: f64, +) -> Option { + if deltas.is_empty() || !deltas.iter().all(|d| d.is_finite()) { + return None; + } + let n = deltas.len() as f64; + let mean = deltas.iter().sum::() / n; + let mean_abs = deltas.iter().map(|d| d.abs()).sum::() / n; + let max_abs = deltas.iter().map(|d| d.abs()).fold(0.0_f64, f64::max); + let variance = deltas.iter().map(|d| (d - mean) * (d - mean)).sum::() / n; + let material = deltas + .iter() + .filter(|d| d.abs() > material_threshold) + .count(); + let large = deltas.iter().filter(|d| d.abs() > large_threshold).count(); + Some(DeltaSummary { + count: deltas.len(), + mean, + mean_abs, + max_abs, + std_dev: variance.sqrt(), + material_fraction: material as f64 / n, + large_fraction: large as f64 / n, + material_threshold, + large_threshold, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn approx(a: f64, b: f64, tol: f64) -> bool { + (a - b).abs() < tol + } + + // ── re-encode drift ───────────────────────────────────────────────── + + /// Disable: skip the `|e_i − e_{i−1}| < EPS` check. The identity codec + /// then reports `converged_at == max_iterations` and `safe == false`. + #[test] + fn identity_round_trip_converges_at_one_and_fills_the_history() { + let r = reencode_drift(0.25, 32, "identity", |x| x); + assert!(r.safe); + assert_eq!(r.converged_at, 1); + assert_eq!(r.error_history.len(), 32, "tail must be filled"); + assert_eq!(r.max_error, 0.0, "0.25 is exact in f32"); + assert_eq!(r.final_error, 0.0); + } + + /// The other side: a codec that multiplies by (1 + 1e-3) on every pass + /// never stabilises. Anti-vacuity: the error grows STRICTLY at every + /// step, so "never converged" is the run's real shape, not a tolerance + /// accident. + #[test] + fn a_multiplicative_drift_never_converges_and_is_unsafe() { + let r = reencode_drift(0.5, 64, "drift", |x| x * 1.001); + assert!(!r.safe); + assert_eq!(r.converged_at, 64); + assert!( + r.error_history.windows(2).all(|w| w[1] > w[0]), + "error must grow monotonically under a multiplicative drift" + ); + assert_eq!(r.final_error, r.max_error); + } + + /// Two-sided on `max_iterations`: a damped codec (x → ½ + ½(x − ½)) + /// converges after MORE than one pass but well within 64; the identical + /// run with the budget set to exactly its convergence point is unsafe. + /// Disable: fill `converged_at` with 0 instead of `i` → the first + /// assertion on `> 1` goes red. + #[test] + fn a_damped_codec_converges_late_and_the_budget_binds() { + let damped = |x: f32| 0.5 + (x - 0.5) * 0.5; + let r = reencode_drift(0.0, 64, "damped", damped); + assert!(r.safe, "{r}"); + assert!( + r.converged_at > 1, + "converged too early: {}", + r.converged_at + ); + assert!(r.converged_at < 64); + // The fixed point is 0.5; the error against the original 0.0 tends + // to 0.5 from below, so the run's max is its final value. + assert!(approx(r.final_error, 0.5, 1e-6), "{r}"); + assert_eq!(r.max_error, r.final_error); + + let tight = reencode_drift(0.0, r.converged_at, "damped", damped); + assert!(!tight.safe, "the budget must bind: {tight}"); + assert_eq!(tight.converged_at, r.converged_at); + } + + #[test] + fn a_zero_budget_proves_nothing() { + let r = reencode_drift(0.3, 0, "identity", |x| x); + assert!(!r.safe); + assert!(r.error_history.is_empty()); + assert_eq!(r.max_error, 0.0); + } + + /// The batch aggregate reports the worst run, the count, and a `false` + /// `all_safe` as soon as one value drifts. Disable: `all_safe = true` + /// unconditionally → red. + #[test] + fn batch_aggregates_the_worst_run_and_the_safe_count() { + let values = [0.1, 0.2, 0.3]; + let b = reencode_batch(&values, |v| { + // Only the middle value drifts. + if approx(v, 0.2, 1e-12) { + reencode_drift(v, 8, "drift", |x| x * 1.01) + } else { + reencode_drift(v, 8, "identity", |x| x) + } + }); + assert!(!b.all_safe); + assert_eq!(b.total, 3); + assert_eq!(b.safe_count, 2); + assert_eq!(b.max_converged_at, 8); + let worst = b.worst.expect("non-empty sweep has a worst run"); + assert_eq!(worst.codec, "drift"); + + let empty = reencode_batch(&[], |v| reencode_drift(v, 8, "identity", |x| x)); + assert!(empty.all_safe); + assert!(empty.worst.is_none()); + assert_eq!(empty.total, 0); + } + + // ── delta summary ─────────────────────────────────────────────────── + + /// Hand-computed: mean 0.04125, mean|δ| 0.04375, max|δ| 0.15, + /// population σ = √(0.0161189/4) ≈ 0.0634801. + #[test] + fn delta_summary_matches_a_hand_computed_fixture() { + let s = delta_summary(&[0.02, -0.005, 0.15, 0.0], 0.01, 0.1).unwrap(); + assert_eq!(s.count, 4); + assert!(approx(s.mean, 0.04125, 1e-12)); + assert!(approx(s.mean_abs, 0.04375, 1e-12)); + assert!(approx(s.max_abs, 0.15, 1e-12)); + assert!(approx(s.std_dev, 0.063_480_1, 1e-6), "σ was {}", s.std_dev); + assert!(approx(s.material_fraction, 0.5, 1e-12)); + assert!(approx(s.large_fraction, 0.25, 1e-12)); + assert_eq!(s.material_threshold, 0.01); + assert_eq!(s.large_threshold, 0.1); + } + + /// The cut-offs are live parameters, not decoration: raising the material + /// threshold above every delta silences the fraction, lowering it to zero + /// admits every non-zero delta. Disable: hardcode 0.01 / 0.1 inside → + /// both halves go red. + #[test] + fn thresholds_are_load_bearing_in_both_directions() { + let deltas = [0.02, -0.005, 0.15, 0.0]; + let strict = delta_summary(&deltas, 0.2, 0.5).unwrap(); + assert_eq!(strict.material_fraction, 0.0); + assert_eq!(strict.large_fraction, 0.0); + let loose = delta_summary(&deltas, 0.0, 0.0).unwrap(); + assert!( + approx(loose.material_fraction, 0.75, 1e-12), + "three non-zero deltas" + ); + assert!(approx(loose.large_fraction, 0.75, 1e-12)); + } + + #[test] + fn empty_or_non_finite_deltas_return_none() { + assert_eq!(delta_summary(&[], 0.01, 0.1), None); + assert_eq!(delta_summary(&[0.1, f64::NAN], 0.01, 0.1), None); + assert_eq!(delta_summary(&[f64::INFINITY], 0.01, 0.1), None); + let e = DeltaSummary::empty(0.01, 0.1); + assert_eq!(e.count, 0); + assert_eq!(e.material_threshold, 0.01); + } +} diff --git a/crates/jc/src/lib.rs b/crates/jc/src/lib.rs index ec8cb1acc..84bb426df 100644 --- a/crates/jc/src/lib.rs +++ b/crates/jc/src/lib.rs @@ -62,6 +62,17 @@ pub mod reliability; // Cohen's d is deliberately out of scope — the effect-size family here is r. pub mod stats; +// Drift statistics (D-TEH-3) — re-encode convergence (`reencode_drift` / +// `reencode_batch`) and correction-delta summaries (`delta_summary`), lifted +// from the thinking-engine lab crate per E-JC-IS-THE-HOME-OF-ALL-CALIBRATED- +// MATH-1. NOT a pillar. The lab keeps the codec glue and calls in here. +pub mod drift; + +// Lens quorum (D-TEH-3) — per-pair `u8` agreement across k lens tables and +// the quorum bands it maps to, plus `cronbach_report` (per-subject variances +// over `reliability::cronbach_alpha`). Lifted with `drift`; NOT a pillar. +pub mod quorum; + // PROBE-SIG-CHECKSUM — depth-2 truncated signature as a replayable // trajectory digest (H.268 probe wave, grades E-WH-TWO-SIDES-SIG-CHECKSUM-1 // leg 2). A probe, not a 12th pillar: intentionally NOT added to the diff --git a/crates/jc/src/quorum.rs b/crates/jc/src/quorum.rs new file mode 100644 index 000000000..5b6da6bc8 --- /dev/null +++ b/crates/jc/src/quorum.rs @@ -0,0 +1,289 @@ +//! Lens quorum — **per-pair agreement across `k` lens tables** and the +//! **quorum level** it maps to, plus a per-subject Cronbach report. +//! +//! NOT a pillar. Lifted out of the thinking-engine lab crate under D-TEH-3 +//! (`thinking-engine-harvest-closure-v1` §1d) per +//! `E-JC-IS-THE-HOME-OF-ALL-CALIBRATED-MATH-1`; the α estimate itself is +//! [`crate::reliability::cronbach_alpha`] — this module never recomputes it. +//! +//! # Two different questions +//! +//! Cronbach α asks about the WHOLE corpus: "do these `k` lenses behave as one +//! scale over all `n` pairs?" The quorum score asks about ONE cell: "how far +//! apart are the `k` lenses on THIS pair?" It is a normalised dispersion, +//! `1 − σ/σ_max`, where `σ_max = 255/2` is the largest population standard +//! deviation a set of `u8` values can have (half at 0, half at 255). It is +//! NOT an α per pair — α is undefined on a single subject — and the lifted +//! source said so; the name here says so too. +//! +//! # Quorum bands +//! +//! [`QuorumLevel::from_score`] cuts the `u8` score at 230 / 179 / 128 — the +//! α bands 0.90 / 0.70 / 0.50 scaled to 255 and rounded up (229.5 → 230, +//! 178.5 → 179, 127.5 → 128). A pair below `Medium` is one the fast cascade +//! should not decide alone. + +use crate::reliability::cronbach_alpha; + +/// Largest population variance of a set of `u8` values: half at 0, half at +/// 255 gives `σ² = (255/2)²`. +pub const U8_MAX_VARIANCE: f64 = 255.0 * 255.0 / 4.0; + +/// Per-pair agreement across `k` square `u8` tables of side `n`. +/// +/// `tables[t][i * n + j]` is lens `t`'s value for pair `(i, j)`. Returns an +/// `n × n` score matrix in `0..=255` — `255` = the lenses coincide on that +/// pair, `0` = maximal disagreement — symmetric, with `255` on the diagonal +/// (a table's self-distance is not a measurement to disagree about). +/// +/// Returns `None` for fewer than two tables, `n < 2`, or any table whose +/// length is not `n * n`. +/// +/// ``` +/// use jc::quorum::pairwise_agreement_u8; +/// let a = [255u8, 100, 100, 255]; +/// let b = [255u8, 100, 100, 255]; +/// let s = pairwise_agreement_u8(&[&a, &b], 2).unwrap(); +/// assert_eq!(s, vec![255, 255, 255, 255]); +/// ``` +pub fn pairwise_agreement_u8(tables: &[&[u8]], n: usize) -> Option> { + let k = tables.len(); + if k < 2 || n < 2 || tables.iter().any(|t| t.len() != n * n) { + return None; + } + let kf = k as f64; + let mut scores = vec![0u8; n * n]; + for i in 0..n { + scores[i * n + i] = 255; + for j in (i + 1)..n { + let idx = i * n + j; + let mean = tables.iter().map(|t| f64::from(t[idx])).sum::() / kf; + let var = tables + .iter() + .map(|t| { + let d = f64::from(t[idx]) - mean; + d * d + }) + .sum::() + / kf; + let agreement = 1.0 - (var / U8_MAX_VARIANCE).sqrt(); + // `agreement` is in [0, 1] by construction (var ≤ U8_MAX_VARIANCE), + // so the rounded product is in 0..=255 and the cast cannot + // truncate; the clamp is belt-and-braces against a rounding tick. + let score = (agreement * 255.0).round().clamp(0.0, 255.0) as u8; + scores[idx] = score; + scores[j * n + i] = score; + } + } + Some(scores) +} + +/// How much the lenses agree on a pair, in the bands the cascade acts on. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum QuorumLevel { + /// Score ≥ 230 (α-equivalent > 0.90): the lenses coincide. Encode + /// confidently. + High, + /// Score 179..=229 (0.70–0.90): mostly agree. Encode, carry a + /// boundary-risk mark. + Medium, + /// Score 128..=178 (0.50–0.70): the lenses see different things. Route to + /// LEAF validation. + Low, + /// Score < 128 (< 0.50): no agreement. The pair is genuinely ambiguous. + Ambiguous, +} + +impl QuorumLevel { + /// Band floors, as `u8` scores. + pub const HIGH_FLOOR: u8 = 230; + /// See [`Self::HIGH_FLOOR`]. + pub const MEDIUM_FLOOR: u8 = 179; + /// See [`Self::HIGH_FLOOR`]. + pub const LOW_FLOOR: u8 = 128; + + /// Classify a [`pairwise_agreement_u8`] score. + pub fn from_score(score: u8) -> Self { + if score >= Self::HIGH_FLOOR { + Self::High + } else if score >= Self::MEDIUM_FLOOR { + Self::Medium + } else if score >= Self::LOW_FLOOR { + Self::Low + } else { + Self::Ambiguous + } + } + + /// Should this pair bypass the fast cascade and be validated at the leaf? + pub fn needs_leaf_validation(self) -> bool { + matches!(self, Self::Low | Self::Ambiguous) + } +} + +/// Cronbach α over the corpus plus the per-subject dispersion that says +/// WHICH pairs the lenses disagree on. +#[derive(Clone, Debug, PartialEq)] +pub struct CronbachReport { + /// [`cronbach_alpha`] over all items and subjects. + pub alpha: f64, + /// Population variance across the `k` items, per subject. + pub subject_variances: Vec, + /// `mean + 1 σ` of `subject_variances`; a subject above it counts as a + /// disagreement. + pub disagreement_threshold: f64, + /// Number of subjects whose across-item variance exceeds the threshold. + pub disagreement_count: usize, + /// `k`. + pub n_items: usize, + /// `n`. + pub n_subjects: usize, +} + +/// Compute α and the per-subject disagreement profile. +/// +/// Same shape and degeneracy contract as [`cronbach_alpha`] (`items[i][s]` = +/// item `i`, subject `s`; `None` wherever α is undefined). +pub fn cronbach_report(items: &[Vec]) -> Option { + let alpha = cronbach_alpha(items)?; + let k = items.len(); + let n = items[0].len(); + let kf = k as f64; + let subject_variances: Vec = (0..n) + .map(|s| { + let mean = items.iter().map(|it| it[s]).sum::() / kf; + items + .iter() + .map(|it| { + let d = it[s] - mean; + d * d + }) + .sum::() + / kf + }) + .collect(); + let nf = n as f64; + let var_mean = subject_variances.iter().sum::() / nf; + let var_sd = (subject_variances + .iter() + .map(|v| (v - var_mean) * (v - var_mean)) + .sum::() + / nf) + .sqrt(); + let disagreement_threshold = var_mean + var_sd; + let disagreement_count = subject_variances + .iter() + .filter(|&&v| v > disagreement_threshold) + .count(); + Some(CronbachReport { + alpha, + subject_variances, + disagreement_threshold, + disagreement_count, + n_items: k, + n_subjects: n, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Anti-vacuity: the off-diagonal cells are asserted, not only the + /// diagonal the function stamps unconditionally. + #[test] + fn coinciding_tables_score_255_everywhere() { + let t = [255u8, 100, 40, 100, 255, 7, 40, 7, 255]; + let s = pairwise_agreement_u8(&[&t, &t, &t], 3).unwrap(); + assert!(s.iter().all(|&v| v == 255), "{s:?}"); + } + + /// Maximal disagreement (0 vs 255) scores 0; the score is symmetric and + /// the diagonal stays 255. Disable: drop the `sqrt` → 0-vs-255 still + /// scores 0 but a 100-vs-150 pair jumps from 205 to 245 (the third + /// assertion). + #[test] + fn maximal_disagreement_scores_zero_and_a_moderate_one_is_scaled_by_sigma() { + let a = [255u8, 0, 100, 0, 255, 0, 100, 0, 255]; + let b = [255u8, 255, 150, 255, 255, 0, 150, 0, 255]; + let s = pairwise_agreement_u8(&[&a, &b], 3).unwrap(); + assert_eq!(s[1], 0, "0 vs 255 is maximal disagreement"); + assert_eq!(s[1], s[3], "symmetric"); + // 100 vs 150: σ = 25, σ_max = 127.5 → 1 − 25/127.5 = 0.80392 → 205. + assert_eq!(s[2], 205, "{s:?}"); + assert_eq!(s[5], 255, "0 vs 0 coincide"); + assert!( + [0usize, 4, 8].iter().all(|&d| s[d] == 255), + "diagonal must be 255" + ); + } + + #[test] + fn degenerate_inputs_return_none() { + let t = [255u8, 1, 1, 255]; + assert_eq!(pairwise_agreement_u8(&[&t], 2), None); // k < 2 + assert_eq!(pairwise_agreement_u8(&[&t, &t], 1), None); // n < 2 + let short = [255u8, 1, 1]; + assert_eq!(pairwise_agreement_u8(&[&t, &short], 2), None); // ragged + } + + /// Both sides of every band floor. + #[test] + fn quorum_bands_cut_exactly_at_their_floors() { + use QuorumLevel::*; + assert_eq!(QuorumLevel::from_score(255), High); + assert_eq!(QuorumLevel::from_score(230), High); + assert_eq!(QuorumLevel::from_score(229), Medium); + assert_eq!(QuorumLevel::from_score(179), Medium); + assert_eq!(QuorumLevel::from_score(178), Low); + assert_eq!(QuorumLevel::from_score(128), Low); + assert_eq!(QuorumLevel::from_score(127), Ambiguous); + assert_eq!(QuorumLevel::from_score(0), Ambiguous); + assert!(!High.needs_leaf_validation()); + assert!(!Medium.needs_leaf_validation()); + assert!(Low.needs_leaf_validation()); + assert!(Ambiguous.needs_leaf_validation()); + } + + /// The report's α IS `reliability::cronbach_alpha` (delegation, not a + /// second formula), and one wild subject is the one flagged. Disable: + /// compare against `var_mean` alone (drop the `+ σ`) → the mild subject + /// at index 1 is flagged too and the count reads 2. + #[test] + fn report_flags_the_one_subject_the_items_disagree_on() { + // 3 items × 5 subjects. Subject 3 is where item 2 breaks rank + // (across-item variance ≈ 5.4); subject 1 is MILDLY disputed + // (variance 2.0 — values 2 ± √3), sitting above the mean variance + // (≈ 1.48) but below mean + σ (≈ 3.59); the other three subjects + // are near-unanimous (≈ 0.002). + let items = vec![ + vec![1.0, 2.0, 3.0, 4.0, 5.0], + vec![1.1, 3.732, 3.0, 4.1, 5.0], + vec![1.0, 0.268, 3.1, 9.0, 5.1], + ]; + let r = cronbach_report(&items).unwrap(); + assert_eq!(r.alpha, cronbach_alpha(&items).unwrap()); + assert_eq!((r.n_items, r.n_subjects), (3, 5)); + assert_eq!(r.subject_variances.len(), 5); + let (argmax, _) = r + .subject_variances + .iter() + .enumerate() + .max_by(|a, b| a.1.total_cmp(b.1)) + .unwrap(); + assert_eq!(argmax, 3); + assert_eq!(r.disagreement_count, 1, "{:?}", r.subject_variances); + // Anti-vacuity for the disable: the mild subject sits ABOVE the mean + // but below mean + σ, so the σ term is what excludes it. + let var_mean = r.subject_variances.iter().sum::() / 5.0; + assert!(r.subject_variances[1] > var_mean); + assert!(r.subject_variances[1] <= r.disagreement_threshold); + } + + #[test] + fn report_is_none_where_alpha_is_undefined() { + assert_eq!(cronbach_report(&[vec![1.0, 2.0]]), None); + let flat = vec![vec![1.0, 2.0], vec![2.0, 1.0]]; + assert_eq!(cronbach_report(&flat), None); + } +} diff --git a/crates/jc/src/reliability.rs b/crates/jc/src/reliability.rs index 55b90487f..c73449ee2 100644 --- a/crates/jc/src/reliability.rs +++ b/crates/jc/src/reliability.rs @@ -542,4 +542,170 @@ mod tests { "icc: expected finite or None, got {v:?}" ); } + // ── D-TEH-3 lift gate ─────────────────────────────────────────────── + // + // E-JC-IS-THE-HOME-OF-ALL-CALIBRATED-MATH-1: a copy elsewhere is lifted + // only if the two implementations agree on a fixture that can DISTINGUISH + // them. Both retired thinking-engine forms are transcribed here verbatim + // (in their own precision / rank convention) so the comparison outlives + // the deleted source and re-runs on every CI pass. + + /// The retired `thinking_engine::cronbach::cronbach_alpha`: same + /// population-variance formula, computed in `f32`, `0.0` on degeneracy. + fn retired_cronbach_f32(items: &[&[f32]]) -> f32 { + fn variance(data: &[f32]) -> f32 { + let n = data.len() as f32; + if n < 2.0 { + return 0.0; + } + let mean = data.iter().sum::() / n; + data.iter().map(|x| (x - mean).powi(2)).sum::() / n + } + let k = items.len(); + if k < 2 { + return 0.0; + } + let n = items[0].len(); + if n < 2 || items.iter().any(|it| it.len() != n) { + return 0.0; + } + let totals: Vec = (0..n) + .map(|s| items.iter().map(|it| it[s]).sum::()) + .collect(); + let var_total = variance(&totals); + if var_total < 1e-10 { + return 0.0; + } + let var_sum: f32 = items.iter().map(|it| variance(it)).sum(); + let kf = k as f32; + (kf / (kf - 1.0)) * (1.0 - var_sum / var_total) + } + + /// The retired `thinking_engine::ground_truth::spearman_rank_correlation`: + /// Pearson on ORDINAL ranks (sort position, ties broken by index), `f32`. + fn retired_spearman_ordinal(a: &[f32], b: &[f32]) -> f32 { + fn ranks(values: &[f32]) -> Vec { + let mut idx: Vec<(usize, f32)> = + values.iter().enumerate().map(|(i, &v)| (i, v)).collect(); + idx.sort_by(|x, y| x.1.partial_cmp(&y.1).unwrap()); + let mut out = vec![0.0f32; values.len()]; + for (rank, &(orig, _)) in idx.iter().enumerate() { + out[orig] = rank as f32; + } + out + } + let n = a.len().min(b.len()); + if n < 2 { + return 0.0; + } + let (ra, rb) = (ranks(a), ranks(b)); + let ma = ra.iter().sum::() / n as f32; + let mb = rb.iter().sum::() / n as f32; + let (mut num, mut da2, mut db2) = (0.0f32, 0.0f32, 0.0f32); + for i in 0..n { + let (da, db) = (ra[i] - ma, rb[i] - mb); + num += da * db; + da2 += da * da; + db2 += db * db; + } + let den = (da2 * db2).sqrt(); + if den > 1e-10 { + num / den + } else { + 0.0 + } + } + + /// Cronbach: the two forms are the SAME formula, so they agree wherever + /// `f32` can carry the arithmetic (the known-value fixture, to f32 + /// tolerance) — and the same fixture shifted by 1e7 shows why the copy + /// dies rather than lifts: the `f32` mean rounds to the nearest unit at + /// that magnitude, the deviations lose their information, and the retired + /// form returns garbage while the `f64` estimate is unchanged. Verdict: + /// LIFT (jc's is the same estimator, in a precision that survives the + /// data). Disable: compute this module's α in `f32` too → the shifted + /// half goes red. + #[test] + fn lift_gate_cronbach_agrees_with_the_retired_f32_form_only_where_f32_survives() { + let base = [ + vec![2.0, 4.0, 3.0, 5.0], + vec![3.0, 5.0, 3.0, 6.0], + vec![1.0, 3.0, 2.0, 4.0], + ]; + let base32: Vec> = base + .iter() + .map(|it| it.iter().map(|&v| v as f32).collect()) + .collect(); + let refs: Vec<&[f32]> = base32.iter().map(Vec::as_slice).collect(); + let ours = cronbach_alpha(&base).unwrap(); + let theirs = f64::from(retired_cronbach_f32(&refs)); + assert!(approx(ours, 0.984_615, 1e-5)); + assert!( + approx(ours, theirs, 1e-5), + "agree on the plain fixture: {ours} vs {theirs}" + ); + + // The distinguishing half: an affine shift leaves α invariant in exact + // arithmetic (variances are shift-free), so the f64 answer must not + // move, and the f32 answer must — that is what proves the fixture + // can tell the two apart. + let shift = 1.0e7; + let shifted: Vec> = base + .iter() + .map(|it| it.iter().map(|v| v + shift).collect()) + .collect(); + let shifted32: Vec> = shifted + .iter() + .map(|it| it.iter().map(|&v| v as f32).collect()) + .collect(); + let refs32: Vec<&[f32]> = shifted32.iter().map(Vec::as_slice).collect(); + let ours_shifted = cronbach_alpha(&shifted).unwrap(); + let theirs_shifted = f64::from(retired_cronbach_f32(&refs32)); + assert!( + approx(ours_shifted, ours, 1e-9), + "f64 α is shift-invariant: {ours_shifted}" + ); + assert!( + !approx(theirs_shifted, ours, 1e-3), + "the retired f32 form must lose the fixture at 1e7: got {theirs_shifted}" + ); + } + + /// Spearman: the retired form is NOT the same estimator — it ranks ties + /// by position instead of averaging them. Tie-free data cannot tell the + /// two apart (both give the same ρ, to f32 tolerance); a single tie does: + /// the ordinal form calls `[1,2,2,3]` a perfect monotone match of + /// `[1,2,3,4]` (ρ = 1), the average-rank form gives the textbook + /// 0.948683. Verdict: PERFECT-IN-JC (the tie correction already here is + /// the fix; the copy dies). Disable: rank ties by position in + /// `average_ranks` → the tie half goes red. + #[test] + fn lift_gate_spearman_tie_fixture_separates_the_retired_ordinal_rank_form() { + // Tie-free: agreement (to f32 tolerance). + let x = [1.0, 2.0, 3.0, 4.0, 5.0]; + let y = [3.0, 1.0, 4.0, 5.0, 2.0]; + let x32: Vec = x.iter().map(|&v| v as f32).collect(); + let y32: Vec = y.iter().map(|&v| v as f32).collect(); + let ours = spearman(&x, &y).unwrap(); + let theirs = f64::from(retired_spearman_ordinal(&x32, &y32)); + assert!(approx(ours, theirs, 1e-6), "tie-free: {ours} vs {theirs}"); + assert!( + ours.abs() < 0.99, + "the tie-free fixture must not be degenerate" + ); + + // One tie: divergence, and jc holds the hand-computed value. + let xt = [1.0, 2.0, 3.0, 4.0]; + let yt = [1.0, 2.0, 2.0, 3.0]; + let xt32: Vec = xt.iter().map(|&v| v as f32).collect(); + let yt32: Vec = yt.iter().map(|&v| v as f32).collect(); + let ours_t = spearman(&xt, &yt).unwrap(); + let theirs_t = f64::from(retired_spearman_ordinal(&xt32, &yt32)); + assert!(approx(ours_t, 0.948_683, 1e-5), "jc: {ours_t}"); + assert!( + approx(theirs_t, 1.0, 1e-6), + "ordinal ranks call the tie a perfect match: {theirs_t}" + ); + assert!(!approx(ours_t, theirs_t, 1e-2)); + } } diff --git a/crates/thinking-engine/src/cronbach.rs b/crates/thinking-engine/src/cronbach.rs deleted file mode 100644 index 95fa0efc5..000000000 --- a/crates/thinking-engine/src/cronbach.rs +++ /dev/null @@ -1,358 +0,0 @@ -//! Cronbach α for multi-lens internal consistency. -//! -//! From HANDOVER_CALIBRATION_SESSION.md (H5): -//! For N sentence pairs, compute distances via all K lenses. -//! Each lens = one "item" in the psychometric instrument. -//! Cronbach α = internal consistency of the multi-lens measurement. -//! -//! Used in two contexts: -//! -//! 1. CALIBRATION (H5 hypothesis): -//! α > 0.90 for similarity → lenses are redundant (use one, save compute) -//! α < 0.70 for relevance → lenses see different things (superposition valuable) -//! -//! 2. ENCODING QUORUM (per centroid pair): -//! During 5-lane table build, compute α across existing baked tables. -//! High α → confident encoding. Low α → boundary_risk = HIGH. -//! The quorum replaces the BF16 ±0.008 heuristic with empirical cross-model test. - -/// Cronbach's alpha for K items measured on N subjects. -/// -/// `items[k][n]` = measurement by lens k for pair n. -/// Returns α in range (-∞, 1.0]. Typically: -/// α > 0.90: excellent internal consistency (lenses agree) -/// α 0.70-0.90: acceptable (mostly agree) -/// α < 0.70: poor (lenses see different things) -/// α < 0.50: unacceptable (no agreement) -pub fn cronbach_alpha(items: &[&[f32]]) -> f32 { - let k = items.len(); - if k < 2 { - return 0.0; - } - let n = items[0].len(); - if n < 2 { - return 0.0; - } - // Verify all items have same length - for item in items { - if item.len() != n { - return 0.0; - } - } - - // Total score per subject (sum across all items) - let totals: Vec = (0..n) - .map(|pair| items.iter().map(|lens| lens[pair]).sum::()) - .collect(); - let var_total = variance(&totals); - - if var_total < 1e-10 { - return 0.0; - } // no variance = undefined - - // Sum of item variances - let var_sum: f32 = items.iter().map(|lens| variance(lens)).sum(); - - // α = (k / (k-1)) × (1 - Σvar_item / var_total) - let kf = k as f32; - (kf / (kf - 1.0)) * (1.0 - var_sum / var_total) -} - -/// Per-pair variance-based agreement score across lens tables. -/// -/// NOTE: This is NOT Cronbach α per pair. It's a normalized variance -/// score measuring how much the lenses agree on each centroid pair. -/// Low variance = high agreement. High variance = disagreement. -/// -/// Returns a score per pair: -/// 255 = low variance (lenses agree) -/// 128 = moderate variance -/// 0 = high variance (lenses disagree — investigate or LEAF validate) -/// -/// For actual Cronbach α use `cronbach_alpha()` on the full corpus. -pub fn variance_agreement_scores( - tables: &[&[u8]], // K tables, each N×N u8 - n: usize, // table dimension (N) -) -> Vec { - let k = tables.len(); - if k < 2 || n < 2 { - return vec![128u8; n * n]; // default medium confidence - } - - let mut scores = vec![0u8; n * n]; - - for i in 0..n { - scores[i * n + i] = 255; // diagonal = perfect agreement (self-distance) - for j in (i + 1)..n { - // Collect this pair's value across all lenses - let values: Vec = tables.iter().map(|t| t[i * n + j] as f32).collect(); - - // Compute agreement: how much do the lenses agree on this pair? - let mean: f32 = values.iter().sum::() / k as f32; - let var: f32 = values.iter().map(|v| (v - mean).powi(2)).sum::() / k as f32; - let max_var = 255.0f32 * 255.0 / 4.0; // max variance for u8 - - // Low variance relative to max = high agreement - let agreement = 1.0 - (var / max_var).sqrt(); - let score = (agreement * 255.0).round().clamp(0.0, 255.0) as u8; - - scores[i * n + j] = score; - scores[j * n + i] = score; - } - } - - scores -} - -/// Interpret a quorum score. -#[derive(Clone, Copy, Debug, PartialEq)] -pub enum QuorumLevel { - /// α > 0.90 — all models agree. Encode confidently. - High, - /// α 0.70-0.90 — mostly agree. Encode with boundary_risk metadata. - Medium, - /// α < 0.70 — models disagree. Mark for LEAF validation. - Low, - /// α < 0.50 — no agreement. This pair is genuinely ambiguous. - Ambiguous, -} - -impl QuorumLevel { - pub fn from_score(score: u8) -> Self { - match score { - 230..=255 => QuorumLevel::High, - 179..=229 => QuorumLevel::Medium, - 128..=178 => QuorumLevel::Low, - _ => QuorumLevel::Ambiguous, - } - } - - /// Should this pair skip the fast cascade and go to LEAF validation? - pub fn needs_leaf_validation(self) -> bool { - matches!(self, QuorumLevel::Low | QuorumLevel::Ambiguous) - } -} - -/// Compute Cronbach α for the full calibration corpus. -/// -/// Each lens provides distances for the same N text pairs. -/// Returns α and per-pair variance (for identifying problematic pairs). -pub struct CronbachResult { - /// Overall Cronbach α across all pairs. - pub alpha: f32, - /// Per-pair variance across lenses (high = lenses disagree on this pair). - pub pair_variances: Vec, - /// Number of pairs where lenses strongly disagree (variance > threshold). - pub disagreement_count: usize, - /// Number of lenses (K). - pub n_lenses: usize, - /// Number of pairs (N). - pub n_pairs: usize, -} - -impl CronbachResult { - pub fn summary(&self) -> String { - let status = if self.alpha > 0.90 { - "EXCELLENT (lenses redundant)" - } else if self.alpha > 0.70 { - "ACCEPTABLE (superposition adds a little)" - } else if self.alpha > 0.50 { - "POOR (lenses see different things — superposition valuable)" - } else { - "UNACCEPTABLE (no agreement — investigate)" - }; - format!( - "Cronbach α = {:.3} [{}]\n {} lenses × {} pairs, {} disagreements ({:.1}%)", - self.alpha, - status, - self.n_lenses, - self.n_pairs, - self.disagreement_count, - self.disagreement_count as f32 / self.n_pairs.max(1) as f32 * 100.0, - ) - } -} - -/// Compute Cronbach α from lens distance vectors. -pub fn cronbach_analysis(lens_distances: &[Vec]) -> CronbachResult { - let k = lens_distances.len(); - let n = lens_distances.first().map(|v| v.len()).unwrap_or(0); - - let refs: Vec<&[f32]> = lens_distances.iter().map(|v| v.as_slice()).collect(); - let alpha = cronbach_alpha(&refs); - - // Per-pair variance - let pair_variances: Vec = (0..n) - .map(|pair| { - let values: Vec = lens_distances.iter().map(|lens| lens[pair]).collect(); - variance(&values) - }) - .collect(); - - // Count high-variance pairs (threshold: > 1 std of pair variances) - let var_mean = pair_variances.iter().sum::() / n.max(1) as f32; - let var_std = variance(&pair_variances).sqrt(); - let threshold = var_mean + var_std; - let disagreement_count = pair_variances.iter().filter(|&&v| v > threshold).count(); - - CronbachResult { - alpha, - pair_variances, - disagreement_count, - n_lenses: k, - n_pairs: n, - } -} - -fn variance(data: &[f32]) -> f32 { - let n = data.len() as f32; - if n < 2.0 { - return 0.0; - } - let mean = data.iter().sum::() / n; - data.iter().map(|x| (x - mean).powi(2)).sum::() / n -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn perfect_agreement() { - // All lenses give identical measurements - let lens1 = vec![0.9, 0.5, 0.1, 0.8]; - let lens2 = vec![0.9, 0.5, 0.1, 0.8]; - let lens3 = vec![0.9, 0.5, 0.1, 0.8]; - let alpha = cronbach_alpha(&[&lens1, &lens2, &lens3]); - assert!( - (alpha - 1.0).abs() < 0.01, - "identical items should give α≈1.0, got {}", - alpha - ); - } - - #[test] - fn no_agreement() { - // Lenses give uncorrelated measurements - let lens1 = vec![0.9, 0.1, 0.5, 0.3]; - let lens2 = vec![0.1, 0.9, 0.3, 0.5]; - let lens3 = vec![0.5, 0.3, 0.9, 0.1]; - let alpha = cronbach_alpha(&[&lens1, &lens2, &lens3]); - assert!(alpha < 0.5, "uncorrelated should give low α, got {}", alpha); - } - - #[test] - fn partial_agreement() { - // Two agree, one disagrees - let lens1 = vec![0.9, 0.7, 0.3, 0.1]; - let lens2 = vec![0.85, 0.65, 0.25, 0.15]; - let lens3 = vec![0.1, 0.3, 0.7, 0.9]; // inverted - let alpha = cronbach_alpha(&[&lens1, &lens2, &lens3]); - assert!(alpha < 0.7, "one inverted should reduce α, got {}", alpha); - } - - #[test] - fn quorum_scores_diagonal() { - let t1 = vec![255u8, 100, 100, 255]; // 2×2 - let t2 = vec![255, 100, 100, 255]; - let scores = variance_agreement_scores(&[&t1, &t2], 2); - assert_eq!(scores[0], 255); // diagonal - assert_eq!(scores[3], 255); // diagonal - assert!(scores[1] > 200); // both agree on 100 - } - - #[test] - fn quorum_scores_disagreement() { - let t1 = vec![255, 200, 200, 255]; // 2×2 - let t2 = vec![255, 50, 50, 255]; // disagrees on off-diagonal - let scores = variance_agreement_scores(&[&t1, &t2], 2); - assert!( - scores[1] < 200, - "disagreement should lower score: {}", - scores[1] - ); - } - - #[test] - fn quorum_level_classification() { - assert_eq!(QuorumLevel::from_score(250), QuorumLevel::High); - assert_eq!(QuorumLevel::from_score(200), QuorumLevel::Medium); - assert_eq!(QuorumLevel::from_score(150), QuorumLevel::Low); - assert_eq!(QuorumLevel::from_score(50), QuorumLevel::Ambiguous); - assert!(!QuorumLevel::High.needs_leaf_validation()); - assert!(QuorumLevel::Low.needs_leaf_validation()); - assert!(QuorumLevel::Ambiguous.needs_leaf_validation()); - } - - #[test] - fn cronbach_analysis_summary() { - let lens1 = vec![0.9, 0.7, 0.5, 0.3, 0.1]; - let lens2 = vec![0.85, 0.65, 0.45, 0.25, 0.15]; - let lens3 = vec![0.88, 0.72, 0.48, 0.28, 0.12]; - let result = cronbach_analysis(&[lens1, lens2, lens3]); - assert!( - result.alpha > 0.90, - "correlated lenses should give high α: {}", - result.alpha - ); - assert_eq!(result.n_lenses, 3); - assert_eq!(result.n_pairs, 5); - eprintln!("{}", result.summary()); - } - - #[test] - fn quorum_on_real_tables() { - // Use the baked Jina, BGE-M3, Reranker tables - use crate::bge_m3_lens::BGE_M3_HDR_TABLE; - use crate::jina_lens::JINA_HDR_TABLE; - use crate::reranker_lens::RERANKER_HDR_TABLE; - - let scores = variance_agreement_scores( - &[ - JINA_HDR_TABLE.as_slice(), - BGE_M3_HDR_TABLE.as_slice(), - RERANKER_HDR_TABLE.as_slice(), - ], - 256, - ); - assert_eq!(scores.len(), 256 * 256); - - // Count quorum levels - let mut high = 0; - let mut med = 0; - let mut low = 0; - let mut amb = 0; - for &s in &scores { - match QuorumLevel::from_score(s) { - QuorumLevel::High => high += 1, - QuorumLevel::Medium => med += 1, - QuorumLevel::Low => low += 1, - QuorumLevel::Ambiguous => amb += 1, - } - } - eprintln!("Quorum on 3 baked lenses (256×256):"); - eprintln!( - " High: {} ({:.1}%)", - high, - high as f32 / scores.len() as f32 * 100.0 - ); - eprintln!( - " Medium: {} ({:.1}%)", - med, - med as f32 / scores.len() as f32 * 100.0 - ); - eprintln!( - " Low: {} ({:.1}%)", - low, - low as f32 / scores.len() as f32 * 100.0 - ); - eprintln!( - " Ambiguous: {} ({:.1}%)", - amb, - amb as f32 / scores.len() as f32 * 100.0 - ); - - // Should have some distribution — not all one level - assert!(high > 0 || med > 0, "should have some agreement"); - } -} From 3847ea714eb4a3247887a59ca866a0c425a2fdac Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 2 Sep 2026 21:44:29 +0000 Subject: [PATCH 2/5] D-TEH-3: lab crate calls jc for its calibration math; cronbach.rs deleted MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - cronbach.rs deleted (cronbach_alpha / variance_agreement_scores / QuorumLevel / cronbach_analysis now live in jc::reliability + jc::quorum). - ground_truth.rs: the private ordinal-rank Spearman and its three tests deleted; calibration::spearman_vs_ground_truth calls jc::reliability::spearman (tie-corrected), keeping its 0.0 fallback. - reencode_safety.rs: glue over jc::drift — the three codec wrappers pass round-trip closures, ReencodeSafety = jc::drift::ReencodeDrift, test_reencode_batch keeps its tuple shape; test_zipper_offsets and all 14 tests unchanged (the x256 proof runs through jc). - silu_correction.rs: CorrectionStats = jc::drift::DeltaSummary; correction_stats is an adapter with the cut-offs named MATERIAL_CORRECTION = 0.01 / LARGE_CORRECTION = 0.1. - examples/certify_jina_v5_7lane.rs: jc::reliability::cronbach_alpha, None -> NaN so an undefined alpha fails every >= verdict. - Cargo.toml: jc path dep; Cargo.lock adds jc and drops entries cargo no longer needs. Lab: cargo check --lib --examples clean except the pre-existing tts_stream_hhtld break (bgz-tensor private fn, untouched); tests for the three modules 23/23. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01PFnYKqw6d7TTiB9cT8eFdK --- crates/thinking-engine/Cargo.lock | 43 +--- crates/thinking-engine/Cargo.toml | 3 + .../examples/certify_jina_v5_7lane.rs | 22 +- crates/thinking-engine/src/ground_truth.rs | 82 +------ crates/thinking-engine/src/lib.rs | 1 - crates/thinking-engine/src/reencode_safety.rs | 228 +++++------------- crates/thinking-engine/src/silu_correction.rs | 61 ++--- 7 files changed, 109 insertions(+), 331 deletions(-) diff --git a/crates/thinking-engine/Cargo.lock b/crates/thinking-engine/Cargo.lock index f74d036e5..95d1c0ace 100644 --- a/crates/thinking-engine/Cargo.lock +++ b/crates/thinking-engine/Cargo.lock @@ -47,12 +47,6 @@ dependencies = [ "libc", ] -[[package]] -name = "arrayvec" -version = "0.7.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" - [[package]] name = "arrow-array" version = "58.4.0" @@ -168,19 +162,6 @@ version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" -[[package]] -name = "blake3" -version = "1.8.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d9e454fc11f76977dc803893aff6304ed33d6a26efae8696573bea74baa27ae" -dependencies = [ - "arrayvec", - "cc", - "cfg-if", - "constant_time_eq", - "cpufeatures", -] - [[package]] name = "bumpalo" version = "3.20.2" @@ -377,12 +358,6 @@ dependencies = [ "tiny-keccak", ] -[[package]] -name = "constant_time_eq" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" - [[package]] name = "core-foundation" version = "0.9.4" @@ -409,15 +384,6 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" -[[package]] -name = "cpufeatures" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" -dependencies = [ - "libc", -] - [[package]] name = "crc32fast" version = "1.5.0" @@ -1387,6 +1353,13 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jc" +version = "0.1.0" +dependencies = [ + "ndarray", +] + [[package]] name = "js-sys" version = "0.3.94" @@ -1565,7 +1538,6 @@ dependencies = [ name = "ndarray" version = "0.17.2" dependencies = [ - "blake3", "fractal", "matrixmultiply", "num-complex", @@ -2398,6 +2370,7 @@ dependencies = [ "half", "hf-hub", "highheelbgz", + "jc", "lance-graph-contract", "ndarray", "rayon", diff --git a/crates/thinking-engine/Cargo.toml b/crates/thinking-engine/Cargo.toml index 37214fc6f..06f3c4f40 100644 --- a/crates/thinking-engine/Cargo.toml +++ b/crates/thinking-engine/Cargo.toml @@ -18,6 +18,9 @@ lance-graph-contract = { path = "../lance-graph-contract" } ndarray = { path = "../../../ndarray", default-features = false, features = ["std"] } bgz-tensor = { path = "../bgz-tensor" } highheelbgz = { path = "../highheelbgz" } +# D-TEH-3: the calibration MATH (Cronbach α, Spearman ρ, re-encode drift, +# correction-delta summaries) lives in jc; this crate keeps the glue. +jc = { path = "../jc" } tokenizers = { version = "0.22", optional = true, features = ["http"] } serde_json = "1.0" serde = { version = "1.0", features = ["derive"] } diff --git a/crates/thinking-engine/examples/certify_jina_v5_7lane.rs b/crates/thinking-engine/examples/certify_jina_v5_7lane.rs index 7ecd50c4c..b9cf76be5 100644 --- a/crates/thinking-engine/examples/certify_jina_v5_7lane.rs +++ b/crates/thinking-engine/examples/certify_jina_v5_7lane.rs @@ -274,11 +274,16 @@ fn main() { let spearman = quality::spearman(&ref_upper, &lane6_upper); let ref_z = z_score_normalize(&ref_upper); let lane6_z = z_score_normalize(&lane6_upper); - let cronbach_a = - thinking_engine::cronbach::cronbach_alpha(&[ref_z.as_slice(), lane6_z.as_slice()]); + // jc returns `None` where α is undefined; NaN fails every `>=` below, + // which is the honest verdict for an undefined estimate. + let cronbach_a = jc::reliability::cronbach_alpha(&[ + ref_z.iter().map(|&v| f64::from(v)).collect(), + lane6_z.iter().map(|&v| f64::from(v)).collect(), + ]) + .unwrap_or(f64::NAN); let lane6_verdict = if pearson >= TARGET_LAB_BF16 && spearman >= TARGET_LAB_BF16 - && (cronbach_a as f64) >= TARGET_LAB_BF16 + && cronbach_a >= TARGET_LAB_BF16 { "PASS" } else { @@ -308,7 +313,7 @@ fn main() { " Cronbach α = {:.4} target {:.4} [{}]", cronbach_a, TARGET_LAB_BF16, - if (cronbach_a as f64) >= TARGET_LAB_BF16 { + if cronbach_a >= TARGET_LAB_BF16 { "pass" } else { "FAIL" @@ -321,7 +326,7 @@ fn main() { target: TARGET_LAB_BF16, pearson, spearman, - cronbach_alpha: cronbach_a as f64, + cronbach_alpha: cronbach_a, verdict: lane6_verdict.to_string(), }); @@ -2024,8 +2029,11 @@ fn measure_lane( // practice when items are on different measurement scales. let ref_z: Vec = z_score_normalize(reference); let lane_z: Vec = z_score_normalize(lane); - let cronbach_a = - thinking_engine::cronbach::cronbach_alpha(&[ref_z.as_slice(), lane_z.as_slice()]) as f64; + let cronbach_a = jc::reliability::cronbach_alpha(&[ + ref_z.iter().map(|&v| f64::from(v)).collect(), + lane_z.iter().map(|&v| f64::from(v)).collect(), + ]) + .unwrap_or(f64::NAN); let metric_value = match primary { "pearson" => pearson, diff --git a/crates/thinking-engine/src/ground_truth.rs b/crates/thinking-engine/src/ground_truth.rs index 17ebf2385..e9e795d2f 100644 --- a/crates/thinking-engine/src/ground_truth.rs +++ b/crates/thinking-engine/src/ground_truth.rs @@ -194,85 +194,21 @@ pub mod calibration { if gt.len() != baked_distances.len() || gt.len() < 2 { return 0.0; } - super::spearman_rank_correlation(>, baked_distances) + // MATH lives in jc (D-TEH-3); tie-corrected average ranks. `None` + // (undefined ρ) keeps the glue's historical `0.0` fallback. + let gt64: Vec = gt.iter().map(|&v| f64::from(v)).collect(); + let bd64: Vec = baked_distances.iter().map(|&v| f64::from(v)).collect(); + jc::reliability::spearman(>64, &bd64).map_or(0.0, |r| r as f32) } } -// -/// Spearman rank correlation between two f32 slices. -pub fn spearman_rank_correlation(a: &[f32], b: &[f32]) -> f32 { - let n = a.len().min(b.len()); - if n < 2 { - return 0.0; - } - let rank_a = ranks(a); - let rank_b = ranks(b); - let mean_a = rank_a.iter().sum::() / n as f32; - let mean_b = rank_b.iter().sum::() / n as f32; - let mut num = 0.0f32; - let mut den_a = 0.0f32; - let mut den_b = 0.0f32; - for i in 0..n { - let da = rank_a[i] - mean_a; - let db = rank_b[i] - mean_b; - num += da * db; - den_a += da * da; - den_b += db * db; - } - let den = (den_a * den_b).sqrt(); - if den > 1e-10 { - num / den - } else { - 0.0 - } -} - -fn ranks(values: &[f32]) -> Vec { - let mut indexed: Vec<(usize, f32)> = values.iter().enumerate().map(|(i, &v)| (i, v)).collect(); - indexed.sort_by(|a, b| a.1.partial_cmp(&b.1).unwrap()); - let mut result = vec![0.0f32; values.len()]; - for (rank, &(orig_idx, _)) in indexed.iter().enumerate() { - result[orig_idx] = rank as f32; - } - result -} +// The Spearman estimator that used to live here moved to +// `jc::reliability::spearman` under D-TEH-3 (tie-corrected; the private +// ordinal-rank copy was numerically wrong under ties — see jc's lift-gate +// test). `calibration::spearman_vs_ground_truth` calls jc. #[cfg(test)] mod tests { - use super::*; - - #[test] - fn spearman_perfect_correlation() { - let a = vec![1.0, 2.0, 3.0, 4.0, 5.0]; - let b = vec![10.0, 20.0, 30.0, 40.0, 50.0]; - let rho = spearman_rank_correlation(&a, &b); - assert!( - (rho - 1.0).abs() < 1e-4, - "perfect correlation should be ~1.0, got {}", - rho - ); - } - - #[test] - fn spearman_inverse_correlation() { - let a = vec![1.0, 2.0, 3.0, 4.0, 5.0]; - let b = vec![50.0, 40.0, 30.0, 20.0, 10.0]; - let rho = spearman_rank_correlation(&a, &b); - assert!( - (rho - (-1.0)).abs() < 1e-4, - "inverse should be ~-1.0, got {}", - rho - ); - } - - #[test] - fn spearman_no_correlation() { - let a = vec![1.0, 2.0, 3.0, 4.0]; - let b = vec![3.0, 1.0, 4.0, 2.0]; - let rho = spearman_rank_correlation(&a, &b); - assert!(rho.abs() < 0.5, "shuffled should be near zero, got {}", rho); - } - #[cfg(feature = "calibration")] mod calibration_tests { use super::super::calibration::*; diff --git a/crates/thinking-engine/src/lib.rs b/crates/thinking-engine/src/lib.rs index bc9b8bd17..54cbe6860 100644 --- a/crates/thinking-engine/src/lib.rs +++ b/crates/thinking-engine/src/lib.rs @@ -27,7 +27,6 @@ pub mod cognitive_trace; pub mod composite_engine; pub mod contract_bridge; pub mod contrastive_learner; -pub mod cronbach; pub mod domino; pub mod dto; pub mod dual_engine; diff --git a/crates/thinking-engine/src/reencode_safety.rs b/crates/thinking-engine/src/reencode_safety.rs index ef25e344a..2ea91d2e9 100644 --- a/crates/thinking-engine/src/reencode_safety.rs +++ b/crates/thinking-engine/src/reencode_safety.rs @@ -8,76 +8,26 @@ //! encode(decode(encode(x))) ≠ encode(x) → drift → unsafe //! //! Goal: prove "x256 re-encode safety" — 256 round-trips with bounded error. +//! +//! **Where the math lives (D-TEH-3).** The drift statistic itself — +//! iterate a round trip, track the error history against the original value, +//! detect convergence, aggregate a sweep — is `jc::drift::{reencode_drift, +//! reencode_batch}`. This module is the GLUE: it knows which codecs to test +//! (BF16, γ+φ, the full chain) and hands each to jc as a round-trip closure. +//! It carries no private copy of the statistic. use bgz_tensor::gamma_phi::{gamma_phi_decode, gamma_phi_encode}; use bgz_tensor::stacked_n::{bf16_to_f32, f32_to_bf16}; +use jc::drift::{reencode_batch, reencode_drift}; -/// Result of a re-encode safety test. -#[derive(Clone, Debug)] -pub struct ReencodeSafety { - /// How many iterations until error stabilized (delta < threshold). - pub converged_at: usize, - /// Maximum error seen across all iterations. - pub max_error: f64, - /// Error at final iteration. - pub final_error: f64, - /// Error history per iteration. - pub error_history: Vec, - /// Is it re-encode safe? (converged within max_iterations) - pub safe: bool, - /// Codec name. - pub codec: String, -} - -impl std::fmt::Display for ReencodeSafety { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{}: {} after {} iterations (max_err={:.2e}, final_err={:.2e})", - self.codec, - if self.safe { "SAFE" } else { "UNSAFE" }, - self.converged_at, - self.max_error, - self.final_error - ) - } -} +/// Result of a re-encode safety test — `jc::drift::ReencodeDrift`. +pub type ReencodeSafety = jc::drift::ReencodeDrift; /// Test BF16 round-trip: f64 → f32 → bf16 → f32 → bf16 → ... → f64 pub fn test_bf16_reencode(value: f64, max_iterations: usize) -> ReencodeSafety { - let mut current = value as f32; - let mut errors = Vec::new(); - let mut converged_at = max_iterations; - - for i in 0..max_iterations { - let encoded = f32_to_bf16(current); - let decoded = bf16_to_f32(encoded); - let error = (decoded as f64 - value).abs(); - errors.push(error); - - // Check convergence: error stopped changing - if i > 0 && (errors[i] - errors[i - 1]).abs() < 1e-15 { - converged_at = i; - // Fill remaining with same error - for _ in (i + 1)..max_iterations { - errors.push(error); - } - break; - } - current = decoded; - } - - let max_error = errors.iter().cloned().fold(0.0f64, f64::max); - let final_error = *errors.last().unwrap_or(&0.0); - - ReencodeSafety { - converged_at, - max_error, - final_error, - error_history: errors, - safe: converged_at < max_iterations, - codec: "BF16".into(), - } + reencode_drift(value, max_iterations, "BF16", |x| { + bf16_to_f32(f32_to_bf16(x)) + }) } /// Test γ+φ round-trip: f64 → gamma_phi_encode → gamma_phi_decode → re-encode → ... @@ -87,37 +37,18 @@ pub fn test_gamma_phi_reencode( phi_scale: f32, max_iterations: usize, ) -> ReencodeSafety { - let mut current = value as f32; - let mut errors = Vec::new(); - let mut converged_at = max_iterations; - - for i in 0..max_iterations { - let encoded = gamma_phi_encode(current, role_gamma, phi_scale); - let decoded = gamma_phi_decode(encoded, role_gamma, phi_scale); - let error = (decoded as f64 - value).abs(); - errors.push(error); - - if i > 0 && (errors[i] - errors[i - 1]).abs() < 1e-15 { - converged_at = i; - for _ in (i + 1)..max_iterations { - errors.push(error); - } - break; - } - current = decoded; - } - - let max_error = errors.iter().cloned().fold(0.0f64, f64::max); - let final_error = *errors.last().unwrap_or(&0.0); - - ReencodeSafety { - converged_at, - max_error, - final_error, - error_history: errors, - safe: converged_at < max_iterations, - codec: format!("γ+φ(γ={},φ={})", role_gamma, phi_scale), - } + reencode_drift( + value, + max_iterations, + format!("γ+φ(γ={},φ={})", role_gamma, phi_scale), + |x| { + gamma_phi_decode( + gamma_phi_encode(x, role_gamma, phi_scale), + role_gamma, + phi_scale, + ) + }, + ) } /// Test full chain: f64 → f32 → bf16 → f32 → gamma_phi_encode → gamma_phi_decode → bf16 → ... @@ -127,92 +58,45 @@ pub fn test_full_chain_reencode( phi_scale: f32, max_iterations: usize, ) -> ReencodeSafety { - let mut current = value as f32; - let mut errors = Vec::new(); - let mut converged_at = max_iterations; - - for i in 0..max_iterations { - // Stage 1: BF16 quantize - let bf16 = f32_to_bf16(current); - let from_bf16 = bf16_to_f32(bf16); - - // Stage 2: γ+φ encode/decode - let gp_encoded = gamma_phi_encode(from_bf16, role_gamma, phi_scale); - let gp_decoded = gamma_phi_decode(gp_encoded, role_gamma, phi_scale); - - // Stage 3: back to BF16 - let re_bf16 = f32_to_bf16(gp_decoded); - let final_val = bf16_to_f32(re_bf16); - - let error = (final_val as f64 - value).abs(); - errors.push(error); - - if i > 0 && (errors[i] - errors[i - 1]).abs() < 1e-15 { - converged_at = i; - for _ in (i + 1)..max_iterations { - errors.push(error); - } - break; - } - current = final_val; - } - - let max_error = errors.iter().cloned().fold(0.0f64, f64::max); - let final_error = *errors.last().unwrap_or(&0.0); - - ReencodeSafety { - converged_at, - max_error, - final_error, - error_history: errors, - safe: converged_at < max_iterations, - codec: format!("BF16+γ+φ(γ={},φ={})", role_gamma, phi_scale), - } + reencode_drift( + value, + max_iterations, + format!("BF16+γ+φ(γ={},φ={})", role_gamma, phi_scale), + |x| { + // Stage 1: BF16 quantize + let from_bf16 = bf16_to_f32(f32_to_bf16(x)); + // Stage 2: γ+φ encode/decode + let gp_decoded = gamma_phi_decode( + gamma_phi_encode(from_bf16, role_gamma, phi_scale), + role_gamma, + phi_scale, + ); + // Stage 3: back to BF16 + bf16_to_f32(f32_to_bf16(gp_decoded)) + }, + ) } /// Test a BATCH of values across the range. Returns (all_safe, worst_case, convergence_stats). +/// +/// Thin adapter over `jc::drift::reencode_batch` that keeps this module's +/// historical tuple shape; an empty sweep reports a placeholder "empty" worst +/// case, as before. pub fn test_reencode_batch( codec_fn: impl Fn(f64) -> ReencodeSafety, test_values: &[f64], ) -> (bool, ReencodeSafety, usize, usize) { - let mut all_safe = true; - let mut worst = None::; - let mut max_converge = 0; - let mut safe_count = 0; - - for &v in test_values { - let result = codec_fn(v); - if result.safe { - safe_count += 1; - } else { - all_safe = false; - } - if result.converged_at > max_converge { - max_converge = result.converged_at; - } - if worst - .as_ref() - .map_or(true, |w| result.max_error > w.max_error) - { - worst = Some(result); - } - } - - ( - all_safe, - worst.unwrap_or_else(|| ReencodeSafety { - converged_at: 0, - max_error: 0.0, - final_error: 0.0, - error_history: vec![], - safe: true, - codec: "empty".into(), - }), - safe_count, - test_values.len(), - ) + let b = reencode_batch(test_values, codec_fn); + let worst = b.worst.unwrap_or_else(|| ReencodeSafety { + converged_at: 0, + max_error: 0.0, + final_error: 0.0, + error_history: vec![], + safe: true, + codec: "empty".into(), + }); + (b.all_safe, worst, b.safe_count, b.total) } - /// Test re-encode safety across multiple zipper offsets. /// /// The golden step (11 mod 17) creates a permutation. diff --git a/crates/thinking-engine/src/silu_correction.rs b/crates/thinking-engine/src/silu_correction.rs index a752e4381..1da4cb3b3 100644 --- a/crates/thinking-engine/src/silu_correction.rs +++ b/crates/thinking-engine/src/silu_correction.rs @@ -191,53 +191,28 @@ pub fn apply_corrections(table: &mut [u8], corrections: &[f32], n: usize) { } // ═══════════════════════════════════════════════════════════════════════════ -// STATISTICS +// STATISTICS — the MATH lives in jc (D-TEH-3); this is the adapter // ═══════════════════════════════════════════════════════════════════════════ -/// Analyze correction magnitude distribution. -#[derive(Debug, Clone)] -pub struct CorrectionStats { - pub count: usize, - pub mean_abs: f32, - pub max_abs: f32, - pub mean: f32, - pub std_dev: f32, - /// Fraction of corrections > 0.01 (material difference). - pub material_fraction: f32, - /// Fraction of corrections > 0.1 (large difference). - pub large_fraction: f32, -} +/// Correction-delta summary. The type is `jc::drift::DeltaSummary` — the +/// descriptive battery (mean, mean |δ|, max |δ|, population σ, fraction above +/// two cut-offs) is calibrated math and lives in jc; this crate only decides +/// which deltas to feed it and which cut-offs mean "material" / "large" here. +pub use jc::drift::DeltaSummary as CorrectionStats; + +/// A correction that changes a cosine by more than this is material. +pub const MATERIAL_CORRECTION: f64 = 0.01; +/// A correction that changes a cosine by more than this is large. +pub const LARGE_CORRECTION: f64 = 0.1; +/// Summarise the corrections of a training sample set. +/// +/// Empty input yields [`CorrectionStats::empty`] (count 0), preserving the +/// historical shape for callers that print a report unconditionally. pub fn correction_stats(samples: &[CorrectionSample]) -> CorrectionStats { - let n = samples.len(); - if n == 0 { - return CorrectionStats { - count: 0, - mean_abs: 0.0, - max_abs: 0.0, - mean: 0.0, - std_dev: 0.0, - material_fraction: 0.0, - large_fraction: 0.0, - }; - } - let corrections: Vec = samples.iter().map(|s| s.correction).collect(); - let mean = corrections.iter().sum::() / n as f32; - let mean_abs = corrections.iter().map(|c| c.abs()).sum::() / n as f32; - let max_abs = corrections.iter().map(|c| c.abs()).fold(0.0f32, f32::max); - let variance = corrections.iter().map(|c| (c - mean).powi(2)).sum::() / n as f32; - let material = corrections.iter().filter(|c| c.abs() > 0.01).count(); - let large = corrections.iter().filter(|c| c.abs() > 0.1).count(); - - CorrectionStats { - count: n, - mean_abs, - max_abs, - mean, - std_dev: variance.sqrt(), - material_fraction: material as f32 / n as f32, - large_fraction: large as f32 / n as f32, - } + let deltas: Vec = samples.iter().map(|s| f64::from(s.correction)).collect(); + jc::drift::delta_summary(&deltas, MATERIAL_CORRECTION, LARGE_CORRECTION) + .unwrap_or_else(|| CorrectionStats::empty(MATERIAL_CORRECTION, LARGE_CORRECTION)) } #[cfg(test)] From be8b1677f5d5d298cc3e2fe544bb015e461a6204 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 2 Sep 2026 21:44:29 +0000 Subject: [PATCH 3/5] =?UTF-8?q?D-TEH-3:=20board=20records=20=E2=80=94=20li?= =?UTF-8?q?ft-gate=20finding,=20inventory=20delta,=20arc=20entry,=20status?= =?UTF-8?q?/plan=20rows,=20TD=20for=20the=20copies=20outside=20jc?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Records the #1142 merge in LATEST_STATE; EPIPHANIES E-THE-LIFT-GATE-FOUND-A-TIE-BLIND-SPEARMAN-1; PR_ARC entry; STATUS_BOARD and plan §5 D-TEH-3 math half Shipped + §3 W2 result addendum; TECH_DEBT TD-RELIABILITY-COPIES-OUTSIDE-JC-1 (ndarray::hpc::reliability and perturbation-sim::stats); SUPERSESSION-INDEX regenerated last. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01PFnYKqw6d7TTiB9cT8eFdK --- .claude/board/EPIPHANIES.md | 18 +++++++++++++ .claude/board/LATEST_STATE.md | 10 +++++++ .claude/board/PR_ARC_INVENTORY.md | 10 +++++++ .claude/board/STATUS_BOARD.md | 2 +- .claude/board/TECH_DEBT.md | 26 +++++++++++++++++++ .../thinking-engine-harvest-closure-v1.md | 4 ++- 6 files changed, 68 insertions(+), 2 deletions(-) diff --git a/.claude/board/EPIPHANIES.md b/.claude/board/EPIPHANIES.md index 44eada80b..e98f45ea9 100644 --- a/.claude/board/EPIPHANIES.md +++ b/.claude/board/EPIPHANIES.md @@ -1,3 +1,21 @@ +## 2026-09-02 — E-THE-LIFT-GATE-FOUND-A-TIE-BLIND-SPEARMAN-1 — the D-TEH-3 comparison on a distinguishing fixture separated a same-formula copy from a wrong-estimator copy, and the tie-free fixtures the lab had used could not have + +**Status:** FINDING (measured, tests committed in `crates/jc/src/reliability.rs`). **Confidence:** High. + +**What the ruling asked.** `E-JC-IS-THE-HOME-OF-ALL-CALIBRATED-MATH-1`: a calibration routine outside jc is LIFTED if correct, or the jc version is PERFECTED and the copy dies — decided on a fixture that can distinguish the two implementations. D-TEH-3 ran that gate on the thinking-engine battery. Four verdicts, two of them not the one a code read would have given. + +**Cronbach α — LIFT, with a precision caveat the fixture had to expose.** `thinking_engine::cronbach::cronbach_alpha` and `jc::reliability::cronbach_alpha` are the same estimator (population variances, `k/(k−1)·(1 − Σσ²/σ²_total)`); on the known-value fixture they agree to `1e-5` (0.984615). That agreement proves nothing by itself, so the same fixture was shifted by `1e7`: α is affine-invariant in exact arithmetic and the `f64` estimate does not move (`< 1e-9`), while the retired `f32` form loses the fixture outright (its mean rounds to the nearest unit at that magnitude and the deviations carry no information). The copy dies; jc's `f64`/`Option` form is the survivor, not because it is "newer" but because the fixture showed which one survives the data. + +**Spearman ρ — PERFECT-IN-JC, and the lab's own tests were structurally blind to it.** `ground_truth::spearman_rank_correlation` ranked ties by sort POSITION (index order), not by average rank. On tie-free data it agrees with jc to `1e-6` — and every one of its three retired unit tests was tie-free. On `x=[1,2,3,4]`, `y=[1,2,2,3]` it returns ρ = 1.000 (a "perfect monotone match"); the average-rank form returns the textbook 0.948683. A calibration battery that compares baked `u8` lens distances against ground-truth cosines will hit ties constantly (256 levels over thousands of pairs), so the retired form was systematically optimistic exactly where it was used. jc already carried the tie correction; nothing was added to it except the comparison test that keeps the retired form on record. + +**Re-encode drift and the correction-delta summary — LIFTED as-is into `jc::drift`.** The convergence statistic (`|e_i − e_{i−1}| < 1e-15` against the ORIGINAL value, tail-filled history, `safe = converged_at < max_iterations`) and the delta battery (mean / mean|δ| / max|δ| / population σ / two threshold fractions) were correct; what changed is their shape — the codec choice (BF16, γ+φ, the full chain) is now a round-trip closure the lab passes in, and the two cut-offs (0.01 / 0.1) are parameters the lab names (`MATERIAL_CORRECTION` / `LARGE_CORRECTION`) instead of constants buried in the statistic. The x256 re-encode proof (14 lab tests) runs unchanged through jc. **The lens quorum** (per-pair `1 − σ/σ_max` over `u8` tables, the 230/179/128 bands, the per-subject Cronbach report) went to `jc::quorum` the same way; it never was an α per pair and its docs now say so in the type name. + +**The generalizable point.** "Agrees on the existing tests" was true for both copies and meant nothing: the tests were the fixtures on which the implementations coincide. The gate had to be run on the input class each estimator would actually see (large offsets for `f32`, ties for ranks). A lift decision made from a code read would have kept the tie-blind Spearman — it looks like the textbook formula. + +**Still outside jc, recorded rather than swept.** `ndarray::hpc::reliability` (tie-aware, `f64`, returns `0.0` on degeneracy) and `perturbation-sim::stats` (a deliberate zero-dep mirror of it) carry the same four estimators. Both are correct in the sense above; neither is jc. Filed as `TD-RELIABILITY-COPIES-OUTSIDE-JC-1` — the ruling's own text names the ndarray direction ("once ndarray is proven bit-exact the same math is re-importable from jc"), which is a deliberate PR with a bit-exactness gate, not a drive-by. + +Refs: `E-JC-IS-THE-HOME-OF-ALL-CALIBRATED-MATH-1`; `thinking-engine-harvest-closure-v1` §1d / §3 W2 / §5 D-TEH-3; `crates/jc/src/{reliability,drift,quorum}.rs`; the falsifiability rule (CLAUDE.md § The falsifiability rule). + ## 2026-09-02 — E-THE-CALIBRATION-GATE-REVERSED-THE-DECLARED-FLOOR-1 — D-TEH-2: the ghost prior landed in the planner, and the pre-registered gate overruled the floor its author had declared **Status:** FINDING (measured, `crates/lance-graph-planner/src/nars/ghost_prior.rs`, test `calibration_gate_picks_the_default_floor`). **Confidence:** High on the fixture; the fixture is one recurrence shape (256 atoms, a 3-atom pattern, 0–30 stale patterns, ages 0–60). diff --git a/.claude/board/LATEST_STATE.md b/.claude/board/LATEST_STATE.md index e265591d0..03671e3c2 100644 --- a/.claude/board/LATEST_STATE.md +++ b/.claude/board/LATEST_STATE.md @@ -1,3 +1,13 @@ +## 2026-09-02 — branch (D-TEH-3, after #1142 merged): calibration math lifted into jc — INVENTORY DELTA + +- MERGED #1142 (D-TEH-2, `3c5f040`): no contract type; planner `nars/ghost_prior.rs` + lab-crate edits + `weather-poc.yml` sibling checkout (see PR_ARC). +- ADDED `jc::drift` — `ReencodeDrift` / `reencode_drift(value, max_iterations, codec, round_trip: FnMut(f32) -> f32)` / `DriftBatch` / `reencode_batch` (the re-encode convergence statistic, codec-agnostic), `DeltaSummary` / `delta_summary(deltas, material_threshold, large_threshold) -> Option` / `DeltaSummary::empty`, `CONVERGENCE_EPS`. 8 tests (identity converges at 1 with a filled tail; multiplicative drift never converges with a strictly growing history; damped codec converges late and the budget binds two-sided; zero budget proves nothing; batch worst/count; hand-computed summary; thresholds load-bearing both ways; empty / non-finite → `None`). +- ADDED `jc::quorum` — `pairwise_agreement_u8(tables, n) -> Option>` (per-pair `1 − σ/σ_max`, `U8_MAX_VARIANCE`), `QuorumLevel { High, Medium, Low, Ambiguous }` with `HIGH_FLOOR`/`MEDIUM_FLOOR`/`LOW_FLOOR` = 230/179/128 and `needs_leaf_validation`, `CronbachReport` / `cronbach_report(items) -> Option` (α by delegation to `reliability::cronbach_alpha` + per-subject variances + the `mean + σ` disagreement count). 6 tests (coincidence → 255 off-diagonal too; 0-vs-255 → 0 and a 100-vs-150 pair → 205; degenerate → `None`; both sides of every band floor; the one disputed subject is the one flagged, with a mild subject above the mean but under `mean + σ` as the anti-vacuity for the disable; `None` where α is). +- ADDED `jc::reliability` tests only — the two D-TEH-3 lift-gate comparisons carrying the retired lab forms verbatim (`retired_cronbach_f32`, `retired_spearman_ordinal`): cronbach agrees on the known-value fixture and loses the `1e7`-shifted one; spearman agrees tie-free and diverges on one tie (1.000 vs 0.948683). No estimator changed. jc lib: 135/135. +- REMOVED (lab crate, excluded) `thinking_engine::cronbach` (`cronbach_alpha`, `variance_agreement_scores`, `QuorumLevel`, `CronbachResult`, `cronbach_analysis`) and `ground_truth::{spearman_rank_correlation, ranks}` with their tests. `reencode_safety.rs` is now glue over `jc::drift` (`pub type ReencodeSafety = jc::drift::ReencodeDrift`; the three codec wrappers pass round-trip closures; `test_reencode_batch` keeps its tuple shape; `test_zipper_offsets` + all 14 tests unchanged and green). `silu_correction.rs`: `pub use jc::drift::DeltaSummary as CorrectionStats`, `correction_stats` is an adapter with the cut-offs named `MATERIAL_CORRECTION = 0.01` / `LARGE_CORRECTION = 0.1`. `ground_truth::calibration::spearman_vs_ground_truth` calls `jc::reliability::spearman` (keeps its `0.0` fallback). `examples/certify_jina_v5_7lane.rs` calls `jc::reliability::cronbach_alpha` (`None` → `NaN`, which fails every `>=` verdict honestly). Lab `Cargo.toml` gains `jc = { path = "../jc" }`; lab lock adds `jc` and drops entries cargo no longer needs. Lab: `cargo check --lib --examples` clean except the pre-existing `tts_stream_hhtld` example break (bgz-tensor private fn, untouched, `TD-THINKING-ENGINE-EXCLUDED-DEBT-1`); lab tests for the three modules 23/23. +- NOT IN THIS DELTA: the `semantic_chunker` / `spiral_segment` halves of D-TEH-3 (falsifier-gated, separate); `silu` and `cosine_f32` stay in the lab (activation + vector glue, not calibrated math); `ndarray::hpc::reliability` and `perturbation-sim::stats` copies (`TD-RELIABILITY-COPIES-OUTSIDE-JC-1`). +- UNCHANGED: every contract type; every existing jc estimator's arithmetic, signature and semantics; the ALU artery files. + ## 2026-09-02 — branch (D-TEH-2, after #1141 merged): ghost prior harvested into the planner — INVENTORY DELTA - MERGED #1141 (D-HOUSE-1, `3102776`): no contract type; planner example `house_differential.rs` + plan/board records (see PR_ARC). diff --git a/.claude/board/PR_ARC_INVENTORY.md b/.claude/board/PR_ARC_INVENTORY.md index 5c0816da4..37b79eab5 100644 --- a/.claude/board/PR_ARC_INVENTORY.md +++ b/.claude/board/PR_ARC_INVENTORY.md @@ -10,6 +10,16 @@ > census §8.3 trap 10: read the body FIRST, then open for write — never > inline both in one expression. +## 2026-09-02 — lance-graph branch `claude/medcare-rs-continue-6nhbxn` (D-TEH-3 PR, after #1142) — calibration MATH → jc; lab copies deleted + +- **Added:** `crates/jc/src/drift.rs` (`ReencodeDrift`, `reencode_drift`, `DriftBatch`, `reencode_batch`, `DeltaSummary`, `delta_summary`, `CONVERGENCE_EPS`; 8 tests), `crates/jc/src/quorum.rs` (`pairwise_agreement_u8`, `U8_MAX_VARIANCE`, `QuorumLevel` + floors, `CronbachReport`, `cronbach_report`; 6 tests), two lift-gate tests in `crates/jc/src/reliability.rs`; `pub mod drift; pub mod quorum;` in jc `lib.rs`. EPIPHANIES `E-THE-LIFT-GATE-FOUND-A-TIE-BLIND-SPEARMAN-1`; TECH_DEBT `TD-RELIABILITY-COPIES-OUTSIDE-JC-1`. +- **Removed:** `crates/thinking-engine/src/cronbach.rs`; `spearman_rank_correlation` + `ranks` (+3 tests) from `ground_truth.rs`; the private statistic bodies in `reencode_safety.rs` and `silu_correction.rs`. +- **Changed (lab crate):** `Cargo.toml` (+`jc` path dep) and `Cargo.lock`; `lib.rs` (no `cronbach`); `reencode_safety.rs` (glue over `jc::drift`, tests verbatim); `silu_correction.rs` (`CorrectionStats` = `jc::drift::DeltaSummary`, named cut-offs); `ground_truth.rs` (calibration glue calls jc); `examples/certify_jina_v5_7lane.rs` (jc α, `Option` → `NaN`). Plan §5 D-TEH-3 → math half Shipped, §3 W2 result addendum; STATUS_BOARD D-TEH-3. +- **Measured:** cronbach — same formula, agree to `1e-5` on the 0.984615 fixture, `f32` copy loses the `1e7`-shifted fixture while `f64` moves `< 1e-9` → LIFT; spearman — retired form ranked ties by position, agrees `1e-6` tie-free, returns 1.000 vs jc 0.948683 on `[1,2,2,3]` → PERFECT-IN-JC (already there); drift + delta summary + quorum → lifted as-is with the codec / cut-offs turned into parameters. jc lib 135/135, new modules clippy `-D warnings` clean, fmt clean; lab 23/23 on the three modules, `check --lib --examples` clean but for the pre-existing `tts_stream_hhtld` break. +- **Locked:** jc is the only home of the four estimators inside lance-graph; the lab carries codec/cut-off choices, never a statistic; the retired forms stay in jc's tests as the comparison record. +- **Deferred:** D-TEH-3's `semantic_chunker` / `spiral_segment` halves (falsifier-gated); the ndarray / perturbation-sim copies (TD, deliberate PR with a bit-exactness gate); the pre-existing lab example break (D-TEH-5 / TD-THINKING-ENGINE-EXCLUDED-DEBT-1). +- **Confidence:** High. + ## 2026-09-02 — lance-graph branch `claude/medcare-rs-continue-6nhbxn` (D-TEH-2 PR, after #1141) — ghost prior harvested into the planner; `ghosts.rs` deleted - **Added:** `crates/lance-graph-planner/src/nars/ghost_prior.rs` — `GhostPrior`, `PriorFloor`, `Trace`, `calibration::{recurrence_fixture, discrimination}` (both `Option` below `FIXTURE_MIN_ATOMS` and above `FIXTURE_MAX_STALE_PATTERNS`), 15 tests (monotone decay to each floor with the inert cycle derived from the constant, decay-constant load-bearing both ways, two-sided free energy under both floors, the calibration gate on the declared default, anti-vacuity that the floors differ, prediction shape, imprint cap, echo rule, independence of two priors); `pub mod ghost_prior` + re-export in `nars/mod.rs`. EPIPHANIES `E-THE-CALIBRATION-GATE-REVERSED-THE-DECLARED-FLOOR-1`. diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index 05fcb4113..a04451b59 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -20,7 +20,7 @@ | D-TEH-0 | census: live footprint measured (one required consumer via `bridge_gate`; one optional via `with-engine`), 51-file fate table, open-row reconciliation, four-wave closure, idea harvest, four rulings asked | **Shipped (plan-only, this PR)** | | D-TEH-1 | W1: `bridge_gate` (seven items) → `lance_graph_contract::bridge_gate`; callcenter re-imports and drops the path dep; thinking-engine keeps a re-export shim | **Shipped 2026-09-02** (before: required dep, 6 crossing sites, dep-drop alone fails 6 × E0433; after: zero thinking-engine deps in callcenter metadata, contract 1303/1303, callcenter 156/156, driver default + `with-engine` green, ALU artery files byte-identical). `with-engine` re-point deferred by stop condition: D-TTV-1 not landed | | D-TEH-2 | W2: ghost prior → planner `nars/ghost_prior.rs` over `WisdomMarker`, per-thought, two-sided falsifiers; crate `ghosts.rs` deleted | **Shipped 2026-09-02** — `GhostPrior`/`PriorFloor` in planner `nars/ghost_prior.rs`, 14 tests incl. the calibration gate (default floor = `Marker`, the gate reversed the first declaration); `ghosts.rs` + `think.rs` deleted; TD-GHOST-ECHO-DUP-1 resolved; consumer D-HOUSE-4 unblocked | -| D-TEH-3 | W2: calibration MATH → `jc` (ruling 4, 2026-09-02: lift if correct, perfect in jc if not; crate copies die); glue stays in the lab crate; `semantic_chunker` / `spiral_segment` decided by their falsifiers | Queued | +| D-TEH-3 | W2: calibration MATH → `jc` (ruling 4, 2026-09-02: lift if correct, perfect in jc if not; crate copies die); glue stays in the lab crate; `semantic_chunker` / `spiral_segment` decided by their falsifiers | **Math half Shipped 2026-09-02** — `jc::drift` (re-encode drift + delta summary) and `jc::quorum` (pairwise agreement, bands, Cronbach report) added; cronbach LIFTED (same formula; `f32` copy loses the `1e7`-shifted fixture), spearman PERFECTED-IN-JC (retired copy was tie-blind: 1.000 vs 0.9487 on one tie); lab `cronbach.rs` deleted, `reencode_safety` / `silu_correction` / `ground_truth` are glue over jc. Open: the `semantic_chunker` / `spiral_segment` falsifier halves | | D-TEH-4 | W3: ENTROPY M8 engine collapse with dtype parity suite; 5 cascade shapes + 3 lens modules collapse | Queued | | D-TEH-5 | W4: residue deleted, crate renamed `thinking-lab` with a `--manifest-path` CI line; §2 rows closed; `TD-THINKING-ENGINE-EXCLUDED-DEBT-1` paid | Queued — closes the chapter | diff --git a/.claude/board/TECH_DEBT.md b/.claude/board/TECH_DEBT.md index 476a12bc1..b3b54f238 100644 --- a/.claude/board/TECH_DEBT.md +++ b/.claude/board/TECH_DEBT.md @@ -1,3 +1,29 @@ +## TD-RELIABILITY-COPIES-OUTSIDE-JC-1 (2026-09-02) — OPEN + +**Two further copies of the Pearson / Spearman / Cronbach α / ICC battery live +outside `jc`.** Found while running the D-TEH-3 lift gate +(`E-THE-LIFT-GATE-FOUND-A-TIE-BLIND-SPEARMAN-1`): + +- `ndarray::hpc::reliability` (`pearson`, `spearman`, `cronbach_alpha`, + `icc_a1`, `FidelityReport`) — tie-aware, `f64`, returns `0.0` on degenerate + input where jc returns `None`. The PRODUCTION copy; consumed by the + edge-codec fidelity work. +- `perturbation-sim::stats` — a self-described zero-dep mirror of the ndarray + copy so that crate's validation harness stays standalone. + +Neither is wrong the way the retired thinking-engine Spearman was (both average +ranks over ties). They are still two more sources of truth for math the ruling +says has ONE home. The ruling itself names the direction for the first — *once +ndarray is proven bit-exact the same math is re-importable from jc* — so the +pay-down is a deliberate PR with a bit-exactness gate (same fixtures as jc's +lift-gate tests, plus the `0.0`-vs-`None` degeneracy contract decided +explicitly), never a drive-by delete. The perturbation-sim mirror follows +whatever the ndarray decision is. + +Owner: whoever next touches `ndarray::hpc::reliability` or `perturbation-sim`. +Refs: `E-JC-IS-THE-HOME-OF-ALL-CALIBRATED-MATH-1`, `crates/jc/src/reliability.rs` +(lift-gate tests), `thinking-engine-harvest-closure-v1` §1d. + ## TD-PILLAR11-SCIENTIFIC-LOOPS-BYPASS-NDARRAY-SIMD-1 (2026-09-02) — OPEN **The debt is not missing SIMD support. It is scientific code bypassing the diff --git a/.claude/plans/thinking-engine-harvest-closure-v1.md b/.claude/plans/thinking-engine-harvest-closure-v1.md index 80f94a452..23a9c0543 100644 --- a/.claude/plans/thinking-engine-harvest-closure-v1.md +++ b/.claude/plans/thinking-engine-harvest-closure-v1.md @@ -182,6 +182,8 @@ axis vocabulary by assertion (AXES_48 goes falsifier-first or stays LAB). **W2 result (2026-09-02, D-TEH-2).** The ghost prior landed as planner `nars/ghost_prior.rs` — per-thought, over `WisdomMarker` / `GhostEcho`, no singleton. The floor question the §1c row left open was decided by the pre-registered calibration gate, and it decided AGAINST the first declaration: `Trace` (source 0.001 + prune) loses all discrimination between a recurrence and a shift once the remembered pattern is older than ~42 cycles, `Marker` (contract 0.1, never pruned) keeps it (0.0188) at the cost of a higher absolute free-energy baseline (0.35 vs 0.07 with 30 stale patterns). Default = `Marker`. The remaining W2 items (math → jc, `semantic_chunker`, `spiral_segment`) are D-TEH-3/-5 and untouched here. +**W2 result (2026-09-02, D-TEH-3, math half).** The calibration battery split by nature as §1d prescribed. MATH went to jc: the re-encode drift statistic and the correction-delta summary as a new `jc::drift` (codec and cut-offs are parameters the lab supplies), the lens quorum and Cronbach report as `jc::quorum`; Cronbach α and Spearman ρ were already in `jc::reliability`, so the gate was a comparison, and it discriminated: the lab's α was the same estimator in `f32` (LIFT — the `f64` form survives a `1e7` offset the `f32` one does not), the lab's ρ was NOT the same estimator (ties ranked by position, ρ = 1.000 where the average-rank form gives 0.948683 — PERFECT-IN-JC, already there). GLUE stayed: `reencode_safety.rs` chooses the codecs, `silu_correction.rs` names its cut-offs, `ground_truth.rs` keeps the loaders; each calls jc and carries no statistic. `cronbach.rs` deleted. Two copies outside the lab (`ndarray::hpc::reliability`, `perturbation-sim::stats`) are recorded as `TD-RELIABILITY-COPIES-OUTSIDE-JC-1`, not swept. The `semantic_chunker` / `spiral_segment` halves of D-TEH-3 remain gated on their falsifiers. + ## 4. The good ideas — what the chapter leaves behind even where the code dies Kept as doctrine with a named present or future home, so the closure is a @@ -209,7 +211,7 @@ harvest and not an amputation. | D-TEH-0 | census + fate table + open-row reconciliation + idea harvest (this plan) | plan + board rows | Shipped (this PR) | | D-TEH-1 | W1: `bridge_gate` (seven items) → `lance_graph_contract::bridge_gate`; callcenter re-imports and drops the path dep; thinking-engine keeps a re-export shim | contract + callcenter | **Shipped 2026-09-02** — edge measured before (required dep, 6 crossing sites, dep-drop fails 6 × E0433) and after (zero thinking-engine deps in callcenter metadata; 1303 + 156 tests, driver default + `with-engine` green). The `with-engine` re-point is NOT part of this wave: D-TTV-1 is Queued and the engine hook still lives in thinking-engine, so there is nothing to re-point it at (stop condition honoured). thinking-engine is now a leaf for every REQUIRED edge; the one remaining edge is the ALU's optional engine hook | | D-TEH-2 | W2: ghost prior harvested as planner `nars/ghost_prior.rs` over `WisdomMarker`, per-thought, with two-sided falsifiers; crate `ghosts.rs` deleted | planner | **Shipped 2026-09-02** — planner `nars/ghost_prior.rs` (`GhostPrior`, `PriorFloor`, `Trace`, `calibration::{recurrence_fixture, discrimination}`; 14 tests); `ghosts.rs` + `examples/think.rs` deleted; lab `persona`/`world_model`/`awareness_dto` re-pointed to `contract::escalation::GhostEcho` (TD-GHOST-ECHO-DUP-1 resolved). Calibration gate REVERSED the first-declared floor: `Marker` (0.1, never pruned) discriminates ≥ `Trace` (0.001) on every fixture row and strictly once the remembered pattern ages past its prune point (disc 0.0188 vs 0.0000 at 30 stale / age 20 and 60); default = `Marker`. Consumer D-HOUSE-4 unblocked | -| D-TEH-3 | W2: calibration MATH → jc (ruling 4: compare, then lift or perfect in jc; crate copies deleted); `semantic_chunker` / `spiral_segment` decided by their falsifiers | jc / deepnsm-v2 / codec home | Queued | +| D-TEH-3 | W2: calibration MATH → jc (ruling 4: compare, then lift or perfect in jc; crate copies deleted); `semantic_chunker` / `spiral_segment` decided by their falsifiers | jc / deepnsm-v2 / codec home | **Math half Shipped 2026-09-02** — new `jc::drift` (`reencode_drift` / `reencode_batch` / `delta_summary`) and `jc::quorum` (`pairwise_agreement_u8` / `QuorumLevel` / `cronbach_report`); the lift gate ran on distinguishing fixtures: cronbach = same estimator (LIFT; the `f32` copy loses a `1e7`-shifted fixture the `f64` form holds to `1e-9`), spearman = the retired copy ranked ties by position (PERFECT-IN-JC: 1.000 vs 0.948683 on `[1,2,2,3]`, indistinguishable on the tie-free fixtures its own tests used). Lab: `cronbach.rs` deleted; `reencode_safety` / `silu_correction` / `ground_truth::calibration` are glue over jc, x256 proof green through jc (14 tests). `semantic_chunker` / `spiral_segment` halves still Queued on their falsifiers | | D-TEH-4 | W3: M8 engine collapse with parity suite; cascade shapes and lens modules collapse | thinking-engine → the one engine | Queued (owns ENTROPY M8) | | D-TEH-5 | W4: residue deleted, crate renamed `thinking-lab` with a CI line; §2 rows closed; TD paid | workspace | Queued — closes the chapter | From cfb9b3b18ec6d9ee4d8418ea975eedc7ccd3312c Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 2 Sep 2026 21:52:12 +0000 Subject: [PATCH 4/5] =?UTF-8?q?D-TEH-3:=20Codex=20round=20=E2=80=94=20odd-?= =?UTF-8?q?k=20quorum=20ceiling,=20f32=20cut-offs,=20invalid-data=20None?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - jc::quorum: max_u8_variance(k) = 255^2 * floor(k/2) * ceil(k/2) / k^2 replaces the even-k constant. The lifted source normalised every k by (255/2)^2, so a maximally split three-lens cell [0, 0, 255] scored 15 instead of 0; perfected in jc per the ruling. New two-sided test (k = 3 maximal -> 0, k = 2 maximal -> 0, k = 3 half-split strictly between) + doctest. 7/7 + doctests green. - lab silu_correction: MATERIAL_CORRECTION / LARGE_CORRECTION are f32 and promoted with the samples, so a correction of exactly 0.1f32 stays 'not more than' 0.1 as the retired code had it; correction_stats returns Option — None for a non-finite correction (an invalid run), Some(empty) only for an empty set. Test pins both; silu_crosscheck adapts (expect + one f32 cast). 10/10 green, both examples compile. - Board: LATEST_STATE / PR_ARC same-PR lines updated. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01PFnYKqw6d7TTiB9cT8eFdK --- .claude/board/LATEST_STATE.md | 4 +- .claude/board/PR_ARC_INVENTORY.md | 3 +- crates/jc/src/quorum.rs | 62 ++++++++++++++++--- .../examples/silu_crosscheck.rs | 5 +- crates/thinking-engine/src/silu_correction.rs | 54 +++++++++++++--- 5 files changed, 104 insertions(+), 24 deletions(-) diff --git a/.claude/board/LATEST_STATE.md b/.claude/board/LATEST_STATE.md index 03671e3c2..078f7cea1 100644 --- a/.claude/board/LATEST_STATE.md +++ b/.claude/board/LATEST_STATE.md @@ -2,9 +2,9 @@ - MERGED #1142 (D-TEH-2, `3c5f040`): no contract type; planner `nars/ghost_prior.rs` + lab-crate edits + `weather-poc.yml` sibling checkout (see PR_ARC). - ADDED `jc::drift` — `ReencodeDrift` / `reencode_drift(value, max_iterations, codec, round_trip: FnMut(f32) -> f32)` / `DriftBatch` / `reencode_batch` (the re-encode convergence statistic, codec-agnostic), `DeltaSummary` / `delta_summary(deltas, material_threshold, large_threshold) -> Option` / `DeltaSummary::empty`, `CONVERGENCE_EPS`. 8 tests (identity converges at 1 with a filled tail; multiplicative drift never converges with a strictly growing history; damped codec converges late and the budget binds two-sided; zero budget proves nothing; batch worst/count; hand-computed summary; thresholds load-bearing both ways; empty / non-finite → `None`). -- ADDED `jc::quorum` — `pairwise_agreement_u8(tables, n) -> Option>` (per-pair `1 − σ/σ_max`, `U8_MAX_VARIANCE`), `QuorumLevel { High, Medium, Low, Ambiguous }` with `HIGH_FLOOR`/`MEDIUM_FLOOR`/`LOW_FLOOR` = 230/179/128 and `needs_leaf_validation`, `CronbachReport` / `cronbach_report(items) -> Option` (α by delegation to `reliability::cronbach_alpha` + per-subject variances + the `mean + σ` disagreement count). 6 tests (coincidence → 255 off-diagonal too; 0-vs-255 → 0 and a 100-vs-150 pair → 205; degenerate → `None`; both sides of every band floor; the one disputed subject is the one flagged, with a mild subject above the mean but under `mean + σ` as the anti-vacuity for the disable; `None` where α is). +- ADDED `jc::quorum` — `pairwise_agreement_u8(tables, n) -> Option>` (per-pair `1 − σ/σ_max(k)`, `max_u8_variance(k)` = `255²·⌊k/2⌋⌈k/2⌉/k²` — the lifted source used the even-`k` ceiling for every `k`, so a maximally split 3-lens cell scored 15 instead of 0; perfected in jc per the ruling, Codex on the lift PR), `QuorumLevel { High, Medium, Low, Ambiguous }` with `HIGH_FLOOR`/`MEDIUM_FLOOR`/`LOW_FLOOR` = 230/179/128 and `needs_leaf_validation`, `CronbachReport` / `cronbach_report(items) -> Option` (α by delegation to `reliability::cronbach_alpha` + per-subject variances + the `mean + σ` disagreement count). 7 tests (coincidence → 255 off-diagonal too; odd-`k` `[0,0,255]` → 0 and `[0,0,128]` strictly between; 0-vs-255 → 0 and a 100-vs-150 pair → 205; degenerate → `None`; both sides of every band floor; the one disputed subject is the one flagged, with a mild subject above the mean but under `mean + σ` as the anti-vacuity for the disable; `None` where α is). - ADDED `jc::reliability` tests only — the two D-TEH-3 lift-gate comparisons carrying the retired lab forms verbatim (`retired_cronbach_f32`, `retired_spearman_ordinal`): cronbach agrees on the known-value fixture and loses the `1e7`-shifted one; spearman agrees tie-free and diverges on one tie (1.000 vs 0.948683). No estimator changed. jc lib: 135/135. -- REMOVED (lab crate, excluded) `thinking_engine::cronbach` (`cronbach_alpha`, `variance_agreement_scores`, `QuorumLevel`, `CronbachResult`, `cronbach_analysis`) and `ground_truth::{spearman_rank_correlation, ranks}` with their tests. `reencode_safety.rs` is now glue over `jc::drift` (`pub type ReencodeSafety = jc::drift::ReencodeDrift`; the three codec wrappers pass round-trip closures; `test_reencode_batch` keeps its tuple shape; `test_zipper_offsets` + all 14 tests unchanged and green). `silu_correction.rs`: `pub use jc::drift::DeltaSummary as CorrectionStats`, `correction_stats` is an adapter with the cut-offs named `MATERIAL_CORRECTION = 0.01` / `LARGE_CORRECTION = 0.1`. `ground_truth::calibration::spearman_vs_ground_truth` calls `jc::reliability::spearman` (keeps its `0.0` fallback). `examples/certify_jina_v5_7lane.rs` calls `jc::reliability::cronbach_alpha` (`None` → `NaN`, which fails every `>=` verdict honestly). Lab `Cargo.toml` gains `jc = { path = "../jc" }`; lab lock adds `jc` and drops entries cargo no longer needs. Lab: `cargo check --lib --examples` clean except the pre-existing `tts_stream_hhtld` example break (bgz-tensor private fn, untouched, `TD-THINKING-ENGINE-EXCLUDED-DEBT-1`); lab tests for the three modules 23/23. +- REMOVED (lab crate, excluded) `thinking_engine::cronbach` (`cronbach_alpha`, `variance_agreement_scores`, `QuorumLevel`, `CronbachResult`, `cronbach_analysis`) and `ground_truth::{spearman_rank_correlation, ranks}` with their tests. `reencode_safety.rs` is now glue over `jc::drift` (`pub type ReencodeSafety = jc::drift::ReencodeDrift`; the three codec wrappers pass round-trip closures; `test_reencode_batch` keeps its tuple shape; `test_zipper_offsets` + all 14 tests unchanged and green). `silu_correction.rs`: `pub use jc::drift::DeltaSummary as CorrectionStats`, `correction_stats -> Option` is an adapter with the cut-offs named `MATERIAL_CORRECTION = 0.01f32` / `LARGE_CORRECTION = 0.1f32` (kept in the samples' precision and promoted with them, so a correction of exactly `0.1f32` stays "not more than" 0.1 — Codex); `None` = non-finite data (an invalid run), `Some(empty)` = genuinely empty (Codex: the first cut folded both into empty). `ground_truth::calibration::spearman_vs_ground_truth` calls `jc::reliability::spearman` (keeps its `0.0` fallback). `examples/certify_jina_v5_7lane.rs` calls `jc::reliability::cronbach_alpha` (`None` → `NaN`, which fails every `>=` verdict honestly). Lab `Cargo.toml` gains `jc = { path = "../jc" }`; lab lock adds `jc` and drops entries cargo no longer needs. Lab: `cargo check --lib --examples` clean except the pre-existing `tts_stream_hhtld` example break (bgz-tensor private fn, untouched, `TD-THINKING-ENGINE-EXCLUDED-DEBT-1`); lab tests for the three modules 23/23. - NOT IN THIS DELTA: the `semantic_chunker` / `spiral_segment` halves of D-TEH-3 (falsifier-gated, separate); `silu` and `cosine_f32` stay in the lab (activation + vector glue, not calibrated math); `ndarray::hpc::reliability` and `perturbation-sim::stats` copies (`TD-RELIABILITY-COPIES-OUTSIDE-JC-1`). - UNCHANGED: every contract type; every existing jc estimator's arithmetic, signature and semantics; the ALU artery files. diff --git a/.claude/board/PR_ARC_INVENTORY.md b/.claude/board/PR_ARC_INVENTORY.md index 37b79eab5..e3a690dce 100644 --- a/.claude/board/PR_ARC_INVENTORY.md +++ b/.claude/board/PR_ARC_INVENTORY.md @@ -12,10 +12,11 @@ ## 2026-09-02 — lance-graph branch `claude/medcare-rs-continue-6nhbxn` (D-TEH-3 PR, after #1142) — calibration MATH → jc; lab copies deleted -- **Added:** `crates/jc/src/drift.rs` (`ReencodeDrift`, `reencode_drift`, `DriftBatch`, `reencode_batch`, `DeltaSummary`, `delta_summary`, `CONVERGENCE_EPS`; 8 tests), `crates/jc/src/quorum.rs` (`pairwise_agreement_u8`, `U8_MAX_VARIANCE`, `QuorumLevel` + floors, `CronbachReport`, `cronbach_report`; 6 tests), two lift-gate tests in `crates/jc/src/reliability.rs`; `pub mod drift; pub mod quorum;` in jc `lib.rs`. EPIPHANIES `E-THE-LIFT-GATE-FOUND-A-TIE-BLIND-SPEARMAN-1`; TECH_DEBT `TD-RELIABILITY-COPIES-OUTSIDE-JC-1`. +- **Added:** `crates/jc/src/drift.rs` (`ReencodeDrift`, `reencode_drift`, `DriftBatch`, `reencode_batch`, `DeltaSummary`, `delta_summary`, `CONVERGENCE_EPS`; 8 tests), `crates/jc/src/quorum.rs` (`pairwise_agreement_u8`, `max_u8_variance`, `QuorumLevel` + floors, `CronbachReport`, `cronbach_report`; 7 tests), two lift-gate tests in `crates/jc/src/reliability.rs`; `pub mod drift; pub mod quorum;` in jc `lib.rs`. EPIPHANIES `E-THE-LIFT-GATE-FOUND-A-TIE-BLIND-SPEARMAN-1`; TECH_DEBT `TD-RELIABILITY-COPIES-OUTSIDE-JC-1`. - **Removed:** `crates/thinking-engine/src/cronbach.rs`; `spearman_rank_correlation` + `ranks` (+3 tests) from `ground_truth.rs`; the private statistic bodies in `reencode_safety.rs` and `silu_correction.rs`. - **Changed (lab crate):** `Cargo.toml` (+`jc` path dep) and `Cargo.lock`; `lib.rs` (no `cronbach`); `reencode_safety.rs` (glue over `jc::drift`, tests verbatim); `silu_correction.rs` (`CorrectionStats` = `jc::drift::DeltaSummary`, named cut-offs); `ground_truth.rs` (calibration glue calls jc); `examples/certify_jina_v5_7lane.rs` (jc α, `Option` → `NaN`). Plan §5 D-TEH-3 → math half Shipped, §3 W2 result addendum; STATUS_BOARD D-TEH-3. - **Measured:** cronbach — same formula, agree to `1e-5` on the 0.984615 fixture, `f32` copy loses the `1e7`-shifted fixture while `f64` moves `< 1e-9` → LIFT; spearman — retired form ranked ties by position, agrees `1e-6` tie-free, returns 1.000 vs jc 0.948683 on `[1,2,2,3]` → PERFECT-IN-JC (already there); drift + delta summary + quorum → lifted as-is with the codec / cut-offs turned into parameters. jc lib 135/135, new modules clippy `-D warnings` clean, fmt clean; lab 23/23 on the three modules, `check --lib --examples` clean but for the pre-existing `tts_stream_hhtld` break. +- **Review folded in (pre-merge, Codex P2 ×3):** odd-`k` quorum normalised by its own attainable variance (`max_u8_variance(k)`; `[0,0,255]` at `k = 3` scored 15 under the lifted even-`k` ceiling, now 0 — a defect the lift PERFECTED rather than carried); the lab cut-offs kept as `f32` and promoted with the samples (exactly `0.1f32` is not `> 0.1`); `correction_stats` returns `Option` — `None` for non-finite corrections, `Some(empty)` only for an empty set. - **Locked:** jc is the only home of the four estimators inside lance-graph; the lab carries codec/cut-off choices, never a statistic; the retired forms stay in jc's tests as the comparison record. - **Deferred:** D-TEH-3's `semantic_chunker` / `spiral_segment` halves (falsifier-gated); the ndarray / perturbation-sim copies (TD, deliberate PR with a bit-exactness gate); the pre-existing lab example break (D-TEH-5 / TD-THINKING-ENGINE-EXCLUDED-DEBT-1). - **Confidence:** High. diff --git a/crates/jc/src/quorum.rs b/crates/jc/src/quorum.rs index 5b6da6bc8..cbe9d36b4 100644 --- a/crates/jc/src/quorum.rs +++ b/crates/jc/src/quorum.rs @@ -11,10 +11,15 @@ //! Cronbach α asks about the WHOLE corpus: "do these `k` lenses behave as one //! scale over all `n` pairs?" The quorum score asks about ONE cell: "how far //! apart are the `k` lenses on THIS pair?" It is a normalised dispersion, -//! `1 − σ/σ_max`, where `σ_max = 255/2` is the largest population standard -//! deviation a set of `u8` values can have (half at 0, half at 255). It is -//! NOT an α per pair — α is undefined on a single subject — and the lifted -//! source said so; the name here says so too. +//! `1 − σ/σ_max(k)`, where `σ_max(k)` is the largest population standard +//! deviation `k` `u8` values can have — `⌊k/2⌋` of them at 0 and `⌈k/2⌉` at +//! 255 (or the reverse), i.e. `σ²_max(k) = 255² · ⌊k/2⌋·⌈k/2⌉ / k²`. For even +//! `k` that is `(255/2)²`; for odd `k` it is strictly smaller, and using the +//! even-`k` ceiling would leave a maximally split three-lens cell scoring 15 +//! instead of 0 (a Codex finding on the lift PR — the lifted source had that +//! defect, perfected here per the ruling). It is NOT an α per pair — α is +//! undefined on a single subject — and the lifted source said so; the name +//! here says so too. //! //! # Quorum bands //! @@ -25,9 +30,25 @@ use crate::reliability::cronbach_alpha; -/// Largest population variance of a set of `u8` values: half at 0, half at -/// 255 gives `σ² = (255/2)²`. -pub const U8_MAX_VARIANCE: f64 = 255.0 * 255.0 / 4.0; +/// Largest population variance `k` `u8` values can have: `⌊k/2⌋` at one +/// extreme and `⌈k/2⌉` at the other, `σ² = 255² · ⌊k/2⌋·⌈k/2⌉ / k²`. Equals +/// `(255/2)²` for even `k` and is strictly smaller for odd `k`; `0.0` for +/// `k < 2` (one value has no dispersion). +/// +/// ``` +/// use jc::quorum::max_u8_variance; +/// assert_eq!(max_u8_variance(2), 255.0 * 255.0 / 4.0); +/// assert_eq!(max_u8_variance(3), 255.0 * 255.0 * 2.0 / 9.0); +/// assert_eq!(max_u8_variance(1), 0.0); +/// ``` +pub fn max_u8_variance(k: usize) -> f64 { + if k < 2 { + return 0.0; + } + let lo = (k / 2) as f64; + let hi = k as f64 - lo; + 255.0 * 255.0 * lo * hi / (k as f64 * k as f64) +} /// Per-pair agreement across `k` square `u8` tables of side `n`. /// @@ -52,6 +73,7 @@ pub fn pairwise_agreement_u8(tables: &[&[u8]], n: usize) -> Option> { return None; } let kf = k as f64; + let max_var = max_u8_variance(k); let mut scores = vec![0u8; n * n]; for i in 0..n { scores[i * n + i] = 255; @@ -66,8 +88,8 @@ pub fn pairwise_agreement_u8(tables: &[&[u8]], n: usize) -> Option> { }) .sum::() / kf; - let agreement = 1.0 - (var / U8_MAX_VARIANCE).sqrt(); - // `agreement` is in [0, 1] by construction (var ≤ U8_MAX_VARIANCE), + let agreement = 1.0 - (var / max_var).sqrt(); + // `agreement` is in [0, 1] by construction (var ≤ max_var for k), // so the rounded product is in 0..=255 and the cast cannot // truncate; the clamp is belt-and-braces against a rounding tick. let score = (agreement * 255.0).round().clamp(0.0, 255.0) as u8; @@ -227,6 +249,28 @@ mod tests { assert_eq!(pairwise_agreement_u8(&[&t, &short], 2), None); // ragged } + /// Odd `k`: a maximally split three-lens cell must score 0, which the + /// even-`k` ceiling cannot deliver (it gives 15 — the Codex finding). + /// Two-sided: a two-lens `[0, 255]` still scores 0 under the same rule, + /// and a three-lens `[0, 0, 128]` lands strictly between. Disable: put + /// `255²/4` back as the ceiling → the first assertion reads 15. + #[test] + fn an_odd_quorum_is_normalised_by_its_own_attainable_variance() { + let a = [255u8, 0, 0, 255]; + let b = [255u8, 0, 0, 255]; + let c = [255u8, 255, 255, 255]; + let s3 = pairwise_agreement_u8(&[&a, &b, &c], 2).unwrap(); + assert_eq!(s3[1], 0, "[0, 0, 255] is maximal for k = 3: {s3:?}"); + let s2 = pairwise_agreement_u8(&[&a, &c], 2).unwrap(); + assert_eq!(s2[1], 0, "[0, 255] is maximal for k = 2"); + let mid = [255u8, 128, 128, 255]; + let s3m = pairwise_agreement_u8(&[&a, &b, &mid], 2).unwrap(); + assert!(s3m[1] > 0 && s3m[1] < 255, "{s3m:?}"); + // The ceiling itself, at the two widths the API is used with. + assert_eq!(max_u8_variance(3), 255.0 * 255.0 * 2.0 / 9.0); + assert_eq!(max_u8_variance(5), 255.0 * 255.0 * 6.0 / 25.0); + } + /// Both sides of every band floor. #[test] fn quorum_bands_cut_exactly_at_their_floors() { diff --git a/crates/thinking-engine/examples/silu_crosscheck.rs b/crates/thinking-engine/examples/silu_crosscheck.rs index 24f55f0fc..06e23e07c 100644 --- a/crates/thinking-engine/examples/silu_crosscheck.rs +++ b/crates/thinking-engine/examples/silu_crosscheck.rs @@ -24,7 +24,8 @@ fn main() { .collect(); let samples = generate_training_data(&gate_centroids, &up_centroids, ¢roids, &probes); - let stats = correction_stats(&samples); + let stats = correction_stats(&samples) + .expect("synthetic corrections are finite; None would mean an invalid run"); eprintln!("Correction stats ({} samples):", stats.count); eprintln!( " Mean |Δ|: {:.4} Material: {:.1}% Large: {:.1}%\n", @@ -36,7 +37,7 @@ fn main() { // 2. Build corrected table let raw_table: Vec = JINA_HDR_TABLE.to_vec(); let mut corrected_table = raw_table.clone(); - let correction_scale = stats.mean_abs.min(0.3); + let correction_scale = stats.mean_abs.min(0.3) as f32; for i in 0..n { for j in 0..n { let idx = i * n + j; diff --git a/crates/thinking-engine/src/silu_correction.rs b/crates/thinking-engine/src/silu_correction.rs index 1da4cb3b3..64f574f2b 100644 --- a/crates/thinking-engine/src/silu_correction.rs +++ b/crates/thinking-engine/src/silu_correction.rs @@ -201,18 +201,30 @@ pub fn apply_corrections(table: &mut [u8], corrections: &[f32], n: usize) { pub use jc::drift::DeltaSummary as CorrectionStats; /// A correction that changes a cosine by more than this is material. -pub const MATERIAL_CORRECTION: f64 = 0.01; -/// A correction that changes a cosine by more than this is large. -pub const LARGE_CORRECTION: f64 = 0.1; +/// +/// Kept in the samples' own precision (`f32`) and promoted alongside them: +/// `f64::from(0.1f32)` is `0.10000000149…`, so a threshold written as +/// `0.1_f64` would count a correction of exactly `0.1f32` as "more than" +/// the cut-off. Promoting the same `f32` literal keeps equality excluded. +pub const MATERIAL_CORRECTION: f32 = 0.01; +/// A correction that changes a cosine by more than this is large. See +/// [`MATERIAL_CORRECTION`] for why this is an `f32`. +pub const LARGE_CORRECTION: f32 = 0.1; /// Summarise the corrections of a training sample set. /// -/// Empty input yields [`CorrectionStats::empty`] (count 0), preserving the -/// historical shape for callers that print a report unconditionally. -pub fn correction_stats(samples: &[CorrectionSample]) -> CorrectionStats { +/// Empty input yields `Some(`[`CorrectionStats::empty`]`)` (count 0) so a +/// report can still be printed. `None` means the data is INVALID — at least +/// one correction is `NaN` / `±∞` (a malformed or overflowed calibration +/// input) — and is deliberately not folded into the empty case: an invalid +/// run must not read as a clean empty one. +pub fn correction_stats(samples: &[CorrectionSample]) -> Option { + let (material, large) = (f64::from(MATERIAL_CORRECTION), f64::from(LARGE_CORRECTION)); + if samples.is_empty() { + return Some(CorrectionStats::empty(material, large)); + } let deltas: Vec = samples.iter().map(|s| f64::from(s.correction)).collect(); - jc::drift::delta_summary(&deltas, MATERIAL_CORRECTION, LARGE_CORRECTION) - .unwrap_or_else(|| CorrectionStats::empty(MATERIAL_CORRECTION, LARGE_CORRECTION)) + jc::drift::delta_summary(&deltas, material, large) } #[cfg(test)] @@ -311,7 +323,7 @@ mod tests { .sum::(); // Self-pairs should have near-zero correction (cos with itself = 1.0 both ways) - let stats = correction_stats(&samples); + let stats = correction_stats(&samples).expect("finite corrections"); eprintln!( "Correction stats: mean_abs={:.4}, max_abs={:.4}, material={:.1}%", stats.mean_abs, @@ -346,7 +358,7 @@ mod tests { } let samples = generate_training_data(&gates, &ups, ¢roids, ¢roids); - let stats = correction_stats(&samples); + let stats = correction_stats(&samples).expect("finite corrections"); eprintln!("\nNarrow gate (reader-lm range):"); eprintln!(" Samples: {}", stats.count); @@ -366,6 +378,28 @@ mod tests { assert!(stats.count > 0); } + /// The two Codex findings on the lift PR, pinned. (1) A correction of + /// exactly `0.1f32` is NOT "more than" the large cut-off — the threshold + /// is promoted from the same `f32`, so equality stays excluded (disable: + /// write the constant as `0.1_f64` → `large_fraction` reads 0.5). + /// (2) A non-finite correction is an INVALID run (`None`), never an + /// empty one; the genuinely empty sample set is `Some` with count 0. + #[test] + fn correction_stats_boundary_and_invalid_data() { + let sample = |c: f32| CorrectionSample { + centroid_i: vec![1.0], + centroid_j: vec![1.0], + correction: c, + }; + let s = correction_stats(&[sample(0.1), sample(0.2)]).unwrap(); + assert_eq!(s.large_fraction, 0.5, "exactly 0.1 is not > 0.1"); + assert_eq!(s.material_fraction, 1.0); + + assert!(correction_stats(&[sample(0.05), sample(f32::NAN)]).is_none()); + let empty = correction_stats(&[]).expect("empty is a valid, empty run"); + assert_eq!(empty.count, 0); + } + #[test] fn test_apply_corrections() { let n = 4; From 8c03eb52e227869160ad799893198c63879b7735 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 2 Sep 2026 22:02:16 +0000 Subject: [PATCH 5/5] fmt: the three lab files #1142 merged unformatted (format check red on main) awareness_dto.rs / persona.rs / world_model.rs were edited in #1142 and never run through rustfmt; the style workflow's format job walks the lab crate too, so main has been red on it since that merge. Formatting only. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01PFnYKqw6d7TTiB9cT8eFdK --- crates/thinking-engine/src/awareness_dto.rs | 2 +- crates/thinking-engine/src/persona.rs | 8 +++++--- crates/thinking-engine/src/world_model.rs | 3 +-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/crates/thinking-engine/src/awareness_dto.rs b/crates/thinking-engine/src/awareness_dto.rs index 503fda4d6..6a3dc3279 100644 --- a/crates/thinking-engine/src/awareness_dto.rs +++ b/crates/thinking-engine/src/awareness_dto.rs @@ -7,8 +7,8 @@ //! ``` use crate::cognitive_stack::{GateState, RungLevel, StyleFamily}; -use lance_graph_contract::escalation::GhostEcho; use crate::meaning_axes::{Archetype, AxisActivation, HdrResonance, Viscosity}; +use lance_graph_contract::escalation::GhostEcho; // ═══════════════════════════════════════════════════════════════════════════ // RESONANCE DTO — the gestalt + user model diff --git a/crates/thinking-engine/src/persona.rs b/crates/thinking-engine/src/persona.rs index 87dff41e3..4ee73afb9 100644 --- a/crates/thinking-engine/src/persona.rs +++ b/crates/thinking-engine/src/persona.rs @@ -20,8 +20,8 @@ use crate::cognitive_stack::EngineStyleExt; use crate::cognitive_stack::{GateState, RungLevel, StyleFamily}; use crate::contract_bridge::{CascadeConfig, FastBusDto}; -use lance_graph_contract::escalation::GhostEcho; use crate::meaning_axes::{Archetype, CouncilWeights, Viscosity}; +use lance_graph_contract::escalation::GhostEcho; // ═══════════════════════════════════════════════════════════════════════════ // PERSONA MODE @@ -452,7 +452,6 @@ impl Agent { ghost_count: u16, dominant_ghost: Option, ) -> SelfModelDto { - let gate = GateState::from_sd(dissonance + self.persona.collapse_bias); let viscosity = match gate { GateState::Flow => Viscosity::Ice, @@ -538,7 +537,10 @@ mod tests { 5, ); let msg = A2AMessage::thought(sender.to_dto(3), "receiver", bus, 0.9); - assert_eq!(msg.from.ghost_count, 3, "the caller-owned count reaches the receiver"); + assert_eq!( + msg.from.ghost_count, 3, + "the caller-owned count reaches the receiver" + ); assert_eq!(msg.to, "receiver"); assert_eq!(msg.resonance_weight, 0.9); assert_eq!(msg.from.mode, PersonaMode::Work); diff --git a/crates/thinking-engine/src/world_model.rs b/crates/thinking-engine/src/world_model.rs index 263511662..dba934475 100644 --- a/crates/thinking-engine/src/world_model.rs +++ b/crates/thinking-engine/src/world_model.rs @@ -12,8 +12,8 @@ //! ``` use crate::cognitive_stack::{GateState, StyleFamily}; -use lance_graph_contract::escalation::GhostEcho; use crate::meaning_axes::{Archetype, HdrResonance, Viscosity}; +use lance_graph_contract::escalation::GhostEcho; // ═══════════════════════════════════════════════════════════════════════════ // SELF STATE — the agent's internal awareness @@ -165,7 +165,6 @@ impl WorldModelDto { trace_count: u16, dominant_trace: Option, ) -> Self { - let hdr = HdrResonance::new( lens_agreement, 1.0 - dissonance,