diff --git a/.github/workflows/rust-test.yml b/.github/workflows/rust-test.yml index 8d4851aab..99bff13d6 100644 --- a/.github/workflows/rust-test.yml +++ b/.github/workflows/rust-test.yml @@ -113,6 +113,15 @@ jobs: run: cargo test --manifest-path crates/lance-graph-contract/Cargo.toml --tests - name: Run contract doctests run: cargo test --manifest-path crates/lance-graph-contract/Cargo.toml --doc + # weather-poc is workspace-EXCLUDED. A new workflow introduced by the + # same pull request cannot gate that pull request because `pull_request` + # workflow selection comes from the base branch. Run both surfaces here + # so D-WXS-4's zero-dep codec and its live NodeRow agreement are tested + # before merge, not one merge later. + - name: Run weather-poc tests (zero-dependency codec) + run: cargo test --manifest-path crates/weather-poc/Cargo.toml + - name: Run weather-poc canonical-row agreement tests + run: cargo test --manifest-path crates/weather-poc/Cargo.toml --features canonical-row # lance-graph-ogar is the workspace-EXCLUDED armed tier: it owns the # mirror<->live-vocab fuses (COUNT_FUSE, per-entry parity, the # authoritative-table roundtrip green light) against the OGAR sibling diff --git a/.github/workflows/weather-poc.yml b/.github/workflows/weather-poc.yml new file mode 100644 index 000000000..bee95ff03 --- /dev/null +++ b/.github/workflows/weather-poc.yml @@ -0,0 +1,41 @@ +name: Weather POC + +on: + pull_request: + paths: + - crates/weather-poc/** + - crates/lance-graph-contract/** + - rust-toolchain + - rust-toolchain.toml + - .github/workflows/weather-poc.yml + push: + branches: + - main + paths: + - crates/weather-poc/** + - crates/lance-graph-contract/** + - .github/workflows/weather-poc.yml + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + test: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Setup Rust + run: | + rustup toolchain install stable + rustup default stable + - name: Test zero-dependency codec path + run: cargo test --manifest-path crates/weather-poc/Cargo.toml + - name: Test live canonical NodeRow agreement path + run: cargo test --manifest-path crates/weather-poc/Cargo.toml --features canonical-row diff --git a/crates/lance-graph-contract/src/ogar_codebook.rs b/crates/lance-graph-contract/src/ogar_codebook.rs index 353ebefd1..efae8e65c 100644 --- a/crates/lance-graph-contract/src/ogar_codebook.rs +++ b/crates/lance-graph-contract/src/ogar_codebook.rs @@ -48,6 +48,8 @@ pub enum ConceptDomain { ProjectMgmt, /// `0x02XX` — commerce / billing / ERP (Odoo ↔ OSB). Commerce, + /// `0x04XX` — Weather / Atmosphere. Shared forecast and atmospheric cells. + Weather, /// `0x07XX` — OSINT (open-source intelligence / Palantir-Gotham). Osint, /// `0x08XX` — OCR (optical character recognition / document extraction). @@ -86,7 +88,8 @@ pub enum ConceptDomain { /// (OGAR canon "256×256 centroid tile", D-BOTHCASC). Mirrors OGAR /// `ogar_vocab::ConceptDomain::Geo`; the parity tests pin `0x0F00 → Geo`. Geo, - /// Any high-byte slot not yet assigned a domain (`0x03XX`–`0x06XX`, `0x10XX`+). + /// Any high-byte slot not yet assigned a domain (`0x03XX`, `0x05XX`–`0x06XX`, + /// `0x10XX`+). Unassigned, } @@ -100,6 +103,7 @@ pub fn canonical_concept_domain(id: u16) -> ConceptDomain { 0x00 => ConceptDomain::Reserved, 0x01 => ConceptDomain::ProjectMgmt, 0x02 => ConceptDomain::Commerce, + 0x04 => ConceptDomain::Weather, 0x07 => ConceptDomain::Osint, 0x08 => ConceptDomain::Ocr, 0x09 => ConceptDomain::Health, @@ -483,6 +487,11 @@ pub const CODEBOOK: &[(&str, u16)] = &[ ("pricelist", 0x0209), ("pricelist_rule", 0x020A), ("unit_of_measure", 0x020B), + // ── 0x04XX — Weather / Atmosphere domain ── + // Canonical cell meanings minted by OGAR #272. W1 field/level/unit slots + // remain ClassView-owned payload structure, not promoted concept rows. + ("weather_cell", 0x0401), + ("weather_static_cell", 0x0402), // ── 0x08XX — OCR domain (document extraction; the Tesseract-rs arc) ── // Class-level container KINDS only (the 5+3-hardened mint discipline): // the concept slots name the container types the OGAR Core resolves — @@ -649,6 +658,7 @@ mod tests { assert_eq!(canonical_concept_domain(0x0000), ConceptDomain::Reserved); assert_eq!(canonical_concept_domain(0x0101), ConceptDomain::ProjectMgmt); assert_eq!(canonical_concept_domain(0x0206), ConceptDomain::Commerce); + assert_eq!(canonical_concept_domain(0x0401), ConceptDomain::Weather); assert_eq!(canonical_concept_domain(0x0700), ConceptDomain::Osint); assert_eq!(canonical_concept_domain(0x0801), ConceptDomain::Ocr); assert_eq!(canonical_concept_domain(0x0901), ConceptDomain::Health); @@ -659,18 +669,13 @@ mod tests { assert_eq!(canonical_concept_domain(0x0D01), ConceptDomain::HR); assert_eq!(canonical_concept_domain(0x0D04), ConceptDomain::HR); assert_eq!(canonical_concept_domain(0x0500), ConceptDomain::Unassigned); - // Genetics (0x0E) operator-allocated 2026-06-26 for CPIC-V3 (was Unassigned). assert_eq!(canonical_concept_domain(0x0E00), ConceptDomain::Genetics); - // Geo (0x0F) allocated with the OSM harvest (OGAR #152; was Unassigned). assert_eq!(canonical_concept_domain(0x0F00), ConceptDomain::Geo); assert_eq!(canonical_concept_domain(0x1000), ConceptDomain::Unassigned); } #[test] fn classid_routes_through_canon_half() { - // The contract classids resolve to the domain their CANON half (the - // HIGH u16 since the P1 flip) encodes — the contract↔OGAR alignment - // (ISS-CLASSID-OGAR-DRIFT). assert_eq!( classid_concept_domain(NodeGuid::CLASSID_PROJECT), ConceptDomain::ProjectMgmt @@ -711,23 +716,20 @@ mod tests { #[test] fn codebook_ids_match_ogar_vocab() { - // Drift guard: these MUST match OGAR `ogar_vocab::CODEBOOK` exactly (the - // wire is the contract). If OGAR moves an id, update BOTH sides together. assert_eq!(canonical_concept_id("project"), Some(0x0101)); assert_eq!(canonical_concept_id("project_work_item"), Some(0x0102)); assert_eq!(canonical_concept_id("project_enabled_module"), Some(0x011A)); assert_eq!(canonical_concept_id("commercial_line_item"), Some(0x0201)); assert_eq!(canonical_concept_id("commercial_document"), Some(0x0202)); assert_eq!(canonical_concept_id("currency_policy"), Some(0x0206)); - // 0x08XX OCR (container kinds; unichar content stays out of the codebook). + assert_eq!(canonical_concept_id("weather_cell"), Some(0x0401)); + assert_eq!(canonical_concept_id("weather_static_cell"), Some(0x0402)); assert_eq!(canonical_concept_id("unicharset"), Some(0x0801)); assert_eq!(canonical_concept_id("charset"), Some(0x0803)); - // 0x09XX Health + 0x0BXX Auth (OGAR #110 minted the AuthStore family). assert_eq!(canonical_concept_id("patient"), Some(0x0901)); assert_eq!(canonical_concept_id("vital_sign"), Some(0x0907)); assert_eq!(canonical_concept_id("auth_store"), Some(0x0B01)); assert_eq!(canonical_concept_id("auth_ory_keto"), Some(0x0B04)); - // 0x0CXX Automation (the MARS/Automation codebook pass minted these in OGAR). assert_eq!(canonical_concept_id("mars_application"), Some(0x0C01)); assert_eq!(canonical_concept_id("knowledge_item"), Some(0x0C05)); assert_eq!(canonical_concept_id("mars_node_template"), Some(0x0C06)); @@ -737,8 +739,6 @@ mod tests { #[test] fn codebook_has_no_duplicate_ids_or_zero_concept_slot() { - // Every id non-zero in its concept slot (CC != 0x00 — root is reserved), - // every id unique, and each id's domain matches its position. let mut seen = std::collections::HashSet::new(); for &(name, id) in CODEBOOK { assert_ne!( @@ -755,9 +755,8 @@ mod tests { let dto = LabelDTO::from_canonical("project_enabled_module").unwrap(); assert_eq!(dto.id, 0x011A); assert_eq!(dto.canonical, "project_enabled_module"); - assert_eq!(dto.id_le(), [0x1A, 0x01]); // LE: low byte (0x1A) first, high (0x01) + assert_eq!(dto.id_le(), [0x1A, 0x01]); assert_eq!(u16::from_le_bytes(dto.id_le()), 0x011A); - // domain reachable from the DTO id assert_eq!(canonical_concept_domain(dto.id), ConceptDomain::ProjectMgmt); assert_eq!( LabelDTO::from_canonical("Issue"), @@ -768,8 +767,6 @@ mod tests { #[test] fn app_prefixes_match_ogar_allocation_table() { - // §2 allocation table — MUST match OGAR `PortSpec::APP_PREFIX` (the - // wire). If OGAR re-allocates a prefix, update BOTH sides together. assert_eq!(AppPrefix::Core.prefix(), 0x0000); assert_eq!(AppPrefix::OpenProject.prefix(), 0x0001); assert_eq!(AppPrefix::Odoo.prefix(), 0x0002); @@ -777,7 +774,6 @@ mod tests { assert_eq!(AppPrefix::Smb.prefix(), 0x0004); assert_eq!(AppPrefix::Healthcare.prefix(), 0x0005); assert_eq!(AppPrefix::Redmine.prefix(), 0x0007); - // round-trips; unallocated slots are None (reserved, cost nothing). for app in [ AppPrefix::Core, AppPrefix::OpenProject, @@ -795,12 +791,9 @@ mod tests { #[test] fn render_classid_composes_decomposes_and_preserves_the_concept_half() { - // Worked examples mirrored from OGAR `ogar_vocab::app` tests — the - // P1 canon-high forms (concept HIGH, prefix LOW). assert_eq!(render_classid(0x0001, 0x0102), 0x0102_0001); - assert_eq!(render_classid(0x0007, 0x0102), 0x0102_0007); // Redmine twin + assert_eq!(render_classid(0x0007, 0x0102), 0x0102_0007); - // MedCare patient — the canonical worked example: 0x0901_0005. let pat = render_classid_for_concept(AppPrefix::Healthcare, "patient").unwrap(); assert_eq!(pat, 0x0901_0005); assert_eq!(classid_app_prefix(pat), 0x0005); @@ -809,48 +802,34 @@ mod tests { AppPrefix::from_prefix(classid_app_prefix(pat)), Some(AppPrefix::Healthcare) ); - // the concept half still routes to its domain under the render prefix. assert_eq!( canonical_concept_domain(classid_concept(pat)), ConceptDomain::Health ); - // Core (prefix=0x0000): the bare concept sits in the CANON (high) half. let core = render_classid(0x0000, 0x0102); assert_eq!(core, (0x0102u32) << 16); assert_eq!(classid_concept(core), 0x0102); - // The render lens never perturbs the CANON concept RBAC keys on. let op = AppPrefix::OpenProject.render(0x0103); let rm = AppPrefix::Redmine.render(0x0103); - assert_ne!( - classid_app_prefix(op), - classid_app_prefix(rm), - "render lenses differ" - ); - assert_eq!( - classid_concept(op), - classid_concept(rm), - "concept is shared" - ); + assert_ne!(classid_app_prefix(op), classid_app_prefix(rm), "render lenses differ"); + assert_eq!(classid_concept(op), classid_concept(rm), "concept is shared"); - // Unpromoted concept → no classid (don't invent one). assert_eq!( render_classid_for_concept(AppPrefix::Healthcare, "nope"), None ); } - // ── D-CCF-0 probes — the one flippable classid composition ──────────── - #[test] fn classid_split_compose_round_trips_under_both_orders() { let samples: &[(u16, u16)] = &[ - (0x0700, 0x0000), // legacy OSINT domain classid halves - (0x0701, 0x1000), // post-flip OSINT:q2 halves + (0x0700, 0x0000), + (0x0701, 0x1000), (0x0A01, 0x1000), (0x0E01, 0x1000), - (0x0901, 0x0005), // Healthcare render pair + (0x0901, 0x0005), (0x0000, 0x0000), (0xFFFF, 0xFFFF), ]; @@ -868,41 +847,27 @@ mod tests { #[test] fn classid_flip_is_involutive_and_p1_pins_target_order() { - // P1 pin: the active order is the target CanonHigh (operator trigger - // 2026-07-02). Un-flipping this const is a migration reversal, never - // a drive-by. assert_eq!(CLASSID_ORDER, ClassidOrder::CanonHigh); - // flip(flip(x)) == x over every wired classid + the post-flip trio. for id in [ - 0x0000_0700u32, // legacy OSINT domain class - 0x1000_0700, // pre-flip OSINT-V3 + 0x0000_0700u32, + 0x1000_0700, 0x1000_0A01, 0x1000_0E00, - 0x0701_1000, // post-flip forms (already valid u32s to flip back) + 0x0701_1000, 0x0A01_1000, 0x0E01_1000, - 0x0005_0901, // Healthcare render classid + 0x0005_0901, 0x0000_0000, 0xFFFF_FFFF, ] { - assert_eq!( - flip_classid(flip_classid(id)), - id, - "flip must be involutive" - ); + assert_eq!(flip_classid(flip_classid(id)), id, "flip must be involutive"); } } #[test] fn classid_route_through_matrix_under_active_and_legacy_order() { - // The boundary matrix (plan §3), post-flip form: under the active - // CanonHigh order every routed reader equals the canon-high masks, - // for every codebook id under every app prefix — and the LEGACY - // (CanonLow) composition stays available under the explicit-order - // API for reading persisted pre-flip ids. for &(_, concept) in CODEBOOK { for prefix in [0x0000u16, 0x0001, 0x0005, 0x1000] { - // Active order: canon (concept) HIGH, custom (prefix) LOW. let id = render_classid(prefix, concept); assert_eq!(id, ((concept as u32) << 16) | (prefix as u32)); assert_eq!(classid_concept(id), concept); @@ -915,15 +880,9 @@ mod tests { "domain routing invariant under the route-through" ); - // Legacy boundary: the explicit CanonLow split still reads a - // persisted pre-flip id exactly as the direct masks did. let legacy = compose_classid_with(ClassidOrder::CanonLow, concept, prefix); assert_eq!(legacy, ((prefix as u32) << 16) | (concept as u32)); - assert_eq!( - split_classid_with(ClassidOrder::CanonLow, legacy), - (concept, prefix) - ); - // And the flip carries a legacy id to its new-form twin. + assert_eq!(split_classid_with(ClassidOrder::CanonLow, legacy), (concept, prefix)); assert_eq!(flip_classid(legacy), id); } } @@ -931,23 +890,17 @@ mod tests { #[test] fn classid_canon_compat_reads_both_stored_forms() { - // New-form ids: compat == strict canon. for id in [0x0901_0005u32, 0x0701_1000, 0x0102_0001, 0x0700_0000] { assert_eq!(classid_canon_compat(id), classid_canon(id)); } - // Persisted pre-flip forms resolve their true canon via the legacy - // fallback: core, render, and V3-marked shapes. - assert_eq!(classid_canon_compat(0x0000_0901), 0x0901); // legacy core - assert_eq!(classid_canon_compat(0x0005_0901), 0x0901); // legacy render - assert_eq!(classid_canon_compat(0x1000_0700), 0x0700); // legacy V3 - assert_eq!(classid_canon_compat(0x0000_0000), 0x0000); // default class + assert_eq!(classid_canon_compat(0x0000_0901), 0x0901); + assert_eq!(classid_canon_compat(0x0005_0901), 0x0901); + assert_eq!(classid_canon_compat(0x1000_0700), 0x0700); + assert_eq!(classid_canon_compat(0x0000_0000), 0x0000); } #[test] fn no_class_collapse_under_canon_high() { - // codex P2 (#627): post-flip, a naive `as u16` reads the CUSTOM half — - // 0x1000 for ALL three V3 classes — collapsing the SoA class_id - // discriminator. The canon half stays distinct; `as u16` does not. let osint = compose_classid_with(ClassidOrder::CanonHigh, 0x0701, 0x1000); let fma = compose_classid_with(ClassidOrder::CanonHigh, 0x0A01, 0x1000); let cpic = compose_classid_with(ClassidOrder::CanonHigh, 0x0E01, 0x1000); @@ -958,12 +911,7 @@ mod tests { split_classid_with(ClassidOrder::CanonHigh, fma).0, split_classid_with(ClassidOrder::CanonHigh, cpic).0, ]; - assert_eq!( - canons, - [0x0701, 0x0A01, 0x0E01], - "canon halves stay distinct" - ); - // The forbidden pattern, demonstrated: `as u16` collapses all three. + assert_eq!(canons, [0x0701, 0x0A01, 0x0E01], "canon halves stay distinct"); assert_eq!( [osint as u16, fma as u16, cpic as u16], [0x1000, 0x1000, 0x1000], diff --git a/crates/lance-graph-ogar/src/lib.rs b/crates/lance-graph-ogar/src/lib.rs index 0f0ff0859..55a538464 100644 --- a/crates/lance-graph-ogar/src/lib.rs +++ b/crates/lance-graph-ogar/src/lib.rs @@ -98,13 +98,11 @@ #![forbid(unsafe_code)] #![warn(missing_docs)] -// ── Full re-export of the OGAR Active-Record crates under stable names ── pub use ogar_adapter_surrealql; pub use ogar_class_view; pub use ogar_ontology; pub use ogar_vocab; -// ── The contract surface OGAR implements + the wire mirror the guard checks ── pub use lance_graph_contract as contract; /// The OGAR active-record `ClassView` projection (`impl @@ -113,22 +111,12 @@ pub use ogar_class_view::OgarClassView; /// The calcified canonical AR shape (attributes + family `Association`s). pub use ogar_vocab::Class; -// ── OGAR-driven tenant port bridges (moved out of lance-graph-ontology, -// which is OGIT and must not depend on ogar-vocab) ── pub mod bridges; -// ── OGAR DO-arm provider: per-class ActionDef manifests with RBAC hardcoded -// into the class (the Türsteher). The action-axis sibling of OgarClassView. ── pub mod actions; pub use actions::OgarActionProvider; pub mod rbac_impl; -// Per-port bridge aliases (`MedcareBridge` / `OpenProjectBridge` / -// `RedmineBridge` / `OdooBridge` / `SmbBridge` / `WoaBridge`) are -// `#[deprecated]` (2026-06-22) — pull the classid via the corresponding -// PortSpec instead. The `Port` types and `UnifiedBridge` harness are -// NOT deprecated. See `docs/CONSUMER-BRIDGE-DEPRECATION.md` + -// AdaWorldAPI/OGAR#95. pub use bridges::{ HealthcarePort, OdooPort, OpenProjectPort, RedminePort, SmbPort, UnifiedBridge, WoaPort, }; @@ -175,6 +163,7 @@ pub mod parity { (O::Reserved, C::Reserved) | (O::ProjectMgmt, C::ProjectMgmt) | (O::Commerce, C::Commerce) + | (O::Weather, C::Weather) | (O::Osint, C::Osint) | (O::Ocr, C::Ocr) | (O::Health, C::Health) @@ -192,7 +181,6 @@ pub mod parity { /// forward (mirror ⊆ OGAR), reverse (OGAR ⊆ mirror), and domain agreement. /// Returns the number of concepts checked. Panics on any divergence. pub fn assert_codebook_parity() -> usize { - // Forward: every mirror entry resolves identically through OGAR's API. for &(concept, id) in mirror::CODEBOOK { assert_eq!( ogar_vocab::canonical_concept_id(concept), @@ -204,8 +192,6 @@ pub mod parity { "domain disagreement for {concept} ({id:#06x})" ); } - // Reverse: every OGAR canonical concept is present in the mirror with the - // same id (no OGAR concept silently missing from the wire mirror). for &(concept, id) in ogar_vocab::class_ids::ALL { assert_eq!( mirror::canonical_concept_id(concept), @@ -228,13 +214,10 @@ pub mod parity { #[test] fn classid_low_u16_is_the_codebook_id() { - // The contract NodeGuid.classid low u16 IS the OGAR codebook id — the - // wire identity the whole separation rests on. use lance_graph_contract::NodeGuid; let project_id = ogar_vocab::canonical_concept_id("project").unwrap(); let guid = NodeGuid::new(u32::from(project_id), 0, 0, 0, 0, 0); assert_eq!(guid.classid() as u16, project_id); - // and it routes to the ProjectMgmt domain on both sides assert!(domains_agree(project_id)); } } @@ -244,16 +227,6 @@ pub mod parity { mod tests { use super::*; - /// The ROUNDTRIP GREEN LIGHT (operator, 2026-07-07): lance-graph does NOT - /// carry ontologies — it only flips this fuse. For every authoritative - /// OGAR capability table (OCR today), assert at ID level that (a) the - /// authority named at least one expected executor ("die Ontologie wurde - /// nicht vergessen"), (b) every subject classid the table binds exists in - /// the wire mirror this crate already guards, (c) the table itself is - /// internally consistent (names unique, non-empty). The consumer-side - /// half of the loop (registration + coverage + classid activation) is - /// asserted in the consumer's own binary via - /// `ogar_vocab::capability_registry::verify_registration`. #[test] fn authoritative_ocr_table_roundtrip_is_green() { use ogar_vocab::ocr_actions; @@ -281,9 +254,6 @@ mod tests { #[test] fn ogar_class_view_implements_contract_class_view() { - // The activation in one line: an OgarClassView IS a contract ClassView, - // so a consumer holding `&dyn ClassView` can be handed the real OGAR AR - // surface. (Compile-time proof; constructing it walks the 32 class fns.) use lance_graph_contract::class_view::ClassView; let view = OgarClassView::new(); let _as_trait: &dyn ClassView = &view; @@ -317,12 +287,6 @@ impl lance_graph_contract::hotplug::CapabilityAuthority for OgarAuthority { .into_iter() .map(|(name, id)| (name.to_string(), id)) .collect(); - // Cross-check the wire mirror for the PLUGGED concepts only. - // `resolve_hotplug` consults OGAR alone, so without this a - // stale mirror activates green here while a mirror-reading - // consumer resolves `None` — the drift class the retired - // COUNT_FUSE guarded (codex P2 on PR #954). This authority is - // the only place both sides are in scope. if let Some(drift) = lance_graph_contract::hotplug::verify_against_mirror(&concepts) { @@ -346,21 +310,11 @@ impl lance_graph_contract::hotplug::CapabilityAuthority for OgarAuthority { mod hotplug_bridge_tests { use lance_graph_contract::hotplug::{CapabilityAuthority, HotPlug}; - /// The generic activation, end to end through the contract socket: the - /// OCR consumer's plug resolves to the 3 vocab rows + 8 capabilities. #[test] fn ocr_hotplug_activates_through_the_contract_socket() { let plug = HotPlug { consumer: "tesseract-ogar", classids: &[0x0805, 0x0808, 0x0809], - // Exactly the actions the three requested classids declare - // (OGAR `ocr_actions`, grown by OGAR #188's structured-document - // v2): textline(0x0805)=1, page_image(0x0808)=7, - // ocr_renderer(0x0809)=4 = 12. `harvest_fields` / - // `detect_page_furniture` are `page_layout`(0x0807), NOT - // requested here, so they stay out. `covered` must match the - // requested classids' action set exactly (activate rejects both - // Uncovered and Undeclared). covered: &[ "recognize_line", "recognize_page", @@ -383,16 +337,6 @@ mod hotplug_bridge_tests { assert_eq!(act.capabilities.len(), 12); } - /// The property the deleted `COUNT_FUSE` could not provide: drift is - /// reported **per plug**, naming the id the consumer actually asked for — - /// not as a global equality assert that fails every build. - /// - /// This is the whole point of the 2026-08-14 migration. Under the fuse, a - /// concept minted in OGAR but not yet mirrored broke `cargo build` for every - /// AR-aware consumer, including ones that never touched it — which is how a - /// production deploy died at const-eval for `osm_street_node`. Under - /// hot-plug, an unplugged concept is INERT, and a plugged-but-unknown one - /// fails loudly, at the consumer, naming itself. #[test] fn an_unknown_classid_drifts_at_the_plug_not_at_the_build() { use lance_graph_contract::hotplug::ActivationDrift; @@ -411,14 +355,6 @@ mod hotplug_bridge_tests { "a plugged classid that is not minted must drift by NAME" ); - // The silent half, and the reason the fuse had to go: a REAL, minted - // concept that this consumer does not plug is inert — it cannot fail - // anything. `osm_street_node` (0x0F0B) is exactly such a concept for an - // OCR consumer, and is the one whose mint took a deploy down. - // - // PIN THE FIXTURE (codex/CodeRabbit on #954): without this the test - // would still pass if `osm_street_node` were removed or renamed — - // silently no longer covering the regression it is named for. assert_eq!( ogar_vocab::canonical_concept_id("osm_street_node"), Some(0x0F0B), @@ -433,10 +369,6 @@ mod hotplug_bridge_tests { let act = auth .activate(&ocr_only) .expect("plugging one id must not be affected by concepts elsewhere in the codebook"); - // Exact equality already proves the whole table was NOT resolved — a - // separate "osm_street_node is absent" assertion would be implied by - // this one, which is the vacuous-assertion shape the repo's own - // falsifiability rule rejects. assert_eq!(act.concepts, vec![("textline".to_string(), 0x0805)]); } } diff --git a/crates/weather-poc/Cargo.toml b/crates/weather-poc/Cargo.toml index b95219aa6..475229286 100644 --- a/crates/weather-poc/Cargo.toml +++ b/crates/weather-poc/Cargo.toml @@ -6,49 +6,30 @@ license = "Apache-2.0" publish = false description = "Zarr -> NodeRow: the ERA5 0.25-degree global grid baked onto the canonical 512-byte SoA row. One cell = one node (HEEL = 16-degree tile, HIP = within-tile), one timestep = one Lance version. Plan: .claude/plans/weather-soa-bake-v1.md" -# Standalone research crate, EXCLUDED from the lance-graph workspace (same -# pattern as jc / sigker / helix / perturbation-sim). Verify with: +# Standalone research crate, EXCLUDED from the lance-graph workspace. Default +# builds remain genuinely zero-dependency: # cargo test --manifest-path crates/weather-poc/Cargo.toml # -# ── ZERO-DEP BY CONSTRUCTION (an orchestrator decision, plan §6.1) ─────────── -# This crate has NO dependencies, not even optional ones, and that is -# deliberate rather than incidental: +# The `canonical-row` feature is a deliberately opt-in AGREEMENT seam to the +# sibling contract crate. It exists so D-WXS-4 can place the W1 facet image in +# the LIVE append-only value carve instead of copying today's free-tail offset +# into this standalone crate. This is different from the historical helix path +# trap: `lance-graph-contract` is a sibling in this repository, while helix's +# mandatory ndarray path may not exist in a clean checkout. # -# * `helix::quantize::RollingFloor` is the shipped carrier for the linear -# 256-level floor this crate needs (plan §5 classifies the arc's Python -# `encode_decode` as a re-derivation of exactly it). But `helix` carries a -# MANDATORY ndarray git dep, and an *optional path* dep is read at manifest -# resolution -- so merely offering `helix = { path = "../helix", optional }` -# would break a clean checkout even with the feature off. That is the -# codex-P2 trap on #504 / the helix-#460 lesson, named in plan §6.3. -# * So `floor.rs` carries the linear formula itself, documented as identical -# to helix's, and the AGREEMENT between the two is a separate deliverable -# run across manifests -- the same shape as D-WXS-12 (jc <-> ndarray), where -# parity is a measured comparison rather than a dependency edge. -# -# If a future wave needs `ndarray::simd`, add it the way perturbation-sim does: -# optional, behind an off-by-default feature, sourced by GIT URL -- never a path. -# All SIMD comes from `ndarray::simd` per the workspace rule, never raw -# intrinsics, and never `ndarray::hpc::*` from a consumer. +# `helix::quantize::RollingFloor` remains intentionally NOT a dependency. The +# linear formula is re-expressed in floor.rs and parity is a measured gate. + +[features] +default = [] +canonical-row = ["dep:lance-graph-contract"] [dependencies] +lance-graph-contract = { path = "../lance-graph-contract", optional = true } -# ── jc AS A DEV-DEPENDENCY, PATH-SOURCED — different case from `helix` above ── -# Bar B6/B7 (plan sec 4 W3, D-WXS-7/D-WXS-8) require the fidelity metric to be -# "computed WITH jc::reliability::spearman" specifically — jc is the -# operator-named authority for this measurement (`.claude/plans/ -# weather-soa-bake-v1.md` sec 5's classification of `spearman(a,b)` as -# SUBSTRATE). This is safe as a PATH dep, unlike helix, for two reasons: -# 1. It is a dev-dependency: resolved only for `cargo test`/examples, NEVER -# for `cargo build --lib` (the default/production build this crate's -# zero-dep note is actually protecting). -# 2. `jc` is itself genuinely zero-external-dep BY DEFAULT (its own -# Cargo.toml: "Zero EXTERNAL deps in production"; its one path-dep, -# `sigker`, sits behind an opt-in, non-default feature) — so pulling it -# in does not chain into ndarray or any other heavy fork, and does not -# reintroduce the codex-P2/#504 trap (that trap is about an OPTIONAL -# dependency still being read at manifest resolution; a plain -# dev-dependency on an already-zero-dep sibling has no such edge). +# Bar B6/B7 require jc::reliability::spearman specifically. jc is itself +# zero-external-dep by default, so this dev edge does not pull the heavy stack +# into production builds. [dev-dependencies] jc = { path = "../jc" } diff --git a/crates/weather-poc/src/bake.rs b/crates/weather-poc/src/bake.rs new file mode 100644 index 000000000..d8a9f57d2 --- /dev/null +++ b/crates/weather-poc/src/bake.rs @@ -0,0 +1,395 @@ +//! W1 weather bake assembly: one ERA5 cell -> canonical key + three L4 facets. +//! +//! This module deliberately stops at the zero-dependency ownership boundary. +//! The default `weather-poc` codec path does not depend on +//! `lance-graph-contract`, so it must not copy the canonical `NodeRow` +//! value-tail offset into a second crate. Instead it emits a +//! [`PackedWeatherCell`] containing the byte-exact key and the three W1 facets. +//! The opt-in canonical adapter places those facets into the live canonical +//! `NodeRow` using the contract-derived free-tail offset. +//! +//! That split is load-bearing: new append-only `ValueTenant`s may move the free +//! tail without changing the 512-byte ABI. A hard-coded row offset here would +//! silently overwrite a later tenant. +//! +//! # Streaming, not a million-object staging vector +//! +//! [`bake_timestep`] walks the 721 x 1440 grid and hands each packed cell to a +//! caller-provided sink immediately. It never allocates a `Vec` of 1,038,240 +//! cells. The sink may assemble `NodeRow`s, feed Arrow batches, hash a receipt, +//! or stop on its own error. + +use std::collections::HashMap; +use std::fmt; + +use crate::floor::CalibratedFloor; +use crate::key::{encode_key, LAT_COUNT, LON_COUNT}; +use crate::lane::{pack_facet, LaneError, FACET_LEN}; +use crate::manifest::{FieldManifest, ManifestEntry}; + +/// Number of W1 L4 facets: F0 surface, F1 850 hPa, F2 500 hPa. +pub const W1_FACET_COUNT: usize = 3; + +/// Bytes in the W1 weather extension image: three 16-byte L4 facets. +pub const W1_IMAGE_LEN: usize = W1_FACET_COUNT * FACET_LEN; + +/// Number of real cells in one ERA5 0.25-degree global timestep. +pub const GLOBAL_CELL_COUNT: usize = LAT_COUNT as usize * LON_COUNT as usize; + +const _: () = assert!(GLOBAL_CELL_COUNT == 1_038_240); +const _: () = assert!(W1_IMAGE_LEN == 48); + +/// One zero-dependency W1 cell image, ready for the canonical-row adapter. +/// +/// This is intentionally *not* a second `NodeRow` type. It carries only the +/// bytes `weather-poc` owns: the canonical key image plus the three weather +/// facets. Edge bytes, existing value tenants, and the current free-tail +/// offset remain the contract crate's responsibility. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PackedWeatherCell { + /// Source latitude index, retained for receipts and sink diagnostics. + pub lat_idx: u16, + /// Source longitude index, retained for receipts and sink diagnostics. + pub lon_idx: u16, + /// Canonical 16-byte key image produced by [`encode_key`]. + pub key: [u8; 16], + /// F0/F1/F2 L4 facets in facet-index order. + pub facets: [[u8; FACET_LEN]; W1_FACET_COUNT], +} + +impl PackedWeatherCell { + /// Flatten the three W1 facets into their contiguous 48-byte extension + /// image. This is useful for hashing/receipts and remains independent of + /// where the live `NodeRow` contract places the extension. + pub fn facet_image(&self) -> [u8; W1_IMAGE_LEN] { + let mut out = [0u8; W1_IMAGE_LEN]; + for (facet, bytes) in self.facets.iter().enumerate() { + let start = facet * FACET_LEN; + out[start..start + FACET_LEN].copy_from_slice(bytes); + } + out + } +} + +/// Errors produced before a packed cell reaches the caller's sink. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum BakeError { + /// `0x0000_0000` belongs to the canonical bootstrap/default ladder and may + /// never label a durable weather row. + ZeroClassId, + /// A caller asked to pack a coordinate outside the real 721 x 1440 grid. + /// This is a hard error even in release builds. `encode_key` itself keeps a + /// debug assertion because it is a low-level codec; the bake is the + /// publication boundary and must not rely on debug-only validation. + GridIndexOutOfRange { + /// Requested latitude index. + lat_idx: u16, + /// Requested longitude index. + lon_idx: u16, + }, + /// Packing one of the three L4 facets failed. + Lane(LaneError), +} + +impl fmt::Display for BakeError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + BakeError::ZeroClassId => f.write_str( + "weather bake refuses classid 0x00000000: it is the canonical bootstrap/default class", + ), + BakeError::GridIndexOutOfRange { lat_idx, lon_idx } => write!( + f, + "weather bake cell ({lat_idx}, {lon_idx}) is outside the real grid 0..{LAT_COUNT} x 0..{LON_COUNT}" + ), + BakeError::Lane(err) => write!(f, "weather facet packing failed: {err}"), + } + } +} + +impl std::error::Error for BakeError {} + +impl From for BakeError { + fn from(value: LaneError) -> Self { + Self::Lane(value) + } +} + +/// Error from a full-grid streaming bake. +#[derive(Debug)] +pub enum BakeStreamError { + /// The weather cell could not be assembled. + Bake(BakeError), + /// The downstream sink rejected an otherwise valid packed cell. + Sink(E), +} + +impl fmt::Display for BakeStreamError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + BakeStreamError::Bake(err) => err.fmt(f), + BakeStreamError::Sink(err) => write!(f, "weather bake sink failed: {err}"), + } + } +} + +impl std::error::Error for BakeStreamError {} + +/// Pack one real grid cell under the W1 manifest. +/// +/// `value_of` is the source boundary. It receives manifest entries rather +/// than numeric slot ordinals, so source adapters may resolve values however +/// they like while slot placement remains entirely manifest-owned. +/// +/// # Errors +/// +/// Refuses the bootstrap classid, rejects indices outside the real grid in all +/// build modes, and propagates all [`LaneError`] variants, including +/// missing/non-finite source values and unknown floors. +pub fn pack_cell( + classid: u32, + lat_idx: u16, + lon_idx: u16, + manifest: &FieldManifest, + floors: &HashMap, + mut value_of: F, +) -> Result +where + F: FnMut(&ManifestEntry) -> Option, +{ + if classid == 0 { + return Err(BakeError::ZeroClassId); + } + if lat_idx >= LAT_COUNT || lon_idx >= LON_COUNT { + return Err(BakeError::GridIndexOutOfRange { lat_idx, lon_idx }); + } + + let key = encode_key(classid, lat_idx, lon_idx); + let mut facets = [[0u8; FACET_LEN]; W1_FACET_COUNT]; + for (facet_idx, facet_bytes) in facets.iter_mut().enumerate() { + *facet_bytes = pack_facet( + classid, + facet_idx as u8, + manifest, + floors, + |entry| value_of(entry), + )?; + } + + Ok(PackedWeatherCell { + lat_idx, + lon_idx, + key, + facets, + }) +} + +/// Stream one complete global timestep through `sink`, one packed cell at a +/// time, in deterministic latitude-major / longitude-minor order. +/// +/// The function returns the number of cells accepted by the sink. A successful +/// full-grid run therefore returns [`GLOBAL_CELL_COUNT`]. No million-cell +/// staging vector is created. +/// +/// # Errors +/// +/// Stops at the first source/packing error or sink error. The caller owns any +/// transactional semantics needed to ensure that a partial sink is not +/// published as a durable timestep. +pub fn bake_timestep( + classid: u32, + manifest: &FieldManifest, + floors: &HashMap, + mut value_of: F, + mut sink: S, +) -> Result> +where + F: FnMut(u16, u16, &ManifestEntry) -> Option, + S: FnMut(PackedWeatherCell) -> Result<(), E>, +{ + if classid == 0 { + return Err(BakeStreamError::Bake(BakeError::ZeroClassId)); + } + + let mut written = 0usize; + for lat_idx in 0..LAT_COUNT { + for lon_idx in 0..LON_COUNT { + let cell = pack_cell( + classid, + lat_idx, + lon_idx, + manifest, + floors, + |entry| value_of(lat_idx, lon_idx, entry), + ) + .map_err(BakeStreamError::Bake)?; + sink(cell).map_err(BakeStreamError::Sink)?; + written += 1; + } + } + Ok(written) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::floor::calibrate; + use crate::manifest::PairByte; + + const HEADER: &str = "facet\tpair\tbyte\tvariable\tlevel_hpa\tunit\tfloor_id"; + const WEATHER_W1_CLASSID: u32 = 0x0401_0009; + + fn manifest(variable0: &str) -> FieldManifest { + FieldManifest::parse(&format!( + "{HEADER}\n\ + 0\t0\tlo\t{variable0}\tsurface\tK\tf0\n\ + 1\t0\tlo\tf1\t850\tK\tf1\n\ + 2\t0\tlo\tf2\t500\tK\tf2\n" + )) + .expect("fixture manifest") + } + + fn floors() -> HashMap { + let sample: Vec = (0..10_000).map(|i| i as f64 / 100.0).collect(); + let floor = calibrate(&sample).expect("fixture floor"); + HashMap::from([ + ("f0".to_string(), floor.clone()), + ("f1".to_string(), floor.clone()), + ("f2".to_string(), floor), + ]) + } + + fn value(entry: &ManifestEntry) -> Option { + match entry.variable.as_str() { + "a" => Some(10.0), + "b" => Some(90.0), + "f1" => Some(20.0), + "f2" => Some(30.0), + _ => None, + } + } + + #[test] + fn zero_classid_is_refused_before_any_row_can_be_published() { + let err = pack_cell(0, 0, 0, &manifest("a"), &floors(), value) + .expect_err("bootstrap classid must be rejected"); + assert_eq!(err, BakeError::ZeroClassId); + } + + #[test] + fn out_of_grid_cell_is_a_release_grade_error_not_a_debug_assert() { + let err = pack_cell( + WEATHER_W1_CLASSID, + LAT_COUNT, + LON_COUNT - 1, + &manifest("a"), + &floors(), + value, + ) + .expect_err("latitude one-past-end must fail"); + assert_eq!( + err, + BakeError::GridIndexOutOfRange { + lat_idx: LAT_COUNT, + lon_idx: LON_COUNT - 1, + } + ); + + let err = pack_cell( + WEATHER_W1_CLASSID, + LAT_COUNT - 1, + LON_COUNT, + &manifest("a"), + &floors(), + value, + ) + .expect_err("longitude one-past-end must fail"); + assert!(matches!(err, BakeError::GridIndexOutOfRange { .. })); + } + + #[test] + fn one_cell_contains_key_and_exactly_three_class_prefixed_facets() { + let cell = pack_cell( + WEATHER_W1_CLASSID, + 720, + 1439, + &manifest("a"), + &floors(), + value, + ) + .expect("last real cell packs"); + + assert_eq!(&cell.key[0..4], &WEATHER_W1_CLASSID.to_le_bytes()); + for facet in &cell.facets { + assert_eq!(&facet[0..4], &WEATHER_W1_CLASSID.to_le_bytes()); + // Only pair 0 low is occupied in the fixture. Everything after it + // is reserved-zero, proving the row image does not invent values. + assert!(facet[5..].iter().all(|b| *b == 0)); + } + assert_eq!(cell.facet_image().len(), 48); + } + + #[test] + fn manifest_mutation_is_load_bearing_end_to_end() { + let a = pack_cell( + WEATHER_W1_CLASSID, + 10, + 20, + &manifest("a"), + &floors(), + value, + ) + .expect("a packs"); + let b = pack_cell( + WEATHER_W1_CLASSID, + 10, + 20, + &manifest("b"), + &floors(), + value, + ) + .expect("b packs"); + + assert_ne!( + a.facet_image(), + b.facet_image(), + "changing the manifest's field identity must change a written byte when the source values differ" + ); + // The changed byte is exactly the slot the manifest resolves, not a + // label or metadata byte smuggled into the payload. + assert_eq!(manifest("a").resolve("a", None), Some((0, 0, PairByte::Lo))); + } + + #[test] + fn sink_failure_stops_the_stream_without_building_a_global_vec() { + #[derive(Debug, PartialEq, Eq)] + struct Stop; + + let mut seen = 0usize; + let mut value_calls = 0usize; + let err = bake_timestep( + WEATHER_W1_CLASSID, + &manifest("a"), + &floors(), + |_lat, _lon, entry| { + value_calls += 1; + value(entry) + }, + |_cell| { + seen += 1; + if seen == 3 { + Err(Stop) + } else { + Ok(()) + } + }, + ) + .expect_err("sink deliberately stops at cell 3"); + + assert_eq!(seen, 3); + assert_eq!(value_calls, 9); + match err { + BakeStreamError::Sink(Stop) => {} + other => panic!("expected sink stop, got {other:?}"), + } + } +} diff --git a/crates/weather-poc/src/canonical.rs b/crates/weather-poc/src/canonical.rs new file mode 100644 index 000000000..fd0106593 --- /dev/null +++ b/crates/weather-poc/src/canonical.rs @@ -0,0 +1,320 @@ +//! Opt-in bridge from the zero-dependency W1 image to the live canonical +//! `lance-graph-contract::canonical_node::NodeRow`. +//! +//! The important property is what is *not* a constant here: the weather +//! extension offset. It is derived from the final descriptor in +//! `VALUE_TENANTS`, so append-only tenants added ahead of weather move this +//! extension automatically without a second source of truth. + +use std::collections::HashMap; +use std::fmt; +use std::io::{self, Write}; + +use lance_graph_contract::canonical_node::{ + EdgeBlock, NodeGuid, NodeRow, ValueSchema, NODE_ROW_STRIDE, VALUE_SLAB_LEN, + VALUE_SLAB_ROW_OFFSET, VALUE_TENANTS, +}; + +use crate::bake::{bake_timestep, BakeError, BakeStreamError, PackedWeatherCell, W1_IMAGE_LEN}; +use crate::floor::CalibratedFloor; +use crate::manifest::{FieldManifest, ManifestEntry}; + +/// Offset of the first currently unassigned byte **within `NodeRow::value`**. +/// +/// Derived from the live append-only tenant table. Weather never owns a copied +/// absolute offset. +pub fn weather_value_offset() -> usize { + let last = VALUE_TENANTS + .last() + .expect("canonical NodeRow must expose at least one value tenant"); + last.row_offset as usize + last.col_bytes_per_row() - VALUE_SLAB_ROW_OFFSET +} + +/// Bytes still available in the canonical value slab after all named tenants. +pub fn weather_tail_capacity() -> usize { + VALUE_SLAB_LEN - weather_value_offset() +} + +/// A canonical-row assembly failure. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum CanonicalRowError { + /// The live contract has grown until W1's three facets no longer fit. + InsufficientTail { + /// W1 bytes required. + needed: usize, + /// Live free-tail capacity. + available: usize, + }, + /// `weather-poc::key` and the real `NodeGuid` no longer agree byte-for-byte. + KeyContractDrift, + /// A facet's classid prefix disagrees with the cell key's classid. + FacetClassIdMismatch { + /// Facet index 0..2. + facet: usize, + /// Classid carried by the cell key. + key_classid: u32, + /// Classid carried by the facet. + facet_classid: u32, + }, +} + +impl fmt::Display for CanonicalRowError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + CanonicalRowError::InsufficientTail { needed, available } => write!( + f, + "W1 weather image needs {needed} value bytes but the live NodeRow free tail has {available}" + ), + CanonicalRowError::KeyContractDrift => f.write_str( + "weather-poc key bytes no longer agree with lance-graph-contract::NodeGuid", + ), + CanonicalRowError::FacetClassIdMismatch { + facet, + key_classid, + facet_classid, + } => write!( + f, + "weather facet {facet} classid 0x{facet_classid:08x} disagrees with key classid 0x{key_classid:08x}" + ), + } + } +} + +impl std::error::Error for CanonicalRowError {} + +/// Error while streaming a whole timestep directly to a byte writer. +#[derive(Debug)] +pub enum CanonicalBakeError { + /// Source/lane assembly failed. + Bake(BakeError), + /// Live canonical-row agreement or capacity failed. + Canonical(CanonicalRowError), + /// The output writer failed. + Io(io::Error), +} + +impl fmt::Display for CanonicalBakeError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + CanonicalBakeError::Bake(err) => err.fmt(f), + CanonicalBakeError::Canonical(err) => err.fmt(f), + CanonicalBakeError::Io(err) => write!(f, "weather slab write failed: {err}"), + } + } +} + +impl std::error::Error for CanonicalBakeError {} + +fn key_as_node_guid(bytes: &[u8; 16]) -> Result { + let classid = u32::from_le_bytes(bytes[0..4].try_into().expect("4-byte classid")); + let heel = u16::from_le_bytes([bytes[4], bytes[5]]); + let hip = u16::from_le_bytes([bytes[6], bytes[7]]); + let twig = u16::from_le_bytes([bytes[8], bytes[9]]); + let family = u32::from_le_bytes([bytes[10], bytes[11], bytes[12], 0]); + let identity = u32::from_le_bytes([bytes[13], bytes[14], bytes[15], 0]); + let guid = NodeGuid::new(classid, heel, hip, twig, family, identity); + if guid.as_bytes() != bytes { + return Err(CanonicalRowError::KeyContractDrift); + } + Ok(guid) +} + +/// Assemble one [`PackedWeatherCell`] into the current canonical 512-byte row. +/// +/// Existing value tenants are left zero. W1's three facets are placed only in +/// the currently free append-only tail, whose offset is derived at runtime from +/// the canonical descriptor table. +pub fn assemble_row(cell: &PackedWeatherCell) -> Result { + let offset = weather_value_offset(); + let available = weather_tail_capacity(); + if W1_IMAGE_LEN > available { + return Err(CanonicalRowError::InsufficientTail { + needed: W1_IMAGE_LEN, + available, + }); + } + + let key = key_as_node_guid(&cell.key)?; + let key_classid = key.classid(); + for (facet, bytes) in cell.facets.iter().enumerate() { + let facet_classid = u32::from_le_bytes(bytes[0..4].try_into().expect("4-byte classid")); + if facet_classid != key_classid { + return Err(CanonicalRowError::FacetClassIdMismatch { + facet, + key_classid, + facet_classid, + }); + } + } + + let mut value = [0u8; VALUE_SLAB_LEN]; + let image = cell.facet_image(); + value[offset..offset + W1_IMAGE_LEN].copy_from_slice(&image); + + Ok(NodeRow { + key, + edges: EdgeBlock::default(), + value, + }) +} + +/// Serialize one canonical row without an unsafe cast. +/// +/// This makes the copy boundary explicit: key, edge block and value slab are +/// copied once into the 512-byte publication image. A later Arrow zero-copy +/// path may replace this only after measurement. +pub fn row_bytes(row: &NodeRow) -> [u8; NODE_ROW_STRIDE] { + let mut out = [0u8; NODE_ROW_STRIDE]; + out[0..16].copy_from_slice(row.key.as_bytes()); + out[16..28].copy_from_slice(&row.edges.in_family); + out[28..32].copy_from_slice(&row.edges.out_family); + out[VALUE_SLAB_ROW_OFFSET..].copy_from_slice(&row.value); + out +} + +/// Stream a complete W1 timestep as canonical 512-byte rows into `writer`. +/// +/// The function creates no global staging vector. It is the production-neutral +/// seam consumed by the existing Lance import/publication path: publication +/// must still be atomic at the dataset/version layer, so callers should write +/// to a temporary/unpublished target and only publish after this returns +/// successfully. +pub fn bake_timestep_to_writer( + classid: u32, + manifest: &FieldManifest, + floors: &HashMap, + mut value_of: F, + writer: &mut W, +) -> Result +where + W: Write, + F: FnMut(u16, u16, &ManifestEntry) -> Option, +{ + enum SinkError { + Canonical(CanonicalRowError), + Io(io::Error), + } + + let result = bake_timestep( + classid, + manifest, + floors, + |lat, lon, entry| value_of(lat, lon, entry), + |cell| { + let row = assemble_row(&cell).map_err(SinkError::Canonical)?; + writer + .write_all(&row_bytes(&row)) + .map_err(SinkError::Io) + }, + ); + + match result { + Ok(rows) => Ok(rows), + Err(BakeStreamError::Bake(err)) => Err(CanonicalBakeError::Bake(err)), + Err(BakeStreamError::Sink(SinkError::Canonical(err))) => { + Err(CanonicalBakeError::Canonical(err)) + } + Err(BakeStreamError::Sink(SinkError::Io(err))) => Err(CanonicalBakeError::Io(err)), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::bake::pack_cell; + use crate::floor::calibrate; + + const HEADER: &str = "facet\tpair\tbyte\tvariable\tlevel_hpa\tunit\tfloor_id"; + const WEATHER_W1_CLASSID: u32 = 0x0401_0009; + + fn fixture_manifest() -> FieldManifest { + FieldManifest::parse(&format!( + "{HEADER}\n\ + 0\t0\tlo\ta\tsurface\tK\tf\n\ + 1\t0\tlo\tb\t850\tK\tf\n\ + 2\t0\tlo\tc\t500\tK\tf\n" + )) + .expect("fixture manifest") + } + + fn fixture_floors() -> HashMap { + let sample: Vec = (0..10_000).map(|i| i as f64 / 100.0).collect(); + HashMap::from([( + "f".to_string(), + calibrate(&sample).expect("fixture floor"), + )]) + } + + #[test] + fn live_contract_places_weather_after_every_named_tenant() { + let manifest = fixture_manifest(); + let floors = fixture_floors(); + let cell = pack_cell(WEATHER_W1_CLASSID, 720, 1439, &manifest, &floors, |entry| { + match entry.variable.as_str() { + "a" => Some(10.0), + "b" => Some(20.0), + "c" => Some(30.0), + _ => None, + } + }) + .expect("cell packs"); + + let row = assemble_row(&cell).expect("canonical row assembles"); + let expected_offset = ValueSchema::Full.tenant_bytes(); + let image = cell.facet_image(); + + assert_eq!(weather_value_offset(), expected_offset); + assert_eq!(row.key.as_bytes(), &cell.key); + assert!(row.edges.in_family.iter().all(|b| *b == 0)); + assert!(row.edges.out_family.iter().all(|b| *b == 0)); + assert!(row.value[..expected_offset].iter().all(|b| *b == 0)); + assert_eq!( + &row.value[expected_offset..expected_offset + W1_IMAGE_LEN], + &image + ); + assert!( + row.value[expected_offset + W1_IMAGE_LEN..] + .iter() + .all(|b| *b == 0) + ); + assert!(VALUE_SLAB_LEN - expected_offset >= W1_IMAGE_LEN); + } + + #[test] + fn canonical_serialization_is_exactly_512_bytes_and_key_agrees() { + let manifest = fixture_manifest(); + let floors = fixture_floors(); + let cell = pack_cell(WEATHER_W1_CLASSID, 123, 456, &manifest, &floors, |_| Some(42.0)) + .expect("cell packs"); + let mut row = assemble_row(&cell).expect("row"); + row.edges.in_family[0] = 0xA5; + row.edges.out_family[3] = 0x5A; + let bytes = row_bytes(&row); + + let expected_offset = ValueSchema::Full.tenant_bytes(); + let image = cell.facet_image(); + let mut expected = [0u8; NODE_ROW_STRIDE]; + expected[0..16].copy_from_slice(&cell.key); + expected[16] = 0xA5; + expected[31] = 0x5A; + let weather_start = VALUE_SLAB_ROW_OFFSET + expected_offset; + expected[weather_start..weather_start + W1_IMAGE_LEN].copy_from_slice(&image); + + assert_eq!(bytes.len(), NODE_ROW_STRIDE); + assert_eq!(bytes, expected); + } + + #[test] + fn facet_classid_tamper_is_rejected() { + let manifest = fixture_manifest(); + let floors = fixture_floors(); + let mut cell = pack_cell(WEATHER_W1_CLASSID, 1, 2, &manifest, &floors, |_| Some(42.0)) + .expect("cell packs"); + cell.facets[1][0..4].copy_from_slice(&0xDEAD_BEEFu32.to_le_bytes()); + + assert!(matches!( + assemble_row(&cell), + Err(CanonicalRowError::FacetClassIdMismatch { facet: 1, .. }) + )); + } +} diff --git a/crates/weather-poc/src/lib.rs b/crates/weather-poc/src/lib.rs index 42546f38f..6804005e8 100644 --- a/crates/weather-poc/src/lib.rs +++ b/crates/weather-poc/src/lib.rs @@ -1,41 +1,42 @@ -//! Zarr → `NodeRow`: the ERA5 0.25° global grid on the canonical SoA row. +//! Zarr -> `NodeRow`: the ERA5 0.25-degree global grid on the canonical SoA row. //! //! **Plan:** `.claude/plans/weather-soa-bake-v1.md`. This crate exists because -//! the weather R&D arc ran entirely as Python over a Zarr file — one variable, -//! four hand-picked 16° boxes, three hand-typed timesteps — and the reason was -//! structural: the Zarr→`NodeRow` path did not exist and no plan specified one. +//! the weather R&D arc ran entirely as Python over a Zarr file and the +//! Zarr-to-row path did not exist. //! //! # The shape, in four facts //! -//! * **One cell is one node.** The ERA5 0.25° grid is 721 × 1440 = +//! * **One cell is one node.** The ERA5 0.25-degree grid is 721 x 1440 = //! **1,038,240 cells**; the canonical row is 512 bytes; so a full global -//! timestep is **0.495 GiB**. All 122 ERA5 fields per cell (17 surface + 7 -//! pressure-level × 13 levels + 14 static) fit at 1 byte/field inside the -//! **292 free slab bytes** — see [`crate::key`] and the plan §2.1 for why the -//! budget is 292 and not 384. -//! * **The key is the address.** `HEEL` = the 16° × 16° tile, `HIP` = position -//! within it, `TWIG` dormant-reserved. A lat/lon grid is the literal-x/y case -//! of the 3×4 cascade, so the arc's hand-picked 16° boxes become **HEEL-prefix -//! range scans** rather than array slicing. -//! * **One timestep is one Lance version.** Per `E-MARKOV-TEMPORAL-STREAM-1`, -//! episodic = Lance versions; a time series is a version-range read. This -//! crate never writes a version writer of its own. -//! * **Nothing in the payload says what a byte means.** The (facet, pair, byte) -//! → (variable, level, unit, floor) mapping is a ClassView-side manifest -//! ([`crate::manifest`]), never a slot in the row — the le-contract §2 -//! slot-purity rule. +//! timestep is **0.495 GiB**. The full 122-field capacity is a proven budget, +//! but W1 deliberately commits only the 22 measured/needed fields across +//! three L4 facets. +//! * **The key is the address.** `HEEL` = the 16-degree tile, `HIP` = position +//! within it, `TWIG` dormant-reserved. The arc's hand-picked boxes become +//! HEEL-prefix range scans rather than array slicing. +//! * **One timestep is one Lance version.** This standalone crate assembles +//! the key + W1 facet image. With feature `canonical-row`, `canonical` +//! places that image into the live `NodeRow` contract and can stream exact +//! 512-byte rows to the existing Lance publication path. No copied value-tail +//! offset and no second dataset-version protocol are introduced. +//! * **Nothing in the payload says what a byte means.** The +//! `(facet, pair, byte) -> (variable, level, unit, floor)` mapping is a +//! ClassView-side manifest ([`crate::manifest`]), never a slot in the row. //! //! # Status //! -//! Wave W1 is under construction. `D-WXS-0` (the classid mint) is **blocked** -//! on an OGAR-side, operator-gated decision, and until it resolves the bake -//! must **refuse to write** rather than emit rows under `0x0000_0000` — that -//! value belongs to the zero-fallback ladder and a dataset carrying it is -//! indistinguishable from a bootstrap row. +//! W1 codec pieces and the streaming cell-image assembler are present. +//! Durable publication still requires a non-zero routable weather classid; +//! `0x0000_0000` is refused because it belongs to the canonical bootstrap +//! fallback ladder. #![forbid(unsafe_code)] #![warn(missing_docs)] +pub mod bake; +/// Agreement bridge to the live 512-byte `NodeRow` contract. +#[cfg(feature = "canonical-row")] +pub mod canonical; pub mod floor; pub mod key; pub mod lane;