diff --git a/crates/ogar-obo/examples/rekey_domain.rs b/crates/ogar-obo/examples/rekey_domain.rs new file mode 100644 index 00000000..257de351 --- /dev/null +++ b/crates/ogar-obo/examples/rekey_domain.rs @@ -0,0 +1,148 @@ +//! **S3 artifact re-key** — carry an existing OBO-core `.soa` from the legacy +//! `0x03` addresses onto the domain-reference tree, WITHOUT re-baking. +//! +//! ```text +//! cargo run -p ogar-obo --example rekey_domain -- +//! ``` +//! +//! # Why a transform and not a re-bake +//! +//! The staging migration's whole premise is that the alias is a computation: +//! only the concept's HIGH byte moves (`disease::mondo 0x0301 -> 0x9101`), +//! everything else — identity rail, edge degrees, entity type, HHTL tiers, +//! row ORDER — is byte-identical. A re-bake from the `.obo` sources would +//! reproduce all of that only if every source and every filter decision of +//! the original bake were reproduced too; the transform instead touches +//! exactly the bytes the flip defines and PROVES the rest unchanged. +//! +//! **Row order is deliberately preserved.** The label slab +//! (`obo_labels.slab`) is positional — same order in, same order out, so the +//! existing slab stays valid against the re-keyed artifact. +//! +//! # What is rewritten +//! +//! - the key classid (bytes `0..4`, u32 LE) of every row; +//! - every edge-lane HEADER classid (`layout::OBO_CORE_ROW.edge_lanes`: +//! 23 lanes x 16 B from row offset 144; each lane `classid(4) + 4xu24`). +//! A lane names its TARGET ontology, so lane headers carry the same five +//! classids as the keys and move by the same map. +//! +//! Everything else is copied verbatim, and the tool verifies that claim on +//! its own output (a byte-diff accounting over the non-classid positions) +//! rather than asserting it. + +use ogar_obo::{NODE_ROW_STRIDE, Namespace, layout}; + +/// The flip, derived from the enum — never a table of literals: resolve the +/// namespace from whatever form the artifact carries (the reader accepts +/// both), re-render under the SAME app prefix. PATO/RO map to themselves. +fn map_classid(classid: u32) -> Option { + let ns = Namespace::from_concept_id((classid >> 16) as u16)?; + Some(ns.render_classid((classid & 0xFFFF) as u16)) +} + +fn main() { + let mut args = std::env::args().skip(1); + let inp = args.next().expect("usage: rekey_domain "); + let out = args.next().expect("usage: rekey_domain "); + + let data = std::fs::read(&inp).unwrap_or_else(|e| panic!("read {inp}: {e}")); + assert!( + data.len().is_multiple_of(NODE_ROW_STRIDE), + "{inp} is not a NodeRow artifact ({} bytes)", + data.len() + ); + let n = data.len() / NODE_ROW_STRIDE; + + let lanes = layout::OBO_CORE_ROW.edge_lanes; + let lane0 = lanes.row_off(); + let lane_count = layout::OBO_CORE_ROW.lane_count; + + let mut rewritten = data.clone(); + let mut keys_moved = 0usize; + let mut keys_kept = 0usize; + let mut lane_headers_moved = 0usize; + let mut lane_headers_kept = 0usize; + let mut unknown_keys = 0usize; + let mut before = std::collections::BTreeMap::::new(); + let mut after = std::collections::BTreeMap::::new(); + + for i in 0..n { + let row = i * NODE_ROW_STRIDE; + let key_cid = u32::from_le_bytes(rewritten[row..row + 4].try_into().unwrap()); + *before.entry(key_cid).or_default() += 1; + match map_classid(key_cid) { + Some(new) => { + if new != key_cid { + keys_moved += 1; + } else { + keys_kept += 1; + } + rewritten[row..row + 4].copy_from_slice(&new.to_le_bytes()); + *after.entry(new).or_default() += 1; + } + None => { + // Not an OBO classid — refuse to guess; counted, kept. + unknown_keys += 1; + *after.entry(key_cid).or_default() += 1; + } + } + for l in 0..lane_count { + let off = row + lane0 + l * 16; + let cid = u32::from_le_bytes(rewritten[off..off + 4].try_into().unwrap()); + if cid == 0 { + continue; // unused lane + } + if let Some(new) = map_classid(cid) { + if new != cid { + lane_headers_moved += 1; + } else { + lane_headers_kept += 1; + } + rewritten[off..off + 4].copy_from_slice(&new.to_le_bytes()); + } + } + } + + assert_eq!( + unknown_keys, 0, + "an OBO-core artifact must carry only OBO keys" + ); + + // ── verify on the OUTPUT, not on intent ───────────────────────────────── + // Every byte outside the rewritten classid positions must be identical. + // Exempt positions within a row: the key classid (bytes 0..4) and each + // edge-lane header classid (first 4 bytes of every 16-byte lane record). + let mut foreign_diffs = 0usize; + for i in 0..n { + let row = i * NODE_ROW_STRIDE; + for b in 0..NODE_ROW_STRIDE { + let is_key_cid = b < 4; + let in_lane_header = match b.checked_sub(lane0) { + Some(rel) => rel < lane_count * 16 && rel % 16 < 4, + None => false, + }; + if !is_key_cid && !in_lane_header && data[row + b] != rewritten[row + b] { + foreign_diffs += 1; + } + } + } + assert_eq!( + foreign_diffs, 0, + "the transform touched bytes outside the classid positions" + ); + + std::fs::write(&out, &rewritten).unwrap_or_else(|e| panic!("write {out}: {e}")); + + println!("rekey_domain: {n} rows, order preserved"); + println!(" keys moved {keys_moved} / kept (PATO/RO) {keys_kept}"); + println!(" lane headers moved {lane_headers_moved} / kept {lane_headers_kept}"); + println!(" classid census before -> after:"); + for (cid, cnt) in &before { + println!(" {cid:#010x} {cnt}"); + } + println!(" --"); + for (cid, cnt) in &after { + println!(" {cid:#010x} {cnt}"); + } +} diff --git a/crates/ogar-obo/src/edges.rs b/crates/ogar-obo/src/edges.rs index 66f727e3..542d05e4 100644 --- a/crates/ogar-obo/src/edges.rs +++ b/crates/ogar-obo/src/edges.rs @@ -431,7 +431,7 @@ mod tests { vec![( Predicate::IsA, Link { - classid: 0x0301_0000, + classid: target_classid(Namespace::Mondo, 0x0000), num: 5015 } )] @@ -456,11 +456,11 @@ mod tests { got, vec![ Link { - classid: 0x0301_0000, + classid: target_classid(Namespace::Mondo, 0x0000), num: 1 }, Link { - classid: 0x0302_0000, + classid: target_classid(Namespace::Hpo, 0x0000), num: 2 } ] @@ -547,7 +547,10 @@ mod tests { // to a reasoner without role composition derives false ancestors, so // the projection must yield ONLY the is_a edge. let mut row = row_with_degrees(&[(Predicate::IsA, 1), (Predicate::PartOf, 1)]); - row[0..16].copy_from_slice(&crate::pack_key(0x0303_0000, 2244)); + row[0..16].copy_from_slice(&crate::pack_key( + target_classid(Namespace::Uberon, 0x0000), + 2244, + )); let mut tg = vec![ (Predicate::IsA, t(Namespace::Uberon, 10)), (Predicate::PartOf, t(Namespace::Uberon, 20)), @@ -563,11 +566,11 @@ mod tests { got, vec![( Link { - classid: 0x0303_0000, + classid: target_classid(Namespace::Uberon, 0x0000), num: 2244 }, Link { - classid: 0x0303_0000, + classid: target_classid(Namespace::Uberon, 0x0000), num: 10 } )], @@ -633,7 +636,10 @@ mod tests { .collect(); assert_eq!( got, - vec![(0x0301_0000, 5015), (0x0303_0000, 955)], + vec![ + (target_classid(Namespace::Mondo, 0x0000), 5015), + (target_classid(Namespace::Uberon, 0x0000), 955), + ], "the three empty slots after the first link must advance the lane, \ not yield phantom links" ); @@ -696,11 +702,12 @@ mod tests { // an oversize CURIE (> u16) must survive: the numeric rides the rail, // high half in `family`. A truncating decode would return 4556 here. let mut row = [0u8; NODE_ROW_STRIDE]; - row[0..16].copy_from_slice(&crate::pack_key(0x0301_0000, 700_092)); + let mondo = target_classid(Namespace::Mondo, 0x0000); + row[0..16].copy_from_slice(&crate::pack_key(mondo, 700_092)); assert_eq!( subject(&row), Link { - classid: 0x0301_0000, + classid: mondo, num: 700_092 } ); diff --git a/crates/ogar-obo/src/lib.rs b/crates/ogar-obo/src/lib.rs index 9e26ce3f..58aa4bce 100644 --- a/crates/ogar-obo/src/lib.rs +++ b/crates/ogar-obo/src/lib.rs @@ -71,20 +71,44 @@ pub const VALUE_OFFSET: usize = 32; pub const ENTITY_TYPE_SLAB_OFFSET: usize = layout::OBO_CORE_ROW.entity_type.slab_off; /// The five OBO-core namespaces this bake carries. +/// +/// # S3 of the staging migration (2026-08-31): the writer emits the DOMAIN tree +/// +/// Since the `0x90..0x9A` domain-reference-tree mint (#292), the retired `0x03` +/// Ontology block is a TERMINAL address space: writers emit the domain form +/// (`disease::mondo = 0x9101`, HIGH byte = compartment, LOW byte = the +/// vocabulary's old `0x03` lo byte carried 1:1), and the legacy `0x03NN` +/// concept stays READABLE ([`Namespace::from_concept_id`] accepts both forms). +/// PATO and RO deliberately do NOT move: PATO's home is the fusion-or-facet +/// question (it is a quality axis, not a domain) and RO is the predicate +/// namespace — neither has a row in the domain table, so both keep their +/// legacy concept until that is decided. +/// +/// **The variant ORDER is a wire contract and never re-sorts:** `ns as u8` +/// (0 MONDO · 1 HP · 2 UBERON · 3 PATO · 4 RO) is the category byte baked +/// into `value[96]` of every row and the `TermId.ns` ordinal. The ids below +/// are therefore no longer monotone in enum order — that property belonged +/// to the retired block, not to the enum. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum Namespace { - /// MONDO — disease (`0x0301`, the `0x03` OBO Ontology domain). + /// MONDO — disease (`0x9101` = `disease::mondo`; legacy `0x0301` readable). Mondo, - /// HPO — human phenotype / clinical symptom (`0x0302`). + /// HPO — human phenotype / clinical symptom (`0x9202` = + /// `phenomenology::hpo`; legacy `0x0302` readable). Hpo, - /// Uberon — anatomy spine (`0x0303`). Cross-references FMA (`ogar-fma`, - /// `0x0A` Anatomy) by edge, not by shared domain byte. + /// Uberon — anatomy spine (`0x9303` = `anatomy::uberon`; legacy `0x0303` + /// readable). Cross-references FMA (`ogar-fma`, `0x0A` Anatomy) by edge, + /// not by shared domain byte; `0x93` is the anatomy ALIAS block — the + /// upstream authority stays `0x0A`, which is why no `ConceptDomain::0x93` + /// variant exists (a deliberate hole, pinned by ogar-vocab's own test). Uberon, - /// PATO — phenotypic quality (`0x0304`). + /// PATO — phenotypic quality (`0x0304`, deliberately NOT migrated — see + /// the enum doc). Pato, - /// RO — relations ontology (predicate classes; `0x0305`). Predicates ride - /// the [`Predicate`] byte palette on edges, not as node classids, but RO - /// term nodes are still baked for completeness. + /// RO — relations ontology (predicate classes; `0x0305`, deliberately NOT + /// migrated — see the enum doc). Predicates ride the [`Predicate`] byte + /// palette on edges, not as node classids, but RO term nodes are still + /// baked for completeness. Ro, } @@ -110,17 +134,17 @@ impl Namespace { /// /// # Why this still exists next to the shared codebook /// - /// Since the 2026-08-22 mint, `ogar_vocab::CODEBOOK` carries all 14 - /// Ontology rows (`0x0301..=0x0306`, `0x0340..=0x0347`), so - /// `concepts_in_domain(ConceptDomain::Ontology)` now returns the FULL - /// domain — that is the discovery surface for "everything ontological". - /// (An earlier revision of this doc said the domain carried zero shared - /// rows by design; the mint reversed that ruling, and the sentence was - /// corrected rather than left to steer consumers into assembling the - /// three producer lists by hand.) This array remains the TYPED - /// five-namespace subset — the `OBO_CORE` band as `Namespace` variants, - /// for callers that want the enum (CURIE prefixes, per-namespace - /// dispatch), not a `(name, id)` row scan. + /// The 2026-08-22 mint put 14 Ontology rows into `ogar_vocab::CODEBOOK`; + /// #290 REVERSED that (operator ruling: plug-and-play — OBO concepts live + /// in their producer, never in the shared codebook), so + /// `concepts_in_domain(ConceptDomain::Ontology)` is EMPTY again and this + /// array is once more the discovery surface for the core five. (This + /// paragraph has now flipped twice; the codebook state, not this prose, + /// is authoritative — measured 2026-08-31: zero `0x03xx` and zero + /// `0x034x` shared rows.) This array remains the TYPED five-namespace + /// subset — the `OBO_CORE` band as `Namespace` variants, for callers + /// that want the enum (CURIE prefixes, per-namespace dispatch), not a + /// `(name, id)` row scan. /// /// Without an enumeration a consumer has three bad options: repeat the list /// locally (the re-implementation [`from_concept_id`](Self::from_concept_id) @@ -181,9 +205,9 @@ impl Namespace { #[must_use] pub const fn concept_id(self) -> u16 { match self { - Namespace::Mondo => 0x0301, - Namespace::Hpo => 0x0302, - Namespace::Uberon => 0x0303, + Namespace::Mondo => 0x9101, + Namespace::Hpo => 0x9202, + Namespace::Uberon => 0x9303, Namespace::Pato => 0x0304, Namespace::Ro => 0x0305, } @@ -208,9 +232,12 @@ impl Namespace { #[must_use] pub const fn from_concept_id(concept: u16) -> Option { Some(match concept { - 0x0301 => Namespace::Mondo, - 0x0302 => Namespace::Hpo, - 0x0303 => Namespace::Uberon, + // Domain form (the writer's form since S3) and the legacy `0x03` + // shadow, both readable — "legacy nur noch lesend". Every baked + // artifact minted before the flip carries the legacy ids. + 0x9101 | 0x0301 => Namespace::Mondo, + 0x9202 | 0x0302 => Namespace::Hpo, + 0x9303 | 0x0303 => Namespace::Uberon, 0x0304 => Namespace::Pato, 0x0305 => Namespace::Ro, _ => return None, @@ -985,18 +1012,24 @@ mod tests { let swept: Vec = (0..=u16::MAX) .filter(|c| super::Namespace::from_concept_id(*c).is_some()) .collect(); - let listed: Vec = super::Namespace::ALL + // RE-PINNED for S3 (2026-08-31): the reader accepts BOTH address + // forms, so the swept set is ALL's ids UNION the legacy shadows of + // the three migrated namespaces — 8 ids for 5 namespaces, each + // shadow resolving to the SAME namespace as its domain form. + let mut expected: Vec = super::Namespace::ALL .iter() .map(|n| n.concept_id()) + .chain([0x0301u16, 0x0302, 0x0303]) .collect(); + expected.sort_unstable(); assert_eq!( - swept, listed, - "ALL and from_concept_id disagree — a namespace was added to one and not the other" + swept, expected, + "from_concept_id must accept exactly ALL's ids plus the three legacy shadows — a namespace was added to one side and not the other" ); - // Anti-vacuity: the sweep really is selective (5 of 65_536), so the + // Anti-vacuity: the sweep really is selective (8 of 65_536), so the // equality above is not two empty sets agreeing. - assert_eq!(swept.len(), super::Namespace::ALL.len()); + assert_eq!(swept.len(), super::Namespace::ALL.len() + 3); assert!(!swept.is_empty(), "the sweep found nothing at all"); assert!( swept.len() < 16, @@ -1004,21 +1037,31 @@ mod tests { swept.len() ); - // Ordered by concept id, as the doc promises — a caller iterating ALL - // walks the block in address order. - assert!( - listed.windows(2).all(|w| w[0] < w[1]), - "ALL is not in ascending concept-id order: {listed:?}" - ); + // The legacy shadow resolves to the SAME namespace as the domain form + // — an alias, never a second meaning. + for (legacy, ns) in [ + (0x0301u16, super::Namespace::Mondo), + (0x0302, super::Namespace::Hpo), + (0x0303, super::Namespace::Uberon), + ] { + assert_eq!(super::Namespace::from_concept_id(legacy), Some(ns)); + } - // Every entry round-trips, and every entry is in the 0x03 block. - for n in super::Namespace::ALL { - assert_eq!(super::Namespace::from_concept_id(n.concept_id()), Some(n)); - assert_eq!( - n.concept_id() >> 8, - 0x03, - "{n:?} is outside the Ontology domain" - ); + // Every entry round-trips, carries its expected address, and the + // enum ORDER stays the wire ordinal (the value[96] category byte) — + // the old "ascending concept-id order" claim belonged to the retired + // 0x03 block and is deliberately NOT re-asserted. + let expected_ids = [ + (super::Namespace::Mondo, 0x9101u16), + (super::Namespace::Hpo, 0x9202), + (super::Namespace::Uberon, 0x9303), + (super::Namespace::Pato, 0x0304), + (super::Namespace::Ro, 0x0305), + ]; + for (i, (n, id)) in expected_ids.into_iter().enumerate() { + assert_eq!(n.concept_id(), id); + assert_eq!(super::Namespace::from_concept_id(id), Some(n)); + assert_eq!(n as usize, i, "the wire ordinal must never re-sort"); assert!(!n.prefix().is_empty()); } } @@ -1098,9 +1141,12 @@ mod tests { let t = TermId::parse("MONDO:0007739").unwrap(); assert_eq!(t.namespace(), Namespace::Mondo); assert_eq!(t.num, 7739); - // canon-high: (0x0301 << 16) | app_prefix ; all five in the 0x03 domain - assert_eq!(Namespace::Mondo.render_classid(0x0000), 0x0301_0000); - assert_eq!(Namespace::Uberon.render_classid(0x00AB), 0x0303_00AB); + // canon-high: (concept << 16) | app_prefix — since the S3 writer flip + // the migrated three carry the DOMAIN form (disease::mondo 0x9101, + // anatomy::uberon 0x9303); the legacy shadows stay readable, which + // the sweep test above pins. + assert_eq!(Namespace::Mondo.render_classid(0x0000), 0x9101_0000); + assert_eq!(Namespace::Uberon.render_classid(0x00AB), 0x9303_00AB); // out-of-scope / malformed assert!(TermId::parse("CHEBI:12345").is_none()); assert!(TermId::parse("MONDO:99999999").is_none()); // > 24-bit diff --git a/crates/ogar-obo/src/registry.rs b/crates/ogar-obo/src/registry.rs index f90b8935..a4f6a9eb 100644 --- a/crates/ogar-obo/src/registry.rs +++ b/crates/ogar-obo/src/registry.rs @@ -139,15 +139,15 @@ impl NsRegistry { pub const OBO_CORE: NsRegistry = NsRegistry::new(&[ NsSpec { prefix: "MONDO", - concept_id: 0x0301, + concept_id: 0x9101, }, NsSpec { prefix: "HP", - concept_id: 0x0302, + concept_id: 0x9202, }, NsSpec { prefix: "UBERON", - concept_id: 0x0303, + concept_id: 0x9303, }, NsSpec { prefix: "PATO", @@ -399,10 +399,17 @@ mod tests { s.concept_id ); } + // Since the S3 writer flip the migrated core ids live in the 0x9X + // domain tree, so "below the spine band" is no longer the core's + // shape. The invariant that MATTERS is unchanged: no core id may sit + // inside the spine block itself (0x0340..=0x0347 + headroom to the + // end of the 0x03 page), and the un-migrated pair (PATO/RO) still + // sits below it. for c in OBO_CORE.specs() { + let in_legacy_page = c.concept_id >> 8 == 0x03; assert!( - c.concept_id < SPINE_BAND_START, - "{} at {:#06X} escaped the reserved core band", + !in_legacy_page || c.concept_id < SPINE_BAND_START, + "{} at {:#06X} collides with the reserved spine band", c.prefix, c.concept_id ); @@ -424,13 +431,16 @@ mod tests { ); } - // Every id stays inside the 0x03 Ontology domain — a spine that ran - // off the end of the block would silently land in another domain. + // Every id stays inside a domain THIS table may address — the spine + // and the un-migrated pair in the legacy 0x03 Ontology page, the + // migrated three in the 0x90..0x9D domain-reference tree (S3 writer + // flip). An id outside both would silently land in a foreign domain, + // which is exactly what this guard exists to catch. for s in OBO_CORE.specs().iter().chain(META_STUDY_SPINE.specs()) { - assert_eq!( - s.concept_id >> 8, - 0x03, - "{} at {:#06X} left the 0x03 Ontology domain", + let hi = (s.concept_id >> 8) as u8; + assert!( + hi == 0x03 || (0x90..=0x9D).contains(&hi), + "{} at {:#06X} sits in a domain this crate does not own", s.prefix, s.concept_id ); diff --git a/crates/ogar-obo/src/spine.rs b/crates/ogar-obo/src/spine.rs index 597cffdf..1ace6cb0 100644 --- a/crates/ogar-obo/src/spine.rs +++ b/crates/ogar-obo/src/spine.rs @@ -217,7 +217,7 @@ mod tests { use super::*; use crate::{Namespace, TermId, edges::pack_edges}; - const MONDO: u32 = 0x0301_0000; + const MONDO: u32 = crate::Namespace::Mondo.render_classid(0x0000); fn row(num: u32, parents: &[u32]) -> Row512 { let mut r = Row512::zeroed(); @@ -308,7 +308,11 @@ mod tests { lens.fillers_of(MONDO, 1, Predicate::HasPhenotype, &mut |c, n| { got.push((c, n)) }); - assert_eq!(got, vec![(0x0302_0000, 42)], "crosses into HPO"); + assert_eq!( + got, + vec![(crate::Namespace::Hpo.render_classid(0x0000), 42)], + "crosses into HPO" + ); // can-stay-silent: a role with no asserted filler yields nothing got.clear(); @@ -332,7 +336,7 @@ mod tests { // anti-vacuity: resolve must be able to say no assert_eq!(lens.resolve(MONDO, 4), None, "absent numeric"); assert_eq!( - lens.resolve(0x0302_0000, 1), + lens.resolve(crate::Namespace::Hpo.render_classid(0x0000), 1), None, "right numeric, wrong class" );