diff --git a/docs/cli.md b/docs/cli.md index 8609b16..65d2ae6 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -125,6 +125,7 @@ tablassert build-fullmap [ARGS] | `--version`, `-v` | str | No | `2026jul22` | BABEL snapshot date to fetch (a RENCI stamp, **not** Tablassert's version) | | `--aria2c`, `-a` | Flag | No | `False` | Opt into the bundled `aria2c` binary from the `[aria2]` extra for resumable segmented downloads (the prebuilt archive **or** BABEL files); fails loud (exit 2, before any download starts) if the extra is missing or unsupported on the current platform, and on a non-zero aria2c exit | | `--force`, `-f` | Flag | No | `False` | Skip the prebuilt download and always rebuild from BABEL outputs | +| `--taxon-allowlist` | Flag | No | `False` | Use the built-in top-100 experimental-taxon allowlist; always build from source BABEL files and never use the unfiltered prebuilt | ```bash # Default: download the prebuilt fullmap.tar.zst for this version and extract it (fast) @@ -133,6 +134,8 @@ tablassert build-fullmap --output /data/fullmap/fullmap.redb tablassert build-fullmap --force --output /data/fullmap/fullmap.redb # Accelerate either download with bundled aria2c (`pip install "tablassert[aria2]"`; the multi-GB prebuilt is the ideal aria2 use case) tablassert build-fullmap --aria2c --output /data/fullmap/fullmap.redb +# Build a smaller source-derived database for the top 100 taxa (does not use the prebuilt archive) +tablassert build-fullmap --taxon-allowlist --output /data/fullmap/experimental.redb ``` By default `build-fullmap` looks for a prebuilt `fullmap.tar.zst` at diff --git a/docs/fullmap.md b/docs/fullmap.md index d668b2a..9e8f885 100644 --- a/docs/fullmap.md +++ b/docs/fullmap.md @@ -23,8 +23,19 @@ tablassert build-fullmap --force # Optional: after `pip install "tablassert[aria2]"`, use bundled aria2c for resumable segmented downloads (the multi-GB prebuilt is the ideal aria2 use case) tablassert build-fullmap --aria2c + +# Optional: source-build a smaller DB retaining the built-in top-100 taxa +# (OrganismTaxon and taxonless rows are not filtered) +tablassert build-fullmap --taxon-allowlist ``` +`--taxon-allowlist` is opt-in. It uses the checked-in `src/tablassert/data/experimental_taxa.yaml` +list, filters non-`OrganismTaxon` synonym rows with valid taxon metadata before interning, and always +builds from the downloaded BABEL sources. It never reuses the unfiltered prebuilt archive. Rows with +no valid taxon metadata and every `OrganismTaxon` row are retained; a row with multiple taxa is kept +when any taxon is allowlisted. The first taxon continues to be stored for existing hydration and +lookup behavior. The built database records its allowlist identity in `META.taxon_allowlist`. + See the [CLI Reference → build-fullmap](cli.md#build-fullmap) for the complete flag table (output path, cache directory, BABEL snapshot version, the optional `--aria2c` / `-a` downloader, and the `--force` / `-f` rebuild flag), their defaults, and more examples. @@ -115,7 +126,7 @@ override only when targeting an unusual machine. | Variable | Default | Description | |----------|---------|-------------| -| `TABLASSERT_FULLMAP_EXCLUDE_PREFIXES` | *(empty)* | Comma-separated CURIE prefixes to drop at build time (e.g. `INCHIKEY,Publication`). Excluding prefixes you never resolve dramatically cuts build time, peak memory, and database size. | +| `TABLASSERT_FULLMAP_EXCLUDE_PREFIXES` | *(empty)* | Comma-separated CURIE prefixes to drop at build time (e.g. `Publication`). Excluding prefixes you never resolve dramatically cuts build time, peak memory, and database size. InChIKey terms are retained unless excluded explicitly. | | `TABLASSERT_FULLMAP_CHUNK_BYTES` | `8388608` (8 MiB) | Byte budget per producer→worker line-chunk. Bounded by bytes (not line count) so chunk memory is fixed even for large synonym records. | | `TABLASSERT_FULLMAP_PRODUCERS` | `clamp(workers/4, 4, #files)` | Number of producer (decompressor) threads. Decompression far outpaces parallel processing, so a handful keeps all workers fed. | | `TABLASSERT_FULLMAP_LOCAL_SPILL_ENTRIES` | `1000000` | Per-worker term-posting buffer size before spilling a sorted run to disk. Lower → less RAM, more run files. | @@ -140,7 +151,7 @@ they hold six tables (see `rust/src/fullmap.rs`): | `categories` | Compact `u16` id → Biolink category string (primary file) | | `sources` | Compact `u8` id → source metadata (name/version) (primary file) | | `curies` | Compact `u32` id → CURIE record (CURIE, preferred name, category, taxon, source) (primary file) | -| `meta` | Schema version tag (`tablassert.fullmap.v5`), the shard count (`shards`), and the BABEL `source_version` used to build the file (primary file) | +| `meta` | Schema version tag (`tablassert.fullmap.v5`), the shard count (`shards`), the BABEL `source_version` used to build the file, and optional allowlist identity (`taxon_allowlist`) (primary file) | The shard files must remain alongside the primary file: lookups discover them as siblings of the resolved primary path. diff --git a/pyproject.toml b/pyproject.toml index 0947f23..145ecf9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,6 +72,7 @@ module-name = "tablassert.rs" bindings = "pyo3" manifest-path = "rust/Cargo.toml" features = ["extension-module"] +include = ["src/tablassert/data/experimental_taxa.yaml"] [project.scripts] tablassert = "tablassert.cli:APP" diff --git a/rust/src/fullmap.rs b/rust/src/fullmap.rs index 9d8e940..9c51659 100644 --- a/rust/src/fullmap.rs +++ b/rust/src/fullmap.rs @@ -294,7 +294,6 @@ fn token_qc(value: &str) -> bool { && !value.contains('\t') && !value.contains('\n') && !value.contains('\r') - && !value.contains("inchikey") && !value.contains("uncharacterized") && !value.contains("hypothetical") } @@ -459,6 +458,15 @@ fn first_category(row: &SynonymRow<'_>) -> String { .to_string() } +fn is_organism_taxon(row: &SynonymRow<'_>) -> bool { + row.types.iter().any(|category| { + category + .trim() + .trim_start_matches("biolink:") + .eq_ignore_ascii_case("OrganismTaxon") + }) +} + fn first_taxon(row: &SynonymRow<'_>) -> i32 { row.taxa .first() @@ -469,6 +477,56 @@ fn first_taxon(row: &SynonymRow<'_>) -> i32 { .unwrap_or(0) } +fn parsed_taxa(row: &SynonymRow<'_>) -> Vec { + row.taxa + .iter() + .filter_map(|taxon| { + let taxon = taxon.trim(); + let local = taxon + .split_once(':') + .filter(|(prefix, _)| prefix.eq_ignore_ascii_case("NCBITaxon")) + .map(|(_, local)| local) + .unwrap_or(taxon); + local.parse::().ok().filter(|id| *id > 0) + }) + .collect() +} + +fn allowlist_identity(taxa: &HashSet) -> String { + let mut ids: Vec = taxa.iter().copied().collect(); + ids.sort_unstable(); + let encoded = ids + .iter() + .map(i32::to_string) + .collect::>() + .join(","); + format!( + "count={};xxh64={:016x}", + ids.len(), + xxh64(encoded.as_bytes(), 0) + ) +} + +fn retain_for_taxon_allowlist(row: &SynonymRow<'_>, allowlist: Option<&HashSet>) -> bool { + let Some(allowlist) = allowlist else { + return true; + }; + if is_organism_taxon(row) { + return true; + } + let mut has_valid_taxon = false; + for taxon in parsed_taxa(row) { + has_valid_taxon = true; + if allowlist.contains(&taxon) { + return true; + } + } + // Taxonless rows remain available for categories whose BABEL synonym rows + // do not carry a positive taxon ID. This includes the conventional + // `NCBITaxon:0` sentinel and malformed/empty taxon values. + !has_valid_taxon +} + fn split_curie(curie: &str) -> Option<(&str, &str)> { curie .split_once(':') @@ -1358,6 +1416,7 @@ struct SynonymShared<'a> { curie_counter: &'a AtomicU32, equivalents: &'a EquivIndex, exclude_prefixes: &'a HashSet, + taxon_allowlist: Option<&'a HashSet>, spill_dir: &'a Path, local_spill: usize, curie_spill: usize, @@ -1393,6 +1452,9 @@ fn process_row( let Some((prefix, local_id)) = split_curie(curie) else { return Ok(()); }; + if !retain_for_taxon_allowlist(row, sh.taxon_allowlist) { + return Ok(()); + } // Skip CURIEs whose prefix the caller excludes (opt-in via // TABLASSERT_FULLMAP_EXCLUDE_PREFIXES) — filtered out downstream anyway. if sh.exclude_prefixes.contains(prefix) { @@ -1623,6 +1685,7 @@ fn process_synonyms( local_spill: usize, curie_spill: usize, exclude_prefixes: &HashSet, + taxon_allowlist: Option<&HashSet>, worker_count: usize, shard_count: usize, chunk_bytes: usize, @@ -1671,6 +1734,7 @@ fn process_synonyms( curie_counter: &curie_counter, equivalents, exclude_prefixes, + taxon_allowlist, spill_dir, local_spill, curie_spill, @@ -1799,6 +1863,7 @@ fn write_final_database( insert_batch: usize, shard_count: usize, progress: Option<&Arc>, + taxon_allowlist_identity: Option<&str>, ) -> PyResult<()> { let build_id = new_build_id(); let build_id_str = build_id.to_string(); @@ -1855,6 +1920,9 @@ fn write_final_database( let shard_count_str = shard_count.to_string(); meta.insert("shards", shard_count_str.as_str()) .map_err(py_err)?; + if let Some(identity) = taxon_allowlist_identity { + meta.insert("taxon_allowlist", identity).map_err(py_err)?; + } } write.commit().map_err(py_err)?; @@ -2229,6 +2297,8 @@ fn build_fullmap_inner( local_spill: usize, curie_spill: usize, exclude_prefixes: HashSet, + taxon_allowlist: Option>, + taxon_allowlist_identity: Option, chunk_bytes: usize, producers: usize, cache_bytes: usize, @@ -2258,6 +2328,7 @@ fn build_fullmap_inner( local_spill, curie_spill, &exclude_prefixes, + taxon_allowlist.as_ref(), worker_count, shard_count, chunk_bytes, @@ -2288,6 +2359,7 @@ fn build_fullmap_inner( insert_batch, shard_count, progress.as_ref(), + taxon_allowlist_identity.as_deref(), )?; // Clean up spill runs on success (left in place on error for inspection). @@ -2297,13 +2369,14 @@ fn build_fullmap_inner( } #[pyfunction] -#[pyo3(signature = (output, classes, synonyms, progress=None))] +#[pyo3(signature = (output, classes, synonyms, progress=None, taxon_allowlist=None))] pub fn build_fullmap_db( py: Python<'_>, output: PathBuf, classes: Vec, synonyms: Vec, progress: Option>, + taxon_allowlist: Option>, ) -> PyResult<()> { if synonyms.is_empty() { return Err(PyValueError::new_err( @@ -2407,6 +2480,9 @@ pub fn build_fullmap_db( PathBuf::from(p) }); + let taxon_allowlist: Option> = + taxon_allowlist.map(|ids| ids.into_iter().filter(|id| *id > 0).collect()); + let taxon_allowlist_identity: Option = taxon_allowlist.as_ref().map(allowlist_identity); let progress = progress.map(|cb| Arc::new(Progress { cb })); // Release the GIL for the whole build so rich's Live display thread can @@ -2422,6 +2498,8 @@ pub fn build_fullmap_db( local_spill, curie_spill, exclude_prefixes, + taxon_allowlist, + taxon_allowlist_identity, chunk_bytes, producers, cache_bytes, @@ -3650,6 +3728,8 @@ mod tests { local_spill, 1_000_000, HashSet::new(), + None, + None, DEFAULT_CHUNK_BYTES, 2, 64 * 1024 * 1024, @@ -3678,10 +3758,12 @@ mod tests { } #[test] - fn token_qc_rejects_banned_tokens() { + fn token_qc_rejects_banned_tokens_but_accepts_inchikey() { assert!(token_qc("brca1")); + assert!(token_qc("inchikey=abc")); assert!(!token_qc("hypothetical protein")); assert!(!token_qc("line\nbreak")); + assert!(!token_qc("tab\tvalue")); } #[test] @@ -4028,6 +4110,7 @@ mod tests { 30, 1_000_000, &HashSet::new(), + None, 4, shard_count, DEFAULT_CHUNK_BYTES, @@ -4085,6 +4168,8 @@ mod tests { 100, 1_000_000, HashSet::new(), + None, + None, DEFAULT_CHUNK_BYTES, 2, 64 * 1024 * 1024, @@ -4102,6 +4187,8 @@ mod tests { 100, 1_000_000, HashSet::new(), + None, + None, DEFAULT_CHUNK_BYTES, 2, 64 * 1024 * 1024, @@ -4519,6 +4606,8 @@ mod tests { 4_000_000, 1_000_000, HashSet::new(), + None, + None, DEFAULT_CHUNK_BYTES, 2, 64 * 1024 * 1024, @@ -4588,7 +4677,7 @@ mod tests { std::env::set_var("TABLASSERT_FULLMAP_SHARDS", "2"); let built = Python::attach(|py| { - build_fullmap_db(py, output.clone(), Vec::new(), vec![synonyms], None) + build_fullmap_db(py, output.clone(), Vec::new(), vec![synonyms], None, None) }); std::env::remove_var("TABLASSERT_FULLMAP_SHARDS"); built.unwrap(); @@ -4649,6 +4738,8 @@ mod tests { 4_000_000, 1_000_000, HashSet::new(), + None, + None, DEFAULT_CHUNK_BYTES, 2, 64 * 1024 * 1024, @@ -5356,6 +5447,8 @@ mod tests { 4_000_000, 2, HashSet::new(), + None, + None, DEFAULT_CHUNK_BYTES, 2, 64 * 1024 * 1024, @@ -5446,6 +5539,8 @@ mod tests { 4_000_000, 1_000_000, exclude, + None, + None, DEFAULT_CHUNK_BYTES, 2, 64 * 1024 * 1024, @@ -5505,6 +5600,8 @@ mod tests { 4_000_000, 1_000_000, HashSet::new(), + None, + None, 8192, 2, 64 * 1024 * 1024, @@ -5546,6 +5643,7 @@ mod tests { Vec::new(), Vec::new(), None, + None, ) .expect_err("empty synonyms should fail"); assert!(err diff --git a/rust/tests/build_golden.rs b/rust/tests/build_golden.rs index 159fac9..f3015e4 100644 --- a/rust/tests/build_golden.rs +++ b/rust/tests/build_golden.rs @@ -285,6 +285,57 @@ fn schema_and_shard_count_are_pinned() { ); } +#[test] +fn taxon_allowlist_metadata_is_stable_and_omitted_for_default_builds() { + let dir = tempfile::tempdir().unwrap(); + let synonyms = dir.path().join("SRC.ndjson"); + write_jsonl( + &synonyms, + &[ + r#"{"curie":"HGNC:1","preferred_name":"Gene","names":["gene"],"types":["Gene"],"taxa":["NCBITaxon:9606"]}"#, + ], + ); + let filtered_a = dir.path().join("filtered_a.redb"); + let filtered_b = dir.path().join("filtered_b.redb"); + let unfiltered = dir.path().join("unfiltered.redb"); + pyo3::Python::initialize(); + pyo3::Python::attach(|py| { + tablassert_rs::build_fullmap_db( + py, + filtered_a.clone(), + vec![], + vec![synonyms.clone()], + None, + Some(vec![10090, 9606]), + ) + .unwrap(); + tablassert_rs::build_fullmap_db( + py, + filtered_b.clone(), + vec![], + vec![synonyms.clone()], + None, + Some(vec![9606, 10090]), + ) + .unwrap(); + tablassert_rs::build_fullmap_db(py, unfiltered.clone(), vec![], vec![synonyms], None, None) + .unwrap(); + }); + + let read_allowlist = |path: &std::path::Path| -> Option { + let db = open_primary_copy(path); + let read = db.begin_read().unwrap(); + let meta = read.open_table(META).unwrap(); + meta.get("taxon_allowlist") + .unwrap() + .map(|value| value.value().to_string()) + }; + let identity = read_allowlist(&filtered_a).unwrap(); + assert_eq!(Some(identity.clone()), read_allowlist(&filtered_b)); + assert_eq!(None, read_allowlist(&unfiltered)); + assert!(identity.starts_with("count=2;xxh64=")); +} + // --------------------------------------------------------------------------- // (f) GZ INPUT — a gzipped synonym file yields identical results to plain. // --------------------------------------------------------------------------- @@ -324,6 +375,7 @@ fn gz_input_matches_plain_input() { vec![classes], vec![synonyms_gz], None, + None, ) .unwrap(); }); @@ -351,8 +403,15 @@ fn empty_synonym_file_builds_empty_db() { write_jsonl(&synonyms, &[]); // 0 rows let output = dir.path().join("fullmap.redb"); pyo3::Python::attach(|py| { - tablassert_rs::build_fullmap_db(py, output.clone(), vec![classes], vec![synonyms], None) - .unwrap(); + tablassert_rs::build_fullmap_db( + py, + output.clone(), + vec![classes], + vec![synonyms], + None, + None, + ) + .unwrap(); }); assert!( @@ -383,7 +442,8 @@ fn synonym_row_with_no_names_indexes_only_curie() { ); let output = dir.path().join("fullmap.redb"); pyo3::Python::attach(|py| { - tablassert_rs::build_fullmap_db(py, output.clone(), vec![], vec![synonyms], None).unwrap(); + tablassert_rs::build_fullmap_db(py, output.clone(), vec![], vec![synonyms], None, None) + .unwrap(); }); let map = term_curie_map(&output); @@ -409,7 +469,8 @@ fn null_preferred_name_falls_back_to_curie() { ); let output = dir.path().join("fullmap.redb"); pyo3::Python::attach(|py| { - tablassert_rs::build_fullmap_db(py, output.clone(), vec![], vec![synonyms], None).unwrap(); + tablassert_rs::build_fullmap_db(py, output.clone(), vec![], vec![synonyms], None, None) + .unwrap(); }); let db = open_primary_copy(&output); @@ -444,8 +505,15 @@ fn class_row_without_equivalents_builds() { ); let output = dir.path().join("fullmap.redb"); pyo3::Python::attach(|py| { - tablassert_rs::build_fullmap_db(py, output.clone(), vec![classes], vec![synonyms], None) - .unwrap(); + tablassert_rs::build_fullmap_db( + py, + output.clone(), + vec![classes], + vec![synonyms], + None, + None, + ) + .unwrap(); }); let map = term_curie_map(&output); diff --git a/rust/tests/common/mod.rs b/rust/tests/common/mod.rs index 0a2a484..5ad9ecd 100644 --- a/rust/tests/common/mod.rs +++ b/rust/tests/common/mod.rs @@ -117,8 +117,15 @@ pub fn build_fixture(dir: &Path) -> PathBuf { write_jsonl(&synonyms, SYNONYM_LINES); let output = dir.join("fullmap.redb"); pyo3::Python::attach(|py| { - tablassert_rs::build_fullmap_db(py, output.clone(), vec![classes], vec![synonyms], None) - .unwrap(); + tablassert_rs::build_fullmap_db( + py, + output.clone(), + vec![classes], + vec![synonyms], + None, + None, + ) + .unwrap(); }); output } diff --git a/src/tablassert/cli.py b/src/tablassert/cli.py index 0db1473..71c0a88 100644 --- a/src/tablassert/cli.py +++ b/src/tablassert/cli.py @@ -8,7 +8,7 @@ from collections.abc import Callable from importlib import import_module from importlib.metadata import version as get_version -from itertools import chain +from itertools import chain, pairwise from multiprocessing import Pool from pathlib import Path from typing import TYPE_CHECKING, Annotated, Any, BinaryIO, Literal @@ -47,6 +47,7 @@ BABEL_EXCLUDE_PREFIXES: tuple[str, ...] = ("Publication", "GeneProteinConflated") BABEL_CLASS_RE: re.Pattern[str] = re.compile(r' Path: @@ -1270,8 +1271,43 @@ def report_progress(downloaded: int, total: int) -> None: download_logger.info("Installed prebuilt fullmap v{release} -> {output}", release=release, output=output) +def load_taxon_allowlist() -> list[int]: + """Load the checked-in experimental-taxon YAML list for an opt-in build.""" + import yaml + + raw: object = yaml.safe_load(TAXON_ALLOWLIST_PATH.read_text(encoding="utf-8")) + if not isinstance(raw, list): + raise ValueError(f"taxon allowlist must be a YAML list: {TAXON_ALLOWLIST_PATH}") + ids: list[int] = [] + frequencies: list[int] = [] + for expected_rank, entry in enumerate(raw, start=1): + if ( + not isinstance(entry, dict) + or entry.get("rank") != expected_rank + or not isinstance(entry.get("taxon_id"), int) + or entry["taxon_id"] <= 0 + or not isinstance(entry.get("frequency"), int) + or entry["frequency"] < 0 + ): + raise ValueError(f"invalid taxon allowlist entry at rank {expected_rank}: {entry!r}") + ids.append(entry["taxon_id"]) + frequencies.append(entry["frequency"]) + if len(ids) != 100 or len(set(ids)) != len(ids): + raise ValueError(f"taxon allowlist must contain 100 unique positive IDs: {TAXON_ALLOWLIST_PATH}") + if any( + left < right or (left == right and left_id > right_id) for (left, left_id), (right, right_id) in pairwise(zip(frequencies, ids, strict=True)) + ): + raise ValueError(f"taxon allowlist ranks are not deterministically ordered: {TAXON_ALLOWLIST_PATH}") + return ids + + def build_fullmap_pipeline( - output: Path, progress: PipelineProgress, cache: Path = Path("./fullmap/downloads"), version: str = BABEL_VERSION, aria2c: bool = False + output: Path, + progress: PipelineProgress, + cache: Path = Path("./fullmap/downloads"), + version: str = BABEL_VERSION, + aria2c: bool = False, + taxon_allowlist: list[int] | None = None, ) -> None: """Build an embedded fullmap redb database from BABEL outputs. @@ -1284,6 +1320,7 @@ def build_fullmap_pipeline( cache: Directory for downloaded BABEL files. version: BABEL version label. aria2c: Use the bundled aria2c binary from the optional ``[aria2]`` extra for downloads when true. + taxon_allowlist: Optional NCBI taxon IDs passed to Rust before interning. """ from tablassert import rs @@ -1326,7 +1363,10 @@ def download_one(filename: str, url: str, destination: Path) -> Path: # Rust drives per-phase progress (equivalents -> synonyms -> writing) via the # callback; the GIL is released during the build so the bar repaints live. on_progress = progress.dynamic_loop("Build") - rs.build_fullmap_db(output, class_files, synonym_files, progress=on_progress) + if taxon_allowlist is None: + rs.build_fullmap_db(output, class_files, synonym_files, progress=on_progress) + else: + rs.build_fullmap_db(output, class_files, synonym_files, progress=on_progress, taxon_allowlist=taxon_allowlist) progress.end_section_task() logger.info( @@ -1345,6 +1385,7 @@ def build_fullmap( version: Annotated[str, cyclopts.Parameter(name=["--version", "-v"])] = BABEL_VERSION, aria2c: Annotated[bool, cyclopts.Parameter(name=["--aria2c", "-a"], negative="")] = False, force: Annotated[bool, cyclopts.Parameter(name=["--force", "-f"], negative="")] = False, + taxon_allowlist: Annotated[bool, cyclopts.Parameter(name="--taxon-allowlist", negative="")] = False, ) -> None: """Build an embedded fullmap redb database, or download a prebuilt one from RENCI. @@ -1353,6 +1394,8 @@ def build_fullmap( extract it — far faster than building from BABEL. If no prebuilt exists for this version (or the download/extract fails), fall back to a from-scratch build. ``--force`` / ``-f`` skips the prebuilt attempt and always builds from BABEL outputs. + ``--taxon-allowlist`` enables the built-in top-100 experimental-taxon filter and always + builds from source BABEL files; it never reuses the unfiltered prebuilt archive. ``--aria2c`` requires the ``[aria2]`` extra, checked before the first download rather than on it, so an unusable flag costs nothing. @@ -1364,10 +1407,12 @@ def build_fullmap( aria2c: Use the bundled aria2c binary from the ``[aria2]`` extra for downloads (prebuilt or BABEL). force: Skip the prebuilt download and always rebuild from BABEL outputs. + taxon_allowlist: Enable the built-in top-100 experimental-taxon filter. """ - # A complete primary redb already on disk means the DB is in place: reuse it. Only - # --force rebuilds once a DB exists, so it is the explicit "fresh build" knob. - if not force and output.is_file() and output.stat().st_size > 0: + allowlist_ids: list[int] | None = load_taxon_allowlist() if taxon_allowlist else None + # Allowlisted outputs must not reuse an unfiltered database already at the path. + # The caller explicitly requested a filtered source build. + if not taxon_allowlist and not force and output.is_file() and output.stat().st_size > 0: print(f"tablassert build-fullmap: fullmap already present at {output}; skipping (use --force to rebuild).", file=sys.stderr) return # Checked here rather than earlier: the reuse path above downloads nothing, so a @@ -1375,10 +1420,13 @@ def build_fullmap( if aria2c and not extras.is_installed("aria2"): print(f"tablassert build-fullmap: --aria2c is unavailable — {aria2_unavailable_detail()}", file=sys.stderr) raise SystemExit(2) - if not force: + if not force and not taxon_allowlist: try: run(2, fetch_prebuilt_fullmap, output, version=version, aria2c=aria2c) return except PrebuiltFullmapUnavailable as exc: logger.warning("Prebuilt fullmap unavailable ({reason}); building from BABEL outputs.", reason=exc) - run(3, build_fullmap_pipeline, output, cache=cache, version=version, aria2c=aria2c) + if allowlist_ids is None: + run(3, build_fullmap_pipeline, output, cache=cache, version=version, aria2c=aria2c) + else: + run(3, build_fullmap_pipeline, output, cache=cache, version=version, aria2c=aria2c, taxon_allowlist=allowlist_ids) diff --git a/src/tablassert/data/experimental_taxa.yaml b/src/tablassert/data/experimental_taxa.yaml new file mode 100644 index 0000000..c6a8cd7 --- /dev/null +++ b/src/tablassert/data/experimental_taxa.yaml @@ -0,0 +1,511 @@ +# Top 100 experimental taxa for the optional fullmap build allowlist. +# +# Generated from non-biolink:OrganismTaxon node in_taxon frequencies in the +# checked-in/generated node corpora of: +# /home/skyeav/Code/ISB/MultiomicsNext +# /home/skyeav/Code/ISB/MultiomicsHarness +# +# Ranking: descending combined node frequency, then ascending numeric NCBI +# taxon ID for deterministic tie-breaking. Counts are retained as provenance +# metadata; the loader uses the taxon_id values. This is an opt-in list and +# deliberately does not filter OrganismTaxon synonym rows. +- rank: 1 + taxon_id: 9606 + frequency: 458410 + multomics_next_frequency: 219402 + multiomics_harness_frequency: 239008 +- rank: 2 + taxon_id: 10090 + frequency: 90244 + multomics_next_frequency: 13715 + multiomics_harness_frequency: 76529 +- rank: 3 + taxon_id: 9913 + frequency: 9582 + multomics_next_frequency: 0 + multiomics_harness_frequency: 9582 +- rank: 4 + taxon_id: 559292 + frequency: 4341 + multomics_next_frequency: 0 + multiomics_harness_frequency: 4341 +- rank: 5 + taxon_id: 3702 + frequency: 653 + multomics_next_frequency: 0 + multiomics_harness_frequency: 653 +- rank: 6 + taxon_id: 185431 + frequency: 240 + multomics_next_frequency: 0 + multiomics_harness_frequency: 240 +- rank: 7 + taxon_id: 69293 + frequency: 58 + multomics_next_frequency: 58 + multiomics_harness_frequency: 0 +- rank: 8 + taxon_id: 7227 + frequency: 40 + multomics_next_frequency: 0 + multiomics_harness_frequency: 40 +- rank: 9 + taxon_id: 208964 + frequency: 34 + multomics_next_frequency: 0 + multiomics_harness_frequency: 34 +- rank: 10 + taxon_id: 386585 + frequency: 34 + multomics_next_frequency: 0 + multiomics_harness_frequency: 34 +- rank: 11 + taxon_id: 1069680 + frequency: 28 + multomics_next_frequency: 0 + multiomics_harness_frequency: 28 +- rank: 12 + taxon_id: 4896 + frequency: 26 + multomics_next_frequency: 0 + multiomics_harness_frequency: 26 +- rank: 13 + taxon_id: 1186058 + frequency: 17 + multomics_next_frequency: 0 + multiomics_harness_frequency: 17 +- rank: 14 + taxon_id: 8030 + frequency: 15 + multomics_next_frequency: 0 + multiomics_harness_frequency: 15 +- rank: 15 + taxon_id: 77519 + frequency: 14 + multomics_next_frequency: 0 + multiomics_harness_frequency: 14 +- rank: 16 + taxon_id: 1094350 + frequency: 14 + multomics_next_frequency: 0 + multiomics_harness_frequency: 14 +- rank: 17 + taxon_id: 8260 + frequency: 13 + multomics_next_frequency: 0 + multiomics_harness_frequency: 13 +- rank: 18 + taxon_id: 76775 + frequency: 13 + multomics_next_frequency: 0 + multiomics_harness_frequency: 13 +- rank: 19 + taxon_id: 1567544 + frequency: 10 + multomics_next_frequency: 0 + multiomics_harness_frequency: 10 +- rank: 20 + taxon_id: 296543 + frequency: 9 + multomics_next_frequency: 0 + multiomics_harness_frequency: 9 +- rank: 21 + taxon_id: 1518534 + frequency: 9 + multomics_next_frequency: 0 + multiomics_harness_frequency: 9 +- rank: 22 + taxon_id: 8078 + frequency: 8 + multomics_next_frequency: 0 + multiomics_harness_frequency: 8 +- rank: 23 + taxon_id: 375899 + frequency: 8 + multomics_next_frequency: 0 + multiomics_harness_frequency: 8 +- rank: 24 + taxon_id: 68895 + frequency: 7 + multomics_next_frequency: 0 + multiomics_harness_frequency: 7 +- rank: 25 + taxon_id: 2492962 + frequency: 7 + multomics_next_frequency: 0 + multiomics_harness_frequency: 7 +- rank: 26 + taxon_id: 69 + frequency: 6 + multomics_next_frequency: 0 + multiomics_harness_frequency: 6 +- rank: 27 + taxon_id: 333 + frequency: 6 + multomics_next_frequency: 0 + multiomics_harness_frequency: 6 +- rank: 28 + taxon_id: 6239 + frequency: 6 + multomics_next_frequency: 0 + multiomics_harness_frequency: 6 +- rank: 29 + taxon_id: 62977 + frequency: 6 + multomics_next_frequency: 0 + multiomics_harness_frequency: 6 +- rank: 30 + taxon_id: 69222 + frequency: 6 + multomics_next_frequency: 0 + multiomics_harness_frequency: 6 +- rank: 31 + taxon_id: 114398 + frequency: 6 + multomics_next_frequency: 0 + multiomics_harness_frequency: 6 +- rank: 32 + taxon_id: 168475 + frequency: 6 + multomics_next_frequency: 0 + multiomics_harness_frequency: 6 +- rank: 33 + taxon_id: 198618 + frequency: 6 + multomics_next_frequency: 0 + multiomics_harness_frequency: 6 +- rank: 34 + taxon_id: 1144522 + frequency: 6 + multomics_next_frequency: 0 + multiomics_harness_frequency: 6 +- rank: 35 + taxon_id: 1194168 + frequency: 6 + multomics_next_frequency: 0 + multiomics_harness_frequency: 6 +- rank: 36 + taxon_id: 1211579 + frequency: 6 + multomics_next_frequency: 0 + multiomics_harness_frequency: 6 +- rank: 37 + taxon_id: 1337664 + frequency: 6 + multomics_next_frequency: 0 + multiomics_harness_frequency: 6 +- rank: 38 + taxon_id: 3240790 + frequency: 6 + multomics_next_frequency: 0 + multiomics_harness_frequency: 6 +- rank: 39 + taxon_id: 292 + frequency: 5 + multomics_next_frequency: 0 + multiomics_harness_frequency: 5 +- rank: 40 + taxon_id: 675 + frequency: 5 + multomics_next_frequency: 0 + multiomics_harness_frequency: 5 +- rank: 41 + taxon_id: 680 + frequency: 5 + multomics_next_frequency: 0 + multiomics_harness_frequency: 5 +- rank: 42 + taxon_id: 964 + frequency: 5 + multomics_next_frequency: 0 + multiomics_harness_frequency: 5 +- rank: 43 + taxon_id: 1019 + frequency: 5 + multomics_next_frequency: 0 + multiomics_harness_frequency: 5 +- rank: 44 + taxon_id: 7091 + frequency: 5 + multomics_next_frequency: 0 + multiomics_harness_frequency: 5 +- rank: 45 + taxon_id: 7092 + frequency: 5 + multomics_next_frequency: 0 + multiomics_harness_frequency: 5 +- rank: 46 + taxon_id: 7113 + frequency: 5 + multomics_next_frequency: 0 + multiomics_harness_frequency: 5 +- rank: 47 + taxon_id: 7130 + frequency: 5 + multomics_next_frequency: 0 + multiomics_harness_frequency: 5 +- rank: 48 + taxon_id: 7137 + frequency: 5 + multomics_next_frequency: 0 + multiomics_harness_frequency: 5 +- rank: 49 + taxon_id: 7141 + frequency: 5 + multomics_next_frequency: 0 + multiomics_harness_frequency: 5 +- rank: 50 + taxon_id: 7213 + frequency: 5 + multomics_next_frequency: 0 + multiomics_harness_frequency: 5 +- rank: 51 + taxon_id: 8022 + frequency: 5 + multomics_next_frequency: 0 + multiomics_harness_frequency: 5 +- rank: 52 + taxon_id: 13037 + frequency: 5 + multomics_next_frequency: 0 + multiomics_harness_frequency: 5 +- rank: 53 + taxon_id: 13191 + frequency: 5 + multomics_next_frequency: 0 + multiomics_harness_frequency: 5 +- rank: 54 + taxon_id: 29058 + frequency: 5 + multomics_next_frequency: 0 + multiomics_harness_frequency: 5 +- rank: 55 + taxon_id: 33412 + frequency: 5 + multomics_next_frequency: 0 + multiomics_harness_frequency: 5 +- rank: 56 + taxon_id: 42275 + frequency: 5 + multomics_next_frequency: 0 + multiomics_harness_frequency: 5 +- rank: 57 + taxon_id: 42288 + frequency: 5 + multomics_next_frequency: 0 + multiomics_harness_frequency: 5 +- rank: 58 + taxon_id: 42677 + frequency: 5 + multomics_next_frequency: 0 + multiomics_harness_frequency: 5 +- rank: 59 + taxon_id: 47958 + frequency: 5 + multomics_next_frequency: 0 + multiomics_harness_frequency: 5 +- rank: 60 + taxon_id: 61647 + frequency: 5 + multomics_next_frequency: 0 + multiomics_harness_frequency: 5 +- rank: 61 + taxon_id: 66420 + frequency: 5 + multomics_next_frequency: 0 + multiomics_harness_frequency: 5 +- rank: 62 + taxon_id: 72036 + frequency: 5 + multomics_next_frequency: 0 + multiomics_harness_frequency: 5 +- rank: 63 + taxon_id: 72248 + frequency: 5 + multomics_next_frequency: 0 + multiomics_harness_frequency: 5 +- rank: 64 + taxon_id: 76193 + frequency: 5 + multomics_next_frequency: 0 + multiomics_harness_frequency: 5 +- rank: 65 + taxon_id: 76194 + frequency: 5 + multomics_next_frequency: 0 + multiomics_harness_frequency: 5 +- rank: 66 + taxon_id: 91739 + frequency: 5 + multomics_next_frequency: 0 + multiomics_harness_frequency: 5 +- rank: 67 + taxon_id: 93504 + frequency: 5 + multomics_next_frequency: 0 + multiomics_harness_frequency: 5 +- rank: 68 + taxon_id: 110368 + frequency: 5 + multomics_next_frequency: 0 + multiomics_harness_frequency: 5 +- rank: 69 + taxon_id: 113334 + frequency: 5 + multomics_next_frequency: 0 + multiomics_harness_frequency: 5 +- rank: 70 + taxon_id: 116150 + frequency: 5 + multomics_next_frequency: 0 + multiomics_harness_frequency: 5 +- rank: 71 + taxon_id: 129554 + frequency: 5 + multomics_next_frequency: 0 + multiomics_harness_frequency: 5 +- rank: 72 + taxon_id: 132476 + frequency: 5 + multomics_next_frequency: 0 + multiomics_harness_frequency: 5 +- rank: 73 + taxon_id: 165597 + frequency: 5 + multomics_next_frequency: 0 + multiomics_harness_frequency: 5 +- rank: 74 + taxon_id: 171585 + frequency: 5 + multomics_next_frequency: 0 + multiomics_harness_frequency: 5 +- rank: 75 + taxon_id: 171605 + frequency: 5 + multomics_next_frequency: 0 + multiomics_harness_frequency: 5 +- rank: 76 + taxon_id: 179879 + frequency: 5 + multomics_next_frequency: 0 + multiomics_harness_frequency: 5 +- rank: 77 + taxon_id: 189913 + frequency: 5 + multomics_next_frequency: 0 + multiomics_harness_frequency: 5 +- rank: 78 + taxon_id: 191418 + frequency: 5 + multomics_next_frequency: 0 + multiomics_harness_frequency: 5 +- rank: 79 + taxon_id: 195709 + frequency: 5 + multomics_next_frequency: 0 + multiomics_harness_frequency: 5 +- rank: 80 + taxon_id: 312309 + frequency: 5 + multomics_next_frequency: 0 + multiomics_harness_frequency: 5 +- rank: 81 + taxon_id: 321846 + frequency: 5 + multomics_next_frequency: 0 + multiomics_harness_frequency: 5 +- rank: 82 + taxon_id: 334116 + frequency: 5 + multomics_next_frequency: 0 + multiomics_harness_frequency: 5 +- rank: 83 + taxon_id: 335848 + frequency: 5 + multomics_next_frequency: 0 + multiomics_harness_frequency: 5 +- rank: 84 + taxon_id: 335849 + frequency: 5 + multomics_next_frequency: 0 + multiomics_harness_frequency: 5 +- rank: 85 + taxon_id: 339670 + frequency: 5 + multomics_next_frequency: 0 + multiomics_harness_frequency: 5 +- rank: 86 + taxon_id: 359387 + frequency: 5 + multomics_next_frequency: 0 + multiomics_harness_frequency: 5 +- rank: 87 + taxon_id: 395600 + frequency: 5 + multomics_next_frequency: 0 + multiomics_harness_frequency: 5 +- rank: 88 + taxon_id: 442694 + frequency: 5 + multomics_next_frequency: 0 + multiomics_harness_frequency: 5 +- rank: 89 + taxon_id: 452646 + frequency: 5 + multomics_next_frequency: 0 + multiomics_harness_frequency: 5 +- rank: 90 + taxon_id: 483199 + frequency: 5 + multomics_next_frequency: 0 + multiomics_harness_frequency: 5 +- rank: 91 + taxon_id: 487832 + frequency: 5 + multomics_next_frequency: 0 + multiomics_harness_frequency: 5 +- rank: 92 + taxon_id: 488447 + frequency: 5 + multomics_next_frequency: 0 + multiomics_harness_frequency: 5 +- rank: 93 + taxon_id: 488729 + frequency: 5 + multomics_next_frequency: 0 + multiomics_harness_frequency: 5 +- rank: 94 + taxon_id: 488731 + frequency: 5 + multomics_next_frequency: 0 + multiomics_harness_frequency: 5 +- rank: 95 + taxon_id: 488732 + frequency: 5 + multomics_next_frequency: 0 + multiomics_harness_frequency: 5 +- rank: 96 + taxon_id: 520877 + frequency: 5 + multomics_next_frequency: 0 + multiomics_harness_frequency: 5 +- rank: 97 + taxon_id: 611301 + frequency: 5 + multomics_next_frequency: 0 + multiomics_harness_frequency: 5 +- rank: 98 + taxon_id: 666685 + frequency: 5 + multomics_next_frequency: 0 + multiomics_harness_frequency: 5 +- rank: 99 + taxon_id: 680683 + frequency: 5 + multomics_next_frequency: 0 + multiomics_harness_frequency: 5 +- rank: 100 + taxon_id: 1074311 + frequency: 5 + multomics_next_frequency: 0 + multiomics_harness_frequency: 5 diff --git a/src/tablassert/rs.pyi b/src/tablassert/rs.pyi index 911b152..db38cbc 100644 --- a/src/tablassert/rs.pyi +++ b/src/tablassert/rs.pyi @@ -5,7 +5,11 @@ from pathlib import Path from typing import Any def build_fullmap_db( - output: Path, classes: list[Path], synonyms: list[Path], progress: Callable[[int, int, int, str], None] | None = None + output: Path, + classes: list[Path], + synonyms: list[Path], + progress: Callable[[int, int, int, str], None] | None = None, + taxon_allowlist: list[int] | None = None, ) -> None: ... def dedup_ndjson( input: Path, output: Path, is_edges: bool, domain: str | None = None, uuid_fields: list[str] | None = None, on_collision: str | None = None diff --git a/tests/test_cover_cli.py b/tests/test_cover_cli.py index dd3e65e..ce455ff 100644 --- a/tests/test_cover_cli.py +++ b/tests/test_cover_cli.py @@ -913,6 +913,15 @@ def _interrupt(archive: Path, output: Path, progress: object = None) -> None: cli._extract_prebuilt_fullmap(tmp_path / "fullmap.tar.zst", tmp_path / "fullmap.redb", on_phase=lambda p: None) +def test_load_taxon_allowlist_is_top_100() -> None: + """The built-in YAML artifact is a deterministic 100-entry positive-ID list.""" + ids = cli.load_taxon_allowlist() + assert len(ids) == 100 + assert len(set(ids)) == 100 + assert ids[:4] == [9606, 10090, 9913, 559292] + assert ids[-1] == 1074311 + + def test_build_fullmap_command_defaults_to_prebuilt_download(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: """Default (no ``--force``) tries the prebuilt download FIRST, not a from-scratch build. @@ -932,6 +941,33 @@ def _fake_run(stages: int, fn: Any, arg: Path, **kwargs: Any) -> None: assert calls == [(2, cli.fetch_prebuilt_fullmap, output, {"version": "v", "aria2c": True})] +def test_build_fullmap_allowlist_skips_prebuilt_and_passes_ids(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Allowlist mode always selects the source build and passes the built-in IDs.""" + calls: list[tuple[Any, ...]] = [] + + def _fake_run(stages: int, fn: Any, arg: Path, **kwargs: Any) -> None: + calls.append((stages, fn, arg, kwargs)) + + monkeypatch.setattr(cli, "run", _fake_run) + output: Path = tmp_path / "filtered.redb" + cli.build_fullmap(output=output, taxon_allowlist=True) + assert len(calls) == 1 + assert calls[0][0] == 3 + assert calls[0][1] is cli.build_fullmap_pipeline + assert calls[0][3]["taxon_allowlist"] == cli.load_taxon_allowlist() + + calls.clear() + output.write_bytes(b"unfiltered-existing") + cli.build_fullmap(output=output, taxon_allowlist=True) + assert len(calls) == 1 + assert calls[0][1] is cli.build_fullmap_pipeline + + monkeypatch.setattr(cli, "fetch_prebuilt_fullmap", lambda *args, **kwargs: (_ for _ in ()).throw(AssertionError("prebuilt must not run"))) + calls.clear() + cli.build_fullmap(output=tmp_path / "forced-filtered.redb", taxon_allowlist=True, force=True) + assert calls[0][1] is cli.build_fullmap_pipeline + + def test_build_fullmap_command_skips_when_output_exists(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None: """A complete existing DB short-circuits everything (no ``run`` call) unless ``--force``.""" output: Path = tmp_path / "fullmap.redb" diff --git a/tests/test_cover_fullmap.py b/tests/test_cover_fullmap.py index ad6669f..856efa7 100644 --- a/tests/test_cover_fullmap.py +++ b/tests/test_cover_fullmap.py @@ -62,6 +62,70 @@ def fullmap_db(tmp_path: Path) -> Path: return output +def test_taxon_allowlist_and_inchikey_paths(tmp_path: Path) -> None: + """The opt-in builder filters taxon-bearing rows but keeps organism/taxonless rows and InChIKeys.""" + synonyms = write_jsonl( + tmp_path / "SRC.ndjson", + [ + {"curie": "HGNC:1", "preferred_name": "human", "names": ["human gene"], "types": ["Gene"], "taxa": ["NCBITaxon:9606"]}, + {"curie": "HGNC:2", "preferred_name": "mouse", "names": ["mouse gene"], "types": ["Gene"], "taxa": ["NCBITaxon:10090"]}, + {"curie": "HGNC:3", "preferred_name": "other", "names": ["other gene"], "types": ["Gene"], "taxa": ["NCBITaxon:999999"]}, + { + "curie": "NCBITaxon:999999", + "preferred_name": "taxon", + "names": ["other taxon"], + "types": ["OrganismTaxon"], + "taxa": ["NCBITaxon:999999"], + }, + { + "curie": "NCBITaxon:888888", + "preferred_name": "multi-category taxon", + "names": ["multi-category taxon"], + "types": ["NamedThing", "biolink:OrganismTaxon"], + "taxa": ["NCBITaxon:888888"], + }, + {"curie": "CHEBI:1", "preferred_name": "untaxed", "names": ["untaxed chemical"], "types": ["ChemicalEntity"], "taxa": []}, + {"curie": "CHEBI:2", "preferred_name": "sentinel", "names": ["sentinel chemical"], "types": ["ChemicalEntity"], "taxa": ["NCBITaxon:0"]}, + { + "curie": "CHEBI:3", + "preferred_name": "malformed", + "names": ["malformed chemical"], + "types": ["ChemicalEntity"], + "taxa": ["not-a-taxon"], + }, + { + "curie": "HGNC:4", + "preferred_name": "multi", + "names": ["multi gene"], + "types": ["Gene"], + "taxa": ["NCBITaxon:999999", "NCBITaxon:9606"], + }, + { + "curie": "HGNC:5", + "preferred_name": "case-insensitive", + "names": ["case-insensitive gene"], + "types": ["Gene"], + "taxa": ["ncbitaxon:9606"], + }, + {"curie": "INCHIKEY:ABC-DEF", "preferred_name": "inchi", "names": ["ABC-DEF"], "types": ["SmallMolecule"], "taxa": ["NCBITaxon:9606"]}, + ], + ) + output = tmp_path / "fullmap.redb" + rs.build_fullmap_db(output, [], [synonyms], taxon_allowlist=[9606]) + assert rs.lookup_fullmap_terms(output, ["human gene"])[0]["CURIE"] == "HGNC:1" + assert rs.lookup_fullmap_terms(output, ["mouse gene"]) == [] + assert rs.lookup_fullmap_terms(output, ["other gene"]) == [] + assert rs.lookup_fullmap_terms(output, ["other taxon"])[0]["CURIE"] == "NCBITaxon:999999" + assert rs.lookup_fullmap_terms(output, ["multi-category taxon"])[0]["CURIE"] == "NCBITaxon:888888" + assert rs.lookup_fullmap_terms(output, ["untaxed chemical"])[0]["CURIE"] == "CHEBI:1" + assert rs.lookup_fullmap_terms(output, ["sentinel chemical"])[0]["CURIE"] == "CHEBI:2" + assert rs.lookup_fullmap_terms(output, ["malformed chemical"])[0]["CURIE"] == "CHEBI:3" + assert rs.lookup_fullmap_terms(output, ["multi gene"])[0]["CURIE"] == "HGNC:4" + assert rs.lookup_fullmap_terms(output, ["case-insensitive gene"])[0]["CURIE"] == "HGNC:5" + assert rs.lookup_fullmap_terms(output, ["abc-def"])[0]["CURIE"] == "INCHIKEY:ABC-DEF" + assert rs.lookup_fullmap_terms(output, ["inchikey:abc-def"])[0]["CURIE"] == "INCHIKEY:ABC-DEF" + + def test_lookup_rows_empty_terms_returns_empty(tmp_path: Path) -> None: """Line 119: ``lookup_rows`` short-circuits to ``[]`` for an empty term list.