feat: add opt-in taxon-filtered fullmaps and InChIKey support

docs/cli.md

@@ -125,6 +125,7 @@ tablassert build-fullmap [ARGS]
 | `--version`, `-v` | str | No | `2026jul22` | BABEL snapshot date to fetch (a RENCI stamp, **not** Tablassert's version) |
 | `--aria2c`, `-a` | Flag | No | `False` | Opt into the bundled `aria2c` binary from the `[aria2]` extra for resumable segmented downloads (the prebuilt archive **or** BABEL files); fails loud (exit 2, before any download starts) if the extra is missing or unsupported on the current platform, and on a non-zero aria2c exit |
 | `--force`, `-f` | Flag | No | `False` | Skip the prebuilt download and always rebuild from BABEL outputs |
+| `--taxon-allowlist` | Flag | No | `False` | Use the built-in top-100 experimental-taxon allowlist; always build from source BABEL files and never use the unfiltered prebuilt |
 
 ```bash
 # Default: download the prebuilt fullmap.tar.zst for this version and extract it (fast)
@@ -133,6 +134,8 @@ tablassert build-fullmap --output /data/fullmap/fullmap.redb
 tablassert build-fullmap --force --output /data/fullmap/fullmap.redb
 # Accelerate either download with bundled aria2c (`pip install "tablassert[aria2]"`; the multi-GB prebuilt is the ideal aria2 use case)
 tablassert build-fullmap --aria2c --output /data/fullmap/fullmap.redb
+# Build a smaller source-derived database for the top 100 taxa (does not use the prebuilt archive)
+tablassert build-fullmap --taxon-allowlist --output /data/fullmap/experimental.redb
 ```
 
 By default `build-fullmap` looks for a prebuilt `fullmap.tar.zst` at

docs/fullmap.md

@@ -23,8 +23,19 @@ tablassert build-fullmap --force
 
 # Optional: after `pip install "tablassert[aria2]"`, use bundled aria2c for resumable segmented downloads (the multi-GB prebuilt is the ideal aria2 use case)
 tablassert build-fullmap --aria2c
+
+# Optional: source-build a smaller DB retaining the built-in top-100 taxa
+# (OrganismTaxon and taxonless rows are not filtered)
+tablassert build-fullmap --taxon-allowlist
 ```
 
+`--taxon-allowlist` is opt-in. It uses the checked-in `src/tablassert/data/experimental_taxa.yaml`
+list, filters non-`OrganismTaxon` synonym rows with valid taxon metadata before interning, and always
+builds from the downloaded BABEL sources. It never reuses the unfiltered prebuilt archive. Rows with
+no valid taxon metadata and every `OrganismTaxon` row are retained; a row with multiple taxa is kept
+when any taxon is allowlisted. The first taxon continues to be stored for existing hydration and
+lookup behavior. The built database records its allowlist identity in `META.taxon_allowlist`.
+
 See the [CLI Reference → build-fullmap](cli.md#build-fullmap) for the complete flag table (output path,
 cache directory, BABEL snapshot version, the optional `--aria2c` / `-a` downloader,
 and the `--force` / `-f` rebuild flag), their defaults, and more examples.
@@ -115,7 +126,7 @@ override only when targeting an unusual machine.
 
 | Variable | Default | Description |
 |----------|---------|-------------|
-| `TABLASSERT_FULLMAP_EXCLUDE_PREFIXES` | *(empty)* | Comma-separated CURIE prefixes to drop at build time (e.g. `INCHIKEY,Publication`). Excluding prefixes you never resolve dramatically cuts build time, peak memory, and database size. |
+| `TABLASSERT_FULLMAP_EXCLUDE_PREFIXES` | *(empty)* | Comma-separated CURIE prefixes to drop at build time (e.g. `Publication`). Excluding prefixes you never resolve dramatically cuts build time, peak memory, and database size. InChIKey terms are retained unless excluded explicitly. |
 | `TABLASSERT_FULLMAP_CHUNK_BYTES` | `8388608` (8 MiB) | Byte budget per producer→worker line-chunk. Bounded by bytes (not line count) so chunk memory is fixed even for large synonym records. |
 | `TABLASSERT_FULLMAP_PRODUCERS` | `clamp(workers/4, 4, #files)` | Number of producer (decompressor) threads. Decompression far outpaces parallel processing, so a handful keeps all workers fed. |
 | `TABLASSERT_FULLMAP_LOCAL_SPILL_ENTRIES` | `1000000` | Per-worker term-posting buffer size before spilling a sorted run to disk. Lower → less RAM, more run files. |
@@ -140,7 +151,7 @@ they hold six tables (see `rust/src/fullmap.rs`):
 | `categories` | Compact `u16` id → Biolink category string (primary file) |
 | `sources` | Compact `u8` id → source metadata (name/version) (primary file) |
 | `curies` | Compact `u32` id → CURIE record (CURIE, preferred name, category, taxon, source) (primary file) |
-| `meta` | Schema version tag (`tablassert.fullmap.v5`), the shard count (`shards`), and the BABEL `source_version` used to build the file (primary file) |
+| `meta` | Schema version tag (`tablassert.fullmap.v5`), the shard count (`shards`), the BABEL `source_version` used to build the file, and optional allowlist identity (`taxon_allowlist`) (primary file) |
 
 The shard files must remain alongside the primary file: lookups discover them as siblings of the
 resolved primary path.

pyproject.toml

@@ -72,6 +72,7 @@ module-name = "tablassert.rs"
 bindings = "pyo3"
 manifest-path = "rust/Cargo.toml"
 features = ["extension-module"]
+include = ["src/tablassert/data/experimental_taxa.yaml"]
 
 [project.scripts]
 tablassert = "tablassert.cli:APP"

rust/src/fullmap.rs

@@ -294,7 +294,6 @@ fn token_qc(value: &str) -> bool {
         && !value.contains('\t')
         && !value.contains('\n')
         && !value.contains('\r')
-        && !value.contains("inchikey")
         && !value.contains("uncharacterized")
         && !value.contains("hypothetical")
 }
@@ -459,6 +458,15 @@ fn first_category(row: &SynonymRow<'_>) -> String {
         .to_string()
 }
 
+fn is_organism_taxon(row: &SynonymRow<'_>) -> bool {
+    row.types.iter().any(|category| {
+        category
+            .trim()
+            .trim_start_matches("biolink:")
+            .eq_ignore_ascii_case("OrganismTaxon")
+    })
+}
+
 fn first_taxon(row: &SynonymRow<'_>) -> i32 {
     row.taxa
         .first()
@@ -469,6 +477,56 @@ fn first_taxon(row: &SynonymRow<'_>) -> i32 {
         .unwrap_or(0)
 }
 
+fn parsed_taxa(row: &SynonymRow<'_>) -> Vec<i32> {
+    row.taxa
+        .iter()
+        .filter_map(|taxon| {
+            let taxon = taxon.trim();
+            let local = taxon
+                .split_once(':')
+                .filter(|(prefix, _)| prefix.eq_ignore_ascii_case("NCBITaxon"))
+                .map(|(_, local)| local)
+                .unwrap_or(taxon);
+            local.parse::<i32>().ok().filter(|id| *id > 0)
+        })
+        .collect()
+}
+
+fn allowlist_identity(taxa: &HashSet<i32>) -> String {
+    let mut ids: Vec<i32> = taxa.iter().copied().collect();
+    ids.sort_unstable();
+    let encoded = ids
+        .iter()
+        .map(i32::to_string)
+        .collect::<Vec<String>>()
+        .join(",");
+    format!(
+        "count={};xxh64={:016x}",
+        ids.len(),
+        xxh64(encoded.as_bytes(), 0)
+    )
+}
+
+fn retain_for_taxon_allowlist(row: &SynonymRow<'_>, allowlist: Option<&HashSet<i32>>) -> bool {
+    let Some(allowlist) = allowlist else {
+        return true;
+    };
+    if is_organism_taxon(row) {
+        return true;
+    }
+    let mut has_valid_taxon = false;
+    for taxon in parsed_taxa(row) {
+        has_valid_taxon = true;
+        if allowlist.contains(&taxon) {
+            return true;
+        }
+    }
+    // Taxonless rows remain available for categories whose BABEL synonym rows
+    // do not carry a positive taxon ID. This includes the conventional
+    // `NCBITaxon:0` sentinel and malformed/empty taxon values.
+    !has_valid_taxon
+}
+
 fn split_curie(curie: &str) -> Option<(&str, &str)> {
     curie
         .split_once(':')
@@ -1358,6 +1416,7 @@ struct SynonymShared<'a> {
     curie_counter: &'a AtomicU32,
     equivalents: &'a EquivIndex,
     exclude_prefixes: &'a HashSet<String>,
+    taxon_allowlist: Option<&'a HashSet<i32>>,
     spill_dir: &'a Path,
     local_spill: usize,
     curie_spill: usize,
@@ -1393,6 +1452,9 @@ fn process_row(
     let Some((prefix, local_id)) = split_curie(curie) else {
         return Ok(());
     };
+    if !retain_for_taxon_allowlist(row, sh.taxon_allowlist) {
+        return Ok(());
+    }
     // Skip CURIEs whose prefix the caller excludes (opt-in via
     // TABLASSERT_FULLMAP_EXCLUDE_PREFIXES) — filtered out downstream anyway.
     if sh.exclude_prefixes.contains(prefix) {
@@ -1623,6 +1685,7 @@ fn process_synonyms(
     local_spill: usize,
     curie_spill: usize,
     exclude_prefixes: &HashSet<String>,
+    taxon_allowlist: Option<&HashSet<i32>>,
     worker_count: usize,
     shard_count: usize,
     chunk_bytes: usize,
@@ -1671,6 +1734,7 @@ fn process_synonyms(
         curie_counter: &curie_counter,
         equivalents,
         exclude_prefixes,
+        taxon_allowlist,
         spill_dir,
         local_spill,
         curie_spill,
@@ -1799,6 +1863,7 @@ fn write_final_database(
     insert_batch: usize,
     shard_count: usize,
     progress: Option<&Arc<Progress>>,
+    taxon_allowlist_identity: Option<&str>,
 ) -> PyResult<()> {
     let build_id = new_build_id();
     let build_id_str = build_id.to_string();
@@ -1855,6 +1920,9 @@ fn write_final_database(
         let shard_count_str = shard_count.to_string();
         meta.insert("shards", shard_count_str.as_str())
             .map_err(py_err)?;
+        if let Some(identity) = taxon_allowlist_identity {
+            meta.insert("taxon_allowlist", identity).map_err(py_err)?;
+        }
     }
     write.commit().map_err(py_err)?;
 
@@ -2229,6 +2297,8 @@ fn build_fullmap_inner(
     local_spill: usize,
     curie_spill: usize,
     exclude_prefixes: HashSet<String>,
+    taxon_allowlist: Option<HashSet<i32>>,
+    taxon_allowlist_identity: Option<String>,
     chunk_bytes: usize,
     producers: usize,
     cache_bytes: usize,
@@ -2258,6 +2328,7 @@ fn build_fullmap_inner(
             local_spill,
             curie_spill,
             &exclude_prefixes,
+            taxon_allowlist.as_ref(),
             worker_count,
             shard_count,
             chunk_bytes,
@@ -2288,6 +2359,7 @@ fn build_fullmap_inner(
             insert_batch,
             shard_count,
             progress.as_ref(),
+            taxon_allowlist_identity.as_deref(),
         )?;
 
         // Clean up spill runs on success (left in place on error for inspection).
@@ -2297,13 +2369,14 @@ fn build_fullmap_inner(
 }
 
 #[pyfunction]
-#[pyo3(signature = (output, classes, synonyms, progress=None))]
+#[pyo3(signature = (output, classes, synonyms, progress=None, taxon_allowlist=None))]
 pub fn build_fullmap_db(
     py: Python<'_>,
     output: PathBuf,
     classes: Vec<PathBuf>,
     synonyms: Vec<PathBuf>,
     progress: Option<Py<PyAny>>,
+    taxon_allowlist: Option<Vec<i32>>,
 ) -> PyResult<()> {
     if synonyms.is_empty() {
         return Err(PyValueError::new_err(
@@ -2407,6 +2480,9 @@ pub fn build_fullmap_db(
             PathBuf::from(p)
         });
 
+    let taxon_allowlist: Option<HashSet<i32>> =
+        taxon_allowlist.map(|ids| ids.into_iter().filter(|id| *id > 0).collect());
+    let taxon_allowlist_identity: Option<String> = taxon_allowlist.as_ref().map(allowlist_identity);
     let progress = progress.map(|cb| Arc::new(Progress { cb }));
 
     // Release the GIL for the whole build so rich's Live display thread can
@@ -2422,6 +2498,8 @@ pub fn build_fullmap_db(
             local_spill,
             curie_spill,
             exclude_prefixes,
+            taxon_allowlist,
+            taxon_allowlist_identity,
             chunk_bytes,
             producers,
             cache_bytes,
@@ -3650,6 +3728,8 @@ mod tests {
             local_spill,
             1_000_000,
             HashSet::new(),
+            None,
+            None,
             DEFAULT_CHUNK_BYTES,
             2,
             64 * 1024 * 1024,
@@ -3678,10 +3758,12 @@ mod tests {
     }
 
     #[test]
-    fn token_qc_rejects_banned_tokens() {
+    fn token_qc_rejects_banned_tokens_but_accepts_inchikey() {
         assert!(token_qc("brca1"));
+        assert!(token_qc("inchikey=abc"));
         assert!(!token_qc("hypothetical protein"));
         assert!(!token_qc("line\nbreak"));
+        assert!(!token_qc("tab\tvalue"));
     }
 
     #[test]
@@ -4028,6 +4110,7 @@ mod tests {
             30,
             1_000_000,
             &HashSet::new(),
+            None,
             4,
             shard_count,
             DEFAULT_CHUNK_BYTES,
@@ -4085,6 +4168,8 @@ mod tests {
             100,
             1_000_000,
             HashSet::new(),
+            None,
+            None,
             DEFAULT_CHUNK_BYTES,
             2,
             64 * 1024 * 1024,
@@ -4102,6 +4187,8 @@ mod tests {
             100,
             1_000_000,
             HashSet::new(),
+            None,
+            None,
             DEFAULT_CHUNK_BYTES,
             2,
             64 * 1024 * 1024,
@@ -4519,6 +4606,8 @@ mod tests {
             4_000_000,
             1_000_000,
             HashSet::new(),
+            None,
+            None,
             DEFAULT_CHUNK_BYTES,
             2,
             64 * 1024 * 1024,
@@ -4588,7 +4677,7 @@ mod tests {
 
         std::env::set_var("TABLASSERT_FULLMAP_SHARDS", "2");
         let built = Python::attach(|py| {
-            build_fullmap_db(py, output.clone(), Vec::new(), vec![synonyms], None)
+            build_fullmap_db(py, output.clone(), Vec::new(), vec![synonyms], None, None)
         });
         std::env::remove_var("TABLASSERT_FULLMAP_SHARDS");
         built.unwrap();
@@ -4649,6 +4738,8 @@ mod tests {
             4_000_000,
             1_000_000,
             HashSet::new(),
+            None,
+            None,
             DEFAULT_CHUNK_BYTES,
             2,
             64 * 1024 * 1024,
@@ -5356,6 +5447,8 @@ mod tests {
             4_000_000,
             2,
             HashSet::new(),
+            None,
+            None,
             DEFAULT_CHUNK_BYTES,
             2,
             64 * 1024 * 1024,
@@ -5446,6 +5539,8 @@ mod tests {
             4_000_000,
             1_000_000,
             exclude,
+            None,
+            None,
             DEFAULT_CHUNK_BYTES,
             2,
             64 * 1024 * 1024,
@@ -5505,6 +5600,8 @@ mod tests {
             4_000_000,
             1_000_000,
             HashSet::new(),
+            None,
+            None,
             8192,
             2,
             64 * 1024 * 1024,
@@ -5546,6 +5643,7 @@ mod tests {
                 Vec::new(),
                 Vec::new(),
                 None,
+                None,
             )
             .expect_err("empty synonyms should fail");
             assert!(err