feat: make section store hashes content-aware

docs/api/utils.md

@@ -22,7 +22,7 @@ Generates an **xxh64** digest (hex string, 16 characters) for arbitrary input, c
 def mkhash(x: Any) -> str
 ```
 
-The input is converted to a string and UTF-8 encoded before hashing. The digest is the content-addressed identity for sections: the per-section parquet store filename (`{hash}.parquet` in `.tablassert/store/`) and the section label in validation errors derive from it. User-facing progress labels truncate it to 8 characters for display.
+The input is converted to a string and UTF-8 encoded before hashing. `mkhash` is a general-purpose digest for arbitrary values, including section configuration; it is not itself a source-file hash. The build pipeline uses the section's `mkhash` value as one component of the build-time store key described below. Configuration-only validation uses this digest without reading source files; build-time progress and section-validation errors use the content-aware key, while source-file hashing errors retain the configuration digest as a fallback label. User-facing progress labels truncate the relevant key to 8 characters for display.
 
 The full 64-bit digest is used deliberately: a 32-bit hash would invite birthday collisions (~50% at ~77k sections) that could silently reuse another section's cached subgraph.
 
@@ -34,6 +34,38 @@ mkhash("hello")  # "26c7827d889f6da3"
 
 **Deterministic:** the same input always produces the same digest.
 
+## file_content_hash()
+
+Generates an **xxh64** digest of a local source file's raw bytes. It resolves the path and raises `SourceFileError` if the path does not exist, is not a regular file, or cannot be read.
+
+```python
+def file_content_hash(
+    path: Path,
+    *,
+    config: Path | None = None,
+    section_label: str | None = None,
+) -> str
+```
+
+This helper does not cache the digest. An edited source file therefore produces a different digest and, in a build, a different section store key. `config` and `section_label` are optional context included in a source-file error when available.
+
+## section_store_key()
+
+Returns the build-time content-aware store key for a section's cached parquet.
+
+```python
+def section_store_key(
+    section: Any,
+    local: Path | None = None,
+    *,
+    content_digest: str | None = None,
+) -> str
+```
+
+For a section with a local source, the key combines the section configuration digest with the source content digest and hashes the result into one 16-character xxh64 value. Callers may provide a precomputed `content_digest` to avoid rereading the file; otherwise `local` is passed to `file_content_hash`. If neither is provided, the function returns the section's plain `mkhash` value. The build pipeline uses the content-aware form for local sources, so changing one source invalidates the cached parquet for each section that reads it without invalidating sections backed by unchanged sources.
+
+The key formula is `mkhash(f"{mkhash(section)}:{content_digest}")`. Build-mode flags add their own filename suffixes (`.head`, `.release`, or `.qc`) after the key; they do not change the digest formula.
+
 ---
 
 ## namespace_uuid()  *(Rust extension: `tablassert.rs`)*

docs/cli.md

@@ -185,8 +185,11 @@ identical at any worker count.
 
 Output is written to `rig.artifact_base_path` (created when missing) as `{name}_{version}.nodes.ndjson`,
 `{name}_{version}.edges.ndjson`, and `{name}_{version}.RIG.yaml`; intermediate parquet lands in
-`.tablassert/store/`. The RIG document is audited in memory before it is written: an invalid
-or incomplete RIG fails the build with `[rig-validation-failed]` and nothing is emitted. See
+`.tablassert/store/`. Build-time store keys incorporate the source-file content, so editing a source
+file rebuilds exactly the sections that read the changed content. The first build after this key-format
+upgrade rebuilds every section once because the keys have changed; orphaned old parquet files are not
+automatically deleted. The RIG document is audited in memory before it is written: an invalid or
+incomplete RIG fails the build with `[rig-validation-failed]` and nothing is emitted. See
 [Graph Configuration](configuration/graph.md).
 
 ??? info "Build progress & stages"

rust/src/lib.rs

@@ -1,11 +1,15 @@
+use pyo3::exceptions::PyOSError;
 use pyo3::prelude::*;
 
 // mimalloc returns memory to the OS far better than glibc malloc under heavy
 // multi-threaded allocation (the synonym phase runs many worker threads each
 // making millions of small String/Vec allocations).  Without it, per-thread
 // malloc arenas retain freed memory and inflate peak RSS several-fold.
 use mimalloc::MiMalloc;
-use xxhash_rust::xxh64::xxh64 as xxh64_digest;
+use std::fs::File;
+use std::io::Read;
+use std::path::PathBuf;
+use xxhash_rust::xxh64::{xxh64 as xxh64_digest, Xxh64};
 
 #[global_allocator]
 static GLOBAL: MiMalloc = MiMalloc;
@@ -26,6 +30,27 @@ fn xxh64(data: &str) -> String {
     format!("{:016x}", xxh64_digest(data.as_bytes(), 0))
 }
 
+/// XXH64 hex digest (seed 0) of a file's raw bytes, streamed in fixed-size chunks.
+#[pyfunction]
+fn xxh64_file(path: PathBuf) -> PyResult<String> {
+    const CHUNK_SIZE: usize = 8 * 1024 * 1024;
+
+    let mut file = File::open(&path)
+        .map_err(|error| PyOSError::new_err(format!("{}: {}", path.display(), error)))?;
+    let mut hasher = Xxh64::new(0);
+    let mut chunk = vec![0; CHUNK_SIZE];
+    loop {
+        let bytes_read = file
+            .read(&mut chunk)
+            .map_err(|error| PyOSError::new_err(format!("{}: {}", path.display(), error)))?;
+        if bytes_read == 0 {
+            break;
+        }
+        hasher.update(&chunk[..bytes_read]);
+    }
+    Ok(format!("{:016x}", hasher.digest()))
+}
+
 // Public re-exports of the fullmap `#[pyfunction]`s so Rust integration tests
 // (`rust/tests/`) and other non-Python embedders can drive the exact production
 // build/read path.  The `fullmap` module itself stays private; only these
@@ -48,12 +73,13 @@ fn rs(module: &Bound<'_, PyModule>) -> PyResult<()> {
     module.add_function(wrap_pyfunction!(ndjson::dedup_ndjson, module)?)?;
     module.add_function(wrap_pyfunction!(uuid::namespace_uuid, module)?)?;
     module.add_function(wrap_pyfunction!(xxh64, module)?)?;
+    module.add_function(wrap_pyfunction!(xxh64_file, module)?)?;
     Ok(())
 }
 
 #[cfg(test)]
 mod tests {
-    use super::xxh64;
+    use super::{xxh64, xxh64_file};
 
     #[test]
     fn xxh64_matches_known_digests() {
@@ -63,4 +89,33 @@ mod tests {
         assert_eq!(xxh64("hello"), "26c7827d889f6da3");
         assert_eq!(xxh64(""), "ef46db3751d8e999");
     }
+
+    #[test]
+    fn xxh64_file_matches_known_digests() {
+        // WHY: pin file hashing to the same seed-0 digest as the string primitive,
+        // including the empty-file identity value and UTF-8 bytes.
+        let dir = tempfile::tempdir().unwrap();
+        let path = dir.path().join("hello");
+        std::fs::write(&path, b"hello").unwrap();
+        assert_eq!(xxh64_file(path).unwrap(), "26c7827d889f6da3");
+
+        let empty = dir.path().join("empty");
+        std::fs::write(&empty, b"").unwrap();
+        assert_eq!(xxh64_file(empty).unwrap(), "ef46db3751d8e999");
+    }
+
+    #[test]
+    fn xxh64_file_reports_io_failures_as_os_error() {
+        // WHY: callers must distinguish I/O failures from valid digests and see
+        // the offending path, including for missing files and directories.
+        let dir = tempfile::tempdir().unwrap();
+        let missing = dir.path().join("missing");
+        let error = xxh64_file(missing.clone()).unwrap_err();
+        assert!(error.to_string().contains(&missing.display().to_string()));
+
+        let error = xxh64_file(dir.path().to_path_buf()).unwrap_err();
+        assert!(error
+            .to_string()
+            .contains(&dir.path().display().to_string()));
+    }
 }

src/tablassert/cli.py

@@ -75,6 +75,44 @@ def _section_store_path(h: str, head: bool = False, release: bool = False, qc: b
     return STORE / f"{h}{suffix}.parquet"
 
 
+def _section_store_key_for_build(
+    section: dict[str, Any], configuration_file: Path, content_hashes: dict[tuple[Path, int, int], str]
+) -> tuple[str, str]:
+    """Return a section's content-aware store key and config-only fallback label.
+
+    The memo belongs to one ``build_graph_pipeline`` call and is invalidated by any
+    ``(resolved path, mtime_ns, size)`` change. Progress and validation errors use the
+    content-aware key; the returned config-only hash labels ``SourceFileError`` when
+    hashing fails before a content-aware key can exist.
+    """
+    from tablassert.utils import file_content_hash, mkhash, section_store_key
+
+    config_hash: str = mkhash(section)
+    source: object = section.get("source")
+    local: object = source.get("local") if isinstance(source, dict) else None
+    if local is None:
+        return section_store_key(section), config_hash
+
+    resolved: Path = Path(str(local)).resolve()
+    config_value: object = section.get("config", "section")
+    section_label: str = f"{Path(str(config_value)).stem} · {config_hash[:8]}"
+    digest: str | None = None
+    try:
+        stat = resolved.stat()
+    except OSError:
+        # Reuse the contextual helper so missing/unreadable paths never leak a raw OS error.
+        digest = file_content_hash(resolved, config=configuration_file, section_label=section_label)
+        return section_store_key(section, content_digest=digest), config_hash
+
+    signature: tuple[Path, int, int] = (resolved, stat.st_mtime_ns, stat.st_size)
+    if digest is None:
+        digest = content_hashes.get(signature)
+    if digest is None:
+        digest = file_content_hash(resolved, config=configuration_file, section_label=section_label)
+        content_hashes[signature] = digest
+    return section_store_key(section, content_digest=digest), config_hash
+
+
 def _load_table_indexed(args: tuple[int, Path]) -> tuple[int, object]:
     """Load one table, tagged with its input index (multiprocessing worker).
 
@@ -184,7 +222,6 @@ def build_graph_pipeline(
     from tablassert.fullmap import fullmap_db_path
     from tablassert.lib import Tcode, compile_graph, compile_subgraph
     from tablassert.progress import flatten_pydantic_error, format_section_compact
-    from tablassert.utils import mkhash
 
     # Stage 1/6: load tables.
     progress.stage("Loading Tables")
@@ -223,8 +260,11 @@ def build_graph_pipeline(
     progress.stage("Building TCode")
     start, advance, _ = progress.section_loop(n, "TCode")
     tcode: list[Tcode] = []
+    # This memo is intentionally scoped to one build: a changed stat signature re-hashes,
+    # while repeated sections pointing to the same unchanged file hash only once.
+    content_hashes: dict[tuple[Path, int, int], str] = {}
     for s in sections:
-        h: str = mkhash(s)
+        h, _ = _section_store_key_for_build(s, configuration_file, content_hashes)
         start(f"{Path(str(s['config'])).stem} · {h[:8]}")
         # Mode flags change the cached parquet's content, so each combination caches
         # to a distinct file and can never quick-exit another mode's build.

src/tablassert/errors.py

@@ -12,6 +12,7 @@
     "graph-validation-failed",
     "section-validation-failed",
     "babel-download-failed",
+    "source-file-unreadable",
     "resolve-bad-specs",
     "config-rows-and-row-slice-conflict",
     "comparison-bad-comparator-type",
@@ -185,3 +186,24 @@ def __init__(self, url: str, retries: int, last_error: BaseException) -> None:
             f"BABEL download failed after {retries} attempts: {url} (last error: {last_error}). Check network connectivity or pin a different BABEL version.",
             code="babel-download-failed",
         )
+
+
+class SourceFileError(TablassertError):
+    """A table section's ``source.local`` file could not be read for content-hashing.
+
+    Notes:
+        ``config`` and ``section_label`` are optional keyword context: the utils layer
+        that first notices the failure often lacks build context, but callers that have
+        it (the CLI's section loop) should pass them so the message names the exact
+        table config and section label (config stem + 8-char section digest).
+    """
+
+    def __init__(self, path: Path, detail: str, *, config: Path | None = None, section_label: str | None = None) -> None:
+        context: str = ""
+        if config is not None and section_label is not None:
+            context = f" (config {config}, section {section_label})"
+        elif config is not None:
+            context = f" (config {config})"
+        elif section_label is not None:
+            context = f" (section {section_label})"
+        super().__init__(f"Source file unreadable{context}: {path} — {detail}", code="source-file-unreadable")

src/tablassert/rs.pyi

@@ -19,3 +19,4 @@ def hydrate_sources(db: Path) -> list[str]: ...
 def lookup_fullmap_terms(db: Path, terms: list[str], return_format: str = "rows") -> list[dict[str, Any]]: ...
 def namespace_uuid(domain: str, values: list[str]) -> str: ...
 def xxh64(data: str) -> str: ...
+def xxh64_file(path: str) -> str: ...

src/tablassert/utils.py

@@ -4,6 +4,7 @@
 from typing import Any
 
 from tablassert import rs
+from tablassert.errors import SourceFileError
 
 BASE: Path = Path("./.tablassert")
 STORE: Path = BASE / "store"
@@ -16,3 +17,37 @@ def mkhash(x: Any) -> str:
     # birthday collisions (~50% at ~77k sections) that would silently reuse another
     # section's cached subgraph; the full 16-hex 64-bit digest avoids that.
     return rs.xxh64(str(x))
+
+
+def file_content_hash(path: Path, *, config: Path | None = None, section_label: str | None = None) -> str:
+    """XXH64 hex digest of a source file's raw bytes, or a loud :class:`SourceFileError`.
+
+    Pure: no caching, so the digest always reflects the file's bytes at call time and an
+    edited source always yields a new section store key. ``config`` / ``section_label``
+    are optional build context forwarded to the error message when available.
+    """
+    resolved: Path = path.resolve()
+    if not resolved.exists():
+        raise SourceFileError(resolved, "no such file", config=config, section_label=section_label)
+    if not resolved.is_file():
+        raise SourceFileError(resolved, "not a regular file", config=config, section_label=section_label)
+    try:
+        return rs.xxh64_file(str(resolved))
+    except OSError as e:
+        raise SourceFileError(resolved, str(e), config=config, section_label=section_label) from e
+
+
+def section_store_key(section: Any, local: Path | None = None, *, content_digest: str | None = None) -> str:
+    """Content-addressed identity for a section's cached parquet store.
+
+    With a ``local`` source file or a precomputed ``content_digest`` the key mixes the
+    section config hash with the file's content digest, so editing the source file
+    invalidates the cache; without one the key is the plain section hash. Always a
+    single 16-hex xxh64 digest. ``content_digest`` lets a caller memoize file reads
+    without changing the key formula.
+    """
+    if content_digest is None:
+        if local is None:
+            return mkhash(section)
+        content_digest = file_content_hash(local)
+    return mkhash(f"{mkhash(section)}:{content_digest}")