P0: Harden deterministic document chunking contract

.github/workflows/test.yml

@@ -0,0 +1,16 @@
+name: test
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - run: python -m pip install --upgrade pip pytest
+      - run: python -m pytest

chunking/chunker.py

@@ -1,19 +1,19 @@
-"""Sentence-aware token-bounded chunking."""
+"""Sentence-aware, approximately token-bounded chunking with whole-sentence overlap."""
 
 from __future__ import annotations
 
+import hashlib
 import re
-import uuid
 from typing import Any
 
 from .types import Chunk
 
 _CHARS_PER_TOKEN = 4
-_BOUNDARY_RE = re.compile(r".*?(?:\. |\n|$)", re.DOTALL)
+_BOUNDARY_RE = re.compile(r".*?(?:(?:[.!?](?=\s|$))|\n|$)", re.DOTALL)
 
 
 class TokenChunker:
-    """Split text into overlapping, approximately token-bounded chunks."""
+    """Split text into approximately token-bounded chunks with sentence overlap."""
 
     def chunk(
         self,
@@ -34,7 +34,7 @@
         if overlap >= max_tokens:
             raise ValueError("overlap must be smaller than max_tokens")
 
-        document_id = document_id or str(uuid.uuid4())
+        document_id = document_id or hashlib.sha256(content.encode("utf-8")).hexdigest()
         max_chars = max_tokens * _CHARS_PER_TOKEN
         overlap_chars = overlap * _CHARS_PER_TOKEN
         segments = self._split_sentences(content)
@@ -56,8 +56,14 @@
             length = end - start
             if length > max_chars:
                 flush()
-                for pos in range(start, end, max_chars):
-                    ranges.append((pos, min(pos + max_chars, end)))
+                stride = max_chars - overlap_chars
+                pos = start
+                while pos < end:
+                    chunk_end = min(pos + max_chars, end)
+                    ranges.append((pos, chunk_end))
+                    if chunk_end >= end:
+                        break
+                    pos += stride
                 continue
 
             if current_start is not None and current_end - current_start + length > max_chars:
@@ -78,6 +84,15 @@
                         current_end = tail[-1][1]
                         current_segments = tail
 
+                # Whole-sentence overlap must never make the next chunk exceed
+                # the configured bound. If the tail does not leave room for the
+                # incoming sentence, sacrifice overlap at this boundary rather
+                # than emitting a redundant tail-only chunk.
+                if current_start is not None and current_end - current_start + length > max_chars:
+                    current_start = None
+                    current_end = 0
+                    current_segments = []
+
             if current_start is None:
                 current_start = start
             current_end = end

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "chunking"
-version = "0.1.0"
+version = "0.1"
 description = "Canonical document-to-chunk capability for FlossWare"
 requires-python = ">=3.10"
 readme = "README.md"

tests/fixtures/README.md

@@ -0,0 +1 @@
+This fixture mirrors the scraping capability's AcquiredResource contract without importing the scraping package.

tests/fixtures/acquired_resource.json

@@ -0,0 +1,9 @@
+{
+  "uri": "file://exports/papers/example.pdf",
+  "media_type": "application/pdf",
+  "content_hash": "sha256:example",
+  "raw_path": "/exports/raw/example.pdf",
+  "size": 12345,
+  "retrieved_at": "2026-09-04T00:00:00+00:00",
+  "discovered_by": "explicit"
+}

tests/test_chunker.py

@@ -1,6 +1,15 @@
 from chunking import TokenChunker
 
 
+def assert_chunk_invariants(text, chunks, max_tokens):
+    assert chunks
+    for sequence, chunk in enumerate(chunks):
+        assert chunk.sequence == sequence
+        assert chunk.start_offset < chunk.end_offset
+        assert text[chunk.start_offset : chunk.end_offset] == chunk.content
+        assert chunk.token_count <= max_tokens
+
+
 def test_empty_content_returns_no_chunks():
     assert TokenChunker().chunk("") == []
 
@@ -14,6 +23,57 @@ def test_chunk_has_contract_and_offsets():
     assert chunks[0].start_offset == 0
     assert chunks[0].end_offset == len(text)
     assert chunks[0].id == "doc-1:0"
+    assert_chunk_invariants(text, chunks, 10)
+
+
+def test_automatic_document_id_is_deterministic():
+    text = "Deterministic Unicode: café 東京."
+    first = TokenChunker().chunk(text, max_tokens=4, overlap=1)
+    second = TokenChunker().chunk(text, max_tokens=4, overlap=1)
+    assert first == second
+    assert first[0].document_id
+    assert_chunk_invariants(text, first, 4)
+
+
+def test_long_segment_uses_requested_overlap():
+    text = "x" * 100
+    chunks = TokenChunker().chunk(text, document_id="long", max_tokens=10, overlap=2)
+    assert len(chunks) > 1
+    for previous, current in zip(chunks, chunks[1:]):
+        assert previous.end_offset - current.start_offset == 8
+        assert current.start_offset < current.end_offset
+    assert_chunk_invariants(text, chunks, 10)
+
+
+def test_sentence_boundaries_include_question_and_exclamation():
+    text = "First? Second! Third."
+    ranges = TokenChunker._split_sentences(text)
+    assert [text[start:end] for start, end in ranges] == ["First?", " Second!", " Third."]
+
+
+def test_sentence_packing_never_exceeds_max_tokens_or_emits_redundant_tail():
+    text = "A" * 15 + "." + " B" * 8 + "." + " C" * 14 + "."
+    chunks = TokenChunker().chunk(text, document_id="bounded", max_tokens=10, overlap=5)
+    assert len(chunks) >= 2
+    assert_chunk_invariants(text, chunks, 10)
+    for previous, current in zip(chunks, chunks[1:]):
+        assert current.content not in previous.content
+        assert previous.content not in current.content
+
+
+def test_unicode_offsets_and_metadata_are_preserved():
+    text = "Unicode café 東京. Next line."
+    metadata = {"uri": "file://exports/papers/example.pdf", "media_type": "application/pdf"}
+    provenance = {"source": "scraping", "content_hash": "abc123"}
+    chunks = TokenChunker().chunk(
+        text, document_id="doc", max_tokens=20, overlap=1, metadata=metadata, provenance=provenance
+    )
+    assert chunks[0].content == text
+    assert chunks[0].start_offset == 0
+    assert chunks[0].end_offset == len(text)
+    assert chunks[0].metadata == metadata
+    assert chunks[0].provenance == provenance
+    assert_chunk_invariants(text, chunks, 20)
 
 
 def test_invalid_configuration_is_rejected():

tests/test_scraping_contract.py

@@ -0,0 +1,32 @@
+import json
+from pathlib import Path
+
+from chunking import TokenChunker
+
+
+FIXTURE = Path(__file__).parent / "fixtures" / "acquired_resource.json"
+
+
+def test_scraping_acquired_resource_contract_can_feed_chunking():
+    resource = json.loads(FIXTURE.read_text(encoding="utf-8"))
+    assert set(resource) == {
+        "uri",
+        "media_type",
+        "content_hash",
+        "raw_path",
+        "size",
+        "retrieved_at",
+        "discovered_by",
+    }
+
+    content = "A paper document. A second paragraph."
+    chunks = TokenChunker().chunk(
+        content,
+        document_id=resource["content_hash"],
+        metadata={"uri": resource["uri"], "media_type": resource["media_type"]},
+        provenance=resource,
+    )
+
+    assert chunks
+    assert chunks[0].document_id == resource["content_hash"]
+    assert chunks[0].provenance == resource