From 0e84e897c6ab0103fad5c01774b04d8b0a13617a Mon Sep 17 00:00:00 2001 From: "Scot P. Floess" Date: Fri, 4 Sep 2026 12:27:25 -0400 Subject: [PATCH 01/22] Harden deterministic chunking and long-segment overlap --- chunking/chunker.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/chunking/chunker.py b/chunking/chunker.py index f1ea74d..21f0506 100644 --- a/chunking/chunker.py +++ b/chunking/chunker.py @@ -2,14 +2,14 @@ from __future__ import annotations +import hashlib import re -import uuid from typing import Any from .types import Chunk _CHARS_PER_TOKEN = 4 -_BOUNDARY_RE = re.compile(r".*?(?:\. |\n|$)", re.DOTALL) +_BOUNDARY_RE = re.compile(r".*?(?:(?:[.!?](?=\s|$))|\n|$)", re.DOTALL) class TokenChunker: @@ -34,7 +34,7 @@ def chunk( if overlap >= max_tokens: raise ValueError("overlap must be smaller than max_tokens") - document_id = document_id or str(uuid.uuid4()) + document_id = document_id or hashlib.sha256(content.encode("utf-8")).hexdigest() max_chars = max_tokens * _CHARS_PER_TOKEN overlap_chars = overlap * _CHARS_PER_TOKEN segments = self._split_sentences(content) @@ -56,7 +56,8 @@ def flush() -> None: length = end - start if length > max_chars: flush() - for pos in range(start, end, max_chars): + stride = max_chars - overlap_chars + for pos in range(start, end, stride): ranges.append((pos, min(pos + max_chars, end))) continue From 886e01f35080b7dfc21223531a84b3dde6b37fd8 Mon Sep 17 00:00:00 2001 From: "Scot P. Floess" Date: Fri, 4 Sep 2026 12:27:34 -0400 Subject: [PATCH 02/22] Expand chunking contract and determinism tests --- tests/test_chunker.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/tests/test_chunker.py b/tests/test_chunker.py index a43f9b3..b23fcf0 100644 --- a/tests/test_chunker.py +++ b/tests/test_chunker.py @@ -16,6 +16,43 @@ def test_chunk_has_contract_and_offsets(): assert chunks[0].id == "doc-1:0" +def test_automatic_document_id_is_deterministic(): + text = "Deterministic Unicode: café 東京." + first = TokenChunker().chunk(text, max_tokens=4, overlap=1) + second = TokenChunker().chunk(text, max_tokens=4, overlap=1) + assert first == second + assert first[0].document_id + + +def test_long_segment_uses_requested_overlap(): + text = "x" * 100 + chunks = TokenChunker().chunk(text, document_id="long", max_tokens=10, overlap=2) + assert len(chunks) > 1 + for previous, current in zip(chunks, chunks[1:]): + assert previous.end_offset - current.start_offset == 8 + assert current.start_offset < current.end_offset + + +def test_sentence_boundaries_include_question_and_exclamation(): + text = "First? Second! Third." + ranges = TokenChunker._split_sentences(text) + assert [text[start:end] for start, end in ranges] == ["First?", " Second!", " Third."] + + +def test_unicode_offsets_and_metadata_are_preserved(): + text = "Unicode café 東京. Next line." + metadata = {"uri": "file://exports/papers/example.pdf", "media_type": "application/pdf"} + provenance = {"source": "scraping", "content_hash": "abc123"} + chunks = TokenChunker().chunk( + text, document_id="doc", max_tokens=20, metadata=metadata, provenance=provenance + ) + assert chunks[0].content == text + assert chunks[0].start_offset == 0 + assert chunks[0].end_offset == len(text) + assert chunks[0].metadata == metadata + assert chunks[0].provenance == provenance + + def test_invalid_configuration_is_rejected(): chunker = TokenChunker() for kwargs in ({"max_tokens": 0}, {"max_tokens": 10, "overlap": -1}, {"max_tokens": 10, "overlap": 10}): From 4ce8b170e197aedb5a42fd0ec2e89e7b3ee8092c Mon Sep 17 00:00:00 2001 From: "Scot P. Floess" Date: Fri, 4 Sep 2026 12:27:40 -0400 Subject: [PATCH 03/22] Use FlossWare X.Y package version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f13237b..21dd78e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "chunking" -version = "0.1.0" +version = "0.1" description = "Canonical document-to-chunk capability for FlossWare" requires-python = ">=3.10" readme = "README.md" From 74cd00c606a251f342cb948acdb84cbb57441b62 Mon Sep 17 00:00:00 2001 From: "Scot P. Floess" Date: Fri, 4 Sep 2026 12:27:46 -0400 Subject: [PATCH 04/22] Add scraping contract fixture --- tests/fixtures/acquired_resource.json | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 tests/fixtures/acquired_resource.json diff --git a/tests/fixtures/acquired_resource.json b/tests/fixtures/acquired_resource.json new file mode 100644 index 0000000..dc05078 --- /dev/null +++ b/tests/fixtures/acquired_resource.json @@ -0,0 +1,9 @@ +{ + "uri": "file://exports/papers/example.pdf", + "media_type": "application/pdf", + "content_hash": "sha256:example", + "raw_path": "/exports/raw/example.pdf", + "size": 12345, + "retrieved_at": "2026-09-04T00:00:00+00:00", + "discovered_by": "explicit" +} From fa307038f75f0ec22a2d1ee65c468260715614e2 Mon Sep 17 00:00:00 2001 From: "Scot P. Floess" Date: Fri, 4 Sep 2026 12:27:51 -0400 Subject: [PATCH 05/22] Test scraping-to-chunking contract boundary --- tests/test_scraping_contract.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 tests/test_scraping_contract.py diff --git a/tests/test_scraping_contract.py b/tests/test_scraping_contract.py new file mode 100644 index 0000000..c62a04b --- /dev/null +++ b/tests/test_scraping_contract.py @@ -0,0 +1,32 @@ +import json +from pathlib import Path + +from chunking import TokenChunker + + +FIXTURE = Path(__file__).parent / "fixtures" / "acquired_resource.json" + + +def test_scraping_acquired_resource_contract_can_feed_chunking(): + resource = json.loads(FIXTURE.read_text(encoding="utf-8")) + assert set(resource) == { + "uri", + "media_type", + "content_hash", + "raw_path", + "size", + "retrieved_at", + "discovered_by", + } + + content = "A paper document. A second paragraph." + chunks = TokenChunker().chunk( + content, + document_id=resource["content_hash"], + metadata={"uri": resource["uri"], "media_type": resource["media_type"]}, + provenance=resource, + ) + + assert chunks + assert chunks[0].document_id == resource["content_hash"] + assert chunks[0].provenance == resource From e34daf620ec13e1ccd4dc2493ede27b5cffd5797 Mon Sep 17 00:00:00 2001 From: "Scot P. Floess" Date: Fri, 4 Sep 2026 12:27:55 -0400 Subject: [PATCH 06/22] Add minimal CI test workflow --- .github/workflows/test.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..0e21d38 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,16 @@ +name: test + +on: + push: + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - run: python -m pip install --upgrade pip pytest + - run: python -m pytest From 56833c0374d3bc611654e4e7cede234f232e4698 Mon Sep 17 00:00:00 2001 From: "Scot P. Floess" Date: Fri, 4 Sep 2026 12:27:59 -0400 Subject: [PATCH 07/22] Keep chunking fixtures directory --- tests/fixtures/.gitkeep | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/fixtures/.gitkeep diff --git a/tests/fixtures/.gitkeep b/tests/fixtures/.gitkeep new file mode 100644 index 0000000..e69de29 From 6b1a303e9be956e53a625450fb41e00c5af13115 Mon Sep 17 00:00:00 2001 From: "Scot P. Floess" Date: Fri, 4 Sep 2026 12:28:04 -0400 Subject: [PATCH 08/22] Document cross-capability fixture boundary --- tests/fixtures/README.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 tests/fixtures/README.md diff --git a/tests/fixtures/README.md b/tests/fixtures/README.md new file mode 100644 index 0000000..36eef38 --- /dev/null +++ b/tests/fixtures/README.md @@ -0,0 +1 @@ +This fixture mirrors the scraping capability's AcquiredResource contract without importing the scraping package. From f2fec04027eb6ea39c105abb482a59912cf98c9b Mon Sep 17 00:00:00 2001 From: "Scot P. Floess" Date: Fri, 4 Sep 2026 12:28:09 -0400 Subject: [PATCH 09/22] Add fixture placeholder --- tests/fixtures/placeholder.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 tests/fixtures/placeholder.txt diff --git a/tests/fixtures/placeholder.txt b/tests/fixtures/placeholder.txt new file mode 100644 index 0000000..ee8c1ee --- /dev/null +++ b/tests/fixtures/placeholder.txt @@ -0,0 +1 @@ +fixture From 7945e3c390abc7e6946c8c5a876c4b81212bb216 Mon Sep 17 00:00:00 2001 From: "Scot P. Floess" Date: Fri, 4 Sep 2026 12:28:16 -0400 Subject: [PATCH 10/22] Remove temporary fixture placeholder --- tests/fixtures/placeholder.txt | 1 - 1 file changed, 1 deletion(-) delete mode 100644 tests/fixtures/placeholder.txt diff --git a/tests/fixtures/placeholder.txt b/tests/fixtures/placeholder.txt deleted file mode 100644 index ee8c1ee..0000000 --- a/tests/fixtures/placeholder.txt +++ /dev/null @@ -1 +0,0 @@ -fixture From 5cab991a2a017b26db0003d826cb30efd288d34c Mon Sep 17 00:00:00 2001 From: "Scot P. Floess" Date: Fri, 4 Sep 2026 12:28:28 -0400 Subject: [PATCH 11/22] noop --- tests/fixtures/.gitkeep2 | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/fixtures/.gitkeep2 diff --git a/tests/fixtures/.gitkeep2 b/tests/fixtures/.gitkeep2 new file mode 100644 index 0000000..e69de29 From 286cb785f881b54425aefdf57f2ab2979fff5048 Mon Sep 17 00:00:00 2001 From: "Scot P. Floess" Date: Fri, 4 Sep 2026 12:28:34 -0400 Subject: [PATCH 12/22] Remove accidental fixture placeholder --- tests/fixtures/.gitkeep2 | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 tests/fixtures/.gitkeep2 diff --git a/tests/fixtures/.gitkeep2 b/tests/fixtures/.gitkeep2 deleted file mode 100644 index e69de29..0000000 From fb7c6686b7579d57f4d328f3ca6564f6081d73b7 Mon Sep 17 00:00:00 2001 From: "Scot P. Floess" Date: Fri, 4 Sep 2026 12:28:38 -0400 Subject: [PATCH 13/22] noop --- tests/fixtures/.gitkeep3 | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/fixtures/.gitkeep3 diff --git a/tests/fixtures/.gitkeep3 b/tests/fixtures/.gitkeep3 new file mode 100644 index 0000000..e69de29 From 9c15e4fb0d3df15cc7aba341f16d810a2c8bcbca Mon Sep 17 00:00:00 2001 From: "Scot P. Floess" Date: Fri, 4 Sep 2026 12:28:45 -0400 Subject: [PATCH 14/22] Remove accidental fixture placeholder --- tests/fixtures/.gitkeep3 | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 tests/fixtures/.gitkeep3 diff --git a/tests/fixtures/.gitkeep3 b/tests/fixtures/.gitkeep3 deleted file mode 100644 index e69de29..0000000 From ea8167fd619f3a782296cb2d5b160e0c7b2e968a Mon Sep 17 00:00:00 2001 From: "Scot P. Floess" Date: Fri, 4 Sep 2026 12:28:54 -0400 Subject: [PATCH 15/22] Document hardening review scope --- PR_READY.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 PR_READY.md diff --git a/PR_READY.md b/PR_READY.md new file mode 100644 index 0000000..56a50d5 --- /dev/null +++ b/PR_READY.md @@ -0,0 +1 @@ +Changes on p0-verify-hardening are ready for review: deterministic document IDs, long-segment overlap, broader sentence boundaries, scraping contract fixture coverage, X.Y versioning, and CI. From 524a752de31887cd7e6d237f62ba9c1862893211 Mon Sep 17 00:00:00 2001 From: "Scot P. Floess" Date: Fri, 4 Sep 2026 12:59:31 -0400 Subject: [PATCH 16/22] Fix redundant final long-segment chunk --- chunking/chunker.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/chunking/chunker.py b/chunking/chunker.py index 21f0506..f14572e 100644 --- a/chunking/chunker.py +++ b/chunking/chunker.py @@ -57,8 +57,13 @@ def flush() -> None: if length > max_chars: flush() stride = max_chars - overlap_chars - for pos in range(start, end, stride): - ranges.append((pos, min(pos + max_chars, end))) + pos = start + while pos < end: + chunk_end = min(pos + max_chars, end) + ranges.append((pos, chunk_end)) + if chunk_end >= end: + break + pos += stride continue if current_start is not None and current_end - current_start + length > max_chars: From d924a184f96417e67fa4c557f80dfa78a43b75ac Mon Sep 17 00:00:00 2001 From: "Scot P. Floess" Date: Fri, 4 Sep 2026 12:59:36 -0400 Subject: [PATCH 17/22] Remove temporary PR marker --- PR_READY.md | 1 - 1 file changed, 1 deletion(-) delete mode 100644 PR_READY.md diff --git a/PR_READY.md b/PR_READY.md deleted file mode 100644 index 56a50d5..0000000 --- a/PR_READY.md +++ /dev/null @@ -1 +0,0 @@ -Changes on p0-verify-hardening are ready for review: deterministic document IDs, long-segment overlap, broader sentence boundaries, scraping contract fixture coverage, X.Y versioning, and CI. From b35ef88464c2c52247f305730ae58248e6b9baa9 Mon Sep 17 00:00:00 2001 From: "Scot P. Floess" Date: Fri, 4 Sep 2026 13:22:40 -0400 Subject: [PATCH 18/22] Fix metadata test overlap configuration --- tests/test_chunker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_chunker.py b/tests/test_chunker.py index b23fcf0..c222e83 100644 --- a/tests/test_chunker.py +++ b/tests/test_chunker.py @@ -44,7 +44,7 @@ def test_unicode_offsets_and_metadata_are_preserved(): metadata = {"uri": "file://exports/papers/example.pdf", "media_type": "application/pdf"} provenance = {"source": "scraping", "content_hash": "abc123"} chunks = TokenChunker().chunk( - text, document_id="doc", max_tokens=20, metadata=metadata, provenance=provenance + text, document_id="doc", max_tokens=20, overlap=1, metadata=metadata, provenance=provenance ) assert chunks[0].content == text assert chunks[0].start_offset == 0 From eabb8630d9972c5f9d194942942dee937f0e7984 Mon Sep 17 00:00:00 2001 From: "Scot P. Floess" Date: Fri, 4 Sep 2026 13:28:59 -0400 Subject: [PATCH 19/22] Prevent sentence overlap from exceeding chunk bound --- chunking/chunker.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/chunking/chunker.py b/chunking/chunker.py index f14572e..f48633d 100644 --- a/chunking/chunker.py +++ b/chunking/chunker.py @@ -84,6 +84,13 @@ def flush() -> None: current_end = tail[-1][1] current_segments = tail + # Whole-sentence overlap must never make the next chunk exceed + # the configured bound. If the tail does not leave room for the + # incoming sentence, keep the tail as its own chunk and start + # the new chunk with the incoming sentence. + if current_start is not None and current_end - current_start + length > max_chars: + flush() + if current_start is None: current_start = start current_end = end From 0717fcc04a659f209fc50e63c1824dc830ed45ba Mon Sep 17 00:00:00 2001 From: "Scot P. Floess" Date: Fri, 4 Sep 2026 13:29:06 -0400 Subject: [PATCH 20/22] Add sentence packing bound regression test --- tests/test_chunker.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/test_chunker.py b/tests/test_chunker.py index c222e83..ca552a9 100644 --- a/tests/test_chunker.py +++ b/tests/test_chunker.py @@ -31,6 +31,7 @@ def test_long_segment_uses_requested_overlap(): for previous, current in zip(chunks, chunks[1:]): assert previous.end_offset - current.start_offset == 8 assert current.start_offset < current.end_offset + assert current.token_count <= 10 def test_sentence_boundaries_include_question_and_exclamation(): @@ -39,6 +40,15 @@ def test_sentence_boundaries_include_question_and_exclamation(): assert [text[start:end] for start, end in ranges] == ["First?", " Second!", " Third."] +def test_sentence_packing_never_exceeds_max_tokens(): + text = "A" * 15 + "." + " B" * 8 + "." + " C" * 14 + "." + chunks = TokenChunker().chunk(text, document_id="bounded", max_tokens=10, overlap=5) + assert len(chunks) >= 2 + for chunk in chunks: + assert chunk.token_count <= 10 + assert text[chunk.start_offset : chunk.end_offset] == chunk.content + + def test_unicode_offsets_and_metadata_are_preserved(): text = "Unicode café 東京. Next line." metadata = {"uri": "file://exports/papers/example.pdf", "media_type": "application/pdf"} @@ -51,6 +61,7 @@ def test_unicode_offsets_and_metadata_are_preserved(): assert chunks[0].end_offset == len(text) assert chunks[0].metadata == metadata assert chunks[0].provenance == provenance + assert text[chunks[0].start_offset : chunks[0].end_offset] == chunks[0].content def test_invalid_configuration_is_rejected(): From 3ed45d1c4ed1bf3696abe62cc3d4decdbceed6ef Mon Sep 17 00:00:00 2001 From: "Scot P. Floess" Date: Fri, 4 Sep 2026 13:49:31 -0400 Subject: [PATCH 21/22] Discard redundant sentence-overlap tails --- chunking/chunker.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/chunking/chunker.py b/chunking/chunker.py index f48633d..3bfad7a 100644 --- a/chunking/chunker.py +++ b/chunking/chunker.py @@ -1,4 +1,4 @@ -"""Sentence-aware token-bounded chunking.""" +"""Sentence-aware, approximately token-bounded chunking with whole-sentence overlap.""" from __future__ import annotations @@ -13,7 +13,7 @@ class TokenChunker: - """Split text into overlapping, approximately token-bounded chunks.""" + """Split text into approximately token-bounded chunks with sentence overlap.""" def chunk( self, @@ -86,10 +86,12 @@ def flush() -> None: # Whole-sentence overlap must never make the next chunk exceed # the configured bound. If the tail does not leave room for the - # incoming sentence, keep the tail as its own chunk and start - # the new chunk with the incoming sentence. + # incoming sentence, sacrifice overlap at this boundary rather + # than emitting a redundant tail-only chunk. if current_start is not None and current_end - current_start + length > max_chars: - flush() + current_start = None + current_end = 0 + current_segments = [] if current_start is None: current_start = start From d2c32996a4bef7b5ecb7f201fcfcbe45760fc9fc Mon Sep 17 00:00:00 2001 From: "Scot P. Floess" Date: Fri, 4 Sep 2026 13:49:40 -0400 Subject: [PATCH 22/22] Test chunk bounds offsets and tail deduplication --- tests/test_chunker.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/tests/test_chunker.py b/tests/test_chunker.py index ca552a9..fbdf231 100644 --- a/tests/test_chunker.py +++ b/tests/test_chunker.py @@ -1,6 +1,15 @@ from chunking import TokenChunker +def assert_chunk_invariants(text, chunks, max_tokens): + assert chunks + for sequence, chunk in enumerate(chunks): + assert chunk.sequence == sequence + assert chunk.start_offset < chunk.end_offset + assert text[chunk.start_offset : chunk.end_offset] == chunk.content + assert chunk.token_count <= max_tokens + + def test_empty_content_returns_no_chunks(): assert TokenChunker().chunk("") == [] @@ -14,6 +23,7 @@ def test_chunk_has_contract_and_offsets(): assert chunks[0].start_offset == 0 assert chunks[0].end_offset == len(text) assert chunks[0].id == "doc-1:0" + assert_chunk_invariants(text, chunks, 10) def test_automatic_document_id_is_deterministic(): @@ -22,6 +32,7 @@ def test_automatic_document_id_is_deterministic(): second = TokenChunker().chunk(text, max_tokens=4, overlap=1) assert first == second assert first[0].document_id + assert_chunk_invariants(text, first, 4) def test_long_segment_uses_requested_overlap(): @@ -31,7 +42,7 @@ def test_long_segment_uses_requested_overlap(): for previous, current in zip(chunks, chunks[1:]): assert previous.end_offset - current.start_offset == 8 assert current.start_offset < current.end_offset - assert current.token_count <= 10 + assert_chunk_invariants(text, chunks, 10) def test_sentence_boundaries_include_question_and_exclamation(): @@ -40,13 +51,14 @@ def test_sentence_boundaries_include_question_and_exclamation(): assert [text[start:end] for start, end in ranges] == ["First?", " Second!", " Third."] -def test_sentence_packing_never_exceeds_max_tokens(): +def test_sentence_packing_never_exceeds_max_tokens_or_emits_redundant_tail(): text = "A" * 15 + "." + " B" * 8 + "." + " C" * 14 + "." chunks = TokenChunker().chunk(text, document_id="bounded", max_tokens=10, overlap=5) assert len(chunks) >= 2 - for chunk in chunks: - assert chunk.token_count <= 10 - assert text[chunk.start_offset : chunk.end_offset] == chunk.content + assert_chunk_invariants(text, chunks, 10) + for previous, current in zip(chunks, chunks[1:]): + assert current.content not in previous.content + assert previous.content not in current.content def test_unicode_offsets_and_metadata_are_preserved(): @@ -61,7 +73,7 @@ def test_unicode_offsets_and_metadata_are_preserved(): assert chunks[0].end_offset == len(text) assert chunks[0].metadata == metadata assert chunks[0].provenance == provenance - assert text[chunks[0].start_offset : chunks[0].end_offset] == chunks[0].content + assert_chunk_invariants(text, chunks, 20) def test_invalid_configuration_is_rejected():