diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..0e21d38 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,16 @@ +name: test + +on: + push: + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - run: python -m pip install --upgrade pip pytest + - run: python -m pytest diff --git a/chunking/chunker.py b/chunking/chunker.py index f1ea74d..3bfad7a 100644 --- a/chunking/chunker.py +++ b/chunking/chunker.py @@ -1,19 +1,19 @@ -"""Sentence-aware token-bounded chunking.""" +"""Sentence-aware, approximately token-bounded chunking with whole-sentence overlap.""" from __future__ import annotations +import hashlib import re -import uuid from typing import Any from .types import Chunk _CHARS_PER_TOKEN = 4 -_BOUNDARY_RE = re.compile(r".*?(?:\. |\n|$)", re.DOTALL) +_BOUNDARY_RE = re.compile(r".*?(?:(?:[.!?](?=\s|$))|\n|$)", re.DOTALL) class TokenChunker: - """Split text into overlapping, approximately token-bounded chunks.""" + """Split text into approximately token-bounded chunks with sentence overlap.""" def chunk( self, @@ -34,7 +34,7 @@ def chunk( if overlap >= max_tokens: raise ValueError("overlap must be smaller than max_tokens") - document_id = document_id or str(uuid.uuid4()) + document_id = document_id or hashlib.sha256(content.encode("utf-8")).hexdigest() max_chars = max_tokens * _CHARS_PER_TOKEN overlap_chars = overlap * _CHARS_PER_TOKEN segments = self._split_sentences(content) @@ -56,8 +56,14 @@ def flush() -> None: length = end - start if length > max_chars: flush() - for pos in range(start, end, max_chars): - ranges.append((pos, min(pos + max_chars, end))) + stride = max_chars - overlap_chars + pos = start + while pos < end: + chunk_end = min(pos + max_chars, end) + ranges.append((pos, chunk_end)) + if chunk_end >= end: + break + pos += stride continue if current_start is not None and current_end - current_start + length > max_chars: @@ -78,6 +84,15 @@ def flush() -> None: current_end = tail[-1][1] current_segments = tail + # Whole-sentence overlap must never make the next chunk exceed + # the configured bound. If the tail does not leave room for the + # incoming sentence, sacrifice overlap at this boundary rather + # than emitting a redundant tail-only chunk. + if current_start is not None and current_end - current_start + length > max_chars: + current_start = None + current_end = 0 + current_segments = [] + if current_start is None: current_start = start current_end = end diff --git a/pyproject.toml b/pyproject.toml index f13237b..21dd78e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "chunking" -version = "0.1.0" +version = "0.1" description = "Canonical document-to-chunk capability for FlossWare" requires-python = ">=3.10" readme = "README.md" diff --git a/tests/fixtures/.gitkeep b/tests/fixtures/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/tests/fixtures/README.md b/tests/fixtures/README.md new file mode 100644 index 0000000..36eef38 --- /dev/null +++ b/tests/fixtures/README.md @@ -0,0 +1 @@ +This fixture mirrors the scraping capability's AcquiredResource contract without importing the scraping package. diff --git a/tests/fixtures/acquired_resource.json b/tests/fixtures/acquired_resource.json new file mode 100644 index 0000000..dc05078 --- /dev/null +++ b/tests/fixtures/acquired_resource.json @@ -0,0 +1,9 @@ +{ + "uri": "file://exports/papers/example.pdf", + "media_type": "application/pdf", + "content_hash": "sha256:example", + "raw_path": "/exports/raw/example.pdf", + "size": 12345, + "retrieved_at": "2026-09-04T00:00:00+00:00", + "discovered_by": "explicit" +} diff --git a/tests/test_chunker.py b/tests/test_chunker.py index a43f9b3..fbdf231 100644 --- a/tests/test_chunker.py +++ b/tests/test_chunker.py @@ -1,6 +1,15 @@ from chunking import TokenChunker +def assert_chunk_invariants(text, chunks, max_tokens): + assert chunks + for sequence, chunk in enumerate(chunks): + assert chunk.sequence == sequence + assert chunk.start_offset < chunk.end_offset + assert text[chunk.start_offset : chunk.end_offset] == chunk.content + assert chunk.token_count <= max_tokens + + def test_empty_content_returns_no_chunks(): assert TokenChunker().chunk("") == [] @@ -14,6 +23,57 @@ def test_chunk_has_contract_and_offsets(): assert chunks[0].start_offset == 0 assert chunks[0].end_offset == len(text) assert chunks[0].id == "doc-1:0" + assert_chunk_invariants(text, chunks, 10) + + +def test_automatic_document_id_is_deterministic(): + text = "Deterministic Unicode: café 東京." + first = TokenChunker().chunk(text, max_tokens=4, overlap=1) + second = TokenChunker().chunk(text, max_tokens=4, overlap=1) + assert first == second + assert first[0].document_id + assert_chunk_invariants(text, first, 4) + + +def test_long_segment_uses_requested_overlap(): + text = "x" * 100 + chunks = TokenChunker().chunk(text, document_id="long", max_tokens=10, overlap=2) + assert len(chunks) > 1 + for previous, current in zip(chunks, chunks[1:]): + assert previous.end_offset - current.start_offset == 8 + assert current.start_offset < current.end_offset + assert_chunk_invariants(text, chunks, 10) + + +def test_sentence_boundaries_include_question_and_exclamation(): + text = "First? Second! Third." + ranges = TokenChunker._split_sentences(text) + assert [text[start:end] for start, end in ranges] == ["First?", " Second!", " Third."] + + +def test_sentence_packing_never_exceeds_max_tokens_or_emits_redundant_tail(): + text = "A" * 15 + "." + " B" * 8 + "." + " C" * 14 + "." + chunks = TokenChunker().chunk(text, document_id="bounded", max_tokens=10, overlap=5) + assert len(chunks) >= 2 + assert_chunk_invariants(text, chunks, 10) + for previous, current in zip(chunks, chunks[1:]): + assert current.content not in previous.content + assert previous.content not in current.content + + +def test_unicode_offsets_and_metadata_are_preserved(): + text = "Unicode café 東京. Next line." + metadata = {"uri": "file://exports/papers/example.pdf", "media_type": "application/pdf"} + provenance = {"source": "scraping", "content_hash": "abc123"} + chunks = TokenChunker().chunk( + text, document_id="doc", max_tokens=20, overlap=1, metadata=metadata, provenance=provenance + ) + assert chunks[0].content == text + assert chunks[0].start_offset == 0 + assert chunks[0].end_offset == len(text) + assert chunks[0].metadata == metadata + assert chunks[0].provenance == provenance + assert_chunk_invariants(text, chunks, 20) def test_invalid_configuration_is_rejected(): diff --git a/tests/test_scraping_contract.py b/tests/test_scraping_contract.py new file mode 100644 index 0000000..c62a04b --- /dev/null +++ b/tests/test_scraping_contract.py @@ -0,0 +1,32 @@ +import json +from pathlib import Path + +from chunking import TokenChunker + + +FIXTURE = Path(__file__).parent / "fixtures" / "acquired_resource.json" + + +def test_scraping_acquired_resource_contract_can_feed_chunking(): + resource = json.loads(FIXTURE.read_text(encoding="utf-8")) + assert set(resource) == { + "uri", + "media_type", + "content_hash", + "raw_path", + "size", + "retrieved_at", + "discovered_by", + } + + content = "A paper document. A second paragraph." + chunks = TokenChunker().chunk( + content, + document_id=resource["content_hash"], + metadata={"uri": resource["uri"], "media_type": resource["media_type"]}, + provenance=resource, + ) + + assert chunks + assert chunks[0].document_id == resource["content_hash"] + assert chunks[0].provenance == resource