From c9777442d9ec9377fddae0a2c02d15a31b268f21 Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Wed, 29 Jul 2026 14:50:42 +0200 Subject: [PATCH 01/15] prototype: ship prebuilt abi3 rtree modules, keep witty as fallback --- hatch_build.py | 156 +++++++++++++++++++++++++++ pyproject.toml | 12 ++- src/spatial_graph/__init__.py | 34 +++++- src/spatial_graph/_rtree/_codegen.py | 41 +++++++ src/spatial_graph/_rtree/_naming.py | 49 +++++++++ src/spatial_graph/_rtree/_specs.py | 40 +++++++ src/spatial_graph/_rtree/rtree.py | 63 ++++++----- tests/test_prebuilt.py | 75 +++++++++++++ 8 files changed, 439 insertions(+), 31 deletions(-) create mode 100644 hatch_build.py create mode 100644 src/spatial_graph/_rtree/_codegen.py create mode 100644 src/spatial_graph/_rtree/_naming.py create mode 100644 src/spatial_graph/_rtree/_specs.py create mode 100644 tests/test_prebuilt.py diff --git a/hatch_build.py b/hatch_build.py new file mode 100644 index 0000000..af48e78 --- /dev/null +++ b/hatch_build.py @@ -0,0 +1,156 @@ +"""Compile RTree variants ahead of time into stable-ABI (abi3) wheels. + +Renders the same pyx wrappers the runtime would JIT-compile (via +`_rtree._codegen`) for every variant in `_rtree._specs`, builds them against +`Py_LIMITED_API`, and force-includes the result as +`spatial_graph/_rtree/_prebuilt/`. One wheel per platform then covers every +supported CPython, and users never need a C compiler for those variants. + +Set `SPATIAL_GRAPH_NO_PREBUILT=1` to build a pure-Python wheel instead. +""" + +from __future__ import annotations + +import os +import sys +import sysconfig +from pathlib import Path +from typing import Any + +from hatchling.builders.hooks.plugin.interface import BuildHookInterface + +# oldest CPython with the buffer protocol (memoryviews) in the limited API +ABI3_MIN = (3, 11) +ABI3_HEX = f"0x{ABI3_MIN[0]:02x}{ABI3_MIN[1]:02x}0000" + +ROOT = Path(__file__).parent +SRC = ROOT / "src" +PKG = "spatial_graph/_rtree/_prebuilt" + + +def _platform_tag() -> str: + return sysconfig.get_platform().replace("-", "_").replace(".", "_") + + +def _stub_spatial_graph_package() -> None: + """Make `spatial_graph.*` submodules importable without running its `__init__`. + + `spatial_graph/__init__.py` pulls in the graph half, and with it witty; the + build only needs the rtree codegen, so we register a bare namespace package + pointing at the source tree instead. + """ + import types + + pkg = types.ModuleType("spatial_graph") + pkg.__path__ = [str(SRC / "spatial_graph")] # type: ignore[attr-defined] + sys.modules.setdefault("spatial_graph", pkg) + + +class PrebuiltRTreeHook(BuildHookInterface): + PLUGIN_NAME = "prebuilt-rtree" + + def initialize(self, version: str, build_data: dict[str, Any]) -> None: + if self.target_name != "wheel": + return + if os.getenv("SPATIAL_GRAPH_NO_PREBUILT"): + return + + # 3.10 lacks the buffer protocol in the limited API, so it gets a plain + # version-specific wheel; 3.11+ all share one abi3 wheel per platform. + abi3 = sys.version_info >= ABI3_MIN + + _stub_spatial_graph_package() + from spatial_graph._rtree._codegen import build_wrapper + from spatial_graph._rtree._naming import module_name + from spatial_graph._rtree._specs import iter_specs + + build_dir = ROOT / "build" / "prebuilt" / f"{_platform_tag()}-{abi3}" + pyx_dir = build_dir / "pyx" + pyx_dir.mkdir(parents=True, exist_ok=True) + + names = [] + for spec in iter_specs(): + name = module_name(spec.cls, spec.item_dtype, spec.coord_dtype, spec.dims) + source = build_wrapper( + spec.cls, spec.item_dtype, spec.coord_dtype, spec.dims + ) + path = pyx_dir / f"{name}.pyx" + # only rewrite when changed, so cythonize can skip unchanged variants + if not path.is_file() or path.read_text() != source: + path.write_text(source) + names.append(name) + + try: + built = self._compile(pyx_dir, names, build_dir, abi3) + except Exception as e: + # Installing from an sdist on a machine with no usable compiler must + # keep working: fall back to a pure-Python wheel that JIT-compiles on + # first use, exactly as before prebuilding existed. CI sets + # SPATIAL_GRAPH_REQUIRE_PREBUILT so this can never pass silently there. + if os.getenv("SPATIAL_GRAPH_REQUIRE_PREBUILT"): + raise + self.app.display_warning( + f"Could not prebuild rtree modules ({e}); building a pure-Python " + "wheel. A C compiler will be needed the first time an RTree is used." + ) + return + + force_include = build_data.setdefault("force_include", {}) + init = build_dir / "__init__.py" + init.write_text("") + force_include[str(init)] = f"{PKG}/__init__.py" + for artifact in built: + force_include[str(artifact)] = f"{PKG}/{artifact.name}" + + build_data["pure_python"] = False + if abi3: + build_data["tag"] = f"cp{ABI3_MIN[0]}{ABI3_MIN[1]}-abi3-{_platform_tag()}" + else: + build_data["infer_tag"] = True + + def _compile( + self, pyx_dir: Path, names: list[str], build_dir: Path, abi3: bool + ) -> list[Path]: + from Cython.Build import cythonize + from setuptools import Distribution, Extension + + rtree_src = SRC / "spatial_graph" / "_rtree" + win = sys.platform == "win32" + extensions = [ + Extension( + f"{PKG.replace('/', '.')}.{name}", + sources=[str(pyx_dir / f"{name}.pyx")], + include_dirs=[str(rtree_src)], + extra_compile_args=["/O2"] if win else ["-O3", "-Wno-unreachable-code"], + define_macros=[ + *([("Py_LIMITED_API", ABI3_HEX)] if abi3 else []), + *([("RTREE_NOATOMICS", "1")] if win else []), + ], + py_limited_api=abi3, + ) + for name in names + ] + + out = build_dir / "lib" + dist = Distribution( + { + "name": "spatial_graph_prebuilt", + "ext_modules": cythonize( + extensions, language_level=3, quiet=True, nthreads=os.cpu_count() + ), + } + ) + cmd = dist.get_command_obj("build_ext") + cmd.build_lib = str(out) + cmd.build_temp = str(build_dir / "temp") + cmd.parallel = os.cpu_count() + cmd.ensure_finalized() + cmd.run() + + built_pkg = out.joinpath(*PKG.split("/")) + artifacts = sorted( + p for p in built_pkg.iterdir() if p.suffix in (".so", ".pyd") + ) + if len(artifacts) != len(names): + raise RuntimeError(f"expected {len(names)} modules, built {len(artifacts)}") + return artifacts diff --git a/pyproject.toml b/pyproject.toml index de66160..67d7c24 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,10 +1,20 @@ [build-system] -requires = ["hatchling", "hatch-vcs"] +requires = [ + "hatchling", + "hatch-vcs", + "Cython>=3.1", + "CT3>=3.3.3", + "numpy", # imported (not linked) while rendering wrappers + "setuptools>=75.8.0", +] build-backend = "hatchling.build" [tool.hatch.version] source = "vcs" +[tool.hatch.build.targets.wheel.hooks.custom] +path = "hatch_build.py" + [project] name = "spatial-graph" dynamic = ["version"] diff --git a/src/spatial_graph/__init__.py b/src/spatial_graph/__init__.py index 4ec1dee..c1ab15a 100644 --- a/src/spatial_graph/__init__.py +++ b/src/spatial_graph/__init__.py @@ -1,4 +1,5 @@ from importlib.metadata import PackageNotFoundError, version +from typing import TYPE_CHECKING, Any try: __version__ = version("spatial_graph") @@ -6,10 +7,25 @@ __version__ = "unknown" -from ._graph import DiGraph, Graph, GraphBase from ._rtree import LineRTree, PointRTree -from ._spatial_graph import SpatialDiGraph, SpatialGraph, SpatialGraphBase -from ._util import create_graph + +if TYPE_CHECKING: + from ._graph import DiGraph, Graph, GraphBase + from ._spatial_graph import SpatialDiGraph, SpatialGraph, SpatialGraphBase + from ._util import create_graph + +# the graph half is always JIT-compiled, and importing it pulls in witty and +# Cheetah. Deferring it keeps `PointRTree`/`LineRTree` -- which ship prebuilt -- +# usable with numpy alone. +_LAZY = { + "DiGraph": "._graph", + "Graph": "._graph", + "GraphBase": "._graph", + "SpatialDiGraph": "._spatial_graph", + "SpatialGraph": "._spatial_graph", + "SpatialGraphBase": "._spatial_graph", + "create_graph": "._util", +} __all__ = [ "DiGraph", @@ -22,3 +38,15 @@ "SpatialGraphBase", "create_graph", ] + + +def __getattr__(name: str) -> Any: + if module := _LAZY.get(name): + import importlib + + return getattr(importlib.import_module(module, __name__), name) + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + +def __dir__() -> list[str]: + return __all__ diff --git a/src/spatial_graph/_rtree/_codegen.py b/src/spatial_graph/_rtree/_codegen.py new file mode 100644 index 0000000..17c349e --- /dev/null +++ b/src/spatial_graph/_rtree/_codegen.py @@ -0,0 +1,41 @@ +"""Rendering of the RTree pyx wrapper. + +Used on the JIT path and by the build hook, so prebuilt and JIT-compiled modules +are always generated from the same source. Requires Cheetah, and is therefore +imported lazily by `rtree.py`. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from Cheetah.Template import Template + +from spatial_graph._dtypes import DType + +from ._naming import SRC_DIR + +if TYPE_CHECKING: + from .rtree import RTree + +TEMPLATE = SRC_DIR / "wrapper_template.pyx" + + +def build_wrapper( + cls: type[RTree], item_dtype: str, coord_dtype: str, dims: int +) -> str: + """Render the pyx wrapper for the given tree parameters.""" + wrapper_template = Template( + file=str(TEMPLATE), + compilerSettings={"directiveStartToken": "%"}, + ) + wrapper_template.item_dtype = DType(item_dtype) + wrapper_template.coord_dtype = DType(coord_dtype) + wrapper_template.dims = dims + wrapper_template.c_distance_function = cls.c_distance_function + wrapper_template.pyx_item_t_declaration = cls.pyx_item_t_declaration + wrapper_template.c_item_t_declaration = cls.c_item_t_declaration + wrapper_template.c_converter_functions = cls.c_converter_functions + wrapper_template.c_equal_function = cls.c_equal_function + + return str(wrapper_template) diff --git a/src/spatial_graph/_rtree/_naming.py b/src/spatial_graph/_rtree/_naming.py new file mode 100644 index 0000000..c048f65 --- /dev/null +++ b/src/spatial_graph/_rtree/_naming.py @@ -0,0 +1,49 @@ +"""Deterministic naming for prebuilt RTree extension modules. + +Shared by the runtime lookup and the build hook, so the two can never disagree. +Deliberately depends only on `_dtypes` -- it sits on the import path of every +`PointRTree`, including installs with neither Cheetah nor witty available. +""" + +from __future__ import annotations + +import hashlib +from pathlib import Path +from typing import TYPE_CHECKING + +from spatial_graph._dtypes import DType + +if TYPE_CHECKING: + from .rtree import RTree + +SRC_DIR = Path(__file__).parent + +# subpackage holding ahead-of-time compiled modules; absent from pure-Python installs +PREBUILT_PACKAGE = f"{__package__}._prebuilt" + + +def _c_name(dtype: DType) -> str: + """Canonical, identifier-safe name for a dtype ("int64", "float", "int64x2").""" + base = dtype.base_c_type.removesuffix("_t") + return f"{base}x{dtype.size}" if dtype.is_array else base + + +def module_name(cls: type[RTree], item_dtype: str, coord_dtype: str, dims: int) -> str: + """Deterministic module name for the given tree parameters. + + Dtypes are canonicalized (so `int` and `int64` agree) and spelled out for + readability. The trailing digest covers the C/pyx code `cls` injects into the + template, so a subclass with custom code can never be served a prebuilt + module compiled from different code. + """ + parts = ( + cls.pyx_item_t_declaration, + cls.c_item_t_declaration, + cls.c_converter_functions, + cls.c_equal_function, + cls.c_distance_function, + ) + digest = hashlib.sha256("\0".join(parts).encode()).hexdigest()[:8] + item = _c_name(DType(item_dtype)) + coord = _c_name(DType(coord_dtype)) + return f"rtree_{item}_{coord}_d{dims}_{digest}" diff --git a/src/spatial_graph/_rtree/_specs.py b/src/spatial_graph/_rtree/_specs.py new file mode 100644 index 0000000..cbfd352 --- /dev/null +++ b/src/spatial_graph/_rtree/_specs.py @@ -0,0 +1,40 @@ +"""The set of RTree variants compiled ahead of time into binary wheels. + +Only `PointRTree` is prebuilt by default: `LineRTree` is only ever used by +`SpatialGraph`, whose graph half is JIT-compiled regardless, so prebuilding it +would double the wheel size without removing anyone's compiler requirement. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, NamedTuple + +from .line_rtree import LineRTree +from .point_rtree import PointRTree + +if TYPE_CHECKING: + from collections.abc import Iterator + + from .rtree import RTree + +ITEM_BASES = ("int64", "uint64") +COORD_DTYPES = ("float32", "float64") +DIMS = (2, 3, 4, 5) +PREBUILT_LINE_TREES = False + + +class Spec(NamedTuple): + cls: type[RTree] + item_dtype: str + coord_dtype: str + dims: int + + +def iter_specs() -> Iterator[Spec]: + """Yield every RTree variant that should be compiled into a wheel.""" + for base in ITEM_BASES: + for coord in COORD_DTYPES: + for dims in DIMS: + yield Spec(PointRTree, base, coord, dims) + if PREBUILT_LINE_TREES: + yield Spec(LineRTree, f"{base}[2]", coord, dims) diff --git a/src/spatial_graph/_rtree/rtree.py b/src/spatial_graph/_rtree/rtree.py index 7bf280b..1d2aaba 100644 --- a/src/spatial_graph/_rtree/rtree.py +++ b/src/spatial_graph/_rtree/rtree.py @@ -1,51 +1,51 @@ from __future__ import annotations +import importlib +import os import sys -from pathlib import Path from typing import ClassVar import numpy as np -import witty -from Cheetah.Template import Template from spatial_graph._dtypes import DType +from ._naming import PREBUILT_PACKAGE, SRC_DIR, module_name + DEFINE_MACROS = [("RTREE_NOATOMICS", "1")] if sys.platform == "win32" else [] if sys.platform == "win32": # pragma: no cover EXTRA_COMPILE_ARGS = ["/O2"] else: EXTRA_COMPILE_ARGS = ["-O3", "-Wno-unreachable-code"] -SRC_DIR = Path(__file__).parent - -def _build_wrapper( +def _load_prebuilt( cls: type[RTree], item_dtype: str, coord_dtype: str, dims: int -) -> str: - ############################################ - # create wrapper from template and compile # - ############################################ - - wrapper_template = Template( - file=str(SRC_DIR / "wrapper_template.pyx"), - compilerSettings={"directiveStartToken": "%"}, - ) - wrapper_template.item_dtype = DType(item_dtype) - wrapper_template.coord_dtype = DType(coord_dtype) - wrapper_template.dims = dims - wrapper_template.c_distance_function = cls.c_distance_function - wrapper_template.pyx_item_t_declaration = cls.pyx_item_t_declaration - wrapper_template.c_item_t_declaration = cls.c_item_t_declaration - wrapper_template.c_converter_functions = cls.c_converter_functions - wrapper_template.c_equal_function = cls.c_equal_function - - return str(wrapper_template) +) -> type | None: + """Return the ahead-of-time compiled tree class, or None if not shipped.""" + if os.getenv("SPATIAL_GRAPH_NO_PREBUILT"): + return None + name = module_name(cls, item_dtype, coord_dtype, dims) + try: + module = importlib.import_module(f"{PREBUILT_PACKAGE}.{name}") + except ImportError: + return None + return module.RTree -def _compile_tree( +def _jit_compile_tree( cls: type[RTree], item_dtype: str, coord_dtype: str, dims: int ) -> type: - wrapper = _build_wrapper(cls, item_dtype, coord_dtype, dims) + """Compile a tree with the system C compiler. + + Only reached for dtype combinations not shipped prebuilt; Cheetah and witty + are imported here so neither is needed by installs that stay on the + prebuilt path. + """ + import witty + + from ._codegen import build_wrapper + + wrapper = build_wrapper(cls, item_dtype, coord_dtype, dims) module = witty.compile_cython( wrapper, depends_on=[ @@ -62,6 +62,15 @@ def _compile_tree( return module.RTree +def _compile_tree( + cls: type[RTree], item_dtype: str, coord_dtype: str, dims: int +) -> type: + tree_cls = _load_prebuilt(cls, item_dtype, coord_dtype, dims) + if tree_cls is None: + tree_cls = _jit_compile_tree(cls, item_dtype, coord_dtype, dims) + return tree_cls + + class RTree: """A generic RTree implementation, compiled on-the-fly during instantiation. diff --git a/tests/test_prebuilt.py b/tests/test_prebuilt.py new file mode 100644 index 0000000..f3d2013 --- /dev/null +++ b/tests/test_prebuilt.py @@ -0,0 +1,75 @@ +"""Tests for ahead-of-time compiled rtree modules. + +The `prebuilt` marked tests only mean something against an installed wheel; in a +source checkout there is no `_prebuilt` subpackage and they are skipped. +""" + +from __future__ import annotations + +import importlib.util + +import numpy as np +import pytest + +from spatial_graph import PointRTree +from spatial_graph._rtree._naming import PREBUILT_PACKAGE, module_name +from spatial_graph._rtree._specs import iter_specs +from spatial_graph._rtree.rtree import _load_prebuilt + +has_prebuilt = importlib.util.find_spec(PREBUILT_PACKAGE) is not None +requires_prebuilt = pytest.mark.skipif( + not has_prebuilt, reason="no prebuilt modules in this install" +) + + +@requires_prebuilt +@pytest.mark.parametrize("spec", list(iter_specs()), ids=str) +def test_every_declared_spec_is_shipped(spec): + """Every variant in `_specs` must actually resolve to a prebuilt module.""" + assert _load_prebuilt(spec.cls, spec.item_dtype, spec.coord_dtype, spec.dims) + + +@requires_prebuilt +def test_prebuilt_is_used_and_correct(): + tree = PointRTree("int64", "float32", 3) + assert "_prebuilt" in type(tree._ctree).__module__ + + items = np.array([10, 20, 30], dtype="int64") + points = np.ascontiguousarray([[0, 0, 0], [1, 1, 1], [9, 9, 9]], dtype="float32") + tree.insert_point_items(items, points) + + lo, hi = np.array([0, 0, 0], "float32"), np.array([2, 2, 2], "float32") + assert sorted(tree.search(lo, hi).ravel().tolist()) == [10, 20] + assert tree.nearest(np.array([8.9, 8.9, 8.9], "float32"), 1).ravel()[0] == 30 + + +def test_dtype_aliases_share_a_module(): + """`int`/`int64` and `float32`/`float` must not compile separate modules.""" + assert module_name(PointRTree, "int", "float32", 3) == module_name( + PointRTree, "int64", "float", 3 + ) + + +@pytest.mark.parametrize( + ("item_dtype", "coord_dtype", "dims"), + [("int32", "float32", 3), ("int64", "float32", 99)], +) +def test_unlisted_combination_falls_back_to_jit(item_dtype, coord_dtype, dims): + assert _load_prebuilt(PointRTree, item_dtype, coord_dtype, dims) is None + + +def test_subclass_with_custom_code_is_not_served_a_prebuilt_module(): + class CustomEquality(PointRTree): + c_equal_function = """ +inline bool equal(const item_t a, const item_t b) { return a == b; } +""" + + assert module_name(CustomEquality, "int64", "float32", 3) != module_name( + PointRTree, "int64", "float32", 3 + ) + assert _load_prebuilt(CustomEquality, "int64", "float32", 3) is None + + +def test_no_prebuilt_env_var_forces_jit(monkeypatch): + monkeypatch.setenv("SPATIAL_GRAPH_NO_PREBUILT", "1") + assert _load_prebuilt(PointRTree, "int64", "float32", 3) is None From 60ddc4085a5305b3624b5758a5fa1238d67e8596 Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Wed, 29 Jul 2026 15:03:01 +0200 Subject: [PATCH 02/15] drop Python 3.10; abi3 build is now unconditional --- hatch_build.py | 33 +++++++++++++++++---------------- pyproject.toml | 8 +++----- 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/hatch_build.py b/hatch_build.py index af48e78..45e7d9a 100644 --- a/hatch_build.py +++ b/hatch_build.py @@ -19,7 +19,10 @@ from hatchling.builders.hooks.plugin.interface import BuildHookInterface -# oldest CPython with the buffer protocol (memoryviews) in the limited API +# The wrappers pass numpy arrays as typed memoryviews, which compile to +# PyObject_GetBuffer/PyBuffer_Release. Those entered the limited API in 3.11 +# (moved from cpython/object.h, excluded under Py_LIMITED_API, to pybuffer.h), +# so 3.11 is the floor for a stable-ABI build -- and matches requires-python. ABI3_MIN = (3, 11) ABI3_HEX = f"0x{ABI3_MIN[0]:02x}{ABI3_MIN[1]:02x}0000" @@ -55,16 +58,19 @@ def initialize(self, version: str, build_data: dict[str, Any]) -> None: if os.getenv("SPATIAL_GRAPH_NO_PREBUILT"): return - # 3.10 lacks the buffer protocol in the limited API, so it gets a plain - # version-specific wheel; 3.11+ all share one abi3 wheel per platform. - abi3 = sys.version_info >= ABI3_MIN + if sys.version_info < ABI3_MIN: + raise RuntimeError( + f"building spatial-graph wheels requires Python >= " + f"{'.'.join(map(str, ABI3_MIN))}; the resulting abi3 wheel then " + f"covers every supported CPython." + ) _stub_spatial_graph_package() from spatial_graph._rtree._codegen import build_wrapper from spatial_graph._rtree._naming import module_name from spatial_graph._rtree._specs import iter_specs - build_dir = ROOT / "build" / "prebuilt" / f"{_platform_tag()}-{abi3}" + build_dir = ROOT / "build" / "prebuilt" / _platform_tag() pyx_dir = build_dir / "pyx" pyx_dir.mkdir(parents=True, exist_ok=True) @@ -81,7 +87,7 @@ def initialize(self, version: str, build_data: dict[str, Any]) -> None: names.append(name) try: - built = self._compile(pyx_dir, names, build_dir, abi3) + built = self._compile(pyx_dir, names, build_dir) except Exception as e: # Installing from an sdist on a machine with no usable compiler must # keep working: fall back to a pure-Python wheel that JIT-compiles on @@ -103,14 +109,9 @@ def initialize(self, version: str, build_data: dict[str, Any]) -> None: force_include[str(artifact)] = f"{PKG}/{artifact.name}" build_data["pure_python"] = False - if abi3: - build_data["tag"] = f"cp{ABI3_MIN[0]}{ABI3_MIN[1]}-abi3-{_platform_tag()}" - else: - build_data["infer_tag"] = True - - def _compile( - self, pyx_dir: Path, names: list[str], build_dir: Path, abi3: bool - ) -> list[Path]: + build_data["tag"] = f"cp{ABI3_MIN[0]}{ABI3_MIN[1]}-abi3-{_platform_tag()}" + + def _compile(self, pyx_dir: Path, names: list[str], build_dir: Path) -> list[Path]: from Cython.Build import cythonize from setuptools import Distribution, Extension @@ -123,10 +124,10 @@ def _compile( include_dirs=[str(rtree_src)], extra_compile_args=["/O2"] if win else ["-O3", "-Wno-unreachable-code"], define_macros=[ - *([("Py_LIMITED_API", ABI3_HEX)] if abi3 else []), + ("Py_LIMITED_API", ABI3_HEX), *([("RTREE_NOATOMICS", "1")] if win else []), ], - py_limited_api=abi3, + py_limited_api=True, ) for name in names ] diff --git a/pyproject.toml b/pyproject.toml index 67d7c24..b4356bc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ name = "spatial-graph" dynamic = ["version"] description = "A spatial graph datastructure for python." readme = "README.md" -requires-python = ">=3.10" +requires-python = ">=3.11" license = { text = "MIT" } authors = [ { email = "funkej@janelia.hhmi.org", name = "Jan Funke" }, @@ -30,7 +30,6 @@ classifiers = [ "Development Status :: 3 - Alpha", "Intended Audience :: Developers", "License :: OSI Approved :: MIT License", - "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", @@ -43,8 +42,7 @@ dependencies = [ "numpy>=2.3.2; python_version >= '3.14'", "numpy>=2.1.0; python_version >= '3.13'", "numpy>=1.26.0; python_version >= '3.12'", - "numpy>=1.23.2; python_version >= '3.11'", - "numpy>=1.21.2", + "numpy>=1.23.2", "setuptools>=75.8.0", "typing_extensions>=4.5.0", # witty<=0.3.1 imports it without declaring it ] @@ -74,7 +72,7 @@ homepage = "https://github.com/funkelab/spatial_graph" repository = "https://github.com/funkelab/spatial_graph" [tool.ruff] -target-version = "py310" +target-version = "py311" line-length = 88 fix = true unsafe-fixes = true From 8a993b57d57982477ee275e0e5d8eda6e4082fc0 Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Wed, 29 Jul 2026 15:33:58 +0200 Subject: [PATCH 03/15] switch build backend to setuptools; merge _specs into _codegen --- hatch_build.py | 157 ------------------ pyproject.toml | 20 ++- setup.py | 125 ++++++++++++++ src/spatial_graph/_rtree/_codegen.py | 45 ++++- src/spatial_graph/_rtree/_naming.py | 7 +- .../_rtree/_prebuilt/__init__.py | 5 + src/spatial_graph/_rtree/_specs.py | 40 ----- src/spatial_graph/_rtree/rtree.py | 5 +- tests/test_prebuilt.py | 10 +- 9 files changed, 190 insertions(+), 224 deletions(-) delete mode 100644 hatch_build.py create mode 100644 setup.py create mode 100644 src/spatial_graph/_rtree/_prebuilt/__init__.py delete mode 100644 src/spatial_graph/_rtree/_specs.py diff --git a/hatch_build.py b/hatch_build.py deleted file mode 100644 index 45e7d9a..0000000 --- a/hatch_build.py +++ /dev/null @@ -1,157 +0,0 @@ -"""Compile RTree variants ahead of time into stable-ABI (abi3) wheels. - -Renders the same pyx wrappers the runtime would JIT-compile (via -`_rtree._codegen`) for every variant in `_rtree._specs`, builds them against -`Py_LIMITED_API`, and force-includes the result as -`spatial_graph/_rtree/_prebuilt/`. One wheel per platform then covers every -supported CPython, and users never need a C compiler for those variants. - -Set `SPATIAL_GRAPH_NO_PREBUILT=1` to build a pure-Python wheel instead. -""" - -from __future__ import annotations - -import os -import sys -import sysconfig -from pathlib import Path -from typing import Any - -from hatchling.builders.hooks.plugin.interface import BuildHookInterface - -# The wrappers pass numpy arrays as typed memoryviews, which compile to -# PyObject_GetBuffer/PyBuffer_Release. Those entered the limited API in 3.11 -# (moved from cpython/object.h, excluded under Py_LIMITED_API, to pybuffer.h), -# so 3.11 is the floor for a stable-ABI build -- and matches requires-python. -ABI3_MIN = (3, 11) -ABI3_HEX = f"0x{ABI3_MIN[0]:02x}{ABI3_MIN[1]:02x}0000" - -ROOT = Path(__file__).parent -SRC = ROOT / "src" -PKG = "spatial_graph/_rtree/_prebuilt" - - -def _platform_tag() -> str: - return sysconfig.get_platform().replace("-", "_").replace(".", "_") - - -def _stub_spatial_graph_package() -> None: - """Make `spatial_graph.*` submodules importable without running its `__init__`. - - `spatial_graph/__init__.py` pulls in the graph half, and with it witty; the - build only needs the rtree codegen, so we register a bare namespace package - pointing at the source tree instead. - """ - import types - - pkg = types.ModuleType("spatial_graph") - pkg.__path__ = [str(SRC / "spatial_graph")] # type: ignore[attr-defined] - sys.modules.setdefault("spatial_graph", pkg) - - -class PrebuiltRTreeHook(BuildHookInterface): - PLUGIN_NAME = "prebuilt-rtree" - - def initialize(self, version: str, build_data: dict[str, Any]) -> None: - if self.target_name != "wheel": - return - if os.getenv("SPATIAL_GRAPH_NO_PREBUILT"): - return - - if sys.version_info < ABI3_MIN: - raise RuntimeError( - f"building spatial-graph wheels requires Python >= " - f"{'.'.join(map(str, ABI3_MIN))}; the resulting abi3 wheel then " - f"covers every supported CPython." - ) - - _stub_spatial_graph_package() - from spatial_graph._rtree._codegen import build_wrapper - from spatial_graph._rtree._naming import module_name - from spatial_graph._rtree._specs import iter_specs - - build_dir = ROOT / "build" / "prebuilt" / _platform_tag() - pyx_dir = build_dir / "pyx" - pyx_dir.mkdir(parents=True, exist_ok=True) - - names = [] - for spec in iter_specs(): - name = module_name(spec.cls, spec.item_dtype, spec.coord_dtype, spec.dims) - source = build_wrapper( - spec.cls, spec.item_dtype, spec.coord_dtype, spec.dims - ) - path = pyx_dir / f"{name}.pyx" - # only rewrite when changed, so cythonize can skip unchanged variants - if not path.is_file() or path.read_text() != source: - path.write_text(source) - names.append(name) - - try: - built = self._compile(pyx_dir, names, build_dir) - except Exception as e: - # Installing from an sdist on a machine with no usable compiler must - # keep working: fall back to a pure-Python wheel that JIT-compiles on - # first use, exactly as before prebuilding existed. CI sets - # SPATIAL_GRAPH_REQUIRE_PREBUILT so this can never pass silently there. - if os.getenv("SPATIAL_GRAPH_REQUIRE_PREBUILT"): - raise - self.app.display_warning( - f"Could not prebuild rtree modules ({e}); building a pure-Python " - "wheel. A C compiler will be needed the first time an RTree is used." - ) - return - - force_include = build_data.setdefault("force_include", {}) - init = build_dir / "__init__.py" - init.write_text("") - force_include[str(init)] = f"{PKG}/__init__.py" - for artifact in built: - force_include[str(artifact)] = f"{PKG}/{artifact.name}" - - build_data["pure_python"] = False - build_data["tag"] = f"cp{ABI3_MIN[0]}{ABI3_MIN[1]}-abi3-{_platform_tag()}" - - def _compile(self, pyx_dir: Path, names: list[str], build_dir: Path) -> list[Path]: - from Cython.Build import cythonize - from setuptools import Distribution, Extension - - rtree_src = SRC / "spatial_graph" / "_rtree" - win = sys.platform == "win32" - extensions = [ - Extension( - f"{PKG.replace('/', '.')}.{name}", - sources=[str(pyx_dir / f"{name}.pyx")], - include_dirs=[str(rtree_src)], - extra_compile_args=["/O2"] if win else ["-O3", "-Wno-unreachable-code"], - define_macros=[ - ("Py_LIMITED_API", ABI3_HEX), - *([("RTREE_NOATOMICS", "1")] if win else []), - ], - py_limited_api=True, - ) - for name in names - ] - - out = build_dir / "lib" - dist = Distribution( - { - "name": "spatial_graph_prebuilt", - "ext_modules": cythonize( - extensions, language_level=3, quiet=True, nthreads=os.cpu_count() - ), - } - ) - cmd = dist.get_command_obj("build_ext") - cmd.build_lib = str(out) - cmd.build_temp = str(build_dir / "temp") - cmd.parallel = os.cpu_count() - cmd.ensure_finalized() - cmd.run() - - built_pkg = out.joinpath(*PKG.split("/")) - artifacts = sorted( - p for p in built_pkg.iterdir() if p.suffix in (".so", ".pyd") - ) - if len(artifacts) != len(names): - raise RuntimeError(f"expected {len(names)} modules, built {len(artifacts)}") - return artifacts diff --git a/pyproject.toml b/pyproject.toml index b4356bc..e223f92 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,19 +1,21 @@ [build-system] requires = [ - "hatchling", - "hatch-vcs", + "setuptools>=77", + "setuptools-scm>=8", "Cython>=3.1", "CT3>=3.3.3", - "numpy", # imported (not linked) while rendering wrappers - "setuptools>=75.8.0", + "numpy", # imported (not linked) while rendering the wrappers ] -build-backend = "hatchling.build" +build-backend = "setuptools.build_meta" + +[tool.setuptools_scm] -[tool.hatch.version] -source = "vcs" +[tool.setuptools.packages.find] +where = ["src"] -[tool.hatch.build.targets.wheel.hooks.custom] -path = "hatch_build.py" +[tool.setuptools.package-data] +# the JIT fallback compiles from these at runtime, so they must ship in the wheel +"*" = ["py.typed", "*.pyx", "*.c", "*.h", "LICENSE*", "*.md"] [project] name = "spatial-graph" diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..67c98d7 --- /dev/null +++ b/setup.py @@ -0,0 +1,125 @@ +"""Compile RTree variants ahead of time into a stable-ABI (abi3) wheel. + +Renders the same pyx wrappers the runtime would JIT-compile (via +`_rtree._codegen`) for every variant in `iter_specs()`, so prebuilt and +JIT-compiled modules can never disagree. One wheel per platform then covers +every supported CPython, and users never need a C compiler for those variants. + +Set `SPATIAL_GRAPH_NO_PREBUILT=1` to build a pure-Python wheel instead, or +`SPATIAL_GRAPH_REQUIRE_PREBUILT=1` (as CI does) to turn a failure to compile +into a hard error rather than a silent fall back to JIT. +""" + +from __future__ import annotations + +import os +import sys +import tempfile +import warnings +from pathlib import Path + +from setuptools import Extension, setup + +ROOT = Path(__file__).parent +SRC = ROOT / "src" +PREBUILT_PKG = "spatial_graph._rtree._prebuilt" + +# The wrappers pass numpy arrays as typed memoryviews, which compile to +# PyObject_GetBuffer/PyBuffer_Release. Those entered the limited API in 3.11 +# (moved from cpython/object.h, excluded under Py_LIMITED_API, to pybuffer.h), +# so 3.11 is the floor for a stable-ABI build -- and matches requires-python. +ABI3_MIN = (3, 11) +ABI3_TAG = f"cp{ABI3_MIN[0]}{ABI3_MIN[1]}" +ABI3_HEX = f"0x{ABI3_MIN[0]:02x}{ABI3_MIN[1]:02x}0000" + +WIN = sys.platform == "win32" + + +def prebuilt_extensions() -> list[Extension]: + """Render every prebuilt RTree variant and declare it as an extension.""" + from Cython.Build import cythonize + + sys.path.insert(0, str(SRC)) + from spatial_graph._rtree._codegen import build_wrapper, iter_specs + from spatial_graph._rtree._naming import module_name + + pyx_dir = ROOT / "build" / "prebuilt-pyx" + pyx_dir.mkdir(parents=True, exist_ok=True) + + extensions = [] + for spec in iter_specs(): + name = module_name(spec.cls, spec.item_dtype, spec.coord_dtype, spec.dims) + source = build_wrapper(spec.cls, spec.item_dtype, spec.coord_dtype, spec.dims) + path = pyx_dir / f"{name}.pyx" + # only rewrite when changed, so cythonize can skip unchanged variants + if not path.is_file() or path.read_text() != source: + path.write_text(source) + extensions.append( + Extension( + f"{PREBUILT_PKG}.{name}", + sources=[str(path)], + include_dirs=[str(SRC / "spatial_graph" / "_rtree")], + extra_compile_args=["/O2"] if WIN else ["-O3", "-Wno-unreachable-code"], + define_macros=[ + ("Py_LIMITED_API", ABI3_HEX), + *([("RTREE_NOATOMICS", "1")] if WIN else []), + ], + py_limited_api=True, + ) + ) + + return cythonize( + extensions, + language_level=3, + quiet=True, + nthreads=0 if WIN else os.cpu_count(), + ) + + +def can_compile() -> bool: + """Whether this machine can build a C extension at all.""" + from distutils.ccompiler import new_compiler + from distutils.sysconfig import customize_compiler + + compiler = new_compiler() + customize_compiler(compiler) # picks up CC/CFLAGS, as build_ext does + with tempfile.TemporaryDirectory() as tmp: + probe = Path(tmp, "probe.c") + probe.write_text("int main(void) { return 0; }\n") + try: + compiler.compile([str(probe)], output_dir=tmp) + except Exception: + return False + return True + + +def should_prebuild() -> bool: + """Whether to compile prebuilt variants into this wheel.""" + if os.getenv("SPATIAL_GRAPH_NO_PREBUILT"): + return False + if os.getenv("SPATIAL_GRAPH_REQUIRE_PREBUILT"): + return True # CI: never let a build silently degrade + if can_compile(): + return True + # Installing from an sdist without a compiler must keep working: fall back to + # a pure-Python wheel that JIT-compiles on first use, as it did before + # prebuilding existed. + warnings.warn( + "No usable C compiler found; building spatial-graph without prebuilt " + "rtree modules. A C compiler will be needed the first time an RTree is " + "used.", + stacklevel=1, + ) + return False + + +if should_prebuild(): + setup( + ext_modules=prebuilt_extensions(), + options={ + "bdist_wheel": {"py_limited_api": ABI3_TAG}, + "build_ext": {"parallel": os.cpu_count()}, + }, + ) +else: + setup(ext_modules=[]) diff --git a/src/spatial_graph/_rtree/_codegen.py b/src/spatial_graph/_rtree/_codegen.py index 17c349e..c1d401e 100644 --- a/src/spatial_graph/_rtree/_codegen.py +++ b/src/spatial_graph/_rtree/_codegen.py @@ -1,24 +1,55 @@ -"""Rendering of the RTree pyx wrapper. +"""What RTree variants get prebuilt, and how their pyx wrappers are rendered. -Used on the JIT path and by the build hook, so prebuilt and JIT-compiled modules -are always generated from the same source. Requires Cheetah, and is therefore -imported lazily by `rtree.py`. +Used on the JIT path and by `setup.py`, so prebuilt and JIT-compiled modules are +always generated from the same source. Requires Cheetah, and is therefore +imported lazily by `rtree.py` -- installs that stay on the prebuilt path need +neither Cheetah nor witty. """ from __future__ import annotations -from typing import TYPE_CHECKING +from pathlib import Path +from typing import TYPE_CHECKING, NamedTuple from Cheetah.Template import Template from spatial_graph._dtypes import DType -from ._naming import SRC_DIR +from .line_rtree import LineRTree +from .point_rtree import PointRTree if TYPE_CHECKING: + from collections.abc import Iterator + from .rtree import RTree -TEMPLATE = SRC_DIR / "wrapper_template.pyx" +TEMPLATE = Path(__file__).parent / "wrapper_template.pyx" + +# Variants compiled ahead of time into binary wheels. Only `PointRTree` by +# default: `LineRTree` is only ever used by `SpatialGraph`, whose graph half is +# JIT-compiled regardless, so prebuilding it would double the wheel size without +# removing anyone's compiler requirement. +ITEM_BASES = ("int64", "uint64") +COORD_DTYPES = ("float32", "float64") +DIMS = (2, 3, 4, 5) +PREBUILT_LINE_TREES = False + + +class Spec(NamedTuple): + cls: type[RTree] + item_dtype: str + coord_dtype: str + dims: int + + +def iter_specs() -> Iterator[Spec]: + """Yield every RTree variant that should be compiled into a wheel.""" + for base in ITEM_BASES: + for coord in COORD_DTYPES: + for dims in DIMS: + yield Spec(PointRTree, base, coord, dims) + if PREBUILT_LINE_TREES: + yield Spec(LineRTree, f"{base}[2]", coord, dims) def build_wrapper( diff --git a/src/spatial_graph/_rtree/_naming.py b/src/spatial_graph/_rtree/_naming.py index c048f65..dcf34eb 100644 --- a/src/spatial_graph/_rtree/_naming.py +++ b/src/spatial_graph/_rtree/_naming.py @@ -1,6 +1,6 @@ """Deterministic naming for prebuilt RTree extension modules. -Shared by the runtime lookup and the build hook, so the two can never disagree. +Shared by the runtime lookup and `setup.py`, so the two can never disagree. Deliberately depends only on `_dtypes` -- it sits on the import path of every `PointRTree`, including installs with neither Cheetah nor witty available. """ @@ -8,7 +8,6 @@ from __future__ import annotations import hashlib -from pathlib import Path from typing import TYPE_CHECKING from spatial_graph._dtypes import DType @@ -16,9 +15,7 @@ if TYPE_CHECKING: from .rtree import RTree -SRC_DIR = Path(__file__).parent - -# subpackage holding ahead-of-time compiled modules; absent from pure-Python installs +# subpackage holding ahead-of-time compiled modules; empty in a source checkout PREBUILT_PACKAGE = f"{__package__}._prebuilt" diff --git a/src/spatial_graph/_rtree/_prebuilt/__init__.py b/src/spatial_graph/_rtree/_prebuilt/__init__.py new file mode 100644 index 0000000..28a9aa6 --- /dev/null +++ b/src/spatial_graph/_rtree/_prebuilt/__init__.py @@ -0,0 +1,5 @@ +"""Ahead-of-time compiled RTree modules, populated at build time by `setup.py`. + +Empty in a plain source checkout: `_load_prebuilt` then finds nothing and every +tree is JIT-compiled, exactly as before prebuilding existed. +""" diff --git a/src/spatial_graph/_rtree/_specs.py b/src/spatial_graph/_rtree/_specs.py deleted file mode 100644 index cbfd352..0000000 --- a/src/spatial_graph/_rtree/_specs.py +++ /dev/null @@ -1,40 +0,0 @@ -"""The set of RTree variants compiled ahead of time into binary wheels. - -Only `PointRTree` is prebuilt by default: `LineRTree` is only ever used by -`SpatialGraph`, whose graph half is JIT-compiled regardless, so prebuilding it -would double the wheel size without removing anyone's compiler requirement. -""" - -from __future__ import annotations - -from typing import TYPE_CHECKING, NamedTuple - -from .line_rtree import LineRTree -from .point_rtree import PointRTree - -if TYPE_CHECKING: - from collections.abc import Iterator - - from .rtree import RTree - -ITEM_BASES = ("int64", "uint64") -COORD_DTYPES = ("float32", "float64") -DIMS = (2, 3, 4, 5) -PREBUILT_LINE_TREES = False - - -class Spec(NamedTuple): - cls: type[RTree] - item_dtype: str - coord_dtype: str - dims: int - - -def iter_specs() -> Iterator[Spec]: - """Yield every RTree variant that should be compiled into a wheel.""" - for base in ITEM_BASES: - for coord in COORD_DTYPES: - for dims in DIMS: - yield Spec(PointRTree, base, coord, dims) - if PREBUILT_LINE_TREES: - yield Spec(LineRTree, f"{base}[2]", coord, dims) diff --git a/src/spatial_graph/_rtree/rtree.py b/src/spatial_graph/_rtree/rtree.py index 1d2aaba..8afb50e 100644 --- a/src/spatial_graph/_rtree/rtree.py +++ b/src/spatial_graph/_rtree/rtree.py @@ -3,13 +3,14 @@ import importlib import os import sys +from pathlib import Path from typing import ClassVar import numpy as np from spatial_graph._dtypes import DType -from ._naming import PREBUILT_PACKAGE, SRC_DIR, module_name +from ._naming import PREBUILT_PACKAGE, module_name DEFINE_MACROS = [("RTREE_NOATOMICS", "1")] if sys.platform == "win32" else [] if sys.platform == "win32": # pragma: no cover @@ -17,6 +18,8 @@ else: EXTRA_COMPILE_ARGS = ["-O3", "-Wno-unreachable-code"] +SRC_DIR = Path(__file__).parent + def _load_prebuilt( cls: type[RTree], item_dtype: str, coord_dtype: str, dims: int diff --git a/tests/test_prebuilt.py b/tests/test_prebuilt.py index f3d2013..ec8c775 100644 --- a/tests/test_prebuilt.py +++ b/tests/test_prebuilt.py @@ -6,17 +6,17 @@ from __future__ import annotations -import importlib.util - import numpy as np import pytest from spatial_graph import PointRTree -from spatial_graph._rtree._naming import PREBUILT_PACKAGE, module_name -from spatial_graph._rtree._specs import iter_specs +from spatial_graph._rtree._codegen import iter_specs +from spatial_graph._rtree._naming import module_name from spatial_graph._rtree.rtree import _load_prebuilt -has_prebuilt = importlib.util.find_spec(PREBUILT_PACKAGE) is not None +# the `_prebuilt` package always exists but is empty in a source checkout, so +# probe for a real module rather than for the package +has_prebuilt = _load_prebuilt(PointRTree, "int64", "float32", 2) is not None requires_prebuilt = pytest.mark.skipif( not has_prebuilt, reason="no prebuilt modules in this install" ) From 4fac7b076ecb704d6c99a8673d573b69ae85095e Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Wed, 29 Jul 2026 15:42:17 +0200 Subject: [PATCH 04/15] ci: drop 3.10, test the built wheel instead of an editable install --- .github/workflows/ci.yml | 16 +++++++++++----- tests/test_prebuilt.py | 17 ++++++++++++++--- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f73cf42..4f2da43 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,17 +29,19 @@ jobs: matrix: # ubuntu: full python range x both resolutions os: [ubuntu-latest] - python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] + python-version: ["3.11", "3.12", "3.13", "3.14"] resolution: [lowest-direct, highest] # windows/macos: only the endpoints, highest resolution include: - - { os: windows-latest, python-version: "3.10", resolution: highest } + - { os: windows-latest, python-version: "3.11", resolution: highest } - { os: windows-latest, python-version: "3.14", resolution: highest } - - { os: macos-latest, python-version: "3.10", resolution: highest } + - { os: macos-latest, python-version: "3.11", resolution: highest } - { os: macos-latest, python-version: "3.14", resolution: highest } env: UV_RESOLUTION: ${{ matrix.resolution }} + # a build that silently falls back to pure-Python must fail, not go green + SPATIAL_GRAPH_REQUIRE_PREBUILT: "1" steps: - uses: actions/checkout@v4 @@ -48,8 +50,12 @@ jobs: python-version: ${{ matrix.python-version }} enable-cache: true cache-dependency-glob: "**/pyproject.toml" + # --no-editable so we test the built wheel, prebuilt rtree modules and all, + # rather than an editable install of src/ + - name: Install as a built wheel + run: uv sync --no-dev --group test --no-editable - name: Test with coverage - run: uv run --no-dev --group test pytest -v --cov=spatial_graph --cov-report=xml + run: uv run --no-sync pytest -v --cov=spatial_graph --cov-report=xml - uses: codecov/codecov-action@v5 with: token: ${{ secrets.CODECOV_TOKEN }} @@ -66,7 +72,7 @@ jobs: enable-cache: true - name: install - run: uv sync --no-dev --group test-codspeed + run: uv sync --no-dev --group test-codspeed --no-editable - name: Run benchmarks uses: CodSpeedHQ/action@v3 diff --git a/tests/test_prebuilt.py b/tests/test_prebuilt.py index ec8c775..5ffa2b3 100644 --- a/tests/test_prebuilt.py +++ b/tests/test_prebuilt.py @@ -1,11 +1,15 @@ """Tests for ahead-of-time compiled rtree modules. -The `prebuilt` marked tests only mean something against an installed wheel; in a -source checkout there is no `_prebuilt` subpackage and they are skipped. +The `requires_prebuilt` tests only mean something against an install that +actually shipped them, and are skipped otherwise -- except when +`SPATIAL_GRAPH_REQUIRE_PREBUILT` is set (as CI does), where their absence is +the very regression we want to catch. """ from __future__ import annotations +import os + import numpy as np import pytest @@ -22,10 +26,17 @@ ) +def test_prebuilt_modules_were_shipped(): + """Guard against a wheel that silently degraded to pure Python.""" + if not os.getenv("SPATIAL_GRAPH_REQUIRE_PREBUILT"): + pytest.skip("SPATIAL_GRAPH_REQUIRE_PREBUILT not set") + assert has_prebuilt, "install shipped no prebuilt rtree modules" + + @requires_prebuilt @pytest.mark.parametrize("spec", list(iter_specs()), ids=str) def test_every_declared_spec_is_shipped(spec): - """Every variant in `_specs` must actually resolve to a prebuilt module.""" + """Every variant in `iter_specs` must actually resolve to a prebuilt module.""" assert _load_prebuilt(spec.cls, spec.item_dtype, spec.coord_dtype, spec.dims) From 2049377a5e63a01a647b96158a1a129469f105b4 Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Wed, 29 Jul 2026 16:22:03 +0200 Subject: [PATCH 05/15] ci: build release wheels with cibuildwheel; prebuild LineRTree too --- .github/workflows/ci.yml | 106 ++++++++++++++++++++++++--- pyproject.toml | 10 +++ src/spatial_graph/_rtree/_codegen.py | 10 +-- 3 files changed, 111 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4f2da43..8debc4b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -79,9 +79,97 @@ jobs: with: run: uv run pytest -W ignore --codspeed -v --color=yes + # One abi3 wheel per platform, covering every supported CPython. Also the only + # thing that produces PyPI-acceptable manylinux tags -- `uv build` alone emits + # `linux_x86_64`, which PyPI rejects. + build-wheels: + name: Wheels ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: + - ubuntu-latest # manylinux x86_64 + - ubuntu-24.04-arm # manylinux aarch64 + - windows-latest # win_amd64 + - macos-13 # macOS x86_64 + - macos-latest # macOS arm64 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # setuptools-scm needs the tags + - uses: pypa/cibuildwheel@v4.1.1 + - uses: actions/upload-artifact@v4 + with: + name: wheels-${{ matrix.os }} + path: wheelhouse/*.whl + + build-sdist: + name: Sdist + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: astral-sh/setup-uv@v6 + - run: uv build --sdist + - uses: actions/upload-artifact@v4 + with: + name: sdist + path: dist/*.tar.gz + + # The claim this whole design rests on: one cp311-abi3 wheel runs on every + # supported CPython, with no compiler and no witty. + test-abi3-wheel: + name: abi3 wheel on py${{ matrix.python-version }} + needs: build-wheels + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.11", "3.12", "3.13", "3.14"] + env: + SPATIAL_GRAPH_REQUIRE_PREBUILT: "1" + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + name: wheels-ubuntu-latest + path: wheelhouse + - uses: astral-sh/setup-uv@v6 + with: + python-version: ${{ matrix.python-version }} + - name: Install the wheel with numpy alone + run: | + uv venv + uv pip install numpy + uv pip install --no-deps wheelhouse/*.whl + - name: Prebuilt rtrees must work without witty, Cheetah or a compiler + run: | + uv run --no-sync python -c " + import sys, numpy as np + try: + import witty; sys.exit('witty present; test is not conclusive') + except ImportError: pass + from spatial_graph import PointRTree + t = PointRTree('int64', 'float32', 3) + t.insert_point_items(np.array([1, 2], dtype='int64'), + np.ascontiguousarray([[0,0,0],[9,9,9]], dtype='float32')) + mod = type(t._ctree).__module__ + assert '_prebuilt' in mod, mod + found = t.search(np.array([0,0,0],'float32'), np.array([1,1,1],'float32')) + assert found.ravel().tolist() == [1], found + print('ok:', mod)" + # the test module imports the codegen (and so Cheetah), so pull the real + # dependency set back in before running the suite + - name: Run the prebuilt test suite against the wheel + run: | + uv pip install wheelhouse/*.whl pytest + uv run --no-sync pytest tests/test_prebuilt.py -v + deploy: name: Deploy - needs: test + needs: [test, test-abi3-wheel, build-sdist] if: success() && startsWith(github.ref, 'refs/tags/') && github.event_name != 'schedule' runs-on: ubuntu-latest @@ -90,17 +178,15 @@ jobs: contents: write steps: - - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 with: - fetch-depth: 0 - - uses: astral-sh/setup-uv@v6 + pattern: wheels-* + path: dist + merge-multiple: true + - uses: actions/download-artifact@v4 with: - python-version: ${{ matrix.python-version }} - enable-cache: true - cache-dependency-glob: "**/pyproject.toml" - - - name: 👷 Build - run: uv build + name: sdist + path: dist - name: 🚢 Publish to PyPI uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/pyproject.toml b/pyproject.toml index e223f92..0dda8d2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,6 +73,16 @@ docs = [ homepage = "https://github.com/funkelab/spatial_graph" repository = "https://github.com/funkelab/spatial_graph" +[tool.cibuildwheel] +# a single abi3 build per platform covers every supported CPython +build = "cp311-*" +# never let a wheel silently degrade to pure Python +environment = { SPATIAL_GRAPH_REQUIRE_PREBUILT = "1" } +test-groups = ["test"] +# these exercise the prebuilt modules in the repaired wheel without needing a +# compiler; cross-version and numpy-only checks live in the CI workflow +test-command = "pytest {project}/tests/test_prebuilt.py -q" + [tool.ruff] target-version = "py311" line-length = 88 diff --git a/src/spatial_graph/_rtree/_codegen.py b/src/spatial_graph/_rtree/_codegen.py index c1d401e..3b30f35 100644 --- a/src/spatial_graph/_rtree/_codegen.py +++ b/src/spatial_graph/_rtree/_codegen.py @@ -25,14 +25,14 @@ TEMPLATE = Path(__file__).parent / "wrapper_template.pyx" -# Variants compiled ahead of time into binary wheels. Only `PointRTree` by -# default: `LineRTree` is only ever used by `SpatialGraph`, whose graph half is -# JIT-compiled regardless, so prebuilding it would double the wheel size without -# removing anyone's compiler requirement. +# Variants compiled ahead of time into binary wheels. `PointRTree` is what makes +# a compiler unnecessary for rtree-only users; `LineRTree` is only reached via +# `SpatialGraph`, whose graph half is JIT-compiled regardless, so prebuilding it +# saves first-use compile time rather than removing a requirement. ITEM_BASES = ("int64", "uint64") COORD_DTYPES = ("float32", "float64") DIMS = (2, 3, 4, 5) -PREBUILT_LINE_TREES = False +PREBUILT_LINE_TREES = True class Spec(NamedTuple): From a74fd2a8b3edb3c9c4152df687fdce2b91c65812 Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Wed, 29 Jul 2026 17:44:16 +0200 Subject: [PATCH 06/15] ci: update macOS runner version to 15 for compatibility --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8debc4b..40e40e4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -92,7 +92,7 @@ jobs: - ubuntu-latest # manylinux x86_64 - ubuntu-24.04-arm # manylinux aarch64 - windows-latest # win_amd64 - - macos-13 # macOS x86_64 + - macos-15-intel # macOS x86_64 - macos-latest # macOS arm64 steps: - uses: actions/checkout@v4 From e8c3ad7a5d59b36c97d6c58fbbacee736c176cc9 Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Fri, 28 Aug 2026 10:28:04 +0200 Subject: [PATCH 07/15] ci: pick the right linux wheel via --find-links instead of a glob --- .github/workflows/ci.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 40e40e4..e094491 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -143,7 +143,9 @@ jobs: run: | uv venv uv pip install numpy - uv pip install --no-deps wheelhouse/*.whl + # cibuildwheel emits both manylinux and musllinux wheels on Linux, so a + # glob is ambiguous; let the resolver pick the one this runner can use + uv pip install --no-deps --no-index --find-links wheelhouse spatial-graph - name: Prebuilt rtrees must work without witty, Cheetah or a compiler run: | uv run --no-sync python -c " @@ -160,11 +162,11 @@ jobs: found = t.search(np.array([0,0,0],'float32'), np.array([1,1,1],'float32')) assert found.ravel().tolist() == [1], found print('ok:', mod)" - # the test module imports the codegen (and so Cheetah), so pull the real - # dependency set back in before running the suite + # the test module imports the codegen (and so Cheetah); add it rather than + # reinstalling, so the wheel under test stays exactly as installed above - name: Run the prebuilt test suite against the wheel run: | - uv pip install wheelhouse/*.whl pytest + uv pip install pytest CT3 uv run --no-sync pytest tests/test_prebuilt.py -v deploy: From 019c7b80d85554ec2dd9da2799e9aa2a70e44f75 Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Fri, 28 Aug 2026 10:30:28 +0200 Subject: [PATCH 08/15] always prebuild both tree classes; drop the PREBUILT_LINE_TREES flag --- src/spatial_graph/_rtree/_codegen.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/spatial_graph/_rtree/_codegen.py b/src/spatial_graph/_rtree/_codegen.py index 3b30f35..35e141b 100644 --- a/src/spatial_graph/_rtree/_codegen.py +++ b/src/spatial_graph/_rtree/_codegen.py @@ -25,14 +25,14 @@ TEMPLATE = Path(__file__).parent / "wrapper_template.pyx" -# Variants compiled ahead of time into binary wheels. `PointRTree` is what makes -# a compiler unnecessary for rtree-only users; `LineRTree` is only reached via -# `SpatialGraph`, whose graph half is JIT-compiled regardless, so prebuilding it -# saves first-use compile time rather than removing a requirement. +# Variants compiled ahead of time into binary wheels. Both tree classes are +# public API and usable on their own, so both are prebuilt: it is what lets an +# rtree-only user install without a C compiler. Trimming the matrix means +# dropping entries below; `SPATIAL_GRAPH_NO_PREBUILT=1` skips prebuilding +# entirely, for machines that cannot compile at build time. ITEM_BASES = ("int64", "uint64") COORD_DTYPES = ("float32", "float64") DIMS = (2, 3, 4, 5) -PREBUILT_LINE_TREES = True class Spec(NamedTuple): @@ -48,8 +48,7 @@ def iter_specs() -> Iterator[Spec]: for coord in COORD_DTYPES: for dims in DIMS: yield Spec(PointRTree, base, coord, dims) - if PREBUILT_LINE_TREES: - yield Spec(LineRTree, f"{base}[2]", coord, dims) + yield Spec(LineRTree, f"{base}[2]", coord, dims) def build_wrapper( From 73b5b86171b0662a3056d3a0bf970e5c0a8d2d6a Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Fri, 28 Aug 2026 11:19:35 +0200 Subject: [PATCH 09/15] address review: setup.py __main__ guard, C-core depends, stronger compiler probe, README - guard setup() so cythonize's worker processes don't re-run the build - declare rtree.c/h + config.h as Extension depends, so edits to the vendored C core actually rebuild the prebuilt modules instead of silently going stale - probe with #include (via get_python_inc) so a box with cc but no dev headers is caught before the real build, and guard the distutils import - package-data patterns needed a src/ prefix to match the vendored C at all - rewrite README's Cross-Platform Support section; it still told users they needed a compiler - reword comments that implied witty/Cheetah were optional: they remain install-time deps, they're just never invoked on the prebuilt path --- .github/workflows/ci.yml | 6 +-- README.md | 34 ++++++++++--- pyproject.toml | 8 ++- setup.py | 74 ++++++++++++++++++---------- src/spatial_graph/_rtree/_codegen.py | 5 +- src/spatial_graph/_rtree/_naming.py | 4 +- src/spatial_graph/_rtree/rtree.py | 12 +++-- 7 files changed, 95 insertions(+), 48 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e094491..3f91131 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -89,9 +89,9 @@ jobs: fail-fast: false matrix: os: - - ubuntu-latest # manylinux x86_64 - - ubuntu-24.04-arm # manylinux aarch64 - - windows-latest # win_amd64 + - ubuntu-latest # manylinux + musllinux x86_64 + - ubuntu-24.04-arm # manylinux + musllinux aarch64 + - windows-latest # win_amd64 + win32 - macos-15-intel # macOS x86_64 - macos-latest # macOS arm64 steps: diff --git a/README.md b/README.md index 95174f2..97e9b71 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,8 @@ where each node has an nD position (in time or space). * attribute access * minimal memory footprint * minimal dependencies - * `cython` / `witty` / `cheetah3` for runtime compilation + * `cython` / `witty` / `cheetah3`, used only when something has to be + compiled at runtime (see Cross-Platform Support) * numpy for array interfaces * PYX API for graph algorithms in C/C++ @@ -116,16 +117,33 @@ A `SpatialGraph` consists of three data structures: ## Cross-Platform Support -`spatial_graph` compiles C/C++ code at runtime, and as such needs access to a -compiler. If you already have one, great! You can use the PyPI package. +`spatial_graph` generates specialized C/C++ for the exact data types you ask +for. Where those types can be known in advance we compile them ahead of time +and ship them in the wheels; everything else is compiled on your machine the +first time it is used, which needs a C compiler. -If you (or your users) don't have a compiler installed, you either need to +**No compiler needed.** The PyPI wheels contain prebuilt `PointRTree` and +`LineRTree` variants for the common combinations: `int64`/`uint64` items, +`float32`/`float64` coordinates, and 2 to 5 dimensions. If your R-tree matches +one of those -- as most do -- nothing is compiled, on any supported Python. -1. Install a compiler. This might be weird for non-technical users. -2. Install `spatial_graph` from `conda-forge`, where we include a compiler - (`clang`) in its dependencies. +**Compiler needed.** Two cases fall back to compiling at runtime: -### Why is this so complicated? +1. `Graph`, `DiGraph`, `SpatialGraph` and `SpatialDiGraph`. Their node and edge + attribute types are only known when you construct the graph, so they cannot + be enumerated ahead of time. +2. R-trees outside the prebuilt set above (an `int32` item type, say, or 6 + dimensions). + +If you or your users need those without a compiler, you can still install +`spatial_graph` from `conda-forge`, where we include a compiler (`clang`) in +its dependencies. + +The wheels are `abi3` (stable ABI) and require Python 3.11 or newer, so one +wheel per platform covers every supported CPython. Python 3.10 users should +pin to a release before this one. + +### Why can't everything be prebuilt? There is no cross-platform C/C++ compiler that we can install using `pip`. [`numba`](https://github.com/numba/numba) is maybe the closest to having solved diff --git a/pyproject.toml b/pyproject.toml index 0dda8d2..7894234 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,8 +14,12 @@ build-backend = "setuptools.build_meta" where = ["src"] [tool.setuptools.package-data] -# the JIT fallback compiles from these at runtime, so they must ship in the wheel -"*" = ["py.typed", "*.pyx", "*.c", "*.h", "LICENSE*", "*.md"] +# The JIT fallback compiles from these at runtime, so they must ship in the +# wheel. `include-package-data` (on by default here) would pick them up from +# the VCS/sdist manifest anyway; listing them keeps that explicit and works +# even when no file finder is available. Patterns are package-relative, so the +# vendored C under `_rtree/src/` and `_graph/src/` needs the `src/` prefix. +"*" = ["py.typed", "*.pyx", "src/*.c", "src/*.h", "src/LICENSE*", "src/*.md"] [project] name = "spatial-graph" diff --git a/setup.py b/setup.py index 67c98d7..f041c35 100644 --- a/setup.py +++ b/setup.py @@ -1,9 +1,10 @@ """Compile RTree variants ahead of time into a stable-ABI (abi3) wheel. Renders the same pyx wrappers the runtime would JIT-compile (via -`_rtree._codegen`) for every variant in `iter_specs()`, so prebuilt and -JIT-compiled modules can never disagree. One wheel per platform then covers -every supported CPython, and users never need a C compiler for those variants. +`_rtree._codegen`) for every variant in `iter_specs()`, so the two paths are +generated from one source. One wheel per platform then covers every supported +CPython, and those variants are never compiled on the user's machine -- so no +C compiler is invoked for them. Set `SPATIAL_GRAPH_NO_PREBUILT=1` to build a pure-Python wheel instead, or `SPATIAL_GRAPH_REQUIRE_PREBUILT=1` (as CI does) to turn a failure to compile @@ -46,6 +47,12 @@ def prebuilt_extensions() -> list[Extension]: pyx_dir = ROOT / "build" / "prebuilt-pyx" pyx_dir.mkdir(parents=True, exist_ok=True) + # the template `#include`s these at compile time, so they never appear in + # `sources`; without `depends` an edit to the C core would leave every + # prebuilt module stale (the JIT path declares the same set to witty) + rtree_dir = SRC / "spatial_graph" / "_rtree" + depends = [str(rtree_dir / "src" / f) for f in ("rtree.c", "rtree.h", "config.h")] + extensions = [] for spec in iter_specs(): name = module_name(spec.cls, spec.item_dtype, spec.coord_dtype, spec.dims) @@ -58,7 +65,8 @@ def prebuilt_extensions() -> list[Extension]: Extension( f"{PREBUILT_PKG}.{name}", sources=[str(path)], - include_dirs=[str(SRC / "spatial_graph" / "_rtree")], + depends=depends, + include_dirs=[str(rtree_dir)], extra_compile_args=["/O2"] if WIN else ["-O3", "-Wno-unreachable-code"], define_macros=[ ("Py_LIMITED_API", ABI3_HEX), @@ -77,19 +85,30 @@ def prebuilt_extensions() -> list[Extension]: def can_compile() -> bool: - """Whether this machine can build a C extension at all.""" - from distutils.ccompiler import new_compiler - from distutils.sysconfig import customize_compiler - - compiler = new_compiler() - customize_compiler(compiler) # picks up CC/CFLAGS, as build_ext does - with tempfile.TemporaryDirectory() as tmp: - probe = Path(tmp, "probe.c") - probe.write_text("int main(void) { return 0; }\n") - try: - compiler.compile([str(probe)], output_dir=tmp) - except Exception: - return False + """Whether this machine can build a C extension at all. + + Includes `Python.h` so that a box with a C compiler but no development + headers -- the usual shape of this failure -- is caught here rather than + part-way through the real build. + """ + try: + from distutils.ccompiler import new_compiler + from distutils.sysconfig import customize_compiler, get_python_inc + + compiler = new_compiler() + customize_compiler(compiler) # picks up CC/CFLAGS, as build_ext does + with tempfile.TemporaryDirectory() as tmp: + probe = Path(tmp, "probe.c") + probe.write_text("#include \nint main(void) { return 0; }\n") + compiler.compile( + [str(probe)], + output_dir=tmp, + # get_python_inc, not sysconfig: inside a venv the latter + # points at the venv, which holds no headers + include_dirs=[get_python_inc()], + ) + except Exception: + return False return True @@ -113,13 +132,14 @@ def should_prebuild() -> bool: return False -if should_prebuild(): - setup( - ext_modules=prebuilt_extensions(), - options={ - "bdist_wheel": {"py_limited_api": ABI3_TAG}, - "build_ext": {"parallel": os.cpu_count()}, - }, - ) -else: - setup(ext_modules=[]) +if __name__ == "__main__": + if should_prebuild(): + setup( + ext_modules=prebuilt_extensions(), + options={ + "bdist_wheel": {"py_limited_api": ABI3_TAG}, + "build_ext": {"parallel": os.cpu_count()}, + }, + ) + else: + setup(ext_modules=[]) diff --git a/src/spatial_graph/_rtree/_codegen.py b/src/spatial_graph/_rtree/_codegen.py index 35e141b..5e09257 100644 --- a/src/spatial_graph/_rtree/_codegen.py +++ b/src/spatial_graph/_rtree/_codegen.py @@ -2,8 +2,9 @@ Used on the JIT path and by `setup.py`, so prebuilt and JIT-compiled modules are always generated from the same source. Requires Cheetah, and is therefore -imported lazily by `rtree.py` -- installs that stay on the prebuilt path need -neither Cheetah nor witty. +imported lazily by `rtree.py`: Cheetah and witty stay installed either way, but +a tree that resolves to a prebuilt module never invokes them, and so never +needs a C compiler. """ from __future__ import annotations diff --git a/src/spatial_graph/_rtree/_naming.py b/src/spatial_graph/_rtree/_naming.py index dcf34eb..d6362dc 100644 --- a/src/spatial_graph/_rtree/_naming.py +++ b/src/spatial_graph/_rtree/_naming.py @@ -31,7 +31,9 @@ def module_name(cls: type[RTree], item_dtype: str, coord_dtype: str, dims: int) Dtypes are canonicalized (so `int` and `int64` agree) and spelled out for readability. The trailing digest covers the C/pyx code `cls` injects into the template, so a subclass with custom code can never be served a prebuilt - module compiled from different code. + module compiled from different code. It deliberately does not cover the + template itself, the vendored C, or the compiler flags: keeping those in + step is the build's job (see `depends` in `setup.py`). """ parts = ( cls.pyx_item_t_declaration, diff --git a/src/spatial_graph/_rtree/rtree.py b/src/spatial_graph/_rtree/rtree.py index 8afb50e..a4457d2 100644 --- a/src/spatial_graph/_rtree/rtree.py +++ b/src/spatial_graph/_rtree/rtree.py @@ -40,9 +40,9 @@ def _jit_compile_tree( ) -> type: """Compile a tree with the system C compiler. - Only reached for dtype combinations not shipped prebuilt; Cheetah and witty - are imported here so neither is needed by installs that stay on the - prebuilt path. + Only reached for dtype combinations not shipped prebuilt. Cheetah and witty + are imported here rather than at module scope so that trees served from a + prebuilt module never invoke them -- and so never need a C compiler. """ import witty @@ -75,8 +75,10 @@ def _compile_tree( class RTree: - """A generic RTree implementation, compiled on-the-fly during - instantiation. + """A generic RTree implementation, specialized for the given types. + + Common type combinations ship precompiled in the wheels; anything else is + compiled on the fly during instantiation, which requires a C compiler. Args: From a664bebce95a8f8be7a4488ddde49be458b08110 Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Fri, 28 Aug 2026 11:39:37 +0200 Subject: [PATCH 10/15] PEP 639 license, lazy metadata builds, env-flag parsing; restore eager __init__ - license = "MIT" + license-files, drop the deprecated classifier - skip rendering/cythonizing 32 variants for metadata-only commands, matched on recognized command names so 'sdist bdist_wheel' still builds - SPATIAL_GRAPH_NO_PREBUILT=0 disabled prebuilding, since any non-empty string is truthy; both readers now share env_flag() so they cannot diverge - the lazy __init__ broke the API docs: griffe inspects __dict__, so the seven deferred names vanished (527 -> 414 documented symbols). Import witty and Cheetah lazily inside _graph.graph_base instead, which keeps the 35ms import and restores full docs --- pyproject.toml | 4 +-- setup.py | 37 +++++++++++++++++++++++--- src/spatial_graph/__init__.py | 34 +++-------------------- src/spatial_graph/_graph/graph_base.py | 12 ++++++--- src/spatial_graph/_rtree/_naming.py | 16 ++++++++++- src/spatial_graph/_rtree/rtree.py | 5 ++-- 6 files changed, 65 insertions(+), 43 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7894234..62d786b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,8 @@ dynamic = ["version"] description = "A spatial graph datastructure for python." readme = "README.md" requires-python = ">=3.11" -license = { text = "MIT" } +license = "MIT" +license-files = ["LICENSE"] authors = [ { email = "funkej@janelia.hhmi.org", name = "Jan Funke" }, { email = "talley.lambert@gmail.com", name = "Talley Lambert" }, @@ -35,7 +36,6 @@ authors = [ classifiers = [ "Development Status :: 3 - Alpha", "Intended Audience :: Developers", - "License :: OSI Approved :: MIT License", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", diff --git a/setup.py b/setup.py index f041c35..7521b28 100644 --- a/setup.py +++ b/setup.py @@ -23,6 +23,10 @@ ROOT = Path(__file__).parent SRC = ROOT / "src" +sys.path.insert(0, str(SRC)) + +from spatial_graph._rtree._naming import env_flag # noqa: E402 + PREBUILT_PKG = "spatial_graph._rtree._prebuilt" # The wrappers pass numpy arrays as typed memoryviews, which compile to @@ -40,7 +44,6 @@ def prebuilt_extensions() -> list[Extension]: """Render every prebuilt RTree variant and declare it as an extension.""" from Cython.Build import cythonize - sys.path.insert(0, str(SRC)) from spatial_graph._rtree._codegen import build_wrapper, iter_specs from spatial_graph._rtree._naming import module_name @@ -112,11 +115,39 @@ def can_compile() -> bool: return True +# Rendering and cythonizing every variant is wasted work for commands that only +# want metadata -- without this, `build --sdist` pays for a full codegen pass. +METADATA_ONLY = {"egg_info", "dist_info", "sdist"} +NEEDS_EXTENSIONS = { + "bdist_egg", + "bdist_wheel", + "build", + "build_ext", + "build_py", + "develop", + "editable_wheel", + "install", +} + + +def metadata_only() -> bool: + """Whether this invocation asks for nothing that needs the extensions. + + Matches on recognized command names only, so option values (`--dist-dir + /tmp/x`) are ignored, and anything unrecognized falls through to building + -- skipping wrongly would silently yield a wheel with no prebuilt modules. + """ + seen = {arg for arg in sys.argv[1:] if arg in METADATA_ONLY | NEEDS_EXTENSIONS} + return bool(seen) and seen <= METADATA_ONLY + + def should_prebuild() -> bool: """Whether to compile prebuilt variants into this wheel.""" - if os.getenv("SPATIAL_GRAPH_NO_PREBUILT"): + if metadata_only(): + return False + if env_flag("SPATIAL_GRAPH_NO_PREBUILT"): return False - if os.getenv("SPATIAL_GRAPH_REQUIRE_PREBUILT"): + if env_flag("SPATIAL_GRAPH_REQUIRE_PREBUILT"): return True # CI: never let a build silently degrade if can_compile(): return True diff --git a/src/spatial_graph/__init__.py b/src/spatial_graph/__init__.py index c1ab15a..4ec1dee 100644 --- a/src/spatial_graph/__init__.py +++ b/src/spatial_graph/__init__.py @@ -1,5 +1,4 @@ from importlib.metadata import PackageNotFoundError, version -from typing import TYPE_CHECKING, Any try: __version__ = version("spatial_graph") @@ -7,25 +6,10 @@ __version__ = "unknown" +from ._graph import DiGraph, Graph, GraphBase from ._rtree import LineRTree, PointRTree - -if TYPE_CHECKING: - from ._graph import DiGraph, Graph, GraphBase - from ._spatial_graph import SpatialDiGraph, SpatialGraph, SpatialGraphBase - from ._util import create_graph - -# the graph half is always JIT-compiled, and importing it pulls in witty and -# Cheetah. Deferring it keeps `PointRTree`/`LineRTree` -- which ship prebuilt -- -# usable with numpy alone. -_LAZY = { - "DiGraph": "._graph", - "Graph": "._graph", - "GraphBase": "._graph", - "SpatialDiGraph": "._spatial_graph", - "SpatialGraph": "._spatial_graph", - "SpatialGraphBase": "._spatial_graph", - "create_graph": "._util", -} +from ._spatial_graph import SpatialDiGraph, SpatialGraph, SpatialGraphBase +from ._util import create_graph __all__ = [ "DiGraph", @@ -38,15 +22,3 @@ "SpatialGraphBase", "create_graph", ] - - -def __getattr__(name: str) -> Any: - if module := _LAZY.get(name): - import importlib - - return getattr(importlib.import_module(module, __name__), name) - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") - - -def __dir__() -> list[str]: - return __all__ diff --git a/src/spatial_graph/_graph/graph_base.py b/src/spatial_graph/_graph/graph_base.py index 52ac4cf..fc4ca90 100644 --- a/src/spatial_graph/_graph/graph_base.py +++ b/src/spatial_graph/_graph/graph_base.py @@ -4,9 +4,6 @@ from pathlib import Path from typing import TYPE_CHECKING, Any -import witty -from Cheetah.Template import Template - from spatial_graph._dtypes import DType from .views import EdgeAttrs, NodeAttrs @@ -75,6 +72,10 @@ def _build_wrapper( if not all(str.isidentifier(name) for name in edge_attr_dtypes): raise ValueError("Edge attribute names must be valid identifiers") + # imported here, not at module scope: it is only needed when something has + # to be compiled, and keeps `import spatial_graph` off Cheetah/Cython + from Cheetah.Template import Template + wrapper_template = Template( file=str(SRC_DIR / "wrapper_template.pyx"), compilerSettings={"directiveStartToken": "%"}, @@ -99,6 +100,11 @@ def _compile_graph( edge_attr_dtypes: Mapping[str, str] | None = None, directed: bool = False, ) -> type: + # graph attribute dtypes are only known at runtime, so this half can never + # be prebuilt; witty is imported here so that merely importing spatial_graph + # does not pull in the compilation toolchain + import witty + wrapper_template = _build_wrapper( node_dtype=node_dtype, node_attr_dtypes=node_attr_dtypes, diff --git a/src/spatial_graph/_rtree/_naming.py b/src/spatial_graph/_rtree/_naming.py index d6362dc..dab348a 100644 --- a/src/spatial_graph/_rtree/_naming.py +++ b/src/spatial_graph/_rtree/_naming.py @@ -1,4 +1,4 @@ -"""Deterministic naming for prebuilt RTree extension modules. +"""Naming and switches for prebuilt RTree extension modules. Shared by the runtime lookup and `setup.py`, so the two can never disagree. Deliberately depends only on `_dtypes` -- it sits on the import path of every @@ -8,6 +8,7 @@ from __future__ import annotations import hashlib +import os from typing import TYPE_CHECKING from spatial_graph._dtypes import DType @@ -18,6 +19,19 @@ # subpackage holding ahead-of-time compiled modules; empty in a source checkout PREBUILT_PACKAGE = f"{__package__}._prebuilt" +_FALSEY = {"", "0", "false", "no", "off"} + + +def env_flag(name: str) -> bool: + """Whether an on/off environment variable is set. + + `SPATIAL_GRAPH_NO_PREBUILT=0` should mean "no, don't skip prebuilding"; + plain truthiness would read it as "yes", since any non-empty string is + true. Both the build and the runtime lookup go through here so they can + never read the same variable differently. + """ + return os.environ.get(name, "").strip().lower() not in _FALSEY + def _c_name(dtype: DType) -> str: """Canonical, identifier-safe name for a dtype ("int64", "float", "int64x2").""" diff --git a/src/spatial_graph/_rtree/rtree.py b/src/spatial_graph/_rtree/rtree.py index a4457d2..3a26465 100644 --- a/src/spatial_graph/_rtree/rtree.py +++ b/src/spatial_graph/_rtree/rtree.py @@ -1,7 +1,6 @@ from __future__ import annotations import importlib -import os import sys from pathlib import Path from typing import ClassVar @@ -10,7 +9,7 @@ from spatial_graph._dtypes import DType -from ._naming import PREBUILT_PACKAGE, module_name +from ._naming import PREBUILT_PACKAGE, env_flag, module_name DEFINE_MACROS = [("RTREE_NOATOMICS", "1")] if sys.platform == "win32" else [] if sys.platform == "win32": # pragma: no cover @@ -25,7 +24,7 @@ def _load_prebuilt( cls: type[RTree], item_dtype: str, coord_dtype: str, dims: int ) -> type | None: """Return the ahead-of-time compiled tree class, or None if not shipped.""" - if os.getenv("SPATIAL_GRAPH_NO_PREBUILT"): + if env_flag("SPATIAL_GRAPH_NO_PREBUILT"): return None name = module_name(cls, item_dtype, coord_dtype, dims) try: From 542f81bd4958dbc35e1ba7220d5c577730de6ba5 Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Fri, 28 Aug 2026 11:45:37 +0200 Subject: [PATCH 11/15] rename env_flag to env_enabled --- setup.py | 6 +++--- src/spatial_graph/_rtree/_naming.py | 4 ++-- src/spatial_graph/_rtree/rtree.py | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/setup.py b/setup.py index 7521b28..a5e5396 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ SRC = ROOT / "src" sys.path.insert(0, str(SRC)) -from spatial_graph._rtree._naming import env_flag # noqa: E402 +from spatial_graph._rtree._naming import env_enabled # noqa: E402 PREBUILT_PKG = "spatial_graph._rtree._prebuilt" @@ -145,9 +145,9 @@ def should_prebuild() -> bool: """Whether to compile prebuilt variants into this wheel.""" if metadata_only(): return False - if env_flag("SPATIAL_GRAPH_NO_PREBUILT"): + if env_enabled("SPATIAL_GRAPH_NO_PREBUILT"): return False - if env_flag("SPATIAL_GRAPH_REQUIRE_PREBUILT"): + if env_enabled("SPATIAL_GRAPH_REQUIRE_PREBUILT"): return True # CI: never let a build silently degrade if can_compile(): return True diff --git a/src/spatial_graph/_rtree/_naming.py b/src/spatial_graph/_rtree/_naming.py index dab348a..5982e43 100644 --- a/src/spatial_graph/_rtree/_naming.py +++ b/src/spatial_graph/_rtree/_naming.py @@ -22,8 +22,8 @@ _FALSEY = {"", "0", "false", "no", "off"} -def env_flag(name: str) -> bool: - """Whether an on/off environment variable is set. +def env_enabled(name: str) -> bool: + """Whether the named on/off environment variable is switched on. `SPATIAL_GRAPH_NO_PREBUILT=0` should mean "no, don't skip prebuilding"; plain truthiness would read it as "yes", since any non-empty string is diff --git a/src/spatial_graph/_rtree/rtree.py b/src/spatial_graph/_rtree/rtree.py index 3a26465..8c79055 100644 --- a/src/spatial_graph/_rtree/rtree.py +++ b/src/spatial_graph/_rtree/rtree.py @@ -9,7 +9,7 @@ from spatial_graph._dtypes import DType -from ._naming import PREBUILT_PACKAGE, env_flag, module_name +from ._naming import PREBUILT_PACKAGE, env_enabled, module_name DEFINE_MACROS = [("RTREE_NOATOMICS", "1")] if sys.platform == "win32" else [] if sys.platform == "win32": # pragma: no cover @@ -24,7 +24,7 @@ def _load_prebuilt( cls: type[RTree], item_dtype: str, coord_dtype: str, dims: int ) -> type | None: """Return the ahead-of-time compiled tree class, or None if not shipped.""" - if env_flag("SPATIAL_GRAPH_NO_PREBUILT"): + if env_enabled("SPATIAL_GRAPH_NO_PREBUILT"): return None name = module_name(cls, item_dtype, coord_dtype, dims) try: From 814ad6b8fe056539ab0b9456c4ce79a7340ceba0 Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Fri, 28 Aug 2026 12:07:08 +0200 Subject: [PATCH 12/15] review: stop setup.py importing the package at module scope Two follow-ups from review. setup.py imported `spatial_graph._rtree._naming` at module scope (after inserting src/ on sys.path), which pulled the whole package -- and with the eager __init__, the graph half too -- into every setuptools command. A single module-scope third-party import anywhere in the package would then break the entire build system, `sdist` and `egg_info` included, with a traceback pointing at the package rather than at setup.py. Verified: adding `import witty` to graph_base.py made `build --sdist` fail at get_requires_for_build_sdist. The import now happens only where the package is genuinely needed, via `_src_on_path()`: in `prebuilt_extensions()`, and in `should_prebuild()` after the `metadata_only()` early return. Metadata-only commands no longer import the package at all -- `spatial_graph` stays out of sys.modules -- while `env_enabled` remains shared with the runtime, so the build and the lookup still cannot read the same variable differently. tests/test_prebuilt.py was the one consumer not converted to `env_enabled`, so SPATIAL_GRAPH_REQUIRE_PREBUILT=0 would skip prebuilding at build time yet still assert it at test time. Co-Authored-By: Claude Opus 5 --- setup.py | 21 +++++++++++++++++---- tests/test_prebuilt.py | 8 +++----- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/setup.py b/setup.py index a5e5396..a17e34f 100644 --- a/setup.py +++ b/setup.py @@ -23,10 +23,6 @@ ROOT = Path(__file__).parent SRC = ROOT / "src" -sys.path.insert(0, str(SRC)) - -from spatial_graph._rtree._naming import env_enabled # noqa: E402 - PREBUILT_PKG = "spatial_graph._rtree._prebuilt" # The wrappers pass numpy arrays as typed memoryviews, which compile to @@ -40,10 +36,23 @@ WIN = sys.platform == "win32" +def _src_on_path() -> None: + """Make the package under `src/` importable, for the helpers shared with it. + + Callers import `spatial_graph` only once they know they need it: importing + it at module scope would make every command -- `sdist` and `egg_info` + included -- fail if anything the package imports is missing from the build + environment. + """ + if str(SRC) not in sys.path: + sys.path.insert(0, str(SRC)) + + def prebuilt_extensions() -> list[Extension]: """Render every prebuilt RTree variant and declare it as an extension.""" from Cython.Build import cythonize + _src_on_path() from spatial_graph._rtree._codegen import build_wrapper, iter_specs from spatial_graph._rtree._naming import module_name @@ -145,6 +154,10 @@ def should_prebuild() -> bool: """Whether to compile prebuilt variants into this wheel.""" if metadata_only(): return False + + _src_on_path() + from spatial_graph._rtree._naming import env_enabled + if env_enabled("SPATIAL_GRAPH_NO_PREBUILT"): return False if env_enabled("SPATIAL_GRAPH_REQUIRE_PREBUILT"): diff --git a/tests/test_prebuilt.py b/tests/test_prebuilt.py index 5ffa2b3..5644f01 100644 --- a/tests/test_prebuilt.py +++ b/tests/test_prebuilt.py @@ -8,14 +8,12 @@ from __future__ import annotations -import os - import numpy as np import pytest from spatial_graph import PointRTree from spatial_graph._rtree._codegen import iter_specs -from spatial_graph._rtree._naming import module_name +from spatial_graph._rtree._naming import env_enabled, module_name from spatial_graph._rtree.rtree import _load_prebuilt # the `_prebuilt` package always exists but is empty in a source checkout, so @@ -28,8 +26,8 @@ def test_prebuilt_modules_were_shipped(): """Guard against a wheel that silently degraded to pure Python.""" - if not os.getenv("SPATIAL_GRAPH_REQUIRE_PREBUILT"): - pytest.skip("SPATIAL_GRAPH_REQUIRE_PREBUILT not set") + if not env_enabled("SPATIAL_GRAPH_REQUIRE_PREBUILT"): + pytest.skip("SPATIAL_GRAPH_REQUIRE_PREBUILT not enabled") assert has_prebuilt, "install shipped no prebuilt rtree modules" From 5bb1fd557210c89db358c4879d4d75e96bc8c059 Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Fri, 28 Aug 2026 14:18:16 +0200 Subject: [PATCH 13/15] correct three inaccurate claims; mirror build_ext's include dirs in the probe - can_compile()'s comment justified get_python_inc with a venv/sysconfig mechanism that does not exist; replaced with the real reason, and the probe now also passes the platform-specific include dir as build_ext does, since that is where pyconfig.h can live and a false negative there silently emits a pure-Python wheel - _naming.py claimed installs may have neither Cheetah nor witty; both are unconditional dependencies, and it contradicted _codegen.py - README implied LineRTree is prebuilt for int64/uint64 items; its items are node pairs, so the prebuilt set is int64[2]/uint64[2] --- README.md | 5 +++-- setup.py | 13 ++++++------- src/spatial_graph/_rtree/_naming.py | 5 +++-- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 97e9b71..3508bc4 100644 --- a/README.md +++ b/README.md @@ -123,8 +123,9 @@ and ship them in the wheels; everything else is compiled on your machine the first time it is used, which needs a C compiler. **No compiler needed.** The PyPI wheels contain prebuilt `PointRTree` and -`LineRTree` variants for the common combinations: `int64`/`uint64` items, -`float32`/`float64` coordinates, and 2 to 5 dimensions. If your R-tree matches +`LineRTree` variants for the common combinations: `float32`/`float64` +coordinates, 2 to 5 dimensions, and `int64`/`uint64` items -- as `int64[2]` / +`uint64[2]` for `LineRTree`, whose items are node pairs. If your R-tree matches one of those -- as most do -- nothing is compiled, on any supported Python. **Compiler needed.** Two cases fall back to compiling at runtime: diff --git a/setup.py b/setup.py index a17e34f..f816e3c 100644 --- a/setup.py +++ b/setup.py @@ -112,13 +112,12 @@ def can_compile() -> bool: with tempfile.TemporaryDirectory() as tmp: probe = Path(tmp, "probe.c") probe.write_text("#include \nint main(void) { return 0; }\n") - compiler.compile( - [str(probe)], - output_dir=tmp, - # get_python_inc, not sysconfig: inside a venv the latter - # points at the venv, which holds no headers - include_dirs=[get_python_inc()], - ) + # mirror build_ext, which passes both include dirs when the + # platform-specific one differs (that is where pyconfig.h can live) + includes = [get_python_inc()] + if (plat_inc := get_python_inc(plat_specific=True)) not in includes: + includes.append(plat_inc) + compiler.compile([str(probe)], output_dir=tmp, include_dirs=includes) except Exception: return False return True diff --git a/src/spatial_graph/_rtree/_naming.py b/src/spatial_graph/_rtree/_naming.py index 5982e43..3b06a99 100644 --- a/src/spatial_graph/_rtree/_naming.py +++ b/src/spatial_graph/_rtree/_naming.py @@ -1,8 +1,9 @@ """Naming and switches for prebuilt RTree extension modules. Shared by the runtime lookup and `setup.py`, so the two can never disagree. -Deliberately depends only on `_dtypes` -- it sits on the import path of every -`PointRTree`, including installs with neither Cheetah nor witty available. +Deliberately depends only on `_dtypes`: it sits on the import path of every +`PointRTree`, and importing Cheetah or witty here would pull the compilation +toolchain into the common case, which is the whole thing prebuilding avoids. """ from __future__ import annotations From 13c971a510b19cdfff7a041d6221fdce8a25b755 Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Fri, 28 Aug 2026 14:27:22 +0200 Subject: [PATCH 14/15] assert no runtime compilation directly, instead of simulating a witty-less install Blocking witty.compile_cython states the actual property -- prebuilt variants must not invoke the compiler -- in the environment users really get, since witty is an unconditional dependency and will always be installed. The old CI step instead simulated its absence, which can never happen, and needed --no-deps/--no-index gymnastics plus a second install to undo them. The check now lives in the test suite, so it runs on every OS and Python in the test matrix rather than only in one Linux job, and it covers all 32 variants. test-abi3-wheel keeps its remaining job: proving one cp311 wheel we actually publish runs on 3.11-3.14. --- .github/workflows/ci.yml | 42 +++++++++++++--------------------------- tests/test_prebuilt.py | 19 +++++++++++++++--- 2 files changed, 29 insertions(+), 32 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3f91131..eefc82e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -118,8 +118,9 @@ jobs: name: sdist path: dist/*.tar.gz - # The claim this whole design rests on: one cp311-abi3 wheel runs on every - # supported CPython, with no compiler and no witty. + # Everything else builds its own wheel; this is the only job that exercises + # the artifacts we would actually publish, and the only one that checks the + # stable-ABI claim -- one cp311 wheel running on every supported CPython. test-abi3-wheel: name: abi3 wheel on py${{ matrix.python-version }} needs: build-wheels @@ -139,35 +140,18 @@ jobs: - uses: astral-sh/setup-uv@v6 with: python-version: ${{ matrix.python-version }} - - name: Install the wheel with numpy alone + - name: Install the built wheel with its real dependencies run: | uv venv - uv pip install numpy - # cibuildwheel emits both manylinux and musllinux wheels on Linux, so a - # glob is ambiguous; let the resolver pick the one this runner can use - uv pip install --no-deps --no-index --find-links wheelhouse spatial-graph - - name: Prebuilt rtrees must work without witty, Cheetah or a compiler - run: | - uv run --no-sync python -c " - import sys, numpy as np - try: - import witty; sys.exit('witty present; test is not conclusive') - except ImportError: pass - from spatial_graph import PointRTree - t = PointRTree('int64', 'float32', 3) - t.insert_point_items(np.array([1, 2], dtype='int64'), - np.ascontiguousarray([[0,0,0],[9,9,9]], dtype='float32')) - mod = type(t._ctree).__module__ - assert '_prebuilt' in mod, mod - found = t.search(np.array([0,0,0],'float32'), np.array([1,1,1],'float32')) - assert found.ravel().tolist() == [1], found - print('ok:', mod)" - # the test module imports the codegen (and so Cheetah); add it rather than - # reinstalling, so the wheel under test stays exactly as installed above - - name: Run the prebuilt test suite against the wheel - run: | - uv pip install pytest CT3 - uv run --no-sync pytest tests/test_prebuilt.py -v + # pin the artifact's exact version so the resolver cannot prefer a + # PyPI release; --find-links then picks the manylinux or musllinux + # wheel this runner can use, and dependencies still come from the index + version=$(ls wheelhouse/*.whl | head -1 | sed -E 's/.*spatial_graph-([^-]+)-cp.*/\1/') + uv pip install --find-links wheelhouse "spatial-graph==$version" pytest + - name: Test it + # test_prebuilt.py asserts the real property directly: constructing a + # prebuilt variant must not reach witty.compile_cython + run: uv run --no-sync pytest tests/test_prebuilt.py -v deploy: name: Deploy diff --git a/tests/test_prebuilt.py b/tests/test_prebuilt.py index 5644f01..db95b37 100644 --- a/tests/test_prebuilt.py +++ b/tests/test_prebuilt.py @@ -10,6 +10,7 @@ import numpy as np import pytest +import witty from spatial_graph import PointRTree from spatial_graph._rtree._codegen import iter_specs @@ -33,9 +34,21 @@ def test_prebuilt_modules_were_shipped(): @requires_prebuilt @pytest.mark.parametrize("spec", list(iter_specs()), ids=str) -def test_every_declared_spec_is_shipped(spec): - """Every variant in `iter_specs` must actually resolve to a prebuilt module.""" - assert _load_prebuilt(spec.cls, spec.item_dtype, spec.coord_dtype, spec.dims) +def test_declared_specs_are_shipped_and_never_compile(spec, monkeypatch): + """Every declared variant must construct without invoking the compiler. + + This is the property prebuilding exists for. Blocking `compile_cython` + asserts it directly, in the environment users actually get -- witty is an + unconditional dependency, so it is always installed; what must not happen + is that it gets *used*. + """ + + def no_compiling(*args, **kwargs): + raise AssertionError(f"{spec} triggered runtime compilation") + + monkeypatch.setattr(witty, "compile_cython", no_compiling) + tree = spec.cls(spec.item_dtype, spec.coord_dtype, spec.dims) + assert "_prebuilt" in type(tree._ctree).__module__ @requires_prebuilt From 06df3dba63a06d95ee345fb88ee366a353c6a00e Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Fri, 28 Aug 2026 14:32:49 +0200 Subject: [PATCH 15/15] simplify the wheel install; harden the deploy job before its first run Installing by package name would have resolved spatial-graph from PyPI: our version is a dev release and resolvers exclude pre-releases by default, so 'uv pip install --find-links wheelhouse spatial-graph' silently installs the last real release (verified: it picked 0.0.7). Naming the manylinux wheel is both simpler than pinning the parsed version and immune to that. deploy has never executed, and only ever will on a tag. Give it a checkout rather than assume action-gh-release works without one, and name build-wheels in needs instead of relying on test-abi3-wheel to pull it in. --- .github/workflows/ci.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eefc82e..258955b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -140,14 +140,13 @@ jobs: - uses: astral-sh/setup-uv@v6 with: python-version: ${{ matrix.python-version }} + # Name the wheel directly: cibuildwheel emits a manylinux and a musllinux + # wheel, so a bare *.whl glob is ambiguous, while installing by package + # name would resolve `spatial-graph` from PyPI instead of this artifact. - name: Install the built wheel with its real dependencies run: | uv venv - # pin the artifact's exact version so the resolver cannot prefer a - # PyPI release; --find-links then picks the manylinux or musllinux - # wheel this runner can use, and dependencies still come from the index - version=$(ls wheelhouse/*.whl | head -1 | sed -E 's/.*spatial_graph-([^-]+)-cp.*/\1/') - uv pip install --find-links wheelhouse "spatial-graph==$version" pytest + uv pip install wheelhouse/*manylinux*.whl pytest - name: Test it # test_prebuilt.py asserts the real property directly: constructing a # prebuilt variant must not reach witty.compile_cython @@ -155,7 +154,7 @@ jobs: deploy: name: Deploy - needs: [test, test-abi3-wheel, build-sdist] + needs: [test, build-wheels, build-sdist, test-abi3-wheel] if: success() && startsWith(github.ref, 'refs/tags/') && github.event_name != 'schedule' runs-on: ubuntu-latest @@ -164,6 +163,7 @@ jobs: contents: write steps: + - uses: actions/checkout@v4 - uses: actions/download-artifact@v4 with: pattern: wheels-*