From efc45889161c2ed55210fcf1ac5a111a8d7f63cf Mon Sep 17 00:00:00 2001 From: Simon Birkholz Date: Tue, 18 Aug 2026 14:31:23 +0200 Subject: [PATCH 1/5] Report missing cuvis functions at import instead of failing the load The extension and the cuvis library are in sync when built, but the library installed on a user's machine can be older and simply not export functions the extension imports. That made `import cuvis_il` fail with an opaque loader error and took down every consumer, including ones that never needed the missing part. The module is now delay-loaded on Windows and lazily bound on Linux, so the load survives. At import the loader reads which cuvis functions the built module needs out of its own PE import table or ELF .dynsym, probes each against the library it loaded, and replaces whatever is absent with a stub that raises RuntimeError naming the symbol. `cuvis_il.missing_symbols` reports them, so a consumer can decide which of its features are affected, and `built_against_version` says which SDK the binding expects. No list of function names is maintained anywhere; the needed set comes from the binary. Verified on Linux by building against the 3.5.3 SDK image and running against 3.4.1, which correctly reported the three dead-pixel-correction functions added since. --- .github/workflows/build_and_test.yml | 41 ++++ CMakeLists.txt | 18 ++ cuvis.swig | 2 +- cuvis_il/__init__.py | 290 ++++++++++++++++++++++++++- pyproject.toml | 2 +- 5 files changed, 346 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index f00cb3a..9ab73e5 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -89,6 +89,27 @@ jobs: echo "Version output: $OUTPUT" echo "$OUTPUT" | grep -q "CUBERT SDK" || (echo "Version check failed!" && exit 1) + - name: Smoke test - symbol introspection + run: | + # Meaningful against a matched SDK: proves the extension can report which cuvis + # functions it needs, and that every one of them is reachable through a Python + # entry point, which is what lets a missing one raise instead of aborting. + python -c " + import cuvis_il + from cuvis_il import cuvis_il as c + needed = cuvis_il._needed_symbols() + assert len(needed) > 100, needed + unguarded = [n for n in needed if cuvis_il._entry_point_for(n) is None] + assert not unguarded, unguarded + print('needs', len(needed), 'cuvis functions; missing:', c.missing_symbols) + print('built against:', c.built_against_version, '| loaded:', c.library_version) + " + SO=$(python -c "from cuvis_il import _cuvis_pyil; print(_cuvis_pyil.__file__)") + if readelf -d "$SO" | grep -q BIND_NOW; then + echo "extension is BIND_NOW: lazy binding is defeated, a missing symbol would fail the import" + exit 1 + fi + setup-windows-deps: needs: prepare runs-on: windows-latest @@ -244,3 +265,23 @@ jobs: $output = python -c "from cuvis_il import cuvis_il; print(cuvis_il.cuvis_version_swig())" Write-Output "Version output: $output" if ($output -notmatch "CUBERT SDK") { Write-Output "Version check failed!"; exit 1 } + + - name: Smoke test - symbol introspection + shell: pwsh + env: + CUVIS: C:\Program Files\Cuvis\bin + run: | + .\venv\Scripts\Activate.ps1 + # Meaningful against a matched SDK: proves the extension can report which cuvis + # functions it needs, and that every one of them is reachable through a Python + # entry point, which is what lets a missing one raise instead of crashing. + python -c @" + import cuvis_il + from cuvis_il import cuvis_il as c + needed = cuvis_il._needed_symbols() + assert len(needed) > 100, needed + unguarded = [n for n in needed if cuvis_il._entry_point_for(n) is None] + assert not unguarded, unguarded + print('needs', len(needed), 'cuvis functions; missing:', c.missing_symbols) + print('built against:', c.built_against_version, '| loaded:', c.library_version) + "@ diff --git a/CMakeLists.txt b/CMakeLists.txt index 8424d8c..f102f8c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -28,4 +28,22 @@ target_link_libraries(${target_name} PRIVATE ${Python_LIBRARIES}) set_property(TARGET ${target_name} PROPERTY SWIG_COMPILE_OPTIONS -doxygen) +# The version of the cuvis library this binding is compiled against, reported at +# import time so a mismatch with the deployed library is visible rather than silent. +target_compile_definitions(${target_name} PRIVATE CUVIS_PYIL_BUILT_VERSION="${Cuvis_VERSION}") + +if(MSVC) + # Delay-load cuvis.dll so a symbol missing from the deployed library no longer + # breaks `import cuvis_il`; the failure moves to the call, where the SEH guard in + # cuvis_il.i turns it into a Python exception. + target_link_options(${target_name} PRIVATE "/DELAYLOAD:cuvis.dll") + target_link_libraries(${target_name} PRIVATE delayimp) + +elseif(UNIX) + # Lazy binding, so an outdated libcuvis.so that is missing a symbol does not fail + # the import; cuvis_il/__init__.py replaces whatever is missing with a raising stub. + # Explicit because distro hardening defaults may otherwise inject -z now. + target_link_options(${target_name} PRIVATE "LINKER:-z,lazy") +endif() + add_custom_command(TARGET ${target_name} POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy "${SWIG_OUTPUT_DIR}/cuvis_il.py" $ ) diff --git a/cuvis.swig b/cuvis.swig index 02e1f31..e4b0a62 160000 --- a/cuvis.swig +++ b/cuvis.swig @@ -1 +1 @@ -Subproject commit 02e1f313ea23c9319ff832e75202bb80908c4f5c +Subproject commit e4b0a625003054e00fd373e8857dbfa6411f7cfb diff --git a/cuvis_il/__init__.py b/cuvis_il/__init__.py index 9da7001..76b4a10 100644 --- a/cuvis_il/__init__.py +++ b/cuvis_il/__init__.py @@ -1,13 +1,47 @@ +"""Loader for the cuvis SWIG binding. + +Beyond setting up the library search path, this module reconciles the extension with +the cuvis library actually installed on the machine. The two are built together and are +always in sync at build time, but the library deployed on a user's machine can be older +and simply not export functions the extension imports. Left alone that fails the import +with an opaque loader error and takes down every consumer, including the ones that never +wanted the missing feature. + +So: the extension is loaded in a way that survives missing symbols (delay-loaded on +Windows, lazily bound on Linux), the set of functions it needs is read back out of the +built module itself, each one is probed against the loaded library, and whatever is +missing is replaced by a stub that raises when called. `missing_symbols` names them, so +a consumer such as cuvis.python can decide which features that breaks. + +Nothing here contains a list of function names; the needed set comes from the binary. +""" +import ctypes import os import platform +import struct import sys +import warnings lib_dir = os.getenv("CUVIS") if lib_dir is None: - print('CUVIS environmental variable is not set!') - sys.exit(1) -if platform.system() == "Windows": + # Raise (do not sys.exit): this module is imported lazily by the SDK, and killing the + # host process on a missing env var would take down consumers that only wanted the + # import-safe cuvis.ipc path. Raising surfaces a clear error at first SDK use instead. + raise ImportError("CUVIS environmental variable is not set!") + +_IS_WINDOWS = platform.system() == "Windows" +if _IS_WINDOWS: os.add_dll_directory(lib_dir) + # cuvis.dll depends on the CUDA runtime (e.g. cublas64_13, npp*_13). Python 3.8+ does + # not search PATH for an extension module's dependencies, so add the CUDA toolkit bin + # dirs explicitly. CUDA_PATH is set by the toolkit installer; CUDA 13 keeps the math + # libs under bin\x64. + _cuda = os.getenv("CUDA_PATH") + if _cuda: + for _sub in ("bin", os.path.join("bin", "x64")): + _d = os.path.join(_cuda, _sub) + if os.path.isdir(_d): + os.add_dll_directory(_d) add_il = os.path.abspath(os.path.dirname(os.path.realpath(__file__))) os.environ['PATH'] += os.pathsep + add_il sys.path.append(str(add_il)) @@ -15,6 +49,252 @@ os.environ['PATH'] = lib_dir + os.pathsep + os.environ['PATH'] else: raise NotImplementedError('Invalid operating system detected!') - # sys.exit(1) -del os, platform, sys \ No newline at end of file + +def _open_cuvis_library(): + """Load the cuvis library by absolute path and return (handle, path). + + Loading it here, before the extension binds anything, pins the library `CUVIS` points + at. That matters on Windows: the delay-load helper calls LoadLibraryExA("cuvis.dll", + NULL, 0), which ignores os.add_dll_directory() and would otherwise pick up any + cuvis.dll reachable through the ordinary search order, the current directory included. + Once ours is in the process under that base name, the helper binds to it. + """ + names = ("cuvis.dll",) if _IS_WINDOWS else ("libcuvis.so",) + loader = ctypes.WinDLL if _IS_WINDOWS else ctypes.CDLL + errors = [] + for name in names: + path = os.path.join(lib_dir, name) + try: + return loader(path), path + except OSError as exc: + errors.append("{}: {}".format(path, exc)) + try: + return loader(name), name # fall back to the platform search order + except OSError as exc: + errors.append("{}: {}".format(name, exc)) + raise ImportError("cuvis library could not be loaded from {}: {}" + .format(lib_dir, "; ".join(errors))) + + +_cuvis_library, _cuvis_library_path = _open_cuvis_library() + + +def _import_extension(): + """Import the extension so that missing symbols do not fail the load. + + Windows needs nothing: cuvis.dll is delay-loaded. Linux does, because CPython dlopens + extensions with RTLD_NOW, which resolves every undefined symbol up front. RTLD_LAZY + defers function symbols, and cuvis.h exports no data symbols, so the whole surface is + covered. numpy is imported first to keep its own extensions off the lazy path. + """ + if _IS_WINDOWS: + from . import cuvis_il + return cuvis_il + try: + import numpy # noqa: F401 + except ImportError: + pass + _previous = sys.getdlopenflags() + # Process-global, not thread-local: a concurrent import in this window also gets lazy + # binding, which is harmless. + sys.setdlopenflags(os.RTLD_LAZY | os.RTLD_LOCAL) + try: + from . import cuvis_il + finally: + sys.setdlopenflags(_previous) + return cuvis_il + + +cuvis_il = _import_extension() + + +def _needed_from_pe(path, dll_prefix="cuvis"): + """Names the module imports from cuvis*.dll, read from its own PE headers. + + Covers both the ordinary import descriptor (data directory 1) and the delay-load one + (directory 13), since delay-loading moves every entry out of the former into the latter. + """ + with open(path, "rb") as handle: + data = handle.read() + coff = struct.unpack_from(" 1 and dirs[1][0]: + pos = offset(dirs[1][0]) + while True: + lookup, _, _, name_rva, address = struct.unpack_from(" 13 and dirs[13][0]: + pos = offset(dirs[13][0]) + while True: + attrs, name_rva, _, _, table = struct.unpack_from("" + if is64: + sh_off, = struct.unpack_from(endian + "Q", data, 0x28) + sh_entsize, sh_num = struct.unpack_from(endian + "HH", data, 0x3A) + sh_fmt, sym_fmt = endian + "IIQQQQIIQQ", endian + "IBBHQQ" + else: + sh_off, = struct.unpack_from(endian + "I", data, 0x20) + sh_entsize, sh_num = struct.unpack_from(endian + "HH", data, 0x2E) + sh_fmt, sym_fmt = endian + "IIIIIIIIII", endian + "IIIBBH" + headers = [struct.unpack_from(sh_fmt, data, sh_off + i * sh_entsize) for i in range(sh_num)] + names = set() + for header in headers: + if header[1] != 11: # SHT_DYNSYM + continue + strtab = headers[header[6]][4] # sh_link -> .dynstr + start, size, entsize = header[4], header[5], header[9] + for pos in range(start, start + size, entsize): + fields = struct.unpack_from(sym_fmt, data, pos) + st_name = fields[0] + st_shndx = fields[3] if is64 else fields[5] + if st_shndx != 0 or not st_name: # keep named SHN_UNDEF entries only + continue + end = data.index(b"\0", strtab + st_name) + names.add(data[strtab + st_name:end].decode("ascii", "replace")) + return names + + +def _entry_point_for(symbol): + """The module attribute through which `symbol` can be reached, or None. + + A wrapped function keeps its own name; the handful that are %ignore'd in the SWIG + interface are reached through a hand-written helper that appends _swig. That is a + naming convention the interface already enforces, not a list of names. + """ + for candidate in (symbol, symbol + "_swig"): + if hasattr(cuvis_il, candidate): + return candidate + return None + + +def _needed_symbols(): + """Function names the built extension expects the cuvis library to export.""" + module_path = cuvis_il._cuvis_pyil.__file__ + if _IS_WINDOWS: + return _needed_from_pe(module_path) + # Undefined ELF symbols do not record which library provides them, so keep the ones + # that correspond to a wrapped entry point. Derived from SWIG's output, not written down. + return {name for name in _undefined_from_elf(module_path) if _entry_point_for(name)} + + +def _stub(symbol, attribute): + def raise_unavailable(*_args, **_kwargs): + raise RuntimeError( + "cuvis: '{}' is not exported by the cuvis library loaded from {}. " + "The installed CUVIS SDK is older than the one this binding was built " + "against ({}).".format(symbol, _cuvis_library_path, + getattr(cuvis_il, "built_against_version", "unknown"))) + raise_unavailable.__name__ = attribute + raise_unavailable.__qualname__ = attribute + return raise_unavailable + + +def _reconcile_with_library(): + """Find what the loaded library does not export, and make calling it raise.""" + try: + needed = _needed_symbols() + except Exception as exc: + # Windows still fails safely: the delay-load guard in cuvis_il.i turns the call + # into a RuntimeError. Linux does not, so say so rather than proceed quietly. + warnings.warn( + "cuvis_il: could not determine which cuvis functions this build needs ({}). " + "A cuvis library missing one of them will {}." + .format(exc, "raise on call" if _IS_WINDOWS else "abort the process"), + RuntimeWarning, stacklevel=2) + cuvis_il.missing_symbols = () + return + + missing = sorted(name for name in needed if not hasattr(_cuvis_library, name)) + unshadowable = [name for name in needed if _entry_point_for(name) is None] + + low_level = cuvis_il._cuvis_pyil + for symbol in missing: + attribute = _entry_point_for(symbol) + if attribute is None: + continue + setattr(cuvis_il, attribute, _stub(symbol, attribute)) + if hasattr(low_level, attribute): + setattr(low_level, attribute, _stub(symbol, attribute)) + + cuvis_il.missing_symbols = tuple(missing) + if missing: + warnings.warn( + "cuvis_il: the cuvis library at {} does not export {}. Calling these raises " + "RuntimeError; everything else works normally." + .format(_cuvis_library_path, ", ".join(missing)), + UserWarning, stacklevel=2) + if unshadowable: + warnings.warn( + "cuvis_il: no Python entry point covers {}, so code reaching them cannot be " + "guarded".format(", ".join(sorted(unshadowable))), + RuntimeWarning, stacklevel=2) + + +def _record_library_version(): + try: + cuvis_il.library_version = cuvis_il.cuvis_version_swig() + except Exception: + cuvis_il.library_version = "" + + +_reconcile_with_library() +_record_library_version() diff --git a/pyproject.toml b/pyproject.toml index c77b7b5..042878a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "cuvis_il" -version = "3.5.3.2" +version = "3.5.3.3" description = "Compiled Python Bindings for the CUVIS SDK." readme = "README.md" requires-python = ">=3.9" From 165a0c3e73fe203d3eee0bc638956e7e8de8abea Mon Sep 17 00:00:00 2001 From: Simon Birkholz Date: Tue, 18 Aug 2026 14:51:45 +0200 Subject: [PATCH 2/5] Make the build-version info language neutral and report the library path The version the binding was compiled against was published through the Python C API, so it existed only for the Python target and needed a not-CSharp guard. It is now an ordinary wrapped function, cuvis_built_against_version(), which every SWIG target language gets for free; verified by building the C# binding from the same interface. The macro is named CUVIS_BINDING_BUILT_VERSION accordingly. The delay-load guard now distinguishes a library that cannot be loaded at all (missing, or its own dependencies such as the CUDA runtime cannot be found) from one that loads but lacks the function, which previously shared one message. cuvis_il.library_path names the library the probe actually opened, so a consumer can report which file it is talking about. --- CMakeLists.txt | 2 +- cuvis.swig | 2 +- cuvis_il/__init__.py | 16 +++++++++++++++- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f102f8c..12e991e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -30,7 +30,7 @@ set_property(TARGET ${target_name} PROPERTY SWIG_COMPILE_OPTIONS -doxygen) # The version of the cuvis library this binding is compiled against, reported at # import time so a mismatch with the deployed library is visible rather than silent. -target_compile_definitions(${target_name} PRIVATE CUVIS_PYIL_BUILT_VERSION="${Cuvis_VERSION}") +target_compile_definitions(${target_name} PRIVATE CUVIS_BINDING_BUILT_VERSION="${Cuvis_VERSION}") if(MSVC) # Delay-load cuvis.dll so a symbol missing from the deployed library no longer diff --git a/cuvis.swig b/cuvis.swig index e4b0a62..1a917e2 160000 --- a/cuvis.swig +++ b/cuvis.swig @@ -1 +1 @@ -Subproject commit e4b0a625003054e00fd373e8857dbfa6411f7cfb +Subproject commit 1a917e25f1e22e0644c1f43210df1c7411205f44 diff --git a/cuvis_il/__init__.py b/cuvis_il/__init__.py index 76b4a10..1fe88ee 100644 --- a/cuvis_il/__init__.py +++ b/cuvis_il/__init__.py @@ -242,7 +242,7 @@ def raise_unavailable(*_args, **_kwargs): "cuvis: '{}' is not exported by the cuvis library loaded from {}. " "The installed CUVIS SDK is older than the one this binding was built " "against ({}).".format(symbol, _cuvis_library_path, - getattr(cuvis_il, "built_against_version", "unknown"))) + _built_against_version())) raise_unavailable.__name__ = attribute raise_unavailable.__qualname__ = attribute return raise_unavailable @@ -289,7 +289,21 @@ def _reconcile_with_library(): RuntimeWarning, stacklevel=2) +def _built_against_version(): + """The cuvis version this binding was compiled against, or "" if unavailable. + + The extension exposes it as an ordinary wrapped function so every target language + can reach it, not just Python. + """ + try: + return cuvis_il.cuvis_built_against_version() + except Exception: + return "" + + def _record_library_version(): + cuvis_il.built_against_version = _built_against_version() + cuvis_il.library_path = _cuvis_library_path try: cuvis_il.library_version = cuvis_il.cuvis_version_swig() except Exception: From db0214a6d73361fd6b085b5f95de1b20c8587b87 Mon Sep 17 00:00:00 2001 From: Simon Birkholz Date: Tue, 18 Aug 2026 15:39:57 +0200 Subject: [PATCH 3/5] updating swig submodule --- cuvis.swig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cuvis.swig b/cuvis.swig index 1a917e2..9f987c7 160000 --- a/cuvis.swig +++ b/cuvis.swig @@ -1 +1 @@ -Subproject commit 1a917e25f1e22e0644c1f43210df1c7411205f44 +Subproject commit 9f987c74d09e4444af3fb803834aebbf4e5b6565 From 52a10ce5e47e0602671165fc49dd096f8ddeeb36 Mon Sep 17 00:00:00 2001 From: Simon Birkholz Date: Wed, 19 Aug 2026 14:12:43 +0200 Subject: [PATCH 4/5] updating swig submodule --- .github/workflows/build_and_test.yml | 8 ++++++ CMakeLists.txt | 4 --- cuvis.swig | 2 +- cuvis_il/__init__.py | 40 ++++++++++++++++++++-------- 4 files changed, 38 insertions(+), 16 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 9ab73e5..15736bc 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -103,6 +103,10 @@ jobs: assert not unguarded, unguarded print('needs', len(needed), 'cuvis functions; missing:', c.missing_symbols) print('built against:', c.built_against_version, '| loaded:', c.library_version) + # Both sides come from the same banner, so a build that lost the define shows up + # here rather than as a silent 'unknown' in someone's bug report. + assert c.built_against_hash and c.library_hash, (c.built_against_version, c.library_version) + assert c.built_against_hash == c.library_hash, 'CI builds and runs the same SDK' " SO=$(python -c "from cuvis_il import _cuvis_pyil; print(_cuvis_pyil.__file__)") if readelf -d "$SO" | grep -q BIND_NOW; then @@ -284,4 +288,8 @@ jobs: assert not unguarded, unguarded print('needs', len(needed), 'cuvis functions; missing:', c.missing_symbols) print('built against:', c.built_against_version, '| loaded:', c.library_version) + # Both sides come from the same banner, so a build that lost the define shows up + # here rather than as a silent 'unknown' in someone's bug report. + assert c.built_against_hash and c.library_hash, (c.built_against_version, c.library_version) + assert c.built_against_hash == c.library_hash, 'CI builds and runs the same SDK' "@ diff --git a/CMakeLists.txt b/CMakeLists.txt index 12e991e..3dfd1e0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -28,10 +28,6 @@ target_link_libraries(${target_name} PRIVATE ${Python_LIBRARIES}) set_property(TARGET ${target_name} PROPERTY SWIG_COMPILE_OPTIONS -doxygen) -# The version of the cuvis library this binding is compiled against, reported at -# import time so a mismatch with the deployed library is visible rather than silent. -target_compile_definitions(${target_name} PRIVATE CUVIS_BINDING_BUILT_VERSION="${Cuvis_VERSION}") - if(MSVC) # Delay-load cuvis.dll so a symbol missing from the deployed library no longer # breaks `import cuvis_il`; the failure moves to the call, where the SEH guard in diff --git a/cuvis.swig b/cuvis.swig index 9f987c7..7afd952 160000 --- a/cuvis.swig +++ b/cuvis.swig @@ -1 +1 @@ -Subproject commit 9f987c74d09e4444af3fb803834aebbf4e5b6565 +Subproject commit 7afd9521ce7733d601cbec10bb2c47c88489cdd2 diff --git a/cuvis_il/__init__.py b/cuvis_il/__init__.py index 1fe88ee..7d7b918 100644 --- a/cuvis_il/__init__.py +++ b/cuvis_il/__init__.py @@ -18,6 +18,7 @@ import ctypes import os import platform +import re import struct import sys import warnings @@ -239,10 +240,11 @@ def _needed_symbols(): def _stub(symbol, attribute): def raise_unavailable(*_args, **_kwargs): raise RuntimeError( - "cuvis: '{}' is not exported by the cuvis library loaded from {}. " - "The installed CUVIS SDK is older than the one this binding was built " - "against ({}).".format(symbol, _cuvis_library_path, - _built_against_version())) + "cuvis: '{}' is not exported by the cuvis library loaded from {}. That " + "library is not the one this binding was built against (built against: {}; " + "loaded: {}).".format(symbol, _cuvis_library_path, + _built_against_version() or "unknown", + _library_version() or "unknown")) raise_unavailable.__name__ = attribute raise_unavailable.__qualname__ = attribute return raise_unavailable @@ -290,10 +292,11 @@ def _reconcile_with_library(): def _built_against_version(): - """The cuvis version this binding was compiled against, or "" if unavailable. + """The cuvis library this binding was compiled against, or "" if unavailable. - The extension exposes it as an ordinary wrapped function so every target language - can reach it, not just Python. + Reported in the same form as the loaded library reports itself, so the two can be + compared directly. The extension exposes it as an ordinary wrapped function so every + target language can reach it, not just Python. """ try: return cuvis_il.cuvis_built_against_version() @@ -301,13 +304,28 @@ def _built_against_version(): return "" +def _library_version(): + try: + return cuvis_il.cuvis_version_swig() + except Exception: + return "" + + +def _hash_from_version(version): + """The build hash out of a 'CUBERT SDK v. X.Y.Z build: ' string, or "".""" + found = re.search(r"build:\s*([0-9a-fA-F]+)", version or "") + return found.group(1) if found else "" + + def _record_library_version(): cuvis_il.built_against_version = _built_against_version() cuvis_il.library_path = _cuvis_library_path - try: - cuvis_il.library_version = cuvis_il.cuvis_version_swig() - except Exception: - cuvis_il.library_version = "" + cuvis_il.library_version = _library_version() + # Both sides are reported in the same form, so the hash that tells two builds of one + # version apart is parsed the same way out of each. A difference is not a failure, + # only a fact worth having when something else is wrong, hence no warning. + cuvis_il.built_against_hash = _hash_from_version(cuvis_il.built_against_version) + cuvis_il.library_hash = _hash_from_version(cuvis_il.library_version) _reconcile_with_library() From 859280ace2ba97ff694e91cb7cc3ce1be3e2a2bb Mon Sep 17 00:00:00 2001 From: Simon Birkholz Date: Wed, 19 Aug 2026 14:31:28 +0200 Subject: [PATCH 5/5] moving dll analysis logic into its own file --- cuvis_il/__init__.py | 260 +++++++++++++------------------------------ cuvis_il/_imports.py | 210 ++++++++++++++++++++++++++++++++++ 2 files changed, 290 insertions(+), 180 deletions(-) create mode 100644 cuvis_il/_imports.py diff --git a/cuvis_il/__init__.py b/cuvis_il/__init__.py index 7d7b918..f0c963d 100644 --- a/cuvis_il/__init__.py +++ b/cuvis_il/__init__.py @@ -1,28 +1,36 @@ """Loader for the cuvis SWIG binding. -Beyond setting up the library search path, this module reconciles the extension with -the cuvis library actually installed on the machine. The two are built together and are +Beyond setting up the library search path, this module reconciles the extension with the +cuvis library actually installed on the machine. The two are built together and are always in sync at build time, but the library deployed on a user's machine can be older and simply not export functions the extension imports. Left alone that fails the import with an opaque loader error and takes down every consumer, including the ones that never wanted the missing feature. -So: the extension is loaded in a way that survives missing symbols (delay-loaded on -Windows, lazily bound on Linux), the set of functions it needs is read back out of the -built module itself, each one is probed against the loaded library, and whatever is -missing is replaced by a stub that raises when called. `missing_symbols` names them, so -a consumer such as cuvis.python can decide which features that breaks. +So, in the order it happens below: the cuvis library is opened by absolute path, the +extension is loaded in a way that survives missing symbols (delay loaded on Windows, +lazily bound on Linux), the functions it needs are read back out of the built module +itself, each is probed against the library, and whatever is missing is replaced by a stub +that raises when called. -Nothing here contains a list of function names; the needed set comes from the binary. +What this publishes on the `cuvis_il` module, for a consumer such as cuvis.python: + + missing_symbols functions the loaded library does not export + library_path the file that was loaded + library_version how that library reports itself + built_against_version how the library this was compiled against reported itself + library_hash the build hash out of each of those two banners, which is + built_against_hash what tells two builds of one version apart """ import ctypes import os import platform import re -import struct import sys import warnings +from ._imports import required_cuvis_symbols + lib_dir = os.getenv("CUVIS") if lib_dir is None: # Raise (do not sys.exit): this module is imported lazily by the SDK, and killing the @@ -53,38 +61,34 @@ def _open_cuvis_library(): - """Load the cuvis library by absolute path and return (handle, path). - - Loading it here, before the extension binds anything, pins the library `CUVIS` points - at. That matters on Windows: the delay-load helper calls LoadLibraryExA("cuvis.dll", - NULL, 0), which ignores os.add_dll_directory() and would otherwise pick up any - cuvis.dll reachable through the ordinary search order, the current directory included. - Once ours is in the process under that base name, the helper binds to it. + """Load the cuvis library and return it together with the path that worked. + + Loading it here, before the extension binds anything, pins the library CUVIS points + at. That matters on Windows: the delay load helper calls + LoadLibraryExA("cuvis.dll", NULL, 0), which ignores os.add_dll_directory and would + otherwise take any cuvis.dll reachable through the ordinary search order, the current + directory included. Once ours is in the process under that base name, the helper + binds to it. """ - names = ("cuvis.dll",) if _IS_WINDOWS else ("libcuvis.so",) - loader = ctypes.WinDLL if _IS_WINDOWS else ctypes.CDLL - errors = [] - for name in names: - path = os.path.join(lib_dir, name) - try: - return loader(path), path - except OSError as exc: - errors.append("{}: {}".format(path, exc)) + name = "cuvis.dll" if _IS_WINDOWS else "libcuvis.so" + load = ctypes.WinDLL if _IS_WINDOWS else ctypes.CDLL + failures = [] + for candidate in (os.path.join(lib_dir, name), name): # ours, then the search order try: - return loader(name), name # fall back to the platform search order + return load(candidate), candidate except OSError as exc: - errors.append("{}: {}".format(name, exc)) + failures.append("{}: {}".format(candidate, exc)) raise ImportError("cuvis library could not be loaded from {}: {}" - .format(lib_dir, "; ".join(errors))) + .format(lib_dir, "; ".join(failures))) _cuvis_library, _cuvis_library_path = _open_cuvis_library() def _import_extension(): - """Import the extension so that missing symbols do not fail the load. + """Import the extension so that a missing symbol does not fail the load. - Windows needs nothing: cuvis.dll is delay-loaded. Linux does, because CPython dlopens + Windows needs nothing: cuvis.dll is delay loaded. Linux does, because CPython dlopens extensions with RTLD_NOW, which resolves every undefined symbol up front. RTLD_LAZY defers function symbols, and cuvis.h exports no data symbols, so the whole surface is covered. numpy is imported first to keep its own extensions off the lazy path. @@ -96,124 +100,20 @@ def _import_extension(): import numpy # noqa: F401 except ImportError: pass - _previous = sys.getdlopenflags() - # Process-global, not thread-local: a concurrent import in this window also gets lazy + previous = sys.getdlopenflags() + # Process global, not thread local: a concurrent import in this window also gets lazy # binding, which is harmless. sys.setdlopenflags(os.RTLD_LAZY | os.RTLD_LOCAL) try: from . import cuvis_il finally: - sys.setdlopenflags(_previous) + sys.setdlopenflags(previous) return cuvis_il cuvis_il = _import_extension() -def _needed_from_pe(path, dll_prefix="cuvis"): - """Names the module imports from cuvis*.dll, read from its own PE headers. - - Covers both the ordinary import descriptor (data directory 1) and the delay-load one - (directory 13), since delay-loading moves every entry out of the former into the latter. - """ - with open(path, "rb") as handle: - data = handle.read() - coff = struct.unpack_from(" 1 and dirs[1][0]: - pos = offset(dirs[1][0]) - while True: - lookup, _, _, name_rva, address = struct.unpack_from(" 13 and dirs[13][0]: - pos = offset(dirs[13][0]) - while True: - attrs, name_rva, _, _, table = struct.unpack_from("" - if is64: - sh_off, = struct.unpack_from(endian + "Q", data, 0x28) - sh_entsize, sh_num = struct.unpack_from(endian + "HH", data, 0x3A) - sh_fmt, sym_fmt = endian + "IIQQQQIIQQ", endian + "IBBHQQ" - else: - sh_off, = struct.unpack_from(endian + "I", data, 0x20) - sh_entsize, sh_num = struct.unpack_from(endian + "HH", data, 0x2E) - sh_fmt, sym_fmt = endian + "IIIIIIIIII", endian + "IIIBBH" - headers = [struct.unpack_from(sh_fmt, data, sh_off + i * sh_entsize) for i in range(sh_num)] - names = set() - for header in headers: - if header[1] != 11: # SHT_DYNSYM - continue - strtab = headers[header[6]][4] # sh_link -> .dynstr - start, size, entsize = header[4], header[5], header[9] - for pos in range(start, start + size, entsize): - fields = struct.unpack_from(sym_fmt, data, pos) - st_name = fields[0] - st_shndx = fields[3] if is64 else fields[5] - if st_shndx != 0 or not st_name: # keep named SHN_UNDEF entries only - continue - end = data.index(b"\0", strtab + st_name) - names.add(data[strtab + st_name:end].decode("ascii", "replace")) - return names - - def _entry_point_for(symbol): """The module attribute through which `symbol` can be reached, or None. @@ -221,23 +121,18 @@ def _entry_point_for(symbol): interface are reached through a hand-written helper that appends _swig. That is a naming convention the interface already enforces, not a list of names. """ - for candidate in (symbol, symbol + "_swig"): - if hasattr(cuvis_il, candidate): - return candidate - return None + return next((name for name in (symbol, symbol + "_swig") + if hasattr(cuvis_il, name)), None) def _needed_symbols(): - """Function names the built extension expects the cuvis library to export.""" - module_path = cuvis_il._cuvis_pyil.__file__ - if _IS_WINDOWS: - return _needed_from_pe(module_path) - # Undefined ELF symbols do not record which library provides them, so keep the ones - # that correspond to a wrapped entry point. Derived from SWIG's output, not written down. - return {name for name in _undefined_from_elf(module_path) if _entry_point_for(name)} + """The cuvis functions the built extension expects the library to export.""" + return required_cuvis_symbols(cuvis_il._cuvis_pyil.__file__, + lambda name: _entry_point_for(name) is not None) def _stub(symbol, attribute): + """A stand-in for an absent function, explaining the absence when it is called.""" def raise_unavailable(*_args, **_kwargs): raise RuntimeError( "cuvis: '{}' is not exported by the cuvis library loaded from {}. That " @@ -250,34 +145,42 @@ def raise_unavailable(*_args, **_kwargs): return raise_unavailable +def _shadow(symbol): + """Replace a symbol's entry points, so reaching it raises instead of crashing.""" + attribute = _entry_point_for(symbol) + if attribute is None: # warned about separately + return + for module in (cuvis_il, cuvis_il._cuvis_pyil): + if hasattr(module, attribute): + setattr(module, attribute, _stub(symbol, attribute)) + + def _reconcile_with_library(): - """Find what the loaded library does not export, and make calling it raise.""" + """Make every function the loaded library does not export raise when it is called. + + :return: the names it does not export, sorted. + """ try: needed = _needed_symbols() except Exception as exc: - # Windows still fails safely: the delay-load guard in cuvis_il.i turns the call + # Windows still fails safely: the delay load guard in cuvis_il.i turns the call # into a RuntimeError. Linux does not, so say so rather than proceed quietly. warnings.warn( "cuvis_il: could not determine which cuvis functions this build needs ({}). " "A cuvis library missing one of them will {}." .format(exc, "raise on call" if _IS_WINDOWS else "abort the process"), RuntimeWarning, stacklevel=2) - cuvis_il.missing_symbols = () - return + return () - missing = sorted(name for name in needed if not hasattr(_cuvis_library, name)) - unshadowable = [name for name in needed if _entry_point_for(name) is None] + missing = tuple(sorted(name for name in needed + if not hasattr(_cuvis_library, name))) + # Always empty on Linux, where the needed set is already narrowed to what is + # reachable through an entry point. It is the Windows case this guards. + unshadowable = sorted(name for name in needed if _entry_point_for(name) is None) - low_level = cuvis_il._cuvis_pyil for symbol in missing: - attribute = _entry_point_for(symbol) - if attribute is None: - continue - setattr(cuvis_il, attribute, _stub(symbol, attribute)) - if hasattr(low_level, attribute): - setattr(low_level, attribute, _stub(symbol, attribute)) - - cuvis_il.missing_symbols = tuple(missing) + _shadow(symbol) + if missing: warnings.warn( "cuvis_il: the cuvis library at {} does not export {}. Calling these raises " @@ -287,8 +190,9 @@ def _reconcile_with_library(): if unshadowable: warnings.warn( "cuvis_il: no Python entry point covers {}, so code reaching them cannot be " - "guarded".format(", ".join(sorted(unshadowable))), + "guarded".format(", ".join(unshadowable)), RuntimeWarning, stacklevel=2) + return missing def _built_against_version(): @@ -311,22 +215,18 @@ def _library_version(): return "" -def _hash_from_version(version): - """The build hash out of a 'CUBERT SDK v. X.Y.Z build: ' string, or "".""" - found = re.search(r"build:\s*([0-9a-fA-F]+)", version or "") +def _build_hash(banner): + """The build hash out of a 'CUBERT SDK v. X.Y.Z build: ' banner, or "".""" + found = re.search(r"build:\s*([0-9a-fA-F]+)", banner or "") return found.group(1) if found else "" -def _record_library_version(): - cuvis_il.built_against_version = _built_against_version() - cuvis_il.library_path = _cuvis_library_path - cuvis_il.library_version = _library_version() - # Both sides are reported in the same form, so the hash that tells two builds of one - # version apart is parsed the same way out of each. A difference is not a failure, - # only a fact worth having when something else is wrong, hence no warning. - cuvis_il.built_against_hash = _hash_from_version(cuvis_il.built_against_version) - cuvis_il.library_hash = _hash_from_version(cuvis_il.library_version) - - -_reconcile_with_library() -_record_library_version() +cuvis_il.missing_symbols = _reconcile_with_library() +cuvis_il.library_path = _cuvis_library_path +cuvis_il.built_against_version = _built_against_version() +cuvis_il.library_version = _library_version() +# Both sides report themselves in the same form, so the hash that tells two builds of one +# version apart is parsed the same way out of each. A difference is not a failure, only a +# fact worth having when something else is wrong, hence no warning. +cuvis_il.built_against_hash = _build_hash(cuvis_il.built_against_version) +cuvis_il.library_hash = _build_hash(cuvis_il.library_version) diff --git a/cuvis_il/_imports.py b/cuvis_il/_imports.py new file mode 100644 index 0000000..ee88549 --- /dev/null +++ b/cuvis_il/_imports.py @@ -0,0 +1,210 @@ +"""Read back what a built extension module expects some other library to provide. + +Nothing here lists cuvis function names. They come out of the compiled module itself, so +they cannot fall out of step with what was actually built, which is the whole point: the +extension and the cuvis library are compiled together, but the library deployed on a +user's machine is a separate thing and may not export everything the extension imports. + +There is no standard library reader for either format, and the shortcuts do not work. +Deriving the set from the SWIG wrapper's own names instead adds every helper compiled +into the extension, none of which the library exports, so they would all be mistaken for +missing. Only the binary says which name comes from where. +""" +import itertools +import struct +from typing import NamedTuple + +_PE_IMPORTS = 1 +_PE_DELAY_IMPORTS = 13 +_PE32_PLUS = 0x20B + +_SHT_DYNSYM = 11 +_SHN_UNDEF = 0 + + +def required_cuvis_symbols(module_path, is_wrapped): + """The cuvis functions the built extension expects the cuvis library to export. + + :param module_path: the compiled extension, ``_cuvis_pyil.pyd`` or ``.so``. + :param is_wrapped: predicate telling whether a name is one this binding wraps. Only + consulted for ELF, see below. + :return: the set of C function names. + + A PE import records which DLL each name comes from, so on Windows the answer is + exact. ELF undefined symbols do not record a provider, so there the wider set of + everything the object expects from outside is narrowed by ``is_wrapped``. + """ + with open(module_path, "rb") as handle: + data = handle.read() + if data[:2] == b"MZ": + return _PortableExecutable(data).imports_from("cuvis") + return {name for name in _Elf(data).undefined_symbols() if is_wrapped(name)} + + +def _identity(rva): + return rva + + +class _Section(NamedTuple): + """A PE section header, of which only the address mapping is of interest.""" + + virtual_size: int + rva: int + raw_size: int + file_offset: int + + @property + def size(self): + # A section holding uninitialised data is larger in memory than on disk, and one + # padded to the file alignment is larger on disk than in memory. + return max(self.virtual_size, self.raw_size) + + def contains(self, rva): + return self.rva <= rva < self.rva + self.size + + +class _PortableExecutable: + """Just enough of a PE image to walk its two import tables.""" + + def __init__(self, data): + self._data = data + coff = struct.unpack_from("" + + if self._wide: + offset, = struct.unpack_from(endian + "Q", data, 0x28) + entry_size, count = struct.unpack_from(endian + "HH", data, 0x3A) + header_layout, self._symbol_layout = endian + "IIQQQQIIQQ", endian + "IBBHQQ" + else: + offset, = struct.unpack_from(endian + "I", data, 0x20) + entry_size, count = struct.unpack_from(endian + "HH", data, 0x2E) + header_layout, self._symbol_layout = endian + "IIIIIIIIII", endian + "IIIBBH" + + self._headers = [ + _SectionHeader(*struct.unpack_from(header_layout, data, offset + i * entry_size)) + for i in range(count)] + + def undefined_symbols(self): + """Named symbols the image expects some other object to provide.""" + return {self._string(header.link, symbol.name) + for header in self._headers if header.type == _SHT_DYNSYM + for symbol in self._symbols(header) + if symbol.section == _SHN_UNDEF and symbol.name} + + def _symbols(self, header): + for at in range(header.offset, header.offset + header.size, header.entry_size): + fields = struct.unpack_from(self._symbol_layout, self._data, at) + # Unlike the section header, the symbol field order differs by width: the + # section index is the fourth field at 64 bit and the sixth at 32. + yield _Symbol(fields[0], fields[3] if self._wide else fields[5]) + + def _string(self, table, offset): + start = self._headers[table].offset + offset + end = self._data.index(b"\0", start) + return self._data[start:end].decode("ascii", "replace")