diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 9f6dec7d1c5802c..46aaa4d1a0b8a19 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -1262,6 +1262,37 @@ class Spec2: origin = "a\x00b" _imp.create_dynamic(Spec2()) + @unittest.skipUnless(_testsinglephase is not None, + 'requires _testsinglephase') + @unittest.skipUnless(os_helper.TESTFN_UNDECODABLE, + 'requires undecodable file names') + def test_import_from_undecodable_path(self): + # gh-155247: the path of the extension module is not encodable + # in UTF-8. + origin = _testsinglephase.__file__ + with os_helper.temp_dir() as tempdir: + subdir = os.path.join(os.fsencode(tempdir), + os_helper.TESTFN_UNDECODABLE) + try: + os.mkdir(subdir) + except OSError: + self.skipTest('undecodable paths are not supported') + path = os.path.join(subdir, os.fsencode(os.path.basename(origin))) + shutil.copyfile(origin, path) + # Import it in a subprocess: the extension module stays loaded, + # and on Windows its file cannot be removed. + script = textwrap.dedent(f""" + import importlib.util + path = {os.fsdecode(path)!a} + spec = importlib.util.spec_from_file_location( + '_testsinglephase', path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + assert module.__name__ == '_testsinglephase', module.__name__ + assert module.__file__ == path, module.__file__ + """) + script_helper.assert_python_ok('-c', script) + def test_create_builtin(self): class Spec: pass diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-08-05-19-10-00.gh-issue-155247.extKey.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-08-05-19-10-00.gh-issue-155247.extKey.rst new file mode 100644 index 000000000000000..1e08323008d5b7a --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-08-05-19-10-00.gh-issue-155247.extKey.rst @@ -0,0 +1,4 @@ +Fix importing an extension module whose path contains characters unencodable +in UTF-8, e.g. undecodable bytes of a file name. Previously it failed with +:exc:`UnicodeEncodeError`, which made it impossible to build or run Python in +a directory with such name. diff --git a/Python/import.c b/Python/import.c index 5ca78a971fa54c6..87476917fa11252 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1276,50 +1276,68 @@ del_extensions_cache_value(void *raw) } } +/* The key of the extensions cache: the raw content of two strings. + + The UTF-8 encoding is not used, because the strings can contain lone + surrogates, e.g. a file name undecodable in the filesystem encoding. */ +struct hashtable_key { + size_t size; /* the total size of the key */ + unsigned char kind1; + unsigned char kind2; + Py_ssize_t len1; + /* followed by the raw content of both strings */ +}; + static void * -hashtable_key_from_2_strings(PyObject *str1, PyObject *str2, const char sep) +hashtable_key_from_2_strings(PyObject *str1, PyObject *str2) { - const char *str1_data = _PyUnicode_AsUTF8NoNUL(str1); - const char *str2_data = _PyUnicode_AsUTF8NoNUL(str2); - if (str1_data == NULL || str2_data == NULL) { - return NULL; - } - Py_ssize_t str1_len = strlen(str1_data); - Py_ssize_t str2_len = strlen(str2_data); + Py_ssize_t len1 = PyUnicode_GET_LENGTH(str1); + Py_ssize_t len2 = PyUnicode_GET_LENGTH(str2); + int kind1 = PyUnicode_KIND(str1); + int kind2 = PyUnicode_KIND(str2); + size_t size1 = (size_t)len1 * kind1; + size_t size2 = (size_t)len2 * kind2; - /* Make sure sep and the NULL byte won't cause an overflow. */ - assert(SIZE_MAX - str1_len - str2_len > 2); - size_t size = str1_len + 1 + str2_len + 1; + assert(SIZE_MAX - sizeof(struct hashtable_key) - size1 > size2); + size_t size = sizeof(struct hashtable_key) + size1 + size2; // XXX Use a buffer if it's a temp value (every case but "set"). - char *key = PyMem_RawMalloc(size); + struct hashtable_key *key = PyMem_RawMalloc(size); if (key == NULL) { PyErr_NoMemory(); return NULL; } - memcpy(key, str1_data, str1_len); - key[str1_len] = sep; - memcpy(key + str1_len + 1, str2_data, str2_len); - key[size - 1] = '\0'; - assert(strlen(key) == size - 1); + /* Clear the padding: the key is hashed and compared as raw bytes. */ + memset(key, 0, sizeof(struct hashtable_key)); + key->size = size; + key->kind1 = (unsigned char)kind1; + key->kind2 = (unsigned char)kind2; + key->len1 = len1; + char *data = (char *)(key + 1); + memcpy(data, PyUnicode_DATA(str1), size1); + memcpy(data + size1, PyUnicode_DATA(str2), size2); return key; } static Py_uhash_t -hashtable_hash_str(const void *key) +hashtable_hash_key(const void *key) { - return Py_HashBuffer(key, strlen((const char *)key)); + return Py_HashBuffer(key, ((const struct hashtable_key *)key)->size); } static int -hashtable_compare_str(const void *key1, const void *key2) +hashtable_compare_key(const void *key1, const void *key2) { - return strcmp((const char *)key1, (const char *)key2) == 0; + size_t size = ((const struct hashtable_key *)key1)->size; + if (size != ((const struct hashtable_key *)key2)->size) { + return 0; + } + return memcmp(key1, key2, size) == 0; } static void -hashtable_destroy_str(void *ptr) +hashtable_destroy_key(void *ptr) { PyMem_RawFree(ptr); } @@ -1359,16 +1377,15 @@ _find_cached_def(PyModuleDef *def) } #endif -#define HTSEP ':' static int _extensions_cache_init(void) { _Py_hashtable_allocator_t alloc = {PyMem_RawMalloc, PyMem_RawFree}; EXTENSIONS.hashtable = _Py_hashtable_new_full( - hashtable_hash_str, - hashtable_compare_str, - hashtable_destroy_str, // key + hashtable_hash_key, + hashtable_compare_key, + hashtable_destroy_key, // key del_extensions_cache_value, // value &alloc ); @@ -1386,7 +1403,7 @@ _extensions_cache_find_unlocked(PyObject *path, PyObject *name, if (EXTENSIONS.hashtable == NULL) { return NULL; } - void *key = hashtable_key_from_2_strings(path, name, HTSEP); + void *key = hashtable_key_from_2_strings(path, name); if (key == NULL) { return NULL; } @@ -1396,7 +1413,7 @@ _extensions_cache_find_unlocked(PyObject *path, PyObject *name, *p_key = key; } else { - hashtable_destroy_str(key); + hashtable_destroy_key(key); } return entry; } @@ -1534,7 +1551,7 @@ _extensions_cache_set(PyObject *path, PyObject *name, finally_oldvalue: extensions_lock_release(); if (key != NULL) { - hashtable_destroy_str(key); + hashtable_destroy_key(key); } return value; @@ -1578,7 +1595,6 @@ _extensions_cache_clear_all(void) EXTENSIONS.hashtable = NULL; } -#undef HTSEP static bool diff --git a/Python/importdl.c b/Python/importdl.c index 537e8d869dc93ca..011612d97a5f48d 100644 --- a/Python/importdl.c +++ b/Python/importdl.c @@ -8,6 +8,7 @@ #include "pycore_moduleobject.h" // _PyModule_GetDefOrNull() #include "pycore_pyerrors.h" // _PyErr_FormatFromCause() #include "pycore_runtime.h" // _Py_ID() +#include "pycore_unicodeobject.h" // _PyUnicode_AsUTF8NoNUL() /***********************************/ @@ -117,7 +118,7 @@ _Py_ext_module_loader_info_init(struct _Py_ext_module_loader_info *p_info, return -1; } - info.newcontext = PyUnicode_AsUTF8(info.name); + info.newcontext = _PyUnicode_AsUTF8NoNUL(info.name); if (info.newcontext == NULL) { _Py_ext_module_loader_info_clear(&info); return -1; @@ -130,6 +131,12 @@ _Py_ext_module_loader_info_init(struct _Py_ext_module_loader_info *p_info, _Py_ext_module_loader_info_clear(&info); return -1; } + if (PyUnicode_FindChar(filename, 0, 0, + PyUnicode_GET_LENGTH(filename), 1) != -1) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + _Py_ext_module_loader_info_clear(&info); + return -1; + } info.filename = Py_NewRef(filename); #ifndef MS_WINDOWS