From b620369312e660b3406cd5f9cedd41e5ed8cd0d0 Mon Sep 17 00:00:00 2001 From: Amin Solhizadeh Date: Thu, 2 Jul 2026 15:33:22 +0200 Subject: [PATCH 01/22] Decouple lxml from xmlsec via shadow copies (#356) python-xmlsec hands lxml's raw libxml2 node pointers straight to xmlsec1. That only works when lxml and xmlsec link the same libxml2 at runtime; when they differ (e.g. lxml's bundled libxml2 vs a system/homebrew one), mixing the two libraries' nodes corrupts memory and segfaults. Rework the template functions to run each xmlsec call on a private "shadow" copy of the element: PyXmlSec_LxmlShadowBegin serializes the element with lxml's own libxml2 and re-parses the bytes with ours, the xmlsec call mutates that copy, and PyXmlSec_LxmlShadowEnd reflects the change back into the live lxml tree. Only bytes ever cross the boundary, never node pointers. Converting a function is four lines (Begin / the unchanged xmlsec call on shadow.root / End) with no per-function callback or context struct. End detects what the call did generically, by tagging pre-existing nodes through the libxml2 _private field, and covers the whole xmlSecTmpl* family: - plain adds graft the new subtree at the position xmlsec chose, - calls that create intermediate ancestors (add_transform's ) graft the topmost new node and return the inner one, - find-or-create calls (ensure_key_info) return the existing live node and mirror any attributes set on it, instead of duplicating it. Reflection dumps the whole mutated copy, not just the new node, so ancestor-declared namespaces and xmlsec's "\n" formatting siblings survive the round-trip and signatures stay byte-identical; the one text slot xmlsec may touch before the new node (parent text / previous sibling tail) is mirrored explicitly. Child indices count exactly the node types lxml exposes as children, so comments/PIs in templates don't skew paths. add_reference, add_transform and ensure_key_info - one per reflect shape - are converted; the rest of template.c is mechanical follow-up. ds.c, enc.c and tree.c still pass raw nodes, so the import-time version guard stays, now with a PYXMLSEC_SKIP_VERSION_CHECK opt-out used to exercise the shadow paths under a mismatch. Validated under a real 2.14<->2.15 libxml2 mismatch: full suite green (288 passed) including the per-test leak detector, plus a 10k-iteration loop over the three converted functions with no crash, no RSS growth and byte-identical output. See developer.md. Co-Authored-By: Claude Fable 5 --- developer.md | 99 +++++++++++++ src/lxml.c | 319 +++++++++++++++++++++++++++++++++++++++- src/lxml.h | 37 +++++ src/template.c | 41 ++++-- tests/test_templates.py | 38 +++++ 5 files changed, 521 insertions(+), 13 deletions(-) create mode 100644 developer.md diff --git a/developer.md b/developer.md new file mode 100644 index 00000000..3a728f75 --- /dev/null +++ b/developer.md @@ -0,0 +1,99 @@ +# Decoupling lxml and xmlsec across libxml2 (#356) + +`python-xmlsec` glues together two libraries that both build on **libxml2**: +lxml (the tree the user edits) and xmlsec1 (the C library that signs/encrypts). +Historically the extension reached into an lxml `_Element` for its raw +`xmlNodePtr` and handed it to xmlsec1. That is only safe when both libraries +link the *same* libxml2 at runtime — and they often don't (lxml wheels bundle +their own). Mixing two libxml2 builds on one tree corrupts memory: segfaults, +double-frees, wrong signatures +([#356](https://github.com/xmlsec/python-xmlsec/issues/356)). The only guard +was refusing to import on a version mismatch (#283). + +## The fix: shadow copies + +Never share nodes; share **bytes**. Each xmlsec call runs on a private, +throwaway copy of the element ("shadow") owned by *our* libxml2, and the +change it makes is reflected back into the live lxml tree afterwards — again +via bytes. Implemented as one pair of helpers in [src/lxml.c](src/lxml.c) +(contract in [src/lxml.h](src/lxml.h)); converting a function is four lines, +with no per-function callback or context struct: + +```c +PyXmlSec_LxmlShadow shadow; +if (PyXmlSec_LxmlShadowBegin(&shadow, node) < 0) goto ON_FAIL; // lxml → bytes → our copy +Py_BEGIN_ALLOW_THREADS; +res = xmlSecTmplSignatureAddReference(shadow.root, ...); // xmlsec mutates the copy +Py_END_ALLOW_THREADS; +result = PyXmlSec_LxmlShadowEnd(&shadow, res, "cannot add reference."); // reflect back, return lxml node +``` + +`Begin` serializes the element with lxml's own `etree.tostring` and re-parses +the bytes with `xmlReadMemory`, tagging every pre-existing node through the +libxml2 `_private` field. `End` walks up from `res` to find the topmost +untagged (= new) node and reflects generically, covering every shape in the +`xmlSecTmpl*` family: + +- **plain add** (`add_reference`): the new subtree is grafted into the live + tree at the same position, located by child-index path. +- **intermediate ancestors** (`add_transform` creating `` around + the ``): the *topmost* new node is grafted; the returned element + is the descendant matching `res`. +- **find-or-create** (`ensure_key_info`): nothing new in the tree — the + existing live element is returned, plus any attributes the call set (`Id`). + +Two serialization details are load-bearing for byte-identical signatures: + +- `End` dumps the **whole** mutated copy (`xmlDocDumpMemory`), not just the new + node, so ancestor-declared namespaces (dsig on ``) and xmlsec's + `"\n"` formatting siblings survive the lxml re-parse with no manual fix-up; + the new node's tail travels with it through `insert()`. +- xmlsec may also emit a `"\n"` *before* the new node (`xmlSecAddChild` / + `AddNextSibling` / `AddPrevSibling`); `End` mirrors that one text slot + (parent `.text` or previous sibling `.tail`) from the copy. + +Child indices count exactly the node types lxml exposes as children (elements, +comments, PIs, entity refs), so paths recorded on the raw copy resolve +identically through lxml's `__getitem__`/`insert`. `End` assumes the xmlsec +call mutates at most one place in the tree — true for all `xmlSecTmpl*` +functions. Whole-document operations (`sign`, `encrypt`) mutate many places +and will need a different reflect step on top of the same Begin machinery. + +> The shadow decouples *lxml* from xmlsec. The extension and `libxmlsec1` +> must still share one libxml2 (wheels/static builds guarantee this). + +## Building & validating under a real mismatch (macOS / homebrew) + +lxml wheels bundle libxml2; build the extension against homebrew's (which +libxmlsec1 links) so only lxml differs — the true #356 scenario. Watch the +three-way trap: the linker prefers the SDK stub `/usr/lib/libxml2.2.dylib`, +so rewrite the runtime dependency after building: + +```sh +rm -rf build/ src/xmlsec.cpython-*-darwin.so +PKG_CONFIG_PATH=/opt/homebrew/opt/libxml2/lib/pkgconfig \ + python setup.py build_ext --inplace --force +install_name_tool -change /usr/lib/libxml2.2.dylib \ + /opt/homebrew/opt/libxml2/lib/libxml2.16.dylib src/xmlsec.cpython-*-darwin.so + +# verify the mismatch is real, then run the suite under it +PYXMLSEC_SKIP_VERSION_CHECK=1 PYTHONPATH=src python -c \ + "import xmlsec; from lxml import etree; \ + print('lxml', etree.LIBXML_VERSION, 'xmlsec', xmlsec.get_libxml_version())" +PYXMLSEC_SKIP_VERSION_CHECK=1 PYTHONPATH=src python -m pytest tests/ +``` + +`PYXMLSEC_SKIP_VERSION_CHECK` bypasses the import-time mismatch guard. It +exists to exercise the shadow paths; it is **unsafe** for every operation +still on the raw-node path, so keep it off in normal use. The guard can only +be relaxed once all node-passing paths are converted. + +## Status + +- ✅ `template.add_reference`, `template.add_transform`, + `template.ensure_key_info` — one per reflect shape. Validated under a real + 2.14 ↔ 2.15 mismatch: full suite green, 10k-iteration loop with no crash, + no leak, byte-identical output. +- ⬜ Rest of `src/template.c` — mechanical: the four-line pattern above. +- ⬜ `src/ds.c` (sign/verify), `src/enc.c` (encrypt/decrypt), `src/tree.c` — + need a whole-document reflect strategy. diff --git a/src/lxml.c b/src/lxml.c index c98e933b..163dee72 100644 --- a/src/lxml.c +++ b/src/lxml.c @@ -102,7 +102,13 @@ static int PyXmlSec_CheckLxmlLibraryVersion(void) { } int PyXmlSec_InitLxmlModule(void) { - if (PyXmlSec_CheckLxmlLibraryVersion() < 0) { + // By default refuse to import when lxml and xmlsec link different libxml2 + // versions: passing raw nodes between the two libraries then corrupts + // memory (https://github.com/xmlsec/python-xmlsec/issues/283). Setting + // PYXMLSEC_SKIP_VERSION_CHECK bypasses the guard — needed to exercise the + // shadow-copy paths (issue #356) under a mismatch, but unsafe for every + // operation that still hands an lxml node to xmlsec. + if (PyXmlSec_CheckLxmlLibraryVersion() < 0 && getenv("PYXMLSEC_SKIP_VERSION_CHECK") == NULL) { PyXmlSec_SetLastError("lxml & xmlsec libxml2 library version mismatch"); return -1; } @@ -129,3 +135,314 @@ int PyXmlSec_LxmlElementConverter(PyObject* o, PyXmlSec_LxmlElementPtr* p) { Py_DECREF(node); return 1; } + +// ---------------------------------------------------------------------------- +// Shadow copies (issue #356) — see lxml.h for the contract. +// +// Both crossings go through serialized bytes: lxml's own etree.tostring / +// etree.fromstring on its side, xmlReadMemory / xmlDocDumpMemory on ours. +// End dumps the *whole* mutated copy, not just the new node: the surrounding +// markup carries the ancestor-declared namespaces and the "\n" formatting +// siblings xmlsec emits, so the reflected result stays byte-identical to the +// old raw-pointer code without any manual namespace or whitespace fix-up. +// ---------------------------------------------------------------------------- + +// etree.tostring(element, with_tail=False) — lxml's own libxml2 walks the +// tree; with_tail keeps the serialization to the element itself. +static PyObject* PyXmlSec_LxmlElementToBytes(PyObject* element) { + PyObject* result = NULL; + PyObject* args = NULL; + PyObject* kwargs = NULL; + PyObject* tostring = NULL; + PyObject* etree = PyImport_ImportModule("lxml.etree"); + if (etree == NULL) { + return NULL; + } + tostring = PyObject_GetAttrString(etree, "tostring"); + Py_DECREF(etree); + if (tostring == NULL) { + return NULL; + } + args = PyTuple_Pack(1, element); + kwargs = Py_BuildValue("{s:O}", "with_tail", Py_False); + if (args != NULL && kwargs != NULL) { + result = PyObject_Call(tostring, args, kwargs); + } + Py_XDECREF(args); + Py_XDECREF(kwargs); + Py_DECREF(tostring); + return result; +} + +// etree.fromstring(data) — the parsed nodes are owned and managed by lxml. +static PyObject* PyXmlSec_LxmlElementFromBytes(PyObject* data) { + PyObject* result; + PyObject* etree = PyImport_ImportModule("lxml.etree"); + if (etree == NULL) { + return NULL; + } + result = PyObject_CallMethod(etree, "fromstring", "O", data); + Py_DECREF(etree); + return result; +} + +// Nodes that exist before the xmlsec call are tagged through the libxml2 +// _private field (never serialized, never touched by the parser or xmlsec); +// whatever is untagged after the call is new. +static const char PyXmlSec_LxmlShadowMarker = 0; +#define PYXMLSEC_SHADOW_MARKED(n) ((n)->_private == (void*)&PyXmlSec_LxmlShadowMarker) + +static void PyXmlSec_LxmlShadowMark(xmlNodePtr node) { + for (; node != NULL; node = node->next) { + node->_private = (void*)&PyXmlSec_LxmlShadowMarker; + if (node->children != NULL) { + PyXmlSec_LxmlShadowMark(node->children); + } + } +} + +// dsig/enc structures never nest anywhere near this deep; the bound just keeps +// the path buffers on the stack and guards against a pathological tree. +#define PYXMLSEC_SHADOW_MAX_DEPTH 64 + +// Index of node among its preceding siblings, counting only the node types +// lxml exposes as children (elements, comments, PIs, entity refs), so indices +// computed here line up with lxml's __getitem__ / insert. +static int PyXmlSec_LxmlShadowChildIndex(xmlNodePtr node) { + int idx = 0; + xmlNodePtr s; + for (s = node->prev; s != NULL; s = s->prev) { + if (_isElement(s)) { + ++idx; + } + } + return idx; +} + +// Records the child indices leading from `top` down to `node` into `path` +// (ordered top-first) and returns the number of steps, or -1 when node is not +// under top or lies too deep. Both trees involved are byte-for-byte copies of +// each other, so a path recorded in one resolves in the other. +static int PyXmlSec_LxmlShadowPathTo(xmlNodePtr node, xmlNodePtr top, int* path) { + int depth = 0; + int d; + xmlNodePtr n = node; + while (n != top) { + if (n->parent == NULL || depth >= PYXMLSEC_SHADOW_MAX_DEPTH) { + return -1; + } + ++depth; + n = n->parent; + } + for (n = node, d = depth; d > 0; n = n->parent) { + path[--d] = PyXmlSec_LxmlShadowChildIndex(n); + } + return depth; +} + +// Walks `path` (child indices) down from `start`. Returns a new reference. +static PyObject* PyXmlSec_LxmlShadowWalk(PyObject* start, const int* path, int depth) { + int i; + PyObject* cur = start; + Py_INCREF(cur); + for (i = 0; i < depth; ++i) { + PyObject* child = PySequence_GetItem(cur, path[i]); + Py_DECREF(cur); + if (child == NULL) { + return NULL; + } + cur = child; + } + return cur; +} + +// Copies the attributes of `src` (a node in the shadow copy) onto the live +// lxml element `dst`: find-or-create calls may set attributes (e.g. Id) on a +// node that already existed, and that is then the only change to reflect. +static int PyXmlSec_LxmlShadowSyncAttributes(xmlNodePtr src, PyObject* dst) { + xmlAttrPtr attr; + for (attr = src->properties; attr != NULL; attr = attr->next) { + PyObject* r = NULL; + xmlChar* value = xmlGetNsProp(src, attr->name, attr->ns != NULL ? attr->ns->href : NULL); + if (value == NULL) { + continue; + } + if (attr->ns != NULL && attr->ns->href != NULL) { + // lxml takes namespaced attribute names in Clark notation. + PyObject* key = PyUnicode_FromFormat("{%s}%s", (const char*)attr->ns->href, (const char*)attr->name); + if (key != NULL) { + r = PyObject_CallMethod(dst, "set", "Os", key, (const char*)value); + Py_DECREF(key); + } + } else { + r = PyObject_CallMethod(dst, "set", "ss", (const char*)attr->name, (const char*)value); + } + xmlFree(value); + if (r == NULL) { + return -1; + } + Py_DECREF(r); + } + return 0; +} + +int PyXmlSec_LxmlShadowBegin(PyXmlSec_LxmlShadow* shadow, PyXmlSec_LxmlElementPtr element) { + PyObject* bytes; + char* data = NULL; + Py_ssize_t size = 0; + + shadow->element = element; + shadow->doc = NULL; + shadow->root = NULL; + + bytes = PyXmlSec_LxmlElementToBytes((PyObject*)element); + if (bytes == NULL || PyBytes_AsStringAndSize(bytes, &data, &size) < 0) { + Py_XDECREF(bytes); + return -1; + } + shadow->doc = xmlReadMemory(data, (int)size, NULL, NULL, XML_PARSE_NONET); + Py_DECREF(bytes); + if (shadow->doc == NULL || (shadow->root = xmlDocGetRootElement(shadow->doc)) == NULL) { + if (shadow->doc != NULL) { + xmlFreeDoc(shadow->doc); + shadow->doc = NULL; + } + PyErr_SetString(PyXmlSec_InternalError, "cannot make a private copy of the element."); + return -1; + } + PyXmlSec_LxmlShadowMark(shadow->doc->children); + return 0; +} + +PyObject* PyXmlSec_LxmlShadowEnd(PyXmlSec_LxmlShadow* shadow, xmlNodePtr res, const char* error) { + PyObject* result = NULL; + PyObject* bytes = NULL; + PyObject* copy_root = NULL; + PyObject* copy_parent = NULL; + PyObject* live_parent = NULL; + PyObject* new_node = NULL; + PyObject* tmp = NULL; + + xmlNodePtr fresh = NULL; // topmost node the xmlsec call created, if any + xmlNodePtr n; + xmlChar* dump = NULL; + int dump_size = 0; + + int path[PYXMLSEC_SHADOW_MAX_DEPTH]; + int rel[PYXMLSEC_SHADOW_MAX_DEPTH]; + int depth, rel_depth, insert_idx; + + if (res == NULL) { + PyXmlSec_SetLastError(error); + goto DONE; + } + + // Everything the xmlsec call created is unmarked; walk up from res to find + // the topmost new node (stays NULL when res already existed before the call). + for (n = res; n != shadow->root && !PYXMLSEC_SHADOW_MARKED(n); n = n->parent) { + if (n->parent == NULL) { + PyErr_SetString(PyXmlSec_InternalError, "unexpected result node."); + goto DONE; + } + fresh = n; + } + + if (fresh == NULL) { + // Find-or-create found: the tree did not grow, so the result is the + // live lxml element in the same position (plus any attributes set). + depth = PyXmlSec_LxmlShadowPathTo(res, shadow->root, path); + if (depth < 0) { + PyErr_SetString(PyXmlSec_InternalError, "unexpected result node."); + goto DONE; + } + result = PyXmlSec_LxmlShadowWalk((PyObject*)shadow->element, path, depth); + if (result != NULL && PyXmlSec_LxmlShadowSyncAttributes(res, result) < 0) { + Py_CLEAR(result); + } + goto DONE; + } + + depth = PyXmlSec_LxmlShadowPathTo(fresh->parent, shadow->root, path); + rel_depth = PyXmlSec_LxmlShadowPathTo(res, fresh, rel); + if (depth < 0 || rel_depth < 0) { + PyErr_SetString(PyXmlSec_InternalError, "unexpected result node."); + goto DONE; + } + insert_idx = PyXmlSec_LxmlShadowChildIndex(fresh); + + xmlDocDumpMemory(shadow->doc, &dump, &dump_size); + if (dump == NULL || dump_size <= 0) { + PyErr_SetString(PyXmlSec_InternalError, "cannot serialize the mutated copy."); + goto DONE; + } + bytes = PyBytes_FromStringAndSize((const char*)dump, (Py_ssize_t)dump_size); + if (bytes == NULL) { + goto DONE; + } + copy_root = PyXmlSec_LxmlElementFromBytes(bytes); + if (copy_root == NULL) { + goto DONE; + } + copy_parent = PyXmlSec_LxmlShadowWalk(copy_root, path, depth); + live_parent = PyXmlSec_LxmlShadowWalk((PyObject*)shadow->element, path, depth); + if (copy_parent == NULL || live_parent == NULL) { + goto DONE; + } + new_node = PySequence_GetItem(copy_parent, insert_idx); + if (new_node == NULL) { + goto DONE; + } + + // Move the new node into the live tree at the position the xmlsec call + // chose; lxml carries the node's tail along and reconciles namespaces. + tmp = PyObject_CallMethod(live_parent, "insert", "iO", insert_idx, new_node); + if (tmp == NULL) { + goto DONE; + } + Py_CLEAR(tmp); + + // xmlsec may also have put a "\n" *before* the new node — the parent's + // text when it is the first child, the previous sibling's tail otherwise; + // mirror that too (a no-op when nothing changed there). + if (insert_idx == 0) { + tmp = PyObject_GetAttrString(copy_parent, "text"); + if (tmp == NULL || PyObject_SetAttrString(live_parent, "text", tmp) < 0) { + goto DONE; + } + Py_CLEAR(tmp); + } else { + PyObject* prev_tail = NULL; + PyObject* copy_prev = PySequence_GetItem(copy_parent, insert_idx - 1); + PyObject* live_prev = PySequence_GetItem(live_parent, insert_idx - 1); + int failed = (copy_prev == NULL || live_prev == NULL + || (prev_tail = PyObject_GetAttrString(copy_prev, "tail")) == NULL + || PyObject_SetAttrString(live_prev, "tail", prev_tail) < 0); + Py_XDECREF(copy_prev); + Py_XDECREF(live_prev); + Py_XDECREF(prev_tail); + if (failed) { + goto DONE; + } + } + + // res may sit below the topmost new node (e.g. the new inside + // a freshly created ); descend to it in the grafted subtree. + result = PyXmlSec_LxmlShadowWalk(new_node, rel, rel_depth); + +DONE: + if (shadow->doc != NULL) { + xmlFreeDoc(shadow->doc); + shadow->doc = NULL; + shadow->root = NULL; + } + if (dump != NULL) { + xmlFree(dump); + } + Py_XDECREF(bytes); + Py_XDECREF(copy_root); + Py_XDECREF(copy_parent); + Py_XDECREF(live_parent); + Py_XDECREF(new_node); + Py_XDECREF(tmp); + return result; +} diff --git a/src/lxml.h b/src/lxml.h index 72050efe..9abeb906 100644 --- a/src/lxml.h +++ b/src/lxml.h @@ -29,6 +29,43 @@ PyXmlSec_LxmlElementPtr PyXmlSec_elementFactory(PyXmlSec_LxmlDocumentPtr doc, xm // converts o to PyObject, None object is not allowed, does not increment ref_counts int PyXmlSec_LxmlElementConverter(PyObject* o, PyXmlSec_LxmlElementPtr* p); +// A "shadow" is a private copy of an lxml element, owned by the libxml2 this +// extension links, so that an xmlsec call never touches a node allocated by +// lxml's (possibly different) libxml2 — only serialized bytes cross between +// the two libraries (https://github.com/xmlsec/python-xmlsec/issues/356). +// +// Usage (see template.c): +// +// PyXmlSec_LxmlShadow shadow; +// if (PyXmlSec_LxmlShadowBegin(&shadow, node) < 0) goto ON_FAIL; +// Py_BEGIN_ALLOW_THREADS; +// res = xmlSecTmplSignatureAddReference(shadow.root, ...); +// Py_END_ALLOW_THREADS; +// result = PyXmlSec_LxmlShadowEnd(&shadow, res, "cannot add reference."); +// +// Begin serializes `element` with lxml's own libxml2 and re-parses the bytes +// with ours into `root`/`doc`. The caller then runs exactly one xmlsec call +// against `root` (nothing else; Python may not run between Begin and End) and +// hands the returned node to End, which reflects whatever the call did back +// into the live lxml tree and returns the lxml element corresponding to that +// node (a new reference), or NULL with an exception set (`error` is raised +// when the node is NULL). End must be called exactly once after a successful +// Begin; it always releases the copy. +// +// The reflection covers the whole xmlSecTmpl* family: a new subtree grafted at +// the position xmlsec chose (including intermediate nodes like +// and the "\n" formatting text around it), or — for find-or-create calls that +// added nothing — the already-existing element plus any attributes the call +// set on it. It assumes the call mutates at most one place in the tree. +typedef struct { + PyXmlSec_LxmlElementPtr element; // borrowed; the live lxml element + xmlDocPtr doc; // the private copy; owned by the shadow + xmlNodePtr root; // doc's root element (the copy of element) +} PyXmlSec_LxmlShadow; + +int PyXmlSec_LxmlShadowBegin(PyXmlSec_LxmlShadow* shadow, PyXmlSec_LxmlElementPtr element); +PyObject* PyXmlSec_LxmlShadowEnd(PyXmlSec_LxmlShadow* shadow, xmlNodePtr res, const char* error); + // get version numbers for libxml2 both compiled and loaded long PyXmlSec_GetLibXmlVersionMajor(); long PyXmlSec_GetLibXmlVersionMinor(); diff --git a/src/template.c b/src/template.c index c6864c2e..75f8bd50 100644 --- a/src/template.c +++ b/src/template.c @@ -91,6 +91,8 @@ static PyObject* PyXmlSec_TemplateAddReference(PyObject* self, PyObject *args, P const char* uri = NULL; const char* type = NULL; xmlNodePtr res; + PyObject* result; + PyXmlSec_LxmlShadow shadow; PYXMLSEC_DEBUG("template add_reference - start"); if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&O!|zzz:add_reference", kwlist, @@ -98,16 +100,21 @@ static PyObject* PyXmlSec_TemplateAddReference(PyObject* self, PyObject *args, P { goto ON_FAIL; } + // The xmlsec call runs on a private copy of `node`, never on lxml's own + // nodes (issue #356); the shadow reflects the new back. + if (PyXmlSec_LxmlShadowBegin(&shadow, node) < 0) { + goto ON_FAIL; + } Py_BEGIN_ALLOW_THREADS; - res = xmlSecTmplSignatureAddReference(node->_c_node, digest->id, XSTR(id), XSTR(uri), XSTR(type)); + res = xmlSecTmplSignatureAddReference(shadow.root, digest->id, XSTR(id), XSTR(uri), XSTR(type)); Py_END_ALLOW_THREADS; - if (res == NULL) { - PyXmlSec_SetLastError("cannot add reference."); + result = PyXmlSec_LxmlShadowEnd(&shadow, res, "cannot add reference."); + if (result == NULL) { goto ON_FAIL; } PYXMLSEC_DEBUG("template add_reference - ok"); - return (PyObject*)PyXmlSec_elementFactory(node->_doc, res); + return result; ON_FAIL: PYXMLSEC_DEBUG("template add_reference - fail"); @@ -129,6 +136,8 @@ static PyObject* PyXmlSec_TemplateAddTransform(PyObject* self, PyObject *args, P PyXmlSec_LxmlElementPtr node = NULL; PyXmlSec_Transform* transform = NULL; xmlNodePtr res; + PyObject* result; + PyXmlSec_LxmlShadow shadow; PYXMLSEC_DEBUG("template add_transform - start"); if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&O!:add_transform", kwlist, @@ -136,16 +145,19 @@ static PyObject* PyXmlSec_TemplateAddTransform(PyObject* self, PyObject *args, P { goto ON_FAIL; } + if (PyXmlSec_LxmlShadowBegin(&shadow, node) < 0) { + goto ON_FAIL; + } Py_BEGIN_ALLOW_THREADS; - res = xmlSecTmplReferenceAddTransform(node->_c_node, transform->id); + res = xmlSecTmplReferenceAddTransform(shadow.root, transform->id); Py_END_ALLOW_THREADS; - if (res == NULL) { - PyXmlSec_SetLastError("cannot add transform."); + result = PyXmlSec_LxmlShadowEnd(&shadow, res, "cannot add transform."); + if (result == NULL) { goto ON_FAIL; } PYXMLSEC_DEBUG("template add_transform - ok"); - return (PyObject*)PyXmlSec_elementFactory(node->_doc, res); + return result; ON_FAIL: PYXMLSEC_DEBUG("template add_transform - fail"); @@ -167,6 +179,8 @@ static PyObject* PyXmlSec_TemplateEnsureKeyInfo(PyObject* self, PyObject *args, PyXmlSec_LxmlElementPtr node = NULL; const char* id = NULL; xmlNodePtr res; + PyObject* result; + PyXmlSec_LxmlShadow shadow; PYXMLSEC_DEBUG("template ensure_key_info - start"); if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&|z:ensure_key_info", kwlist, PyXmlSec_LxmlElementConverter, &node, &id)) @@ -174,16 +188,19 @@ static PyObject* PyXmlSec_TemplateEnsureKeyInfo(PyObject* self, PyObject *args, goto ON_FAIL; } + if (PyXmlSec_LxmlShadowBegin(&shadow, node) < 0) { + goto ON_FAIL; + } Py_BEGIN_ALLOW_THREADS; - res = xmlSecTmplSignatureEnsureKeyInfo(node->_c_node, XSTR(id)); + res = xmlSecTmplSignatureEnsureKeyInfo(shadow.root, XSTR(id)); Py_END_ALLOW_THREADS; - if (res == NULL) { - PyXmlSec_SetLastError("cannot ensure key info."); + result = PyXmlSec_LxmlShadowEnd(&shadow, res, "cannot ensure key info."); + if (result == NULL) { goto ON_FAIL; } PYXMLSEC_DEBUG("template ensure_key_info - ok"); - return (PyObject*)PyXmlSec_elementFactory(node->_doc, res); + return result; ON_FAIL: PYXMLSEC_DEBUG("template ensure_key_info - fail"); diff --git a/tests/test_templates.py b/tests/test_templates.py index bbf7f42d..09567fce 100644 --- a/tests/test_templates.py +++ b/tests/test_templates.py @@ -36,6 +36,18 @@ def test_ensure_key_info(self): ki = xmlsec.template.ensure_key_info(sign, id='Id') self.assertEqual('Id', ki.get('Id')) + def test_ensure_key_info_existing(self): + root = self.load_xml('doc.xml') + sign = xmlsec.template.create(root, c14n_method=consts.TransformExclC14N, sign_method=consts.TransformRsaSha1) + ki = xmlsec.template.ensure_key_info(sign) + self.assertIs(ki.getroottree().getroot(), root) + # the second call finds the existing node instead of adding another one, + # but still applies the requested id to it + ki2 = xmlsec.template.ensure_key_info(sign, id='Id') + self.assertIs(ki2, ki) + self.assertEqual('Id', ki.get('Id')) + self.assertEqual(1, sum(1 for n in sign if n.tag == f'{{{consts.DSigNs}}}KeyInfo')) + def test_ensure_key_info_fail(self): with self.assertRaisesRegex(xmlsec.Error, 'cannot ensure key info.'): xmlsec.template.ensure_key_info(etree.fromstring(b''), id='Id') @@ -92,6 +104,32 @@ def test_add_reference_fail(self): with self.assertRaisesRegex(xmlsec.Error, 'cannot add reference.'): xmlsec.template.add_reference(etree.Element('root'), consts.TransformSha1) + def test_add_transform(self): + root = self.load_xml('doc.xml') + sign = xmlsec.template.create(root, c14n_method=consts.TransformExclC14N, sign_method=consts.TransformRsaSha1) + ref = xmlsec.template.add_reference(sign, consts.TransformSha1, uri='URI') + tr = xmlsec.template.add_transform(ref, consts.TransformEnveloped) + # the returned node is live in the original tree, inside the + # wrapper the first call creates before + self.assertIs(tr.getroottree().getroot(), root) + transforms = tr.getparent() + self.assertEqual(f'{{{consts.DSigNs}}}Transforms', transforms.tag) + self.assertIs(ref[0], transforms) + self.assertEqual(f'{{{consts.DSigNs}}}DigestMethod', ref[1].tag) + + def test_add_transform_existing_transforms(self): + root = self.load_xml('doc.xml') + sign = xmlsec.template.create(root, c14n_method=consts.TransformExclC14N, sign_method=consts.TransformRsaSha1) + ref = xmlsec.template.add_reference(sign, consts.TransformSha1, uri='URI') + tr = xmlsec.template.add_transform(ref, consts.TransformEnveloped) + # a second transform lands in the existing wrapper, after the first + tr2 = xmlsec.template.add_transform(ref, consts.TransformExclC14N) + transforms = tr.getparent() + self.assertIs(tr2.getparent(), transforms) + self.assertEqual(2, len(transforms)) + self.assertIs(transforms[0], tr) + self.assertIs(transforms[1], tr2) + def test_add_transform_bad_args(self): with self.assertRaises(TypeError): xmlsec.template.add_transform('', consts.TransformSha1) From 9a7b1a65eecce21e164dcd82ca8331ec1b2ea0ff Mon Sep 17 00:00:00 2001 From: Amin Solhizadeh Date: Thu, 2 Jul 2026 16:02:40 +0200 Subject: [PATCH 02/22] Add step-by-step guide for converting functions to shadow copies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit developer.md explains why the shadow copy exists and how the reflection works; converting-functions.md is the operational companion: pick a function, classify the xmlSecTmpl* call against the shapes the reflect covers (including the int-returning and detached-create shapes that need extra care), apply the mechanical binding edit, and the test / mismatch-validation checklist — including the tests/base.py leak detector gotcha. Co-Authored-By: Claude Fable 5 --- converting-functions.md | 128 ++++++++++++++++++++++++++++++++++++++++ developer.md | 3 +- 2 files changed, 130 insertions(+), 1 deletion(-) create mode 100644 converting-functions.md diff --git a/converting-functions.md b/converting-functions.md new file mode 100644 index 00000000..6818e7db --- /dev/null +++ b/converting-functions.md @@ -0,0 +1,128 @@ +# Converting a function to the shadow-copy pattern — step by step + +This is the how-to companion to [developer.md](developer.md) (which explains +*why* the shadow copy exists and how the reflection works). Follow these steps +to move one more binding off the raw-node path for +[#356](https://github.com/xmlsec/python-xmlsec/issues/356). + +## 1. Pick a function + +Anything still touching `node->_c_node` / `node->_doc->_c_doc` is on the raw +path: + +```sh +grep -n '_c_node\|_c_doc' src/template.c +``` + +## 2. Classify the xmlsec call + +Read the `xmlSecTmpl*` function it wraps (xmlsec1's `src/templates.c`) and +match it to a row: + +| Shape | Examples | Covered? | +|---|---|---| +| Adds a subtree under the element (any position, possibly with intermediate nodes) | `add_key_name`, `add_key_value`, `add_x509_data`, `x509_data_add_*`, `add_encrypted_key` | ✅ just follow step 3 | +| Find-or-create, may set attributes on the existing node | `encrypted_data_ensure_key_info`, `encrypted_data_ensure_cipher_value` | ✅ just follow step 3 | +| Returns a status `int`, mutates one child | `transform_add_c14n_inclusive_namespaces` | ✅ with a twist: after the call, locate the affected child on the copy (here: the `` under `shadow.root`) and pass *that* to `End`; discard the returned element and return `None` | +| Returns a **detached** node — the element argument only supplies the document | `create`, `encrypted_data_create` | ❌ `End` would raise "unexpected result node"; needs its own reflect (no graft — parse the standalone result with lxml and return it) | +| Reads or mutates the **whole document** | `ds.c` sign/verify, `enc.c` encrypt/decrypt, `tree.c` | ❌ needs a whole-document reflect strategy | + +`res` does not have to be the topmost node the call created — `End` walks up +to the topmost new ancestor itself. Any node inside the new subtree works. + +## 3. Edit the binding + +The change is mechanical; `add_key_name` as the worked example: + +```c + PyXmlSec_LxmlElementPtr node = NULL; + const char* name = NULL; + xmlNodePtr res; ++ PyObject* result; ++ PyXmlSec_LxmlShadow shadow; + + PYXMLSEC_DEBUG("template add_key_name - start"); + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&|z:add_key_name", kwlist, + PyXmlSec_LxmlElementConverter, &node, &name)) + { + goto ON_FAIL; + } + ++ if (PyXmlSec_LxmlShadowBegin(&shadow, node) < 0) { ++ goto ON_FAIL; ++ } + Py_BEGIN_ALLOW_THREADS; +- res = xmlSecTmplKeyInfoAddKeyName(node->_c_node, XSTR(name)); ++ res = xmlSecTmplKeyInfoAddKeyName(shadow.root, XSTR(name)); + Py_END_ALLOW_THREADS; +- if (res == NULL) { +- PyXmlSec_SetLastError("cannot add key name."); ++ result = PyXmlSec_LxmlShadowEnd(&shadow, res, "cannot add key name."); ++ if (result == NULL) { + goto ON_FAIL; + } + + PYXMLSEC_DEBUG("template add_key_name - ok"); +- return (PyObject*)PyXmlSec_elementFactory(node->_doc, res); ++ return result; +``` + +Rules the pattern must keep: + +- swap `node->_c_node` for `shadow.root` and change **nothing else** about the + xmlsec call or its error string; +- run **exactly one** xmlsec call between `Begin` and `End`, and no Python + code (the `Py_*_ALLOW_THREADS` pair is fine — the call is pure C); +- call `End` **exactly once** after a successful `Begin`; it frees the copy on + every path, including when `res == NULL`. + +## 4. Build + +```sh +python setup.py build_ext --inplace --force +PYTHONPATH=src python -m pytest tests/ +``` + +(On a homebrew Mac the plain build links mismatched libxml2s — see +"Building & validating under a real mismatch" in [developer.md](developer.md) +for the `PKG_CONFIG_PATH` + `install_name_tool` recipe.) + +## 5. Add a targeted test + +Existing tests cover return values; add one asserting the *reflection*, in +`tests/test_templates.py`: + +- the returned node is live in the caller's tree + (`self.assertIs(kn.getroottree().getroot(), root)`) and at the position + xmlsec puts it; +- for find-or-create: a second call returns the same element + (`assertIs`), sets the requested attributes on it, and does not duplicate it. + +Beware the leak detector in `tests/base.py`: it reruns each test with +`gc.disable()` and fails on monotonic object-count growth, which plain +allocation churn can trigger with no real leak. Keep each test small (split +rather than combine scenarios), prefer `assertIs(parent[0], tr)` over building +lists to compare, and check stability with a few rounds of: + +```sh +PYXMLSEC_TEST_ITERATIONS=50 PYTHONPATH=src python -m pytest tests/test_templates.py +``` + +## 6. Validate under a real libxml2 mismatch + +Build per the developer.md recipe so lxml and the extension report different +libxml2 versions, then: + +```sh +PYXMLSEC_SKIP_VERSION_CHECK=1 PYTHONPATH=src python -m pytest tests/ +``` + +For anything non-trivial, also loop the converted function ~10k times under +the mismatch and watch `ru_maxrss` stays flat and the serialized output stays +byte-identical between iterations. + +## 7. Record it + +Move the function to the ✅ list in [developer.md](developer.md)'s Status +section. Once nothing passes raw nodes anymore, the import-time version guard +can be relaxed. diff --git a/developer.md b/developer.md index 3a728f75..321dc4df 100644 --- a/developer.md +++ b/developer.md @@ -94,6 +94,7 @@ be relaxed once all node-passing paths are converted. `template.ensure_key_info` — one per reflect shape. Validated under a real 2.14 ↔ 2.15 mismatch: full suite green, 10k-iteration loop with no crash, no leak, byte-identical output. -- ⬜ Rest of `src/template.c` — mechanical: the four-line pattern above. +- ⬜ Rest of `src/template.c` — mechanical: follow the step-by-step guide in + [converting-functions.md](converting-functions.md). - ⬜ `src/ds.c` (sign/verify), `src/enc.c` (encrypt/decrypt), `src/tree.c` — need a whole-document reflect strategy. From e939f1e14538b24645707064c32f391c36afa9c1 Mon Sep 17 00:00:00 2001 From: Amin Solhizadeh Date: Thu, 2 Jul 2026 16:06:13 +0200 Subject: [PATCH 03/22] Add high-level summary of the shadow-copy solution for #356 The problem, the shadow-copy idea, what the change consists of, why this design replaced the first (op/ctx) attempt, validation results, and what remains. Entry point to developer.md (design detail) and converting-functions.md (rollout how-to). Co-Authored-By: Claude Fable 5 --- 356-summary.md | 97 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 356-summary.md diff --git a/356-summary.md b/356-summary.md new file mode 100644 index 00000000..27b15932 --- /dev/null +++ b/356-summary.md @@ -0,0 +1,97 @@ +# Shadow copies for #356 — what was done, at a high level + +**TL;DR** — `python-xmlsec` crashes when `lxml` and `xmlsec1` are built +against different `libxml2` versions, because it passes raw libxml2 node +pointers between them. This branch makes each xmlsec call run on a private +*copy* of the element and reflects the result back afterwards, so only +serialized bytes ever cross the boundary. Three template functions — one per +mutation shape — are converted; converting the rest is a four-line edit each. + +## The problem + +`python-xmlsec` glues together two libraries that both build on **libxml2**: +lxml (the XML tree the user edits in Python) and xmlsec1 (the C library that +signs/encrypts). The extension reaches into an lxml `_Element` for its raw +`xmlNodePtr` and hands it to xmlsec1. That is only safe when both libraries +link the *same* libxml2 at runtime — and they often don't, because lxml wheels +bundle their own. Two libxml2 builds touching one tree means mismatched struct +layouts and allocators: segfaults, double-frees, wrong signatures +([#356](https://github.com/xmlsec/python-xmlsec/issues/356)). The only +existing mitigation was refusing to import on a version mismatch (#283). + +## The idea: shadow copies + +Bytes have no ABI. So instead of sharing nodes, each converted binding now: + +``` + lxml element ──(lxml's libxml2 serializes)──► bytes + bytes ──(our libxml2 parses)──► private "shadow" copy + xmlsec mutates the shadow (it never sees an lxml node) + shadow ──(our libxml2 dumps)──► bytes ──(lxml parses)──► change grafted + into the live tree +``` + +The user-visible behaviour is unchanged: the input element gains exactly what +xmlsec added, the returned node is live in their tree (so incremental building +like `add_transform(ref, ...)` keeps working), and the serialized output stays +byte-identical — namespaces and xmlsec's `"\n"` formatting included. + +## What the change consists of + +- **One helper pair** in `src/lxml.c`/`src/lxml.h`: + `PyXmlSec_LxmlShadowBegin` (element → shadow copy) and + `PyXmlSec_LxmlShadowEnd` (reflect the mutation back, return the lxml node). + `Begin` tags every pre-existing node of the copy through libxml2's private + field, which lets `End` *discover* what the call did instead of being told — + that is what makes the reflection generic. +- **Three bindings converted** in `src/template.c`, deliberately one per + mutation shape so the helper is proven against all of them: + - `add_reference` — plain "add a subtree", + - `add_transform` — also creates an intermediate `` wrapper at a + chosen position, + - `ensure_key_info` — find-or-create: returns the existing node (attributes + synced) instead of duplicating it. + A conversion is four lines: `Begin` / the unchanged xmlsec call on + `shadow.root` / `End` — no per-function callback or context struct. +- **An escape hatch** for development: `PYXMLSEC_SKIP_VERSION_CHECK` bypasses + the import-time mismatch guard so the converted paths can be exercised under + a real mismatch. The guard itself stays on by default. +- **Tests** in `tests/test_templates.py` asserting the reflection semantics + (liveness, position, no duplication on repeated ensure). +- **Two docs**: [developer.md](developer.md) — the design and the build/ + validation recipe; [converting-functions.md](converting-functions.md) — the + step-by-step guide for converting the remaining functions. + +## Why this design (vs. the first attempt) + +An earlier branch (`fix/356-decouple-add-reference`) proved the +serialize-across-the-boundary idea but required a callback function plus a +context struct per converted binding, and its reflection only handled the +"append exactly one child" shape — `ensure_key_info` and `add_transform` +would have needed helper extensions. The shadow design inverts control: the +call site stays a plain xmlsec call, and the helper works out what changed by +diffing tagged vs. untagged nodes. Result: less code overall, zero +per-function boilerplate, and all `xmlSecTmpl*` shapes covered by one +mechanism. + +## Validation + +Exercised under a **real** libxml2 mismatch (lxml bundling 2.14.6, extension + +libxmlsec1 on homebrew 2.15.3): + +- full test suite: **288 passed, 6 skipped**, including the per-test leak + detector, across repeated runs; +- 10,000-iteration loop over all three converted functions: no crash, no RSS + growth, byte-identical output every iteration. + +> Caveat (unchanged from before): this decouples *lxml* from xmlsec. The +> extension and `libxmlsec1` must still share one libxml2 — which wheels and +> static builds guarantee. + +## What's left + +The rest of `src/template.c` is a mechanical rollout of the four-line pattern +(see the guide). `src/ds.c` (sign/verify), `src/enc.c` (encrypt/decrypt) and +`src/tree.c` operate on whole documents and need a reflect strategy of their +own on top of the same `Begin` machinery. The import-time version guard can +only be relaxed once every node-passing path is converted. From 72a78cb3e66b9328b79436cc0bf002524f525052 Mon Sep 17 00:00:00 2001 From: Amin Solhizadeh Date: Thu, 2 Jul 2026 21:34:06 +0200 Subject: [PATCH 04/22] Skip the shadow copy when lxml links the same libxml2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The import guard only lets matched libxml2 versions run, and for them the old direct behavior — xmlsec mutating lxml's nodes — is safe; that is what shipped for years. Yet every converted function paid the full shadow round-trip (serialize with lxml, re-parse, dump, re-parse with lxml) even in that case, four serializations per call for zero safety benefit. That is noise for the small xmlSecTmpl* trees, but would become a real regression when the pattern reaches sign/encrypt on whole documents. Make Begin/End dual-path, decided once at import: on matched versions Begin aliases the live _c_node (no copy) and End just wraps the node xmlsec returned, machine-identical to the pre-shadow code; the shadow round-trip activates only under a mismatch — or when PYXMLSEC_FORCE_SHADOW is set, which CI now uses to run the suite a second time so the shadow path stays exercised on matched builds. Call sites cannot tell the difference, and every function converted later inherits both paths. While in there, resolve lxml.etree's tostring/fromstring once at module init instead of importing lxml.etree on every shadow crossing. Benchmark (matched static build, create + add_reference + add_transform + ensure_key_info per iteration): fast path 8.6us vs shadow 72.3us, ~8x. Validated three ways, with byte-identical template output across the two paths: full suite on the fast path and on PYXMLSEC_FORCE_SHADOW=1 (matched static build, 300 passed / 6 skipped each), and full suite under a real 2.14 vs 2.15 mismatch (288 passed, 6 skipped). Co-Authored-By: Claude Fable 5 --- .github/scripts/manylinux_build_and_test.sh | 5 ++ .github/workflows/linuxbrew.yml | 3 + .github/workflows/macosx.yml | 8 +++ 356-summary.md | 6 ++ developer.md | 24 +++++++ src/lxml.c | 75 ++++++++++++++------- src/lxml.h | 7 ++ 7 files changed, 103 insertions(+), 25 deletions(-) diff --git a/.github/scripts/manylinux_build_and_test.sh b/.github/scripts/manylinux_build_and_test.sh index ce8301a7..7d526a06 100644 --- a/.github/scripts/manylinux_build_and_test.sh +++ b/.github/scripts/manylinux_build_and_test.sh @@ -56,6 +56,11 @@ echo "== [container] Step: Install test dependencies ==" echo "== [container] Step: Run tests ==" /opt/python/${PY_ABI}/bin/pytest -v --color=yes +# Step: Run tests again on the shadow-copy path (issue #356), which the +# matched libxml2 build would otherwise never exercise. +echo "== [container] Step: Run tests (forced shadow path) ==" +PYXMLSEC_FORCE_SHADOW=1 /opt/python/${PY_ABI}/bin/pytest -v --color=yes + # Step: Fix mounted workspace file ownership on host echo "== [container] Step: Fix mounted workspace file ownership on host ==" chown -R "${HOST_UID}:${HOST_GID}" dist wheelhouse build libs || true diff --git a/.github/workflows/linuxbrew.yml b/.github/workflows/linuxbrew.yml index 51b0db1e..c4ab34b0 100644 --- a/.github/workflows/linuxbrew.yml +++ b/.github/workflows/linuxbrew.yml @@ -48,3 +48,6 @@ jobs: pip3 install --upgrade --no-binary=lxml -r requirements-test.txt pip3 install xmlsec --only-binary=xmlsec --no-index --find-links=dist/ pytest -v --color=yes + # Same suite on the shadow-copy path (issue #356), which the + # matched libxml2 build would otherwise never exercise. + PYXMLSEC_FORCE_SHADOW=1 pytest -v --color=yes diff --git a/.github/workflows/macosx.yml b/.github/workflows/macosx.yml index c9d8034e..db765ed3 100644 --- a/.github/workflows/macosx.yml +++ b/.github/workflows/macosx.yml @@ -87,6 +87,14 @@ jobs: run: | coverage run -m pytest -v --color=yes + # Same suite on the shadow-copy path (issue #356), which the matched + # libxml2 build would otherwise never exercise. + - name: Run tests (forced shadow path) + env: + PYXMLSEC_FORCE_SHADOW: "1" + run: | + pytest -v --color=yes + - name: Report coverage to codecov if: matrix.static_deps != 'static' run: | diff --git a/356-summary.md b/356-summary.md index 27b15932..95019b92 100644 --- a/356-summary.md +++ b/356-summary.md @@ -53,6 +53,12 @@ byte-identical — namespaces and xmlsec's `"\n"` formatting included. synced) instead of duplicating it. A conversion is four lines: `Begin` / the unchanged xmlsec call on `shadow.root` / `End` — no per-function callback or context struct. +- **A fast path** decided once at import: when lxml links the same libxml2 as + the extension (the only configuration the import guard lets run today), + `Begin`/`End` skip the copy entirely and behave exactly like the old direct + code — zero overhead. The shadow round-trip activates under a mismatch, or + with `PYXMLSEC_FORCE_SHADOW=1`, which CI uses to keep the shadow path + exercised on matched libraries. - **An escape hatch** for development: `PYXMLSEC_SKIP_VERSION_CHECK` bypasses the import-time mismatch guard so the converted paths can be exercised under a real mismatch. The guard itself stays on by default. diff --git a/developer.md b/developer.md index 321dc4df..417cb787 100644 --- a/developer.md +++ b/developer.md @@ -59,6 +59,26 @@ call mutates at most one place in the tree — true for all `xmlSecTmpl*` functions. Whole-document operations (`sign`, `encrypt`) mutate many places and will need a different reflect step on top of the same Begin machinery. +## The fast path: shadows only when needed + +Copying is pointless when lxml links the same libxml2 as the extension — the +raw-node behavior that shipped for years is safe then, and it is the only +configuration the import guard currently lets run. So `Begin`/`End` are +dual-path, decided once at import: + +- **matched versions** (the guard passed): `Begin` aliases the live + `_c_node` into `shadow.root` with no serialization, and `End` just wraps + the node xmlsec returned — machine-identical to the pre-shadow code, zero + overhead; +- **mismatch** (import allowed via `PYXMLSEC_SKIP_VERSION_CHECK` today, + automatic once everything is converted), **or `PYXMLSEC_FORCE_SHADOW` set**: + the full shadow round-trip described above. + +Call sites cannot tell the difference; every converted function inherits both +paths. `PYXMLSEC_FORCE_SHADOW` exists so CI keeps the shadow path exercised on +matched libraries (see the test matrix), where it must also pass the full +suite. + > The shadow decouples *lxml* from xmlsec. The extension and `libxmlsec1` > must still share one libxml2 (wheels/static builds guarantee this). @@ -88,6 +108,10 @@ exists to exercise the shadow paths; it is **unsafe** for every operation still on the raw-node path, so keep it off in normal use. The guard can only be relaxed once all node-passing paths are converted. +On a *matched* build (no mismatch available), run the suite twice instead: +once plain (fast path) and once with `PYXMLSEC_FORCE_SHADOW=1` (shadow path +on matched libraries — safe everywhere, so the whole suite must pass). + ## Status - ✅ `template.add_reference`, `template.add_transform`, diff --git a/src/lxml.c b/src/lxml.c index 163dee72..b6021b66 100644 --- a/src/lxml.c +++ b/src/lxml.c @@ -101,6 +101,15 @@ static int PyXmlSec_CheckLxmlLibraryVersion(void) { return result; } +// Non-zero when converted functions must run their xmlsec call on a shadow +// copy instead of directly on lxml's nodes; decided once at import, below. +static int PyXmlSec_LxmlShadowActive = 1; + +// The two lxml.etree callables the shadow crossings use, resolved once at +// import and kept for the lifetime of the process. +static PyObject* PyXmlSec_LxmlEtreeToString; +static PyObject* PyXmlSec_LxmlEtreeFromString; + int PyXmlSec_InitLxmlModule(void) { // By default refuse to import when lxml and xmlsec link different libxml2 // versions: passing raw nodes between the two libraries then corrupts @@ -108,11 +117,30 @@ int PyXmlSec_InitLxmlModule(void) { // PYXMLSEC_SKIP_VERSION_CHECK bypasses the guard — needed to exercise the // shadow-copy paths (issue #356) under a mismatch, but unsafe for every // operation that still hands an lxml node to xmlsec. - if (PyXmlSec_CheckLxmlLibraryVersion() < 0 && getenv("PYXMLSEC_SKIP_VERSION_CHECK") == NULL) { + int mismatch = PyXmlSec_CheckLxmlLibraryVersion() < 0; + if (mismatch && getenv("PYXMLSEC_SKIP_VERSION_CHECK") == NULL) { PyXmlSec_SetLastError("lxml & xmlsec libxml2 library version mismatch"); return -1; } + // Matched versions are the long-standing status quo: xmlsec may mutate + // lxml's nodes directly, so converted functions skip the copy (the + // Begin/End fast path). Shadows turn on under a mismatch — or always with + // PYXMLSEC_FORCE_SHADOW, the knob CI uses to exercise the shadow path on + // matched libraries. + PyXmlSec_LxmlShadowActive = mismatch || getenv("PYXMLSEC_FORCE_SHADOW") != NULL; + + PyObject* etree = PyImport_ImportModule("lxml.etree"); + if (etree == NULL) { + return -1; + } + PyXmlSec_LxmlEtreeToString = PyObject_GetAttrString(etree, "tostring"); + PyXmlSec_LxmlEtreeFromString = PyObject_GetAttrString(etree, "fromstring"); + Py_DECREF(etree); + if (PyXmlSec_LxmlEtreeToString == NULL || PyXmlSec_LxmlEtreeFromString == NULL) { + return -1; + } + return import_lxml__etree(); } @@ -151,39 +179,19 @@ int PyXmlSec_LxmlElementConverter(PyObject* o, PyXmlSec_LxmlElementPtr* p) { // tree; with_tail keeps the serialization to the element itself. static PyObject* PyXmlSec_LxmlElementToBytes(PyObject* element) { PyObject* result = NULL; - PyObject* args = NULL; - PyObject* kwargs = NULL; - PyObject* tostring = NULL; - PyObject* etree = PyImport_ImportModule("lxml.etree"); - if (etree == NULL) { - return NULL; - } - tostring = PyObject_GetAttrString(etree, "tostring"); - Py_DECREF(etree); - if (tostring == NULL) { - return NULL; - } - args = PyTuple_Pack(1, element); - kwargs = Py_BuildValue("{s:O}", "with_tail", Py_False); + PyObject* args = PyTuple_Pack(1, element); + PyObject* kwargs = Py_BuildValue("{s:O}", "with_tail", Py_False); if (args != NULL && kwargs != NULL) { - result = PyObject_Call(tostring, args, kwargs); + result = PyObject_Call(PyXmlSec_LxmlEtreeToString, args, kwargs); } Py_XDECREF(args); Py_XDECREF(kwargs); - Py_DECREF(tostring); return result; } // etree.fromstring(data) — the parsed nodes are owned and managed by lxml. static PyObject* PyXmlSec_LxmlElementFromBytes(PyObject* data) { - PyObject* result; - PyObject* etree = PyImport_ImportModule("lxml.etree"); - if (etree == NULL) { - return NULL; - } - result = PyObject_CallMethod(etree, "fromstring", "O", data); - Py_DECREF(etree); - return result; + return PyObject_CallFunctionObjArgs(PyXmlSec_LxmlEtreeFromString, data, NULL); } // Nodes that exist before the xmlsec call are tagged through the libxml2 @@ -295,6 +303,14 @@ int PyXmlSec_LxmlShadowBegin(PyXmlSec_LxmlShadow* shadow, PyXmlSec_LxmlElementPt shadow->doc = NULL; shadow->root = NULL; + // Fast path: lxml links the same libxml2 (the import guard passed), so + // xmlsec can mutate lxml's nodes directly and no copy is needed; End sees + // doc == NULL and just wraps the result node. + if (!PyXmlSec_LxmlShadowActive) { + shadow->root = element->_c_node; + return 0; + } + bytes = PyXmlSec_LxmlElementToBytes((PyObject*)element); if (bytes == NULL || PyBytes_AsStringAndSize(bytes, &data, &size) < 0) { Py_XDECREF(bytes); @@ -332,6 +348,15 @@ PyObject* PyXmlSec_LxmlShadowEnd(PyXmlSec_LxmlShadow* shadow, xmlNodePtr res, co int rel[PYXMLSEC_SHADOW_MAX_DEPTH]; int depth, rel_depth, insert_idx; + // Fast path (no copy was made): res is a node in the live lxml tree. + if (shadow->doc == NULL) { + if (res == NULL) { + PyXmlSec_SetLastError(error); + return NULL; + } + return (PyObject*)PyXmlSec_elementFactory(shadow->element->_doc, res); + } + if (res == NULL) { PyXmlSec_SetLastError(error); goto DONE; diff --git a/src/lxml.h b/src/lxml.h index 9abeb906..a9609e66 100644 --- a/src/lxml.h +++ b/src/lxml.h @@ -52,6 +52,13 @@ int PyXmlSec_LxmlElementConverter(PyObject* o, PyXmlSec_LxmlElementPtr* p); // when the node is NULL). End must be called exactly once after a successful // Begin; it always releases the copy. // +// Fast path: when lxml links the same libxml2 as this extension (the +// import-time version check passed), no copy is needed — Begin aliases the +// live node into `root` (leaving `doc` NULL) and End just wraps the result, +// which is the long-standing direct behavior with zero overhead. Setting +// PYXMLSEC_FORCE_SHADOW in the environment forces the shadow path even on +// matched libraries; CI uses it to keep that path exercised. +// // The reflection covers the whole xmlSecTmpl* family: a new subtree grafted at // the position xmlsec chose (including intermediate nodes like // and the "\n" formatting text around it), or — for find-or-create calls that From 92d68ad3a595ce05e190003bc4e8e0bb875013fa Mon Sep 17 00:00:00 2001 From: Amin Solhizadeh Date: Tue, 7 Jul 2026 16:05:05 +0200 Subject: [PATCH 05/22] Convert every remaining raw-node crossing to shadow copies (#356) Roll the shadow pattern out beyond templates so no binding hands an lxml node to xmlsec anymore (fast path unchanged on matched libxml2): - template.c: all remaining functions, incl. the create shape (BeginNewDoc/EndNewDoc builds the detached template in a private doc) and the status-int C14N helper (FindFresh locates the created node). - tree.c: finders map results back by path (EndFind, None on not-found; find_parent shadows the whole tree); add_ids records id-attribute specs instead of writing lxml's ID hash with our libxml2. - ds.c: sign/verify run on a whole-document copy (BeginDoc) with the recorded IDs replayed so #id references resolve; sign reflects all mutation sites (DigestValue/SignatureValue/KeyInfo) via ReflectAll, verify just discards the copy. - enc.c: encrypt_binary/encrypt_uri reflect the mutated template; encrypt_xml/decrypt re-parse everything into one copy and reflect the replacement through lxml (element, content, or returned bytes). Replacing the document root cannot be expressed through lxml's API and raises a clear error on the shadow path. ReflectAll is two-phase (prefetch payloads from the re-parsed copy in its final state, then apply to the live tree in document order) because each graft moves a node out of the copy and would invalidate the indices later sites resolve through. Two template tests now attach the created template before asserting liveness: a shadow-created template lives in its own document until grafted, as lxml cannot express the raw path's "detached node inside an existing document". Validated under a real 2.14<->2.15 mismatch (full suite, 10k-iteration sign/verify/encrypt/decrypt loop, flat RSS, byte-identical output) and on a matched static wheel with and without PYXMLSEC_FORCE_SHADOW. Co-Authored-By: Claude Fable 5 --- converting-functions.md | 15 +- developer.md | 85 ++++- src/ds.c | 67 +++- src/enc.c | 290 +++++++++++++++- src/lxml.c | 745 +++++++++++++++++++++++++++++++++++++++- src/lxml.h | 79 ++++- src/template.c | 238 +++++++++---- src/tree.c | 77 ++++- tests/test_templates.py | 8 + 9 files changed, 1498 insertions(+), 106 deletions(-) diff --git a/converting-functions.md b/converting-functions.md index 6818e7db..a3906a3c 100644 --- a/converting-functions.md +++ b/converting-functions.md @@ -19,13 +19,18 @@ grep -n '_c_node\|_c_doc' src/template.c Read the `xmlSecTmpl*` function it wraps (xmlsec1's `src/templates.c`) and match it to a row: -| Shape | Examples | Covered? | +| Shape | Examples | How | |---|---|---| | Adds a subtree under the element (any position, possibly with intermediate nodes) | `add_key_name`, `add_key_value`, `add_x509_data`, `x509_data_add_*`, `add_encrypted_key` | ✅ just follow step 3 | -| Find-or-create, may set attributes on the existing node | `encrypted_data_ensure_key_info`, `encrypted_data_ensure_cipher_value` | ✅ just follow step 3 | -| Returns a status `int`, mutates one child | `transform_add_c14n_inclusive_namespaces` | ✅ with a twist: after the call, locate the affected child on the copy (here: the `` under `shadow.root`) and pass *that* to `End`; discard the returned element and return `None` | -| Returns a **detached** node — the element argument only supplies the document | `create`, `encrypted_data_create` | ❌ `End` would raise "unexpected result node"; needs its own reflect (no graft — parse the standalone result with lxml and return it) | -| Reads or mutates the **whole document** | `ds.c` sign/verify, `enc.c` encrypt/decrypt, `tree.c` | ❌ needs a whole-document reflect strategy | +| Find-or-create, may set attributes on the existing node | `encrypted_data_ensure_key_info`, `encrypted_data_ensure_cipher_value` | ✅ just follow step 3 (a call that can also *rename the prefix* of the existing node uses `EndReplace` instead of `End`) | +| Returns a status `int`, mutates one child | `transform_add_c14n_inclusive_namespaces` | ✅ pass `PyXmlSec_LxmlShadowFindFresh(&shadow)` to `End`; discard the returned element and return `None` | +| Returns a **detached** node — the element argument only supplies the document | `create`, `encrypted_data_create` | ✅ `BeginNewDoc`/`EndNewDoc`: the call runs against a private document and the result comes back as a new detached lxml element | +| Read-only search | `tree.c` find_child/find_node (subtree), find_parent (whole doc) | ✅ `Begin` (or `BeginDoc` when the search leaves the subtree) + `EndFind`, which returns `None` on not-found | +| Reads or mutates the **whole document**, possibly at several places | `ds.c` sign (whole-doc + multi-site), verify (read-only) | ✅ `BeginDoc` + `ReplayIds`, then `ReflectAll` (sign) or `Discard` (verify) | +| **Replaces** nodes | `enc.c` encrypt_xml/decrypt (`encrypt_binary`/`encrypt_uri` are template-shaped: `Begin` + `ReflectAll`) | ✅ `BeginDoc`, remove the consumed live node/content first, then `ReflectAll` grafts the replacement (`_setroot` when the document root itself was replaced) | + +All shapes are converted; `developer.md`'s "Beyond templates" section explains +each reflect. This guide stays as the recipe should new bindings appear. `res` does not have to be the topmost node the call created — `End` walks up to the topmost new ancestor itself. Any node inside the new subtree works. diff --git a/developer.md b/developer.md index 417cb787..77c2c7d7 100644 --- a/developer.md +++ b/developer.md @@ -56,8 +56,54 @@ Child indices count exactly the node types lxml exposes as children (elements, comments, PIs, entity refs), so paths recorded on the raw copy resolve identically through lxml's `__getitem__`/`insert`. `End` assumes the xmlsec call mutates at most one place in the tree — true for all `xmlSecTmpl*` -functions. Whole-document operations (`sign`, `encrypt`) mutate many places -and will need a different reflect step on top of the same Begin machinery. +functions. + +## Beyond templates: the whole-code rollout + +Every binding that used to hand a raw lxml node to xmlsec now goes through a +shadow. The extra shapes (all in [src/lxml.c](src/lxml.c), contracts in +[src/lxml.h](src/lxml.h)): + +- **Create** (`template.create`, `encrypted_data_create`): + `BeginNewDoc`/`EndNewDoc`. The call builds a *detached* subtree and only + needs a document to allocate in — a private one on the shadow path. The + result comes back as a new detached lxml element (in its own document; lxml + moves it when the caller grafts it), instead of the raw path's "detached + node inside the source document", which lxml's API cannot express. +- **Finders** (`tree.find_child`/`find_node`/`find_parent`): `EndFind` maps + the found copy node back by path and returns `None` on not-found. + `find_parent` walks upward, so it uses the whole-document Begin. +- **Whole-document** (`sign`, `verify`, `decrypt`, `find_parent`): + `BeginDoc` serializes `element.getroottree()` (comments/PIs outside the + root and the internal DTD subset survive), records the element's position + through lxml's API, and hands back the copy's counterpart node. +- **Multi-site reflect** (`sign`, `encrypt_binary`, `encrypt_uri`): + `ReflectAll` scans the copy for *every* topmost untagged node and grafts + each back — new subtrees via `insert`, new/changed text (DigestValue, + SignatureValue) via the text slots. Two-phase: payloads are fetched from + the re-parsed copy while it is still in its final state, then applied to + the live tree in document order (a graft moves a node out of the re-parsed + copy, which would invalidate later fetches). +- **Replacement reflect** (`encrypt_xml`, `decrypt`): encryption/decryption + *replace* nodes, so the live target (or its content, or the document root + via `_setroot`) is removed first and `ReflectAll` grafts what took its + place. `encrypt_xml` re-serializes the template into the same shadow doc + (`ImportElement`); a template attached inside the target's own tree is + therefore copied, not moved. `verify` needs no reflect at all — `Discard` + just frees the copy. +- **ID registration** (`tree.add_ids`, `SignatureContext.register_id`): these + used to write lxml's ID hash with our libxml2. Under the shadow they record + the id-attribute specs in a registry keyed by document identity + (`RecordId`), and every `BeginDoc` replays them onto the copy + (`ReplayIds`) so `#id` references resolve. The replay scans the whole copy + for the recorded attribute names — a superset of the raw registration. The + registry holds no strong references to documents (lxml objects refuse weak + references, so entries are validated by a stored `_c_doc` address and + capped in size). +- **Prefix rename** (`encrypted_data_ensure_key_info(ns=...)` on an existing + KeyInfo): `EndReplace` swaps the live element for the copy's version, since + lxml cannot rename a prefix in place; the returned element is then a new + object rather than the original proxy. ## The fast path: shadows only when needed @@ -114,11 +160,30 @@ on matched libraries — safe everywhere, so the whole suite must pass). ## Status -- ✅ `template.add_reference`, `template.add_transform`, - `template.ensure_key_info` — one per reflect shape. Validated under a real - 2.14 ↔ 2.15 mismatch: full suite green, 10k-iteration loop with no crash, - no leak, byte-identical output. -- ⬜ Rest of `src/template.c` — mechanical: follow the step-by-step guide in - [converting-functions.md](converting-functions.md). -- ⬜ `src/ds.c` (sign/verify), `src/enc.c` (encrypt/decrypt), `src/tree.c` — - need a whole-document reflect strategy. +- ✅ All of `src/template.c` (create, references, transforms, key info, x509, + encrypted data, C14N namespaces). +- ✅ `src/tree.c` (find_child/find_node/find_parent, add_ids). +- ✅ `src/ds.c` (register_id, sign, verify; the binary operations never + touched nodes). +- ✅ `src/enc.c` (encrypt_binary, encrypt_uri, encrypt_xml, decrypt). +- Validated under a real 2.14 ↔ 2.15 mismatch: full suite green, 10k-iteration + sign/verify/encrypt/decrypt loop with no crash, no leak, byte-identical + output; and on a matched static build: full suite green on both the fast + path and `PYXMLSEC_FORCE_SHADOW=1`. +- ⬜ Endgame: turn the import-time guard into a mode switch (mismatch sets + the shadow flag instead of refusing to import) and retire + `PYXMLSEC_SKIP_VERSION_CHECK`. That is the actual user-facing resolution + of #356, kept as its own change. + +Known, deliberate divergences on the shadow path (all invisible to the +documented API): created templates live in their own document until grafted; +`encrypted_data_ensure_key_info(ns=...)` on an existing KeyInfo returns a new +element object; `register_id` skips the live duplicate-id check (it runs per +copy instead); `encrypt_xml` copies rather than moves a template that is +attached inside the target tree; signature/encryption contexts keep no live +result nodes after the call (they never usefully did). One hard limitation: +operations that would replace the **document root** (encrypting the root +element, decrypting a root `EncryptedData`) raise `xmlsec.Error` — lxml's API +cannot swap a document's root, and morphing it in place would rewrite +namespace prefixes, breaking signatures over the content. Re-parse the +document or work on a subelement instead. diff --git a/src/ds.c b/src/ds.c index d0b4bdf9..73299342 100644 --- a/src/ds.c +++ b/src/ds.c @@ -146,6 +146,39 @@ static PyObject* PyXmlSec_SignatureContextRegisterId(PyObject* self, PyObject* a goto ON_FAIL; } + // Shadow mode: never touch lxml's document (its ID hash) with our libxml2 + // (issue #356). Validate through lxml's API and record the spec; the + // whole-document shadows (sign/verify/decrypt) replay it onto their + // private copies. The duplicate-id check runs there, per copy. + if (PyXmlSec_LxmlShadowIsActive()) { + PyObject* key; + PyObject* value; + if (id_ns != NULL) { + key = PyUnicode_FromFormat("{%s}%s", id_ns, id_attr); + } else { + key = PyUnicode_FromString(id_attr); + } + if (key == NULL) { + goto ON_FAIL; + } + value = PyObject_CallMethod((PyObject*)node, "get", "O", key); + Py_DECREF(key); + if (value == NULL) { + goto ON_FAIL; + } + if (value == Py_None) { + Py_DECREF(value); + PyErr_SetString(PyXmlSec_Error, "missing attribute."); + goto ON_FAIL; + } + Py_DECREF(value); + if (PyXmlSec_LxmlShadowRecordId(node, id_attr, id_ns) < 0) { + goto ON_FAIL; + } + PYXMLSEC_DEBUGF("%p: register id - ok", self); + Py_RETURN_NONE; + } + if (id_ns != NULL) { attr = xmlHasNsProp(node->_c_node, XSTR(id_attr), XSTR(id_ns)); } else { @@ -189,6 +222,8 @@ static PyObject* PyXmlSec_SignatureContextSign(PyObject* self, PyObject* args, P PyXmlSec_SignatureContext* ctx = (PyXmlSec_SignatureContext*)self; PyXmlSec_LxmlElementPtr node = NULL; + xmlNodePtr target; + PyXmlSec_LxmlShadow shadow; int rv; PYXMLSEC_DEBUGF("%p: sign - start", self); @@ -196,14 +231,30 @@ static PyObject* PyXmlSec_SignatureContextSign(PyObject* self, PyObject* args, P goto ON_FAIL; } + // References (URI="", "#id") reach anywhere in the document, so the + // shadow covers the whole tree (issue #356); registered IDs are replayed + // onto the copy so they resolve. Signing fills several places inside + // (DigestValue, SignatureValue, KeyInfo), all reflected by + // the multi-site reflection. + if (PyXmlSec_LxmlShadowBeginDoc(&shadow, node, &target) < 0) { + goto ON_FAIL; + } + if (PyXmlSec_LxmlShadowReplayIds(&shadow) < 0) { + PyXmlSec_LxmlShadowDiscard(&shadow); + goto ON_FAIL; + } Py_BEGIN_ALLOW_THREADS; - rv = xmlSecDSigCtxSign(ctx->handle, node->_c_node); + rv = xmlSecDSigCtxSign(ctx->handle, target); PYXMLSEC_DUMP(xmlSecDSigCtxDebugDump, ctx->handle); Py_END_ALLOW_THREADS; if (rv < 0) { + PyXmlSec_LxmlShadowDiscard(&shadow); PyXmlSec_SetLastError("failed to sign"); goto ON_FAIL; } + if (PyXmlSec_LxmlShadowReflectAll(&shadow) < 0) { + goto ON_FAIL; + } PYXMLSEC_DEBUGF("%p: sign - ok", self); Py_RETURN_NONE; @@ -224,6 +275,8 @@ static PyObject* PyXmlSec_SignatureContextVerify(PyObject* self, PyObject* args, PyXmlSec_SignatureContext* ctx = (PyXmlSec_SignatureContext*)self; PyXmlSec_LxmlElementPtr node = NULL; + xmlNodePtr target; + PyXmlSec_LxmlShadow shadow; int rv; PYXMLSEC_DEBUGF("%p: verify - start", self); @@ -231,10 +284,20 @@ static PyObject* PyXmlSec_SignatureContextVerify(PyObject* self, PyObject* args, goto ON_FAIL; } + // Verification is read-only: whole-document shadow, replayed IDs, and no + // reflection at all — the copy is simply discarded. + if (PyXmlSec_LxmlShadowBeginDoc(&shadow, node, &target) < 0) { + goto ON_FAIL; + } + if (PyXmlSec_LxmlShadowReplayIds(&shadow) < 0) { + PyXmlSec_LxmlShadowDiscard(&shadow); + goto ON_FAIL; + } Py_BEGIN_ALLOW_THREADS; - rv = xmlSecDSigCtxVerify(ctx->handle, node->_c_node); + rv = xmlSecDSigCtxVerify(ctx->handle, target); PYXMLSEC_DUMP(xmlSecDSigCtxDebugDump, ctx->handle); Py_END_ALLOW_THREADS; + PyXmlSec_LxmlShadowDiscard(&shadow); if (rv < 0) { PyXmlSec_SetLastError("failed to verify"); diff --git a/src/enc.c b/src/enc.c index 42195dd3..4a0128ef 100644 --- a/src/enc.c +++ b/src/enc.c @@ -166,6 +166,7 @@ static PyObject* PyXmlSec_EncryptionContextEncryptBinary(PyObject* self, PyObjec PyXmlSec_LxmlElementPtr template = NULL; const char* data = NULL; Py_ssize_t data_size = 0; + PyXmlSec_LxmlShadow shadow; int rv; PYXMLSEC_DEBUGF("%p: encrypt_binary - start", self); @@ -175,15 +176,25 @@ static PyObject* PyXmlSec_EncryptionContextEncryptBinary(PyObject* self, PyObjec goto ON_FAIL; } + // The encryption fills several places inside the template subtree + // (CipherValue, KeyInfo/EncryptedKey); the multi-site reflection carries + // them all back (issue #356). + if (PyXmlSec_LxmlShadowBegin(&shadow, template) < 0) { + goto ON_FAIL; + } Py_BEGIN_ALLOW_THREADS; - rv = xmlSecEncCtxBinaryEncrypt(ctx->handle, template->_c_node, (const xmlSecByte*)data, (xmlSecSize)data_size); + rv = xmlSecEncCtxBinaryEncrypt(ctx->handle, shadow.root, (const xmlSecByte*)data, (xmlSecSize)data_size); PYXMLSEC_DUMP(xmlSecEncCtxDebugDump, ctx->handle); Py_END_ALLOW_THREADS; if (rv < 0) { + PyXmlSec_LxmlShadowDiscard(&shadow); PyXmlSec_SetLastError("failed to encrypt binary"); goto ON_FAIL; } + if (PyXmlSec_LxmlShadowReflectAll(&shadow) < 0) { + goto ON_FAIL; + } Py_INCREF(template); PYXMLSEC_DEBUGF("%p: encrypt_binary - ok", self); @@ -214,6 +225,144 @@ static void PyXmlSec_ClearReplacedNodes(xmlSecEncCtxPtr ctx, PyXmlSec_LxmlDocume ctx->replacedNodeList = NULL; } +// Shadow-path body of encrypt_xml (issue #356): the target document and the +// template are both re-parsed into one private copy, the encryption runs +// there, and the replacement is reflected back through lxml — the fresh +// takes the target's place (`Type=Element`) or its content +// (`Type=Content`). One divergence from the raw path: a template that is +// *attached* inside the target's own tree is copied, not moved, so it also +// remains at its original position. +static PyObject* PyXmlSec_EncryptionContextEncryptXmlShadow(PyXmlSec_EncryptionContext* ctx, PyXmlSec_LxmlElementPtr template, PyXmlSec_LxmlElementPtr node) { + PyXmlSec_LxmlShadow shadow; + xmlNodePtr target = NULL; + xmlNodePtr tmpl_copy; + PyObject* type_value = NULL; + PyObject* parent = NULL; + PyObject* result = NULL; + PyObject* tmp = NULL; + const char* type_str; + long idx = -1; + int is_content = 0; + int rv; + + type_value = PyObject_CallMethod((PyObject*)template, "get", "s", "Type"); + if (type_value == NULL) { + return NULL; + } + type_str = type_value == Py_None ? NULL : PyUnicode_AsUTF8(type_value); + if (type_str == NULL || !(strcmp(type_str, (const char*)xmlSecTypeEncElement) == 0 + || strcmp(type_str, (const char*)xmlSecTypeEncContent) == 0)) { + PyErr_SetString(PyXmlSec_Error, "unsupported `Type`, it should be `element` or `content`"); + goto ON_FAIL; + } + is_content = strcmp(type_str, (const char*)xmlSecTypeEncContent) == 0; + + parent = PyObject_CallMethod((PyObject*)node, "getparent", NULL); + if (parent == NULL) { + goto ON_FAIL; + } + if (parent != Py_None) { + tmp = PyObject_CallMethod(parent, "index", "O", node); + if (tmp == NULL) { + goto ON_FAIL; + } + idx = PyLong_AsLong(tmp); + Py_CLEAR(tmp); + if (idx < 0) { + goto ON_FAIL; + } + } + + if (PyXmlSec_LxmlShadowBeginDoc(&shadow, node, &target) < 0) { + goto ON_FAIL; + } + tmpl_copy = PyXmlSec_LxmlShadowImportElement(&shadow, template); + if (tmpl_copy == NULL) { + PyXmlSec_LxmlShadowDiscard(&shadow); + goto ON_FAIL; + } + + Py_BEGIN_ALLOW_THREADS; + rv = xmlSecEncCtxXmlEncrypt(ctx->handle, tmpl_copy, target); + PYXMLSEC_DUMP(xmlSecEncCtxDebugDump, ctx->handle); + Py_END_ALLOW_THREADS; + + if (rv < 0) { + // still detached means the encryption never consumed our template copy + if (tmpl_copy->parent == NULL) { + xmlFreeNode(tmpl_copy); + } + PyXmlSec_LxmlShadowDiscard(&shadow); + PyXmlSec_SetLastError("failed to encrypt xml"); + goto ON_FAIL; + } + + if (parent == Py_None && !is_content) { + // The encryption replaced the copy's root, which cannot be reflected: + // lxml's API offers no way to swap a document's root element (and + // morphing it in place would rewrite namespace prefixes, breaking + // signatures over the content). The live tree is untouched. Encrypt a + // subelement instead, or re-parse the document. + PyXmlSec_LxmlShadowDiscard(&shadow); + PyErr_SetString(PyXmlSec_Error, + "encrypting the document root is not supported when lxml and xmlsec use different libxml2 libraries"); + goto ON_FAIL; + } + + if (is_content) { + // the node stays; its old content was consumed and replaced by the + // fresh , which the reflection grafts back in + Py_ssize_t len = PyObject_Length((PyObject*)node); + if (len < 0) { + PyXmlSec_LxmlShadowDiscard(&shadow); + goto ON_FAIL; + } + while (len-- > 0) { + PyObject* child = PySequence_GetItem((PyObject*)node, 0); + tmp = child != NULL ? PyObject_CallMethod((PyObject*)node, "remove", "O", child) : NULL; + Py_XDECREF(child); + if (tmp == NULL) { + PyXmlSec_LxmlShadowDiscard(&shadow); + goto ON_FAIL; + } + Py_CLEAR(tmp); + } + if (PyXmlSec_LxmlShadowReflectAll(&shadow) < 0) { + goto ON_FAIL; + } + result = PySequence_GetItem((PyObject*)node, 0); + if (result == NULL) { + goto ON_FAIL; + } + } else { + // Type=Element: the node itself was consumed and replaced + tmp = PyObject_CallMethod(parent, "remove", "O", node); + if (tmp == NULL) { + PyXmlSec_LxmlShadowDiscard(&shadow); + goto ON_FAIL; + } + Py_CLEAR(tmp); + if (PyXmlSec_LxmlShadowReflectAll(&shadow) < 0) { + goto ON_FAIL; + } + result = PySequence_GetItem(parent, (Py_ssize_t)idx); + if (result == NULL) { + goto ON_FAIL; + } + } + + Py_DECREF(type_value); + Py_XDECREF(parent); + return result; + +ON_FAIL: + Py_XDECREF(type_value); + Py_XDECREF(parent); + Py_XDECREF(tmp); + Py_XDECREF(result); + return NULL; +} + static const char PyXmlSec_EncryptionContextEncryptXml__doc__[] = \ "encrypt_xml(template, node) -> lxml.etree._Element\n" "Encrypts ``node`` using ``template``.\n\n" @@ -243,6 +392,16 @@ static PyObject* PyXmlSec_EncryptionContextEncryptXml(PyObject* self, PyObject* { goto ON_FAIL; } + + if (PyXmlSec_LxmlShadowIsActive()) { + PyObject* result = PyXmlSec_EncryptionContextEncryptXmlShadow(ctx, template, node); + if (result == NULL) { + goto ON_FAIL; + } + PYXMLSEC_DEBUGF("%p: encrypt_xml - ok", self); + return result; + } + tmpType = xmlGetProp(template->_c_node, XSTR("Type")); if (tmpType == NULL || !(xmlStrEqual(tmpType, xmlSecTypeEncElement) || xmlStrEqual(tmpType, xmlSecTypeEncContent))) { PyErr_SetString(PyXmlSec_Error, "unsupported `Type`, it should be `element` or `content`"); @@ -316,6 +475,7 @@ static PyObject* PyXmlSec_EncryptionContextEncryptUri(PyObject* self, PyObject* PyXmlSec_EncryptionContext* ctx = (PyXmlSec_EncryptionContext*)self; PyXmlSec_LxmlElementPtr template = NULL; const char* uri = NULL; + PyXmlSec_LxmlShadow shadow; int rv; PYXMLSEC_DEBUGF("%p: encrypt_uri - start", self); @@ -323,15 +483,22 @@ static PyObject* PyXmlSec_EncryptionContextEncryptUri(PyObject* self, PyObject* goto ON_FAIL; } + if (PyXmlSec_LxmlShadowBegin(&shadow, template) < 0) { + goto ON_FAIL; + } Py_BEGIN_ALLOW_THREADS; - rv = xmlSecEncCtxUriEncrypt(ctx->handle, template->_c_node, (const xmlSecByte*)uri); + rv = xmlSecEncCtxUriEncrypt(ctx->handle, shadow.root, (const xmlSecByte*)uri); PYXMLSEC_DUMP(xmlSecEncCtxDebugDump, ctx->handle); Py_END_ALLOW_THREADS; if (rv < 0) { + PyXmlSec_LxmlShadowDiscard(&shadow); PyXmlSec_SetLastError("failed to encrypt URI"); goto ON_FAIL; } + if (PyXmlSec_LxmlShadowReflectAll(&shadow) < 0) { + goto ON_FAIL; + } PYXMLSEC_DEBUGF("%p: encrypt_uri - ok", self); Py_INCREF(template); return (PyObject*)template; @@ -340,6 +507,116 @@ static PyObject* PyXmlSec_EncryptionContextEncryptUri(PyObject* self, PyObject* return NULL; } +// Shadow-path body of decrypt (issue #356): whole-document copy (with the +// registered IDs replayed, for RetrievalMethod references), decryption on the +// copy, then a replacement reflect — the decrypted subtree or content takes +// the 's place in the live tree. Binary results need no +// reflection at all. +static PyObject* PyXmlSec_EncryptionContextDecryptShadow(PyXmlSec_EncryptionContext* ctx, PyXmlSec_LxmlElementPtr node) { + PyXmlSec_LxmlShadow shadow; + xmlNodePtr target = NULL; + PyObject* parent = NULL; + PyObject* result = NULL; + PyObject* tmp = NULL; + long idx = -1; + xmlChar* ttype; + int not_content; + int rv; + + parent = PyObject_CallMethod((PyObject*)node, "getparent", NULL); + if (parent == NULL) { + return NULL; + } + if (parent != Py_None) { + tmp = PyObject_CallMethod(parent, "index", "O", node); + if (tmp == NULL) { + goto ON_FAIL; + } + idx = PyLong_AsLong(tmp); + Py_CLEAR(tmp); + if (idx < 0) { + goto ON_FAIL; + } + } + + if (PyXmlSec_LxmlShadowBeginDoc(&shadow, node, &target) < 0) { + goto ON_FAIL; + } + if (PyXmlSec_LxmlShadowReplayIds(&shadow) < 0) { + PyXmlSec_LxmlShadowDiscard(&shadow); + goto ON_FAIL; + } + + // the Type decides the reflect shape; read it from the copy before the + // decryption consumes the node + ttype = xmlGetProp(target, XSTR("Type")); + not_content = (ttype == NULL || !xmlStrEqual(ttype, xmlSecTypeEncContent)); + xmlFree(ttype); + + Py_BEGIN_ALLOW_THREADS; + ctx->handle->mode = xmlSecCheckNodeName(target, xmlSecNodeEncryptedKey, xmlSecEncNs) ? xmlEncCtxModeEncryptedKey : xmlEncCtxModeEncryptedData; + PYXMLSEC_DEBUGF("mode: %d", ctx->handle->mode); + rv = xmlSecEncCtxDecrypt(ctx->handle, target); + PYXMLSEC_DUMP(xmlSecEncCtxDebugDump, ctx->handle); + Py_END_ALLOW_THREADS; + + if (rv < 0) { + PyXmlSec_LxmlShadowDiscard(&shadow); + PyXmlSec_SetLastError("failed to decrypt"); + goto ON_FAIL; + } + + if (!ctx->handle->resultReplaced) { + PyXmlSec_LxmlShadowDiscard(&shadow); + Py_DECREF(parent); + PYXMLSEC_DEBUGF("%p: binary.decrypt - ok", ctx); + return PyBytes_FromStringAndSize( + (const char*)xmlSecBufferGetData(ctx->handle->result), + (Py_ssize_t)xmlSecBufferGetSize(ctx->handle->result) + ); + } + + if (parent == Py_None) { + // Decryption replaced the document root; lxml's API offers no way to + // swap a document's root element, so this cannot be reflected (the + // live tree is untouched). Re-parse the document into a wrapper or + // decrypt a non-root EncryptedData instead. + PyXmlSec_LxmlShadowDiscard(&shadow); + PyErr_SetString(PyXmlSec_Error, + "decrypting the document root is not supported when lxml and xmlsec use different libxml2 libraries"); + goto ON_FAIL; + } else { + // the node was consumed; the reflection grafts whatever replaced it + tmp = PyObject_CallMethod(parent, "remove", "O", node); + if (tmp == NULL) { + PyXmlSec_LxmlShadowDiscard(&shadow); + goto ON_FAIL; + } + Py_CLEAR(tmp); + if (PyXmlSec_LxmlShadowReflectAll(&shadow) < 0) { + goto ON_FAIL; + } + if (not_content) { + result = PySequence_GetItem(parent, (Py_ssize_t)idx); + if (result == NULL) { + goto ON_FAIL; + } + } else { + result = parent; + Py_INCREF(result); + } + } + + Py_DECREF(parent); + return result; + +ON_FAIL: + Py_XDECREF(parent); + Py_XDECREF(tmp); + Py_XDECREF(result); + return NULL; +} + static const char PyXmlSec_EncryptionContextDecrypt__doc__[] = \ "decrypt(node)\n" "Decrypts ``node`` (an ``EncryptedData`` or ``EncryptedKey`` element) and returns the result. " @@ -372,6 +649,15 @@ static PyObject* PyXmlSec_EncryptionContextDecrypt(PyObject* self, PyObject* arg goto ON_FAIL; } + if (PyXmlSec_LxmlShadowIsActive()) { + PyObject* result = PyXmlSec_EncryptionContextDecryptShadow(ctx, node); + if (result == NULL) { + goto ON_FAIL; + } + PYXMLSEC_DEBUGF("%p: decrypt - ok", self); + return result; + } + xparent = node->_c_node->parent; if (xparent != NULL && !PyXmlSec_IsElement(xparent)) { xparent = NULL; diff --git a/src/lxml.c b/src/lxml.c index b6021b66..e266a017 100644 --- a/src/lxml.c +++ b/src/lxml.c @@ -110,6 +110,14 @@ static int PyXmlSec_LxmlShadowActive = 1; static PyObject* PyXmlSec_LxmlEtreeToString; static PyObject* PyXmlSec_LxmlEtreeFromString; +// Shadow-mode ID registry: maps the identity of an lxml document to the list +// of id-attribute specs registered for it (see PyXmlSec_LxmlShadowRecordId). +static PyObject* PyXmlSec_LxmlShadowIdRegistry; + +int PyXmlSec_LxmlShadowIsActive(void) { + return PyXmlSec_LxmlShadowActive; +} + int PyXmlSec_InitLxmlModule(void) { // By default refuse to import when lxml and xmlsec link different libxml2 // versions: passing raw nodes between the two libraries then corrupts @@ -141,6 +149,11 @@ int PyXmlSec_InitLxmlModule(void) { return -1; } + PyXmlSec_LxmlShadowIdRegistry = PyDict_New(); + if (PyXmlSec_LxmlShadowIdRegistry == NULL) { + return -1; + } + return import_lxml__etree(); } @@ -209,9 +222,10 @@ static void PyXmlSec_LxmlShadowMark(xmlNodePtr node) { } } -// dsig/enc structures never nest anywhere near this deep; the bound just keeps -// the path buffers on the stack and guards against a pathological tree. -#define PYXMLSEC_SHADOW_MAX_DEPTH 64 +// Paths now span whole user documents (BeginDoc), not just dsig/enc +// structures; the bound keeps the path buffers on the stack and guards +// against a pathological tree. +#define PYXMLSEC_SHADOW_MAX_DEPTH 128 // Index of node among its preceding siblings, counting only the node types // lxml exposes as children (elements, comments, PIs, entity refs), so indices @@ -300,6 +314,7 @@ int PyXmlSec_LxmlShadowBegin(PyXmlSec_LxmlShadow* shadow, PyXmlSec_LxmlElementPt Py_ssize_t size = 0; shadow->element = element; + shadow->owned = NULL; shadow->doc = NULL; shadow->root = NULL; @@ -455,19 +470,743 @@ PyObject* PyXmlSec_LxmlShadowEnd(PyXmlSec_LxmlShadow* shadow, xmlNodePtr res, co result = PyXmlSec_LxmlShadowWalk(new_node, rel, rel_depth); DONE: + PyXmlSec_LxmlShadowDiscard(shadow); + if (dump != NULL) { + xmlFree(dump); + } + Py_XDECREF(bytes); + Py_XDECREF(copy_root); + Py_XDECREF(copy_parent); + Py_XDECREF(live_parent); + Py_XDECREF(new_node); + Py_XDECREF(tmp); + return result; +} + +void PyXmlSec_LxmlShadowDiscard(PyXmlSec_LxmlShadow* shadow) { if (shadow->doc != NULL) { xmlFreeDoc(shadow->doc); shadow->doc = NULL; shadow->root = NULL; } + Py_CLEAR(shadow->owned); +} + +// ---------------------------------------------------------------------------- +// Doc-level shadows and multi-site reflection — the rollout of the shadow +// pattern beyond templates (sign/verify, encrypt/decrypt, tree finders). +// Contracts in lxml.h. +// ---------------------------------------------------------------------------- + +// Walks `path` (child indices, top-first) down from `start` on a raw copy, +// counting children exactly like PyXmlSec_LxmlShadowChildIndex. +static xmlNodePtr PyXmlSec_LxmlShadowWalkNode(xmlNodePtr start, const int* path, int depth) { + int i; + xmlNodePtr n = start; + for (i = 0; i < depth; ++i) { + int idx = path[i]; + xmlNodePtr c; + for (c = n->children; c != NULL; c = c->next) { + if (_isElement(c) && idx-- == 0) { + break; + } + } + if (c == NULL) { + return NULL; + } + n = c; + } + return n; +} + +int PyXmlSec_LxmlShadowBeginDoc(PyXmlSec_LxmlShadow* shadow, PyXmlSec_LxmlElementPtr element, xmlNodePtr* target) { + PyObject* cur = NULL; + PyObject* tree = NULL; + PyObject* bytes = NULL; + char* data = NULL; + Py_ssize_t size = 0; + int path[PYXMLSEC_SHADOW_MAX_DEPTH]; + int depth = 0; + int i; + + shadow->element = element; + shadow->owned = NULL; + shadow->doc = NULL; + shadow->root = NULL; + *target = NULL; + + if (!PyXmlSec_LxmlShadowActive) { + shadow->root = element->_c_node; + *target = element->_c_node; + return 0; + } + + // Record the element's position in its tree through lxml's own API (the + // shadow path never walks lxml's raw nodes), ascending to the top; `cur` + // ends as the live root element and `path` (reversed below) leads back + // down to `element`. + cur = (PyObject*)element; + Py_INCREF(cur); + for (;;) { + PyObject* parent = PyObject_CallMethod(cur, "getparent", NULL); + PyObject* index; + long idx; + if (parent == NULL) { + goto ON_FAIL; + } + if (parent == Py_None) { + Py_DECREF(parent); + break; + } + index = PyObject_CallMethod(parent, "index", "O", cur); + if (index == NULL) { + Py_DECREF(parent); + goto ON_FAIL; + } + idx = PyLong_AsLong(index); + Py_DECREF(index); + if (idx < 0 || depth >= PYXMLSEC_SHADOW_MAX_DEPTH) { + Py_DECREF(parent); + if (!PyErr_Occurred()) { + PyErr_SetString(PyXmlSec_InternalError, "the document is nested too deeply."); + } + goto ON_FAIL; + } + path[depth++] = (int)idx; + Py_DECREF(cur); + cur = parent; + } + for (i = 0; i < depth / 2; ++i) { // collected bottom-up; reverse + int t = path[i]; + path[i] = path[depth - 1 - i]; + path[depth - 1 - i] = t; + } + + // Serialize the whole tree, not just the root element, so comments/PIs + // outside the root and the internal DTD subset (declared IDs) survive + // into the copy. + tree = PyObject_CallMethod(cur, "getroottree", NULL); + if (tree == NULL) { + goto ON_FAIL; + } + bytes = PyObject_CallFunctionObjArgs(PyXmlSec_LxmlEtreeToString, tree, NULL); + Py_CLEAR(tree); + if (bytes == NULL || PyBytes_AsStringAndSize(bytes, &data, &size) < 0) { + goto ON_FAIL; + } + shadow->doc = xmlReadMemory(data, (int)size, NULL, NULL, XML_PARSE_NONET); + Py_CLEAR(bytes); + if (shadow->doc == NULL || (shadow->root = xmlDocGetRootElement(shadow->doc)) == NULL) { + PyErr_SetString(PyXmlSec_InternalError, "cannot make a private copy of the document."); + goto ON_FAIL; + } + PyXmlSec_LxmlShadowMark(shadow->doc->children); + + *target = PyXmlSec_LxmlShadowWalkNode(shadow->root, path, depth); + if (*target == NULL) { + PyErr_SetString(PyXmlSec_InternalError, "cannot locate the element in the private copy."); + goto ON_FAIL; + } + + // The reflect helpers map paths from the copy root, so the live element + // they start from must be the live root. + shadow->element = (PyXmlSec_LxmlElementPtr)cur; + shadow->owned = cur; + return 0; + +ON_FAIL: + Py_XDECREF(cur); + Py_XDECREF(tree); + Py_XDECREF(bytes); + if (shadow->doc != NULL) { + xmlFreeDoc(shadow->doc); + shadow->doc = NULL; + shadow->root = NULL; + } + return -1; +} + +xmlDocPtr PyXmlSec_LxmlShadowBeginNewDoc(PyXmlSec_LxmlShadow* shadow, PyXmlSec_LxmlElementPtr element) { + shadow->element = element; + shadow->owned = NULL; + shadow->doc = NULL; + shadow->root = NULL; + + // Fast path: allocate the detached subtree straight in the element's own + // document, as the raw code always did. + if (!PyXmlSec_LxmlShadowActive) { + return element->_doc->_c_doc; + } + shadow->doc = xmlNewDoc((const xmlChar*)"1.0"); + if (shadow->doc == NULL) { + PyErr_SetString(PyXmlSec_InternalError, "cannot create a private document."); + return NULL; + } + return shadow->doc; +} + +PyObject* PyXmlSec_LxmlShadowEndNewDoc(PyXmlSec_LxmlShadow* shadow, xmlNodePtr res, const char* error) { + PyObject* result = NULL; + PyObject* bytes = NULL; + xmlChar* dump = NULL; + int dump_size = 0; + + if (shadow->doc == NULL) { // fast path + if (res == NULL) { + PyXmlSec_SetLastError(error); + return NULL; + } + return (PyObject*)PyXmlSec_elementFactory(shadow->element->_doc, res); + } + + if (res == NULL) { + PyXmlSec_SetLastError(error); + goto DONE; + } + // The call left `res` detached inside our private doc; root it there so + // the whole subtree serializes (and gets freed with the doc), then hand + // the bytes to lxml. The result is a new detached element — lxml gives it + // a document of its own, and moves it when the caller grafts it into a + // tree, just like the raw code's detached node. + xmlDocSetRootElement(shadow->doc, res); + xmlDocDumpMemory(shadow->doc, &dump, &dump_size); + if (dump == NULL || dump_size <= 0) { + PyErr_SetString(PyXmlSec_InternalError, "cannot serialize the created node."); + goto DONE; + } + bytes = PyBytes_FromStringAndSize((const char*)dump, (Py_ssize_t)dump_size); + if (bytes != NULL) { + result = PyXmlSec_LxmlElementFromBytes(bytes); + } + +DONE: + PyXmlSec_LxmlShadowDiscard(shadow); if (dump != NULL) { xmlFree(dump); } Py_XDECREF(bytes); + return result; +} + +PyObject* PyXmlSec_LxmlShadowEndFind(PyXmlSec_LxmlShadow* shadow, xmlNodePtr res) { + PyObject* result; + int path[PYXMLSEC_SHADOW_MAX_DEPTH]; + int depth; + + if (shadow->doc == NULL) { // fast path: res is a live node + if (res == NULL) { + Py_RETURN_NONE; + } + return (PyObject*)PyXmlSec_elementFactory(shadow->element->_doc, res); + } + if (res == NULL) { + PyXmlSec_LxmlShadowDiscard(shadow); + Py_RETURN_NONE; + } + depth = PyXmlSec_LxmlShadowPathTo(res, shadow->root, path); + if (depth < 0) { + PyErr_SetString(PyXmlSec_InternalError, "unexpected result node."); + PyXmlSec_LxmlShadowDiscard(shadow); + return NULL; + } + result = PyXmlSec_LxmlShadowWalk((PyObject*)shadow->element, path, depth); + PyXmlSec_LxmlShadowDiscard(shadow); + return result; +} + +PyObject* PyXmlSec_LxmlShadowEndReplace(PyXmlSec_LxmlShadow* shadow, xmlNodePtr res, const char* error) { + PyObject* copy_root = NULL; + PyObject* copy_parent = NULL; + PyObject* live_parent = NULL; + PyObject* live_old = NULL; + PyObject* new_node = NULL; + PyObject* tmp = NULL; + PyObject* result = NULL; + int path[PYXMLSEC_SHADOW_MAX_DEPTH]; + int depth; + int idx; + + // Everything except "shadow copy in play and res pre-existed, below the + // root" is exactly the general End: the fast path mutates live nodes in + // place, a fresh res reflects through the graft, errors raise, and for + // res == root there is no live parent to graft into (only attributes can + // change there, which End's sync covers). + if (shadow->doc == NULL || res == NULL || !PYXMLSEC_SHADOW_MARKED(res) || res == shadow->root) { + return PyXmlSec_LxmlShadowEnd(shadow, res, error); + } + + depth = PyXmlSec_LxmlShadowPathTo(res->parent, shadow->root, path); + if (depth < 0) { + PyErr_SetString(PyXmlSec_InternalError, "unexpected result node."); + goto DONE; + } + idx = PyXmlSec_LxmlShadowChildIndex(res); + + copy_root = PyXmlSec_LxmlShadowDumpCopy(shadow); + if (copy_root == NULL) { + goto DONE; + } + copy_parent = PyXmlSec_LxmlShadowWalk(copy_root, path, depth); + live_parent = PyXmlSec_LxmlShadowWalk((PyObject*)shadow->element, path, depth); + if (copy_parent == NULL || live_parent == NULL) { + goto DONE; + } + new_node = PySequence_GetItem(copy_parent, idx); + live_old = PySequence_GetItem(live_parent, idx); + if (new_node == NULL || live_old == NULL) { + goto DONE; + } + // Swap the live element for the copy's version; the tail travels with the + // inserted node (same content — the call does not touch it). + tmp = PyObject_CallMethod(live_parent, "remove", "O", live_old); + if (tmp == NULL) { + goto DONE; + } + Py_CLEAR(tmp); + tmp = PyObject_CallMethod(live_parent, "insert", "iO", idx, new_node); + if (tmp == NULL) { + goto DONE; + } + Py_CLEAR(tmp); + result = PySequence_GetItem(live_parent, idx); + +DONE: + PyXmlSec_LxmlShadowDiscard(shadow); Py_XDECREF(copy_root); Py_XDECREF(copy_parent); Py_XDECREF(live_parent); + Py_XDECREF(live_old); Py_XDECREF(new_node); Py_XDECREF(tmp); return result; } + +xmlNodePtr PyXmlSec_LxmlShadowFindFresh(PyXmlSec_LxmlShadow* shadow) { + xmlNodePtr n; + + if (shadow->doc == NULL) { // fast path: End just wraps the node + return shadow->root; + } + n = shadow->root->children; + while (n != NULL) { + if (!PYXMLSEC_SHADOW_MARKED(n)) { + if (_isElement(n)) { + return n; + } + n = n->next; // fresh formatting text; the element follows it + continue; + } + if (n->children != NULL) { + n = n->children; + continue; + } + while (n != shadow->root && n->next == NULL) { + n = n->parent; + } + n = (n == shadow->root) ? NULL : n->next; + } + return shadow->root; // nothing created; End takes its find-or-create path +} + +PyObject* PyXmlSec_LxmlShadowDumpCopy(PyXmlSec_LxmlShadow* shadow) { + PyObject* bytes = NULL; + PyObject* result = NULL; + xmlChar* dump = NULL; + int dump_size = 0; + + xmlDocDumpMemory(shadow->doc, &dump, &dump_size); + if (dump == NULL || dump_size <= 0) { + PyErr_SetString(PyXmlSec_InternalError, "cannot serialize the mutated copy."); + goto DONE; + } + bytes = PyBytes_FromStringAndSize((const char*)dump, (Py_ssize_t)dump_size); + if (bytes != NULL) { + result = PyXmlSec_LxmlElementFromBytes(bytes); + } +DONE: + if (dump != NULL) { + xmlFree(dump); + } + Py_XDECREF(bytes); + return result; +} + +xmlNodePtr PyXmlSec_LxmlShadowImportElement(PyXmlSec_LxmlShadow* shadow, PyXmlSec_LxmlElementPtr element) { + PyObject* bytes; + char* data = NULL; + Py_ssize_t size = 0; + xmlDocPtr tdoc; + xmlNodePtr troot; + xmlNodePtr result = NULL; + + bytes = PyXmlSec_LxmlElementToBytes((PyObject*)element); + if (bytes == NULL || PyBytes_AsStringAndSize(bytes, &data, &size) < 0) { + Py_XDECREF(bytes); + return NULL; + } + tdoc = xmlReadMemory(data, (int)size, NULL, NULL, XML_PARSE_NONET); + Py_DECREF(bytes); + if (tdoc == NULL || (troot = xmlDocGetRootElement(tdoc)) == NULL) { + if (tdoc != NULL) { + xmlFreeDoc(tdoc); + } + PyErr_SetString(PyXmlSec_InternalError, "cannot make a private copy of the element."); + return NULL; + } + // Copy into the shadow doc (both trees are ours). The copy stays + // untagged: from the reflection's point of view whatever the xmlsec call + // grafts of it is a node "the call created". + result = xmlDocCopyNode(troot, shadow->doc, 1); + xmlFreeDoc(tdoc); + if (result == NULL) { + PyErr_SetString(PyXmlSec_InternalError, "cannot make a private copy of the element."); + } + return result; +} + +// A mutation site found in the copy: either a fresh node to graft (elements, +// comments, PIs) or a text slot whose value changed. The reflection is +// two-phase — first every site's payload is fetched from the re-parsed copy +// while it is still in its final, untouched state, then everything is applied +// to the live tree in document order (each graft moves a node out of the +// re-parsed copy, which would invalidate later fetches, and each live insert +// makes the later, larger indices valid). +typedef struct { + int depth; + int path[PYXMLSEC_SHADOW_MAX_DEPTH]; // to the site's parent, from the copy root + int idx; // element-index of the site among its siblings + int is_element; // grafts `node`; otherwise the site is a text slot only + int mirror; // reflect the text slot before/at idx + PyObject* node; // phase 1: the node to graft, from the re-parsed copy + PyObject* slot; // phase 1: the slot's new value (str or None) +} PyXmlSec_LxmlShadowSite; + +typedef struct { + PyXmlSec_LxmlShadowSite* items; + int count; + int capacity; + xmlNodePtr top; // the copy's root element — origin of all paths +} PyXmlSec_LxmlShadowSiteList; + +static int PyXmlSec_LxmlShadowSiteAppend(PyXmlSec_LxmlShadowSiteList* list, xmlNodePtr node, int is_element, int prev_elem_fresh) { + PyXmlSec_LxmlShadowSite* site; + + if (list->count == list->capacity) { + int capacity = list->capacity == 0 ? 8 : list->capacity * 2; + PyXmlSec_LxmlShadowSite* items = (PyXmlSec_LxmlShadowSite*)PyMem_Realloc(list->items, capacity * sizeof(*items)); + if (items == NULL) { + PyErr_NoMemory(); + return -1; + } + list->items = items; + list->capacity = capacity; + } + site = &list->items[list->count]; + site->depth = PyXmlSec_LxmlShadowPathTo(node->parent, list->top, site->path); + if (site->depth < 0) { + PyErr_SetString(PyXmlSec_InternalError, "unexpected mutation site."); + return -1; + } + site->idx = PyXmlSec_LxmlShadowChildIndex(node); + site->is_element = is_element; + // The slot before a fresh element travels with the preceding fresh + // element's graft (it is that element's tail) — mirror it only otherwise. + site->mirror = site->idx == 0 || !prev_elem_fresh; + site->node = NULL; + site->slot = NULL; + ++list->count; + return 0; +} + +// Walks the pre-existing (marked) structure of the copy in document order, +// recording every fresh site. Fresh subtrees are grafted wholesale, so the +// scan does not descend into them. +static int PyXmlSec_LxmlShadowCollectSites(PyXmlSec_LxmlShadowSiteList* list, xmlNodePtr parent) { + xmlNodePtr n; + int last_elem_fresh = 0; + + for (n = parent->children; n != NULL; n = n->next) { + if (PYXMLSEC_SHADOW_MARKED(n)) { + if (_isElement(n)) { + last_elem_fresh = 0; + if (n->type == XML_ELEMENT_NODE && n->children != NULL + && PyXmlSec_LxmlShadowCollectSites(list, n) < 0) { + return -1; + } + } + continue; + } + if (_isElement(n)) { + if (PyXmlSec_LxmlShadowSiteAppend(list, n, 1, last_elem_fresh) < 0) { + return -1; + } + last_elem_fresh = 1; + } else if (n->type == XML_TEXT_NODE || n->type == XML_CDATA_SECTION_NODE) { + // A fresh text after a fresh element is that element's tail + // (travels with the graft); adjacent fresh texts share one slot. + if (!last_elem_fresh + && !(n->prev != NULL && !PYXMLSEC_SHADOW_MARKED(n->prev) + && (n->prev->type == XML_TEXT_NODE || n->prev->type == XML_CDATA_SECTION_NODE)) + && PyXmlSec_LxmlShadowSiteAppend(list, n, 0, 0) < 0) { + return -1; + } + } + } + return 0; +} + +int PyXmlSec_LxmlShadowReflectAll(PyXmlSec_LxmlShadow* shadow) { + PyXmlSec_LxmlShadowSiteList list = {NULL, 0, 0, NULL}; + PyObject* copy_root = NULL; + int i; + int rv = -1; + + if (shadow->doc == NULL) { // fast path: xmlsec already mutated the live tree + Py_CLEAR(shadow->owned); + return 0; + } + + // Re-fetch the root: replacement operations may have swapped nodes at the + // top. A fresh root means the call replaced the root itself — the callers + // covering that (enc.c) reflect it explicitly and never get here. + list.top = xmlDocGetRootElement(shadow->doc); + if (list.top == NULL || !PYXMLSEC_SHADOW_MARKED(list.top)) { + PyErr_SetString(PyXmlSec_InternalError, "unexpected mutation site."); + goto DONE; + } + if (PyXmlSec_LxmlShadowCollectSites(&list, list.top) < 0) { + goto DONE; + } + if (list.count == 0) { + rv = 0; + goto DONE; + } + copy_root = PyXmlSec_LxmlShadowDumpCopy(shadow); + if (copy_root == NULL) { + goto DONE; + } + + // Phase 1: fetch every payload from the re-parsed copy (still final state). + for (i = 0; i < list.count; ++i) { + PyXmlSec_LxmlShadowSite* site = &list.items[i]; + PyObject* copy_parent = PyXmlSec_LxmlShadowWalk(copy_root, site->path, site->depth); + if (copy_parent == NULL) { + goto DONE; + } + if (site->is_element) { + site->node = PySequence_GetItem(copy_parent, site->idx); + } + if (site->mirror) { + if (site->idx == 0) { + site->slot = PyObject_GetAttrString(copy_parent, "text"); + } else { + PyObject* prev = PySequence_GetItem(copy_parent, site->idx - 1); + if (prev != NULL) { + site->slot = PyObject_GetAttrString(prev, "tail"); + Py_DECREF(prev); + } + } + } + Py_DECREF(copy_parent); + if ((site->is_element && site->node == NULL) || (site->mirror && site->slot == NULL)) { + goto DONE; + } + } + + // Phase 2: apply to the live tree in document order. + for (i = 0; i < list.count; ++i) { + PyXmlSec_LxmlShadowSite* site = &list.items[i]; + int failed = 0; + PyObject* live_parent = PyXmlSec_LxmlShadowWalk((PyObject*)shadow->element, site->path, site->depth); + if (live_parent == NULL) { + goto DONE; + } + if (site->mirror) { + if (site->idx == 0) { + failed = PyObject_SetAttrString(live_parent, "text", site->slot) < 0; + } else { + PyObject* live_prev = PySequence_GetItem(live_parent, site->idx - 1); + failed = live_prev == NULL || PyObject_SetAttrString(live_prev, "tail", site->slot) < 0; + Py_XDECREF(live_prev); + } + } + if (!failed && site->is_element) { + PyObject* tmp = PyObject_CallMethod(live_parent, "insert", "iO", site->idx, site->node); + failed = tmp == NULL; + Py_XDECREF(tmp); + } + Py_DECREF(live_parent); + if (failed) { + goto DONE; + } + } + rv = 0; + +DONE: + for (i = 0; i < list.count; ++i) { + Py_XDECREF(list.items[i].node); + Py_XDECREF(list.items[i].slot); + } + PyMem_Free(list.items); + Py_XDECREF(copy_root); + PyXmlSec_LxmlShadowDiscard(shadow); + return rv; +} + +// ---------------------------------------------------------------------------- +// Shadow-mode ID registry. +// +// register_id / add_ids used to write straight into lxml's document (its ID +// hash) with our libxml2 — exactly the cross-library access the shadow +// forbids. Under the shadow they record the id-attribute specs here instead, +// and every whole-document Begin replays them onto the private copy so that +// #id references resolve during sign/verify/decrypt. +// +// The registry cannot hold strong references to lxml documents (that would +// pin whole trees forever) and lxml's classes refuse weak references, so +// entries are keyed by the _Document object's address with the underlying +// xmlDoc pointer stored alongside as a staleness check: an entry is trusted +// only while both addresses match, and is replaced when the address has been +// reused by a different document. A size cap bounds growth from dead +// documents whose addresses never get reused. +// ---------------------------------------------------------------------------- + +#define PYXMLSEC_SHADOW_ID_REGISTRY_CAP 4096 + +int PyXmlSec_LxmlShadowRecordId(PyXmlSec_LxmlElementPtr element, const char* name, const char* ns) { + PyObject* key = NULL; + PyObject* cdoc = NULL; + PyObject* created = NULL; + PyObject* spec = NULL; + PyObject* entry; + PyObject* specs; + int contains; + int result = -1; + + key = PyLong_FromVoidPtr((void*)element->_doc); + cdoc = PyLong_FromVoidPtr((void*)element->_doc->_c_doc); + if (key == NULL || cdoc == NULL) { + goto DONE; + } + + entry = PyDict_GetItem(PyXmlSec_LxmlShadowIdRegistry, key); // borrowed + if (entry != NULL) { + int eq = PyObject_RichCompareBool(PyTuple_GET_ITEM(entry, 0), cdoc, Py_EQ); + if (eq < 0) { + goto DONE; + } + if (!eq) { + entry = NULL; // the address was reused by another document + } + } + if (entry == NULL) { + if (PyDict_Size(PyXmlSec_LxmlShadowIdRegistry) >= PYXMLSEC_SHADOW_ID_REGISTRY_CAP) { + PyObject* k; + PyObject* v; + Py_ssize_t pos = 0; + if (PyDict_Next(PyXmlSec_LxmlShadowIdRegistry, &pos, &k, &v) + && PyDict_DelItem(PyXmlSec_LxmlShadowIdRegistry, k) < 0) { + goto DONE; + } + } + specs = PyList_New(0); + if (specs == NULL) { + goto DONE; + } + created = PyTuple_Pack(2, cdoc, specs); + Py_DECREF(specs); + if (created == NULL || PyDict_SetItem(PyXmlSec_LxmlShadowIdRegistry, key, created) < 0) { + goto DONE; + } + entry = created; + } + + spec = Py_BuildValue("(sz)", name, ns); + if (spec == NULL) { + goto DONE; + } + specs = PyTuple_GET_ITEM(entry, 1); + contains = PySequence_Contains(specs, spec); + if (contains < 0 || (!contains && PyList_Append(specs, spec) < 0)) { + goto DONE; + } + result = 0; + +DONE: + Py_XDECREF(key); + Py_XDECREF(cdoc); + Py_XDECREF(created); + Py_XDECREF(spec); + return result; +} + +// Registers every attribute named `name` (under `ns` when given) in the copy +// as an XML ID — a superset of the fast path's registrations (single node for +// register_id, subtree for add_ids), which is the safe direction: it mirrors +// what xmlSecAddIDs does from the root. +static void PyXmlSec_LxmlShadowApplyIdSpec(xmlDocPtr doc, xmlNodePtr n, const xmlChar* name, const xmlChar* ns) { + for (; n != NULL; n = n->next) { + if (n->type == XML_ELEMENT_NODE) { + xmlAttrPtr attr = ns != NULL ? xmlHasNsProp(n, name, ns) : xmlHasProp(n, name); + if (attr != NULL && attr->children != NULL) { + xmlChar* value = xmlNodeListGetString(doc, attr->children, 1); + if (value != NULL) { + if (xmlGetID(doc, value) == NULL) { + xmlAddID(NULL, doc, value, attr); + } + xmlFree(value); + } + } + if (n->children != NULL) { + PyXmlSec_LxmlShadowApplyIdSpec(doc, n->children, name, ns); + } + } + } +} + +int PyXmlSec_LxmlShadowReplayIds(PyXmlSec_LxmlShadow* shadow) { + PyObject* key; + PyObject* cdoc; + PyObject* entry; + PyObject* specs; + Py_ssize_t i, n; + int eq; + + if (shadow->doc == NULL) { // fast path: the live document carries its own IDs + return 0; + } + key = PyLong_FromVoidPtr((void*)shadow->element->_doc); + if (key == NULL) { + return -1; + } + entry = PyDict_GetItem(PyXmlSec_LxmlShadowIdRegistry, key); // borrowed + Py_DECREF(key); + if (entry == NULL) { + return 0; + } + cdoc = PyLong_FromVoidPtr((void*)shadow->element->_doc->_c_doc); + if (cdoc == NULL) { + return -1; + } + eq = PyObject_RichCompareBool(PyTuple_GET_ITEM(entry, 0), cdoc, Py_EQ); + Py_DECREF(cdoc); + if (eq < 0) { + return -1; + } + if (!eq) { + return 0; // stale entry from a dead document at the same address + } + + specs = PyTuple_GET_ITEM(entry, 1); + n = PyList_GET_SIZE(specs); + for (i = 0; i < n; ++i) { + PyObject* spec = PyList_GET_ITEM(specs, i); + const char* name = PyUnicode_AsUTF8(PyTuple_GET_ITEM(spec, 0)); + const char* ns = PyTuple_GET_ITEM(spec, 1) == Py_None ? NULL : PyUnicode_AsUTF8(PyTuple_GET_ITEM(spec, 1)); + if (name == NULL || (ns == NULL && PyErr_Occurred())) { + return -1; + } + PyXmlSec_LxmlShadowApplyIdSpec(shadow->doc, shadow->doc->children, (const xmlChar*)name, (const xmlChar*)ns); + } + return 0; +} diff --git a/src/lxml.h b/src/lxml.h index a9609e66..b255d2ba 100644 --- a/src/lxml.h +++ b/src/lxml.h @@ -63,9 +63,11 @@ int PyXmlSec_LxmlElementConverter(PyObject* o, PyXmlSec_LxmlElementPtr* p); // the position xmlsec chose (including intermediate nodes like // and the "\n" formatting text around it), or — for find-or-create calls that // added nothing — the already-existing element plus any attributes the call -// set on it. It assumes the call mutates at most one place in the tree. +// set on it. It assumes the call mutates at most one place in the tree; +// multi-site calls (sign, encrypt) use ReflectAll below instead. typedef struct { - PyXmlSec_LxmlElementPtr element; // borrowed; the live lxml element + PyXmlSec_LxmlElementPtr element; // borrowed; the live lxml element (BeginDoc: the live root) + PyObject* owned; // reference released when the shadow ends (BeginDoc's root proxy) xmlDocPtr doc; // the private copy; owned by the shadow xmlNodePtr root; // doc's root element (the copy of element) } PyXmlSec_LxmlShadow; @@ -73,6 +75,79 @@ typedef struct { int PyXmlSec_LxmlShadowBegin(PyXmlSec_LxmlShadow* shadow, PyXmlSec_LxmlElementPtr element); PyObject* PyXmlSec_LxmlShadowEnd(PyXmlSec_LxmlShadow* shadow, xmlNodePtr res, const char* error); +// Non-zero when the shadow path is on (mismatched libxml2 or PYXMLSEC_FORCE_SHADOW). +// Only the few call sites whose *semantics* differ per mode (ID registration, +// encrypt/decrypt replacement) may branch on this; everything else goes through +// the Begin/End pairs, which encapsulate both paths. +int PyXmlSec_LxmlShadowIsActive(void); + +// Whole-document shadow, for xmlsec calls that read or mutate beyond the +// element's subtree (sign/verify, encrypt/decrypt, find_parent). Serializes the +// element's whole tree (element.getroottree()), so ID references resolve and +// comments/PIs outside the root survive. `*target` receives the copy's node +// corresponding to `element` (the live node itself on the fast path); +// `shadow.element` becomes the live *root*, so the End/Reflect helpers map +// copy paths from the copy root onto it. +int PyXmlSec_LxmlShadowBeginDoc(PyXmlSec_LxmlShadow* shadow, PyXmlSec_LxmlElementPtr element, xmlNodePtr* target); + +// Create-shape (template.create, encrypted_data_create): the xmlsec call only +// needs a document to allocate a *detached* subtree in. Begin returns that +// document — the element's own on the fast path, a private empty one on the +// shadow path — and End reflects the result as a new detached lxml element +// (no live-tree graft). NULL from Begin means an exception is set. +xmlDocPtr PyXmlSec_LxmlShadowBeginNewDoc(PyXmlSec_LxmlShadow* shadow, PyXmlSec_LxmlElementPtr element); +PyObject* PyXmlSec_LxmlShadowEndNewDoc(PyXmlSec_LxmlShadow* shadow, xmlNodePtr res, const char* error); + +// End for read-only finders: maps `res` (a pre-existing node in the copy) back +// to the live lxml element; returns None when res is NULL (not found — no +// exception). Always releases the copy. +PyObject* PyXmlSec_LxmlShadowEndFind(PyXmlSec_LxmlShadow* shadow, xmlNodePtr res); + +// End for find-or-create calls that may mutate the found node in ways the +// attribute sync cannot express (encrypted_data_ensure_key_info renames the +// namespace prefix): a fresh `res` behaves exactly like End; a pre-existing +// one is reflected by *replacing* the live element with the copy's version, +// so the returned element is a new object rather than the original proxy. +PyObject* PyXmlSec_LxmlShadowEndReplace(PyXmlSec_LxmlShadow* shadow, xmlNodePtr res, const char* error); + +// First node the xmlsec call created (document order), for calls that return +// only a status int: pass the result to End. Falls back to shadow->root when +// the call created nothing (End then takes its find-or-create path). +xmlNodePtr PyXmlSec_LxmlShadowFindFresh(PyXmlSec_LxmlShadow* shadow); + +// Multi-site End (sign, encrypt_binary/uri and the replacement reflects): +// scans the copy for *all* topmost nodes the xmlsec call created and grafts +// each back into the live tree — new subtrees via insert, new/changed text via +// the text slots. Returns 0, or -1 with an exception set. Like End, it always +// releases the copy; there is no result node (the call sites know what to +// return). No-op on the fast path. +int PyXmlSec_LxmlShadowReflectAll(PyXmlSec_LxmlShadow* shadow); + +// Serializes the mutated copy and re-parses it with lxml, returning the root +// element of that detached parse (new reference). Used by the replacement +// reflects (enc.c) when the copy's root itself was replaced. Does not release +// the copy. +PyObject* PyXmlSec_LxmlShadowDumpCopy(PyXmlSec_LxmlShadow* shadow); + +// Re-serializes `element` (a live lxml element) into the shadow's private copy +// as a fresh (untagged) detached subtree — encrypt_xml's template import. +// Shadow path only (shadow->doc != NULL). +xmlNodePtr PyXmlSec_LxmlShadowImportElement(PyXmlSec_LxmlShadow* shadow, PyXmlSec_LxmlElementPtr element); + +// Releases the copy without reflecting anything (verify, error paths). +// Safe to call after any successful Begin*; End* call it internally. +void PyXmlSec_LxmlShadowDiscard(PyXmlSec_LxmlShadow* shadow); + +// Shadow-mode ID registration (issue #356): register_id/add_ids cannot write +// lxml's ID hash with our libxml2, so they record the id-attribute specs per +// document here, and every whole-document Begin replays them onto the copy +// (ReplayIds) so that #id references resolve during sign/verify/decrypt. +// The registry is keyed by the lxml document's identity; replay scans the +// whole copy for the recorded attribute names (a superset of the single-node +// registration on the fast path — see lxml.c). +int PyXmlSec_LxmlShadowRecordId(PyXmlSec_LxmlElementPtr element, const char* name, const char* ns); +int PyXmlSec_LxmlShadowReplayIds(PyXmlSec_LxmlShadow* shadow); + // get version numbers for libxml2 both compiled and loaded long PyXmlSec_GetLibXmlVersionMajor(); long PyXmlSec_GetLibXmlVersionMinor(); diff --git a/src/template.c b/src/template.c index 75f8bd50..5cfa55d0 100644 --- a/src/template.c +++ b/src/template.c @@ -42,6 +42,9 @@ static PyObject* PyXmlSec_TemplateCreate(PyObject* self, PyObject *args, PyObjec const char* id = NULL; const char* ns = NULL; xmlNodePtr res; + xmlDocPtr tdoc; + PyObject* result; + PyXmlSec_LxmlShadow shadow; PYXMLSEC_DEBUG("template create - start"); if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&O!O!|zz:create", kwlist, @@ -50,16 +53,22 @@ static PyObject* PyXmlSec_TemplateCreate(PyObject* self, PyObject *args, PyObjec goto ON_FAIL; } + // `node` only supplies the document the detached template is built in; + // the create-shape shadow provides a private one instead (issue #356). + tdoc = PyXmlSec_LxmlShadowBeginNewDoc(&shadow, node); + if (tdoc == NULL) { + goto ON_FAIL; + } Py_BEGIN_ALLOW_THREADS; - res = xmlSecTmplSignatureCreateNsPref(node->_doc->_c_doc, c14n->id, sign->id, XSTR(id), XSTR(ns)); + res = xmlSecTmplSignatureCreateNsPref(tdoc, c14n->id, sign->id, XSTR(id), XSTR(ns)); Py_END_ALLOW_THREADS; - if (res == NULL) { - PyXmlSec_SetLastError("cannot create template."); + result = PyXmlSec_LxmlShadowEndNewDoc(&shadow, res, "cannot create template."); + if (result == NULL) { goto ON_FAIL; } PYXMLSEC_DEBUG("template create - ok"); - return (PyObject*)PyXmlSec_elementFactory(node->_doc, res); + return result; ON_FAIL: PYXMLSEC_DEBUG("template create - fail"); @@ -222,6 +231,8 @@ static PyObject* PyXmlSec_TemplateAddKeyName(PyObject* self, PyObject *args, PyO PyXmlSec_LxmlElementPtr node = NULL; const char* name = NULL; xmlNodePtr res; + PyObject* result; + PyXmlSec_LxmlShadow shadow; PYXMLSEC_DEBUG("template add_key_name - start"); if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&|z:add_key_name", kwlist, PyXmlSec_LxmlElementConverter, &node, &name)) @@ -229,16 +240,19 @@ static PyObject* PyXmlSec_TemplateAddKeyName(PyObject* self, PyObject *args, PyO goto ON_FAIL; } + if (PyXmlSec_LxmlShadowBegin(&shadow, node) < 0) { + goto ON_FAIL; + } Py_BEGIN_ALLOW_THREADS; - res = xmlSecTmplKeyInfoAddKeyName(node->_c_node, XSTR(name)); + res = xmlSecTmplKeyInfoAddKeyName(shadow.root, XSTR(name)); Py_END_ALLOW_THREADS; - if (res == NULL) { - PyXmlSec_SetLastError("cannot add key name."); + result = PyXmlSec_LxmlShadowEnd(&shadow, res, "cannot add key name."); + if (result == NULL) { goto ON_FAIL; } PYXMLSEC_DEBUG("template add_key_name - ok"); - return (PyObject*)PyXmlSec_elementFactory(node->_doc, res); + return result; ON_FAIL: PYXMLSEC_DEBUG("template add_key_name - fail"); @@ -257,6 +271,8 @@ static PyObject* PyXmlSec_TemplateAddKeyValue(PyObject* self, PyObject *args, Py PyXmlSec_LxmlElementPtr node = NULL; xmlNodePtr res; + PyObject* result; + PyXmlSec_LxmlShadow shadow; PYXMLSEC_DEBUG("template add_key_value - start"); if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&:add_key_value", kwlist, PyXmlSec_LxmlElementConverter, &node)) @@ -264,16 +280,19 @@ static PyObject* PyXmlSec_TemplateAddKeyValue(PyObject* self, PyObject *args, Py goto ON_FAIL; } + if (PyXmlSec_LxmlShadowBegin(&shadow, node) < 0) { + goto ON_FAIL; + } Py_BEGIN_ALLOW_THREADS; - res = xmlSecTmplKeyInfoAddKeyValue(node->_c_node); + res = xmlSecTmplKeyInfoAddKeyValue(shadow.root); Py_END_ALLOW_THREADS; - if (res == NULL) { - PyXmlSec_SetLastError("cannot add key value."); + result = PyXmlSec_LxmlShadowEnd(&shadow, res, "cannot add key value."); + if (result == NULL) { goto ON_FAIL; } PYXMLSEC_DEBUG("template add_key_name - ok"); - return (PyObject*)PyXmlSec_elementFactory(node->_doc, res); + return result; ON_FAIL: PYXMLSEC_DEBUG("template add_key_name - fail"); @@ -292,6 +311,8 @@ static PyObject* PyXmlSec_TemplateAddX509Data(PyObject* self, PyObject *args, Py PyXmlSec_LxmlElementPtr node = NULL; xmlNodePtr res; + PyObject* result; + PyXmlSec_LxmlShadow shadow; PYXMLSEC_DEBUG("template add_x509_data - start"); if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&:add_x509_data", kwlist, PyXmlSec_LxmlElementConverter, &node)) @@ -299,16 +320,19 @@ static PyObject* PyXmlSec_TemplateAddX509Data(PyObject* self, PyObject *args, Py goto ON_FAIL; } + if (PyXmlSec_LxmlShadowBegin(&shadow, node) < 0) { + goto ON_FAIL; + } Py_BEGIN_ALLOW_THREADS; - res = xmlSecTmplKeyInfoAddX509Data(node->_c_node); + res = xmlSecTmplKeyInfoAddX509Data(shadow.root); Py_END_ALLOW_THREADS; - if (res == NULL) { - PyXmlSec_SetLastError("cannot add x509 data."); + result = PyXmlSec_LxmlShadowEnd(&shadow, res, "cannot add x509 data."); + if (result == NULL) { goto ON_FAIL; } PYXMLSEC_DEBUG("template add_x509_data - ok"); - return (PyObject*)PyXmlSec_elementFactory(node->_doc, res); + return result; ON_FAIL: PYXMLSEC_DEBUG("template add_x509_data - fail"); @@ -327,6 +351,8 @@ static PyObject* PyXmlSec_TemplateAddX509DataAddIssuerSerial(PyObject* self, PyO PyXmlSec_LxmlElementPtr node = NULL; xmlNodePtr res; + PyObject* result; + PyXmlSec_LxmlShadow shadow; PYXMLSEC_DEBUG("template x509_data_add_issuer_serial - start"); if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&:x509_data_add_issuer_serial", kwlist, @@ -334,16 +360,19 @@ static PyObject* PyXmlSec_TemplateAddX509DataAddIssuerSerial(PyObject* self, PyO { goto ON_FAIL; } + if (PyXmlSec_LxmlShadowBegin(&shadow, node) < 0) { + goto ON_FAIL; + } Py_BEGIN_ALLOW_THREADS; - res = xmlSecTmplX509DataAddIssuerSerial(node->_c_node); + res = xmlSecTmplX509DataAddIssuerSerial(shadow.root); Py_END_ALLOW_THREADS; - if (res == NULL) { - PyXmlSec_SetLastError("cannot add x509 issuer serial."); + result = PyXmlSec_LxmlShadowEnd(&shadow, res, "cannot add x509 issuer serial."); + if (result == NULL) { goto ON_FAIL; } PYXMLSEC_DEBUG("template x509_data_add_issuer_serial - ok"); - return (PyObject*)PyXmlSec_elementFactory(node->_doc, res); + return result; ON_FAIL: PYXMLSEC_DEBUG("template x509_data_add_issuer_serial - fail"); @@ -365,6 +394,8 @@ static PyObject* PyXmlSec_TemplateAddX509DataIssuerSerialAddIssuerName(PyObject* PyXmlSec_LxmlElementPtr node = NULL; const char* name = NULL; xmlNodePtr res; + PyObject* result; + PyXmlSec_LxmlShadow shadow; PYXMLSEC_DEBUG("template x509_issuer_serial_add_issuer_name - start"); if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&|z:x509_issuer_serial_add_issuer_name", kwlist, @@ -373,16 +404,19 @@ static PyObject* PyXmlSec_TemplateAddX509DataIssuerSerialAddIssuerName(PyObject* goto ON_FAIL; } + if (PyXmlSec_LxmlShadowBegin(&shadow, node) < 0) { + goto ON_FAIL; + } Py_BEGIN_ALLOW_THREADS; - res = xmlSecTmplX509IssuerSerialAddIssuerName(node->_c_node, XSTR(name)); + res = xmlSecTmplX509IssuerSerialAddIssuerName(shadow.root, XSTR(name)); Py_END_ALLOW_THREADS; - if (res == NULL) { - PyXmlSec_SetLastError("cannot add x509 issuer serial name."); + result = PyXmlSec_LxmlShadowEnd(&shadow, res, "cannot add x509 issuer serial name."); + if (result == NULL) { goto ON_FAIL; } PYXMLSEC_DEBUG("template x509_issuer_serial_add_issuer_name - ok"); - return (PyObject*)PyXmlSec_elementFactory(node->_doc, res); + return result; ON_FAIL: PYXMLSEC_DEBUG("template x509_issuer_serial_add_issuer_name - fail"); @@ -404,6 +438,8 @@ static PyObject* PyXmlSec_TemplateAddX509DataIssuerSerialAddIssuerSerialNumber(P PyXmlSec_LxmlElementPtr node = NULL; const char* serial = NULL; xmlNodePtr res; + PyObject* result; + PyXmlSec_LxmlShadow shadow; PYXMLSEC_DEBUG("template x509_issuer_serial_add_serial_number - start"); if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&|z:x509_issuer_serial_add_serial_number", kwlist, @@ -412,16 +448,19 @@ static PyObject* PyXmlSec_TemplateAddX509DataIssuerSerialAddIssuerSerialNumber(P goto ON_FAIL; } + if (PyXmlSec_LxmlShadowBegin(&shadow, node) < 0) { + goto ON_FAIL; + } Py_BEGIN_ALLOW_THREADS; - res = xmlSecTmplX509IssuerSerialAddSerialNumber(node->_c_node, XSTR(serial)); + res = xmlSecTmplX509IssuerSerialAddSerialNumber(shadow.root, XSTR(serial)); Py_END_ALLOW_THREADS; - if (res == NULL) { - PyXmlSec_SetLastError("cannot add x509 issuer serial number."); + result = PyXmlSec_LxmlShadowEnd(&shadow, res, "cannot add x509 issuer serial number."); + if (result == NULL) { goto ON_FAIL; } PYXMLSEC_DEBUG("template x509_issuer_serial_add_serial_number - ok"); - return (PyObject*)PyXmlSec_elementFactory(node->_doc, res); + return result; ON_FAIL: PYXMLSEC_DEBUG("template x509_issuer_serial_add_serial_number - fail"); @@ -440,6 +479,8 @@ static PyObject* PyXmlSec_TemplateAddX509DataAddSubjectName(PyObject* self, PyOb PyXmlSec_LxmlElementPtr node = NULL; xmlNodePtr res; + PyObject* result; + PyXmlSec_LxmlShadow shadow; PYXMLSEC_DEBUG("template x509_data_add_subject_name - start"); if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&:x509_data_add_subject_name", kwlist, @@ -448,16 +489,19 @@ static PyObject* PyXmlSec_TemplateAddX509DataAddSubjectName(PyObject* self, PyOb goto ON_FAIL; } + if (PyXmlSec_LxmlShadowBegin(&shadow, node) < 0) { + goto ON_FAIL; + } Py_BEGIN_ALLOW_THREADS; - res = xmlSecTmplX509DataAddSubjectName(node->_c_node); + res = xmlSecTmplX509DataAddSubjectName(shadow.root); Py_END_ALLOW_THREADS; - if (res == NULL) { - PyXmlSec_SetLastError("cannot add x509 subject name."); + result = PyXmlSec_LxmlShadowEnd(&shadow, res, "cannot add x509 subject name."); + if (result == NULL) { goto ON_FAIL; } PYXMLSEC_DEBUG("template x509_data_add_subject_name - ok"); - return (PyObject*)PyXmlSec_elementFactory(node->_doc, res); + return result; ON_FAIL: PYXMLSEC_DEBUG("template x509_data_add_subject_name - fail"); @@ -476,6 +520,8 @@ static PyObject* PyXmlSec_TemplateAddX509DataAddSKI(PyObject* self, PyObject *ar PyXmlSec_LxmlElementPtr node = NULL; xmlNodePtr res; + PyObject* result; + PyXmlSec_LxmlShadow shadow; PYXMLSEC_DEBUG("template x509_data_add_ski - start"); if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&:x509_data_add_ski", kwlist, @@ -484,16 +530,19 @@ static PyObject* PyXmlSec_TemplateAddX509DataAddSKI(PyObject* self, PyObject *ar goto ON_FAIL; } + if (PyXmlSec_LxmlShadowBegin(&shadow, node) < 0) { + goto ON_FAIL; + } Py_BEGIN_ALLOW_THREADS; - res = xmlSecTmplX509DataAddSKI(node->_c_node); + res = xmlSecTmplX509DataAddSKI(shadow.root); Py_END_ALLOW_THREADS; - if (res == NULL) { - PyXmlSec_SetLastError("cannot add x509 SKI."); + result = PyXmlSec_LxmlShadowEnd(&shadow, res, "cannot add x509 SKI."); + if (result == NULL) { goto ON_FAIL; } PYXMLSEC_DEBUG("template x509_data_add_ski - ok"); - return (PyObject*)PyXmlSec_elementFactory(node->_doc, res); + return result; ON_FAIL: PYXMLSEC_DEBUG("template x509_data_add_ski - fail"); @@ -512,6 +561,8 @@ static PyObject* PyXmlSec_TemplateAddX509DataAddCertificate(PyObject* self, PyOb PyXmlSec_LxmlElementPtr node = NULL; xmlNodePtr res; + PyObject* result; + PyXmlSec_LxmlShadow shadow; PYXMLSEC_DEBUG("template x509_data_add_certificate - start"); if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&:x509_data_add_certificate", kwlist, @@ -520,16 +571,19 @@ static PyObject* PyXmlSec_TemplateAddX509DataAddCertificate(PyObject* self, PyOb goto ON_FAIL; } + if (PyXmlSec_LxmlShadowBegin(&shadow, node) < 0) { + goto ON_FAIL; + } Py_BEGIN_ALLOW_THREADS; - res = xmlSecTmplX509DataAddCertificate(node->_c_node); + res = xmlSecTmplX509DataAddCertificate(shadow.root); Py_END_ALLOW_THREADS; - if (res == NULL) { - PyXmlSec_SetLastError("cannot add x509 certificate."); + result = PyXmlSec_LxmlShadowEnd(&shadow, res, "cannot add x509 certificate."); + if (result == NULL) { goto ON_FAIL; } PYXMLSEC_DEBUG("template x509_data_add_certificate - ok"); - return (PyObject*)PyXmlSec_elementFactory(node->_doc, res); + return result; ON_FAIL: PYXMLSEC_DEBUG("template x509_data_add_certificate - fail"); @@ -548,6 +602,8 @@ static PyObject* PyXmlSec_TemplateAddX509DataAddCRL(PyObject* self, PyObject *ar PyXmlSec_LxmlElementPtr node = NULL; xmlNodePtr res; + PyObject* result; + PyXmlSec_LxmlShadow shadow; PYXMLSEC_DEBUG("template x509_data_add_crl - start"); if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&:x509_data_add_crl", kwlist, @@ -556,16 +612,19 @@ static PyObject* PyXmlSec_TemplateAddX509DataAddCRL(PyObject* self, PyObject *ar goto ON_FAIL; } + if (PyXmlSec_LxmlShadowBegin(&shadow, node) < 0) { + goto ON_FAIL; + } Py_BEGIN_ALLOW_THREADS; - res = xmlSecTmplX509DataAddCRL(node->_c_node); + res = xmlSecTmplX509DataAddCRL(shadow.root); Py_END_ALLOW_THREADS; - if (res == NULL) { - PyXmlSec_SetLastError("cannot add x509 CRL."); + result = PyXmlSec_LxmlShadowEnd(&shadow, res, "cannot add x509 CRL."); + if (result == NULL) { goto ON_FAIL; } PYXMLSEC_DEBUG("template x509_data_add_crl - ok"); - return (PyObject*)PyXmlSec_elementFactory(node->_doc, res); + return result; ON_FAIL: PYXMLSEC_DEBUG("template x509_data_add_crl - fail"); @@ -596,6 +655,8 @@ static PyObject* PyXmlSec_TemplateAddEncryptedKey(PyObject* self, PyObject *args const char* type = NULL; const char* recipient = NULL; xmlNodePtr res; + PyObject* result; + PyXmlSec_LxmlShadow shadow; PYXMLSEC_DEBUG("template add_encrypted_key - start"); if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&O!|zzz:add_encrypted_key", kwlist, @@ -604,16 +665,19 @@ static PyObject* PyXmlSec_TemplateAddEncryptedKey(PyObject* self, PyObject *args goto ON_FAIL; } + if (PyXmlSec_LxmlShadowBegin(&shadow, node) < 0) { + goto ON_FAIL; + } Py_BEGIN_ALLOW_THREADS; - res = xmlSecTmplKeyInfoAddEncryptedKey(node->_c_node, method->id, XSTR(id), XSTR(type), XSTR(recipient)); + res = xmlSecTmplKeyInfoAddEncryptedKey(shadow.root, method->id, XSTR(id), XSTR(type), XSTR(recipient)); Py_END_ALLOW_THREADS; - if (res == NULL) { - PyXmlSec_SetLastError("cannot add encrypted key."); + result = PyXmlSec_LxmlShadowEnd(&shadow, res, "cannot add encrypted key."); + if (result == NULL) { goto ON_FAIL; } PYXMLSEC_DEBUG("template add_encrypted_key - ok"); - return (PyObject*)PyXmlSec_elementFactory(node->_doc, res); + return result; ON_FAIL: PYXMLSEC_DEBUG("template add_encrypted_key - fail"); @@ -650,6 +714,9 @@ static PyObject* PyXmlSec_TemplateCreateEncryptedData(PyObject* self, PyObject * const char* encoding = NULL; const char* ns = NULL; xmlNodePtr res; + xmlDocPtr tdoc; + PyObject* result; + PyXmlSec_LxmlShadow shadow; PYXMLSEC_DEBUG("template encrypted_data_create - start"); if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&O!|zzzzz:encrypted_data_create", kwlist, @@ -658,19 +725,24 @@ static PyObject* PyXmlSec_TemplateCreateEncryptedData(PyObject* self, PyObject * goto ON_FAIL; } - Py_BEGIN_ALLOW_THREADS; - res = xmlSecTmplEncDataCreate(node->_doc->_c_doc, method->id, XSTR(id), XSTR(type), XSTR(mime_type), XSTR(encoding)); - Py_END_ALLOW_THREADS; - if (res == NULL) { - PyXmlSec_SetLastError("cannot create encrypted data."); + tdoc = PyXmlSec_LxmlShadowBeginNewDoc(&shadow, node); + if (tdoc == NULL) { goto ON_FAIL; } - if (ns != NULL) { + Py_BEGIN_ALLOW_THREADS; + res = xmlSecTmplEncDataCreate(tdoc, method->id, XSTR(id), XSTR(type), XSTR(mime_type), XSTR(encoding)); + // the prefix rewrite happens before End so the serialization carries it + if (res != NULL && ns != NULL) { res->ns->prefix = xmlStrdup(XSTR(ns)); } + Py_END_ALLOW_THREADS; + result = PyXmlSec_LxmlShadowEndNewDoc(&shadow, res, "cannot create encrypted data."); + if (result == NULL) { + goto ON_FAIL; + } PYXMLSEC_DEBUG("template encrypted_data_create - ok"); - return (PyObject*)PyXmlSec_elementFactory(node->_doc, res); + return result; ON_FAIL: PYXMLSEC_DEBUG("template encrypted_data_create - fail"); @@ -695,6 +767,8 @@ static PyObject* PyXmlSec_TemplateEncryptedDataEnsureKeyInfo(PyObject* self, PyO const char* id = NULL; const char* ns = NULL; xmlNodePtr res; + PyObject* result; + PyXmlSec_LxmlShadow shadow; PYXMLSEC_DEBUG("template encrypted_data_ensure_key_info - start"); if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&|zz:encrypted_data_ensure_key_info", kwlist, @@ -703,19 +777,29 @@ static PyObject* PyXmlSec_TemplateEncryptedDataEnsureKeyInfo(PyObject* self, PyO goto ON_FAIL; } - Py_BEGIN_ALLOW_THREADS; - res = xmlSecTmplEncDataEnsureKeyInfo(node->_c_node, XSTR(id)); - Py_END_ALLOW_THREADS; - if (res == NULL) { - PyXmlSec_SetLastError("cannot ensure key info for encrypted data."); + if (PyXmlSec_LxmlShadowBegin(&shadow, node) < 0) { goto ON_FAIL; } - if (ns != NULL) { + Py_BEGIN_ALLOW_THREADS; + res = xmlSecTmplEncDataEnsureKeyInfo(shadow.root, XSTR(id)); + // the prefix rewrite lands before End so the reflection carries it + if (res != NULL && ns != NULL) { res->ns->prefix = xmlStrdup(XSTR(ns)); } + Py_END_ALLOW_THREADS; + if (ns != NULL) { + // renaming the prefix of a KeyInfo that already existed needs the + // replace reflect — attribute sync cannot express it + result = PyXmlSec_LxmlShadowEndReplace(&shadow, res, "cannot ensure key info for encrypted data."); + } else { + result = PyXmlSec_LxmlShadowEnd(&shadow, res, "cannot ensure key info for encrypted data."); + } + if (result == NULL) { + goto ON_FAIL; + } PYXMLSEC_DEBUG("template encrypted_data_ensure_key_info - ok"); - return (PyObject*)PyXmlSec_elementFactory(node->_doc, res); + return result; ON_FAIL: PYXMLSEC_DEBUG("template encrypted_data_ensure_key_info - fail"); @@ -734,6 +818,8 @@ static PyObject* PyXmlSec_TemplateEncryptedDataEnsureCipherValue(PyObject* self, PyXmlSec_LxmlElementPtr node = NULL; xmlNodePtr res; + PyObject* result; + PyXmlSec_LxmlShadow shadow; PYXMLSEC_DEBUG("template encrypted_data_ensure_cipher_value - start"); if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&:encrypted_data_ensure_cipher_value", kwlist, @@ -742,16 +828,19 @@ static PyObject* PyXmlSec_TemplateEncryptedDataEnsureCipherValue(PyObject* self, goto ON_FAIL; } + if (PyXmlSec_LxmlShadowBegin(&shadow, node) < 0) { + goto ON_FAIL; + } Py_BEGIN_ALLOW_THREADS; - res = xmlSecTmplEncDataEnsureCipherValue(node->_c_node); + res = xmlSecTmplEncDataEnsureCipherValue(shadow.root); Py_END_ALLOW_THREADS; - if (res == NULL) { - PyXmlSec_SetLastError("cannot ensure cipher value for encrypted data."); + result = PyXmlSec_LxmlShadowEnd(&shadow, res, "cannot ensure cipher value for encrypted data."); + if (result == NULL) { goto ON_FAIL; } PYXMLSEC_DEBUG("template encrypted_data_ensure_cipher_value - ok"); - return (PyObject*)PyXmlSec_elementFactory(node->_doc, res); + return result; ON_FAIL: PYXMLSEC_DEBUG("template encrypted_data_ensure_cipher_value - fail"); @@ -771,8 +860,10 @@ static PyObject* PyXmlSec_TemplateTransformAddC14NInclNamespaces(PyObject* self, PyXmlSec_LxmlElementPtr node = NULL; PyObject* prefixes = NULL; PyObject* sep; + PyObject* result; int res; const char* c_prefixes; + PyXmlSec_LxmlShadow shadow; // transform_add_c14n_inclusive_namespaces PYXMLSEC_DEBUG("template encrypted_data_ensure_cipher_value - start"); @@ -799,13 +890,26 @@ static PyObject* PyXmlSec_TemplateTransformAddC14NInclNamespaces(PyObject* self, c_prefixes = PyUnicode_AsUTF8(prefixes); + if (PyXmlSec_LxmlShadowBegin(&shadow, node) < 0) { + goto ON_FAIL; + } Py_BEGIN_ALLOW_THREADS; - res = xmlSecTmplTransformAddC14NInclNamespaces(node->_c_node, XSTR(c_prefixes)); + res = xmlSecTmplTransformAddC14NInclNamespaces(shadow.root, XSTR(c_prefixes)); Py_END_ALLOW_THREADS; if (res != 0) { + PyXmlSec_LxmlShadowDiscard(&shadow); PyXmlSec_SetLastError("cannot add 'inclusive' namespaces to the ExcC14N transform node"); goto ON_FAIL; } + // The call only reports a status; locate the it + // created on the copy and let End graft it back (the returned element is + // not part of this function's interface). + result = PyXmlSec_LxmlShadowEnd(&shadow, PyXmlSec_LxmlShadowFindFresh(&shadow), + "cannot add 'inclusive' namespaces to the ExcC14N transform node"); + if (result == NULL) { + goto ON_FAIL; + } + Py_DECREF(result); Py_DECREF(prefixes); PYXMLSEC_DEBUG("transform_add_c14n_inclusive_namespaces - ok"); diff --git a/src/tree.c b/src/tree.c index 37cae785..d9e7f7a2 100644 --- a/src/tree.c +++ b/src/tree.c @@ -33,6 +33,8 @@ static PyObject* PyXmlSec_TreeFindChild(PyObject* self, PyObject *args, PyObject const char* name = NULL; const char* ns = (const char*)xmlSecDSigNs; xmlNodePtr res; + PyObject* result; + PyXmlSec_LxmlShadow shadow; PYXMLSEC_DEBUG("tree find_child - start"); if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&s|s:find_child", kwlist, @@ -41,15 +43,21 @@ static PyObject* PyXmlSec_TreeFindChild(PyObject* self, PyObject *args, PyObject goto ON_FAIL; } + // Read-only, but still ABI-unsafe on raw nodes: the search runs on a + // shadow copy and the found node is mapped back by path (issue #356). + if (PyXmlSec_LxmlShadowBegin(&shadow, node) < 0) { + goto ON_FAIL; + } Py_BEGIN_ALLOW_THREADS; - res = xmlSecFindChild(node->_c_node, XSTR(name), XSTR(ns)); + res = xmlSecFindChild(shadow.root, XSTR(name), XSTR(ns)); Py_END_ALLOW_THREADS; + result = PyXmlSec_LxmlShadowEndFind(&shadow, res); + if (result == NULL) { + goto ON_FAIL; + } PYXMLSEC_DEBUG("tree find_child - ok"); - if (res == NULL) { - Py_RETURN_NONE; - } - return (PyObject*)PyXmlSec_elementFactory(node->_doc, res); + return result; ON_FAIL: PYXMLSEC_DEBUG("tree find_child - fail"); @@ -74,6 +82,9 @@ static PyObject* PyXmlSec_TreeFindParent(PyObject* self, PyObject *args, PyObjec const char* name = NULL; const char* ns = (const char*)xmlSecDSigNs; xmlNodePtr res; + xmlNodePtr target; + PyObject* result; + PyXmlSec_LxmlShadow shadow; PYXMLSEC_DEBUG("tree find_parent - start"); if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&s|s:find_parent", kwlist, @@ -82,15 +93,21 @@ static PyObject* PyXmlSec_TreeFindParent(PyObject* self, PyObject *args, PyObjec goto ON_FAIL; } + // The search walks upward, so the shadow must cover the whole tree; the + // call then starts from the copy's counterpart of `node`. + if (PyXmlSec_LxmlShadowBeginDoc(&shadow, node, &target) < 0) { + goto ON_FAIL; + } Py_BEGIN_ALLOW_THREADS; - res = xmlSecFindParent(node->_c_node, XSTR(name), XSTR(ns)); + res = xmlSecFindParent(target, XSTR(name), XSTR(ns)); Py_END_ALLOW_THREADS; + result = PyXmlSec_LxmlShadowEndFind(&shadow, res); + if (result == NULL) { + goto ON_FAIL; + } PYXMLSEC_DEBUG("tree find_parent - ok"); - if (res == NULL) { - Py_RETURN_NONE; - } - return (PyObject*)PyXmlSec_elementFactory(node->_doc, res); + return result; ON_FAIL: PYXMLSEC_DEBUG("tree find_parent - fail"); @@ -115,6 +132,8 @@ static PyObject* PyXmlSec_TreeFindNode(PyObject* self, PyObject *args, PyObject const char* name = NULL; const char* ns = (const char*)xmlSecDSigNs; xmlNodePtr res; + PyObject* result; + PyXmlSec_LxmlShadow shadow; PYXMLSEC_DEBUG("tree find_node - start"); if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&s|s:find_node", kwlist, @@ -123,15 +142,19 @@ static PyObject* PyXmlSec_TreeFindNode(PyObject* self, PyObject *args, PyObject goto ON_FAIL; } + if (PyXmlSec_LxmlShadowBegin(&shadow, node) < 0) { + goto ON_FAIL; + } Py_BEGIN_ALLOW_THREADS; - res = xmlSecFindNode(node->_c_node, XSTR(name), XSTR(ns)); + res = xmlSecFindNode(shadow.root, XSTR(name), XSTR(ns)); Py_END_ALLOW_THREADS; + result = PyXmlSec_LxmlShadowEndFind(&shadow, res); + if (result == NULL) { + goto ON_FAIL; + } PYXMLSEC_DEBUG("tree find_node - ok"); - if (res == NULL) { - Py_RETURN_NONE; - } - return (PyObject*)PyXmlSec_elementFactory(node->_doc, res); + return result; ON_FAIL: PYXMLSEC_DEBUG("tree find_node - fail"); @@ -170,6 +193,30 @@ static PyObject* PyXmlSec_TreeAddIds(PyObject* self, PyObject *args, PyObject *k n = PyObject_Length(ids); if (n < 0) goto ON_FAIL; + // Shadow mode: registering IDs on lxml's document with our libxml2 is + // exactly the cross-library write issue #356 forbids. Record the + // attribute names instead; every whole-document shadow (sign, verify, + // decrypt) replays them onto its private copy. The replay scans the whole + // copy rather than just this subtree — a superset of the registration. + if (PyXmlSec_LxmlShadowIsActive()) { + for (i = 0; i < n; ++i) { + const char* name; + key = PyLong_FromSsize_t(i); + if (key == NULL) goto ON_FAIL; + tmp = PyObject_GetItem(ids, key); + Py_DECREF(key); + if (tmp == NULL) goto ON_FAIL; + name = PyUnicode_AsUTF8(tmp); + if (name == NULL || PyXmlSec_LxmlShadowRecordId(node, name, NULL) < 0) { + Py_DECREF(tmp); + goto ON_FAIL; + } + Py_DECREF(tmp); + } + PYXMLSEC_DEBUG("tree add_ids - ok"); + Py_RETURN_NONE; + } + list = (const xmlChar**)xmlMalloc(sizeof(xmlChar*) * (n + 1)); if (list == NULL) { PyErr_SetString(PyExc_MemoryError, "no memory"); diff --git a/tests/test_templates.py b/tests/test_templates.py index 09567fce..70176772 100644 --- a/tests/test_templates.py +++ b/tests/test_templates.py @@ -39,6 +39,10 @@ def test_ensure_key_info(self): def test_ensure_key_info_existing(self): root = self.load_xml('doc.xml') sign = xmlsec.template.create(root, c14n_method=consts.TransformExclC14N, sign_method=consts.TransformRsaSha1) + # attach the template the way real callers do; a created template is + # detached (on the shadow path it also lives in its own document + # until grafted, so liveness is only observable through `root`) + root.append(sign) ki = xmlsec.template.ensure_key_info(sign) self.assertIs(ki.getroottree().getroot(), root) # the second call finds the existing node instead of adding another one, @@ -107,6 +111,10 @@ def test_add_reference_fail(self): def test_add_transform(self): root = self.load_xml('doc.xml') sign = xmlsec.template.create(root, c14n_method=consts.TransformExclC14N, sign_method=consts.TransformRsaSha1) + # attach the template the way real callers do; a created template is + # detached (on the shadow path it also lives in its own document + # until grafted, so liveness is only observable through `root`) + root.append(sign) ref = xmlsec.template.add_reference(sign, consts.TransformSha1, uri='URI') tr = xmlsec.template.add_transform(ref, consts.TransformEnveloped) # the returned node is live in the original tree, inside the From 8a487f2af93e0b6aa7f70832170be2f18387efc3 Mon Sep 17 00:00:00 2001 From: Amin Solhizadeh Date: Fri, 4 Sep 2026 13:19:19 +0200 Subject: [PATCH 06/22] Unify the shadow reflection engine and prune its API (#356) Review pass over the shadow-copy branch. The design stands; the helper layer had grown two reflection engines (End for single-site template calls, ReflectAll for multi-site sign/encrypt) plus five single-use entry points. Collapse them into one engine so every binding uses the same few lines: - One reflection walk: fresh nodes are grafted at their child index, and every parent that gained a node has its text slots (.text and the children's .tail) synced wholesale from the re-parsed copy. The sync replaces the old per-element "mirror" heuristic and also covers text the call removed (encrypt Type=Content), which no fresh-node scan can see. - End maps the result node back after reflecting: grafted, or found (attributes synced; a renamed prefix swaps in the copy's version). It serves every Begin flavour, so EndNewDoc and EndReplace go away. - Reflect(shadow, rv, error) ends status-returning calls in one line (sign, encrypt_binary, encrypt_uri, C14N inclusive namespaces); FindFresh is gone. - BeginDoc replays the registered IDs itself; ReplayIds and DumpCopy become static. Header: 15 -> 10 functions. - Our re-parse uses XML_PARSE_HUGE and a cached huge_tree lxml parser, so a CipherValue above libxml2's 10 MB text-node limit reflects (12 MB encrypt_binary/decrypt round trip verified); path depth 256. - The enc.c shadow bodies clear XMLSEC_ENC_RETURN_REPLACED_NODE explicitly: xmlsec must free replaced nodes with our libxml2 before the copy is discarded (it already did, implicitly). Also fixes a pre-existing crash on the raw path, found by the new tests: encrypt_xml with Type=Content on text or mixed content put text nodes into xmlsec's replaced-node list, and PyXmlSec_ClearReplacedNodes handed them to lxml's elementFactory; lxml frees the text siblings that follow an element when its proxy is released, so the next list entry was freed under our feet. Each node is now severed from the chain before release and non-element nodes are freed directly. Tests: decrypt of Type=Content with whitespace around EncryptedData and with mixed content, register_id sign/verify round trip, prefix rename on a live KeyInfo. Docs consolidated into developer.md (356-summary.md and converting-functions.md removed). Validated: real 2.14.6/2.15.3 mismatch 292 passed / 6 skipped, 10k loop with flat RSS and byte-identical output; matched static wheel 304 / 6 on both the fast path and PYXMLSEC_FORCE_SHADOW=1. --- 356-summary.md | 103 --- converting-functions.md | 133 ---- developer.md | 371 +++++++---- src/ds.c | 27 +- src/enc.c | 58 +- src/lxml.c | 1315 +++++++++++++++++++-------------------- src/lxml.h | 153 +++-- src/template.c | 28 +- tests/test_ds.py | 24 + tests/test_enc.py | 34 + tests/test_templates.py | 13 + 11 files changed, 1072 insertions(+), 1187 deletions(-) delete mode 100644 356-summary.md delete mode 100644 converting-functions.md diff --git a/356-summary.md b/356-summary.md deleted file mode 100644 index 95019b92..00000000 --- a/356-summary.md +++ /dev/null @@ -1,103 +0,0 @@ -# Shadow copies for #356 — what was done, at a high level - -**TL;DR** — `python-xmlsec` crashes when `lxml` and `xmlsec1` are built -against different `libxml2` versions, because it passes raw libxml2 node -pointers between them. This branch makes each xmlsec call run on a private -*copy* of the element and reflects the result back afterwards, so only -serialized bytes ever cross the boundary. Three template functions — one per -mutation shape — are converted; converting the rest is a four-line edit each. - -## The problem - -`python-xmlsec` glues together two libraries that both build on **libxml2**: -lxml (the XML tree the user edits in Python) and xmlsec1 (the C library that -signs/encrypts). The extension reaches into an lxml `_Element` for its raw -`xmlNodePtr` and hands it to xmlsec1. That is only safe when both libraries -link the *same* libxml2 at runtime — and they often don't, because lxml wheels -bundle their own. Two libxml2 builds touching one tree means mismatched struct -layouts and allocators: segfaults, double-frees, wrong signatures -([#356](https://github.com/xmlsec/python-xmlsec/issues/356)). The only -existing mitigation was refusing to import on a version mismatch (#283). - -## The idea: shadow copies - -Bytes have no ABI. So instead of sharing nodes, each converted binding now: - -``` - lxml element ──(lxml's libxml2 serializes)──► bytes - bytes ──(our libxml2 parses)──► private "shadow" copy - xmlsec mutates the shadow (it never sees an lxml node) - shadow ──(our libxml2 dumps)──► bytes ──(lxml parses)──► change grafted - into the live tree -``` - -The user-visible behaviour is unchanged: the input element gains exactly what -xmlsec added, the returned node is live in their tree (so incremental building -like `add_transform(ref, ...)` keeps working), and the serialized output stays -byte-identical — namespaces and xmlsec's `"\n"` formatting included. - -## What the change consists of - -- **One helper pair** in `src/lxml.c`/`src/lxml.h`: - `PyXmlSec_LxmlShadowBegin` (element → shadow copy) and - `PyXmlSec_LxmlShadowEnd` (reflect the mutation back, return the lxml node). - `Begin` tags every pre-existing node of the copy through libxml2's private - field, which lets `End` *discover* what the call did instead of being told — - that is what makes the reflection generic. -- **Three bindings converted** in `src/template.c`, deliberately one per - mutation shape so the helper is proven against all of them: - - `add_reference` — plain "add a subtree", - - `add_transform` — also creates an intermediate `` wrapper at a - chosen position, - - `ensure_key_info` — find-or-create: returns the existing node (attributes - synced) instead of duplicating it. - A conversion is four lines: `Begin` / the unchanged xmlsec call on - `shadow.root` / `End` — no per-function callback or context struct. -- **A fast path** decided once at import: when lxml links the same libxml2 as - the extension (the only configuration the import guard lets run today), - `Begin`/`End` skip the copy entirely and behave exactly like the old direct - code — zero overhead. The shadow round-trip activates under a mismatch, or - with `PYXMLSEC_FORCE_SHADOW=1`, which CI uses to keep the shadow path - exercised on matched libraries. -- **An escape hatch** for development: `PYXMLSEC_SKIP_VERSION_CHECK` bypasses - the import-time mismatch guard so the converted paths can be exercised under - a real mismatch. The guard itself stays on by default. -- **Tests** in `tests/test_templates.py` asserting the reflection semantics - (liveness, position, no duplication on repeated ensure). -- **Two docs**: [developer.md](developer.md) — the design and the build/ - validation recipe; [converting-functions.md](converting-functions.md) — the - step-by-step guide for converting the remaining functions. - -## Why this design (vs. the first attempt) - -An earlier branch (`fix/356-decouple-add-reference`) proved the -serialize-across-the-boundary idea but required a callback function plus a -context struct per converted binding, and its reflection only handled the -"append exactly one child" shape — `ensure_key_info` and `add_transform` -would have needed helper extensions. The shadow design inverts control: the -call site stays a plain xmlsec call, and the helper works out what changed by -diffing tagged vs. untagged nodes. Result: less code overall, zero -per-function boilerplate, and all `xmlSecTmpl*` shapes covered by one -mechanism. - -## Validation - -Exercised under a **real** libxml2 mismatch (lxml bundling 2.14.6, extension + -libxmlsec1 on homebrew 2.15.3): - -- full test suite: **288 passed, 6 skipped**, including the per-test leak - detector, across repeated runs; -- 10,000-iteration loop over all three converted functions: no crash, no RSS - growth, byte-identical output every iteration. - -> Caveat (unchanged from before): this decouples *lxml* from xmlsec. The -> extension and `libxmlsec1` must still share one libxml2 — which wheels and -> static builds guarantee. - -## What's left - -The rest of `src/template.c` is a mechanical rollout of the four-line pattern -(see the guide). `src/ds.c` (sign/verify), `src/enc.c` (encrypt/decrypt) and -`src/tree.c` operate on whole documents and need a reflect strategy of their -own on top of the same `Begin` machinery. The import-time version guard can -only be relaxed once every node-passing path is converted. diff --git a/converting-functions.md b/converting-functions.md deleted file mode 100644 index a3906a3c..00000000 --- a/converting-functions.md +++ /dev/null @@ -1,133 +0,0 @@ -# Converting a function to the shadow-copy pattern — step by step - -This is the how-to companion to [developer.md](developer.md) (which explains -*why* the shadow copy exists and how the reflection works). Follow these steps -to move one more binding off the raw-node path for -[#356](https://github.com/xmlsec/python-xmlsec/issues/356). - -## 1. Pick a function - -Anything still touching `node->_c_node` / `node->_doc->_c_doc` is on the raw -path: - -```sh -grep -n '_c_node\|_c_doc' src/template.c -``` - -## 2. Classify the xmlsec call - -Read the `xmlSecTmpl*` function it wraps (xmlsec1's `src/templates.c`) and -match it to a row: - -| Shape | Examples | How | -|---|---|---| -| Adds a subtree under the element (any position, possibly with intermediate nodes) | `add_key_name`, `add_key_value`, `add_x509_data`, `x509_data_add_*`, `add_encrypted_key` | ✅ just follow step 3 | -| Find-or-create, may set attributes on the existing node | `encrypted_data_ensure_key_info`, `encrypted_data_ensure_cipher_value` | ✅ just follow step 3 (a call that can also *rename the prefix* of the existing node uses `EndReplace` instead of `End`) | -| Returns a status `int`, mutates one child | `transform_add_c14n_inclusive_namespaces` | ✅ pass `PyXmlSec_LxmlShadowFindFresh(&shadow)` to `End`; discard the returned element and return `None` | -| Returns a **detached** node — the element argument only supplies the document | `create`, `encrypted_data_create` | ✅ `BeginNewDoc`/`EndNewDoc`: the call runs against a private document and the result comes back as a new detached lxml element | -| Read-only search | `tree.c` find_child/find_node (subtree), find_parent (whole doc) | ✅ `Begin` (or `BeginDoc` when the search leaves the subtree) + `EndFind`, which returns `None` on not-found | -| Reads or mutates the **whole document**, possibly at several places | `ds.c` sign (whole-doc + multi-site), verify (read-only) | ✅ `BeginDoc` + `ReplayIds`, then `ReflectAll` (sign) or `Discard` (verify) | -| **Replaces** nodes | `enc.c` encrypt_xml/decrypt (`encrypt_binary`/`encrypt_uri` are template-shaped: `Begin` + `ReflectAll`) | ✅ `BeginDoc`, remove the consumed live node/content first, then `ReflectAll` grafts the replacement (`_setroot` when the document root itself was replaced) | - -All shapes are converted; `developer.md`'s "Beyond templates" section explains -each reflect. This guide stays as the recipe should new bindings appear. - -`res` does not have to be the topmost node the call created — `End` walks up -to the topmost new ancestor itself. Any node inside the new subtree works. - -## 3. Edit the binding - -The change is mechanical; `add_key_name` as the worked example: - -```c - PyXmlSec_LxmlElementPtr node = NULL; - const char* name = NULL; - xmlNodePtr res; -+ PyObject* result; -+ PyXmlSec_LxmlShadow shadow; - - PYXMLSEC_DEBUG("template add_key_name - start"); - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&|z:add_key_name", kwlist, - PyXmlSec_LxmlElementConverter, &node, &name)) - { - goto ON_FAIL; - } - -+ if (PyXmlSec_LxmlShadowBegin(&shadow, node) < 0) { -+ goto ON_FAIL; -+ } - Py_BEGIN_ALLOW_THREADS; -- res = xmlSecTmplKeyInfoAddKeyName(node->_c_node, XSTR(name)); -+ res = xmlSecTmplKeyInfoAddKeyName(shadow.root, XSTR(name)); - Py_END_ALLOW_THREADS; -- if (res == NULL) { -- PyXmlSec_SetLastError("cannot add key name."); -+ result = PyXmlSec_LxmlShadowEnd(&shadow, res, "cannot add key name."); -+ if (result == NULL) { - goto ON_FAIL; - } - - PYXMLSEC_DEBUG("template add_key_name - ok"); -- return (PyObject*)PyXmlSec_elementFactory(node->_doc, res); -+ return result; -``` - -Rules the pattern must keep: - -- swap `node->_c_node` for `shadow.root` and change **nothing else** about the - xmlsec call or its error string; -- run **exactly one** xmlsec call between `Begin` and `End`, and no Python - code (the `Py_*_ALLOW_THREADS` pair is fine — the call is pure C); -- call `End` **exactly once** after a successful `Begin`; it frees the copy on - every path, including when `res == NULL`. - -## 4. Build - -```sh -python setup.py build_ext --inplace --force -PYTHONPATH=src python -m pytest tests/ -``` - -(On a homebrew Mac the plain build links mismatched libxml2s — see -"Building & validating under a real mismatch" in [developer.md](developer.md) -for the `PKG_CONFIG_PATH` + `install_name_tool` recipe.) - -## 5. Add a targeted test - -Existing tests cover return values; add one asserting the *reflection*, in -`tests/test_templates.py`: - -- the returned node is live in the caller's tree - (`self.assertIs(kn.getroottree().getroot(), root)`) and at the position - xmlsec puts it; -- for find-or-create: a second call returns the same element - (`assertIs`), sets the requested attributes on it, and does not duplicate it. - -Beware the leak detector in `tests/base.py`: it reruns each test with -`gc.disable()` and fails on monotonic object-count growth, which plain -allocation churn can trigger with no real leak. Keep each test small (split -rather than combine scenarios), prefer `assertIs(parent[0], tr)` over building -lists to compare, and check stability with a few rounds of: - -```sh -PYXMLSEC_TEST_ITERATIONS=50 PYTHONPATH=src python -m pytest tests/test_templates.py -``` - -## 6. Validate under a real libxml2 mismatch - -Build per the developer.md recipe so lxml and the extension report different -libxml2 versions, then: - -```sh -PYXMLSEC_SKIP_VERSION_CHECK=1 PYTHONPATH=src python -m pytest tests/ -``` - -For anything non-trivial, also loop the converted function ~10k times under -the mismatch and watch `ru_maxrss` stays flat and the serialized output stays -byte-identical between iterations. - -## 7. Record it - -Move the function to the ✅ list in [developer.md](developer.md)'s Status -section. Once nothing passes raw nodes anymore, the import-time version guard -can be relaxed. diff --git a/developer.md b/developer.md index 77c2c7d7..551de773 100644 --- a/developer.md +++ b/developer.md @@ -1,134 +1,247 @@ # Decoupling lxml and xmlsec across libxml2 (#356) +**TL;DR** — `python-xmlsec` crashes when `lxml` and `xmlsec1` are built +against different `libxml2` versions, because it passes raw libxml2 node +pointers between them. Every xmlsec call now runs on a private *copy* of the +element ("shadow") owned by our libxml2, and the change it makes is reflected +back into the live lxml tree afterwards, so only serialized bytes ever cross +the boundary. Converting a binding is four lines; when the libxml2 versions +match, the copy is skipped and the old direct code runs unchanged. + +## The problem + `python-xmlsec` glues together two libraries that both build on **libxml2**: -lxml (the tree the user edits) and xmlsec1 (the C library that signs/encrypts). -Historically the extension reached into an lxml `_Element` for its raw -`xmlNodePtr` and handed it to xmlsec1. That is only safe when both libraries -link the *same* libxml2 at runtime — and they often don't (lxml wheels bundle -their own). Mixing two libxml2 builds on one tree corrupts memory: segfaults, +lxml (the tree the user edits in Python) and xmlsec1 (the C library that +signs/encrypts). Historically the extension reached into an lxml `_Element` +for its raw `xmlNodePtr` and handed it to xmlsec1. That is only safe when both +libraries link the *same* libxml2 at runtime — and they often don't, because +lxml wheels bundle their own. Two libxml2 builds touching one tree means +mismatched struct layouts, dictionaries and allocators: segfaults, double-frees, wrong signatures -([#356](https://github.com/xmlsec/python-xmlsec/issues/356)). The only guard -was refusing to import on a version mismatch (#283). +([#356](https://github.com/xmlsec/python-xmlsec/issues/356)). The only +mitigation so far was refusing to import on a version mismatch (#283). ## The fix: shadow copies -Never share nodes; share **bytes**. Each xmlsec call runs on a private, -throwaway copy of the element ("shadow") owned by *our* libxml2, and the -change it makes is reflected back into the live lxml tree afterwards — again -via bytes. Implemented as one pair of helpers in [src/lxml.c](src/lxml.c) -(contract in [src/lxml.h](src/lxml.h)); converting a function is four lines, -with no per-function callback or context struct: +Bytes have no ABI. Each binding therefore does: + +```text + lxml element ──(lxml's libxml2 serializes)──► bytes + bytes ──(our libxml2 parses)──► private "shadow" copy + xmlsec mutates the shadow (it never sees an lxml node) + shadow ──(our libxml2 dumps)──► bytes ──(lxml parses)──► changes grafted + into the live tree +``` + +The user-visible behaviour is unchanged: the input element gains exactly what +xmlsec added, the returned node is live in the caller's tree (incremental +building like `add_transform(ref, ...)` keeps working, and proxies the caller +holds stay valid), and the serialized output is byte-identical — namespaces +and xmlsec's `"\n"` formatting included. + +The whole mechanism lives in [src/lxml.c](src/lxml.c), with the contract in +[src/lxml.h](src/lxml.h). A binding looks like this (`add_reference`): ```c PyXmlSec_LxmlShadow shadow; -if (PyXmlSec_LxmlShadowBegin(&shadow, node) < 0) goto ON_FAIL; // lxml → bytes → our copy +if (PyXmlSec_LxmlShadowBegin(&shadow, node) < 0) goto ON_FAIL; // lxml → bytes → our copy Py_BEGIN_ALLOW_THREADS; -res = xmlSecTmplSignatureAddReference(shadow.root, ...); // xmlsec mutates the copy +res = xmlSecTmplSignatureAddReference(shadow.root, ...); // xmlsec mutates the copy Py_END_ALLOW_THREADS; result = PyXmlSec_LxmlShadowEnd(&shadow, res, "cannot add reference."); // reflect back, return lxml node ``` -`Begin` serializes the element with lxml's own `etree.tostring` and re-parses -the bytes with `xmlReadMemory`, tagging every pre-existing node through the -libxml2 `_private` field. `End` walks up from `res` to find the topmost -untagged (= new) node and reflects generically, covering every shape in the -`xmlSecTmpl*` family: +No per-function callback or context struct: the call site is the plain xmlsec +call, and the helper works out what the call did. + +## The API + +Three `Begin` flavours make the copy, four `End` functions consume it. Every +End always releases the copy, on success and on error. + +| Function | Use | +| --- | --- | +| `Begin(&shadow, element)` | copy of the element's subtree; `shadow.root` is the copy | +| `BeginDoc(&shadow, element, &target)` | copy of the element's whole document, for calls that follow references or walk upward; `target` is the copy's counterpart of `element`; registered IDs are replayed onto the copy | +| `BeginNewDoc(&shadow, element)` → `xmlDocPtr` | no copy at all: a private document for calls that only build a *detached* subtree (`create`) | +| `End(&shadow, res, error)` → element | the lxml element for a result node — grafted into the live tree if the call created it, or the existing live element (with the attributes / prefix the call changed) if it found it; NULL `res` raises `error` | +| `EndFind(&shadow, res)` → element or `None` | the same for read-only finders; NULL is "not found" | +| `Reflect(&shadow, rv, error)` → int | for calls returning only a status: `rv < 0` raises `error`, otherwise every change is reflected | +| `Discard(&shadow)` | release without reflecting (read-only calls, error paths before End) | + +Plus three helpers for the call sites whose *semantics* differ per mode: +`IsActive()` (which path is on), `ImportElement()` (encrypt_xml's template +import into the shadow document) and `RecordId()` (ID registration, below). + +Which pair a binding uses follows from what the xmlsec call does: + +| The xmlsec call ... | Begin | End | Bindings | +| --- | --- | --- | --- | +| adds or finds a node under the element | `Begin` | `End` | every `xmlSecTmpl*` add / ensure call | +| builds a detached subtree, needs only a document | `BeginNewDoc` | `End` | `create`, `encrypted_data_create` | +| searches the subtree, read-only | `Begin` | `EndFind` | `find_child`, `find_node` | +| searches upward, read-only | `BeginDoc` | `EndFind` | `find_parent` | +| mutates the subtree, returns a status | `Begin` | `Reflect` | `transform_add_c14n_inclusive_namespaces`, `encrypt_binary`, `encrypt_uri` | +| mutates anywhere in the document, returns a status | `BeginDoc` | `Reflect` | `sign` | +| reads the document, returns a status | `BeginDoc` | `Discard` | `verify` | +| *replaces* the element or its content | `BeginDoc` | remove the consumed live node/content, then `Reflect` | `encrypt_xml`, `decrypt` | + +Rules every call site must keep: + +- swap `node->_c_node` for `shadow.root` (or `target`) and change **nothing + else** about the xmlsec call or its error string; +- run **exactly one** xmlsec call between Begin and End, and no Python code + in between (the `Py_*_ALLOW_THREADS` pair is fine — the call is pure C); +- call exactly one End function after a successful Begin. + +## How the reflection works -- **plain add** (`add_reference`): the new subtree is grafted into the live - tree at the same position, located by child-index path. -- **intermediate ancestors** (`add_transform` creating `` around - the ``): the *topmost* new node is grafted; the returned element - is the descendant matching `res`. -- **find-or-create** (`ensure_key_info`): nothing new in the tree — the - existing live element is returned, plus any attributes the call set (`Id`). +`Begin` serializes the element with lxml's own `etree.tostring`, re-parses +the bytes with `xmlReadMemory`, and tags every node of the copy through the +libxml2 `_private` field (never serialized, never touched by the parser or +xmlsec). After the call, whatever is untagged is what the call created. The +reflection then walks the tagged structure of the copy in document order and +records two kinds of *site*: + +- **graft** — a fresh node (element, comment, PI) to insert at its child + index. Fresh subtrees are grafted wholesale; the scan never descends into + them. +- **sync** — a tagged parent that gained any fresh node (element or text) + gets its text slots (its `.text` and each child's `.tail`) copied over from + the re-parsed copy. That covers everything xmlsec does to text: the `"\n"` + formatting around a new node, values filled into empty elements + (`DigestValue`, `SignatureValue`), and content it removed (encrypt + `Type=Content`) — a removal leaves no fresh node behind, so only a + wholesale sync can see it. + +Sites are addressed by **child-index paths** from the copy root, counting +exactly the node types lxml exposes as children (elements, comments, PIs, +entity refs), so a path recorded on the raw copy resolves identically through +lxml's `__getitem__` / `insert` on the live tree. The reflection is +**two-phase**: every payload is fetched from the re-parsed copy first, while +it is still in its final state, then everything is applied to the live tree in +document order (a graft moves a node out of the re-parsed copy, which would +invalidate later fetches; each live insert makes the later, larger indices +valid; a parent's sync is recorded after its grafts). Two serialization details are load-bearing for byte-identical signatures: +the **whole** copy is dumped (`xmlDocDumpMemory`), not just the fresh nodes, +so ancestor-declared namespaces and the formatting siblings survive the lxml +re-parse without any manual fix-up; and lxml's `insert` carries a node's tail +along and reconciles namespaces against the live ancestry. -- `End` dumps the **whole** mutated copy (`xmlDocDumpMemory`), not just the new - node, so ancestor-declared namespaces (dsig on ``) and xmlsec's - `"\n"` formatting siblings survive the lxml re-parse with no manual fix-up; - the new node's tail travels with it through `insert()`. -- xmlsec may also emit a `"\n"` *before* the new node (`xmlSecAddChild` / - `AddNextSibling` / `AddPrevSibling`); `End` mirrors that one text slot - (parent `.text` or previous sibling `.tail`) from the copy. - -Child indices count exactly the node types lxml exposes as children (elements, -comments, PIs, entity refs), so paths recorded on the raw copy resolve -identically through lxml's `__getitem__`/`insert`. `End` assumes the xmlsec -call mutates at most one place in the tree — true for all `xmlSecTmpl*` -functions. - -## Beyond templates: the whole-code rollout - -Every binding that used to hand a raw lxml node to xmlsec now goes through a -shadow. The extra shapes (all in [src/lxml.c](src/lxml.c), contracts in -[src/lxml.h](src/lxml.h)): - -- **Create** (`template.create`, `encrypted_data_create`): - `BeginNewDoc`/`EndNewDoc`. The call builds a *detached* subtree and only - needs a document to allocate in — a private one on the shadow path. The - result comes back as a new detached lxml element (in its own document; lxml - moves it when the caller grafts it), instead of the raw path's "detached - node inside the source document", which lxml's API cannot express. -- **Finders** (`tree.find_child`/`find_node`/`find_parent`): `EndFind` maps - the found copy node back by path and returns `None` on not-found. - `find_parent` walks upward, so it uses the whole-document Begin. -- **Whole-document** (`sign`, `verify`, `decrypt`, `find_parent`): - `BeginDoc` serializes `element.getroottree()` (comments/PIs outside the - root and the internal DTD subset survive), records the element's position - through lxml's API, and hands back the copy's counterpart node. -- **Multi-site reflect** (`sign`, `encrypt_binary`, `encrypt_uri`): - `ReflectAll` scans the copy for *every* topmost untagged node and grafts - each back — new subtrees via `insert`, new/changed text (DigestValue, - SignatureValue) via the text slots. Two-phase: payloads are fetched from - the re-parsed copy while it is still in its final state, then applied to - the live tree in document order (a graft moves a node out of the re-parsed - copy, which would invalidate later fetches). -- **Replacement reflect** (`encrypt_xml`, `decrypt`): encryption/decryption - *replace* nodes, so the live target (or its content, or the document root - via `_setroot`) is removed first and `ReflectAll` grafts what took its - place. `encrypt_xml` re-serializes the template into the same shadow doc - (`ImportElement`); a template attached inside the target's own tree is - therefore copied, not moved. `verify` needs no reflect at all — `Discard` - just frees the copy. -- **ID registration** (`tree.add_ids`, `SignatureContext.register_id`): these - used to write lxml's ID hash with our libxml2. Under the shadow they record - the id-attribute specs in a registry keyed by document identity - (`RecordId`), and every `BeginDoc` replays them onto the copy - (`ReplayIds`) so `#id` references resolve. The replay scans the whole copy - for the recorded attribute names — a superset of the raw registration. The - registry holds no strong references to documents (lxml objects refuse weak - references, so entries are validated by a stored `_c_doc` address and - capped in size). -- **Prefix rename** (`encrypted_data_ensure_key_info(ns=...)` on an existing - KeyInfo): `EndReplace` swaps the live element for the copy's version, since - lxml cannot rename a prefix in place; the returned element is then a new - object rather than the original proxy. +`End` then maps the result node back: it records the path of `res` in the +copy before reflecting, and after the reflection the live tree mirrors the +copy's element structure, so the same path resolves to the live counterpart — +whether the call created it (now grafted) or found it. For a found node the +tree did not grow there, so the live element is returned with the attributes +the call set (`Id`) synced onto it; a renamed namespace prefix +(`encrypted_data_ensure_key_info(ns=...)` on an existing `KeyInfo`) has no +lxml API, so the live element is swapped for the copy's version and the +caller gets a new proxy object. + +`BeginDoc` records the element's position through lxml's API (`getparent` / +`index`), serializes `element.getroottree()` — comments/PIs outside the root +and the internal DTD subset survive — and hands back the copy's counterpart; +`shadow.element` becomes the live *root*, which is where the reflection maps +paths onto. `BeginNewDoc` creates an empty private document; `End` roots the +detached result there, dumps it and returns it as a new detached lxml element +(in a document of its own until grafted; lxml moves it when the caller +appends it, like the raw path's detached node). ## The fast path: shadows only when needed Copying is pointless when lxml links the same libxml2 as the extension — the -raw-node behavior that shipped for years is safe then, and it is the only -configuration the import guard currently lets run. So `Begin`/`End` are +raw-node behaviour that shipped for years is safe then, and it is the only +configuration the import guard currently lets run. `Begin`/`End` are dual-path, decided once at import: - **matched versions** (the guard passed): `Begin` aliases the live - `_c_node` into `shadow.root` with no serialization, and `End` just wraps - the node xmlsec returned — machine-identical to the pre-shadow code, zero - overhead; + `_c_node` into `shadow.root` with no serialization, `End` just wraps the + node xmlsec returned, `Reflect` does nothing — machine-identical to the + pre-shadow code, zero overhead; - **mismatch** (import allowed via `PYXMLSEC_SKIP_VERSION_CHECK` today, - automatic once everything is converted), **or `PYXMLSEC_FORCE_SHADOW` set**: - the full shadow round-trip described above. + automatic once the guard becomes a mode switch), **or `PYXMLSEC_FORCE_SHADOW` + set**: the full shadow round-trip. + +Call sites cannot tell the difference. `PYXMLSEC_FORCE_SHADOW` exists so CI +keeps the shadow path exercised on matched libraries (the workflows run the +suite twice), where it must also pass the full suite. Measured cost of the +shadow path per template call: about 8x (72 µs vs 8.6 µs for create + +add_reference + add_transform + ensure_key_info); whole-document operations +scale with document size. -Call sites cannot tell the difference; every converted function inherits both -paths. `PYXMLSEC_FORCE_SHADOW` exists so CI keeps the shadow path exercised on -matched libraries (see the test matrix), where it must also pass the full -suite. +The re-parse on our side uses `XML_PARSE_HUGE` and the lxml side a cached +`XMLParser(huge_tree=True)`, so a `CipherValue` above libxml2's 10 MB +text-node limit (large `encrypt_binary` payloads) or a document the user +parsed with `huge_tree` still reflects. That is safe: what gets parsed is +lxml's own dump of a tree it already parsed. > The shadow decouples *lxml* from xmlsec. The extension and `libxmlsec1` -> must still share one libxml2 (wheels/static builds guarantee this). +> must still share one libxml2 (wheels and static builds guarantee this). + +## ID registration under the shadow + +`SignatureContext.register_id` and `tree.add_ids` used to write lxml's ID +hash with our libxml2 — exactly the cross-library access the shadow forbids. +Under the shadow they record the id-attribute specs in a registry keyed by +document identity (`RecordId`), and every `BeginDoc` replays them onto its +copy so that `#id` references resolve during sign/verify/decrypt. The replay +scans the whole copy for the recorded attribute names — a superset of the +single-node registration on the raw path, mirroring what `xmlSecAddIDs` does +from the root. The registry holds no strong references to documents (lxml's +classes refuse weak references), so entries are validated by a stored +`_c_doc` address and capped in size. The two bindings are the only places, +together with encrypt_xml/decrypt's replacement bodies, that branch on +`IsActive()`. + +## Converting a binding + +1. Classify the xmlsec call with the table above. +2. Edit: swap `node->_c_node` for `shadow.root`, wrap the call in the + matching Begin/End pair, keep the error string. +3. Build and run the suite (see below). Add a test asserting the + *reflection*: the returned node is live in the caller's tree + (`assertIs(node.getroottree().getroot(), root)`) and at the position + xmlsec puts it; for find-or-create, a second call returns the same element. +4. Validate under a real libxml2 mismatch, and on a matched build with + `PYXMLSEC_FORCE_SHADOW=1`. -## Building & validating under a real mismatch (macOS / homebrew) +Beware the leak detector in `tests/base.py`: it reruns each test with +`gc.disable()` and fails on monotonic object-count growth, which plain +allocation churn can trigger with no real leak. Keep each test small (split +rather than combine scenarios), prefer `assertIs(parent[0], tr)` over +building lists to compare, and check stability with +`PYXMLSEC_TEST_ITERATIONS=50 PYTHONPATH=src python -m pytest tests/`. + +## Known divergences and limitations (shadow path only) + +All invisible to the documented API: + +- created templates (`create`, `encrypted_data_create`) live in their own + document until grafted; +- `encrypted_data_ensure_key_info(ns=...)` on an existing `KeyInfo` returns + a new element object rather than the original proxy; +- `register_id` skips the live duplicate-id check (it runs per copy instead) + and, without `id_ns`, looks the attribute up namespace-strictly where the + raw `xmlHasProp` is namespace-agnostic; +- `encrypt_xml` *copies* a template that is attached inside the target tree + rather than moving it, so it also remains at its original position; +- signature/encryption contexts keep no live result nodes after the call + (they never usefully did); +- documents nested deeper than 256 levels (only possible with `huge_tree`) + are refused with an internal error. + +One hard limitation: operations that would replace the **document root** +(encrypting the root element with `Type=Element`, decrypting a root +`EncryptedData`) raise `xmlsec.Error` — lxml's API cannot swap a document's +root, and morphing it in place would rewrite namespace prefixes, breaking +signatures over the content. Re-parse the document into a wrapper or work on +a subelement instead. + +## Building & validating + +### Under a real mismatch (macOS / homebrew) lxml wheels bundle libxml2; build the extension against homebrew's (which libxmlsec1 links) so only lxml differs — the true #356 scenario. Watch the @@ -149,41 +262,33 @@ PYXMLSEC_SKIP_VERSION_CHECK=1 PYTHONPATH=src python -c \ PYXMLSEC_SKIP_VERSION_CHECK=1 PYTHONPATH=src python -m pytest tests/ ``` -`PYXMLSEC_SKIP_VERSION_CHECK` bypasses the import-time mismatch guard. It -exists to exercise the shadow paths; it is **unsafe** for every operation -still on the raw-node path, so keep it off in normal use. The guard can only -be relaxed once all node-passing paths are converted. +`PYXMLSEC_SKIP_VERSION_CHECK` bypasses the import-time mismatch guard so the +shadow paths can be exercised; keep it off in normal use until the guard +becomes a mode switch. For anything non-trivial, also loop the converted +function ~10k times under the mismatch and check `ru_maxrss` stays flat and +the serialized output stays byte-identical between iterations. + +### On a matched build (static wheel) -On a *matched* build (no mismatch available), run the suite twice instead: -once plain (fast path) and once with `PYXMLSEC_FORCE_SHADOW=1` (shadow path -on matched libraries — safe everywhere, so the whole suite must pass). +`PYXMLSEC_STATIC_DEPS=true python -m build --wheel` bundles a libxml2 matched +to lxml's wheels; install it into a venv with wheel lxml +(`pip install --no-deps --force-reinstall dist/*.whl`) and run the suite twice +— plain (fast path) and with `PYXMLSEC_FORCE_SHADOW=1` (shadow path on +matched libraries; safe everywhere, so the whole suite must pass). Gotcha: +setuptools reuses stale objects from `build/`, so `rm -rf build/lib.* +build/temp.*` before switching between the dynamic in-place build and the +static wheel, or the wheel silently ships the old dynamically linked module +(it is then ~50 KB instead of several MB). ## Status -- ✅ All of `src/template.c` (create, references, transforms, key info, x509, - encrypted data, C14N namespaces). -- ✅ `src/tree.c` (find_child/find_node/find_parent, add_ids). -- ✅ `src/ds.c` (register_id, sign, verify; the binary operations never - touched nodes). -- ✅ `src/enc.c` (encrypt_binary, encrypt_uri, encrypt_xml, decrypt). -- Validated under a real 2.14 ↔ 2.15 mismatch: full suite green, 10k-iteration - sign/verify/encrypt/decrypt loop with no crash, no leak, byte-identical - output; and on a matched static build: full suite green on both the fast +- ✅ Every binding that hands a node to xmlsec goes through a shadow: + all of `src/template.c`, `src/tree.c`, `src/ds.c` and `src/enc.c`. +- ✅ Validated under a real 2.14 ↔ 2.15 mismatch (full suite, 10k-iteration + sign/verify/encrypt/decrypt loop with flat RSS and byte-identical output, + 12 MB binary round trip) and on a matched static build on both the fast path and `PYXMLSEC_FORCE_SHADOW=1`. -- ⬜ Endgame: turn the import-time guard into a mode switch (mismatch sets - the shadow flag instead of refusing to import) and retire - `PYXMLSEC_SKIP_VERSION_CHECK`. That is the actual user-facing resolution - of #356, kept as its own change. - -Known, deliberate divergences on the shadow path (all invisible to the -documented API): created templates live in their own document until grafted; -`encrypted_data_ensure_key_info(ns=...)` on an existing KeyInfo returns a new -element object; `register_id` skips the live duplicate-id check (it runs per -copy instead); `encrypt_xml` copies rather than moves a template that is -attached inside the target tree; signature/encryption contexts keep no live -result nodes after the call (they never usefully did). One hard limitation: -operations that would replace the **document root** (encrypting the root -element, decrypting a root `EncryptedData`) raise `xmlsec.Error` — lxml's API -cannot swap a document's root, and morphing it in place would rewrite -namespace prefixes, breaking signatures over the content. Re-parse the -document or work on a subelement instead. +- ⬜ Endgame, kept as its own change: turn the import-time guard into a mode + switch (a mismatch sets the shadow flag instead of refusing to import) and + retire `PYXMLSEC_SKIP_VERSION_CHECK`. That is the user-facing resolution of + #356. diff --git a/src/ds.c b/src/ds.c index 73299342..eff7a8a6 100644 --- a/src/ds.c +++ b/src/ds.c @@ -232,27 +232,18 @@ static PyObject* PyXmlSec_SignatureContextSign(PyObject* self, PyObject* args, P } // References (URI="", "#id") reach anywhere in the document, so the - // shadow covers the whole tree (issue #356); registered IDs are replayed - // onto the copy so they resolve. Signing fills several places inside - // (DigestValue, SignatureValue, KeyInfo), all reflected by - // the multi-site reflection. + // shadow covers the whole tree (issue #356), with the registered IDs + // replayed onto the copy. Signing fills several places inside + // (DigestValue, SignatureValue, KeyInfo); the reflect carries + // them all back. if (PyXmlSec_LxmlShadowBeginDoc(&shadow, node, &target) < 0) { goto ON_FAIL; } - if (PyXmlSec_LxmlShadowReplayIds(&shadow) < 0) { - PyXmlSec_LxmlShadowDiscard(&shadow); - goto ON_FAIL; - } Py_BEGIN_ALLOW_THREADS; rv = xmlSecDSigCtxSign(ctx->handle, target); PYXMLSEC_DUMP(xmlSecDSigCtxDebugDump, ctx->handle); Py_END_ALLOW_THREADS; - if (rv < 0) { - PyXmlSec_LxmlShadowDiscard(&shadow); - PyXmlSec_SetLastError("failed to sign"); - goto ON_FAIL; - } - if (PyXmlSec_LxmlShadowReflectAll(&shadow) < 0) { + if (PyXmlSec_LxmlShadowReflect(&shadow, rv, "failed to sign") < 0) { goto ON_FAIL; } PYXMLSEC_DEBUGF("%p: sign - ok", self); @@ -284,15 +275,11 @@ static PyObject* PyXmlSec_SignatureContextVerify(PyObject* self, PyObject* args, goto ON_FAIL; } - // Verification is read-only: whole-document shadow, replayed IDs, and no - // reflection at all — the copy is simply discarded. + // Verification is read-only: whole-document shadow (with the registered + // IDs replayed) and no reflection at all — the copy is simply discarded. if (PyXmlSec_LxmlShadowBeginDoc(&shadow, node, &target) < 0) { goto ON_FAIL; } - if (PyXmlSec_LxmlShadowReplayIds(&shadow) < 0) { - PyXmlSec_LxmlShadowDiscard(&shadow); - goto ON_FAIL; - } Py_BEGIN_ALLOW_THREADS; rv = xmlSecDSigCtxVerify(ctx->handle, target); PYXMLSEC_DUMP(xmlSecDSigCtxDebugDump, ctx->handle); diff --git a/src/enc.c b/src/enc.c index 4a0128ef..2f684e5b 100644 --- a/src/enc.c +++ b/src/enc.c @@ -177,8 +177,8 @@ static PyObject* PyXmlSec_EncryptionContextEncryptBinary(PyObject* self, PyObjec } // The encryption fills several places inside the template subtree - // (CipherValue, KeyInfo/EncryptedKey); the multi-site reflection carries - // them all back (issue #356). + // (CipherValue, KeyInfo/EncryptedKey); the reflect carries them all back + // (issue #356). if (PyXmlSec_LxmlShadowBegin(&shadow, template) < 0) { goto ON_FAIL; } @@ -187,12 +187,7 @@ static PyObject* PyXmlSec_EncryptionContextEncryptBinary(PyObject* self, PyObjec PYXMLSEC_DUMP(xmlSecEncCtxDebugDump, ctx->handle); Py_END_ALLOW_THREADS; - if (rv < 0) { - PyXmlSec_LxmlShadowDiscard(&shadow); - PyXmlSec_SetLastError("failed to encrypt binary"); - goto ON_FAIL; - } - if (PyXmlSec_LxmlShadowReflectAll(&shadow) < 0) { + if (PyXmlSec_LxmlShadowReflect(&shadow, rv, "failed to encrypt binary") < 0) { goto ON_FAIL; } Py_INCREF(template); @@ -214,12 +209,22 @@ static void PyXmlSec_ClearReplacedNodes(xmlSecEncCtxPtr ctx, PyXmlSec_LxmlDocume while (n != NULL) { PYXMLSEC_DEBUGF("clear replaced node %p", n); nn = n->next; - // if n has references, it will not be deleted - elem = (PyXmlSec_LxmlElementPtr)PyXmlSec_elementFactory(doc, n); - if (NULL == elem) + // Sever the chain first: lxml releases an element together with the + // text siblings that follow it, which would free the next node of + // this list under our feet (Type=Content replaces text nodes too). + n->next = NULL; + n->prev = NULL; + if (PyXmlSec_IsElement(n)) { + // if n has references, it will not be deleted + elem = (PyXmlSec_LxmlElementPtr)PyXmlSec_elementFactory(doc, n); + if (NULL == elem) + xmlFreeNode(n); + else + Py_DECREF(elem); + } else { + // text and CDATA nodes never have lxml proxies xmlFreeNode(n); - else - Py_DECREF(elem); + } n = nn; } ctx->replacedNodeList = NULL; @@ -282,6 +287,11 @@ static PyObject* PyXmlSec_EncryptionContextEncryptXmlShadow(PyXmlSec_EncryptionC goto ON_FAIL; } + // The replaced nodes belong to the private copy: xmlsec must free them + // itself (with our libxml2) rather than hand them back, because the copy + // is discarded right after and nothing could release them later. + ctx->handle->flags &= ~XMLSEC_ENC_RETURN_REPLACED_NODE; + Py_BEGIN_ALLOW_THREADS; rv = xmlSecEncCtxXmlEncrypt(ctx->handle, tmpl_copy, target); PYXMLSEC_DUMP(xmlSecEncCtxDebugDump, ctx->handle); @@ -327,7 +337,7 @@ static PyObject* PyXmlSec_EncryptionContextEncryptXmlShadow(PyXmlSec_EncryptionC } Py_CLEAR(tmp); } - if (PyXmlSec_LxmlShadowReflectAll(&shadow) < 0) { + if (PyXmlSec_LxmlShadowReflect(&shadow, 0, NULL) < 0) { goto ON_FAIL; } result = PySequence_GetItem((PyObject*)node, 0); @@ -342,7 +352,7 @@ static PyObject* PyXmlSec_EncryptionContextEncryptXmlShadow(PyXmlSec_EncryptionC goto ON_FAIL; } Py_CLEAR(tmp); - if (PyXmlSec_LxmlShadowReflectAll(&shadow) < 0) { + if (PyXmlSec_LxmlShadowReflect(&shadow, 0, NULL) < 0) { goto ON_FAIL; } result = PySequence_GetItem(parent, (Py_ssize_t)idx); @@ -491,12 +501,7 @@ static PyObject* PyXmlSec_EncryptionContextEncryptUri(PyObject* self, PyObject* PYXMLSEC_DUMP(xmlSecEncCtxDebugDump, ctx->handle); Py_END_ALLOW_THREADS; - if (rv < 0) { - PyXmlSec_LxmlShadowDiscard(&shadow); - PyXmlSec_SetLastError("failed to encrypt URI"); - goto ON_FAIL; - } - if (PyXmlSec_LxmlShadowReflectAll(&shadow) < 0) { + if (PyXmlSec_LxmlShadowReflect(&shadow, rv, "failed to encrypt URI") < 0) { goto ON_FAIL; } PYXMLSEC_DEBUGF("%p: encrypt_uri - ok", self); @@ -542,10 +547,6 @@ static PyObject* PyXmlSec_EncryptionContextDecryptShadow(PyXmlSec_EncryptionCont if (PyXmlSec_LxmlShadowBeginDoc(&shadow, node, &target) < 0) { goto ON_FAIL; } - if (PyXmlSec_LxmlShadowReplayIds(&shadow) < 0) { - PyXmlSec_LxmlShadowDiscard(&shadow); - goto ON_FAIL; - } // the Type decides the reflect shape; read it from the copy before the // decryption consumes the node @@ -553,6 +554,11 @@ static PyObject* PyXmlSec_EncryptionContextDecryptShadow(PyXmlSec_EncryptionCont not_content = (ttype == NULL || !xmlStrEqual(ttype, xmlSecTypeEncContent)); xmlFree(ttype); + // The replaced node belongs to the private copy: xmlsec must free it + // itself (with our libxml2) rather than hand it back, because the copy + // is discarded right after and nothing could release it later. + ctx->handle->flags &= ~XMLSEC_ENC_RETURN_REPLACED_NODE; + Py_BEGIN_ALLOW_THREADS; ctx->handle->mode = xmlSecCheckNodeName(target, xmlSecNodeEncryptedKey, xmlSecEncNs) ? xmlEncCtxModeEncryptedKey : xmlEncCtxModeEncryptedData; PYXMLSEC_DEBUGF("mode: %d", ctx->handle->mode); @@ -593,7 +599,7 @@ static PyObject* PyXmlSec_EncryptionContextDecryptShadow(PyXmlSec_EncryptionCont goto ON_FAIL; } Py_CLEAR(tmp); - if (PyXmlSec_LxmlShadowReflectAll(&shadow) < 0) { + if (PyXmlSec_LxmlShadowReflect(&shadow, 0, NULL) < 0) { goto ON_FAIL; } if (not_content) { diff --git a/src/lxml.c b/src/lxml.c index e266a017..d666ce69 100644 --- a/src/lxml.c +++ b/src/lxml.c @@ -105,10 +105,11 @@ static int PyXmlSec_CheckLxmlLibraryVersion(void) { // copy instead of directly on lxml's nodes; decided once at import, below. static int PyXmlSec_LxmlShadowActive = 1; -// The two lxml.etree callables the shadow crossings use, resolved once at -// import and kept for the lifetime of the process. +// The lxml.etree callables the shadow crossings use, resolved once at import +// and kept for the lifetime of the process. static PyObject* PyXmlSec_LxmlEtreeToString; static PyObject* PyXmlSec_LxmlEtreeFromString; +static PyObject* PyXmlSec_LxmlEtreeParser; // Shadow-mode ID registry: maps the identity of an lxml document to the list // of id-attribute specs registered for it (see PyXmlSec_LxmlShadowRecordId). @@ -118,13 +119,32 @@ int PyXmlSec_LxmlShadowIsActive(void) { return PyXmlSec_LxmlShadowActive; } +// etree.XMLParser(huge_tree=True), the parser for lxml's side of the reflect +// crossing. huge_tree lifts libxml2's 10 MB text-node limit, which lxml's +// default parser keeps, so a CipherValue above it (large encrypt_binary +// payloads) or a document the user parsed with huge_tree still reflects. It +// is safe here: what gets parsed is this extension's own dump of a tree lxml +// has already parsed. +static PyObject* PyXmlSec_LxmlNewParser(PyObject* etree) { + PyObject* result = NULL; + PyObject* cls = PyObject_GetAttrString(etree, "XMLParser"); + PyObject* args = PyTuple_New(0); + PyObject* kwargs = Py_BuildValue("{s:O}", "huge_tree", Py_True); + if (cls != NULL && args != NULL && kwargs != NULL) { + result = PyObject_Call(cls, args, kwargs); + } + Py_XDECREF(cls); + Py_XDECREF(args); + Py_XDECREF(kwargs); + return result; +} + int PyXmlSec_InitLxmlModule(void) { // By default refuse to import when lxml and xmlsec link different libxml2 // versions: passing raw nodes between the two libraries then corrupts // memory (https://github.com/xmlsec/python-xmlsec/issues/283). Setting // PYXMLSEC_SKIP_VERSION_CHECK bypasses the guard — needed to exercise the - // shadow-copy paths (issue #356) under a mismatch, but unsafe for every - // operation that still hands an lxml node to xmlsec. + // shadow-copy paths (issue #356) under a mismatch. int mismatch = PyXmlSec_CheckLxmlLibraryVersion() < 0; if (mismatch && getenv("PYXMLSEC_SKIP_VERSION_CHECK") == NULL) { PyXmlSec_SetLastError("lxml & xmlsec libxml2 library version mismatch"); @@ -144,8 +164,9 @@ int PyXmlSec_InitLxmlModule(void) { } PyXmlSec_LxmlEtreeToString = PyObject_GetAttrString(etree, "tostring"); PyXmlSec_LxmlEtreeFromString = PyObject_GetAttrString(etree, "fromstring"); + PyXmlSec_LxmlEtreeParser = PyXmlSec_LxmlNewParser(etree); Py_DECREF(etree); - if (PyXmlSec_LxmlEtreeToString == NULL || PyXmlSec_LxmlEtreeFromString == NULL) { + if (PyXmlSec_LxmlEtreeToString == NULL || PyXmlSec_LxmlEtreeFromString == NULL || PyXmlSec_LxmlEtreeParser == NULL) { return -1; } @@ -182,10 +203,11 @@ int PyXmlSec_LxmlElementConverter(PyObject* o, PyXmlSec_LxmlElementPtr* p) { // // Both crossings go through serialized bytes: lxml's own etree.tostring / // etree.fromstring on its side, xmlReadMemory / xmlDocDumpMemory on ours. -// End dumps the *whole* mutated copy, not just the new node: the surrounding -// markup carries the ancestor-declared namespaces and the "\n" formatting -// siblings xmlsec emits, so the reflected result stays byte-identical to the -// old raw-pointer code without any manual namespace or whitespace fix-up. +// Begin tags every node of the copy through the libxml2 _private field, so +// that after the xmlsec call whatever is untagged is what the call created; +// the reflection then grafts exactly those nodes (and the text slots the +// call filled) into the live lxml tree, addressed by child-index paths that +// resolve identically in both trees. // ---------------------------------------------------------------------------- // etree.tostring(element, with_tail=False) — lxml's own libxml2 walks the @@ -202,9 +224,36 @@ static PyObject* PyXmlSec_LxmlElementToBytes(PyObject* element) { return result; } -// etree.fromstring(data) — the parsed nodes are owned and managed by lxml. +// etree.fromstring(data, parser) — the parsed nodes are owned and managed by lxml. static PyObject* PyXmlSec_LxmlElementFromBytes(PyObject* data) { - return PyObject_CallFunctionObjArgs(PyXmlSec_LxmlEtreeFromString, data, NULL); + return PyObject_CallFunctionObjArgs(PyXmlSec_LxmlEtreeFromString, data, PyXmlSec_LxmlEtreeParser, NULL); +} + +// Our side of the crossing. NONET as always; HUGE lifts the 10 MB text-node +// and nesting limits so that nothing lxml accepted — or a CipherValue xmlsec +// is about to produce — is refused here. The bytes are lxml's own dump of an +// already-parsed tree, so the relaxed limits add no attack surface. +#define PYXMLSEC_SHADOW_PARSE_OPTIONS (XML_PARSE_NONET | XML_PARSE_HUGE) + +// Parses `bytes` into a private document with a root element, or returns +// NULL with an exception set (`error` for parse failures). +static xmlDocPtr PyXmlSec_LxmlShadowParse(PyObject* bytes, const char* error) { + char* data = NULL; + Py_ssize_t size = 0; + xmlDocPtr doc; + + if (PyBytes_AsStringAndSize(bytes, &data, &size) < 0) { + return NULL; + } + doc = xmlReadMemory(data, (int)size, NULL, NULL, PYXMLSEC_SHADOW_PARSE_OPTIONS); + if (doc == NULL || xmlDocGetRootElement(doc) == NULL) { + if (doc != NULL) { + xmlFreeDoc(doc); + } + PyErr_SetString(PyXmlSec_InternalError, error); + return NULL; + } + return doc; } // Nodes that exist before the xmlsec call are tagged through the libxml2 @@ -222,10 +271,10 @@ static void PyXmlSec_LxmlShadowMark(xmlNodePtr node) { } } -// Paths now span whole user documents (BeginDoc), not just dsig/enc -// structures; the bound keeps the path buffers on the stack and guards -// against a pathological tree. -#define PYXMLSEC_SHADOW_MAX_DEPTH 128 +// Paths span whole user documents (BeginDoc). 256 is libxml2's default +// nesting limit, so any tree a default lxml parser accepted fits; deeper +// (huge_tree) documents fail cleanly instead of overrunning the buffers. +#define PYXMLSEC_SHADOW_MAX_DEPTH 256 // Index of node among its preceding siblings, counting only the node types // lxml exposes as children (elements, comments, PIs, entity refs), so indices @@ -262,7 +311,8 @@ static int PyXmlSec_LxmlShadowPathTo(xmlNodePtr node, xmlNodePtr top, int* path) return depth; } -// Walks `path` (child indices) down from `start`. Returns a new reference. +// Walks `path` (child indices) down from `start`, an lxml element. Returns a +// new reference. static PyObject* PyXmlSec_LxmlShadowWalk(PyObject* start, const int* path, int depth) { int i; PyObject* cur = start; @@ -278,40 +328,101 @@ static PyObject* PyXmlSec_LxmlShadowWalk(PyObject* start, const int* path, int d return cur; } -// Copies the attributes of `src` (a node in the shadow copy) onto the live -// lxml element `dst`: find-or-create calls may set attributes (e.g. Id) on a -// node that already existed, and that is then the only change to reflect. +// The same walk on a raw copy, counting children exactly like ChildIndex. +static xmlNodePtr PyXmlSec_LxmlShadowWalkNode(xmlNodePtr start, const int* path, int depth) { + int i; + xmlNodePtr n = start; + for (i = 0; i < depth; ++i) { + int idx = path[i]; + xmlNodePtr c; + for (c = n->children; c != NULL; c = c->next) { + if (_isElement(c) && idx-- == 0) { + break; + } + } + if (c == NULL) { + return NULL; + } + n = c; + } + return n; +} + +// Serializes the copy in its current state and re-parses it with lxml, +// returning the root element of that detached parse (new reference). The +// *whole* copy is dumped, not just the changed nodes: the surrounding markup +// carries the ancestor-declared namespaces and the "\n" formatting siblings +// xmlsec emits, so whatever gets grafted from the re-parse stays +// byte-identical to the raw-pointer code without any manual namespace or +// whitespace fix-up. +static PyObject* PyXmlSec_LxmlShadowDumpCopy(PyXmlSec_LxmlShadow* shadow) { + PyObject* bytes = NULL; + PyObject* result = NULL; + xmlChar* dump = NULL; + int dump_size = 0; + + xmlDocDumpMemory(shadow->doc, &dump, &dump_size); + if (dump == NULL || dump_size <= 0) { + PyErr_SetString(PyXmlSec_InternalError, "cannot serialize the private copy."); + goto DONE; + } + bytes = PyBytes_FromStringAndSize((const char*)dump, (Py_ssize_t)dump_size); + if (bytes != NULL) { + result = PyXmlSec_LxmlElementFromBytes(bytes); + } +DONE: + if (dump != NULL) { + xmlFree(dump); + } + Py_XDECREF(bytes); + return result; +} + +// Copies the attributes of `src` (a node in the copy) onto the live lxml +// element `dst`, touching only the ones that differ: find-or-create calls may +// set attributes (e.g. Id) on a node that already existed. static int PyXmlSec_LxmlShadowSyncAttributes(xmlNodePtr src, PyObject* dst) { xmlAttrPtr attr; for (attr = src->properties; attr != NULL; attr = attr->next) { - PyObject* r = NULL; + PyObject* key = NULL; + PyObject* val = NULL; + PyObject* cur = NULL; + int same = -1; xmlChar* value = xmlGetNsProp(src, attr->name, attr->ns != NULL ? attr->ns->href : NULL); if (value == NULL) { continue; } if (attr->ns != NULL && attr->ns->href != NULL) { // lxml takes namespaced attribute names in Clark notation. - PyObject* key = PyUnicode_FromFormat("{%s}%s", (const char*)attr->ns->href, (const char*)attr->name); - if (key != NULL) { - r = PyObject_CallMethod(dst, "set", "Os", key, (const char*)value); - Py_DECREF(key); - } + key = PyUnicode_FromFormat("{%s}%s", (const char*)attr->ns->href, (const char*)attr->name); } else { - r = PyObject_CallMethod(dst, "set", "ss", (const char*)attr->name, (const char*)value); + key = PyUnicode_FromString((const char*)attr->name); } + val = PyUnicode_FromString((const char*)value); xmlFree(value); - if (r == NULL) { + if (key != NULL && val != NULL) { + cur = PyObject_CallMethod(dst, "get", "O", key); + if (cur != NULL) { + same = PyObject_RichCompareBool(cur, val, Py_EQ); + } + } + if (same == 0) { + PyObject* r = PyObject_CallMethod(dst, "set", "OO", key, val); + same = r != NULL ? 1 : -1; + Py_XDECREF(r); + } + Py_XDECREF(key); + Py_XDECREF(val); + Py_XDECREF(cur); + if (same < 0) { return -1; } - Py_DECREF(r); } return 0; } int PyXmlSec_LxmlShadowBegin(PyXmlSec_LxmlShadow* shadow, PyXmlSec_LxmlElementPtr element) { PyObject* bytes; - char* data = NULL; - Py_ssize_t size = 0; shadow->element = element; shadow->owned = NULL; @@ -327,204 +438,207 @@ int PyXmlSec_LxmlShadowBegin(PyXmlSec_LxmlShadow* shadow, PyXmlSec_LxmlElementPt } bytes = PyXmlSec_LxmlElementToBytes((PyObject*)element); - if (bytes == NULL || PyBytes_AsStringAndSize(bytes, &data, &size) < 0) { - Py_XDECREF(bytes); + if (bytes == NULL) { return -1; } - shadow->doc = xmlReadMemory(data, (int)size, NULL, NULL, XML_PARSE_NONET); + shadow->doc = PyXmlSec_LxmlShadowParse(bytes, "cannot make a private copy of the element."); Py_DECREF(bytes); - if (shadow->doc == NULL || (shadow->root = xmlDocGetRootElement(shadow->doc)) == NULL) { - if (shadow->doc != NULL) { - xmlFreeDoc(shadow->doc); - shadow->doc = NULL; - } - PyErr_SetString(PyXmlSec_InternalError, "cannot make a private copy of the element."); + if (shadow->doc == NULL) { return -1; } + shadow->root = xmlDocGetRootElement(shadow->doc); PyXmlSec_LxmlShadowMark(shadow->doc->children); return 0; } -PyObject* PyXmlSec_LxmlShadowEnd(PyXmlSec_LxmlShadow* shadow, xmlNodePtr res, const char* error) { - PyObject* result = NULL; - PyObject* bytes = NULL; - PyObject* copy_root = NULL; - PyObject* copy_parent = NULL; - PyObject* live_parent = NULL; - PyObject* new_node = NULL; - PyObject* tmp = NULL; - - xmlNodePtr fresh = NULL; // topmost node the xmlsec call created, if any - xmlNodePtr n; - xmlChar* dump = NULL; - int dump_size = 0; - - int path[PYXMLSEC_SHADOW_MAX_DEPTH]; - int rel[PYXMLSEC_SHADOW_MAX_DEPTH]; - int depth, rel_depth, insert_idx; +xmlDocPtr PyXmlSec_LxmlShadowBeginNewDoc(PyXmlSec_LxmlShadow* shadow, PyXmlSec_LxmlElementPtr element) { + shadow->element = element; + shadow->owned = NULL; + shadow->doc = NULL; + shadow->root = NULL; - // Fast path (no copy was made): res is a node in the live lxml tree. + // Fast path: allocate the detached subtree straight in the element's own + // document, as the raw code always did. + if (!PyXmlSec_LxmlShadowActive) { + return element->_doc->_c_doc; + } + shadow->doc = xmlNewDoc((const xmlChar*)"1.0"); if (shadow->doc == NULL) { - if (res == NULL) { - PyXmlSec_SetLastError(error); - return NULL; - } - return (PyObject*)PyXmlSec_elementFactory(shadow->element->_doc, res); + PyErr_SetString(PyXmlSec_InternalError, "cannot create a private document."); + return NULL; } + return shadow->doc; +} - if (res == NULL) { - PyXmlSec_SetLastError(error); - goto DONE; +void PyXmlSec_LxmlShadowDiscard(PyXmlSec_LxmlShadow* shadow) { + if (shadow->doc != NULL) { + xmlFreeDoc(shadow->doc); + shadow->doc = NULL; + shadow->root = NULL; } + Py_CLEAR(shadow->owned); +} - // Everything the xmlsec call created is unmarked; walk up from res to find - // the topmost new node (stays NULL when res already existed before the call). - for (n = res; n != shadow->root && !PYXMLSEC_SHADOW_MARKED(n); n = n->parent) { - if (n->parent == NULL) { - PyErr_SetString(PyXmlSec_InternalError, "unexpected result node."); - goto DONE; - } - fresh = n; - } +// ---------------------------------------------------------------------------- +// Shadow-mode ID registry. +// +// register_id / add_ids used to write straight into lxml's document (its ID +// hash) with our libxml2 — exactly the cross-library access the shadow +// forbids. Under the shadow they record the id-attribute specs here instead, +// and every whole-document Begin replays them onto the private copy so that +// #id references resolve during sign/verify/decrypt. +// +// The registry cannot hold strong references to lxml documents (that would +// pin whole trees forever) and lxml's classes refuse weak references, so +// entries are keyed by the _Document object's address with the underlying +// xmlDoc pointer stored alongside as a staleness check: an entry is trusted +// only while both addresses match, and is replaced when the address has been +// reused by a different document. A size cap bounds growth from dead +// documents whose addresses never get reused. +// ---------------------------------------------------------------------------- - if (fresh == NULL) { - // Find-or-create found: the tree did not grow, so the result is the - // live lxml element in the same position (plus any attributes set). - depth = PyXmlSec_LxmlShadowPathTo(res, shadow->root, path); - if (depth < 0) { - PyErr_SetString(PyXmlSec_InternalError, "unexpected result node."); - goto DONE; - } - result = PyXmlSec_LxmlShadowWalk((PyObject*)shadow->element, path, depth); - if (result != NULL && PyXmlSec_LxmlShadowSyncAttributes(res, result) < 0) { - Py_CLEAR(result); - } - goto DONE; - } +#define PYXMLSEC_SHADOW_ID_REGISTRY_CAP 4096 - depth = PyXmlSec_LxmlShadowPathTo(fresh->parent, shadow->root, path); - rel_depth = PyXmlSec_LxmlShadowPathTo(res, fresh, rel); - if (depth < 0 || rel_depth < 0) { - PyErr_SetString(PyXmlSec_InternalError, "unexpected result node."); - goto DONE; - } - insert_idx = PyXmlSec_LxmlShadowChildIndex(fresh); +int PyXmlSec_LxmlShadowRecordId(PyXmlSec_LxmlElementPtr element, const char* name, const char* ns) { + PyObject* key = NULL; + PyObject* cdoc = NULL; + PyObject* created = NULL; + PyObject* spec = NULL; + PyObject* entry; + PyObject* specs; + int contains; + int result = -1; - xmlDocDumpMemory(shadow->doc, &dump, &dump_size); - if (dump == NULL || dump_size <= 0) { - PyErr_SetString(PyXmlSec_InternalError, "cannot serialize the mutated copy."); - goto DONE; - } - bytes = PyBytes_FromStringAndSize((const char*)dump, (Py_ssize_t)dump_size); - if (bytes == NULL) { - goto DONE; - } - copy_root = PyXmlSec_LxmlElementFromBytes(bytes); - if (copy_root == NULL) { - goto DONE; - } - copy_parent = PyXmlSec_LxmlShadowWalk(copy_root, path, depth); - live_parent = PyXmlSec_LxmlShadowWalk((PyObject*)shadow->element, path, depth); - if (copy_parent == NULL || live_parent == NULL) { - goto DONE; - } - new_node = PySequence_GetItem(copy_parent, insert_idx); - if (new_node == NULL) { + key = PyLong_FromVoidPtr((void*)element->_doc); + cdoc = PyLong_FromVoidPtr((void*)element->_doc->_c_doc); + if (key == NULL || cdoc == NULL) { goto DONE; } - // Move the new node into the live tree at the position the xmlsec call - // chose; lxml carries the node's tail along and reconciles namespaces. - tmp = PyObject_CallMethod(live_parent, "insert", "iO", insert_idx, new_node); - if (tmp == NULL) { - goto DONE; + entry = PyDict_GetItem(PyXmlSec_LxmlShadowIdRegistry, key); // borrowed + if (entry != NULL) { + int eq = PyObject_RichCompareBool(PyTuple_GET_ITEM(entry, 0), cdoc, Py_EQ); + if (eq < 0) { + goto DONE; + } + if (!eq) { + entry = NULL; // the address was reused by another document + } } - Py_CLEAR(tmp); - - // xmlsec may also have put a "\n" *before* the new node — the parent's - // text when it is the first child, the previous sibling's tail otherwise; - // mirror that too (a no-op when nothing changed there). - if (insert_idx == 0) { - tmp = PyObject_GetAttrString(copy_parent, "text"); - if (tmp == NULL || PyObject_SetAttrString(live_parent, "text", tmp) < 0) { + if (entry == NULL) { + if (PyDict_Size(PyXmlSec_LxmlShadowIdRegistry) >= PYXMLSEC_SHADOW_ID_REGISTRY_CAP) { + PyObject* k; + PyObject* v; + Py_ssize_t pos = 0; + if (PyDict_Next(PyXmlSec_LxmlShadowIdRegistry, &pos, &k, &v) + && PyDict_DelItem(PyXmlSec_LxmlShadowIdRegistry, k) < 0) { + goto DONE; + } + } + specs = PyList_New(0); + if (specs == NULL) { goto DONE; } - Py_CLEAR(tmp); - } else { - PyObject* prev_tail = NULL; - PyObject* copy_prev = PySequence_GetItem(copy_parent, insert_idx - 1); - PyObject* live_prev = PySequence_GetItem(live_parent, insert_idx - 1); - int failed = (copy_prev == NULL || live_prev == NULL - || (prev_tail = PyObject_GetAttrString(copy_prev, "tail")) == NULL - || PyObject_SetAttrString(live_prev, "tail", prev_tail) < 0); - Py_XDECREF(copy_prev); - Py_XDECREF(live_prev); - Py_XDECREF(prev_tail); - if (failed) { + created = PyTuple_Pack(2, cdoc, specs); + Py_DECREF(specs); + if (created == NULL || PyDict_SetItem(PyXmlSec_LxmlShadowIdRegistry, key, created) < 0) { goto DONE; } + entry = created; } - // res may sit below the topmost new node (e.g. the new inside - // a freshly created ); descend to it in the grafted subtree. - result = PyXmlSec_LxmlShadowWalk(new_node, rel, rel_depth); + spec = Py_BuildValue("(sz)", name, ns); + if (spec == NULL) { + goto DONE; + } + specs = PyTuple_GET_ITEM(entry, 1); + contains = PySequence_Contains(specs, spec); + if (contains < 0 || (!contains && PyList_Append(specs, spec) < 0)) { + goto DONE; + } + result = 0; DONE: - PyXmlSec_LxmlShadowDiscard(shadow); - if (dump != NULL) { - xmlFree(dump); - } - Py_XDECREF(bytes); - Py_XDECREF(copy_root); - Py_XDECREF(copy_parent); - Py_XDECREF(live_parent); - Py_XDECREF(new_node); - Py_XDECREF(tmp); + Py_XDECREF(key); + Py_XDECREF(cdoc); + Py_XDECREF(created); + Py_XDECREF(spec); return result; } -void PyXmlSec_LxmlShadowDiscard(PyXmlSec_LxmlShadow* shadow) { - if (shadow->doc != NULL) { - xmlFreeDoc(shadow->doc); - shadow->doc = NULL; - shadow->root = NULL; +// Registers every attribute named `name` (under `ns` when given) in the copy +// as an XML ID — a superset of the fast path's registrations (single node for +// register_id, subtree for add_ids), which is the safe direction: it mirrors +// what xmlSecAddIDs does from the root. +static void PyXmlSec_LxmlShadowApplyIdSpec(xmlDocPtr doc, xmlNodePtr n, const xmlChar* name, const xmlChar* ns) { + for (; n != NULL; n = n->next) { + if (n->type == XML_ELEMENT_NODE) { + xmlAttrPtr attr = ns != NULL ? xmlHasNsProp(n, name, ns) : xmlHasProp(n, name); + if (attr != NULL && attr->children != NULL) { + xmlChar* value = xmlNodeListGetString(doc, attr->children, 1); + if (value != NULL) { + if (xmlGetID(doc, value) == NULL) { + xmlAddID(NULL, doc, value, attr); + } + xmlFree(value); + } + } + if (n->children != NULL) { + PyXmlSec_LxmlShadowApplyIdSpec(doc, n->children, name, ns); + } + } } - Py_CLEAR(shadow->owned); } -// ---------------------------------------------------------------------------- -// Doc-level shadows and multi-site reflection — the rollout of the shadow -// pattern beyond templates (sign/verify, encrypt/decrypt, tree finders). -// Contracts in lxml.h. -// ---------------------------------------------------------------------------- - -// Walks `path` (child indices, top-first) down from `start` on a raw copy, -// counting children exactly like PyXmlSec_LxmlShadowChildIndex. -static xmlNodePtr PyXmlSec_LxmlShadowWalkNode(xmlNodePtr start, const int* path, int depth) { - int i; - xmlNodePtr n = start; - for (i = 0; i < depth; ++i) { - int idx = path[i]; - xmlNodePtr c; - for (c = n->children; c != NULL; c = c->next) { - if (_isElement(c) && idx-- == 0) { - break; - } - } - if (c == NULL) { - return NULL; +// Replays the specs recorded for the shadow's live document onto the copy. +static int PyXmlSec_LxmlShadowReplayIds(PyXmlSec_LxmlShadow* shadow) { + PyObject* key; + PyObject* cdoc; + PyObject* entry; + PyObject* specs; + Py_ssize_t i, n; + int eq; + + key = PyLong_FromVoidPtr((void*)shadow->element->_doc); + if (key == NULL) { + return -1; + } + entry = PyDict_GetItem(PyXmlSec_LxmlShadowIdRegistry, key); // borrowed + Py_DECREF(key); + if (entry == NULL) { + return 0; + } + cdoc = PyLong_FromVoidPtr((void*)shadow->element->_doc->_c_doc); + if (cdoc == NULL) { + return -1; + } + eq = PyObject_RichCompareBool(PyTuple_GET_ITEM(entry, 0), cdoc, Py_EQ); + Py_DECREF(cdoc); + if (eq < 0) { + return -1; + } + if (!eq) { + return 0; // stale entry from a dead document at the same address + } + + specs = PyTuple_GET_ITEM(entry, 1); + n = PyList_GET_SIZE(specs); + for (i = 0; i < n; ++i) { + PyObject* spec = PyList_GET_ITEM(specs, i); + const char* name = PyUnicode_AsUTF8(PyTuple_GET_ITEM(spec, 0)); + const char* ns = PyTuple_GET_ITEM(spec, 1) == Py_None ? NULL : PyUnicode_AsUTF8(PyTuple_GET_ITEM(spec, 1)); + if (name == NULL || (ns == NULL && PyErr_Occurred())) { + return -1; } - n = c; + PyXmlSec_LxmlShadowApplyIdSpec(shadow->doc, shadow->doc->children, (const xmlChar*)name, (const xmlChar*)ns); } - return n; + return 0; } int PyXmlSec_LxmlShadowBeginDoc(PyXmlSec_LxmlShadow* shadow, PyXmlSec_LxmlElementPtr element, xmlNodePtr* target) { PyObject* cur = NULL; PyObject* tree = NULL; PyObject* bytes = NULL; - char* data = NULL; - Py_ssize_t size = 0; int path[PYXMLSEC_SHADOW_MAX_DEPTH]; int depth = 0; int i; @@ -591,15 +705,15 @@ int PyXmlSec_LxmlShadowBeginDoc(PyXmlSec_LxmlShadow* shadow, PyXmlSec_LxmlElemen } bytes = PyObject_CallFunctionObjArgs(PyXmlSec_LxmlEtreeToString, tree, NULL); Py_CLEAR(tree); - if (bytes == NULL || PyBytes_AsStringAndSize(bytes, &data, &size) < 0) { + if (bytes == NULL) { goto ON_FAIL; } - shadow->doc = xmlReadMemory(data, (int)size, NULL, NULL, XML_PARSE_NONET); + shadow->doc = PyXmlSec_LxmlShadowParse(bytes, "cannot make a private copy of the document."); Py_CLEAR(bytes); - if (shadow->doc == NULL || (shadow->root = xmlDocGetRootElement(shadow->doc)) == NULL) { - PyErr_SetString(PyXmlSec_InternalError, "cannot make a private copy of the document."); + if (shadow->doc == NULL) { goto ON_FAIL; } + shadow->root = xmlDocGetRootElement(shadow->doc); PyXmlSec_LxmlShadowMark(shadow->doc->children); *target = PyXmlSec_LxmlShadowWalkNode(shadow->root, path, depth); @@ -608,113 +722,278 @@ int PyXmlSec_LxmlShadowBeginDoc(PyXmlSec_LxmlShadow* shadow, PyXmlSec_LxmlElemen goto ON_FAIL; } - // The reflect helpers map paths from the copy root, so the live element + // The End functions map paths from the copy root, so the live element // they start from must be the live root. shadow->element = (PyXmlSec_LxmlElementPtr)cur; shadow->owned = cur; + cur = NULL; + + // The registered IDs live in lxml's document, which the copy knows + // nothing about; replay them so that #id references resolve. + if (PyXmlSec_LxmlShadowReplayIds(shadow) < 0) { + goto ON_FAIL; + } return 0; ON_FAIL: Py_XDECREF(cur); Py_XDECREF(tree); Py_XDECREF(bytes); - if (shadow->doc != NULL) { - xmlFreeDoc(shadow->doc); - shadow->doc = NULL; - shadow->root = NULL; - } + PyXmlSec_LxmlShadowDiscard(shadow); + *target = NULL; return -1; } -xmlDocPtr PyXmlSec_LxmlShadowBeginNewDoc(PyXmlSec_LxmlShadow* shadow, PyXmlSec_LxmlElementPtr element) { - shadow->element = element; - shadow->owned = NULL; - shadow->doc = NULL; - shadow->root = NULL; +// ---------------------------------------------------------------------------- +// The reflection: everything the xmlsec call did to the copy, applied to the +// live tree through lxml's Python API. +// +// Two kinds of site are recorded while walking the pre-existing (marked) +// structure of the copy in document order: +// +// graft — a fresh node (element, comment, PI) to insert at its child +// index. Fresh subtrees are grafted wholesale; the scan never +// descends into them. +// sync — a marked parent that gained any fresh node (element or text) +// gets its text slots (its .text and each child's .tail) copied +// over from the re-parsed copy. That covers everything xmlsec does +// to text: the "\n" formatting around a new node, values filled +// into empty elements (DigestValue), and content it removed +// (encrypt Type=Content) — a removal leaves no fresh node behind, +// so only a wholesale sync can see it. +// +// The reflection is two-phase: first every site's payload is fetched from +// the re-parsed copy while it is still in its final, untouched state, then +// everything is applied to the live tree in document order (each graft moves +// a node out of the re-parsed copy, which would invalidate later fetches, +// and each live insert makes the later, larger indices valid; a parent's +// sync is recorded after its grafts, so it runs once they are in place). +// ---------------------------------------------------------------------------- - // Fast path: allocate the detached subtree straight in the element's own - // document, as the raw code always did. - if (!PyXmlSec_LxmlShadowActive) { - return element->_doc->_c_doc; +enum { PYXMLSEC_SHADOW_SITE_GRAFT, PYXMLSEC_SHADOW_SITE_SYNC }; + +typedef struct { + int kind; + int depth; + int path[PYXMLSEC_SHADOW_MAX_DEPTH]; // graft: to the fresh node's parent; sync: to the parent itself + int idx; // graft: child index of the fresh node + PyObject* value; // phase 1: the node to graft, or the [text, tail, ...] list to sync +} PyXmlSec_LxmlShadowSite; + +typedef struct { + PyXmlSec_LxmlShadowSite* items; + int count; + int capacity; + xmlNodePtr top; // the copy's root element — origin of all paths +} PyXmlSec_LxmlShadowSiteList; + +static int PyXmlSec_LxmlShadowSiteAppend(PyXmlSec_LxmlShadowSiteList* list, xmlNodePtr node, int kind) { + PyXmlSec_LxmlShadowSite* site; + int graft = kind == PYXMLSEC_SHADOW_SITE_GRAFT; + + if (list->count == list->capacity) { + int capacity = list->capacity == 0 ? 8 : list->capacity * 2; + PyXmlSec_LxmlShadowSite* items = (PyXmlSec_LxmlShadowSite*)PyMem_Realloc(list->items, capacity * sizeof(*items)); + if (items == NULL) { + PyErr_NoMemory(); + return -1; + } + list->items = items; + list->capacity = capacity; } - shadow->doc = xmlNewDoc((const xmlChar*)"1.0"); - if (shadow->doc == NULL) { - PyErr_SetString(PyXmlSec_InternalError, "cannot create a private document."); - return NULL; + site = &list->items[list->count]; + site->kind = kind; + site->depth = PyXmlSec_LxmlShadowPathTo(graft ? node->parent : node, list->top, site->path); + if (site->depth < 0) { + PyErr_SetString(PyXmlSec_InternalError, "unexpected mutation site."); + return -1; } - return shadow->doc; + site->idx = graft ? PyXmlSec_LxmlShadowChildIndex(node) : 0; + site->value = NULL; + ++list->count; + return 0; } -PyObject* PyXmlSec_LxmlShadowEndNewDoc(PyXmlSec_LxmlShadow* shadow, xmlNodePtr res, const char* error) { - PyObject* result = NULL; - PyObject* bytes = NULL; - xmlChar* dump = NULL; - int dump_size = 0; +// Walks the marked structure of the copy in document order, recording a +// graft for every fresh node and, after them, a sync for their parent. +static int PyXmlSec_LxmlShadowCollectSites(PyXmlSec_LxmlShadowSiteList* list, xmlNodePtr parent) { + xmlNodePtr n; + int fresh = 0; - if (shadow->doc == NULL) { // fast path - if (res == NULL) { - PyXmlSec_SetLastError(error); - return NULL; + for (n = parent->children; n != NULL; n = n->next) { + if (PYXMLSEC_SHADOW_MARKED(n)) { + if (n->type == XML_ELEMENT_NODE && n->children != NULL + && PyXmlSec_LxmlShadowCollectSites(list, n) < 0) { + return -1; + } + continue; } - return (PyObject*)PyXmlSec_elementFactory(shadow->element->_doc, res); + fresh = 1; + if (_isElement(n) && PyXmlSec_LxmlShadowSiteAppend(list, n, PYXMLSEC_SHADOW_SITE_GRAFT) < 0) { + return -1; + } + } + if (fresh && PyXmlSec_LxmlShadowSiteAppend(list, parent, PYXMLSEC_SHADOW_SITE_SYNC) < 0) { + return -1; } + return 0; +} - if (res == NULL) { - PyXmlSec_SetLastError(error); - goto DONE; +// The text slots of an lxml element as the list [text, tail of child 0, +// tail of child 1, ...] (new reference). +static PyObject* PyXmlSec_LxmlShadowGetTextSlots(PyObject* parent) { + PyObject* slots = PyList_New(0); + PyObject* item; + Py_ssize_t i, n; + int failed; + + if (slots == NULL) { + return NULL; } - // The call left `res` detached inside our private doc; root it there so - // the whole subtree serializes (and gets freed with the doc), then hand - // the bytes to lxml. The result is a new detached element — lxml gives it - // a document of its own, and moves it when the caller grafts it into a - // tree, just like the raw code's detached node. - xmlDocSetRootElement(shadow->doc, res); - xmlDocDumpMemory(shadow->doc, &dump, &dump_size); - if (dump == NULL || dump_size <= 0) { - PyErr_SetString(PyXmlSec_InternalError, "cannot serialize the created node."); - goto DONE; + item = PyObject_GetAttrString(parent, "text"); + failed = item == NULL || PyList_Append(slots, item) < 0; + Py_XDECREF(item); + n = failed ? -1 : PyObject_Length(parent); + for (i = 0; i < n; ++i) { + PyObject* child = PySequence_GetItem(parent, i); + item = child != NULL ? PyObject_GetAttrString(child, "tail") : NULL; + failed = item == NULL || PyList_Append(slots, item) < 0; + Py_XDECREF(child); + Py_XDECREF(item); + if (failed) { + break; + } } - bytes = PyBytes_FromStringAndSize((const char*)dump, (Py_ssize_t)dump_size); - if (bytes != NULL) { - result = PyXmlSec_LxmlElementFromBytes(bytes); + if (failed || n < 0) { + Py_DECREF(slots); + return NULL; } + return slots; +} -DONE: - PyXmlSec_LxmlShadowDiscard(shadow); - if (dump != NULL) { - xmlFree(dump); +// Applies a slot list to an lxml element whose children already mirror the +// copy's (the grafts under it have been applied). +static int PyXmlSec_LxmlShadowSetTextSlots(PyObject* parent, PyObject* slots) { + Py_ssize_t i; + Py_ssize_t n = PyObject_Length(parent); + + if (n < 0) { + return -1; } - Py_XDECREF(bytes); - return result; + if (n + 1 != PyList_GET_SIZE(slots)) { + PyErr_SetString(PyXmlSec_InternalError, "unexpected mutation site."); + return -1; + } + if (PyObject_SetAttrString(parent, "text", PyList_GET_ITEM(slots, 0)) < 0) { + return -1; + } + for (i = 0; i < n; ++i) { + PyObject* child = PySequence_GetItem(parent, i); + int failed = child == NULL || PyObject_SetAttrString(child, "tail", PyList_GET_ITEM(slots, i + 1)) < 0; + Py_XDECREF(child); + if (failed) { + return -1; + } + } + return 0; } -PyObject* PyXmlSec_LxmlShadowEndFind(PyXmlSec_LxmlShadow* shadow, xmlNodePtr res) { - PyObject* result; - int path[PYXMLSEC_SHADOW_MAX_DEPTH]; - int depth; +// Applies every change in the copy to the live tree. Does not release the +// copy. Returns 0, or -1 with an exception set. +static int PyXmlSec_LxmlShadowReflectSites(PyXmlSec_LxmlShadow* shadow) { + PyXmlSec_LxmlShadowSiteList list = {NULL, 0, 0, NULL}; + PyObject* copy_root = NULL; + int i; + int rv = -1; - if (shadow->doc == NULL) { // fast path: res is a live node - if (res == NULL) { - Py_RETURN_NONE; + // Re-fetch the root: replacement operations may swap nodes at the top. A + // fresh root means the call replaced the root itself, which cannot be + // reflected (lxml cannot swap a document's root); the callers that can + // hit that (enc.c) check for it before getting here. + list.top = xmlDocGetRootElement(shadow->doc); + if (list.top == NULL || !PYXMLSEC_SHADOW_MARKED(list.top)) { + PyErr_SetString(PyXmlSec_InternalError, "unexpected mutation site."); + goto DONE; + } + if (PyXmlSec_LxmlShadowCollectSites(&list, list.top) < 0) { + goto DONE; + } + if (list.count == 0) { + rv = 0; + goto DONE; + } + copy_root = PyXmlSec_LxmlShadowDumpCopy(shadow); + if (copy_root == NULL) { + goto DONE; + } + + // Phase 1: fetch every payload from the re-parsed copy (still final state). + for (i = 0; i < list.count; ++i) { + PyXmlSec_LxmlShadowSite* site = &list.items[i]; + PyObject* copy_parent = PyXmlSec_LxmlShadowWalk(copy_root, site->path, site->depth); + if (copy_parent == NULL) { + goto DONE; + } + if (site->kind == PYXMLSEC_SHADOW_SITE_GRAFT) { + site->value = PySequence_GetItem(copy_parent, site->idx); + } else { + site->value = PyXmlSec_LxmlShadowGetTextSlots(copy_parent); + } + Py_DECREF(copy_parent); + if (site->value == NULL) { + goto DONE; } - return (PyObject*)PyXmlSec_elementFactory(shadow->element->_doc, res); } - if (res == NULL) { - PyXmlSec_LxmlShadowDiscard(shadow); - Py_RETURN_NONE; + + // Phase 2: apply to the live tree in document order. + for (i = 0; i < list.count; ++i) { + PyXmlSec_LxmlShadowSite* site = &list.items[i]; + PyObject* live_parent = PyXmlSec_LxmlShadowWalk((PyObject*)shadow->element, site->path, site->depth); + int failed; + if (live_parent == NULL) { + goto DONE; + } + if (site->kind == PYXMLSEC_SHADOW_SITE_GRAFT) { + // lxml carries the node's tail along and reconciles namespaces. + PyObject* tmp = PyObject_CallMethod(live_parent, "insert", "iO", site->idx, site->value); + failed = tmp == NULL; + Py_XDECREF(tmp); + } else { + failed = PyXmlSec_LxmlShadowSetTextSlots(live_parent, site->value) < 0; + } + Py_DECREF(live_parent); + if (failed) { + goto DONE; + } } - depth = PyXmlSec_LxmlShadowPathTo(res, shadow->root, path); - if (depth < 0) { - PyErr_SetString(PyXmlSec_InternalError, "unexpected result node."); - PyXmlSec_LxmlShadowDiscard(shadow); - return NULL; + rv = 0; + +DONE: + for (i = 0; i < list.count; ++i) { + Py_XDECREF(list.items[i].value); } - result = PyXmlSec_LxmlShadowWalk((PyObject*)shadow->element, path, depth); + PyMem_Free(list.items); + Py_XDECREF(copy_root); + return rv; +} + +// Create shape (BeginNewDoc): the call left `res` detached inside the +// private document. Root it there so the whole subtree serializes (and is +// freed with the doc) and hand the bytes to lxml: the result is a new +// detached element in a document of its own — lxml moves it when the caller +// grafts it into a tree, just like the raw path's detached node. +static PyObject* PyXmlSec_LxmlShadowEndDetached(PyXmlSec_LxmlShadow* shadow, xmlNodePtr res) { + PyObject* result; + xmlDocSetRootElement(shadow->doc, res); + result = PyXmlSec_LxmlShadowDumpCopy(shadow); PyXmlSec_LxmlShadowDiscard(shadow); return result; } -PyObject* PyXmlSec_LxmlShadowEndReplace(PyXmlSec_LxmlShadow* shadow, xmlNodePtr res, const char* error) { +// Swaps the live element at `path` for the copy's version of it (the tail +// travels with the inserted node; the call does not touch it). Returns the +// new live element (new reference). +static PyObject* PyXmlSec_LxmlShadowSwapLive(PyXmlSec_LxmlShadow* shadow, const int* path, int depth) { PyObject* copy_root = NULL; PyObject* copy_parent = NULL; PyObject* live_parent = NULL; @@ -722,32 +1001,14 @@ PyObject* PyXmlSec_LxmlShadowEndReplace(PyXmlSec_LxmlShadow* shadow, xmlNodePtr PyObject* new_node = NULL; PyObject* tmp = NULL; PyObject* result = NULL; - int path[PYXMLSEC_SHADOW_MAX_DEPTH]; - int depth; - int idx; - - // Everything except "shadow copy in play and res pre-existed, below the - // root" is exactly the general End: the fast path mutates live nodes in - // place, a fresh res reflects through the graft, errors raise, and for - // res == root there is no live parent to graft into (only attributes can - // change there, which End's sync covers). - if (shadow->doc == NULL || res == NULL || !PYXMLSEC_SHADOW_MARKED(res) || res == shadow->root) { - return PyXmlSec_LxmlShadowEnd(shadow, res, error); - } - - depth = PyXmlSec_LxmlShadowPathTo(res->parent, shadow->root, path); - if (depth < 0) { - PyErr_SetString(PyXmlSec_InternalError, "unexpected result node."); - goto DONE; - } - idx = PyXmlSec_LxmlShadowChildIndex(res); + int idx = path[depth - 1]; copy_root = PyXmlSec_LxmlShadowDumpCopy(shadow); if (copy_root == NULL) { goto DONE; } - copy_parent = PyXmlSec_LxmlShadowWalk(copy_root, path, depth); - live_parent = PyXmlSec_LxmlShadowWalk((PyObject*)shadow->element, path, depth); + copy_parent = PyXmlSec_LxmlShadowWalk(copy_root, path, depth - 1); + live_parent = PyXmlSec_LxmlShadowWalk((PyObject*)shadow->element, path, depth - 1); if (copy_parent == NULL || live_parent == NULL) { goto DONE; } @@ -756,8 +1017,6 @@ PyObject* PyXmlSec_LxmlShadowEndReplace(PyXmlSec_LxmlShadow* shadow, xmlNodePtr if (new_node == NULL || live_old == NULL) { goto DONE; } - // Swap the live element for the copy's version; the tail travels with the - // inserted node (same content — the call does not touch it). tmp = PyObject_CallMethod(live_parent, "remove", "O", live_old); if (tmp == NULL) { goto DONE; @@ -771,7 +1030,6 @@ PyObject* PyXmlSec_LxmlShadowEndReplace(PyXmlSec_LxmlShadow* shadow, xmlNodePtr result = PySequence_GetItem(live_parent, idx); DONE: - PyXmlSec_LxmlShadowDiscard(shadow); Py_XDECREF(copy_root); Py_XDECREF(copy_parent); Py_XDECREF(live_parent); @@ -781,432 +1039,155 @@ PyObject* PyXmlSec_LxmlShadowEndReplace(PyXmlSec_LxmlShadow* shadow, xmlNodePtr return result; } -xmlNodePtr PyXmlSec_LxmlShadowFindFresh(PyXmlSec_LxmlShadow* shadow) { - xmlNodePtr n; +// Find-or-create found an existing node: the tree did not grow there, so the +// result is the live element at the same path, plus whatever the call changed +// on it. Attributes (Id) are synced in place. A renamed namespace prefix +// (encrypted_data_ensure_key_info(ns=...)) has no lxml API, so the live +// element is swapped for the copy's version and the caller gets a new proxy +// object — except for the shadow root, which has no live parent to swap +// under (only attributes can change there). +static PyObject* PyXmlSec_LxmlShadowEndFound(PyXmlSec_LxmlShadow* shadow, xmlNodePtr res, const int* path, int depth) { + PyObject* live = NULL; + PyObject* live_prefix = NULL; + PyObject* copy_prefix = NULL; + PyObject* result = NULL; + int same; - if (shadow->doc == NULL) { // fast path: End just wraps the node - return shadow->root; - } - n = shadow->root->children; - while (n != NULL) { - if (!PYXMLSEC_SHADOW_MARKED(n)) { - if (_isElement(n)) { - return n; - } - n = n->next; // fresh formatting text; the element follows it - continue; - } - if (n->children != NULL) { - n = n->children; - continue; - } - while (n != shadow->root && n->next == NULL) { - n = n->parent; - } - n = (n == shadow->root) ? NULL : n->next; + live = PyXmlSec_LxmlShadowWalk((PyObject*)shadow->element, path, depth); + if (live == NULL) { + return NULL; } - return shadow->root; // nothing created; End takes its find-or-create path -} - -PyObject* PyXmlSec_LxmlShadowDumpCopy(PyXmlSec_LxmlShadow* shadow) { - PyObject* bytes = NULL; - PyObject* result = NULL; - xmlChar* dump = NULL; - int dump_size = 0; - - xmlDocDumpMemory(shadow->doc, &dump, &dump_size); - if (dump == NULL || dump_size <= 0) { - PyErr_SetString(PyXmlSec_InternalError, "cannot serialize the mutated copy."); + live_prefix = PyObject_GetAttrString(live, "prefix"); + if (live_prefix == NULL) { goto DONE; } - bytes = PyBytes_FromStringAndSize((const char*)dump, (Py_ssize_t)dump_size); - if (bytes != NULL) { - result = PyXmlSec_LxmlElementFromBytes(bytes); + if (res->ns != NULL && res->ns->prefix != NULL) { + copy_prefix = PyUnicode_FromString((const char*)res->ns->prefix); + } else { + copy_prefix = Py_None; + Py_INCREF(copy_prefix); } -DONE: - if (dump != NULL) { - xmlFree(dump); + if (copy_prefix == NULL) { + goto DONE; } - Py_XDECREF(bytes); - return result; -} - -xmlNodePtr PyXmlSec_LxmlShadowImportElement(PyXmlSec_LxmlShadow* shadow, PyXmlSec_LxmlElementPtr element) { - PyObject* bytes; - char* data = NULL; - Py_ssize_t size = 0; - xmlDocPtr tdoc; - xmlNodePtr troot; - xmlNodePtr result = NULL; - - bytes = PyXmlSec_LxmlElementToBytes((PyObject*)element); - if (bytes == NULL || PyBytes_AsStringAndSize(bytes, &data, &size) < 0) { - Py_XDECREF(bytes); - return NULL; + same = PyObject_RichCompareBool(live_prefix, copy_prefix, Py_EQ); + if (same < 0) { + goto DONE; } - tdoc = xmlReadMemory(data, (int)size, NULL, NULL, XML_PARSE_NONET); - Py_DECREF(bytes); - if (tdoc == NULL || (troot = xmlDocGetRootElement(tdoc)) == NULL) { - if (tdoc != NULL) { - xmlFreeDoc(tdoc); + if (same || depth == 0) { + if (PyXmlSec_LxmlShadowSyncAttributes(res, live) == 0) { + result = live; + live = NULL; } - PyErr_SetString(PyXmlSec_InternalError, "cannot make a private copy of the element."); - return NULL; - } - // Copy into the shadow doc (both trees are ours). The copy stays - // untagged: from the reflection's point of view whatever the xmlsec call - // grafts of it is a node "the call created". - result = xmlDocCopyNode(troot, shadow->doc, 1); - xmlFreeDoc(tdoc); - if (result == NULL) { - PyErr_SetString(PyXmlSec_InternalError, "cannot make a private copy of the element."); + } else { + result = PyXmlSec_LxmlShadowSwapLive(shadow, path, depth); } + +DONE: + Py_XDECREF(live); + Py_XDECREF(live_prefix); + Py_XDECREF(copy_prefix); return result; } -// A mutation site found in the copy: either a fresh node to graft (elements, -// comments, PIs) or a text slot whose value changed. The reflection is -// two-phase — first every site's payload is fetched from the re-parsed copy -// while it is still in its final, untouched state, then everything is applied -// to the live tree in document order (each graft moves a node out of the -// re-parsed copy, which would invalidate later fetches, and each live insert -// makes the later, larger indices valid). -typedef struct { +PyObject* PyXmlSec_LxmlShadowEnd(PyXmlSec_LxmlShadow* shadow, xmlNodePtr res, const char* error) { + PyObject* result = NULL; + int path[PYXMLSEC_SHADOW_MAX_DEPTH]; int depth; - int path[PYXMLSEC_SHADOW_MAX_DEPTH]; // to the site's parent, from the copy root - int idx; // element-index of the site among its siblings - int is_element; // grafts `node`; otherwise the site is a text slot only - int mirror; // reflect the text slot before/at idx - PyObject* node; // phase 1: the node to graft, from the re-parsed copy - PyObject* slot; // phase 1: the slot's new value (str or None) -} PyXmlSec_LxmlShadowSite; -typedef struct { - PyXmlSec_LxmlShadowSite* items; - int count; - int capacity; - xmlNodePtr top; // the copy's root element — origin of all paths -} PyXmlSec_LxmlShadowSiteList; - -static int PyXmlSec_LxmlShadowSiteAppend(PyXmlSec_LxmlShadowSiteList* list, xmlNodePtr node, int is_element, int prev_elem_fresh) { - PyXmlSec_LxmlShadowSite* site; - - if (list->count == list->capacity) { - int capacity = list->capacity == 0 ? 8 : list->capacity * 2; - PyXmlSec_LxmlShadowSite* items = (PyXmlSec_LxmlShadowSite*)PyMem_Realloc(list->items, capacity * sizeof(*items)); - if (items == NULL) { - PyErr_NoMemory(); - return -1; - } - list->items = items; - list->capacity = capacity; - } - site = &list->items[list->count]; - site->depth = PyXmlSec_LxmlShadowPathTo(node->parent, list->top, site->path); - if (site->depth < 0) { - PyErr_SetString(PyXmlSec_InternalError, "unexpected mutation site."); - return -1; + if (res == NULL) { + PyXmlSec_LxmlShadowDiscard(shadow); + PyXmlSec_SetLastError(error); + return NULL; } - site->idx = PyXmlSec_LxmlShadowChildIndex(node); - site->is_element = is_element; - // The slot before a fresh element travels with the preceding fresh - // element's graft (it is that element's tail) — mirror it only otherwise. - site->mirror = site->idx == 0 || !prev_elem_fresh; - site->node = NULL; - site->slot = NULL; - ++list->count; - return 0; -} - -// Walks the pre-existing (marked) structure of the copy in document order, -// recording every fresh site. Fresh subtrees are grafted wholesale, so the -// scan does not descend into them. -static int PyXmlSec_LxmlShadowCollectSites(PyXmlSec_LxmlShadowSiteList* list, xmlNodePtr parent) { - xmlNodePtr n; - int last_elem_fresh = 0; - - for (n = parent->children; n != NULL; n = n->next) { - if (PYXMLSEC_SHADOW_MARKED(n)) { - if (_isElement(n)) { - last_elem_fresh = 0; - if (n->type == XML_ELEMENT_NODE && n->children != NULL - && PyXmlSec_LxmlShadowCollectSites(list, n) < 0) { - return -1; - } - } - continue; - } - if (_isElement(n)) { - if (PyXmlSec_LxmlShadowSiteAppend(list, n, 1, last_elem_fresh) < 0) { - return -1; - } - last_elem_fresh = 1; - } else if (n->type == XML_TEXT_NODE || n->type == XML_CDATA_SECTION_NODE) { - // A fresh text after a fresh element is that element's tail - // (travels with the graft); adjacent fresh texts share one slot. - if (!last_elem_fresh - && !(n->prev != NULL && !PYXMLSEC_SHADOW_MARKED(n->prev) - && (n->prev->type == XML_TEXT_NODE || n->prev->type == XML_CDATA_SECTION_NODE)) - && PyXmlSec_LxmlShadowSiteAppend(list, n, 0, 0) < 0) { - return -1; - } - } + // Fast path (no copy was made): res is a node in the live lxml tree. + if (shadow->doc == NULL) { + PyXmlSec_LxmlShadowDiscard(shadow); + return (PyObject*)PyXmlSec_elementFactory(shadow->element->_doc, res); } - return 0; -} - -int PyXmlSec_LxmlShadowReflectAll(PyXmlSec_LxmlShadow* shadow) { - PyXmlSec_LxmlShadowSiteList list = {NULL, 0, 0, NULL}; - PyObject* copy_root = NULL; - int i; - int rv = -1; - - if (shadow->doc == NULL) { // fast path: xmlsec already mutated the live tree - Py_CLEAR(shadow->owned); - return 0; + if (shadow->root == NULL) { + return PyXmlSec_LxmlShadowEndDetached(shadow, res); } - // Re-fetch the root: replacement operations may have swapped nodes at the - // top. A fresh root means the call replaced the root itself — the callers - // covering that (enc.c) reflect it explicitly and never get here. - list.top = xmlDocGetRootElement(shadow->doc); - if (list.top == NULL || !PYXMLSEC_SHADOW_MARKED(list.top)) { - PyErr_SetString(PyXmlSec_InternalError, "unexpected mutation site."); - goto DONE; - } - if (PyXmlSec_LxmlShadowCollectSites(&list, list.top) < 0) { - goto DONE; - } - if (list.count == 0) { - rv = 0; + depth = PyXmlSec_LxmlShadowPathTo(res, shadow->root, path); + if (depth < 0) { + PyErr_SetString(PyXmlSec_InternalError, "unexpected result node."); goto DONE; } - copy_root = PyXmlSec_LxmlShadowDumpCopy(shadow); - if (copy_root == NULL) { + // Reflect first: the live tree then mirrors the copy's element structure, + // so the path of `res` in the copy resolves to its live counterpart — + // whether the call created it (now grafted) or found it. + if (PyXmlSec_LxmlShadowReflectSites(shadow) < 0) { goto DONE; } - - // Phase 1: fetch every payload from the re-parsed copy (still final state). - for (i = 0; i < list.count; ++i) { - PyXmlSec_LxmlShadowSite* site = &list.items[i]; - PyObject* copy_parent = PyXmlSec_LxmlShadowWalk(copy_root, site->path, site->depth); - if (copy_parent == NULL) { - goto DONE; - } - if (site->is_element) { - site->node = PySequence_GetItem(copy_parent, site->idx); - } - if (site->mirror) { - if (site->idx == 0) { - site->slot = PyObject_GetAttrString(copy_parent, "text"); - } else { - PyObject* prev = PySequence_GetItem(copy_parent, site->idx - 1); - if (prev != NULL) { - site->slot = PyObject_GetAttrString(prev, "tail"); - Py_DECREF(prev); - } - } - } - Py_DECREF(copy_parent); - if ((site->is_element && site->node == NULL) || (site->mirror && site->slot == NULL)) { - goto DONE; - } - } - - // Phase 2: apply to the live tree in document order. - for (i = 0; i < list.count; ++i) { - PyXmlSec_LxmlShadowSite* site = &list.items[i]; - int failed = 0; - PyObject* live_parent = PyXmlSec_LxmlShadowWalk((PyObject*)shadow->element, site->path, site->depth); - if (live_parent == NULL) { - goto DONE; - } - if (site->mirror) { - if (site->idx == 0) { - failed = PyObject_SetAttrString(live_parent, "text", site->slot) < 0; - } else { - PyObject* live_prev = PySequence_GetItem(live_parent, site->idx - 1); - failed = live_prev == NULL || PyObject_SetAttrString(live_prev, "tail", site->slot) < 0; - Py_XDECREF(live_prev); - } - } - if (!failed && site->is_element) { - PyObject* tmp = PyObject_CallMethod(live_parent, "insert", "iO", site->idx, site->node); - failed = tmp == NULL; - Py_XDECREF(tmp); - } - Py_DECREF(live_parent); - if (failed) { - goto DONE; - } + if (PYXMLSEC_SHADOW_MARKED(res)) { + result = PyXmlSec_LxmlShadowEndFound(shadow, res, path, depth); + } else { + result = PyXmlSec_LxmlShadowWalk((PyObject*)shadow->element, path, depth); } - rv = 0; DONE: - for (i = 0; i < list.count; ++i) { - Py_XDECREF(list.items[i].node); - Py_XDECREF(list.items[i].slot); - } - PyMem_Free(list.items); - Py_XDECREF(copy_root); PyXmlSec_LxmlShadowDiscard(shadow); - return rv; + return result; } -// ---------------------------------------------------------------------------- -// Shadow-mode ID registry. -// -// register_id / add_ids used to write straight into lxml's document (its ID -// hash) with our libxml2 — exactly the cross-library access the shadow -// forbids. Under the shadow they record the id-attribute specs here instead, -// and every whole-document Begin replays them onto the private copy so that -// #id references resolve during sign/verify/decrypt. -// -// The registry cannot hold strong references to lxml documents (that would -// pin whole trees forever) and lxml's classes refuse weak references, so -// entries are keyed by the _Document object's address with the underlying -// xmlDoc pointer stored alongside as a staleness check: an entry is trusted -// only while both addresses match, and is replaced when the address has been -// reused by a different document. A size cap bounds growth from dead -// documents whose addresses never get reused. -// ---------------------------------------------------------------------------- - -#define PYXMLSEC_SHADOW_ID_REGISTRY_CAP 4096 - -int PyXmlSec_LxmlShadowRecordId(PyXmlSec_LxmlElementPtr element, const char* name, const char* ns) { - PyObject* key = NULL; - PyObject* cdoc = NULL; - PyObject* created = NULL; - PyObject* spec = NULL; - PyObject* entry; - PyObject* specs; - int contains; - int result = -1; - - key = PyLong_FromVoidPtr((void*)element->_doc); - cdoc = PyLong_FromVoidPtr((void*)element->_doc->_c_doc); - if (key == NULL || cdoc == NULL) { - goto DONE; - } +PyObject* PyXmlSec_LxmlShadowEndFind(PyXmlSec_LxmlShadow* shadow, xmlNodePtr res) { + PyObject* result = NULL; + int path[PYXMLSEC_SHADOW_MAX_DEPTH]; + int depth; - entry = PyDict_GetItem(PyXmlSec_LxmlShadowIdRegistry, key); // borrowed - if (entry != NULL) { - int eq = PyObject_RichCompareBool(PyTuple_GET_ITEM(entry, 0), cdoc, Py_EQ); - if (eq < 0) { - goto DONE; - } - if (!eq) { - entry = NULL; // the address was reused by another document - } - } - if (entry == NULL) { - if (PyDict_Size(PyXmlSec_LxmlShadowIdRegistry) >= PYXMLSEC_SHADOW_ID_REGISTRY_CAP) { - PyObject* k; - PyObject* v; - Py_ssize_t pos = 0; - if (PyDict_Next(PyXmlSec_LxmlShadowIdRegistry, &pos, &k, &v) - && PyDict_DelItem(PyXmlSec_LxmlShadowIdRegistry, k) < 0) { - goto DONE; - } - } - specs = PyList_New(0); - if (specs == NULL) { - goto DONE; - } - created = PyTuple_Pack(2, cdoc, specs); - Py_DECREF(specs); - if (created == NULL || PyDict_SetItem(PyXmlSec_LxmlShadowIdRegistry, key, created) < 0) { - goto DONE; - } - entry = created; + if (res == NULL) { + PyXmlSec_LxmlShadowDiscard(shadow); + Py_RETURN_NONE; } - - spec = Py_BuildValue("(sz)", name, ns); - if (spec == NULL) { - goto DONE; + if (shadow->doc == NULL) { // fast path: res is a live node + PyXmlSec_LxmlShadowDiscard(shadow); + return (PyObject*)PyXmlSec_elementFactory(shadow->element->_doc, res); } - specs = PyTuple_GET_ITEM(entry, 1); - contains = PySequence_Contains(specs, spec); - if (contains < 0 || (!contains && PyList_Append(specs, spec) < 0)) { - goto DONE; + depth = PyXmlSec_LxmlShadowPathTo(res, shadow->root, path); + if (depth < 0) { + PyErr_SetString(PyXmlSec_InternalError, "unexpected result node."); + } else { + result = PyXmlSec_LxmlShadowWalk((PyObject*)shadow->element, path, depth); } - result = 0; - -DONE: - Py_XDECREF(key); - Py_XDECREF(cdoc); - Py_XDECREF(created); - Py_XDECREF(spec); + PyXmlSec_LxmlShadowDiscard(shadow); return result; } -// Registers every attribute named `name` (under `ns` when given) in the copy -// as an XML ID — a superset of the fast path's registrations (single node for -// register_id, subtree for add_ids), which is the safe direction: it mirrors -// what xmlSecAddIDs does from the root. -static void PyXmlSec_LxmlShadowApplyIdSpec(xmlDocPtr doc, xmlNodePtr n, const xmlChar* name, const xmlChar* ns) { - for (; n != NULL; n = n->next) { - if (n->type == XML_ELEMENT_NODE) { - xmlAttrPtr attr = ns != NULL ? xmlHasNsProp(n, name, ns) : xmlHasProp(n, name); - if (attr != NULL && attr->children != NULL) { - xmlChar* value = xmlNodeListGetString(doc, attr->children, 1); - if (value != NULL) { - if (xmlGetID(doc, value) == NULL) { - xmlAddID(NULL, doc, value, attr); - } - xmlFree(value); - } - } - if (n->children != NULL) { - PyXmlSec_LxmlShadowApplyIdSpec(doc, n->children, name, ns); - } +int PyXmlSec_LxmlShadowReflect(PyXmlSec_LxmlShadow* shadow, int rv, const char* error) { + if (rv < 0) { + PyXmlSec_LxmlShadowDiscard(shadow); + if (error != NULL) { + PyXmlSec_SetLastError(error); } + return -1; } + rv = shadow->doc != NULL ? PyXmlSec_LxmlShadowReflectSites(shadow) : 0; + PyXmlSec_LxmlShadowDiscard(shadow); + return rv; } -int PyXmlSec_LxmlShadowReplayIds(PyXmlSec_LxmlShadow* shadow) { - PyObject* key; - PyObject* cdoc; - PyObject* entry; - PyObject* specs; - Py_ssize_t i, n; - int eq; +xmlNodePtr PyXmlSec_LxmlShadowImportElement(PyXmlSec_LxmlShadow* shadow, PyXmlSec_LxmlElementPtr element) { + PyObject* bytes; + xmlDocPtr tdoc; + xmlNodePtr result; - if (shadow->doc == NULL) { // fast path: the live document carries its own IDs - return 0; - } - key = PyLong_FromVoidPtr((void*)shadow->element->_doc); - if (key == NULL) { - return -1; - } - entry = PyDict_GetItem(PyXmlSec_LxmlShadowIdRegistry, key); // borrowed - Py_DECREF(key); - if (entry == NULL) { - return 0; - } - cdoc = PyLong_FromVoidPtr((void*)shadow->element->_doc->_c_doc); - if (cdoc == NULL) { - return -1; - } - eq = PyObject_RichCompareBool(PyTuple_GET_ITEM(entry, 0), cdoc, Py_EQ); - Py_DECREF(cdoc); - if (eq < 0) { - return -1; + bytes = PyXmlSec_LxmlElementToBytes((PyObject*)element); + if (bytes == NULL) { + return NULL; } - if (!eq) { - return 0; // stale entry from a dead document at the same address + tdoc = PyXmlSec_LxmlShadowParse(bytes, "cannot make a private copy of the element."); + Py_DECREF(bytes); + if (tdoc == NULL) { + return NULL; } - - specs = PyTuple_GET_ITEM(entry, 1); - n = PyList_GET_SIZE(specs); - for (i = 0; i < n; ++i) { - PyObject* spec = PyList_GET_ITEM(specs, i); - const char* name = PyUnicode_AsUTF8(PyTuple_GET_ITEM(spec, 0)); - const char* ns = PyTuple_GET_ITEM(spec, 1) == Py_None ? NULL : PyUnicode_AsUTF8(PyTuple_GET_ITEM(spec, 1)); - if (name == NULL || (ns == NULL && PyErr_Occurred())) { - return -1; - } - PyXmlSec_LxmlShadowApplyIdSpec(shadow->doc, shadow->doc->children, (const xmlChar*)name, (const xmlChar*)ns); + // Copy into the shadow doc (both trees are ours). The copy stays + // untagged: from the reflection's point of view whatever the xmlsec call + // grafts of it is a node "the call created". + result = xmlDocCopyNode(xmlDocGetRootElement(tdoc), shadow->doc, 1); + xmlFreeDoc(tdoc); + if (result == NULL) { + PyErr_SetString(PyXmlSec_InternalError, "cannot make a private copy of the element."); } - return 0; + return result; } diff --git a/src/lxml.h b/src/lxml.h index b255d2ba..532f6a06 100644 --- a/src/lxml.h +++ b/src/lxml.h @@ -34,7 +34,7 @@ int PyXmlSec_LxmlElementConverter(PyObject* o, PyXmlSec_LxmlElementPtr* p); // lxml's (possibly different) libxml2 — only serialized bytes cross between // the two libraries (https://github.com/xmlsec/python-xmlsec/issues/356). // -// Usage (see template.c): +// Every binding follows the same four lines (see template.c): // // PyXmlSec_LxmlShadow shadow; // if (PyXmlSec_LxmlShadowBegin(&shadow, node) < 0) goto ON_FAIL; @@ -43,110 +43,97 @@ int PyXmlSec_LxmlElementConverter(PyObject* o, PyXmlSec_LxmlElementPtr* p); // Py_END_ALLOW_THREADS; // result = PyXmlSec_LxmlShadowEnd(&shadow, res, "cannot add reference."); // -// Begin serializes `element` with lxml's own libxml2 and re-parses the bytes -// with ours into `root`/`doc`. The caller then runs exactly one xmlsec call -// against `root` (nothing else; Python may not run between Begin and End) and -// hands the returned node to End, which reflects whatever the call did back -// into the live lxml tree and returns the lxml element corresponding to that -// node (a new reference), or NULL with an exception set (`error` is raised -// when the node is NULL). End must be called exactly once after a successful -// Begin; it always releases the copy. +// Three Begin flavours make the copy: of the element's subtree (Begin), of +// its whole document (BeginDoc — for calls that follow references or walk +// upward), or of nothing (BeginNewDoc — for calls that only need a document +// to build a detached subtree in). The caller then runs exactly one xmlsec +// call against the copy (nothing else; Python may not run between Begin and +// End) and hands its result to one of the End functions, which reflect every +// change the call made back into the live lxml tree and always release the +// copy: +// +// End returns the lxml element for a result node (new reference): +// grafted into the live tree if the call created it, or the +// existing live element (with the attributes / prefix the call +// changed) if it already existed. NULL raises `error`. +// EndFind the same for read-only finders; a NULL result is None, not an +// error. +// Reflect for calls that return only a status: rv < 0 raises `error`, +// otherwise the changes are reflected. +// Discard releases the copy without reflecting (read-only calls such as +// verify, and error paths before End). // // Fast path: when lxml links the same libxml2 as this extension (the -// import-time version check passed), no copy is needed — Begin aliases the +// import-time version check passed), no copy is made — Begin aliases the // live node into `root` (leaving `doc` NULL) and End just wraps the result, -// which is the long-standing direct behavior with zero overhead. Setting +// which is the long-standing direct behaviour with zero overhead. Setting // PYXMLSEC_FORCE_SHADOW in the environment forces the shadow path even on // matched libraries; CI uses it to keep that path exercised. -// -// The reflection covers the whole xmlSecTmpl* family: a new subtree grafted at -// the position xmlsec chose (including intermediate nodes like -// and the "\n" formatting text around it), or — for find-or-create calls that -// added nothing — the already-existing element plus any attributes the call -// set on it. It assumes the call mutates at most one place in the tree; -// multi-site calls (sign, encrypt) use ReflectAll below instead. typedef struct { - PyXmlSec_LxmlElementPtr element; // borrowed; the live lxml element (BeginDoc: the live root) + PyXmlSec_LxmlElementPtr element; // borrowed; the live element copy paths start from (BeginDoc: the live root) PyObject* owned; // reference released when the shadow ends (BeginDoc's root proxy) - xmlDocPtr doc; // the private copy; owned by the shadow - xmlNodePtr root; // doc's root element (the copy of element) + xmlDocPtr doc; // the private copy, owned by the shadow; NULL on the fast path + xmlNodePtr root; // doc's root element, the copy of `element`; NULL for BeginNewDoc } PyXmlSec_LxmlShadow; +// Subtree copy: `shadow.root` is the copy of `element`. int PyXmlSec_LxmlShadowBegin(PyXmlSec_LxmlShadow* shadow, PyXmlSec_LxmlElementPtr element); -PyObject* PyXmlSec_LxmlShadowEnd(PyXmlSec_LxmlShadow* shadow, xmlNodePtr res, const char* error); -// Non-zero when the shadow path is on (mismatched libxml2 or PYXMLSEC_FORCE_SHADOW). -// Only the few call sites whose *semantics* differ per mode (ID registration, -// encrypt/decrypt replacement) may branch on this; everything else goes through -// the Begin/End pairs, which encapsulate both paths. -int PyXmlSec_LxmlShadowIsActive(void); - -// Whole-document shadow, for xmlsec calls that read or mutate beyond the -// element's subtree (sign/verify, encrypt/decrypt, find_parent). Serializes the -// element's whole tree (element.getroottree()), so ID references resolve and -// comments/PIs outside the root survive. `*target` receives the copy's node -// corresponding to `element` (the live node itself on the fast path); -// `shadow.element` becomes the live *root*, so the End/Reflect helpers map -// copy paths from the copy root onto it. +// Whole-document copy, for calls that read or mutate beyond the element's +// subtree (sign/verify, encrypt/decrypt, find_parent). Serializes +// element.getroottree() — comments/PIs outside the root and the internal DTD +// subset survive — and replays the IDs registered for the document +// (RecordId) onto the copy so that #id references resolve. `*target` +// receives the copy's counterpart of `element` (the live node itself on the +// fast path); `shadow.root` / `shadow.element` become the copy root / the +// live root, which is what the End functions map paths between. int PyXmlSec_LxmlShadowBeginDoc(PyXmlSec_LxmlShadow* shadow, PyXmlSec_LxmlElementPtr element, xmlNodePtr* target); -// Create-shape (template.create, encrypted_data_create): the xmlsec call only -// needs a document to allocate a *detached* subtree in. Begin returns that -// document — the element's own on the fast path, a private empty one on the -// shadow path — and End reflects the result as a new detached lxml element -// (no live-tree graft). NULL from Begin means an exception is set. +// Create shape (template.create, encrypted_data_create): the call only needs +// a document to build a *detached* subtree in. Returns that document — the +// element's own on the fast path, a private empty one on the shadow path — +// or NULL with an exception set. End then returns the result as a new +// detached lxml element (in a document of its own until it is grafted; the +// raw path's "detached node inside the source document" has no lxml +// equivalent). xmlDocPtr PyXmlSec_LxmlShadowBeginNewDoc(PyXmlSec_LxmlShadow* shadow, PyXmlSec_LxmlElementPtr element); -PyObject* PyXmlSec_LxmlShadowEndNewDoc(PyXmlSec_LxmlShadow* shadow, xmlNodePtr res, const char* error); -// End for read-only finders: maps `res` (a pre-existing node in the copy) back -// to the live lxml element; returns None when res is NULL (not found — no -// exception). Always releases the copy. +// Reflects every change the call made and returns the lxml element for +// `res` (new reference), or NULL with an exception set (`error` is raised +// when res is NULL). Works after any Begin flavour. +PyObject* PyXmlSec_LxmlShadowEnd(PyXmlSec_LxmlShadow* shadow, xmlNodePtr res, const char* error); + +// End for read-only finders: maps `res` (a pre-existing node in the copy) +// back to the live element; returns None when res is NULL (not found). PyObject* PyXmlSec_LxmlShadowEndFind(PyXmlSec_LxmlShadow* shadow, xmlNodePtr res); -// End for find-or-create calls that may mutate the found node in ways the -// attribute sync cannot express (encrypted_data_ensure_key_info renames the -// namespace prefix): a fresh `res` behaves exactly like End; a pre-existing -// one is reflected by *replacing* the live element with the copy's version, -// so the returned element is a new object rather than the original proxy. -PyObject* PyXmlSec_LxmlShadowEndReplace(PyXmlSec_LxmlShadow* shadow, xmlNodePtr res, const char* error); - -// First node the xmlsec call created (document order), for calls that return -// only a status int: pass the result to End. Falls back to shadow->root when -// the call created nothing (End then takes its find-or-create path). -xmlNodePtr PyXmlSec_LxmlShadowFindFresh(PyXmlSec_LxmlShadow* shadow); - -// Multi-site End (sign, encrypt_binary/uri and the replacement reflects): -// scans the copy for *all* topmost nodes the xmlsec call created and grafts -// each back into the live tree — new subtrees via insert, new/changed text via -// the text slots. Returns 0, or -1 with an exception set. Like End, it always -// releases the copy; there is no result node (the call sites know what to -// return). No-op on the fast path. -int PyXmlSec_LxmlShadowReflectAll(PyXmlSec_LxmlShadow* shadow); - -// Serializes the mutated copy and re-parses it with lxml, returning the root -// element of that detached parse (new reference). Used by the replacement -// reflects (enc.c) when the copy's root itself was replaced. Does not release -// the copy. -PyObject* PyXmlSec_LxmlShadowDumpCopy(PyXmlSec_LxmlShadow* shadow); - -// Re-serializes `element` (a live lxml element) into the shadow's private copy -// as a fresh (untagged) detached subtree — encrypt_xml's template import. -// Shadow path only (shadow->doc != NULL). -xmlNodePtr PyXmlSec_LxmlShadowImportElement(PyXmlSec_LxmlShadow* shadow, PyXmlSec_LxmlElementPtr element); +// End for calls that return only a status (sign, encrypt, ...): rv < 0 +// raises `error` and returns -1; otherwise every change the call made is +// reflected. Returns 0, or -1 with an exception set. No-op on the fast path +// except for raising. +int PyXmlSec_LxmlShadowReflect(PyXmlSec_LxmlShadow* shadow, int rv, const char* error); -// Releases the copy without reflecting anything (verify, error paths). -// Safe to call after any successful Begin*; End* call it internally. +// Releases the copy without reflecting anything (verify, error paths before +// End). Safe to call after any successful Begin; the End functions call it. void PyXmlSec_LxmlShadowDiscard(PyXmlSec_LxmlShadow* shadow); +// Non-zero when the shadow path is on (mismatched libxml2 or +// PYXMLSEC_FORCE_SHADOW). Only the few call sites whose *semantics* differ +// per mode (ID registration, encrypt/decrypt replacement) may branch on +// this; everything else goes through Begin/End, which encapsulate both paths. +int PyXmlSec_LxmlShadowIsActive(void); + +// Re-serializes `element` (a live lxml element) into the shadow's private +// copy as a fresh (untagged) detached subtree — encrypt_xml's template +// import. Shadow path only (shadow->doc != NULL). +xmlNodePtr PyXmlSec_LxmlShadowImportElement(PyXmlSec_LxmlShadow* shadow, PyXmlSec_LxmlElementPtr element); + // Shadow-mode ID registration (issue #356): register_id/add_ids cannot write -// lxml's ID hash with our libxml2, so they record the id-attribute specs per -// document here, and every whole-document Begin replays them onto the copy -// (ReplayIds) so that #id references resolve during sign/verify/decrypt. -// The registry is keyed by the lxml document's identity; replay scans the -// whole copy for the recorded attribute names (a superset of the single-node -// registration on the fast path — see lxml.c). +// lxml's ID hash with our libxml2, so they record the id-attribute spec for +// the element's document here, and every BeginDoc replays the recorded specs +// onto its copy. The replay scans the whole copy for the recorded attribute +// names — a superset of the single-node registration on the fast path. int PyXmlSec_LxmlShadowRecordId(PyXmlSec_LxmlElementPtr element, const char* name, const char* ns); -int PyXmlSec_LxmlShadowReplayIds(PyXmlSec_LxmlShadow* shadow); // get version numbers for libxml2 both compiled and loaded long PyXmlSec_GetLibXmlVersionMajor(); diff --git a/src/template.c b/src/template.c index 5cfa55d0..ff85dd8b 100644 --- a/src/template.c +++ b/src/template.c @@ -62,7 +62,7 @@ static PyObject* PyXmlSec_TemplateCreate(PyObject* self, PyObject *args, PyObjec Py_BEGIN_ALLOW_THREADS; res = xmlSecTmplSignatureCreateNsPref(tdoc, c14n->id, sign->id, XSTR(id), XSTR(ns)); Py_END_ALLOW_THREADS; - result = PyXmlSec_LxmlShadowEndNewDoc(&shadow, res, "cannot create template."); + result = PyXmlSec_LxmlShadowEnd(&shadow, res, "cannot create template."); if (result == NULL) { goto ON_FAIL; } @@ -736,7 +736,7 @@ static PyObject* PyXmlSec_TemplateCreateEncryptedData(PyObject* self, PyObject * res->ns->prefix = xmlStrdup(XSTR(ns)); } Py_END_ALLOW_THREADS; - result = PyXmlSec_LxmlShadowEndNewDoc(&shadow, res, "cannot create encrypted data."); + result = PyXmlSec_LxmlShadowEnd(&shadow, res, "cannot create encrypted data."); if (result == NULL) { goto ON_FAIL; } @@ -787,13 +787,7 @@ static PyObject* PyXmlSec_TemplateEncryptedDataEnsureKeyInfo(PyObject* self, PyO res->ns->prefix = xmlStrdup(XSTR(ns)); } Py_END_ALLOW_THREADS; - if (ns != NULL) { - // renaming the prefix of a KeyInfo that already existed needs the - // replace reflect — attribute sync cannot express it - result = PyXmlSec_LxmlShadowEndReplace(&shadow, res, "cannot ensure key info for encrypted data."); - } else { - result = PyXmlSec_LxmlShadowEnd(&shadow, res, "cannot ensure key info for encrypted data."); - } + result = PyXmlSec_LxmlShadowEnd(&shadow, res, "cannot ensure key info for encrypted data."); if (result == NULL) { goto ON_FAIL; } @@ -860,7 +854,6 @@ static PyObject* PyXmlSec_TemplateTransformAddC14NInclNamespaces(PyObject* self, PyXmlSec_LxmlElementPtr node = NULL; PyObject* prefixes = NULL; PyObject* sep; - PyObject* result; int res; const char* c_prefixes; PyXmlSec_LxmlShadow shadow; @@ -896,20 +889,11 @@ static PyObject* PyXmlSec_TemplateTransformAddC14NInclNamespaces(PyObject* self, Py_BEGIN_ALLOW_THREADS; res = xmlSecTmplTransformAddC14NInclNamespaces(shadow.root, XSTR(c_prefixes)); Py_END_ALLOW_THREADS; - if (res != 0) { - PyXmlSec_LxmlShadowDiscard(&shadow); - PyXmlSec_SetLastError("cannot add 'inclusive' namespaces to the ExcC14N transform node"); - goto ON_FAIL; - } - // The call only reports a status; locate the it - // created on the copy and let End graft it back (the returned element is - // not part of this function's interface). - result = PyXmlSec_LxmlShadowEnd(&shadow, PyXmlSec_LxmlShadowFindFresh(&shadow), - "cannot add 'inclusive' namespaces to the ExcC14N transform node"); - if (result == NULL) { + // the call only reports a status; the reflect grafts the + // it created back into the live tree + if (PyXmlSec_LxmlShadowReflect(&shadow, res, "cannot add 'inclusive' namespaces to the ExcC14N transform node") < 0) { goto ON_FAIL; } - Py_DECREF(result); Py_DECREF(prefixes); PYXMLSEC_DEBUG("transform_add_c14n_inclusive_namespaces - ok"); diff --git a/tests/test_ds.py b/tests/test_ds.py index dd0657d3..3463a9ac 100644 --- a/tests/test_ds.py +++ b/tests/test_ds.py @@ -188,6 +188,30 @@ def test_sign_case5(self): expected_xml_file = 'sign5-out.xml' self.assertEqual(self.load_xml(expected_xml_file), root) + def test_sign_and_verify_with_registered_id(self): + """Should resolve a #id reference registered through register_id (not add_ids) on sign and verify.""" + root = self.load_xml('sign4-in.xml') + ctx = xmlsec.SignatureContext() + ctx.register_id(root, 'ID') + sign = xmlsec.template.create(root, consts.TransformExclC14N, consts.TransformRsaSha1, ns='ds') + root.append(sign) + ref = xmlsec.template.add_reference(sign, consts.TransformSha1, uri='#' + root.get('ID')) + xmlsec.template.add_transform(ref, consts.TransformEnveloped) + xmlsec.template.add_transform(ref, consts.TransformExclC14N) + ki = xmlsec.template.ensure_key_info(sign) + xmlsec.template.add_x509_data(ki) + + ctx.key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem) + ctx.key.load_cert_from_file(self.path('rsacert.pem'), consts.KeyDataFormatPem) + ctx.key.name = 'rsakey.pem' + ctx.sign(sign) + self.assertEqual(self.load_xml('sign4-out.xml'), root) + + verify_ctx = xmlsec.SignatureContext() + verify_ctx.register_id(root, 'ID') + verify_ctx.key = xmlsec.Key.from_file(self.path('rsapub.pem'), format=consts.KeyDataFormatPem) + verify_ctx.verify(sign) + def test_sign_binary_bad_args(self): ctx = xmlsec.SignatureContext() ctx.key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem) diff --git a/tests/test_enc.py b/tests/test_enc.py index 41f78d74..3204eb20 100644 --- a/tests/test_enc.py +++ b/tests/test_enc.py @@ -229,6 +229,40 @@ def check_decrypt(self, i): self.assertIsNotNone(decrypted) self.assertEqual(self.load_xml(f'enc{i}-in.xml'), root) + def encrypt_content(self, xml, path): + """Encrypts the content of the element at ``path`` with a fresh session key.""" + root = etree.fromstring(xml) + key = xmlsec.Key.generate(consts.KeyDataAes, 128, consts.KeyDataTypeSession) + enc_data = xmlsec.template.encrypted_data_create(root, consts.TransformAes128Cbc, type=consts.TypeEncContent, ns='xenc') + xmlsec.template.encrypted_data_ensure_cipher_value(enc_data) + ctx = xmlsec.EncryptionContext() + ctx.key = key + ctx.encrypt_xml(enc_data, root.find(path)) + return root, key + + def test_decrypt_content_text_between_whitespace(self): + # in a pretty-printed document the decrypted text lands between the + # whitespace that surrounded ; the parent's text must + # carry all three pieces + root, key = self.encrypt_content(b'secret', 'Password') + pretty = etree.fromstring(etree.tostring(root, pretty_print=True)) + enc_data = xmlsec.tree.find_node(pretty, consts.NodeEncryptedData, consts.EncNs) + ctx = xmlsec.EncryptionContext() + ctx.key = key + password = ctx.decrypt(enc_data) + self.assertIs(password, pretty[0]) + self.assertEqual('\n secret\n ', password.text) + self.assertEqual(0, len(password)) + + def test_decrypt_content_mixed(self): + root, key = self.encrypt_content(b'hello world end', 'Box') + pretty = etree.fromstring(etree.tostring(root, pretty_print=True)) + enc_data = xmlsec.tree.find_node(pretty, consts.NodeEncryptedData, consts.EncNs) + ctx = xmlsec.EncryptionContext() + ctx.key = key + box = ctx.decrypt(enc_data) + self.assertEqual(b'\n hello world end\n ', etree.tostring(box, with_tail=False)) + def test_decrypt_bad_args(self): ctx = xmlsec.EncryptionContext() with self.assertRaises(TypeError): diff --git a/tests/test_templates.py b/tests/test_templates.py index 70176772..59af6c6f 100644 --- a/tests/test_templates.py +++ b/tests/test_templates.py @@ -239,6 +239,19 @@ def test_encrypted_data_ensure_key_info(self): self.assertEqual('Id', ki2.get('Id')) self.assertEqual('test', ki2.prefix) + def test_encrypted_data_ensure_key_info_rename_prefix(self): + root = self.load_xml('doc.xml') + enc = xmlsec.template.encrypted_data_create(root, method=consts.TransformDes3Cbc) + root.append(enc) + xmlsec.template.encrypted_data_ensure_key_info(enc) + # renaming the prefix of the KeyInfo that already exists keeps it + # live, in place, and unique + ki = xmlsec.template.encrypted_data_ensure_key_info(enc, ns='test') + self.assertIs(ki.getroottree().getroot(), root) + self.assertEqual('test', ki.prefix) + self.assertIs(enc[1], ki) + self.assertEqual(1, sum(1 for n in enc if n.tag == f'{{{consts.DSigNs}}}KeyInfo')) + def test_encrypted_data_ensure_key_info_bad_args(self): with self.assertRaises(TypeError): xmlsec.template.encrypted_data_ensure_key_info('') From 66a674235eba3ee3a71cba55b947ff0005cb73bf Mon Sep 17 00:00:00 2001 From: Amin Solhizadeh Date: Fri, 4 Sep 2026 13:38:42 +0200 Subject: [PATCH 07/22] Guard the shadow invariants with a source-level test (#356) An audit of every binding that accepts an lxml element (30 of them) confirmed that all of them run their xmlsec call on a shadow copy when the shadow mode is on: 26 through the Begin/End helpers, and four (register_id, add_ids, encrypt_xml, decrypt) through an explicit IsActive() switch whose raw body only runs on matched libxml2. Nothing was left to convert, but the invariant was enforced by review alone: on matched libraries a binding that bypassed the switch would still pass the test suite. tests/test_shadow_audit.py now scans src/*.c and fails when a function that takes an lxml element neither calls a PyXmlSec_LxmlShadowBegin* helper nor is one of the listed dual-body functions, when a dual-body function touches a raw node before consulting IsActive(), or when ->_c_node / ->_c_doc appears outside the allowlisted functions (the dual bodies, the helpers' fast-path branches and the ID registry). Two tests prove the checker flags synthetic bad code and one guards the scanner against a broken regex. developer.md states the rules. --- developer.md | 9 +++ tests/test_shadow_audit.py | 151 +++++++++++++++++++++++++++++++++++++ 2 files changed, 160 insertions(+) create mode 100644 tests/test_shadow_audit.py diff --git a/developer.md b/developer.md index 551de773..dc765b74 100644 --- a/developer.md +++ b/developer.md @@ -94,6 +94,15 @@ Rules every call site must keep: in between (the `Py_*_ALLOW_THREADS` pair is fine — the call is pure C); - call exactly one End function after a successful Begin. +**Invariant, enforced by `tests/test_shadow_audit.py`:** every C function that +accepts an lxml element (`PyXmlSec_LxmlElementConverter`) either calls a +`PyXmlSec_LxmlShadowBegin*` helper or is one of the four dual-body functions +(`register_id`, `add_ids`, `encrypt_xml`, `decrypt`), which must consult +`IsActive()` before touching a raw node; and `->_c_node` / `->_c_doc` appear +only in those four, in the helpers' fast-path branches, and in the ID registry +(which uses the addresses as keys only). The test scans `src/*.c`, so a raw +access anywhere else fails the suite on both paths. + ## How the reflection works `Begin` serializes the element with lxml's own `etree.tostring`, re-parses diff --git a/tests/test_shadow_audit.py b/tests/test_shadow_audit.py new file mode 100644 index 00000000..0254b197 --- /dev/null +++ b/tests/test_shadow_audit.py @@ -0,0 +1,151 @@ +"""Source-level guard for the shadow-copy invariants (issue #356). + +The extension may hand lxml's raw libxml2 nodes to xmlsec only on the fast path, when both +link the same libxml2. ``developer.md`` describes the design; this module scans ``src/*.c`` and +checks the two rules that keep every binding on the shadow path whenever it is on: + +1. every C function that accepts an lxml element (it uses ``PyXmlSec_LxmlElementConverter``) + either runs its xmlsec call through a ``PyXmlSec_LxmlShadowBegin*`` helper, or is one of + the dual-body functions in ``DUAL_BODY_FUNCTIONS``, which must consult + ``PyXmlSec_LxmlShadowIsActive()`` before touching a raw node; +2. raw node access (``->_c_node`` / ``->_c_doc``) appears only inside the functions listed in + ``RAW_ACCESS_ALLOWED``. + +Adding a function to either list is a deliberate design decision; see developer.md. +""" + +import glob +import os +import re +import unittest +from collections.abc import Iterator + +SRC_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'src') + +# Bindings with a raw body for the fast path and a shadow body behind IsActive(). +DUAL_BODY_FUNCTIONS = frozenset( + { + 'PyXmlSec_SignatureContextRegisterId', + 'PyXmlSec_TreeAddIds', + 'PyXmlSec_EncryptionContextEncryptXml', + 'PyXmlSec_EncryptionContextDecrypt', + } +) + +# Functions allowed to dereference lxml's raw node/document pointers: the dual bodies above, +# the fast-path branches of the shadow helpers, and the ID registry (addresses used as keys). +RAW_ACCESS_ALLOWED = DUAL_BODY_FUNCTIONS | frozenset( + { + 'PyXmlSec_LxmlShadowBegin', + 'PyXmlSec_LxmlShadowBeginDoc', + 'PyXmlSec_LxmlShadowBeginNewDoc', + 'PyXmlSec_LxmlShadowEnd', + 'PyXmlSec_LxmlShadowEndFind', + 'PyXmlSec_LxmlShadowRecordId', + 'PyXmlSec_LxmlShadowReplayIds', + } +) + +# A function definition at column 0: `[static ][*] PyXmlSec_(`; prototypes end in ';'. +FUNCTION_DEF = re.compile(r'^(?:static\s+)?[\w\s]+?\**\s*\**(PyXmlSec_\w+)\s*\(') +RAW_ACCESS = re.compile(r'->_c_(?:node|doc)\b') +BEGIN_CALL = re.compile(r'\bPyXmlSec_LxmlShadowBegin\w*\s*\(') +CONVERTER = 'PyXmlSec_LxmlElementConverter' +IS_ACTIVE = 'PyXmlSec_LxmlShadowIsActive()' + + +def _functions(source: str) -> Iterator[tuple[str, list[str]]]: + """Yields (name, [lines]) for every PyXmlSec_* function defined in the C source.""" + name = None + body: list[str] = [] + for line in source.splitlines(): + match = FUNCTION_DEF.match(line) + if match and not line.rstrip().endswith(';'): + if name is not None: + yield name, body + name, body = match.group(1), [] + elif name is not None: + body.append(line) + if name is not None: + yield name, body + + +def violations(source: str, filename: str = '') -> list[str]: + """Returns a description of every rule violation in one C source file.""" + found: list[str] = [] + for name, body in _functions(source): + where = f'{filename}:{name}' + raw_lines = [i for i, line in enumerate(body) if RAW_ACCESS.search(line)] + active_lines = [i for i, line in enumerate(body) if IS_ACTIVE in line] + takes_element = any(CONVERTER in line for line in body) + begins = any(BEGIN_CALL.search(line) for line in body) + + if name in DUAL_BODY_FUNCTIONS: + if not active_lines: + found.append(f'{where}: dual-body function never consults {IS_ACTIVE}') + elif raw_lines and raw_lines[0] < active_lines[0]: + found.append(f'{where}: raw node access before {IS_ACTIVE}') + elif takes_element and not begins: + found.append(f'{where}: takes an lxml element but never calls a PyXmlSec_LxmlShadowBegin* helper') + + if raw_lines and name not in RAW_ACCESS_ALLOWED: + found.append(f'{where}: raw node access (->_c_node / ->_c_doc) outside the allowed functions') + return found + + +@unittest.skipUnless(os.path.isdir(SRC_DIR), 'C sources not available (installed package)') +class TestShadowAudit(unittest.TestCase): + def sources(self) -> Iterator[tuple[str, str]]: + files = sorted(glob.glob(os.path.join(SRC_DIR, '*.c'))) + self.assertTrue(files, f'no C sources under {SRC_DIR}') + for path in files: + with open(path, encoding='utf-8') as f: + yield os.path.basename(path), f.read() + + def test_scanner_sees_the_bindings(self) -> None: + # guards the scanner itself: a broken regex would make the sources look clean + names: set[str] = set() + raw_files: set[str] = set() + for filename, source in self.sources(): + for name, body in _functions(source): + if any(CONVERTER in line for line in body): + names.add(name) + if any(RAW_ACCESS.search(line) for line in body): + raw_files.add(filename) + self.assertGreaterEqual(len(names), 30) + self.assertTrue(names.issuperset({'PyXmlSec_TemplateAddReference', 'PyXmlSec_SignatureContextSign'})) + self.assertTrue(names.issuperset(DUAL_BODY_FUNCTIONS)) + self.assertEqual({'ds.c', 'enc.c', 'lxml.c', 'tree.c'}, raw_files) + + def test_every_binding_goes_through_the_shadow(self) -> None: + found: list[str] = [] + for filename, source in self.sources(): + found.extend(violations(source, filename)) + self.assertEqual([], found, '\n'.join(found)) + + def test_checker_flags_a_raw_binding(self) -> None: + bad = ( + 'static PyObject* PyXmlSec_Bad(PyObject* self, PyObject* args) {\n' + ' PyXmlSec_LxmlElementPtr node = NULL;\n' + ' if (!PyArg_ParseTuple(args, "O&:bad", PyXmlSec_LxmlElementConverter, &node)) return NULL;\n' + ' return (PyObject*)PyXmlSec_elementFactory(node->_doc, xmlSecFindChild(node->_c_node, NULL, NULL));\n' + '}\n' + ) + found = violations(bad, 'bad.c') + self.assertEqual(2, len(found), found) + self.assertIn('never calls a PyXmlSec_LxmlShadowBegin*', found[0]) + self.assertIn('outside the allowed functions', found[1]) + + def test_checker_flags_raw_access_before_the_switch(self) -> None: + bad = ( + 'static PyObject* PyXmlSec_TreeAddIds(PyObject* self, PyObject* args) {\n' + ' PyXmlSec_LxmlElementPtr node = NULL;\n' + ' if (!PyArg_ParseTuple(args, "O&:add_ids", PyXmlSec_LxmlElementConverter, &node)) return NULL;\n' + ' xmlDocPtr doc = node->_doc->_c_doc;\n' + ' if (PyXmlSec_LxmlShadowIsActive()) Py_RETURN_NONE;\n' + ' return NULL;\n' + '}\n' + ) + found = violations(bad, 'bad.c') + self.assertEqual(1, len(found), found) + self.assertIn('raw node access before', found[0]) From a3b1fce72e8f311e8dea10cfd1f6ab322598e799 Mon Sep 17 00:00:00 2001 From: Amin Solhizadeh Date: Fri, 4 Sep 2026 14:29:56 +0200 Subject: [PATCH 08/22] Reflect root replacement by morphing the live root in place (#356) The shadow path refused to encrypt the document root with Type=Element and to decrypt a root EncryptedData, since lxml offers no way to swap a document's root element (_ElementTree._setroot only rebinds that one Python object). The reflection now morphs the live root element in place into the re-parsed replacement through lxml's public API: it is emptied, stripped of its namespace declarations, given exactly the replacement's (a temporary child pins the default namespace, which the tag setter never declares and cleanup_namespaces would drop as unused), then renamed and refilled. Both shadow bodies in enc.c return the node itself for a root, matching the raw path's contract of returning the new root. A root replaced by anything but a single element is refused with xmlsec.Error. --- developer.md | 20 +++-- src/enc.c | 63 ++++++++-------- src/lxml.c | 183 ++++++++++++++++++++++++++++++++++++++++++++-- src/lxml.h | 5 +- tests/test_enc.py | 41 +++++++++++ 5 files changed, 263 insertions(+), 49 deletions(-) diff --git a/developer.md b/developer.md index dc765b74..835dd59b 100644 --- a/developer.md +++ b/developer.md @@ -239,14 +239,18 @@ All invisible to the documented API: - signature/encryption contexts keep no live result nodes after the call (they never usefully did); - documents nested deeper than 256 levels (only possible with `huge_tree`) - are refused with an internal error. - -One hard limitation: operations that would replace the **document root** -(encrypting the root element with `Type=Element`, decrypting a root -`EncryptedData`) raise `xmlsec.Error` — lxml's API cannot swap a document's -root, and morphing it in place would rewrite namespace prefixes, breaking -signatures over the content. Re-parse the document into a wrapper or work on -a subelement instead. + are refused with an internal error; +- operations that replace the **document root** (encrypting the root element + with `Type=Element`, decrypting a root `EncryptedData`) morph the live root + element in place into the replacement, because lxml's API cannot swap a + document's root (`_ElementTree._setroot` only rebinds that one Python + object). The result is the same as on the raw path — the returned element + is the new root, with the replacement's own namespace declarations and + document-level siblings intact — except that the caller's root proxy (and + any `_ElementTree` holding it) *becomes* the replacement instead of going + stale as a detached copy of the old root. A root replaced by anything but a + single element (a `Type=Content` decryption of the root) is refused with + `xmlsec.Error`. ## Building & validating diff --git a/src/enc.c b/src/enc.c index 2f684e5b..4d932034 100644 --- a/src/enc.c +++ b/src/enc.c @@ -233,7 +233,8 @@ static void PyXmlSec_ClearReplacedNodes(xmlSecEncCtxPtr ctx, PyXmlSec_LxmlDocume // Shadow-path body of encrypt_xml (issue #356): the target document and the // template are both re-parsed into one private copy, the encryption runs // there, and the replacement is reflected back through lxml — the fresh -// takes the target's place (`Type=Element`) or its content +// takes the target's place (`Type=Element`; the document +// root is morphed in place, as lxml cannot swap it) or its content // (`Type=Content`). One divergence from the raw path: a template that is // *attached* inside the target's own tree is copied, not moved, so it also // remains at its original position. @@ -307,18 +308,6 @@ static PyObject* PyXmlSec_EncryptionContextEncryptXmlShadow(PyXmlSec_EncryptionC goto ON_FAIL; } - if (parent == Py_None && !is_content) { - // The encryption replaced the copy's root, which cannot be reflected: - // lxml's API offers no way to swap a document's root element (and - // morphing it in place would rewrite namespace prefixes, breaking - // signatures over the content). The live tree is untouched. Encrypt a - // subelement instead, or re-parse the document. - PyXmlSec_LxmlShadowDiscard(&shadow); - PyErr_SetString(PyXmlSec_Error, - "encrypting the document root is not supported when lxml and xmlsec use different libxml2 libraries"); - goto ON_FAIL; - } - if (is_content) { // the node stays; its old content was consumed and replaced by the // fresh , which the reflection grafts back in @@ -344,6 +333,15 @@ static PyObject* PyXmlSec_EncryptionContextEncryptXmlShadow(PyXmlSec_EncryptionC if (result == NULL) { goto ON_FAIL; } + } else if (parent == Py_None) { + // Type=Element on the document root: the reflection morphs the live + // root in place into the fresh , so the node itself + // is the result — the new root, as on the raw path + if (PyXmlSec_LxmlShadowReflect(&shadow, 0, NULL) < 0) { + goto ON_FAIL; + } + result = (PyObject*)node; + Py_INCREF(result); } else { // Type=Element: the node itself was consumed and replaced tmp = PyObject_CallMethod(parent, "remove", "O", node); @@ -582,35 +580,32 @@ static PyObject* PyXmlSec_EncryptionContextDecryptShadow(PyXmlSec_EncryptionCont ); } - if (parent == Py_None) { - // Decryption replaced the document root; lxml's API offers no way to - // swap a document's root element, so this cannot be reflected (the - // live tree is untouched). Re-parse the document into a wrapper or - // decrypt a non-root EncryptedData instead. - PyXmlSec_LxmlShadowDiscard(&shadow); - PyErr_SetString(PyXmlSec_Error, - "decrypting the document root is not supported when lxml and xmlsec use different libxml2 libraries"); - goto ON_FAIL; - } else { - // the node was consumed; the reflection grafts whatever replaced it + // the node was consumed; the reflection grafts whatever replaced it — or, + // for the document root (no parent to remove it from), morphs the node + // itself into the replacement, which is then the new root: the raw path's + // result too + if (parent != Py_None) { tmp = PyObject_CallMethod(parent, "remove", "O", node); if (tmp == NULL) { PyXmlSec_LxmlShadowDiscard(&shadow); goto ON_FAIL; } Py_CLEAR(tmp); - if (PyXmlSec_LxmlShadowReflect(&shadow, 0, NULL) < 0) { + } + if (PyXmlSec_LxmlShadowReflect(&shadow, 0, NULL) < 0) { + goto ON_FAIL; + } + if (parent == Py_None) { + result = (PyObject*)node; + Py_INCREF(result); + } else if (not_content) { + result = PySequence_GetItem(parent, (Py_ssize_t)idx); + if (result == NULL) { goto ON_FAIL; } - if (not_content) { - result = PySequence_GetItem(parent, (Py_ssize_t)idx); - if (result == NULL) { - goto ON_FAIL; - } - } else { - result = parent; - Py_INCREF(result); - } + } else { + result = parent; + Py_INCREF(result); } Py_DECREF(parent); diff --git a/src/lxml.c b/src/lxml.c index d666ce69..cafbacf2 100644 --- a/src/lxml.c +++ b/src/lxml.c @@ -109,6 +109,7 @@ static int PyXmlSec_LxmlShadowActive = 1; // and kept for the lifetime of the process. static PyObject* PyXmlSec_LxmlEtreeToString; static PyObject* PyXmlSec_LxmlEtreeFromString; +static PyObject* PyXmlSec_LxmlEtreeCleanupNamespaces; static PyObject* PyXmlSec_LxmlEtreeParser; // Shadow-mode ID registry: maps the identity of an lxml document to the list @@ -164,9 +165,11 @@ int PyXmlSec_InitLxmlModule(void) { } PyXmlSec_LxmlEtreeToString = PyObject_GetAttrString(etree, "tostring"); PyXmlSec_LxmlEtreeFromString = PyObject_GetAttrString(etree, "fromstring"); + PyXmlSec_LxmlEtreeCleanupNamespaces = PyObject_GetAttrString(etree, "cleanup_namespaces"); PyXmlSec_LxmlEtreeParser = PyXmlSec_LxmlNewParser(etree); Py_DECREF(etree); - if (PyXmlSec_LxmlEtreeToString == NULL || PyXmlSec_LxmlEtreeFromString == NULL || PyXmlSec_LxmlEtreeParser == NULL) { + if (PyXmlSec_LxmlEtreeToString == NULL || PyXmlSec_LxmlEtreeFromString == NULL + || PyXmlSec_LxmlEtreeCleanupNamespaces == NULL || PyXmlSec_LxmlEtreeParser == NULL) { return -1; } @@ -898,6 +901,151 @@ static int PyXmlSec_LxmlShadowSetTextSlots(PyObject* parent, PyObject* slots) { return 0; } +// etree.cleanup_namespaces(element, top_nsmap=..., keep_ns_prefixes=...); +// either keyword may be NULL to leave it out. +static int PyXmlSec_LxmlCleanupNamespaces(PyObject* element, PyObject* top_nsmap, PyObject* keep) { + PyObject* args = PyTuple_Pack(1, element); + PyObject* kwargs = PyDict_New(); + PyObject* result = NULL; + + if (args != NULL && kwargs != NULL + && (top_nsmap == NULL || PyDict_SetItemString(kwargs, "top_nsmap", top_nsmap) == 0) + && (keep == NULL || PyDict_SetItemString(kwargs, "keep_ns_prefixes", keep) == 0)) { + result = PyObject_Call(PyXmlSec_LxmlEtreeCleanupNamespaces, args, kwargs); + } + Py_XDECREF(args); + Py_XDECREF(kwargs); + Py_XDECREF(result); + return result != NULL ? 0 : -1; +} + +// The call replaced the copy's root element itself — encrypt_xml with +// Type=Element on the document root, decrypt of a root . +// lxml offers no way to swap a document's root element (_ElementTree._setroot +// only rebinds that one Python object; the document keeps its root), so the +// live element is morphed in place into `fresh`, the re-parsed replacement, +// through lxml's public API: emptied and stripped of every namespace +// declaration, given exactly the declarations of `fresh` (a temporary child +// pins the default namespace, which cleanup_namespaces would otherwise drop +// as unused before the tag can use it — the tag setter itself never declares +// a default namespace), then renamed and refilled. The children go through +// lxml's usual namespace reconciliation, like every graft. The live proxy +// thus *becomes* the replacement, where the raw path leaves the caller's old +// root proxy (and any _ElementTree holding it) detached and stale. +static int PyXmlSec_LxmlShadowMorphRoot(PyObject* live, PyObject* fresh) { + PyObject* tag = NULL; + PyObject* tail = NULL; + PyObject* nsmap = NULL; + PyObject* keep = NULL; + PyObject* pin = NULL; + PyObject* attrib = NULL; + PyObject* fresh_attrib = NULL; + PyObject* text = NULL; + PyObject* children = NULL; + PyObject* tmp = NULL; + PyObject* key; + PyObject* href; + Py_ssize_t pos = 0; + const char* local; + int rv = -1; + + // Empty the element and give it a namespace-free name (the tail is not + // the call's to change), so that every old declaration is unused and + // cleanup_namespaces drops it — a conflicting old prefix would otherwise + // block the new declaration. + tag = PyObject_GetAttrString(fresh, "tag"); + tail = PyObject_GetAttrString(live, "tail"); + if (tag == NULL || tail == NULL || (local = PyUnicode_AsUTF8(tag)) == NULL) { + goto DONE; + } + if (strchr(local, '}') != NULL) { + local = strchr(local, '}') + 1; + } + tmp = PyObject_CallMethod(live, "clear", NULL); + if (tmp == NULL || PyObject_SetAttrString(live, "tail", tail) < 0) { + goto DONE; + } + Py_CLEAR(tmp); + tmp = PyUnicode_FromString(local); + if (tmp == NULL || PyObject_SetAttrString(live, "tag", tmp) < 0) { + goto DONE; + } + Py_CLEAR(tmp); + if (PyXmlSec_LxmlCleanupNamespaces(live, NULL, NULL) < 0) { + goto DONE; + } + + // Declare exactly the replacement's namespaces. + nsmap = PyObject_GetAttrString(fresh, "nsmap"); + keep = PyList_New(0); + if (nsmap == NULL || keep == NULL) { + goto DONE; + } + if (!PyDict_Check(nsmap)) { + PyErr_SetString(PyXmlSec_InternalError, "unexpected nsmap."); + goto DONE; + } + while (PyDict_Next(nsmap, &pos, &key, &href)) { + if (key == Py_None) { + tmp = PyUnicode_FromFormat("{%U}pin", href); + pin = tmp != NULL ? PyObject_CallMethod(live, "makeelement", "O", tmp) : NULL; + Py_CLEAR(tmp); + tmp = pin != NULL ? PyObject_CallMethod(live, "append", "O", pin) : NULL; + if (tmp == NULL) { + goto DONE; + } + Py_CLEAR(tmp); + } else if (PyList_Append(keep, key) < 0) { + goto DONE; + } + } + if (PyXmlSec_LxmlCleanupNamespaces(live, nsmap, keep) < 0) { + goto DONE; + } + + // Rename and refill. + if (PyObject_SetAttrString(live, "tag", tag) < 0) { + goto DONE; + } + attrib = PyObject_GetAttrString(live, "attrib"); + fresh_attrib = PyObject_GetAttrString(fresh, "attrib"); + tmp = attrib != NULL && fresh_attrib != NULL ? PyObject_CallMethod(attrib, "update", "O", fresh_attrib) : NULL; + if (tmp == NULL) { + goto DONE; + } + Py_CLEAR(tmp); + if (pin != NULL) { + tmp = PyObject_CallMethod(live, "remove", "O", pin); + if (tmp == NULL) { + goto DONE; + } + Py_CLEAR(tmp); + } + text = PyObject_GetAttrString(fresh, "text"); + if (text == NULL || PyObject_SetAttrString(live, "text", text) < 0) { + goto DONE; + } + children = PySequence_List(fresh); + tmp = children != NULL ? PyObject_CallMethod(live, "extend", "O", children) : NULL; + if (tmp == NULL) { + goto DONE; + } + rv = 0; + +DONE: + Py_XDECREF(tag); + Py_XDECREF(tail); + Py_XDECREF(nsmap); + Py_XDECREF(keep); + Py_XDECREF(pin); + Py_XDECREF(attrib); + Py_XDECREF(fresh_attrib); + Py_XDECREF(text); + Py_XDECREF(children); + Py_XDECREF(tmp); + return rv; +} + // Applies every change in the copy to the live tree. Does not release the // copy. Returns 0, or -1 with an exception set. static int PyXmlSec_LxmlShadowReflectSites(PyXmlSec_LxmlShadow* shadow) { @@ -906,15 +1054,38 @@ static int PyXmlSec_LxmlShadowReflectSites(PyXmlSec_LxmlShadow* shadow) { int i; int rv = -1; - // Re-fetch the root: replacement operations may swap nodes at the top. A - // fresh root means the call replaced the root itself, which cannot be - // reflected (lxml cannot swap a document's root); the callers that can - // hit that (enc.c) check for it before getting here. + // Re-fetch the root: replacement operations may swap nodes at the top. list.top = xmlDocGetRootElement(shadow->doc); - if (list.top == NULL || !PYXMLSEC_SHADOW_MARKED(list.top)) { + if (list.top == NULL) { PyErr_SetString(PyXmlSec_InternalError, "unexpected mutation site."); goto DONE; } + if (!PYXMLSEC_SHADOW_MARKED(list.top)) { + // A fresh root: the call replaced the root itself, so there are no + // sites to graft — the live root is morphed into it wholesale. lxml + // holds one element (plus comments and PIs) at document level, so a + // root replaced by anything else (a Type=Content decryption of the + // root) cannot be reflected. + xmlNodePtr n; + int elements = 0; + int others = 0; + for (n = shadow->doc->children; n != NULL; n = n->next) { + if (n->type == XML_ELEMENT_NODE) { + ++elements; + } else if (n->type != XML_COMMENT_NODE && n->type != XML_PI_NODE && n->type != XML_DTD_NODE) { + ++others; + } + } + if (elements != 1 || others != 0) { + PyErr_SetString(PyXmlSec_Error, "the document root was replaced by content that is not a single element"); + goto DONE; + } + copy_root = PyXmlSec_LxmlShadowDumpCopy(shadow); + if (copy_root != NULL) { + rv = PyXmlSec_LxmlShadowMorphRoot((PyObject*)shadow->element, copy_root); + } + goto DONE; + } if (PyXmlSec_LxmlShadowCollectSites(&list, list.top) < 0) { goto DONE; } diff --git a/src/lxml.h b/src/lxml.h index 532f6a06..5da1ac0f 100644 --- a/src/lxml.h +++ b/src/lxml.h @@ -59,7 +59,10 @@ int PyXmlSec_LxmlElementConverter(PyObject* o, PyXmlSec_LxmlElementPtr* p); // EndFind the same for read-only finders; a NULL result is None, not an // error. // Reflect for calls that return only a status: rv < 0 raises `error`, -// otherwise the changes are reflected. +// otherwise the changes are reflected. A call that replaced the +// copy's root element (encrypt/decrypt of the document root) +// morphs the live element in place into the replacement, since +// lxml cannot swap a document's root. // Discard releases the copy without reflecting (read-only calls such as // verify, and error paths before End). // diff --git a/tests/test_enc.py b/tests/test_enc.py index 3204eb20..8db004b5 100644 --- a/tests/test_enc.py +++ b/tests/test_enc.py @@ -1,3 +1,4 @@ +import io import tempfile from lxml import etree @@ -82,6 +83,32 @@ def test_encrypt_xml(self): cipher_value = xmlsec.tree.find_node(ki, consts.NodeCipherValue, consts.EncNs) self.assertIsNotNone(cipher_value) + def test_encrypt_xml_root(self): + # Type=Element on the document root replaces the root itself: the new + # root keeps the template's namespace prefix and the document-level + # siblings, and decrypting it restores the document in place + xml = b'ttail' + root = etree.parse(io.BytesIO(xml)).getroot() + key = xmlsec.Key.generate(consts.KeyDataAes, 128, consts.KeyDataTypeSession) + enc_data = xmlsec.template.encrypted_data_create(root, consts.TransformAes128Cbc, type=consts.TypeEncElement, ns='xenc') + xmlsec.template.encrypted_data_ensure_cipher_value(enc_data) + ctx = xmlsec.EncryptionContext() + ctx.key = key + encrypted = ctx.encrypt_xml(enc_data, root) + self.assertEqual(f'{{{consts.EncNs}}}{consts.NodeEncryptedData}', encrypted.tag) + self.assertEqual('xenc', encrypted.prefix) + self.assertIsNone(encrypted.getparent()) + self.assertIs(encrypted.getroottree().getroot(), encrypted) + self.assertTrue(etree.tostring(encrypted.getroottree()).startswith(b't', '.') + enc_data = etree.fromstring(etree.tostring(root[0])) + self.assertIsNone(enc_data.getparent()) + ctx = xmlsec.EncryptionContext() + ctx.key = key + decrypted = ctx.decrypt(enc_data) + self.assertEqual('x', decrypted.tag) + self.assertEqual('t', decrypted.text) + self.assertIsNone(decrypted.getparent()) + self.assertIs(decrypted.getroottree().getroot(), decrypted) + def test_decrypt_content_text_between_whitespace(self): # in a pretty-printed document the decrypted text lands between the # whitespace that surrounded ; the parent's text must From 6cc57b263bb6df4a8db4e627dd4f6092f8a539cc Mon Sep 17 00:00:00 2001 From: Amin Solhizadeh Date: Fri, 4 Sep 2026 14:39:15 +0200 Subject: [PATCH 09/22] Match id attributes by local name under the shadow (#356) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The shadow's register_id validated the attribute with node.get(), which only finds an unqualified attribute, while the fast path's xmlHasProp() matches by local name whatever the namespace. A valid register_id(node, "Id") for a namespaced Id could therefore raise "missing attribute." in shadow mode only. Without id_ns the lookup now scans the element's attribute names and compares local names, as xmlHasProp does — and as the ID replay onto the copy already did. Co-Authored-By: Claude Opus 5 --- developer.md | 4 +-- src/ds.c | 78 ++++++++++++++++++++++++++++++++++++++---------- tests/test_ds.py | 8 +++++ 3 files changed, 72 insertions(+), 18 deletions(-) diff --git a/developer.md b/developer.md index 835dd59b..c412c402 100644 --- a/developer.md +++ b/developer.md @@ -231,9 +231,7 @@ All invisible to the documented API: document until grafted; - `encrypted_data_ensure_key_info(ns=...)` on an existing `KeyInfo` returns a new element object rather than the original proxy; -- `register_id` skips the live duplicate-id check (it runs per copy instead) - and, without `id_ns`, looks the attribute up namespace-strictly where the - raw `xmlHasProp` is namespace-agnostic; +- `register_id` skips the live duplicate-id check (it runs per copy instead); - `encrypt_xml` *copies* a template that is attached inside the target tree rather than moving it, so it also remains at its original position; - signature/encryption contexts keep no live result nodes after the call diff --git a/src/ds.c b/src/ds.c index eff7a8a6..b1288a45 100644 --- a/src/ds.c +++ b/src/ds.c @@ -119,6 +119,50 @@ static int PyXmlSec_SignatureContextKeySet(PyObject* self, PyObject* value, void return 0; } +// Non-zero when `node` carries an attribute whose *local* name is `name`, +// whatever its namespace, -1 with an exception set on failure. The fast +// path's xmlHasProp() matches that way, while lxml's node.get(name) only +// finds an unqualified attribute; the shadow's validation goes through this +// so that both modes accept the same calls. +static int PyXmlSec_LxmlHasAttrByLocalName(PyXmlSec_LxmlElementPtr node, const char* name) { + PyObject* keys; + PyObject* seq; + Py_ssize_t i; + Py_ssize_t n; + int found = 0; + + keys = PyObject_CallMethod((PyObject*)node, "keys", NULL); + if (keys == NULL) { + return -1; + } + seq = PySequence_Fast(keys, "unexpected attribute names."); + Py_DECREF(keys); + if (seq == NULL) { + return -1; + } + + n = PySequence_Fast_GET_SIZE(seq); + for (i = 0; i < n && !found; ++i) { + PyObject* key = PySequence_Fast_GET_ITEM(seq, i); // borrowed + const char* local; + const char* end; + if (!PyUnicode_Check(key) || (local = PyUnicode_AsUTF8(key)) == NULL) { + if (!PyErr_Occurred()) { + PyErr_SetString(PyXmlSec_InternalError, "unexpected attribute name."); + } + Py_DECREF(seq); + return -1; + } + // lxml spells a namespaced attribute "{href}local". + if (local[0] == '{' && (end = strchr(local, '}')) != NULL) { + local = end + 1; + } + found = strcmp(local, name) == 0; + } + Py_DECREF(seq); + return found; +} + static const char PyXmlSec_SignatureContextRegisterId__doc__[] = \ "register_id(node, id_attr = 'ID', id_ns = None) -> None\n" "Registers new id.\n\n" @@ -151,27 +195,31 @@ static PyObject* PyXmlSec_SignatureContextRegisterId(PyObject* self, PyObject* a // whole-document shadows (sign/verify/decrypt) replay it onto their // private copies. The duplicate-id check runs there, per copy. if (PyXmlSec_LxmlShadowIsActive()) { - PyObject* key; - PyObject* value; + int found; if (id_ns != NULL) { - key = PyUnicode_FromFormat("{%s}%s", id_ns, id_attr); + PyObject* key = PyUnicode_FromFormat("{%s}%s", id_ns, id_attr); + PyObject* value; + if (key == NULL) { + goto ON_FAIL; + } + value = PyObject_CallMethod((PyObject*)node, "get", "O", key); + Py_DECREF(key); + if (value == NULL) { + goto ON_FAIL; + } + found = value != Py_None; + Py_DECREF(value); } else { - key = PyUnicode_FromString(id_attr); - } - if (key == NULL) { - goto ON_FAIL; - } - value = PyObject_CallMethod((PyObject*)node, "get", "O", key); - Py_DECREF(key); - if (value == NULL) { - goto ON_FAIL; + // As xmlHasProp() does on the fast path: by local name. + found = PyXmlSec_LxmlHasAttrByLocalName(node, id_attr); + if (found < 0) { + goto ON_FAIL; + } } - if (value == Py_None) { - Py_DECREF(value); + if (!found) { PyErr_SetString(PyXmlSec_Error, "missing attribute."); goto ON_FAIL; } - Py_DECREF(value); if (PyXmlSec_LxmlShadowRecordId(node, id_attr, id_ns) < 0) { goto ON_FAIL; } diff --git a/tests/test_ds.py b/tests/test_ds.py index 3463a9ac..f5a535a9 100644 --- a/tests/test_ds.py +++ b/tests/test_ds.py @@ -55,6 +55,14 @@ def test_register_id_bad_args(self): with self.assertRaises(TypeError): ctx.register_id('') + def test_register_id_matches_namespaced_attribute_by_local_name(self): + """Should accept a namespaced id attribute when no id_ns is given, as xmlHasProp does.""" + ctx = xmlsec.SignatureContext() + root = self.load_xml('sign_template.xml') + sign = xmlsec.template.create(root, consts.TransformExclC14N, consts.TransformRsaSha1) + sign.set('{http://www.example.org/ns}Id', 'sig-1') + ctx.register_id(sign, 'Id') + def test_register_id_with_namespace_without_attribute(self): ctx = xmlsec.SignatureContext() root = self.load_xml('sign_template.xml') From 4e9e56bb8d07ec20384f85801f7bd2473418250b Mon Sep 17 00:00:00 2001 From: Amin Solhizadeh Date: Fri, 4 Sep 2026 15:02:03 +0200 Subject: [PATCH 10/22] Never evict a live id registration (#356) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The shadow's id registry held no reference to the documents it recorded, so entries could only be validated by a stored _c_doc address and the dict was capped at 4096 to bound growth from dead documents. At the cap it deleted an arbitrary entry, which could be a live document's: a long-running process that keeps an early signed document and registers ids for 4096 later ones silently stopped resolving that document's #id references. An entry now keeps a strong reference to its _Document. The key, the document's address, can then never go stale — the entry itself keeps that address occupied — so the _c_doc check is gone; and liveness becomes decidable without weak references, which lxml's classes refuse: when the registry holds the only reference to a document, nothing can present that document to a binding again, so the entry is dead. Every new registration first drops those entries, which bounds both the registry and the documents it pins by the documents still in use. No cap, and nothing reachable is ever evicted. Co-Authored-By: Claude Opus 5 --- developer.md | 14 +++--- src/lxml.c | 88 +++++++++++++++++++------------------- tests/test_ds.py | 27 ++++++++++++ tests/test_shadow_audit.py | 6 +-- 4 files changed, 80 insertions(+), 55 deletions(-) diff --git a/developer.md b/developer.md index c412c402..f0039c9d 100644 --- a/developer.md +++ b/developer.md @@ -99,9 +99,8 @@ accepts an lxml element (`PyXmlSec_LxmlElementConverter`) either calls a `PyXmlSec_LxmlShadowBegin*` helper or is one of the four dual-body functions (`register_id`, `add_ids`, `encrypt_xml`, `decrypt`), which must consult `IsActive()` before touching a raw node; and `->_c_node` / `->_c_doc` appear -only in those four, in the helpers' fast-path branches, and in the ID registry -(which uses the addresses as keys only). The test scans `src/*.c`, so a raw -access anywhere else fails the suite on both paths. +only in those four and in the helpers' fast-path branches. The test scans +`src/*.c`, so a raw access anywhere else fails the suite on both paths. ## How the reflection works @@ -198,9 +197,12 @@ document identity (`RecordId`), and every `BeginDoc` replays them onto its copy so that `#id` references resolve during sign/verify/decrypt. The replay scans the whole copy for the recorded attribute names — a superset of the single-node registration on the raw path, mirroring what `xmlSecAddIDs` does -from the root. The registry holds no strong references to documents (lxml's -classes refuse weak references), so entries are validated by a stored -`_c_doc` address and capped in size. The two bindings are the only places, +from the root. lxml's classes refuse weak references, so an entry keeps a +strong reference to its document instead: the key (the document's address) +can then never go stale, and a document referenced by nothing but the +registry is provably unreachable, so its entry — and the document with it — +is dropped before the next registration. The registry therefore tracks the +documents still in use and never evicts a live one. The two bindings are the only places, together with encrypt_xml/decrypt's replacement bodies, that branch on `IsActive()`. diff --git a/src/lxml.c b/src/lxml.c index cafbacf2..71f54cbc 100644 --- a/src/lxml.c +++ b/src/lxml.c @@ -491,20 +491,49 @@ void PyXmlSec_LxmlShadowDiscard(PyXmlSec_LxmlShadow* shadow) { // and every whole-document Begin replays them onto the private copy so that // #id references resolve during sign/verify/decrypt. // -// The registry cannot hold strong references to lxml documents (that would -// pin whole trees forever) and lxml's classes refuse weak references, so -// entries are keyed by the _Document object's address with the underlying -// xmlDoc pointer stored alongside as a staleness check: an entry is trusted -// only while both addresses match, and is replaced when the address has been -// reused by a different document. A size cap bounds growth from dead -// documents whose addresses never get reused. +// An entry is keyed by the _Document object's address and holds a strong +// reference to that document, so the key can never go stale: the address +// cannot be reused while the entry keeps the object alive. That reference +// also makes liveness decidable without weak references (lxml's classes +// refuse those): when the registry holds the *only* reference to a document, +// nothing can hand that document to a binding again, so the entry is dead and +// is dropped. Pruning runs before every new registration, which bounds the +// registry — and the documents it pins — by the documents still in use. No +// live registration is ever evicted. // ---------------------------------------------------------------------------- -#define PYXMLSEC_SHADOW_ID_REGISTRY_CAP 4096 +// Drops the entries of documents nobody but the registry still references. +// Best effort: on an allocation failure the entries simply survive until the +// next registration prunes them. +static void PyXmlSec_LxmlShadowPruneIdRegistry(void) { + PyObject* dead; + PyObject* key; + PyObject* entry; + Py_ssize_t pos = 0; + Py_ssize_t i; + + dead = PyList_New(0); + if (dead == NULL) { + PyErr_Clear(); + return; + } + // The dict cannot be mutated while iterating it, so collect first. + while (PyDict_Next(PyXmlSec_LxmlShadowIdRegistry, &pos, &key, &entry)) { + if (Py_REFCNT(PyTuple_GET_ITEM(entry, 0)) == 1 && PyList_Append(dead, key) < 0) { + PyErr_Clear(); + break; + } + } + for (i = 0; i < PyList_GET_SIZE(dead); ++i) { + if (PyDict_DelItem(PyXmlSec_LxmlShadowIdRegistry, PyList_GET_ITEM(dead, i)) < 0) { + PyErr_Clear(); + } + } + Py_DECREF(dead); +} int PyXmlSec_LxmlShadowRecordId(PyXmlSec_LxmlElementPtr element, const char* name, const char* ns) { PyObject* key = NULL; - PyObject* cdoc = NULL; PyObject* created = NULL; PyObject* spec = NULL; PyObject* entry; @@ -513,36 +542,18 @@ int PyXmlSec_LxmlShadowRecordId(PyXmlSec_LxmlElementPtr element, const char* nam int result = -1; key = PyLong_FromVoidPtr((void*)element->_doc); - cdoc = PyLong_FromVoidPtr((void*)element->_doc->_c_doc); - if (key == NULL || cdoc == NULL) { + if (key == NULL) { goto DONE; } entry = PyDict_GetItem(PyXmlSec_LxmlShadowIdRegistry, key); // borrowed - if (entry != NULL) { - int eq = PyObject_RichCompareBool(PyTuple_GET_ITEM(entry, 0), cdoc, Py_EQ); - if (eq < 0) { - goto DONE; - } - if (!eq) { - entry = NULL; // the address was reused by another document - } - } if (entry == NULL) { - if (PyDict_Size(PyXmlSec_LxmlShadowIdRegistry) >= PYXMLSEC_SHADOW_ID_REGISTRY_CAP) { - PyObject* k; - PyObject* v; - Py_ssize_t pos = 0; - if (PyDict_Next(PyXmlSec_LxmlShadowIdRegistry, &pos, &k, &v) - && PyDict_DelItem(PyXmlSec_LxmlShadowIdRegistry, k) < 0) { - goto DONE; - } - } + PyXmlSec_LxmlShadowPruneIdRegistry(); specs = PyList_New(0); if (specs == NULL) { goto DONE; } - created = PyTuple_Pack(2, cdoc, specs); + created = PyTuple_Pack(2, (PyObject*)element->_doc, specs); Py_DECREF(specs); if (created == NULL || PyDict_SetItem(PyXmlSec_LxmlShadowIdRegistry, key, created) < 0) { goto DONE; @@ -563,7 +574,6 @@ int PyXmlSec_LxmlShadowRecordId(PyXmlSec_LxmlElementPtr element, const char* nam DONE: Py_XDECREF(key); - Py_XDECREF(cdoc); Py_XDECREF(created); Py_XDECREF(spec); return result; @@ -596,33 +606,21 @@ static void PyXmlSec_LxmlShadowApplyIdSpec(xmlDocPtr doc, xmlNodePtr n, const xm // Replays the specs recorded for the shadow's live document onto the copy. static int PyXmlSec_LxmlShadowReplayIds(PyXmlSec_LxmlShadow* shadow) { PyObject* key; - PyObject* cdoc; PyObject* entry; PyObject* specs; Py_ssize_t i, n; - int eq; key = PyLong_FromVoidPtr((void*)shadow->element->_doc); if (key == NULL) { return -1; } + // The entry, if any, belongs to this very document: the registry's own + // reference keeps the address from being reused by another one. entry = PyDict_GetItem(PyXmlSec_LxmlShadowIdRegistry, key); // borrowed Py_DECREF(key); if (entry == NULL) { return 0; } - cdoc = PyLong_FromVoidPtr((void*)shadow->element->_doc->_c_doc); - if (cdoc == NULL) { - return -1; - } - eq = PyObject_RichCompareBool(PyTuple_GET_ITEM(entry, 0), cdoc, Py_EQ); - Py_DECREF(cdoc); - if (eq < 0) { - return -1; - } - if (!eq) { - return 0; // stale entry from a dead document at the same address - } specs = PyTuple_GET_ITEM(entry, 1); n = PyList_GET_SIZE(specs); diff --git a/tests/test_ds.py b/tests/test_ds.py index f5a535a9..ccce3a6b 100644 --- a/tests/test_ds.py +++ b/tests/test_ds.py @@ -1,5 +1,7 @@ import unittest +from lxml import etree + import xmlsec from tests import base @@ -367,3 +369,28 @@ def test_set_enabled_key_data_bad_list(self): ctx.key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem) with self.assertRaisesRegex(TypeError, 'expected list of KeyData constants.'): ctx.set_enabled_key_data('foo') + + +class TestIdRegistryLifetime(base.TestMemoryLeaks): + """Guards the lifetime rule of the shadow-mode id registry (issue #356).""" + + # The document churn below is this test's point; the leak harness would repeat it ten times. + iterations = 0 + + # Far more documents than any bounded registry would have held at once. + OTHER_DOCUMENTS = 5000 + + def test_registration_survives_other_live_documents(self): + """Registering ids for other documents must never drop a live document's own registration.""" + root = self.load_xml('sign4-out.xml') + ctx = xmlsec.SignatureContext() + ctx.register_id(root, 'ID') + + # All kept alive, so none of these registrations may be traded for another. + others = [etree.fromstring(f'') for i in range(self.OTHER_DOCUMENTS)] + for other in others: + ctx.register_id(other, 'ID') + + sign = xmlsec.tree.find_node(root, consts.NodeSignature) + ctx.key = xmlsec.Key.from_file(self.path('rsapub.pem'), format=consts.KeyDataFormatPem) + ctx.verify(sign) # resolves #ID through the registration made before the churn diff --git a/tests/test_shadow_audit.py b/tests/test_shadow_audit.py index 0254b197..8329f2a6 100644 --- a/tests/test_shadow_audit.py +++ b/tests/test_shadow_audit.py @@ -32,8 +32,8 @@ } ) -# Functions allowed to dereference lxml's raw node/document pointers: the dual bodies above, -# the fast-path branches of the shadow helpers, and the ID registry (addresses used as keys). +# Functions allowed to dereference lxml's raw node/document pointers: the dual bodies above +# and the fast-path branches of the shadow helpers. RAW_ACCESS_ALLOWED = DUAL_BODY_FUNCTIONS | frozenset( { 'PyXmlSec_LxmlShadowBegin', @@ -41,8 +41,6 @@ 'PyXmlSec_LxmlShadowBeginNewDoc', 'PyXmlSec_LxmlShadowEnd', 'PyXmlSec_LxmlShadowEndFind', - 'PyXmlSec_LxmlShadowRecordId', - 'PyXmlSec_LxmlShadowReplayIds', } ) From 302c85144040ec0dcbf5642818c96541b460c8aa Mon Sep 17 00:00:00 2001 From: Amin Solhizadeh Date: Fri, 4 Sep 2026 15:06:52 +0200 Subject: [PATCH 11/22] Read code, not comments, in the shadow audit (#356) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The audit matched raw source lines, so a comment naming PyXmlSec_LxmlShadowBegin or ->_c_node — the comments that explain those very crossings — counted as the code itself. A binding could lose its Begin call and still pass the invariant because a comment beside it mentioned one, and a raw access inside an error string would have been reported as a real one. Comments and string/char literal bodies are now blanked (lines and columns preserved) before either rule is applied. Co-Authored-By: Claude Opus 5 --- tests/test_shadow_audit.py | 54 +++++++++++++++++++++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) diff --git a/tests/test_shadow_audit.py b/tests/test_shadow_audit.py index 8329f2a6..fcf1d617 100644 --- a/tests/test_shadow_audit.py +++ b/tests/test_shadow_audit.py @@ -11,6 +11,8 @@ 2. raw node access (``->_c_node`` / ``->_c_doc``) appears only inside the functions listed in ``RAW_ACCESS_ALLOWED``. +Comments and string literals are blanked first, so only real code counts for either rule. + Adding a function to either list is a deliberate design decision; see developer.md. """ @@ -52,11 +54,45 @@ IS_ACTIVE = 'PyXmlSec_LxmlShadowIsActive()' +def _blank_comments_and_literals(source: str) -> str: + """Blanks the body of every comment and string/char literal, keeping lines and columns. + + The checks below match raw text, so without this a comment explaining ``->_c_node`` or + naming a ``PyXmlSec_LxmlShadowBegin*`` helper would read as the code itself — a binding + could lose its guard and still pass because of the comment describing it. + """ + out: list[str] = [] + i, n = 0, len(source) + while i < n: + pair = source[i : i + 2] + if pair in ('//', '/*'): + end = source.find('\n', i) if pair == '//' else source.find('*/', i + 2) + 2 + if end < 2: # unterminated: the rest of the file is comment + end = n + out.append(''.join('\n' if c == '\n' else ' ' for c in source[i:end])) + i = end + elif source[i] in '"\'': + quote = source[i] + out.append(quote) + i += 1 + while i < n and source[i] != quote: + step = 2 if source[i] == '\\' and i + 1 < n else 1 + out.append(''.join('\n' if c == '\n' else ' ' for c in source[i : i + step])) + i += step + if i < n: + out.append(quote) + i += 1 + else: + out.append(source[i]) + i += 1 + return ''.join(out) + + def _functions(source: str) -> Iterator[tuple[str, list[str]]]: """Yields (name, [lines]) for every PyXmlSec_* function defined in the C source.""" name = None body: list[str] = [] - for line in source.splitlines(): + for line in _blank_comments_and_literals(source).splitlines(): match = FUNCTION_DEF.match(line) if match and not line.rstrip().endswith(';'): if name is not None: @@ -134,6 +170,22 @@ def test_checker_flags_a_raw_binding(self) -> None: self.assertIn('never calls a PyXmlSec_LxmlShadowBegin*', found[0]) self.assertIn('outside the allowed functions', found[1]) + def test_checker_reads_code_not_comments(self) -> None: + # the guard named in a comment does not guard anything, and a mention in a literal is not access + bad = ( + 'static PyObject* PyXmlSec_Bad(PyObject* self, PyObject* args) {\n' + ' PyXmlSec_LxmlElementPtr node = NULL;\n' + ' if (!PyArg_ParseTuple(args, "O&:bad", PyXmlSec_LxmlElementConverter, &node)) return NULL;\n' + ' // PyXmlSec_LxmlShadowBegin(&shadow, node, "cannot copy.") belongs here\n' + ' /* and node->_c_node would then be shadow.root */\n' + ' PyErr_SetString(PyXmlSec_Error, "node->_c_node is not for xmlsec.");\n' + ' return NULL;\n' + '}\n' + ) + found = violations(bad, 'bad.c') + self.assertEqual(1, len(found), found) + self.assertIn('never calls a PyXmlSec_LxmlShadowBegin*', found[0]) + def test_checker_flags_raw_access_before_the_switch(self) -> None: bad = ( 'static PyObject* PyXmlSec_TreeAddIds(PyObject* self, PyObject* args) {\n' From 97cb9934e37fcf4260e16c2925ba27ffb9805e69 Mon Sep 17 00:00:00 2001 From: Amin Solhizadeh Date: Fri, 4 Sep 2026 15:09:00 +0200 Subject: [PATCH 12/22] Drop the source-level shadow audit (#356) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The scan enforced the shadow invariants by pattern-matching src/*.c, which made it a parser of C that it never was: a comment or literal read as code until the previous commit, and any body it failed to delimit was attributed to its neighbour. The invariants stay — developer.md now states them as a review rule, with the grep that lists every raw crossing — but they are no longer asserted by the test suite. Co-Authored-By: Claude Opus 5 --- developer.md | 8 +- tests/test_shadow_audit.py | 201 ------------------------------------- 2 files changed, 4 insertions(+), 205 deletions(-) delete mode 100644 tests/test_shadow_audit.py diff --git a/developer.md b/developer.md index f0039c9d..eb82370a 100644 --- a/developer.md +++ b/developer.md @@ -94,13 +94,13 @@ Rules every call site must keep: in between (the `Py_*_ALLOW_THREADS` pair is fine — the call is pure C); - call exactly one End function after a successful Begin. -**Invariant, enforced by `tests/test_shadow_audit.py`:** every C function that -accepts an lxml element (`PyXmlSec_LxmlElementConverter`) either calls a +**Invariant, to check by review:** every C function that accepts an lxml +element (`PyXmlSec_LxmlElementConverter`) either calls a `PyXmlSec_LxmlShadowBegin*` helper or is one of the four dual-body functions (`register_id`, `add_ids`, `encrypt_xml`, `decrypt`), which must consult `IsActive()` before touching a raw node; and `->_c_node` / `->_c_doc` appear -only in those four and in the helpers' fast-path branches. The test scans -`src/*.c`, so a raw access anywhere else fails the suite on both paths. +only in those four and in the helpers' fast-path branches. A quick +`grep -n '\->_c_\(node\|doc\)' src/*.c` lists every crossing to check. ## How the reflection works diff --git a/tests/test_shadow_audit.py b/tests/test_shadow_audit.py deleted file mode 100644 index fcf1d617..00000000 --- a/tests/test_shadow_audit.py +++ /dev/null @@ -1,201 +0,0 @@ -"""Source-level guard for the shadow-copy invariants (issue #356). - -The extension may hand lxml's raw libxml2 nodes to xmlsec only on the fast path, when both -link the same libxml2. ``developer.md`` describes the design; this module scans ``src/*.c`` and -checks the two rules that keep every binding on the shadow path whenever it is on: - -1. every C function that accepts an lxml element (it uses ``PyXmlSec_LxmlElementConverter``) - either runs its xmlsec call through a ``PyXmlSec_LxmlShadowBegin*`` helper, or is one of - the dual-body functions in ``DUAL_BODY_FUNCTIONS``, which must consult - ``PyXmlSec_LxmlShadowIsActive()`` before touching a raw node; -2. raw node access (``->_c_node`` / ``->_c_doc``) appears only inside the functions listed in - ``RAW_ACCESS_ALLOWED``. - -Comments and string literals are blanked first, so only real code counts for either rule. - -Adding a function to either list is a deliberate design decision; see developer.md. -""" - -import glob -import os -import re -import unittest -from collections.abc import Iterator - -SRC_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'src') - -# Bindings with a raw body for the fast path and a shadow body behind IsActive(). -DUAL_BODY_FUNCTIONS = frozenset( - { - 'PyXmlSec_SignatureContextRegisterId', - 'PyXmlSec_TreeAddIds', - 'PyXmlSec_EncryptionContextEncryptXml', - 'PyXmlSec_EncryptionContextDecrypt', - } -) - -# Functions allowed to dereference lxml's raw node/document pointers: the dual bodies above -# and the fast-path branches of the shadow helpers. -RAW_ACCESS_ALLOWED = DUAL_BODY_FUNCTIONS | frozenset( - { - 'PyXmlSec_LxmlShadowBegin', - 'PyXmlSec_LxmlShadowBeginDoc', - 'PyXmlSec_LxmlShadowBeginNewDoc', - 'PyXmlSec_LxmlShadowEnd', - 'PyXmlSec_LxmlShadowEndFind', - } -) - -# A function definition at column 0: `[static ][*] PyXmlSec_(`; prototypes end in ';'. -FUNCTION_DEF = re.compile(r'^(?:static\s+)?[\w\s]+?\**\s*\**(PyXmlSec_\w+)\s*\(') -RAW_ACCESS = re.compile(r'->_c_(?:node|doc)\b') -BEGIN_CALL = re.compile(r'\bPyXmlSec_LxmlShadowBegin\w*\s*\(') -CONVERTER = 'PyXmlSec_LxmlElementConverter' -IS_ACTIVE = 'PyXmlSec_LxmlShadowIsActive()' - - -def _blank_comments_and_literals(source: str) -> str: - """Blanks the body of every comment and string/char literal, keeping lines and columns. - - The checks below match raw text, so without this a comment explaining ``->_c_node`` or - naming a ``PyXmlSec_LxmlShadowBegin*`` helper would read as the code itself — a binding - could lose its guard and still pass because of the comment describing it. - """ - out: list[str] = [] - i, n = 0, len(source) - while i < n: - pair = source[i : i + 2] - if pair in ('//', '/*'): - end = source.find('\n', i) if pair == '//' else source.find('*/', i + 2) + 2 - if end < 2: # unterminated: the rest of the file is comment - end = n - out.append(''.join('\n' if c == '\n' else ' ' for c in source[i:end])) - i = end - elif source[i] in '"\'': - quote = source[i] - out.append(quote) - i += 1 - while i < n and source[i] != quote: - step = 2 if source[i] == '\\' and i + 1 < n else 1 - out.append(''.join('\n' if c == '\n' else ' ' for c in source[i : i + step])) - i += step - if i < n: - out.append(quote) - i += 1 - else: - out.append(source[i]) - i += 1 - return ''.join(out) - - -def _functions(source: str) -> Iterator[tuple[str, list[str]]]: - """Yields (name, [lines]) for every PyXmlSec_* function defined in the C source.""" - name = None - body: list[str] = [] - for line in _blank_comments_and_literals(source).splitlines(): - match = FUNCTION_DEF.match(line) - if match and not line.rstrip().endswith(';'): - if name is not None: - yield name, body - name, body = match.group(1), [] - elif name is not None: - body.append(line) - if name is not None: - yield name, body - - -def violations(source: str, filename: str = '') -> list[str]: - """Returns a description of every rule violation in one C source file.""" - found: list[str] = [] - for name, body in _functions(source): - where = f'{filename}:{name}' - raw_lines = [i for i, line in enumerate(body) if RAW_ACCESS.search(line)] - active_lines = [i for i, line in enumerate(body) if IS_ACTIVE in line] - takes_element = any(CONVERTER in line for line in body) - begins = any(BEGIN_CALL.search(line) for line in body) - - if name in DUAL_BODY_FUNCTIONS: - if not active_lines: - found.append(f'{where}: dual-body function never consults {IS_ACTIVE}') - elif raw_lines and raw_lines[0] < active_lines[0]: - found.append(f'{where}: raw node access before {IS_ACTIVE}') - elif takes_element and not begins: - found.append(f'{where}: takes an lxml element but never calls a PyXmlSec_LxmlShadowBegin* helper') - - if raw_lines and name not in RAW_ACCESS_ALLOWED: - found.append(f'{where}: raw node access (->_c_node / ->_c_doc) outside the allowed functions') - return found - - -@unittest.skipUnless(os.path.isdir(SRC_DIR), 'C sources not available (installed package)') -class TestShadowAudit(unittest.TestCase): - def sources(self) -> Iterator[tuple[str, str]]: - files = sorted(glob.glob(os.path.join(SRC_DIR, '*.c'))) - self.assertTrue(files, f'no C sources under {SRC_DIR}') - for path in files: - with open(path, encoding='utf-8') as f: - yield os.path.basename(path), f.read() - - def test_scanner_sees_the_bindings(self) -> None: - # guards the scanner itself: a broken regex would make the sources look clean - names: set[str] = set() - raw_files: set[str] = set() - for filename, source in self.sources(): - for name, body in _functions(source): - if any(CONVERTER in line for line in body): - names.add(name) - if any(RAW_ACCESS.search(line) for line in body): - raw_files.add(filename) - self.assertGreaterEqual(len(names), 30) - self.assertTrue(names.issuperset({'PyXmlSec_TemplateAddReference', 'PyXmlSec_SignatureContextSign'})) - self.assertTrue(names.issuperset(DUAL_BODY_FUNCTIONS)) - self.assertEqual({'ds.c', 'enc.c', 'lxml.c', 'tree.c'}, raw_files) - - def test_every_binding_goes_through_the_shadow(self) -> None: - found: list[str] = [] - for filename, source in self.sources(): - found.extend(violations(source, filename)) - self.assertEqual([], found, '\n'.join(found)) - - def test_checker_flags_a_raw_binding(self) -> None: - bad = ( - 'static PyObject* PyXmlSec_Bad(PyObject* self, PyObject* args) {\n' - ' PyXmlSec_LxmlElementPtr node = NULL;\n' - ' if (!PyArg_ParseTuple(args, "O&:bad", PyXmlSec_LxmlElementConverter, &node)) return NULL;\n' - ' return (PyObject*)PyXmlSec_elementFactory(node->_doc, xmlSecFindChild(node->_c_node, NULL, NULL));\n' - '}\n' - ) - found = violations(bad, 'bad.c') - self.assertEqual(2, len(found), found) - self.assertIn('never calls a PyXmlSec_LxmlShadowBegin*', found[0]) - self.assertIn('outside the allowed functions', found[1]) - - def test_checker_reads_code_not_comments(self) -> None: - # the guard named in a comment does not guard anything, and a mention in a literal is not access - bad = ( - 'static PyObject* PyXmlSec_Bad(PyObject* self, PyObject* args) {\n' - ' PyXmlSec_LxmlElementPtr node = NULL;\n' - ' if (!PyArg_ParseTuple(args, "O&:bad", PyXmlSec_LxmlElementConverter, &node)) return NULL;\n' - ' // PyXmlSec_LxmlShadowBegin(&shadow, node, "cannot copy.") belongs here\n' - ' /* and node->_c_node would then be shadow.root */\n' - ' PyErr_SetString(PyXmlSec_Error, "node->_c_node is not for xmlsec.");\n' - ' return NULL;\n' - '}\n' - ) - found = violations(bad, 'bad.c') - self.assertEqual(1, len(found), found) - self.assertIn('never calls a PyXmlSec_LxmlShadowBegin*', found[0]) - - def test_checker_flags_raw_access_before_the_switch(self) -> None: - bad = ( - 'static PyObject* PyXmlSec_TreeAddIds(PyObject* self, PyObject* args) {\n' - ' PyXmlSec_LxmlElementPtr node = NULL;\n' - ' if (!PyArg_ParseTuple(args, "O&:add_ids", PyXmlSec_LxmlElementConverter, &node)) return NULL;\n' - ' xmlDocPtr doc = node->_doc->_c_doc;\n' - ' if (PyXmlSec_LxmlShadowIsActive()) Py_RETURN_NONE;\n' - ' return NULL;\n' - '}\n' - ) - found = violations(bad, 'bad.c') - self.assertEqual(1, len(found), found) - self.assertIn('raw node access before', found[0]) From 38174b9285b31b6681bacaf0fe680f9f84840ded Mon Sep 17 00:00:00 2001 From: Amin Solhizadeh Date: Fri, 4 Sep 2026 16:06:42 +0200 Subject: [PATCH 13/22] Replay id registrations at the node they were registered for (#356) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The shadow's id registry recorded only the attribute name and namespace, so the replay registered every matching attribute of the private copy. That is not a harmless superset: an element the caller never registered, sharing the id value and coming first, claims the value at xmlGetID and the intended element is skipped — a "#id" signature reference then resolves to content the caller never vouched for. The registry now keeps the registered elements themselves, and the replay applies each spec at the copy's counterpart of its element: that node alone for register_id, its subtree for add_ids (the scope xmlSecAddIDs walks). Liveness stays decidable without weak references — every element proxy holds a reference to its document, so an entry is dead when the document's reference count is exactly what the registry holds and nothing else holds its elements. Two other faithfulness gaps in the copy, from the same review: - the whole-document and subtree parses dropped the source document's base URI; they now carry docinfo.URL, so relative references resolve where they did before the copy. - the reflection recorded a text-slot sync only for a parent that gained a fresh node, so a call that *removed* a node and left nothing fresh behind (an EncryptedData decrypting to empty content) went unnoticed. Each pre-existing node is now tagged with its child count, and a parent whose tagged children changed is synced too. Co-Authored-By: Claude Opus 5 --- developer.md | 27 +-- src/ds.c | 3 +- src/lxml.c | 468 +++++++++++++++++++++++++++++++++++------------ src/lxml.h | 21 ++- src/tree.c | 6 +- tests/test_ds.py | 48 +++++ 6 files changed, 437 insertions(+), 136 deletions(-) diff --git a/developer.md b/developer.md index eb82370a..e6c9bb01 100644 --- a/developer.md +++ b/developer.md @@ -194,17 +194,22 @@ lxml's own dump of a tree it already parsed. hash with our libxml2 — exactly the cross-library access the shadow forbids. Under the shadow they record the id-attribute specs in a registry keyed by document identity (`RecordId`), and every `BeginDoc` replays them onto its -copy so that `#id` references resolve during sign/verify/decrypt. The replay -scans the whole copy for the recorded attribute names — a superset of the -single-node registration on the raw path, mirroring what `xmlSecAddIDs` does -from the root. lxml's classes refuse weak references, so an entry keeps a -strong reference to its document instead: the key (the document's address) -can then never go stale, and a document referenced by nothing but the -registry is provably unreachable, so its entry — and the document with it — -is dropped before the next registration. The registry therefore tracks the -documents still in use and never evicts a live one. The two bindings are the only places, -together with encrypt_xml/decrypt's replacement bodies, that branch on -`IsActive()`. +copy so that `#id` references resolve during sign/verify/decrypt. An entry +keeps the registered elements themselves, so a spec is replayed at exactly +the node it was registered for — that node alone for `register_id`, its +subtree for `add_ids`, the scope `xmlSecAddIDs` walks. Registering every +matching attribute of the copy instead would be unsafe, not merely generous: +an unrelated element sharing the id value would claim it first and a `#id` +reference could then resolve to content the caller never registered. lxml's +classes refuse weak references, so the entry keeps strong references (to the +document and to those elements) instead: the key (the document's address) can +then never go stale, and since every element proxy holds a reference to its +document, a document whose reference count is exactly what the registry holds +— and whose registered elements nothing else holds — is provably unreachable, +so its entry, and the document with it, is dropped before the next +registration. The registry therefore tracks the documents still in use and +never evicts a live one. The two bindings are the only places, together with +encrypt_xml/decrypt's replacement bodies, that branch on `IsActive()`. ## Converting a binding diff --git a/src/ds.c b/src/ds.c index b1288a45..9d5d2692 100644 --- a/src/ds.c +++ b/src/ds.c @@ -220,7 +220,8 @@ static PyObject* PyXmlSec_SignatureContextRegisterId(PyObject* self, PyObject* a PyErr_SetString(PyXmlSec_Error, "missing attribute."); goto ON_FAIL; } - if (PyXmlSec_LxmlShadowRecordId(node, id_attr, id_ns) < 0) { + // Scope 0: this node alone, exactly what the fast path registers. + if (PyXmlSec_LxmlShadowRecordId(node, id_attr, id_ns, 0) < 0) { goto ON_FAIL; } PYXMLSEC_DEBUGF("%p: register id - ok", self); diff --git a/src/lxml.c b/src/lxml.c index 71f54cbc..3484fed3 100644 --- a/src/lxml.c +++ b/src/lxml.c @@ -18,6 +18,8 @@ #include #include +#include + #define XMLSEC_EXTRACT_VERSION(x, y) ((x / (y)) % 100) #define XMLSEC_EXTRACT_MAJOR(x) XMLSEC_EXTRACT_VERSION(x, 100 * 100) @@ -239,8 +241,9 @@ static PyObject* PyXmlSec_LxmlElementFromBytes(PyObject* data) { #define PYXMLSEC_SHADOW_PARSE_OPTIONS (XML_PARSE_NONET | XML_PARSE_HUGE) // Parses `bytes` into a private document with a root element, or returns -// NULL with an exception set (`error` for parse failures). -static xmlDocPtr PyXmlSec_LxmlShadowParse(PyObject* bytes, const char* error) { +// NULL with an exception set (`error` for parse failures). `url` is the base +// URI the copy gets (see PyXmlSec_LxmlDocumentUrl); NULL when there is none. +static xmlDocPtr PyXmlSec_LxmlShadowParse(PyObject* bytes, const char* url, const char* error) { char* data = NULL; Py_ssize_t size = 0; xmlDocPtr doc; @@ -248,7 +251,7 @@ static xmlDocPtr PyXmlSec_LxmlShadowParse(PyObject* bytes, const char* error) { if (PyBytes_AsStringAndSize(bytes, &data, &size) < 0) { return NULL; } - doc = xmlReadMemory(data, (int)size, NULL, NULL, PYXMLSEC_SHADOW_PARSE_OPTIONS); + doc = xmlReadMemory(data, (int)size, url, NULL, PYXMLSEC_SHADOW_PARSE_OPTIONS); if (doc == NULL || xmlDocGetRootElement(doc) == NULL) { if (doc != NULL) { xmlFreeDoc(doc); @@ -259,19 +262,109 @@ static xmlDocPtr PyXmlSec_LxmlShadowParse(PyObject* bytes, const char* error) { return doc; } +// The URL of the document `tree` belongs to (lxml's `docinfo.URL`), so that +// the private copy carries the same base URI as the document it copies — +// what libxml2 resolves relative references against (an XSLT transform's +// document()/import, an xml:base, a DTD or entity system id). A copy parsed +// without one would resolve them against the process' working directory +// instead. Stores a new reference to the owning string in *holder (NULL when +// the document has no URL) and its UTF-8 in *url; returns 0, or -1 with an +// exception set. +static int PyXmlSec_LxmlDocumentUrl(PyObject* tree, PyObject** holder, const char** url) { + PyObject* info; + PyObject* value; + + *holder = NULL; + *url = NULL; + info = PyObject_GetAttrString(tree, "docinfo"); + if (info == NULL) { + return -1; + } + value = PyObject_GetAttrString(info, "URL"); + Py_DECREF(info); + if (value == NULL) { + return -1; + } + if (!PyUnicode_Check(value)) { // None for a document parsed from memory + Py_DECREF(value); + return 0; + } + *url = PyUnicode_AsUTF8(value); + if (*url == NULL) { + Py_DECREF(value); + return -1; + } + *holder = value; + return 0; +} + +// The same for the document `element` hangs in. +static int PyXmlSec_LxmlElementDocumentUrl(PyObject* element, PyObject** holder, const char** url) { + PyObject* tree = PyObject_CallMethod(element, "getroottree", NULL); + int rv; + + *holder = NULL; + *url = NULL; + if (tree == NULL) { + return -1; + } + rv = PyXmlSec_LxmlDocumentUrl(tree, holder, url); + Py_DECREF(tree); + return rv; +} + // Nodes that exist before the xmlsec call are tagged through the libxml2 -// _private field (never serialized, never touched by the parser or xmlsec); -// whatever is untagged after the call is new. -static const char PyXmlSec_LxmlShadowMarker = 0; -#define PYXMLSEC_SHADOW_MARKED(n) ((n)->_private == (void*)&PyXmlSec_LxmlShadowMarker) +// _private field (never serialized, never touched by the parser or xmlsec): +// it is pointed at the shadow's own tag array, so whatever is untagged after +// the call is what the call created. Each tag also records the child count +// the node had, which is what makes a *removal* visible — a call can delete a +// node and leave nothing fresh behind (an that decrypts to +// empty content), and only the changed count shows it happened. A tag +// pointer is compared against the array bounds, never dereferenced blindly: +// a foreign _private (lxml's own proxy, when both libraries are the same +// libxml2) simply falls outside. +#define PYXMLSEC_SHADOW_TAGGED(shadow, n) \ + ((uintptr_t)(n)->_private >= (uintptr_t)(shadow)->tags \ + && (uintptr_t)(n)->_private < (uintptr_t)((shadow)->tags + (shadow)->ntags)) +#define PYXMLSEC_SHADOW_TAG(shadow, n) \ + (PYXMLSEC_SHADOW_TAGGED(shadow, n) ? (PyXmlSec_LxmlShadowTag*)(n)->_private : NULL) + +static int PyXmlSec_LxmlShadowCountNodes(xmlNodePtr node) { + int count = 0; + for (; node != NULL; node = node->next) { + count += 1 + PyXmlSec_LxmlShadowCountNodes(node->children); + } + return count; +} -static void PyXmlSec_LxmlShadowMark(xmlNodePtr node) { +// Hands out `shadow->tags` in document order; returns the next free slot. +static int PyXmlSec_LxmlShadowTagNodes(PyXmlSec_LxmlShadow* shadow, xmlNodePtr node, int next) { for (; node != NULL; node = node->next) { - node->_private = (void*)&PyXmlSec_LxmlShadowMarker; - if (node->children != NULL) { - PyXmlSec_LxmlShadowMark(node->children); + PyXmlSec_LxmlShadowTag* tag = &shadow->tags[next++]; + xmlNodePtr child; + tag->children = 0; + for (child = node->children; child != NULL; child = child->next) { + ++tag->children; } + node->_private = (void*)tag; + next = PyXmlSec_LxmlShadowTagNodes(shadow, node->children, next); + } + return next; +} + +// Tags every node of the freshly parsed copy. Returns 0, or -1 with an +// exception set. +static int PyXmlSec_LxmlShadowMark(PyXmlSec_LxmlShadow* shadow) { + int count = PyXmlSec_LxmlShadowCountNodes(shadow->doc->children); + + shadow->tags = (PyXmlSec_LxmlShadowTag*)PyMem_Malloc((count > 0 ? count : 1) * sizeof(*shadow->tags)); + if (shadow->tags == NULL) { + PyErr_NoMemory(); + return -1; } + shadow->ntags = count; + PyXmlSec_LxmlShadowTagNodes(shadow, shadow->doc->children, 0); + return 0; } // Paths span whole user documents (BeginDoc). 256 is libxml2's default @@ -351,6 +444,59 @@ static xmlNodePtr PyXmlSec_LxmlShadowWalkNode(xmlNodePtr start, const int* path, return n; } +// Records the child indices leading from the top of `node`'s live tree down +// to `node` into `path` (ordered top-first), through lxml's own API — the +// shadow path never walks lxml's raw nodes. Returns the depth and stores the +// top element in *top (new reference), or -1 with an exception set. +static int PyXmlSec_LxmlLivePathTo(PyObject* node, int* path, PyObject** top) { + PyObject* cur = node; + int depth = 0; + int i; + + *top = NULL; + Py_INCREF(cur); + for (;;) { + PyObject* parent = PyObject_CallMethod(cur, "getparent", NULL); + PyObject* index; + long idx; + if (parent == NULL) { + goto ON_FAIL; + } + if (parent == Py_None) { + Py_DECREF(parent); + break; + } + index = PyObject_CallMethod(parent, "index", "O", cur); + if (index == NULL) { + Py_DECREF(parent); + goto ON_FAIL; + } + idx = PyLong_AsLong(index); + Py_DECREF(index); + if (idx < 0 || depth >= PYXMLSEC_SHADOW_MAX_DEPTH) { + Py_DECREF(parent); + if (!PyErr_Occurred()) { + PyErr_SetString(PyXmlSec_InternalError, "the document is nested too deeply."); + } + goto ON_FAIL; + } + path[depth++] = (int)idx; + Py_DECREF(cur); + cur = parent; + } + for (i = 0; i < depth / 2; ++i) { // collected bottom-up; reverse + int t = path[i]; + path[i] = path[depth - 1 - i]; + path[depth - 1 - i] = t; + } + *top = cur; + return depth; + +ON_FAIL: + Py_DECREF(cur); + return -1; +} + // Serializes the copy in its current state and re-parses it with lxml, // returning the root element of that detached parse (new reference). The // *whole* copy is dumped, not just the changed nodes: the surrounding markup @@ -426,11 +572,15 @@ static int PyXmlSec_LxmlShadowSyncAttributes(xmlNodePtr src, PyObject* dst) { int PyXmlSec_LxmlShadowBegin(PyXmlSec_LxmlShadow* shadow, PyXmlSec_LxmlElementPtr element) { PyObject* bytes; + PyObject* url_holder = NULL; + const char* url = NULL; shadow->element = element; shadow->owned = NULL; shadow->doc = NULL; shadow->root = NULL; + shadow->tags = NULL; + shadow->ntags = 0; // Fast path: lxml links the same libxml2 (the import guard passed), so // xmlsec can mutate lxml's nodes directly and no copy is needed; End sees @@ -444,14 +594,18 @@ int PyXmlSec_LxmlShadowBegin(PyXmlSec_LxmlShadow* shadow, PyXmlSec_LxmlElementPt if (bytes == NULL) { return -1; } - shadow->doc = PyXmlSec_LxmlShadowParse(bytes, "cannot make a private copy of the element."); + if (PyXmlSec_LxmlElementDocumentUrl((PyObject*)element, &url_holder, &url) < 0) { + Py_DECREF(bytes); + return -1; + } + shadow->doc = PyXmlSec_LxmlShadowParse(bytes, url, "cannot make a private copy of the element."); Py_DECREF(bytes); + Py_XDECREF(url_holder); if (shadow->doc == NULL) { return -1; } shadow->root = xmlDocGetRootElement(shadow->doc); - PyXmlSec_LxmlShadowMark(shadow->doc->children); - return 0; + return PyXmlSec_LxmlShadowMark(shadow); } xmlDocPtr PyXmlSec_LxmlShadowBeginNewDoc(PyXmlSec_LxmlShadow* shadow, PyXmlSec_LxmlElementPtr element) { @@ -459,6 +613,8 @@ xmlDocPtr PyXmlSec_LxmlShadowBeginNewDoc(PyXmlSec_LxmlShadow* shadow, PyXmlSec_L shadow->owned = NULL; shadow->doc = NULL; shadow->root = NULL; + shadow->tags = NULL; + shadow->ntags = 0; // Fast path: allocate the detached subtree straight in the element's own // document, as the raw code always did. @@ -479,6 +635,9 @@ void PyXmlSec_LxmlShadowDiscard(PyXmlSec_LxmlShadow* shadow) { shadow->doc = NULL; shadow->root = NULL; } + PyMem_Free(shadow->tags); + shadow->tags = NULL; + shadow->ntags = 0; Py_CLEAR(shadow->owned); } @@ -491,17 +650,50 @@ void PyXmlSec_LxmlShadowDiscard(PyXmlSec_LxmlShadow* shadow) { // and every whole-document Begin replays them onto the private copy so that // #id references resolve during sign/verify/decrypt. // -// An entry is keyed by the _Document object's address and holds a strong -// reference to that document, so the key can never go stale: the address -// cannot be reused while the entry keeps the object alive. That reference -// also makes liveness decidable without weak references (lxml's classes -// refuse those): when the registry holds the *only* reference to a document, -// nothing can hand that document to a binding again, so the entry is dead and -// is dropped. Pruning runs before every new registration, which bounds the -// registry — and the documents it pins — by the documents still in use. No -// live registration is ever evicted. +// An entry is the tuple (document, nodes, specs), keyed by the _Document +// object's address: `nodes` holds the registered elements themselves and +// `specs` the (attribute name, namespace, node index, subtree) records. The +// entry keeps a strong reference to the document and to every registered +// element, so the key can never go stale — the address cannot be reused while +// the entry keeps the object alive. +// +// Keeping the elements is also what makes the replay faithful: a spec is +// applied to the very node it was registered for (and, for add_ids, that +// node's subtree), never to every same-named attribute in the document. +// Registering the whole document would let an unrelated element sharing the +// id value claim it first, so a `#id` reference — the URI a signature covers +// — could resolve to content the caller never registered. +// +// Those references also make liveness decidable without weak references +// (lxml's classes refuse those): every element proxy holds a reference to its +// document, so when the document's reference count is exactly what the +// registry itself holds (the entry plus one per registered element) and no +// registered element is referenced anywhere else, nothing can hand that +// document to a binding again — the entry is dead and is dropped. Pruning +// runs before every new registration, which bounds the registry — and the +// documents it pins — by the documents still in use. No live registration is +// ever evicted. // ---------------------------------------------------------------------------- +enum { PYXMLSEC_ID_ENTRY_DOC, PYXMLSEC_ID_ENTRY_NODES, PYXMLSEC_ID_ENTRY_SPECS }; + +// Non-zero when nothing outside the registry can reach the entry's document. +static int PyXmlSec_LxmlShadowIdEntryIsDead(PyObject* entry) { + PyObject* nodes = PyTuple_GET_ITEM(entry, PYXMLSEC_ID_ENTRY_NODES); + Py_ssize_t n = PyList_GET_SIZE(nodes); + Py_ssize_t i; + + if (Py_REFCNT(PyTuple_GET_ITEM(entry, PYXMLSEC_ID_ENTRY_DOC)) != n + 1) { + return 0; + } + for (i = 0; i < n; ++i) { + if (Py_REFCNT(PyList_GET_ITEM(nodes, i)) != 1) { + return 0; + } + } + return 1; +} + // Drops the entries of documents nobody but the registry still references. // Best effort: on an allocation failure the entries simply survive until the // next registration prunes them. @@ -519,7 +711,7 @@ static void PyXmlSec_LxmlShadowPruneIdRegistry(void) { } // The dict cannot be mutated while iterating it, so collect first. while (PyDict_Next(PyXmlSec_LxmlShadowIdRegistry, &pos, &key, &entry)) { - if (Py_REFCNT(PyTuple_GET_ITEM(entry, 0)) == 1 && PyList_Append(dead, key) < 0) { + if (PyXmlSec_LxmlShadowIdEntryIsDead(entry) && PyList_Append(dead, key) < 0) { PyErr_Clear(); break; } @@ -532,12 +724,15 @@ static void PyXmlSec_LxmlShadowPruneIdRegistry(void) { Py_DECREF(dead); } -int PyXmlSec_LxmlShadowRecordId(PyXmlSec_LxmlElementPtr element, const char* name, const char* ns) { +int PyXmlSec_LxmlShadowRecordId(PyXmlSec_LxmlElementPtr element, const char* name, const char* ns, int subtree) { PyObject* key = NULL; PyObject* created = NULL; PyObject* spec = NULL; PyObject* entry; + PyObject* nodes; PyObject* specs; + Py_ssize_t idx; + Py_ssize_t i; int contains; int result = -1; @@ -549,23 +744,41 @@ int PyXmlSec_LxmlShadowRecordId(PyXmlSec_LxmlElementPtr element, const char* nam entry = PyDict_GetItem(PyXmlSec_LxmlShadowIdRegistry, key); // borrowed if (entry == NULL) { PyXmlSec_LxmlShadowPruneIdRegistry(); + nodes = PyList_New(0); specs = PyList_New(0); - if (specs == NULL) { - goto DONE; + if (nodes != NULL && specs != NULL) { + created = PyTuple_Pack(3, (PyObject*)element->_doc, nodes, specs); } - created = PyTuple_Pack(2, (PyObject*)element->_doc, specs); - Py_DECREF(specs); + Py_XDECREF(nodes); + Py_XDECREF(specs); if (created == NULL || PyDict_SetItem(PyXmlSec_LxmlShadowIdRegistry, key, created) < 0) { goto DONE; } entry = created; } - spec = Py_BuildValue("(sz)", name, ns); + // One reference per registered element, so that the liveness test can + // account for exactly the references the registry itself holds. + nodes = PyTuple_GET_ITEM(entry, PYXMLSEC_ID_ENTRY_NODES); + specs = PyTuple_GET_ITEM(entry, PYXMLSEC_ID_ENTRY_SPECS); + idx = -1; + for (i = 0; i < PyList_GET_SIZE(nodes); ++i) { + if (PyList_GET_ITEM(nodes, i) == (PyObject*)element) { + idx = i; + break; + } + } + if (idx < 0) { + if (PyList_Append(nodes, (PyObject*)element) < 0) { + goto DONE; + } + idx = PyList_GET_SIZE(nodes) - 1; + } + + spec = Py_BuildValue("(szni)", name, ns, idx, subtree); if (spec == NULL) { goto DONE; } - specs = PyTuple_GET_ITEM(entry, 1); contains = PySequence_Contains(specs, spec); if (contains < 0 || (!contains && PyList_Append(specs, spec) < 0)) { goto DONE; @@ -579,35 +792,56 @@ int PyXmlSec_LxmlShadowRecordId(PyXmlSec_LxmlElementPtr element, const char* nam return result; } -// Registers every attribute named `name` (under `ns` when given) in the copy -// as an XML ID — a superset of the fast path's registrations (single node for -// register_id, subtree for add_ids), which is the safe direction: it mirrors -// what xmlSecAddIDs does from the root. -static void PyXmlSec_LxmlShadowApplyIdSpec(xmlDocPtr doc, xmlNodePtr n, const xmlChar* name, const xmlChar* ns) { - for (; n != NULL; n = n->next) { - if (n->type == XML_ELEMENT_NODE) { - xmlAttrPtr attr = ns != NULL ? xmlHasNsProp(n, name, ns) : xmlHasProp(n, name); - if (attr != NULL && attr->children != NULL) { - xmlChar* value = xmlNodeListGetString(doc, attr->children, 1); - if (value != NULL) { - if (xmlGetID(doc, value) == NULL) { - xmlAddID(NULL, doc, value, attr); - } - xmlFree(value); - } - } - if (n->children != NULL) { - PyXmlSec_LxmlShadowApplyIdSpec(doc, n->children, name, ns); - } +// Registers `node`'s `name` attribute (in `ns` when given) as an XML ID, the +// way xmlSecAddIDs does: the first registration of a value wins. +static void PyXmlSec_LxmlShadowAddId(xmlDocPtr doc, xmlNodePtr node, const xmlChar* name, const xmlChar* ns) { + xmlAttrPtr attr = ns != NULL ? xmlHasNsProp(node, name, ns) : xmlHasProp(node, name); + xmlChar* value; + + if (attr == NULL || attr->children == NULL) { + return; + } + value = xmlNodeListGetString(doc, attr->children, 1); + if (value == NULL) { + return; + } + if (xmlGetID(doc, value) == NULL) { + xmlAddID(NULL, doc, value, attr); + } + xmlFree(value); +} + +// `node`, its siblings and their descendants. +static void PyXmlSec_LxmlShadowAddIdsBelow(xmlDocPtr doc, xmlNodePtr node, const xmlChar* name, const xmlChar* ns) { + for (; node != NULL; node = node->next) { + if (node->type == XML_ELEMENT_NODE) { + PyXmlSec_LxmlShadowAddId(doc, node, name, ns); + PyXmlSec_LxmlShadowAddIdsBelow(doc, node->children, name, ns); } } } -// Replays the specs recorded for the shadow's live document onto the copy. +// Applies one recorded spec to `node`, the copy's counterpart of the element +// it was registered for: that node alone (register_id), or the subtree rooted +// at it (add_ids, which is the scope xmlSecAddIDs walks). +static void PyXmlSec_LxmlShadowApplyIdSpec(xmlDocPtr doc, xmlNodePtr node, const xmlChar* name, const xmlChar* ns, int subtree) { + if (node == NULL || node->type != XML_ELEMENT_NODE) { + return; + } + PyXmlSec_LxmlShadowAddId(doc, node, name, ns); + if (subtree) { + PyXmlSec_LxmlShadowAddIdsBelow(doc, node->children, name, ns); + } +} + +// Replays the specs recorded for the shadow's live document onto the copy, +// each at the copy's counterpart of the element it was registered for. static int PyXmlSec_LxmlShadowReplayIds(PyXmlSec_LxmlShadow* shadow) { PyObject* key; PyObject* entry; + PyObject* nodes; PyObject* specs; + int path[PYXMLSEC_SHADOW_MAX_DEPTH]; Py_ssize_t i, n; key = PyLong_FromVoidPtr((void*)shadow->element->_doc); @@ -622,16 +856,33 @@ static int PyXmlSec_LxmlShadowReplayIds(PyXmlSec_LxmlShadow* shadow) { return 0; } - specs = PyTuple_GET_ITEM(entry, 1); + nodes = PyTuple_GET_ITEM(entry, PYXMLSEC_ID_ENTRY_NODES); + specs = PyTuple_GET_ITEM(entry, PYXMLSEC_ID_ENTRY_SPECS); n = PyList_GET_SIZE(specs); for (i = 0; i < n; ++i) { - PyObject* spec = PyList_GET_ITEM(specs, i); + PyObject* spec = PyList_GET_ITEM(specs, i); // (name, ns, node index, subtree) + PyObject* node = PyList_GET_ITEM(nodes, PyLong_AsSsize_t(PyTuple_GET_ITEM(spec, 2))); + PyObject* top = NULL; const char* name = PyUnicode_AsUTF8(PyTuple_GET_ITEM(spec, 0)); const char* ns = PyTuple_GET_ITEM(spec, 1) == Py_None ? NULL : PyUnicode_AsUTF8(PyTuple_GET_ITEM(spec, 1)); + int subtree = PyObject_IsTrue(PyTuple_GET_ITEM(spec, 3)); + int depth; + if (name == NULL || (ns == NULL && PyErr_Occurred())) { return -1; } - PyXmlSec_LxmlShadowApplyIdSpec(shadow->doc, shadow->doc->children, (const xmlChar*)name, (const xmlChar*)ns); + depth = PyXmlSec_LxmlLivePathTo(node, path, &top); + if (depth < 0) { + return -1; + } + // lxml hands out one proxy per node, so identity settles whether the + // element still hangs under the root being copied; a registration for + // an element that has since left this tree applies to nothing here. + if (top == (PyObject*)shadow->element) { + PyXmlSec_LxmlShadowApplyIdSpec(shadow->doc, PyXmlSec_LxmlShadowWalkNode(shadow->root, path, depth), + (const xmlChar*)name, (const xmlChar*)ns, subtree); + } + Py_DECREF(top); } return 0; } @@ -640,14 +891,17 @@ int PyXmlSec_LxmlShadowBeginDoc(PyXmlSec_LxmlShadow* shadow, PyXmlSec_LxmlElemen PyObject* cur = NULL; PyObject* tree = NULL; PyObject* bytes = NULL; + PyObject* url_holder = NULL; + const char* url = NULL; int path[PYXMLSEC_SHADOW_MAX_DEPTH]; - int depth = 0; - int i; + int depth; shadow->element = element; shadow->owned = NULL; shadow->doc = NULL; shadow->root = NULL; + shadow->tags = NULL; + shadow->ntags = 0; *target = NULL; if (!PyXmlSec_LxmlShadowActive) { @@ -656,45 +910,11 @@ int PyXmlSec_LxmlShadowBeginDoc(PyXmlSec_LxmlShadow* shadow, PyXmlSec_LxmlElemen return 0; } - // Record the element's position in its tree through lxml's own API (the - // shadow path never walks lxml's raw nodes), ascending to the top; `cur` - // ends as the live root element and `path` (reversed below) leads back - // down to `element`. - cur = (PyObject*)element; - Py_INCREF(cur); - for (;;) { - PyObject* parent = PyObject_CallMethod(cur, "getparent", NULL); - PyObject* index; - long idx; - if (parent == NULL) { - goto ON_FAIL; - } - if (parent == Py_None) { - Py_DECREF(parent); - break; - } - index = PyObject_CallMethod(parent, "index", "O", cur); - if (index == NULL) { - Py_DECREF(parent); - goto ON_FAIL; - } - idx = PyLong_AsLong(index); - Py_DECREF(index); - if (idx < 0 || depth >= PYXMLSEC_SHADOW_MAX_DEPTH) { - Py_DECREF(parent); - if (!PyErr_Occurred()) { - PyErr_SetString(PyXmlSec_InternalError, "the document is nested too deeply."); - } - goto ON_FAIL; - } - path[depth++] = (int)idx; - Py_DECREF(cur); - cur = parent; - } - for (i = 0; i < depth / 2; ++i) { // collected bottom-up; reverse - int t = path[i]; - path[i] = path[depth - 1 - i]; - path[depth - 1 - i] = t; + // `cur` ends as the live root element and `path` leads back down to + // `element`. + depth = PyXmlSec_LxmlLivePathTo((PyObject*)element, path, &cur); + if (depth < 0) { + goto ON_FAIL; } // Serialize the whole tree, not just the root element, so comments/PIs @@ -705,17 +925,20 @@ int PyXmlSec_LxmlShadowBeginDoc(PyXmlSec_LxmlShadow* shadow, PyXmlSec_LxmlElemen goto ON_FAIL; } bytes = PyObject_CallFunctionObjArgs(PyXmlSec_LxmlEtreeToString, tree, NULL); - Py_CLEAR(tree); - if (bytes == NULL) { + if (bytes == NULL || PyXmlSec_LxmlDocumentUrl(tree, &url_holder, &url) < 0) { goto ON_FAIL; } - shadow->doc = PyXmlSec_LxmlShadowParse(bytes, "cannot make a private copy of the document."); + Py_CLEAR(tree); + shadow->doc = PyXmlSec_LxmlShadowParse(bytes, url, "cannot make a private copy of the document."); Py_CLEAR(bytes); + Py_CLEAR(url_holder); if (shadow->doc == NULL) { goto ON_FAIL; } shadow->root = xmlDocGetRootElement(shadow->doc); - PyXmlSec_LxmlShadowMark(shadow->doc->children); + if (PyXmlSec_LxmlShadowMark(shadow) < 0) { + goto ON_FAIL; + } *target = PyXmlSec_LxmlShadowWalkNode(shadow->root, path, depth); if (*target == NULL) { @@ -740,6 +963,7 @@ int PyXmlSec_LxmlShadowBeginDoc(PyXmlSec_LxmlShadow* shadow, PyXmlSec_LxmlElemen Py_XDECREF(cur); Py_XDECREF(tree); Py_XDECREF(bytes); + Py_XDECREF(url_holder); PyXmlSec_LxmlShadowDiscard(shadow); *target = NULL; return -1; @@ -755,13 +979,15 @@ int PyXmlSec_LxmlShadowBeginDoc(PyXmlSec_LxmlShadow* shadow, PyXmlSec_LxmlElemen // graft — a fresh node (element, comment, PI) to insert at its child // index. Fresh subtrees are grafted wholesale; the scan never // descends into them. -// sync — a marked parent that gained any fresh node (element or text) -// gets its text slots (its .text and each child's .tail) copied -// over from the re-parsed copy. That covers everything xmlsec does -// to text: the "\n" formatting around a new node, values filled -// into empty elements (DigestValue), and content it removed -// (encrypt Type=Content) — a removal leaves no fresh node behind, -// so only a wholesale sync can see it. +// sync — a tagged parent whose children changed — it gained a fresh node +// (element or text), or lost a tagged one — gets its text slots +// (its .text and each child's .tail) copied over from the re-parsed +// copy. That covers everything xmlsec does to text: the "\n" +// formatting around a new node, values filled into empty elements +// (DigestValue), and content it removed (encrypt Type=Content) — +// a removal leaves no fresh node behind, so only the tagged child +// count shows that it happened, and only a wholesale sync can carry +// it across. // // The reflection is two-phase: first every site's payload is fetched from // the re-parsed copy while it is still in its final, untouched state, then @@ -785,7 +1011,8 @@ typedef struct { PyXmlSec_LxmlShadowSite* items; int count; int capacity; - xmlNodePtr top; // the copy's root element — origin of all paths + xmlNodePtr top; // the copy's root element — origin of all paths + PyXmlSec_LxmlShadow* shadow; // borrowed; owns the tags the walk reads } PyXmlSec_LxmlShadowSiteList; static int PyXmlSec_LxmlShadowSiteAppend(PyXmlSec_LxmlShadowSiteList* list, xmlNodePtr node, int kind) { @@ -815,16 +1042,20 @@ static int PyXmlSec_LxmlShadowSiteAppend(PyXmlSec_LxmlShadowSiteList* list, xmlN return 0; } -// Walks the marked structure of the copy in document order, recording a -// graft for every fresh node and, after them, a sync for their parent. +// Walks the tagged structure of the copy in document order, recording a +// graft for every fresh node and, after them, a sync for their parent — and +// for a parent that no longer holds the children it was tagged with, whose +// text slots are the only trace the removal left. static int PyXmlSec_LxmlShadowCollectSites(PyXmlSec_LxmlShadowSiteList* list, xmlNodePtr parent) { + PyXmlSec_LxmlShadowTag* tag = PYXMLSEC_SHADOW_TAG(list->shadow, parent); xmlNodePtr n; int fresh = 0; + int tagged = 0; for (n = parent->children; n != NULL; n = n->next) { - if (PYXMLSEC_SHADOW_MARKED(n)) { - if (n->type == XML_ELEMENT_NODE && n->children != NULL - && PyXmlSec_LxmlShadowCollectSites(list, n) < 0) { + if (PYXMLSEC_SHADOW_TAGGED(list->shadow, n)) { + ++tagged; + if (n->type == XML_ELEMENT_NODE && PyXmlSec_LxmlShadowCollectSites(list, n) < 0) { return -1; } continue; @@ -834,7 +1065,8 @@ static int PyXmlSec_LxmlShadowCollectSites(PyXmlSec_LxmlShadowSiteList* list, xm return -1; } } - if (fresh && PyXmlSec_LxmlShadowSiteAppend(list, parent, PYXMLSEC_SHADOW_SITE_SYNC) < 0) { + if ((fresh || (tag != NULL && tag->children != tagged)) + && PyXmlSec_LxmlShadowSiteAppend(list, parent, PYXMLSEC_SHADOW_SITE_SYNC) < 0) { return -1; } return 0; @@ -1047,7 +1279,7 @@ static int PyXmlSec_LxmlShadowMorphRoot(PyObject* live, PyObject* fresh) { // Applies every change in the copy to the live tree. Does not release the // copy. Returns 0, or -1 with an exception set. static int PyXmlSec_LxmlShadowReflectSites(PyXmlSec_LxmlShadow* shadow) { - PyXmlSec_LxmlShadowSiteList list = {NULL, 0, 0, NULL}; + PyXmlSec_LxmlShadowSiteList list = {NULL, 0, 0, NULL, shadow}; PyObject* copy_root = NULL; int i; int rv = -1; @@ -1058,7 +1290,7 @@ static int PyXmlSec_LxmlShadowReflectSites(PyXmlSec_LxmlShadow* shadow) { PyErr_SetString(PyXmlSec_InternalError, "unexpected mutation site."); goto DONE; } - if (!PYXMLSEC_SHADOW_MARKED(list.top)) { + if (!PYXMLSEC_SHADOW_TAGGED(shadow, list.top)) { // A fresh root: the call replaced the root itself, so there are no // sites to graft — the live root is morphed into it wholesale. lxml // holds one element (plus comments and PIs) at document level, so a @@ -1289,7 +1521,7 @@ PyObject* PyXmlSec_LxmlShadowEnd(PyXmlSec_LxmlShadow* shadow, xmlNodePtr res, co if (PyXmlSec_LxmlShadowReflectSites(shadow) < 0) { goto DONE; } - if (PYXMLSEC_SHADOW_MARKED(res)) { + if (PYXMLSEC_SHADOW_TAGGED(shadow, res)) { result = PyXmlSec_LxmlShadowEndFound(shadow, res, path, depth); } else { result = PyXmlSec_LxmlShadowWalk((PyObject*)shadow->element, path, depth); @@ -1345,7 +1577,7 @@ xmlNodePtr PyXmlSec_LxmlShadowImportElement(PyXmlSec_LxmlShadow* shadow, PyXmlSe if (bytes == NULL) { return NULL; } - tdoc = PyXmlSec_LxmlShadowParse(bytes, "cannot make a private copy of the element."); + tdoc = PyXmlSec_LxmlShadowParse(bytes, NULL, "cannot make a private copy of the element."); Py_DECREF(bytes); if (tdoc == NULL) { return NULL; diff --git a/src/lxml.h b/src/lxml.h index 5da1ac0f..b7b7a5aa 100644 --- a/src/lxml.h +++ b/src/lxml.h @@ -72,11 +72,22 @@ int PyXmlSec_LxmlElementConverter(PyObject* o, PyXmlSec_LxmlElementPtr* p); // which is the long-standing direct behaviour with zero overhead. Setting // PYXMLSEC_FORCE_SHADOW in the environment forces the shadow path even on // matched libraries; CI uses it to keep that path exercised. + +// One per node of the copy that existed before the xmlsec call: the tag the +// node's libxml2 _private field points at, holding the child count that node +// had. Untagged after the call means the call created the node; a changed +// child count means it removed (or moved) one. +typedef struct { + int children; +} PyXmlSec_LxmlShadowTag; + typedef struct { PyXmlSec_LxmlElementPtr element; // borrowed; the live element copy paths start from (BeginDoc: the live root) PyObject* owned; // reference released when the shadow ends (BeginDoc's root proxy) xmlDocPtr doc; // the private copy, owned by the shadow; NULL on the fast path xmlNodePtr root; // doc's root element, the copy of `element`; NULL for BeginNewDoc + PyXmlSec_LxmlShadowTag* tags; // one per pre-existing node of `doc`, owned by the shadow + int ntags; } PyXmlSec_LxmlShadow; // Subtree copy: `shadow.root` is the copy of `element`. @@ -134,9 +145,13 @@ xmlNodePtr PyXmlSec_LxmlShadowImportElement(PyXmlSec_LxmlShadow* shadow, PyXmlSe // Shadow-mode ID registration (issue #356): register_id/add_ids cannot write // lxml's ID hash with our libxml2, so they record the id-attribute spec for // the element's document here, and every BeginDoc replays the recorded specs -// onto its copy. The replay scans the whole copy for the recorded attribute -// names — a superset of the single-node registration on the fast path. -int PyXmlSec_LxmlShadowRecordId(PyXmlSec_LxmlElementPtr element, const char* name, const char* ns); +// onto its copy. The registry keeps `element` itself, so the replay applies a +// spec to exactly the node it was registered for — `subtree` extends it to +// the node's descendants, as add_ids (xmlSecAddIDs) does, while register_id +// registers the one node. Registering every matching attribute of the +// document instead would let an unrelated element with the same id value win +// the lookup and steer a `#id` reference away from the registered one. +int PyXmlSec_LxmlShadowRecordId(PyXmlSec_LxmlElementPtr element, const char* name, const char* ns, int subtree); // get version numbers for libxml2 both compiled and loaded long PyXmlSec_GetLibXmlVersionMajor(); diff --git a/src/tree.c b/src/tree.c index d9e7f7a2..8b70534f 100644 --- a/src/tree.c +++ b/src/tree.c @@ -196,8 +196,8 @@ static PyObject* PyXmlSec_TreeAddIds(PyObject* self, PyObject *args, PyObject *k // Shadow mode: registering IDs on lxml's document with our libxml2 is // exactly the cross-library write issue #356 forbids. Record the // attribute names instead; every whole-document shadow (sign, verify, - // decrypt) replays them onto its private copy. The replay scans the whole - // copy rather than just this subtree — a superset of the registration. + // decrypt) replays them onto its private copy, over the subtree rooted at + // `node` — the scope xmlSecAddIDs walks below. if (PyXmlSec_LxmlShadowIsActive()) { for (i = 0; i < n; ++i) { const char* name; @@ -207,7 +207,7 @@ static PyObject* PyXmlSec_TreeAddIds(PyObject* self, PyObject *args, PyObject *k Py_DECREF(key); if (tmp == NULL) goto ON_FAIL; name = PyUnicode_AsUTF8(tmp); - if (name == NULL || PyXmlSec_LxmlShadowRecordId(node, name, NULL) < 0) { + if (name == NULL || PyXmlSec_LxmlShadowRecordId(node, name, NULL, 1) < 0) { Py_DECREF(tmp); goto ON_FAIL; } diff --git a/tests/test_ds.py b/tests/test_ds.py index ccce3a6b..ac6573c3 100644 --- a/tests/test_ds.py +++ b/tests/test_ds.py @@ -222,6 +222,54 @@ def test_sign_and_verify_with_registered_id(self): verify_ctx.key = xmlsec.Key.from_file(self.path('rsapub.pem'), format=consts.KeyDataFormatPem) verify_ctx.verify(sign) + # A document where an unregistered element carries the same id value as the + # registered one, and comes first. Only the registered element may answer + # the "#dup" reference (issue #356). + DUPLICATE_ID_XML = ( + b'\n' + b' decoy\n' + b' real\n' + b'\n' + ) + + def sign_duplicate_id(self, register): + """Signs a "#dup" reference over the document above, registering the ids with `register`.""" + root = etree.fromstring(self.DUPLICATE_ID_XML) + ctx = xmlsec.SignatureContext() + register(root) + sign = xmlsec.template.create(root, consts.TransformExclC14N, consts.TransformRsaSha1, ns='ds') + root.append(sign) + ref = xmlsec.template.add_reference(sign, consts.TransformSha1, uri='#dup') + xmlsec.template.add_transform(ref, consts.TransformExclC14N) + ctx.key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem) + ctx.sign(sign) + return etree.tostring(root) + + def verify_duplicate_id(self, signed, register, tamper): + """Re-parses `signed`, rewrites the text of the `tamper` element and verifies.""" + root = etree.fromstring(signed) + register(root) + root.find(tamper).text = 'tampered' + ctx = xmlsec.SignatureContext() + ctx.key = xmlsec.Key.from_file(self.path('rsapub.pem'), format=consts.KeyDataFormatPem) + ctx.verify(root.find('dsig:Signature', namespaces=base.ns)) + + def assert_covers_registered_element_only(self, register): + signed = self.sign_duplicate_id(register) + # the decoy is not what the reference resolved to, so it is not covered + self.verify_duplicate_id(signed, register, 'Decoy/Data') + # the registered element is + with self.assertRaises(xmlsec.VerificationError): + self.verify_duplicate_id(signed, register, 'Scope/Real/Data') + + def test_register_id_covers_only_the_registered_node(self): + """register_id registers the node it is given, not every element with that attribute.""" + self.assert_covers_registered_element_only(lambda root: xmlsec.SignatureContext().register_id(root.find('Scope/Real'), 'ID')) + + def test_add_ids_covers_only_the_given_subtree(self): + """add_ids registers the subtree it is given, not the whole document.""" + self.assert_covers_registered_element_only(lambda root: xmlsec.tree.add_ids(root.find('Scope'), ['ID'])) + def test_sign_binary_bad_args(self): ctx = xmlsec.SignatureContext() ctx.key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem) From a4f83e326e43c25105beed3c1fd50efd6ae42da8 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 4 Sep 2026 14:08:34 +0000 Subject: [PATCH 14/22] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/test_ds.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_ds.py b/tests/test_ds.py index ac6573c3..d7b44d70 100644 --- a/tests/test_ds.py +++ b/tests/test_ds.py @@ -264,7 +264,9 @@ def assert_covers_registered_element_only(self, register): def test_register_id_covers_only_the_registered_node(self): """register_id registers the node it is given, not every element with that attribute.""" - self.assert_covers_registered_element_only(lambda root: xmlsec.SignatureContext().register_id(root.find('Scope/Real'), 'ID')) + self.assert_covers_registered_element_only( + lambda root: xmlsec.SignatureContext().register_id(root.find('Scope/Real'), 'ID') + ) def test_add_ids_covers_only_the_given_subtree(self): """add_ids registers the subtree it is given, not the whole document.""" From 9c36ae772b17996ff3393b0751e1d53f76f7d34a Mon Sep 17 00:00:00 2001 From: Amin Solhizadeh Date: Fri, 4 Sep 2026 17:26:40 +0200 Subject: [PATCH 15/22] Keep entity references and adopted nodes straight under the shadow (#356) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five findings from the review of the shadow-copy work: - A subtree of a document that declares entities could not be copied at all: `tostring(element)` emits `&name;` without the internal subset that declares it, so every subtree `Begin` — the public finders included — failed with "cannot make a private copy of the element" on a tree parsed with `resolve_entities=False`. Begin now copies the whole document when there is an internal subset and cuts the copy back to the element, so the declarations travel with the references. - The reflect parser expanded entity references, so the re-parse of the copy had a different child structure than the copy the sites were collected from and the child-index paths addressed the wrong nodes (IndexError on a graft past an `_Entity` sibling). It now parses with `resolve_entities=False`. - `Begin` leaked the private document when tagging failed: callers read a negative result as "no shadow to discard". It now discards through one failure path, as BeginDoc already did. - The id registry's liveness test assumed every registered proxy still references the entry's document. lxml lets an element be adopted into another tree, and the resulting offset either pinned the old entry forever (50k adoptions: 24 -> 71 MiB, now flat) or, when an unrelated reference made up the difference, pruned a registration for a document still in use. The expected count now comes from each proxy's current owner, and re-registering an adopted node vacates its old slot. - `add_ids` recorded each name before validating the rest, so `add_ids(node, ['ID', 1])` raised but left `ID` registered — where the fast path validates the whole list before touching the document. The names are materialized and validated first, and the recording is now all-or-nothing. Regression tests cover the four reachable ones; all four fail on the pre-fix build. Suite: 304 passed under the mismatch, 316 on the matched wheel plain and with PYXMLSEC_FORCE_SHADOW=1. --- developer.md | 7 + src/lxml.c | 281 ++++++++++++++++++++++++++++++++-------- src/lxml.h | 10 +- src/tree.c | 22 ++-- tests/test_ds.py | 29 +++++ tests/test_templates.py | 9 ++ tests/test_tree.py | 33 +++++ 7 files changed, 330 insertions(+), 61 deletions(-) diff --git a/developer.md b/developer.md index e6c9bb01..034da532 100644 --- a/developer.md +++ b/developer.md @@ -245,6 +245,13 @@ All invisible to the documented API: (they never usefully did); - documents nested deeper than 256 levels (only possible with `huge_tree`) are refused with an internal error; +- a subtree of a document that declares entities in its internal subset + (`resolve_entities=False`) is copied by copying the whole document and + cutting it back to the element, since the subtree's `&name;` references + need their declarations; `encrypt_xml` templates are still serialized on + their own, so a *template* carrying unresolved entity references is not + supported (signing and encryption of such a document are refused on both + paths anyway — libxml2's c14n rejects entity-reference nodes); - operations that replace the **document root** (encrypting the root element with `Type=Element`, decrypting a root `EncryptedData`) morph the live root element in place into the replacement, because lxml's API cannot swap a diff --git a/src/lxml.c b/src/lxml.c index 3484fed3..35789f5b 100644 --- a/src/lxml.c +++ b/src/lxml.c @@ -122,17 +122,27 @@ int PyXmlSec_LxmlShadowIsActive(void) { return PyXmlSec_LxmlShadowActive; } -// etree.XMLParser(huge_tree=True), the parser for lxml's side of the reflect -// crossing. huge_tree lifts libxml2's 10 MB text-node limit, which lxml's -// default parser keeps, so a CipherValue above it (large encrypt_binary -// payloads) or a document the user parsed with huge_tree still reflects. It -// is safe here: what gets parsed is this extension's own dump of a tree lxml -// has already parsed. +// etree.XMLParser(huge_tree=True, resolve_entities=False), the parser for +// lxml's side of the reflect crossing. +// +// huge_tree lifts libxml2's 10 MB text-node limit, which lxml's default +// parser keeps, so a CipherValue above it (large encrypt_binary payloads) or +// a document the user parsed with huge_tree still reflects. +// +// resolve_entities=False keeps the entity references of a tree the caller +// parsed that way (lxml's `_Entity` children) as references: our own parse of +// the copy leaves them unexpanded too, so expanding them here would give the +// re-parse a different child structure than the copy the sites were collected +// from, and the child-index paths would address the wrong nodes. A tree whose +// entities were already resolved carries none to keep. +// +// Both are safe here: what gets parsed is this extension's own dump of a tree +// lxml has already parsed. static PyObject* PyXmlSec_LxmlNewParser(PyObject* etree) { PyObject* result = NULL; PyObject* cls = PyObject_GetAttrString(etree, "XMLParser"); PyObject* args = PyTuple_New(0); - PyObject* kwargs = Py_BuildValue("{s:O}", "huge_tree", Py_True); + PyObject* kwargs = Py_BuildValue("{s:O,s:O}", "huge_tree", Py_True, "resolve_entities", Py_False); if (cls != NULL && args != NULL && kwargs != NULL) { result = PyObject_Call(cls, args, kwargs); } @@ -298,19 +308,25 @@ static int PyXmlSec_LxmlDocumentUrl(PyObject* tree, PyObject** holder, const cha return 0; } -// The same for the document `element` hangs in. -static int PyXmlSec_LxmlElementDocumentUrl(PyObject* element, PyObject** holder, const char** url) { - PyObject* tree = PyObject_CallMethod(element, "getroottree", NULL); - int rv; +// Whether `tree` carries an internal DTD subset — the only place a document +// lxml parsed can declare the entities its `&name;` references name, and so +// what decides whether a subtree may be serialized on its own (see Begin). +static int PyXmlSec_LxmlDocumentHasInternalDtd(PyObject* tree, int* dtd) { + PyObject* info = PyObject_GetAttrString(tree, "docinfo"); + PyObject* value; - *holder = NULL; - *url = NULL; - if (tree == NULL) { + *dtd = 0; + if (info == NULL) { return -1; } - rv = PyXmlSec_LxmlDocumentUrl(tree, holder, url); - Py_DECREF(tree); - return rv; + value = PyObject_GetAttrString(info, "internalDTD"); + Py_DECREF(info); + if (value == NULL) { + return -1; + } + *dtd = value != Py_None; + Py_DECREF(value); + return 0; } // Nodes that exist before the xmlsec call are tagged through the libxml2 @@ -570,10 +586,47 @@ static int PyXmlSec_LxmlShadowSyncAttributes(xmlNodePtr src, PyObject* dst) { return 0; } +// Makes the copy's counterpart of the element at `path` the copy's root and +// drops the rest of the document, keeping its internal subset. Used when the +// document declares entities: the subtree cannot be serialized on its own +// then (its `&name;` references would be undefined), so the whole document is +// copied and cut down here. The node is copied before it is re-rooted, which +// is what makes the cut safe — libxml2 redeclares on the copy the namespaces +// it inherited from the ancestors that are about to go, and rebinds its +// entity references to the copy's own declarations. Returns 0, or -1 with an +// exception set. +static int PyXmlSec_LxmlShadowReroot(PyXmlSec_LxmlShadow* shadow, const int* path, int depth) { + xmlNodePtr target = PyXmlSec_LxmlShadowWalkNode(xmlDocGetRootElement(shadow->doc), path, depth); + xmlNodePtr copy; + xmlNodePtr old; + + if (target == NULL) { + PyErr_SetString(PyXmlSec_InternalError, "cannot locate the element in the private copy."); + return -1; + } + if (depth == 0) { // the element is the document root already + return 0; + } + copy = xmlDocCopyNode(target, shadow->doc, 1); + if (copy == NULL) { + PyErr_SetString(PyXmlSec_InternalError, "cannot make a private copy of the element."); + return -1; + } + old = xmlDocSetRootElement(shadow->doc, copy); + if (old != NULL) { + xmlFreeNode(old); + } + return 0; +} + int PyXmlSec_LxmlShadowBegin(PyXmlSec_LxmlShadow* shadow, PyXmlSec_LxmlElementPtr element) { - PyObject* bytes; + PyObject* tree = NULL; + PyObject* bytes = NULL; PyObject* url_holder = NULL; const char* url = NULL; + int path[PYXMLSEC_SHADOW_MAX_DEPTH]; + int depth = 0; + int dtd = 0; shadow->element = element; shadow->owned = NULL; @@ -590,22 +643,55 @@ int PyXmlSec_LxmlShadowBegin(PyXmlSec_LxmlShadow* shadow, PyXmlSec_LxmlElementPt return 0; } - bytes = PyXmlSec_LxmlElementToBytes((PyObject*)element); - if (bytes == NULL) { - return -1; + tree = PyObject_CallMethod((PyObject*)element, "getroottree", NULL); + if (tree == NULL) { + goto ON_FAIL; } - if (PyXmlSec_LxmlElementDocumentUrl((PyObject*)element, &url_holder, &url) < 0) { - Py_DECREF(bytes); - return -1; + if (PyXmlSec_LxmlDocumentUrl(tree, &url_holder, &url) < 0 + || PyXmlSec_LxmlDocumentHasInternalDtd(tree, &dtd) < 0) { + goto ON_FAIL; + } + + if (dtd) { + // The subtree may hold entity references, and their declarations live + // in the document's internal subset — serializing the element alone + // would leave them undefined and the parse would fail. Copy the whole + // document, declarations included, and cut it back to the element. + PyObject* live_top = NULL; + depth = PyXmlSec_LxmlLivePathTo((PyObject*)element, path, &live_top); + Py_XDECREF(live_top); + if (depth < 0) { + goto ON_FAIL; + } + bytes = PyObject_CallFunctionObjArgs(PyXmlSec_LxmlEtreeToString, tree, NULL); + } else { + bytes = PyXmlSec_LxmlElementToBytes((PyObject*)element); + } + Py_CLEAR(tree); + if (bytes == NULL) { + goto ON_FAIL; } shadow->doc = PyXmlSec_LxmlShadowParse(bytes, url, "cannot make a private copy of the element."); - Py_DECREF(bytes); - Py_XDECREF(url_holder); + Py_CLEAR(bytes); + Py_CLEAR(url_holder); if (shadow->doc == NULL) { - return -1; + goto ON_FAIL; + } + if (dtd && PyXmlSec_LxmlShadowReroot(shadow, path, depth) < 0) { + goto ON_FAIL; } shadow->root = xmlDocGetRootElement(shadow->doc); - return PyXmlSec_LxmlShadowMark(shadow); + if (PyXmlSec_LxmlShadowMark(shadow) < 0) { + goto ON_FAIL; + } + return 0; + +ON_FAIL: + Py_XDECREF(tree); + Py_XDECREF(bytes); + Py_XDECREF(url_holder); + PyXmlSec_LxmlShadowDiscard(shadow); + return -1; } xmlDocPtr PyXmlSec_LxmlShadowBeginNewDoc(PyXmlSec_LxmlShadow* shadow, PyXmlSec_LxmlElementPtr element) { @@ -665,33 +751,84 @@ void PyXmlSec_LxmlShadowDiscard(PyXmlSec_LxmlShadow* shadow) { // — could resolve to content the caller never registered. // // Those references also make liveness decidable without weak references -// (lxml's classes refuse those): every element proxy holds a reference to its -// document, so when the document's reference count is exactly what the -// registry itself holds (the entry plus one per registered element) and no -// registered element is referenced anywhere else, nothing can hand that -// document to a binding again — the entry is dead and is dropped. Pruning -// runs before every new registration, which bounds the registry — and the -// documents it pins — by the documents still in use. No live registration is -// ever evicted. +// (lxml's classes refuse those): every element proxy holds a reference to the +// document it hangs in, so when the document's reference count is exactly +// what the registry itself holds (the entry plus one per registered element +// still in it) and no registered element is referenced anywhere else, nothing +// can hand that document to a binding again — the entry is dead and is +// dropped. Pruning runs before every new registration, which bounds the +// registry — and the documents it pins — by the documents still in use. No +// live registration is ever evicted. +// +// An element need not stay in the document it was registered for: lxml lets +// one be adopted into another tree, and its proxy then references that other +// document. The count is therefore taken from each proxy's *current* owner, +// and re-registering an adopted node vacates its slot in the entry it came +// from (ForgetIdNode), so a node is never held by two entries at once and a +// reference count of one still means "the registry alone". // ---------------------------------------------------------------------------- enum { PYXMLSEC_ID_ENTRY_DOC, PYXMLSEC_ID_ENTRY_NODES, PYXMLSEC_ID_ENTRY_SPECS }; // Non-zero when nothing outside the registry can reach the entry's document. static int PyXmlSec_LxmlShadowIdEntryIsDead(PyObject* entry) { + PyObject* doc = PyTuple_GET_ITEM(entry, PYXMLSEC_ID_ENTRY_DOC); PyObject* nodes = PyTuple_GET_ITEM(entry, PYXMLSEC_ID_ENTRY_NODES); Py_ssize_t n = PyList_GET_SIZE(nodes); + Py_ssize_t held = 1; // the entry's own reference Py_ssize_t i; - if (Py_REFCNT(PyTuple_GET_ITEM(entry, PYXMLSEC_ID_ENTRY_DOC)) != n + 1) { - return 0; - } for (i = 0; i < n; ++i) { - if (Py_REFCNT(PyList_GET_ITEM(nodes, i)) != 1) { + PyObject* node = PyList_GET_ITEM(nodes, i); + if (node == Py_None) { // vacated slot: the node was registered elsewhere + continue; + } + if (Py_REFCNT(node) != 1) { return 0; } + if ((PyObject*)((PyXmlSec_LxmlElementPtr)node)->_doc == doc) { + ++held; + } + } + return Py_REFCNT(doc) == held; +} + +// Drops `element` from every entry but `keep`. A node registered again after +// being adopted into another document is no longer part of the tree its old +// entry stands for, and its slot there must stop holding it: the liveness +// test above counts on a registered proxy being held by one entry only. The +// slot is vacated rather than removed, so the indices the surviving specs +// carry stay valid; the specs that pointed at it go. Best effort — a failure +// only leaves an entry alive longer than needed. +static void PyXmlSec_LxmlShadowForgetIdNode(PyObject* keep, PyObject* element) { + PyObject* key; + PyObject* entry; + Py_ssize_t pos = 0; + + while (PyDict_Next(PyXmlSec_LxmlShadowIdRegistry, &pos, &key, &entry)) { + PyObject* nodes; + PyObject* specs; + Py_ssize_t i; + if (entry == keep) { + continue; + } + nodes = PyTuple_GET_ITEM(entry, PYXMLSEC_ID_ENTRY_NODES); + specs = PyTuple_GET_ITEM(entry, PYXMLSEC_ID_ENTRY_SPECS); + for (i = 0; i < PyList_GET_SIZE(nodes); ++i) { + Py_ssize_t j; + if (PyList_GET_ITEM(nodes, i) != element) { + continue; + } + for (j = PyList_GET_SIZE(specs) - 1; j >= 0; --j) { + PyObject* spec = PyList_GET_ITEM(specs, j); + if (PyLong_AsSsize_t(PyTuple_GET_ITEM(spec, 2)) == i && PyList_SetSlice(specs, j, j + 1, NULL) < 0) { + PyErr_Clear(); + } + } + Py_INCREF(Py_None); + PyList_SetItem(nodes, i, Py_None); // steals the reference, releases the node + } } - return 1; } // Drops the entries of documents nobody but the registry still references. @@ -724,16 +861,19 @@ static void PyXmlSec_LxmlShadowPruneIdRegistry(void) { Py_DECREF(dead); } -int PyXmlSec_LxmlShadowRecordId(PyXmlSec_LxmlElementPtr element, const char* name, const char* ns, int subtree) { +int PyXmlSec_LxmlShadowRecordIds(PyXmlSec_LxmlElementPtr element, PyObject* names, const char* ns, int subtree) { PyObject* key = NULL; PyObject* created = NULL; PyObject* spec = NULL; - PyObject* entry; + PyObject* entry = NULL; PyObject* nodes; PyObject* specs; + Py_ssize_t nnodes = 0; + Py_ssize_t nspecs = 0; Py_ssize_t idx; Py_ssize_t i; int contains; + int fresh = 0; int result = -1; key = PyLong_FromVoidPtr((void*)element->_doc); @@ -755,14 +895,17 @@ int PyXmlSec_LxmlShadowRecordId(PyXmlSec_LxmlElementPtr element, const char* nam goto DONE; } entry = created; + fresh = 1; } // One reference per registered element, so that the liveness test can // account for exactly the references the registry itself holds. nodes = PyTuple_GET_ITEM(entry, PYXMLSEC_ID_ENTRY_NODES); specs = PyTuple_GET_ITEM(entry, PYXMLSEC_ID_ENTRY_SPECS); + nnodes = PyList_GET_SIZE(nodes); + nspecs = PyList_GET_SIZE(specs); idx = -1; - for (i = 0; i < PyList_GET_SIZE(nodes); ++i) { + for (i = 0; i < nnodes; ++i) { if (PyList_GET_ITEM(nodes, i) == (PyObject*)element) { idx = i; break; @@ -772,26 +915,57 @@ int PyXmlSec_LxmlShadowRecordId(PyXmlSec_LxmlElementPtr element, const char* nam if (PyList_Append(nodes, (PyObject*)element) < 0) { goto DONE; } - idx = PyList_GET_SIZE(nodes) - 1; + idx = nnodes; + PyXmlSec_LxmlShadowForgetIdNode(entry, (PyObject*)element); } - spec = Py_BuildValue("(szni)", name, ns, idx, subtree); - if (spec == NULL) { - goto DONE; - } - contains = PySequence_Contains(specs, spec); - if (contains < 0 || (!contains && PyList_Append(specs, spec) < 0)) { - goto DONE; + for (i = 0; i < PyList_GET_SIZE(names); ++i) { + spec = Py_BuildValue("(Ozni)", PyList_GET_ITEM(names, i), ns, idx, subtree); + if (spec == NULL) { + goto DONE; + } + contains = PySequence_Contains(specs, spec); + if (contains < 0 || (!contains && PyList_Append(specs, spec) < 0)) { + goto DONE; + } + Py_CLEAR(spec); } result = 0; DONE: + // All of the call's names are recorded or none of them are: a caller that + // hands over a bad list must not find part of it registered. Rolling back + // must not clobber the failure that caused it, hence the fetch/restore. + if (result < 0 && entry != NULL) { + PyObject* type; + PyObject* value; + PyObject* tb; + PyErr_Fetch(&type, &value, &tb); + if (fresh ? PyDict_DelItem(PyXmlSec_LxmlShadowIdRegistry, key) < 0 + : (PyList_SetSlice(PyTuple_GET_ITEM(entry, PYXMLSEC_ID_ENTRY_SPECS), nspecs, PY_SSIZE_T_MAX, NULL) < 0 + || PyList_SetSlice(PyTuple_GET_ITEM(entry, PYXMLSEC_ID_ENTRY_NODES), nnodes, PY_SSIZE_T_MAX, NULL) < 0)) { + PyErr_Clear(); + } + PyErr_Restore(type, value, tb); + } Py_XDECREF(key); Py_XDECREF(created); Py_XDECREF(spec); return result; } +int PyXmlSec_LxmlShadowRecordId(PyXmlSec_LxmlElementPtr element, const char* name, const char* ns, int subtree) { + PyObject* names = Py_BuildValue("[s]", name); + int rv; + + if (names == NULL) { + return -1; + } + rv = PyXmlSec_LxmlShadowRecordIds(element, names, ns, subtree); + Py_DECREF(names); + return rv; +} + // Registers `node`'s `name` attribute (in `ns` when given) as an XML ID, the // way xmlSecAddIDs does: the first registration of a value wins. static void PyXmlSec_LxmlShadowAddId(xmlDocPtr doc, xmlNodePtr node, const xmlChar* name, const xmlChar* ns) { @@ -871,6 +1045,9 @@ static int PyXmlSec_LxmlShadowReplayIds(PyXmlSec_LxmlShadow* shadow) { if (name == NULL || (ns == NULL && PyErr_Occurred())) { return -1; } + if (node == Py_None) { // vacated slot: the node is registered elsewhere now + continue; + } depth = PyXmlSec_LxmlLivePathTo(node, path, &top); if (depth < 0) { return -1; diff --git a/src/lxml.h b/src/lxml.h index b7b7a5aa..c868f311 100644 --- a/src/lxml.h +++ b/src/lxml.h @@ -90,7 +90,9 @@ typedef struct { int ntags; } PyXmlSec_LxmlShadow; -// Subtree copy: `shadow.root` is the copy of `element`. +// Subtree copy: `shadow.root` is the copy of `element`. When the document +// declares entities, the whole document is copied and then cut back to the +// element, so that the subtree's entity references keep their declarations. int PyXmlSec_LxmlShadowBegin(PyXmlSec_LxmlShadow* shadow, PyXmlSec_LxmlElementPtr element); // Whole-document copy, for calls that read or mutate beyond the element's @@ -153,6 +155,12 @@ xmlNodePtr PyXmlSec_LxmlShadowImportElement(PyXmlSec_LxmlShadow* shadow, PyXmlSe // the lookup and steer a `#id` reference away from the registered one. int PyXmlSec_LxmlShadowRecordId(PyXmlSec_LxmlElementPtr element, const char* name, const char* ns, int subtree); +// The same for a whole list of attribute names (`names`, a list of str), for +// add_ids: either every name is recorded or, if anything fails, none is — +// the fast path likewise builds its complete list before it touches the +// document, so a bad list leaves no half-applied registration behind. +int PyXmlSec_LxmlShadowRecordIds(PyXmlSec_LxmlElementPtr element, PyObject* names, const char* ns, int subtree); + // get version numbers for libxml2 both compiled and loaded long PyXmlSec_GetLibXmlVersionMajor(); long PyXmlSec_GetLibXmlVersionMinor(); diff --git a/src/tree.c b/src/tree.c index 8b70534f..1a8470d9 100644 --- a/src/tree.c +++ b/src/tree.c @@ -199,20 +199,26 @@ static PyObject* PyXmlSec_TreeAddIds(PyObject* self, PyObject *args, PyObject *k // decrypt) replays them onto its private copy, over the subtree rooted at // `node` — the scope xmlSecAddIDs walks below. if (PyXmlSec_LxmlShadowIsActive()) { + // Materialize and validate the whole list before recording any of it, + // as the fast path below does before it calls xmlSecAddIDs: a bad + // item must leave nothing registered. + PyObject* names = PyList_New(0); + int rv; + if (names == NULL) goto ON_FAIL; for (i = 0; i < n; ++i) { - const char* name; key = PyLong_FromSsize_t(i); - if (key == NULL) goto ON_FAIL; - tmp = PyObject_GetItem(ids, key); - Py_DECREF(key); - if (tmp == NULL) goto ON_FAIL; - name = PyUnicode_AsUTF8(tmp); - if (name == NULL || PyXmlSec_LxmlShadowRecordId(node, name, NULL, 1) < 0) { - Py_DECREF(tmp); + tmp = key != NULL ? PyObject_GetItem(ids, key) : NULL; + Py_XDECREF(key); + if (tmp == NULL || PyUnicode_AsUTF8(tmp) == NULL || PyList_Append(names, tmp) < 0) { + Py_XDECREF(tmp); + Py_DECREF(names); goto ON_FAIL; } Py_DECREF(tmp); } + rv = PyXmlSec_LxmlShadowRecordIds(node, names, NULL, 1); + Py_DECREF(names); + if (rv < 0) goto ON_FAIL; PYXMLSEC_DEBUG("tree add_ids - ok"); Py_RETURN_NONE; } diff --git a/tests/test_ds.py b/tests/test_ds.py index d7b44d70..f1fb76b3 100644 --- a/tests/test_ds.py +++ b/tests/test_ds.py @@ -272,6 +272,16 @@ def test_add_ids_covers_only_the_given_subtree(self): """add_ids registers the subtree it is given, not the whole document.""" self.assert_covers_registered_element_only(lambda root: xmlsec.tree.add_ids(root.find('Scope'), ['ID'])) + def test_add_ids_records_nothing_when_an_item_is_not_a_name(self): + """A rejected add_ids leaves no part of its list registered, so the "#dup" reference resolves to nothing.""" + + def register(root): + with self.assertRaises(TypeError): + xmlsec.tree.add_ids(root.find('Scope'), ['ID', 1]) + + with self.assertRaises(xmlsec.Error): + self.sign_duplicate_id(register) + def test_sign_binary_bad_args(self): ctx = xmlsec.SignatureContext() ctx.key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem) @@ -444,3 +454,22 @@ def test_registration_survives_other_live_documents(self): sign = xmlsec.tree.find_node(root, consts.NodeSignature) ctx.key = xmlsec.Key.from_file(self.path('rsapub.pem'), format=consts.KeyDataFormatPem) ctx.verify(sign) # resolves #ID through the registration made before the churn + + def test_registration_survives_a_sibling_being_adopted_away(self): + """An element moved into another document must not make its old document's registrations look collectable.""" + root = etree.fromstring(b'x') + ctx = xmlsec.SignatureContext() + moved = root.find('Moved') + ctx.register_id(moved, 'ID') + ctx.register_id(root.find('Stays'), 'ID') + + etree.fromstring(b'').append(moved) # `moved` now references the other document + del moved + ctx.register_id(etree.fromstring(b''), 'ID') # a new document prunes the registry + + sign = xmlsec.template.create(root, consts.TransformExclC14N, consts.TransformRsaSha1, ns='ds') + root.append(sign) + ref = xmlsec.template.add_reference(sign, consts.TransformSha1, uri='#stays') + xmlsec.template.add_transform(ref, consts.TransformExclC14N) + ctx.key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem) + ctx.sign(sign) # resolves #stays only if `root`'s registrations survived the prune diff --git a/tests/test_templates.py b/tests/test_templates.py index 59af6c6f..efe8b9f0 100644 --- a/tests/test_templates.py +++ b/tests/test_templates.py @@ -98,6 +98,15 @@ def test_add_reference(self): for a in ('Id', 'URI', 'Type'): self.assertEqual(a, ref.get(a)) + def test_add_reference_beside_an_entity_reference(self): + """A graft must count entity references as children, exactly as lxml's insert does (issue #356).""" + root = etree.fromstring(b' ]>\n', etree.XMLParser(resolve_entities=False)) + sign = xmlsec.template.create(root, c14n_method=consts.TransformExclC14N, sign_method=consts.TransformRsaSha1) + root.append(sign) + sign[0].insert(0, etree.Entity('greet')) # SignedInfo now leads with an entity reference + ref = xmlsec.template.add_reference(sign, consts.TransformSha1, uri='#abc') + self.assertIs(ref, sign[0][3]) + def test_add_reference_bad_args(self): with self.assertRaises(TypeError): xmlsec.template.add_reference('', consts.TransformSha1) diff --git a/tests/test_tree.py b/tests/test_tree.py index 5e80a60a..cabc30fb 100644 --- a/tests/test_tree.py +++ b/tests/test_tree.py @@ -1,3 +1,5 @@ +from lxml import etree + import xmlsec from tests import base @@ -40,6 +42,37 @@ def test_add_ids(self): root = self.load_xml('sign_template.xml') xmlsec.tree.add_ids(root, ['id1', 'id2', 'id3']) + # A document whose entity declarations live in its internal subset, parsed the way callers who + # refuse entity expansion do. The subtree cannot be serialized on its own then (issue #356). + ENTITY_XML = ( + b' ]>\n' + b'' + b'&greet;' + ) + + def load_entity_xml(self): + root = etree.fromstring(self.ENTITY_XML, etree.XMLParser(resolve_entities=False)) + return root, etree.tostring(root.getroottree()) + + def test_find_child_keeps_entity_references(self): + """The finders must work on a document that declares entities, and leave its references alone.""" + root, before = self.load_entity_xml() + sign = xmlsec.tree.find_child(root, consts.NodeSignature, consts.DSigNs) + self.assertEqual(consts.NodeSignature, sign.tag.partition('}')[2]) + self.assertEqual(before, etree.tostring(root.getroottree())) + + def test_find_node_keeps_entity_references(self): + root, before = self.load_entity_xml() + si = xmlsec.tree.find_node(root, consts.NodeSignedInfo, consts.DSigNs) + self.assertEqual(consts.NodeSignedInfo, si.tag.partition('}')[2]) + self.assertEqual(before, etree.tostring(root.getroottree())) + + def test_find_parent_keeps_entity_references(self): + root, before = self.load_entity_xml() + si = xmlsec.tree.find_node(root, consts.NodeSignedInfo, consts.DSigNs) + self.assertIs(root[0], xmlsec.tree.find_parent(si, consts.NodeSignature)) + self.assertEqual(before, etree.tostring(root.getroottree())) + def test_add_ids_bad_args(self): with self.assertRaises(TypeError): xmlsec.tree.add_ids('', []) From 7e279cf71e8a34331c44ab07e5608989b9ae5e45 Mon Sep 17 00:00:00 2001 From: Amin Solhizadeh Date: Fri, 4 Sep 2026 18:14:00 +0200 Subject: [PATCH 16/22] Move an attached encrypt_xml template, fingerprint text (#356) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two findings from the review of the shadow path. `encrypt_xml` encrypts a copy of the template, so a template attached in the target's own document stayed where it was and the result gained a second, empty — a tree shape that depended on which libxml2 the extension is linked against. The raw path hands the template node itself to xmlsec, which moves it. The live template is now unlinked after the reflection when it is still under the live document root, with its tail text re-homed onto the previous sibling (or the parent's text) the way libxml2's xmlReplaceNode leaves it. The mutation detector recorded a sync for a parent that gained a fresh node or lost a tagged one, but not for a text node rewritten in place. Writing a value into an element goes through xmlNodeSetContent, which frees the old text node and parses a fresh one, so re-signing over an existing DigestValue was already caught; appending to a text node (xmlNodeAddContent onto a trailing text child) is not. Each tag now carries an FNV-1a fingerprint of a text node's content, so the invariant holds without depending on that libxml2 internal. Co-Authored-By: Claude Opus 5 --- developer.md | 28 +++++++---- src/enc.c | 120 ++++++++++++++++++++++++++++++++++++++++++++-- src/lxml.c | 49 ++++++++++++++----- src/lxml.h | 6 ++- tests/test_enc.py | 44 +++++++++++++++++ 5 files changed, 221 insertions(+), 26 deletions(-) diff --git a/developer.md b/developer.md index 034da532..d68fc979 100644 --- a/developer.md +++ b/developer.md @@ -114,13 +114,21 @@ records two kinds of *site*: - **graft** — a fresh node (element, comment, PI) to insert at its child index. Fresh subtrees are grafted wholesale; the scan never descends into them. -- **sync** — a tagged parent that gained any fresh node (element or text) - gets its text slots (its `.text` and each child's `.tail`) copied over from - the re-parsed copy. That covers everything xmlsec does to text: the `"\n"` - formatting around a new node, values filled into empty elements - (`DigestValue`, `SignatureValue`), and content it removed (encrypt - `Type=Content`) — a removal leaves no fresh node behind, so only a - wholesale sync can see it. +- **sync** — a tagged parent whose children changed gets its text slots (its + `.text` and each child's `.tail`) copied over from the re-parsed copy. That + covers everything xmlsec does to text: the `"\n"` formatting around a new + node, values filled into empty elements (`DigestValue`, `SignatureValue`), + and content it removed (encrypt `Type=Content`). A tag therefore records + more than "this node existed": it also holds the node's child count, since + a removal leaves no fresh node behind and only the changed count shows it + happened, and an FNV-1a fingerprint of a text node's own content, since + appending to a text node (`xmlNodeAddContent` onto a trailing text child) + changes neither the node nor the count. Writing an element's value goes + through `xmlNodeSetContent`, which frees the old text node and parses a + fresh one, so re-signing over an existing `DigestValue` is caught by the + fresh-node rule; the fingerprint is what keeps the invariant ("a parent + whose children changed gets a sync") from depending on that libxml2 + internal. Sites are addressed by **child-index paths** from the copy root, counting exactly the node types lxml exposes as children (elements, comments, PIs, @@ -239,8 +247,10 @@ All invisible to the documented API: - `encrypted_data_ensure_key_info(ns=...)` on an existing `KeyInfo` returns a new element object rather than the original proxy; - `register_id` skips the live duplicate-id check (it runs per copy instead); -- `encrypt_xml` *copies* a template that is attached inside the target tree - rather than moving it, so it also remains at its original position; +- `encrypt_xml` encrypts a *copy* of the template, so the caller's template + proxy is not the returned element; a template attached in the target's own + document is unlinked afterwards (keeping its tail text where libxml2 would + leave it), so the resulting tree matches the raw path's move; - signature/encryption contexts keep no live result nodes after the call (they never usefully did); - documents nested deeper than 256 levels (only possible with `huge_tree`) diff --git a/src/enc.c b/src/enc.c index 4d932034..dd1e84b5 100644 --- a/src/enc.c +++ b/src/enc.c @@ -230,14 +230,124 @@ static void PyXmlSec_ClearReplacedNodes(xmlSecEncCtxPtr ctx, PyXmlSec_LxmlDocume ctx->replacedNodeList = NULL; } +// The raw path hands `xmlSecEncCtxXmlEncrypt` the template node itself +// whenever it belongs to the target's document, and xmlsec *moves* it into +// the target's place. The shadow path encrypts a copy of it instead, so +// without this the live tree would keep the original where it was and the +// document would end up with a second, empty — a tree shape +// that depended on which libxml2 the extension is linked against. Unlink it +// once the reflection is done, leaving its tail text behind the way libxml2's +// `xmlReplaceNode` does, since lxml drops an element's tail together with the +// element. A detached template, or one in another document, is copied on both +// paths and stays where it is. Returns 0, or -1 with an exception set. +static int PyXmlSec_EncryptionContextDropMovedTemplate(PyXmlSec_LxmlElementPtr template, PyXmlSec_LxmlElementPtr node) { + PyObject* parent = NULL; + PyObject* tree = NULL; + PyObject* root = NULL; + PyObject* top = NULL; + PyObject* tail = NULL; + PyObject* prev = NULL; + PyObject* slot = NULL; + PyObject* tmp = NULL; + const char* name = "tail"; + int rv = -1; + + parent = PyObject_CallMethod((PyObject*)template, "getparent", NULL); + if (parent == NULL) { + goto DONE; + } + if (parent == Py_None) { + rv = 0; + goto DONE; + } + + // Only a template still hanging under the live document root is one the + // raw path would have moved; one carried off inside the subtree the + // encryption replaced, or one belonging to another document, is not. + tree = PyObject_CallMethod((PyObject*)node, "getroottree", NULL); + root = tree != NULL ? PyObject_CallMethod(tree, "getroot", NULL) : NULL; + if (root == NULL) { + goto DONE; + } + top = parent; + Py_INCREF(top); + for (;;) { + tmp = PyObject_CallMethod(top, "getparent", NULL); + if (tmp == NULL) { + goto DONE; + } + if (tmp == Py_None) { + Py_CLEAR(tmp); + break; + } + Py_DECREF(top); + top = tmp; + tmp = NULL; + } + if (top != root) { + rv = 0; + goto DONE; + } + + tail = PyObject_GetAttrString((PyObject*)template, "tail"); + if (tail == NULL) { + goto DONE; + } + if (tail != Py_None) { + prev = PyObject_CallMethod((PyObject*)template, "getprevious", NULL); + if (prev == NULL) { + goto DONE; + } + if (prev == Py_None) { + // first child: the text libxml2 would leave behind belongs to the + // parent's own text slot + Py_DECREF(prev); + Py_INCREF(parent); + prev = parent; + name = "text"; + } + slot = PyObject_GetAttrString(prev, name); + if (slot == NULL) { + goto DONE; + } + if (slot != Py_None) { + tmp = PyUnicode_Concat(slot, tail); + if (tmp == NULL) { + goto DONE; + } + Py_DECREF(tail); + tail = tmp; + tmp = NULL; + } + if (PyObject_SetAttrString(prev, name, tail) < 0) { + goto DONE; + } + } + tmp = PyObject_CallMethod(parent, "remove", "O", template); + if (tmp == NULL) { + goto DONE; + } + rv = 0; + +DONE: + Py_XDECREF(parent); + Py_XDECREF(tree); + Py_XDECREF(root); + Py_XDECREF(top); + Py_XDECREF(tail); + Py_XDECREF(prev); + Py_XDECREF(slot); + Py_XDECREF(tmp); + return rv; +} + // Shadow-path body of encrypt_xml (issue #356): the target document and the // template are both re-parsed into one private copy, the encryption runs // there, and the replacement is reflected back through lxml — the fresh // takes the target's place (`Type=Element`; the document // root is morphed in place, as lxml cannot swap it) or its content -// (`Type=Content`). One divergence from the raw path: a template that is -// *attached* inside the target's own tree is copied, not moved, so it also -// remains at its original position. +// (`Type=Content`). The live template is unlinked afterwards when the raw +// path would have moved it. static PyObject* PyXmlSec_EncryptionContextEncryptXmlShadow(PyXmlSec_EncryptionContext* ctx, PyXmlSec_LxmlElementPtr template, PyXmlSec_LxmlElementPtr node) { PyXmlSec_LxmlShadow shadow; xmlNodePtr target = NULL; @@ -359,6 +469,10 @@ static PyObject* PyXmlSec_EncryptionContextEncryptXmlShadow(PyXmlSec_EncryptionC } } + if (PyXmlSec_EncryptionContextDropMovedTemplate(template, node) < 0) { + goto ON_FAIL; + } + Py_DECREF(type_value); Py_XDECREF(parent); return result; diff --git a/src/lxml.c b/src/lxml.c index 35789f5b..1c4d0be8 100644 --- a/src/lxml.c +++ b/src/lxml.c @@ -335,7 +335,10 @@ static int PyXmlSec_LxmlDocumentHasInternalDtd(PyObject* tree, int* dtd) { // the call is what the call created. Each tag also records the child count // the node had, which is what makes a *removal* visible — a call can delete a // node and leave nothing fresh behind (an that decrypts to -// empty content), and only the changed count shows it happened. A tag +// empty content), and only the changed count shows it happened — and a +// fingerprint of the node's content, which is what makes an *in-place* text +// rewrite visible: appending to a text node (xmlNodeAddContent onto a +// trailing text child) leaves both the node and the count untouched. A tag // pointer is compared against the array bounds, never dereferenced blindly: // a foreign _private (lxml's own proxy, when both libraries are the same // libxml2) simply falls outside. @@ -353,6 +356,21 @@ static int PyXmlSec_LxmlShadowCountNodes(xmlNodePtr node) { return count; } +// FNV-1a over the content a text or CDATA node carries; 0 for every other +// node type, whose content the reflection does not carry back on its own. +static unsigned long long PyXmlSec_LxmlShadowContentPrint(xmlNodePtr node) { + unsigned long long fingerprint = 14695981039346656037ULL; + const xmlChar* p; + + if (node->type != XML_TEXT_NODE && node->type != XML_CDATA_SECTION_NODE) { + return 0; + } + for (p = node->content; p != NULL && *p != 0; ++p) { + fingerprint = (fingerprint ^ (unsigned long long)*p) * 1099511628211ULL; + } + return fingerprint; +} + // Hands out `shadow->tags` in document order; returns the next free slot. static int PyXmlSec_LxmlShadowTagNodes(PyXmlSec_LxmlShadow* shadow, xmlNodePtr node, int next) { for (; node != NULL; node = node->next) { @@ -362,6 +380,7 @@ static int PyXmlSec_LxmlShadowTagNodes(PyXmlSec_LxmlShadow* shadow, xmlNodePtr n for (child = node->children; child != NULL; child = child->next) { ++tag->children; } + tag->content = PyXmlSec_LxmlShadowContentPrint(node); node->_private = (void*)tag; next = PyXmlSec_LxmlShadowTagNodes(shadow, node->children, next); } @@ -1157,14 +1176,14 @@ int PyXmlSec_LxmlShadowBeginDoc(PyXmlSec_LxmlShadow* shadow, PyXmlSec_LxmlElemen // index. Fresh subtrees are grafted wholesale; the scan never // descends into them. // sync — a tagged parent whose children changed — it gained a fresh node -// (element or text), or lost a tagged one — gets its text slots -// (its .text and each child's .tail) copied over from the re-parsed -// copy. That covers everything xmlsec does to text: the "\n" -// formatting around a new node, values filled into empty elements -// (DigestValue), and content it removed (encrypt Type=Content) — -// a removal leaves no fresh node behind, so only the tagged child -// count shows that it happened, and only a wholesale sync can carry -// it across. +// (element or text), lost a tagged one, or had a tagged text child +// rewritten in place — gets its text slots (its .text and each +// child's .tail) copied over from the re-parsed copy. That covers +// everything xmlsec does to text: the "\n" formatting around a new +// node, values filled into empty elements (DigestValue), and +// content it removed (encrypt Type=Content) — a removal leaves no +// fresh node behind, so only the tagged child count shows that it +// happened, and only a wholesale sync can carry it across. // // The reflection is two-phase: first every site's payload is fetched from // the re-parsed copy while it is still in its final, untouched state, then @@ -1222,16 +1241,22 @@ static int PyXmlSec_LxmlShadowSiteAppend(PyXmlSec_LxmlShadowSiteList* list, xmlN // Walks the tagged structure of the copy in document order, recording a // graft for every fresh node and, after them, a sync for their parent — and // for a parent that no longer holds the children it was tagged with, whose -// text slots are the only trace the removal left. +// text slots are the only trace the removal left, or one whose own text +// survived the call but with different content. static int PyXmlSec_LxmlShadowCollectSites(PyXmlSec_LxmlShadowSiteList* list, xmlNodePtr parent) { PyXmlSec_LxmlShadowTag* tag = PYXMLSEC_SHADOW_TAG(list->shadow, parent); xmlNodePtr n; int fresh = 0; int tagged = 0; + int rewritten = 0; for (n = parent->children; n != NULL; n = n->next) { - if (PYXMLSEC_SHADOW_TAGGED(list->shadow, n)) { + PyXmlSec_LxmlShadowTag* child_tag = PYXMLSEC_SHADOW_TAG(list->shadow, n); + if (child_tag != NULL) { ++tagged; + if (child_tag->content != PyXmlSec_LxmlShadowContentPrint(n)) { + rewritten = 1; + } if (n->type == XML_ELEMENT_NODE && PyXmlSec_LxmlShadowCollectSites(list, n) < 0) { return -1; } @@ -1242,7 +1267,7 @@ static int PyXmlSec_LxmlShadowCollectSites(PyXmlSec_LxmlShadowSiteList* list, xm return -1; } } - if ((fresh || (tag != NULL && tag->children != tagged)) + if ((fresh || rewritten || (tag != NULL && tag->children != tagged)) && PyXmlSec_LxmlShadowSiteAppend(list, parent, PYXMLSEC_SHADOW_SITE_SYNC) < 0) { return -1; } diff --git a/src/lxml.h b/src/lxml.h index c868f311..2ba40094 100644 --- a/src/lxml.h +++ b/src/lxml.h @@ -75,10 +75,12 @@ int PyXmlSec_LxmlElementConverter(PyObject* o, PyXmlSec_LxmlElementPtr* p); // One per node of the copy that existed before the xmlsec call: the tag the // node's libxml2 _private field points at, holding the child count that node -// had. Untagged after the call means the call created the node; a changed -// child count means it removed (or moved) one. +// had and a fingerprint of the content it held. Untagged after the call means +// the call created the node; a changed child count means it removed (or +// moved) one; a changed fingerprint means it rewrote a text node in place. typedef struct { int children; + unsigned long long content; } PyXmlSec_LxmlShadowTag; typedef struct { diff --git a/tests/test_enc.py b/tests/test_enc.py index 8db004b5..b2f3599c 100644 --- a/tests/test_enc.py +++ b/tests/test_enc.py @@ -109,6 +109,50 @@ def test_encrypt_xml_root(self): self.assertIs(decrypted.getroottree().getroot(), decrypted) self.assertEqual(xml, etree.tostring(decrypted.getroottree())) + def encrypt_with_attached_template(self, enc_type, index): + # A template that hangs in the target's own document is *moved* into + # the target's place, whichever path runs: the document must not end + # up with a second, empty where the template was, and + # the text that followed it must stay behind (issue #356). + root = etree.fromstring(b'\n \n secret\n \n') + enc_data = xmlsec.template.encrypted_data_create(root, consts.TransformAes128Cbc, type=enc_type, ns='xenc') + xmlsec.template.encrypted_data_ensure_cipher_value(enc_data) + root.insert(index, enc_data) + enc_data.tail = '\n tail\n ' + + ctx = xmlsec.EncryptionContext() + ctx.key = xmlsec.Key.generate(consts.KeyDataAes, 128, consts.KeyDataTypeSession) + encrypted = ctx.encrypt_xml(enc_data, root.find('Data')) + + self.assertEqual(f'{{{consts.EncNs}}}{consts.NodeEncryptedData}', encrypted.tag) + self.assertEqual(1, len(root.findall(f'.//{{{consts.EncNs}}}{consts.NodeEncryptedData}'))) + self.assertIn('tail', etree.tostring(root).decode()) + + def test_encrypt_xml_attached_template_element(self): + self.encrypt_with_attached_template(consts.TypeEncElement, 0) + self.encrypt_with_attached_template(consts.TypeEncElement, 2) + + def test_encrypt_xml_attached_template_content(self): + self.encrypt_with_attached_template(consts.TypeEncContent, 0) + self.encrypt_with_attached_template(consts.TypeEncContent, 2) + + def test_encrypt_binary_over_a_filled_cipher_value(self): + # The reflection must carry a rewritten text value back, not only the + # first one written into an empty element (issue #356). + root = self.load_xml('enc1-in.xml') + enc_data = xmlsec.template.encrypted_data_create(root, consts.TransformAes128Cbc, type=consts.TypeEncContent, ns='xenc') + xmlsec.template.encrypted_data_ensure_cipher_value(enc_data) + + def encrypt(data): + ctx = xmlsec.EncryptionContext() + ctx.key = xmlsec.Key.generate(consts.KeyDataAes, 128, consts.KeyDataTypeSession) + ctx.encrypt_binary(enc_data, data) + return xmlsec.tree.find_node(enc_data, consts.NodeCipherValue, consts.EncNs).text + + first = encrypt(b'first') + self.assertIsNotNone(first) + self.assertNotEqual(first, encrypt(b'a rather different payload')) + def test_encrypt_xml_bad_args(self): ctx = xmlsec.EncryptionContext() with self.assertRaises(TypeError): From 3367b3941b6d1def759b5fe50dd056d07ad9cd99 Mon Sep 17 00:00:00 2001 From: Amin Solhizadeh Date: Fri, 4 Sep 2026 18:58:08 +0200 Subject: [PATCH 17/22] Refuse a duplicate id at register_id under the shadow (#356) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One finding from the review of the shadow path. `register_id` raises "duplicated id." on the raw path when the id value it is asked to register is already registered for another attribute, and under the shadow it silently recorded the spec instead: the replay's first-wins rule then skipped it, so the call succeeded while the caller's `#id` reference resolved to whichever element claimed the value first — an earlier registration, a DTD-declared id attribute or an xml:id. The check is back at the call that makes the registration, where the fast path has it, rather than at the replay: raising from a later sign/verify would report the collision from the wrong call and then from every later call on that document. `xmlGetID(doc, value) != attr` is assembled from the two places a registration can live under the shadow — what lxml's own parse declared, read back through XPath's id(), which crosses the library boundary as strings and elements only, and what earlier register_id or add_ids calls recorded in the registry. Re-registering the same attribute stays the no-op the fast path performs, and add_ids keeps xmlSecAddIDs' own first-wins semantics, which never raise. Co-Authored-By: Claude Opus 5 --- developer.md | 23 +++- src/ds.c | 78 +------------ src/lxml.c | 278 ++++++++++++++++++++++++++++++++++++++++++++++- src/lxml.h | 16 ++- tests/test_ds.py | 34 ++++++ 5 files changed, 343 insertions(+), 86 deletions(-) diff --git a/developer.md b/developer.md index d68fc979..dd4601a5 100644 --- a/developer.md +++ b/developer.md @@ -71,7 +71,7 @@ End always releases the copy, on success and on error. Plus three helpers for the call sites whose *semantics* differ per mode: `IsActive()` (which path is on), `ImportElement()` (encrypt_xml's template -import into the shadow document) and `RecordId()` (ID registration, below). +import into the shadow document) and `RegisterId()` (ID registration, below). Which pair a binding uses follows from what the xmlsec call does: @@ -201,7 +201,7 @@ lxml's own dump of a tree it already parsed. `SignatureContext.register_id` and `tree.add_ids` used to write lxml's ID hash with our libxml2 — exactly the cross-library access the shadow forbids. Under the shadow they record the id-attribute specs in a registry keyed by -document identity (`RecordId`), and every `BeginDoc` replays them onto its +document identity (`RegisterId`, `RecordIds`), and every `BeginDoc` replays them onto its copy so that `#id` references resolve during sign/verify/decrypt. An entry keeps the registered elements themselves, so a spec is replayed at exactly the node it was registered for — that node alone for `register_id`, its @@ -219,6 +219,19 @@ registration. The registry therefore tracks the documents still in use and never evicts a live one. The two bindings are the only places, together with encrypt_xml/decrypt's replacement bodies, that branch on `IsActive()`. +`register_id` still refuses a duplicate id the way the fast path does, and at +the same call: `xmlGetID(doc, value) != attr` — the test that raises +`duplicated id.` — is assembled from the two places a registration can live +under the shadow. What lxml's own parse declared (a DTD id attribute, an +`xml:id`) is read back through XPath's `id()`, the one door into lxml's id +hash that passes nothing but strings and elements; what earlier +`register_id`/`add_ids` calls claimed is read from the registry, a subtree +spec through one XPath over its scope. Deferring the check to the replay +instead would raise from the wrong call — a later `sign`, and then from every +later call on that document — and would leave the caller believing a +registration took that can never win the lookup. `add_ids` keeps its own +semantics: `xmlSecAddIDs` registers first-wins and never raises. + ## Converting a binding 1. Classify the xmlsec call with the table above. @@ -246,7 +259,11 @@ All invisible to the documented API: document until grafted; - `encrypted_data_ensure_key_info(ns=...)` on an existing `KeyInfo` returns a new element object rather than the original proxy; -- `register_id` skips the live duplicate-id check (it runs per copy instead); +- `register_id`'s duplicate-id check cannot see an id value that contains + whitespace among lxml's declared ids (XPath's `id()` would read it as a + list of ids), so such a value is compared against the registry alone; a + registration whose element has since been adopted into another document + claims nothing, as at replay; - `encrypt_xml` encrypts a *copy* of the template, so the caller's template proxy is not the returned element; a template attached in the target's own document is unlinked afterwards (keeping its tail text where libxml2 would diff --git a/src/ds.c b/src/ds.c index 9d5d2692..904e609c 100644 --- a/src/ds.c +++ b/src/ds.c @@ -119,50 +119,6 @@ static int PyXmlSec_SignatureContextKeySet(PyObject* self, PyObject* value, void return 0; } -// Non-zero when `node` carries an attribute whose *local* name is `name`, -// whatever its namespace, -1 with an exception set on failure. The fast -// path's xmlHasProp() matches that way, while lxml's node.get(name) only -// finds an unqualified attribute; the shadow's validation goes through this -// so that both modes accept the same calls. -static int PyXmlSec_LxmlHasAttrByLocalName(PyXmlSec_LxmlElementPtr node, const char* name) { - PyObject* keys; - PyObject* seq; - Py_ssize_t i; - Py_ssize_t n; - int found = 0; - - keys = PyObject_CallMethod((PyObject*)node, "keys", NULL); - if (keys == NULL) { - return -1; - } - seq = PySequence_Fast(keys, "unexpected attribute names."); - Py_DECREF(keys); - if (seq == NULL) { - return -1; - } - - n = PySequence_Fast_GET_SIZE(seq); - for (i = 0; i < n && !found; ++i) { - PyObject* key = PySequence_Fast_GET_ITEM(seq, i); // borrowed - const char* local; - const char* end; - if (!PyUnicode_Check(key) || (local = PyUnicode_AsUTF8(key)) == NULL) { - if (!PyErr_Occurred()) { - PyErr_SetString(PyXmlSec_InternalError, "unexpected attribute name."); - } - Py_DECREF(seq); - return -1; - } - // lxml spells a namespaced attribute "{href}local". - if (local[0] == '{' && (end = strchr(local, '}')) != NULL) { - local = end + 1; - } - found = strcmp(local, name) == 0; - } - Py_DECREF(seq); - return found; -} - static const char PyXmlSec_SignatureContextRegisterId__doc__[] = \ "register_id(node, id_attr = 'ID', id_ns = None) -> None\n" "Registers new id.\n\n" @@ -191,37 +147,11 @@ static PyObject* PyXmlSec_SignatureContextRegisterId(PyObject* self, PyObject* a } // Shadow mode: never touch lxml's document (its ID hash) with our libxml2 - // (issue #356). Validate through lxml's API and record the spec; the - // whole-document shadows (sign/verify/decrypt) replay it onto their - // private copies. The duplicate-id check runs there, per copy. + // (issue #356). The registration is validated through lxml's API and + // recorded instead; the whole-document shadows (sign/verify/decrypt) + // replay it onto their private copies. if (PyXmlSec_LxmlShadowIsActive()) { - int found; - if (id_ns != NULL) { - PyObject* key = PyUnicode_FromFormat("{%s}%s", id_ns, id_attr); - PyObject* value; - if (key == NULL) { - goto ON_FAIL; - } - value = PyObject_CallMethod((PyObject*)node, "get", "O", key); - Py_DECREF(key); - if (value == NULL) { - goto ON_FAIL; - } - found = value != Py_None; - Py_DECREF(value); - } else { - // As xmlHasProp() does on the fast path: by local name. - found = PyXmlSec_LxmlHasAttrByLocalName(node, id_attr); - if (found < 0) { - goto ON_FAIL; - } - } - if (!found) { - PyErr_SetString(PyXmlSec_Error, "missing attribute."); - goto ON_FAIL; - } - // Scope 0: this node alone, exactly what the fast path registers. - if (PyXmlSec_LxmlShadowRecordId(node, id_attr, id_ns, 0) < 0) { + if (PyXmlSec_LxmlShadowRegisterId(node, id_attr, id_ns) < 0) { goto ON_FAIL; } PYXMLSEC_DEBUGF("%p: register id - ok", self); diff --git a/src/lxml.c b/src/lxml.c index 1c4d0be8..af77dea3 100644 --- a/src/lxml.c +++ b/src/lxml.c @@ -115,7 +115,7 @@ static PyObject* PyXmlSec_LxmlEtreeCleanupNamespaces; static PyObject* PyXmlSec_LxmlEtreeParser; // Shadow-mode ID registry: maps the identity of an lxml document to the list -// of id-attribute specs registered for it (see PyXmlSec_LxmlShadowRecordId). +// of id-attribute specs registered for it (see PyXmlSec_LxmlShadowRegisterId). static PyObject* PyXmlSec_LxmlShadowIdRegistry; int PyXmlSec_LxmlShadowIsActive(void) { @@ -880,6 +880,250 @@ static void PyXmlSec_LxmlShadowPruneIdRegistry(void) { Py_DECREF(dead); } +// The value of `element`'s `name` attribute (in `ns` when given) as a str, a +// new reference; Py_None when the element carries no such attribute. Without +// a namespace the lookup goes by *local* name, whatever namespace the +// attribute is in, because that is how the fast path matches: both +// xmlHasProp() and xmlSecAddIDs() compare names only, where lxml's +// element.get(name) would find an unqualified attribute alone. +static PyObject* PyXmlSec_LxmlAttrValue(PyXmlSec_LxmlElementPtr element, const char* name, const char* ns) { + PyObject* items; + PyObject* seq; + PyObject* found = NULL; + Py_ssize_t i; + Py_ssize_t n; + + if (ns != NULL) { + // lxml spells a namespaced attribute "{href}local". + PyObject* key = PyUnicode_FromFormat("{%s}%s", ns, name); + if (key == NULL) { + return NULL; + } + found = PyObject_CallMethod((PyObject*)element, "get", "O", key); + Py_DECREF(key); + return found; + } + + items = PyObject_CallMethod((PyObject*)element, "items", NULL); + if (items == NULL) { + return NULL; + } + seq = PySequence_Fast(items, "unexpected attributes."); + Py_DECREF(items); + if (seq == NULL) { + return NULL; + } + + n = PySequence_Fast_GET_SIZE(seq); + for (i = 0; i < n && found == NULL; ++i) { + PyObject* item = PySequence_Fast_GET_ITEM(seq, i); // borrowed (name, value) + PyObject* key; + const char* local; + const char* end; + + if (!PyTuple_Check(item) || PyTuple_GET_SIZE(item) != 2) { + PyErr_SetString(PyXmlSec_InternalError, "unexpected attribute."); + break; + } + key = PyTuple_GET_ITEM(item, 0); // borrowed + if (!PyUnicode_Check(key) || (local = PyUnicode_AsUTF8(key)) == NULL) { + if (!PyErr_Occurred()) { + PyErr_SetString(PyXmlSec_InternalError, "unexpected attribute name."); + } + break; + } + if (local[0] == '{' && (end = strchr(local, '}')) != NULL) { + local = end + 1; + } + if (strcmp(local, name) == 0) { + found = PyTuple_GET_ITEM(item, 1); // borrowed + Py_INCREF(found); + } + } + Py_DECREF(seq); + if (found == NULL && !PyErr_Occurred()) { + Py_INCREF(Py_None); + found = Py_None; + } + return found; +} + +// node.xpath(expr, n=name, v=value) — `name` is optional. The id lookups +// below need XPath variables rather than an interpolated expression, so that +// an id value never gets read as expression syntax. +static PyObject* PyXmlSec_LxmlXPath(PyObject* node, const char* expr, PyObject* name, PyObject* value) { + PyObject* method; + PyObject* args = NULL; + PyObject* kwargs = NULL; + PyObject* rv = NULL; + + method = PyObject_GetAttrString(node, "xpath"); + if (method == NULL) { + return NULL; + } + args = Py_BuildValue("(s)", expr); + kwargs = PyDict_New(); + if (args == NULL || kwargs == NULL || PyDict_SetItemString(kwargs, "v", value) < 0 + || (name != NULL && PyDict_SetItemString(kwargs, "n", name) < 0)) { + goto DONE; + } + rv = PyObject_Call(method, args, kwargs); + +DONE: + Py_DECREF(method); + Py_XDECREF(args); + Py_XDECREF(kwargs); + return rv; +} + +// Non-zero when `value` is registered as an XML ID in lxml's own document — +// a DTD-declared id attribute, or an xml:id, both of which lxml's libxml2 +// entered into the document's id hash when it parsed the tree — for an +// element other than `element`. That hash belongs to lxml's library and +// cannot be read from here; XPath's id() is the one door into it that takes +// and returns nothing but strings and elements. +// +// id() reads its argument as a whitespace-separated list of ids, so a value +// carrying whitespace would probe the wrong strings entirely: such a value +// is left to the registry check below, which compares whole values. +static int PyXmlSec_LxmlShadowIdIsDeclared(PyXmlSec_LxmlElementPtr element, PyObject* value) { + PyObject* matches; + Py_ssize_t i; + Py_ssize_t n; + const char* text; + Py_ssize_t size; + int taken = 0; + + text = PyUnicode_AsUTF8AndSize(value, &size); + if (text == NULL) { + return -1; + } + if (size == 0 || strcspn(text, " \t\r\n") != (size_t)size) { + return 0; + } + + matches = PyXmlSec_LxmlXPath((PyObject*)element, "id($v)", NULL, value); + if (matches == NULL) { + return -1; + } + n = PySequence_Size(matches); + for (i = 0; i < n && !taken; ++i) { + PyObject* match = PySequence_GetItem(matches, i); + if (match == NULL) { + Py_DECREF(matches); + return -1; + } + // lxml hands out one proxy per node, so identity is node identity. + taken = match != (PyObject*)element; + Py_DECREF(match); + } + Py_DECREF(matches); + return n < 0 ? -1 : taken; +} + +// Non-zero when `value` is already registered as an XML ID for anything but +// `element`'s own `name` attribute — what the fast path finds when it tests +// `xmlGetID(doc, value) != attr` and raises "duplicated id.". Under the +// shadow the registrations live in two places: lxml's document (above), and +// this registry, holding what earlier register_id/add_ids calls recorded and +// every whole-document Begin replays. A registration for the very attribute +// being registered is not a collision — the fast path's `tmpAttr == attr` +// leaves the document alone and returns. +static int PyXmlSec_LxmlShadowIdIsTaken(PyXmlSec_LxmlElementPtr element, const char* name, PyObject* value) { + PyObject* key; + PyObject* entry; + PyObject* nodes; + PyObject* specs; + Py_ssize_t i; + int taken; + + taken = PyXmlSec_LxmlShadowIdIsDeclared(element, value); + if (taken != 0) { + return taken; + } + + key = PyLong_FromVoidPtr((void*)element->_doc); + if (key == NULL) { + return -1; + } + entry = PyDict_GetItem(PyXmlSec_LxmlShadowIdRegistry, key); // borrowed + Py_DECREF(key); + if (entry == NULL) { + return 0; + } + + nodes = PyTuple_GET_ITEM(entry, PYXMLSEC_ID_ENTRY_NODES); + specs = PyTuple_GET_ITEM(entry, PYXMLSEC_ID_ENTRY_SPECS); + for (i = 0; i < PyList_GET_SIZE(specs); ++i) { + PyObject* spec = PyList_GET_ITEM(specs, i); // (name, ns, node index, subtree) + PyObject* node = PyList_GET_ITEM(nodes, PyLong_AsSsize_t(PyTuple_GET_ITEM(spec, 2))); + PyObject* spec_ns = PyTuple_GET_ITEM(spec, 1); + const char* spec_name = PyUnicode_AsUTF8(PyTuple_GET_ITEM(spec, 0)); + int subtree = PyObject_IsTrue(PyTuple_GET_ITEM(spec, 3)); + int mine; + + if (spec_name == NULL || subtree < 0) { + return -1; + } + // The same attribute of the same element: registering it again is + // the no-op the fast path performs, whoever recorded it first. The + // names alone decide it, as xmlHasProp's own matching does. + mine = strcmp(spec_name, name) == 0; + // A vacated slot, or an element adopted into another tree: neither + // stands for anything in this document, and the replay skips both. + if (node == Py_None || ((PyXmlSec_LxmlElementPtr)node)->_doc != element->_doc) { + continue; + } + if (subtree) { + // add_ids: the spec claims the value for whichever element of the + // subtree carries it, xmlSecAddIDs matching by name alone. + PyObject* matches = PyXmlSec_LxmlXPath(node, "descendant-or-self::*[@*[local-name()=$n]=$v]", + PyTuple_GET_ITEM(spec, 0), value); + Py_ssize_t j; + Py_ssize_t n; + if (matches == NULL) { + return -1; + } + n = PySequence_Size(matches); + for (j = 0; j < n && !taken; ++j) { + PyObject* match = PySequence_GetItem(matches, j); + if (match == NULL) { + Py_DECREF(matches); + return -1; + } + taken = !(mine && match == (PyObject*)element); + Py_DECREF(match); + } + Py_DECREF(matches); + if (n < 0) { + return -1; + } + } else { + const char* spec_href = spec_ns == Py_None ? NULL : PyUnicode_AsUTF8(spec_ns); + PyObject* other; + int same; + + if (spec_href == NULL && spec_ns != Py_None) { + return -1; + } + other = PyXmlSec_LxmlAttrValue((PyXmlSec_LxmlElementPtr)node, spec_name, spec_href); + if (other == NULL) { + return -1; + } + same = PyObject_RichCompareBool(other, value, Py_EQ); + Py_DECREF(other); + if (same < 0) { + return -1; + } + taken = same && !(mine && node == (PyObject*)element); + } + if (taken) { + return 1; + } + } + return 0; +} + int PyXmlSec_LxmlShadowRecordIds(PyXmlSec_LxmlElementPtr element, PyObject* names, const char* ns, int subtree) { PyObject* key = NULL; PyObject* created = NULL; @@ -973,14 +1217,40 @@ int PyXmlSec_LxmlShadowRecordIds(PyXmlSec_LxmlElementPtr element, PyObject* name return result; } -int PyXmlSec_LxmlShadowRecordId(PyXmlSec_LxmlElementPtr element, const char* name, const char* ns, int subtree) { - PyObject* names = Py_BuildValue("[s]", name); +int PyXmlSec_LxmlShadowRegisterId(PyXmlSec_LxmlElementPtr element, const char* name, const char* ns) { + PyObject* value = PyXmlSec_LxmlAttrValue(element, name, ns); + PyObject* names; + int taken; int rv; + if (value == NULL) { + return -1; + } + if (value == Py_None) { + Py_DECREF(value); + PyErr_SetString(PyXmlSec_Error, "missing attribute."); + return -1; + } + taken = PyXmlSec_LxmlShadowIdIsTaken(element, name, value); + Py_DECREF(value); + if (taken < 0) { + return -1; + } + if (taken) { + // The requested attribute cannot win the lookup, so the registration + // is refused rather than recorded and silently skipped at replay: + // the caller would otherwise never learn that its `#id` reference + // resolves to the element that claimed the value first. + PyErr_SetString(PyXmlSec_Error, "duplicated id."); + return -1; + } + + // Scope 0: this node alone, exactly what the fast path registers. + names = Py_BuildValue("[s]", name); if (names == NULL) { return -1; } - rv = PyXmlSec_LxmlShadowRecordIds(element, names, ns, subtree); + rv = PyXmlSec_LxmlShadowRecordIds(element, names, ns, 0); Py_DECREF(names); return rv; } diff --git a/src/lxml.h b/src/lxml.h index 2ba40094..9448db5f 100644 --- a/src/lxml.h +++ b/src/lxml.h @@ -101,10 +101,10 @@ int PyXmlSec_LxmlShadowBegin(PyXmlSec_LxmlShadow* shadow, PyXmlSec_LxmlElementPt // subtree (sign/verify, encrypt/decrypt, find_parent). Serializes // element.getroottree() — comments/PIs outside the root and the internal DTD // subset survive — and replays the IDs registered for the document -// (RecordId) onto the copy so that #id references resolve. `*target` -// receives the copy's counterpart of `element` (the live node itself on the -// fast path); `shadow.root` / `shadow.element` become the copy root / the -// live root, which is what the End functions map paths between. +// (RegisterId/RecordIds) onto the copy so that #id references resolve. +// `*target` receives the copy's counterpart of `element` (the live node +// itself on the fast path); `shadow.root` / `shadow.element` become the copy +// root / the live root, which is what the End functions map paths between. int PyXmlSec_LxmlShadowBeginDoc(PyXmlSec_LxmlShadow* shadow, PyXmlSec_LxmlElementPtr element, xmlNodePtr* target); // Create shape (template.create, encrypted_data_create): the call only needs @@ -155,7 +155,13 @@ xmlNodePtr PyXmlSec_LxmlShadowImportElement(PyXmlSec_LxmlShadow* shadow, PyXmlSe // registers the one node. Registering every matching attribute of the // document instead would let an unrelated element with the same id value win // the lookup and steer a `#id` reference away from the registered one. -int PyXmlSec_LxmlShadowRecordId(PyXmlSec_LxmlElementPtr element, const char* name, const char* ns, int subtree); +// +// This is the whole of register_id under the shadow: it validates what the +// fast path validates — "missing attribute." for an absent one, "duplicated +// id." when the value is already registered for another attribute — and +// records the spec for the node alone. Returns 0, or -1 with the exception +// the fast path would have raised. +int PyXmlSec_LxmlShadowRegisterId(PyXmlSec_LxmlElementPtr element, const char* name, const char* ns); // The same for a whole list of attribute names (`names`, a list of str), for // add_ids: either every name is recorded or, if anything fails, none is — diff --git a/tests/test_ds.py b/tests/test_ds.py index f1fb76b3..b623584e 100644 --- a/tests/test_ds.py +++ b/tests/test_ds.py @@ -65,6 +65,40 @@ def test_register_id_matches_namespaced_attribute_by_local_name(self): sign.set('{http://www.example.org/ns}Id', 'sig-1') ctx.register_id(sign, 'Id') + def test_register_id_rejects_an_id_value_already_registered(self): + """A value another registration claimed cannot be registered again: only one attribute can win the id lookup.""" + ctx = xmlsec.SignatureContext() + root = etree.fromstring(b'') + ctx.register_id(root[0], 'ID') + with self.assertRaisesRegex(xmlsec.Error, 'duplicated id.'): + ctx.register_id(root[1], 'Id') + + def test_register_id_rejects_an_id_value_the_document_declares(self): + """An xml:id the document was parsed with already claims the value, so another attribute for it is refused.""" + root = etree.fromstring(b'') + with self.assertRaisesRegex(xmlsec.Error, 'duplicated id.'): + xmlsec.SignatureContext().register_id(root[1], 'ID') + + def test_register_id_rejects_an_id_value_add_ids_claimed(self): + """Registrations made over a subtree by add_ids claim their values too.""" + root = etree.fromstring(b'') + xmlsec.tree.add_ids(root[0], ['ID']) + with self.assertRaisesRegex(xmlsec.Error, 'duplicated id.'): + xmlsec.SignatureContext().register_id(root[1], 'Id') + + def test_register_id_accepts_the_same_attribute_twice(self): + """Re-registering an attribute is the no-op the id lookup already resolves; only another claim is a duplicate.""" + ctx = xmlsec.SignatureContext() + root = etree.fromstring(b'') + ctx.register_id(root[0], 'ID') + ctx.register_id(root[0], 'ID') + + def test_register_id_accepts_an_attribute_add_ids_registered(self): + """add_ids already registered this very attribute, which the fast path's `tmpAttr == attr` accepts.""" + root = etree.fromstring(b'') + xmlsec.tree.add_ids(root[0], ['ID']) + xmlsec.SignatureContext().register_id(root[0][0], 'ID') + def test_register_id_with_namespace_without_attribute(self): ctx = xmlsec.SignatureContext() root = self.load_xml('sign_template.xml') From 3a50b0422a5ac24208810cc87ccf5925fc9c2edf Mon Sep 17 00:00:00 2001 From: Amin Solhizadeh Date: Fri, 4 Sep 2026 21:04:55 +0200 Subject: [PATCH 18/22] Bound the copy's walks, load an external DTD subset (#356) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two findings from the review of the shadow path. The walks over the private copy — CountNodes, TagNodes, CollectSites — recurse once per level of nesting and were bounded only by what the parse that produced the copy accepted. libxml2 2.14 and later cap a parse at 2048 levels even under XML_PARSE_HUGE, so on those the copy can never be deeper; older libxml2 lifts its cap entirely under HUGE, and lxml can build and serialize a tree far deeper than it would parse, so a document built element by element reached the walks with no ceiling at all and overran the C stack (checked against 2.9.13: SIGSEGV at 400000 levels on the main thread, at 20000 on a 512 KB thread stack). The walks now carry a depth and refuse a document nested deeper than 2048 levels, the same ceiling modern libxml2 enforces, so the failure is clean and identical across libxml2 versions. CollectSites is checked as well as Mark: the call being reflected can graft subtrees of its own into the copy. The copy was parsed without loading any external DTD subset, so the ID attributes such a DTD types were untyped in it and a #id reference over one failed to resolve where the raw path signs — the serialization keeps only the DOCTYPE reference to the declarations. The copy is now parsed with XML_PARSE_DTDLOAD exactly when lxml itself loaded an external subset (docinfo.externalDTD), fetching the same local file the document names, resolved against the base URI the copy already carries, with the network still off. Never DTDATTR: libxml2 fills in defaulted attributes only under that flag, and the copy has to stay what lxml serialized. Suite: 314 passed / 6 skipped under the mismatch, 326 on the matched wheel plain and with PYXMLSEC_FORCE_SHADOW=1; the external-DTD test fails on the pre-fix build. Co-Authored-By: Claude Opus 5 --- developer.md | 17 +++++++- src/lxml.c | 98 ++++++++++++++++++++++++++++++++++-------- tests/data/id_attr.dtd | 3 ++ tests/test_ds.py | 21 +++++++++ tests/test_tree.py | 13 ++++++ 5 files changed, 131 insertions(+), 21 deletions(-) create mode 100644 tests/data/id_attr.dtd diff --git a/developer.md b/developer.md index dd4601a5..201b6379 100644 --- a/developer.md +++ b/developer.md @@ -270,8 +270,21 @@ All invisible to the documented API: leave it), so the resulting tree matches the raw path's move; - signature/encryption contexts keep no live result nodes after the call (they never usefully did); -- documents nested deeper than 256 levels (only possible with `huge_tree`) - are refused with an internal error; +- a mutation site deeper than 256 levels is refused with an internal error, + and so is a document nested deeper than 2048 levels — the ceiling libxml2 + 2.14 and later put on a parse even under `XML_PARSE_HUGE`. The copy's own + walks enforce that ceiling for themselves, since an older libxml2 lifts its + cap entirely under `HUGE` (2.9.13 parses a 200000-level document) and lxml + can hand over a tree that was never parsed on its side at all; +- when the source document loaded an external DTD subset (`load_dtd=True`), + the copy is parsed with `XML_PARSE_DTDLOAD`, so that the IDs the DTD + declares type the copy's attributes as well and `#id` references over them + resolve. What is fetched is the local file the document's own DOCTYPE + names, resolved against the same base URI, with the network still off and + attribute defaulting (`XML_PARSE_DTDATTR`) still off — the copy must stay + what lxml serialized. A DTD lxml obtained through a Python resolver of its + own is invisible to that parse, so the ids it declares are not carried + across; - a subtree of a document that declares entities in its internal subset (`resolve_entities=False`) is copied by copying the whole document and cutting it back to the element, since the subtree's `&name;` references diff --git a/src/lxml.c b/src/lxml.c index af77dea3..ac617eee 100644 --- a/src/lxml.c +++ b/src/lxml.c @@ -253,7 +253,15 @@ static PyObject* PyXmlSec_LxmlElementFromBytes(PyObject* data) { // Parses `bytes` into a private document with a root element, or returns // NULL with an exception set (`error` for parse failures). `url` is the base // URI the copy gets (see PyXmlSec_LxmlDocumentUrl); NULL when there is none. -static xmlDocPtr PyXmlSec_LxmlShadowParse(PyObject* bytes, const char* url, const char* error) { +// `external_dtd` says the source document loaded an external subset: the dump +// keeps only the DOCTYPE reference to it, and its declarations are what typed +// the document's ID attributes, so the copy has to load it too or resolve no +// #id reference the DTD declared. DTDLOAD alone, never DTDATTR — libxml2 +// fills in defaulted attributes only under the latter, and the copy must stay +// what lxml serialized. NONET keeps the fetch to the local file the source +// document already named, resolved against the same base URI. +static xmlDocPtr PyXmlSec_LxmlShadowParse(PyObject* bytes, const char* url, int external_dtd, const char* error) { + int options = PYXMLSEC_SHADOW_PARSE_OPTIONS | (external_dtd ? XML_PARSE_DTDLOAD : 0); char* data = NULL; Py_ssize_t size = 0; xmlDocPtr doc; @@ -261,7 +269,7 @@ static xmlDocPtr PyXmlSec_LxmlShadowParse(PyObject* bytes, const char* url, cons if (PyBytes_AsStringAndSize(bytes, &data, &size) < 0) { return NULL; } - doc = xmlReadMemory(data, (int)size, url, NULL, PYXMLSEC_SHADOW_PARSE_OPTIONS); + doc = xmlReadMemory(data, (int)size, url, NULL, options); if (doc == NULL || xmlDocGetRootElement(doc) == NULL) { if (doc != NULL) { xmlFreeDoc(doc); @@ -308,23 +316,36 @@ static int PyXmlSec_LxmlDocumentUrl(PyObject* tree, PyObject** holder, const cha return 0; } -// Whether `tree` carries an internal DTD subset — the only place a document -// lxml parsed can declare the entities its `&name;` references name, and so -// what decides whether a subtree may be serialized on its own (see Begin). -static int PyXmlSec_LxmlDocumentHasInternalDtd(PyObject* tree, int* dtd) { +// The DTD subsets of the document `tree` belongs to. The internal one (lxml +// reports one for any DOCTYPE declaration) is where a document lxml parsed +// declares the entities its `&name;` references name, and so decides whether +// a subtree may be serialized on its own (see Begin). The external one is +// reported only when lxml actually loaded it — the caller asked for it with +// load_dtd — and the copy then has to load it as well, since its declarations +// are what typed the document's ID attributes (see PyXmlSec_LxmlShadowParse). +// Returns 0, or -1 with an exception set. +static int PyXmlSec_LxmlDocumentSubsets(PyObject* tree, int* internal, int* external) { PyObject* info = PyObject_GetAttrString(tree, "docinfo"); PyObject* value; - *dtd = 0; + *internal = 0; + *external = 0; if (info == NULL) { return -1; } value = PyObject_GetAttrString(info, "internalDTD"); + if (value == NULL) { + Py_DECREF(info); + return -1; + } + *internal = value != Py_None; + Py_DECREF(value); + value = PyObject_GetAttrString(info, "externalDTD"); Py_DECREF(info); if (value == NULL) { return -1; } - *dtd = value != Py_None; + *external = value != Py_None; Py_DECREF(value); return 0; } @@ -348,10 +369,31 @@ static int PyXmlSec_LxmlDocumentHasInternalDtd(PyObject* tree, int* dtd) { #define PYXMLSEC_SHADOW_TAG(shadow, n) \ (PYXMLSEC_SHADOW_TAGGED(shadow, n) ? (PyXmlSec_LxmlShadowTag*)(n)->_private : NULL) -static int PyXmlSec_LxmlShadowCountNodes(xmlNodePtr node) { +// The walks over the copy's own structure — this one, TagNodes and +// CollectSites — recurse once per level of nesting, so they need a ceiling of +// their own. libxml2 2.14 and later refuse a parse deeper than 2048 levels +// even under XML_PARSE_HUGE, so a copy parsed by one of those can never reach +// it; older libxml2 lifts its cap entirely under HUGE (checked: 2.9.13 parses +// a 200000-level document, and the unguarded walk then overruns the C stack), +// and lxml hands over trees that were never parsed on its side at all — one +// built element by element has no limit whatsoever. Anything deeper fails +// cleanly here rather than crashing; sites deeper than +// PYXMLSEC_SHADOW_MAX_DEPTH are refused later anyway. +#define PYXMLSEC_SHADOW_MAX_NESTING 2048 + +// Number of nodes in the list at `node` and under it, or -1 when the tree is +// nested deeper than the walks go. +static int PyXmlSec_LxmlShadowCountNodes(xmlNodePtr node, int depth) { int count = 0; + if (depth >= PYXMLSEC_SHADOW_MAX_NESTING) { + return -1; + } for (; node != NULL; node = node->next) { - count += 1 + PyXmlSec_LxmlShadowCountNodes(node->children); + int children = PyXmlSec_LxmlShadowCountNodes(node->children, depth + 1); + if (children < 0) { + return -1; + } + count += 1 + children; } return count; } @@ -372,6 +414,8 @@ static unsigned long long PyXmlSec_LxmlShadowContentPrint(xmlNodePtr node) { } // Hands out `shadow->tags` in document order; returns the next free slot. +// Runs only after CountNodes succeeded, so the tree it walks is within +// PYXMLSEC_SHADOW_MAX_NESTING and this recursion is bounded with it. static int PyXmlSec_LxmlShadowTagNodes(PyXmlSec_LxmlShadow* shadow, xmlNodePtr node, int next) { for (; node != NULL; node = node->next) { PyXmlSec_LxmlShadowTag* tag = &shadow->tags[next++]; @@ -390,8 +434,12 @@ static int PyXmlSec_LxmlShadowTagNodes(PyXmlSec_LxmlShadow* shadow, xmlNodePtr n // Tags every node of the freshly parsed copy. Returns 0, or -1 with an // exception set. static int PyXmlSec_LxmlShadowMark(PyXmlSec_LxmlShadow* shadow) { - int count = PyXmlSec_LxmlShadowCountNodes(shadow->doc->children); + int count = PyXmlSec_LxmlShadowCountNodes(shadow->doc->children, 0); + if (count < 0) { + PyErr_SetString(PyXmlSec_InternalError, "the document is nested too deeply."); + return -1; + } shadow->tags = (PyXmlSec_LxmlShadowTag*)PyMem_Malloc((count > 0 ? count : 1) * sizeof(*shadow->tags)); if (shadow->tags == NULL) { PyErr_NoMemory(); @@ -646,6 +694,7 @@ int PyXmlSec_LxmlShadowBegin(PyXmlSec_LxmlShadow* shadow, PyXmlSec_LxmlElementPt int path[PYXMLSEC_SHADOW_MAX_DEPTH]; int depth = 0; int dtd = 0; + int extdtd = 0; shadow->element = element; shadow->owned = NULL; @@ -667,7 +716,7 @@ int PyXmlSec_LxmlShadowBegin(PyXmlSec_LxmlShadow* shadow, PyXmlSec_LxmlElementPt goto ON_FAIL; } if (PyXmlSec_LxmlDocumentUrl(tree, &url_holder, &url) < 0 - || PyXmlSec_LxmlDocumentHasInternalDtd(tree, &dtd) < 0) { + || PyXmlSec_LxmlDocumentSubsets(tree, &dtd, &extdtd) < 0) { goto ON_FAIL; } @@ -690,7 +739,7 @@ int PyXmlSec_LxmlShadowBegin(PyXmlSec_LxmlShadow* shadow, PyXmlSec_LxmlElementPt if (bytes == NULL) { goto ON_FAIL; } - shadow->doc = PyXmlSec_LxmlShadowParse(bytes, url, "cannot make a private copy of the element."); + shadow->doc = PyXmlSec_LxmlShadowParse(bytes, url, extdtd, "cannot make a private copy of the element."); Py_CLEAR(bytes); Py_CLEAR(url_holder); if (shadow->doc == NULL) { @@ -1361,6 +1410,8 @@ int PyXmlSec_LxmlShadowBeginDoc(PyXmlSec_LxmlShadow* shadow, PyXmlSec_LxmlElemen const char* url = NULL; int path[PYXMLSEC_SHADOW_MAX_DEPTH]; int depth; + int dtd = 0; + int extdtd = 0; shadow->element = element; shadow->owned = NULL; @@ -1390,12 +1441,15 @@ int PyXmlSec_LxmlShadowBeginDoc(PyXmlSec_LxmlShadow* shadow, PyXmlSec_LxmlElemen if (tree == NULL) { goto ON_FAIL; } + // The whole tree is dumped either way here, so only the external subset + // matters: it has to be loaded for the copy to know the IDs it declares. bytes = PyObject_CallFunctionObjArgs(PyXmlSec_LxmlEtreeToString, tree, NULL); - if (bytes == NULL || PyXmlSec_LxmlDocumentUrl(tree, &url_holder, &url) < 0) { + if (bytes == NULL || PyXmlSec_LxmlDocumentUrl(tree, &url_holder, &url) < 0 + || PyXmlSec_LxmlDocumentSubsets(tree, &dtd, &extdtd) < 0) { goto ON_FAIL; } Py_CLEAR(tree); - shadow->doc = PyXmlSec_LxmlShadowParse(bytes, url, "cannot make a private copy of the document."); + shadow->doc = PyXmlSec_LxmlShadowParse(bytes, url, extdtd, "cannot make a private copy of the document."); Py_CLEAR(bytes); Py_CLEAR(url_holder); if (shadow->doc == NULL) { @@ -1513,13 +1567,19 @@ static int PyXmlSec_LxmlShadowSiteAppend(PyXmlSec_LxmlShadowSiteList* list, xmlN // for a parent that no longer holds the children it was tagged with, whose // text slots are the only trace the removal left, or one whose own text // survived the call but with different content. -static int PyXmlSec_LxmlShadowCollectSites(PyXmlSec_LxmlShadowSiteList* list, xmlNodePtr parent) { +static int PyXmlSec_LxmlShadowCollectSites(PyXmlSec_LxmlShadowSiteList* list, xmlNodePtr parent, int depth) { PyXmlSec_LxmlShadowTag* tag = PYXMLSEC_SHADOW_TAG(list->shadow, parent); xmlNodePtr n; int fresh = 0; int tagged = 0; int rewritten = 0; + // Checked here too, and not only in Mark: the call being reflected can + // have grafted whole subtrees of its own into the copy since. + if (depth >= PYXMLSEC_SHADOW_MAX_NESTING) { + PyErr_SetString(PyXmlSec_InternalError, "the document is nested too deeply."); + return -1; + } for (n = parent->children; n != NULL; n = n->next) { PyXmlSec_LxmlShadowTag* child_tag = PYXMLSEC_SHADOW_TAG(list->shadow, n); if (child_tag != NULL) { @@ -1527,7 +1587,7 @@ static int PyXmlSec_LxmlShadowCollectSites(PyXmlSec_LxmlShadowSiteList* list, xm if (child_tag->content != PyXmlSec_LxmlShadowContentPrint(n)) { rewritten = 1; } - if (n->type == XML_ELEMENT_NODE && PyXmlSec_LxmlShadowCollectSites(list, n) < 0) { + if (n->type == XML_ELEMENT_NODE && PyXmlSec_LxmlShadowCollectSites(list, n, depth + 1) < 0) { return -1; } continue; @@ -1788,7 +1848,7 @@ static int PyXmlSec_LxmlShadowReflectSites(PyXmlSec_LxmlShadow* shadow) { } goto DONE; } - if (PyXmlSec_LxmlShadowCollectSites(&list, list.top) < 0) { + if (PyXmlSec_LxmlShadowCollectSites(&list, list.top, 0) < 0) { goto DONE; } if (list.count == 0) { @@ -2049,7 +2109,7 @@ xmlNodePtr PyXmlSec_LxmlShadowImportElement(PyXmlSec_LxmlShadow* shadow, PyXmlSe if (bytes == NULL) { return NULL; } - tdoc = PyXmlSec_LxmlShadowParse(bytes, NULL, "cannot make a private copy of the element."); + tdoc = PyXmlSec_LxmlShadowParse(bytes, NULL, 0, "cannot make a private copy of the element."); Py_DECREF(bytes); if (tdoc == NULL) { return NULL; diff --git a/tests/data/id_attr.dtd b/tests/data/id_attr.dtd new file mode 100644 index 00000000..083b3d9f --- /dev/null +++ b/tests/data/id_attr.dtd @@ -0,0 +1,3 @@ + + diff --git a/tests/test_ds.py b/tests/test_ds.py index b623584e..308f0cbe 100644 --- a/tests/test_ds.py +++ b/tests/test_ds.py @@ -256,6 +256,27 @@ def test_sign_and_verify_with_registered_id(self): verify_ctx.key = xmlsec.Key.from_file(self.path('rsapub.pem'), format=consts.KeyDataFormatPem) verify_ctx.verify(sign) + # A document whose id attribute is typed by an external DTD subset — the + # declarations live in a file the DOCTYPE names, and only a parse that + # loads it knows "#ext" resolves to the Node (issue #356). + EXTERNAL_DTD_XML = b'\nsigned\n' + + def test_sign_and_verify_with_an_id_an_external_dtd_declares(self): + """Should resolve a #id reference whose id attribute an external subset the caller loaded declares.""" + root = etree.fromstring(self.EXTERNAL_DTD_XML, etree.XMLParser(load_dtd=True), base_url=self.path('doc.xml')) + sign = xmlsec.template.create(root, consts.TransformExclC14N, consts.TransformRsaSha1, ns='ds') + root.append(sign) + ref = xmlsec.template.add_reference(sign, consts.TransformSha1, uri='#ext') + xmlsec.template.add_transform(ref, consts.TransformExclC14N) + + ctx = xmlsec.SignatureContext() + ctx.key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem) + ctx.sign(sign) + + verify_ctx = xmlsec.SignatureContext() + verify_ctx.key = xmlsec.Key.from_file(self.path('rsapub.pem'), format=consts.KeyDataFormatPem) + verify_ctx.verify(sign) + # A document where an unregistered element carries the same id value as the # registered one, and comes first. Only the registered element may answer # the "#dup" reference (issue #356). diff --git a/tests/test_tree.py b/tests/test_tree.py index cabc30fb..fd9f6777 100644 --- a/tests/test_tree.py +++ b/tests/test_tree.py @@ -73,6 +73,19 @@ def test_find_parent_keeps_entity_references(self): self.assertIs(root[0], xmlsec.tree.find_parent(si, consts.NodeSignature)) self.assertEqual(before, etree.tostring(root.getroottree())) + def test_deeply_nested_document_fails_cleanly(self): + """A tree nested deeper than the private copy walks must be refused, never crash (issue #356).""" + root = etree.Element('Root') + deepest = root + for _ in range(3000): # past the 2048 levels libxml2 2.14+ parses, and the walks' own ceiling + deepest = etree.SubElement(deepest, 'a') + etree.SubElement(deepest, f'{{{consts.DSigNs}}}Signature') + try: + found = xmlsec.tree.find_node(root, consts.NodeSignature, consts.DSigNs) + except xmlsec.InternalError: + return # the shadow path refuses a document this deep + self.assertEqual(consts.NodeSignature, found.tag.partition('}')[2]) + def test_add_ids_bad_args(self): with self.assertRaises(TypeError): xmlsec.tree.add_ids('', []) From 48e8d0d30b7e56aa714116cdad66bd6c0dbc8eae Mon Sep 17 00:00:00 2001 From: Amin Solhizadeh Date: Fri, 4 Sep 2026 21:39:08 +0200 Subject: [PATCH 19/22] Tell two same-valued attributes of one element apart (#356) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit register_id's shadow body decided "the attribute being registered is already the declared one" from the *element* XPath id() returned. An element can carry the value twice: answers N whichever attribute is asked about, so register_id(N, 'ID') recorded a spec where the fast path registers ID, finds xml:id holding the value and raises "duplicated id.". The registry half had the same shape, by local name only: a spec for a:Id and a call for Id are two attributes, and the second cannot win the lookup either. Both halves now compare attributes. PyXmlSec_LxmlAttrValue became AttrFind, which also reports the lxml key of the attribute xmlHasProp/xmlHasNsProp would pick, and a registry spec is resolved through it on the node it was recorded for. For the declared half, a match on the element itself settles nothing when two of its attributes carry the value, and no lxml API names the declared attribute — id() names elements, and an ATTLIST without an ELEMENT leaves lxml's DTD objects empty. It is named instead by copying the document the way a whole-document shadow copies it (same base URL, same subsets) and reading that copy's own id hash. Only a value already declared for the element, on an element carrying it twice, pays for that copy. Nine collision shapes now answer identically on the raw path, the forced shadow and the mismatch build: xml:id beside ID, a DTD-declared ID beside a twin attribute (both directions), and namespaced/ unqualified registry pairs in both orders. Tests in tests/test_ds.py: the two "rejects" cases fail on the pre-fix build; the two "accepts" cases guard the no-op against an over-eager check. Validation: 318 passed / 6 skipped on the mismatch build, also at PYXMLSEC_TEST_ITERATIONS=50; 330 / 6 on the matched static wheel, plain and with PYXMLSEC_FORCE_SHADOW=1; a 10k loop of no-op plus refused registration over the DTD document with RSS 23.3 -> 24.7 MiB. Co-Authored-By: Claude Opus 5 --- developer.md | 17 ++- src/lxml.c | 276 ++++++++++++++++++++++++++++++++++++++--------- tests/test_ds.py | 27 +++++ 3 files changed, 269 insertions(+), 51 deletions(-) diff --git a/developer.md b/developer.md index 201b6379..1263d902 100644 --- a/developer.md +++ b/developer.md @@ -226,7 +226,22 @@ under the shadow. What lxml's own parse declared (a DTD id attribute, an `xml:id`) is read back through XPath's `id()`, the one door into lxml's id hash that passes nothing but strings and elements; what earlier `register_id`/`add_ids` calls claimed is read from the registry, a subtree -spec through one XPath over its scope. Deferring the check to the replay +spec through one XPath over its scope. + +Both halves compare *attributes*, not elements: `` +answers `N` to `id('dup')` whichever attribute is asked about, while the fast +path registers `ID`, finds `xml:id` holding the value and raises. A registry +spec is therefore resolved to the attribute its `xmlHasProp`/`xmlHasNsProp` +would pick, and a match on the element itself only settles the declared half +when a single attribute of the element carries the value — otherwise the +declared attribute has to be named outright, which no lxml API does (`id()` +names elements, and an `ATTLIST` without an `ELEMENT` leaves lxml's DTD +objects empty). It is then named by copying the document the way a +whole-document shadow copies it — same base URL, same subsets — and reading +that copy's own id hash, a copy the registration is refused or recorded +against anyway. Only a value already declared for the element pays for it. + +Deferring the check to the replay instead would raise from the wrong call — a later `sign`, and then from every later call on that document — and would leave the caller believing a registration took that can never win the lookup. `add_ids` keeps its own diff --git a/src/lxml.c b/src/lxml.c index ac617eee..44a8eb5c 100644 --- a/src/lxml.c +++ b/src/lxml.c @@ -929,27 +929,42 @@ static void PyXmlSec_LxmlShadowPruneIdRegistry(void) { Py_DECREF(dead); } -// The value of `element`'s `name` attribute (in `ns` when given) as a str, a -// new reference; Py_None when the element carries no such attribute. Without -// a namespace the lookup goes by *local* name, whatever namespace the -// attribute is in, because that is how the fast path matches: both -// xmlHasProp() and xmlSecAddIDs() compare names only, where lxml's +// The attribute of `element` the fast path would pick for `name` (in `ns` +// when given): its value as a str, a new reference, and — when `key` is not +// NULL — the lxml key naming it ("{href}local", or "local" unqualified), +// also a new reference. Py_None (and *key left NULL) when the element +// carries no such attribute. Without a namespace the lookup goes by *local* +// name, whatever namespace the attribute is in, and takes the first such +// attribute in document order, because that is how the fast path matches: +// both xmlHasProp() and xmlSecAddIDs() compare names only, where lxml's // element.get(name) would find an unqualified attribute alone. -static PyObject* PyXmlSec_LxmlAttrValue(PyXmlSec_LxmlElementPtr element, const char* name, const char* ns) { +// +// The key is what tells two same-valued attributes of one element apart, so +// that "the very attribute being registered" stays a question about the +// attribute rather than about the element carrying it. +static PyObject* PyXmlSec_LxmlAttrFind(PyXmlSec_LxmlElementPtr element, const char* name, const char* ns, PyObject** key) { PyObject* items; PyObject* seq; PyObject* found = NULL; Py_ssize_t i; Py_ssize_t n; + if (key != NULL) { + *key = NULL; + } + if (ns != NULL) { // lxml spells a namespaced attribute "{href}local". - PyObject* key = PyUnicode_FromFormat("{%s}%s", ns, name); - if (key == NULL) { + PyObject* qname = PyUnicode_FromFormat("{%s}%s", ns, name); + if (qname == NULL) { return NULL; } - found = PyObject_CallMethod((PyObject*)element, "get", "O", key); - Py_DECREF(key); + found = PyObject_CallMethod((PyObject*)element, "get", "O", qname); + if (found != NULL && found != Py_None && key != NULL) { + *key = qname; + } else { + Py_DECREF(qname); + } return found; } @@ -966,7 +981,7 @@ static PyObject* PyXmlSec_LxmlAttrValue(PyXmlSec_LxmlElementPtr element, const c n = PySequence_Fast_GET_SIZE(seq); for (i = 0; i < n && found == NULL; ++i) { PyObject* item = PySequence_Fast_GET_ITEM(seq, i); // borrowed (name, value) - PyObject* key; + PyObject* qname; const char* local; const char* end; @@ -974,8 +989,8 @@ static PyObject* PyXmlSec_LxmlAttrValue(PyXmlSec_LxmlElementPtr element, const c PyErr_SetString(PyXmlSec_InternalError, "unexpected attribute."); break; } - key = PyTuple_GET_ITEM(item, 0); // borrowed - if (!PyUnicode_Check(key) || (local = PyUnicode_AsUTF8(key)) == NULL) { + qname = PyTuple_GET_ITEM(item, 0); // borrowed + if (!PyUnicode_Check(qname) || (local = PyUnicode_AsUTF8(qname)) == NULL) { if (!PyErr_Occurred()) { PyErr_SetString(PyXmlSec_InternalError, "unexpected attribute name."); } @@ -987,6 +1002,10 @@ static PyObject* PyXmlSec_LxmlAttrValue(PyXmlSec_LxmlElementPtr element, const c if (strcmp(local, name) == 0) { found = PyTuple_GET_ITEM(item, 1); // borrowed Py_INCREF(found); + if (key != NULL) { + *key = qname; + Py_INCREF(qname); + } } } Py_DECREF(seq); @@ -1025,22 +1044,117 @@ static PyObject* PyXmlSec_LxmlXPath(PyObject* node, const char* expr, PyObject* return rv; } +// The number of `element`'s attributes whose value is `value`, or -1. +static Py_ssize_t PyXmlSec_LxmlAttrsWithValue(PyXmlSec_LxmlElementPtr element, PyObject* value) { + PyObject* values = PyObject_CallMethod((PyObject*)element, "values", NULL); + PyObject* seq; + Py_ssize_t i; + Py_ssize_t n; + Py_ssize_t found = 0; + + if (values == NULL) { + return -1; + } + seq = PySequence_Fast(values, "unexpected attributes."); + Py_DECREF(values); + if (seq == NULL) { + return -1; + } + n = PySequence_Fast_GET_SIZE(seq); + for (i = 0; i < n; ++i) { + int same = PyObject_RichCompareBool(PySequence_Fast_GET_ITEM(seq, i), value, Py_EQ); + if (same < 0) { + Py_DECREF(seq); + return -1; + } + found += same; + } + Py_DECREF(seq); + return found; +} + +// The lxml key of the attribute lxml's parse declared as the XML ID for +// `value` — a new reference, Py_None when there is none. XPath's id() names +// the element carrying it but never the attribute, and the DTD that typed it +// may declare no element at all (an ATTLIST alone leaves lxml's DTD objects +// empty), so the only way to name the attribute is to ask a libxml2 we may +// read from: the document is copied the way a whole-document shadow copies +// it — same base URL, same subsets — and its own id hash consulted. +static PyObject* PyXmlSec_LxmlShadowDeclaredIdKey(PyXmlSec_LxmlElementPtr element, PyObject* value) { + PyObject* tree = NULL; + PyObject* bytes = NULL; + PyObject* url_holder = NULL; + PyObject* key = NULL; + const char* url = NULL; + const char* text; + xmlDocPtr doc = NULL; + xmlAttrPtr attr; + int dtd = 0; + int extdtd = 0; + + text = PyUnicode_AsUTF8(value); + if (text == NULL) { + goto DONE; + } + tree = PyObject_CallMethod((PyObject*)element, "getroottree", NULL); + if (tree == NULL) { + goto DONE; + } + bytes = PyObject_CallFunctionObjArgs(PyXmlSec_LxmlEtreeToString, tree, NULL); + if (bytes == NULL || PyXmlSec_LxmlDocumentUrl(tree, &url_holder, &url) < 0 + || PyXmlSec_LxmlDocumentSubsets(tree, &dtd, &extdtd) < 0) { + goto DONE; + } + doc = PyXmlSec_LxmlShadowParse(bytes, url, extdtd, "cannot make a private copy of the document."); + if (doc == NULL) { + goto DONE; + } + attr = xmlGetID(doc, XSTR(text)); + if (attr == NULL) { + Py_INCREF(Py_None); + key = Py_None; + } else if (attr->ns != NULL && attr->ns->href != NULL) { + key = PyUnicode_FromFormat("{%s}%s", (const char*)attr->ns->href, (const char*)attr->name); + } else { + key = PyUnicode_FromString((const char*)attr->name); + } + +DONE: + if (doc != NULL) { + xmlFreeDoc(doc); + } + Py_XDECREF(tree); + Py_XDECREF(bytes); + Py_XDECREF(url_holder); + return key; +} + // Non-zero when `value` is registered as an XML ID in lxml's own document — // a DTD-declared id attribute, or an xml:id, both of which lxml's libxml2 -// entered into the document's id hash when it parsed the tree — for an -// element other than `element`. That hash belongs to lxml's library and -// cannot be read from here; XPath's id() is the one door into it that takes -// and returns nothing but strings and elements. +// entered into the document's id hash when it parsed the tree — for anything +// but `element`'s own `key` attribute. That hash belongs to lxml's library +// and cannot be read from here; XPath's id() is the one door into it that +// takes and returns nothing but strings and elements. +// +// id() names an element, not an attribute, and an element can carry the same +// value twice: `` answers `N` whichever of the two +// is asked about, where the fast path registers `ID` and finds `xml:id` +// holding the value already. So a match on `element` itself only settles it +// when a single attribute of it carries the value — otherwise the declared +// attribute has to be named outright. // // id() reads its argument as a whitespace-separated list of ids, so a value // carrying whitespace would probe the wrong strings entirely: such a value // is left to the registry check below, which compares whole values. -static int PyXmlSec_LxmlShadowIdIsDeclared(PyXmlSec_LxmlElementPtr element, PyObject* value) { +static int PyXmlSec_LxmlShadowIdIsDeclared(PyXmlSec_LxmlElementPtr element, PyObject* key, PyObject* value) { PyObject* matches; + PyObject* declared; Py_ssize_t i; Py_ssize_t n; + Py_ssize_t carriers; const char* text; Py_ssize_t size; + int self = 0; int taken = 0; text = PyUnicode_AsUTF8AndSize(value, &size); @@ -1064,39 +1178,102 @@ static int PyXmlSec_LxmlShadowIdIsDeclared(PyXmlSec_LxmlElementPtr element, PyOb } // lxml hands out one proxy per node, so identity is node identity. taken = match != (PyObject*)element; + self = self || match == (PyObject*)element; Py_DECREF(match); } Py_DECREF(matches); - return n < 0 ? -1 : taken; + if (n < 0) { + return -1; + } + if (taken || !self) { + return taken; + } + + // The declared id is on this element. Only one attribute of it holds the + // value, and that attribute is the one being registered (it holds the + // value by construction), so this is the fast path's `tmpAttr == attr`. + carriers = PyXmlSec_LxmlAttrsWithValue(element, value); + if (carriers < 0) { + return -1; + } + if (carriers < 2) { + return 0; + } + + declared = PyXmlSec_LxmlShadowDeclaredIdKey(element, value); + if (declared == NULL) { + return -1; + } + // A copy that declares no id at all leaves the value unclaimed in the + // world the shadow runs in, which is the world the registration is for. + if (declared == Py_None) { + taken = 0; + } else { + int same = PyObject_RichCompareBool(declared, key, Py_EQ); + taken = same < 0 ? -1 : !same; + } + Py_DECREF(declared); + return taken; +} + +// Non-zero when a recorded spec — `name` (in `ns` when given) applied to +// `node` — claims `value` for an attribute other than `element`'s `key`, the +// one being registered. The spec is resolved to the attribute it actually +// names, the way the replay's xmlHasProp/xmlHasNsProp resolves it: two +// attributes of one element can carry the same value under different names, +// and only the attribute settles whether this is the fast path's harmless +// `tmpAttr == attr` or its "duplicated id.". +static int PyXmlSec_LxmlShadowSpecClaims(PyXmlSec_LxmlElementPtr node, const char* name, const char* ns, + PyXmlSec_LxmlElementPtr element, PyObject* key, PyObject* value) { + PyObject* other_key = NULL; + PyObject* other = PyXmlSec_LxmlAttrFind(node, name, ns, &other_key); + int same; + int mine; + + if (other == NULL) { + return -1; + } + same = PyObject_RichCompareBool(other, value, Py_EQ); + Py_DECREF(other); + if (same <= 0) { + Py_XDECREF(other_key); + return same; + } + mine = node == element && other_key != NULL && PyObject_RichCompareBool(other_key, key, Py_EQ); + Py_XDECREF(other_key); + if (PyErr_Occurred()) { + return -1; + } + return !mine; } // Non-zero when `value` is already registered as an XML ID for anything but -// `element`'s own `name` attribute — what the fast path finds when it tests +// `element`'s own `key` attribute — what the fast path finds when it tests // `xmlGetID(doc, value) != attr` and raises "duplicated id.". Under the // shadow the registrations live in two places: lxml's document (above), and // this registry, holding what earlier register_id/add_ids calls recorded and // every whole-document Begin replays. A registration for the very attribute // being registered is not a collision — the fast path's `tmpAttr == attr` // leaves the document alone and returns. -static int PyXmlSec_LxmlShadowIdIsTaken(PyXmlSec_LxmlElementPtr element, const char* name, PyObject* value) { - PyObject* key; +static int PyXmlSec_LxmlShadowIdIsTaken(PyXmlSec_LxmlElementPtr element, PyObject* key, PyObject* value) { + PyObject* doc_key; PyObject* entry; PyObject* nodes; PyObject* specs; Py_ssize_t i; int taken; - taken = PyXmlSec_LxmlShadowIdIsDeclared(element, value); + taken = PyXmlSec_LxmlShadowIdIsDeclared(element, key, value); if (taken != 0) { return taken; } - key = PyLong_FromVoidPtr((void*)element->_doc); - if (key == NULL) { + doc_key = PyLong_FromVoidPtr((void*)element->_doc); + if (doc_key == NULL) { return -1; } - entry = PyDict_GetItem(PyXmlSec_LxmlShadowIdRegistry, key); // borrowed - Py_DECREF(key); + entry = PyDict_GetItem(PyXmlSec_LxmlShadowIdRegistry, doc_key); // borrowed + Py_DECREF(doc_key); if (entry == NULL) { return 0; } @@ -1108,16 +1285,16 @@ static int PyXmlSec_LxmlShadowIdIsTaken(PyXmlSec_LxmlElementPtr element, const c PyObject* node = PyList_GET_ITEM(nodes, PyLong_AsSsize_t(PyTuple_GET_ITEM(spec, 2))); PyObject* spec_ns = PyTuple_GET_ITEM(spec, 1); const char* spec_name = PyUnicode_AsUTF8(PyTuple_GET_ITEM(spec, 0)); + const char* spec_href; int subtree = PyObject_IsTrue(PyTuple_GET_ITEM(spec, 3)); - int mine; if (spec_name == NULL || subtree < 0) { return -1; } - // The same attribute of the same element: registering it again is - // the no-op the fast path performs, whoever recorded it first. The - // names alone decide it, as xmlHasProp's own matching does. - mine = strcmp(spec_name, name) == 0; + spec_href = spec_ns == Py_None ? NULL : PyUnicode_AsUTF8(spec_ns); + if (spec_href == NULL && spec_ns != Py_None) { + return -1; + } // A vacated slot, or an element adopted into another tree: neither // stands for anything in this document, and the replay skips both. if (node == Py_None || ((PyXmlSec_LxmlElementPtr)node)->_doc != element->_doc) { @@ -1140,31 +1317,24 @@ static int PyXmlSec_LxmlShadowIdIsTaken(PyXmlSec_LxmlElementPtr element, const c Py_DECREF(matches); return -1; } - taken = !(mine && match == (PyObject*)element); + taken = PyXmlSec_LxmlShadowSpecClaims((PyXmlSec_LxmlElementPtr)match, spec_name, spec_href, + element, key, value); Py_DECREF(match); + if (taken < 0) { + Py_DECREF(matches); + return -1; + } } Py_DECREF(matches); if (n < 0) { return -1; } } else { - const char* spec_href = spec_ns == Py_None ? NULL : PyUnicode_AsUTF8(spec_ns); - PyObject* other; - int same; - - if (spec_href == NULL && spec_ns != Py_None) { - return -1; - } - other = PyXmlSec_LxmlAttrValue((PyXmlSec_LxmlElementPtr)node, spec_name, spec_href); - if (other == NULL) { - return -1; - } - same = PyObject_RichCompareBool(other, value, Py_EQ); - Py_DECREF(other); - if (same < 0) { + taken = PyXmlSec_LxmlShadowSpecClaims((PyXmlSec_LxmlElementPtr)node, spec_name, spec_href, + element, key, value); + if (taken < 0) { return -1; } - taken = same && !(mine && node == (PyObject*)element); } if (taken) { return 1; @@ -1267,7 +1437,8 @@ int PyXmlSec_LxmlShadowRecordIds(PyXmlSec_LxmlElementPtr element, PyObject* name } int PyXmlSec_LxmlShadowRegisterId(PyXmlSec_LxmlElementPtr element, const char* name, const char* ns) { - PyObject* value = PyXmlSec_LxmlAttrValue(element, name, ns); + PyObject* key = NULL; + PyObject* value = PyXmlSec_LxmlAttrFind(element, name, ns, &key); PyObject* names; int taken; int rv; @@ -1277,11 +1448,16 @@ int PyXmlSec_LxmlShadowRegisterId(PyXmlSec_LxmlElementPtr element, const char* n } if (value == Py_None) { Py_DECREF(value); + Py_XDECREF(key); PyErr_SetString(PyXmlSec_Error, "missing attribute."); return -1; } - taken = PyXmlSec_LxmlShadowIdIsTaken(element, name, value); + // `key` names the attribute the fast path would hand to xmlAddID; every + // collision test below asks whether the value is claimed by some *other* + // attribute than that one. + taken = PyXmlSec_LxmlShadowIdIsTaken(element, key, value); Py_DECREF(value); + Py_XDECREF(key); if (taken < 0) { return -1; } diff --git a/tests/test_ds.py b/tests/test_ds.py index 308f0cbe..358ab9ae 100644 --- a/tests/test_ds.py +++ b/tests/test_ds.py @@ -99,6 +99,33 @@ def test_register_id_accepts_an_attribute_add_ids_registered(self): xmlsec.tree.add_ids(root[0], ['ID']) xmlsec.SignatureContext().register_id(root[0][0], 'ID') + def test_register_id_rejects_a_value_a_sibling_attribute_declares(self): + """The declared id of an element can sit on another of its attributes, which claims the value all the same.""" + root = etree.fromstring(b'') + with self.assertRaisesRegex(xmlsec.Error, 'duplicated id.'): + xmlsec.SignatureContext().register_id(root[0], 'ID') + + def test_register_id_accepts_the_declared_attribute_beside_a_twin(self): + """A second attribute repeating the value is not the declared one, so registering the declared one is the no-op.""" + xml = b'\n\n' + root = etree.fromstring(xml, etree.XMLParser(load_dtd=True), base_url=self.path('doc.xml')) + xmlsec.SignatureContext().register_id(root[0], 'ID') + + def test_register_id_rejects_a_value_a_namespaced_registration_claimed(self): + """Two attributes of one element differing only in namespace are two attributes: the second cannot win the lookup.""" + ctx = xmlsec.SignatureContext() + root = etree.fromstring(b'') + ctx.register_id(root[0], 'Id', id_ns='urn:a') + with self.assertRaisesRegex(xmlsec.Error, 'duplicated id.'): + ctx.register_id(root[0], 'Id') + + def test_register_id_accepts_the_same_namespaced_attribute_twice(self): + """The no-op holds for a namespaced attribute too, which the key comparison must not read as a collision.""" + ctx = xmlsec.SignatureContext() + root = etree.fromstring(b'') + ctx.register_id(root[0], 'Id', id_ns='urn:a') + ctx.register_id(root[0], 'Id', id_ns='urn:a') + def test_register_id_with_namespace_without_attribute(self): ctx = xmlsec.SignatureContext() root = self.load_xml('sign_template.xml') From ea7dacea6f3e31ab483d1d3b17d2cdf6453cab2e Mon Sep 17 00:00:00 2001 From: Amin Solhizadeh Date: Fri, 4 Sep 2026 21:50:26 +0200 Subject: [PATCH 20/22] Skip the external-DTD tests where nothing may load one (#356) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The sdist job fails on `test_sign_and_verify_with_an_id_an_external_dtd_ declares`, added with the external-subset fix in 3a50b04 and never yet through CI. Ubuntu 22.04 ships libxmlsec1 1.2.33 with the XXE patch backported, and that xmlsec installs its no-XXE external entity loader globally at xmlSecInit — so importing xmlsec refuses lxml its own `load_dtd=True` parse, well before any shadow exists. libxml2 is matched in that job (lxml is built with --no-binary), so the raw path is what runs: no declaration is made, `#ext` resolves to nothing and the sign fails. The test's premise, not its subject, is what the environment removes. Both tests that need a loaded subset now go through `parse_with_external_dtd`, which skips when `docinfo.externalDTD` comes back None — the same signal `PyXmlSec_LxmlDocumentSubsets` reads to decide whether the copy should load one. Verified by hiding tests/data/id_attr.dtd: 4 skips, no failures. 318 passed / 6 skipped on the mismatch build; 330 / 6 on the matched static wheel, plain and with PYXMLSEC_FORCE_SHADOW=1. Co-Authored-By: Claude Opus 5 --- tests/test_ds.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/tests/test_ds.py b/tests/test_ds.py index 358ab9ae..7e0999be 100644 --- a/tests/test_ds.py +++ b/tests/test_ds.py @@ -108,8 +108,7 @@ def test_register_id_rejects_a_value_a_sibling_attribute_declares(self): def test_register_id_accepts_the_declared_attribute_beside_a_twin(self): """A second attribute repeating the value is not the declared one, so registering the declared one is the no-op.""" xml = b'\n\n' - root = etree.fromstring(xml, etree.XMLParser(load_dtd=True), base_url=self.path('doc.xml')) - xmlsec.SignatureContext().register_id(root[0], 'ID') + xmlsec.SignatureContext().register_id(self.parse_with_external_dtd(xml)[0], 'ID') def test_register_id_rejects_a_value_a_namespaced_registration_claimed(self): """Two attributes of one element differing only in namespace are two attributes: the second cannot win the lookup.""" @@ -288,9 +287,23 @@ def test_sign_and_verify_with_registered_id(self): # loads it knows "#ext" resolves to the Node (issue #356). EXTERNAL_DTD_XML = b'\nsigned\n' + def parse_with_external_dtd(self, xml): + """Parse `xml` with its external subset loaded, or skip the test when this build forbids that. + + xmlsec installs a no-XXE external entity loader globally at + ``xmlSecInit`` (1.2.34 and later, and the patched 1.2.33 some + distributions ship), so merely importing xmlsec can refuse lxml its + own ``load_dtd=True`` parse. Nothing then types the id, on either + path, and there is no declaration left for the copy to carry across. + """ + root = etree.fromstring(xml, etree.XMLParser(load_dtd=True), base_url=self.path('doc.xml')) + if root.getroottree().docinfo.externalDTD is None: + self.skipTest('this build refuses to load an external DTD subset') + return root + def test_sign_and_verify_with_an_id_an_external_dtd_declares(self): """Should resolve a #id reference whose id attribute an external subset the caller loaded declares.""" - root = etree.fromstring(self.EXTERNAL_DTD_XML, etree.XMLParser(load_dtd=True), base_url=self.path('doc.xml')) + root = self.parse_with_external_dtd(self.EXTERNAL_DTD_XML) sign = xmlsec.template.create(root, consts.TransformExclC14N, consts.TransformRsaSha1, ns='ds') root.append(sign) ref = xmlsec.template.add_reference(sign, consts.TransformSha1, uri='#ext') From b86e624853c1dd3f0abf89e0aac99ea21ece193f Mon Sep 17 00:00:00 2001 From: Amin Solhizadeh Date: Sat, 5 Sep 2026 09:56:44 +0200 Subject: [PATCH 21/22] Copy a subtree its document lost as an unlinked node (#356) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PyXmlSec_LxmlLivePathTo stops where getparent() returns None, but an element removed from its tree is not a document root: lxml leaves such a subtree pointing at the document it left, whose getroottree() still serializes that document — now without the subtree in it. BeginDoc therefore copied a document that does not hold the element, recorded depth 0, and handed xmlsec the copy's *root* instead. Signing a template taken out of sign1-in.xml (its URI="" reference then digests the document without the signature in it, which is the point of taking it out) succeeds on the raw path and failed with Error(1, 'failed to sign') under the shadow; find_parent was worse, answering for the wrong tree with no error at all. The copy now reproduces the shape the raw path works on. The document is copied as before, for the references, and the removed subtree is copied into it as an unlinked node beside its tree (shadow.unlinked): that node is what shadow.root / shadow.element and every path map between, what the marking tags, what the reflection re-parses (xmlSaveTree of the subtree — the document dump does not hold it) and what Discard frees, since the document does not own it. Registered ids are replayed once per live top, the subtree's own and the document's, because a #id reference from the subtree into the document it left resolves on the raw path too. Such a node cannot be replaced on either path — libxml2 needs a parent to put the replacement in. Begin's whole-document detour for internal-subset documents had the same blind spot (the dump cannot hold the subtree, and the path re-rooted the wrong node); it now takes that detour only when the element really hangs under the document's root. Also: test_encrypt_uri never called encrypt_uri — it called encrypt_binary with the file:// URI — so the shadow path of encrypt_uri had no successful test at all. Verified byte-identical to the raw path for sign+verify of a removed template, a removed ancestor, a #id reference into the document it left, find_parent in both directions, decrypt, and an encrypt/decrypt Type=Content round trip. Five new tests, each failing on the shadow path before the fix and passing on the raw path. 323 passed / 6 skipped on the mismatch build (also at PYXMLSEC_TEST_ITERATIONS=50), 335 / 6 on the matched static wheel plain and with PYXMLSEC_FORCE_SHADOW=1; a 10k removed-template sign+verify loop holds RSS at 24.3 → 26.1 MiB with a stable digest. Co-Authored-By: Claude Opus 5 --- developer.md | 21 ++++-- src/lxml.c | 144 +++++++++++++++++++++++++++++++++++----- src/lxml.h | 7 ++ tests/test_ds.py | 45 +++++++++++++ tests/test_enc.py | 23 ++++++- tests/test_templates.py | 13 ++++ tests/test_tree.py | 9 +++ 7 files changed, 242 insertions(+), 20 deletions(-) diff --git a/developer.md b/developer.md index 1263d902..c426147a 100644 --- a/developer.md +++ b/developer.md @@ -160,7 +160,20 @@ caller gets a new proxy object. `index`), serializes `element.getroottree()` — comments/PIs outside the root and the internal DTD subset survive — and hands back the copy's counterpart; `shadow.element` becomes the live *root*, which is where the reflection maps -paths onto. `BeginNewDoc` creates an empty private document; `End` roots the +paths onto. + +An element **removed from its document** is a shape of its own: lxml leaves +such a subtree pointing at the document it left, and so does the raw path — +xmlsec works on a node outside the tree whose `doc` still answers its `#id` +references (a template taken out of a document and signed with `URI=""` +digests that document, without itself in it). The copy holds both: the +document is copied as always, and the removed subtree is copied into it as an +*unlinked* node beside its tree (`shadow.unlinked`), which is then what +`shadow.root` / `shadow.element` and every path map between. Registered IDs +are replayed twice, once for each of the two live tops. Such a node cannot be +replaced (libxml2 needs a parent to put the replacement in), on either path. + +`BeginNewDoc` creates an empty private document; `End` roots the detached result there, dumps it and returns it as a new detached lxml element (in a document of its own until grafted; lxml moves it when the caller appends it, like the raw path's detached node). @@ -303,9 +316,9 @@ All invisible to the documented API: - a subtree of a document that declares entities in its internal subset (`resolve_entities=False`) is copied by copying the whole document and cutting it back to the element, since the subtree's `&name;` references - need their declarations; `encrypt_xml` templates are still serialized on - their own, so a *template* carrying unresolved entity references is not - supported (signing and encryption of such a document are refused on both + need their declarations; `encrypt_xml` templates and subtrees removed from + their document are still serialized on their own, so a *template* — or a + removed subtree — carrying unresolved entity references is not supported (signing and encryption of such a document are refused on both paths anyway — libxml2's c14n rejects entity-reference nodes); - operations that replace the **document root** (encrypting the root element with `Type=Element`, decrypting a root `EncryptedData`) morph the live root diff --git a/src/lxml.c b/src/lxml.c index 44a8eb5c..545ac2c2 100644 --- a/src/lxml.c +++ b/src/lxml.c @@ -17,6 +17,7 @@ #include #include #include +#include #include @@ -435,18 +436,26 @@ static int PyXmlSec_LxmlShadowTagNodes(PyXmlSec_LxmlShadow* shadow, xmlNodePtr n // exception set. static int PyXmlSec_LxmlShadowMark(PyXmlSec_LxmlShadow* shadow) { int count = PyXmlSec_LxmlShadowCountNodes(shadow->doc->children, 0); + // The unlinked subtree is in the document but not in its tree, so the + // walk over the tree does not reach it; it is pre-existing all the same. + int unlinked = shadow->unlinked ? PyXmlSec_LxmlShadowCountNodes(shadow->root, 0) : 0; + int next; - if (count < 0) { + if (count < 0 || unlinked < 0) { PyErr_SetString(PyXmlSec_InternalError, "the document is nested too deeply."); return -1; } + count += unlinked; shadow->tags = (PyXmlSec_LxmlShadowTag*)PyMem_Malloc((count > 0 ? count : 1) * sizeof(*shadow->tags)); if (shadow->tags == NULL) { PyErr_NoMemory(); return -1; } shadow->ntags = count; - PyXmlSec_LxmlShadowTagNodes(shadow, shadow->doc->children, 0); + next = PyXmlSec_LxmlShadowTagNodes(shadow, shadow->doc->children, 0); + if (shadow->unlinked) { + PyXmlSec_LxmlShadowTagNodes(shadow, shadow->root, next); + } return 0; } @@ -593,6 +602,33 @@ static PyObject* PyXmlSec_LxmlShadowDumpCopy(PyXmlSec_LxmlShadow* shadow) { xmlChar* dump = NULL; int dump_size = 0; + if (shadow->unlinked) { + // The paths the reflection carries are relative to the unlinked + // subtree, so that is what has to be re-parsed — the document dump + // does not hold it. + xmlBufferPtr buffer = xmlBufferCreate(); + xmlSaveCtxtPtr save = buffer != NULL ? xmlSaveToBuffer(buffer, "UTF-8", XML_SAVE_NO_DECL) : NULL; + int failed = 1; + + if (save != NULL) { + failed = xmlSaveTree(save, shadow->root) < 0; + failed = xmlSaveClose(save) < 0 || failed; + } + if (!failed) { + bytes = PyBytes_FromStringAndSize((const char*)xmlBufferContent(buffer), (Py_ssize_t)xmlBufferLength(buffer)); + if (bytes != NULL) { + result = PyXmlSec_LxmlElementFromBytes(bytes); + } + Py_XDECREF(bytes); + } else { + PyErr_SetString(PyXmlSec_InternalError, "cannot serialize the private copy."); + } + if (buffer != NULL) { + xmlBufferFree(buffer); + } + return result; + } + xmlDocDumpMemory(shadow->doc, &dump, &dump_size); if (dump == NULL || dump_size <= 0) { PyErr_SetString(PyXmlSec_InternalError, "cannot serialize the private copy."); @@ -695,6 +731,7 @@ int PyXmlSec_LxmlShadowBegin(PyXmlSec_LxmlShadow* shadow, PyXmlSec_LxmlElementPt int depth = 0; int dtd = 0; int extdtd = 0; + int whole = 0; shadow->element = element; shadow->owned = NULL; @@ -702,6 +739,7 @@ int PyXmlSec_LxmlShadowBegin(PyXmlSec_LxmlShadow* shadow, PyXmlSec_LxmlElementPt shadow->root = NULL; shadow->tags = NULL; shadow->ntags = 0; + shadow->unlinked = 0; // Fast path: lxml links the same libxml2 (the import guard passed), so // xmlsec can mutate lxml's nodes directly and no copy is needed; End sees @@ -725,12 +763,22 @@ int PyXmlSec_LxmlShadowBegin(PyXmlSec_LxmlShadow* shadow, PyXmlSec_LxmlElementPt // in the document's internal subset — serializing the element alone // would leave them undefined and the parse would fail. Copy the whole // document, declarations included, and cut it back to the element. + // Unless the element was removed from that document: the dump would + // not hold it at all, and the element alone is all there is to copy. PyObject* live_top = NULL; + PyObject* live_root = PyObject_CallMethod(tree, "getroot", NULL); + if (live_root == NULL) { + goto ON_FAIL; + } depth = PyXmlSec_LxmlLivePathTo((PyObject*)element, path, &live_top); + whole = depth >= 0 && live_top == live_root; Py_XDECREF(live_top); + Py_DECREF(live_root); if (depth < 0) { goto ON_FAIL; } + } + if (whole) { bytes = PyObject_CallFunctionObjArgs(PyXmlSec_LxmlEtreeToString, tree, NULL); } else { bytes = PyXmlSec_LxmlElementToBytes((PyObject*)element); @@ -745,7 +793,7 @@ int PyXmlSec_LxmlShadowBegin(PyXmlSec_LxmlShadow* shadow, PyXmlSec_LxmlElementPt if (shadow->doc == NULL) { goto ON_FAIL; } - if (dtd && PyXmlSec_LxmlShadowReroot(shadow, path, depth) < 0) { + if (whole && PyXmlSec_LxmlShadowReroot(shadow, path, depth) < 0) { goto ON_FAIL; } shadow->root = xmlDocGetRootElement(shadow->doc); @@ -769,6 +817,7 @@ xmlDocPtr PyXmlSec_LxmlShadowBeginNewDoc(PyXmlSec_LxmlShadow* shadow, PyXmlSec_L shadow->root = NULL; shadow->tags = NULL; shadow->ntags = 0; + shadow->unlinked = 0; // Fast path: allocate the detached subtree straight in the element's own // document, as the raw code always did. @@ -785,9 +834,15 @@ xmlDocPtr PyXmlSec_LxmlShadowBeginNewDoc(PyXmlSec_LxmlShadow* shadow, PyXmlSec_L void PyXmlSec_LxmlShadowDiscard(PyXmlSec_LxmlShadow* shadow) { if (shadow->doc != NULL) { + if (shadow->unlinked && shadow->root != NULL) { + // Outside the document's tree: freeing the document does not + // reach it. + xmlFreeNode(shadow->root); + } xmlFreeDoc(shadow->doc); shadow->doc = NULL; shadow->root = NULL; + shadow->unlinked = 0; } PyMem_Free(shadow->tags); shadow->tags = NULL; @@ -1523,8 +1578,10 @@ static void PyXmlSec_LxmlShadowApplyIdSpec(xmlDocPtr doc, xmlNodePtr node, const } // Replays the specs recorded for the shadow's live document onto the copy, -// each at the copy's counterpart of the element it was registered for. -static int PyXmlSec_LxmlShadowReplayIds(PyXmlSec_LxmlShadow* shadow) { +// each at the copy's counterpart of the element it was registered for: a spec +// recorded for an element under the live `live_top` is applied under its +// counterpart `copy_top`. Returns 0, or -1 with an exception set. +static int PyXmlSec_LxmlShadowReplayIds(PyXmlSec_LxmlShadow* shadow, PyObject* live_top, xmlNodePtr copy_top) { PyObject* key; PyObject* entry; PyObject* nodes; @@ -1567,10 +1624,10 @@ static int PyXmlSec_LxmlShadowReplayIds(PyXmlSec_LxmlShadow* shadow) { return -1; } // lxml hands out one proxy per node, so identity settles whether the - // element still hangs under the root being copied; a registration for - // an element that has since left this tree applies to nothing here. - if (top == (PyObject*)shadow->element) { - PyXmlSec_LxmlShadowApplyIdSpec(shadow->doc, PyXmlSec_LxmlShadowWalkNode(shadow->root, path, depth), + // element still hangs under the tree being replayed; a registration + // for an element that has since left it applies to nothing here. + if (top == live_top) { + PyXmlSec_LxmlShadowApplyIdSpec(shadow->doc, PyXmlSec_LxmlShadowWalkNode(copy_top, path, depth), (const xmlChar*)name, (const xmlChar*)ns, subtree); } Py_DECREF(top); @@ -1578,8 +1635,38 @@ static int PyXmlSec_LxmlShadowReplayIds(PyXmlSec_LxmlShadow* shadow) { return 0; } +// Copies the live subtree `element` — removed from its document, so the copy +// of that document does not hold it either — into the private document, as an +// unlinked node beside its tree. That is the shape the raw path hands xmlsec: +// a node outside the tree whose `doc` still answers `#id` references. +// Returns 0, or -1 with an exception set. +static int PyXmlSec_LxmlShadowUnlinkedCopy(PyXmlSec_LxmlShadow* shadow, PyObject* element) { + PyObject* bytes = PyXmlSec_LxmlElementToBytes(element); + xmlDocPtr tdoc; + xmlNodePtr copy; + + if (bytes == NULL) { + return -1; + } + tdoc = PyXmlSec_LxmlShadowParse(bytes, NULL, 0, "cannot make a private copy of the element."); + Py_DECREF(bytes); + if (tdoc == NULL) { + return -1; + } + copy = xmlDocCopyNode(xmlDocGetRootElement(tdoc), shadow->doc, 1); + xmlFreeDoc(tdoc); + if (copy == NULL) { + PyErr_SetString(PyXmlSec_InternalError, "cannot make a private copy of the element."); + return -1; + } + shadow->root = copy; + shadow->unlinked = 1; + return 0; +} + int PyXmlSec_LxmlShadowBeginDoc(PyXmlSec_LxmlShadow* shadow, PyXmlSec_LxmlElementPtr element, xmlNodePtr* target) { PyObject* cur = NULL; + PyObject* live_root = NULL; PyObject* tree = NULL; PyObject* bytes = NULL; PyObject* url_holder = NULL; @@ -1595,6 +1682,7 @@ int PyXmlSec_LxmlShadowBeginDoc(PyXmlSec_LxmlShadow* shadow, PyXmlSec_LxmlElemen shadow->root = NULL; shadow->tags = NULL; shadow->ntags = 0; + shadow->unlinked = 0; *target = NULL; if (!PyXmlSec_LxmlShadowActive) { @@ -1617,6 +1705,13 @@ int PyXmlSec_LxmlShadowBeginDoc(PyXmlSec_LxmlShadow* shadow, PyXmlSec_LxmlElemen if (tree == NULL) { goto ON_FAIL; } + // `cur` is the document's root element unless `element` — or an ancestor + // of it — was removed from the tree: lxml leaves such a subtree pointing + // at the document it left, which the dump below therefore does not hold. + live_root = PyObject_CallMethod(tree, "getroot", NULL); + if (live_root == NULL) { + goto ON_FAIL; + } // The whole tree is dumped either way here, so only the external subset // matters: it has to be loaded for the copy to know the IDs it declares. bytes = PyObject_CallFunctionObjArgs(PyXmlSec_LxmlEtreeToString, tree, NULL); @@ -1632,6 +1727,9 @@ int PyXmlSec_LxmlShadowBeginDoc(PyXmlSec_LxmlShadow* shadow, PyXmlSec_LxmlElemen goto ON_FAIL; } shadow->root = xmlDocGetRootElement(shadow->doc); + if (live_root != cur && PyXmlSec_LxmlShadowUnlinkedCopy(shadow, cur) < 0) { + goto ON_FAIL; + } if (PyXmlSec_LxmlShadowMark(shadow) < 0) { goto ON_FAIL; } @@ -1649,14 +1747,23 @@ int PyXmlSec_LxmlShadowBeginDoc(PyXmlSec_LxmlShadow* shadow, PyXmlSec_LxmlElemen cur = NULL; // The registered IDs live in lxml's document, which the copy knows - // nothing about; replay them so that #id references resolve. - if (PyXmlSec_LxmlShadowReplayIds(shadow) < 0) { + // nothing about; replay them so that #id references resolve. An unlinked + // subtree carries its own registrations, and the document it left keeps + // the rest — a reference from the subtree into that document resolves on + // the raw path too. + if (PyXmlSec_LxmlShadowReplayIds(shadow, (PyObject*)shadow->element, shadow->root) < 0) { goto ON_FAIL; } + if (shadow->unlinked + && PyXmlSec_LxmlShadowReplayIds(shadow, live_root, xmlDocGetRootElement(shadow->doc)) < 0) { + goto ON_FAIL; + } + Py_CLEAR(live_root); return 0; ON_FAIL: Py_XDECREF(cur); + Py_XDECREF(live_root); Py_XDECREF(tree); Py_XDECREF(bytes); Py_XDECREF(url_holder); @@ -1993,20 +2100,27 @@ static int PyXmlSec_LxmlShadowReflectSites(PyXmlSec_LxmlShadow* shadow) { int rv = -1; // Re-fetch the root: replacement operations may swap nodes at the top. - list.top = xmlDocGetRootElement(shadow->doc); + // An unlinked subtree cannot be replaced at all (libxml2 needs a parent + // to put the replacement in), so its top is the one the shadow made. + list.top = shadow->unlinked ? shadow->root : xmlDocGetRootElement(shadow->doc); if (list.top == NULL) { PyErr_SetString(PyXmlSec_InternalError, "unexpected mutation site."); goto DONE; } if (!PYXMLSEC_SHADOW_TAGGED(shadow, list.top)) { + xmlNodePtr n; + int elements = 0; + int others = 0; + + if (shadow->unlinked) { + PyErr_SetString(PyXmlSec_InternalError, "unexpected mutation site."); + goto DONE; + } // A fresh root: the call replaced the root itself, so there are no // sites to graft — the live root is morphed into it wholesale. lxml // holds one element (plus comments and PIs) at document level, so a // root replaced by anything else (a Type=Content decryption of the // root) cannot be reflected. - xmlNodePtr n; - int elements = 0; - int others = 0; for (n = shadow->doc->children; n != NULL; n = n->next) { if (n->type == XML_ELEMENT_NODE) { ++elements; diff --git a/src/lxml.h b/src/lxml.h index 9448db5f..01599818 100644 --- a/src/lxml.h +++ b/src/lxml.h @@ -90,6 +90,8 @@ typedef struct { xmlNodePtr root; // doc's root element, the copy of `element`; NULL for BeginNewDoc PyXmlSec_LxmlShadowTag* tags; // one per pre-existing node of `doc`, owned by the shadow int ntags; + int unlinked; // `root` hangs in `doc` outside its tree (BeginDoc of an element + // removed from its document); the shadow frees it itself } PyXmlSec_LxmlShadow; // Subtree copy: `shadow.root` is the copy of `element`. When the document @@ -105,6 +107,11 @@ int PyXmlSec_LxmlShadowBegin(PyXmlSec_LxmlShadow* shadow, PyXmlSec_LxmlElementPt // `*target` receives the copy's counterpart of `element` (the live node // itself on the fast path); `shadow.root` / `shadow.element` become the copy // root / the live root, which is what the End functions map paths between. +// An element removed from its document keeps pointing at it, and so does the +// raw path — xmlsec works on the unlinked node while the document it left +// still answers its `#id` references. The copy holds both: the document, and +// the unlinked subtree copied into it beside the tree (`shadow.unlinked`, +// with `shadow.root` / `shadow.element` the subtree's top on either side). int PyXmlSec_LxmlShadowBeginDoc(PyXmlSec_LxmlShadow* shadow, PyXmlSec_LxmlElementPtr element, xmlNodePtr* target); // Create shape (template.create, encrypted_data_create): the call only needs diff --git a/tests/test_ds.py b/tests/test_ds.py index 7e0999be..97e69129 100644 --- a/tests/test_ds.py +++ b/tests/test_ds.py @@ -282,6 +282,51 @@ def test_sign_and_verify_with_registered_id(self): verify_ctx.key = xmlsec.Key.from_file(self.path('rsapub.pem'), format=consts.KeyDataFormatPem) verify_ctx.verify(sign) + def test_sign_and_verify_a_template_removed_from_its_document(self): + """A template taken out of its tree still signs: lxml leaves it pointing at the document it left, + and that is where its URI="" reference resolves — on the raw path too (issue #356).""" + root = self.load_xml('sign1-in.xml') + sign = xmlsec.tree.find_node(root, consts.NodeSignature, consts.DSigNs) + sign.getparent().remove(sign) + + ctx = xmlsec.SignatureContext() + ctx.key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem) + ctx.sign(sign) + # the signature is written into the removed subtree, and nothing is + # written into the document it covers + self.assertTrue(xmlsec.tree.find_node(sign, consts.NodeSignatureValue, consts.DSigNs).text) + self.assertIsNone(xmlsec.tree.find_node(root, consts.NodeSignature, consts.DSigNs)) + + verify_ctx = xmlsec.SignatureContext() + verify_ctx.key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem) + verify_ctx.verify(sign) + + # the digest covers that document: changing it invalidates the signature + root.find('{urn:envelope}Data').text = 'tampered' + tampered_ctx = xmlsec.SignatureContext() + tampered_ctx.key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem) + with self.assertRaises(xmlsec.VerificationError): + tampered_ctx.verify(sign) + + def test_sign_and_verify_a_removed_template_against_a_registered_id(self): + """A #id reference from a removed subtree resolves in the document it left (issue #356).""" + root = self.load_xml('sign4-in.xml') + sign = xmlsec.template.create(root, consts.TransformExclC14N, consts.TransformRsaSha1, ns='ds') + root.append(sign) + ref = xmlsec.template.add_reference(sign, consts.TransformSha1, uri='#' + root.get('ID')) + xmlsec.template.add_transform(ref, consts.TransformExclC14N) + root.remove(sign) + + ctx = xmlsec.SignatureContext() + ctx.register_id(root, 'ID') + ctx.key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem) + ctx.sign(sign) + + verify_ctx = xmlsec.SignatureContext() + verify_ctx.register_id(root, 'ID') + verify_ctx.key = xmlsec.Key.from_file(self.path('rsapub.pem'), format=consts.KeyDataFormatPem) + verify_ctx.verify(sign) + # A document whose id attribute is typed by an external DTD subset — the # declarations live in a file the DOCTYPE names, and only a parse that # loads it knows "#ext" resolves to the Node (issue #356). diff --git a/tests/test_enc.py b/tests/test_enc.py index b2f3599c..a97f8823 100644 --- a/tests/test_enc.py +++ b/tests/test_enc.py @@ -238,7 +238,7 @@ def test_encrypt_uri(self): with tempfile.NamedTemporaryFile(delete=False) as tmpfile: tmpfile.write(b'test') - encrypted = ctx.encrypt_binary(enc_data, 'file://' + tmpfile.name) + encrypted = ctx.encrypt_uri(enc_data, 'file://' + tmpfile.name) self.assertIsNotNone(encrypted) self.assertEqual(f'{{{consts.EncNs}}}{consts.NodeEncryptedData}', encrypted.tag) @@ -254,6 +254,27 @@ def test_encrypt_uri(self): cipher_value = xmlsec.tree.find_node(ki, consts.NodeCipherValue, consts.EncNs) self.assertIsNotNone(cipher_value) + def test_encrypt_and_decrypt_content_of_a_subtree_removed_from_its_document(self): + """A subtree taken out of its tree is encrypted and decrypted in place, as on the raw path (issue #356).""" + root = etree.fromstring(b'hello x tail') + data = root[0] + root.remove(data) + before = etree.tostring(data) + + key = xmlsec.Key.generate(consts.KeyDataAes, 128, consts.KeyDataTypeSession) + enc_data = xmlsec.template.encrypted_data_create(data, consts.TransformAes128Cbc, type=consts.TypeEncContent, ns='xenc') + xmlsec.template.encrypted_data_ensure_cipher_value(enc_data) + ctx = xmlsec.EncryptionContext() + ctx.key = key + ctx.encrypt_xml(enc_data, data) + self.assertEqual(b'', etree.tostring(root)) + self.assertIsNone(data.text) + + dec_ctx = xmlsec.EncryptionContext() + dec_ctx.key = key + dec_ctx.decrypt(data[0]) + self.assertEqual(before, etree.tostring(data)) + def test_encrypt_uri_bad_args(self): ctx = xmlsec.EncryptionContext() with self.assertRaises(TypeError): diff --git a/tests/test_templates.py b/tests/test_templates.py index efe8b9f0..ce9f1d4d 100644 --- a/tests/test_templates.py +++ b/tests/test_templates.py @@ -107,6 +107,19 @@ def test_add_reference_beside_an_entity_reference(self): ref = xmlsec.template.add_reference(sign, consts.TransformSha1, uri='#abc') self.assertIs(ref, sign[0][3]) + def test_add_reference_in_a_subtree_removed_from_an_entity_document(self): + """The whole-document copy an internal subset forces cannot hold a subtree the document lost (issue #356).""" + root = etree.fromstring( + b' ]>\n&greet;', + etree.XMLParser(resolve_entities=False), + ) + holder = root[1] + root.remove(holder) + sign = xmlsec.template.create(holder, c14n_method=consts.TransformExclC14N, sign_method=consts.TransformRsaSha1) + holder.append(sign) + ref = xmlsec.template.add_reference(sign, consts.TransformSha1, uri='#abc') + self.assertIs(ref, sign[0][2]) + def test_add_reference_bad_args(self): with self.assertRaises(TypeError): xmlsec.template.add_reference('', consts.TransformSha1) diff --git a/tests/test_tree.py b/tests/test_tree.py index fd9f6777..f40c69b6 100644 --- a/tests/test_tree.py +++ b/tests/test_tree.py @@ -24,6 +24,15 @@ def test_find_parent(self): self.assertIs(root, xmlsec.tree.find_parent(si, consts.NodeSignature)) self.assertIsNone(xmlsec.tree.find_parent(root, consts.NodeSignedInfo)) + def test_find_parent_in_a_removed_subtree(self): + """The walk upward stops at the top of a subtree removed from its document (issue #356).""" + root = self.load_xml('sign1-in.xml') + sign = xmlsec.tree.find_node(root, consts.NodeSignature, consts.DSigNs) + si = xmlsec.tree.find_child(sign, consts.NodeSignedInfo, consts.DSigNs) + sign.getparent().remove(sign) + self.assertIs(sign, xmlsec.tree.find_parent(si, consts.NodeSignature, consts.DSigNs)) + self.assertIsNone(xmlsec.tree.find_parent(si, 'Envelope', 'urn:envelope')) + def test_find_parent_bad_args(self): with self.assertRaises(TypeError): xmlsec.tree.find_parent('', 0, True) From ff6a0a6bc460a75b0194d032e1f1e0f70ffa9403 Mon Sep 17 00:00:00 2001 From: Amin Solhizadeh Date: Sat, 5 Sep 2026 11:09:05 +0200 Subject: [PATCH 22/22] Snapshot add_ids' scope, retire dead registrations (#356) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two findings on the shadow-mode id registry. add_ids recorded one spec per attribute name over the *scope*, and the replay re-walked that scope when something was signed — so the ids a signature could resolve were the ones the tree carried by then, not the ones xmlSecAddIDs had registered at the call. An element appended to the scope afterwards became resolvable where the fast path had never registered it; and the walk applied one name at a time across the whole scope, where xmlSecAddIDs takes the elements in document order and the names within each element. With and add_ids(root, ['A', 'B']), the raw path signs Y and the shadow signed X: both happily, over different content. RecordIds now expands the scope where the registration happens: one spec per element carrying one of the names, element by element in document order, names in the caller's order. Specs are single-node throughout — AddIdsBelow, the replay's subtree walk and the duplicate check's XPath probe over a scope are gone — so the shadow registers exactly the attributes the caller registered, never one that appeared later. A registration also outlived its element. The registry holds the element proxy (lxml refuses weak references) and released the slot only with the whole document entry, so a document that registers and drops temporary elements grew without bound, and kept claiming their id values: registering a value a dropped element had carried raised "duplicated id." where the fast path, whose id entry dies with the attribute, accepts it. A slot is now retired when the registry holds the only reference to the proxy, the tree it hangs in is not its document's, and no other proxy remains anywhere in that tree — exactly when lxml frees such a subtree and libxml2 drops the id entries of the attributes in it. Vacated slots are reused, so neither the elements nor the list grow, and the sweep runs before every registration and before every duplicate check. What this does not reproduce is lxml clearing its own id entry whenever an element is *moved* — even within the one document, and for a whole subtree when an ancestor moves (verified against lxml's id hash). The registry cannot observe a move, so a #id the fast path stops resolving keeps resolving under the shadow, to the element it was registered for and never to another one. Documented with the other divergences. Cost, worst case (2000 id-bearing elements): add_ids 0.000 -> 0.035 s, the sign that follows 11 -> 14 ms, and 2000 register_id calls on that document 0.54 -> 1.18 s, each call now sweeping 2000 registrations. 328 passed / 6 skipped on the mismatch build, also at PYXMLSEC_TEST_ITERATIONS=50; 340 / 6 on the matched static wheel, plain and with PYXMLSEC_FORCE_SHADOW=1. 10k sign+verify with an element registered and dropped per iteration: RSS 26.2 -> 26.3 MiB, output byte-identical throughout; 20k registration churn on one document 25.9 -> 25.9 MiB, where it was 27.2 -> 34.0 before. Three new tests fail on the shadow path before the fix and pass on the raw path; two more guard against retiring a registration that is still live. --- developer.md | 40 ++++- src/lxml.c | 405 +++++++++++++++++++++++++++++++++-------------- src/lxml.h | 14 +- src/tree.c | 6 +- tests/test_ds.py | 60 +++++++ 5 files changed, 393 insertions(+), 132 deletions(-) diff --git a/developer.md b/developer.md index c426147a..97625d5a 100644 --- a/developer.md +++ b/developer.md @@ -217,18 +217,35 @@ Under the shadow they record the id-attribute specs in a registry keyed by document identity (`RegisterId`, `RecordIds`), and every `BeginDoc` replays them onto its copy so that `#id` references resolve during sign/verify/decrypt. An entry keeps the registered elements themselves, so a spec is replayed at exactly -the node it was registered for — that node alone for `register_id`, its -subtree for `add_ids`, the scope `xmlSecAddIDs` walks. Registering every -matching attribute of the copy instead would be unsafe, not merely generous: -an unrelated element sharing the id value would claim it first and a `#id` -reference could then resolve to content the caller never registered. lxml's +the node it was registered for. Registering every matching attribute of the +copy instead would be unsafe, not merely generous: an unrelated element +sharing the id value would claim it first and a `#id` reference could then +resolve to content the caller never registered. + +`add_ids` covers a whole scope, and that scope is walked **at the call**: +`RecordIds` expands it into one spec per element carrying one of the names, +element by element in document order and, within an element, in the order of +the names — the registration `xmlSecAddIDs` makes, at the moment it makes it. +Leaving the scope to be walked at the replay would register whatever the tree +had become by the time something was signed: an element that grew the +attribute, or joined the scope, after the call would resolve under the shadow +where the fast path never registered it, and two elements claiming one value +under different names would be ordered by name rather than by document +order — a `#id` covering different content on the two paths. lxml's classes refuse weak references, so the entry keeps strong references (to the document and to those elements) instead: the key (the document's address) can then never go stale, and since every element proxy holds a reference to its document, a document whose reference count is exactly what the registry holds — and whose registered elements nothing else holds — is provably unreachable, so its entry, and the document with it, is dropped before the next -registration. The registry therefore tracks the documents still in use and +registration. Registrations are retired one by one on the same principle: a +slot whose proxy the registry alone holds, hanging in a tree that is not its +document's, cannot be reached again — lxml keeps an unlinked subtree only for +as long as a proxy remains somewhere in it, and the fast path's id entry dies +at exactly that moment too, when libxml2 frees the attribute. The slot is +vacated (and reused by the next registration), so registering and dropping +elements on a long-lived document neither grows the registry nor keeps their +values claimed. The registry therefore tracks the documents still in use and never evicts a live one. The two bindings are the only places, together with encrypt_xml/decrypt's replacement bodies, that branch on `IsActive()`. @@ -238,8 +255,8 @@ the same call: `xmlGetID(doc, value) != attr` — the test that raises under the shadow. What lxml's own parse declared (a DTD id attribute, an `xml:id`) is read back through XPath's `id()`, the one door into lxml's id hash that passes nothing but strings and elements; what earlier -`register_id`/`add_ids` calls claimed is read from the registry, a subtree -spec through one XPath over its scope. +`register_id`/`add_ids` calls claimed is read from the registry, spec by +spec. Both halves compare *attributes*, not elements: `` answers `N` to `id('dup')` whichever attribute is asked about, while the fast @@ -292,6 +309,13 @@ All invisible to the documented API: list of ids), so such a value is compared against the registry alone; a registration whose element has since been adopted into another document claims nothing, as at replay; +- a registration follows the element it was made for. lxml drops its own id + entry whenever an element is *moved* — even within the one document, and + for a whole subtree when an ancestor moves — which the registry cannot + observe, so a `#id` the fast path stops resolving after such a move keeps + resolving under the shadow. It resolves to the registered element, never to + another one: the shadow registers exactly the attributes the caller + registered; - `encrypt_xml` encrypts a *copy* of the template, so the caller's template proxy is not the returned element; a template attached in the target's own document is unlinked afterwards (keeping its tail text where libxml2 would diff --git a/src/lxml.c b/src/lxml.c index 545ac2c2..7b75d768 100644 --- a/src/lxml.c +++ b/src/lxml.c @@ -861,17 +861,23 @@ void PyXmlSec_LxmlShadowDiscard(PyXmlSec_LxmlShadow* shadow) { // // An entry is the tuple (document, nodes, specs), keyed by the _Document // object's address: `nodes` holds the registered elements themselves and -// `specs` the (attribute name, namespace, node index, subtree) records. The -// entry keeps a strong reference to the document and to every registered -// element, so the key can never go stale — the address cannot be reused while -// the entry keeps the object alive. +// `specs` the (attribute name, namespace, node index) records. The entry +// keeps a strong reference to the document and to every registered element, +// so the key can never go stale — the address cannot be reused while the +// entry keeps the object alive. // // Keeping the elements is also what makes the replay faithful: a spec is -// applied to the very node it was registered for (and, for add_ids, that -// node's subtree), never to every same-named attribute in the document. -// Registering the whole document would let an unrelated element sharing the -// id value claim it first, so a `#id` reference — the URI a signature covers -// — could resolve to content the caller never registered. +// applied to the very node it was registered for, never to every same-named +// attribute in the document. Registering the whole document would let an +// unrelated element sharing the id value claim it first, so a `#id` +// reference — the URI a signature covers — could resolve to content the +// caller never registered. +// +// A registration covering a subtree (add_ids) is expanded into one spec per +// element carrying the attribute *at the call*, which is the snapshot +// xmlSecAddIDs takes of the scope it walks: an element that only later grows +// the attribute, or only later joins the scope, was never registered by the +// fast path and must not become resolvable under the shadow either. // // Those references also make liveness decidable without weak references // (lxml's classes refuse those): every element proxy holds a reference to the @@ -916,13 +922,29 @@ static int PyXmlSec_LxmlShadowIdEntryIsDead(PyObject* entry) { return Py_REFCNT(doc) == held; } +// Releases the node held in slot `i` of `entry`: the specs that named the +// slot go with it, and the slot is set to None rather than removed, so the +// indices the surviving specs carry stay valid — the next registration +// reuses it. Best effort; a failure only keeps a spec alive longer. +static void PyXmlSec_LxmlShadowVacateIdSlot(PyObject* entry, Py_ssize_t i) { + PyObject* nodes = PyTuple_GET_ITEM(entry, PYXMLSEC_ID_ENTRY_NODES); + PyObject* specs = PyTuple_GET_ITEM(entry, PYXMLSEC_ID_ENTRY_SPECS); + Py_ssize_t j; + + for (j = PyList_GET_SIZE(specs) - 1; j >= 0; --j) { + PyObject* spec = PyList_GET_ITEM(specs, j); + if (PyLong_AsSsize_t(PyTuple_GET_ITEM(spec, 2)) == i && PyList_SetSlice(specs, j, j + 1, NULL) < 0) { + PyErr_Clear(); + } + } + Py_INCREF(Py_None); + PyList_SetItem(nodes, i, Py_None); // steals the reference, releases the node +} + // Drops `element` from every entry but `keep`. A node registered again after // being adopted into another document is no longer part of the tree its old // entry stands for, and its slot there must stop holding it: the liveness -// test above counts on a registered proxy being held by one entry only. The -// slot is vacated rather than removed, so the indices the surviving specs -// carry stay valid; the specs that pointed at it go. Best effort — a failure -// only leaves an entry alive longer than needed. +// test above counts on a registered proxy being held by one entry only. static void PyXmlSec_LxmlShadowForgetIdNode(PyObject* keep, PyObject* element) { PyObject* key; PyObject* entry; @@ -930,30 +952,182 @@ static void PyXmlSec_LxmlShadowForgetIdNode(PyObject* keep, PyObject* element) { while (PyDict_Next(PyXmlSec_LxmlShadowIdRegistry, &pos, &key, &entry)) { PyObject* nodes; - PyObject* specs; Py_ssize_t i; if (entry == keep) { continue; } nodes = PyTuple_GET_ITEM(entry, PYXMLSEC_ID_ENTRY_NODES); - specs = PyTuple_GET_ITEM(entry, PYXMLSEC_ID_ENTRY_SPECS); for (i = 0; i < PyList_GET_SIZE(nodes); ++i) { - Py_ssize_t j; - if (PyList_GET_ITEM(nodes, i) != element) { - continue; - } - for (j = PyList_GET_SIZE(specs) - 1; j >= 0; --j) { - PyObject* spec = PyList_GET_ITEM(specs, j); - if (PyLong_AsSsize_t(PyTuple_GET_ITEM(spec, 2)) == i && PyList_SetSlice(specs, j, j + 1, NULL) < 0) { - PyErr_Clear(); - } + if (PyList_GET_ITEM(nodes, i) == element) { + PyXmlSec_LxmlShadowVacateIdSlot(entry, i); } - Py_INCREF(Py_None); - PyList_SetItem(nodes, i, Py_None); // steals the reference, releases the node } } } +// The root element of the document `node` belongs to — the one lxml answers +// with even for a node that has left the tree. New reference, or NULL. +static PyObject* PyXmlSec_LxmlDocumentRoot(PyObject* node) { + PyObject* tree = PyObject_CallMethod(node, "getroottree", NULL); + PyObject* root = tree == NULL ? NULL : PyObject_CallMethod(tree, "getroot", NULL); + + Py_XDECREF(tree); + return root; +} + +// Non-zero when nothing but the registry can reach `node` again: the registry +// holds the only reference to the proxy, and the tree the node hangs in is +// not the document's (`root` is that document's root element) — an unlinked +// subtree, which lxml keeps alive only for as long as a proxy remains +// somewhere in it. On the fast path such a subtree is freed at that same +// moment, and libxml2 drops the ID entries of the attributes it takes with +// it, so a slot that answers yes here stands for a registration the fast path +// no longer has either. Best effort: anything that goes wrong answers +// "reachable", which only keeps the slot. +static int PyXmlSec_LxmlShadowIdNodeIsDead(PyObject* node, PyObject* root) { + PyObject* top; + PyObject* nodes = NULL; + Py_ssize_t i; + int dead = 0; + + if (Py_REFCNT(node) != 1) { + return 0; + } + // The top of the tree `node` hangs in, through lxml's own API. + top = node; + Py_INCREF(top); + for (;;) { + PyObject* parent = PyObject_CallMethod(top, "getparent", NULL); + if (parent == NULL) { + goto DONE; + } + if (parent == Py_None) { + Py_DECREF(parent); + break; + } + Py_DECREF(top); + top = parent; + } + // Still in the document's own tree, which is reachable through the + // document itself, whoever holds that; only the entry's own liveness test + // can retire one of those. + if (top == root) { + goto DONE; + } + // lxml hands out at most one proxy per node, so a reference held by + // anyone but this walk and the registry is a way back to `node`: from any + // node of the tree, `getparent()` and the children lead to every other. + nodes = PyObject_CallMethod(top, "xpath", "s", + "descendant-or-self::*" + "|descendant-or-self::comment()" + "|descendant-or-self::processing-instruction()"); + if (nodes == NULL || !PyList_Check(nodes)) { + goto DONE; + } + dead = 1; + for (i = 0; i < PyList_GET_SIZE(nodes) && dead; ++i) { + PyObject* item = PyList_GET_ITEM(nodes, i); // borrowed + Py_ssize_t held = 1; // the list's own reference + if (item == node) { + ++held; // the registry's + } + if (item == top) { + ++held; // the walk above + } + dead = Py_REFCNT(item) <= held; + } + +DONE: + Py_XDECREF(nodes); + Py_DECREF(top); + PyErr_Clear(); + return dead; +} + +// Vacates every slot no spec names any more. +static void PyXmlSec_LxmlShadowCompactIdNodes(PyObject* entry) { + PyObject* nodes = PyTuple_GET_ITEM(entry, PYXMLSEC_ID_ENTRY_NODES); + PyObject* specs = PyTuple_GET_ITEM(entry, PYXMLSEC_ID_ENTRY_SPECS); + Py_ssize_t i; + Py_ssize_t j; + + for (i = 0; i < PyList_GET_SIZE(nodes); ++i) { + int named = 0; + if (PyList_GET_ITEM(nodes, i) == Py_None) { + continue; + } + for (j = 0; j < PyList_GET_SIZE(specs) && !named; ++j) { + named = PyLong_AsSsize_t(PyTuple_GET_ITEM(PyList_GET_ITEM(specs, j), 2)) == i; + } + if (!named) { + PyXmlSec_LxmlShadowVacateIdSlot(entry, i); + } + } +} + +// Vacates the slots of nodes nothing can reach any more. Registration churn +// on a long-lived document — elements created, registered, dropped — would +// otherwise pin every one of those elements, and its subtree, for as long as +// the document lives, where the fast path's own registration dies with the +// element. Runs before every registration, so the entry stays bounded by the +// registrations that can still be replayed. +static void PyXmlSec_LxmlShadowReclaimIdNodes(PyObject* entry) { + PyObject* doc = PyTuple_GET_ITEM(entry, PYXMLSEC_ID_ENTRY_DOC); + PyObject* nodes = PyTuple_GET_ITEM(entry, PYXMLSEC_ID_ENTRY_NODES); + PyObject* root = NULL; // this document's root element, one lookup for the whole pass + Py_ssize_t i; + + for (i = 0; i < PyList_GET_SIZE(nodes); ++i) { + PyObject* node = PyList_GET_ITEM(nodes, i); // borrowed + // Anything the registry does not hold alone is reachable, and a node + // adopted into another document stands for nothing here anyway. + if (node == Py_None || Py_REFCNT(node) != 1 + || (PyObject*)((PyXmlSec_LxmlElementPtr)node)->_doc != doc) { + continue; + } + if (root == NULL && (root = PyXmlSec_LxmlDocumentRoot(node)) == NULL) { + PyErr_Clear(); + break; + } + if (PyXmlSec_LxmlShadowIdNodeIsDead(node, root)) { + PyXmlSec_LxmlShadowVacateIdSlot(entry, i); + } + } + Py_XDECREF(root); +} + +// The entry's slot holding `node`, reused from a vacated one or appended. +// Returns the index, or -1 with an exception set. +static Py_ssize_t PyXmlSec_LxmlShadowIdSlot(PyObject* entry, PyObject* node) { + PyObject* nodes = PyTuple_GET_ITEM(entry, PYXMLSEC_ID_ENTRY_NODES); + Py_ssize_t n = PyList_GET_SIZE(nodes); + Py_ssize_t vacant = -1; + Py_ssize_t i; + + for (i = 0; i < n; ++i) { + PyObject* item = PyList_GET_ITEM(nodes, i); // borrowed + if (item == node) { + return i; // already held here, and so already dropped elsewhere + } + if (item == Py_None && vacant < 0) { + vacant = i; + } + } + if (vacant < 0) { + if (PyList_Append(nodes, node) < 0) { + return -1; + } + vacant = n; + } else { + Py_INCREF(node); + PyList_SetItem(nodes, vacant, node); // steals the reference + } + // One reference per registered element, so that the liveness test can + // account for exactly the references the registry itself holds. + PyXmlSec_LxmlShadowForgetIdNode(entry, node); + return vacant; +} + // Drops the entries of documents nobody but the registry still references. // Best effort: on an allocation failure the entries simply survive until the // next registration prunes them. @@ -1332,18 +1506,21 @@ static int PyXmlSec_LxmlShadowIdIsTaken(PyXmlSec_LxmlElementPtr element, PyObjec if (entry == NULL) { return 0; } + // A registration whose element nothing can reach any more claims nothing: + // the fast path lost that attribute — and libxml2's id entry for it — + // when lxml freed the element. + PyXmlSec_LxmlShadowReclaimIdNodes(entry); nodes = PyTuple_GET_ITEM(entry, PYXMLSEC_ID_ENTRY_NODES); specs = PyTuple_GET_ITEM(entry, PYXMLSEC_ID_ENTRY_SPECS); for (i = 0; i < PyList_GET_SIZE(specs); ++i) { - PyObject* spec = PyList_GET_ITEM(specs, i); // (name, ns, node index, subtree) + PyObject* spec = PyList_GET_ITEM(specs, i); // (name, ns, node index) PyObject* node = PyList_GET_ITEM(nodes, PyLong_AsSsize_t(PyTuple_GET_ITEM(spec, 2))); PyObject* spec_ns = PyTuple_GET_ITEM(spec, 1); const char* spec_name = PyUnicode_AsUTF8(PyTuple_GET_ITEM(spec, 0)); const char* spec_href; - int subtree = PyObject_IsTrue(PyTuple_GET_ITEM(spec, 3)); - if (spec_name == NULL || subtree < 0) { + if (spec_name == NULL) { return -1; } spec_href = spec_ns == Py_None ? NULL : PyUnicode_AsUTF8(spec_ns); @@ -1355,41 +1532,10 @@ static int PyXmlSec_LxmlShadowIdIsTaken(PyXmlSec_LxmlElementPtr element, PyObjec if (node == Py_None || ((PyXmlSec_LxmlElementPtr)node)->_doc != element->_doc) { continue; } - if (subtree) { - // add_ids: the spec claims the value for whichever element of the - // subtree carries it, xmlSecAddIDs matching by name alone. - PyObject* matches = PyXmlSec_LxmlXPath(node, "descendant-or-self::*[@*[local-name()=$n]=$v]", - PyTuple_GET_ITEM(spec, 0), value); - Py_ssize_t j; - Py_ssize_t n; - if (matches == NULL) { - return -1; - } - n = PySequence_Size(matches); - for (j = 0; j < n && !taken; ++j) { - PyObject* match = PySequence_GetItem(matches, j); - if (match == NULL) { - Py_DECREF(matches); - return -1; - } - taken = PyXmlSec_LxmlShadowSpecClaims((PyXmlSec_LxmlElementPtr)match, spec_name, spec_href, - element, key, value); - Py_DECREF(match); - if (taken < 0) { - Py_DECREF(matches); - return -1; - } - } - Py_DECREF(matches); - if (n < 0) { - return -1; - } - } else { - taken = PyXmlSec_LxmlShadowSpecClaims((PyXmlSec_LxmlElementPtr)node, spec_name, spec_href, - element, key, value); - if (taken < 0) { - return -1; - } + taken = PyXmlSec_LxmlShadowSpecClaims((PyXmlSec_LxmlElementPtr)node, spec_name, spec_href, + element, key, value); + if (taken < 0) { + return -1; } if (taken) { return 1; @@ -1398,29 +1544,47 @@ static int PyXmlSec_LxmlShadowIdIsTaken(PyXmlSec_LxmlElementPtr element, PyObjec return 0; } +// The elements a registration covers, in document order: `element` alone, or +// — for add_ids — the subtree rooted at it, the scope xmlSecAddIDs walks. +// New reference to a list, or NULL with an exception set. +static PyObject* PyXmlSec_LxmlShadowIdTargets(PyXmlSec_LxmlElementPtr element, int subtree) { + PyObject* targets; + + if (!subtree) { + return Py_BuildValue("[O]", (PyObject*)element); + } + targets = PyObject_CallMethod((PyObject*)element, "xpath", "s", "descendant-or-self::*"); + if (targets != NULL && !PyList_Check(targets)) { + PyErr_SetString(PyXmlSec_InternalError, "unexpected elements."); + Py_CLEAR(targets); + } + return targets; +} + int PyXmlSec_LxmlShadowRecordIds(PyXmlSec_LxmlElementPtr element, PyObject* names, const char* ns, int subtree) { PyObject* key = NULL; PyObject* created = NULL; + PyObject* targets = NULL; PyObject* spec = NULL; PyObject* entry = NULL; PyObject* nodes; PyObject* specs; Py_ssize_t nnodes = 0; Py_ssize_t nspecs = 0; - Py_ssize_t idx; + Py_ssize_t t; Py_ssize_t i; - int contains; int fresh = 0; int result = -1; + targets = PyXmlSec_LxmlShadowIdTargets(element, subtree); key = PyLong_FromVoidPtr((void*)element->_doc); - if (key == NULL) { + if (targets == NULL || key == NULL) { goto DONE; } + PyXmlSec_LxmlShadowPruneIdRegistry(); entry = PyDict_GetItem(PyXmlSec_LxmlShadowIdRegistry, key); // borrowed if (entry == NULL) { - PyXmlSec_LxmlShadowPruneIdRegistry(); nodes = PyList_New(0); specs = PyList_New(0); if (nodes != NULL && specs != NULL) { @@ -1433,39 +1597,54 @@ int PyXmlSec_LxmlShadowRecordIds(PyXmlSec_LxmlElementPtr element, PyObject* name } entry = created; fresh = 1; + } else { + PyXmlSec_LxmlShadowReclaimIdNodes(entry); } - // One reference per registered element, so that the liveness test can - // account for exactly the references the registry itself holds. nodes = PyTuple_GET_ITEM(entry, PYXMLSEC_ID_ENTRY_NODES); specs = PyTuple_GET_ITEM(entry, PYXMLSEC_ID_ENTRY_SPECS); nnodes = PyList_GET_SIZE(nodes); nspecs = PyList_GET_SIZE(specs); - idx = -1; - for (i = 0; i < nnodes; ++i) { - if (PyList_GET_ITEM(nodes, i) == (PyObject*)element) { - idx = i; - break; - } - } - if (idx < 0) { - if (PyList_Append(nodes, (PyObject*)element) < 0) { - goto DONE; - } - idx = nnodes; - PyXmlSec_LxmlShadowForgetIdNode(entry, (PyObject*)element); - } - for (i = 0; i < PyList_GET_SIZE(names); ++i) { - spec = Py_BuildValue("(Ozni)", PyList_GET_ITEM(names, i), ns, idx, subtree); - if (spec == NULL) { - goto DONE; - } - contains = PySequence_Contains(specs, spec); - if (contains < 0 || (!contains && PyList_Append(specs, spec) < 0)) { - goto DONE; + // One spec per attribute the scope carries *now*, element by element in + // document order and, within an element, in the order of `names`: the + // registration xmlSecAddIDs makes, taken at the call rather than left to + // the replay, which would see whatever the tree had become by then. + for (t = 0; t < PyList_GET_SIZE(targets); ++t) { + PyObject* target = PyList_GET_ITEM(targets, t); // borrowed + Py_ssize_t idx = -1; + for (i = 0; i < PyList_GET_SIZE(names); ++i) { + PyObject* name = PyList_GET_ITEM(names, i); // borrowed + const char* text = PyUnicode_AsUTF8(name); + PyObject* value; + int carried; + int contains; + + if (text == NULL) { + goto DONE; + } + value = PyXmlSec_LxmlAttrFind((PyXmlSec_LxmlElementPtr)target, text, ns, NULL); + if (value == NULL) { + goto DONE; + } + carried = value != Py_None; + Py_DECREF(value); + if (!carried) { + continue; + } + if (idx < 0 && (idx = PyXmlSec_LxmlShadowIdSlot(entry, target)) < 0) { + goto DONE; + } + spec = Py_BuildValue("(Ozn)", name, ns, idx); + if (spec == NULL) { + goto DONE; + } + contains = PySequence_Contains(specs, spec); + if (contains < 0 || (!contains && PyList_Append(specs, spec) < 0)) { + goto DONE; + } + Py_CLEAR(spec); } - Py_CLEAR(spec); } result = 0; @@ -1478,15 +1657,24 @@ int PyXmlSec_LxmlShadowRecordIds(PyXmlSec_LxmlElementPtr element, PyObject* name PyObject* value; PyObject* tb; PyErr_Fetch(&type, &value, &tb); - if (fresh ? PyDict_DelItem(PyXmlSec_LxmlShadowIdRegistry, key) < 0 - : (PyList_SetSlice(PyTuple_GET_ITEM(entry, PYXMLSEC_ID_ENTRY_SPECS), nspecs, PY_SSIZE_T_MAX, NULL) < 0 - || PyList_SetSlice(PyTuple_GET_ITEM(entry, PYXMLSEC_ID_ENTRY_NODES), nnodes, PY_SSIZE_T_MAX, NULL) < 0)) { - PyErr_Clear(); + if (fresh) { + if (PyDict_DelItem(PyXmlSec_LxmlShadowIdRegistry, key) < 0) { + PyErr_Clear(); + } + } else { + if (PyList_SetSlice(PyTuple_GET_ITEM(entry, PYXMLSEC_ID_ENTRY_SPECS), nspecs, PY_SSIZE_T_MAX, NULL) < 0 + || PyList_SetSlice(PyTuple_GET_ITEM(entry, PYXMLSEC_ID_ENTRY_NODES), nnodes, PY_SSIZE_T_MAX, NULL) < 0) { + PyErr_Clear(); + } + // A slot reused for one of the rolled-back specs holds a node no + // surviving spec names any more. + PyXmlSec_LxmlShadowCompactIdNodes(entry); } PyErr_Restore(type, value, tb); } Py_XDECREF(key); Py_XDECREF(created); + Py_XDECREF(targets); Py_XDECREF(spec); return result; } @@ -1554,27 +1742,13 @@ static void PyXmlSec_LxmlShadowAddId(xmlDocPtr doc, xmlNodePtr node, const xmlCh xmlFree(value); } -// `node`, its siblings and their descendants. -static void PyXmlSec_LxmlShadowAddIdsBelow(xmlDocPtr doc, xmlNodePtr node, const xmlChar* name, const xmlChar* ns) { - for (; node != NULL; node = node->next) { - if (node->type == XML_ELEMENT_NODE) { - PyXmlSec_LxmlShadowAddId(doc, node, name, ns); - PyXmlSec_LxmlShadowAddIdsBelow(doc, node->children, name, ns); - } - } -} - // Applies one recorded spec to `node`, the copy's counterpart of the element -// it was registered for: that node alone (register_id), or the subtree rooted -// at it (add_ids, which is the scope xmlSecAddIDs walks). -static void PyXmlSec_LxmlShadowApplyIdSpec(xmlDocPtr doc, xmlNodePtr node, const xmlChar* name, const xmlChar* ns, int subtree) { +// it was registered for. +static void PyXmlSec_LxmlShadowApplyIdSpec(xmlDocPtr doc, xmlNodePtr node, const xmlChar* name, const xmlChar* ns) { if (node == NULL || node->type != XML_ELEMENT_NODE) { return; } PyXmlSec_LxmlShadowAddId(doc, node, name, ns); - if (subtree) { - PyXmlSec_LxmlShadowAddIdsBelow(doc, node->children, name, ns); - } } // Replays the specs recorded for the shadow's live document onto the copy, @@ -1605,12 +1779,11 @@ static int PyXmlSec_LxmlShadowReplayIds(PyXmlSec_LxmlShadow* shadow, PyObject* l specs = PyTuple_GET_ITEM(entry, PYXMLSEC_ID_ENTRY_SPECS); n = PyList_GET_SIZE(specs); for (i = 0; i < n; ++i) { - PyObject* spec = PyList_GET_ITEM(specs, i); // (name, ns, node index, subtree) + PyObject* spec = PyList_GET_ITEM(specs, i); // (name, ns, node index) PyObject* node = PyList_GET_ITEM(nodes, PyLong_AsSsize_t(PyTuple_GET_ITEM(spec, 2))); PyObject* top = NULL; const char* name = PyUnicode_AsUTF8(PyTuple_GET_ITEM(spec, 0)); const char* ns = PyTuple_GET_ITEM(spec, 1) == Py_None ? NULL : PyUnicode_AsUTF8(PyTuple_GET_ITEM(spec, 1)); - int subtree = PyObject_IsTrue(PyTuple_GET_ITEM(spec, 3)); int depth; if (name == NULL || (ns == NULL && PyErr_Occurred())) { @@ -1628,7 +1801,7 @@ static int PyXmlSec_LxmlShadowReplayIds(PyXmlSec_LxmlShadow* shadow, PyObject* l // for an element that has since left it applies to nothing here. if (top == live_top) { PyXmlSec_LxmlShadowApplyIdSpec(shadow->doc, PyXmlSec_LxmlShadowWalkNode(copy_top, path, depth), - (const xmlChar*)name, (const xmlChar*)ns, subtree); + (const xmlChar*)name, (const xmlChar*)ns); } Py_DECREF(top); } diff --git a/src/lxml.h b/src/lxml.h index 01599818..f0c1e9a8 100644 --- a/src/lxml.h +++ b/src/lxml.h @@ -157,11 +157,10 @@ xmlNodePtr PyXmlSec_LxmlShadowImportElement(PyXmlSec_LxmlShadow* shadow, PyXmlSe // lxml's ID hash with our libxml2, so they record the id-attribute spec for // the element's document here, and every BeginDoc replays the recorded specs // onto its copy. The registry keeps `element` itself, so the replay applies a -// spec to exactly the node it was registered for — `subtree` extends it to -// the node's descendants, as add_ids (xmlSecAddIDs) does, while register_id -// registers the one node. Registering every matching attribute of the -// document instead would let an unrelated element with the same id value win -// the lookup and steer a `#id` reference away from the registered one. +// spec to exactly the node it was registered for. Registering every matching +// attribute of the document instead would let an unrelated element with the +// same id value win the lookup and steer a `#id` reference away from the +// registered one. // // This is the whole of register_id under the shadow: it validates what the // fast path validates — "missing attribute." for an absent one, "duplicated @@ -174,6 +173,11 @@ int PyXmlSec_LxmlShadowRegisterId(PyXmlSec_LxmlElementPtr element, const char* n // add_ids: either every name is recorded or, if anything fails, none is — // the fast path likewise builds its complete list before it touches the // document, so a bad list leaves no half-applied registration behind. +// `subtree` extends the registration to the node's descendants, the scope +// add_ids (xmlSecAddIDs) walks; it is expanded here and now into one spec per +// element carrying one of the names, which is the snapshot xmlSecAddIDs takes +// — an element that grows the attribute, or joins the scope, after the call +// was never registered by the fast path either. int PyXmlSec_LxmlShadowRecordIds(PyXmlSec_LxmlElementPtr element, PyObject* names, const char* ns, int subtree); // get version numbers for libxml2 both compiled and loaded diff --git a/src/tree.c b/src/tree.c index 1a8470d9..de089b4a 100644 --- a/src/tree.c +++ b/src/tree.c @@ -195,9 +195,9 @@ static PyObject* PyXmlSec_TreeAddIds(PyObject* self, PyObject *args, PyObject *k // Shadow mode: registering IDs on lxml's document with our libxml2 is // exactly the cross-library write issue #356 forbids. Record the - // attribute names instead; every whole-document shadow (sign, verify, - // decrypt) replays them onto its private copy, over the subtree rooted at - // `node` — the scope xmlSecAddIDs walks below. + // attributes the subtree rooted at `node` — the scope xmlSecAddIDs walks + // below — carries right now instead; every whole-document shadow (sign, + // verify, decrypt) replays those onto its private copy. if (PyXmlSec_LxmlShadowIsActive()) { // Materialize and validate the whole list before recording any of it, // as the fast path below does before it calls xmlSecAddIDs: a bad diff --git a/tests/test_ds.py b/tests/test_ds.py index 97e69129..3796b4f7 100644 --- a/tests/test_ds.py +++ b/tests/test_ds.py @@ -132,6 +132,35 @@ def test_register_id_with_namespace_without_attribute(self): with self.assertRaisesRegex(xmlsec.Error, 'missing attribute.'): ctx.register_id(sign, 'Id', id_ns='foo') + def sign_id_reference(self, root, uri): + """Signs `root` with a single reference to `uri` and returns the digest of that reference.""" + sign = xmlsec.template.create(root, consts.TransformExclC14N, consts.TransformRsaSha1, ns='ds') + root.append(sign) + ref = xmlsec.template.add_reference(sign, consts.TransformSha1, uri=uri) + xmlsec.template.add_transform(ref, consts.TransformExclC14N) + ctx = xmlsec.SignatureContext() + ctx.key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem) + ctx.sign(sign) + return xmlsec.tree.find_node(sign, consts.NodeDigestValue, consts.DSigNs).text + + def test_add_ids_ignores_an_id_that_joins_the_scope_later(self): + """add_ids registers what the scope carries at the call, which is where xmlSecAddIDs walks it.""" + root = etree.fromstring(b'') + xmlsec.tree.add_ids(root, ['ID']) + etree.SubElement(root, 'B', {'ID': 'b'}) + with self.assertRaisesRegex(xmlsec.Error, 'failed to sign'): + self.sign_id_reference(root, '#b') + + def test_add_ids_claims_a_value_in_document_order(self): + """xmlSecAddIDs walks the scope element by element, the names within each: claims "v" first.""" + xml = b'yx' + root = etree.fromstring(xml) + xmlsec.tree.add_ids(root, ['A', 'B']) + both = self.sign_id_reference(root, '#v') + root = etree.fromstring(xml) + xmlsec.tree.add_ids(root, ['B']) + self.assertEqual(both, self.sign_id_reference(root, '#v')) + def test_sign_bad_args(self): ctx = xmlsec.SignatureContext() ctx.key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem) @@ -595,6 +624,37 @@ def test_registration_survives_other_live_documents(self): ctx.key = xmlsec.Key.from_file(self.path('rsapub.pem'), format=consts.KeyDataFormatPem) ctx.verify(sign) # resolves #ID through the registration made before the churn + def test_registration_dies_with_its_element(self): + """A dropped element takes its registration with it: libxml2 drops the id entry when it frees the attribute.""" + root = etree.fromstring(b'') + ctx = xmlsec.SignatureContext() + for _ in range(2): # the second registration claims the value the first one did + node = etree.SubElement(root, 'T', {'ID': 'x'}) + ctx.register_id(node, 'ID') + root.remove(node) + del node + + def test_registration_survives_its_element_being_removed(self): + """An element out of its document is not gone: lxml keeps the subtree, and the id entry stands.""" + root = etree.fromstring(b'') + ctx = xmlsec.SignatureContext() + removed = root[0] + ctx.register_id(removed, 'ID') + root.remove(removed) + with self.assertRaisesRegex(xmlsec.Error, 'duplicated id.'): + ctx.register_id(root[0], 'ID') + + def test_registration_survives_a_descendant_being_held(self): + """A proxy anywhere in a removed subtree keeps it alive, registration included.""" + root = etree.fromstring(b'') + ctx = xmlsec.SignatureContext() + ctx.register_id(root[0], 'ID') + inner = root[0][0] + root.remove(inner.getparent()) + with self.assertRaisesRegex(xmlsec.Error, 'duplicated id.'): + ctx.register_id(root[0], 'ID') + del inner + def test_registration_survives_a_sibling_being_adopted_away(self): """An element moved into another document must not make its old document's registrations look collectable.""" root = etree.fromstring(b'x')