diff --git a/.github/scripts/manylinux_build_and_test.sh b/.github/scripts/manylinux_build_and_test.sh
index ce8301a7..7d526a06 100644
--- a/.github/scripts/manylinux_build_and_test.sh
+++ b/.github/scripts/manylinux_build_and_test.sh
@@ -56,6 +56,11 @@ echo "== [container] Step: Install test dependencies =="
echo "== [container] Step: Run tests =="
/opt/python/${PY_ABI}/bin/pytest -v --color=yes
+# Step: Run tests again on the shadow-copy path (issue #356), which the
+# matched libxml2 build would otherwise never exercise.
+echo "== [container] Step: Run tests (forced shadow path) =="
+PYXMLSEC_FORCE_SHADOW=1 /opt/python/${PY_ABI}/bin/pytest -v --color=yes
+
# Step: Fix mounted workspace file ownership on host
echo "== [container] Step: Fix mounted workspace file ownership on host =="
chown -R "${HOST_UID}:${HOST_GID}" dist wheelhouse build libs || true
diff --git a/.github/workflows/linuxbrew.yml b/.github/workflows/linuxbrew.yml
index 51b0db1e..c4ab34b0 100644
--- a/.github/workflows/linuxbrew.yml
+++ b/.github/workflows/linuxbrew.yml
@@ -48,3 +48,6 @@ jobs:
pip3 install --upgrade --no-binary=lxml -r requirements-test.txt
pip3 install xmlsec --only-binary=xmlsec --no-index --find-links=dist/
pytest -v --color=yes
+ # Same suite on the shadow-copy path (issue #356), which the
+ # matched libxml2 build would otherwise never exercise.
+ PYXMLSEC_FORCE_SHADOW=1 pytest -v --color=yes
diff --git a/.github/workflows/macosx.yml b/.github/workflows/macosx.yml
index c9d8034e..db765ed3 100644
--- a/.github/workflows/macosx.yml
+++ b/.github/workflows/macosx.yml
@@ -87,6 +87,14 @@ jobs:
run: |
coverage run -m pytest -v --color=yes
+ # Same suite on the shadow-copy path (issue #356), which the matched
+ # libxml2 build would otherwise never exercise.
+ - name: Run tests (forced shadow path)
+ env:
+ PYXMLSEC_FORCE_SHADOW: "1"
+ run: |
+ pytest -v --color=yes
+
- name: Report coverage to codecov
if: matrix.static_deps != 'static'
run: |
diff --git a/developer.md b/developer.md
new file mode 100644
index 00000000..97625d5a
--- /dev/null
+++ b/developer.md
@@ -0,0 +1,411 @@
+# Decoupling lxml and xmlsec across libxml2 (#356)
+
+**TL;DR** — `python-xmlsec` crashes when `lxml` and `xmlsec1` are built
+against different `libxml2` versions, because it passes raw libxml2 node
+pointers between them. Every xmlsec call now runs on a private *copy* of the
+element ("shadow") owned by our libxml2, and the change it makes is reflected
+back into the live lxml tree afterwards, so only serialized bytes ever cross
+the boundary. Converting a binding is four lines; when the libxml2 versions
+match, the copy is skipped and the old direct code runs unchanged.
+
+## The problem
+
+`python-xmlsec` glues together two libraries that both build on **libxml2**:
+lxml (the tree the user edits in Python) and xmlsec1 (the C library that
+signs/encrypts). Historically the extension reached into an lxml `_Element`
+for its raw `xmlNodePtr` and handed it to xmlsec1. That is only safe when both
+libraries link the *same* libxml2 at runtime — and they often don't, because
+lxml wheels bundle their own. Two libxml2 builds touching one tree means
+mismatched struct layouts, dictionaries and allocators: segfaults,
+double-frees, wrong signatures
+([#356](https://github.com/xmlsec/python-xmlsec/issues/356)). The only
+mitigation so far was refusing to import on a version mismatch (#283).
+
+## The fix: shadow copies
+
+Bytes have no ABI. Each binding therefore does:
+
+```text
+ lxml element ──(lxml's libxml2 serializes)──► bytes
+ bytes ──(our libxml2 parses)──► private "shadow" copy
+ xmlsec mutates the shadow (it never sees an lxml node)
+ shadow ──(our libxml2 dumps)──► bytes ──(lxml parses)──► changes grafted
+ into the live tree
+```
+
+The user-visible behaviour is unchanged: the input element gains exactly what
+xmlsec added, the returned node is live in the caller's tree (incremental
+building like `add_transform(ref, ...)` keeps working, and proxies the caller
+holds stay valid), and the serialized output is byte-identical — namespaces
+and xmlsec's `"\n"` formatting included.
+
+The whole mechanism lives in [src/lxml.c](src/lxml.c), with the contract in
+[src/lxml.h](src/lxml.h). A binding looks like this (`add_reference`):
+
+```c
+PyXmlSec_LxmlShadow shadow;
+if (PyXmlSec_LxmlShadowBegin(&shadow, node) < 0) goto ON_FAIL; // lxml → bytes → our copy
+Py_BEGIN_ALLOW_THREADS;
+res = xmlSecTmplSignatureAddReference(shadow.root, ...); // xmlsec mutates the copy
+Py_END_ALLOW_THREADS;
+result = PyXmlSec_LxmlShadowEnd(&shadow, res, "cannot add reference."); // reflect back, return lxml node
+```
+
+No per-function callback or context struct: the call site is the plain xmlsec
+call, and the helper works out what the call did.
+
+## The API
+
+Three `Begin` flavours make the copy, four `End` functions consume it. Every
+End always releases the copy, on success and on error.
+
+| Function | Use |
+| --- | --- |
+| `Begin(&shadow, element)` | copy of the element's subtree; `shadow.root` is the copy |
+| `BeginDoc(&shadow, element, &target)` | copy of the element's whole document, for calls that follow references or walk upward; `target` is the copy's counterpart of `element`; registered IDs are replayed onto the copy |
+| `BeginNewDoc(&shadow, element)` → `xmlDocPtr` | no copy at all: a private document for calls that only build a *detached* subtree (`create`) |
+| `End(&shadow, res, error)` → element | the lxml element for a result node — grafted into the live tree if the call created it, or the existing live element (with the attributes / prefix the call changed) if it found it; NULL `res` raises `error` |
+| `EndFind(&shadow, res)` → element or `None` | the same for read-only finders; NULL is "not found" |
+| `Reflect(&shadow, rv, error)` → int | for calls returning only a status: `rv < 0` raises `error`, otherwise every change is reflected |
+| `Discard(&shadow)` | release without reflecting (read-only calls, error paths before End) |
+
+Plus three helpers for the call sites whose *semantics* differ per mode:
+`IsActive()` (which path is on), `ImportElement()` (encrypt_xml's template
+import into the shadow document) and `RegisterId()` (ID registration, below).
+
+Which pair a binding uses follows from what the xmlsec call does:
+
+| The xmlsec call ... | Begin | End | Bindings |
+| --- | --- | --- | --- |
+| adds or finds a node under the element | `Begin` | `End` | every `xmlSecTmpl*` add / ensure call |
+| builds a detached subtree, needs only a document | `BeginNewDoc` | `End` | `create`, `encrypted_data_create` |
+| searches the subtree, read-only | `Begin` | `EndFind` | `find_child`, `find_node` |
+| searches upward, read-only | `BeginDoc` | `EndFind` | `find_parent` |
+| mutates the subtree, returns a status | `Begin` | `Reflect` | `transform_add_c14n_inclusive_namespaces`, `encrypt_binary`, `encrypt_uri` |
+| mutates anywhere in the document, returns a status | `BeginDoc` | `Reflect` | `sign` |
+| reads the document, returns a status | `BeginDoc` | `Discard` | `verify` |
+| *replaces* the element or its content | `BeginDoc` | remove the consumed live node/content, then `Reflect` | `encrypt_xml`, `decrypt` |
+
+Rules every call site must keep:
+
+- swap `node->_c_node` for `shadow.root` (or `target`) and change **nothing
+ else** about the xmlsec call or its error string;
+- run **exactly one** xmlsec call between Begin and End, and no Python code
+ in between (the `Py_*_ALLOW_THREADS` pair is fine — the call is pure C);
+- call exactly one End function after a successful Begin.
+
+**Invariant, to check by review:** every C function that accepts an lxml
+element (`PyXmlSec_LxmlElementConverter`) either calls a
+`PyXmlSec_LxmlShadowBegin*` helper or is one of the four dual-body functions
+(`register_id`, `add_ids`, `encrypt_xml`, `decrypt`), which must consult
+`IsActive()` before touching a raw node; and `->_c_node` / `->_c_doc` appear
+only in those four and in the helpers' fast-path branches. A quick
+`grep -n '\->_c_\(node\|doc\)' src/*.c` lists every crossing to check.
+
+## How the reflection works
+
+`Begin` serializes the element with lxml's own `etree.tostring`, re-parses
+the bytes with `xmlReadMemory`, and tags every node of the copy through the
+libxml2 `_private` field (never serialized, never touched by the parser or
+xmlsec). After the call, whatever is untagged is what the call created. The
+reflection then walks the tagged structure of the copy in document order and
+records two kinds of *site*:
+
+- **graft** — a fresh node (element, comment, PI) to insert at its child
+ index. Fresh subtrees are grafted wholesale; the scan never descends into
+ them.
+- **sync** — a tagged parent whose children changed gets its text slots (its
+ `.text` and each child's `.tail`) copied over from the re-parsed copy. That
+ covers everything xmlsec does to text: the `"\n"` formatting around a new
+ node, values filled into empty elements (`DigestValue`, `SignatureValue`),
+ and content it removed (encrypt `Type=Content`). A tag therefore records
+ more than "this node existed": it also holds the node's child count, since
+ a removal leaves no fresh node behind and only the changed count shows it
+ happened, and an FNV-1a fingerprint of a text node's own content, since
+ appending to a text node (`xmlNodeAddContent` onto a trailing text child)
+ changes neither the node nor the count. Writing an element's value goes
+ through `xmlNodeSetContent`, which frees the old text node and parses a
+ fresh one, so re-signing over an existing `DigestValue` is caught by the
+ fresh-node rule; the fingerprint is what keeps the invariant ("a parent
+ whose children changed gets a sync") from depending on that libxml2
+ internal.
+
+Sites are addressed by **child-index paths** from the copy root, counting
+exactly the node types lxml exposes as children (elements, comments, PIs,
+entity refs), so a path recorded on the raw copy resolves identically through
+lxml's `__getitem__` / `insert` on the live tree. The reflection is
+**two-phase**: every payload is fetched from the re-parsed copy first, while
+it is still in its final state, then everything is applied to the live tree in
+document order (a graft moves a node out of the re-parsed copy, which would
+invalidate later fetches; each live insert makes the later, larger indices
+valid; a parent's sync is recorded after its grafts).
+
+Two serialization details are load-bearing for byte-identical signatures:
+the **whole** copy is dumped (`xmlDocDumpMemory`), not just the fresh nodes,
+so ancestor-declared namespaces and the formatting siblings survive the lxml
+re-parse without any manual fix-up; and lxml's `insert` carries a node's tail
+along and reconciles namespaces against the live ancestry.
+
+`End` then maps the result node back: it records the path of `res` in the
+copy before reflecting, and after the reflection the live tree mirrors the
+copy's element structure, so the same path resolves to the live counterpart —
+whether the call created it (now grafted) or found it. For a found node the
+tree did not grow there, so the live element is returned with the attributes
+the call set (`Id`) synced onto it; a renamed namespace prefix
+(`encrypted_data_ensure_key_info(ns=...)` on an existing `KeyInfo`) has no
+lxml API, so the live element is swapped for the copy's version and the
+caller gets a new proxy object.
+
+`BeginDoc` records the element's position through lxml's API (`getparent` /
+`index`), serializes `element.getroottree()` — comments/PIs outside the root
+and the internal DTD subset survive — and hands back the copy's counterpart;
+`shadow.element` becomes the live *root*, which is where the reflection maps
+paths onto.
+
+An element **removed from its document** is a shape of its own: lxml leaves
+such a subtree pointing at the document it left, and so does the raw path —
+xmlsec works on a node outside the tree whose `doc` still answers its `#id`
+references (a template taken out of a document and signed with `URI=""`
+digests that document, without itself in it). The copy holds both: the
+document is copied as always, and the removed subtree is copied into it as an
+*unlinked* node beside its tree (`shadow.unlinked`), which is then what
+`shadow.root` / `shadow.element` and every path map between. Registered IDs
+are replayed twice, once for each of the two live tops. Such a node cannot be
+replaced (libxml2 needs a parent to put the replacement in), on either path.
+
+`BeginNewDoc` creates an empty private document; `End` roots the
+detached result there, dumps it and returns it as a new detached lxml element
+(in a document of its own until grafted; lxml moves it when the caller
+appends it, like the raw path's detached node).
+
+## The fast path: shadows only when needed
+
+Copying is pointless when lxml links the same libxml2 as the extension — the
+raw-node behaviour that shipped for years is safe then, and it is the only
+configuration the import guard currently lets run. `Begin`/`End` are
+dual-path, decided once at import:
+
+- **matched versions** (the guard passed): `Begin` aliases the live
+ `_c_node` into `shadow.root` with no serialization, `End` just wraps the
+ node xmlsec returned, `Reflect` does nothing — machine-identical to the
+ pre-shadow code, zero overhead;
+- **mismatch** (import allowed via `PYXMLSEC_SKIP_VERSION_CHECK` today,
+ automatic once the guard becomes a mode switch), **or `PYXMLSEC_FORCE_SHADOW`
+ set**: the full shadow round-trip.
+
+Call sites cannot tell the difference. `PYXMLSEC_FORCE_SHADOW` exists so CI
+keeps the shadow path exercised on matched libraries (the workflows run the
+suite twice), where it must also pass the full suite. Measured cost of the
+shadow path per template call: about 8x (72 µs vs 8.6 µs for create +
+add_reference + add_transform + ensure_key_info); whole-document operations
+scale with document size.
+
+The re-parse on our side uses `XML_PARSE_HUGE` and the lxml side a cached
+`XMLParser(huge_tree=True)`, so a `CipherValue` above libxml2's 10 MB
+text-node limit (large `encrypt_binary` payloads) or a document the user
+parsed with `huge_tree` still reflects. That is safe: what gets parsed is
+lxml's own dump of a tree it already parsed.
+
+> The shadow decouples *lxml* from xmlsec. The extension and `libxmlsec1`
+> must still share one libxml2 (wheels and static builds guarantee this).
+
+## ID registration under the shadow
+
+`SignatureContext.register_id` and `tree.add_ids` used to write lxml's ID
+hash with our libxml2 — exactly the cross-library access the shadow forbids.
+Under the shadow they record the id-attribute specs in a registry keyed by
+document identity (`RegisterId`, `RecordIds`), and every `BeginDoc` replays them onto its
+copy so that `#id` references resolve during sign/verify/decrypt. An entry
+keeps the registered elements themselves, so a spec is replayed at exactly
+the node it was registered for. Registering every matching attribute of the
+copy instead would be unsafe, not merely generous: an unrelated element
+sharing the id value would claim it first and a `#id` reference could then
+resolve to content the caller never registered.
+
+`add_ids` covers a whole scope, and that scope is walked **at the call**:
+`RecordIds` expands it into one spec per element carrying one of the names,
+element by element in document order and, within an element, in the order of
+the names — the registration `xmlSecAddIDs` makes, at the moment it makes it.
+Leaving the scope to be walked at the replay would register whatever the tree
+had become by the time something was signed: an element that grew the
+attribute, or joined the scope, after the call would resolve under the shadow
+where the fast path never registered it, and two elements claiming one value
+under different names would be ordered by name rather than by document
+order — a `#id` covering different content on the two paths. lxml's
+classes refuse weak references, so the entry keeps strong references (to the
+document and to those elements) instead: the key (the document's address) can
+then never go stale, and since every element proxy holds a reference to its
+document, a document whose reference count is exactly what the registry holds
+— and whose registered elements nothing else holds — is provably unreachable,
+so its entry, and the document with it, is dropped before the next
+registration. Registrations are retired one by one on the same principle: a
+slot whose proxy the registry alone holds, hanging in a tree that is not its
+document's, cannot be reached again — lxml keeps an unlinked subtree only for
+as long as a proxy remains somewhere in it, and the fast path's id entry dies
+at exactly that moment too, when libxml2 frees the attribute. The slot is
+vacated (and reused by the next registration), so registering and dropping
+elements on a long-lived document neither grows the registry nor keeps their
+values claimed. The registry therefore tracks the documents still in use and
+never evicts a live one. The two bindings are the only places, together with
+encrypt_xml/decrypt's replacement bodies, that branch on `IsActive()`.
+
+`register_id` still refuses a duplicate id the way the fast path does, and at
+the same call: `xmlGetID(doc, value) != attr` — the test that raises
+`duplicated id.` — is assembled from the two places a registration can live
+under the shadow. What lxml's own parse declared (a DTD id attribute, an
+`xml:id`) is read back through XPath's `id()`, the one door into lxml's id
+hash that passes nothing but strings and elements; what earlier
+`register_id`/`add_ids` calls claimed is read from the registry, spec by
+spec.
+
+Both halves compare *attributes*, not elements: ``
+answers `N` to `id('dup')` whichever attribute is asked about, while the fast
+path registers `ID`, finds `xml:id` holding the value and raises. A registry
+spec is therefore resolved to the attribute its `xmlHasProp`/`xmlHasNsProp`
+would pick, and a match on the element itself only settles the declared half
+when a single attribute of the element carries the value — otherwise the
+declared attribute has to be named outright, which no lxml API does (`id()`
+names elements, and an `ATTLIST` without an `ELEMENT` leaves lxml's DTD
+objects empty). It is then named by copying the document the way a
+whole-document shadow copies it — same base URL, same subsets — and reading
+that copy's own id hash, a copy the registration is refused or recorded
+against anyway. Only a value already declared for the element pays for it.
+
+Deferring the check to the replay
+instead would raise from the wrong call — a later `sign`, and then from every
+later call on that document — and would leave the caller believing a
+registration took that can never win the lookup. `add_ids` keeps its own
+semantics: `xmlSecAddIDs` registers first-wins and never raises.
+
+## Converting a binding
+
+1. Classify the xmlsec call with the table above.
+2. Edit: swap `node->_c_node` for `shadow.root`, wrap the call in the
+ matching Begin/End pair, keep the error string.
+3. Build and run the suite (see below). Add a test asserting the
+ *reflection*: the returned node is live in the caller's tree
+ (`assertIs(node.getroottree().getroot(), root)`) and at the position
+ xmlsec puts it; for find-or-create, a second call returns the same element.
+4. Validate under a real libxml2 mismatch, and on a matched build with
+ `PYXMLSEC_FORCE_SHADOW=1`.
+
+Beware the leak detector in `tests/base.py`: it reruns each test with
+`gc.disable()` and fails on monotonic object-count growth, which plain
+allocation churn can trigger with no real leak. Keep each test small (split
+rather than combine scenarios), prefer `assertIs(parent[0], tr)` over
+building lists to compare, and check stability with
+`PYXMLSEC_TEST_ITERATIONS=50 PYTHONPATH=src python -m pytest tests/`.
+
+## Known divergences and limitations (shadow path only)
+
+All invisible to the documented API:
+
+- created templates (`create`, `encrypted_data_create`) live in their own
+ document until grafted;
+- `encrypted_data_ensure_key_info(ns=...)` on an existing `KeyInfo` returns
+ a new element object rather than the original proxy;
+- `register_id`'s duplicate-id check cannot see an id value that contains
+ whitespace among lxml's declared ids (XPath's `id()` would read it as a
+ list of ids), so such a value is compared against the registry alone; a
+ registration whose element has since been adopted into another document
+ claims nothing, as at replay;
+- a registration follows the element it was made for. lxml drops its own id
+ entry whenever an element is *moved* — even within the one document, and
+ for a whole subtree when an ancestor moves — which the registry cannot
+ observe, so a `#id` the fast path stops resolving after such a move keeps
+ resolving under the shadow. It resolves to the registered element, never to
+ another one: the shadow registers exactly the attributes the caller
+ registered;
+- `encrypt_xml` encrypts a *copy* of the template, so the caller's template
+ proxy is not the returned element; a template attached in the target's own
+ document is unlinked afterwards (keeping its tail text where libxml2 would
+ leave it), so the resulting tree matches the raw path's move;
+- signature/encryption contexts keep no live result nodes after the call
+ (they never usefully did);
+- a mutation site deeper than 256 levels is refused with an internal error,
+ and so is a document nested deeper than 2048 levels — the ceiling libxml2
+ 2.14 and later put on a parse even under `XML_PARSE_HUGE`. The copy's own
+ walks enforce that ceiling for themselves, since an older libxml2 lifts its
+ cap entirely under `HUGE` (2.9.13 parses a 200000-level document) and lxml
+ can hand over a tree that was never parsed on its side at all;
+- when the source document loaded an external DTD subset (`load_dtd=True`),
+ the copy is parsed with `XML_PARSE_DTDLOAD`, so that the IDs the DTD
+ declares type the copy's attributes as well and `#id` references over them
+ resolve. What is fetched is the local file the document's own DOCTYPE
+ names, resolved against the same base URI, with the network still off and
+ attribute defaulting (`XML_PARSE_DTDATTR`) still off — the copy must stay
+ what lxml serialized. A DTD lxml obtained through a Python resolver of its
+ own is invisible to that parse, so the ids it declares are not carried
+ across;
+- a subtree of a document that declares entities in its internal subset
+ (`resolve_entities=False`) is copied by copying the whole document and
+ cutting it back to the element, since the subtree's `&name;` references
+ need their declarations; `encrypt_xml` templates and subtrees removed from
+ their document are still serialized on their own, so a *template* — or a
+ removed subtree — carrying unresolved entity references is not supported (signing and encryption of such a document are refused on both
+ paths anyway — libxml2's c14n rejects entity-reference nodes);
+- operations that replace the **document root** (encrypting the root element
+ with `Type=Element`, decrypting a root `EncryptedData`) morph the live root
+ element in place into the replacement, because lxml's API cannot swap a
+ document's root (`_ElementTree._setroot` only rebinds that one Python
+ object). The result is the same as on the raw path — the returned element
+ is the new root, with the replacement's own namespace declarations and
+ document-level siblings intact — except that the caller's root proxy (and
+ any `_ElementTree` holding it) *becomes* the replacement instead of going
+ stale as a detached copy of the old root. A root replaced by anything but a
+ single element (a `Type=Content` decryption of the root) is refused with
+ `xmlsec.Error`.
+
+## Building & validating
+
+### Under a real mismatch (macOS / homebrew)
+
+lxml wheels bundle libxml2; build the extension against homebrew's (which
+libxmlsec1 links) so only lxml differs — the true #356 scenario. Watch the
+three-way trap: the linker prefers the SDK stub `/usr/lib/libxml2.2.dylib`,
+so rewrite the runtime dependency after building:
+
+```sh
+rm -rf build/ src/xmlsec.cpython-*-darwin.so
+PKG_CONFIG_PATH=/opt/homebrew/opt/libxml2/lib/pkgconfig \
+ python setup.py build_ext --inplace --force
+install_name_tool -change /usr/lib/libxml2.2.dylib \
+ /opt/homebrew/opt/libxml2/lib/libxml2.16.dylib src/xmlsec.cpython-*-darwin.so
+
+# verify the mismatch is real, then run the suite under it
+PYXMLSEC_SKIP_VERSION_CHECK=1 PYTHONPATH=src python -c \
+ "import xmlsec; from lxml import etree; \
+ print('lxml', etree.LIBXML_VERSION, 'xmlsec', xmlsec.get_libxml_version())"
+PYXMLSEC_SKIP_VERSION_CHECK=1 PYTHONPATH=src python -m pytest tests/
+```
+
+`PYXMLSEC_SKIP_VERSION_CHECK` bypasses the import-time mismatch guard so the
+shadow paths can be exercised; keep it off in normal use until the guard
+becomes a mode switch. For anything non-trivial, also loop the converted
+function ~10k times under the mismatch and check `ru_maxrss` stays flat and
+the serialized output stays byte-identical between iterations.
+
+### On a matched build (static wheel)
+
+`PYXMLSEC_STATIC_DEPS=true python -m build --wheel` bundles a libxml2 matched
+to lxml's wheels; install it into a venv with wheel lxml
+(`pip install --no-deps --force-reinstall dist/*.whl`) and run the suite twice
+— plain (fast path) and with `PYXMLSEC_FORCE_SHADOW=1` (shadow path on
+matched libraries; safe everywhere, so the whole suite must pass). Gotcha:
+setuptools reuses stale objects from `build/`, so `rm -rf build/lib.*
+build/temp.*` before switching between the dynamic in-place build and the
+static wheel, or the wheel silently ships the old dynamically linked module
+(it is then ~50 KB instead of several MB).
+
+## Status
+
+- ✅ Every binding that hands a node to xmlsec goes through a shadow:
+ all of `src/template.c`, `src/tree.c`, `src/ds.c` and `src/enc.c`.
+- ✅ Validated under a real 2.14 ↔ 2.15 mismatch (full suite, 10k-iteration
+ sign/verify/encrypt/decrypt loop with flat RSS and byte-identical output,
+ 12 MB binary round trip) and on a matched static build on both the fast
+ path and `PYXMLSEC_FORCE_SHADOW=1`.
+- ⬜ Endgame, kept as its own change: turn the import-time guard into a mode
+ switch (a mismatch sets the shadow flag instead of refusing to import) and
+ retire `PYXMLSEC_SKIP_VERSION_CHECK`. That is the user-facing resolution of
+ #356.
diff --git a/src/ds.c b/src/ds.c
index d0b4bdf9..904e609c 100644
--- a/src/ds.c
+++ b/src/ds.c
@@ -146,6 +146,18 @@ static PyObject* PyXmlSec_SignatureContextRegisterId(PyObject* self, PyObject* a
goto ON_FAIL;
}
+ // Shadow mode: never touch lxml's document (its ID hash) with our libxml2
+ // (issue #356). The registration is validated through lxml's API and
+ // recorded instead; the whole-document shadows (sign/verify/decrypt)
+ // replay it onto their private copies.
+ if (PyXmlSec_LxmlShadowIsActive()) {
+ if (PyXmlSec_LxmlShadowRegisterId(node, id_attr, id_ns) < 0) {
+ goto ON_FAIL;
+ }
+ PYXMLSEC_DEBUGF("%p: register id - ok", self);
+ Py_RETURN_NONE;
+ }
+
if (id_ns != NULL) {
attr = xmlHasNsProp(node->_c_node, XSTR(id_attr), XSTR(id_ns));
} else {
@@ -189,6 +201,8 @@ static PyObject* PyXmlSec_SignatureContextSign(PyObject* self, PyObject* args, P
PyXmlSec_SignatureContext* ctx = (PyXmlSec_SignatureContext*)self;
PyXmlSec_LxmlElementPtr node = NULL;
+ xmlNodePtr target;
+ PyXmlSec_LxmlShadow shadow;
int rv;
PYXMLSEC_DEBUGF("%p: sign - start", self);
@@ -196,12 +210,19 @@ static PyObject* PyXmlSec_SignatureContextSign(PyObject* self, PyObject* args, P
goto ON_FAIL;
}
+ // References (URI="", "#id") reach anywhere in the document, so the
+ // shadow covers the whole tree (issue #356), with the registered IDs
+ // replayed onto the copy. Signing fills several places inside
+ // (DigestValue, SignatureValue, KeyInfo); the reflect carries
+ // them all back.
+ if (PyXmlSec_LxmlShadowBeginDoc(&shadow, node, &target) < 0) {
+ goto ON_FAIL;
+ }
Py_BEGIN_ALLOW_THREADS;
- rv = xmlSecDSigCtxSign(ctx->handle, node->_c_node);
+ rv = xmlSecDSigCtxSign(ctx->handle, target);
PYXMLSEC_DUMP(xmlSecDSigCtxDebugDump, ctx->handle);
Py_END_ALLOW_THREADS;
- if (rv < 0) {
- PyXmlSec_SetLastError("failed to sign");
+ if (PyXmlSec_LxmlShadowReflect(&shadow, rv, "failed to sign") < 0) {
goto ON_FAIL;
}
PYXMLSEC_DEBUGF("%p: sign - ok", self);
@@ -224,6 +245,8 @@ static PyObject* PyXmlSec_SignatureContextVerify(PyObject* self, PyObject* args,
PyXmlSec_SignatureContext* ctx = (PyXmlSec_SignatureContext*)self;
PyXmlSec_LxmlElementPtr node = NULL;
+ xmlNodePtr target;
+ PyXmlSec_LxmlShadow shadow;
int rv;
PYXMLSEC_DEBUGF("%p: verify - start", self);
@@ -231,10 +254,16 @@ static PyObject* PyXmlSec_SignatureContextVerify(PyObject* self, PyObject* args,
goto ON_FAIL;
}
+ // Verification is read-only: whole-document shadow (with the registered
+ // IDs replayed) and no reflection at all — the copy is simply discarded.
+ if (PyXmlSec_LxmlShadowBeginDoc(&shadow, node, &target) < 0) {
+ goto ON_FAIL;
+ }
Py_BEGIN_ALLOW_THREADS;
- rv = xmlSecDSigCtxVerify(ctx->handle, node->_c_node);
+ rv = xmlSecDSigCtxVerify(ctx->handle, target);
PYXMLSEC_DUMP(xmlSecDSigCtxDebugDump, ctx->handle);
Py_END_ALLOW_THREADS;
+ PyXmlSec_LxmlShadowDiscard(&shadow);
if (rv < 0) {
PyXmlSec_SetLastError("failed to verify");
diff --git a/src/enc.c b/src/enc.c
index 42195dd3..dd1e84b5 100644
--- a/src/enc.c
+++ b/src/enc.c
@@ -166,6 +166,7 @@ static PyObject* PyXmlSec_EncryptionContextEncryptBinary(PyObject* self, PyObjec
PyXmlSec_LxmlElementPtr template = NULL;
const char* data = NULL;
Py_ssize_t data_size = 0;
+ PyXmlSec_LxmlShadow shadow;
int rv;
PYXMLSEC_DEBUGF("%p: encrypt_binary - start", self);
@@ -175,13 +176,18 @@ static PyObject* PyXmlSec_EncryptionContextEncryptBinary(PyObject* self, PyObjec
goto ON_FAIL;
}
+ // The encryption fills several places inside the template subtree
+ // (CipherValue, KeyInfo/EncryptedKey); the reflect carries them all back
+ // (issue #356).
+ if (PyXmlSec_LxmlShadowBegin(&shadow, template) < 0) {
+ goto ON_FAIL;
+ }
Py_BEGIN_ALLOW_THREADS;
- rv = xmlSecEncCtxBinaryEncrypt(ctx->handle, template->_c_node, (const xmlSecByte*)data, (xmlSecSize)data_size);
+ rv = xmlSecEncCtxBinaryEncrypt(ctx->handle, shadow.root, (const xmlSecByte*)data, (xmlSecSize)data_size);
PYXMLSEC_DUMP(xmlSecEncCtxDebugDump, ctx->handle);
Py_END_ALLOW_THREADS;
- if (rv < 0) {
- PyXmlSec_SetLastError("failed to encrypt binary");
+ if (PyXmlSec_LxmlShadowReflect(&shadow, rv, "failed to encrypt binary") < 0) {
goto ON_FAIL;
}
Py_INCREF(template);
@@ -203,17 +209,282 @@ static void PyXmlSec_ClearReplacedNodes(xmlSecEncCtxPtr ctx, PyXmlSec_LxmlDocume
while (n != NULL) {
PYXMLSEC_DEBUGF("clear replaced node %p", n);
nn = n->next;
- // if n has references, it will not be deleted
- elem = (PyXmlSec_LxmlElementPtr)PyXmlSec_elementFactory(doc, n);
- if (NULL == elem)
+ // Sever the chain first: lxml releases an element together with the
+ // text siblings that follow it, which would free the next node of
+ // this list under our feet (Type=Content replaces text nodes too).
+ n->next = NULL;
+ n->prev = NULL;
+ if (PyXmlSec_IsElement(n)) {
+ // if n has references, it will not be deleted
+ elem = (PyXmlSec_LxmlElementPtr)PyXmlSec_elementFactory(doc, n);
+ if (NULL == elem)
+ xmlFreeNode(n);
+ else
+ Py_DECREF(elem);
+ } else {
+ // text and CDATA nodes never have lxml proxies
xmlFreeNode(n);
- else
- Py_DECREF(elem);
+ }
n = nn;
}
ctx->replacedNodeList = NULL;
}
+// The raw path hands `xmlSecEncCtxXmlEncrypt` the template node itself
+// whenever it belongs to the target's document, and xmlsec *moves* it into
+// the target's place. The shadow path encrypts a copy of it instead, so
+// without this the live tree would keep the original where it was and the
+// document would end up with a second, empty — a tree shape
+// that depended on which libxml2 the extension is linked against. Unlink it
+// once the reflection is done, leaving its tail text behind the way libxml2's
+// `xmlReplaceNode` does, since lxml drops an element's tail together with the
+// element. A detached template, or one in another document, is copied on both
+// paths and stays where it is. Returns 0, or -1 with an exception set.
+static int PyXmlSec_EncryptionContextDropMovedTemplate(PyXmlSec_LxmlElementPtr template, PyXmlSec_LxmlElementPtr node) {
+ PyObject* parent = NULL;
+ PyObject* tree = NULL;
+ PyObject* root = NULL;
+ PyObject* top = NULL;
+ PyObject* tail = NULL;
+ PyObject* prev = NULL;
+ PyObject* slot = NULL;
+ PyObject* tmp = NULL;
+ const char* name = "tail";
+ int rv = -1;
+
+ parent = PyObject_CallMethod((PyObject*)template, "getparent", NULL);
+ if (parent == NULL) {
+ goto DONE;
+ }
+ if (parent == Py_None) {
+ rv = 0;
+ goto DONE;
+ }
+
+ // Only a template still hanging under the live document root is one the
+ // raw path would have moved; one carried off inside the subtree the
+ // encryption replaced, or one belonging to another document, is not.
+ tree = PyObject_CallMethod((PyObject*)node, "getroottree", NULL);
+ root = tree != NULL ? PyObject_CallMethod(tree, "getroot", NULL) : NULL;
+ if (root == NULL) {
+ goto DONE;
+ }
+ top = parent;
+ Py_INCREF(top);
+ for (;;) {
+ tmp = PyObject_CallMethod(top, "getparent", NULL);
+ if (tmp == NULL) {
+ goto DONE;
+ }
+ if (tmp == Py_None) {
+ Py_CLEAR(tmp);
+ break;
+ }
+ Py_DECREF(top);
+ top = tmp;
+ tmp = NULL;
+ }
+ if (top != root) {
+ rv = 0;
+ goto DONE;
+ }
+
+ tail = PyObject_GetAttrString((PyObject*)template, "tail");
+ if (tail == NULL) {
+ goto DONE;
+ }
+ if (tail != Py_None) {
+ prev = PyObject_CallMethod((PyObject*)template, "getprevious", NULL);
+ if (prev == NULL) {
+ goto DONE;
+ }
+ if (prev == Py_None) {
+ // first child: the text libxml2 would leave behind belongs to the
+ // parent's own text slot
+ Py_DECREF(prev);
+ Py_INCREF(parent);
+ prev = parent;
+ name = "text";
+ }
+ slot = PyObject_GetAttrString(prev, name);
+ if (slot == NULL) {
+ goto DONE;
+ }
+ if (slot != Py_None) {
+ tmp = PyUnicode_Concat(slot, tail);
+ if (tmp == NULL) {
+ goto DONE;
+ }
+ Py_DECREF(tail);
+ tail = tmp;
+ tmp = NULL;
+ }
+ if (PyObject_SetAttrString(prev, name, tail) < 0) {
+ goto DONE;
+ }
+ }
+ tmp = PyObject_CallMethod(parent, "remove", "O", template);
+ if (tmp == NULL) {
+ goto DONE;
+ }
+ rv = 0;
+
+DONE:
+ Py_XDECREF(parent);
+ Py_XDECREF(tree);
+ Py_XDECREF(root);
+ Py_XDECREF(top);
+ Py_XDECREF(tail);
+ Py_XDECREF(prev);
+ Py_XDECREF(slot);
+ Py_XDECREF(tmp);
+ return rv;
+}
+
+// Shadow-path body of encrypt_xml (issue #356): the target document and the
+// template are both re-parsed into one private copy, the encryption runs
+// there, and the replacement is reflected back through lxml — the fresh
+// takes the target's place (`Type=Element`; the document
+// root is morphed in place, as lxml cannot swap it) or its content
+// (`Type=Content`). The live template is unlinked afterwards when the raw
+// path would have moved it.
+static PyObject* PyXmlSec_EncryptionContextEncryptXmlShadow(PyXmlSec_EncryptionContext* ctx, PyXmlSec_LxmlElementPtr template, PyXmlSec_LxmlElementPtr node) {
+ PyXmlSec_LxmlShadow shadow;
+ xmlNodePtr target = NULL;
+ xmlNodePtr tmpl_copy;
+ PyObject* type_value = NULL;
+ PyObject* parent = NULL;
+ PyObject* result = NULL;
+ PyObject* tmp = NULL;
+ const char* type_str;
+ long idx = -1;
+ int is_content = 0;
+ int rv;
+
+ type_value = PyObject_CallMethod((PyObject*)template, "get", "s", "Type");
+ if (type_value == NULL) {
+ return NULL;
+ }
+ type_str = type_value == Py_None ? NULL : PyUnicode_AsUTF8(type_value);
+ if (type_str == NULL || !(strcmp(type_str, (const char*)xmlSecTypeEncElement) == 0
+ || strcmp(type_str, (const char*)xmlSecTypeEncContent) == 0)) {
+ PyErr_SetString(PyXmlSec_Error, "unsupported `Type`, it should be `element` or `content`");
+ goto ON_FAIL;
+ }
+ is_content = strcmp(type_str, (const char*)xmlSecTypeEncContent) == 0;
+
+ parent = PyObject_CallMethod((PyObject*)node, "getparent", NULL);
+ if (parent == NULL) {
+ goto ON_FAIL;
+ }
+ if (parent != Py_None) {
+ tmp = PyObject_CallMethod(parent, "index", "O", node);
+ if (tmp == NULL) {
+ goto ON_FAIL;
+ }
+ idx = PyLong_AsLong(tmp);
+ Py_CLEAR(tmp);
+ if (idx < 0) {
+ goto ON_FAIL;
+ }
+ }
+
+ if (PyXmlSec_LxmlShadowBeginDoc(&shadow, node, &target) < 0) {
+ goto ON_FAIL;
+ }
+ tmpl_copy = PyXmlSec_LxmlShadowImportElement(&shadow, template);
+ if (tmpl_copy == NULL) {
+ PyXmlSec_LxmlShadowDiscard(&shadow);
+ goto ON_FAIL;
+ }
+
+ // The replaced nodes belong to the private copy: xmlsec must free them
+ // itself (with our libxml2) rather than hand them back, because the copy
+ // is discarded right after and nothing could release them later.
+ ctx->handle->flags &= ~XMLSEC_ENC_RETURN_REPLACED_NODE;
+
+ Py_BEGIN_ALLOW_THREADS;
+ rv = xmlSecEncCtxXmlEncrypt(ctx->handle, tmpl_copy, target);
+ PYXMLSEC_DUMP(xmlSecEncCtxDebugDump, ctx->handle);
+ Py_END_ALLOW_THREADS;
+
+ if (rv < 0) {
+ // still detached means the encryption never consumed our template copy
+ if (tmpl_copy->parent == NULL) {
+ xmlFreeNode(tmpl_copy);
+ }
+ PyXmlSec_LxmlShadowDiscard(&shadow);
+ PyXmlSec_SetLastError("failed to encrypt xml");
+ goto ON_FAIL;
+ }
+
+ if (is_content) {
+ // the node stays; its old content was consumed and replaced by the
+ // fresh , which the reflection grafts back in
+ Py_ssize_t len = PyObject_Length((PyObject*)node);
+ if (len < 0) {
+ PyXmlSec_LxmlShadowDiscard(&shadow);
+ goto ON_FAIL;
+ }
+ while (len-- > 0) {
+ PyObject* child = PySequence_GetItem((PyObject*)node, 0);
+ tmp = child != NULL ? PyObject_CallMethod((PyObject*)node, "remove", "O", child) : NULL;
+ Py_XDECREF(child);
+ if (tmp == NULL) {
+ PyXmlSec_LxmlShadowDiscard(&shadow);
+ goto ON_FAIL;
+ }
+ Py_CLEAR(tmp);
+ }
+ if (PyXmlSec_LxmlShadowReflect(&shadow, 0, NULL) < 0) {
+ goto ON_FAIL;
+ }
+ result = PySequence_GetItem((PyObject*)node, 0);
+ if (result == NULL) {
+ goto ON_FAIL;
+ }
+ } else if (parent == Py_None) {
+ // Type=Element on the document root: the reflection morphs the live
+ // root in place into the fresh , so the node itself
+ // is the result — the new root, as on the raw path
+ if (PyXmlSec_LxmlShadowReflect(&shadow, 0, NULL) < 0) {
+ goto ON_FAIL;
+ }
+ result = (PyObject*)node;
+ Py_INCREF(result);
+ } else {
+ // Type=Element: the node itself was consumed and replaced
+ tmp = PyObject_CallMethod(parent, "remove", "O", node);
+ if (tmp == NULL) {
+ PyXmlSec_LxmlShadowDiscard(&shadow);
+ goto ON_FAIL;
+ }
+ Py_CLEAR(tmp);
+ if (PyXmlSec_LxmlShadowReflect(&shadow, 0, NULL) < 0) {
+ goto ON_FAIL;
+ }
+ result = PySequence_GetItem(parent, (Py_ssize_t)idx);
+ if (result == NULL) {
+ goto ON_FAIL;
+ }
+ }
+
+ if (PyXmlSec_EncryptionContextDropMovedTemplate(template, node) < 0) {
+ goto ON_FAIL;
+ }
+
+ Py_DECREF(type_value);
+ Py_XDECREF(parent);
+ return result;
+
+ON_FAIL:
+ Py_XDECREF(type_value);
+ Py_XDECREF(parent);
+ Py_XDECREF(tmp);
+ Py_XDECREF(result);
+ return NULL;
+}
+
static const char PyXmlSec_EncryptionContextEncryptXml__doc__[] = \
"encrypt_xml(template, node) -> lxml.etree._Element\n"
"Encrypts ``node`` using ``template``.\n\n"
@@ -243,6 +514,16 @@ static PyObject* PyXmlSec_EncryptionContextEncryptXml(PyObject* self, PyObject*
{
goto ON_FAIL;
}
+
+ if (PyXmlSec_LxmlShadowIsActive()) {
+ PyObject* result = PyXmlSec_EncryptionContextEncryptXmlShadow(ctx, template, node);
+ if (result == NULL) {
+ goto ON_FAIL;
+ }
+ PYXMLSEC_DEBUGF("%p: encrypt_xml - ok", self);
+ return result;
+ }
+
tmpType = xmlGetProp(template->_c_node, XSTR("Type"));
if (tmpType == NULL || !(xmlStrEqual(tmpType, xmlSecTypeEncElement) || xmlStrEqual(tmpType, xmlSecTypeEncContent))) {
PyErr_SetString(PyXmlSec_Error, "unsupported `Type`, it should be `element` or `content`");
@@ -316,6 +597,7 @@ static PyObject* PyXmlSec_EncryptionContextEncryptUri(PyObject* self, PyObject*
PyXmlSec_EncryptionContext* ctx = (PyXmlSec_EncryptionContext*)self;
PyXmlSec_LxmlElementPtr template = NULL;
const char* uri = NULL;
+ PyXmlSec_LxmlShadow shadow;
int rv;
PYXMLSEC_DEBUGF("%p: encrypt_uri - start", self);
@@ -323,13 +605,15 @@ static PyObject* PyXmlSec_EncryptionContextEncryptUri(PyObject* self, PyObject*
goto ON_FAIL;
}
+ if (PyXmlSec_LxmlShadowBegin(&shadow, template) < 0) {
+ goto ON_FAIL;
+ }
Py_BEGIN_ALLOW_THREADS;
- rv = xmlSecEncCtxUriEncrypt(ctx->handle, template->_c_node, (const xmlSecByte*)uri);
+ rv = xmlSecEncCtxUriEncrypt(ctx->handle, shadow.root, (const xmlSecByte*)uri);
PYXMLSEC_DUMP(xmlSecEncCtxDebugDump, ctx->handle);
Py_END_ALLOW_THREADS;
- if (rv < 0) {
- PyXmlSec_SetLastError("failed to encrypt URI");
+ if (PyXmlSec_LxmlShadowReflect(&shadow, rv, "failed to encrypt URI") < 0) {
goto ON_FAIL;
}
PYXMLSEC_DEBUGF("%p: encrypt_uri - ok", self);
@@ -340,6 +624,114 @@ static PyObject* PyXmlSec_EncryptionContextEncryptUri(PyObject* self, PyObject*
return NULL;
}
+// Shadow-path body of decrypt (issue #356): whole-document copy (with the
+// registered IDs replayed, for RetrievalMethod references), decryption on the
+// copy, then a replacement reflect — the decrypted subtree or content takes
+// the 's place in the live tree. Binary results need no
+// reflection at all.
+static PyObject* PyXmlSec_EncryptionContextDecryptShadow(PyXmlSec_EncryptionContext* ctx, PyXmlSec_LxmlElementPtr node) {
+ PyXmlSec_LxmlShadow shadow;
+ xmlNodePtr target = NULL;
+ PyObject* parent = NULL;
+ PyObject* result = NULL;
+ PyObject* tmp = NULL;
+ long idx = -1;
+ xmlChar* ttype;
+ int not_content;
+ int rv;
+
+ parent = PyObject_CallMethod((PyObject*)node, "getparent", NULL);
+ if (parent == NULL) {
+ return NULL;
+ }
+ if (parent != Py_None) {
+ tmp = PyObject_CallMethod(parent, "index", "O", node);
+ if (tmp == NULL) {
+ goto ON_FAIL;
+ }
+ idx = PyLong_AsLong(tmp);
+ Py_CLEAR(tmp);
+ if (idx < 0) {
+ goto ON_FAIL;
+ }
+ }
+
+ if (PyXmlSec_LxmlShadowBeginDoc(&shadow, node, &target) < 0) {
+ goto ON_FAIL;
+ }
+
+ // the Type decides the reflect shape; read it from the copy before the
+ // decryption consumes the node
+ ttype = xmlGetProp(target, XSTR("Type"));
+ not_content = (ttype == NULL || !xmlStrEqual(ttype, xmlSecTypeEncContent));
+ xmlFree(ttype);
+
+ // The replaced node belongs to the private copy: xmlsec must free it
+ // itself (with our libxml2) rather than hand it back, because the copy
+ // is discarded right after and nothing could release it later.
+ ctx->handle->flags &= ~XMLSEC_ENC_RETURN_REPLACED_NODE;
+
+ Py_BEGIN_ALLOW_THREADS;
+ ctx->handle->mode = xmlSecCheckNodeName(target, xmlSecNodeEncryptedKey, xmlSecEncNs) ? xmlEncCtxModeEncryptedKey : xmlEncCtxModeEncryptedData;
+ PYXMLSEC_DEBUGF("mode: %d", ctx->handle->mode);
+ rv = xmlSecEncCtxDecrypt(ctx->handle, target);
+ PYXMLSEC_DUMP(xmlSecEncCtxDebugDump, ctx->handle);
+ Py_END_ALLOW_THREADS;
+
+ if (rv < 0) {
+ PyXmlSec_LxmlShadowDiscard(&shadow);
+ PyXmlSec_SetLastError("failed to decrypt");
+ goto ON_FAIL;
+ }
+
+ if (!ctx->handle->resultReplaced) {
+ PyXmlSec_LxmlShadowDiscard(&shadow);
+ Py_DECREF(parent);
+ PYXMLSEC_DEBUGF("%p: binary.decrypt - ok", ctx);
+ return PyBytes_FromStringAndSize(
+ (const char*)xmlSecBufferGetData(ctx->handle->result),
+ (Py_ssize_t)xmlSecBufferGetSize(ctx->handle->result)
+ );
+ }
+
+ // the node was consumed; the reflection grafts whatever replaced it — or,
+ // for the document root (no parent to remove it from), morphs the node
+ // itself into the replacement, which is then the new root: the raw path's
+ // result too
+ if (parent != Py_None) {
+ tmp = PyObject_CallMethod(parent, "remove", "O", node);
+ if (tmp == NULL) {
+ PyXmlSec_LxmlShadowDiscard(&shadow);
+ goto ON_FAIL;
+ }
+ Py_CLEAR(tmp);
+ }
+ if (PyXmlSec_LxmlShadowReflect(&shadow, 0, NULL) < 0) {
+ goto ON_FAIL;
+ }
+ if (parent == Py_None) {
+ result = (PyObject*)node;
+ Py_INCREF(result);
+ } else if (not_content) {
+ result = PySequence_GetItem(parent, (Py_ssize_t)idx);
+ if (result == NULL) {
+ goto ON_FAIL;
+ }
+ } else {
+ result = parent;
+ Py_INCREF(result);
+ }
+
+ Py_DECREF(parent);
+ return result;
+
+ON_FAIL:
+ Py_XDECREF(parent);
+ Py_XDECREF(tmp);
+ Py_XDECREF(result);
+ return NULL;
+}
+
static const char PyXmlSec_EncryptionContextDecrypt__doc__[] = \
"decrypt(node)\n"
"Decrypts ``node`` (an ``EncryptedData`` or ``EncryptedKey`` element) and returns the result. "
@@ -372,6 +764,15 @@ static PyObject* PyXmlSec_EncryptionContextDecrypt(PyObject* self, PyObject* arg
goto ON_FAIL;
}
+ if (PyXmlSec_LxmlShadowIsActive()) {
+ PyObject* result = PyXmlSec_EncryptionContextDecryptShadow(ctx, node);
+ if (result == NULL) {
+ goto ON_FAIL;
+ }
+ PYXMLSEC_DEBUGF("%p: decrypt - ok", self);
+ return result;
+ }
+
xparent = node->_c_node->parent;
if (xparent != NULL && !PyXmlSec_IsElement(xparent)) {
xparent = NULL;
diff --git a/src/lxml.c b/src/lxml.c
index c98e933b..7b75d768 100644
--- a/src/lxml.c
+++ b/src/lxml.c
@@ -17,6 +17,9 @@
#include
#include
#include
+#include
+
+#include
#define XMLSEC_EXTRACT_VERSION(x, y) ((x / (y)) % 100)
@@ -101,12 +104,93 @@ static int PyXmlSec_CheckLxmlLibraryVersion(void) {
return result;
}
+// Non-zero when converted functions must run their xmlsec call on a shadow
+// copy instead of directly on lxml's nodes; decided once at import, below.
+static int PyXmlSec_LxmlShadowActive = 1;
+
+// The lxml.etree callables the shadow crossings use, resolved once at import
+// and kept for the lifetime of the process.
+static PyObject* PyXmlSec_LxmlEtreeToString;
+static PyObject* PyXmlSec_LxmlEtreeFromString;
+static PyObject* PyXmlSec_LxmlEtreeCleanupNamespaces;
+static PyObject* PyXmlSec_LxmlEtreeParser;
+
+// Shadow-mode ID registry: maps the identity of an lxml document to the list
+// of id-attribute specs registered for it (see PyXmlSec_LxmlShadowRegisterId).
+static PyObject* PyXmlSec_LxmlShadowIdRegistry;
+
+int PyXmlSec_LxmlShadowIsActive(void) {
+ return PyXmlSec_LxmlShadowActive;
+}
+
+// etree.XMLParser(huge_tree=True, resolve_entities=False), the parser for
+// lxml's side of the reflect crossing.
+//
+// huge_tree lifts libxml2's 10 MB text-node limit, which lxml's default
+// parser keeps, so a CipherValue above it (large encrypt_binary payloads) or
+// a document the user parsed with huge_tree still reflects.
+//
+// resolve_entities=False keeps the entity references of a tree the caller
+// parsed that way (lxml's `_Entity` children) as references: our own parse of
+// the copy leaves them unexpanded too, so expanding them here would give the
+// re-parse a different child structure than the copy the sites were collected
+// from, and the child-index paths would address the wrong nodes. A tree whose
+// entities were already resolved carries none to keep.
+//
+// Both are safe here: what gets parsed is this extension's own dump of a tree
+// lxml has already parsed.
+static PyObject* PyXmlSec_LxmlNewParser(PyObject* etree) {
+ PyObject* result = NULL;
+ PyObject* cls = PyObject_GetAttrString(etree, "XMLParser");
+ PyObject* args = PyTuple_New(0);
+ PyObject* kwargs = Py_BuildValue("{s:O,s:O}", "huge_tree", Py_True, "resolve_entities", Py_False);
+ if (cls != NULL && args != NULL && kwargs != NULL) {
+ result = PyObject_Call(cls, args, kwargs);
+ }
+ Py_XDECREF(cls);
+ Py_XDECREF(args);
+ Py_XDECREF(kwargs);
+ return result;
+}
+
int PyXmlSec_InitLxmlModule(void) {
- if (PyXmlSec_CheckLxmlLibraryVersion() < 0) {
+ // By default refuse to import when lxml and xmlsec link different libxml2
+ // versions: passing raw nodes between the two libraries then corrupts
+ // memory (https://github.com/xmlsec/python-xmlsec/issues/283). Setting
+ // PYXMLSEC_SKIP_VERSION_CHECK bypasses the guard — needed to exercise the
+ // shadow-copy paths (issue #356) under a mismatch.
+ int mismatch = PyXmlSec_CheckLxmlLibraryVersion() < 0;
+ if (mismatch && getenv("PYXMLSEC_SKIP_VERSION_CHECK") == NULL) {
PyXmlSec_SetLastError("lxml & xmlsec libxml2 library version mismatch");
return -1;
}
+ // Matched versions are the long-standing status quo: xmlsec may mutate
+ // lxml's nodes directly, so converted functions skip the copy (the
+ // Begin/End fast path). Shadows turn on under a mismatch — or always with
+ // PYXMLSEC_FORCE_SHADOW, the knob CI uses to exercise the shadow path on
+ // matched libraries.
+ PyXmlSec_LxmlShadowActive = mismatch || getenv("PYXMLSEC_FORCE_SHADOW") != NULL;
+
+ PyObject* etree = PyImport_ImportModule("lxml.etree");
+ if (etree == NULL) {
+ return -1;
+ }
+ PyXmlSec_LxmlEtreeToString = PyObject_GetAttrString(etree, "tostring");
+ PyXmlSec_LxmlEtreeFromString = PyObject_GetAttrString(etree, "fromstring");
+ PyXmlSec_LxmlEtreeCleanupNamespaces = PyObject_GetAttrString(etree, "cleanup_namespaces");
+ PyXmlSec_LxmlEtreeParser = PyXmlSec_LxmlNewParser(etree);
+ Py_DECREF(etree);
+ if (PyXmlSec_LxmlEtreeToString == NULL || PyXmlSec_LxmlEtreeFromString == NULL
+ || PyXmlSec_LxmlEtreeCleanupNamespaces == NULL || PyXmlSec_LxmlEtreeParser == NULL) {
+ return -1;
+ }
+
+ PyXmlSec_LxmlShadowIdRegistry = PyDict_New();
+ if (PyXmlSec_LxmlShadowIdRegistry == NULL) {
+ return -1;
+ }
+
return import_lxml__etree();
}
@@ -129,3 +213,2377 @@ int PyXmlSec_LxmlElementConverter(PyObject* o, PyXmlSec_LxmlElementPtr* p) {
Py_DECREF(node);
return 1;
}
+
+// ----------------------------------------------------------------------------
+// Shadow copies (issue #356) — see lxml.h for the contract.
+//
+// Both crossings go through serialized bytes: lxml's own etree.tostring /
+// etree.fromstring on its side, xmlReadMemory / xmlDocDumpMemory on ours.
+// Begin tags every node of the copy through the libxml2 _private field, so
+// that after the xmlsec call whatever is untagged is what the call created;
+// the reflection then grafts exactly those nodes (and the text slots the
+// call filled) into the live lxml tree, addressed by child-index paths that
+// resolve identically in both trees.
+// ----------------------------------------------------------------------------
+
+// etree.tostring(element, with_tail=False) — lxml's own libxml2 walks the
+// tree; with_tail keeps the serialization to the element itself.
+static PyObject* PyXmlSec_LxmlElementToBytes(PyObject* element) {
+ PyObject* result = NULL;
+ PyObject* args = PyTuple_Pack(1, element);
+ PyObject* kwargs = Py_BuildValue("{s:O}", "with_tail", Py_False);
+ if (args != NULL && kwargs != NULL) {
+ result = PyObject_Call(PyXmlSec_LxmlEtreeToString, args, kwargs);
+ }
+ Py_XDECREF(args);
+ Py_XDECREF(kwargs);
+ return result;
+}
+
+// etree.fromstring(data, parser) — the parsed nodes are owned and managed by lxml.
+static PyObject* PyXmlSec_LxmlElementFromBytes(PyObject* data) {
+ return PyObject_CallFunctionObjArgs(PyXmlSec_LxmlEtreeFromString, data, PyXmlSec_LxmlEtreeParser, NULL);
+}
+
+// Our side of the crossing. NONET as always; HUGE lifts the 10 MB text-node
+// and nesting limits so that nothing lxml accepted — or a CipherValue xmlsec
+// is about to produce — is refused here. The bytes are lxml's own dump of an
+// already-parsed tree, so the relaxed limits add no attack surface.
+#define PYXMLSEC_SHADOW_PARSE_OPTIONS (XML_PARSE_NONET | XML_PARSE_HUGE)
+
+// Parses `bytes` into a private document with a root element, or returns
+// NULL with an exception set (`error` for parse failures). `url` is the base
+// URI the copy gets (see PyXmlSec_LxmlDocumentUrl); NULL when there is none.
+// `external_dtd` says the source document loaded an external subset: the dump
+// keeps only the DOCTYPE reference to it, and its declarations are what typed
+// the document's ID attributes, so the copy has to load it too or resolve no
+// #id reference the DTD declared. DTDLOAD alone, never DTDATTR — libxml2
+// fills in defaulted attributes only under the latter, and the copy must stay
+// what lxml serialized. NONET keeps the fetch to the local file the source
+// document already named, resolved against the same base URI.
+static xmlDocPtr PyXmlSec_LxmlShadowParse(PyObject* bytes, const char* url, int external_dtd, const char* error) {
+ int options = PYXMLSEC_SHADOW_PARSE_OPTIONS | (external_dtd ? XML_PARSE_DTDLOAD : 0);
+ char* data = NULL;
+ Py_ssize_t size = 0;
+ xmlDocPtr doc;
+
+ if (PyBytes_AsStringAndSize(bytes, &data, &size) < 0) {
+ return NULL;
+ }
+ doc = xmlReadMemory(data, (int)size, url, NULL, options);
+ if (doc == NULL || xmlDocGetRootElement(doc) == NULL) {
+ if (doc != NULL) {
+ xmlFreeDoc(doc);
+ }
+ PyErr_SetString(PyXmlSec_InternalError, error);
+ return NULL;
+ }
+ return doc;
+}
+
+// The URL of the document `tree` belongs to (lxml's `docinfo.URL`), so that
+// the private copy carries the same base URI as the document it copies —
+// what libxml2 resolves relative references against (an XSLT transform's
+// document()/import, an xml:base, a DTD or entity system id). A copy parsed
+// without one would resolve them against the process' working directory
+// instead. Stores a new reference to the owning string in *holder (NULL when
+// the document has no URL) and its UTF-8 in *url; returns 0, or -1 with an
+// exception set.
+static int PyXmlSec_LxmlDocumentUrl(PyObject* tree, PyObject** holder, const char** url) {
+ PyObject* info;
+ PyObject* value;
+
+ *holder = NULL;
+ *url = NULL;
+ info = PyObject_GetAttrString(tree, "docinfo");
+ if (info == NULL) {
+ return -1;
+ }
+ value = PyObject_GetAttrString(info, "URL");
+ Py_DECREF(info);
+ if (value == NULL) {
+ return -1;
+ }
+ if (!PyUnicode_Check(value)) { // None for a document parsed from memory
+ Py_DECREF(value);
+ return 0;
+ }
+ *url = PyUnicode_AsUTF8(value);
+ if (*url == NULL) {
+ Py_DECREF(value);
+ return -1;
+ }
+ *holder = value;
+ return 0;
+}
+
+// The DTD subsets of the document `tree` belongs to. The internal one (lxml
+// reports one for any DOCTYPE declaration) is where a document lxml parsed
+// declares the entities its `&name;` references name, and so decides whether
+// a subtree may be serialized on its own (see Begin). The external one is
+// reported only when lxml actually loaded it — the caller asked for it with
+// load_dtd — and the copy then has to load it as well, since its declarations
+// are what typed the document's ID attributes (see PyXmlSec_LxmlShadowParse).
+// Returns 0, or -1 with an exception set.
+static int PyXmlSec_LxmlDocumentSubsets(PyObject* tree, int* internal, int* external) {
+ PyObject* info = PyObject_GetAttrString(tree, "docinfo");
+ PyObject* value;
+
+ *internal = 0;
+ *external = 0;
+ if (info == NULL) {
+ return -1;
+ }
+ value = PyObject_GetAttrString(info, "internalDTD");
+ if (value == NULL) {
+ Py_DECREF(info);
+ return -1;
+ }
+ *internal = value != Py_None;
+ Py_DECREF(value);
+ value = PyObject_GetAttrString(info, "externalDTD");
+ Py_DECREF(info);
+ if (value == NULL) {
+ return -1;
+ }
+ *external = value != Py_None;
+ Py_DECREF(value);
+ return 0;
+}
+
+// Nodes that exist before the xmlsec call are tagged through the libxml2
+// _private field (never serialized, never touched by the parser or xmlsec):
+// it is pointed at the shadow's own tag array, so whatever is untagged after
+// the call is what the call created. Each tag also records the child count
+// the node had, which is what makes a *removal* visible — a call can delete a
+// node and leave nothing fresh behind (an that decrypts to
+// empty content), and only the changed count shows it happened — and a
+// fingerprint of the node's content, which is what makes an *in-place* text
+// rewrite visible: appending to a text node (xmlNodeAddContent onto a
+// trailing text child) leaves both the node and the count untouched. A tag
+// pointer is compared against the array bounds, never dereferenced blindly:
+// a foreign _private (lxml's own proxy, when both libraries are the same
+// libxml2) simply falls outside.
+#define PYXMLSEC_SHADOW_TAGGED(shadow, n) \
+ ((uintptr_t)(n)->_private >= (uintptr_t)(shadow)->tags \
+ && (uintptr_t)(n)->_private < (uintptr_t)((shadow)->tags + (shadow)->ntags))
+#define PYXMLSEC_SHADOW_TAG(shadow, n) \
+ (PYXMLSEC_SHADOW_TAGGED(shadow, n) ? (PyXmlSec_LxmlShadowTag*)(n)->_private : NULL)
+
+// The walks over the copy's own structure — this one, TagNodes and
+// CollectSites — recurse once per level of nesting, so they need a ceiling of
+// their own. libxml2 2.14 and later refuse a parse deeper than 2048 levels
+// even under XML_PARSE_HUGE, so a copy parsed by one of those can never reach
+// it; older libxml2 lifts its cap entirely under HUGE (checked: 2.9.13 parses
+// a 200000-level document, and the unguarded walk then overruns the C stack),
+// and lxml hands over trees that were never parsed on its side at all — one
+// built element by element has no limit whatsoever. Anything deeper fails
+// cleanly here rather than crashing; sites deeper than
+// PYXMLSEC_SHADOW_MAX_DEPTH are refused later anyway.
+#define PYXMLSEC_SHADOW_MAX_NESTING 2048
+
+// Number of nodes in the list at `node` and under it, or -1 when the tree is
+// nested deeper than the walks go.
+static int PyXmlSec_LxmlShadowCountNodes(xmlNodePtr node, int depth) {
+ int count = 0;
+ if (depth >= PYXMLSEC_SHADOW_MAX_NESTING) {
+ return -1;
+ }
+ for (; node != NULL; node = node->next) {
+ int children = PyXmlSec_LxmlShadowCountNodes(node->children, depth + 1);
+ if (children < 0) {
+ return -1;
+ }
+ count += 1 + children;
+ }
+ return count;
+}
+
+// FNV-1a over the content a text or CDATA node carries; 0 for every other
+// node type, whose content the reflection does not carry back on its own.
+static unsigned long long PyXmlSec_LxmlShadowContentPrint(xmlNodePtr node) {
+ unsigned long long fingerprint = 14695981039346656037ULL;
+ const xmlChar* p;
+
+ if (node->type != XML_TEXT_NODE && node->type != XML_CDATA_SECTION_NODE) {
+ return 0;
+ }
+ for (p = node->content; p != NULL && *p != 0; ++p) {
+ fingerprint = (fingerprint ^ (unsigned long long)*p) * 1099511628211ULL;
+ }
+ return fingerprint;
+}
+
+// Hands out `shadow->tags` in document order; returns the next free slot.
+// Runs only after CountNodes succeeded, so the tree it walks is within
+// PYXMLSEC_SHADOW_MAX_NESTING and this recursion is bounded with it.
+static int PyXmlSec_LxmlShadowTagNodes(PyXmlSec_LxmlShadow* shadow, xmlNodePtr node, int next) {
+ for (; node != NULL; node = node->next) {
+ PyXmlSec_LxmlShadowTag* tag = &shadow->tags[next++];
+ xmlNodePtr child;
+ tag->children = 0;
+ for (child = node->children; child != NULL; child = child->next) {
+ ++tag->children;
+ }
+ tag->content = PyXmlSec_LxmlShadowContentPrint(node);
+ node->_private = (void*)tag;
+ next = PyXmlSec_LxmlShadowTagNodes(shadow, node->children, next);
+ }
+ return next;
+}
+
+// Tags every node of the freshly parsed copy. Returns 0, or -1 with an
+// exception set.
+static int PyXmlSec_LxmlShadowMark(PyXmlSec_LxmlShadow* shadow) {
+ int count = PyXmlSec_LxmlShadowCountNodes(shadow->doc->children, 0);
+ // The unlinked subtree is in the document but not in its tree, so the
+ // walk over the tree does not reach it; it is pre-existing all the same.
+ int unlinked = shadow->unlinked ? PyXmlSec_LxmlShadowCountNodes(shadow->root, 0) : 0;
+ int next;
+
+ if (count < 0 || unlinked < 0) {
+ PyErr_SetString(PyXmlSec_InternalError, "the document is nested too deeply.");
+ return -1;
+ }
+ count += unlinked;
+ shadow->tags = (PyXmlSec_LxmlShadowTag*)PyMem_Malloc((count > 0 ? count : 1) * sizeof(*shadow->tags));
+ if (shadow->tags == NULL) {
+ PyErr_NoMemory();
+ return -1;
+ }
+ shadow->ntags = count;
+ next = PyXmlSec_LxmlShadowTagNodes(shadow, shadow->doc->children, 0);
+ if (shadow->unlinked) {
+ PyXmlSec_LxmlShadowTagNodes(shadow, shadow->root, next);
+ }
+ return 0;
+}
+
+// Paths span whole user documents (BeginDoc). 256 is libxml2's default
+// nesting limit, so any tree a default lxml parser accepted fits; deeper
+// (huge_tree) documents fail cleanly instead of overrunning the buffers.
+#define PYXMLSEC_SHADOW_MAX_DEPTH 256
+
+// Index of node among its preceding siblings, counting only the node types
+// lxml exposes as children (elements, comments, PIs, entity refs), so indices
+// computed here line up with lxml's __getitem__ / insert.
+static int PyXmlSec_LxmlShadowChildIndex(xmlNodePtr node) {
+ int idx = 0;
+ xmlNodePtr s;
+ for (s = node->prev; s != NULL; s = s->prev) {
+ if (_isElement(s)) {
+ ++idx;
+ }
+ }
+ return idx;
+}
+
+// Records the child indices leading from `top` down to `node` into `path`
+// (ordered top-first) and returns the number of steps, or -1 when node is not
+// under top or lies too deep. Both trees involved are byte-for-byte copies of
+// each other, so a path recorded in one resolves in the other.
+static int PyXmlSec_LxmlShadowPathTo(xmlNodePtr node, xmlNodePtr top, int* path) {
+ int depth = 0;
+ int d;
+ xmlNodePtr n = node;
+ while (n != top) {
+ if (n->parent == NULL || depth >= PYXMLSEC_SHADOW_MAX_DEPTH) {
+ return -1;
+ }
+ ++depth;
+ n = n->parent;
+ }
+ for (n = node, d = depth; d > 0; n = n->parent) {
+ path[--d] = PyXmlSec_LxmlShadowChildIndex(n);
+ }
+ return depth;
+}
+
+// Walks `path` (child indices) down from `start`, an lxml element. Returns a
+// new reference.
+static PyObject* PyXmlSec_LxmlShadowWalk(PyObject* start, const int* path, int depth) {
+ int i;
+ PyObject* cur = start;
+ Py_INCREF(cur);
+ for (i = 0; i < depth; ++i) {
+ PyObject* child = PySequence_GetItem(cur, path[i]);
+ Py_DECREF(cur);
+ if (child == NULL) {
+ return NULL;
+ }
+ cur = child;
+ }
+ return cur;
+}
+
+// The same walk on a raw copy, counting children exactly like ChildIndex.
+static xmlNodePtr PyXmlSec_LxmlShadowWalkNode(xmlNodePtr start, const int* path, int depth) {
+ int i;
+ xmlNodePtr n = start;
+ for (i = 0; i < depth; ++i) {
+ int idx = path[i];
+ xmlNodePtr c;
+ for (c = n->children; c != NULL; c = c->next) {
+ if (_isElement(c) && idx-- == 0) {
+ break;
+ }
+ }
+ if (c == NULL) {
+ return NULL;
+ }
+ n = c;
+ }
+ return n;
+}
+
+// Records the child indices leading from the top of `node`'s live tree down
+// to `node` into `path` (ordered top-first), through lxml's own API — the
+// shadow path never walks lxml's raw nodes. Returns the depth and stores the
+// top element in *top (new reference), or -1 with an exception set.
+static int PyXmlSec_LxmlLivePathTo(PyObject* node, int* path, PyObject** top) {
+ PyObject* cur = node;
+ int depth = 0;
+ int i;
+
+ *top = NULL;
+ Py_INCREF(cur);
+ for (;;) {
+ PyObject* parent = PyObject_CallMethod(cur, "getparent", NULL);
+ PyObject* index;
+ long idx;
+ if (parent == NULL) {
+ goto ON_FAIL;
+ }
+ if (parent == Py_None) {
+ Py_DECREF(parent);
+ break;
+ }
+ index = PyObject_CallMethod(parent, "index", "O", cur);
+ if (index == NULL) {
+ Py_DECREF(parent);
+ goto ON_FAIL;
+ }
+ idx = PyLong_AsLong(index);
+ Py_DECREF(index);
+ if (idx < 0 || depth >= PYXMLSEC_SHADOW_MAX_DEPTH) {
+ Py_DECREF(parent);
+ if (!PyErr_Occurred()) {
+ PyErr_SetString(PyXmlSec_InternalError, "the document is nested too deeply.");
+ }
+ goto ON_FAIL;
+ }
+ path[depth++] = (int)idx;
+ Py_DECREF(cur);
+ cur = parent;
+ }
+ for (i = 0; i < depth / 2; ++i) { // collected bottom-up; reverse
+ int t = path[i];
+ path[i] = path[depth - 1 - i];
+ path[depth - 1 - i] = t;
+ }
+ *top = cur;
+ return depth;
+
+ON_FAIL:
+ Py_DECREF(cur);
+ return -1;
+}
+
+// Serializes the copy in its current state and re-parses it with lxml,
+// returning the root element of that detached parse (new reference). The
+// *whole* copy is dumped, not just the changed nodes: the surrounding markup
+// carries the ancestor-declared namespaces and the "\n" formatting siblings
+// xmlsec emits, so whatever gets grafted from the re-parse stays
+// byte-identical to the raw-pointer code without any manual namespace or
+// whitespace fix-up.
+static PyObject* PyXmlSec_LxmlShadowDumpCopy(PyXmlSec_LxmlShadow* shadow) {
+ PyObject* bytes = NULL;
+ PyObject* result = NULL;
+ xmlChar* dump = NULL;
+ int dump_size = 0;
+
+ if (shadow->unlinked) {
+ // The paths the reflection carries are relative to the unlinked
+ // subtree, so that is what has to be re-parsed — the document dump
+ // does not hold it.
+ xmlBufferPtr buffer = xmlBufferCreate();
+ xmlSaveCtxtPtr save = buffer != NULL ? xmlSaveToBuffer(buffer, "UTF-8", XML_SAVE_NO_DECL) : NULL;
+ int failed = 1;
+
+ if (save != NULL) {
+ failed = xmlSaveTree(save, shadow->root) < 0;
+ failed = xmlSaveClose(save) < 0 || failed;
+ }
+ if (!failed) {
+ bytes = PyBytes_FromStringAndSize((const char*)xmlBufferContent(buffer), (Py_ssize_t)xmlBufferLength(buffer));
+ if (bytes != NULL) {
+ result = PyXmlSec_LxmlElementFromBytes(bytes);
+ }
+ Py_XDECREF(bytes);
+ } else {
+ PyErr_SetString(PyXmlSec_InternalError, "cannot serialize the private copy.");
+ }
+ if (buffer != NULL) {
+ xmlBufferFree(buffer);
+ }
+ return result;
+ }
+
+ xmlDocDumpMemory(shadow->doc, &dump, &dump_size);
+ if (dump == NULL || dump_size <= 0) {
+ PyErr_SetString(PyXmlSec_InternalError, "cannot serialize the private copy.");
+ goto DONE;
+ }
+ bytes = PyBytes_FromStringAndSize((const char*)dump, (Py_ssize_t)dump_size);
+ if (bytes != NULL) {
+ result = PyXmlSec_LxmlElementFromBytes(bytes);
+ }
+DONE:
+ if (dump != NULL) {
+ xmlFree(dump);
+ }
+ Py_XDECREF(bytes);
+ return result;
+}
+
+// Copies the attributes of `src` (a node in the copy) onto the live lxml
+// element `dst`, touching only the ones that differ: find-or-create calls may
+// set attributes (e.g. Id) on a node that already existed.
+static int PyXmlSec_LxmlShadowSyncAttributes(xmlNodePtr src, PyObject* dst) {
+ xmlAttrPtr attr;
+ for (attr = src->properties; attr != NULL; attr = attr->next) {
+ PyObject* key = NULL;
+ PyObject* val = NULL;
+ PyObject* cur = NULL;
+ int same = -1;
+ xmlChar* value = xmlGetNsProp(src, attr->name, attr->ns != NULL ? attr->ns->href : NULL);
+ if (value == NULL) {
+ continue;
+ }
+ if (attr->ns != NULL && attr->ns->href != NULL) {
+ // lxml takes namespaced attribute names in Clark notation.
+ key = PyUnicode_FromFormat("{%s}%s", (const char*)attr->ns->href, (const char*)attr->name);
+ } else {
+ key = PyUnicode_FromString((const char*)attr->name);
+ }
+ val = PyUnicode_FromString((const char*)value);
+ xmlFree(value);
+ if (key != NULL && val != NULL) {
+ cur = PyObject_CallMethod(dst, "get", "O", key);
+ if (cur != NULL) {
+ same = PyObject_RichCompareBool(cur, val, Py_EQ);
+ }
+ }
+ if (same == 0) {
+ PyObject* r = PyObject_CallMethod(dst, "set", "OO", key, val);
+ same = r != NULL ? 1 : -1;
+ Py_XDECREF(r);
+ }
+ Py_XDECREF(key);
+ Py_XDECREF(val);
+ Py_XDECREF(cur);
+ if (same < 0) {
+ return -1;
+ }
+ }
+ return 0;
+}
+
+// Makes the copy's counterpart of the element at `path` the copy's root and
+// drops the rest of the document, keeping its internal subset. Used when the
+// document declares entities: the subtree cannot be serialized on its own
+// then (its `&name;` references would be undefined), so the whole document is
+// copied and cut down here. The node is copied before it is re-rooted, which
+// is what makes the cut safe — libxml2 redeclares on the copy the namespaces
+// it inherited from the ancestors that are about to go, and rebinds its
+// entity references to the copy's own declarations. Returns 0, or -1 with an
+// exception set.
+static int PyXmlSec_LxmlShadowReroot(PyXmlSec_LxmlShadow* shadow, const int* path, int depth) {
+ xmlNodePtr target = PyXmlSec_LxmlShadowWalkNode(xmlDocGetRootElement(shadow->doc), path, depth);
+ xmlNodePtr copy;
+ xmlNodePtr old;
+
+ if (target == NULL) {
+ PyErr_SetString(PyXmlSec_InternalError, "cannot locate the element in the private copy.");
+ return -1;
+ }
+ if (depth == 0) { // the element is the document root already
+ return 0;
+ }
+ copy = xmlDocCopyNode(target, shadow->doc, 1);
+ if (copy == NULL) {
+ PyErr_SetString(PyXmlSec_InternalError, "cannot make a private copy of the element.");
+ return -1;
+ }
+ old = xmlDocSetRootElement(shadow->doc, copy);
+ if (old != NULL) {
+ xmlFreeNode(old);
+ }
+ return 0;
+}
+
+int PyXmlSec_LxmlShadowBegin(PyXmlSec_LxmlShadow* shadow, PyXmlSec_LxmlElementPtr element) {
+ PyObject* tree = NULL;
+ PyObject* bytes = NULL;
+ PyObject* url_holder = NULL;
+ const char* url = NULL;
+ int path[PYXMLSEC_SHADOW_MAX_DEPTH];
+ int depth = 0;
+ int dtd = 0;
+ int extdtd = 0;
+ int whole = 0;
+
+ shadow->element = element;
+ shadow->owned = NULL;
+ shadow->doc = NULL;
+ shadow->root = NULL;
+ shadow->tags = NULL;
+ shadow->ntags = 0;
+ shadow->unlinked = 0;
+
+ // Fast path: lxml links the same libxml2 (the import guard passed), so
+ // xmlsec can mutate lxml's nodes directly and no copy is needed; End sees
+ // doc == NULL and just wraps the result node.
+ if (!PyXmlSec_LxmlShadowActive) {
+ shadow->root = element->_c_node;
+ return 0;
+ }
+
+ tree = PyObject_CallMethod((PyObject*)element, "getroottree", NULL);
+ if (tree == NULL) {
+ goto ON_FAIL;
+ }
+ if (PyXmlSec_LxmlDocumentUrl(tree, &url_holder, &url) < 0
+ || PyXmlSec_LxmlDocumentSubsets(tree, &dtd, &extdtd) < 0) {
+ goto ON_FAIL;
+ }
+
+ if (dtd) {
+ // The subtree may hold entity references, and their declarations live
+ // in the document's internal subset — serializing the element alone
+ // would leave them undefined and the parse would fail. Copy the whole
+ // document, declarations included, and cut it back to the element.
+ // Unless the element was removed from that document: the dump would
+ // not hold it at all, and the element alone is all there is to copy.
+ PyObject* live_top = NULL;
+ PyObject* live_root = PyObject_CallMethod(tree, "getroot", NULL);
+ if (live_root == NULL) {
+ goto ON_FAIL;
+ }
+ depth = PyXmlSec_LxmlLivePathTo((PyObject*)element, path, &live_top);
+ whole = depth >= 0 && live_top == live_root;
+ Py_XDECREF(live_top);
+ Py_DECREF(live_root);
+ if (depth < 0) {
+ goto ON_FAIL;
+ }
+ }
+ if (whole) {
+ bytes = PyObject_CallFunctionObjArgs(PyXmlSec_LxmlEtreeToString, tree, NULL);
+ } else {
+ bytes = PyXmlSec_LxmlElementToBytes((PyObject*)element);
+ }
+ Py_CLEAR(tree);
+ if (bytes == NULL) {
+ goto ON_FAIL;
+ }
+ shadow->doc = PyXmlSec_LxmlShadowParse(bytes, url, extdtd, "cannot make a private copy of the element.");
+ Py_CLEAR(bytes);
+ Py_CLEAR(url_holder);
+ if (shadow->doc == NULL) {
+ goto ON_FAIL;
+ }
+ if (whole && PyXmlSec_LxmlShadowReroot(shadow, path, depth) < 0) {
+ goto ON_FAIL;
+ }
+ shadow->root = xmlDocGetRootElement(shadow->doc);
+ if (PyXmlSec_LxmlShadowMark(shadow) < 0) {
+ goto ON_FAIL;
+ }
+ return 0;
+
+ON_FAIL:
+ Py_XDECREF(tree);
+ Py_XDECREF(bytes);
+ Py_XDECREF(url_holder);
+ PyXmlSec_LxmlShadowDiscard(shadow);
+ return -1;
+}
+
+xmlDocPtr PyXmlSec_LxmlShadowBeginNewDoc(PyXmlSec_LxmlShadow* shadow, PyXmlSec_LxmlElementPtr element) {
+ shadow->element = element;
+ shadow->owned = NULL;
+ shadow->doc = NULL;
+ shadow->root = NULL;
+ shadow->tags = NULL;
+ shadow->ntags = 0;
+ shadow->unlinked = 0;
+
+ // Fast path: allocate the detached subtree straight in the element's own
+ // document, as the raw code always did.
+ if (!PyXmlSec_LxmlShadowActive) {
+ return element->_doc->_c_doc;
+ }
+ shadow->doc = xmlNewDoc((const xmlChar*)"1.0");
+ if (shadow->doc == NULL) {
+ PyErr_SetString(PyXmlSec_InternalError, "cannot create a private document.");
+ return NULL;
+ }
+ return shadow->doc;
+}
+
+void PyXmlSec_LxmlShadowDiscard(PyXmlSec_LxmlShadow* shadow) {
+ if (shadow->doc != NULL) {
+ if (shadow->unlinked && shadow->root != NULL) {
+ // Outside the document's tree: freeing the document does not
+ // reach it.
+ xmlFreeNode(shadow->root);
+ }
+ xmlFreeDoc(shadow->doc);
+ shadow->doc = NULL;
+ shadow->root = NULL;
+ shadow->unlinked = 0;
+ }
+ PyMem_Free(shadow->tags);
+ shadow->tags = NULL;
+ shadow->ntags = 0;
+ Py_CLEAR(shadow->owned);
+}
+
+// ----------------------------------------------------------------------------
+// Shadow-mode ID registry.
+//
+// register_id / add_ids used to write straight into lxml's document (its ID
+// hash) with our libxml2 — exactly the cross-library access the shadow
+// forbids. Under the shadow they record the id-attribute specs here instead,
+// and every whole-document Begin replays them onto the private copy so that
+// #id references resolve during sign/verify/decrypt.
+//
+// An entry is the tuple (document, nodes, specs), keyed by the _Document
+// object's address: `nodes` holds the registered elements themselves and
+// `specs` the (attribute name, namespace, node index) records. The entry
+// keeps a strong reference to the document and to every registered element,
+// so the key can never go stale — the address cannot be reused while the
+// entry keeps the object alive.
+//
+// Keeping the elements is also what makes the replay faithful: a spec is
+// applied to the very node it was registered for, never to every same-named
+// attribute in the document. Registering the whole document would let an
+// unrelated element sharing the id value claim it first, so a `#id`
+// reference — the URI a signature covers — could resolve to content the
+// caller never registered.
+//
+// A registration covering a subtree (add_ids) is expanded into one spec per
+// element carrying the attribute *at the call*, which is the snapshot
+// xmlSecAddIDs takes of the scope it walks: an element that only later grows
+// the attribute, or only later joins the scope, was never registered by the
+// fast path and must not become resolvable under the shadow either.
+//
+// Those references also make liveness decidable without weak references
+// (lxml's classes refuse those): every element proxy holds a reference to the
+// document it hangs in, so when the document's reference count is exactly
+// what the registry itself holds (the entry plus one per registered element
+// still in it) and no registered element is referenced anywhere else, nothing
+// can hand that document to a binding again — the entry is dead and is
+// dropped. Pruning runs before every new registration, which bounds the
+// registry — and the documents it pins — by the documents still in use. No
+// live registration is ever evicted.
+//
+// An element need not stay in the document it was registered for: lxml lets
+// one be adopted into another tree, and its proxy then references that other
+// document. The count is therefore taken from each proxy's *current* owner,
+// and re-registering an adopted node vacates its slot in the entry it came
+// from (ForgetIdNode), so a node is never held by two entries at once and a
+// reference count of one still means "the registry alone".
+// ----------------------------------------------------------------------------
+
+enum { PYXMLSEC_ID_ENTRY_DOC, PYXMLSEC_ID_ENTRY_NODES, PYXMLSEC_ID_ENTRY_SPECS };
+
+// Non-zero when nothing outside the registry can reach the entry's document.
+static int PyXmlSec_LxmlShadowIdEntryIsDead(PyObject* entry) {
+ PyObject* doc = PyTuple_GET_ITEM(entry, PYXMLSEC_ID_ENTRY_DOC);
+ PyObject* nodes = PyTuple_GET_ITEM(entry, PYXMLSEC_ID_ENTRY_NODES);
+ Py_ssize_t n = PyList_GET_SIZE(nodes);
+ Py_ssize_t held = 1; // the entry's own reference
+ Py_ssize_t i;
+
+ for (i = 0; i < n; ++i) {
+ PyObject* node = PyList_GET_ITEM(nodes, i);
+ if (node == Py_None) { // vacated slot: the node was registered elsewhere
+ continue;
+ }
+ if (Py_REFCNT(node) != 1) {
+ return 0;
+ }
+ if ((PyObject*)((PyXmlSec_LxmlElementPtr)node)->_doc == doc) {
+ ++held;
+ }
+ }
+ return Py_REFCNT(doc) == held;
+}
+
+// Releases the node held in slot `i` of `entry`: the specs that named the
+// slot go with it, and the slot is set to None rather than removed, so the
+// indices the surviving specs carry stay valid — the next registration
+// reuses it. Best effort; a failure only keeps a spec alive longer.
+static void PyXmlSec_LxmlShadowVacateIdSlot(PyObject* entry, Py_ssize_t i) {
+ PyObject* nodes = PyTuple_GET_ITEM(entry, PYXMLSEC_ID_ENTRY_NODES);
+ PyObject* specs = PyTuple_GET_ITEM(entry, PYXMLSEC_ID_ENTRY_SPECS);
+ Py_ssize_t j;
+
+ for (j = PyList_GET_SIZE(specs) - 1; j >= 0; --j) {
+ PyObject* spec = PyList_GET_ITEM(specs, j);
+ if (PyLong_AsSsize_t(PyTuple_GET_ITEM(spec, 2)) == i && PyList_SetSlice(specs, j, j + 1, NULL) < 0) {
+ PyErr_Clear();
+ }
+ }
+ Py_INCREF(Py_None);
+ PyList_SetItem(nodes, i, Py_None); // steals the reference, releases the node
+}
+
+// Drops `element` from every entry but `keep`. A node registered again after
+// being adopted into another document is no longer part of the tree its old
+// entry stands for, and its slot there must stop holding it: the liveness
+// test above counts on a registered proxy being held by one entry only.
+static void PyXmlSec_LxmlShadowForgetIdNode(PyObject* keep, PyObject* element) {
+ PyObject* key;
+ PyObject* entry;
+ Py_ssize_t pos = 0;
+
+ while (PyDict_Next(PyXmlSec_LxmlShadowIdRegistry, &pos, &key, &entry)) {
+ PyObject* nodes;
+ Py_ssize_t i;
+ if (entry == keep) {
+ continue;
+ }
+ nodes = PyTuple_GET_ITEM(entry, PYXMLSEC_ID_ENTRY_NODES);
+ for (i = 0; i < PyList_GET_SIZE(nodes); ++i) {
+ if (PyList_GET_ITEM(nodes, i) == element) {
+ PyXmlSec_LxmlShadowVacateIdSlot(entry, i);
+ }
+ }
+ }
+}
+
+// The root element of the document `node` belongs to — the one lxml answers
+// with even for a node that has left the tree. New reference, or NULL.
+static PyObject* PyXmlSec_LxmlDocumentRoot(PyObject* node) {
+ PyObject* tree = PyObject_CallMethod(node, "getroottree", NULL);
+ PyObject* root = tree == NULL ? NULL : PyObject_CallMethod(tree, "getroot", NULL);
+
+ Py_XDECREF(tree);
+ return root;
+}
+
+// Non-zero when nothing but the registry can reach `node` again: the registry
+// holds the only reference to the proxy, and the tree the node hangs in is
+// not the document's (`root` is that document's root element) — an unlinked
+// subtree, which lxml keeps alive only for as long as a proxy remains
+// somewhere in it. On the fast path such a subtree is freed at that same
+// moment, and libxml2 drops the ID entries of the attributes it takes with
+// it, so a slot that answers yes here stands for a registration the fast path
+// no longer has either. Best effort: anything that goes wrong answers
+// "reachable", which only keeps the slot.
+static int PyXmlSec_LxmlShadowIdNodeIsDead(PyObject* node, PyObject* root) {
+ PyObject* top;
+ PyObject* nodes = NULL;
+ Py_ssize_t i;
+ int dead = 0;
+
+ if (Py_REFCNT(node) != 1) {
+ return 0;
+ }
+ // The top of the tree `node` hangs in, through lxml's own API.
+ top = node;
+ Py_INCREF(top);
+ for (;;) {
+ PyObject* parent = PyObject_CallMethod(top, "getparent", NULL);
+ if (parent == NULL) {
+ goto DONE;
+ }
+ if (parent == Py_None) {
+ Py_DECREF(parent);
+ break;
+ }
+ Py_DECREF(top);
+ top = parent;
+ }
+ // Still in the document's own tree, which is reachable through the
+ // document itself, whoever holds that; only the entry's own liveness test
+ // can retire one of those.
+ if (top == root) {
+ goto DONE;
+ }
+ // lxml hands out at most one proxy per node, so a reference held by
+ // anyone but this walk and the registry is a way back to `node`: from any
+ // node of the tree, `getparent()` and the children lead to every other.
+ nodes = PyObject_CallMethod(top, "xpath", "s",
+ "descendant-or-self::*"
+ "|descendant-or-self::comment()"
+ "|descendant-or-self::processing-instruction()");
+ if (nodes == NULL || !PyList_Check(nodes)) {
+ goto DONE;
+ }
+ dead = 1;
+ for (i = 0; i < PyList_GET_SIZE(nodes) && dead; ++i) {
+ PyObject* item = PyList_GET_ITEM(nodes, i); // borrowed
+ Py_ssize_t held = 1; // the list's own reference
+ if (item == node) {
+ ++held; // the registry's
+ }
+ if (item == top) {
+ ++held; // the walk above
+ }
+ dead = Py_REFCNT(item) <= held;
+ }
+
+DONE:
+ Py_XDECREF(nodes);
+ Py_DECREF(top);
+ PyErr_Clear();
+ return dead;
+}
+
+// Vacates every slot no spec names any more.
+static void PyXmlSec_LxmlShadowCompactIdNodes(PyObject* entry) {
+ PyObject* nodes = PyTuple_GET_ITEM(entry, PYXMLSEC_ID_ENTRY_NODES);
+ PyObject* specs = PyTuple_GET_ITEM(entry, PYXMLSEC_ID_ENTRY_SPECS);
+ Py_ssize_t i;
+ Py_ssize_t j;
+
+ for (i = 0; i < PyList_GET_SIZE(nodes); ++i) {
+ int named = 0;
+ if (PyList_GET_ITEM(nodes, i) == Py_None) {
+ continue;
+ }
+ for (j = 0; j < PyList_GET_SIZE(specs) && !named; ++j) {
+ named = PyLong_AsSsize_t(PyTuple_GET_ITEM(PyList_GET_ITEM(specs, j), 2)) == i;
+ }
+ if (!named) {
+ PyXmlSec_LxmlShadowVacateIdSlot(entry, i);
+ }
+ }
+}
+
+// Vacates the slots of nodes nothing can reach any more. Registration churn
+// on a long-lived document — elements created, registered, dropped — would
+// otherwise pin every one of those elements, and its subtree, for as long as
+// the document lives, where the fast path's own registration dies with the
+// element. Runs before every registration, so the entry stays bounded by the
+// registrations that can still be replayed.
+static void PyXmlSec_LxmlShadowReclaimIdNodes(PyObject* entry) {
+ PyObject* doc = PyTuple_GET_ITEM(entry, PYXMLSEC_ID_ENTRY_DOC);
+ PyObject* nodes = PyTuple_GET_ITEM(entry, PYXMLSEC_ID_ENTRY_NODES);
+ PyObject* root = NULL; // this document's root element, one lookup for the whole pass
+ Py_ssize_t i;
+
+ for (i = 0; i < PyList_GET_SIZE(nodes); ++i) {
+ PyObject* node = PyList_GET_ITEM(nodes, i); // borrowed
+ // Anything the registry does not hold alone is reachable, and a node
+ // adopted into another document stands for nothing here anyway.
+ if (node == Py_None || Py_REFCNT(node) != 1
+ || (PyObject*)((PyXmlSec_LxmlElementPtr)node)->_doc != doc) {
+ continue;
+ }
+ if (root == NULL && (root = PyXmlSec_LxmlDocumentRoot(node)) == NULL) {
+ PyErr_Clear();
+ break;
+ }
+ if (PyXmlSec_LxmlShadowIdNodeIsDead(node, root)) {
+ PyXmlSec_LxmlShadowVacateIdSlot(entry, i);
+ }
+ }
+ Py_XDECREF(root);
+}
+
+// The entry's slot holding `node`, reused from a vacated one or appended.
+// Returns the index, or -1 with an exception set.
+static Py_ssize_t PyXmlSec_LxmlShadowIdSlot(PyObject* entry, PyObject* node) {
+ PyObject* nodes = PyTuple_GET_ITEM(entry, PYXMLSEC_ID_ENTRY_NODES);
+ Py_ssize_t n = PyList_GET_SIZE(nodes);
+ Py_ssize_t vacant = -1;
+ Py_ssize_t i;
+
+ for (i = 0; i < n; ++i) {
+ PyObject* item = PyList_GET_ITEM(nodes, i); // borrowed
+ if (item == node) {
+ return i; // already held here, and so already dropped elsewhere
+ }
+ if (item == Py_None && vacant < 0) {
+ vacant = i;
+ }
+ }
+ if (vacant < 0) {
+ if (PyList_Append(nodes, node) < 0) {
+ return -1;
+ }
+ vacant = n;
+ } else {
+ Py_INCREF(node);
+ PyList_SetItem(nodes, vacant, node); // steals the reference
+ }
+ // One reference per registered element, so that the liveness test can
+ // account for exactly the references the registry itself holds.
+ PyXmlSec_LxmlShadowForgetIdNode(entry, node);
+ return vacant;
+}
+
+// Drops the entries of documents nobody but the registry still references.
+// Best effort: on an allocation failure the entries simply survive until the
+// next registration prunes them.
+static void PyXmlSec_LxmlShadowPruneIdRegistry(void) {
+ PyObject* dead;
+ PyObject* key;
+ PyObject* entry;
+ Py_ssize_t pos = 0;
+ Py_ssize_t i;
+
+ dead = PyList_New(0);
+ if (dead == NULL) {
+ PyErr_Clear();
+ return;
+ }
+ // The dict cannot be mutated while iterating it, so collect first.
+ while (PyDict_Next(PyXmlSec_LxmlShadowIdRegistry, &pos, &key, &entry)) {
+ if (PyXmlSec_LxmlShadowIdEntryIsDead(entry) && PyList_Append(dead, key) < 0) {
+ PyErr_Clear();
+ break;
+ }
+ }
+ for (i = 0; i < PyList_GET_SIZE(dead); ++i) {
+ if (PyDict_DelItem(PyXmlSec_LxmlShadowIdRegistry, PyList_GET_ITEM(dead, i)) < 0) {
+ PyErr_Clear();
+ }
+ }
+ Py_DECREF(dead);
+}
+
+// The attribute of `element` the fast path would pick for `name` (in `ns`
+// when given): its value as a str, a new reference, and — when `key` is not
+// NULL — the lxml key naming it ("{href}local", or "local" unqualified),
+// also a new reference. Py_None (and *key left NULL) when the element
+// carries no such attribute. Without a namespace the lookup goes by *local*
+// name, whatever namespace the attribute is in, and takes the first such
+// attribute in document order, because that is how the fast path matches:
+// both xmlHasProp() and xmlSecAddIDs() compare names only, where lxml's
+// element.get(name) would find an unqualified attribute alone.
+//
+// The key is what tells two same-valued attributes of one element apart, so
+// that "the very attribute being registered" stays a question about the
+// attribute rather than about the element carrying it.
+static PyObject* PyXmlSec_LxmlAttrFind(PyXmlSec_LxmlElementPtr element, const char* name, const char* ns, PyObject** key) {
+ PyObject* items;
+ PyObject* seq;
+ PyObject* found = NULL;
+ Py_ssize_t i;
+ Py_ssize_t n;
+
+ if (key != NULL) {
+ *key = NULL;
+ }
+
+ if (ns != NULL) {
+ // lxml spells a namespaced attribute "{href}local".
+ PyObject* qname = PyUnicode_FromFormat("{%s}%s", ns, name);
+ if (qname == NULL) {
+ return NULL;
+ }
+ found = PyObject_CallMethod((PyObject*)element, "get", "O", qname);
+ if (found != NULL && found != Py_None && key != NULL) {
+ *key = qname;
+ } else {
+ Py_DECREF(qname);
+ }
+ return found;
+ }
+
+ items = PyObject_CallMethod((PyObject*)element, "items", NULL);
+ if (items == NULL) {
+ return NULL;
+ }
+ seq = PySequence_Fast(items, "unexpected attributes.");
+ Py_DECREF(items);
+ if (seq == NULL) {
+ return NULL;
+ }
+
+ n = PySequence_Fast_GET_SIZE(seq);
+ for (i = 0; i < n && found == NULL; ++i) {
+ PyObject* item = PySequence_Fast_GET_ITEM(seq, i); // borrowed (name, value)
+ PyObject* qname;
+ const char* local;
+ const char* end;
+
+ if (!PyTuple_Check(item) || PyTuple_GET_SIZE(item) != 2) {
+ PyErr_SetString(PyXmlSec_InternalError, "unexpected attribute.");
+ break;
+ }
+ qname = PyTuple_GET_ITEM(item, 0); // borrowed
+ if (!PyUnicode_Check(qname) || (local = PyUnicode_AsUTF8(qname)) == NULL) {
+ if (!PyErr_Occurred()) {
+ PyErr_SetString(PyXmlSec_InternalError, "unexpected attribute name.");
+ }
+ break;
+ }
+ if (local[0] == '{' && (end = strchr(local, '}')) != NULL) {
+ local = end + 1;
+ }
+ if (strcmp(local, name) == 0) {
+ found = PyTuple_GET_ITEM(item, 1); // borrowed
+ Py_INCREF(found);
+ if (key != NULL) {
+ *key = qname;
+ Py_INCREF(qname);
+ }
+ }
+ }
+ Py_DECREF(seq);
+ if (found == NULL && !PyErr_Occurred()) {
+ Py_INCREF(Py_None);
+ found = Py_None;
+ }
+ return found;
+}
+
+// node.xpath(expr, n=name, v=value) — `name` is optional. The id lookups
+// below need XPath variables rather than an interpolated expression, so that
+// an id value never gets read as expression syntax.
+static PyObject* PyXmlSec_LxmlXPath(PyObject* node, const char* expr, PyObject* name, PyObject* value) {
+ PyObject* method;
+ PyObject* args = NULL;
+ PyObject* kwargs = NULL;
+ PyObject* rv = NULL;
+
+ method = PyObject_GetAttrString(node, "xpath");
+ if (method == NULL) {
+ return NULL;
+ }
+ args = Py_BuildValue("(s)", expr);
+ kwargs = PyDict_New();
+ if (args == NULL || kwargs == NULL || PyDict_SetItemString(kwargs, "v", value) < 0
+ || (name != NULL && PyDict_SetItemString(kwargs, "n", name) < 0)) {
+ goto DONE;
+ }
+ rv = PyObject_Call(method, args, kwargs);
+
+DONE:
+ Py_DECREF(method);
+ Py_XDECREF(args);
+ Py_XDECREF(kwargs);
+ return rv;
+}
+
+// The number of `element`'s attributes whose value is `value`, or -1.
+static Py_ssize_t PyXmlSec_LxmlAttrsWithValue(PyXmlSec_LxmlElementPtr element, PyObject* value) {
+ PyObject* values = PyObject_CallMethod((PyObject*)element, "values", NULL);
+ PyObject* seq;
+ Py_ssize_t i;
+ Py_ssize_t n;
+ Py_ssize_t found = 0;
+
+ if (values == NULL) {
+ return -1;
+ }
+ seq = PySequence_Fast(values, "unexpected attributes.");
+ Py_DECREF(values);
+ if (seq == NULL) {
+ return -1;
+ }
+ n = PySequence_Fast_GET_SIZE(seq);
+ for (i = 0; i < n; ++i) {
+ int same = PyObject_RichCompareBool(PySequence_Fast_GET_ITEM(seq, i), value, Py_EQ);
+ if (same < 0) {
+ Py_DECREF(seq);
+ return -1;
+ }
+ found += same;
+ }
+ Py_DECREF(seq);
+ return found;
+}
+
+// The lxml key of the attribute lxml's parse declared as the XML ID for
+// `value` — a new reference, Py_None when there is none. XPath's id() names
+// the element carrying it but never the attribute, and the DTD that typed it
+// may declare no element at all (an ATTLIST alone leaves lxml's DTD objects
+// empty), so the only way to name the attribute is to ask a libxml2 we may
+// read from: the document is copied the way a whole-document shadow copies
+// it — same base URL, same subsets — and its own id hash consulted.
+static PyObject* PyXmlSec_LxmlShadowDeclaredIdKey(PyXmlSec_LxmlElementPtr element, PyObject* value) {
+ PyObject* tree = NULL;
+ PyObject* bytes = NULL;
+ PyObject* url_holder = NULL;
+ PyObject* key = NULL;
+ const char* url = NULL;
+ const char* text;
+ xmlDocPtr doc = NULL;
+ xmlAttrPtr attr;
+ int dtd = 0;
+ int extdtd = 0;
+
+ text = PyUnicode_AsUTF8(value);
+ if (text == NULL) {
+ goto DONE;
+ }
+ tree = PyObject_CallMethod((PyObject*)element, "getroottree", NULL);
+ if (tree == NULL) {
+ goto DONE;
+ }
+ bytes = PyObject_CallFunctionObjArgs(PyXmlSec_LxmlEtreeToString, tree, NULL);
+ if (bytes == NULL || PyXmlSec_LxmlDocumentUrl(tree, &url_holder, &url) < 0
+ || PyXmlSec_LxmlDocumentSubsets(tree, &dtd, &extdtd) < 0) {
+ goto DONE;
+ }
+ doc = PyXmlSec_LxmlShadowParse(bytes, url, extdtd, "cannot make a private copy of the document.");
+ if (doc == NULL) {
+ goto DONE;
+ }
+ attr = xmlGetID(doc, XSTR(text));
+ if (attr == NULL) {
+ Py_INCREF(Py_None);
+ key = Py_None;
+ } else if (attr->ns != NULL && attr->ns->href != NULL) {
+ key = PyUnicode_FromFormat("{%s}%s", (const char*)attr->ns->href, (const char*)attr->name);
+ } else {
+ key = PyUnicode_FromString((const char*)attr->name);
+ }
+
+DONE:
+ if (doc != NULL) {
+ xmlFreeDoc(doc);
+ }
+ Py_XDECREF(tree);
+ Py_XDECREF(bytes);
+ Py_XDECREF(url_holder);
+ return key;
+}
+
+// Non-zero when `value` is registered as an XML ID in lxml's own document —
+// a DTD-declared id attribute, or an xml:id, both of which lxml's libxml2
+// entered into the document's id hash when it parsed the tree — for anything
+// but `element`'s own `key` attribute. That hash belongs to lxml's library
+// and cannot be read from here; XPath's id() is the one door into it that
+// takes and returns nothing but strings and elements.
+//
+// id() names an element, not an attribute, and an element can carry the same
+// value twice: `` answers `N` whichever of the two
+// is asked about, where the fast path registers `ID` and finds `xml:id`
+// holding the value already. So a match on `element` itself only settles it
+// when a single attribute of it carries the value — otherwise the declared
+// attribute has to be named outright.
+//
+// id() reads its argument as a whitespace-separated list of ids, so a value
+// carrying whitespace would probe the wrong strings entirely: such a value
+// is left to the registry check below, which compares whole values.
+static int PyXmlSec_LxmlShadowIdIsDeclared(PyXmlSec_LxmlElementPtr element, PyObject* key, PyObject* value) {
+ PyObject* matches;
+ PyObject* declared;
+ Py_ssize_t i;
+ Py_ssize_t n;
+ Py_ssize_t carriers;
+ const char* text;
+ Py_ssize_t size;
+ int self = 0;
+ int taken = 0;
+
+ text = PyUnicode_AsUTF8AndSize(value, &size);
+ if (text == NULL) {
+ return -1;
+ }
+ if (size == 0 || strcspn(text, " \t\r\n") != (size_t)size) {
+ return 0;
+ }
+
+ matches = PyXmlSec_LxmlXPath((PyObject*)element, "id($v)", NULL, value);
+ if (matches == NULL) {
+ return -1;
+ }
+ n = PySequence_Size(matches);
+ for (i = 0; i < n && !taken; ++i) {
+ PyObject* match = PySequence_GetItem(matches, i);
+ if (match == NULL) {
+ Py_DECREF(matches);
+ return -1;
+ }
+ // lxml hands out one proxy per node, so identity is node identity.
+ taken = match != (PyObject*)element;
+ self = self || match == (PyObject*)element;
+ Py_DECREF(match);
+ }
+ Py_DECREF(matches);
+ if (n < 0) {
+ return -1;
+ }
+ if (taken || !self) {
+ return taken;
+ }
+
+ // The declared id is on this element. Only one attribute of it holds the
+ // value, and that attribute is the one being registered (it holds the
+ // value by construction), so this is the fast path's `tmpAttr == attr`.
+ carriers = PyXmlSec_LxmlAttrsWithValue(element, value);
+ if (carriers < 0) {
+ return -1;
+ }
+ if (carriers < 2) {
+ return 0;
+ }
+
+ declared = PyXmlSec_LxmlShadowDeclaredIdKey(element, value);
+ if (declared == NULL) {
+ return -1;
+ }
+ // A copy that declares no id at all leaves the value unclaimed in the
+ // world the shadow runs in, which is the world the registration is for.
+ if (declared == Py_None) {
+ taken = 0;
+ } else {
+ int same = PyObject_RichCompareBool(declared, key, Py_EQ);
+ taken = same < 0 ? -1 : !same;
+ }
+ Py_DECREF(declared);
+ return taken;
+}
+
+// Non-zero when a recorded spec — `name` (in `ns` when given) applied to
+// `node` — claims `value` for an attribute other than `element`'s `key`, the
+// one being registered. The spec is resolved to the attribute it actually
+// names, the way the replay's xmlHasProp/xmlHasNsProp resolves it: two
+// attributes of one element can carry the same value under different names,
+// and only the attribute settles whether this is the fast path's harmless
+// `tmpAttr == attr` or its "duplicated id.".
+static int PyXmlSec_LxmlShadowSpecClaims(PyXmlSec_LxmlElementPtr node, const char* name, const char* ns,
+ PyXmlSec_LxmlElementPtr element, PyObject* key, PyObject* value) {
+ PyObject* other_key = NULL;
+ PyObject* other = PyXmlSec_LxmlAttrFind(node, name, ns, &other_key);
+ int same;
+ int mine;
+
+ if (other == NULL) {
+ return -1;
+ }
+ same = PyObject_RichCompareBool(other, value, Py_EQ);
+ Py_DECREF(other);
+ if (same <= 0) {
+ Py_XDECREF(other_key);
+ return same;
+ }
+ mine = node == element && other_key != NULL && PyObject_RichCompareBool(other_key, key, Py_EQ);
+ Py_XDECREF(other_key);
+ if (PyErr_Occurred()) {
+ return -1;
+ }
+ return !mine;
+}
+
+// Non-zero when `value` is already registered as an XML ID for anything but
+// `element`'s own `key` attribute — what the fast path finds when it tests
+// `xmlGetID(doc, value) != attr` and raises "duplicated id.". Under the
+// shadow the registrations live in two places: lxml's document (above), and
+// this registry, holding what earlier register_id/add_ids calls recorded and
+// every whole-document Begin replays. A registration for the very attribute
+// being registered is not a collision — the fast path's `tmpAttr == attr`
+// leaves the document alone and returns.
+static int PyXmlSec_LxmlShadowIdIsTaken(PyXmlSec_LxmlElementPtr element, PyObject* key, PyObject* value) {
+ PyObject* doc_key;
+ PyObject* entry;
+ PyObject* nodes;
+ PyObject* specs;
+ Py_ssize_t i;
+ int taken;
+
+ taken = PyXmlSec_LxmlShadowIdIsDeclared(element, key, value);
+ if (taken != 0) {
+ return taken;
+ }
+
+ doc_key = PyLong_FromVoidPtr((void*)element->_doc);
+ if (doc_key == NULL) {
+ return -1;
+ }
+ entry = PyDict_GetItem(PyXmlSec_LxmlShadowIdRegistry, doc_key); // borrowed
+ Py_DECREF(doc_key);
+ if (entry == NULL) {
+ return 0;
+ }
+ // A registration whose element nothing can reach any more claims nothing:
+ // the fast path lost that attribute — and libxml2's id entry for it —
+ // when lxml freed the element.
+ PyXmlSec_LxmlShadowReclaimIdNodes(entry);
+
+ nodes = PyTuple_GET_ITEM(entry, PYXMLSEC_ID_ENTRY_NODES);
+ specs = PyTuple_GET_ITEM(entry, PYXMLSEC_ID_ENTRY_SPECS);
+ for (i = 0; i < PyList_GET_SIZE(specs); ++i) {
+ PyObject* spec = PyList_GET_ITEM(specs, i); // (name, ns, node index)
+ PyObject* node = PyList_GET_ITEM(nodes, PyLong_AsSsize_t(PyTuple_GET_ITEM(spec, 2)));
+ PyObject* spec_ns = PyTuple_GET_ITEM(spec, 1);
+ const char* spec_name = PyUnicode_AsUTF8(PyTuple_GET_ITEM(spec, 0));
+ const char* spec_href;
+
+ if (spec_name == NULL) {
+ return -1;
+ }
+ spec_href = spec_ns == Py_None ? NULL : PyUnicode_AsUTF8(spec_ns);
+ if (spec_href == NULL && spec_ns != Py_None) {
+ return -1;
+ }
+ // A vacated slot, or an element adopted into another tree: neither
+ // stands for anything in this document, and the replay skips both.
+ if (node == Py_None || ((PyXmlSec_LxmlElementPtr)node)->_doc != element->_doc) {
+ continue;
+ }
+ taken = PyXmlSec_LxmlShadowSpecClaims((PyXmlSec_LxmlElementPtr)node, spec_name, spec_href,
+ element, key, value);
+ if (taken < 0) {
+ return -1;
+ }
+ if (taken) {
+ return 1;
+ }
+ }
+ return 0;
+}
+
+// The elements a registration covers, in document order: `element` alone, or
+// — for add_ids — the subtree rooted at it, the scope xmlSecAddIDs walks.
+// New reference to a list, or NULL with an exception set.
+static PyObject* PyXmlSec_LxmlShadowIdTargets(PyXmlSec_LxmlElementPtr element, int subtree) {
+ PyObject* targets;
+
+ if (!subtree) {
+ return Py_BuildValue("[O]", (PyObject*)element);
+ }
+ targets = PyObject_CallMethod((PyObject*)element, "xpath", "s", "descendant-or-self::*");
+ if (targets != NULL && !PyList_Check(targets)) {
+ PyErr_SetString(PyXmlSec_InternalError, "unexpected elements.");
+ Py_CLEAR(targets);
+ }
+ return targets;
+}
+
+int PyXmlSec_LxmlShadowRecordIds(PyXmlSec_LxmlElementPtr element, PyObject* names, const char* ns, int subtree) {
+ PyObject* key = NULL;
+ PyObject* created = NULL;
+ PyObject* targets = NULL;
+ PyObject* spec = NULL;
+ PyObject* entry = NULL;
+ PyObject* nodes;
+ PyObject* specs;
+ Py_ssize_t nnodes = 0;
+ Py_ssize_t nspecs = 0;
+ Py_ssize_t t;
+ Py_ssize_t i;
+ int fresh = 0;
+ int result = -1;
+
+ targets = PyXmlSec_LxmlShadowIdTargets(element, subtree);
+ key = PyLong_FromVoidPtr((void*)element->_doc);
+ if (targets == NULL || key == NULL) {
+ goto DONE;
+ }
+
+ PyXmlSec_LxmlShadowPruneIdRegistry();
+ entry = PyDict_GetItem(PyXmlSec_LxmlShadowIdRegistry, key); // borrowed
+ if (entry == NULL) {
+ nodes = PyList_New(0);
+ specs = PyList_New(0);
+ if (nodes != NULL && specs != NULL) {
+ created = PyTuple_Pack(3, (PyObject*)element->_doc, nodes, specs);
+ }
+ Py_XDECREF(nodes);
+ Py_XDECREF(specs);
+ if (created == NULL || PyDict_SetItem(PyXmlSec_LxmlShadowIdRegistry, key, created) < 0) {
+ goto DONE;
+ }
+ entry = created;
+ fresh = 1;
+ } else {
+ PyXmlSec_LxmlShadowReclaimIdNodes(entry);
+ }
+
+ nodes = PyTuple_GET_ITEM(entry, PYXMLSEC_ID_ENTRY_NODES);
+ specs = PyTuple_GET_ITEM(entry, PYXMLSEC_ID_ENTRY_SPECS);
+ nnodes = PyList_GET_SIZE(nodes);
+ nspecs = PyList_GET_SIZE(specs);
+
+ // One spec per attribute the scope carries *now*, element by element in
+ // document order and, within an element, in the order of `names`: the
+ // registration xmlSecAddIDs makes, taken at the call rather than left to
+ // the replay, which would see whatever the tree had become by then.
+ for (t = 0; t < PyList_GET_SIZE(targets); ++t) {
+ PyObject* target = PyList_GET_ITEM(targets, t); // borrowed
+ Py_ssize_t idx = -1;
+ for (i = 0; i < PyList_GET_SIZE(names); ++i) {
+ PyObject* name = PyList_GET_ITEM(names, i); // borrowed
+ const char* text = PyUnicode_AsUTF8(name);
+ PyObject* value;
+ int carried;
+ int contains;
+
+ if (text == NULL) {
+ goto DONE;
+ }
+ value = PyXmlSec_LxmlAttrFind((PyXmlSec_LxmlElementPtr)target, text, ns, NULL);
+ if (value == NULL) {
+ goto DONE;
+ }
+ carried = value != Py_None;
+ Py_DECREF(value);
+ if (!carried) {
+ continue;
+ }
+ if (idx < 0 && (idx = PyXmlSec_LxmlShadowIdSlot(entry, target)) < 0) {
+ goto DONE;
+ }
+ spec = Py_BuildValue("(Ozn)", name, ns, idx);
+ if (spec == NULL) {
+ goto DONE;
+ }
+ contains = PySequence_Contains(specs, spec);
+ if (contains < 0 || (!contains && PyList_Append(specs, spec) < 0)) {
+ goto DONE;
+ }
+ Py_CLEAR(spec);
+ }
+ }
+ result = 0;
+
+DONE:
+ // All of the call's names are recorded or none of them are: a caller that
+ // hands over a bad list must not find part of it registered. Rolling back
+ // must not clobber the failure that caused it, hence the fetch/restore.
+ if (result < 0 && entry != NULL) {
+ PyObject* type;
+ PyObject* value;
+ PyObject* tb;
+ PyErr_Fetch(&type, &value, &tb);
+ if (fresh) {
+ if (PyDict_DelItem(PyXmlSec_LxmlShadowIdRegistry, key) < 0) {
+ PyErr_Clear();
+ }
+ } else {
+ if (PyList_SetSlice(PyTuple_GET_ITEM(entry, PYXMLSEC_ID_ENTRY_SPECS), nspecs, PY_SSIZE_T_MAX, NULL) < 0
+ || PyList_SetSlice(PyTuple_GET_ITEM(entry, PYXMLSEC_ID_ENTRY_NODES), nnodes, PY_SSIZE_T_MAX, NULL) < 0) {
+ PyErr_Clear();
+ }
+ // A slot reused for one of the rolled-back specs holds a node no
+ // surviving spec names any more.
+ PyXmlSec_LxmlShadowCompactIdNodes(entry);
+ }
+ PyErr_Restore(type, value, tb);
+ }
+ Py_XDECREF(key);
+ Py_XDECREF(created);
+ Py_XDECREF(targets);
+ Py_XDECREF(spec);
+ return result;
+}
+
+int PyXmlSec_LxmlShadowRegisterId(PyXmlSec_LxmlElementPtr element, const char* name, const char* ns) {
+ PyObject* key = NULL;
+ PyObject* value = PyXmlSec_LxmlAttrFind(element, name, ns, &key);
+ PyObject* names;
+ int taken;
+ int rv;
+
+ if (value == NULL) {
+ return -1;
+ }
+ if (value == Py_None) {
+ Py_DECREF(value);
+ Py_XDECREF(key);
+ PyErr_SetString(PyXmlSec_Error, "missing attribute.");
+ return -1;
+ }
+ // `key` names the attribute the fast path would hand to xmlAddID; every
+ // collision test below asks whether the value is claimed by some *other*
+ // attribute than that one.
+ taken = PyXmlSec_LxmlShadowIdIsTaken(element, key, value);
+ Py_DECREF(value);
+ Py_XDECREF(key);
+ if (taken < 0) {
+ return -1;
+ }
+ if (taken) {
+ // The requested attribute cannot win the lookup, so the registration
+ // is refused rather than recorded and silently skipped at replay:
+ // the caller would otherwise never learn that its `#id` reference
+ // resolves to the element that claimed the value first.
+ PyErr_SetString(PyXmlSec_Error, "duplicated id.");
+ return -1;
+ }
+
+ // Scope 0: this node alone, exactly what the fast path registers.
+ names = Py_BuildValue("[s]", name);
+ if (names == NULL) {
+ return -1;
+ }
+ rv = PyXmlSec_LxmlShadowRecordIds(element, names, ns, 0);
+ Py_DECREF(names);
+ return rv;
+}
+
+// Registers `node`'s `name` attribute (in `ns` when given) as an XML ID, the
+// way xmlSecAddIDs does: the first registration of a value wins.
+static void PyXmlSec_LxmlShadowAddId(xmlDocPtr doc, xmlNodePtr node, const xmlChar* name, const xmlChar* ns) {
+ xmlAttrPtr attr = ns != NULL ? xmlHasNsProp(node, name, ns) : xmlHasProp(node, name);
+ xmlChar* value;
+
+ if (attr == NULL || attr->children == NULL) {
+ return;
+ }
+ value = xmlNodeListGetString(doc, attr->children, 1);
+ if (value == NULL) {
+ return;
+ }
+ if (xmlGetID(doc, value) == NULL) {
+ xmlAddID(NULL, doc, value, attr);
+ }
+ xmlFree(value);
+}
+
+// Applies one recorded spec to `node`, the copy's counterpart of the element
+// it was registered for.
+static void PyXmlSec_LxmlShadowApplyIdSpec(xmlDocPtr doc, xmlNodePtr node, const xmlChar* name, const xmlChar* ns) {
+ if (node == NULL || node->type != XML_ELEMENT_NODE) {
+ return;
+ }
+ PyXmlSec_LxmlShadowAddId(doc, node, name, ns);
+}
+
+// Replays the specs recorded for the shadow's live document onto the copy,
+// each at the copy's counterpart of the element it was registered for: a spec
+// recorded for an element under the live `live_top` is applied under its
+// counterpart `copy_top`. Returns 0, or -1 with an exception set.
+static int PyXmlSec_LxmlShadowReplayIds(PyXmlSec_LxmlShadow* shadow, PyObject* live_top, xmlNodePtr copy_top) {
+ PyObject* key;
+ PyObject* entry;
+ PyObject* nodes;
+ PyObject* specs;
+ int path[PYXMLSEC_SHADOW_MAX_DEPTH];
+ Py_ssize_t i, n;
+
+ key = PyLong_FromVoidPtr((void*)shadow->element->_doc);
+ if (key == NULL) {
+ return -1;
+ }
+ // The entry, if any, belongs to this very document: the registry's own
+ // reference keeps the address from being reused by another one.
+ entry = PyDict_GetItem(PyXmlSec_LxmlShadowIdRegistry, key); // borrowed
+ Py_DECREF(key);
+ if (entry == NULL) {
+ return 0;
+ }
+
+ nodes = PyTuple_GET_ITEM(entry, PYXMLSEC_ID_ENTRY_NODES);
+ specs = PyTuple_GET_ITEM(entry, PYXMLSEC_ID_ENTRY_SPECS);
+ n = PyList_GET_SIZE(specs);
+ for (i = 0; i < n; ++i) {
+ PyObject* spec = PyList_GET_ITEM(specs, i); // (name, ns, node index)
+ PyObject* node = PyList_GET_ITEM(nodes, PyLong_AsSsize_t(PyTuple_GET_ITEM(spec, 2)));
+ PyObject* top = NULL;
+ const char* name = PyUnicode_AsUTF8(PyTuple_GET_ITEM(spec, 0));
+ const char* ns = PyTuple_GET_ITEM(spec, 1) == Py_None ? NULL : PyUnicode_AsUTF8(PyTuple_GET_ITEM(spec, 1));
+ int depth;
+
+ if (name == NULL || (ns == NULL && PyErr_Occurred())) {
+ return -1;
+ }
+ if (node == Py_None) { // vacated slot: the node is registered elsewhere now
+ continue;
+ }
+ depth = PyXmlSec_LxmlLivePathTo(node, path, &top);
+ if (depth < 0) {
+ return -1;
+ }
+ // lxml hands out one proxy per node, so identity settles whether the
+ // element still hangs under the tree being replayed; a registration
+ // for an element that has since left it applies to nothing here.
+ if (top == live_top) {
+ PyXmlSec_LxmlShadowApplyIdSpec(shadow->doc, PyXmlSec_LxmlShadowWalkNode(copy_top, path, depth),
+ (const xmlChar*)name, (const xmlChar*)ns);
+ }
+ Py_DECREF(top);
+ }
+ return 0;
+}
+
+// Copies the live subtree `element` — removed from its document, so the copy
+// of that document does not hold it either — into the private document, as an
+// unlinked node beside its tree. That is the shape the raw path hands xmlsec:
+// a node outside the tree whose `doc` still answers `#id` references.
+// Returns 0, or -1 with an exception set.
+static int PyXmlSec_LxmlShadowUnlinkedCopy(PyXmlSec_LxmlShadow* shadow, PyObject* element) {
+ PyObject* bytes = PyXmlSec_LxmlElementToBytes(element);
+ xmlDocPtr tdoc;
+ xmlNodePtr copy;
+
+ if (bytes == NULL) {
+ return -1;
+ }
+ tdoc = PyXmlSec_LxmlShadowParse(bytes, NULL, 0, "cannot make a private copy of the element.");
+ Py_DECREF(bytes);
+ if (tdoc == NULL) {
+ return -1;
+ }
+ copy = xmlDocCopyNode(xmlDocGetRootElement(tdoc), shadow->doc, 1);
+ xmlFreeDoc(tdoc);
+ if (copy == NULL) {
+ PyErr_SetString(PyXmlSec_InternalError, "cannot make a private copy of the element.");
+ return -1;
+ }
+ shadow->root = copy;
+ shadow->unlinked = 1;
+ return 0;
+}
+
+int PyXmlSec_LxmlShadowBeginDoc(PyXmlSec_LxmlShadow* shadow, PyXmlSec_LxmlElementPtr element, xmlNodePtr* target) {
+ PyObject* cur = NULL;
+ PyObject* live_root = NULL;
+ PyObject* tree = NULL;
+ PyObject* bytes = NULL;
+ PyObject* url_holder = NULL;
+ const char* url = NULL;
+ int path[PYXMLSEC_SHADOW_MAX_DEPTH];
+ int depth;
+ int dtd = 0;
+ int extdtd = 0;
+
+ shadow->element = element;
+ shadow->owned = NULL;
+ shadow->doc = NULL;
+ shadow->root = NULL;
+ shadow->tags = NULL;
+ shadow->ntags = 0;
+ shadow->unlinked = 0;
+ *target = NULL;
+
+ if (!PyXmlSec_LxmlShadowActive) {
+ shadow->root = element->_c_node;
+ *target = element->_c_node;
+ return 0;
+ }
+
+ // `cur` ends as the live root element and `path` leads back down to
+ // `element`.
+ depth = PyXmlSec_LxmlLivePathTo((PyObject*)element, path, &cur);
+ if (depth < 0) {
+ goto ON_FAIL;
+ }
+
+ // Serialize the whole tree, not just the root element, so comments/PIs
+ // outside the root and the internal DTD subset (declared IDs) survive
+ // into the copy.
+ tree = PyObject_CallMethod(cur, "getroottree", NULL);
+ if (tree == NULL) {
+ goto ON_FAIL;
+ }
+ // `cur` is the document's root element unless `element` — or an ancestor
+ // of it — was removed from the tree: lxml leaves such a subtree pointing
+ // at the document it left, which the dump below therefore does not hold.
+ live_root = PyObject_CallMethod(tree, "getroot", NULL);
+ if (live_root == NULL) {
+ goto ON_FAIL;
+ }
+ // The whole tree is dumped either way here, so only the external subset
+ // matters: it has to be loaded for the copy to know the IDs it declares.
+ bytes = PyObject_CallFunctionObjArgs(PyXmlSec_LxmlEtreeToString, tree, NULL);
+ if (bytes == NULL || PyXmlSec_LxmlDocumentUrl(tree, &url_holder, &url) < 0
+ || PyXmlSec_LxmlDocumentSubsets(tree, &dtd, &extdtd) < 0) {
+ goto ON_FAIL;
+ }
+ Py_CLEAR(tree);
+ shadow->doc = PyXmlSec_LxmlShadowParse(bytes, url, extdtd, "cannot make a private copy of the document.");
+ Py_CLEAR(bytes);
+ Py_CLEAR(url_holder);
+ if (shadow->doc == NULL) {
+ goto ON_FAIL;
+ }
+ shadow->root = xmlDocGetRootElement(shadow->doc);
+ if (live_root != cur && PyXmlSec_LxmlShadowUnlinkedCopy(shadow, cur) < 0) {
+ goto ON_FAIL;
+ }
+ if (PyXmlSec_LxmlShadowMark(shadow) < 0) {
+ goto ON_FAIL;
+ }
+
+ *target = PyXmlSec_LxmlShadowWalkNode(shadow->root, path, depth);
+ if (*target == NULL) {
+ PyErr_SetString(PyXmlSec_InternalError, "cannot locate the element in the private copy.");
+ goto ON_FAIL;
+ }
+
+ // The End functions map paths from the copy root, so the live element
+ // they start from must be the live root.
+ shadow->element = (PyXmlSec_LxmlElementPtr)cur;
+ shadow->owned = cur;
+ cur = NULL;
+
+ // The registered IDs live in lxml's document, which the copy knows
+ // nothing about; replay them so that #id references resolve. An unlinked
+ // subtree carries its own registrations, and the document it left keeps
+ // the rest — a reference from the subtree into that document resolves on
+ // the raw path too.
+ if (PyXmlSec_LxmlShadowReplayIds(shadow, (PyObject*)shadow->element, shadow->root) < 0) {
+ goto ON_FAIL;
+ }
+ if (shadow->unlinked
+ && PyXmlSec_LxmlShadowReplayIds(shadow, live_root, xmlDocGetRootElement(shadow->doc)) < 0) {
+ goto ON_FAIL;
+ }
+ Py_CLEAR(live_root);
+ return 0;
+
+ON_FAIL:
+ Py_XDECREF(cur);
+ Py_XDECREF(live_root);
+ Py_XDECREF(tree);
+ Py_XDECREF(bytes);
+ Py_XDECREF(url_holder);
+ PyXmlSec_LxmlShadowDiscard(shadow);
+ *target = NULL;
+ return -1;
+}
+
+// ----------------------------------------------------------------------------
+// The reflection: everything the xmlsec call did to the copy, applied to the
+// live tree through lxml's Python API.
+//
+// Two kinds of site are recorded while walking the pre-existing (marked)
+// structure of the copy in document order:
+//
+// graft — a fresh node (element, comment, PI) to insert at its child
+// index. Fresh subtrees are grafted wholesale; the scan never
+// descends into them.
+// sync — a tagged parent whose children changed — it gained a fresh node
+// (element or text), lost a tagged one, or had a tagged text child
+// rewritten in place — gets its text slots (its .text and each
+// child's .tail) copied over from the re-parsed copy. That covers
+// everything xmlsec does to text: the "\n" formatting around a new
+// node, values filled into empty elements (DigestValue), and
+// content it removed (encrypt Type=Content) — a removal leaves no
+// fresh node behind, so only the tagged child count shows that it
+// happened, and only a wholesale sync can carry it across.
+//
+// The reflection is two-phase: first every site's payload is fetched from
+// the re-parsed copy while it is still in its final, untouched state, then
+// everything is applied to the live tree in document order (each graft moves
+// a node out of the re-parsed copy, which would invalidate later fetches,
+// and each live insert makes the later, larger indices valid; a parent's
+// sync is recorded after its grafts, so it runs once they are in place).
+// ----------------------------------------------------------------------------
+
+enum { PYXMLSEC_SHADOW_SITE_GRAFT, PYXMLSEC_SHADOW_SITE_SYNC };
+
+typedef struct {
+ int kind;
+ int depth;
+ int path[PYXMLSEC_SHADOW_MAX_DEPTH]; // graft: to the fresh node's parent; sync: to the parent itself
+ int idx; // graft: child index of the fresh node
+ PyObject* value; // phase 1: the node to graft, or the [text, tail, ...] list to sync
+} PyXmlSec_LxmlShadowSite;
+
+typedef struct {
+ PyXmlSec_LxmlShadowSite* items;
+ int count;
+ int capacity;
+ xmlNodePtr top; // the copy's root element — origin of all paths
+ PyXmlSec_LxmlShadow* shadow; // borrowed; owns the tags the walk reads
+} PyXmlSec_LxmlShadowSiteList;
+
+static int PyXmlSec_LxmlShadowSiteAppend(PyXmlSec_LxmlShadowSiteList* list, xmlNodePtr node, int kind) {
+ PyXmlSec_LxmlShadowSite* site;
+ int graft = kind == PYXMLSEC_SHADOW_SITE_GRAFT;
+
+ if (list->count == list->capacity) {
+ int capacity = list->capacity == 0 ? 8 : list->capacity * 2;
+ PyXmlSec_LxmlShadowSite* items = (PyXmlSec_LxmlShadowSite*)PyMem_Realloc(list->items, capacity * sizeof(*items));
+ if (items == NULL) {
+ PyErr_NoMemory();
+ return -1;
+ }
+ list->items = items;
+ list->capacity = capacity;
+ }
+ site = &list->items[list->count];
+ site->kind = kind;
+ site->depth = PyXmlSec_LxmlShadowPathTo(graft ? node->parent : node, list->top, site->path);
+ if (site->depth < 0) {
+ PyErr_SetString(PyXmlSec_InternalError, "unexpected mutation site.");
+ return -1;
+ }
+ site->idx = graft ? PyXmlSec_LxmlShadowChildIndex(node) : 0;
+ site->value = NULL;
+ ++list->count;
+ return 0;
+}
+
+// Walks the tagged structure of the copy in document order, recording a
+// graft for every fresh node and, after them, a sync for their parent — and
+// for a parent that no longer holds the children it was tagged with, whose
+// text slots are the only trace the removal left, or one whose own text
+// survived the call but with different content.
+static int PyXmlSec_LxmlShadowCollectSites(PyXmlSec_LxmlShadowSiteList* list, xmlNodePtr parent, int depth) {
+ PyXmlSec_LxmlShadowTag* tag = PYXMLSEC_SHADOW_TAG(list->shadow, parent);
+ xmlNodePtr n;
+ int fresh = 0;
+ int tagged = 0;
+ int rewritten = 0;
+
+ // Checked here too, and not only in Mark: the call being reflected can
+ // have grafted whole subtrees of its own into the copy since.
+ if (depth >= PYXMLSEC_SHADOW_MAX_NESTING) {
+ PyErr_SetString(PyXmlSec_InternalError, "the document is nested too deeply.");
+ return -1;
+ }
+ for (n = parent->children; n != NULL; n = n->next) {
+ PyXmlSec_LxmlShadowTag* child_tag = PYXMLSEC_SHADOW_TAG(list->shadow, n);
+ if (child_tag != NULL) {
+ ++tagged;
+ if (child_tag->content != PyXmlSec_LxmlShadowContentPrint(n)) {
+ rewritten = 1;
+ }
+ if (n->type == XML_ELEMENT_NODE && PyXmlSec_LxmlShadowCollectSites(list, n, depth + 1) < 0) {
+ return -1;
+ }
+ continue;
+ }
+ fresh = 1;
+ if (_isElement(n) && PyXmlSec_LxmlShadowSiteAppend(list, n, PYXMLSEC_SHADOW_SITE_GRAFT) < 0) {
+ return -1;
+ }
+ }
+ if ((fresh || rewritten || (tag != NULL && tag->children != tagged))
+ && PyXmlSec_LxmlShadowSiteAppend(list, parent, PYXMLSEC_SHADOW_SITE_SYNC) < 0) {
+ return -1;
+ }
+ return 0;
+}
+
+// The text slots of an lxml element as the list [text, tail of child 0,
+// tail of child 1, ...] (new reference).
+static PyObject* PyXmlSec_LxmlShadowGetTextSlots(PyObject* parent) {
+ PyObject* slots = PyList_New(0);
+ PyObject* item;
+ Py_ssize_t i, n;
+ int failed;
+
+ if (slots == NULL) {
+ return NULL;
+ }
+ item = PyObject_GetAttrString(parent, "text");
+ failed = item == NULL || PyList_Append(slots, item) < 0;
+ Py_XDECREF(item);
+ n = failed ? -1 : PyObject_Length(parent);
+ for (i = 0; i < n; ++i) {
+ PyObject* child = PySequence_GetItem(parent, i);
+ item = child != NULL ? PyObject_GetAttrString(child, "tail") : NULL;
+ failed = item == NULL || PyList_Append(slots, item) < 0;
+ Py_XDECREF(child);
+ Py_XDECREF(item);
+ if (failed) {
+ break;
+ }
+ }
+ if (failed || n < 0) {
+ Py_DECREF(slots);
+ return NULL;
+ }
+ return slots;
+}
+
+// Applies a slot list to an lxml element whose children already mirror the
+// copy's (the grafts under it have been applied).
+static int PyXmlSec_LxmlShadowSetTextSlots(PyObject* parent, PyObject* slots) {
+ Py_ssize_t i;
+ Py_ssize_t n = PyObject_Length(parent);
+
+ if (n < 0) {
+ return -1;
+ }
+ if (n + 1 != PyList_GET_SIZE(slots)) {
+ PyErr_SetString(PyXmlSec_InternalError, "unexpected mutation site.");
+ return -1;
+ }
+ if (PyObject_SetAttrString(parent, "text", PyList_GET_ITEM(slots, 0)) < 0) {
+ return -1;
+ }
+ for (i = 0; i < n; ++i) {
+ PyObject* child = PySequence_GetItem(parent, i);
+ int failed = child == NULL || PyObject_SetAttrString(child, "tail", PyList_GET_ITEM(slots, i + 1)) < 0;
+ Py_XDECREF(child);
+ if (failed) {
+ return -1;
+ }
+ }
+ return 0;
+}
+
+// etree.cleanup_namespaces(element, top_nsmap=..., keep_ns_prefixes=...);
+// either keyword may be NULL to leave it out.
+static int PyXmlSec_LxmlCleanupNamespaces(PyObject* element, PyObject* top_nsmap, PyObject* keep) {
+ PyObject* args = PyTuple_Pack(1, element);
+ PyObject* kwargs = PyDict_New();
+ PyObject* result = NULL;
+
+ if (args != NULL && kwargs != NULL
+ && (top_nsmap == NULL || PyDict_SetItemString(kwargs, "top_nsmap", top_nsmap) == 0)
+ && (keep == NULL || PyDict_SetItemString(kwargs, "keep_ns_prefixes", keep) == 0)) {
+ result = PyObject_Call(PyXmlSec_LxmlEtreeCleanupNamespaces, args, kwargs);
+ }
+ Py_XDECREF(args);
+ Py_XDECREF(kwargs);
+ Py_XDECREF(result);
+ return result != NULL ? 0 : -1;
+}
+
+// The call replaced the copy's root element itself — encrypt_xml with
+// Type=Element on the document root, decrypt of a root .
+// lxml offers no way to swap a document's root element (_ElementTree._setroot
+// only rebinds that one Python object; the document keeps its root), so the
+// live element is morphed in place into `fresh`, the re-parsed replacement,
+// through lxml's public API: emptied and stripped of every namespace
+// declaration, given exactly the declarations of `fresh` (a temporary child
+// pins the default namespace, which cleanup_namespaces would otherwise drop
+// as unused before the tag can use it — the tag setter itself never declares
+// a default namespace), then renamed and refilled. The children go through
+// lxml's usual namespace reconciliation, like every graft. The live proxy
+// thus *becomes* the replacement, where the raw path leaves the caller's old
+// root proxy (and any _ElementTree holding it) detached and stale.
+static int PyXmlSec_LxmlShadowMorphRoot(PyObject* live, PyObject* fresh) {
+ PyObject* tag = NULL;
+ PyObject* tail = NULL;
+ PyObject* nsmap = NULL;
+ PyObject* keep = NULL;
+ PyObject* pin = NULL;
+ PyObject* attrib = NULL;
+ PyObject* fresh_attrib = NULL;
+ PyObject* text = NULL;
+ PyObject* children = NULL;
+ PyObject* tmp = NULL;
+ PyObject* key;
+ PyObject* href;
+ Py_ssize_t pos = 0;
+ const char* local;
+ int rv = -1;
+
+ // Empty the element and give it a namespace-free name (the tail is not
+ // the call's to change), so that every old declaration is unused and
+ // cleanup_namespaces drops it — a conflicting old prefix would otherwise
+ // block the new declaration.
+ tag = PyObject_GetAttrString(fresh, "tag");
+ tail = PyObject_GetAttrString(live, "tail");
+ if (tag == NULL || tail == NULL || (local = PyUnicode_AsUTF8(tag)) == NULL) {
+ goto DONE;
+ }
+ if (strchr(local, '}') != NULL) {
+ local = strchr(local, '}') + 1;
+ }
+ tmp = PyObject_CallMethod(live, "clear", NULL);
+ if (tmp == NULL || PyObject_SetAttrString(live, "tail", tail) < 0) {
+ goto DONE;
+ }
+ Py_CLEAR(tmp);
+ tmp = PyUnicode_FromString(local);
+ if (tmp == NULL || PyObject_SetAttrString(live, "tag", tmp) < 0) {
+ goto DONE;
+ }
+ Py_CLEAR(tmp);
+ if (PyXmlSec_LxmlCleanupNamespaces(live, NULL, NULL) < 0) {
+ goto DONE;
+ }
+
+ // Declare exactly the replacement's namespaces.
+ nsmap = PyObject_GetAttrString(fresh, "nsmap");
+ keep = PyList_New(0);
+ if (nsmap == NULL || keep == NULL) {
+ goto DONE;
+ }
+ if (!PyDict_Check(nsmap)) {
+ PyErr_SetString(PyXmlSec_InternalError, "unexpected nsmap.");
+ goto DONE;
+ }
+ while (PyDict_Next(nsmap, &pos, &key, &href)) {
+ if (key == Py_None) {
+ tmp = PyUnicode_FromFormat("{%U}pin", href);
+ pin = tmp != NULL ? PyObject_CallMethod(live, "makeelement", "O", tmp) : NULL;
+ Py_CLEAR(tmp);
+ tmp = pin != NULL ? PyObject_CallMethod(live, "append", "O", pin) : NULL;
+ if (tmp == NULL) {
+ goto DONE;
+ }
+ Py_CLEAR(tmp);
+ } else if (PyList_Append(keep, key) < 0) {
+ goto DONE;
+ }
+ }
+ if (PyXmlSec_LxmlCleanupNamespaces(live, nsmap, keep) < 0) {
+ goto DONE;
+ }
+
+ // Rename and refill.
+ if (PyObject_SetAttrString(live, "tag", tag) < 0) {
+ goto DONE;
+ }
+ attrib = PyObject_GetAttrString(live, "attrib");
+ fresh_attrib = PyObject_GetAttrString(fresh, "attrib");
+ tmp = attrib != NULL && fresh_attrib != NULL ? PyObject_CallMethod(attrib, "update", "O", fresh_attrib) : NULL;
+ if (tmp == NULL) {
+ goto DONE;
+ }
+ Py_CLEAR(tmp);
+ if (pin != NULL) {
+ tmp = PyObject_CallMethod(live, "remove", "O", pin);
+ if (tmp == NULL) {
+ goto DONE;
+ }
+ Py_CLEAR(tmp);
+ }
+ text = PyObject_GetAttrString(fresh, "text");
+ if (text == NULL || PyObject_SetAttrString(live, "text", text) < 0) {
+ goto DONE;
+ }
+ children = PySequence_List(fresh);
+ tmp = children != NULL ? PyObject_CallMethod(live, "extend", "O", children) : NULL;
+ if (tmp == NULL) {
+ goto DONE;
+ }
+ rv = 0;
+
+DONE:
+ Py_XDECREF(tag);
+ Py_XDECREF(tail);
+ Py_XDECREF(nsmap);
+ Py_XDECREF(keep);
+ Py_XDECREF(pin);
+ Py_XDECREF(attrib);
+ Py_XDECREF(fresh_attrib);
+ Py_XDECREF(text);
+ Py_XDECREF(children);
+ Py_XDECREF(tmp);
+ return rv;
+}
+
+// Applies every change in the copy to the live tree. Does not release the
+// copy. Returns 0, or -1 with an exception set.
+static int PyXmlSec_LxmlShadowReflectSites(PyXmlSec_LxmlShadow* shadow) {
+ PyXmlSec_LxmlShadowSiteList list = {NULL, 0, 0, NULL, shadow};
+ PyObject* copy_root = NULL;
+ int i;
+ int rv = -1;
+
+ // Re-fetch the root: replacement operations may swap nodes at the top.
+ // An unlinked subtree cannot be replaced at all (libxml2 needs a parent
+ // to put the replacement in), so its top is the one the shadow made.
+ list.top = shadow->unlinked ? shadow->root : xmlDocGetRootElement(shadow->doc);
+ if (list.top == NULL) {
+ PyErr_SetString(PyXmlSec_InternalError, "unexpected mutation site.");
+ goto DONE;
+ }
+ if (!PYXMLSEC_SHADOW_TAGGED(shadow, list.top)) {
+ xmlNodePtr n;
+ int elements = 0;
+ int others = 0;
+
+ if (shadow->unlinked) {
+ PyErr_SetString(PyXmlSec_InternalError, "unexpected mutation site.");
+ goto DONE;
+ }
+ // A fresh root: the call replaced the root itself, so there are no
+ // sites to graft — the live root is morphed into it wholesale. lxml
+ // holds one element (plus comments and PIs) at document level, so a
+ // root replaced by anything else (a Type=Content decryption of the
+ // root) cannot be reflected.
+ for (n = shadow->doc->children; n != NULL; n = n->next) {
+ if (n->type == XML_ELEMENT_NODE) {
+ ++elements;
+ } else if (n->type != XML_COMMENT_NODE && n->type != XML_PI_NODE && n->type != XML_DTD_NODE) {
+ ++others;
+ }
+ }
+ if (elements != 1 || others != 0) {
+ PyErr_SetString(PyXmlSec_Error, "the document root was replaced by content that is not a single element");
+ goto DONE;
+ }
+ copy_root = PyXmlSec_LxmlShadowDumpCopy(shadow);
+ if (copy_root != NULL) {
+ rv = PyXmlSec_LxmlShadowMorphRoot((PyObject*)shadow->element, copy_root);
+ }
+ goto DONE;
+ }
+ if (PyXmlSec_LxmlShadowCollectSites(&list, list.top, 0) < 0) {
+ goto DONE;
+ }
+ if (list.count == 0) {
+ rv = 0;
+ goto DONE;
+ }
+ copy_root = PyXmlSec_LxmlShadowDumpCopy(shadow);
+ if (copy_root == NULL) {
+ goto DONE;
+ }
+
+ // Phase 1: fetch every payload from the re-parsed copy (still final state).
+ for (i = 0; i < list.count; ++i) {
+ PyXmlSec_LxmlShadowSite* site = &list.items[i];
+ PyObject* copy_parent = PyXmlSec_LxmlShadowWalk(copy_root, site->path, site->depth);
+ if (copy_parent == NULL) {
+ goto DONE;
+ }
+ if (site->kind == PYXMLSEC_SHADOW_SITE_GRAFT) {
+ site->value = PySequence_GetItem(copy_parent, site->idx);
+ } else {
+ site->value = PyXmlSec_LxmlShadowGetTextSlots(copy_parent);
+ }
+ Py_DECREF(copy_parent);
+ if (site->value == NULL) {
+ goto DONE;
+ }
+ }
+
+ // Phase 2: apply to the live tree in document order.
+ for (i = 0; i < list.count; ++i) {
+ PyXmlSec_LxmlShadowSite* site = &list.items[i];
+ PyObject* live_parent = PyXmlSec_LxmlShadowWalk((PyObject*)shadow->element, site->path, site->depth);
+ int failed;
+ if (live_parent == NULL) {
+ goto DONE;
+ }
+ if (site->kind == PYXMLSEC_SHADOW_SITE_GRAFT) {
+ // lxml carries the node's tail along and reconciles namespaces.
+ PyObject* tmp = PyObject_CallMethod(live_parent, "insert", "iO", site->idx, site->value);
+ failed = tmp == NULL;
+ Py_XDECREF(tmp);
+ } else {
+ failed = PyXmlSec_LxmlShadowSetTextSlots(live_parent, site->value) < 0;
+ }
+ Py_DECREF(live_parent);
+ if (failed) {
+ goto DONE;
+ }
+ }
+ rv = 0;
+
+DONE:
+ for (i = 0; i < list.count; ++i) {
+ Py_XDECREF(list.items[i].value);
+ }
+ PyMem_Free(list.items);
+ Py_XDECREF(copy_root);
+ return rv;
+}
+
+// Create shape (BeginNewDoc): the call left `res` detached inside the
+// private document. Root it there so the whole subtree serializes (and is
+// freed with the doc) and hand the bytes to lxml: the result is a new
+// detached element in a document of its own — lxml moves it when the caller
+// grafts it into a tree, just like the raw path's detached node.
+static PyObject* PyXmlSec_LxmlShadowEndDetached(PyXmlSec_LxmlShadow* shadow, xmlNodePtr res) {
+ PyObject* result;
+ xmlDocSetRootElement(shadow->doc, res);
+ result = PyXmlSec_LxmlShadowDumpCopy(shadow);
+ PyXmlSec_LxmlShadowDiscard(shadow);
+ return result;
+}
+
+// Swaps the live element at `path` for the copy's version of it (the tail
+// travels with the inserted node; the call does not touch it). Returns the
+// new live element (new reference).
+static PyObject* PyXmlSec_LxmlShadowSwapLive(PyXmlSec_LxmlShadow* shadow, const int* path, int depth) {
+ PyObject* copy_root = NULL;
+ PyObject* copy_parent = NULL;
+ PyObject* live_parent = NULL;
+ PyObject* live_old = NULL;
+ PyObject* new_node = NULL;
+ PyObject* tmp = NULL;
+ PyObject* result = NULL;
+ int idx = path[depth - 1];
+
+ copy_root = PyXmlSec_LxmlShadowDumpCopy(shadow);
+ if (copy_root == NULL) {
+ goto DONE;
+ }
+ copy_parent = PyXmlSec_LxmlShadowWalk(copy_root, path, depth - 1);
+ live_parent = PyXmlSec_LxmlShadowWalk((PyObject*)shadow->element, path, depth - 1);
+ if (copy_parent == NULL || live_parent == NULL) {
+ goto DONE;
+ }
+ new_node = PySequence_GetItem(copy_parent, idx);
+ live_old = PySequence_GetItem(live_parent, idx);
+ if (new_node == NULL || live_old == NULL) {
+ goto DONE;
+ }
+ tmp = PyObject_CallMethod(live_parent, "remove", "O", live_old);
+ if (tmp == NULL) {
+ goto DONE;
+ }
+ Py_CLEAR(tmp);
+ tmp = PyObject_CallMethod(live_parent, "insert", "iO", idx, new_node);
+ if (tmp == NULL) {
+ goto DONE;
+ }
+ Py_CLEAR(tmp);
+ result = PySequence_GetItem(live_parent, idx);
+
+DONE:
+ Py_XDECREF(copy_root);
+ Py_XDECREF(copy_parent);
+ Py_XDECREF(live_parent);
+ Py_XDECREF(live_old);
+ Py_XDECREF(new_node);
+ Py_XDECREF(tmp);
+ return result;
+}
+
+// Find-or-create found an existing node: the tree did not grow there, so the
+// result is the live element at the same path, plus whatever the call changed
+// on it. Attributes (Id) are synced in place. A renamed namespace prefix
+// (encrypted_data_ensure_key_info(ns=...)) has no lxml API, so the live
+// element is swapped for the copy's version and the caller gets a new proxy
+// object — except for the shadow root, which has no live parent to swap
+// under (only attributes can change there).
+static PyObject* PyXmlSec_LxmlShadowEndFound(PyXmlSec_LxmlShadow* shadow, xmlNodePtr res, const int* path, int depth) {
+ PyObject* live = NULL;
+ PyObject* live_prefix = NULL;
+ PyObject* copy_prefix = NULL;
+ PyObject* result = NULL;
+ int same;
+
+ live = PyXmlSec_LxmlShadowWalk((PyObject*)shadow->element, path, depth);
+ if (live == NULL) {
+ return NULL;
+ }
+ live_prefix = PyObject_GetAttrString(live, "prefix");
+ if (live_prefix == NULL) {
+ goto DONE;
+ }
+ if (res->ns != NULL && res->ns->prefix != NULL) {
+ copy_prefix = PyUnicode_FromString((const char*)res->ns->prefix);
+ } else {
+ copy_prefix = Py_None;
+ Py_INCREF(copy_prefix);
+ }
+ if (copy_prefix == NULL) {
+ goto DONE;
+ }
+ same = PyObject_RichCompareBool(live_prefix, copy_prefix, Py_EQ);
+ if (same < 0) {
+ goto DONE;
+ }
+ if (same || depth == 0) {
+ if (PyXmlSec_LxmlShadowSyncAttributes(res, live) == 0) {
+ result = live;
+ live = NULL;
+ }
+ } else {
+ result = PyXmlSec_LxmlShadowSwapLive(shadow, path, depth);
+ }
+
+DONE:
+ Py_XDECREF(live);
+ Py_XDECREF(live_prefix);
+ Py_XDECREF(copy_prefix);
+ return result;
+}
+
+PyObject* PyXmlSec_LxmlShadowEnd(PyXmlSec_LxmlShadow* shadow, xmlNodePtr res, const char* error) {
+ PyObject* result = NULL;
+ int path[PYXMLSEC_SHADOW_MAX_DEPTH];
+ int depth;
+
+ if (res == NULL) {
+ PyXmlSec_LxmlShadowDiscard(shadow);
+ PyXmlSec_SetLastError(error);
+ return NULL;
+ }
+ // Fast path (no copy was made): res is a node in the live lxml tree.
+ if (shadow->doc == NULL) {
+ PyXmlSec_LxmlShadowDiscard(shadow);
+ return (PyObject*)PyXmlSec_elementFactory(shadow->element->_doc, res);
+ }
+ if (shadow->root == NULL) {
+ return PyXmlSec_LxmlShadowEndDetached(shadow, res);
+ }
+
+ depth = PyXmlSec_LxmlShadowPathTo(res, shadow->root, path);
+ if (depth < 0) {
+ PyErr_SetString(PyXmlSec_InternalError, "unexpected result node.");
+ goto DONE;
+ }
+ // Reflect first: the live tree then mirrors the copy's element structure,
+ // so the path of `res` in the copy resolves to its live counterpart —
+ // whether the call created it (now grafted) or found it.
+ if (PyXmlSec_LxmlShadowReflectSites(shadow) < 0) {
+ goto DONE;
+ }
+ if (PYXMLSEC_SHADOW_TAGGED(shadow, res)) {
+ result = PyXmlSec_LxmlShadowEndFound(shadow, res, path, depth);
+ } else {
+ result = PyXmlSec_LxmlShadowWalk((PyObject*)shadow->element, path, depth);
+ }
+
+DONE:
+ PyXmlSec_LxmlShadowDiscard(shadow);
+ return result;
+}
+
+PyObject* PyXmlSec_LxmlShadowEndFind(PyXmlSec_LxmlShadow* shadow, xmlNodePtr res) {
+ PyObject* result = NULL;
+ int path[PYXMLSEC_SHADOW_MAX_DEPTH];
+ int depth;
+
+ if (res == NULL) {
+ PyXmlSec_LxmlShadowDiscard(shadow);
+ Py_RETURN_NONE;
+ }
+ if (shadow->doc == NULL) { // fast path: res is a live node
+ PyXmlSec_LxmlShadowDiscard(shadow);
+ return (PyObject*)PyXmlSec_elementFactory(shadow->element->_doc, res);
+ }
+ depth = PyXmlSec_LxmlShadowPathTo(res, shadow->root, path);
+ if (depth < 0) {
+ PyErr_SetString(PyXmlSec_InternalError, "unexpected result node.");
+ } else {
+ result = PyXmlSec_LxmlShadowWalk((PyObject*)shadow->element, path, depth);
+ }
+ PyXmlSec_LxmlShadowDiscard(shadow);
+ return result;
+}
+
+int PyXmlSec_LxmlShadowReflect(PyXmlSec_LxmlShadow* shadow, int rv, const char* error) {
+ if (rv < 0) {
+ PyXmlSec_LxmlShadowDiscard(shadow);
+ if (error != NULL) {
+ PyXmlSec_SetLastError(error);
+ }
+ return -1;
+ }
+ rv = shadow->doc != NULL ? PyXmlSec_LxmlShadowReflectSites(shadow) : 0;
+ PyXmlSec_LxmlShadowDiscard(shadow);
+ return rv;
+}
+
+xmlNodePtr PyXmlSec_LxmlShadowImportElement(PyXmlSec_LxmlShadow* shadow, PyXmlSec_LxmlElementPtr element) {
+ PyObject* bytes;
+ xmlDocPtr tdoc;
+ xmlNodePtr result;
+
+ bytes = PyXmlSec_LxmlElementToBytes((PyObject*)element);
+ if (bytes == NULL) {
+ return NULL;
+ }
+ tdoc = PyXmlSec_LxmlShadowParse(bytes, NULL, 0, "cannot make a private copy of the element.");
+ Py_DECREF(bytes);
+ if (tdoc == NULL) {
+ return NULL;
+ }
+ // Copy into the shadow doc (both trees are ours). The copy stays
+ // untagged: from the reflection's point of view whatever the xmlsec call
+ // grafts of it is a node "the call created".
+ result = xmlDocCopyNode(xmlDocGetRootElement(tdoc), shadow->doc, 1);
+ xmlFreeDoc(tdoc);
+ if (result == NULL) {
+ PyErr_SetString(PyXmlSec_InternalError, "cannot make a private copy of the element.");
+ }
+ return result;
+}
diff --git a/src/lxml.h b/src/lxml.h
index 72050efe..f0c1e9a8 100644
--- a/src/lxml.h
+++ b/src/lxml.h
@@ -29,6 +29,157 @@ PyXmlSec_LxmlElementPtr PyXmlSec_elementFactory(PyXmlSec_LxmlDocumentPtr doc, xm
// converts o to PyObject, None object is not allowed, does not increment ref_counts
int PyXmlSec_LxmlElementConverter(PyObject* o, PyXmlSec_LxmlElementPtr* p);
+// A "shadow" is a private copy of an lxml element, owned by the libxml2 this
+// extension links, so that an xmlsec call never touches a node allocated by
+// lxml's (possibly different) libxml2 — only serialized bytes cross between
+// the two libraries (https://github.com/xmlsec/python-xmlsec/issues/356).
+//
+// Every binding follows the same four lines (see template.c):
+//
+// PyXmlSec_LxmlShadow shadow;
+// if (PyXmlSec_LxmlShadowBegin(&shadow, node) < 0) goto ON_FAIL;
+// Py_BEGIN_ALLOW_THREADS;
+// res = xmlSecTmplSignatureAddReference(shadow.root, ...);
+// Py_END_ALLOW_THREADS;
+// result = PyXmlSec_LxmlShadowEnd(&shadow, res, "cannot add reference.");
+//
+// Three Begin flavours make the copy: of the element's subtree (Begin), of
+// its whole document (BeginDoc — for calls that follow references or walk
+// upward), or of nothing (BeginNewDoc — for calls that only need a document
+// to build a detached subtree in). The caller then runs exactly one xmlsec
+// call against the copy (nothing else; Python may not run between Begin and
+// End) and hands its result to one of the End functions, which reflect every
+// change the call made back into the live lxml tree and always release the
+// copy:
+//
+// End returns the lxml element for a result node (new reference):
+// grafted into the live tree if the call created it, or the
+// existing live element (with the attributes / prefix the call
+// changed) if it already existed. NULL raises `error`.
+// EndFind the same for read-only finders; a NULL result is None, not an
+// error.
+// Reflect for calls that return only a status: rv < 0 raises `error`,
+// otherwise the changes are reflected. A call that replaced the
+// copy's root element (encrypt/decrypt of the document root)
+// morphs the live element in place into the replacement, since
+// lxml cannot swap a document's root.
+// Discard releases the copy without reflecting (read-only calls such as
+// verify, and error paths before End).
+//
+// Fast path: when lxml links the same libxml2 as this extension (the
+// import-time version check passed), no copy is made — Begin aliases the
+// live node into `root` (leaving `doc` NULL) and End just wraps the result,
+// which is the long-standing direct behaviour with zero overhead. Setting
+// PYXMLSEC_FORCE_SHADOW in the environment forces the shadow path even on
+// matched libraries; CI uses it to keep that path exercised.
+
+// One per node of the copy that existed before the xmlsec call: the tag the
+// node's libxml2 _private field points at, holding the child count that node
+// had and a fingerprint of the content it held. Untagged after the call means
+// the call created the node; a changed child count means it removed (or
+// moved) one; a changed fingerprint means it rewrote a text node in place.
+typedef struct {
+ int children;
+ unsigned long long content;
+} PyXmlSec_LxmlShadowTag;
+
+typedef struct {
+ PyXmlSec_LxmlElementPtr element; // borrowed; the live element copy paths start from (BeginDoc: the live root)
+ PyObject* owned; // reference released when the shadow ends (BeginDoc's root proxy)
+ xmlDocPtr doc; // the private copy, owned by the shadow; NULL on the fast path
+ xmlNodePtr root; // doc's root element, the copy of `element`; NULL for BeginNewDoc
+ PyXmlSec_LxmlShadowTag* tags; // one per pre-existing node of `doc`, owned by the shadow
+ int ntags;
+ int unlinked; // `root` hangs in `doc` outside its tree (BeginDoc of an element
+ // removed from its document); the shadow frees it itself
+} PyXmlSec_LxmlShadow;
+
+// Subtree copy: `shadow.root` is the copy of `element`. When the document
+// declares entities, the whole document is copied and then cut back to the
+// element, so that the subtree's entity references keep their declarations.
+int PyXmlSec_LxmlShadowBegin(PyXmlSec_LxmlShadow* shadow, PyXmlSec_LxmlElementPtr element);
+
+// Whole-document copy, for calls that read or mutate beyond the element's
+// subtree (sign/verify, encrypt/decrypt, find_parent). Serializes
+// element.getroottree() — comments/PIs outside the root and the internal DTD
+// subset survive — and replays the IDs registered for the document
+// (RegisterId/RecordIds) onto the copy so that #id references resolve.
+// `*target` receives the copy's counterpart of `element` (the live node
+// itself on the fast path); `shadow.root` / `shadow.element` become the copy
+// root / the live root, which is what the End functions map paths between.
+// An element removed from its document keeps pointing at it, and so does the
+// raw path — xmlsec works on the unlinked node while the document it left
+// still answers its `#id` references. The copy holds both: the document, and
+// the unlinked subtree copied into it beside the tree (`shadow.unlinked`,
+// with `shadow.root` / `shadow.element` the subtree's top on either side).
+int PyXmlSec_LxmlShadowBeginDoc(PyXmlSec_LxmlShadow* shadow, PyXmlSec_LxmlElementPtr element, xmlNodePtr* target);
+
+// Create shape (template.create, encrypted_data_create): the call only needs
+// a document to build a *detached* subtree in. Returns that document — the
+// element's own on the fast path, a private empty one on the shadow path —
+// or NULL with an exception set. End then returns the result as a new
+// detached lxml element (in a document of its own until it is grafted; the
+// raw path's "detached node inside the source document" has no lxml
+// equivalent).
+xmlDocPtr PyXmlSec_LxmlShadowBeginNewDoc(PyXmlSec_LxmlShadow* shadow, PyXmlSec_LxmlElementPtr element);
+
+// Reflects every change the call made and returns the lxml element for
+// `res` (new reference), or NULL with an exception set (`error` is raised
+// when res is NULL). Works after any Begin flavour.
+PyObject* PyXmlSec_LxmlShadowEnd(PyXmlSec_LxmlShadow* shadow, xmlNodePtr res, const char* error);
+
+// End for read-only finders: maps `res` (a pre-existing node in the copy)
+// back to the live element; returns None when res is NULL (not found).
+PyObject* PyXmlSec_LxmlShadowEndFind(PyXmlSec_LxmlShadow* shadow, xmlNodePtr res);
+
+// End for calls that return only a status (sign, encrypt, ...): rv < 0
+// raises `error` and returns -1; otherwise every change the call made is
+// reflected. Returns 0, or -1 with an exception set. No-op on the fast path
+// except for raising.
+int PyXmlSec_LxmlShadowReflect(PyXmlSec_LxmlShadow* shadow, int rv, const char* error);
+
+// Releases the copy without reflecting anything (verify, error paths before
+// End). Safe to call after any successful Begin; the End functions call it.
+void PyXmlSec_LxmlShadowDiscard(PyXmlSec_LxmlShadow* shadow);
+
+// Non-zero when the shadow path is on (mismatched libxml2 or
+// PYXMLSEC_FORCE_SHADOW). Only the few call sites whose *semantics* differ
+// per mode (ID registration, encrypt/decrypt replacement) may branch on
+// this; everything else goes through Begin/End, which encapsulate both paths.
+int PyXmlSec_LxmlShadowIsActive(void);
+
+// Re-serializes `element` (a live lxml element) into the shadow's private
+// copy as a fresh (untagged) detached subtree — encrypt_xml's template
+// import. Shadow path only (shadow->doc != NULL).
+xmlNodePtr PyXmlSec_LxmlShadowImportElement(PyXmlSec_LxmlShadow* shadow, PyXmlSec_LxmlElementPtr element);
+
+// Shadow-mode ID registration (issue #356): register_id/add_ids cannot write
+// lxml's ID hash with our libxml2, so they record the id-attribute spec for
+// the element's document here, and every BeginDoc replays the recorded specs
+// onto its copy. The registry keeps `element` itself, so the replay applies a
+// spec to exactly the node it was registered for. Registering every matching
+// attribute of the document instead would let an unrelated element with the
+// same id value win the lookup and steer a `#id` reference away from the
+// registered one.
+//
+// This is the whole of register_id under the shadow: it validates what the
+// fast path validates — "missing attribute." for an absent one, "duplicated
+// id." when the value is already registered for another attribute — and
+// records the spec for the node alone. Returns 0, or -1 with the exception
+// the fast path would have raised.
+int PyXmlSec_LxmlShadowRegisterId(PyXmlSec_LxmlElementPtr element, const char* name, const char* ns);
+
+// The same for a whole list of attribute names (`names`, a list of str), for
+// add_ids: either every name is recorded or, if anything fails, none is —
+// the fast path likewise builds its complete list before it touches the
+// document, so a bad list leaves no half-applied registration behind.
+// `subtree` extends the registration to the node's descendants, the scope
+// add_ids (xmlSecAddIDs) walks; it is expanded here and now into one spec per
+// element carrying one of the names, which is the snapshot xmlSecAddIDs takes
+// — an element that grows the attribute, or joins the scope, after the call
+// was never registered by the fast path either.
+int PyXmlSec_LxmlShadowRecordIds(PyXmlSec_LxmlElementPtr element, PyObject* names, const char* ns, int subtree);
+
// get version numbers for libxml2 both compiled and loaded
long PyXmlSec_GetLibXmlVersionMajor();
long PyXmlSec_GetLibXmlVersionMinor();
diff --git a/src/template.c b/src/template.c
index c6864c2e..ff85dd8b 100644
--- a/src/template.c
+++ b/src/template.c
@@ -42,6 +42,9 @@ static PyObject* PyXmlSec_TemplateCreate(PyObject* self, PyObject *args, PyObjec
const char* id = NULL;
const char* ns = NULL;
xmlNodePtr res;
+ xmlDocPtr tdoc;
+ PyObject* result;
+ PyXmlSec_LxmlShadow shadow;
PYXMLSEC_DEBUG("template create - start");
if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&O!O!|zz:create", kwlist,
@@ -50,16 +53,22 @@ static PyObject* PyXmlSec_TemplateCreate(PyObject* self, PyObject *args, PyObjec
goto ON_FAIL;
}
+ // `node` only supplies the document the detached template is built in;
+ // the create-shape shadow provides a private one instead (issue #356).
+ tdoc = PyXmlSec_LxmlShadowBeginNewDoc(&shadow, node);
+ if (tdoc == NULL) {
+ goto ON_FAIL;
+ }
Py_BEGIN_ALLOW_THREADS;
- res = xmlSecTmplSignatureCreateNsPref(node->_doc->_c_doc, c14n->id, sign->id, XSTR(id), XSTR(ns));
+ res = xmlSecTmplSignatureCreateNsPref(tdoc, c14n->id, sign->id, XSTR(id), XSTR(ns));
Py_END_ALLOW_THREADS;
- if (res == NULL) {
- PyXmlSec_SetLastError("cannot create template.");
+ result = PyXmlSec_LxmlShadowEnd(&shadow, res, "cannot create template.");
+ if (result == NULL) {
goto ON_FAIL;
}
PYXMLSEC_DEBUG("template create - ok");
- return (PyObject*)PyXmlSec_elementFactory(node->_doc, res);
+ return result;
ON_FAIL:
PYXMLSEC_DEBUG("template create - fail");
@@ -91,6 +100,8 @@ static PyObject* PyXmlSec_TemplateAddReference(PyObject* self, PyObject *args, P
const char* uri = NULL;
const char* type = NULL;
xmlNodePtr res;
+ PyObject* result;
+ PyXmlSec_LxmlShadow shadow;
PYXMLSEC_DEBUG("template add_reference - start");
if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&O!|zzz:add_reference", kwlist,
@@ -98,16 +109,21 @@ static PyObject* PyXmlSec_TemplateAddReference(PyObject* self, PyObject *args, P
{
goto ON_FAIL;
}
+ // The xmlsec call runs on a private copy of `node`, never on lxml's own
+ // nodes (issue #356); the shadow reflects the new back.
+ if (PyXmlSec_LxmlShadowBegin(&shadow, node) < 0) {
+ goto ON_FAIL;
+ }
Py_BEGIN_ALLOW_THREADS;
- res = xmlSecTmplSignatureAddReference(node->_c_node, digest->id, XSTR(id), XSTR(uri), XSTR(type));
+ res = xmlSecTmplSignatureAddReference(shadow.root, digest->id, XSTR(id), XSTR(uri), XSTR(type));
Py_END_ALLOW_THREADS;
- if (res == NULL) {
- PyXmlSec_SetLastError("cannot add reference.");
+ result = PyXmlSec_LxmlShadowEnd(&shadow, res, "cannot add reference.");
+ if (result == NULL) {
goto ON_FAIL;
}
PYXMLSEC_DEBUG("template add_reference - ok");
- return (PyObject*)PyXmlSec_elementFactory(node->_doc, res);
+ return result;
ON_FAIL:
PYXMLSEC_DEBUG("template add_reference - fail");
@@ -129,6 +145,8 @@ static PyObject* PyXmlSec_TemplateAddTransform(PyObject* self, PyObject *args, P
PyXmlSec_LxmlElementPtr node = NULL;
PyXmlSec_Transform* transform = NULL;
xmlNodePtr res;
+ PyObject* result;
+ PyXmlSec_LxmlShadow shadow;
PYXMLSEC_DEBUG("template add_transform - start");
if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&O!:add_transform", kwlist,
@@ -136,16 +154,19 @@ static PyObject* PyXmlSec_TemplateAddTransform(PyObject* self, PyObject *args, P
{
goto ON_FAIL;
}
+ if (PyXmlSec_LxmlShadowBegin(&shadow, node) < 0) {
+ goto ON_FAIL;
+ }
Py_BEGIN_ALLOW_THREADS;
- res = xmlSecTmplReferenceAddTransform(node->_c_node, transform->id);
+ res = xmlSecTmplReferenceAddTransform(shadow.root, transform->id);
Py_END_ALLOW_THREADS;
- if (res == NULL) {
- PyXmlSec_SetLastError("cannot add transform.");
+ result = PyXmlSec_LxmlShadowEnd(&shadow, res, "cannot add transform.");
+ if (result == NULL) {
goto ON_FAIL;
}
PYXMLSEC_DEBUG("template add_transform - ok");
- return (PyObject*)PyXmlSec_elementFactory(node->_doc, res);
+ return result;
ON_FAIL:
PYXMLSEC_DEBUG("template add_transform - fail");
@@ -167,6 +188,8 @@ static PyObject* PyXmlSec_TemplateEnsureKeyInfo(PyObject* self, PyObject *args,
PyXmlSec_LxmlElementPtr node = NULL;
const char* id = NULL;
xmlNodePtr res;
+ PyObject* result;
+ PyXmlSec_LxmlShadow shadow;
PYXMLSEC_DEBUG("template ensure_key_info - start");
if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&|z:ensure_key_info", kwlist, PyXmlSec_LxmlElementConverter, &node, &id))
@@ -174,16 +197,19 @@ static PyObject* PyXmlSec_TemplateEnsureKeyInfo(PyObject* self, PyObject *args,
goto ON_FAIL;
}
+ if (PyXmlSec_LxmlShadowBegin(&shadow, node) < 0) {
+ goto ON_FAIL;
+ }
Py_BEGIN_ALLOW_THREADS;
- res = xmlSecTmplSignatureEnsureKeyInfo(node->_c_node, XSTR(id));
+ res = xmlSecTmplSignatureEnsureKeyInfo(shadow.root, XSTR(id));
Py_END_ALLOW_THREADS;
- if (res == NULL) {
- PyXmlSec_SetLastError("cannot ensure key info.");
+ result = PyXmlSec_LxmlShadowEnd(&shadow, res, "cannot ensure key info.");
+ if (result == NULL) {
goto ON_FAIL;
}
PYXMLSEC_DEBUG("template ensure_key_info - ok");
- return (PyObject*)PyXmlSec_elementFactory(node->_doc, res);
+ return result;
ON_FAIL:
PYXMLSEC_DEBUG("template ensure_key_info - fail");
@@ -205,6 +231,8 @@ static PyObject* PyXmlSec_TemplateAddKeyName(PyObject* self, PyObject *args, PyO
PyXmlSec_LxmlElementPtr node = NULL;
const char* name = NULL;
xmlNodePtr res;
+ PyObject* result;
+ PyXmlSec_LxmlShadow shadow;
PYXMLSEC_DEBUG("template add_key_name - start");
if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&|z:add_key_name", kwlist, PyXmlSec_LxmlElementConverter, &node, &name))
@@ -212,16 +240,19 @@ static PyObject* PyXmlSec_TemplateAddKeyName(PyObject* self, PyObject *args, PyO
goto ON_FAIL;
}
+ if (PyXmlSec_LxmlShadowBegin(&shadow, node) < 0) {
+ goto ON_FAIL;
+ }
Py_BEGIN_ALLOW_THREADS;
- res = xmlSecTmplKeyInfoAddKeyName(node->_c_node, XSTR(name));
+ res = xmlSecTmplKeyInfoAddKeyName(shadow.root, XSTR(name));
Py_END_ALLOW_THREADS;
- if (res == NULL) {
- PyXmlSec_SetLastError("cannot add key name.");
+ result = PyXmlSec_LxmlShadowEnd(&shadow, res, "cannot add key name.");
+ if (result == NULL) {
goto ON_FAIL;
}
PYXMLSEC_DEBUG("template add_key_name - ok");
- return (PyObject*)PyXmlSec_elementFactory(node->_doc, res);
+ return result;
ON_FAIL:
PYXMLSEC_DEBUG("template add_key_name - fail");
@@ -240,6 +271,8 @@ static PyObject* PyXmlSec_TemplateAddKeyValue(PyObject* self, PyObject *args, Py
PyXmlSec_LxmlElementPtr node = NULL;
xmlNodePtr res;
+ PyObject* result;
+ PyXmlSec_LxmlShadow shadow;
PYXMLSEC_DEBUG("template add_key_value - start");
if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&:add_key_value", kwlist, PyXmlSec_LxmlElementConverter, &node))
@@ -247,16 +280,19 @@ static PyObject* PyXmlSec_TemplateAddKeyValue(PyObject* self, PyObject *args, Py
goto ON_FAIL;
}
+ if (PyXmlSec_LxmlShadowBegin(&shadow, node) < 0) {
+ goto ON_FAIL;
+ }
Py_BEGIN_ALLOW_THREADS;
- res = xmlSecTmplKeyInfoAddKeyValue(node->_c_node);
+ res = xmlSecTmplKeyInfoAddKeyValue(shadow.root);
Py_END_ALLOW_THREADS;
- if (res == NULL) {
- PyXmlSec_SetLastError("cannot add key value.");
+ result = PyXmlSec_LxmlShadowEnd(&shadow, res, "cannot add key value.");
+ if (result == NULL) {
goto ON_FAIL;
}
PYXMLSEC_DEBUG("template add_key_name - ok");
- return (PyObject*)PyXmlSec_elementFactory(node->_doc, res);
+ return result;
ON_FAIL:
PYXMLSEC_DEBUG("template add_key_name - fail");
@@ -275,6 +311,8 @@ static PyObject* PyXmlSec_TemplateAddX509Data(PyObject* self, PyObject *args, Py
PyXmlSec_LxmlElementPtr node = NULL;
xmlNodePtr res;
+ PyObject* result;
+ PyXmlSec_LxmlShadow shadow;
PYXMLSEC_DEBUG("template add_x509_data - start");
if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&:add_x509_data", kwlist, PyXmlSec_LxmlElementConverter, &node))
@@ -282,16 +320,19 @@ static PyObject* PyXmlSec_TemplateAddX509Data(PyObject* self, PyObject *args, Py
goto ON_FAIL;
}
+ if (PyXmlSec_LxmlShadowBegin(&shadow, node) < 0) {
+ goto ON_FAIL;
+ }
Py_BEGIN_ALLOW_THREADS;
- res = xmlSecTmplKeyInfoAddX509Data(node->_c_node);
+ res = xmlSecTmplKeyInfoAddX509Data(shadow.root);
Py_END_ALLOW_THREADS;
- if (res == NULL) {
- PyXmlSec_SetLastError("cannot add x509 data.");
+ result = PyXmlSec_LxmlShadowEnd(&shadow, res, "cannot add x509 data.");
+ if (result == NULL) {
goto ON_FAIL;
}
PYXMLSEC_DEBUG("template add_x509_data - ok");
- return (PyObject*)PyXmlSec_elementFactory(node->_doc, res);
+ return result;
ON_FAIL:
PYXMLSEC_DEBUG("template add_x509_data - fail");
@@ -310,6 +351,8 @@ static PyObject* PyXmlSec_TemplateAddX509DataAddIssuerSerial(PyObject* self, PyO
PyXmlSec_LxmlElementPtr node = NULL;
xmlNodePtr res;
+ PyObject* result;
+ PyXmlSec_LxmlShadow shadow;
PYXMLSEC_DEBUG("template x509_data_add_issuer_serial - start");
if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&:x509_data_add_issuer_serial", kwlist,
@@ -317,16 +360,19 @@ static PyObject* PyXmlSec_TemplateAddX509DataAddIssuerSerial(PyObject* self, PyO
{
goto ON_FAIL;
}
+ if (PyXmlSec_LxmlShadowBegin(&shadow, node) < 0) {
+ goto ON_FAIL;
+ }
Py_BEGIN_ALLOW_THREADS;
- res = xmlSecTmplX509DataAddIssuerSerial(node->_c_node);
+ res = xmlSecTmplX509DataAddIssuerSerial(shadow.root);
Py_END_ALLOW_THREADS;
- if (res == NULL) {
- PyXmlSec_SetLastError("cannot add x509 issuer serial.");
+ result = PyXmlSec_LxmlShadowEnd(&shadow, res, "cannot add x509 issuer serial.");
+ if (result == NULL) {
goto ON_FAIL;
}
PYXMLSEC_DEBUG("template x509_data_add_issuer_serial - ok");
- return (PyObject*)PyXmlSec_elementFactory(node->_doc, res);
+ return result;
ON_FAIL:
PYXMLSEC_DEBUG("template x509_data_add_issuer_serial - fail");
@@ -348,6 +394,8 @@ static PyObject* PyXmlSec_TemplateAddX509DataIssuerSerialAddIssuerName(PyObject*
PyXmlSec_LxmlElementPtr node = NULL;
const char* name = NULL;
xmlNodePtr res;
+ PyObject* result;
+ PyXmlSec_LxmlShadow shadow;
PYXMLSEC_DEBUG("template x509_issuer_serial_add_issuer_name - start");
if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&|z:x509_issuer_serial_add_issuer_name", kwlist,
@@ -356,16 +404,19 @@ static PyObject* PyXmlSec_TemplateAddX509DataIssuerSerialAddIssuerName(PyObject*
goto ON_FAIL;
}
+ if (PyXmlSec_LxmlShadowBegin(&shadow, node) < 0) {
+ goto ON_FAIL;
+ }
Py_BEGIN_ALLOW_THREADS;
- res = xmlSecTmplX509IssuerSerialAddIssuerName(node->_c_node, XSTR(name));
+ res = xmlSecTmplX509IssuerSerialAddIssuerName(shadow.root, XSTR(name));
Py_END_ALLOW_THREADS;
- if (res == NULL) {
- PyXmlSec_SetLastError("cannot add x509 issuer serial name.");
+ result = PyXmlSec_LxmlShadowEnd(&shadow, res, "cannot add x509 issuer serial name.");
+ if (result == NULL) {
goto ON_FAIL;
}
PYXMLSEC_DEBUG("template x509_issuer_serial_add_issuer_name - ok");
- return (PyObject*)PyXmlSec_elementFactory(node->_doc, res);
+ return result;
ON_FAIL:
PYXMLSEC_DEBUG("template x509_issuer_serial_add_issuer_name - fail");
@@ -387,6 +438,8 @@ static PyObject* PyXmlSec_TemplateAddX509DataIssuerSerialAddIssuerSerialNumber(P
PyXmlSec_LxmlElementPtr node = NULL;
const char* serial = NULL;
xmlNodePtr res;
+ PyObject* result;
+ PyXmlSec_LxmlShadow shadow;
PYXMLSEC_DEBUG("template x509_issuer_serial_add_serial_number - start");
if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&|z:x509_issuer_serial_add_serial_number", kwlist,
@@ -395,16 +448,19 @@ static PyObject* PyXmlSec_TemplateAddX509DataIssuerSerialAddIssuerSerialNumber(P
goto ON_FAIL;
}
+ if (PyXmlSec_LxmlShadowBegin(&shadow, node) < 0) {
+ goto ON_FAIL;
+ }
Py_BEGIN_ALLOW_THREADS;
- res = xmlSecTmplX509IssuerSerialAddSerialNumber(node->_c_node, XSTR(serial));
+ res = xmlSecTmplX509IssuerSerialAddSerialNumber(shadow.root, XSTR(serial));
Py_END_ALLOW_THREADS;
- if (res == NULL) {
- PyXmlSec_SetLastError("cannot add x509 issuer serial number.");
+ result = PyXmlSec_LxmlShadowEnd(&shadow, res, "cannot add x509 issuer serial number.");
+ if (result == NULL) {
goto ON_FAIL;
}
PYXMLSEC_DEBUG("template x509_issuer_serial_add_serial_number - ok");
- return (PyObject*)PyXmlSec_elementFactory(node->_doc, res);
+ return result;
ON_FAIL:
PYXMLSEC_DEBUG("template x509_issuer_serial_add_serial_number - fail");
@@ -423,6 +479,8 @@ static PyObject* PyXmlSec_TemplateAddX509DataAddSubjectName(PyObject* self, PyOb
PyXmlSec_LxmlElementPtr node = NULL;
xmlNodePtr res;
+ PyObject* result;
+ PyXmlSec_LxmlShadow shadow;
PYXMLSEC_DEBUG("template x509_data_add_subject_name - start");
if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&:x509_data_add_subject_name", kwlist,
@@ -431,16 +489,19 @@ static PyObject* PyXmlSec_TemplateAddX509DataAddSubjectName(PyObject* self, PyOb
goto ON_FAIL;
}
+ if (PyXmlSec_LxmlShadowBegin(&shadow, node) < 0) {
+ goto ON_FAIL;
+ }
Py_BEGIN_ALLOW_THREADS;
- res = xmlSecTmplX509DataAddSubjectName(node->_c_node);
+ res = xmlSecTmplX509DataAddSubjectName(shadow.root);
Py_END_ALLOW_THREADS;
- if (res == NULL) {
- PyXmlSec_SetLastError("cannot add x509 subject name.");
+ result = PyXmlSec_LxmlShadowEnd(&shadow, res, "cannot add x509 subject name.");
+ if (result == NULL) {
goto ON_FAIL;
}
PYXMLSEC_DEBUG("template x509_data_add_subject_name - ok");
- return (PyObject*)PyXmlSec_elementFactory(node->_doc, res);
+ return result;
ON_FAIL:
PYXMLSEC_DEBUG("template x509_data_add_subject_name - fail");
@@ -459,6 +520,8 @@ static PyObject* PyXmlSec_TemplateAddX509DataAddSKI(PyObject* self, PyObject *ar
PyXmlSec_LxmlElementPtr node = NULL;
xmlNodePtr res;
+ PyObject* result;
+ PyXmlSec_LxmlShadow shadow;
PYXMLSEC_DEBUG("template x509_data_add_ski - start");
if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&:x509_data_add_ski", kwlist,
@@ -467,16 +530,19 @@ static PyObject* PyXmlSec_TemplateAddX509DataAddSKI(PyObject* self, PyObject *ar
goto ON_FAIL;
}
+ if (PyXmlSec_LxmlShadowBegin(&shadow, node) < 0) {
+ goto ON_FAIL;
+ }
Py_BEGIN_ALLOW_THREADS;
- res = xmlSecTmplX509DataAddSKI(node->_c_node);
+ res = xmlSecTmplX509DataAddSKI(shadow.root);
Py_END_ALLOW_THREADS;
- if (res == NULL) {
- PyXmlSec_SetLastError("cannot add x509 SKI.");
+ result = PyXmlSec_LxmlShadowEnd(&shadow, res, "cannot add x509 SKI.");
+ if (result == NULL) {
goto ON_FAIL;
}
PYXMLSEC_DEBUG("template x509_data_add_ski - ok");
- return (PyObject*)PyXmlSec_elementFactory(node->_doc, res);
+ return result;
ON_FAIL:
PYXMLSEC_DEBUG("template x509_data_add_ski - fail");
@@ -495,6 +561,8 @@ static PyObject* PyXmlSec_TemplateAddX509DataAddCertificate(PyObject* self, PyOb
PyXmlSec_LxmlElementPtr node = NULL;
xmlNodePtr res;
+ PyObject* result;
+ PyXmlSec_LxmlShadow shadow;
PYXMLSEC_DEBUG("template x509_data_add_certificate - start");
if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&:x509_data_add_certificate", kwlist,
@@ -503,16 +571,19 @@ static PyObject* PyXmlSec_TemplateAddX509DataAddCertificate(PyObject* self, PyOb
goto ON_FAIL;
}
+ if (PyXmlSec_LxmlShadowBegin(&shadow, node) < 0) {
+ goto ON_FAIL;
+ }
Py_BEGIN_ALLOW_THREADS;
- res = xmlSecTmplX509DataAddCertificate(node->_c_node);
+ res = xmlSecTmplX509DataAddCertificate(shadow.root);
Py_END_ALLOW_THREADS;
- if (res == NULL) {
- PyXmlSec_SetLastError("cannot add x509 certificate.");
+ result = PyXmlSec_LxmlShadowEnd(&shadow, res, "cannot add x509 certificate.");
+ if (result == NULL) {
goto ON_FAIL;
}
PYXMLSEC_DEBUG("template x509_data_add_certificate - ok");
- return (PyObject*)PyXmlSec_elementFactory(node->_doc, res);
+ return result;
ON_FAIL:
PYXMLSEC_DEBUG("template x509_data_add_certificate - fail");
@@ -531,6 +602,8 @@ static PyObject* PyXmlSec_TemplateAddX509DataAddCRL(PyObject* self, PyObject *ar
PyXmlSec_LxmlElementPtr node = NULL;
xmlNodePtr res;
+ PyObject* result;
+ PyXmlSec_LxmlShadow shadow;
PYXMLSEC_DEBUG("template x509_data_add_crl - start");
if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&:x509_data_add_crl", kwlist,
@@ -539,16 +612,19 @@ static PyObject* PyXmlSec_TemplateAddX509DataAddCRL(PyObject* self, PyObject *ar
goto ON_FAIL;
}
+ if (PyXmlSec_LxmlShadowBegin(&shadow, node) < 0) {
+ goto ON_FAIL;
+ }
Py_BEGIN_ALLOW_THREADS;
- res = xmlSecTmplX509DataAddCRL(node->_c_node);
+ res = xmlSecTmplX509DataAddCRL(shadow.root);
Py_END_ALLOW_THREADS;
- if (res == NULL) {
- PyXmlSec_SetLastError("cannot add x509 CRL.");
+ result = PyXmlSec_LxmlShadowEnd(&shadow, res, "cannot add x509 CRL.");
+ if (result == NULL) {
goto ON_FAIL;
}
PYXMLSEC_DEBUG("template x509_data_add_crl - ok");
- return (PyObject*)PyXmlSec_elementFactory(node->_doc, res);
+ return result;
ON_FAIL:
PYXMLSEC_DEBUG("template x509_data_add_crl - fail");
@@ -579,6 +655,8 @@ static PyObject* PyXmlSec_TemplateAddEncryptedKey(PyObject* self, PyObject *args
const char* type = NULL;
const char* recipient = NULL;
xmlNodePtr res;
+ PyObject* result;
+ PyXmlSec_LxmlShadow shadow;
PYXMLSEC_DEBUG("template add_encrypted_key - start");
if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&O!|zzz:add_encrypted_key", kwlist,
@@ -587,16 +665,19 @@ static PyObject* PyXmlSec_TemplateAddEncryptedKey(PyObject* self, PyObject *args
goto ON_FAIL;
}
+ if (PyXmlSec_LxmlShadowBegin(&shadow, node) < 0) {
+ goto ON_FAIL;
+ }
Py_BEGIN_ALLOW_THREADS;
- res = xmlSecTmplKeyInfoAddEncryptedKey(node->_c_node, method->id, XSTR(id), XSTR(type), XSTR(recipient));
+ res = xmlSecTmplKeyInfoAddEncryptedKey(shadow.root, method->id, XSTR(id), XSTR(type), XSTR(recipient));
Py_END_ALLOW_THREADS;
- if (res == NULL) {
- PyXmlSec_SetLastError("cannot add encrypted key.");
+ result = PyXmlSec_LxmlShadowEnd(&shadow, res, "cannot add encrypted key.");
+ if (result == NULL) {
goto ON_FAIL;
}
PYXMLSEC_DEBUG("template add_encrypted_key - ok");
- return (PyObject*)PyXmlSec_elementFactory(node->_doc, res);
+ return result;
ON_FAIL:
PYXMLSEC_DEBUG("template add_encrypted_key - fail");
@@ -633,6 +714,9 @@ static PyObject* PyXmlSec_TemplateCreateEncryptedData(PyObject* self, PyObject *
const char* encoding = NULL;
const char* ns = NULL;
xmlNodePtr res;
+ xmlDocPtr tdoc;
+ PyObject* result;
+ PyXmlSec_LxmlShadow shadow;
PYXMLSEC_DEBUG("template encrypted_data_create - start");
if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&O!|zzzzz:encrypted_data_create", kwlist,
@@ -641,19 +725,24 @@ static PyObject* PyXmlSec_TemplateCreateEncryptedData(PyObject* self, PyObject *
goto ON_FAIL;
}
- Py_BEGIN_ALLOW_THREADS;
- res = xmlSecTmplEncDataCreate(node->_doc->_c_doc, method->id, XSTR(id), XSTR(type), XSTR(mime_type), XSTR(encoding));
- Py_END_ALLOW_THREADS;
- if (res == NULL) {
- PyXmlSec_SetLastError("cannot create encrypted data.");
+ tdoc = PyXmlSec_LxmlShadowBeginNewDoc(&shadow, node);
+ if (tdoc == NULL) {
goto ON_FAIL;
}
- if (ns != NULL) {
+ Py_BEGIN_ALLOW_THREADS;
+ res = xmlSecTmplEncDataCreate(tdoc, method->id, XSTR(id), XSTR(type), XSTR(mime_type), XSTR(encoding));
+ // the prefix rewrite happens before End so the serialization carries it
+ if (res != NULL && ns != NULL) {
res->ns->prefix = xmlStrdup(XSTR(ns));
}
+ Py_END_ALLOW_THREADS;
+ result = PyXmlSec_LxmlShadowEnd(&shadow, res, "cannot create encrypted data.");
+ if (result == NULL) {
+ goto ON_FAIL;
+ }
PYXMLSEC_DEBUG("template encrypted_data_create - ok");
- return (PyObject*)PyXmlSec_elementFactory(node->_doc, res);
+ return result;
ON_FAIL:
PYXMLSEC_DEBUG("template encrypted_data_create - fail");
@@ -678,6 +767,8 @@ static PyObject* PyXmlSec_TemplateEncryptedDataEnsureKeyInfo(PyObject* self, PyO
const char* id = NULL;
const char* ns = NULL;
xmlNodePtr res;
+ PyObject* result;
+ PyXmlSec_LxmlShadow shadow;
PYXMLSEC_DEBUG("template encrypted_data_ensure_key_info - start");
if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&|zz:encrypted_data_ensure_key_info", kwlist,
@@ -686,19 +777,23 @@ static PyObject* PyXmlSec_TemplateEncryptedDataEnsureKeyInfo(PyObject* self, PyO
goto ON_FAIL;
}
- Py_BEGIN_ALLOW_THREADS;
- res = xmlSecTmplEncDataEnsureKeyInfo(node->_c_node, XSTR(id));
- Py_END_ALLOW_THREADS;
- if (res == NULL) {
- PyXmlSec_SetLastError("cannot ensure key info for encrypted data.");
+ if (PyXmlSec_LxmlShadowBegin(&shadow, node) < 0) {
goto ON_FAIL;
}
- if (ns != NULL) {
+ Py_BEGIN_ALLOW_THREADS;
+ res = xmlSecTmplEncDataEnsureKeyInfo(shadow.root, XSTR(id));
+ // the prefix rewrite lands before End so the reflection carries it
+ if (res != NULL && ns != NULL) {
res->ns->prefix = xmlStrdup(XSTR(ns));
}
+ Py_END_ALLOW_THREADS;
+ result = PyXmlSec_LxmlShadowEnd(&shadow, res, "cannot ensure key info for encrypted data.");
+ if (result == NULL) {
+ goto ON_FAIL;
+ }
PYXMLSEC_DEBUG("template encrypted_data_ensure_key_info - ok");
- return (PyObject*)PyXmlSec_elementFactory(node->_doc, res);
+ return result;
ON_FAIL:
PYXMLSEC_DEBUG("template encrypted_data_ensure_key_info - fail");
@@ -717,6 +812,8 @@ static PyObject* PyXmlSec_TemplateEncryptedDataEnsureCipherValue(PyObject* self,
PyXmlSec_LxmlElementPtr node = NULL;
xmlNodePtr res;
+ PyObject* result;
+ PyXmlSec_LxmlShadow shadow;
PYXMLSEC_DEBUG("template encrypted_data_ensure_cipher_value - start");
if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&:encrypted_data_ensure_cipher_value", kwlist,
@@ -725,16 +822,19 @@ static PyObject* PyXmlSec_TemplateEncryptedDataEnsureCipherValue(PyObject* self,
goto ON_FAIL;
}
+ if (PyXmlSec_LxmlShadowBegin(&shadow, node) < 0) {
+ goto ON_FAIL;
+ }
Py_BEGIN_ALLOW_THREADS;
- res = xmlSecTmplEncDataEnsureCipherValue(node->_c_node);
+ res = xmlSecTmplEncDataEnsureCipherValue(shadow.root);
Py_END_ALLOW_THREADS;
- if (res == NULL) {
- PyXmlSec_SetLastError("cannot ensure cipher value for encrypted data.");
+ result = PyXmlSec_LxmlShadowEnd(&shadow, res, "cannot ensure cipher value for encrypted data.");
+ if (result == NULL) {
goto ON_FAIL;
}
PYXMLSEC_DEBUG("template encrypted_data_ensure_cipher_value - ok");
- return (PyObject*)PyXmlSec_elementFactory(node->_doc, res);
+ return result;
ON_FAIL:
PYXMLSEC_DEBUG("template encrypted_data_ensure_cipher_value - fail");
@@ -756,6 +856,7 @@ static PyObject* PyXmlSec_TemplateTransformAddC14NInclNamespaces(PyObject* self,
PyObject* sep;
int res;
const char* c_prefixes;
+ PyXmlSec_LxmlShadow shadow;
// transform_add_c14n_inclusive_namespaces
PYXMLSEC_DEBUG("template encrypted_data_ensure_cipher_value - start");
@@ -782,11 +883,15 @@ static PyObject* PyXmlSec_TemplateTransformAddC14NInclNamespaces(PyObject* self,
c_prefixes = PyUnicode_AsUTF8(prefixes);
+ if (PyXmlSec_LxmlShadowBegin(&shadow, node) < 0) {
+ goto ON_FAIL;
+ }
Py_BEGIN_ALLOW_THREADS;
- res = xmlSecTmplTransformAddC14NInclNamespaces(node->_c_node, XSTR(c_prefixes));
+ res = xmlSecTmplTransformAddC14NInclNamespaces(shadow.root, XSTR(c_prefixes));
Py_END_ALLOW_THREADS;
- if (res != 0) {
- PyXmlSec_SetLastError("cannot add 'inclusive' namespaces to the ExcC14N transform node");
+ // the call only reports a status; the reflect grafts the
+ // it created back into the live tree
+ if (PyXmlSec_LxmlShadowReflect(&shadow, res, "cannot add 'inclusive' namespaces to the ExcC14N transform node") < 0) {
goto ON_FAIL;
}
diff --git a/src/tree.c b/src/tree.c
index 37cae785..de089b4a 100644
--- a/src/tree.c
+++ b/src/tree.c
@@ -33,6 +33,8 @@ static PyObject* PyXmlSec_TreeFindChild(PyObject* self, PyObject *args, PyObject
const char* name = NULL;
const char* ns = (const char*)xmlSecDSigNs;
xmlNodePtr res;
+ PyObject* result;
+ PyXmlSec_LxmlShadow shadow;
PYXMLSEC_DEBUG("tree find_child - start");
if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&s|s:find_child", kwlist,
@@ -41,15 +43,21 @@ static PyObject* PyXmlSec_TreeFindChild(PyObject* self, PyObject *args, PyObject
goto ON_FAIL;
}
+ // Read-only, but still ABI-unsafe on raw nodes: the search runs on a
+ // shadow copy and the found node is mapped back by path (issue #356).
+ if (PyXmlSec_LxmlShadowBegin(&shadow, node) < 0) {
+ goto ON_FAIL;
+ }
Py_BEGIN_ALLOW_THREADS;
- res = xmlSecFindChild(node->_c_node, XSTR(name), XSTR(ns));
+ res = xmlSecFindChild(shadow.root, XSTR(name), XSTR(ns));
Py_END_ALLOW_THREADS;
+ result = PyXmlSec_LxmlShadowEndFind(&shadow, res);
+ if (result == NULL) {
+ goto ON_FAIL;
+ }
PYXMLSEC_DEBUG("tree find_child - ok");
- if (res == NULL) {
- Py_RETURN_NONE;
- }
- return (PyObject*)PyXmlSec_elementFactory(node->_doc, res);
+ return result;
ON_FAIL:
PYXMLSEC_DEBUG("tree find_child - fail");
@@ -74,6 +82,9 @@ static PyObject* PyXmlSec_TreeFindParent(PyObject* self, PyObject *args, PyObjec
const char* name = NULL;
const char* ns = (const char*)xmlSecDSigNs;
xmlNodePtr res;
+ xmlNodePtr target;
+ PyObject* result;
+ PyXmlSec_LxmlShadow shadow;
PYXMLSEC_DEBUG("tree find_parent - start");
if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&s|s:find_parent", kwlist,
@@ -82,15 +93,21 @@ static PyObject* PyXmlSec_TreeFindParent(PyObject* self, PyObject *args, PyObjec
goto ON_FAIL;
}
+ // The search walks upward, so the shadow must cover the whole tree; the
+ // call then starts from the copy's counterpart of `node`.
+ if (PyXmlSec_LxmlShadowBeginDoc(&shadow, node, &target) < 0) {
+ goto ON_FAIL;
+ }
Py_BEGIN_ALLOW_THREADS;
- res = xmlSecFindParent(node->_c_node, XSTR(name), XSTR(ns));
+ res = xmlSecFindParent(target, XSTR(name), XSTR(ns));
Py_END_ALLOW_THREADS;
+ result = PyXmlSec_LxmlShadowEndFind(&shadow, res);
+ if (result == NULL) {
+ goto ON_FAIL;
+ }
PYXMLSEC_DEBUG("tree find_parent - ok");
- if (res == NULL) {
- Py_RETURN_NONE;
- }
- return (PyObject*)PyXmlSec_elementFactory(node->_doc, res);
+ return result;
ON_FAIL:
PYXMLSEC_DEBUG("tree find_parent - fail");
@@ -115,6 +132,8 @@ static PyObject* PyXmlSec_TreeFindNode(PyObject* self, PyObject *args, PyObject
const char* name = NULL;
const char* ns = (const char*)xmlSecDSigNs;
xmlNodePtr res;
+ PyObject* result;
+ PyXmlSec_LxmlShadow shadow;
PYXMLSEC_DEBUG("tree find_node - start");
if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&s|s:find_node", kwlist,
@@ -123,15 +142,19 @@ static PyObject* PyXmlSec_TreeFindNode(PyObject* self, PyObject *args, PyObject
goto ON_FAIL;
}
+ if (PyXmlSec_LxmlShadowBegin(&shadow, node) < 0) {
+ goto ON_FAIL;
+ }
Py_BEGIN_ALLOW_THREADS;
- res = xmlSecFindNode(node->_c_node, XSTR(name), XSTR(ns));
+ res = xmlSecFindNode(shadow.root, XSTR(name), XSTR(ns));
Py_END_ALLOW_THREADS;
+ result = PyXmlSec_LxmlShadowEndFind(&shadow, res);
+ if (result == NULL) {
+ goto ON_FAIL;
+ }
PYXMLSEC_DEBUG("tree find_node - ok");
- if (res == NULL) {
- Py_RETURN_NONE;
- }
- return (PyObject*)PyXmlSec_elementFactory(node->_doc, res);
+ return result;
ON_FAIL:
PYXMLSEC_DEBUG("tree find_node - fail");
@@ -170,6 +193,36 @@ static PyObject* PyXmlSec_TreeAddIds(PyObject* self, PyObject *args, PyObject *k
n = PyObject_Length(ids);
if (n < 0) goto ON_FAIL;
+ // Shadow mode: registering IDs on lxml's document with our libxml2 is
+ // exactly the cross-library write issue #356 forbids. Record the
+ // attributes the subtree rooted at `node` — the scope xmlSecAddIDs walks
+ // below — carries right now instead; every whole-document shadow (sign,
+ // verify, decrypt) replays those onto its private copy.
+ if (PyXmlSec_LxmlShadowIsActive()) {
+ // Materialize and validate the whole list before recording any of it,
+ // as the fast path below does before it calls xmlSecAddIDs: a bad
+ // item must leave nothing registered.
+ PyObject* names = PyList_New(0);
+ int rv;
+ if (names == NULL) goto ON_FAIL;
+ for (i = 0; i < n; ++i) {
+ key = PyLong_FromSsize_t(i);
+ tmp = key != NULL ? PyObject_GetItem(ids, key) : NULL;
+ Py_XDECREF(key);
+ if (tmp == NULL || PyUnicode_AsUTF8(tmp) == NULL || PyList_Append(names, tmp) < 0) {
+ Py_XDECREF(tmp);
+ Py_DECREF(names);
+ goto ON_FAIL;
+ }
+ Py_DECREF(tmp);
+ }
+ rv = PyXmlSec_LxmlShadowRecordIds(node, names, NULL, 1);
+ Py_DECREF(names);
+ if (rv < 0) goto ON_FAIL;
+ PYXMLSEC_DEBUG("tree add_ids - ok");
+ Py_RETURN_NONE;
+ }
+
list = (const xmlChar**)xmlMalloc(sizeof(xmlChar*) * (n + 1));
if (list == NULL) {
PyErr_SetString(PyExc_MemoryError, "no memory");
diff --git a/tests/data/id_attr.dtd b/tests/data/id_attr.dtd
new file mode 100644
index 00000000..083b3d9f
--- /dev/null
+++ b/tests/data/id_attr.dtd
@@ -0,0 +1,3 @@
+
+
diff --git a/tests/test_ds.py b/tests/test_ds.py
index dd0657d3..3796b4f7 100644
--- a/tests/test_ds.py
+++ b/tests/test_ds.py
@@ -1,5 +1,7 @@
import unittest
+from lxml import etree
+
import xmlsec
from tests import base
@@ -55,6 +57,74 @@ def test_register_id_bad_args(self):
with self.assertRaises(TypeError):
ctx.register_id('')
+ def test_register_id_matches_namespaced_attribute_by_local_name(self):
+ """Should accept a namespaced id attribute when no id_ns is given, as xmlHasProp does."""
+ ctx = xmlsec.SignatureContext()
+ root = self.load_xml('sign_template.xml')
+ sign = xmlsec.template.create(root, consts.TransformExclC14N, consts.TransformRsaSha1)
+ sign.set('{http://www.example.org/ns}Id', 'sig-1')
+ ctx.register_id(sign, 'Id')
+
+ def test_register_id_rejects_an_id_value_already_registered(self):
+ """A value another registration claimed cannot be registered again: only one attribute can win the id lookup."""
+ ctx = xmlsec.SignatureContext()
+ root = etree.fromstring(b'')
+ ctx.register_id(root[0], 'ID')
+ with self.assertRaisesRegex(xmlsec.Error, 'duplicated id.'):
+ ctx.register_id(root[1], 'Id')
+
+ def test_register_id_rejects_an_id_value_the_document_declares(self):
+ """An xml:id the document was parsed with already claims the value, so another attribute for it is refused."""
+ root = etree.fromstring(b'')
+ with self.assertRaisesRegex(xmlsec.Error, 'duplicated id.'):
+ xmlsec.SignatureContext().register_id(root[1], 'ID')
+
+ def test_register_id_rejects_an_id_value_add_ids_claimed(self):
+ """Registrations made over a subtree by add_ids claim their values too."""
+ root = etree.fromstring(b'')
+ xmlsec.tree.add_ids(root[0], ['ID'])
+ with self.assertRaisesRegex(xmlsec.Error, 'duplicated id.'):
+ xmlsec.SignatureContext().register_id(root[1], 'Id')
+
+ def test_register_id_accepts_the_same_attribute_twice(self):
+ """Re-registering an attribute is the no-op the id lookup already resolves; only another claim is a duplicate."""
+ ctx = xmlsec.SignatureContext()
+ root = etree.fromstring(b'')
+ ctx.register_id(root[0], 'ID')
+ ctx.register_id(root[0], 'ID')
+
+ def test_register_id_accepts_an_attribute_add_ids_registered(self):
+ """add_ids already registered this very attribute, which the fast path's `tmpAttr == attr` accepts."""
+ root = etree.fromstring(b'')
+ xmlsec.tree.add_ids(root[0], ['ID'])
+ xmlsec.SignatureContext().register_id(root[0][0], 'ID')
+
+ def test_register_id_rejects_a_value_a_sibling_attribute_declares(self):
+ """The declared id of an element can sit on another of its attributes, which claims the value all the same."""
+ root = etree.fromstring(b'')
+ with self.assertRaisesRegex(xmlsec.Error, 'duplicated id.'):
+ xmlsec.SignatureContext().register_id(root[0], 'ID')
+
+ def test_register_id_accepts_the_declared_attribute_beside_a_twin(self):
+ """A second attribute repeating the value is not the declared one, so registering the declared one is the no-op."""
+ xml = b'\n\n'
+ xmlsec.SignatureContext().register_id(self.parse_with_external_dtd(xml)[0], 'ID')
+
+ def test_register_id_rejects_a_value_a_namespaced_registration_claimed(self):
+ """Two attributes of one element differing only in namespace are two attributes: the second cannot win the lookup."""
+ ctx = xmlsec.SignatureContext()
+ root = etree.fromstring(b'')
+ ctx.register_id(root[0], 'Id', id_ns='urn:a')
+ with self.assertRaisesRegex(xmlsec.Error, 'duplicated id.'):
+ ctx.register_id(root[0], 'Id')
+
+ def test_register_id_accepts_the_same_namespaced_attribute_twice(self):
+ """The no-op holds for a namespaced attribute too, which the key comparison must not read as a collision."""
+ ctx = xmlsec.SignatureContext()
+ root = etree.fromstring(b'')
+ ctx.register_id(root[0], 'Id', id_ns='urn:a')
+ ctx.register_id(root[0], 'Id', id_ns='urn:a')
+
def test_register_id_with_namespace_without_attribute(self):
ctx = xmlsec.SignatureContext()
root = self.load_xml('sign_template.xml')
@@ -62,6 +132,35 @@ def test_register_id_with_namespace_without_attribute(self):
with self.assertRaisesRegex(xmlsec.Error, 'missing attribute.'):
ctx.register_id(sign, 'Id', id_ns='foo')
+ def sign_id_reference(self, root, uri):
+ """Signs `root` with a single reference to `uri` and returns the digest of that reference."""
+ sign = xmlsec.template.create(root, consts.TransformExclC14N, consts.TransformRsaSha1, ns='ds')
+ root.append(sign)
+ ref = xmlsec.template.add_reference(sign, consts.TransformSha1, uri=uri)
+ xmlsec.template.add_transform(ref, consts.TransformExclC14N)
+ ctx = xmlsec.SignatureContext()
+ ctx.key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem)
+ ctx.sign(sign)
+ return xmlsec.tree.find_node(sign, consts.NodeDigestValue, consts.DSigNs).text
+
+ def test_add_ids_ignores_an_id_that_joins_the_scope_later(self):
+ """add_ids registers what the scope carries at the call, which is where xmlSecAddIDs walks it."""
+ root = etree.fromstring(b'')
+ xmlsec.tree.add_ids(root, ['ID'])
+ etree.SubElement(root, 'B', {'ID': 'b'})
+ with self.assertRaisesRegex(xmlsec.Error, 'failed to sign'):
+ self.sign_id_reference(root, '#b')
+
+ def test_add_ids_claims_a_value_in_document_order(self):
+ """xmlSecAddIDs walks the scope element by element, the names within each: claims "v" first."""
+ xml = b'yx'
+ root = etree.fromstring(xml)
+ xmlsec.tree.add_ids(root, ['A', 'B'])
+ both = self.sign_id_reference(root, '#v')
+ root = etree.fromstring(xml)
+ xmlsec.tree.add_ids(root, ['B'])
+ self.assertEqual(both, self.sign_id_reference(root, '#v'))
+
def test_sign_bad_args(self):
ctx = xmlsec.SignatureContext()
ctx.key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem)
@@ -188,6 +287,170 @@ def test_sign_case5(self):
expected_xml_file = 'sign5-out.xml'
self.assertEqual(self.load_xml(expected_xml_file), root)
+ def test_sign_and_verify_with_registered_id(self):
+ """Should resolve a #id reference registered through register_id (not add_ids) on sign and verify."""
+ root = self.load_xml('sign4-in.xml')
+ ctx = xmlsec.SignatureContext()
+ ctx.register_id(root, 'ID')
+ sign = xmlsec.template.create(root, consts.TransformExclC14N, consts.TransformRsaSha1, ns='ds')
+ root.append(sign)
+ ref = xmlsec.template.add_reference(sign, consts.TransformSha1, uri='#' + root.get('ID'))
+ xmlsec.template.add_transform(ref, consts.TransformEnveloped)
+ xmlsec.template.add_transform(ref, consts.TransformExclC14N)
+ ki = xmlsec.template.ensure_key_info(sign)
+ xmlsec.template.add_x509_data(ki)
+
+ ctx.key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem)
+ ctx.key.load_cert_from_file(self.path('rsacert.pem'), consts.KeyDataFormatPem)
+ ctx.key.name = 'rsakey.pem'
+ ctx.sign(sign)
+ self.assertEqual(self.load_xml('sign4-out.xml'), root)
+
+ verify_ctx = xmlsec.SignatureContext()
+ verify_ctx.register_id(root, 'ID')
+ verify_ctx.key = xmlsec.Key.from_file(self.path('rsapub.pem'), format=consts.KeyDataFormatPem)
+ verify_ctx.verify(sign)
+
+ def test_sign_and_verify_a_template_removed_from_its_document(self):
+ """A template taken out of its tree still signs: lxml leaves it pointing at the document it left,
+ and that is where its URI="" reference resolves — on the raw path too (issue #356)."""
+ root = self.load_xml('sign1-in.xml')
+ sign = xmlsec.tree.find_node(root, consts.NodeSignature, consts.DSigNs)
+ sign.getparent().remove(sign)
+
+ ctx = xmlsec.SignatureContext()
+ ctx.key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem)
+ ctx.sign(sign)
+ # the signature is written into the removed subtree, and nothing is
+ # written into the document it covers
+ self.assertTrue(xmlsec.tree.find_node(sign, consts.NodeSignatureValue, consts.DSigNs).text)
+ self.assertIsNone(xmlsec.tree.find_node(root, consts.NodeSignature, consts.DSigNs))
+
+ verify_ctx = xmlsec.SignatureContext()
+ verify_ctx.key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem)
+ verify_ctx.verify(sign)
+
+ # the digest covers that document: changing it invalidates the signature
+ root.find('{urn:envelope}Data').text = 'tampered'
+ tampered_ctx = xmlsec.SignatureContext()
+ tampered_ctx.key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem)
+ with self.assertRaises(xmlsec.VerificationError):
+ tampered_ctx.verify(sign)
+
+ def test_sign_and_verify_a_removed_template_against_a_registered_id(self):
+ """A #id reference from a removed subtree resolves in the document it left (issue #356)."""
+ root = self.load_xml('sign4-in.xml')
+ sign = xmlsec.template.create(root, consts.TransformExclC14N, consts.TransformRsaSha1, ns='ds')
+ root.append(sign)
+ ref = xmlsec.template.add_reference(sign, consts.TransformSha1, uri='#' + root.get('ID'))
+ xmlsec.template.add_transform(ref, consts.TransformExclC14N)
+ root.remove(sign)
+
+ ctx = xmlsec.SignatureContext()
+ ctx.register_id(root, 'ID')
+ ctx.key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem)
+ ctx.sign(sign)
+
+ verify_ctx = xmlsec.SignatureContext()
+ verify_ctx.register_id(root, 'ID')
+ verify_ctx.key = xmlsec.Key.from_file(self.path('rsapub.pem'), format=consts.KeyDataFormatPem)
+ verify_ctx.verify(sign)
+
+ # A document whose id attribute is typed by an external DTD subset — the
+ # declarations live in a file the DOCTYPE names, and only a parse that
+ # loads it knows "#ext" resolves to the Node (issue #356).
+ EXTERNAL_DTD_XML = b'\nsigned\n'
+
+ def parse_with_external_dtd(self, xml):
+ """Parse `xml` with its external subset loaded, or skip the test when this build forbids that.
+
+ xmlsec installs a no-XXE external entity loader globally at
+ ``xmlSecInit`` (1.2.34 and later, and the patched 1.2.33 some
+ distributions ship), so merely importing xmlsec can refuse lxml its
+ own ``load_dtd=True`` parse. Nothing then types the id, on either
+ path, and there is no declaration left for the copy to carry across.
+ """
+ root = etree.fromstring(xml, etree.XMLParser(load_dtd=True), base_url=self.path('doc.xml'))
+ if root.getroottree().docinfo.externalDTD is None:
+ self.skipTest('this build refuses to load an external DTD subset')
+ return root
+
+ def test_sign_and_verify_with_an_id_an_external_dtd_declares(self):
+ """Should resolve a #id reference whose id attribute an external subset the caller loaded declares."""
+ root = self.parse_with_external_dtd(self.EXTERNAL_DTD_XML)
+ sign = xmlsec.template.create(root, consts.TransformExclC14N, consts.TransformRsaSha1, ns='ds')
+ root.append(sign)
+ ref = xmlsec.template.add_reference(sign, consts.TransformSha1, uri='#ext')
+ xmlsec.template.add_transform(ref, consts.TransformExclC14N)
+
+ ctx = xmlsec.SignatureContext()
+ ctx.key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem)
+ ctx.sign(sign)
+
+ verify_ctx = xmlsec.SignatureContext()
+ verify_ctx.key = xmlsec.Key.from_file(self.path('rsapub.pem'), format=consts.KeyDataFormatPem)
+ verify_ctx.verify(sign)
+
+ # A document where an unregistered element carries the same id value as the
+ # registered one, and comes first. Only the registered element may answer
+ # the "#dup" reference (issue #356).
+ DUPLICATE_ID_XML = (
+ b'\n'
+ b' decoy\n'
+ b' real\n'
+ b'\n'
+ )
+
+ def sign_duplicate_id(self, register):
+ """Signs a "#dup" reference over the document above, registering the ids with `register`."""
+ root = etree.fromstring(self.DUPLICATE_ID_XML)
+ ctx = xmlsec.SignatureContext()
+ register(root)
+ sign = xmlsec.template.create(root, consts.TransformExclC14N, consts.TransformRsaSha1, ns='ds')
+ root.append(sign)
+ ref = xmlsec.template.add_reference(sign, consts.TransformSha1, uri='#dup')
+ xmlsec.template.add_transform(ref, consts.TransformExclC14N)
+ ctx.key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem)
+ ctx.sign(sign)
+ return etree.tostring(root)
+
+ def verify_duplicate_id(self, signed, register, tamper):
+ """Re-parses `signed`, rewrites the text of the `tamper` element and verifies."""
+ root = etree.fromstring(signed)
+ register(root)
+ root.find(tamper).text = 'tampered'
+ ctx = xmlsec.SignatureContext()
+ ctx.key = xmlsec.Key.from_file(self.path('rsapub.pem'), format=consts.KeyDataFormatPem)
+ ctx.verify(root.find('dsig:Signature', namespaces=base.ns))
+
+ def assert_covers_registered_element_only(self, register):
+ signed = self.sign_duplicate_id(register)
+ # the decoy is not what the reference resolved to, so it is not covered
+ self.verify_duplicate_id(signed, register, 'Decoy/Data')
+ # the registered element is
+ with self.assertRaises(xmlsec.VerificationError):
+ self.verify_duplicate_id(signed, register, 'Scope/Real/Data')
+
+ def test_register_id_covers_only_the_registered_node(self):
+ """register_id registers the node it is given, not every element with that attribute."""
+ self.assert_covers_registered_element_only(
+ lambda root: xmlsec.SignatureContext().register_id(root.find('Scope/Real'), 'ID')
+ )
+
+ def test_add_ids_covers_only_the_given_subtree(self):
+ """add_ids registers the subtree it is given, not the whole document."""
+ self.assert_covers_registered_element_only(lambda root: xmlsec.tree.add_ids(root.find('Scope'), ['ID']))
+
+ def test_add_ids_records_nothing_when_an_item_is_not_a_name(self):
+ """A rejected add_ids leaves no part of its list registered, so the "#dup" reference resolves to nothing."""
+
+ def register(root):
+ with self.assertRaises(TypeError):
+ xmlsec.tree.add_ids(root.find('Scope'), ['ID', 1])
+
+ with self.assertRaises(xmlsec.Error):
+ self.sign_duplicate_id(register)
+
def test_sign_binary_bad_args(self):
ctx = xmlsec.SignatureContext()
ctx.key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem)
@@ -335,3 +598,78 @@ def test_set_enabled_key_data_bad_list(self):
ctx.key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem)
with self.assertRaisesRegex(TypeError, 'expected list of KeyData constants.'):
ctx.set_enabled_key_data('foo')
+
+
+class TestIdRegistryLifetime(base.TestMemoryLeaks):
+ """Guards the lifetime rule of the shadow-mode id registry (issue #356)."""
+
+ # The document churn below is this test's point; the leak harness would repeat it ten times.
+ iterations = 0
+
+ # Far more documents than any bounded registry would have held at once.
+ OTHER_DOCUMENTS = 5000
+
+ def test_registration_survives_other_live_documents(self):
+ """Registering ids for other documents must never drop a live document's own registration."""
+ root = self.load_xml('sign4-out.xml')
+ ctx = xmlsec.SignatureContext()
+ ctx.register_id(root, 'ID')
+
+ # All kept alive, so none of these registrations may be traded for another.
+ others = [etree.fromstring(f'') for i in range(self.OTHER_DOCUMENTS)]
+ for other in others:
+ ctx.register_id(other, 'ID')
+
+ sign = xmlsec.tree.find_node(root, consts.NodeSignature)
+ ctx.key = xmlsec.Key.from_file(self.path('rsapub.pem'), format=consts.KeyDataFormatPem)
+ ctx.verify(sign) # resolves #ID through the registration made before the churn
+
+ def test_registration_dies_with_its_element(self):
+ """A dropped element takes its registration with it: libxml2 drops the id entry when it frees the attribute."""
+ root = etree.fromstring(b'')
+ ctx = xmlsec.SignatureContext()
+ for _ in range(2): # the second registration claims the value the first one did
+ node = etree.SubElement(root, 'T', {'ID': 'x'})
+ ctx.register_id(node, 'ID')
+ root.remove(node)
+ del node
+
+ def test_registration_survives_its_element_being_removed(self):
+ """An element out of its document is not gone: lxml keeps the subtree, and the id entry stands."""
+ root = etree.fromstring(b'')
+ ctx = xmlsec.SignatureContext()
+ removed = root[0]
+ ctx.register_id(removed, 'ID')
+ root.remove(removed)
+ with self.assertRaisesRegex(xmlsec.Error, 'duplicated id.'):
+ ctx.register_id(root[0], 'ID')
+
+ def test_registration_survives_a_descendant_being_held(self):
+ """A proxy anywhere in a removed subtree keeps it alive, registration included."""
+ root = etree.fromstring(b'')
+ ctx = xmlsec.SignatureContext()
+ ctx.register_id(root[0], 'ID')
+ inner = root[0][0]
+ root.remove(inner.getparent())
+ with self.assertRaisesRegex(xmlsec.Error, 'duplicated id.'):
+ ctx.register_id(root[0], 'ID')
+ del inner
+
+ def test_registration_survives_a_sibling_being_adopted_away(self):
+ """An element moved into another document must not make its old document's registrations look collectable."""
+ root = etree.fromstring(b'x')
+ ctx = xmlsec.SignatureContext()
+ moved = root.find('Moved')
+ ctx.register_id(moved, 'ID')
+ ctx.register_id(root.find('Stays'), 'ID')
+
+ etree.fromstring(b'').append(moved) # `moved` now references the other document
+ del moved
+ ctx.register_id(etree.fromstring(b''), 'ID') # a new document prunes the registry
+
+ sign = xmlsec.template.create(root, consts.TransformExclC14N, consts.TransformRsaSha1, ns='ds')
+ root.append(sign)
+ ref = xmlsec.template.add_reference(sign, consts.TransformSha1, uri='#stays')
+ xmlsec.template.add_transform(ref, consts.TransformExclC14N)
+ ctx.key = xmlsec.Key.from_file(self.path('rsakey.pem'), format=consts.KeyDataFormatPem)
+ ctx.sign(sign) # resolves #stays only if `root`'s registrations survived the prune
diff --git a/tests/test_enc.py b/tests/test_enc.py
index 41f78d74..a97f8823 100644
--- a/tests/test_enc.py
+++ b/tests/test_enc.py
@@ -1,3 +1,4 @@
+import io
import tempfile
from lxml import etree
@@ -82,6 +83,76 @@ def test_encrypt_xml(self):
cipher_value = xmlsec.tree.find_node(ki, consts.NodeCipherValue, consts.EncNs)
self.assertIsNotNone(cipher_value)
+ def test_encrypt_xml_root(self):
+ # Type=Element on the document root replaces the root itself: the new
+ # root keeps the template's namespace prefix and the document-level
+ # siblings, and decrypting it restores the document in place
+ xml = b'ttail'
+ root = etree.parse(io.BytesIO(xml)).getroot()
+ key = xmlsec.Key.generate(consts.KeyDataAes, 128, consts.KeyDataTypeSession)
+ enc_data = xmlsec.template.encrypted_data_create(root, consts.TransformAes128Cbc, type=consts.TypeEncElement, ns='xenc')
+ xmlsec.template.encrypted_data_ensure_cipher_value(enc_data)
+ ctx = xmlsec.EncryptionContext()
+ ctx.key = key
+ encrypted = ctx.encrypt_xml(enc_data, root)
+ self.assertEqual(f'{{{consts.EncNs}}}{consts.NodeEncryptedData}', encrypted.tag)
+ self.assertEqual('xenc', encrypted.prefix)
+ self.assertIsNone(encrypted.getparent())
+ self.assertIs(encrypted.getroottree().getroot(), encrypted)
+ self.assertTrue(etree.tostring(encrypted.getroottree()).startswith(b' where the template was, and
+ # the text that followed it must stay behind (issue #356).
+ root = etree.fromstring(b'\n \n secret\n \n')
+ enc_data = xmlsec.template.encrypted_data_create(root, consts.TransformAes128Cbc, type=enc_type, ns='xenc')
+ xmlsec.template.encrypted_data_ensure_cipher_value(enc_data)
+ root.insert(index, enc_data)
+ enc_data.tail = '\n tail\n '
+
+ ctx = xmlsec.EncryptionContext()
+ ctx.key = xmlsec.Key.generate(consts.KeyDataAes, 128, consts.KeyDataTypeSession)
+ encrypted = ctx.encrypt_xml(enc_data, root.find('Data'))
+
+ self.assertEqual(f'{{{consts.EncNs}}}{consts.NodeEncryptedData}', encrypted.tag)
+ self.assertEqual(1, len(root.findall(f'.//{{{consts.EncNs}}}{consts.NodeEncryptedData}')))
+ self.assertIn('tail', etree.tostring(root).decode())
+
+ def test_encrypt_xml_attached_template_element(self):
+ self.encrypt_with_attached_template(consts.TypeEncElement, 0)
+ self.encrypt_with_attached_template(consts.TypeEncElement, 2)
+
+ def test_encrypt_xml_attached_template_content(self):
+ self.encrypt_with_attached_template(consts.TypeEncContent, 0)
+ self.encrypt_with_attached_template(consts.TypeEncContent, 2)
+
+ def test_encrypt_binary_over_a_filled_cipher_value(self):
+ # The reflection must carry a rewritten text value back, not only the
+ # first one written into an empty element (issue #356).
+ root = self.load_xml('enc1-in.xml')
+ enc_data = xmlsec.template.encrypted_data_create(root, consts.TransformAes128Cbc, type=consts.TypeEncContent, ns='xenc')
+ xmlsec.template.encrypted_data_ensure_cipher_value(enc_data)
+
+ def encrypt(data):
+ ctx = xmlsec.EncryptionContext()
+ ctx.key = xmlsec.Key.generate(consts.KeyDataAes, 128, consts.KeyDataTypeSession)
+ ctx.encrypt_binary(enc_data, data)
+ return xmlsec.tree.find_node(enc_data, consts.NodeCipherValue, consts.EncNs).text
+
+ first = encrypt(b'first')
+ self.assertIsNotNone(first)
+ self.assertNotEqual(first, encrypt(b'a rather different payload'))
+
def test_encrypt_xml_bad_args(self):
ctx = xmlsec.EncryptionContext()
with self.assertRaises(TypeError):
@@ -167,7 +238,7 @@ def test_encrypt_uri(self):
with tempfile.NamedTemporaryFile(delete=False) as tmpfile:
tmpfile.write(b'test')
- encrypted = ctx.encrypt_binary(enc_data, 'file://' + tmpfile.name)
+ encrypted = ctx.encrypt_uri(enc_data, 'file://' + tmpfile.name)
self.assertIsNotNone(encrypted)
self.assertEqual(f'{{{consts.EncNs}}}{consts.NodeEncryptedData}', encrypted.tag)
@@ -183,6 +254,27 @@ def test_encrypt_uri(self):
cipher_value = xmlsec.tree.find_node(ki, consts.NodeCipherValue, consts.EncNs)
self.assertIsNotNone(cipher_value)
+ def test_encrypt_and_decrypt_content_of_a_subtree_removed_from_its_document(self):
+ """A subtree taken out of its tree is encrypted and decrypted in place, as on the raw path (issue #356)."""
+ root = etree.fromstring(b'hello x tail')
+ data = root[0]
+ root.remove(data)
+ before = etree.tostring(data)
+
+ key = xmlsec.Key.generate(consts.KeyDataAes, 128, consts.KeyDataTypeSession)
+ enc_data = xmlsec.template.encrypted_data_create(data, consts.TransformAes128Cbc, type=consts.TypeEncContent, ns='xenc')
+ xmlsec.template.encrypted_data_ensure_cipher_value(enc_data)
+ ctx = xmlsec.EncryptionContext()
+ ctx.key = key
+ ctx.encrypt_xml(enc_data, data)
+ self.assertEqual(b'', etree.tostring(root))
+ self.assertIsNone(data.text)
+
+ dec_ctx = xmlsec.EncryptionContext()
+ dec_ctx.key = key
+ dec_ctx.decrypt(data[0])
+ self.assertEqual(before, etree.tostring(data))
+
def test_encrypt_uri_bad_args(self):
ctx = xmlsec.EncryptionContext()
with self.assertRaises(TypeError):
@@ -229,6 +321,54 @@ def check_decrypt(self, i):
self.assertIsNotNone(decrypted)
self.assertEqual(self.load_xml(f'enc{i}-in.xml'), root)
+ def encrypt_content(self, xml, path):
+ """Encrypts the content of the element at ``path`` with a fresh session key."""
+ root = etree.fromstring(xml)
+ key = xmlsec.Key.generate(consts.KeyDataAes, 128, consts.KeyDataTypeSession)
+ enc_data = xmlsec.template.encrypted_data_create(root, consts.TransformAes128Cbc, type=consts.TypeEncContent, ns='xenc')
+ xmlsec.template.encrypted_data_ensure_cipher_value(enc_data)
+ ctx = xmlsec.EncryptionContext()
+ ctx.key = key
+ ctx.encrypt_xml(enc_data, root.find(path))
+ return root, key
+
+ def test_decrypt_root_content(self):
+ # a root EncryptedData of Type=Content gives way to its decrypted
+ # content, which becomes the new document root
+ root, key = self.encrypt_content(b't', '.')
+ enc_data = etree.fromstring(etree.tostring(root[0]))
+ self.assertIsNone(enc_data.getparent())
+ ctx = xmlsec.EncryptionContext()
+ ctx.key = key
+ decrypted = ctx.decrypt(enc_data)
+ self.assertEqual('x', decrypted.tag)
+ self.assertEqual('t', decrypted.text)
+ self.assertIsNone(decrypted.getparent())
+ self.assertIs(decrypted.getroottree().getroot(), decrypted)
+
+ def test_decrypt_content_text_between_whitespace(self):
+ # in a pretty-printed document the decrypted text lands between the
+ # whitespace that surrounded ; the parent's text must
+ # carry all three pieces
+ root, key = self.encrypt_content(b'secret', 'Password')
+ pretty = etree.fromstring(etree.tostring(root, pretty_print=True))
+ enc_data = xmlsec.tree.find_node(pretty, consts.NodeEncryptedData, consts.EncNs)
+ ctx = xmlsec.EncryptionContext()
+ ctx.key = key
+ password = ctx.decrypt(enc_data)
+ self.assertIs(password, pretty[0])
+ self.assertEqual('\n secret\n ', password.text)
+ self.assertEqual(0, len(password))
+
+ def test_decrypt_content_mixed(self):
+ root, key = self.encrypt_content(b'hello world end', 'Box')
+ pretty = etree.fromstring(etree.tostring(root, pretty_print=True))
+ enc_data = xmlsec.tree.find_node(pretty, consts.NodeEncryptedData, consts.EncNs)
+ ctx = xmlsec.EncryptionContext()
+ ctx.key = key
+ box = ctx.decrypt(enc_data)
+ self.assertEqual(b'\n hello world end\n ', etree.tostring(box, with_tail=False))
+
def test_decrypt_bad_args(self):
ctx = xmlsec.EncryptionContext()
with self.assertRaises(TypeError):
diff --git a/tests/test_templates.py b/tests/test_templates.py
index bbf7f42d..ce9f1d4d 100644
--- a/tests/test_templates.py
+++ b/tests/test_templates.py
@@ -36,6 +36,22 @@ def test_ensure_key_info(self):
ki = xmlsec.template.ensure_key_info(sign, id='Id')
self.assertEqual('Id', ki.get('Id'))
+ def test_ensure_key_info_existing(self):
+ root = self.load_xml('doc.xml')
+ sign = xmlsec.template.create(root, c14n_method=consts.TransformExclC14N, sign_method=consts.TransformRsaSha1)
+ # attach the template the way real callers do; a created template is
+ # detached (on the shadow path it also lives in its own document
+ # until grafted, so liveness is only observable through `root`)
+ root.append(sign)
+ ki = xmlsec.template.ensure_key_info(sign)
+ self.assertIs(ki.getroottree().getroot(), root)
+ # the second call finds the existing node instead of adding another one,
+ # but still applies the requested id to it
+ ki2 = xmlsec.template.ensure_key_info(sign, id='Id')
+ self.assertIs(ki2, ki)
+ self.assertEqual('Id', ki.get('Id'))
+ self.assertEqual(1, sum(1 for n in sign if n.tag == f'{{{consts.DSigNs}}}KeyInfo'))
+
def test_ensure_key_info_fail(self):
with self.assertRaisesRegex(xmlsec.Error, 'cannot ensure key info.'):
xmlsec.template.ensure_key_info(etree.fromstring(b''), id='Id')
@@ -82,6 +98,28 @@ def test_add_reference(self):
for a in ('Id', 'URI', 'Type'):
self.assertEqual(a, ref.get(a))
+ def test_add_reference_beside_an_entity_reference(self):
+ """A graft must count entity references as children, exactly as lxml's insert does (issue #356)."""
+ root = etree.fromstring(b' ]>\n', etree.XMLParser(resolve_entities=False))
+ sign = xmlsec.template.create(root, c14n_method=consts.TransformExclC14N, sign_method=consts.TransformRsaSha1)
+ root.append(sign)
+ sign[0].insert(0, etree.Entity('greet')) # SignedInfo now leads with an entity reference
+ ref = xmlsec.template.add_reference(sign, consts.TransformSha1, uri='#abc')
+ self.assertIs(ref, sign[0][3])
+
+ def test_add_reference_in_a_subtree_removed_from_an_entity_document(self):
+ """The whole-document copy an internal subset forces cannot hold a subtree the document lost (issue #356)."""
+ root = etree.fromstring(
+ b' ]>\n&greet;',
+ etree.XMLParser(resolve_entities=False),
+ )
+ holder = root[1]
+ root.remove(holder)
+ sign = xmlsec.template.create(holder, c14n_method=consts.TransformExclC14N, sign_method=consts.TransformRsaSha1)
+ holder.append(sign)
+ ref = xmlsec.template.add_reference(sign, consts.TransformSha1, uri='#abc')
+ self.assertIs(ref, sign[0][2])
+
def test_add_reference_bad_args(self):
with self.assertRaises(TypeError):
xmlsec.template.add_reference('', consts.TransformSha1)
@@ -92,6 +130,36 @@ def test_add_reference_fail(self):
with self.assertRaisesRegex(xmlsec.Error, 'cannot add reference.'):
xmlsec.template.add_reference(etree.Element('root'), consts.TransformSha1)
+ def test_add_transform(self):
+ root = self.load_xml('doc.xml')
+ sign = xmlsec.template.create(root, c14n_method=consts.TransformExclC14N, sign_method=consts.TransformRsaSha1)
+ # attach the template the way real callers do; a created template is
+ # detached (on the shadow path it also lives in its own document
+ # until grafted, so liveness is only observable through `root`)
+ root.append(sign)
+ ref = xmlsec.template.add_reference(sign, consts.TransformSha1, uri='URI')
+ tr = xmlsec.template.add_transform(ref, consts.TransformEnveloped)
+ # the returned node is live in the original tree, inside the
+ # wrapper the first call creates before
+ self.assertIs(tr.getroottree().getroot(), root)
+ transforms = tr.getparent()
+ self.assertEqual(f'{{{consts.DSigNs}}}Transforms', transforms.tag)
+ self.assertIs(ref[0], transforms)
+ self.assertEqual(f'{{{consts.DSigNs}}}DigestMethod', ref[1].tag)
+
+ def test_add_transform_existing_transforms(self):
+ root = self.load_xml('doc.xml')
+ sign = xmlsec.template.create(root, c14n_method=consts.TransformExclC14N, sign_method=consts.TransformRsaSha1)
+ ref = xmlsec.template.add_reference(sign, consts.TransformSha1, uri='URI')
+ tr = xmlsec.template.add_transform(ref, consts.TransformEnveloped)
+ # a second transform lands in the existing wrapper, after the first
+ tr2 = xmlsec.template.add_transform(ref, consts.TransformExclC14N)
+ transforms = tr.getparent()
+ self.assertIs(tr2.getparent(), transforms)
+ self.assertEqual(2, len(transforms))
+ self.assertIs(transforms[0], tr)
+ self.assertIs(transforms[1], tr2)
+
def test_add_transform_bad_args(self):
with self.assertRaises(TypeError):
xmlsec.template.add_transform('', consts.TransformSha1)
@@ -193,6 +261,19 @@ def test_encrypted_data_ensure_key_info(self):
self.assertEqual('Id', ki2.get('Id'))
self.assertEqual('test', ki2.prefix)
+ def test_encrypted_data_ensure_key_info_rename_prefix(self):
+ root = self.load_xml('doc.xml')
+ enc = xmlsec.template.encrypted_data_create(root, method=consts.TransformDes3Cbc)
+ root.append(enc)
+ xmlsec.template.encrypted_data_ensure_key_info(enc)
+ # renaming the prefix of the KeyInfo that already exists keeps it
+ # live, in place, and unique
+ ki = xmlsec.template.encrypted_data_ensure_key_info(enc, ns='test')
+ self.assertIs(ki.getroottree().getroot(), root)
+ self.assertEqual('test', ki.prefix)
+ self.assertIs(enc[1], ki)
+ self.assertEqual(1, sum(1 for n in enc if n.tag == f'{{{consts.DSigNs}}}KeyInfo'))
+
def test_encrypted_data_ensure_key_info_bad_args(self):
with self.assertRaises(TypeError):
xmlsec.template.encrypted_data_ensure_key_info('')
diff --git a/tests/test_tree.py b/tests/test_tree.py
index 5e80a60a..f40c69b6 100644
--- a/tests/test_tree.py
+++ b/tests/test_tree.py
@@ -1,3 +1,5 @@
+from lxml import etree
+
import xmlsec
from tests import base
@@ -22,6 +24,15 @@ def test_find_parent(self):
self.assertIs(root, xmlsec.tree.find_parent(si, consts.NodeSignature))
self.assertIsNone(xmlsec.tree.find_parent(root, consts.NodeSignedInfo))
+ def test_find_parent_in_a_removed_subtree(self):
+ """The walk upward stops at the top of a subtree removed from its document (issue #356)."""
+ root = self.load_xml('sign1-in.xml')
+ sign = xmlsec.tree.find_node(root, consts.NodeSignature, consts.DSigNs)
+ si = xmlsec.tree.find_child(sign, consts.NodeSignedInfo, consts.DSigNs)
+ sign.getparent().remove(sign)
+ self.assertIs(sign, xmlsec.tree.find_parent(si, consts.NodeSignature, consts.DSigNs))
+ self.assertIsNone(xmlsec.tree.find_parent(si, 'Envelope', 'urn:envelope'))
+
def test_find_parent_bad_args(self):
with self.assertRaises(TypeError):
xmlsec.tree.find_parent('', 0, True)
@@ -40,6 +51,50 @@ def test_add_ids(self):
root = self.load_xml('sign_template.xml')
xmlsec.tree.add_ids(root, ['id1', 'id2', 'id3'])
+ # A document whose entity declarations live in its internal subset, parsed the way callers who
+ # refuse entity expansion do. The subtree cannot be serialized on its own then (issue #356).
+ ENTITY_XML = (
+ b' ]>\n'
+ b''
+ b'&greet;'
+ )
+
+ def load_entity_xml(self):
+ root = etree.fromstring(self.ENTITY_XML, etree.XMLParser(resolve_entities=False))
+ return root, etree.tostring(root.getroottree())
+
+ def test_find_child_keeps_entity_references(self):
+ """The finders must work on a document that declares entities, and leave its references alone."""
+ root, before = self.load_entity_xml()
+ sign = xmlsec.tree.find_child(root, consts.NodeSignature, consts.DSigNs)
+ self.assertEqual(consts.NodeSignature, sign.tag.partition('}')[2])
+ self.assertEqual(before, etree.tostring(root.getroottree()))
+
+ def test_find_node_keeps_entity_references(self):
+ root, before = self.load_entity_xml()
+ si = xmlsec.tree.find_node(root, consts.NodeSignedInfo, consts.DSigNs)
+ self.assertEqual(consts.NodeSignedInfo, si.tag.partition('}')[2])
+ self.assertEqual(before, etree.tostring(root.getroottree()))
+
+ def test_find_parent_keeps_entity_references(self):
+ root, before = self.load_entity_xml()
+ si = xmlsec.tree.find_node(root, consts.NodeSignedInfo, consts.DSigNs)
+ self.assertIs(root[0], xmlsec.tree.find_parent(si, consts.NodeSignature))
+ self.assertEqual(before, etree.tostring(root.getroottree()))
+
+ def test_deeply_nested_document_fails_cleanly(self):
+ """A tree nested deeper than the private copy walks must be refused, never crash (issue #356)."""
+ root = etree.Element('Root')
+ deepest = root
+ for _ in range(3000): # past the 2048 levels libxml2 2.14+ parses, and the walks' own ceiling
+ deepest = etree.SubElement(deepest, 'a')
+ etree.SubElement(deepest, f'{{{consts.DSigNs}}}Signature')
+ try:
+ found = xmlsec.tree.find_node(root, consts.NodeSignature, consts.DSigNs)
+ except xmlsec.InternalError:
+ return # the shadow path refuses a document this deep
+ self.assertEqual(consts.NodeSignature, found.tag.partition('}')[2])
+
def test_add_ids_bad_args(self):
with self.assertRaises(TypeError):
xmlsec.tree.add_ids('', [])