From 452ebf6c7fe0e5a1557f2dcf62e7ddc1a86b303f Mon Sep 17 00:00:00 2001 From: Ray Walker Date: Tue, 29 Sep 2026 15:02:29 +1000 Subject: [PATCH] docs(saas-api): specify cache-key path encoding + path-encoding test vectors (LAB-2879) The spec documented /v1/cache/{key} and its /ttl and /lock sub-resources without saying how {key} is placed in the path. Add a normative Cache-Key Path Encoding section: one percent-encoded segment with unreserved characters raw, client-side rejection of the reserved keys (., .., health, ttl, lock), a single server-side decode, and interop defined on the decoded key. test-vectors/path-encoding.json pins the rules; tools/path-encoding-verify.py checks the reference encoding, the exact reserved set, and the encodeURIComponent alternates, with a mutation self-test run first in verify.yml. cache-key-format.md Server-Side Requirements links the new section and records the empty-key, single-dot and empty-remainder rejects. The feature matrix gains a per-SDK path-encoding row. --- .github/workflows/verify.yml | 3 + README.md | 4 +- changelog.d/20260929_lab-2879.md | 16 ++++ sdk-feature-matrix.md | 7 +- spec/cache-key-format.md | 8 +- spec/saas-api.md | 28 +++++++ test-vectors/path-encoding.json | 105 ++++++++++++++++++++++++++ tools/path-encoding-verify.py | 125 +++++++++++++++++++++++++++++++ 8 files changed, 287 insertions(+), 9 deletions(-) create mode 100644 changelog.d/20260929_lab-2879.md create mode 100644 test-vectors/path-encoding.json create mode 100644 tools/path-encoding-verify.py diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 2727592..ad38f64 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -65,6 +65,9 @@ jobs: - name: File-backend format verify (stdlib only) run: python3 tools/file-backend-reference.py + - name: Cache-key path-encoding verify (stdlib only; mutation self-test first) + run: python3 tools/path-encoding-verify.py + # LAB-1202: the mutation suite runs first so the lz4 allocation guard # cannot silently degrade to reporting OK (same rule as the version-floors # suite below). diff --git a/README.md b/README.md index 97d89bd..047f050 100644 --- a/README.md +++ b/README.md @@ -67,7 +67,7 @@ layer's own store/retrieve flows are specified in | [spec/cache-key-format.md](spec/cache-key-format.md) | Cache key generation algorithm — Blake2b-256, argument normalization, cross-SDK key strategy | | [spec/wire-format.md](spec/wire-format.md) | ByteStorage envelope — LZ4 block compression, xxHash3-64 integrity, decompression bomb protection | | [spec/encryption.md](spec/encryption.md) | AES-256-GCM encryption, HKDF-SHA256 key derivation, AAD v0x03, counter-based nonces, key rotation | -| [spec/saas-api.md](spec/saas-api.md) | REST API endpoints, binary wire protocol, error codes, metrics headers | +| [spec/saas-api.md](spec/saas-api.md) | REST API endpoints, cache-key path encoding, binary wire protocol, error codes, metrics headers | | [spec/interop-mode.md](spec/interop-mode.md) | Cross-SDK cache sharing — language-neutral key format, canonical argument normalization *(normative; shipped opt-in in all three SDKs — see the [feature matrix](sdk-feature-matrix.md#compliance-status) for per-SDK version floors)* | | [spec/interop-v2.md](spec/interop-v2.md) | Interop v2 compressed-values profile — opt-in LZ4-block + AES-256-GCM cross-SDK values *(DRAFT; no SDK implements it yet)* | | [spec/file-backend-format.md](spec/file-backend-format.md) | Shared local File backend filename, header, expiry, and fail-closed flag negotiation | @@ -127,7 +127,7 @@ An SDK is protocol-compliant when: 1. [Interop-mode](spec/interop-mode.md) key generation produces identical keys for identical inputs across all languages (auto-mode keys embed language-specific function identity and are not cross-SDK by design) 2. Interop-mode values encode and decode per the canonical vectors; where the SDK implements the ByteStorage envelope, it deserializes any spec-conformant envelope 3. Encrypted interop-mode payloads can be decrypted by any SDK with the same master key and tenant ID -4. SaaS API integration follows the documented endpoint contracts +4. SaaS API integration follows the documented endpoint contracts, including [cache-key path encoding](spec/saas-api.md#cache-key-path-encoding) (`test-vectors/path-encoding.json`) Test vectors are published in [`test-vectors/`](test-vectors/) as JSON files. diff --git a/changelog.d/20260929_lab-2879.md b/changelog.d/20260929_lab-2879.md new file mode 100644 index 0000000..cb1d656 --- /dev/null +++ b/changelog.d/20260929_lab-2879.md @@ -0,0 +1,16 @@ +### SaaS API — cache-key path encoding specified (LAB-2879) + +- [`spec/saas-api.md`](spec/saas-api.md#cache-key-path-encoding) gains a normative + **Cache-Key Path Encoding** section: how a cache key is carried as the `{key}` path + segment of `/v1/cache/{key}`, `/ttl` and `/lock`, including the reserved keys clients + must reject before building the URL. +- New [`test-vectors/path-encoding.json`](test-vectors/path-encoding.json) pins those rules + as `key → encoded → decoded` rows, CI-verified by + [`tools/path-encoding-verify.py`](tools/path-encoding-verify.py) (stdlib, mutation + self-test first). +- [`spec/cache-key-format.md`](spec/cache-key-format.md#server-side-requirements) + Server-Side Requirements now links the path-encoding section and records three rejects + the server already enforces: an empty key, a key of exactly `.`, and a namespaced key + with an empty remainder. +- [`sdk-feature-matrix.md`](sdk-feature-matrix.md) Compliance Status gains a path-encoding + row. diff --git a/sdk-feature-matrix.md b/sdk-feature-matrix.md index 13d79ab..1ec21f2 100644 --- a/sdk-feature-matrix.md +++ b/sdk-feature-matrix.md @@ -297,8 +297,9 @@ its spec: | Storage container (auto mode)¹⁵ | CK v3 frame (Python-internal) | Plain MessagePack (`rmp` named) — no envelope | Bare ByteStorage envelope (default) | — | | Encryption (AES-256-GCM) | ✅ Compliant | ✅ Canonical (cachekit-core) | ✅ Compliant | ⚠️ Untested | | AAD v0x03 | ✅ Compliant (5 components — every auto serializer appends `original_type`; interop mode is the sole 4-component path) | ✅ Compliant (4 components) | ✅ Compliant (4 components) | ❌ Not implemented | -| SaaS API | ✅ Compliant | ✅ Compliant (CachekitIO backend) | ✅ Compliant | ❌ Not implemented | -| Test vectors in CI¹⁶ | ✅ interop/v1 (full set, incl. AAD + encryption through the real stack) — fixture 1.1.0 (`ns`/`nsapi` namespace reservation) in [cachekit-py#350](https://github.com/cachekit-io/cachekit-py/pull/350), unreleased; `decode-bounds.json` vendored + CI-executed since [cachekit-py#276](https://github.com/cachekit-io/cachekit-py/pull/276) (LAB-2503) — ⚠️ asserts rejection and a peak-memory budget, not that the structural guard rejected (the [Decode bounds](spec/interop-mode.md#decode-bounds) MUST) | ✅ interop/v1 (full set) since [#33](https://github.com/cachekit-io/cachekit-rs/pull/33) — fixture 1.1.0 in [cachekit-rs#89](https://github.com/cachekit-io/cachekit-rs/pull/89), unreleased; `decode-bounds.json` vendored + CI-executed since [cachekit-rs#73](https://github.com/cachekit-io/cachekit-rs/pull/73) (LAB-2503; default CI green on `main`) — ⚠️ asserts the error type only, not that the structural guard rejected | ✅ interop/v1 (full set, incl. its key vectors) + inline Python-generated AAD-construction and encryption (decrypt-Python-ciphertext) vectors — fixture 1.1.0 in [cachekit-ts#143](https://github.com/cachekit-io/cachekit-ts/pull/143), unreleased; decode bounds enforced ([#112](https://github.com/cachekit-io/cachekit-ts/pull/112)); `decode-bounds.json` vendored + CI-executed since [cachekit-ts#121](https://github.com/cachekit-io/cachekit-ts/pull/121) (LAB-2737), asserting the guard error each vector trips (pre-scan, or the event size cap ahead of it) — ⚠️ except the envelope entry point (`cachekit-core-ts` `unpack`), which has no guard to assert (see the Wire format row) | ⚠️ Pending | +| SaaS API | ✅ Compliant except path encoding (row below) | ✅ Compliant (CachekitIO backend) | ✅ Compliant | ❌ Not implemented | +| SaaS API — cache-key path encoding ([spec](spec/saas-api.md#cache-key-path-encoding)) | ⚠️ Partial — rules 1/3/4 since 0.18.0+ ([cachekit-py#279](https://github.com/cachekit-io/cachekit-py/pull/279)); rule 2 not implemented | ✅ Compliant on `main`, unreleased ([cachekit-rs#76](https://github.com/cachekit-io/cachekit-rs/pull/76)) | ✅ Compliant on `main`, unreleased ([cachekit-ts#118](https://github.com/cachekit-io/cachekit-ts/pull/118)) | ❌ Not implemented | +| Test vectors in CI¹⁶ | ✅ interop/v1 (full set, incl. AAD + encryption through the real stack) — fixture 1.1.0 (`ns`/`nsapi` namespace reservation) in [cachekit-py#350](https://github.com/cachekit-io/cachekit-py/pull/350), unreleased; `decode-bounds.json` vendored + CI-executed since [cachekit-py#276](https://github.com/cachekit-io/cachekit-py/pull/276) (LAB-2503) — ⚠️ asserts rejection and a peak-memory budget, not that the structural guard rejected (the [Decode bounds](spec/interop-mode.md#decode-bounds) MUST) | ✅ interop/v1 (full set) since [#33](https://github.com/cachekit-io/cachekit-rs/pull/33) — fixture 1.1.0 in [cachekit-rs#89](https://github.com/cachekit-io/cachekit-rs/pull/89), unreleased; `decode-bounds.json` vendored + CI-executed since [cachekit-rs#73](https://github.com/cachekit-io/cachekit-rs/pull/73) (LAB-2503; default CI green on `main`) — ⚠️ asserts the error type only, not that the structural guard rejected | ✅ interop/v1 (full set, incl. its key vectors) + inline Python-generated AAD-construction and encryption (decrypt-Python-ciphertext) vectors — fixture 1.1.0 in [cachekit-ts#143](https://github.com/cachekit-io/cachekit-ts/pull/143), unreleased; decode bounds enforced ([#112](https://github.com/cachekit-io/cachekit-ts/pull/112)); `decode-bounds.json` vendored + CI-executed since [cachekit-ts#121](https://github.com/cachekit-io/cachekit-ts/pull/121) (LAB-2737), asserting the guard error each vector trips (pre-scan, or the event size cap ahead of it) — ⚠️ except the envelope entry point (`cachekit-core-ts` `unpack`), which has no guard to assert (see the Wire format row); `path-encoding.json` vendored + CI-executed since [cachekit-ts#118](https://github.com/cachekit-io/cachekit-ts/pull/118), unreleased | ⚠️ Pending | | Interop mode ([spec](spec/interop-mode.md), opt-in) | ✅ Released — PyPI 0.14.0+¹⁷ ([#220](https://github.com/cachekit-io/cachekit-py/pull/220)) | ✅ Released — crates.io 0.4.0+ ([#33](https://github.com/cachekit-io/cachekit-rs/pull/33)) | ✅ Released — npm 0.1.3+ ([#71](https://github.com/cachekit-io/cachekit-ts/pull/71)) | ❌ Not implemented | > [!NOTE] @@ -306,7 +307,7 @@ its spec: > > ¹⁵ Auto-mode **stored bytes** are SDK-internal and differ per SDK — see [wire-format.md → SDK Storage Containers](spec/wire-format.md#sdk-storage-containers-auto-mode). Python stores the ByteStorage envelope *inside* its CK v3 frame; `cachekit-rs` does not use the envelope for values at all (it uses `cachekit-core` only for encryption). Cross-SDK value compatibility is exclusively an [interop-mode](spec/interop-mode.md) property (protocol#11). > -> ¹⁶ "Test vectors in CI" = vectors the SDK's own default CI executes. Beyond the SDKs, this repo's `verify.yml` CI-verifies `interop-mode.json`, `encryption.json`, `python-frame.json`, `file-backend.json` ([`tools/file-backend-reference.py`](tools/file-backend-reference.py)), and — since LAB-423 — `wire-format.json` ([`tools/wire-format-reference.py`](tools/wire-format-reference.py)), and — since LAB-2503 — `decode-bounds.json` ([`tools/decode-bounds-reference.py`](tools/decode-bounds-reference.py), `verify` in both the stdlib and the optional-deps legs) against reference implementations. The SDKs vendor earlier revisions of `decode-bounds.json`, all labelled `1.0.0`: cachekit-py and cachekit-ts the 13-reject / 2-accept revision, cachekit-rs a 10-reject / 2-accept one. This repo's file is `1.1.0` (17 reject, 3 accept); vectors newer than an SDK's copy run only here until that SDK re-vendors. `cache-keys.json` (regenerated by cachekit-py v0.12.0, byte-identical to the v0.5.0 originals) is vendored and CI-verified in cachekit-py since [cachekit-py#229](https://github.com/cachekit-io/cachekit-py/pull/229) (LAB-425). +> ¹⁶ "Test vectors in CI" = vectors the SDK's own default CI executes. Beyond the SDKs, this repo's `verify.yml` CI-verifies `interop-mode.json`, `encryption.json`, `python-frame.json`, `file-backend.json` ([`tools/file-backend-reference.py`](tools/file-backend-reference.py)), and — since LAB-423 — `wire-format.json` ([`tools/wire-format-reference.py`](tools/wire-format-reference.py)), and — since LAB-2503 — `decode-bounds.json` ([`tools/decode-bounds-reference.py`](tools/decode-bounds-reference.py), `verify` in both the stdlib and the optional-deps legs), and — since LAB-2879 — `path-encoding.json` ([`tools/path-encoding-verify.py`](tools/path-encoding-verify.py)) against reference implementations. The SDKs vendor earlier revisions of `decode-bounds.json`, all labelled `1.0.0`: cachekit-py and cachekit-ts the 13-reject / 2-accept revision, cachekit-rs a 10-reject / 2-accept one. This repo's file is `1.1.0` (17 reject, 3 accept); vectors newer than an SDK's copy run only here until that SDK re-vendors. `cache-keys.json` (regenerated by cachekit-py v0.12.0, byte-identical to the v0.5.0 originals) is vendored and CI-verified in cachekit-py since [cachekit-py#229](https://github.com/cachekit-io/cachekit-py/pull/229) (LAB-425). > > ¹⁷ Version cells are **floors** (`X+`), not snapshots — they stay true as new versions publish; check the registry for the current release. Python's floor is the first *installable* one: interop merged under the `v0.13.0` tag, but neither `0.12.0` nor `0.13.0` was ever published to PyPI, so `0.14.0` is the earliest PyPI release containing interop mode. Do not "correct" this to 0.13.0 from the cachekit-py changelog alone. diff --git a/spec/cache-key-format.md b/spec/cache-key-format.md index 3eb6c07..e8adf8f 100644 --- a/spec/cache-key-format.md +++ b/spec/cache-key-format.md @@ -185,11 +185,11 @@ enforces is security-relevant (per `saas` issue #91 / SRP refactor): | Check | Rule | | :--- | :--- | -| Transport | Key is percent-encoded into the URL path; the server decodes it once. | -| Length | Decoded key ≤ 400 characters. | +| Transport | Key is percent-encoded into one URL path segment; the server decodes it once. See [saas-api.md → Cache-Key Path Encoding](saas-api.md#cache-key-path-encoding). | +| Length | Decoded key is 1–400 characters. | | Charset | `[a-zA-Z0-9_.:-]` only — no `/` (sub-resource routing), no `%`, no control chars. | -| Traversal | `..` is rejected anywhere in the key. | -| Namespace | Keys starting `ns:{namespace}:` or `nsapi:{namespace}:` must have a namespace of 1–64 chars of `[a-zA-Z0-9_-]`. Keys without either prefix scope to the `default` namespace. | +| Traversal | `..` is rejected anywhere in the key, and so is a key of exactly `.`. | +| Namespace | A key starting `ns:` or `nsapi:` must be `{prefix}:{namespace}:{rest}`, with a namespace of 1–64 chars of `[a-zA-Z0-9_-]` and a non-empty `{rest}`; any other key with either prefix is rejected. Keys without either prefix scope to the `default` namespace. | | Write spaces | `ns:` keys are mutable only by SDK (`ck_sdk_`) API keys; `nsapi:` keys only by direct (`ck_api_`) API keys. Reads are open to both. Legacy `ck_live_` keys predate the split and are exempt from it — they may write either class. No server-side retirement date is set for `ck_live_`. | | Default namespace | Keys with neither prefix (TypeScript/Rust `{ns}:{hash}`, [Interop Mode](interop-mode.md) keys, bare hashes) are an **open** write space: any key class may write them, so the intra-tenant write-space isolation above does not protect them. Per-key namespace grants still apply — an API key restricted to named namespaces must include `default` to read or write unprefixed keys. | diff --git a/spec/saas-api.md b/spec/saas-api.md index 23b142f..5484a95 100644 --- a/spec/saas-api.md +++ b/spec/saas-api.md @@ -17,6 +17,7 @@ - [Overview](#overview) - [Authentication](#authentication) - [Content Type](#content-type) +- [Cache-Key Path Encoding](#cache-key-path-encoding) - [Cache Endpoints](#cache-endpoints) - [Stale-While-Revalidate](#stale-while-revalidate) - [Lock Endpoints](#lock-endpoints) @@ -81,6 +82,33 @@ Content-Type: application/octet-stream --- +## Cache-Key Path Encoding + +Every endpoint below carries the cache key as a path segment — `/v1/cache/{key}`, `/v1/cache/{key}/ttl`, `/v1/cache/{key}/lock`. The key is caller-controlled (each SDK's `key=` escape hatch accepts an arbitrary string), so how it is placed in the path is a security boundary, not a formatting detail: an unencoded key can escape `/v1/cache/` and deliver the bearer token to a different route (CWE-22 — cachekit-py shipped exactly that until [cachekit-py#279](https://github.com/cachekit-io/cachekit-py/pull/279)). MUST, MUST NOT, SHOULD and MAY are used as in RFC 2119. + +### Encoding rules + +**1. One segment, percent-encoded.** `{key}` MUST be exactly one path segment. Clients MUST percent-encode the key's UTF-8 bytes (RFC 3986 §2.1) so that only unreserved characters — `ALPHA / DIGIT / "-" / "." / "_" / "~"` — appear raw, and MUST NOT percent-encode an unreserved character (RFC 3986 §2.3): `.` is sent as `.`, never `%2E`. Every other byte MUST be sent as `%HH` — the delimiters `/ ? # %`, `:` (a canonical key carries six), space (`%20`, never `+`), every byte ≥ `0x80` — with one tolerance: the sub-delims `! * ' ( )` MAY be left raw as a set, all five raw (the `encodeURIComponent` form) or all five encoded, never a mix (rule 4). Hex digits SHOULD be uppercase (RFC 3986 §2.1); the server decodes either case. Reference encoders: Python `urllib.parse.quote(key, safe="")`, Rust `urlencoding::encode`, JavaScript `encodeURIComponent`. + +**2. Reserved segments MUST be rejected client-side.** A key of exactly `.` or `..` survives rule 1 unchanged (`.` is unreserved) and is a *dot segment*: URL parsers remove it before routing — `/v1/cache/..` becomes `/v1/`, `/v1/cache/../ttl` becomes `/v1/ttl` — so the request lands on a different route, still carrying `Authorization`, and never reaches the key validator. Percent-encoding the dots does not help. The server parses the request URL under the WHATWG URL Standard, which treats an ASCII-case-insensitive `%2e` as a single-dot segment and `%2e%2e`, `.%2e`, `%2e.` as double-dot segments (URL Standard §4.1), so `%2E%2E` is collapsed *server-side* even when the client's own parser (RFC 3986 §5.2.4, e.g. `httpx`) sent it intact; WHATWG clients (`fetch`/undici, browsers, the Workers runtime, rust-url and therefore `reqwest`) collapse it before sending. **No wire form of a `.` or `..` key reaches the validator from any client.** The literal segments `health`, `ttl` and `lock` are route tokens at this level — `/v1/cache/health` is the health endpoint, and a final `ttl` or `lock` segment selects the sub-resource — so a key equal to one of those words is routed elsewhere or read as an empty key. + +Therefore clients MUST reject a key that is exactly `.`, `..`, `health`, `ttl` or `lock` before building the URL, surfacing a client-side error; servers MUST NOT be relied on to compensate. Under rule 1 each of these keys encodes to itself, so checking the key and checking its encoding are the same test. Only an *entirely*-dot segment is a dot segment: `a:..`, `..a`, `x..y` are inert and MUST be sent per rule 1 with their dots raw. Canonical and interop keys always contain `:` and never meet this rule. Because every conformant client hard-codes this reserved set, servers MUST NOT add a route under `/v1/cache/` beyond `{key}`, `{key}/ttl`, `{key}/lock` and `health` without a protocol version bump. + +Conformance tests MUST assert that every `reject: true` vector raises before a URL is built. They MUST assert every transmittable vector on the *parsed* request path (`new URL(u).pathname`, `Url::parse(u)?.path()`, `httpx.Request.url.raw_path`), not on the un-parsed template string — a template-string test passes while the traversal ships. + +**3. The server decodes exactly once.** After the WHATWG parse of rule 2, the server splits the path on raw `/`, then percent-decodes the key segment once (`decodeURIComponent`-equivalent; a malformed escape is `400 Bad Request`) and validates the *decoded* key against the key format's [Server-Side Requirements](cache-key-format.md#server-side-requirements): a key failing the Length, Charset, Traversal or Namespace check is `400 Bad Request`; a write-space or namespace-grant violation is `403 Forbidden` ([Authentication](#authentication)). Consequences clients MUST honour: + +- Clients MUST NOT double-encode. A literal `%` in a key is sent as `%25` once; `%2525` decodes to `%25`, a different key. +- An encoded `%2F` never becomes a segment boundary: the split on raw `/` happens *before* decoding, so `a%2Fb` reaches the validator as `a/b` and is rejected by the charset rule. A conformant client can neither traverse nor store a key containing `/`. + +**4. Interop is defined on the decoded key.** `encodeURIComponent` leaves the sub-delims `! * ' ( )` raw (legal `pchar` in a path segment; they decode to themselves); `quote(safe="")` and `urlencoding::encode` emit `%21 %2A %27 %28 %29`. Both forms are conformant because the server-side key is identical after the single decode. Cross-SDK key equality is therefore a property of the **decoded** key, not of the wire bytes in general — but every key the server accepts is drawn from `[A-Za-z0-9_.:-]`, on which all three reference encoders agree (`:` → `%3A`, the rest raw). Every canonical auto-mode key and every [interop-mode](interop-mode.md) key is thus byte-identical on the wire across SDKs; the variance set only ever appears in keys the server rejects. + +### Test vectors + +[`test-vectors/path-encoding.json`](../test-vectors/path-encoding.json) pins these rules as `key → encoded → decoded` rows; its `contract` field defines the row semantics and travels with every vendored copy. + +--- + ## Cache Endpoints All cache endpoints are prefixed with `/v1/cache/`. diff --git a/test-vectors/path-encoding.json b/test-vectors/path-encoding.json new file mode 100644 index 0000000..70b8f51 --- /dev/null +++ b/test-vectors/path-encoding.json @@ -0,0 +1,105 @@ +{ + "version": "1.0.0", + "generator": "urllib.parse.quote(key, safe=\"\")", + "spec": "spec/saas-api.md § Cache-Key Path Encoding", + "ci_verification": "tools/path-encoding-verify.py (stdlib only; mutation self-test first)", + "contract": "`encoded` is the single `{key}` path segment in the reference form (spec rule 1); `decoded` is the key the server sees after its single percent-decode and equals `key` in every transmittable row (spec rules 3-4). Rows with `reject: true` are the reserved segments of spec rule 2 (`.`, `..`, `health`, `ttl`, `lock`): a conformant client raises before building the URL, so `encoded` and `decoded` are null. `encoded_alternates` lists the `encodeURIComponent` form (`! * ' ( )` raw) where it differs from `encoded`, and is absent otherwise; reject rows carry none. An SDK conforms when its encoding of `key` is in `[encoded] + encoded_alternates` (a missing `encoded_alternates` is `[]`).", + "vectors": [ + { + "key": "ns:test:func:__main__.get_user:args:3870b2ea5735ae639ded9450ef117768db676f037bec636503796c5b81095153:1s", + "encoded": "ns%3Atest%3Afunc%3A__main__.get_user%3Aargs%3A3870b2ea5735ae639ded9450ef117768db676f037bec636503796c5b81095153%3A1s", + "decoded": "ns:test:func:__main__.get_user:args:3870b2ea5735ae639ded9450ef117768db676f037bec636503796c5b81095153:1s", + "note": "Canonical 7-segment auto-mode key (test-vectors/cache-keys.json `single_integer`). Only `:` is encoded; identical bytes from all three reference encoders." + }, + { + "key": "default:../../admin", + "encoded": "default%3A..%2F..%2Fadmin", + "decoded": "default:../../admin", + "note": "Embedded traversal: every `/` is `%2F`, so no `../` boundary exists for a URL parser to collapse. Server decodes once, then rejects (`/` outside charset; `..` substring)." + }, + { + "key": "x/../../health", + "encoded": "x%2F..%2F..%2Fhealth", + "decoded": "x/../../health", + "note": "Traversal aimed at the `/v1/cache/health` route token; inert once `/` is `%2F`. Server rejects the decoded key (charset)." + }, + { + "key": "k?x=1#f", + "encoded": "k%3Fx%3D1%23f", + "decoded": "k?x=1#f", + "note": "Query/fragment injection: `?` and `#` MUST be encoded or the client's URL parser truncates the key and emits a query string. Server rejects the decoded key (charset)." + }, + { + "key": "a b", + "encoded": "a%20b", + "decoded": "a b", + "note": "Space is `%20` in a path segment, never `+` (form-encoding, which the server does not decode). Server rejects the decoded key (charset)." + }, + { + "key": "100%", + "encoded": "100%25", + "decoded": "100%", + "note": "A literal `%` is encoded exactly once; the server decodes once and sees `100%`, then rejects it (charset). `%2525` would decode to `100%25`, a different key." + }, + { + "key": ".", + "encoded": null, + "decoded": null, + "reject": true, + "note": "Dot segment: the plain encoding `.` is removed by every URL parser, and `%2E` is collapsed by the server's WHATWG parse (`/v1/cache/%2E` → `/v1/cache/`). No wire form reaches the validator." + }, + { + "key": "..", + "encoded": null, + "decoded": null, + "reject": true, + "note": "Dot segment: unencoded, `/v1/cache/..` collapses to `/v1/` and `/v1/cache/../ttl` to `/v1/ttl`; `%2E%2E` is collapsed the same way by the server's WHATWG parse. No wire form reaches the validator." + }, + { + "key": "health", + "encoded": null, + "decoded": null, + "reject": true, + "note": "Route token: `/v1/cache/health` is the health endpoint, so a GET for this key would return the health payload as a cache hit." + }, + { + "key": "ttl", + "encoded": null, + "decoded": null, + "reject": true, + "note": "Route token: a final `ttl` segment selects the TTL sub-resource, so `/v1/cache/ttl` is read as an empty key plus `/ttl`." + }, + { + "key": "lock", + "encoded": null, + "decoded": null, + "reject": true, + "note": "Route token: a final `lock` segment selects the lock sub-resource, so `/v1/cache/lock` is read as an empty key plus `/lock`." + }, + { + "key": "a:..", + "encoded": "a%3A..", + "decoded": "a:..", + "note": "Trailing dots but NOT an all-dot segment: not a dot segment under either parsing model, so the dots stay raw. Server rejects the decoded `..` substring." + }, + { + "key": "..a", + "encoded": "..a", + "decoded": "..a", + "note": "Leading dots, not an all-dot segment: sent verbatim (all characters unreserved). Server rejects the decoded `..` substring." + }, + { + "key": "ns:key", + "encoded": "ns%3Akey", + "decoded": "ns:key", + "note": "`:` → `%3A`, decoded once server-side. The server then rejects it: a key starting `ns:` must be `ns:{namespace}:{rest}` with a non-empty `{rest}` (spec/cache-key-format.md § Server-Side Requirements, Namespace), and this one has no `{rest}`." + }, + { + "key": "f(x)!*'", + "encoded": "f%28x%29%21%2A%27", + "decoded": "f(x)!*'", + "encoded_alternates": ["f(x)!*'"], + "note": "Encoder-variance row (spec rule 4): `quote(safe=\"\")` and `urlencoding::encode` produce `encoded`; `encodeURIComponent` produces the alternate. Both decode to `key`. Server rejects the decoded key (charset)." + } + ] +} diff --git a/tools/path-encoding-verify.py b/tools/path-encoding-verify.py new file mode 100644 index 0000000..b2b2195 --- /dev/null +++ b/tools/path-encoding-verify.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +"""Validate test-vectors/path-encoding.json with Python stdlib. + +spec/saas-api.md § Cache-Key Path Encoding. A transmittable row's `encoded` must be +the reference form (`quote(key, safe="")`) and decode once back to `key`; its +`encoded_alternates` must be exactly the `encodeURIComponent` form where that differs. +The reject rows must be exactly the reserved keys of spec rule 2, with no wire form. A +mutation self-test runs first so the guard cannot degrade to silently reporting OK, and +each mutation must trip the guard it names (same doctrine as +tools/test_wire_format_reference.py). +""" + +from __future__ import annotations + +import copy +import json +import logging +import sys +from collections.abc import Callable +from pathlib import Path +from urllib.parse import quote + +ROOT = Path(__file__).resolve().parents[1] +VECTORS = ROOT / "test-vectors" / "path-encoding.json" + +# spec rule 2: `.`/`..` are dot segments; `health`/`ttl`/`lock` are route tokens at the +# `/v1/cache/` level. Each encodes to itself under rule 1. +RESERVED_SEGMENTS = {".", "..", "health", "ttl", "lock"} +# The sub-delims encodeURIComponent leaves raw (spec rule 4); quote() with these as safe +# is byte-for-byte encodeURIComponent. +ENCODE_URI_COMPONENT_SAFE = "!*'()" + + +def check(condition: bool, name: str, detail: str) -> None: + """Fail closed even under ``python -O`` (asserts would be stripped).""" + if not condition: + raise ValueError(f"{name}: {detail}") + + +def verify(document: dict) -> int: + vectors = document["vectors"] + keys = [vector["key"] for vector in vectors] + check(len(keys) == len(set(keys)), "vectors", "duplicate key") + for vector in vectors: + check(isinstance(vector.get("reject", False), bool), repr(vector["key"]), "reject must be absent or a bool") + rejected = {vector["key"] for vector in vectors if vector.get("reject") is True} + check(rejected == RESERVED_SEGMENTS, "vectors", f"reject rows {sorted(rejected)} != reserved set {sorted(RESERVED_SEGMENTS)}") + + for vector in vectors: + key = vector["key"] + name = repr(key) + if key in RESERVED_SEGMENTS: + check(vector["encoded"] is None and vector["decoded"] is None, name, "reject row carries a wire form") + check("encoded_alternates" not in vector, name, "reject row carries encoded_alternates") + continue + reference = quote(key, safe="") + check(vector["decoded"] == key, name, "decoded != key (interop is defined on the decoded key)") + check(vector["encoded"] == reference, name, f"encoded {vector['encoded']!r} != reference {reference!r}") + uri_component = quote(key, safe=ENCODE_URI_COMPONENT_SAFE) + expected = [uri_component] if uri_component != reference else [] + alternates = vector.get("encoded_alternates", []) + check(alternates == expected, name, f"encoded_alternates {alternates!r} != encodeURIComponent form {expected!r}") + return len(vectors) + + +def self_test(document: dict) -> None: + """Each poisoned copy must trip the guard it names — otherwise verify() is toothless.""" + + def row(vectors: list, key: str) -> dict: + return next(v for v in vectors if v["key"] == key) + + def set_field(key: str, field: str, value: object) -> Callable[[list], None]: + return lambda v: row(v, key).__setitem__(field, value) + + def drop_row(key: str) -> Callable[[list], None]: + return lambda v: v.remove(row(v, key)) + + # label: (poison, substring the tripped guard's message must contain) + mutations = { + "encoded drift": (set_field("x/../../health", "encoded", "x/..%2F..%2Fhealth"), "!= reference"), + "decoded drift": (set_field("ns:key", "decoded", "ns:kex"), "decoded != key"), + "duplicate key": (lambda v: v.append(copy.deepcopy(row(v, "ns:key"))), "duplicate key"), + "reject row dropped": (drop_row("health"), "!= reserved set"), + "reject flag on transmittable key": (set_field("a:..", "reject", True), "!= reserved set"), + "non-bool reject flag": (set_field("a:..", "reject", 1), "absent or a bool"), + "reserved key not flagged": (set_field("..", "reject", False), "!= reserved set"), + "reject row with wire form": (set_field("..", "encoded", "%2E%2E"), "carries a wire form"), + "reject row with alternates": (set_field("lock", "encoded_alternates", ["lock"]), "carries encoded_alternates"), + "alternate over-encoded": (set_field("f(x)!*'", "encoded_alternates", ["%66(x)!*'"]), "!= encodeURIComponent form"), + "alternate missing": (set_field("f(x)!*'", "encoded_alternates", []), "!= encodeURIComponent form"), + "alternate where none differs": (set_field("ns:key", "encoded_alternates", ["ns%3Akey"]), "!= encodeURIComponent form"), + } + for label, (mutate, expected) in mutations.items(): + poisoned = copy.deepcopy(document) + mutate(poisoned["vectors"]) + try: + verify(poisoned) + except ValueError as exc: + check(expected in str(exc), "self-test", f"mutation {label!r} tripped the wrong guard: {exc}") + continue + raise ValueError(f"self-test: mutation {label!r} was not rejected") + + +def main() -> None: + try: + document = json.loads(VECTORS.read_text(encoding="utf-8")) + except OSError as exc: + sys.exit(f"cannot read {VECTORS}: {exc}") + except json.JSONDecodeError as exc: + sys.exit(f"invalid JSON in {VECTORS}: {exc}") + + try: + self_test(document) + count = verify(document) + except ValueError as exc: + sys.exit(f"invalid vector file: {exc}") + except (KeyError, TypeError, AttributeError, StopIteration) as exc: + sys.exit(f"invalid vector file: malformed structure ({exc!r})") + + logging.info("validated %d path-encoding vectors (self-test passed)", count) + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO, format="%(message)s") + main()