From eac9b1b000c8faa56c19cc4cd9df83ba1ad688e9 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Wed, 16 Sep 2026 03:00:08 +0000 Subject: [PATCH 1/4] feat(reporting): persist managed delivery reconciliation records --- README.md | 2 +- docs/reporting-ledger-migration.md | 17 +- docs/reporting-reconciliation-storage.md | 184 ++++++ src/adcp/reporting/evidence.py | 215 +++++++ src/adcp/reporting/ledger/__init__.py | 73 +++ src/adcp/reporting/ledger/_delivery_state.py | 594 ++++++++++++++++++ src/adcp/reporting/ledger/delivery.py | 502 +++++++++++++++ src/adcp/reporting/ledger/delivery_models.py | 476 ++++++++++++++ src/adcp/reporting/ledger/delivery_pg.py | 286 +++++++++ src/adcp/reporting/ledger/models.py | 51 +- src/adcp/reporting/ledger/pg.py | 116 +++- src/adcp/reporting/ledger/producer.py | 18 +- .../reporting/ledger/reporting_ledger.sql | 2 +- .../reporting_ledger_reconciliation.sql | 169 +++++ src/adcp/reporting/ledger/status.py | 2 + src/adcp/reporting/ledger/store.py | 106 +++- .../reporting/_reconciliation_support.py | 239 +++++++ tests/conformance/reporting/conftest.py | 3 + .../test_reporting_currency_migration.py | 9 +- .../test_reporting_generation_migration.py | 15 +- ...est_reporting_reconciliation_boundaries.py | 556 ++++++++++++++++ ...test_reporting_reconciliation_migration.py | 272 ++++++++ .../test_reporting_reconciliation_store.py | 403 ++++++++++++ ...st_reporting_reconciliation_transitions.py | 391 ++++++++++++ tests/fixtures/reporting_ledger_1171.sql | 295 +++++++++ .../reporting_reconciliation_records.py | 125 ++++ 26 files changed, 5066 insertions(+), 55 deletions(-) create mode 100644 docs/reporting-reconciliation-storage.md create mode 100644 src/adcp/reporting/evidence.py create mode 100644 src/adcp/reporting/ledger/_delivery_state.py create mode 100644 src/adcp/reporting/ledger/delivery.py create mode 100644 src/adcp/reporting/ledger/delivery_models.py create mode 100644 src/adcp/reporting/ledger/delivery_pg.py create mode 100644 src/adcp/reporting/ledger/reporting_ledger_reconciliation.sql create mode 100644 tests/conformance/reporting/_reconciliation_support.py create mode 100644 tests/conformance/reporting/conftest.py create mode 100644 tests/conformance/reporting/test_reporting_reconciliation_boundaries.py create mode 100644 tests/conformance/reporting/test_reporting_reconciliation_migration.py create mode 100644 tests/conformance/reporting/test_reporting_reconciliation_store.py create mode 100644 tests/conformance/reporting/test_reporting_reconciliation_transitions.py create mode 100644 tests/fixtures/reporting_ledger_1171.sql create mode 100644 tests/type_checks/reporting_reconciliation_records.py diff --git a/README.md b/README.md index 9fc00e55a..a6716bca1 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ This README serves both sides of an AdCP integration. Jump to what you're doing: - **Connect as a buyer** → [Quick Start: Test Helpers](#quick-start-test-helpers) and [Quick Start: Distributed Operations](#quick-start-distributed-operations). Entry point: `from adcp import ADCPClient, AgentConfig`; start with the `client.simple.*` API. - **Build a seller / agent** → [Building an AdCP Agent](#building-an-adcp-agent). Entry point: `from adcp.server import ADCPHandler, serve`; use the [production seller path](docs/production-seller.md) when adding tenants, durable tasks, and webhooks. -- **Run Reliable Reporting** → [Account currencies](docs/reporting-currency.md) and [ledger migrations](docs/reporting-ledger-migration.md). +- **Run Reliable Reporting** → [Account currencies](docs/reporting-currency.md), [ledger migrations](docs/reporting-ledger-migration.md), and the [optional reconciliation storage contract](docs/reporting-reconciliation-storage.md). - **Understand the type system & imports** → [Type Safety](#type-safety) (import surface, partial modules, cold-start note). - **Test against reference agents** → [Quick Start: Test Helpers](#quick-start-test-helpers) and [Test Helpers](#test-helpers). Entry point: `from adcp.testing import test_agent, creative_agent`. diff --git a/docs/reporting-ledger-migration.md b/docs/reporting-ledger-migration.md index dd5c91c61..9a3fa7cf9 100644 --- a/docs/reporting-ledger-migration.md +++ b/docs/reporting-ledger-migration.md @@ -49,27 +49,34 @@ obligation IDs, the named obligations must exist in the requested account. mixing old and new workers is unsafe once accounts reuse a config ID. 2. With the upgraded SDK, run `await store.create_schema()` before starting reporting work. It creates missing tables and applies the bundled - `reporting_ledger_account_generations.sql` and - `reporting_ledger_obligation_currency.sql` migrations in one transaction. + `reporting_ledger_account_generations.sql`, + `reporting_ledger_obligation_currency.sql`, and + `reporting_ledger_reconciliation.sql` migrations in one transaction. 3. Restart reporting work with the upgraded SDK on every instance. For deployments managed by a migration tool, the standalone migration is [`reporting_ledger_account_generations.sql`](../src/adcp/reporting/ledger/reporting_ledger_account_generations.sql). It upgrades an existing beta.15 ledger by itself, including in autocommit mode. -For a combined bootstrap and upgrade, run all three bundled files in one transaction: +For a combined bootstrap and upgrade, run all four bundled files in one transaction: ```sh psql "$REPORTING_DATABASE_URL" --set=ON_ERROR_STOP=1 --single-transaction \ -f src/adcp/reporting/ledger/reporting_ledger.sql \ -f src/adcp/reporting/ledger/reporting_ledger_account_generations.sql \ - -f src/adcp/reporting/ledger/reporting_ledger_obligation_currency.sql + -f src/adcp/reporting/ledger/reporting_ledger_obligation_currency.sql \ + -f src/adcp/reporting/ledger/reporting_ledger_reconciliation.sql ``` -Use the ledger's existing `search_path` and a role that owns its tables. Both +Use the ledger's existing `search_path` and a role that owns its tables. All SQL migrations are also included as resources in the installed SDK's `adcp.reporting.ledger` package. Running only `CREATE TABLE IF NOT EXISTS` leaves the old primary key in place and does not perform this upgrade. +The reconciliation migration adds empty optional evidence tables and nullable, +immutable managed digest/total evidence on revisions and adjustments. It does not infer historical +evidence or enable a delivery tier. See the [storage contract](reporting-reconciliation-storage.md) +for the records, migration invariants, and deferred writer/handler work. + The migration inspects the primary-key columns under an exclusive table lock and replaces the beta.15 global primary key with the account-qualified key. It preserves the constraint's name, including a renamed beta.15 key. Existing diff --git a/docs/reporting-reconciliation-storage.md b/docs/reporting-reconciliation-storage.md new file mode 100644 index 000000000..0c8540466 --- /dev/null +++ b/docs/reporting-reconciliation-storage.md @@ -0,0 +1,184 @@ +# Seller reconciliation storage foundation + +This is the durable storage half of #1167, based on #1174/#1169 and #1175/#1171. +It does **not** implement or advertise Managed Delivery or Reconciled Billing. +`ReportingStatusHandler` still projects Core only. Destination writers, +authenticated receipt handlers, batch idempotency, complete reconciliation health, +capability checks, and the transactional notification outbox remain follow-ups. + +## Public replacement contracts + +`adcp.reporting.ledger` exports `ReportingDestinationStore`, +`ReportingMaterializationStore`, `ReportingReceiptStore`, and their combined +`ReportingReconciliationStore`. They are additive protocols; an existing +`ReportingLedgerStore` implementation need not implement them. + +`InMemoryReportingReconciliationStore` and `PgReportingReconciliationStore` also +implement the Core store protocol. Their constructors are the existing Core +constructors: an optional clock, and a caller-owned connection pool for PostgreSQL. +No destination, credential resolver, receipt client, writer, or notification sender +is required at construction or startup. Existing Core classes remain usable. + +The methods return `(immutable_record, recorded)`. An exact retry returns the +retained record and `False`; reuse of an identity with different content raises +`LedgerConflictError`. Its `code` is a stable classification; messages do not echo +input, provider responses, SQL details, or credentials. Receipt `received_at` is +assigned by the store. Retry either the original input or the returned record; +changes to a supplied retained timestamp also conflict. + +| Record | Write | Identity and transition | +| --- | --- | --- | +| `ReportingDestinationBinding` | `put_destination_binding` | Consumer plus typed account/configuration generation; immutable | +| `ReportingObligationDeliveryRecord` | `bind_obligation_delivery` | Account, consumer, obligation; freezes the existing obligation currency and retention floor | +| `ReportingMaterializationAttempt` | `commit_materialization_attempt` | Account, consumer, materialization ID; consecutive attempts per obligation/revision | +| `ReportingMaterializationRecord` | `commit_materialization` | The attempt's single terminal outcome: available, delivered, or failed | +| `ReportingMaterializationCheck` | `record_materialization_check` | Account, consumer, check ID; append-only, ordered readability/corruption observations | +| `ReportingRevisionReceiptRecord` | `record_revision_receipt` | Account, consumer, receipt ID; one current leaf per obligation/revision across all attempts | +| `ReportingAdjustmentReceiptRecord` | `record_adjustment_receipt` | Same receipt ID namespace; one current leaf per account/consumer/adjustment | + +Receipt replacement names the current rejected leaf. Missing, stale, cross-account, +and cross-consumer replacement pointers have the same unavailable result. An accepted +leaf is terminal. History is never edited to mark a receipt superseded: current +leaves and terminal acceptance keys are derived from the retained graph. Revision +and adjustment receipt IDs share a namespace, as the batched wire request requires. + +## Trusted inputs and credential boundary + +These are low-level seller storage contracts, not an authentication boundary. +Only trusted account/configuration ingestion may call `put_destination_binding` +and `bind_obligation_delivery`. The later receipt handler must derive +`ReportingDeliveryPrincipal` and `ReportingDeliveryScope.consumer_id` from transport +authentication and authorize the account; the receipt body cannot grant that access. +The bindings then constrain every materialization and receipt lookup and write. +`get_destination_binding` requires both caller and typed generation; +`get_obligation_delivery` requires the full consumer/account/obligation scope. + +Use `ReportingConfigurationGenerationKey` from #1169 throughout. A binding adds +the consumer principal to that account-qualified generation. Configuration IDs, +materialization IDs, destination references, and receipt IDs may be reused by +independent principals/accounts without aliasing their retained state. Core revision, +obligation, and adjustment IDs retain the predecessor's global uniqueness constraints; +every new reference to them includes its account, including database foreign keys. + +`trusted_binding_ref` names an immutable trusted configuration and is omitted from +wire projections and record reprs. It is **not** a credential, an authorization grant, +or a mutable `latest` pointer. A later trusted resolver may obtain rotating credentials +behind that reference while preserving the destination/configuration identity. No +resolver output enters these records. There is no arbitrary metadata dictionary or +provider response field. Public metadata uses closed, frozen record types, tuples, +safe failure classifications and inert references. URL credentials, signed query +strings, bearer material and private keys are refused without echoing the input. +Adapters must supply public identifiers; syntax checks cannot establish +the provenance of an otherwise opaque identifier. +Native versions retain their decoded provider value, including characters such as +`/`, `+`, and `=`. They are not entity IDs, URLs, or URI-encoded object references. + +## Evidence and financial ordering + +`ReportingRevisionRecord.canonical_content_digest` optionally holds a frozen +`ReportingCanonicalDigest`. This is the **managed logical-row** digest under an +immutable canonicalization contract, distinct from Core's fixed +`revision_content_sha256`. Neither a destination response nor an observed consumer +digest establishes the expected digest. The trusted publisher must verify the pinned +contract and compute it before committing the revision. This PR stores that fact; +the later writer must perform and attest actual destination verification. +New managed revision commits also verify Core's row binding and include their +finality timestamps and canonicalization contract in immutable replay identity. +`ReportingRevisionRecord.managed_control_totals` retains expected +`ReportingControlTotalRecord` values before destination work; it must be present +for materialization attempts, including an explicit empty tuple for no totals. +`ReportingAdjustmentRecord.managed_control_total_deltas` similarly retains the +complete expected delta evidence used in the adjustment digest. A trusted publisher +supplies these from its pinned definition and verified content. Their names/values +must match the existing Core pairs. They are optional for Core-only records. +Pass those expected totals as `control_total_evidence` to +`revision_content_sha256()` when preparing a new managed revision. This binds +the full totals actually exposed by status and exact reads under the same Core +four-field hash rule. Omitting the argument preserves the existing Core hashes. + +Terminal success checks row count, exact named totals, the selected profile, every +provided canonical digest, declared format, reader compatibility, method/resource +kind, verification path, physical-object membership and native-version evidence. +`ReportingControlTotalRecord` retains value type and unit as well as name and value; +changing a receipt's monetary unit is a changed statement, never a cached success. +Declared monetary units are checked against #1171's frozen obligation currency +and pinned definition; omitted units inherit that context. Nonmonetary units and +explicit types (including a decimal total represented by `"5"`) remain unchanged. +Unknown historical currency or missing expected managed totals fails closed. + +One logical revision may fan out to another destination obligation only when its +account, frozen period and resolved buy/package scope, coverage, definition and +currency match exactly. That does not grant another principal access: each +consumer still requires its own trusted binding and obligation delivery record. + +Billing requires an official configuration, consumer receipts, and canonical digest +verification. Other managed profiles retain their narrower assurance: native commits +and manifest checksums do not assert cryptographic equality of logical rows. +Available and delivered claims follow the frozen binding's success status. + +Adjustment acceptance verifies the complete adjustment's JCS/SHA-256 evidence, +including optional `reason_detail`, and its exact official revision. Correction +observation must be no earlier than finalization and no later than creation; receipt +observation follows creation. Periods must be ordered. These are evidence records, +not permission to post accounting entries, reopen books, change invoices, or settle. + +Two wire boundaries need attention in the completion PR: + +* The adjustment schema defines an independent receipt chain and does not require + prior acceptance of the official revision receipt. The store records those two + facts independently; completion must require both. An accepted adjustment alone + cannot establish reconciled completion. +* The existing buyer `_select_current` can report `AMBIGUOUS_REVISION_CHAIN` when a + retained snapshot and a separate official revision coexist, while Core's producer + forbids an official revision from superseding a snapshot. This PR preserves that + financial finality rule. The completion PR must resolve reader selection against + the protocol before claiming a complete multi-finality lifecycle. + +## Retention, snapshots, and future notifications + +`read_reconciliation_snapshot(caller=...)` reads retained records at a Core ledger +boundary. Reusing its `boundary` excludes later outcomes, receipts and storage +checks. `get_materialization` returns the attempt, terminal outcome, binding and +readability history; `get_receipt` requires an explicit account/consumer key. +Expiry and corruption never erase immutable evidence or acceptance identity. +`readable_at()` applies the retained checks and exact resource expiry, rather than +changing the materialization's terminal wire state. A successful resource must last +through both the obligation floor and readiness plus the frozen retention contract. +There is no purge or unchecked history-repair API. + +The opt-in `materialization_to_wire`, `receipt_to_wire`, `revision_to_wire`, and +`adjustment_to_wire` helpers support generated wire models. They are not mounted by +the Core status handler. Snapshot records are a lower-level storage read, not an +unbounded wire response; bounded status pagination and tier-correct counts belong +to the completion PR. + +Each new record appends a distinct, scoped ledger change in the same transaction. +Pending attempts and terminal outcomes have separate change kinds, so a snapshot +cannot acquire terminal evidence committed after its boundary. Exact retries append +nothing. Both stores share the transition validator; PostgreSQL serializes with the +existing account lock and additionally uses a conditional receipt-head update. +Database triggers reject evidence updates/deletes and terminal-head replacement. +This is the transaction in which #1168 can insert its outbox row; there is no adopter +callback or after-commit webhook send in this PR. + +## Migration and operational limits + +Run `create_schema()` or all four bundled SQL resources in one transaction: +`reporting_ledger.sql`, `reporting_ledger_account_generations.sql`, +`reporting_ledger_obligation_currency.sql`, then `reporting_ledger_reconciliation.sql`. +The last migration is also one atomic statement in autocommit mode. All migrations +share the schema advisory lock. Drain older writers before upgrading. + +The migration adds nullable managed digest/total evidence with no default/backfill, +immutable record and receipt-head tables, and account-qualified reference indexes. +Literal beta.15 and #1171 upgrades preserve pre-existing rows, hashes, currency, leases, +issue history and feed sequence numbers. Unknown currency/digest/total history stays +unknown. Existing Core replay hashes do not change. A different replay of an +existing adjustment now conflicts instead of silently returning its old content. + +Table/index creation and prerequisite migrations take locks; production-sized +duration is not benchmarked. The reference stores load one consumer's retained +record set to validate transitions. Large histories need indexed queries or a +conforming replacement store before production rollout. Only SDK store operations +are supported writers; database owners can always circumvent application invariants +by disabling constraints. PostgreSQL connection ownership remains with the adopter. diff --git a/src/adcp/reporting/evidence.py b/src/adcp/reporting/evidence.py new file mode 100644 index 000000000..35e46e79e --- /dev/null +++ b/src/adcp/reporting/evidence.py @@ -0,0 +1,215 @@ +"""Small immutable public evidence values, independent of transports and credentials.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import ClassVar, Literal +from urllib.parse import urlsplit + +from pydantic import ConfigDict + + +def native_version_reference(value: str) -> str: + """Retain a decoded, publicly classified native version without URI encoding. + + Native versions are not entity IDs: for example, ``/``, ``+`` and ``=`` are + valid characters. The trusted adapter must establish that the value is public; + this guard rejects recognizable authorization material, not opaque secrets. + """ + if ( + type(value) is not str + or not 1 <= len(value) <= 1024 + or not value.isprintable() + or "://" in value + or re.search( + r"(?i)(?:bearer\s|password|secret|token|signature|credential|private.key|-----BEGIN)", + value, + ) + ): + raise ValueError("native version evidence requires a non-secret decoded public reference") + return value + + +def public_reference(value: str, *, maximum: int = 1024, path: bool = False) -> str: + """Accept inert identifiers, never URLs, query strings or authentication material. + + These are trusted public labels, not an arbitrary provider response sanitiser. + Adapters needing a richer provider identifier must retain it behind the trusted + binding and publish an opaque label. Error messages never interpolate input. + """ + pattern = r"[A-Za-z0-9_.:/-]+" if path else r"[A-Za-z0-9_.:-]+" + if ( + not isinstance(value, str) + or not 1 <= len(value) <= maximum + or re.fullmatch(pattern, value) is None + or "://" in value + or value.startswith("/") + or any(part == ".." for part in value.split("/")) + or re.search( + r"(?i)(?:bearer|password|secret|token|signature|credential|private.key)", value + ) + ): + raise ValueError("reporting metadata requires a non-secret public reference") + return value + + +def sha256_value(value: str) -> str: + if not isinstance(value, str) or re.fullmatch(r"[a-fA-F0-9]{64}", value) is None: + raise ValueError("reporting evidence requires a SHA-256 digest") + return value + + +def aware_utc(value: datetime) -> datetime: + if not isinstance(value, datetime) or value.tzinfo is None: + raise ValueError("reporting evidence requires an aware timestamp") + return value.astimezone(timezone.utc) + + +@dataclass(frozen=True, slots=True) +class ReportingControlTotalRecord: + """An exact expected or observed wire total, with its declared type and unit.""" + + name: str + value: str + value_type: Literal["integer", "decimal"] + unit: str | None = None + __pydantic_config__: ClassVar[ConfigDict] = ConfigDict( + extra="forbid", hide_input_in_errors=True + ) + + def __post_init__(self) -> None: + if ( + any(type(value) is not str for value in (self.name, self.value, self.value_type)) + or (self.unit is not None and type(self.unit) is not str) + or self.value_type not in {"integer", "decimal"} + ): + raise ValueError("control total evidence requires immutable typed values") + public_reference(self.name, maximum=128) + if re.fullmatch(r"[A-Za-z][A-Za-z0-9_.:-]{0,127}", self.name) is None: + raise ValueError("reporting total names require public metric identifiers") + pattern = ( + r"-?(?:0|[1-9][0-9]*)" + if self.value_type == "integer" + else r"-?(?:0|[1-9][0-9]*)(?:\.[0-9]+)?" + ) + if re.fullmatch(pattern, self.value) is None: + raise ValueError("control totals require canonical numeric strings matching their type") + if self.unit is not None: + public_reference(self.unit, maximum=32) + + def to_wire(self) -> dict[str, str]: + result = {"name": self.name, "value": self.value, "value_type": self.value_type} + if self.unit is not None: + result["unit"] = self.unit + return result + + @classmethod + def from_wire(cls, value: object) -> ReportingControlTotalRecord: + if ( + not isinstance(value, dict) + or not {"name", "value", "value_type"} <= set(value) + or set(value) - {"name", "value", "value_type", "unit"} + ): + raise ValueError("invalid retained control total evidence") + return cls(value["name"], value["value"], value["value_type"], value.get("unit")) + + +def freeze_control_totals( + totals: tuple[ReportingControlTotalRecord, ...], pairs: tuple[tuple[str, str], ...] +) -> tuple[ReportingControlTotalRecord, ...]: + if not isinstance(totals, (tuple, list)) or any( + type(item) is not ReportingControlTotalRecord for item in totals + ): + raise ValueError("control total evidence requires closed immutable records") + result = tuple(totals) + if len({item.name for item in result}) != len(result) or sorted( + (item.name, item.value) for item in result + ) != sorted(pairs): + raise ValueError("control total evidence must match the retained names and values") + return result + + +@dataclass(frozen=True, slots=True) +class ReportingCanonicalDigest: + """A trusted publisher's logical-row digest under a pinned managed contract. + + This is distinct from Core's fixed ``revision_content_sha256``. A future + managed publisher must verify the pinned contract and compute this value + before destination work; destination output cannot establish the expectation. + """ + + value: str + canonicalization_id: str + canonicalization_uri: str + canonicalization_sha256: str + + __pydantic_config__: ClassVar[ConfigDict] = ConfigDict( + extra="forbid", hide_input_in_errors=True + ) + + def __post_init__(self) -> None: + if any( + type(value) is not str + for value in ( + self.value, + self.canonicalization_id, + self.canonicalization_uri, + self.canonicalization_sha256, + ) + ): + raise ValueError("canonical evidence requires immutable string values") + sha256_value(self.value) + sha256_value(self.canonicalization_sha256) + public_reference(self.canonicalization_id, maximum=128) + uri = self.canonicalization_uri + try: + parsed = urlsplit(uri) + valid = ( + parsed.scheme == "https" + and parsed.hostname is not None + and re.fullmatch(r"(?:[A-Za-z0-9-]+\.)+[A-Za-z]{2,}", parsed.hostname) + and not parsed.username + and not parsed.password + and not parsed.query + and not parsed.fragment + and uri.isascii() + and not re.search(r"[\s\\%]", uri) + and not re.search(r"(?i)(token|secret|signature|credential)", uri) + ) + except (TypeError, ValueError): + valid = False + if not valid: + raise ValueError("canonicalization requires a public HTTPS contract URI") + + def to_wire(self) -> dict[str, str]: + return { + "algorithm": "sha256", + "value": self.value, + "canonicalization_id": self.canonicalization_id, + "canonicalization_uri": self.canonicalization_uri, + "canonicalization_sha256": self.canonicalization_sha256, + } + + @classmethod + def from_wire(cls, value: object) -> ReportingCanonicalDigest: + if ( + not isinstance(value, dict) + or set(value) + != { + "algorithm", + "value", + "canonicalization_id", + "canonicalization_uri", + "canonicalization_sha256", + } + or value.get("algorithm") != "sha256" + ): + raise ValueError("invalid retained canonical evidence") + return cls( + value["value"], + value["canonicalization_id"], + value["canonicalization_uri"], + value["canonicalization_sha256"], + ) diff --git a/src/adcp/reporting/ledger/__init__.py b/src/adcp/reporting/ledger/__init__.py index 8769ce372..8a972cc44 100644 --- a/src/adcp/reporting/ledger/__init__.py +++ b/src/adcp/reporting/ledger/__init__.py @@ -59,11 +59,14 @@ from __future__ import annotations +from typing import TYPE_CHECKING + from adcp.reporting.currency import ( ReportingCurrencyError, require_single_currency, validate_currency, ) +from adcp.reporting.evidence import ReportingCanonicalDigest from adcp.reporting.ledger.consumer_status import ( ConsumerMismatch, ConsumerStatusDisabledError, @@ -74,6 +77,38 @@ project_consumer_mismatch, stale_received_grace_deadline, ) +from adcp.reporting.ledger.delivery import ( + InMemoryReportingReconciliationStore, + ReportingDestinationStore, + ReportingMaterializationStore, + ReportingMaterializationView, + ReportingReceiptStore, + ReportingReconciliationSnapshot, + ReportingReconciliationStore, + adjustment_to_wire, + materialization_to_wire, + receipt_to_wire, + revision_to_wire, +) +from adcp.reporting.ledger.delivery_models import ( + ReportingAdjustmentReceiptRecord, + ReportingControlTotalRecord, + ReportingDeliveryPrincipal, + ReportingDeliveryRecord, + ReportingDeliveryScope, + ReportingDestinationBinding, + ReportingMaterializationAttempt, + ReportingMaterializationCheck, + ReportingMaterializationKey, + ReportingMaterializationRecord, + ReportingObligationDeliveryRecord, + ReportingPhysicalChecksum, + ReportingReceiptKey, + ReportingReceiptRecord, + ReportingResourceRecord, + ReportingRevisionReceiptRecord, + ReportingVerificationRecord, +) from adcp.reporting.ledger.health import ( ObligationProjection, aggregate_reporting_health, @@ -131,6 +166,10 @@ reject_reserved_authoritative_party, ) +if TYPE_CHECKING: + from adcp.reporting.ledger.delivery_pg import PgReportingReconciliationStore + from adcp.reporting.ledger.pg import PgReportingLedgerStore + __all__ = [ "ConsumerMismatch", "ConsumerStatusDisabledError", @@ -140,6 +179,7 @@ "CurrencyResolver", "FixedCurrencyResolver", "InMemoryReportingLedgerStore", + "InMemoryReportingReconciliationStore", "LeasedConfiguration", "LedgerChange", "LedgerConflictError", @@ -147,31 +187,57 @@ "LedgerSnapshot", "ObligationProjection", "PgReportingLedgerStore", + "PgReportingReconciliationStore", "ProducerOfferings", + "ReportingAdjustmentReceiptRecord", "ReportingAdjustmentRecord", + "ReportingCanonicalDigest", "ReportingConfiguration", "ReportingConfigurationGenerationKey", + "ReportingControlTotalRecord", "ReportingCurrencyError", "ReportingDefinitionBinding", "ReportingDeliveryEscalation", + "ReportingDeliveryPrincipal", + "ReportingDeliveryRecord", + "ReportingDeliveryScope", + "ReportingDestinationBinding", + "ReportingDestinationStore", "ReportingFinality", "ReportingHealth", "ReportingIssue", "ReportingIssueLifecycle", "ReportingIssueStateValue", "ReportingLedgerStore", + "ReportingMaterializationAttempt", + "ReportingMaterializationCheck", + "ReportingMaterializationKey", + "ReportingMaterializationRecord", + "ReportingMaterializationStore", + "ReportingMaterializationView", "ReportingMismatchCode", + "ReportingObligationDeliveryRecord", "ReportingObligationRecord", "ReportingPeriodBoundary", + "ReportingPhysicalChecksum", "ReportingProducer", "ReportingProductionStatus", + "ReportingReceiptKey", + "ReportingReceiptRecord", + "ReportingReceiptStore", + "ReportingReconciliationSnapshot", + "ReportingReconciliationStore", + "ReportingResourceRecord", + "ReportingRevisionReceiptRecord", "ReportingRevisionRecord", "ReportingRowPage", "ReportingScheduleSpec", "ReportingStatusCaller", "ReportingStatusHandler", "ReportingStatusView", + "ReportingVerificationRecord", "WorkerTurn", + "adjustment_to_wire", "aggregate_reporting_health", "derive_period", "check_issue_state_transition", @@ -183,11 +249,14 @@ "issue_id_for", "issue_is_retirable", "issue_id_for_occurrence", + "materialization_to_wire", "project_consumer_mismatch", "project_obligation_health", + "receipt_to_wire", "reject_reserved_authoritative_party", "require_single_currency", "revision_content_sha256", + "revision_to_wire", "stale_received_grace_deadline", "validate_currency", ] @@ -199,4 +268,8 @@ def __getattr__(name: str) -> object: from adcp.reporting.ledger.pg import PgReportingLedgerStore return PgReportingLedgerStore + if name == "PgReportingReconciliationStore": + from adcp.reporting.ledger.delivery_pg import PgReportingReconciliationStore + + return PgReportingReconciliationStore raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/src/adcp/reporting/ledger/_delivery_state.py b/src/adcp/reporting/ledger/_delivery_state.py new file mode 100644 index 000000000..cd2dd1021 --- /dev/null +++ b/src/adcp/reporting/ledger/_delivery_state.py @@ -0,0 +1,594 @@ +"""One transition contract shared by memory and PostgreSQL, with no I/O.""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import asdict, dataclass, replace +from datetime import datetime, timedelta +from typing import Annotated, Any, NoReturn, TypeVar + +from pydantic import Field, TypeAdapter, ValidationError + +from adcp.reporting.canonical_json import canonical_json_utf8_v1 +from adcp.reporting.currency import require_frozen_currency +from adcp.reporting.evidence import aware_utc +from adcp.reporting.ledger.delivery_models import ( + ReportingAdjustmentReceiptRecord, + ReportingControlTotalRecord, + ReportingDeliveryPrincipal, + ReportingDeliveryRecord, + ReportingDestinationBinding, + ReportingMaterializationAttempt, + ReportingMaterializationCheck, + ReportingMaterializationRecord, + ReportingObligationDeliveryRecord, + ReportingReceiptRecord, + ReportingRevisionReceiptRecord, +) +from adcp.reporting.ledger.models import ( + ReportingAdjustmentRecord, + ReportingConfiguration, + ReportingObligationRecord, + ReportingRevisionRecord, +) +from adcp.reporting.ledger.store import ( + LedgerConflictError, + validate_adjustment_currency, + validate_managed_total_units, +) + +RecordT = TypeVar("RecordT", bound=ReportingDeliveryRecord) +_ADAPTER: TypeAdapter[ReportingDeliveryRecord] = TypeAdapter( + Annotated[ReportingDeliveryRecord, Field(discriminator="kind")] +) +_RECEIPTS = (ReportingRevisionReceiptRecord, ReportingAdjustmentReceiptRecord) + + +def fail(code: str) -> NoReturn: + # All errors are closed classifications. Never attach provider exceptions, + # raw input, identifiers, a database detail string, or credential resolver state. + raise LedgerConflictError(code, "reporting record violates the retained evidence contract") + + +def unavailable() -> NoReturn: + raise LedgerConflictError("REPORTING_RECORD_UNAVAILABLE", "reporting record is unavailable") + + +def iso(value: datetime) -> str: + return aware_utc(value).isoformat().replace("+00:00", "Z") + + +def payload(record: ReportingDeliveryRecord) -> dict[str, Any]: + # Dataclass fields only: never __dict__, a provider object, or a wire extras bag. + result: dict[str, Any] = json.loads(json.dumps(asdict(record), default=iso)) + return result + + +def decode_record(value: object) -> ReportingDeliveryRecord: + result: ReportingDeliveryRecord | None = None + try: + result = _ADAPTER.validate_python(value) + except (ValidationError, ValueError, TypeError): + pass + if result is None: + fail("INVALID_REPORTING_RECORD") + if payload(result) != value: + # Predecessor dataclasses may otherwise ignore unknown nested fields. + # Never hash or echo a raw untrusted payload while checking its shape. + fail("INVALID_REPORTING_RECORD") + return result + + +def fingerprint(record: ReportingDeliveryRecord) -> str: + return hashlib.sha256(canonical_json_utf8_v1(payload(record))).hexdigest() + + +def principal(record: ReportingDeliveryRecord) -> ReportingDeliveryPrincipal: + return ( + record.principal + if isinstance(record, ReportingDestinationBinding) + else record.scope.principal + ) + + +def record_identity(record: ReportingDeliveryRecord) -> tuple[str, str]: + if isinstance(record, ReportingDestinationBinding): + generation = record.generation_key + return record.kind, hashlib.sha256(canonical_json_utf8_v1(asdict(generation))).hexdigest() + if isinstance(record, ReportingObligationDeliveryRecord): + return record.kind, record.scope.reporting_obligation_id + if isinstance(record, ReportingMaterializationCheck): + return record.kind, record.check_id + if isinstance(record, _RECEIPTS): + # One namespace across both receipt arrays, as required by the batch schema. + return "receipt", record.reporting_receipt_id + return record.kind, record.reporting_materialization_id + + +def change_id(record: ReportingDeliveryRecord) -> str: + who = principal(record) + return hashlib.sha256( + canonical_json_utf8_v1([who.account_id, who.consumer_id, *record_identity(record)]) + ).hexdigest() + + +def receipt_chain(record: ReportingReceiptRecord) -> str: + if isinstance(record, ReportingAdjustmentReceiptRecord): + parts = ["adjustment", record.reporting_adjustment_id] + else: + parts = ["revision", record.scope.reporting_obligation_id, record.reporting_revision_id] + return hashlib.sha256(canonical_json_utf8_v1(parts)).hexdigest() + + +def current_receipt( + records: tuple[ReportingDeliveryRecord, ...], requested: ReportingReceiptRecord +) -> ReportingReceiptRecord | None: + chain = [ + item + for item in records + if isinstance(item, _RECEIPTS) and receipt_chain(item) == receipt_chain(requested) + ] + superseded = {item.supersedes_reporting_receipt_id for item in chain} + leaves = [item for item in chain if item.reporting_receipt_id not in superseded] + if chain and len(leaves) != 1: + fail("REPORTING_HISTORY_CORRUPT") + return leaves[0] if leaves else None + + +def replay( + record: ReportingDeliveryRecord, records: tuple[ReportingDeliveryRecord, ...] +) -> ReportingDeliveryRecord | None: + existing = next( + (item for item in records if record_identity(item) == record_identity(record)), None + ) + if existing is None: + return None + comparison = existing + if ( + isinstance(record, _RECEIPTS) + and isinstance(existing, _RECEIPTS) + and record.received_at is None + ): + comparison = replace(existing, received_at=None) + if fingerprint(comparison) != fingerprint(record): + fail("REPORTING_IDENTITY_CONFLICT") + return existing + + +@dataclass(frozen=True) +class DeliveryContext: + configuration: ReportingConfiguration | None = None + obligation: ReportingObligationRecord | None = None + revision: ReportingRevisionRecord | None = None + revision_obligation: ReportingObligationRecord | None = None + adjustment: ReportingAdjustmentRecord | None = None + + +def validate_transition( + record: ReportingDeliveryRecord, + records: tuple[ReportingDeliveryRecord, ...], + context: DeliveryContext, + now: datetime, +) -> ReportingDeliveryRecord: + """Validate only new writes. Call replay first, including after retention loss.""" + now = aware_utc(now) + who = principal(record) + if isinstance(record, ReportingDestinationBinding): + config = context.configuration + if config is None or config.generation_key != record.generation_key: + unavailable() + if config.feed_purpose != record.feed_purpose: + fail("CONFIGURATION_BINDING_MISMATCH") + if record.created_at > now: + fail("REPORTING_TIME_INVALID") + if config.feed_purpose == "billing" and record.verification_profile != "canonical_digest": + fail("BILLING_REQUIRES_CANONICAL_DIGEST") + if config.feed_purpose == "billing" and ( + record.reconciliation_mode != "consumer_receipt" + or config.required_finality != "official" + ): + fail("BILLING_REQUIRES_OFFICIAL_RECEIPTS") + return record + + binding = next( + ( + item + for item in records + if isinstance(item, ReportingDestinationBinding) + and item.generation_key == record.scope.generation_key + ), + None, + ) + obligation = context.obligation + if ( + binding is None + or obligation is None + or obligation.account_id != who.account_id + or ( + obligation.reporting_obligation_id != record.scope.reporting_obligation_id + or obligation.generation_key != record.scope.generation_key + ) + ): + unavailable() + currency = require_frozen_currency(obligation.currency) + + if isinstance(record, ReportingObligationDeliveryRecord): + if record.currency != currency: + fail("CURRENCY_MISMATCH") + if ( + record.created_at < max(binding.created_at, obligation.created_at) + or record.created_at > now + ): + fail("REPORTING_TIME_INVALID") + if record.resource_retained_until < record.created_at + timedelta( + days=binding.resource_retention_days + ): + fail("RESOURCE_RETENTION_INVALID") + return record + + delivery = next( + ( + item + for item in records + if isinstance(item, ReportingObligationDeliveryRecord) and item.scope == record.scope + ), + None, + ) + if delivery is None: + unavailable() + if delivery.currency != currency: + fail("CURRENCY_MISMATCH") + + if isinstance(record, ReportingMaterializationCheck): + target = _materialization(records, record.reporting_materialization_id) + if target is None or target.scope != record.scope or target.status == "failed": + unavailable() + assert target.resource is not None + latest = latest_check(records, record.reporting_materialization_id) + if ( + record.checked_at < target.completed_at + or record.checked_at > now + or (latest is not None and record.checked_at <= latest.checked_at) + ): + fail("REPORTING_TIME_INVALID") + if record.state == "readable" and record.checked_at >= target.resource.expires_at: + fail("RESOURCE_RETENTION_INVALID") + return record + + revision = context.revision + revision_id = ( + record.adjusts_reporting_revision_id + if isinstance(record, ReportingAdjustmentReceiptRecord) + else record.reporting_revision_id + ) + if ( + revision is None + or revision.account_id != who.account_id + or revision.reporting_revision_id != revision_id + or not same_revision_scope(context.revision_obligation, obligation) + ): + unavailable() + + if isinstance(record, ReportingMaterializationAttempt): + revision_control_totals(revision, obligation) + if ( + record.created_at < max(delivery.created_at, revision.created_at) + or record.created_at > now + ): + fail("REPORTING_TIME_INVALID") + siblings = [ + item + for item in records + if isinstance(item, ReportingMaterializationAttempt) + and item.scope == record.scope + and item.reporting_revision_id == revision_id + ] + if record.attempt != len(siblings) + 1: + fail("MATERIALIZATION_ATTEMPT_CONFLICT") + if ( + binding.verification_profile == "canonical_digest" + and revision.canonical_content_digest is None + ): + fail("REVISION_CANONICAL_EVIDENCE_REQUIRED") + if not revision.readable: + fail("REVISION_UNREADABLE") + return record + + if isinstance(record, ReportingMaterializationRecord): + attempt = next( + ( + item + for item in records + if isinstance(item, ReportingMaterializationAttempt) + and item.reporting_materialization_id == record.reporting_materialization_id + ), + None, + ) + if ( + attempt is None + or attempt.scope != record.scope + or attempt.reporting_revision_id != revision_id + ): + unavailable() + if record.completed_at < attempt.created_at or record.completed_at > now: + fail("REPORTING_TIME_INVALID") + if record.status != "failed": + _verify_materialization(record, binding, delivery, revision, obligation) + return record + + if binding.reconciliation_mode != "consumer_receipt": + fail("RECEIPTS_NOT_ENABLED") + if record.received_at is not None: + fail("RECEIVED_AT_READ_ONLY") + if record.observed_at > now: + fail("REPORTING_TIME_INVALID") + if isinstance(record, ReportingRevisionReceiptRecord): + _verify_receipt(record, records, revision) + else: + adjustment = context.adjustment + if ( + adjustment is None + or adjustment.account_id != who.account_id + or ( + adjustment.reporting_adjustment_id != record.reporting_adjustment_id + or adjustment.adjusts_reporting_revision_id != revision_id + ) + ): + unavailable() + validate_adjustment_currency(obligation, adjustment) + if revision.finality != "official" or revision.finalized_at is None: + fail("ADJUSTMENT_REQUIRES_OFFICIAL") + if not ( + obligation.period.end <= revision.finalized_at <= revision.created_at + and revision.finalized_at <= adjustment.correction_observed_at <= adjustment.created_at + and adjustment.accounting_period_start < adjustment.accounting_period_end + and adjustment.created_at <= record.observed_at + ): + fail("ADJUSTMENT_ORDER_INVALID") + if record.status == "accepted" and record.observed_adjustment_sha256 != adjustment_sha256( + adjustment + ): + fail("ADJUSTMENT_DIGEST_MISMATCH") + + leaf = current_receipt(records, record) + if leaf is not None and leaf.status == "accepted": + fail("ACCEPTED_RECEIPT_TERMINAL") + if leaf is None: + if record.supersedes_reporting_receipt_id is not None: + unavailable() + elif record.supersedes_reporting_receipt_id != leaf.reporting_receipt_id: + # Missing/stale/cross-caller pointers have the same result. + unavailable() + return replace(record, received_at=now) + + +def same_revision_scope( + origin: ReportingObligationRecord | None, target: ReportingObligationRecord +) -> bool: + """Destination-independent content may fan out only over the exact frozen slice.""" + return bool( + origin is not None + and origin.account_id == target.account_id + and origin.report_definition_id == target.report_definition_id + and origin.reporting_profile == target.reporting_profile + and origin.definition == target.definition + and origin.currency == target.currency + and origin.period.start == target.period.start + and origin.period.end == target.period.end + and origin.period.source_timezone == target.period.source_timezone + and origin.scope_resolved_at == target.scope_resolved_at + and sorted(origin.media_buy_ids) == sorted(target.media_buy_ids) + and sorted(origin.package_ids) == sorted(target.package_ids) + and origin.coverage_status == target.coverage_status + ) + + +def _verify_materialization( + record: ReportingMaterializationRecord, + binding: ReportingDestinationBinding, + delivery: ReportingObligationDeliveryRecord, + revision: ReportingRevisionRecord, + obligation: ReportingObligationRecord, +) -> None: + resource, verification = record.resource, record.verification + assert resource is not None and verification is not None + if record.status != binding.success_status: + fail("MATERIALIZATION_STATUS_MISMATCH") + if verification.verified_at != record.completed_at: + fail("VERIFICATION_TIME_MISMATCH") + if resource.expires_at < max( + delivery.resource_retained_until, + record.completed_at + timedelta(days=binding.resource_retention_days), + ): + fail("RESOURCE_RETENTION_INVALID") + if ( + verification.verification_profile != binding.verification_profile + or verification.verified_format != binding.format + ): + fail("VERIFICATION_PROFILE_MISMATCH") + if resource.reader_compatibility != binding.reader_compatibility: + fail("READER_COMPATIBILITY_MISMATCH") + if verification.row_count != revision.row_count or sorted( + verification.control_totals, key=lambda item: item.name + ) != sorted(revision_control_totals(revision, obligation), key=lambda item: item.name): + fail("VERIFICATION_TOTALS_MISMATCH") + digest = verification.canonical_content_digest + if (digest is not None and digest != revision.canonical_content_digest) or ( + binding.verification_profile == "canonical_digest" and digest is None + ): + fail("VERIFICATION_DIGEST_MISMATCH") + expected_kind = { + "file_transfer": "manifest", + "dataset_share": "dataset", + "warehouse_materialization": "warehouse_relation", + }[binding.method] + if resource.kind != expected_kind: + fail("VERIFICATION_METHOD_MISMATCH") + path = verification.verification_path + if ( + (binding.method == "dataset_share" and path != "representative_consumer") + or (binding.method == "warehouse_materialization" and path != "destination") + or (record.status == "delivered" and path != "destination") + or (binding.method == "warehouse_materialization" and record.status != "delivered") + ): + fail("VERIFICATION_PATH_MISMATCH") + checksums = verification.physical_checksums + if binding.method == "file_transfer" and (not checksums or not resource.object_refs): + fail("PHYSICAL_CHECKSUMS_REQUIRED") + if len({(item.object_ref, item.algorithm) for item in checksums}) != len(checksums) or any( + item.object_ref not in resource.object_refs for item in checksums + ): + fail("PHYSICAL_CHECKSUM_BINDING_MISMATCH") + if binding.method == "file_transfer" and {item.object_ref for item in checksums} != set( + resource.object_refs + ): + fail("PHYSICAL_CHECKSUM_BINDING_MISMATCH") + if binding.verification_profile == "manifest_checksums" and ( + resource.kind != "manifest" or resource.manifest_sha256 is None or not checksums + ): + fail("PHYSICAL_CHECKSUMS_REQUIRED") + if ( + binding.verification_profile == "native_commit" + or verification.native_version_ref is not None + ): + if ( + verification.native_version_ref is None + or verification.native_version_ref != resource.native_version_ref + or verification.native_observed_through != path + ): + fail("NATIVE_COMMIT_MISMATCH") + + +def _materialization( + records: tuple[ReportingDeliveryRecord, ...], materialization_id: str +) -> ReportingMaterializationRecord | None: + return next( + ( + item + for item in records + if isinstance(item, ReportingMaterializationRecord) + and item.reporting_materialization_id == materialization_id + ), + None, + ) + + +def latest_check( + records: tuple[ReportingDeliveryRecord, ...], + materialization_id: str, + *, + at: datetime | None = None, +) -> ReportingMaterializationCheck | None: + return max( + ( + item + for item in records + if isinstance(item, ReportingMaterializationCheck) + and item.reporting_materialization_id == materialization_id + and (at is None or item.checked_at <= at) + ), + key=lambda item: item.checked_at, + default=None, + ) + + +def _verify_receipt( + record: ReportingRevisionReceiptRecord, + records: tuple[ReportingDeliveryRecord, ...], + revision: ReportingRevisionRecord, +) -> None: + target = _materialization(records, record.reporting_materialization_id) + if ( + target is None + or target.scope != record.scope + or ( + target.reporting_revision_id != record.reporting_revision_id + or target.status == "failed" + ) + ): + unavailable() + assert target.resource is not None and target.verification is not None + if record.verification_profile != target.verification.verification_profile: + fail("RECEIPT_PROFILE_MISMATCH") + if record.observed_at < target.completed_at: + fail("REPORTING_TIME_INVALID") + if record.status != "accepted": + return + if record.observed_row_count != revision.row_count or sorted( + record.observed_control_totals, key=lambda item: item.name + ) != sorted(target.verification.control_totals, key=lambda item: item.name): + fail("RECEIPT_TOTALS_MISMATCH") + profile = record.verification_profile + if ( + ( + record.observed_canonical_content_digest is not None + and record.observed_canonical_content_digest != revision.canonical_content_digest + ) + or ( + record.observed_manifest_sha256 is not None + and record.observed_manifest_sha256 != target.resource.manifest_sha256 + ) + or ( + record.observed_native_version_ref is not None + and record.observed_native_version_ref != target.resource.native_version_ref + ) + or ( + profile == "canonical_digest" + and ( + record.observed_canonical_content_digest is None + or record.observed_canonical_content_digest != revision.canonical_content_digest + ) + ) + or ( + profile == "manifest_checksums" + and record.observed_manifest_sha256 != target.resource.manifest_sha256 + ) + or ( + profile == "native_commit" + and record.observed_native_version_ref != target.resource.native_version_ref + ) + ): + fail("RECEIPT_EVIDENCE_MISMATCH") + check = latest_check(records, record.reporting_materialization_id, at=record.observed_at) + if record.observed_at >= target.resource.expires_at or ( + check is not None and check.state != "readable" + ): + fail("MATERIALIZATION_UNREADABLE") + + +def totals_to_wire(totals: tuple[ReportingControlTotalRecord, ...]) -> list[dict[str, str]]: + return [item.to_wire() for item in totals] + + +def revision_control_totals( + revision: ReportingRevisionRecord, obligation: ReportingObligationRecord +) -> tuple[ReportingControlTotalRecord, ...]: + if revision.managed_control_totals is None: + fail("REVISION_TOTAL_EVIDENCE_REQUIRED") + validate_managed_total_units(obligation, revision.managed_control_totals) + return revision.managed_control_totals + + +def adjustment_payload(adjustment: ReportingAdjustmentRecord) -> dict[str, Any]: + if not adjustment.managed_control_total_deltas: + fail("ADJUSTMENT_TOTAL_EVIDENCE_REQUIRED") + result: dict[str, Any] = { + "reporting_adjustment_id": adjustment.reporting_adjustment_id, + "adjusts_reporting_revision_id": adjustment.adjusts_reporting_revision_id, + "reason_code": adjustment.reason_code, + "accounting_period": { + "start": iso(adjustment.accounting_period_start), + "end": iso(adjustment.accounting_period_end), + }, + "control_total_deltas": totals_to_wire(adjustment.managed_control_total_deltas), + "correction_observed_at": iso(adjustment.correction_observed_at), + "created_at": iso(adjustment.created_at), + } + if adjustment.reason_detail is not None: + result["reason_detail"] = adjustment.reason_detail + return result + + +def adjustment_sha256(adjustment: ReportingAdjustmentRecord) -> str: + return hashlib.sha256(canonical_json_utf8_v1(adjustment_payload(adjustment))).hexdigest() diff --git a/src/adcp/reporting/ledger/delivery.py b/src/adcp/reporting/ledger/delivery.py new file mode 100644 index 000000000..6b65799ed --- /dev/null +++ b/src/adcp/reporting/ledger/delivery.py @@ -0,0 +1,502 @@ +"""Optional durable seller contracts for future destination writers and receipt handlers. + +These stores persist evidence only. They do not resolve credentials, perform +external writes, mount tasks, or advertise Managed Delivery/Reconciled Billing. +The existing Core store protocol and construction contract remain unchanged. +""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass +from datetime import datetime +from typing import Any, Protocol, cast, runtime_checkable + +from adcp.reporting.ledger._delivery_state import ( + DeliveryContext, + RecordT, + adjustment_payload, + adjustment_sha256, + change_id, + decode_record, + iso, + payload, + principal, + replay, + revision_control_totals, + totals_to_wire, + unavailable, + validate_transition, +) +from adcp.reporting.ledger.delivery_models import ( + ReportingAdjustmentReceiptRecord, + ReportingDeliveryPrincipal, + ReportingDeliveryRecord, + ReportingDeliveryScope, + ReportingDestinationBinding, + ReportingMaterializationAttempt, + ReportingMaterializationCheck, + ReportingMaterializationKey, + ReportingMaterializationRecord, + ReportingObligationDeliveryRecord, + ReportingReceiptKey, + ReportingReceiptRecord, + ReportingRevisionReceiptRecord, +) +from adcp.reporting.ledger.models import ( + LedgerSnapshot, + ReportingAdjustmentRecord, + ReportingConfigurationGenerationKey, + ReportingObligationRecord, + ReportingRevisionRecord, +) +from adcp.reporting.ledger.store import InMemoryReportingLedgerStore + + +@runtime_checkable +class ReportingDestinationStore(Protocol): + """Trusted configuration ingestion. Never accept a destination from a receipt body.""" + + async def put_destination_binding( + self, record: ReportingDestinationBinding + ) -> tuple[ReportingDestinationBinding, bool]: ... + + async def bind_obligation_delivery( + self, record: ReportingObligationDeliveryRecord + ) -> tuple[ReportingObligationDeliveryRecord, bool]: ... + + async def get_destination_binding( + self, + *, + caller: ReportingDeliveryPrincipal, + generation_key: ReportingConfigurationGenerationKey, + ) -> ReportingDestinationBinding | None: ... + + async def get_obligation_delivery( + self, scope: ReportingDeliveryScope + ) -> ReportingObligationDeliveryRecord | None: ... + + +@runtime_checkable +class ReportingMaterializationStore(Protocol): + async def commit_materialization_attempt( + self, record: ReportingMaterializationAttempt + ) -> tuple[ReportingMaterializationAttempt, bool]: ... + + async def commit_materialization( + self, record: ReportingMaterializationRecord + ) -> tuple[ReportingMaterializationRecord, bool]: ... + + async def record_materialization_check( + self, record: ReportingMaterializationCheck + ) -> tuple[ReportingMaterializationCheck, bool]: ... + + async def get_materialization( + self, key: ReportingMaterializationKey + ) -> ReportingMaterializationView | None: ... + + +@runtime_checkable +class ReportingReceiptStore(Protocol): + """Transport-derived scope is mandatory. Exact retries precede temporal checks. + + ``received_at`` is assigned by the store; callers retry the immutable input + or the returned record. Both receipt kinds share the same scoped ID namespace. + """ + + async def record_revision_receipt( + self, record: ReportingRevisionReceiptRecord + ) -> tuple[ReportingRevisionReceiptRecord, bool]: ... + + async def record_adjustment_receipt( + self, record: ReportingAdjustmentReceiptRecord + ) -> tuple[ReportingAdjustmentReceiptRecord, bool]: ... + + async def get_receipt(self, key: ReportingReceiptKey) -> ReportingReceiptRecord | None: ... + + +@runtime_checkable +class ReportingReconciliationStore( + ReportingDestinationStore, ReportingMaterializationStore, ReportingReceiptStore, Protocol +): + async def read_reconciliation_snapshot( + self, + *, + caller: ReportingDeliveryPrincipal, + boundary: LedgerSnapshot | None = None, + ) -> ReportingReconciliationSnapshot: ... + + +@dataclass(frozen=True, slots=True) +class ReportingMaterializationView: + attempt: ReportingMaterializationAttempt + binding: ReportingDestinationBinding + outcome: ReportingMaterializationRecord | None + checks: tuple[ReportingMaterializationCheck, ...] + + @property + def check(self) -> ReportingMaterializationCheck | None: + return max(self.checks, key=lambda item: item.checked_at, default=None) + + def readable_at(self, at: datetime) -> bool: + outcome = self.outcome + check = max( + (item for item in self.checks if item.checked_at <= at), + key=lambda item: item.checked_at, + default=None, + ) + return bool( + outcome is not None + and outcome.status in {"available", "delivered"} + and outcome.resource is not None + and outcome.completed_at <= at < outcome.resource.expires_at + and (check is None or check.state == "readable") + ) + + def to_wire(self) -> dict[str, Any]: + """An inert projection for future handlers. Never exposes the trusted reference.""" + attempt, binding, outcome = self.attempt, self.binding, self.outcome + generation = attempt.scope.generation_key + result: dict[str, Any] = { + "reporting_materialization_id": attempt.reporting_materialization_id, + "reporting_revision_id": attempt.reporting_revision_id, + "reporting_obligation_id": attempt.scope.reporting_obligation_id, + "delivery_config_id": generation.delivery_config_id, + "delivery_config_version": generation.delivery_config_version, + "destination_ref": binding.destination_ref, + "feed_purpose": binding.feed_purpose, + "method": binding.method, + "transport": binding.transport, + "attempt": attempt.attempt, + "status": outcome.status if outcome is not None else "pending", + "created_at": iso(attempt.created_at), + } + if outcome is None: + return result + if outcome.status == "failed": + result.update(failed_at=iso(outcome.completed_at), failure_code=outcome.failure_code) + return result + resource, verification = outcome.resource, outcome.verification + assert resource is not None and verification is not None + descriptor = { + key: value + for key, value in asdict(resource).items() + if value is not None and key != "object_refs" + } + descriptor["expires_at"] = iso(resource.expires_at) + descriptor["reader_compatibility"] = list(resource.reader_compatibility) + if resource.kind == "manifest": + descriptor["manifest_version"] = "1.0" + evidence: dict[str, Any] = { + "verified_at": iso(verification.verified_at), + "verification_path": verification.verification_path, + "verification_profile": verification.verification_profile, + "row_count": verification.row_count, + "control_totals": totals_to_wire(verification.control_totals), + } + if verification.canonical_content_digest is not None: + evidence["canonical_content_digest"] = verification.canonical_content_digest.to_wire() + if verification.physical_checksums: + evidence["physical_checksums"] = [ + asdict(item) for item in verification.physical_checksums + ] + if verification.native_version_ref is not None: + evidence["native_commit_evidence"] = { + "native_version_ref": verification.native_version_ref, + "observed_through": verification.native_observed_through, + } + result.update( + ready_at=iso(outcome.completed_at), resource=descriptor, verification=evidence + ) + return result + + +@dataclass(frozen=True, slots=True) +class ReportingReconciliationSnapshot: + caller: ReportingDeliveryPrincipal + boundary: LedgerSnapshot + records: tuple[ReportingDeliveryRecord, ...] + + def materialization( + self, key: ReportingMaterializationKey + ) -> ReportingMaterializationView | None: + if key.principal != self.caller: + return None + attempt = next( + ( + item + for item in self.records + if isinstance(item, ReportingMaterializationAttempt) and item.key == key + ), + None, + ) + if attempt is None: + return None + binding = next( + ( + item + for item in self.records + if isinstance(item, ReportingDestinationBinding) + and item.generation_key == attempt.scope.generation_key + ), + None, + ) + if binding is None: + unavailable() + outcome = next( + ( + item + for item in self.records + if isinstance(item, ReportingMaterializationRecord) and item.key == key + ), + None, + ) + return ReportingMaterializationView( + attempt, + binding, + outcome, + tuple( + item + for item in self.records + if isinstance(item, ReportingMaterializationCheck) + and item.reporting_materialization_id == key.reporting_materialization_id + ), + ) + + @property + def current_receipts(self) -> tuple[ReportingReceiptRecord, ...]: + receipts = tuple( + item + for item in self.records + if isinstance(item, (ReportingRevisionReceiptRecord, ReportingAdjustmentReceiptRecord)) + ) + replaced = {item.supersedes_reporting_receipt_id for item in receipts} + return tuple(item for item in receipts if item.reporting_receipt_id not in replaced) + + @property + def terminal_acceptances(self) -> tuple[ReportingReceiptKey, ...]: + return tuple(item.key for item in self.current_receipts if item.status == "accepted") + + +class _ReconciliationOperations: + """Typed forwarding shared by the two storage mechanisms; not an adopter hook.""" + + async def _commit(self, record: RecordT) -> tuple[RecordT, bool]: + raise NotImplementedError + + async def put_destination_binding( + self, record: ReportingDestinationBinding + ) -> tuple[ReportingDestinationBinding, bool]: + return await self._commit(record) + + async def get_destination_binding( + self, + *, + caller: ReportingDeliveryPrincipal, + generation_key: ReportingConfigurationGenerationKey, + ) -> ReportingDestinationBinding | None: + if caller.account_id != generation_key.account_id: + return None + snapshot = await self.read_reconciliation_snapshot(caller=caller) + return next( + ( + item + for item in snapshot.records + if isinstance(item, ReportingDestinationBinding) + and item.generation_key == generation_key + ), + None, + ) + + async def get_obligation_delivery( + self, scope: ReportingDeliveryScope + ) -> ReportingObligationDeliveryRecord | None: + snapshot = await self.read_reconciliation_snapshot(caller=scope.principal) + return next( + ( + item + for item in snapshot.records + if isinstance(item, ReportingObligationDeliveryRecord) and item.scope == scope + ), + None, + ) + + async def bind_obligation_delivery( + self, record: ReportingObligationDeliveryRecord + ) -> tuple[ReportingObligationDeliveryRecord, bool]: + return await self._commit(record) + + async def commit_materialization_attempt( + self, record: ReportingMaterializationAttempt + ) -> tuple[ReportingMaterializationAttempt, bool]: + return await self._commit(record) + + async def commit_materialization( + self, record: ReportingMaterializationRecord + ) -> tuple[ReportingMaterializationRecord, bool]: + return await self._commit(record) + + async def record_materialization_check( + self, record: ReportingMaterializationCheck + ) -> tuple[ReportingMaterializationCheck, bool]: + return await self._commit(record) + + async def record_revision_receipt( + self, record: ReportingRevisionReceiptRecord + ) -> tuple[ReportingRevisionReceiptRecord, bool]: + return await self._commit(record) + + async def record_adjustment_receipt( + self, record: ReportingAdjustmentReceiptRecord + ) -> tuple[ReportingAdjustmentReceiptRecord, bool]: + return await self._commit(record) + + async def read_reconciliation_snapshot( + self, *, caller: ReportingDeliveryPrincipal, boundary: LedgerSnapshot | None = None + ) -> ReportingReconciliationSnapshot: + raise NotImplementedError + + async def get_materialization( + self, key: ReportingMaterializationKey + ) -> ReportingMaterializationView | None: + snapshot = await self.read_reconciliation_snapshot(caller=key.principal) + return snapshot.materialization(key) + + async def get_receipt(self, key: ReportingReceiptKey) -> ReportingReceiptRecord | None: + snapshot = await self.read_reconciliation_snapshot(caller=key.principal) + return next( + ( + item + for item in snapshot.records + if isinstance( + item, (ReportingRevisionReceiptRecord, ReportingAdjustmentReceiptRecord) + ) + and item.key == key + ), + None, + ) + + +class InMemoryReportingReconciliationStore(InMemoryReportingLedgerStore, _ReconciliationOperations): + """Optional reference extension. No destination/receipt services at construction.""" + + async def _commit(self, record: RecordT) -> tuple[RecordT, bool]: + candidate = decode_record(payload(record)) + who = principal(candidate) + async with self._lock: + retained = self._retained_delivery_records() + records = tuple(item[2] for item in retained if item[1] == who) + existing = replay(candidate, records) + if existing is not None: + return cast(RecordT, existing), False + context = self._delivery_context(candidate) + stored = validate_transition(candidate, records, context, self._clock()) + self._append(who.account_id, stored.kind, change_id(stored)) + retained.append((self._sequence, who, stored)) + return cast(RecordT, stored), True + + def _retained_delivery_records( + self, + ) -> list[tuple[int, ReportingDeliveryPrincipal, ReportingDeliveryRecord]]: + # Lazily allocated so construction keeps Core's exact component surface. + if not hasattr(self, "_delivery_records"): + self._delivery_records: list[ + tuple[int, ReportingDeliveryPrincipal, ReportingDeliveryRecord] + ] = [] + return self._delivery_records + + def _delivery_context(self, record: ReportingDeliveryRecord) -> DeliveryContext: + if isinstance(record, ReportingDestinationBinding): + return DeliveryContext(configuration=self._configurations.get(record.generation_key)) + obligation = self._obligations.get(record.scope.reporting_obligation_id) + revision_id = getattr(record, "reporting_revision_id", None) + if isinstance(record, ReportingAdjustmentReceiptRecord): + revision_id = record.adjusts_reporting_revision_id + revision = self._revisions.get(revision_id) if revision_id is not None else None + return DeliveryContext( + obligation=obligation, + revision=revision, + revision_obligation=( + self._obligations.get(revision.reporting_obligation_id) if revision else None + ), + adjustment=( + self._adjustments.get(record.reporting_adjustment_id) + if isinstance(record, ReportingAdjustmentReceiptRecord) + else None + ), + ) + + async def read_reconciliation_snapshot( + self, *, caller: ReportingDeliveryPrincipal, boundary: LedgerSnapshot | None = None + ) -> ReportingReconciliationSnapshot: + if boundary is None: + boundary = await self.open_snapshot( + account_id=caller.account_id, filters_fingerprint=caller.consumer_id + ) + if boundary.account_id != caller.account_id: + unavailable() + async with self._lock: + return ReportingReconciliationSnapshot( + caller, + boundary, + tuple( + record + for sequence, owner, record in self._retained_delivery_records() + if owner == caller and sequence <= boundary.max_sequence + ), + ) + + +def materialization_to_wire( + view: ReportingMaterializationView, *, obligation: ReportingObligationRecord +) -> dict[str, Any]: + if view.attempt.scope.generation_key != obligation.generation_key or ( + view.attempt.scope.reporting_obligation_id != obligation.reporting_obligation_id + ): + unavailable() + result = view.to_wire() + result["feed_purpose"] = obligation.feed_purpose + return result + + +def receipt_to_wire(record: ReportingReceiptRecord) -> dict[str, Any]: + result = payload(record) + result.pop("scope") + result.pop("kind") + if isinstance(record, ReportingRevisionReceiptRecord): + result["reporting_obligation_id"] = record.scope.reporting_obligation_id + result["observed_control_totals"] = totals_to_wire(record.observed_control_totals) + if record.observed_canonical_content_digest is not None: + result["observed_canonical_content_digest"] = ( + record.observed_canonical_content_digest.to_wire() + ) + return { + key: value + for key, value in result.items() + if value is not None and not (key == "rejection_codes" and not value) + } + + +def adjustment_to_wire(adjustment: ReportingAdjustmentRecord) -> dict[str, Any]: + """Complete immutable adjustment evidence, including reason_detail in its digest.""" + return { + **adjustment_payload(adjustment), + "canonical_adjustment_sha256": adjustment_sha256(adjustment), + } + + +def revision_to_wire( + revision: ReportingRevisionRecord, *, obligation: ReportingObligationRecord +) -> dict[str, Any]: + """Opt-in evidence projection; Core's mounted handler remains unchanged.""" + from adcp.reporting.ledger.status import _revision_to_wire + + if ( + revision.account_id != obligation.account_id + or revision.reporting_obligation_id != obligation.reporting_obligation_id + ): + unavailable() + result = _revision_to_wire(revision, obligation) + result["control_totals"] = totals_to_wire(revision_control_totals(revision, obligation)) + if revision.canonical_content_digest is not None: + result["canonical_content_digest"] = revision.canonical_content_digest.to_wire() + return result diff --git a/src/adcp/reporting/ledger/delivery_models.py b/src/adcp/reporting/ledger/delivery_models.py new file mode 100644 index 000000000..0b0fa78da --- /dev/null +++ b/src/adcp/reporting/ledger/delivery_models.py @@ -0,0 +1,476 @@ +"""Retained seller delivery/reconciliation facts. No provider clients or wire blobs. + +Every collection is a tuple, every nested value is frozen, and there is no open +metadata bag. The trusted binding reference selects configuration in a separate +credential resolver; it is never itself a credential or an authorization grant. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field, fields +from datetime import datetime +from types import UnionType +from typing import Any, ClassVar, Literal, TypeAlias, Union, get_args, get_origin, get_type_hints + +from pydantic import ConfigDict + +from adcp.reporting.currency import validate_currency +from adcp.reporting.evidence import ( + ReportingCanonicalDigest, + aware_utc, + native_version_reference, + public_reference, + sha256_value, +) +from adcp.reporting.evidence import ReportingControlTotalRecord as ReportingControlTotalRecord +from adcp.reporting.ledger.models import ReportingConfigurationGenerationKey + +DeliveryMethod = Literal["file_transfer", "dataset_share", "warehouse_materialization"] +ReportingFormat = Literal["jsonl", "csv", "parquet", "avro", "orc"] +VerificationProfile = Literal["canonical_digest", "manifest_checksums", "native_commit"] +VerificationPath = Literal["producer", "representative_consumer", "destination"] +ReceiptStatus = Literal["accepted", "rejected"] +MaterializationFailure = Literal[ + "WRITE_FAILED", "VERIFICATION_FAILED", "CONTENT_CORRUPT", "RESOURCE_UNAVAILABLE" +] + + +class _ClosedValue: + __slots__ = () + __pydantic_config__: ClassVar[ConfigDict] = ConfigDict( + extra="forbid", hide_input_in_errors=True + ) + + +def _hints(cls: type[Any]) -> dict[str, Any]: + return get_type_hints(cls) + + +def _freeze(value: Any, annotation: Any) -> Any: + origin, args = get_origin(annotation), get_args(annotation) + if origin in (UnionType, Union): + for member in args: + try: + return _freeze(value, member) + except ValueError: + continue + elif origin is Literal: + if value in args and type(value) is type(args[0]): + return value + elif origin is tuple and isinstance(value, (tuple, list)): + if len(args) == 2 and args[1] is Ellipsis: + return tuple(_freeze(item, args[0]) for item in value) + if len(args) == len(value): + return tuple(_freeze(item, expected) for item, expected in zip(value, args)) + elif type(value) is annotation: + return value + raise ValueError("reporting records require closed immutable typed values") + + +def _freeze_fields(record: Any) -> None: + hints = _hints(type(record)) + for item in fields(record): + object.__setattr__(record, item.name, _freeze(getattr(record, item.name), hints[item.name])) + + +def _positive(value: int, *, zero: bool = False) -> None: + if type(value) is not int or value < (0 if zero else 1): + raise ValueError("reporting evidence has an invalid count") + + +def _codes(values: tuple[str, ...]) -> tuple[str, ...]: + values = tuple(values) + if len(set(values)) != len(values) or any( + not isinstance(value, str) or re.fullmatch(r"[A-Z][A-Z0-9_]{0,127}", value) is None + for value in values + ): + raise ValueError("reporting rejection codes require unique safe classifications") + return values + + +@dataclass(frozen=True, slots=True) +class ReportingDeliveryPrincipal(_ClosedValue): + """Trusted account and consumer identity, resolved from authenticated transport.""" + + account_id: str + consumer_id: str + + def __post_init__(self) -> None: + _freeze_fields(self) + public_reference(self.account_id, maximum=255) + public_reference(self.consumer_id, maximum=255) + + +@dataclass(frozen=True, slots=True) +class ReportingDeliveryScope(_ClosedValue): + generation_key: ReportingConfigurationGenerationKey + consumer_id: str + reporting_obligation_id: str + + def __post_init__(self) -> None: + _freeze_fields(self) + if type(self.generation_key) is not ReportingConfigurationGenerationKey: + raise ValueError("reporting scope requires the typed configuration generation") + self.principal + public_reference(self.generation_key.delivery_config_id, maximum=64) + _positive(self.generation_key.delivery_config_version) + public_reference(self.reporting_obligation_id, maximum=255) + + @property + def principal(self) -> ReportingDeliveryPrincipal: + return ReportingDeliveryPrincipal(self.generation_key.account_id, self.consumer_id) + + +@dataclass(frozen=True, slots=True) +class ReportingMaterializationKey(_ClosedValue): + principal: ReportingDeliveryPrincipal + reporting_materialization_id: str + + def __post_init__(self) -> None: + _freeze_fields(self) + public_reference(self.reporting_materialization_id, maximum=255) + + +@dataclass(frozen=True, slots=True) +class ReportingReceiptKey(_ClosedValue): + principal: ReportingDeliveryPrincipal + reporting_receipt_id: str + + def __post_init__(self) -> None: + _freeze_fields(self) + public_reference(self.reporting_receipt_id, maximum=255) + if len(self.reporting_receipt_id) < 16: + raise ValueError("a reporting receipt identifier requires at least 16 characters") + + +@dataclass(frozen=True, slots=True) +class ReportingDestinationBinding(_ClosedValue): + """Immutable public portion of one trusted principal/configuration binding. + + ``trusted_binding_ref`` identifies an immutable trusted configuration, not a + mutable 'latest' alias. Credentials are resolved later behind that reference. + Storing this record neither configures a writer nor advertises a capability. + """ + + generation_key: ReportingConfigurationGenerationKey + consumer_id: str + destination_ref: str + trusted_binding_ref: str = field(repr=False) + method: DeliveryMethod + transport: str + verification_profile: VerificationProfile + reconciliation_mode: Literal["delivery_only", "consumer_receipt"] + feed_purpose: Literal["pacing", "analytics", "billing"] + resource_retention_days: int + created_at: datetime + format: ReportingFormat | None = None + reader_compatibility: tuple[str, ...] = () + success_status: Literal["available", "delivered"] = "available" + kind: Literal["destination_binding"] = field(default="destination_binding", kw_only=True) + + def __post_init__(self) -> None: + _freeze_fields(self) + ReportingDeliveryScope(self.generation_key, self.consumer_id, "binding") + public_reference(self.destination_ref, maximum=255) + public_reference(self.trusted_binding_ref, maximum=255) + if re.fullmatch(r"[a-z][a-z0-9_.-]{0,63}", self.transport) is None: + raise ValueError("reporting transport requires a public protocol label") + public_reference(self.transport, maximum=64) + _positive(self.resource_retention_days) + object.__setattr__(self, "created_at", aware_utc(self.created_at)) + object.__setattr__( + self, + "reader_compatibility", + tuple(public_reference(value, maximum=128) for value in self.reader_compatibility), + ) + if len(set(self.reader_compatibility)) != len(self.reader_compatibility): + raise ValueError("reader compatibility requirements must be unique") + if self.method == "file_transfer" and self.format is None: + raise ValueError("file transfer requires a declared format") + if self.method == "warehouse_materialization" and self.success_status != "delivered": + raise ValueError("warehouse materialization requires destination delivery") + if self.method == "dataset_share" and self.success_status != "available": + raise ValueError("dataset share requires representative-consumer availability") + if self.feed_purpose == "billing" and self.verification_profile != "canonical_digest": + raise ValueError("billing receipts require canonical digest verification") + + @property + def principal(self) -> ReportingDeliveryPrincipal: + return ReportingDeliveryPrincipal(self.generation_key.account_id, self.consumer_id) + + +@dataclass(frozen=True, slots=True) +class ReportingObligationDeliveryRecord(_ClosedValue): + scope: ReportingDeliveryScope + currency: str + resource_retained_until: datetime + created_at: datetime + kind: Literal["obligation_delivery"] = field(default="obligation_delivery", kw_only=True) + + def __post_init__(self) -> None: + _freeze_fields(self) + validate_currency(self.currency) + object.__setattr__(self, "resource_retained_until", aware_utc(self.resource_retained_until)) + object.__setattr__(self, "created_at", aware_utc(self.created_at)) + + +@dataclass(frozen=True, slots=True) +class ReportingMaterializationAttempt(_ClosedValue): + scope: ReportingDeliveryScope + reporting_revision_id: str + reporting_materialization_id: str + attempt: int + created_at: datetime + kind: Literal["materialization_attempt"] = field( + default="materialization_attempt", kw_only=True + ) + + def __post_init__(self) -> None: + _freeze_fields(self) + public_reference(self.reporting_revision_id, maximum=255) + public_reference(self.reporting_materialization_id, maximum=255) + _positive(self.attempt) + object.__setattr__(self, "created_at", aware_utc(self.created_at)) + + @property + def key(self) -> ReportingMaterializationKey: + return ReportingMaterializationKey(self.scope.principal, self.reporting_materialization_id) + + +@dataclass(frozen=True, slots=True) +class ReportingResourceRecord(_ClosedValue): + resource_ref: str + kind: Literal["manifest", "dataset", "warehouse_relation"] + location: str + immutability: Literal["immutable_location", "native_version"] + expires_at: datetime + native_version_ref: str | None = None + manifest_sha256: str | None = None + object_refs: tuple[str, ...] = () + reader_compatibility: tuple[str, ...] = () + + def __post_init__(self) -> None: + _freeze_fields(self) + public_reference(self.resource_ref, maximum=255) + public_reference(self.location, maximum=2048, path=True) + if self.native_version_ref is not None: + native_version_reference(self.native_version_ref) + if self.manifest_sha256 is not None: + sha256_value(self.manifest_sha256) + if self.kind == "manifest" and self.manifest_sha256 is None: + raise ValueError("manifest resources require the exact manifest digest") + if self.immutability == "native_version" and self.native_version_ref is None: + raise ValueError("native-version resources require an immutable version") + object.__setattr__(self, "expires_at", aware_utc(self.expires_at)) + object.__setattr__( + self, + "object_refs", + tuple(public_reference(value, path=True) for value in self.object_refs), + ) + object.__setattr__( + self, + "reader_compatibility", + tuple(public_reference(value, maximum=128) for value in self.reader_compatibility), + ) + if len(set(self.object_refs)) != len(self.object_refs): + raise ValueError("resource object references must be unique") + if len(set(self.reader_compatibility)) != len(self.reader_compatibility): + raise ValueError("reader compatibility requirements must be unique") + + +@dataclass(frozen=True, slots=True) +class ReportingPhysicalChecksum(_ClosedValue): + object_ref: str + algorithm: Literal["sha256", "sha512"] + value: str + + def __post_init__(self) -> None: + _freeze_fields(self) + public_reference(self.object_ref, path=True) + size = 64 if self.algorithm == "sha256" else 128 + if ( + not isinstance(self.value, str) + or re.fullmatch(rf"[a-fA-F0-9]{{{size}}}", self.value) is None + ): + raise ValueError("physical checksum length must match its algorithm") + + +def _unique_totals(totals: tuple[ReportingControlTotalRecord, ...]) -> None: + if len({item.name for item in totals}) != len(totals): + raise ValueError("reporting control total names must be unique") + + +@dataclass(frozen=True, slots=True) +class ReportingVerificationRecord(_ClosedValue): + verified_at: datetime + verification_path: VerificationPath + verification_profile: VerificationProfile + row_count: int + control_totals: tuple[ReportingControlTotalRecord, ...] + canonical_content_digest: ReportingCanonicalDigest | None = None + physical_checksums: tuple[ReportingPhysicalChecksum, ...] = () + native_version_ref: str | None = None + native_observed_through: Literal["representative_consumer", "destination"] | None = None + verified_format: ReportingFormat | None = None + + def __post_init__(self) -> None: + _freeze_fields(self) + object.__setattr__(self, "verified_at", aware_utc(self.verified_at)) + _positive(self.row_count, zero=True) + _unique_totals(self.control_totals) + object.__setattr__(self, "physical_checksums", tuple(self.physical_checksums)) + if self.native_version_ref is not None: + native_version_reference(self.native_version_ref) + if self.native_observed_through is not None and self.native_version_ref is None: + raise ValueError("native observation paths require version evidence") + + +@dataclass(frozen=True, slots=True) +class ReportingMaterializationRecord(_ClosedValue): + """One terminal outcome. The pending attempt is retained separately.""" + + scope: ReportingDeliveryScope + reporting_revision_id: str + reporting_materialization_id: str + status: Literal["available", "delivered", "failed"] + completed_at: datetime + resource: ReportingResourceRecord | None = None + verification: ReportingVerificationRecord | None = None + failure_code: MaterializationFailure | None = None + kind: Literal["materialization"] = field(default="materialization", kw_only=True) + + def __post_init__(self) -> None: + _freeze_fields(self) + public_reference(self.reporting_revision_id, maximum=255) + public_reference(self.reporting_materialization_id, maximum=255) + object.__setattr__(self, "completed_at", aware_utc(self.completed_at)) + if self.status == "failed": + if ( + self.failure_code is None + or self.resource is not None + or self.verification is not None + ): + raise ValueError( + "failed materializations retain only a safe failure classification" + ) + elif self.resource is None or self.verification is None or self.failure_code is not None: + raise ValueError( + "successful materializations require resource and verification evidence" + ) + + @property + def key(self) -> ReportingMaterializationKey: + return ReportingMaterializationKey(self.scope.principal, self.reporting_materialization_id) + + +@dataclass(frozen=True, slots=True) +class ReportingMaterializationCheck(_ClosedValue): + """Append-only storage observations; they never mutate terminal evidence.""" + + scope: ReportingDeliveryScope + reporting_materialization_id: str + check_id: str + state: Literal["readable", "unavailable", "corrupt"] + checked_at: datetime + kind: Literal["materialization_check"] = field(default="materialization_check", kw_only=True) + + def __post_init__(self) -> None: + _freeze_fields(self) + public_reference(self.reporting_materialization_id, maximum=255) + public_reference(self.check_id, maximum=255) + object.__setattr__(self, "checked_at", aware_utc(self.checked_at)) + + +@dataclass(frozen=True, slots=True) +class ReportingRevisionReceiptRecord(_ClosedValue): + scope: ReportingDeliveryScope + reporting_receipt_id: str + reporting_revision_id: str + reporting_materialization_id: str + status: ReceiptStatus + verification_profile: VerificationProfile + observed_row_count: int + observed_control_totals: tuple[ReportingControlTotalRecord, ...] + observed_at: datetime + supersedes_reporting_receipt_id: str | None = None + observed_canonical_content_digest: ReportingCanonicalDigest | None = None + observed_manifest_sha256: str | None = None + observed_native_version_ref: str | None = None + consumer_commit_ref: str | None = None + rejection_codes: tuple[str, ...] = () + received_at: datetime | None = None + kind: Literal["revision_receipt"] = field(default="revision_receipt", kw_only=True) + + def __post_init__(self) -> None: + _freeze_fields(self) + self.key + public_reference(self.reporting_revision_id, maximum=255) + public_reference(self.reporting_materialization_id, maximum=255) + _positive(self.observed_row_count, zero=True) + _unique_totals(self.observed_control_totals) + if self.observed_manifest_sha256 is not None: + sha256_value(self.observed_manifest_sha256) + if self.observed_native_version_ref is not None: + native_version_reference(self.observed_native_version_ref) + if self.consumer_commit_ref is not None: + public_reference(self.consumer_commit_ref, maximum=512) + _receipt_fields(self) + + @property + def key(self) -> ReportingReceiptKey: + return ReportingReceiptKey(self.scope.principal, self.reporting_receipt_id) + + +@dataclass(frozen=True, slots=True) +class ReportingAdjustmentReceiptRecord(_ClosedValue): + scope: ReportingDeliveryScope + reporting_receipt_id: str + reporting_adjustment_id: str + adjusts_reporting_revision_id: str + status: ReceiptStatus + observed_adjustment_sha256: str + observed_at: datetime + supersedes_reporting_receipt_id: str | None = None + rejection_codes: tuple[str, ...] = () + received_at: datetime | None = None + kind: Literal["adjustment_receipt"] = field(default="adjustment_receipt", kw_only=True) + + def __post_init__(self) -> None: + _freeze_fields(self) + self.key + public_reference(self.reporting_adjustment_id, maximum=255) + public_reference(self.adjusts_reporting_revision_id, maximum=255) + sha256_value(self.observed_adjustment_sha256) + _receipt_fields(self) + + @property + def key(self) -> ReportingReceiptKey: + return ReportingReceiptKey(self.scope.principal, self.reporting_receipt_id) + + +ReportingReceiptRecord: TypeAlias = ( + ReportingRevisionReceiptRecord | ReportingAdjustmentReceiptRecord +) + + +def _receipt_fields(record: ReportingReceiptRecord) -> None: + if record.supersedes_reporting_receipt_id is not None: + ReportingReceiptKey(record.scope.principal, record.supersedes_reporting_receipt_id) + object.__setattr__(record, "observed_at", aware_utc(record.observed_at)) + if record.received_at is not None: + object.__setattr__(record, "received_at", aware_utc(record.received_at)) + object.__setattr__(record, "rejection_codes", _codes(record.rejection_codes)) + if (record.status == "rejected") != bool(record.rejection_codes): + raise ValueError("only rejected receipts carry rejection codes") + + +ReportingDeliveryRecord: TypeAlias = ( + ReportingDestinationBinding + | ReportingObligationDeliveryRecord + | ReportingMaterializationAttempt + | ReportingMaterializationRecord + | ReportingMaterializationCheck + | ReportingRevisionReceiptRecord + | ReportingAdjustmentReceiptRecord +) diff --git a/src/adcp/reporting/ledger/delivery_pg.py b/src/adcp/reporting/ledger/delivery_pg.py new file mode 100644 index 000000000..d407c1996 --- /dev/null +++ b/src/adcp/reporting/ledger/delivery_pg.py @@ -0,0 +1,286 @@ +"""PostgreSQL reconciliation extension, sharing the Core ledger transaction/feed.""" + +from __future__ import annotations + +from typing import Any, cast + +from adcp.reporting.ledger._delivery_state import ( + DeliveryContext, + RecordT, + change_id, + decode_record, + fail, + fingerprint, + payload, + principal, + receipt_chain, + record_identity, + replay, + unavailable, + validate_transition, +) +from adcp.reporting.ledger.delivery import ( + ReportingReconciliationSnapshot, + _ReconciliationOperations, +) +from adcp.reporting.ledger.delivery_models import ( + ReportingAdjustmentReceiptRecord, + ReportingDeliveryPrincipal, + ReportingDeliveryRecord, + ReportingDestinationBinding, + ReportingMaterializationAttempt, + ReportingReceiptRecord, + ReportingRevisionReceiptRecord, +) +from adcp.reporting.ledger.models import LedgerSnapshot +from adcp.reporting.ledger.pg import ( + _ADJUSTMENT_COLUMNS, + _OBLIGATION_COLUMNS, + _REVISION_COLUMNS, + PgReportingLedgerStore, + _adjustment_from_row, + _configuration_from_row, + _json, + _obligation_from_row, + _revision_from_row, +) + + +class PgReportingReconciliationStore(PgReportingLedgerStore, _ReconciliationOperations): + """Evidence and receipt heads commit with the feed, including autocommit pools. + + All state decisions run under the same per-account transaction lock as Core + writes. Receipt replacement additionally uses a conditional head update; + accepted heads and immutable evidence are protected by database triggers. + """ + + async def _commit(self, record: RecordT) -> tuple[RecordT, bool]: + candidate = decode_record(payload(record)) + who = principal(candidate) + async with self._pool.connection() as connection: + async with connection.transaction(): + await self._lock_account(connection, who.account_id) + records = await self._records(connection, who) + existing = replay(candidate, records) + if existing is not None: + return cast(RecordT, existing), False + context = await self._delivery_context(connection, candidate) + if self._clock is not None: + now = self._clock() + else: + time_row = await ( + await connection.execute("SELECT clock_timestamp()") + ).fetchone() + assert time_row is not None + now = time_row[0] + stored = validate_transition(candidate, records, context, now) + await self._insert(connection, stored) + if isinstance( + stored, (ReportingRevisionReceiptRecord, ReportingAdjustmentReceiptRecord) + ): + await self._advance_receipt(connection, stored) + await self._append_change( + connection, who.account_id, stored.kind, change_id(stored) + ) + return cast(RecordT, stored), True + + async def _records( + self, connection: Any, who: ReportingDeliveryPrincipal, maximum: int | None = None + ) -> tuple[ReportingDeliveryRecord, ...]: + rows = await ( + await connection.execute( + "SELECT r.payload, r.content_sha256, c.seq, r.namespace, r.record_id," + " r.record_kind, r.change_id FROM reporting_reconciliation_records r" + " LEFT JOIN reporting_ledger_changes c ON c.account_id = r.account_id" + " AND c.record_kind = r.record_kind AND c.record_id = r.change_id" + " WHERE r.account_id = %s AND r.consumer_id = %s" + " AND (c.seq IS NULL OR %s::bigint IS NULL OR c.seq <= %s::bigint) ORDER BY c.seq", + (who.account_id, who.consumer_id, maximum, maximum), + ) + ).fetchall() + records = tuple(decode_record(row[0]) for row in rows) + if any( + principal(record) != who + or fingerprint(record) != row[1] + or row[2] is None + or record_identity(record) != (row[3], row[4]) + or record.kind != row[5] + or change_id(record) != row[6] + for record, row in zip(records, rows) + ): + fail("REPORTING_HISTORY_CORRUPT") + return records + + async def _delivery_context( + self, connection: Any, record: ReportingDeliveryRecord + ) -> DeliveryContext: + who = principal(record) + if isinstance(record, ReportingDestinationBinding): + generation = record.generation_key + row = await ( + await connection.execute( + "SELECT delivery_config_id, delivery_config_version, account_id," + " report_definition_id, reporting_profile, feed_purpose, required_finality," + " account_timezone, schedule, media_buy_ids, activated_at, deactivated_at," + " automated_recovery_seconds, status_retention_days, definition," + " authoritative_party" + " FROM reporting_configurations WHERE account_id = %s" + " AND delivery_config_id = %s AND delivery_config_version = %s", + ( + who.account_id, + generation.delivery_config_id, + generation.delivery_config_version, + ), + ) + ).fetchone() + return DeliveryContext(configuration=_configuration_from_row(row) if row else None) + obligation_row = await ( + await connection.execute( + f"SELECT {_OBLIGATION_COLUMNS} FROM reporting_obligations" # noqa: S608 # nosec B608 + " WHERE account_id = %s AND reporting_obligation_id = %s", + (who.account_id, record.scope.reporting_obligation_id), + ) + ).fetchone() + revision_id = getattr(record, "reporting_revision_id", None) + if isinstance(record, ReportingAdjustmentReceiptRecord): + revision_id = record.adjusts_reporting_revision_id + revision_row = await ( + await connection.execute( + f"SELECT {_REVISION_COLUMNS} FROM reporting_revisions" # noqa: S608 # nosec B608 + " WHERE account_id = %s AND reporting_revision_id = %s", + (who.account_id, revision_id), + ) + ).fetchone() + adjustment_row = None + revision_obligation_row = None + if revision_row is not None: + revision_obligation_row = await ( + await connection.execute( + f"SELECT {_OBLIGATION_COLUMNS} FROM reporting_obligations" # noqa: S608 # nosec B608 + " WHERE account_id = %s AND reporting_obligation_id = %s", + (who.account_id, revision_row[2]), + ) + ).fetchone() + if isinstance(record, ReportingAdjustmentReceiptRecord): + adjustment_row = await ( + await connection.execute( + f"SELECT {_ADJUSTMENT_COLUMNS} FROM reporting_adjustments" # noqa: S608 # nosec B608 + " WHERE account_id = %s AND reporting_adjustment_id = %s", + (who.account_id, record.reporting_adjustment_id), + ) + ).fetchone() + return DeliveryContext( + obligation=_obligation_from_row(obligation_row) if obligation_row else None, + revision=_revision_from_row(revision_row) if revision_row else None, + revision_obligation=( + _obligation_from_row(revision_obligation_row) if revision_obligation_row else None + ), + adjustment=_adjustment_from_row(adjustment_row) if adjustment_row else None, + ) + + async def _insert(self, connection: Any, record: ReportingDeliveryRecord) -> None: + who = principal(record) + namespace, record_id = record_identity(record) + generation = ( + record.generation_key + if isinstance(record, ReportingDestinationBinding) + else record.scope.generation_key + ) + receipt = ( + record + if isinstance( + record, (ReportingRevisionReceiptRecord, ReportingAdjustmentReceiptRecord) + ) + else None + ) + await connection.execute( + "INSERT INTO reporting_reconciliation_records" + " (account_id, consumer_id, namespace, record_id, record_kind, delivery_config_id," + " delivery_config_version, reporting_obligation_id, reporting_revision_id," + " reporting_materialization_id, reporting_adjustment_id, attempt_number," + " receipt_chain_key, receipt_status, supersedes_receipt_id, payload," + " content_sha256, change_id)" + " VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s," + " %s::jsonb, %s, %s)", + ( + who.account_id, + who.consumer_id, + namespace, + record_id, + record.kind, + generation.delivery_config_id, + generation.delivery_config_version, + ( + None + if isinstance(record, ReportingDestinationBinding) + else record.scope.reporting_obligation_id + ), + ( + record.adjusts_reporting_revision_id + if isinstance(record, ReportingAdjustmentReceiptRecord) + else getattr(record, "reporting_revision_id", None) + ), + getattr(record, "reporting_materialization_id", None), + ( + record.reporting_adjustment_id + if isinstance(record, ReportingAdjustmentReceiptRecord) + else None + ), + record.attempt if isinstance(record, ReportingMaterializationAttempt) else None, + receipt_chain(receipt) if receipt is not None else None, + receipt.status if receipt is not None else None, + receipt.supersedes_reporting_receipt_id if receipt is not None else None, + _json(payload(record)), + fingerprint(record), + change_id(record), + ), + ) + + async def _advance_receipt(self, connection: Any, record: ReportingReceiptRecord) -> None: + who = principal(record) + chain = receipt_chain(record) + if record.supersedes_reporting_receipt_id is None: + cursor = await connection.execute( + "INSERT INTO reporting_receipt_heads" + " (account_id, consumer_id, chain_key, receipt_id, receipt_status)" + " VALUES (%s, %s, %s, %s, %s)" + " ON CONFLICT (account_id, consumer_id, chain_key) DO NOTHING", + ( + who.account_id, + who.consumer_id, + chain, + record.reporting_receipt_id, + record.status, + ), + ) + else: + cursor = await connection.execute( + "UPDATE reporting_receipt_heads SET receipt_id = %s, receipt_status = %s," + " supersedes_receipt_id = %s" + " WHERE account_id = %s AND consumer_id = %s AND chain_key = %s" + " AND receipt_id = %s AND receipt_status = 'rejected'", + ( + record.reporting_receipt_id, + record.status, + record.supersedes_reporting_receipt_id, + who.account_id, + who.consumer_id, + chain, + record.supersedes_reporting_receipt_id, + ), + ) + if cursor.rowcount != 1: + unavailable() + + async def read_reconciliation_snapshot( + self, *, caller: ReportingDeliveryPrincipal, boundary: LedgerSnapshot | None = None + ) -> ReportingReconciliationSnapshot: + if boundary is None: + boundary = await self.open_snapshot( + account_id=caller.account_id, filters_fingerprint=caller.consumer_id + ) + if boundary.account_id != caller.account_id: + unavailable() + async with self._pool.connection() as connection: + records = await self._records(connection, caller, boundary.max_sequence) + return ReportingReconciliationSnapshot(caller, boundary, records) diff --git a/src/adcp/reporting/ledger/models.py b/src/adcp/reporting/ledger/models.py index 40d83bafe..6a8cded52 100644 --- a/src/adcp/reporting/ledger/models.py +++ b/src/adcp/reporting/ledger/models.py @@ -28,6 +28,11 @@ from typing import Any, Literal from adcp.reporting.currency import validate_currency, validate_currency_units +from adcp.reporting.evidence import ( + ReportingCanonicalDigest, + ReportingControlTotalRecord, + freeze_control_totals, +) __all__ = [ "ConsumerStatusRecord", @@ -58,7 +63,19 @@ ReportingFinality = Literal["snapshot", "official"] ReportingHealth = Literal["healthy", "waiting", "delayed", "action_required", "complete"] ReportingProductionStatus = Literal["not_due", "pending", "published", "failed"] -LedgerRecordKind = Literal["obligation", "revision", "adjustment", "consumer_status"] +LedgerRecordKind = Literal[ + "obligation", + "revision", + "adjustment", + "consumer_status", + "destination_binding", + "obligation_delivery", + "materialization_attempt", + "materialization", + "materialization_check", + "revision_receipt", + "adjustment_receipt", +] #: The five values a consumer may state about one expected period. AdCP #: 3.2.0-rc.3 adds ``content_mismatch``: reporting that arrived and parsed but @@ -417,6 +434,8 @@ def __post_init__(self) -> None: *self.definition.monetary_control_total_units, ), ) + object.__setattr__(self, "media_buy_ids", tuple(self.media_buy_ids)) + object.__setattr__(self, "package_ids", tuple(self.package_ids)) if _utc(self.scope_resolved_at) != _utc(self.period.end): raise ValueError( "scope_resolved_at must equal the period end; the denominator froze there " @@ -459,8 +478,26 @@ class ReportingRevisionRecord: readable_at_commit: bool = True source_publication_id: str | None = None source_manifest_sha256: str | None = None + # Optional managed evidence supplied by a trusted publisher before any + # destination work. Core neither computes nor requires this contract. + canonical_content_digest: ReportingCanonicalDigest | None = None + managed_control_totals: tuple[ReportingControlTotalRecord, ...] | None = None def __post_init__(self) -> None: + if ( + self.canonical_content_digest is not None + and type(self.canonical_content_digest) is not ReportingCanonicalDigest + ): + raise ValueError("managed revision evidence requires an immutable canonical digest") + object.__setattr__( + self, "control_totals", tuple(tuple(item) for item in self.control_totals) + ) + if self.managed_control_totals is not None: + object.__setattr__( + self, + "managed_control_totals", + freeze_control_totals(self.managed_control_totals, self.control_totals), + ) if self.finality == "official": if not (self.finality_basis and self.finality_policy_id and self.finalized_at): raise ValueError( @@ -503,6 +540,18 @@ class ReportingAdjustmentRecord: correction_observed_at: datetime created_at: datetime reason_detail: str | None = None + managed_control_total_deltas: tuple[ReportingControlTotalRecord, ...] | None = None + + def __post_init__(self) -> None: + object.__setattr__( + self, "control_total_deltas", tuple(tuple(item) for item in self.control_total_deltas) + ) + if self.managed_control_total_deltas is not None: + object.__setattr__( + self, + "managed_control_total_deltas", + freeze_control_totals(self.managed_control_total_deltas, self.control_total_deltas), + ) @dataclass(frozen=True) diff --git a/src/adcp/reporting/ledger/pg.py b/src/adcp/reporting/ledger/pg.py index cbd94f849..dc3eaa20d 100644 --- a/src/adcp/reporting/ledger/pg.py +++ b/src/adcp/reporting/ledger/pg.py @@ -26,8 +26,9 @@ :meth:`create_schema` creates or upgrades the schema transactionally, including the account-qualified configuration primary key for beta.15 installations. The raw DDL ships in :file:`reporting_ledger.sql` followed by -:file:`reporting_ledger_account_generations.sql` and -:file:`reporting_ledger_obligation_currency.sql`; run all three in one transaction +:file:`reporting_ledger_account_generations.sql`, +:file:`reporting_ledger_obligation_currency.sql` and +:file:`reporting_ledger_reconciliation.sql`; run all four in one transaction when using Alembic, Flyway, or psql. See :file:`docs/reporting-ledger-migration.md` for deployment and compatibility notes. @@ -73,12 +74,14 @@ import hashlib import json from collections.abc import Callable, Sequence +from copy import deepcopy from datetime import datetime, timedelta, timezone from pathlib import Path from typing import TYPE_CHECKING, Any, ClassVar, Literal from adcp.reporting.canonical_json import canonical_json_utf8_v1 from adcp.reporting.currency import require_frozen_currency +from adcp.reporting.evidence import ReportingCanonicalDigest, ReportingControlTotalRecord from adcp.reporting.ledger.health import issue_id_for_occurrence from adcp.reporting.ledger.models import ( ConsumerStatusRecord, @@ -103,8 +106,10 @@ decode_cursor, encode_cursor, issue_is_retirable, + managed_revision_metadata, reject_reserved_authoritative_party, validate_adjustment_currency, + validate_managed_revision_rows, validate_revision_currency, ) @@ -126,6 +131,7 @@ _DDL_PATH = Path(__file__).parent / "reporting_ledger.sql" _ACCOUNT_GENERATIONS_DDL_PATH = Path(__file__).parent / "reporting_ledger_account_generations.sql" _CURRENCY_DDL_PATH = Path(__file__).parent / "reporting_ledger_obligation_currency.sql" +_RECONCILIATION_DDL_PATH = Path(__file__).parent / "reporting_ledger_reconciliation.sql" __all__ = ["PG_AVAILABLE", "PgReportingLedgerStore"] @@ -176,6 +182,7 @@ async def create_schema(self) -> None: await connection.execute(_DDL_PATH.read_text()) await connection.execute(_ACCOUNT_GENERATIONS_DDL_PATH.read_text()) await connection.execute(_CURRENCY_DDL_PATH.read_text()) + await connection.execute(_RECONCILIATION_DDL_PATH.read_text()) # -- change feed ------------------------------------------------------ @@ -420,35 +427,31 @@ async def find_obligation( async def commit_revision( self, revision: ReportingRevisionRecord, rows: Sequence[dict[str, Any]] ) -> ReportingRevisionRecord: + rows = tuple(deepcopy(row) for row in rows) if revision.row_count != len(rows): raise LedgerConflictError( "ROW_COUNT_MISMATCH", f"revision declares {revision.row_count} rows but {len(rows)} were supplied", ) + validate_managed_revision_rows(revision, rows) digest = _fingerprint(_revision_payload(revision)) - async with self._pool.connection() as connection: + async with self._pool.connection() as connection, connection.transaction(): + await self._lock_account(connection, revision.account_id) existing = await ( await connection.execute( - "SELECT content_sha256 FROM reporting_revisions" + f"SELECT {_REVISION_COLUMNS}, content_sha256 FROM reporting_revisions" # noqa: S608 # nosec B608 " WHERE reporting_revision_id = %s AND account_id = %s", (revision.reporting_revision_id, revision.account_id), ) ).fetchone() if existing is not None: - if existing[0] != digest: + if existing[-1] != digest: raise LedgerConflictError( "REVISION_IMMUTABLE", f"revision {revision.reporting_revision_id} already exists with " "different content; a restatement is a new revision", ) - stored = await self.get_revision( - account_id=revision.account_id, - reporting_revision_id=revision.reporting_revision_id, - ) - assert stored is not None - return stored - - await self._lock_account(connection, revision.account_id) + return _revision_from_row(existing[:-1]) obligation = await ( await connection.execute( f"SELECT {_OBLIGATION_COLUMNS} FROM reporting_obligations" # noqa: S608 # nosec B608 @@ -464,6 +467,7 @@ async def commit_revision( validate_revision_currency(_obligation_from_row(obligation), revision, rows) if revision.supersedes_reporting_revision_id: await self._require_current_leaf(connection, revision) + write_error = None try: await connection.execute( "INSERT INTO reporting_revisions" @@ -472,9 +476,9 @@ async def commit_revision( " data_through, created_at, supersedes_reporting_revision_id," " finality_basis, finality_policy_id, finalized_at, readable," " readable_at_commit, source_publication_id, source_manifest_sha256," - " content_sha256)" + " content_sha256, canonical_content_digest, managed_control_totals)" " VALUES (%s, %s, %s, %s, %s, %s, %s::jsonb, %s, %s, %s, %s, %s, %s, %s," - " %s, %s, %s, %s, %s)", + " %s, %s, %s, %s, %s, %s::jsonb, %s::jsonb)", ( revision.reporting_revision_id, revision.account_id, @@ -495,10 +499,22 @@ async def commit_revision( revision.source_publication_id, revision.source_manifest_sha256, digest, + ( + _json(revision.canonical_content_digest.to_wire()) + if revision.canonical_content_digest is not None + else None + ), + ( + _json([item.to_wire() for item in revision.managed_control_totals]) + if revision.managed_control_totals is not None + else None + ), ), ) except Exception as error: # psycopg raises UniqueViolation subclasses - raise _translate_integrity_error(error) from error + write_error = _translate_integrity_error(error) + if write_error is not None: + raise write_error if rows: await connection.cursor().executemany( "INSERT INTO reporting_revision_rows" @@ -627,7 +643,7 @@ async def set_revision_readable( async def commit_adjustment( self, adjustment: ReportingAdjustmentRecord ) -> ReportingAdjustmentRecord: - async with self._pool.connection() as connection: + async with self._pool.connection() as connection, connection.transaction(): await self._lock_account(connection, adjustment.account_id) existing = await ( await connection.execute( @@ -637,7 +653,12 @@ async def commit_adjustment( ) ).fetchone() if existing is not None: - return _adjustment_from_row(existing) + stored = _adjustment_from_row(existing) + if stored != adjustment: + raise LedgerConflictError( + "ADJUSTMENT_IMMUTABLE", "adjustment content is immutable" + ) + return stored revision = await ( await connection.execute( "SELECT finality, reporting_obligation_id FROM reporting_revisions" @@ -673,8 +694,8 @@ async def commit_adjustment( " (reporting_adjustment_id, account_id, adjusts_reporting_revision_id," " reason_code, reason_detail, accounting_period_start," " accounting_period_end, control_total_deltas, correction_observed_at," - " created_at)" - " VALUES (%s, %s, %s, %s, %s, %s, %s, %s::jsonb, %s, %s)" + " created_at, managed_control_total_deltas)" + " VALUES (%s, %s, %s, %s, %s, %s, %s, %s::jsonb, %s, %s, %s::jsonb)" " ON CONFLICT (reporting_adjustment_id) DO NOTHING" " RETURNING reporting_adjustment_id", ( @@ -688,16 +709,24 @@ async def commit_adjustment( _json([[name, value] for name, value in adjustment.control_total_deltas]), adjustment.correction_observed_at, adjustment.created_at, + ( + _json( + [item.to_wire() for item in adjustment.managed_control_total_deltas] + ) + if adjustment.managed_control_total_deltas is not None + else None + ), ), ) ).fetchone() - if inserted is not None: - await self._append_change( - connection, - adjustment.account_id, - "adjustment", - adjustment.reporting_adjustment_id, - ) + if inserted is None: + raise LedgerConflictError("ADJUSTMENT_UNAVAILABLE", "adjustment is unavailable") + await self._append_change( + connection, + adjustment.account_id, + "adjustment", + adjustment.reporting_adjustment_id, + ) return adjustment async def list_adjustments( @@ -1099,6 +1128,8 @@ async def read_page( async def _resolve( self, connection: Any, account_id: str, kind: str, record_id: str ) -> Any | None: + if kind not in _RESOLVERS: + return None # Higher-tier records are never projected by Core. table, columns, key, builder = _RESOLVERS[kind] row = await ( await connection.execute( @@ -1220,6 +1251,8 @@ def _translate_integrity_error(error: Exception) -> LedgerConflictError: "an official revision already exists for this obligation; publish a later source " "correction as an adjustment", ) + if "reporting_revisions_pkey" in text: + return LedgerConflictError("REVISION_NOT_FOUND", "no such revision for this account") if "reporting_revisions_one_successor" in text: return LedgerConflictError( "SUPERSEDES_STALE", @@ -1259,13 +1292,14 @@ def _translate_integrity_error(error: Exception) -> LedgerConflictError: "reporting_revision_id, account_id, reporting_obligation_id, finality," " revision_content_sha256, row_count, control_totals, observed_at, data_through," " created_at, supersedes_reporting_revision_id, finality_basis, finality_policy_id," - " finalized_at, readable, readable_at_commit, source_publication_id, source_manifest_sha256" + " finalized_at, readable, readable_at_commit, source_publication_id, source_manifest_sha256," + " canonical_content_digest, managed_control_totals" ) _ADJUSTMENT_COLUMNS = ( "reporting_adjustment_id, account_id, adjusts_reporting_revision_id, reason_code," " reason_detail, accounting_period_start, accounting_period_end, control_total_deltas," - " correction_observed_at, created_at" + " correction_observed_at, created_at, managed_control_total_deltas" ) _STATUS_COLUMNS = ( @@ -1416,6 +1450,14 @@ def _revision_from_row(row: Sequence[Any]) -> ReportingRevisionRecord: readable_at_commit=row[15], source_publication_id=row[16], source_manifest_sha256=row[17], + canonical_content_digest=( + ReportingCanonicalDigest.from_wire(row[18]) if row[18] is not None else None + ), + managed_control_totals=( + tuple(ReportingControlTotalRecord.from_wire(item) for item in row[19]) + if row[19] is not None + else None + ), ) @@ -1431,6 +1473,11 @@ def _adjustment_from_row(row: Sequence[Any]) -> ReportingAdjustmentRecord: control_total_deltas=tuple((name, value) for name, value in (row[7] or [])), correction_observed_at=_utc(row[8]), created_at=_utc(row[9]), + managed_control_total_deltas=( + tuple(ReportingControlTotalRecord.from_wire(item) for item in row[10]) + if row[10] is not None + else None + ), ) @@ -1488,7 +1535,7 @@ def _consumer_status_payload(status: ConsumerStatusRecord) -> dict[str, Any]: def _revision_payload(revision: ReportingRevisionRecord) -> dict[str, Any]: - return { + payload = { "finality": revision.finality, "revision_content_sha256": revision.revision_content_sha256, "row_count": revision.row_count, @@ -1496,6 +1543,15 @@ def _revision_payload(revision: ReportingRevisionRecord) -> dict[str, Any]: "obligation": revision.reporting_obligation_id, "supersedes": revision.supersedes_reporting_revision_id, } + if revision.canonical_content_digest is not None: + payload["canonical_content_digest"] = revision.canonical_content_digest.to_wire() + if revision.managed_control_totals is not None: + payload["managed_control_totals"] = [ + item.to_wire() for item in revision.managed_control_totals + ] + if revision.canonical_content_digest is not None or revision.managed_control_totals is not None: + payload["managed_metadata"] = managed_revision_metadata(revision) + return payload _RESOLVERS: dict[str, tuple[str, str, str, Any]] = { diff --git a/src/adcp/reporting/ledger/producer.py b/src/adcp/reporting/ledger/producer.py index 3c192dfc4..ea180f5a9 100644 --- a/src/adcp/reporting/ledger/producer.py +++ b/src/adcp/reporting/ledger/producer.py @@ -43,6 +43,7 @@ require_frozen_currency, validate_currency, ) +from adcp.reporting.evidence import ReportingControlTotalRecord, freeze_control_totals from adcp.reporting.ledger.models import ( ReportingConfiguration, ReportingDeliveryEscalation, @@ -123,6 +124,7 @@ def revision_content_sha256( row_count: int, control_totals: Sequence[tuple[str, str]], reporting_rows: Sequence[dict[str, Any]], + control_total_evidence: Sequence[ReportingControlTotalRecord] | None = None, ) -> str: """The Core revision binding: JCS over the four bound fields, SHA-256. @@ -132,15 +134,25 @@ def revision_content_sha256( ``{reporting_revision_id, row_count, control_totals, reporting_rows}`` -- nothing about storage, materialization, or delivery, which is what keeps Core's digest distinct from the Managed Delivery canonicalization contract. + + New managed publishers supply ``control_total_evidence`` to bind the exact + type/unit-bearing totals exposed by status and exact reads. Omitting it keeps + the existing Core pair projection and all previously retained hashes intact. """ + totals = ( + [ + item.to_wire() + for item in freeze_control_totals(tuple(control_total_evidence), tuple(control_totals)) + ] + if control_total_evidence is not None + else [{"name": name, "value": value} for name, value in control_totals] + ) return hashlib.sha256( canonical_json_utf8_v1( { "reporting_revision_id": reporting_revision_id, "row_count": row_count, - "control_totals": [ - {"name": name, "value": value} for name, value in control_totals - ], + "control_totals": totals, "reporting_rows": [dict(row) for row in reporting_rows], } ) diff --git a/src/adcp/reporting/ledger/reporting_ledger.sql b/src/adcp/reporting/ledger/reporting_ledger.sql index 3d391954f..f4c76c4f5 100644 --- a/src/adcp/reporting/ledger/reporting_ledger.sql +++ b/src/adcp/reporting/ledger/reporting_ledger.sql @@ -1,7 +1,7 @@ -- AdCP Reliable Reporting ledger — durable obligations, revisions, and status. -- -- Run this followed by reporting_ledger_account_generations.sql and --- reporting_ledger_obligation_currency.sql in ONE +-- reporting_ledger_obligation_currency.sql and reporting_ledger_reconciliation.sql in ONE -- transaction (psql --single-transaction -f ... -f ...), or call -- PgReportingLedgerStore.create_schema(). CREATE TABLE IF NOT EXISTS alone -- does not upgrade the global configuration primary key from 8.0.0-beta.15. diff --git a/src/adcp/reporting/ledger/reporting_ledger_reconciliation.sql b/src/adcp/reporting/ledger/reporting_ledger_reconciliation.sql new file mode 100644 index 000000000..1c2f930af --- /dev/null +++ b/src/adcp/reporting/ledger/reporting_ledger_reconciliation.sql @@ -0,0 +1,169 @@ +-- Storage foundation for #1167. Apply after the account-generation and currency +-- migrations, with old writers drained. This single DO is atomic in autocommit. +-- No legacy evidence is inferred, hashed again, or backfilled. +DO $reconciliation$ +DECLARE + evidence_type OID; + evidence_default TEXT; + evidence_nullable BOOLEAN; + evidence_table TEXT; + evidence_column TEXT; +BEGIN + PERFORM pg_advisory_xact_lock(hashtext('adcp.reporting.schema'), hashtext(current_schema())); + -- Resolve prerequisite columns/keys before installing any extension tables. + PERFORM currency FROM reporting_obligations LIMIT 0; + ALTER TABLE reporting_revisions ADD COLUMN IF NOT EXISTS canonical_content_digest JSONB; + ALTER TABLE reporting_revisions ADD COLUMN IF NOT EXISTS managed_control_totals JSONB; + ALTER TABLE reporting_adjustments ADD COLUMN IF NOT EXISTS managed_control_total_deltas JSONB; + FOR evidence_table, evidence_column IN VALUES + ('reporting_revisions', 'canonical_content_digest'), + ('reporting_revisions', 'managed_control_totals'), + ('reporting_adjustments', 'managed_control_total_deltas') + LOOP + SELECT a.atttypid, pg_get_expr(d.adbin, d.adrelid), NOT a.attnotnull + INTO evidence_type, evidence_default, evidence_nullable + FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid = a.attrelid AND d.adnum = a.attnum + WHERE a.attrelid = evidence_table::regclass AND a.attname = evidence_column; + IF evidence_type <> 'jsonb'::regtype OR evidence_default IS NOT NULL OR NOT evidence_nullable THEN + RAISE EXCEPTION 'Unexpected reporting evidence column'; + END IF; + END LOOP; + + CREATE OR REPLACE FUNCTION reporting_canonical_evidence_immutable() + RETURNS TRIGGER LANGUAGE plpgsql AS $function$ + BEGIN + IF NEW.canonical_content_digest IS DISTINCT FROM OLD.canonical_content_digest + OR NEW.managed_control_totals IS DISTINCT FROM OLD.managed_control_totals THEN + RAISE EXCEPTION 'reporting canonical evidence is immutable' USING ERRCODE = '23514'; + END IF; + RETURN NEW; + END + $function$; + IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid = 'reporting_revisions'::regclass + AND tgname = 'reporting_canonical_evidence_immutable' AND NOT tgisinternal) THEN + CREATE TRIGGER reporting_canonical_evidence_immutable + BEFORE UPDATE OF canonical_content_digest, managed_control_totals ON reporting_revisions + FOR EACH ROW EXECUTE FUNCTION reporting_canonical_evidence_immutable(); + END IF; + + CREATE OR REPLACE FUNCTION reporting_adjustment_evidence_immutable() + RETURNS TRIGGER LANGUAGE plpgsql AS $function$ + BEGIN + IF NEW.managed_control_total_deltas IS DISTINCT FROM OLD.managed_control_total_deltas THEN + RAISE EXCEPTION 'reporting adjustment evidence is immutable' USING ERRCODE = '23514'; + END IF; + RETURN NEW; + END + $function$; + IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid = 'reporting_adjustments'::regclass + AND tgname = 'reporting_adjustment_evidence_immutable' AND NOT tgisinternal) THEN + CREATE TRIGGER reporting_adjustment_evidence_immutable + BEFORE UPDATE OF managed_control_total_deltas ON reporting_adjustments + FOR EACH ROW EXECUTE FUNCTION reporting_adjustment_evidence_immutable(); + END IF; + + -- The older Core identifiers remain globally unique. Every new relationship + -- nevertheless uses the account in its foreign key; a global ID is no grant. + CREATE UNIQUE INDEX IF NOT EXISTS reporting_obligations_account_identity + ON reporting_obligations(account_id, reporting_obligation_id); + CREATE UNIQUE INDEX IF NOT EXISTS reporting_revisions_account_identity + ON reporting_revisions(account_id, reporting_revision_id); + CREATE UNIQUE INDEX IF NOT EXISTS reporting_adjustments_account_identity + ON reporting_adjustments(account_id, reporting_adjustment_id); + + -- Each payload is the SDK's closed, frozen record shape, not a generated wire + -- object or provider response. Identity/join/transition fields are explicit. + CREATE TABLE IF NOT EXISTS reporting_reconciliation_records ( + account_id TEXT COLLATE "C" NOT NULL, + consumer_id TEXT COLLATE "C" NOT NULL, + namespace TEXT COLLATE "C" NOT NULL, + record_id TEXT COLLATE "C" NOT NULL, + record_kind TEXT NOT NULL CHECK (record_kind IN ( + 'destination_binding', 'obligation_delivery', 'materialization_attempt', + 'materialization', 'materialization_check', 'revision_receipt', 'adjustment_receipt')), + delivery_config_id TEXT COLLATE "C" NOT NULL, + delivery_config_version INTEGER NOT NULL, + reporting_obligation_id TEXT COLLATE "C", + reporting_revision_id TEXT COLLATE "C", + reporting_materialization_id TEXT COLLATE "C", + reporting_adjustment_id TEXT COLLATE "C", + attempt_number INTEGER CHECK (attempt_number > 0), + receipt_chain_key TEXT COLLATE "C", + receipt_status TEXT CHECK (receipt_status IN ('accepted', 'rejected')), + supersedes_receipt_id TEXT COLLATE "C", + payload JSONB NOT NULL CHECK (jsonb_typeof(payload) = 'object'), + content_sha256 TEXT COLLATE "C" NOT NULL CHECK (content_sha256 ~ '^[a-f0-9]{64}$'), + change_id TEXT COLLATE "C" NOT NULL, + PRIMARY KEY (account_id, consumer_id, namespace, record_id), + UNIQUE (account_id, record_kind, change_id), + CHECK (namespace = CASE WHEN record_kind IN ('revision_receipt', 'adjustment_receipt') + THEN 'receipt' ELSE record_kind END), + CHECK ((record_kind IN ('revision_receipt', 'adjustment_receipt')) = (receipt_status IS NOT NULL)), + CHECK ((receipt_status IS NOT NULL) = (receipt_chain_key IS NOT NULL)), + CHECK ((record_kind = 'materialization_attempt') = (attempt_number IS NOT NULL)), + CHECK (payload->>'kind' = record_kind), + FOREIGN KEY (account_id, delivery_config_id, delivery_config_version) + REFERENCES reporting_configurations(account_id, delivery_config_id, delivery_config_version), + FOREIGN KEY (account_id, reporting_obligation_id) + REFERENCES reporting_obligations(account_id, reporting_obligation_id), + FOREIGN KEY (account_id, reporting_revision_id) + REFERENCES reporting_revisions(account_id, reporting_revision_id), + FOREIGN KEY (account_id, reporting_adjustment_id) + REFERENCES reporting_adjustments(account_id, reporting_adjustment_id), + FOREIGN KEY (account_id, consumer_id, namespace, supersedes_receipt_id) + REFERENCES reporting_reconciliation_records(account_id, consumer_id, namespace, record_id) + ); + CREATE UNIQUE INDEX IF NOT EXISTS reporting_materialization_attempt_identity + ON reporting_reconciliation_records(account_id, consumer_id, reporting_obligation_id, + reporting_revision_id, attempt_number) + WHERE record_kind = 'materialization_attempt'; + CREATE UNIQUE INDEX IF NOT EXISTS reporting_receipt_one_successor + ON reporting_reconciliation_records(account_id, consumer_id, supersedes_receipt_id) + WHERE supersedes_receipt_id IS NOT NULL; + + CREATE TABLE IF NOT EXISTS reporting_receipt_heads ( + account_id TEXT COLLATE "C" NOT NULL, + consumer_id TEXT COLLATE "C" NOT NULL, + chain_key TEXT COLLATE "C" NOT NULL, + namespace TEXT NOT NULL DEFAULT 'receipt' CHECK (namespace = 'receipt'), + receipt_id TEXT COLLATE "C" NOT NULL, + receipt_status TEXT NOT NULL CHECK (receipt_status IN ('accepted', 'rejected')), + supersedes_receipt_id TEXT COLLATE "C", + PRIMARY KEY (account_id, consumer_id, chain_key), + FOREIGN KEY (account_id, consumer_id, namespace, receipt_id) + REFERENCES reporting_reconciliation_records(account_id, consumer_id, namespace, record_id) + ); + + CREATE OR REPLACE FUNCTION reporting_reconciliation_immutable() + RETURNS TRIGGER LANGUAGE plpgsql AS $function$ + BEGIN + RAISE EXCEPTION 'reporting reconciliation evidence is append-only' USING ERRCODE = '23514'; + END + $function$; + CREATE OR REPLACE FUNCTION reporting_receipt_terminal() + RETURNS TRIGGER LANGUAGE plpgsql AS $function$ + BEGIN + IF TG_OP = 'DELETE' OR OLD.receipt_status = 'accepted' + OR NEW.account_id <> OLD.account_id OR NEW.consumer_id <> OLD.consumer_id + OR NEW.chain_key <> OLD.chain_key + OR NEW.supersedes_receipt_id IS DISTINCT FROM OLD.receipt_id THEN + RAISE EXCEPTION 'reporting receipt replacement is invalid' USING ERRCODE = '23514'; + END IF; + RETURN NEW; + END + $function$; + IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid = 'reporting_reconciliation_records'::regclass + AND tgname = 'reporting_reconciliation_immutable' AND NOT tgisinternal) THEN + CREATE TRIGGER reporting_reconciliation_immutable + BEFORE UPDATE OR DELETE ON reporting_reconciliation_records + FOR EACH ROW EXECUTE FUNCTION reporting_reconciliation_immutable(); + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid = 'reporting_receipt_heads'::regclass + AND tgname = 'reporting_receipt_terminal' AND NOT tgisinternal) THEN + CREATE TRIGGER reporting_receipt_terminal + BEFORE UPDATE OR DELETE ON reporting_receipt_heads + FOR EACH ROW EXECUTE FUNCTION reporting_receipt_terminal(); + END IF; +END +$reconciliation$; diff --git a/src/adcp/reporting/ledger/status.py b/src/adcp/reporting/ledger/status.py index 11b251a09..2a2e1ff6f 100644 --- a/src/adcp/reporting/ledger/status.py +++ b/src/adcp/reporting/ledger/status.py @@ -871,6 +871,8 @@ def _revision_to_wire( ], "created_at": _iso(revision.created_at), } + if revision.managed_control_totals is not None: + payload["control_totals"] = [item.to_wire() for item in revision.managed_control_totals] if obligation is not None: payload.update( report_definition_id=obligation.report_definition_id, diff --git a/src/adcp/reporting/ledger/store.py b/src/adcp/reporting/ledger/store.py index 4f9a9ca16..0ded97290 100644 --- a/src/adcp/reporting/ledger/store.py +++ b/src/adcp/reporting/ledger/store.py @@ -36,6 +36,7 @@ import hashlib import json from collections.abc import Callable, Sequence +from copy import deepcopy from dataclasses import dataclass, replace from datetime import datetime, timezone from typing import Any, Literal, Protocol, runtime_checkable @@ -47,6 +48,7 @@ validate_currency_units, validate_monetary_content, ) +from adcp.reporting.evidence import ReportingControlTotalRecord from adcp.reporting.ledger.health import issue_id_for_occurrence from adcp.reporting.ledger.models import ( ConsumerStatusRecord, @@ -444,16 +446,64 @@ def _revision_identity(revision: ReportingRevisionRecord) -> str: Content plus binding, excluding readability -- which is mutable state about storage, not about what was published. """ - return _fingerprint( - { - "finality": revision.finality, - "revision_content_sha256": revision.revision_content_sha256, - "row_count": revision.row_count, - "control_totals": [list(item) for item in revision.control_totals], - "obligation": revision.reporting_obligation_id, - "supersedes": revision.supersedes_reporting_revision_id, - } + payload = { + "finality": revision.finality, + "revision_content_sha256": revision.revision_content_sha256, + "row_count": revision.row_count, + "control_totals": [list(item) for item in revision.control_totals], + "obligation": revision.reporting_obligation_id, + "supersedes": revision.supersedes_reporting_revision_id, + } + if revision.canonical_content_digest is not None: + payload["canonical_content_digest"] = revision.canonical_content_digest.to_wire() + if revision.managed_control_totals is not None: + payload["managed_control_totals"] = [ + item.to_wire() for item in revision.managed_control_totals + ] + if revision.canonical_content_digest is not None or revision.managed_control_totals is not None: + payload["managed_metadata"] = managed_revision_metadata(revision) + return _fingerprint(payload) + + +def managed_revision_metadata(revision: ReportingRevisionRecord) -> dict[str, Any]: + """New evidence is exact; omit this block entirely for legacy replay hashes.""" + return { + "account_id": revision.account_id, + "created_at": _utc(revision.created_at).isoformat(), + "observed_at": _utc(revision.observed_at).isoformat(), + "data_through": _utc(revision.data_through).isoformat() if revision.data_through else None, + "finality_basis": revision.finality_basis, + "finality_policy_id": revision.finality_policy_id, + "finalized_at": _utc(revision.finalized_at).isoformat() if revision.finalized_at else None, + "readable_at_commit": revision.readable_at_commit, + "source_publication_id": revision.source_publication_id, + "source_manifest_sha256": revision.source_manifest_sha256, + } + + +def validate_managed_revision_rows( + revision: ReportingRevisionRecord, rows: Sequence[dict[str, Any]] +) -> None: + """Managed expectations must start from an internally consistent Core revision. + + This checks Core's existing binding only. It does not derive or substitute the + separate managed canonical digest or fetch its pinned contract. + """ + if revision.canonical_content_digest is None and revision.managed_control_totals is None: + return + from adcp.reporting.ledger.producer import revision_content_sha256 + + actual = revision_content_sha256( + reporting_revision_id=revision.reporting_revision_id, + row_count=revision.row_count, + control_totals=revision.control_totals, + reporting_rows=rows, + control_total_evidence=revision.managed_control_totals, ) + if actual != revision.revision_content_sha256: + raise LedgerConflictError( + "REVISION_CONTENT_MISMATCH", "revision row content does not match its immutable binding" + ) def _consumer_status_identity(status: ConsumerStatusRecord) -> str: @@ -663,6 +713,7 @@ async def find_obligation( async def commit_revision( self, revision: ReportingRevisionRecord, rows: Sequence[dict[str, Any]] ) -> ReportingRevisionRecord: + rows = tuple(deepcopy(row) for row in rows) # Checked first, exactly as PgReportingLedgerStore does: a caller whose # rows do not match its own declared count must get the same code from # both stores, not whichever invariant that store happens to reach. @@ -671,10 +722,15 @@ async def commit_revision( "ROW_COUNT_MISMATCH", f"revision declares {revision.row_count} rows but {len(rows)} were supplied", ) + validate_managed_revision_rows(revision, rows) async with self._lock: identity = _revision_identity(revision) existing = self._revisions.get(revision.reporting_revision_id) if existing is not None: + if existing.account_id != revision.account_id: + raise LedgerConflictError( + "REVISION_NOT_FOUND", "no such revision for this account" + ) if self._revision_identity[revision.reporting_revision_id] != identity: raise LedgerConflictError( "REVISION_IMMUTABLE", @@ -766,7 +822,7 @@ async def read_revision_rows( window = rows[offset : offset + limit] has_more = offset + limit < len(rows) return ReportingRowPage( - rows=tuple(dict(row) for row in window), + rows=tuple(deepcopy(row) for row in window), total_count=len(rows), has_more=has_more, cursor=( @@ -793,6 +849,12 @@ async def commit_adjustment( async with self._lock: existing = self._adjustments.get(adjustment.reporting_adjustment_id) if existing is not None: + if existing.account_id != adjustment.account_id: + raise LedgerConflictError("ADJUSTMENT_UNAVAILABLE", "adjustment is unavailable") + if existing != adjustment: + raise LedgerConflictError( + "ADJUSTMENT_IMMUTABLE", "adjustment content is immutable" + ) return existing revision = self._revisions.get(adjustment.adjusts_reporting_revision_id) if revision is None or revision.account_id != adjustment.account_id: @@ -1081,7 +1143,11 @@ def _resolve(self, kind: LedgerRecordKind, record_id: str) -> Any: return self._revisions.get(record_id) if kind == "adjustment": return self._adjustments.get(record_id) - return self._statuses.get(record_id) + if kind == "consumer_status": + return self._statuses.get(record_id) + # Optional delivery records have their own retained snapshot reader. + # Core status must not count or expose them, even in a shared ledger. + return None def _in_scope( self, @@ -1230,6 +1296,8 @@ def validate_revision_currency( metric_units=definition.monetary_metric_units if definition else (), total_units=definition.monetary_control_total_units if definition else (), ) + if revision.managed_control_totals is not None: + validate_managed_total_units(obligation, revision.managed_control_totals) def validate_adjustment_currency( @@ -1245,3 +1313,19 @@ def validate_adjustment_currency( for name, value in adjustment.control_total_deltas: if name in units: monetary_decimal(value) + if adjustment.managed_control_total_deltas is not None: + validate_managed_total_units(obligation, adjustment.managed_control_total_deltas) + + +def validate_managed_total_units( + obligation: ReportingObligationRecord, totals: tuple[ReportingControlTotalRecord, ...] +) -> None: + units = {"spend": require_frozen_currency(obligation.currency)} + if obligation.definition is not None: + units.update(obligation.definition.monetary_metric_units) + units.update(obligation.definition.monetary_control_total_units) + for total in totals: + if total.name in units and total.unit is not None and total.unit != units[total.name]: + raise LedgerConflictError( + "CURRENCY_MISMATCH", "control total evidence disagrees with frozen currency" + ) diff --git a/tests/conformance/reporting/_reconciliation_support.py b/tests/conformance/reporting/_reconciliation_support.py new file mode 100644 index 000000000..b6ebf5698 --- /dev/null +++ b/tests/conformance/reporting/_reconciliation_support.py @@ -0,0 +1,239 @@ +"""One deterministic seller scenario for both reconciliation storage mechanisms.""" + +from __future__ import annotations + +import hashlib +from collections.abc import AsyncIterator +from dataclasses import dataclass, replace +from datetime import datetime, timedelta +from typing import Literal, TypeAlias + +import pytest + +from adcp.reporting.canonical_json import canonical_json_utf8_v1 +from adcp.reporting.ledger import ( + InMemoryReportingReconciliationStore, + PgReportingReconciliationStore, + ReportingCanonicalDigest, + ReportingControlTotalRecord, + ReportingDeliveryScope, + ReportingDestinationBinding, + ReportingFinality, + ReportingMaterializationAttempt, + ReportingMaterializationRecord, + ReportingObligationDeliveryRecord, + ReportingObligationRecord, + ReportingPhysicalChecksum, + ReportingResourceRecord, + ReportingRevisionReceiptRecord, + ReportingRevisionRecord, + ReportingVerificationRecord, + revision_content_sha256, +) +from adcp.reporting.ledger.delivery_models import DeliveryMethod, VerificationProfile + +from ._generation_support import ( + END, + NOW, + START, + configuration, + isolated_reporting_pool, + obligation_for, +) + +Store: TypeAlias = InMemoryReportingReconciliationStore | PgReportingReconciliationStore + + +@dataclass +class Clock: + now: datetime = NOW + + def __call__(self) -> datetime: + return self.now + + +@pytest.fixture(params=["memory", "postgres"]) +async def reconciliation_store( + request: pytest.FixtureRequest, +) -> AsyncIterator[tuple[Store, Clock]]: + clock = Clock() + if request.param == "memory": + yield InMemoryReportingReconciliationStore(clock=clock), clock + else: + async with isolated_reporting_pool() as pool: + store = PgReportingReconciliationStore(pool=pool, clock=clock) + await store.create_schema() + yield store, clock + + +@dataclass(frozen=True) +class Scenario: + binding: ReportingDestinationBinding + delivery: ReportingObligationDeliveryRecord + obligation: ReportingObligationRecord + revision: ReportingRevisionRecord + attempt: ReportingMaterializationAttempt + outcome: ReportingMaterializationRecord + receipt: ReportingRevisionReceiptRecord + + +async def scenario( + store: Store, + *, + account_id: str = "acct_a", + consumer_id: str = "buyer", + method: DeliveryMethod = "file_transfer", + profile: VerificationProfile = "canonical_digest", + billing: bool = True, + finality: ReportingFinality = "official", + reconciliation_mode: Literal["delivery_only", "consumer_receipt"] = "consumer_receipt", + control_total_evidence: tuple[ReportingControlTotalRecord, ...] | None = None, +) -> Scenario: + feed = "billing" if billing else "analytics" + config = replace(configuration(account_id), feed_purpose=feed, required_finality=finality) + await store.put_configuration(config) + obligation = replace(obligation_for(config), currency="EUR") + await store.commit_obligation(obligation) + scope = ReportingDeliveryScope( + config.generation_key, consumer_id, obligation.reporting_obligation_id + ) + binding = ReportingDestinationBinding( + generation_key=config.generation_key, + consumer_id=consumer_id, + destination_ref="destination-generation-1", + trusted_binding_ref="trusted-binding-1", + method=method, + transport="test-storage", + verification_profile=profile, + reconciliation_mode=reconciliation_mode, + feed_purpose=feed, + resource_retention_days=400, + created_at=START, + format="jsonl" if method == "file_transfer" else None, + reader_compatibility=("jsonl-v1",) if method == "file_transfer" else ("table-v1",), + success_status="delivered" if method == "warehouse_materialization" else "available", + ) + await store.put_destination_binding(binding) + delivery = ReportingObligationDeliveryRecord(scope, "EUR", END + timedelta(days=400), END) + await store.bind_obligation_delivery(delivery) + rows = [ + { + "media_buy_id": obligation.media_buy_ids[0], + "impressions": 5, + "spend": "12.50", + "currency": "EUR", + } + ] + totals = (("impressions", "5"), ("spend", "12.50")) + total_records = ( + ReportingControlTotalRecord("impressions", "5", "integer"), + ReportingControlTotalRecord("spend", "12.50", "decimal", "EUR"), + ) + if control_total_evidence is not None: + total_records = control_total_evidence + revision_id = f"revision-{account_id}" + digest = ReportingCanonicalDigest( + value=hashlib.sha256(canonical_json_utf8_v1(rows)).hexdigest(), + canonicalization_id="rows-v1", + canonicalization_uri="https://contracts.example.test/rows-v1.json", + canonicalization_sha256="b" * 64, + ) + revision = ReportingRevisionRecord( + reporting_revision_id=revision_id, + account_id=account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + finality=finality, + revision_content_sha256=revision_content_sha256( + reporting_revision_id=revision_id, + row_count=1, + control_totals=totals, + reporting_rows=rows, + control_total_evidence=total_records, + ), + row_count=1, + control_totals=totals, + observed_at=END, + data_through=END, + created_at=END + timedelta(seconds=1), + finality_basis="source_final" if finality == "official" else None, + finality_policy_id="policy-v1" if finality == "official" else None, + finalized_at=END if finality == "official" else None, + canonical_content_digest=digest, + managed_control_totals=total_records, + ) + await store.commit_revision(revision, rows) + attempt = ReportingMaterializationAttempt( + scope, revision_id, "materialization-1", 1, END + timedelta(seconds=2) + ) + await store.commit_materialization_attempt(attempt) + completed = END + timedelta(seconds=3) + resource = ReportingResourceRecord( + resource_ref="resource-1", + kind={ + "file_transfer": "manifest", + "dataset_share": "dataset", + "warehouse_materialization": "warehouse_relation", + }[method], + location="reports/official/manifest.json", + immutability="immutable_location" if method == "file_transfer" else "native_version", + expires_at=completed + timedelta(days=400), + manifest_sha256="c" * 64 if method == "file_transfer" else None, + native_version_ref=( + "version-1" if method != "file_transfer" or profile == "native_commit" else None + ), + object_refs=("reports/official/part-000.jsonl",) if method == "file_transfer" else (), + reader_compatibility=binding.reader_compatibility, + ) + verification = ReportingVerificationRecord( + verified_at=completed, + verification_path={ + "file_transfer": "producer", + "dataset_share": "representative_consumer", + "warehouse_materialization": "destination", + }[method], + verification_profile=profile, + row_count=1, + control_totals=total_records, + canonical_content_digest=digest if profile == "canonical_digest" else None, + physical_checksums=( + (ReportingPhysicalChecksum(resource.object_refs[0], "sha256", "d" * 64),) + if method == "file_transfer" + else () + ), + native_version_ref="version-1" if profile == "native_commit" else None, + native_observed_through=( + ("representative_consumer" if method == "dataset_share" else "destination") + if profile == "native_commit" + else None + ), + verified_format=binding.format, + ) + outcome = ReportingMaterializationRecord( + scope, + revision_id, + attempt.reporting_materialization_id, + binding.success_status, + completed, + resource=resource, + verification=verification, + ) + receipt = ReportingRevisionReceiptRecord( + scope=scope, + reporting_receipt_id="receipt-first-0001", + reporting_revision_id=revision_id, + reporting_materialization_id=attempt.reporting_materialization_id, + status="accepted", + verification_profile=profile, + observed_row_count=1, + observed_control_totals=total_records, + observed_canonical_content_digest=digest if profile == "canonical_digest" else None, + observed_manifest_sha256=( + resource.manifest_sha256 if profile == "manifest_checksums" else None + ), + observed_native_version_ref=( + resource.native_version_ref if profile == "native_commit" else None + ), + observed_at=END + timedelta(seconds=4), + consumer_commit_ref="load-0001", + ) + return Scenario(binding, delivery, obligation, revision, attempt, outcome, receipt) diff --git a/tests/conformance/reporting/conftest.py b/tests/conformance/reporting/conftest.py new file mode 100644 index 000000000..85e9ba903 --- /dev/null +++ b/tests/conformance/reporting/conftest.py @@ -0,0 +1,3 @@ +"""Shared storage state-machine fixture, registered for reporting conformance.""" + +from ._reconciliation_support import reconciliation_store as reconciliation_store diff --git a/tests/conformance/reporting/test_reporting_currency_migration.py b/tests/conformance/reporting/test_reporting_currency_migration.py index 5c1a5d3c8..e25725826 100644 --- a/tests/conformance/reporting/test_reporting_currency_migration.py +++ b/tests/conformance/reporting/test_reporting_currency_migration.py @@ -98,10 +98,15 @@ async def test_migration_preserves_evidence_and_quarantines_unknown_currency( *(raw_upgrade(pool) for _ in range(3)), ) after = await retained(pool) - # The ONLY record change is the explicit unknown column; all hashes, - # rows, units, issues, leases and sequence numbers are otherwise exact. + # Additive unknown evidence columns; all existing hashes, rows, units, + # issues, leases and sequence numbers are otherwise exact. for record in before["reporting_obligations"]: record["currency"] = None + for record in before["reporting_revisions"]: + record["canonical_content_digest"] = None + record["managed_control_totals"] = None + for record in before["reporting_adjustments"]: + record["managed_control_total_deltas"] = None assert after == before assert (await _primary_key(pool))[ 3 diff --git a/tests/conformance/reporting/test_reporting_generation_migration.py b/tests/conformance/reporting/test_reporting_generation_migration.py index b953e9968..d810a0f44 100644 --- a/tests/conformance/reporting/test_reporting_generation_migration.py +++ b/tests/conformance/reporting/test_reporting_generation_migration.py @@ -80,6 +80,16 @@ async def _retained_rows(pool: AsyncConnectionPool) -> dict[str, list[Any]]: for record in result[table]: if record.get("currency") is None: record.pop("currency", None) + if table == "reporting_revisions": + for record in result[table]: + if record.get("canonical_content_digest") is None: + record.pop("canonical_content_digest", None) + if record.get("managed_control_totals") is None: + record.pop("managed_control_totals", None) + if table == "reporting_adjustments": + for record in result[table]: + if record.get("managed_control_total_deltas") is None: + record.pop("managed_control_total_deltas", None) return result @@ -140,7 +150,10 @@ async def test_beta15_upgrade_preserves_all_evidence_and_survives_concurrent_boo == "PRIMARY KEY (account_id, delivery_config_id, delivery_config_version)" ) assert await _retained_rows(pool) == before - assert await _other_constraints(pool) == constraints + # Later additive migrations may add constraints, but cannot replace or + # alter any of these pre-existing physical constraint/index identities. + for original, upgraded in zip(constraints, await _other_constraints(pool)): + assert set(original) <= set(upgraded) await asyncio.gather(_raw_upgrade(pool), PgReportingLedgerStore(pool=pool).create_schema()) assert await _primary_key(pool) == upgraded_key diff --git a/tests/conformance/reporting/test_reporting_reconciliation_boundaries.py b/tests/conformance/reporting/test_reporting_reconciliation_boundaries.py new file mode 100644 index 000000000..2ef5f7a7f --- /dev/null +++ b/tests/conformance/reporting/test_reporting_reconciliation_boundaries.py @@ -0,0 +1,556 @@ +"""Shared evidence, privacy and retention boundaries, independent of persistence.""" + +from __future__ import annotations + +import asyncio +import hashlib +import json +import subprocess +import sys +from dataclasses import asdict, replace +from datetime import timedelta + +import pytest + +from adcp.reporting.canonical_json import canonical_json_utf8_v1 +from adcp.reporting.ledger import ( + LedgerConflictError, + ReportingAdjustmentReceiptRecord, + ReportingAdjustmentRecord, + ReportingControlTotalRecord, + ReportingDeliveryPrincipal, + ReportingMaterializationCheck, + ReportingMaterializationKey, + ReportingReceiptKey, + ReportingStatusCaller, + ReportingStatusHandler, + adjustment_to_wire, + revision_content_sha256, + revision_to_wire, +) +from adcp.types import ReportingMaterialization, ReportingReceipt, ReportingRevision + +from ._generation_support import END, NOW +from ._reconciliation_support import Clock, Store, scenario + + +async def test_each_immutable_record_rejects_changed_replay( + reconciliation_store: tuple[Store, Clock], +) -> None: + store, _ = reconciliation_store + s = await scenario(store) + await store.commit_materialization(s.outcome) + check = ReportingMaterializationCheck( + s.attempt.scope, s.attempt.reporting_materialization_id, "check-immutable", "readable", NOW + ) + await store.record_materialization_check(check) + receipt, _ = await store.record_revision_receipt(s.receipt) + candidates = [ + ( + store.put_destination_binding, + replace(s.binding, trusted_binding_ref="trusted-generation-2"), + ), + ( + store.bind_obligation_delivery, + replace(s.delivery, resource_retained_until=NOW + timedelta(days=900)), + ), + (store.commit_materialization_attempt, replace(s.attempt, attempt=2)), + (store.record_materialization_check, replace(check, state="corrupt")), + (store.record_revision_receipt, replace(receipt, received_at=NOW + timedelta(seconds=1))), + ] + for write, changed in candidates: + with pytest.raises(LedgerConflictError) as error: + await write(changed) + assert error.value.code == "REPORTING_IDENTITY_CONFLICT" + assert ( + await store.get_destination_binding( + caller=s.binding.principal, generation_key=s.binding.generation_key + ) + == s.binding + ) + assert await store.get_obligation_delivery(s.delivery.scope) == s.delivery + + +async def test_native_versions_preserve_decoded_value_and_require_exact_path_evidence( + reconciliation_store: tuple[Store, Clock], +) -> None: + store, _ = reconciliation_store + s = await scenario( + store, method="warehouse_materialization", profile="native_commit", billing=False + ) + native = "/decoded+native=version/" + "a" * 800 + resource = replace(s.outcome.resource, native_version_ref=native) + verification = replace(s.outcome.verification, native_version_ref=native) + outcome = replace(s.outcome, resource=resource, verification=verification) + for wrong in [ + replace(verification, native_version_ref="different-version"), + replace(verification, native_observed_through="representative_consumer"), + ]: + with pytest.raises(LedgerConflictError) as error: + await store.commit_materialization(replace(outcome, verification=wrong)) + assert error.value.code == "NATIVE_COMMIT_MISMATCH" + await store.commit_materialization(outcome) + view = await store.get_materialization(s.attempt.key) + projected = ReportingMaterialization.model_validate(view.to_wire()) + assert projected.resource.native_version_ref.root == native + with pytest.raises(LedgerConflictError) as error: + await store.record_revision_receipt(s.receipt) + assert error.value.code == "RECEIPT_EVIDENCE_MISMATCH" + receipt, _ = await store.record_revision_receipt( + replace(s.receipt, observed_native_version_ref=native) + ) + from adcp.reporting.ledger import receipt_to_wire + + assert ( + ReportingReceipt.model_validate(receipt_to_wire(receipt)).observed_native_version_ref.root + == native + ) + + +async def test_competing_attempt_ids_cannot_claim_the_same_ordinal( + reconciliation_store: tuple[Store, Clock], +) -> None: + store, _ = reconciliation_store + s = await scenario(store) + results = await asyncio.gather( + *( + store.commit_materialization_attempt( + replace(s.attempt, reporting_materialization_id=f"competing-attempt-{i}", attempt=2) + ) + for i in range(8) + ), + return_exceptions=True, + ) + assert sum(isinstance(result, tuple) for result in results) == 1 + assert all( + isinstance(result, tuple) + or ( + isinstance(result, LedgerConflictError) + and result.code == "MATERIALIZATION_ATTEMPT_CONFLICT" + ) + for result in results + ) + + +async def test_cross_account_and_principal_keys_are_indistinguishable_from_absence( + reconciliation_store: tuple[Store, Clock], +) -> None: + store, _ = reconciliation_store + owner = await scenario(store) + other = await scenario(store, account_id="acct_b", consumer_id="another-buyer") + await store.commit_materialization(owner.outcome) + await store.record_revision_receipt(owner.receipt) + for caller in [ + ReportingDeliveryPrincipal("unknown-account", "buyer"), + ReportingDeliveryPrincipal("acct_a", "unknown-consumer"), + ]: + assert ( + await store.get_destination_binding( + caller=caller, generation_key=owner.binding.generation_key + ) + is None + ) + assert ( + await store.get_obligation_delivery( + replace( + owner.delivery.scope, + consumer_id=caller.consumer_id, + generation_key=replace( + owner.binding.generation_key, account_id=caller.account_id + ), + ) + ) + is None + ) + for target in [owner.attempt.reporting_materialization_id, "missing-materialization"]: + assert ( + await store.get_materialization(ReportingMaterializationKey(caller, target)) is None + ) + for target in [owner.receipt.reporting_receipt_id, "missing-receipt-0001"]: + assert await store.get_receipt(ReportingReceiptKey(caller, target)) is None + # This principal has a valid account binding, but the foreign revision and + # obligation must still have exactly the same result as nonexistent IDs. + for field, existing, missing in [ + ("reporting_revision_id", owner.revision.reporting_revision_id, "missing-revision"), + ("reporting_obligation_id", owner.obligation.reporting_obligation_id, "missing-obligation"), + ]: + errors = [] + for target in [existing, missing]: + attempt = replace( + other.attempt, reporting_materialization_id="foreign-attempt", attempt=2 + ) + if field == "reporting_revision_id": + attempt = replace(attempt, reporting_revision_id=target) + else: + attempt = replace( + attempt, scope=replace(attempt.scope, reporting_obligation_id=target) + ) + with pytest.raises(LedgerConflictError) as error: + await store.commit_materialization_attempt(attempt) + errors.append((error.value.code, str(error.value))) + assert errors == [("REPORTING_RECORD_UNAVAILABLE", "reporting record is unavailable")] * 2 + + +async def test_receipt_kinds_share_identity_but_never_replacement_chains( + reconciliation_store: tuple[Store, Clock], +) -> None: + store, _ = reconciliation_store + s = await scenario(store) + await store.commit_materialization(s.outcome) + rejected, _ = await store.record_revision_receipt( + replace(s.receipt, status="rejected", rejection_codes=("LOAD_FAILED",)) + ) + adjustment = ReportingAdjustmentRecord( + "adjustment-namespace", + "acct_a", + s.revision.reporting_revision_id, + "source_correction", + END, + END + timedelta(days=30), + (("spend", "-1.50"),), + END + timedelta(seconds=5), + END + timedelta(seconds=6), + managed_control_total_deltas=( + ReportingControlTotalRecord("spend", "-1.50", "decimal", "EUR"), + ), + ) + await store.commit_adjustment(adjustment) + receipt = ReportingAdjustmentReceiptRecord( + s.attempt.scope, + rejected.reporting_receipt_id, + adjustment.reporting_adjustment_id, + s.revision.reporting_revision_id, + "accepted", + adjustment_to_wire(adjustment)["canonical_adjustment_sha256"], + END + timedelta(seconds=7), + ) + with pytest.raises(LedgerConflictError) as error: + await store.record_adjustment_receipt(receipt) + assert error.value.code == "REPORTING_IDENTITY_CONFLICT" + with pytest.raises(LedgerConflictError) as error: + await store.record_adjustment_receipt( + replace( + receipt, + reporting_receipt_id="adjustment-own-receipt-0001", + supersedes_reporting_receipt_id=rejected.reporting_receipt_id, + ) + ) + assert error.value.code == "REPORTING_RECORD_UNAVAILABLE" + for caller in ["other-consumer", "missing-consumer"]: + with pytest.raises(LedgerConflictError) as error: + await store.record_adjustment_receipt( + replace(receipt, scope=replace(receipt.scope, consumer_id=caller)) + ) + assert error.value.code == "REPORTING_RECORD_UNAVAILABLE" + + +async def test_readability_history_and_receipts_use_observation_time( + reconciliation_store: tuple[Store, Clock], +) -> None: + store, clock = reconciliation_store + s = await scenario(store) + await store.commit_materialization(s.outcome) + clock.now = END + timedelta(seconds=20) + corrupt = ReportingMaterializationCheck( + s.attempt.scope, + s.attempt.reporting_materialization_id, + "corrupt-check", + "corrupt", + END + timedelta(seconds=8), + ) + await store.record_materialization_check(corrupt) + observed_bad = replace(s.receipt, observed_at=END + timedelta(seconds=9)) + with pytest.raises(LedgerConflictError) as error: + await store.record_revision_receipt(observed_bad) + assert error.value.code == "MATERIALIZATION_UNREADABLE" + repaired = replace( + corrupt, check_id="repaired-check", state="readable", checked_at=END + timedelta(seconds=10) + ) + await store.record_materialization_check(repaired) + with pytest.raises(LedgerConflictError) as error: + await store.record_materialization_check(replace(corrupt, check_id="out-of-order-check")) + assert error.value.code == "REPORTING_TIME_INVALID" + view = await store.get_materialization(s.attempt.key) + assert view.readable_at(END + timedelta(seconds=7)) + assert not view.readable_at(END + timedelta(seconds=9)) + assert view.readable_at(END + timedelta(seconds=11)) + assert not view.readable_at(view.outcome.resource.expires_at) + accepted, _ = await store.record_revision_receipt( + replace(s.receipt, observed_at=END + timedelta(seconds=11)) + ) + clock.now = view.outcome.resource.expires_at + with pytest.raises(LedgerConflictError) as error: + await store.record_materialization_check( + replace(repaired, check_id="expired-check", checked_at=clock.now) + ) + assert error.value.code == "RESOURCE_RETENTION_INVALID" + assert await store.get_receipt(accepted.key) == accepted + + +async def test_caller_owned_collections_are_frozen_before_retention( + reconciliation_store: tuple[Store, Clock], +) -> None: + store, _ = reconciliation_store + s = await scenario(store) + totals = list(s.outcome.verification.control_totals) + checksums = list(s.outcome.verification.physical_checksums) + objects = list(s.outcome.resource.object_refs) + evidence = replace(s.outcome.verification, control_totals=totals, physical_checksums=checksums) + outcome = replace( + s.outcome, verification=evidence, resource=replace(s.outcome.resource, object_refs=objects) + ) + totals.clear() + checksums.clear() + objects.clear() + assert await store.commit_materialization(outcome) == (s.outcome, True) + with pytest.raises(ValueError) as error: + replace(s.receipt, observed_canonical_content_digest={"credential": "MUST_NOT_RETAIN"}) + assert "MUST_NOT_RETAIN" not in str(error.value) + snapshot = await store.read_reconciliation_snapshot(caller=s.attempt.scope.principal) + assert "MUST_NOT_RETAIN" not in json.dumps(asdict(snapshot), default=str) + + +async def test_managed_revision_replay_binds_rows_metadata_and_canonical_contract( + reconciliation_store: tuple[Store, Clock], +) -> None: + store, _ = reconciliation_store + s = await scenario(store) + page = await store.read_revision_rows( + account_id="acct_a", reporting_revision_id=s.revision.reporting_revision_id + ) + assert await store.commit_revision(s.revision, page.rows) == s.revision + writes = await asyncio.gather(*(store.commit_revision(s.revision, page.rows) for _ in range(6))) + assert all(item == s.revision for item in writes) + with pytest.raises(LedgerConflictError) as error: + await store.commit_revision(s.revision, [{**page.rows[0], "spend": "15.00"}]) + assert error.value.code == "REVISION_CONTENT_MISMATCH" + for changed in [ + replace(s.revision, finalized_at=END + timedelta(seconds=1)), + replace( + s.revision, + canonical_content_digest=replace( + s.revision.canonical_content_digest, canonicalization_id="different-contract" + ), + ), + ]: + with pytest.raises(LedgerConflictError) as error: + await store.commit_revision(changed, page.rows) + assert error.value.code == "REVISION_IMMUTABLE" + # Memory must not hand out mutable references to retained row content. + page.rows[0]["spend"] = "999.00" + retained = await store.read_revision_rows( + account_id="acct_a", reporting_revision_id=s.revision.reporting_revision_id + ) + assert retained.rows[0]["spend"] == "12.50" + + +async def test_expected_total_types_and_nonmonetary_units_survive_storage( + reconciliation_store: tuple[Store, Clock], +) -> None: + store, _ = reconciliation_store + totals = ( + ReportingControlTotalRecord("impressions", "5", "decimal", "impressions"), + ReportingControlTotalRecord("spend", "12.50", "decimal", "EUR"), + ) + s = await scenario(store, control_total_evidence=totals) + await store.commit_materialization(s.outcome) + receipt, _ = await store.record_revision_receipt(s.receipt) + retained = await store.get_revision( + account_id="acct_a", reporting_revision_id=s.revision.reporting_revision_id + ) + assert retained.managed_control_totals == totals + projected = ReportingRevision.model_validate( + revision_to_wire(retained, obligation=s.obligation) + ).model_dump(mode="json", exclude_none=True) + assert projected["control_totals"] == [total.to_wire() for total in totals] + assert receipt.observed_control_totals == totals + rows = ( + await store.read_revision_rows( + account_id="acct_a", reporting_revision_id=retained.reporting_revision_id + ) + ).rows + digest_input = { + "reporting_revision_id": retained.reporting_revision_id, + "row_count": retained.row_count, + "control_totals": projected["control_totals"], + "reporting_rows": list(rows), + } + assert ( + hashlib.sha256(canonical_json_utf8_v1(digest_input)).hexdigest() + == retained.revision_content_sha256 + ) + core = await ReportingStatusHandler(store).handle( + {"view": "periods"}, caller=ReportingStatusCaller("acct_a", "buyer") + ) + assert core["revisions"][0]["control_totals"] == projected["control_totals"] + assert core["materializations"] == core["receipts"] == [] + changed_totals = (replace(totals[0], value_type="integer"), totals[1]) + with pytest.raises(LedgerConflictError) as error: + await store.commit_revision( + replace( + retained, + managed_control_totals=changed_totals, + revision_content_sha256=revision_content_sha256( + reporting_revision_id=retained.reporting_revision_id, + row_count=retained.row_count, + control_totals=retained.control_totals, + reporting_rows=rows, + control_total_evidence=changed_totals, + ), + ), + rows, + ) + assert error.value.code == "REVISION_IMMUTABLE" + + +async def test_adjustment_hash_retains_declared_type_unit_and_rejects_changed_replay( + reconciliation_store: tuple[Store, Clock], +) -> None: + store, _ = reconciliation_store + s = await scenario(store) + total = ReportingControlTotalRecord("spend", "-1", "decimal", "EUR") + adjustment = ReportingAdjustmentRecord( + "typed-adjustment", + "acct_a", + s.revision.reporting_revision_id, + "source_correction", + END, + END + timedelta(days=30), + (("spend", "-1"),), + END + timedelta(seconds=5), + END + timedelta(seconds=6), + managed_control_total_deltas=(total,), + ) + retained = await store.commit_adjustment(adjustment) + assert retained.managed_control_total_deltas == (total,) + wire = adjustment_to_wire(retained) + assert wire["control_total_deltas"] == [total.to_wire()] + changed = replace( + adjustment, managed_control_total_deltas=(replace(total, value_type="integer"),) + ) + assert ( + adjustment_to_wire(changed)["canonical_adjustment_sha256"] + != wire["canonical_adjustment_sha256"] + ) + with pytest.raises(LedgerConflictError) as error: + await store.commit_adjustment(changed) + assert error.value.code == "ADJUSTMENT_IMMUTABLE" + with pytest.raises(LedgerConflictError) as error: + await store.commit_adjustment( + replace( + adjustment, + reporting_adjustment_id="wrong-currency-adjustment", + managed_control_total_deltas=(replace(total, unit="USD"),), + ) + ) + assert error.value.code == "CURRENCY_MISMATCH" + + +async def test_zero_row_revision_is_verifiable_and_missing_canonical_evidence_is_not( + reconciliation_store: tuple[Store, Clock], +) -> None: + store, _ = reconciliation_store + s = await scenario(store, billing=False, finality="snapshot") + totals = (("impressions", "0"), ("spend", "0.00")) + evidence = ( + ReportingControlTotalRecord("impressions", "0", "integer"), + ReportingControlTotalRecord("spend", "0.00", "decimal", "EUR"), + ) + digest = replace(s.revision.canonical_content_digest, value=hashlib.sha256(b"[]").hexdigest()) + zero = replace( + s.revision, + reporting_revision_id="empty-revision", + row_count=0, + control_totals=totals, + canonical_content_digest=digest, + managed_control_totals=evidence, + supersedes_reporting_revision_id=s.revision.reporting_revision_id, + revision_content_sha256=revision_content_sha256( + reporting_revision_id="empty-revision", + row_count=0, + control_totals=totals, + reporting_rows=[], + control_total_evidence=evidence, + ), + ) + await store.commit_revision(zero, []) + attempt = replace( + s.attempt, + reporting_revision_id=zero.reporting_revision_id, + reporting_materialization_id="empty-materialization", + ) + await store.commit_materialization_attempt(attempt) + outcome = replace( + s.outcome, + reporting_revision_id=zero.reporting_revision_id, + reporting_materialization_id=attempt.reporting_materialization_id, + resource=replace( + s.outcome.resource, + resource_ref="empty-resource", + location="reports/empty/manifest.json", + ), + verification=replace( + s.outcome.verification, + row_count=0, + control_totals=evidence, + canonical_content_digest=digest, + ), + ) + await store.commit_materialization(outcome) + accepted, _ = await store.record_revision_receipt( + replace( + s.receipt, + reporting_receipt_id="empty-revision-receipt-0001", + reporting_revision_id=zero.reporting_revision_id, + reporting_materialization_id=attempt.reporting_materialization_id, + observed_row_count=0, + observed_control_totals=evidence, + observed_canonical_content_digest=digest, + ) + ) + assert accepted.observed_row_count == 0 + unverified = replace( + zero, + reporting_revision_id="legacy-revision-without-digest", + canonical_content_digest=None, + supersedes_reporting_revision_id=zero.reporting_revision_id, + revision_content_sha256=revision_content_sha256( + reporting_revision_id="legacy-revision-without-digest", + row_count=0, + control_totals=totals, + reporting_rows=[], + control_total_evidence=evidence, + ), + ) + await store.commit_revision(unverified, []) + with pytest.raises(LedgerConflictError) as error: + await store.commit_materialization_attempt( + replace( + attempt, + reporting_revision_id=unverified.reporting_revision_id, + reporting_materialization_id="unverified-materialization", + ) + ) + assert error.value.code == "REVISION_CANONICAL_EVIDENCE_REQUIRED" + + +def test_core_import_and_startup_with_optional_dependencies_unavailable() -> None: + script = """ +import importlib.abc +import sys + +class NoOptionalProviders(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if fullname.split('.')[0] in {'psycopg', 'psycopg_pool', 'boto3', 'google'}: + raise ImportError('optional provider dependency is unavailable') + +sys.meta_path.insert(0, NoOptionalProviders()) +from adcp.reporting.ledger import ( + InMemoryReportingLedgerStore, InMemoryReportingReconciliationStore, ReportingStatusHandler, +) +ReportingStatusHandler(InMemoryReportingLedgerStore()) +ReportingStatusHandler(InMemoryReportingReconciliationStore()) +assert 'adcp.reporting.ledger.delivery_pg' not in sys.modules +""" + result = subprocess.run([sys.executable, "-c", script], capture_output=True, text=True) + assert result.returncode == 0, result.stderr diff --git a/tests/conformance/reporting/test_reporting_reconciliation_migration.py b/tests/conformance/reporting/test_reporting_reconciliation_migration.py new file mode 100644 index 000000000..19907f0db --- /dev/null +++ b/tests/conformance/reporting/test_reporting_reconciliation_migration.py @@ -0,0 +1,272 @@ +"""Literal beta.15/#1171 upgrades, database enforcement, and restart persistence.""" + +from __future__ import annotations + +import asyncio +import hashlib +import os +from dataclasses import replace +from importlib.resources import files +from pathlib import Path +from typing import Any + +import pytest + +from adcp.reporting.ledger import LedgerConflictError, PgReportingReconciliationStore + +from ._generation_support import NOW, isolated_reporting_pool +from ._reconciliation_support import scenario +from .test_reporting_currency_migration import retained + +FIXTURES = Path(__file__).resolve().parents[2] / "fixtures" +RESOURCES = files("adcp.reporting.ledger") +MIGRATION = RESOURCES.joinpath("reporting_ledger_reconciliation.sql") + + +def test_literal_stacked_schema_fixture() -> None: + # #1175 head / reviewed #1171 base 7bd5b8f6, without synthesized DDL edits. + assert hashlib.sha256((FIXTURES / "reporting_ledger_1171.sql").read_bytes()).hexdigest() == ( + "65d9b44e220f1828b48beb32a44334b956fbf27081bed72390de1c526e67df4e" + ) + + +@pytest.mark.parametrize("source", ["reporting_ledger_beta15.sql", "reporting_ledger_1171.sql"]) +@pytest.mark.parametrize("autocommit", [False, True]) +async def test_upgrade_preserves_history_and_is_concurrent_idempotent( + source: str, autocommit: bool +) -> None: + async with isolated_reporting_pool(autocommit=autocommit) as pool: + async with pool.connection() as connection: + await connection.execute((FIXTURES / source).read_text()) + await connection.execute((FIXTURES / "reporting_ledger_beta15_data.sql").read_text()) + if source == "reporting_ledger_1171.sql": + # Simulate a currency explicitly inserted by the predecessor, + # before installing its update-protection trigger. + await connection.execute("UPDATE reporting_obligations SET currency = 'EUR'") + await connection.execute( + RESOURCES.joinpath("reporting_ledger_account_generations.sql").read_text() + ) + await connection.execute( + RESOURCES.joinpath("reporting_ledger_obligation_currency.sql").read_text() + ) + before = await retained(pool) + stores = [PgReportingReconciliationStore(pool=pool, clock=lambda: NOW) for _ in range(6)] + await asyncio.gather(*(store.create_schema() for store in stores)) + expected = before + for record in expected["reporting_obligations"]: + record.setdefault("currency", None) + for record in expected["reporting_revisions"]: + record["canonical_content_digest"] = None + record["managed_control_totals"] = None + for record in expected["reporting_adjustments"]: + record["managed_control_total_deltas"] = None + assert await retained(pool) == expected + async with pool.connection() as connection: + await connection.execute(MIGRATION.read_text()) + keys_before = await ( + await connection.execute( + "SELECT oid, conname, pg_get_constraintdef(oid) FROM pg_constraint" + " WHERE connamespace = current_schema()::regnamespace ORDER BY oid" + ) + ).fetchall() + assert ( + await ( + await connection.execute( + "SELECT count(*) FROM reporting_reconciliation_records" + ) + ).fetchone() + )[0] == 0 + await stores[0].create_schema() + async with pool.connection() as connection: + assert ( + await ( + await connection.execute( + "SELECT oid, conname, pg_get_constraintdef(oid) FROM pg_constraint" + " WHERE connamespace = current_schema()::regnamespace ORDER BY oid" + ) + ).fetchall() + == keys_before + ) + assert await retained(pool) == expected + import psycopg + + for statement in [ + "UPDATE reporting_revisions SET managed_control_totals = '[]'::jsonb", + "UPDATE reporting_adjustments SET managed_control_total_deltas = '[]'::jsonb", + ]: + with pytest.raises(psycopg.errors.CheckViolation): + async with pool.connection() as connection: + await connection.execute(statement) + s = await scenario(stores[0], account_id="new-account") + await stores[0].commit_materialization(s.outcome) + receipt, _ = await stores[0].record_revision_receipt(s.receipt) + assert await stores[1].get_receipt(receipt.key) == receipt + + +@pytest.mark.parametrize("autocommit", [False, True]) +async def test_failed_extension_upgrade_rolls_back_prerequisites(autocommit: bool) -> None: + pytest.importorskip("psycopg") + import psycopg + + async with isolated_reporting_pool(autocommit=autocommit) as pool: + async with pool.connection() as connection: + await connection.execute((FIXTURES / "reporting_ledger_beta15.sql").read_text()) + await connection.execute((FIXTURES / "reporting_ledger_beta15_data.sql").read_text()) + await connection.execute( + "CREATE TABLE reporting_reconciliation_records (adopter_marker TEXT)" + ) + before = await retained(pool) + with pytest.raises(psycopg.Error): + await PgReportingReconciliationStore(pool=pool).create_schema() + assert await retained(pool) == before + async with pool.connection() as connection: + row = await ( + await connection.execute( + "SELECT count(*) FROM pg_attribute" + " WHERE attrelid = 'reporting_revisions'::regclass" + " AND attname = 'canonical_content_digest' AND NOT attisdropped" + ) + ).fetchone() + assert row[0] == 0 + + +async def test_new_evidence_and_acceptance_are_database_immutable() -> None: + pytest.importorskip("psycopg") + import psycopg + + async with isolated_reporting_pool() as pool: + store = PgReportingReconciliationStore(pool=pool, clock=lambda: NOW) + await store.create_schema() + s = await scenario(store) + await store.commit_materialization(s.outcome) + receipt, _ = await store.record_revision_receipt(s.receipt) + for statement in [ + "UPDATE reporting_reconciliation_records SET payload = '{}'::jsonb", + "DELETE FROM reporting_reconciliation_records", + "UPDATE reporting_receipt_heads SET receipt_status = 'rejected'", + "DELETE FROM reporting_receipt_heads", + "UPDATE reporting_revisions SET canonical_content_digest = NULL", + "UPDATE reporting_revisions SET managed_control_totals = NULL", + ]: + with pytest.raises(psycopg.errors.CheckViolation): + async with pool.connection() as connection: + await connection.execute(statement) + assert await store.get_receipt(receipt.key) == receipt + + +async def test_accepted_records_survive_all_application_connections_closing() -> None: + pytest.importorskip("psycopg_pool") + from psycopg_pool import AsyncConnectionPool + + async with isolated_reporting_pool() as pool: + store = PgReportingReconciliationStore(pool=pool, clock=lambda: NOW) + await store.create_schema() + s = await scenario(store) + await store.commit_materialization(s.outcome) + receipt, _ = await store.record_revision_receipt(s.receipt) + async with pool.connection() as connection: + schema = (await (await connection.execute("SELECT current_schema()")).fetchone())[0] + await pool.close() + async with AsyncConnectionPool( + os.environ["ADCP_PG_TEST_URL"], + kwargs={"options": f"-csearch_path={schema}"}, + open=False, + ) as restarted_pool: + await restarted_pool.wait() + restarted = PgReportingReconciliationStore(pool=restarted_pool, clock=lambda: NOW) + assert await restarted.record_revision_receipt(s.receipt) == (receipt, False) + assert await restarted.get_receipt(receipt.key) == receipt + with pytest.raises(LedgerConflictError) as error: + await restarted.record_revision_receipt( + replace(s.receipt, reporting_receipt_id="receipt-after-restart-0002") + ) + assert error.value.code == "ACCEPTED_RECEIPT_TERMINAL" + + +@pytest.mark.parametrize("autocommit", [False, True]) +async def test_record_receipt_head_and_feed_rollback_together(autocommit: bool) -> None: + async with isolated_reporting_pool(autocommit=autocommit) as pool: + store = PgReportingReconciliationStore(pool=pool, clock=lambda: NOW) + await store.create_schema() + s = await scenario(store) + await store.commit_materialization(s.outcome) + + class FailingStore(PgReportingReconciliationStore): + async def _append_change( + self, connection: Any, account_id: str, kind: Any, record_id: str + ) -> None: + raise RuntimeError("injected transaction failure") + + failing = FailingStore(pool=pool, clock=lambda: NOW) + before = await store.read_reconciliation_snapshot(caller=s.attempt.scope.principal) + with pytest.raises(RuntimeError, match="injected transaction failure"): + await failing.record_revision_receipt(s.receipt) + assert await store.get_receipt(s.receipt.key) is None + after = await store.read_reconciliation_snapshot(caller=s.attempt.scope.principal) + assert after.records == before.records + assert after.boundary.max_sequence == before.boundary.max_sequence + async with pool.connection() as connection: + assert ( + await ( + await connection.execute("SELECT count(*) FROM reporting_receipt_heads") + ).fetchone() + )[0] == 0 + assert (await store.record_revision_receipt(s.receipt))[1] + + +@pytest.mark.parametrize( + "corruption", + ["unknown_field", "nested_unknown_field", "generation_extra", "fingerprint", "missing_feed"], +) +async def test_corrupt_retained_evidence_fails_closed_without_echoing_payload( + corruption: str, +) -> None: + async with isolated_reporting_pool() as pool: + store = PgReportingReconciliationStore(pool=pool, clock=lambda: NOW) + await store.create_schema() + s = await scenario(store) + await store.commit_materialization(s.outcome) + async with pool.connection() as connection: + # Deliberately simulate damaged storage using the database-owner role. + # The supported SDK path cannot update/delete these immutable rows. + await connection.execute( + "ALTER TABLE reporting_reconciliation_records" + " DISABLE TRIGGER reporting_reconciliation_immutable" + ) + if corruption == "missing_feed": + await connection.execute( + "DELETE FROM reporting_ledger_changes WHERE record_kind = 'materialization'" + ) + elif corruption == "fingerprint": + await connection.execute( + "UPDATE reporting_reconciliation_records SET content_sha256 = %s" + " WHERE record_kind = 'materialization'", + ("0" * 64,), + ) + else: + path = ( + "{credential}" + if corruption == "unknown_field" + else "{verification,canonical_content_digest,credential}" + ) + if corruption == "generation_extra": + path = "{scope,generation_key,credential}" + await connection.execute( + "UPDATE reporting_reconciliation_records" + " SET payload = jsonb_set(payload, %s::text[], %s::jsonb)" + " WHERE record_kind = 'materialization'", + (path, '"MUST_NOT_RETAIN_PROVIDER_RESPONSE"'), + ) + await connection.execute( + "ALTER TABLE reporting_reconciliation_records" + " ENABLE TRIGGER reporting_reconciliation_immutable" + ) + for read in [ + store.get_materialization(s.attempt.key), + store.record_revision_receipt(s.receipt), + ]: + with pytest.raises(LedgerConflictError) as error: + await read + assert error.value.code in {"INVALID_REPORTING_RECORD", "REPORTING_HISTORY_CORRUPT"} + assert "MUST_NOT_RETAIN" not in str(error.value) + assert error.value.__cause__ is None and error.value.__context__ is None diff --git a/tests/conformance/reporting/test_reporting_reconciliation_store.py b/tests/conformance/reporting/test_reporting_reconciliation_store.py new file mode 100644 index 000000000..cc86d9721 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_reconciliation_store.py @@ -0,0 +1,403 @@ +"""The same state machine, including races and hostile references, on memory and real PG.""" + +from __future__ import annotations + +import asyncio +import hashlib +import json +from dataclasses import FrozenInstanceError, asdict, replace +from datetime import timedelta + +import pytest + +from adcp.reporting.canonical_json import canonical_json_utf8_v1 +from adcp.reporting.ledger import ( + LedgerConflictError, + ReportingAdjustmentReceiptRecord, + ReportingAdjustmentRecord, + ReportingControlTotalRecord, + ReportingDeliveryPrincipal, + ReportingMaterializationCheck, + ReportingMaterializationKey, + ReportingReceiptKey, + ReportingStatusCaller, + ReportingStatusHandler, + adjustment_to_wire, + materialization_to_wire, + receipt_to_wire, + revision_to_wire, +) +from adcp.types import ( + GetReportingStatusResponse, + ReportingMaterialization, + ReportingReceipt, + ReportingRevision, +) + +from ._generation_support import END, NOW +from ._reconciliation_support import Clock, Store, scenario + + +async def test_attempt_outcome_replay_and_generated_projections( + reconciliation_store: tuple[Store, Clock], +) -> None: + store, _ = reconciliation_store + s = await scenario(store) + before = await store.read_reconciliation_snapshot(caller=s.attempt.scope.principal) + pending = before.materialization(s.attempt.key) + assert pending is not None and pending.outcome is None and not pending.readable_at(NOW) + assert ReportingMaterialization.model_validate(pending.to_wire()).status.value == "pending" + stored, created = await store.commit_materialization(s.outcome) + assert created and stored == s.outcome + assert await store.commit_materialization(s.outcome) == (stored, False) + assert await store.commit_materialization_attempt(s.attempt) == (s.attempt, False) + assert await store.put_destination_binding(s.binding) == (s.binding, False) + assert await store.bind_obligation_delivery(s.delivery) == (s.delivery, False) + with pytest.raises(LedgerConflictError, match="retained evidence") as conflict: + await store.commit_materialization( + replace(s.outcome, completed_at=END + timedelta(seconds=5)) + ) + assert conflict.value.code == "REPORTING_IDENTITY_CONFLICT" + view = await store.get_materialization(s.attempt.key) + assert view is not None and view.readable_at(NOW) + projection = materialization_to_wire(view, obligation=s.obligation) + assert ReportingMaterialization.model_validate(projection).verification.row_count == 1 + assert s.binding.trusted_binding_ref not in json.dumps(projection) + ReportingRevision.model_validate(revision_to_wire(s.revision, obligation=s.obligation)) + historical = await store.read_reconciliation_snapshot( + caller=s.attempt.scope.principal, boundary=before.boundary + ) + assert historical == before + + +@pytest.mark.parametrize( + "mutation,code", + [ + ("rows", "VERIFICATION_TOTALS_MISMATCH"), + ("totals", "VERIFICATION_TOTALS_MISMATCH"), + ("digest", "VERIFICATION_DIGEST_MISMATCH"), + ("format", "VERIFICATION_PROFILE_MISMATCH"), + ("profile", "VERIFICATION_PROFILE_MISMATCH"), + ("time", "VERIFICATION_TIME_MISMATCH"), + ("path", "MATERIALIZATION_STATUS_MISMATCH"), + ("checksums", "PHYSICAL_CHECKSUMS_REQUIRED"), + ("objects", "PHYSICAL_CHECKSUM_BINDING_MISMATCH"), + ("retention", "RESOURCE_RETENTION_INVALID"), + ("reader", "READER_COMPATIBILITY_MISMATCH"), + ], +) +async def test_verification_gate( + reconciliation_store: tuple[Store, Clock], mutation: str, code: str +) -> None: + store, _ = reconciliation_store + s = await scenario(store) + v, r = s.outcome.verification, s.outcome.resource + assert v is not None and r is not None and v.canonical_content_digest is not None + outcome = s.outcome + if mutation == "rows": + v = replace(v, row_count=2) + if mutation == "totals": + v = replace( + v, control_totals=(replace(v.control_totals[0], value="6"), v.control_totals[1]) + ) + if mutation == "digest": + v = replace(v, canonical_content_digest=replace(v.canonical_content_digest, value="0" * 64)) + if mutation == "format": + v = replace(v, verified_format="csv") + if mutation == "profile": + v = replace(v, verification_profile="native_commit") + if mutation == "time": + v = replace(v, verified_at=END) + if mutation == "path": + outcome = replace(outcome, status="delivered") + if mutation == "checksums": + v = replace(v, physical_checksums=()) + if mutation == "objects": + r = replace(r, object_refs=("another/part.jsonl",)) + if mutation == "retention": + r = replace(r, expires_at=NOW) + if mutation == "reader": + r = replace(r, reader_compatibility=()) + with pytest.raises(LedgerConflictError) as error: + await store.commit_materialization(replace(outcome, verification=v, resource=r)) + assert error.value.code == code + assert (await store.get_materialization(s.attempt.key)).outcome is None + await store.commit_materialization(s.outcome) + + +async def test_rejected_replacement_and_terminal_acceptance( + reconciliation_store: tuple[Store, Clock], +) -> None: + store, _ = reconciliation_store + s = await scenario(store) + await store.commit_materialization(s.outcome) + rejected = replace( + s.receipt, status="rejected", observed_row_count=99, rejection_codes=("ROW_COUNT_MISMATCH",) + ) + first, written = await store.record_revision_receipt(rejected) + assert written and first.received_at == NOW + assert await store.record_revision_receipt(rejected) == (first, False) + replacement = replace( + s.receipt, + reporting_receipt_id="receipt-replacement-0002", + supersedes_reporting_receipt_id=rejected.reporting_receipt_id, + ) + accepted, written = await store.record_revision_receipt(replacement) + assert written + assert await store.record_revision_receipt(first) == (first, False) + assert await store.record_revision_receipt(replacement) == (accepted, False) + assert await store.record_revision_receipt(accepted) == (accepted, False) + ReportingReceipt.model_validate(receipt_to_wire(first)) + ReportingReceipt.model_validate(receipt_to_wire(accepted)) + with pytest.raises(LedgerConflictError) as conflict: + await store.record_revision_receipt( + replace( + replacement, + reporting_receipt_id="receipt-third-0003", + supersedes_reporting_receipt_id=accepted.reporting_receipt_id, + ) + ) + assert conflict.value.code == "ACCEPTED_RECEIPT_TERMINAL" + with pytest.raises(LedgerConflictError) as conflict: + await store.record_revision_receipt( + replace(replacement, consumer_commit_ref="another-load") + ) + assert conflict.value.code == "REPORTING_IDENTITY_CONFLICT" + snapshot = await store.read_reconciliation_snapshot(caller=s.attempt.scope.principal) + assert snapshot.current_receipts == (accepted,) + assert snapshot.terminal_acceptances == (accepted.key,) + + +async def test_retry_materialization_does_not_reset_receipt_chain( + reconciliation_store: tuple[Store, Clock], +) -> None: + store, _ = reconciliation_store + s = await scenario(store) + await store.commit_materialization(s.outcome) + rejected, _ = await store.record_revision_receipt( + replace(s.receipt, status="rejected", rejection_codes=("CONTENT_MISMATCH",)) + ) + retry = replace(s.attempt, reporting_materialization_id="materialization-2", attempt=2) + await store.commit_materialization_attempt(retry) + await store.commit_materialization( + replace(s.outcome, reporting_materialization_id=retry.reporting_materialization_id) + ) + receipt = replace( + s.receipt, + reporting_receipt_id="receipt-for-retry-0002", + reporting_materialization_id=retry.reporting_materialization_id, + ) + with pytest.raises(LedgerConflictError) as error: + await store.record_revision_receipt(receipt) + assert error.value.code == "REPORTING_RECORD_UNAVAILABLE" + accepted, _ = await store.record_revision_receipt( + replace(receipt, supersedes_reporting_receipt_id=rejected.reporting_receipt_id) + ) + assert accepted.status == "accepted" + + +async def test_corruption_and_retention_preserve_receipts_and_snapshots( + reconciliation_store: tuple[Store, Clock], +) -> None: + store, clock = reconciliation_store + s = await scenario(store) + await store.commit_materialization(s.outcome) + accepted, _ = await store.record_revision_receipt(s.receipt) + before = await store.read_reconciliation_snapshot(caller=s.attempt.scope.principal) + corrupt = ReportingMaterializationCheck( + s.attempt.scope, s.attempt.reporting_materialization_id, "check-1", "corrupt", NOW + ) + assert (await store.record_materialization_check(corrupt))[1] + assert not (await store.record_materialization_check(corrupt))[1] + assert not (await store.get_materialization(s.attempt.key)).readable_at(NOW) + assert ( + await store.read_reconciliation_snapshot( + caller=s.attempt.scope.principal, boundary=before.boundary + ) + ) == before + assert before.materialization(s.attempt.key).readable_at(NOW) + clock.now = NOW + timedelta(days=500) + await store.set_revision_readable( + account_id=s.revision.account_id, + reporting_revision_id=s.revision.reporting_revision_id, + readable=False, + ) + assert await store.record_revision_receipt(s.receipt) == (accepted, False) + assert await store.commit_materialization(s.outcome) == (s.outcome, False) + assert await store.get_receipt(accepted.key) == accepted + assert not (await store.get_materialization(s.attempt.key)).readable_at(clock.now) + + +async def test_concurrent_workers_converge_without_terminal_races( + reconciliation_store: tuple[Store, Clock], +) -> None: + store, _ = reconciliation_store + s = await scenario(store) + writes = await asyncio.gather(*(store.commit_materialization(s.outcome) for _ in range(12))) + assert sum(created for _, created in writes) == 1 + rejected, _ = await store.record_revision_receipt( + replace(s.receipt, status="rejected", rejection_codes=("CONTENT_MISMATCH",)) + ) + receipts = [ + replace( + s.receipt, + reporting_receipt_id=f"receipt-concurrent-{i:04}", + supersedes_reporting_receipt_id=rejected.reporting_receipt_id, + ) + for i in range(12) + ] + outcomes = await asyncio.gather( + *(store.record_revision_receipt(item) for item in receipts), return_exceptions=True + ) + assert sum(isinstance(item, tuple) for item in outcomes) == 1 + assert all( + isinstance(item, tuple) or isinstance(item, LedgerConflictError) for item in outcomes + ) + snapshot = await store.read_reconciliation_snapshot(caller=s.attempt.scope.principal) + assert len(snapshot.terminal_acceptances) == 1 + assert len([r for r in snapshot.records if r.kind == "revision_receipt"]) == 2 + + +async def test_account_principal_isolation_and_shared_identifiers( + reconciliation_store: tuple[Store, Clock], +) -> None: + store, _ = reconciliation_store + first = await scenario(store) + other_account = await scenario(store, account_id="acct_b") + other_principal = await scenario(store, consumer_id="other-buyer") + for s in [first, other_account, other_principal]: + await store.commit_materialization(s.outcome) + await store.record_revision_receipt(s.receipt) + for s in [first, other_account, other_principal]: + records = ( + await store.read_reconciliation_snapshot(caller=s.attempt.scope.principal) + ).records + assert all( + (r.principal if hasattr(r, "principal") else r.scope.principal) + == s.attempt.scope.principal + for r in records + ) + assert (await store.get_receipt(s.receipt.key)).scope == s.receipt.scope + stranger = ReportingDeliveryPrincipal("acct_a", "stranger") + assert ( + await store.get_receipt(ReportingReceiptKey(stranger, first.receipt.reporting_receipt_id)) + is None + ) + assert ( + await store.get_materialization( + ReportingMaterializationKey(stranger, first.attempt.reporting_materialization_id) + ) + is None + ) + errors = [] + for revision_id in [first.revision.reporting_revision_id, "unknown-revision"]: + with pytest.raises(LedgerConflictError) as error: + await store.record_revision_receipt( + replace( + first.receipt, + scope=replace(first.attempt.scope, consumer_id="stranger"), + reporting_revision_id=revision_id, + ) + ) + errors.append((error.value.code, str(error.value))) + assert errors[0] == errors[1] + + +async def test_adjustment_digest_finality_and_independent_receipt_chain( + reconciliation_store: tuple[Store, Clock], +) -> None: + store, _ = reconciliation_store + s = await scenario(store) + adjustment = ReportingAdjustmentRecord( + "adjustment-1", + s.obligation.account_id, + s.revision.reporting_revision_id, + "source_correction", + END, + END + timedelta(days=30), + (("spend", "-1.50"),), + END + timedelta(seconds=5), + END + timedelta(seconds=6), + reason_detail="Late correction", + managed_control_total_deltas=( + ReportingControlTotalRecord("spend", "-1.50", "decimal", "EUR"), + ), + ) + await store.commit_adjustment(adjustment) + wire = adjustment_to_wire(adjustment) + expected = wire.pop("canonical_adjustment_sha256") + assert expected == hashlib.sha256(canonical_json_utf8_v1(wire)).hexdigest() + wire["canonical_adjustment_sha256"] = expected + GetReportingStatusResponse.model_validate({"status": "completed", "adjustments": [wire]}) + record = ReportingAdjustmentReceiptRecord( + s.attempt.scope, + "adjustment-receipt-0001", + adjustment.reporting_adjustment_id, + s.revision.reporting_revision_id, + "accepted", + expected, + END + timedelta(seconds=7), + ) + # The schema defines independent evidence; it does not require acceptance + # of the official receipt first. Completion still needs both receipts. + accepted, _ = await store.record_adjustment_receipt(record) + GetReportingStatusResponse.model_validate( + {"status": "completed", "adjustment_receipts": [receipt_to_wire(accepted)]} + ) + assert await store.record_adjustment_receipt(record) == (accepted, False) + with pytest.raises(LedgerConflictError) as error: + await store.record_adjustment_receipt( + replace( + record, + reporting_receipt_id="adjustment-receipt-0002", + supersedes_reporting_receipt_id=record.reporting_receipt_id, + ) + ) + assert error.value.code == "ACCEPTED_RECEIPT_TERMINAL" + await store.commit_materialization(s.outcome) + await store.record_revision_receipt(s.receipt) + assert ( + len( + ( + await store.read_reconciliation_snapshot(caller=s.attempt.scope.principal) + ).terminal_acceptances + ) + == 2 + ) + + +async def test_core_handler_keeps_empty_higher_tier_projection( + reconciliation_store: tuple[Store, Clock], +) -> None: + store, _ = reconciliation_store + s = await scenario(store) + await store.commit_materialization(s.outcome) + await store.record_revision_receipt(s.receipt) + result = await ReportingStatusHandler(store).handle( + {"view": "periods"}, caller=ReportingStatusCaller("acct_a", "buyer") + ) + assert result["materializations"] == result["receipts"] == [] + assert result["periods"][0]["reconciliation_mode"] == "delivery_only" + + +async def test_immutable_records_and_public_metadata_rejection( + reconciliation_store: tuple[Store, Clock], caplog: pytest.LogCaptureFixture +) -> None: + store, _ = reconciliation_store + s = await scenario(store) + assert s.outcome.resource is not None + with pytest.raises((FrozenInstanceError, AttributeError)): + s.outcome.status = "failed" + sentinel = "Bearer DO_NOT_RETAIN_123" + for metadata in [ + sentinel, + "https://example.test/data?X-Amz-Signature=DO_NOT_RETAIN_123", + "password=test", + ]: + with pytest.raises(ValueError) as error: + replace(s.outcome.resource, location=metadata) + assert metadata not in str(error.value) + assert metadata not in caplog.text + assert "DO_NOT_RETAIN_123" not in str(error.value) + assert "DO_NOT_RETAIN_123" not in caplog.text + assert "DO_NOT_RETAIN_123" not in json.dumps(asdict(s.outcome), default=str) diff --git a/tests/conformance/reporting/test_reporting_reconciliation_transitions.py b/tests/conformance/reporting/test_reporting_reconciliation_transitions.py new file mode 100644 index 000000000..f6609960e --- /dev/null +++ b/tests/conformance/reporting/test_reporting_reconciliation_transitions.py @@ -0,0 +1,391 @@ +"""Additional shared financial-state and wire/buyer constraints for both stores.""" + +from __future__ import annotations + +import asyncio +from dataclasses import replace +from datetime import timedelta + +import pytest + +from adcp.reporting import ReportingLedger, evaluate_reporting_ledger +from adcp.reporting.ledger import ( + LedgerConflictError, + ReportingAdjustmentReceiptRecord, + ReportingAdjustmentRecord, + ReportingControlTotalRecord, + ReportingDeliveryScope, + ReportingMaterializationRecord, + ReportingStatusCaller, + ReportingStatusHandler, + adjustment_to_wire, + receipt_to_wire, + revision_content_sha256, + revision_to_wire, +) +from adcp.reporting.ledger.delivery_models import DeliveryMethod, VerificationProfile +from adcp.types import GetReportingStatusResponse, ReportingReceipt + +from ._generation_support import END, NOW, configuration +from ._reconciliation_support import Clock, Store, scenario + + +@pytest.mark.parametrize( + "method,profile", + [ + ("file_transfer", "canonical_digest"), + ("file_transfer", "manifest_checksums"), + ("dataset_share", "canonical_digest"), + ("dataset_share", "native_commit"), + ("warehouse_materialization", "canonical_digest"), + ("warehouse_materialization", "native_commit"), + ], +) +async def test_generated_projections_are_accepted_by_buyer_reconciler( + reconciliation_store: tuple[Store, Clock], method: DeliveryMethod, profile: VerificationProfile +) -> None: + store, _ = reconciliation_store + s = await scenario(store, method=method, profile=profile, billing=profile == "canonical_digest") + await store.commit_materialization(s.outcome) + receipt, _ = await store.record_revision_receipt(s.receipt) + view = await store.get_materialization(s.attempt.key) + assert view is not None and view.readable_at(NOW) + # Construct a future periods projection locally. The real mounted handler + # deliberately continues serving Core with empty higher-tier arrays. + result = await ReportingStatusHandler(store).handle( + {"view": "periods"}, caller=ReportingStatusCaller("acct_a", "buyer") + ) + result["periods"][0].update( + destination_ref=s.binding.destination_ref, + reconciliation_mode="consumer_receipt", + reconciliation_status="accepted", + materialization_count=1, + successful_materialization_count=1, + receipt_count=1, + accepted_receipt_count=1, + resource_retained_until=s.delivery.resource_retained_until.isoformat(), + ) + result["revisions"] = [revision_to_wire(s.revision, obligation=s.obligation)] + result["materializations"] = [view.to_wire()] + result["receipts"] = [receipt_to_wire(receipt)] + result["pagination"]["total_count"] = 4 + response = GetReportingStatusResponse.model_validate(result) + ledger = ReportingLedger( + response.ledger_snapshot_id, + response.ledger_as_of, + response.account_id, + response.scope, + response.periods, + response.revisions, + response.materializations, + response.receipts, + ) + verdict = evaluate_reporting_ledger(ledger, expected_periods=[], now=NOW) + assert verdict.definitive, verdict.obligations + + +@pytest.mark.parametrize( + "field", ["currency", "value_type", "decimal_spelling", "digest", "row_count"] +) +async def test_acceptance_requires_exact_evidence_and_rejection_preserves_disagreement( + reconciliation_store: tuple[Store, Clock], field: str +) -> None: + store, _ = reconciliation_store + s = await scenario(store) + await store.commit_materialization(s.outcome) + receipt = s.receipt + totals = list(receipt.observed_control_totals) + if field == "currency": + totals[1] = replace(totals[1], unit="USD") + if field == "value_type": + totals[0] = replace(totals[0], value_type="decimal") + if field == "decimal_spelling": + totals[1] = replace(totals[1], value="12.5") + receipt = replace(receipt, observed_control_totals=tuple(totals)) + if field == "digest": + receipt = replace( + receipt, + observed_canonical_content_digest=replace( + receipt.observed_canonical_content_digest, value="0" * 64 + ), + ) + if field == "row_count": + receipt = replace(receipt, observed_row_count=0) + with pytest.raises(LedgerConflictError) as error: + await store.record_revision_receipt(receipt) + assert error.value.code in {"RECEIPT_EVIDENCE_MISMATCH", "RECEIPT_TOTALS_MISMATCH"} + rejected, _ = await store.record_revision_receipt( + replace(receipt, status="rejected", rejection_codes=("EVIDENCE_MISMATCH",)) + ) + ReportingReceipt.model_validate(receipt_to_wire(rejected)) + replacement = replace( + s.receipt, + reporting_receipt_id="receipt-corrected-0002", + supersedes_reporting_receipt_id=rejected.reporting_receipt_id, + ) + accepted, _ = await store.record_revision_receipt(replacement) + assert accepted.status == "accepted" + + +async def test_rejected_leaf_must_be_current_and_accepted_leaf_survives_another_attempt( + reconciliation_store: tuple[Store, Clock], +) -> None: + store, _ = reconciliation_store + s = await scenario(store) + await store.commit_materialization(s.outcome) + r1, _ = await store.record_revision_receipt( + replace(s.receipt, status="rejected", rejection_codes=("LOAD_FAILED",)) + ) + r2_input = replace( + r1, + received_at=None, + reporting_receipt_id="receipt-rejected-0002", + supersedes_reporting_receipt_id=r1.reporting_receipt_id, + ) + r2, _ = await store.record_revision_receipt(r2_input) + with pytest.raises(LedgerConflictError) as stale: + await store.record_revision_receipt( + replace( + s.receipt, + reporting_receipt_id="receipt-stale-0003", + supersedes_reporting_receipt_id=r1.reporting_receipt_id, + ) + ) + assert stale.value.code == "REPORTING_RECORD_UNAVAILABLE" + accepted, _ = await store.record_revision_receipt( + replace( + s.receipt, + reporting_receipt_id="receipt-accepted-0003", + supersedes_reporting_receipt_id=r2.reporting_receipt_id, + ) + ) + a2 = replace(s.attempt, reporting_materialization_id="materialization-2", attempt=2) + await store.commit_materialization_attempt(a2) + await store.commit_materialization( + replace(s.outcome, reporting_materialization_id=a2.reporting_materialization_id) + ) + with pytest.raises(LedgerConflictError) as terminal: + await store.record_revision_receipt( + replace( + s.receipt, + reporting_receipt_id="receipt-after-retry-0004", + reporting_materialization_id=a2.reporting_materialization_id, + ) + ) + assert terminal.value.code == "ACCEPTED_RECEIPT_TERMINAL" + assert await store.get_receipt(accepted.key) == accepted + + +async def test_failed_attempt_is_terminal_and_retry_has_a_new_identity( + reconciliation_store: tuple[Store, Clock], +) -> None: + store, _ = reconciliation_store + s = await scenario(store) + failed = ReportingMaterializationRecord( + s.attempt.scope, + s.attempt.reporting_revision_id, + s.attempt.reporting_materialization_id, + "failed", + s.outcome.completed_at, + failure_code="CONTENT_CORRUPT", + ) + await store.commit_materialization(failed) + assert await store.commit_materialization(failed) == (failed, False) + with pytest.raises(LedgerConflictError) as error: + await store.commit_materialization(s.outcome) + assert error.value.code == "REPORTING_IDENTITY_CONFLICT" + with pytest.raises(LedgerConflictError) as error: + await store.record_revision_receipt(s.receipt) + assert error.value.code == "REPORTING_RECORD_UNAVAILABLE" + # Many workers proposing the same next attempt converge on a single append. + retry = replace(s.attempt, reporting_materialization_id="materialization-retry", attempt=2) + results = await asyncio.gather(*(store.commit_materialization_attempt(retry) for _ in range(8))) + assert sum(written for _, written in results) == 1 + with pytest.raises(LedgerConflictError) as error: + await store.commit_materialization_attempt( + replace(retry, reporting_materialization_id="duplicate-attempt") + ) + assert error.value.code == "MATERIALIZATION_ATTEMPT_CONFLICT" + + +async def test_new_snapshot_revision_has_its_own_terminal_acceptance( + reconciliation_store: tuple[Store, Clock], +) -> None: + store, _ = reconciliation_store + s = await scenario(store, billing=False, finality="snapshot") + await store.commit_materialization(s.outcome) + first, _ = await store.record_revision_receipt(s.receipt) + rows = ( + await store.read_revision_rows( + account_id="acct_a", reporting_revision_id=s.revision.reporting_revision_id + ) + ).rows + revision = replace( + s.revision, + reporting_revision_id="snapshot-restatement", + supersedes_reporting_revision_id=s.revision.reporting_revision_id, + revision_content_sha256=revision_content_sha256( + reporting_revision_id="snapshot-restatement", + row_count=1, + control_totals=s.revision.control_totals, + reporting_rows=rows, + control_total_evidence=s.revision.managed_control_totals, + ), + ) + await store.commit_revision(revision, rows) + attempt = replace( + s.attempt, + reporting_revision_id=revision.reporting_revision_id, + reporting_materialization_id="restated-materialization", + ) + await store.commit_materialization_attempt(attempt) + await store.commit_materialization( + replace( + s.outcome, + reporting_revision_id=revision.reporting_revision_id, + reporting_materialization_id=attempt.reporting_materialization_id, + ) + ) + second, _ = await store.record_revision_receipt( + replace( + s.receipt, + reporting_receipt_id="restated-receipt-0002", + reporting_revision_id=revision.reporting_revision_id, + reporting_materialization_id=attempt.reporting_materialization_id, + ) + ) + snapshot = await store.read_reconciliation_snapshot(caller=s.attempt.scope.principal) + assert snapshot.terminal_acceptances == (first.key, second.key) + + +@pytest.mark.parametrize( + "violation", + ["before_finalization", "after_creation", "period", "receipt_before_creation", "digest"], +) +async def test_adjustment_ordering_and_digest_disagreement( + reconciliation_store: tuple[Store, Clock], violation: str +) -> None: + store, _ = reconciliation_store + s = await scenario(store) + adjustment = ReportingAdjustmentRecord( + "adjustment-order", + "acct_a", + s.revision.reporting_revision_id, + "source_correction", + END, + END + timedelta(days=30), + (("spend", "-1.50"),), + END + timedelta(seconds=5), + END + timedelta(seconds=6), + managed_control_total_deltas=( + ReportingControlTotalRecord("spend", "-1.50", "decimal", "EUR"), + ), + ) + if violation == "before_finalization": + adjustment = replace(adjustment, correction_observed_at=END - timedelta(seconds=1)) + if violation == "after_creation": + adjustment = replace(adjustment, correction_observed_at=END + timedelta(seconds=8)) + if violation == "period": + adjustment = replace(adjustment, accounting_period_end=END) + await store.commit_adjustment(adjustment) + digest = adjustment_to_wire(adjustment)["canonical_adjustment_sha256"] + record = ReportingAdjustmentReceiptRecord( + s.attempt.scope, + "adjustment-order-receipt-1", + adjustment.reporting_adjustment_id, + s.revision.reporting_revision_id, + "accepted", + digest, + END + timedelta(seconds=7), + ) + if violation == "receipt_before_creation": + record = replace(record, observed_at=END) + if violation == "digest": + record = replace(record, observed_adjustment_sha256="0" * 64) + with pytest.raises(LedgerConflictError) as error: + await store.record_adjustment_receipt(record) + assert error.value.code == ( + "ADJUSTMENT_DIGEST_MISMATCH" if violation == "digest" else "ADJUSTMENT_ORDER_INVALID" + ) + if violation == "digest": + rejected, _ = await store.record_adjustment_receipt( + replace(record, status="rejected", rejection_codes=("DIGEST_MISMATCH",)) + ) + accepted, _ = await store.record_adjustment_receipt( + replace( + record, + reporting_receipt_id="adjustment-order-receipt-2", + observed_adjustment_sha256=digest, + supersedes_reporting_receipt_id=rejected.reporting_receipt_id, + ) + ) + assert accepted.status == "accepted" + with pytest.raises(LedgerConflictError) as replay: + await store.record_adjustment_receipt( + replace(accepted, observed_adjustment_sha256="f" * 64) + ) + assert replay.value.code == "REPORTING_IDENTITY_CONFLICT" + + +@pytest.mark.parametrize("different_scope", [False, True]) +async def test_revision_fanout_requires_exact_frozen_logical_scope( + reconciliation_store: tuple[Store, Clock], different_scope: bool +) -> None: + store, _ = reconciliation_store + s = await scenario(store) + generation = replace(s.binding.generation_key, delivery_config_id="second-destination") + config = replace( + configuration(), + delivery_config_id=generation.delivery_config_id, + feed_purpose="billing", + required_finality="official", + ) + await store.put_configuration(config) + obligation = replace( + s.obligation, + reporting_obligation_id="obligation-second-destination", + delivery_config_id=generation.delivery_config_id, + media_buy_ids=("another-buy",) if different_scope else s.obligation.media_buy_ids, + ) + await store.commit_obligation(obligation) + await store.put_destination_binding( + replace(s.binding, generation_key=generation, destination_ref="destination-generation-2") + ) + scope = ReportingDeliveryScope(generation, "buyer", obligation.reporting_obligation_id) + await store.bind_obligation_delivery(replace(s.delivery, scope=scope)) + attempt = replace(s.attempt, scope=scope, reporting_materialization_id="fanout-materialization") + if different_scope: + with pytest.raises(LedgerConflictError) as error: + await store.commit_materialization_attempt(attempt) + assert error.value.code == "REPORTING_RECORD_UNAVAILABLE" + else: + await store.commit_materialization_attempt(attempt) + await store.commit_materialization( + replace( + s.outcome, + scope=scope, + reporting_materialization_id=attempt.reporting_materialization_id, + ) + ) + receipt, _ = await store.record_revision_receipt( + replace( + s.receipt, + scope=scope, + reporting_receipt_id="fanout-receipt-0001", + reporting_materialization_id=attempt.reporting_materialization_id, + ) + ) + assert receipt.scope == scope + + +async def test_core_and_managed_delivery_only_do_not_require_receipt_components( + reconciliation_store: tuple[Store, Clock], +) -> None: + store, _ = reconciliation_store + # Construction and create_schema have already succeeded without any writer, + # resolver, receipt client or outbox being supplied by the fixture. + s = await scenario(store, billing=False, reconciliation_mode="delivery_only") + await store.commit_materialization(s.outcome) + with pytest.raises(LedgerConflictError) as error: + await store.record_revision_receipt(s.receipt) + assert error.value.code == "RECEIPTS_NOT_ENABLED" diff --git a/tests/fixtures/reporting_ledger_1171.sql b/tests/fixtures/reporting_ledger_1171.sql new file mode 100644 index 000000000..3d391954f --- /dev/null +++ b/tests/fixtures/reporting_ledger_1171.sql @@ -0,0 +1,295 @@ +-- AdCP Reliable Reporting ledger — durable obligations, revisions, and status. +-- +-- Run this followed by reporting_ledger_account_generations.sql and +-- reporting_ledger_obligation_currency.sql in ONE +-- transaction (psql --single-transaction -f ... -f ...), or call +-- PgReportingLedgerStore.create_schema(). CREATE TABLE IF NOT EXISTS alone +-- does not upgrade the global configuration primary key from 8.0.0-beta.15. +-- See docs/reporting-ledger-migration.md before upgrading live workers. +-- +-- COLLATE "C" on identifier columns avoids locale-dependent case folding — on +-- some locales "Buy-A" and "buy-a" compare equal, which would collapse two +-- distinct accounts or revisions into one. "C" is the byte-for-byte comparison +-- reporting evidence actually requires. +-- +-- Every immutable write also appends to reporting_ledger_changes in the same +-- transaction. That feed is what makes `changes_after` exact: a consumer that +-- persists a checkpoint and replays from it cannot miss a record or see the +-- same record twice under a different identity. + +-- Serialize bootstrap/upgrade before touching catalog objects: IF NOT EXISTS +-- by itself can still race another CREATE TABLE on an empty schema. The +-- standalone account-generations migration uses the same advisory lock. +SELECT pg_advisory_xact_lock(hashtext('adcp.reporting.schema'), hashtext(current_schema())); + +CREATE TABLE IF NOT EXISTS reporting_configurations ( + delivery_config_id TEXT COLLATE "C" NOT NULL, + delivery_config_version INTEGER NOT NULL, + account_id TEXT COLLATE "C" NOT NULL, + report_definition_id TEXT COLLATE "C" NOT NULL, + reporting_profile TEXT NOT NULL, + feed_purpose TEXT NOT NULL, + required_finality TEXT NOT NULL, + account_timezone TEXT NOT NULL DEFAULT 'UTC', + schedule JSONB NOT NULL, + media_buy_ids JSONB NOT NULL DEFAULT '[]'::jsonb, + activated_at TIMESTAMPTZ, + deactivated_at TIMESTAMPTZ, + automated_recovery_seconds DOUBLE PRECISION NOT NULL DEFAULT 21600, + status_retention_days INTEGER NOT NULL DEFAULT 400, + -- Content-addressed report definition and row schema (URI + digest). Core + -- wire records are self-describing; without this a retained revision + -- cannot name what it was produced under. + definition JSONB, + -- Binds the whole generation so a re-put with changed content is a + -- detectable conflict rather than a silent edit of retained evidence. + content_sha256 TEXT COLLATE "C" NOT NULL, + -- Period-close leasing. A worker that dies mid-close releases its work by + -- expiry instead of wedging the period forever. + lease_worker_id TEXT COLLATE "C", + lease_expires_at TIMESTAMPTZ, + PRIMARY KEY (account_id, delivery_config_id, delivery_config_version) +); + +CREATE INDEX IF NOT EXISTS reporting_configurations_account_idx + ON reporting_configurations (account_id); + +-- Supports "lease the least recently worked generation" without a full scan. +CREATE INDEX IF NOT EXISTS reporting_configurations_lease_idx + ON reporting_configurations (lease_expires_at NULLS FIRST); + +CREATE TABLE IF NOT EXISTS reporting_obligations ( + reporting_obligation_id TEXT COLLATE "C" NOT NULL PRIMARY KEY, + account_id TEXT COLLATE "C" NOT NULL, + delivery_config_id TEXT COLLATE "C" NOT NULL, + delivery_config_version INTEGER NOT NULL, + report_definition_id TEXT COLLATE "C" NOT NULL, + reporting_profile TEXT NOT NULL, + feed_purpose TEXT NOT NULL, + period_key TEXT NOT NULL, + period_start TIMESTAMPTZ NOT NULL, + period_end TIMESTAMPTZ NOT NULL, + source_timezone TEXT NOT NULL, + expected_at TIMESTAMPTZ NOT NULL, + scope_resolved_at TIMESTAMPTZ NOT NULL, + automated_recovery_deadline_at TIMESTAMPTZ NOT NULL, + required_finality TEXT NOT NULL, + coverage_status TEXT NOT NULL DEFAULT 'full', + media_buy_ids JSONB NOT NULL DEFAULT '[]'::jsonb, + package_ids JSONB NOT NULL DEFAULT '[]'::jsonb, + schedule JSONB NOT NULL, + -- Frozen with the obligation, not read from the live configuration: a + -- definition that changes later must not retroactively re-describe a + -- period that already closed. + definition JSONB, + created_at TIMESTAMPTZ NOT NULL, + -- NULL is unknown legacy currency, never an implicit USD default. The + -- currency migration installs validation and immutability on all schemas. + currency TEXT COLLATE "C" +); + +-- One obligation per logical period. Without this, two workers racing a period +-- close could commit two obligations and a seller could quietly publish twice +-- and pick a winner. +CREATE UNIQUE INDEX IF NOT EXISTS reporting_obligations_period_key + ON reporting_obligations + (account_id, delivery_config_id, delivery_config_version, period_start, period_end); + +CREATE INDEX IF NOT EXISTS reporting_obligations_account_idx + ON reporting_obligations (account_id, period_end DESC); + +CREATE TABLE IF NOT EXISTS reporting_revisions ( + reporting_revision_id TEXT COLLATE "C" NOT NULL PRIMARY KEY, + account_id TEXT COLLATE "C" NOT NULL, + reporting_obligation_id TEXT COLLATE "C" NOT NULL + REFERENCES reporting_obligations (reporting_obligation_id), + finality TEXT NOT NULL, + revision_content_sha256 TEXT COLLATE "C" NOT NULL, + row_count BIGINT NOT NULL, + control_totals JSONB NOT NULL DEFAULT '[]'::jsonb, + observed_at TIMESTAMPTZ NOT NULL, + data_through TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL, + supersedes_reporting_revision_id TEXT COLLATE "C", + finality_basis TEXT, + finality_policy_id TEXT, + finalized_at TIMESTAMPTZ, + -- Core promises a committed revision stays readable for + -- status_retention_days. `readable` records reality; `readable_at_commit` + -- remembers that it once was, so the health projection can name the + -- revision that opened the gap rather than an arbitrary one. + readable BOOLEAN NOT NULL DEFAULT TRUE, + readable_at_commit BOOLEAN NOT NULL DEFAULT TRUE, + source_publication_id TEXT COLLATE "C", + source_manifest_sha256 TEXT COLLATE "C", + -- Binds what was published, excluding mutable readability. + content_sha256 TEXT COLLATE "C" NOT NULL +); + +-- An official revision is terminal: at most one per obligation. A later source +-- correction is an adjustment, never a second official close. +CREATE UNIQUE INDEX IF NOT EXISTS reporting_revisions_one_official + ON reporting_revisions (reporting_obligation_id) + WHERE finality = 'official'; + +-- Supersession must not fork: a given revision may be superseded at most once. +CREATE UNIQUE INDEX IF NOT EXISTS reporting_revisions_one_successor + ON reporting_revisions (supersedes_reporting_revision_id) + WHERE supersedes_reporting_revision_id IS NOT NULL; + +CREATE INDEX IF NOT EXISTS reporting_revisions_obligation_idx + ON reporting_revisions (account_id, reporting_obligation_id); + +CREATE TABLE IF NOT EXISTS reporting_revision_rows ( + reporting_revision_id TEXT COLLATE "C" NOT NULL + REFERENCES reporting_revisions (reporting_revision_id), + ordinal BIGINT NOT NULL, + row_payload JSONB NOT NULL, + PRIMARY KEY (reporting_revision_id, ordinal) +); + +CREATE TABLE IF NOT EXISTS reporting_adjustments ( + reporting_adjustment_id TEXT COLLATE "C" NOT NULL PRIMARY KEY, + account_id TEXT COLLATE "C" NOT NULL, + adjusts_reporting_revision_id TEXT COLLATE "C" NOT NULL + REFERENCES reporting_revisions (reporting_revision_id), + reason_code TEXT NOT NULL, + reason_detail TEXT, + accounting_period_start TIMESTAMPTZ NOT NULL, + accounting_period_end TIMESTAMPTZ NOT NULL, + control_total_deltas JSONB NOT NULL DEFAULT '[]'::jsonb, + correction_observed_at TIMESTAMPTZ NOT NULL, + created_at TIMESTAMPTZ NOT NULL +); + +CREATE INDEX IF NOT EXISTS reporting_adjustments_revision_idx + ON reporting_adjustments (account_id, adjusts_reporting_revision_id); + +-- PREVIEW: sync_reporting_status. Created unconditionally because DDL is +-- cheap and a migration mid-rollout is not; the ingest that writes here is off +-- until an adopter enables it. See adcp/reporting/ledger/consumer_status.py. +CREATE TABLE IF NOT EXISTS reporting_consumer_statuses ( + reporting_status_id TEXT COLLATE "C" NOT NULL PRIMARY KEY, + account_id TEXT COLLATE "C" NOT NULL, + -- Derived from authenticated transport, never from the request body. + consumer_id TEXT COLLATE "C" NOT NULL, + delivery_config_id TEXT COLLATE "C" NOT NULL, + delivery_config_version INTEGER NOT NULL, + report_definition_id TEXT COLLATE "C" NOT NULL, + period_start TIMESTAMPTZ NOT NULL, + period_end TIMESTAMPTZ NOT NULL, + period_source_timezone TEXT NOT NULL, + consumer_status TEXT NOT NULL, + status_as_of TIMESTAMPTZ NOT NULL, + recorded_at TIMESTAMPTZ NOT NULL, + supersedes_reporting_status_id TEXT COLLATE "C", + -- Nullable by design: obligation_missing is filed precisely when the + -- seller's ledger omitted the period, so requiring a seller-issued + -- obligation id would make the first missing report invisible again. + reporting_obligation_id TEXT COLLATE "C", + reporting_revision_id TEXT COLLATE "C", + observed_revision_content_sha256 TEXT COLLATE "C", + failure_code TEXT, + consumer_commit_ref TEXT, + seller_ledger_snapshot_id TEXT, + seller_ledger_as_of TIMESTAMPTZ, + superseded BOOLEAN NOT NULL DEFAULT FALSE, + content_sha256 TEXT COLLATE "C" NOT NULL +); + +-- Exactly one unsuperseded leaf per logical chain. This is what makes +-- supersession atomic: a concurrent update naming a stale leaf hits this +-- constraint instead of forking the chain, so a successful retry cannot erase +-- a recorded outage. +CREATE UNIQUE INDEX IF NOT EXISTS reporting_consumer_statuses_one_leaf + ON reporting_consumer_statuses + (account_id, consumer_id, delivery_config_id, delivery_config_version, + report_definition_id, period_start, period_end) + WHERE superseded = FALSE; + +CREATE UNIQUE INDEX IF NOT EXISTS reporting_consumer_statuses_one_successor + ON reporting_consumer_statuses (supersedes_reporting_status_id) + WHERE supersedes_reporting_status_id IS NOT NULL; + +CREATE INDEX IF NOT EXISTS reporting_consumer_statuses_chain_idx + ON reporting_consumer_statuses (account_id, consumer_id, period_end DESC); + +-- AdCP 3.2.0-rc.3 additions to tables created by an earlier SDK. ADD COLUMN +-- IF NOT EXISTS keeps create_schema() an upgrade path, not just a bootstrap: +-- an adopter that installed the rc.2 schema gets these on the next boot +-- without a hand-written migration, and a fresh install is unaffected. +ALTER TABLE reporting_consumer_statuses + ADD COLUMN IF NOT EXISTS mismatch_code TEXT; + +ALTER TABLE reporting_configurations + ADD COLUMN IF NOT EXISTS authoritative_party TEXT NOT NULL DEFAULT 'seller'; + +-- Durable issue lifecycle. Every other issue in this ledger has a *derived* +-- identity, because the conditions they name are monotone for one immutable +-- obligation: once a qualifying revision is associated, REPORT_OVERDUE cannot +-- recur. A consumer mismatch is not monotone -- the buyer can supersede, the +-- seller can restate, the disagreement can clear and come back -- and rc.3 +-- requires opened_at to survive every re-emission because it anchors the +-- escalation clock. A derived timestamp would reset on each poll and an +-- unattended mismatch would never escalate. +-- +-- issue_key identifies the condition; issue_id identifies one occurrence of +-- it. Retirement bumps generation, so a recurrence after resolved/waived gets +-- a new issue_id and a new opened_at, while an unresolved condition keeps both +-- across a severity change from delayed to action_required. +CREATE TABLE IF NOT EXISTS reporting_issue_lifecycle ( + issue_key TEXT COLLATE "C" NOT NULL, + account_id TEXT COLLATE "C" NOT NULL, + generation INTEGER NOT NULL, + issue_id TEXT COLLATE "C" NOT NULL, + -- Caller-scoped issues (every consumer mismatch) carry the consumer whose + -- statement caused them; NULL is a seller-wide condition. + consumer_id TEXT COLLATE "C", + opened_at TIMESTAMPTZ NOT NULL, + issue_state TEXT NOT NULL DEFAULT 'open', + -- Inert correlation text for the party's own tracker. Never dereferenced. + external_ref TEXT, + retired_at TIMESTAMPTZ, + PRIMARY KEY (account_id, issue_key, generation) +); + +-- At most one live occurrence per condition. 'waived' counts as live: waiving +-- records an agreement to stop *acting*, not a finding that the reporting is +-- fine, so the occurrence must keep blocking a new one -- otherwise the next +-- poll would open a fresh occurrence and republish the very issue the parties +-- agreed to stop acting on. Only 'resolved' frees the condition to recur. +-- +-- This is also what makes +-- ensure_issue_opened() safe under concurrent reads: two readers of the same +-- condition collide on this index and converge on one row instead of opening +-- two occurrences with two different opened_at values -- which would give the +-- same disagreement two escalation clocks. +CREATE UNIQUE INDEX IF NOT EXISTS reporting_issue_lifecycle_one_live + ON reporting_issue_lifecycle (account_id, issue_key) + WHERE issue_state IN ('open', 'acknowledged', 'waived'); + +CREATE UNIQUE INDEX IF NOT EXISTS reporting_issue_lifecycle_issue_id + ON reporting_issue_lifecycle (issue_id); + +-- The per-account change feed. `seq` orders every immutable record across +-- kinds so `changes_after` is exact. +-- +-- Appends take a transaction-scoped advisory lock on the account, which makes +-- sequence order equal commit order *within an account*. Without it, a +-- transaction that grabbed a low sequence but committed late would be invisible +-- to a consumer that had already checkpointed past it — a silently lost record, +-- which is the one failure a reporting ledger must not have. Contention is +-- per-account and the appends are short. +CREATE TABLE IF NOT EXISTS reporting_ledger_changes ( + seq BIGSERIAL NOT NULL PRIMARY KEY, + account_id TEXT COLLATE "C" NOT NULL, + record_kind TEXT NOT NULL, + record_id TEXT COLLATE "C" NOT NULL, + committed_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE UNIQUE INDEX IF NOT EXISTS reporting_ledger_changes_record + ON reporting_ledger_changes (account_id, record_kind, record_id); + +CREATE INDEX IF NOT EXISTS reporting_ledger_changes_feed_idx + ON reporting_ledger_changes (account_id, seq); diff --git a/tests/type_checks/reporting_reconciliation_records.py b/tests/type_checks/reporting_reconciliation_records.py new file mode 100644 index 000000000..be0199899 --- /dev/null +++ b/tests/type_checks/reporting_reconciliation_records.py @@ -0,0 +1,125 @@ +"""Adopters can replace each durable seam without adding dependencies to Core.""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import replace +from datetime import datetime, timezone +from typing import Any + +from psycopg_pool import AsyncConnectionPool + +from adcp.reporting.ledger import ( + InMemoryReportingLedgerStore, + InMemoryReportingReconciliationStore, + PgReportingReconciliationStore, + ReportingCanonicalDigest, + ReportingConfigurationGenerationKey, + ReportingControlTotalRecord, + ReportingDeliveryScope, + ReportingDestinationBinding, + ReportingDestinationStore, + ReportingLedgerStore, + ReportingMaterializationAttempt, + ReportingMaterializationRecord, + ReportingMaterializationStore, + ReportingMaterializationView, + ReportingObligationDeliveryRecord, + ReportingReceiptKey, + ReportingReceiptRecord, + ReportingReceiptStore, + ReportingReconciliationStore, + ReportingRevisionReceiptRecord, + ReportingRevisionRecord, + revision_content_sha256, +) + + +def core_only() -> ReportingLedgerStore: + return InMemoryReportingLedgerStore() + + +def reference() -> ReportingReconciliationStore: + return InMemoryReportingReconciliationStore() + + +def persistent(pool: AsyncConnectionPool) -> ReportingReconciliationStore: + return PgReportingReconciliationStore(pool=pool) + + +def trusted_publisher_evidence( + revision: ReportingRevisionRecord, + digest: ReportingCanonicalDigest, + expected_totals: tuple[ReportingControlTotalRecord, ...], + rows: Sequence[dict[str, Any]], +) -> ReportingRevisionRecord: + return replace( + revision, + canonical_content_digest=digest, + managed_control_totals=expected_totals, + revision_content_sha256=revision_content_sha256( + reporting_revision_id=revision.reporting_revision_id, + row_count=revision.row_count, + control_totals=revision.control_totals, + reporting_rows=rows, + control_total_evidence=expected_totals, + ), + ) + + +async def trusted_configuration_ingest( + store: ReportingDestinationStore, + *, + generation: ReportingConfigurationGenerationKey, + consumer_id: str, + obligation_id: str, + retained_until: datetime, +) -> None: + now = datetime.now(timezone.utc) + binding = ReportingDestinationBinding( + generation_key=generation, + consumer_id=consumer_id, + destination_ref="destination-generation-1", + trusted_binding_ref="trusted-binding-1", + method="warehouse_materialization", + transport="warehouse", + verification_profile="canonical_digest", + reconciliation_mode="consumer_receipt", + feed_purpose="billing", + resource_retention_days=400, + created_at=now, + success_status="delivered", + ) + stored_binding, recorded = await store.put_destination_binding(binding) + scope = ReportingDeliveryScope(stored_binding.generation_key, consumer_id, obligation_id) + delivery = ReportingObligationDeliveryRecord(scope, "EUR", retained_until, now) + await store.bind_obligation_delivery(delivery) + assert isinstance(recorded, bool) + + +async def destination_writer_completion( + store: ReportingMaterializationStore, + attempt: ReportingMaterializationAttempt, + verified_result: ReportingMaterializationRecord, +) -> ReportingMaterializationView | None: + # A writer resolves credentials through its trusted binding outside these + # records. Only its immutable attempt and verified public evidence cross here. + await store.commit_materialization_attempt(attempt) + await store.commit_materialization(verified_result) + return await store.get_materialization(attempt.key) + + +async def receipt_handler_storage( + store: ReportingReceiptStore, authenticated_receipt: ReportingRevisionReceiptRecord +) -> tuple[ReportingReceiptKey, bool]: + stored, recorded = await store.record_revision_receipt(authenticated_receipt) + total: ReportingControlTotalRecord = stored.observed_control_totals[0] + unit: str | None = total.unit + assert unit is None or isinstance(unit, str) + return stored.key, recorded + + +async def retained_receipt( + store: ReportingReceiptStore, key: ReportingReceiptKey +) -> ReportingReceiptRecord | None: + return await store.get_receipt(key) From 89674803f128868e4c8069f4b22f97dccf5836ac Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Wed, 16 Sep 2026 03:27:16 +0000 Subject: [PATCH 2/4] test(reporting): cover independent revision histories and receipt races --- docs/reporting-reconciliation-storage.md | 7 + ...test_reporting_reconciliation_migration.py | 2 +- .../test_reporting_reconciliation_store.py | 151 ++++++++++++++++- ...st_reporting_reconciliation_transitions.py | 153 ++++++++++++++++++ 4 files changed, 304 insertions(+), 9 deletions(-) diff --git a/docs/reporting-reconciliation-storage.md b/docs/reporting-reconciliation-storage.md index 0c8540466..4f82a5f82 100644 --- a/docs/reporting-reconciliation-storage.md +++ b/docs/reporting-reconciliation-storage.md @@ -42,6 +42,13 @@ leaf is terminal. History is never edited to mark a receipt superseded: current leaves and terminal acceptance keys are derived from the retained graph. Revision and adjustment receipt IDs share a namespace, as the batched wire request requires. +A retained snapshot and a later official revision coexist as independent histories. +The official revision does not supersede the snapshot. Each revision starts its own +materialization attempt sequence and receipt chain, so both can have attempt `1` +and a terminal acceptance. Accepting the official receipt leaves an earlier rejected +snapshot receipt replaceable; neither chain changes the other's retained evidence. +There is no mutable current-revision pointer in these storage contracts. + ## Trusted inputs and credential boundary These are low-level seller storage contracts, not an authentication boundary. diff --git a/tests/conformance/reporting/test_reporting_reconciliation_migration.py b/tests/conformance/reporting/test_reporting_reconciliation_migration.py index 19907f0db..6f2e72526 100644 --- a/tests/conformance/reporting/test_reporting_reconciliation_migration.py +++ b/tests/conformance/reporting/test_reporting_reconciliation_migration.py @@ -24,7 +24,7 @@ def test_literal_stacked_schema_fixture() -> None: - # #1175 head / reviewed #1171 base 7bd5b8f6, without synthesized DDL edits. + # Literal #1175 bootstrap, unchanged at corrected #1171 head ff584b6f. assert hashlib.sha256((FIXTURES / "reporting_ledger_1171.sql").read_bytes()).hexdigest() == ( "65d9b44e220f1828b48beb32a44334b956fbf27081bed72390de1c526e67df4e" ) diff --git a/tests/conformance/reporting/test_reporting_reconciliation_store.py b/tests/conformance/reporting/test_reporting_reconciliation_store.py index cc86d9721..c8e72c662 100644 --- a/tests/conformance/reporting/test_reporting_reconciliation_store.py +++ b/tests/conformance/reporting/test_reporting_reconciliation_store.py @@ -228,21 +228,28 @@ async def test_corruption_and_retention_preserve_receipts_and_snapshots( assert not (await store.get_materialization(s.attempt.key)).readable_at(clock.now) +@pytest.mark.parametrize("has_predecessor", [False, True]) +@pytest.mark.parametrize("status", ["accepted", "rejected"]) async def test_concurrent_workers_converge_without_terminal_races( - reconciliation_store: tuple[Store, Clock], + reconciliation_store: tuple[Store, Clock], has_predecessor: bool, status: str ) -> None: store, _ = reconciliation_store s = await scenario(store) writes = await asyncio.gather(*(store.commit_materialization(s.outcome) for _ in range(12))) assert sum(created for _, created in writes) == 1 - rejected, _ = await store.record_revision_receipt( - replace(s.receipt, status="rejected", rejection_codes=("CONTENT_MISMATCH",)) - ) + predecessor = None + if has_predecessor: + rejected, _ = await store.record_revision_receipt( + replace(s.receipt, status="rejected", rejection_codes=("CONTENT_MISMATCH",)) + ) + predecessor = rejected.reporting_receipt_id receipts = [ replace( s.receipt, reporting_receipt_id=f"receipt-concurrent-{i:04}", - supersedes_reporting_receipt_id=rejected.reporting_receipt_id, + status=status, + rejection_codes=("LOAD_FAILED",) if status == "rejected" else (), + supersedes_reporting_receipt_id=predecessor, ) for i in range(12) ] @@ -251,11 +258,139 @@ async def test_concurrent_workers_converge_without_terminal_races( ) assert sum(isinstance(item, tuple) for item in outcomes) == 1 assert all( - isinstance(item, tuple) or isinstance(item, LedgerConflictError) for item in outcomes + isinstance(item, tuple) + or ( + isinstance(item, LedgerConflictError) + and item.code + == ( + "ACCEPTED_RECEIPT_TERMINAL" + if status == "accepted" + else "REPORTING_RECORD_UNAVAILABLE" + ) + ) + for item in outcomes ) snapshot = await store.read_reconciliation_snapshot(caller=s.attempt.scope.principal) - assert len(snapshot.terminal_acceptances) == 1 - assert len([r for r in snapshot.records if r.kind == "revision_receipt"]) == 2 + assert len(snapshot.current_receipts) == 1 + assert len(snapshot.terminal_acceptances) == (1 if status == "accepted" else 0) + assert len([r for r in snapshot.records if r.kind == "revision_receipt"]) == ( + 2 if has_predecessor else 1 + ) + if status == "rejected": + accepted, written = await store.record_revision_receipt( + replace( + s.receipt, + reporting_receipt_id="receipt-after-replacement-race", + supersedes_reporting_receipt_id=snapshot.current_receipts[0].reporting_receipt_id, + ) + ) + assert written and accepted.status == "accepted" + + +async def test_competing_materialization_outcomes_keep_one_terminal_fact( + reconciliation_store: tuple[Store, Clock], +) -> None: + store, _ = reconciliation_store + s = await scenario(store) + failed = replace( + s.outcome, + status="failed", + resource=None, + verification=None, + failure_code="WRITE_FAILED", + ) + outcomes = await asyncio.gather( + *(store.commit_materialization(item) for item in [s.outcome, failed] * 4), + return_exceptions=True, + ) + written = [item for item in outcomes if isinstance(item, tuple)] + assert len(written) == 4 + assert sum(created for _, created in written) == 1 + assert all(item[0] == written[0][0] for item in written) + conflicts = [item for item in outcomes if isinstance(item, Exception)] + assert len(conflicts) == 4 + assert all( + isinstance(item, LedgerConflictError) and item.code == "REPORTING_IDENTITY_CONFLICT" + for item in conflicts + ) + assert (await store.get_materialization(s.attempt.key)).outcome == written[0][0] + + +async def test_repair_appends_evidence_and_advances_checkpoint_without_rewriting_history( + reconciliation_store: tuple[Store, Clock], +) -> None: + store, _ = reconciliation_store + s = await scenario(store) + await store.commit_materialization(s.outcome) + caller = s.attempt.scope.principal + handler = ReportingStatusHandler(store) + status_caller = ReportingStatusCaller(caller.account_id, caller.consumer_id) + core_before = await handler.handle({"view": "periods"}, caller=status_caller) + before = await store.read_reconciliation_snapshot(caller=caller) + corrupt = ReportingMaterializationCheck( + s.attempt.scope, + s.attempt.reporting_materialization_id, + "checkpoint-corrupt", + "corrupt", + END + timedelta(seconds=8), + ) + await store.record_materialization_check(corrupt) + rejected, _ = await store.record_revision_receipt( + replace( + s.receipt, + status="rejected", + rejection_codes=("CONTENT_CORRUPT",), + observed_at=END + timedelta(seconds=9), + ) + ) + blocked = await store.read_reconciliation_snapshot(caller=caller) + repaired = replace( + corrupt, + check_id="checkpoint-repaired", + state="readable", + checked_at=END + timedelta(seconds=10), + ) + await store.record_materialization_check(repaired) + accepted_input = replace( + s.receipt, + reporting_receipt_id="checkpoint-repaired-receipt", + supersedes_reporting_receipt_id=rejected.reporting_receipt_id, + observed_at=END + timedelta(seconds=11), + ) + accepted, _ = await store.record_revision_receipt(accepted_input) + after = await store.read_reconciliation_snapshot(caller=caller) + assert after.records == before.records + (corrupt, rejected, repaired, accepted) + assert ( + before.boundary.max_sequence < blocked.boundary.max_sequence < after.boundary.max_sequence + ) + assert before.materialization(s.attempt.key).readable_at(NOW) + assert not blocked.materialization(s.attempt.key).readable_at(NOW) + assert after.materialization(s.attempt.key).readable_at(NOW) + for old in (before, blocked): + assert await store.read_reconciliation_snapshot(caller=caller, boundary=old.boundary) == old + for check in (corrupt, repaired): + assert await store.record_materialization_check(check) == (check, False) + assert await store.record_revision_receipt(accepted_input) == (accepted, False) + assert await store.read_reconciliation_snapshot(caller=caller) == after + # Core consumes the shared checkpoint without counting or exposing optional records. + core_after = await handler.handle( + {"view": "periods", "changes_after": core_before["changes_checkpoint"]}, + caller=status_caller, + ) + assert core_after["changes_checkpoint"] != core_before["changes_checkpoint"] + assert core_after["pagination"]["total_count"] == 0 + assert core_after["periods"] == core_after["revisions"] == [] + assert core_after["materializations"] == core_after["receipts"] == [] + stranger = ReportingDeliveryPrincipal(caller.account_id, "another-consumer") + assert ( + await store.read_reconciliation_snapshot(caller=stranger, boundary=after.boundary) + ).records == () + with pytest.raises(LedgerConflictError) as error: + await store.read_reconciliation_snapshot( + caller=ReportingDeliveryPrincipal("another-account", caller.consumer_id), + boundary=after.boundary, + ) + assert error.value.code == "REPORTING_RECORD_UNAVAILABLE" async def test_account_principal_isolation_and_shared_identifiers( diff --git a/tests/conformance/reporting/test_reporting_reconciliation_transitions.py b/tests/conformance/reporting/test_reporting_reconciliation_transitions.py index f6609960e..6ae2cf014 100644 --- a/tests/conformance/reporting/test_reporting_reconciliation_transitions.py +++ b/tests/conformance/reporting/test_reporting_reconciliation_transitions.py @@ -3,12 +3,14 @@ from __future__ import annotations import asyncio +import hashlib from dataclasses import replace from datetime import timedelta import pytest from adcp.reporting import ReportingLedger, evaluate_reporting_ledger +from adcp.reporting.canonical_json import canonical_json_utf8_v1 from adcp.reporting.ledger import ( LedgerConflictError, ReportingAdjustmentReceiptRecord, @@ -258,6 +260,157 @@ async def test_new_snapshot_revision_has_its_own_terminal_acceptance( assert snapshot.terminal_acceptances == (first.key, second.key) +async def test_snapshot_and_later_official_keep_independent_delivery_and_receipt_histories( + reconciliation_store: tuple[Store, Clock], +) -> None: + store, _ = reconciliation_store + s = await scenario(store, billing=False, finality="snapshot") + await store.commit_materialization(s.outcome) + rejected, _ = await store.record_revision_receipt( + replace(s.receipt, status="rejected", rejection_codes=("LOAD_FAILED",)) + ) + earlier = await store.read_reconciliation_snapshot(caller=s.attempt.scope.principal) + snapshot_rows = ( + await store.read_revision_rows( + account_id="acct_a", reporting_revision_id=s.revision.reporting_revision_id + ) + ).rows + official_rows = [{**snapshot_rows[0], "spend": "13.50"}] + official_totals = ( + s.revision.managed_control_totals[0], + replace(s.revision.managed_control_totals[1], value="13.50"), + ) + totals = tuple((item.name, item.value) for item in official_totals) + digest = replace( + s.revision.canonical_content_digest, + value=hashlib.sha256(canonical_json_utf8_v1(official_rows)).hexdigest(), + ) + official = replace( + s.revision, + reporting_revision_id="later-official-revision", + finality="official", + finality_basis="source_final", + finality_policy_id="policy-v1", + finalized_at=END + timedelta(seconds=5), + observed_at=END + timedelta(seconds=5), + created_at=END + timedelta(seconds=6), + control_totals=totals, + managed_control_totals=official_totals, + canonical_content_digest=digest, + revision_content_sha256=revision_content_sha256( + reporting_revision_id="later-official-revision", + row_count=1, + control_totals=totals, + reporting_rows=official_rows, + control_total_evidence=official_totals, + ), + ) + assert official.supersedes_reporting_revision_id is None + with pytest.raises(ValueError, match="supersede"): + replace(official, supersedes_reporting_revision_id=s.revision.reporting_revision_id) + await store.commit_revision(official, official_rows) + attempt = replace( + s.attempt, + reporting_revision_id=official.reporting_revision_id, + reporting_materialization_id="official-materialization", + created_at=END + timedelta(seconds=7), + ) + assert attempt.attempt == s.attempt.attempt == 1 + await store.commit_materialization_attempt(attempt) + completed_at = END + timedelta(seconds=8) + outcome = replace( + s.outcome, + reporting_revision_id=official.reporting_revision_id, + reporting_materialization_id=attempt.reporting_materialization_id, + completed_at=completed_at, + resource=replace( + s.outcome.resource, + resource_ref="official-resource", + location="reports/later-official/manifest.json", + manifest_sha256="e" * 64, + object_refs=("reports/later-official/part-000.jsonl",), + expires_at=completed_at + timedelta(days=400), + ), + verification=replace( + s.outcome.verification, + verified_at=completed_at, + control_totals=official_totals, + canonical_content_digest=digest, + physical_checksums=( + replace( + s.outcome.verification.physical_checksums[0], + object_ref="reports/later-official/part-000.jsonl", + value="f" * 64, + ), + ), + ), + ) + await store.commit_materialization(outcome) + receipt = replace( + s.receipt, + reporting_receipt_id="official-receipt-0001", + reporting_revision_id=official.reporting_revision_id, + reporting_materialization_id=attempt.reporting_materialization_id, + observed_control_totals=official_totals, + observed_canonical_content_digest=digest, + observed_at=END + timedelta(seconds=9), + consumer_commit_ref="official-load-0001", + ) + with pytest.raises(LedgerConflictError) as error: + await store.record_revision_receipt( + replace(receipt, supersedes_reporting_receipt_id=rejected.reporting_receipt_id) + ) + assert error.value.code == "REPORTING_RECORD_UNAVAILABLE" + official_accepted, _ = await store.record_revision_receipt(receipt) + # Accepting the later official leaves the snapshot's rejected chain repairable. + snapshot_accepted, _ = await store.record_revision_receipt( + replace( + s.receipt, + reporting_receipt_id="snapshot-repaired-receipt-0002", + supersedes_reporting_receipt_id=rejected.reporting_receipt_id, + observed_at=END + timedelta(seconds=10), + ) + ) + history = await store.read_reconciliation_snapshot(caller=s.attempt.scope.principal) + assert history.current_receipts == (official_accepted, snapshot_accepted) + assert history.terminal_acceptances == (official_accepted.key, snapshot_accepted.key) + assert history.materialization(s.attempt.key).outcome == s.outcome + assert history.materialization(attempt.key).outcome == outcome + assert ( + await store.read_revision_rows( + account_id="acct_a", reporting_revision_id=s.revision.reporting_revision_id + ) + ).rows == snapshot_rows + assert await store.list_revisions( + account_id="acct_a", reporting_obligation_id=s.obligation.reporting_obligation_id + ) == (s.revision, official) + assert ( + await store.read_reconciliation_snapshot( + caller=s.attempt.scope.principal, boundary=earlier.boundary + ) + == earlier + ) + for accepted in (official_accepted, snapshot_accepted): + assert await store.record_revision_receipt(accepted) == (accepted, False) + with pytest.raises(LedgerConflictError) as error: + await store.record_revision_receipt( + replace( + accepted, + received_at=None, + reporting_receipt_id=accepted.reporting_receipt_id + "-successor", + supersedes_reporting_receipt_id=accepted.reporting_receipt_id, + ) + ) + assert error.value.code == "ACCEPTED_RECEIPT_TERMINAL" + assert await store.read_reconciliation_snapshot(caller=s.attempt.scope.principal) == history + core = await ReportingStatusHandler(store).handle( + {"view": "periods"}, caller=ReportingStatusCaller("acct_a", "buyer") + ) + GetReportingStatusResponse.model_validate(core) + assert {item["finality"] for item in core["revisions"]} == {"snapshot", "official"} + assert core["materializations"] == core["receipts"] == [] + + @pytest.mark.parametrize( "violation", ["before_finalization", "after_creation", "period", "receipt_before_creation", "digest"], From c44095e3b457c44931eacc8a0da21ddf442229fd Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Wed, 16 Sep 2026 04:29:27 +0000 Subject: [PATCH 3/4] fix(reporting): enforce exact reconciliation graph and isolated feeds --- docs/reporting-reconciliation-storage.md | 115 +++- src/adcp/reporting/_reconcile.py | 22 +- src/adcp/reporting/evidence.py | 112 +++- src/adcp/reporting/ledger/__init__.py | 20 + src/adcp/reporting/ledger/_delivery_state.py | 84 ++- src/adcp/reporting/ledger/delivery.py | 145 +++- src/adcp/reporting/ledger/delivery_changes.py | 318 +++++++++ src/adcp/reporting/ledger/delivery_models.py | 71 +- src/adcp/reporting/ledger/delivery_pg.py | 361 ++++++---- src/adcp/reporting/ledger/models.py | 14 +- src/adcp/reporting/ledger/pg.py | 3 +- .../reporting_ledger_reconciliation.sql | 474 +++++++++++++ src/adcp/reporting/ledger/store.py | 2 +- .../reporting/_reconciliation_support.py | 24 +- .../test_reporting_reconciliation_changes.py | 439 ++++++++++++ ...test_reporting_reconciliation_migration.py | 232 ++++++- .../test_reporting_reconciliation_profiles.py | 172 +++++ .../test_reporting_reconciliation_sql.py | 623 ++++++++++++++++++ .../test_reporting_reconciliation_store.py | 8 +- ...st_reporting_reconciliation_transitions.py | 22 +- ...eporting_ledger_reconciliation_initial.sql | 169 +++++ tests/test_reporting_reconciliation.py | 105 ++- .../reporting_reconciliation_records.py | 36 + 23 files changed, 3264 insertions(+), 307 deletions(-) create mode 100644 src/adcp/reporting/ledger/delivery_changes.py create mode 100644 tests/conformance/reporting/test_reporting_reconciliation_changes.py create mode 100644 tests/conformance/reporting/test_reporting_reconciliation_profiles.py create mode 100644 tests/conformance/reporting/test_reporting_reconciliation_sql.py create mode 100644 tests/fixtures/reporting_ledger_reconciliation_initial.sql diff --git a/docs/reporting-reconciliation-storage.md b/docs/reporting-reconciliation-storage.md index 4f82a5f82..26689a090 100644 --- a/docs/reporting-reconciliation-storage.md +++ b/docs/reporting-reconciliation-storage.md @@ -48,6 +48,11 @@ materialization attempt sequence and receipt chain, so both can have attempt `1` and a terminal acceptance. Accepting the official receipt leaves an earlier rejected snapshot receipt replaceable; neither chain changes the other's retained evidence. There is no mutable current-revision pointer in these storage contracts. +Consumers select the obligation's unique current revision before selecting a +materialization. An official revision wins when present. Otherwise the unique +unsuperseded snapshot is current. An unmaterialized, pending, or failed current +revision never falls back to an older materialized snapshot; multiple current +candidates fail closed. ## Trusted inputs and credential boundary @@ -77,8 +82,13 @@ safe failure classifications and inert references. URL credentials, signed query strings, bearer material and private keys are refused without echoing the input. Adapters must supply public identifiers; syntax checks cannot establish the provenance of an otherwise opaque identifier. -Native versions retain their decoded provider value, including characters such as -`/`, `+`, and `=`. They are not entity IDs, URLs, or URI-encoded object references. +Validators follow each field's wire contract: seller-issued entity IDs retain +their schema grammar; provider locations, consumer load/commit IDs, reader feature +labels, decoded object keys and native versions retain spaces, `/`, `+`, `=` and +Unicode without rewriting. Object keys must be destination-relative. All these +fields reject recognizable credentials and URLs, including encoded forms, without +echoing the input. Native versions remain separate from object keys and are only +URI-encoded when a later adapter constructs its provider request. ## Evidence and financial ordering @@ -113,38 +123,44 @@ and pinned definition; omitted units inherit that context. Nonmonetary units and explicit types (including a decimal total represented by `"5"`) remain unchanged. Unknown historical currency or missing expected managed totals fails closed. -One logical revision may fan out to another destination obligation only when its -account, frozen period and resolved buy/package scope, coverage, definition and -currency match exactly. That does not grant another principal access: each -consumer still requires its own trusted binding and obligation delivery record. +A revision belongs to exactly one frozen obligation identity. Every attempt, +outcome and receipt must retain that account, configuration generation and +obligation, even if another obligation has identical content, scope and currency. +This slice has no obligation aliases or cross-obligation fan-out. Each consumer +also requires its own trusted binding and obligation delivery record. Billing requires an official configuration, consumer receipts, and canonical digest verification. Other managed profiles retain their narrower assurance: native commits and manifest checksums do not assert cryptographic equality of logical rows. Available and delivered claims follow the frozen binding's success status. +| Method | Canonical digest | Manifest checksums | Native commit | +| --- | --- | --- | --- | +| File transfer | Supported | Supported | Supported | +| Dataset share | Supported | Rejected at binding | Supported | +| Warehouse materialization | Supported | Rejected at binding | Supported | + +Billing permits only canonical digest verification. Native commit requires +`resource.immutability == "native_version"`, identical resource/verification version +references, and an exact representative-consumer or destination observation path. +File transfer additionally requires the committed manifest and all object checksums. + Adjustment acceptance verifies the complete adjustment's JCS/SHA-256 evidence, including optional `reason_detail`, and its exact official revision. Correction observation must be no earlier than finalization and no later than creation; receipt observation follows creation. Periods must be ordered. These are evidence records, not permission to post accounting entries, reopen books, change invoices, or settle. -Two wire boundaries need attention in the completion PR: - -* The adjustment schema defines an independent receipt chain and does not require - prior acceptance of the official revision receipt. The store records those two - facts independently; completion must require both. An accepted adjustment alone - cannot establish reconciled completion. -* The existing buyer `_select_current` can report `AMBIGUOUS_REVISION_CHAIN` when a - retained snapshot and a separate official revision coexist, while Core's producer - forbids an official revision from superseding a snapshot. This PR preserves that - financial finality rule. The completion PR must resolve reader selection against - the protocol before claiming a complete multi-finality lifecycle. +The adjustment schema defines an independent receipt chain and does not require +prior acceptance of the official revision receipt. The store records those two +facts independently; completion must require both. An accepted adjustment alone +cannot establish reconciled completion. ## Retention, snapshots, and future notifications -`read_reconciliation_snapshot(caller=...)` reads retained records at a Core ledger -boundary. Reusing its `boundary` excludes later outcomes, receipts and storage +`read_reconciliation_snapshot(caller=...)` reads retained records at an independent +account/consumer boundary, typed as `ReportingReconciliationSnapshotToken`. +Reusing its `boundary` excludes later outcomes, receipts and storage checks. `get_materialization` returns the attempt, terminal outcome, binding and readability history; `get_receipt` requires an explicit account/consumer key. Expiry and corruption never erase immutable evidence or acceptance identity. @@ -153,18 +169,58 @@ changing the materialization's terminal wire state. A successful resource must l through both the obligation floor and readiness plus the frozen retention contract. There is no purge or unchecked history-repair API. +`ReportingReconciliationFeedStore` is a separate, optional replacement protocol; +neither existing Core nor external reconciliation stores need new methods. +Both reference stores implement `read_reconciliation_changes(caller=..., limit=...)`. +It returns immutable `(sequence, record)` changes, a fixed boundary, an optional +continuation cursor, and a checkpoint only on the final page. Every record kind, including checks and both +receipt kinds, appears exactly once in sequence order when continuing from the +last consumed position. Interleaving writes appear on the next walk. Foreign +accounts and consumers never contribute records or change another principal's +snapshot ID, count, cursor, checkpoint, or next sequence. Reconciliation writes +also leave Core's sequence, snapshot ID, record counts and checkpoint unchanged. + +Persist a partial page's records and `ReportingReconciliationCursor` together; +`changes_checkpoint` is `None` until the walk is complete. `cursor=page.cursor` +continues the frozen walk across a process restart. Persist the final page with +its `ReportingReconciliationCheckpoint`, then use `changes_after=checkpoint` to +open the next walk. Repeating a position intentionally replays immutable records. +`ReportingReconciliationFilter` supports record kinds and an exact obligation ID. +Caller scope and filters apply before counting and limiting; PostgreSQL keyset-pages +by the caller-local sequence under the frozen upper bound. + +Opaque tokens bind account, consumer, feed version, filter, lower/upper bounds and +the last emitted sequence/record key. Tokens carry no authority: caller identity +always comes from authenticated transport and is revalidated against retained +records. A Core checkpoint, foreign scope, changed filter, invalid bounds, or +inconsistent last key is rejected. Core's `LedgerRecordKind`, `LedgerPage`, required +store protocols and wire projection remain unchanged. The early PR spelling +`ReportingReconciliationChangeStore` is an alias for the optional feed protocol. + The opt-in `materialization_to_wire`, `receipt_to_wire`, `revision_to_wire`, and `adjustment_to_wire` helpers support generated wire models. They are not mounted by the Core status handler. Snapshot records are a lower-level storage read, not an unbounded wire response; bounded status pagination and tier-correct counts belong to the completion PR. -Each new record appends a distinct, scoped ledger change in the same transaction. +Each new record appends a distinct change in `reporting_reconciliation_changes` +and advances `reporting_reconciliation_heads` for its account/consumer in the same +transaction. The memory store publishes the evidence and caller-local sequence +with one assignment under its lock. Core's change feed is separate. Pending attempts and terminal outcomes have separate change kinds, so a snapshot cannot acquire terminal evidence committed after its boundary. Exact retries append nothing. Both stores share the transition validator; PostgreSQL serializes with the -existing account lock and additionally uses a conditional receipt-head update. -Database triggers reject evidence updates/deletes and terminal-head replacement. +existing account lock. Database triggers validate principal, generation, obligation, +revision, attempt and materialization references; receipt chains require the exact +current rejected predecessor. An insert advances its receipt head atomically in +the database, including for direct SQL writers. Composite foreign keys bind the +exact publication and adjustment graph. Bidirectional composite foreign keys require +the record and exact feed identity together; a deferred head reference and ordinal +guards prevent missing, skipped or reassigned sequences. Reads validate both sides +of the scoped feed/record join before boundary/count/limit so damage cannot disappear +through filtering. Stored identity columns are compared against the decoded payload. +Updates/deletes and terminal-head replacement are rejected; referenced +Core publications stay frozen while leases and readability remain operational. This is the transaction in which #1168 can insert its outbox row; there is no adopter callback or after-commit webhook send in this PR. @@ -177,15 +233,24 @@ The last migration is also one atomic statement in autocommit mode. All migratio share the schema advisory lock. Drain older writers before upgrading. The migration adds nullable managed digest/total evidence with no default/backfill, -immutable record and receipt-head tables, and account-qualified reference indexes. +immutable record, receipt-head, feed and caller-head tables, and account-qualified reference indexes. Literal beta.15 and #1171 upgrades preserve pre-existing rows, hashes, currency, leases, issue history and feed sequence numbers. Unknown currency/digest/total history stays unknown. Existing Core replay hashes do not change. A different replay of an existing adjustment now conflicts instead of silently returning its old content. +Upgrading the initial reconciliation schema preserves every existing record, +receipt head and legacy Core-feed row. It projects the retained legacy change order +into dense, independent account/consumer sequences without changing original hashes +or timestamps. The new reconciliation tokens do not reuse legacy/Core checkpoints; +start with a reconciliation snapshot or initial feed walk. The migration validates +all retained graph edges and feed identities and rejects incomplete history atomically; +it never repairs missing evidence or reassigns an obligation identity. Table/index creation and prerequisite migrations take locks; production-sized duration is not benchmarked. The reference stores load one consumer's retained -record set to validate transitions. Large histories need indexed queries or a -conforming replacement store before production rollout. Only SDK store operations +record set to validate transitions. Feed pages use indexed keyset reads, with scoped +integrity/count scans; these reads share the account transaction lock with writes. +Large histories need benchmarks or a conforming replacement store before production rollout. +Only SDK store operations are supported writers; database owners can always circumvent application invariants by disabling constraints. PostgreSQL connection ownership remains with the adopter. diff --git a/src/adcp/reporting/_reconcile.py b/src/adcp/reporting/_reconcile.py index 294a5977e..69143a092 100644 --- a/src/adcp/reporting/_reconcile.py +++ b/src/adcp/reporting/_reconcile.py @@ -441,9 +441,8 @@ def _select_current( item for item in ledger.revisions if ( - item.reporting_revision_id in revision_ids - if managed_delivery - else _revision_matches_obligation(item, obligation) + _revision_matches_obligation(item, obligation) + or (managed_delivery and item.reporting_revision_id in revision_ids) ) ] receipts = [ @@ -483,19 +482,30 @@ def _select_current( if item.supersedes_reporting_revision_id } candidate_ids = {item.reporting_revision_id for item in candidates} + if any(not _revision_matches_obligation(item, obligation) for item in candidates) or any( + item.reporting_revision_id in revision_ids + and item.reporting_obligation_id != obligation.reporting_obligation_id + for item in ledger.materializations + ): + reasons.append("REVISION_SCOPE_MISMATCH") if any( item.supersedes_reporting_revision_id and item.supersedes_reporting_revision_id not in candidate_ids for item in candidates ): reasons.append("INCOMPLETE_REVISION_CHAIN") - current = [item for item in candidates if item.reporting_revision_id not in superseded] + # Publication selection precedes destination selection. An official close + # coexists with retained snapshots; it does not supersede their histories. + # A newer unmaterialized publication must never reveal an older snapshot as + # the current deliverable merely because that snapshot has a ready resource. + official = [item for item in candidates if _enum(item.finality) == "official"] + current = official or [ + item for item in candidates if item.reporting_revision_id not in superseded + ] if len(current) != 1: reasons.append("MISSING_CURRENT_REVISION" if not current else "AMBIGUOUS_REVISION_CHAIN") return None, None, reasons revision = current[0] - if any(not _revision_matches_obligation(item, obligation) for item in candidates): - reasons.append("REVISION_SCOPE_MISMATCH") if ( not _coverage_is_full(obligation.coverage, obligation.media_buy_ids) or obligation.coverage.evaluated_at != obligation.scope_resolved_at diff --git a/src/adcp/reporting/evidence.py b/src/adcp/reporting/evidence.py index 35e46e79e..9281025ac 100644 --- a/src/adcp/reporting/evidence.py +++ b/src/adcp/reporting/evidence.py @@ -6,55 +6,101 @@ from dataclasses import dataclass from datetime import datetime, timezone from typing import ClassVar, Literal -from urllib.parse import urlsplit +from urllib.parse import unquote, urlsplit from pydantic import ConfigDict -def native_version_reference(value: str) -> str: - """Retain a decoded, publicly classified native version without URI encoding. +def _public_text(value: str, *, maximum: int) -> str: + """Reject recognizable credentials without normalizing a provider's identity. - Native versions are not entity IDs: for example, ``/``, ``+`` and ``=`` are - valid characters. The trusted adapter must establish that the value is public; - this guard rejects recognizable authorization material, not opaque secrets. + Provenance still belongs to the trusted adapter: no syntax check can decide + whether an arbitrary opaque value is a secret. Inspect percent-decoded forms + as well, so encoding cannot hide a URL or credential from this boundary. """ if ( type(value) is not str - or not 1 <= len(value) <= 1024 + or not 1 <= len(value) <= maximum or not value.isprintable() - or "://" in value - or re.search( - r"(?i)(?:bearer\s|password|secret|token|signature|credential|private.key|-----BEGIN)", - value, - ) + or not value.strip() ): - raise ValueError("native version evidence requires a non-secret decoded public reference") + raise ValueError("reporting metadata requires non-secret public text") + inspected = value + while True: + if ( + not inspected.isprintable() + or "://" in inspected + or inspected.startswith("//") + or re.search( + r"(?i)(?:bearer|password|secret|token|signature|credential|private.key|" + r"authorization|api[ _-]?key|access[ _-]?key|-----BEGIN|" + r"(?:^|\s)(?:https?|ftp|file|data|mailto|s3|gs):|" + r"[^\s/:]+:[^\s/]+@)", + inspected, + ) + or re.search(r"(?:^|[^A-Za-z0-9])(?:AKIA|ASIA)[A-Z0-9]{16}\b", inspected) + or re.search(r"\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+", inspected) + ): + raise ValueError("reporting metadata requires non-secret public text") + decoded = unquote(inspected) + if decoded == inspected: + break + inspected = decoded return value -def public_reference(value: str, *, maximum: int = 1024, path: bool = False) -> str: - """Accept inert identifiers, never URLs, query strings or authentication material. +def reporting_identifier(value: str, *, maximum: int = 255) -> str: + """Seller-issued wire identities use the schema's deliberately narrow grammar.""" + _public_text(value, maximum=maximum) + if re.fullmatch(r"[A-Za-z0-9_.:-]+", value) is None: + raise ValueError("reporting identity requires a public protocol identifier") + return value - These are trusted public labels, not an arbitrary provider response sanitiser. - Adapters needing a richer provider identifier must retain it behind the trusted - binding and publish an opaque label. Error messages never interpolate input. - """ - pattern = r"[A-Za-z0-9_.:/-]+" if path else r"[A-Za-z0-9_.:-]+" + +def principal_reference(value: str) -> str: + """Trusted public account/consumer identity, not a protocol entity ID.""" + return _public_text(value, maximum=255) + + +def destination_reference(value: str) -> str: + """Immutable public destination/configuration reference, with the wire length bound.""" + return _public_text(value, maximum=255) + + +def resource_location(value: str) -> str: + """Public provider-native relation/share/object name, kept byte-for-byte.""" + return _public_text(value, maximum=2048) + + +def file_object_reference(value: str) -> str: + """Decoded destination-relative object key, never a URL or version query.""" + _public_text(value, maximum=1024) if ( - not isinstance(value, str) - or not 1 <= len(value) <= maximum - or re.fullmatch(pattern, value) is None - or "://" in value - or value.startswith("/") - or any(part == ".." for part in value.split("/")) - or re.search( - r"(?i)(?:bearer|password|secret|token|signature|credential|private.key)", value - ) + value.startswith(("/", "\\")) + or "\\" in value + or "?" in value + or "#" in value + or ".." in value.split("/") ): - raise ValueError("reporting metadata requires a non-secret public reference") + raise ValueError("reporting object evidence requires a destination-relative decoded key") return value +def native_version_reference(value: str) -> str: + """Decoded provider-native immutable version, including spaces, '/', '+' and '='.""" + return _public_text(value, maximum=1024) + + +def consumer_commit_reference(value: str) -> str: + """Public checkpoint, transaction or load ID; no protocol entity-ID grammar.""" + return _public_text(value, maximum=512) + + +def reader_feature_reference(value: str) -> str: + """Public reader/format requirement, which may be a provider's feature label.""" + return _public_text(value, maximum=128) + + def sha256_value(value: str) -> str: if not isinstance(value, str) or re.fullmatch(r"[a-fA-F0-9]{64}", value) is None: raise ValueError("reporting evidence requires a SHA-256 digest") @@ -86,7 +132,7 @@ def __post_init__(self) -> None: or self.value_type not in {"integer", "decimal"} ): raise ValueError("control total evidence requires immutable typed values") - public_reference(self.name, maximum=128) + reporting_identifier(self.name, maximum=128) if re.fullmatch(r"[A-Za-z][A-Za-z0-9_.:-]{0,127}", self.name) is None: raise ValueError("reporting total names require public metric identifiers") pattern = ( @@ -97,7 +143,7 @@ def __post_init__(self) -> None: if re.fullmatch(pattern, self.value) is None: raise ValueError("control totals require canonical numeric strings matching their type") if self.unit is not None: - public_reference(self.unit, maximum=32) + _public_text(self.unit, maximum=32) def to_wire(self) -> dict[str, str]: result = {"name": self.name, "value": self.value, "value_type": self.value_type} @@ -162,7 +208,7 @@ def __post_init__(self) -> None: raise ValueError("canonical evidence requires immutable string values") sha256_value(self.value) sha256_value(self.canonicalization_sha256) - public_reference(self.canonicalization_id, maximum=128) + _public_text(self.canonicalization_id, maximum=128) uri = self.canonicalization_uri try: parsed = urlsplit(uri) diff --git a/src/adcp/reporting/ledger/__init__.py b/src/adcp/reporting/ledger/__init__.py index 8a972cc44..dac7f8299 100644 --- a/src/adcp/reporting/ledger/__init__.py +++ b/src/adcp/reporting/ledger/__init__.py @@ -90,6 +90,16 @@ receipt_to_wire, revision_to_wire, ) +from adcp.reporting.ledger.delivery_changes import ( + ReportingReconciliationChange, + ReportingReconciliationChangeStore, + ReportingReconciliationCheckpoint, + ReportingReconciliationCursor, + ReportingReconciliationFeedStore, + ReportingReconciliationFilter, + ReportingReconciliationPage, + ReportingReconciliationSnapshotToken, +) from adcp.reporting.ledger.delivery_models import ( ReportingAdjustmentReceiptRecord, ReportingControlTotalRecord, @@ -105,6 +115,7 @@ ReportingPhysicalChecksum, ReportingReceiptKey, ReportingReceiptRecord, + ReportingReconciliationRecordKind, ReportingResourceRecord, ReportingRevisionReceiptRecord, ReportingVerificationRecord, @@ -226,6 +237,15 @@ "ReportingReceiptRecord", "ReportingReceiptStore", "ReportingReconciliationSnapshot", + "ReportingReconciliationChange", + "ReportingReconciliationChangeStore", + "ReportingReconciliationCheckpoint", + "ReportingReconciliationCursor", + "ReportingReconciliationFeedStore", + "ReportingReconciliationFilter", + "ReportingReconciliationPage", + "ReportingReconciliationSnapshotToken", + "ReportingReconciliationRecordKind", "ReportingReconciliationStore", "ReportingResourceRecord", "ReportingRevisionReceiptRecord", diff --git a/src/adcp/reporting/ledger/_delivery_state.py b/src/adcp/reporting/ledger/_delivery_state.py index cd2dd1021..a385a5610 100644 --- a/src/adcp/reporting/ledger/_delivery_state.py +++ b/src/adcp/reporting/ledger/_delivery_state.py @@ -121,6 +121,63 @@ def receipt_chain(record: ReportingReceiptRecord) -> str: return hashlib.sha256(canonical_json_utf8_v1(parts)).hexdigest() +def storage_identity(record: ReportingDeliveryRecord) -> tuple[str | int | None, ...]: + """Every indexed/joined/transition column, in the PostgreSQL schema order.""" + who = principal(record) + if isinstance(record, ReportingDestinationBinding): + generation = record.generation_key + obligation_id = None + else: + generation = record.scope.generation_key + obligation_id = record.scope.reporting_obligation_id + receipt: ReportingReceiptRecord | None = record if isinstance(record, _RECEIPTS) else None + revision_id = None + if isinstance(record, ReportingAdjustmentReceiptRecord): + revision_id = record.adjusts_reporting_revision_id + elif isinstance( + record, + ( + ReportingMaterializationAttempt, + ReportingMaterializationRecord, + ReportingRevisionReceiptRecord, + ), + ): + revision_id = record.reporting_revision_id + materialization_id = ( + record.reporting_materialization_id + if isinstance( + record, + ( + ReportingMaterializationAttempt, + ReportingMaterializationRecord, + ReportingMaterializationCheck, + ReportingRevisionReceiptRecord, + ), + ) + else None + ) + return ( + who.account_id, + who.consumer_id, + *record_identity(record), + record.kind, + generation.delivery_config_id, + generation.delivery_config_version, + obligation_id, + revision_id, + materialization_id, + ( + record.reporting_adjustment_id + if isinstance(record, ReportingAdjustmentReceiptRecord) + else None + ), + record.attempt if isinstance(record, ReportingMaterializationAttempt) else None, + receipt_chain(receipt) if receipt is not None else None, + receipt.status if receipt is not None else None, + receipt.supersedes_reporting_receipt_id if receipt is not None else None, + ) + + def current_receipt( records: tuple[ReportingDeliveryRecord, ...], requested: ReportingReceiptRecord ) -> ReportingReceiptRecord | None: @@ -161,7 +218,6 @@ class DeliveryContext: configuration: ReportingConfiguration | None = None obligation: ReportingObligationRecord | None = None revision: ReportingRevisionRecord | None = None - revision_obligation: ReportingObligationRecord | None = None adjustment: ReportingAdjustmentRecord | None = None @@ -266,7 +322,7 @@ def validate_transition( revision is None or revision.account_id != who.account_id or revision.reporting_revision_id != revision_id - or not same_revision_scope(context.revision_obligation, obligation) + or revision.reporting_obligation_id != obligation.reporting_obligation_id ): unavailable() @@ -363,27 +419,6 @@ def validate_transition( return replace(record, received_at=now) -def same_revision_scope( - origin: ReportingObligationRecord | None, target: ReportingObligationRecord -) -> bool: - """Destination-independent content may fan out only over the exact frozen slice.""" - return bool( - origin is not None - and origin.account_id == target.account_id - and origin.report_definition_id == target.report_definition_id - and origin.reporting_profile == target.reporting_profile - and origin.definition == target.definition - and origin.currency == target.currency - and origin.period.start == target.period.start - and origin.period.end == target.period.end - and origin.period.source_timezone == target.period.source_timezone - and origin.scope_resolved_at == target.scope_resolved_at - and sorted(origin.media_buy_ids) == sorted(target.media_buy_ids) - and sorted(origin.package_ids) == sorted(target.package_ids) - and origin.coverage_status == target.coverage_status - ) - - def _verify_materialization( record: ReportingMaterializationRecord, binding: ReportingDestinationBinding, @@ -453,7 +488,8 @@ def _verify_materialization( or verification.native_version_ref is not None ): if ( - verification.native_version_ref is None + resource.immutability != "native_version" + or verification.native_version_ref is None or verification.native_version_ref != resource.native_version_ref or verification.native_observed_through != path ): diff --git a/src/adcp/reporting/ledger/delivery.py b/src/adcp/reporting/ledger/delivery.py index 6b65799ed..101a06707 100644 --- a/src/adcp/reporting/ledger/delivery.py +++ b/src/adcp/reporting/ledger/delivery.py @@ -18,6 +18,7 @@ adjustment_sha256, change_id, decode_record, + fail, iso, payload, principal, @@ -27,6 +28,18 @@ unavailable, validate_transition, ) +from adcp.reporting.ledger.delivery_changes import ( + ReportingReconciliationChange, + ReportingReconciliationCheckpoint, + ReportingReconciliationCursor, + ReportingReconciliationFilter, + ReportingReconciliationPage, + ReportingReconciliationSnapshotToken, + change_boundary, + change_page, + read_position, + validate_boundary, +) from adcp.reporting.ledger.delivery_models import ( ReportingAdjustmentReceiptRecord, ReportingDeliveryPrincipal, @@ -43,13 +56,12 @@ ReportingRevisionReceiptRecord, ) from adcp.reporting.ledger.models import ( - LedgerSnapshot, ReportingAdjustmentRecord, ReportingConfigurationGenerationKey, ReportingObligationRecord, ReportingRevisionRecord, ) -from adcp.reporting.ledger.store import InMemoryReportingLedgerStore +from adcp.reporting.ledger.store import InMemoryReportingLedgerStore, LedgerConflictError @runtime_checkable @@ -122,7 +134,7 @@ async def read_reconciliation_snapshot( self, *, caller: ReportingDeliveryPrincipal, - boundary: LedgerSnapshot | None = None, + boundary: ReportingReconciliationSnapshotToken | None = None, ) -> ReportingReconciliationSnapshot: ... @@ -213,7 +225,7 @@ def to_wire(self) -> dict[str, Any]: @dataclass(frozen=True, slots=True) class ReportingReconciliationSnapshot: caller: ReportingDeliveryPrincipal - boundary: LedgerSnapshot + boundary: ReportingReconciliationSnapshotToken records: tuple[ReportingDeliveryRecord, ...] def materialization( @@ -351,7 +363,10 @@ async def record_adjustment_receipt( return await self._commit(record) async def read_reconciliation_snapshot( - self, *, caller: ReportingDeliveryPrincipal, boundary: LedgerSnapshot | None = None + self, + *, + caller: ReportingDeliveryPrincipal, + boundary: ReportingReconciliationSnapshotToken | None = None, ) -> ReportingReconciliationSnapshot: raise NotImplementedError @@ -379,31 +394,57 @@ async def get_receipt(self, key: ReportingReceiptKey) -> ReportingReceiptRecord class InMemoryReportingReconciliationStore(InMemoryReportingLedgerStore, _ReconciliationOperations): """Optional reference extension. No destination/receipt services at construction.""" + _delivery_records: list[tuple[int, ReportingDeliveryPrincipal, ReportingDeliveryRecord]] + async def _commit(self, record: RecordT) -> tuple[RecordT, bool]: candidate = decode_record(payload(record)) who = principal(candidate) async with self._lock: - retained = self._retained_delivery_records() - records = tuple(item[2] for item in retained if item[1] == who) + records = tuple(item.record for item in self._caller_changes(who)) existing = replay(candidate, records) if existing is not None: return cast(RecordT, existing), False context = self._delivery_context(candidate) stored = validate_transition(candidate, records, context, self._clock()) - self._append(who.account_id, stored.kind, change_id(stored)) - retained.append((self._sequence, who, stored)) + self._append_reconciliation_change(stored) return cast(RecordT, stored), True + def _append_reconciliation_change(self, record: ReportingDeliveryRecord) -> None: + who = principal(record) + retained = self._retained_delivery_records() + sequence = sum(owner == who for _, owner, _ in retained) + 1 + # One assignment publishes the record, its feed row, and its local head. + # Core's sequence and change list never participate in this transaction. + self._delivery_records = [*retained, (sequence, who, record)] + def _retained_delivery_records( self, ) -> list[tuple[int, ReportingDeliveryPrincipal, ReportingDeliveryRecord]]: # Lazily allocated so construction keeps Core's exact component surface. if not hasattr(self, "_delivery_records"): - self._delivery_records: list[ - tuple[int, ReportingDeliveryPrincipal, ReportingDeliveryRecord] - ] = [] + self._delivery_records = [] return self._delivery_records + def _caller_changes( + self, caller: ReportingDeliveryPrincipal + ) -> tuple[ReportingReconciliationChange, ...]: + retained = [ + item + for item in self._retained_delivery_records() + if item[1] == caller or principal(item[2]) == caller + ] + if any( + type(sequence) is not int + or sequence != ordinal + or owner != caller + or principal(record) != caller + for ordinal, (sequence, owner, record) in enumerate(retained, start=1) + ): + fail("REPORTING_HISTORY_CORRUPT") + return tuple( + ReportingReconciliationChange(sequence, record) for sequence, _, record in retained + ) + def _delivery_context(self, record: ReportingDeliveryRecord) -> DeliveryContext: if isinstance(record, ReportingDestinationBinding): return DeliveryContext(configuration=self._configurations.get(record.generation_key)) @@ -415,9 +456,6 @@ def _delivery_context(self, record: ReportingDeliveryRecord) -> DeliveryContext: return DeliveryContext( obligation=obligation, revision=revision, - revision_obligation=( - self._obligations.get(revision.reporting_obligation_id) if revision else None - ), adjustment=( self._adjustments.get(record.reporting_adjustment_id) if isinstance(record, ReportingAdjustmentReceiptRecord) @@ -426,23 +464,76 @@ def _delivery_context(self, record: ReportingDeliveryRecord) -> DeliveryContext: ) async def read_reconciliation_snapshot( - self, *, caller: ReportingDeliveryPrincipal, boundary: LedgerSnapshot | None = None + self, + *, + caller: ReportingDeliveryPrincipal, + boundary: ReportingReconciliationSnapshotToken | None = None, ) -> ReportingReconciliationSnapshot: - if boundary is None: - boundary = await self.open_snapshot( - account_id=caller.account_id, filters_fingerprint=caller.consumer_id - ) - if boundary.account_id != caller.account_id: - unavailable() async with self._lock: + changes = self._caller_changes(caller) + if boundary is None: + boundary = change_boundary(caller, len(changes), self._clock()) + if ( + type(boundary) is not ReportingReconciliationSnapshotToken + or boundary.caller != caller + or boundary.min_sequence != 0 + or boundary.filters != ReportingReconciliationFilter() + ): + unavailable() + validate_boundary(caller, boundary, len(changes)) + if boundary.total_count != boundary.max_sequence: + fail("REPORTING_HISTORY_CORRUPT") return ReportingReconciliationSnapshot( caller, boundary, - tuple( - record - for sequence, owner, record in self._retained_delivery_records() - if owner == caller and sequence <= boundary.max_sequence - ), + tuple(item.record for item in changes if item.sequence <= boundary.max_sequence), + ) + + async def read_reconciliation_changes( + self, + *, + caller: ReportingDeliveryPrincipal, + changes_after: ReportingReconciliationCheckpoint | None = None, + cursor: ReportingReconciliationCursor | None = None, + limit: int = 100, + filters: ReportingReconciliationFilter = ReportingReconciliationFilter(), + ) -> ReportingReconciliationPage: + after, boundary, last_key = read_position(caller, changes_after, cursor, limit, filters) + async with self._lock: + records = self._caller_changes(caller) + if boundary is None: + boundary = change_boundary( + caller, + len(records), + self._clock(), + after=after, + total_count=sum( + item.sequence > after and filters.matches(item.record) for item in records + ), + filters=filters, + ) + validate_boundary(caller, boundary, len(records)) + if last_key is not None and not any( + item.sequence == after + and change_id(item.record) == last_key + and filters.matches(item.record) + for item in records + ): + raise LedgerConflictError("INVALID_CHECKPOINT", "reconciliation key is unavailable") + changes = tuple( + item + for item in records + if boundary.min_sequence < item.sequence <= boundary.max_sequence + and filters.matches(item.record) + ) + if len(changes) != boundary.total_count: + fail("REPORTING_HISTORY_CORRUPT") + return change_page( + caller, + boundary, + after, + tuple(item for item in changes if item.sequence > after), + limit, ) diff --git a/src/adcp/reporting/ledger/delivery_changes.py b/src/adcp/reporting/ledger/delivery_changes.py new file mode 100644 index 000000000..d4221d230 --- /dev/null +++ b/src/adcp/reporting/ledger/delivery_changes.py @@ -0,0 +1,318 @@ +"""Consumer-scoped incremental reconciliation reads, independent of Core cursors.""" + +from __future__ import annotations + +import hashlib +from dataclasses import asdict, dataclass +from datetime import datetime +from typing import Any, NewType, Protocol, get_args, runtime_checkable + +from adcp.reporting.canonical_json import canonical_json_utf8_v1 +from adcp.reporting.evidence import aware_utc, reporting_identifier +from adcp.reporting.ledger._delivery_state import change_id, fail, principal +from adcp.reporting.ledger.delivery_models import ( + ReportingDeliveryPrincipal, + ReportingDeliveryRecord, + ReportingDestinationBinding, + ReportingReconciliationRecordKind, +) +from adcp.reporting.ledger.store import LedgerConflictError, decode_cursor, encode_cursor + +_FEED = "reporting-reconciliation-v1" +ReportingReconciliationCursor = NewType("ReportingReconciliationCursor", str) +ReportingReconciliationCheckpoint = NewType("ReportingReconciliationCheckpoint", str) + + +def _is_record_kind(value: object) -> bool: + return type(value) is str and value in get_args(ReportingReconciliationRecordKind) + + +@dataclass(frozen=True, slots=True) +class ReportingReconciliationFilter: + record_kinds: tuple[ReportingReconciliationRecordKind, ...] = () + reporting_obligation_id: str | None = None + + def __post_init__(self) -> None: + if type(self.record_kinds) is not tuple or any( + not _is_record_kind(kind) for kind in self.record_kinds + ): + raise ValueError("invalid reconciliation record filter") + object.__setattr__(self, "record_kinds", tuple(sorted(set(self.record_kinds)))) + if self.reporting_obligation_id is not None: + reporting_identifier(self.reporting_obligation_id) + + @property + def fingerprint(self) -> str: + return hashlib.sha256(canonical_json_utf8_v1(asdict(self))).hexdigest() + + def matches(self, record: ReportingDeliveryRecord) -> bool: + return (not self.record_kinds or record.kind in self.record_kinds) and ( + self.reporting_obligation_id is None + or ( + not isinstance(record, ReportingDestinationBinding) + and record.scope.reporting_obligation_id == self.reporting_obligation_id + ) + ) + + +@dataclass(frozen=True, slots=True) +class ReportingReconciliationSnapshotToken: + """A frozen boundary in one principal's feed, never a Core ledger sequence.""" + + caller: ReportingDeliveryPrincipal + snapshot_id: str + ledger_as_of: datetime + min_sequence: int + max_sequence: int + total_count: int + filters: ReportingReconciliationFilter + + @property + def account_id(self) -> str: + return self.caller.account_id + + @property + def consumer_id(self) -> str: + return self.caller.consumer_id + + +@dataclass(frozen=True, slots=True) +class ReportingReconciliationChange: + sequence: int + record: ReportingDeliveryRecord + + +@dataclass(frozen=True, slots=True) +class ReportingReconciliationPage: + caller: ReportingDeliveryPrincipal + boundary: ReportingReconciliationSnapshotToken + changes: tuple[ReportingReconciliationChange, ...] + has_more: bool + cursor: ReportingReconciliationCursor | None + changes_checkpoint: ReportingReconciliationCheckpoint | None + + @property + def total_count(self) -> int: + return self.boundary.total_count + + +@runtime_checkable +class ReportingReconciliationFeedStore(Protocol): + """An optional seam; existing Core and reconciliation protocols are unchanged. + + Persist partial pages with their cursor. Only the final page issues a + checkpoint, which opens a new walk via ``changes_after``. Tokens bind caller, + version, filters, bounds and the last emitted key, and survive reader restarts. + Neither token is an authorization grant or a Core status checkpoint. + """ + + async def read_reconciliation_changes( + self, + *, + caller: ReportingDeliveryPrincipal, + changes_after: ReportingReconciliationCheckpoint | None = None, + cursor: ReportingReconciliationCursor | None = None, + limit: int = 100, + filters: ReportingReconciliationFilter = ReportingReconciliationFilter(), + ) -> ReportingReconciliationPage: ... + + +# Retain the early storage-slice spelling for callers already evaluating the PR. +ReportingReconciliationChangeStore = ReportingReconciliationFeedStore + + +def _token_scope( + caller: ReportingDeliveryPrincipal, filters: ReportingReconciliationFilter +) -> dict[str, str | int]: + return { + "feed": _FEED, + "version": 1, + "account": caller.account_id, + "consumer": caller.consumer_id, + "filter": filters.fingerprint, + } + + +def _decode_position( + value: str, caller: ReportingDeliveryPrincipal, filters: ReportingReconciliationFilter +) -> dict[str, Any]: + decoded: dict[str, Any] | None = None + try: + if type(value) is str and 0 < len(value) <= 4096: + decoded = decode_cursor(value) + except (LedgerConflictError, TypeError, ValueError): + pass + if ( + decoded is None + or type(decoded.get("version")) is not int + or any(decoded.get(k) != v for k, v in _token_scope(caller, filters).items()) + ): + raise LedgerConflictError("INVALID_CHECKPOINT", "reconciliation position is unavailable") + return decoded + + +def change_boundary( + caller: ReportingDeliveryPrincipal, + maximum: int, + as_of: datetime, + *, + after: int = 0, + total_count: int | None = None, + filters: ReportingReconciliationFilter = ReportingReconciliationFilter(), +) -> ReportingReconciliationSnapshotToken: + count = maximum - after if total_count is None else total_count + if ( + any(type(value) is not int for value in (after, maximum, count)) + or not 0 <= after <= maximum + or not 0 <= count <= maximum - after + ): + raise LedgerConflictError("INVALID_CHECKPOINT", "invalid reconciliation boundary") + identity = hashlib.sha256( + canonical_json_utf8_v1([_token_scope(caller, filters), after, maximum, count]) + ).hexdigest() + return ReportingReconciliationSnapshotToken( + caller, "rprc_" + identity[:32], aware_utc(as_of), after, maximum, count, filters + ) + + +def validate_boundary( + caller: ReportingDeliveryPrincipal, + boundary: ReportingReconciliationSnapshotToken, + maximum: int, +) -> None: + valid = False + try: + if ( + type(boundary) is ReportingReconciliationSnapshotToken + and boundary.caller == caller + and type(boundary.filters) is ReportingReconciliationFilter + ): + expected = change_boundary( + caller, + boundary.max_sequence, + boundary.ledger_as_of, + after=boundary.min_sequence, + total_count=boundary.total_count, + filters=boundary.filters, + ) + valid = boundary == expected and boundary.max_sequence <= maximum + except (ValueError, TypeError, LedgerConflictError): + pass + if not valid: + raise LedgerConflictError("INVALID_CHECKPOINT", "reconciliation boundary is unavailable") + + +def read_position( + caller: ReportingDeliveryPrincipal, + changes_after: ReportingReconciliationCheckpoint | None, + cursor: ReportingReconciliationCursor | None, + limit: int, + filters: ReportingReconciliationFilter, +) -> tuple[int, ReportingReconciliationSnapshotToken | None, str | None]: + if ( + type(limit) is not int + or not 1 <= limit <= 1000 + or (changes_after is not None and cursor is not None) + or type(caller) is not ReportingDeliveryPrincipal + or type(filters) is not ReportingReconciliationFilter + ): + raise LedgerConflictError("INVALID_CHECKPOINT", "invalid reconciliation page request") + if cursor is None and changes_after is None: + return 0, None, None + position = _decode_position( + cursor if cursor is not None else changes_after or "", caller, filters + ) + sequence = position.get("seq") + expected = {*_token_scope(caller, filters), "type", "seq"} + if cursor is not None: + expected.update(("after", "through", "as_of", "count", "snapshot", "key")) + if ( + set(position) != expected + or type(sequence) is not int + or sequence < 0 + or position["type"] != ("cursor" if cursor is not None else "checkpoint") + ): + raise LedgerConflictError("INVALID_CHECKPOINT", "invalid reconciliation position") + if cursor is None: + return sequence, None, None + through, after, as_of = position["through"], position["after"], position["as_of"] + boundary = None + if ( + type(through) is int + and type(after) is int + and 0 <= after < sequence < through + and type(as_of) is str + and type(position["count"]) is int + and type(position["key"]) is str + and len(position["key"]) == 64 + ): + try: + boundary = change_boundary( + caller, + through, + datetime.fromisoformat(as_of), + after=after, + total_count=position["count"], + filters=filters, + ) + except (ValueError, LedgerConflictError): + pass + if boundary is None or position["snapshot"] != boundary.snapshot_id: + raise LedgerConflictError("INVALID_CHECKPOINT", "invalid reconciliation boundary") + return sequence, boundary, position["key"] + + +def change_page( + caller: ReportingDeliveryPrincipal, + boundary: ReportingReconciliationSnapshotToken, + after: int, + changes: tuple[ReportingReconciliationChange, ...], + limit: int, +) -> ReportingReconciliationPage: + validate_boundary(caller, boundary, boundary.max_sequence) + if not boundary.min_sequence <= after <= boundary.max_sequence: + raise LedgerConflictError("INVALID_CHECKPOINT", "reconciliation position is unavailable") + previous = after + for change in changes: + if ( + type(change.sequence) is not int + or not previous < change.sequence <= boundary.max_sequence + or principal(change.record) != caller + or not boundary.filters.matches(change.record) + ): + fail("REPORTING_HISTORY_CORRUPT") + previous = change.sequence + window, more = changes[:limit], len(changes) > limit + position = _token_scope(caller, boundary.filters) + return ReportingReconciliationPage( + caller, + boundary, + window, + more, + ( + ReportingReconciliationCursor( + encode_cursor( + { + **position, + "type": "cursor", + "seq": window[-1].sequence, + "key": change_id(window[-1].record), + "after": boundary.min_sequence, + "through": boundary.max_sequence, + "as_of": boundary.ledger_as_of.isoformat(), + "count": boundary.total_count, + "snapshot": boundary.snapshot_id, + } + ) + ) + if more + else None + ), + ( + None + if more + else ReportingReconciliationCheckpoint( + encode_cursor({**position, "type": "checkpoint", "seq": boundary.max_sequence}) + ) + ), + ) diff --git a/src/adcp/reporting/ledger/delivery_models.py b/src/adcp/reporting/ledger/delivery_models.py index 0b0fa78da..9fc33172b 100644 --- a/src/adcp/reporting/ledger/delivery_models.py +++ b/src/adcp/reporting/ledger/delivery_models.py @@ -19,8 +19,14 @@ from adcp.reporting.evidence import ( ReportingCanonicalDigest, aware_utc, + consumer_commit_reference, + destination_reference, + file_object_reference, native_version_reference, - public_reference, + principal_reference, + reader_feature_reference, + reporting_identifier, + resource_location, sha256_value, ) from adcp.reporting.evidence import ReportingControlTotalRecord as ReportingControlTotalRecord @@ -31,6 +37,15 @@ VerificationProfile = Literal["canonical_digest", "manifest_checksums", "native_commit"] VerificationPath = Literal["producer", "representative_consumer", "destination"] ReceiptStatus = Literal["accepted", "rejected"] +ReportingReconciliationRecordKind = Literal[ + "destination_binding", + "obligation_delivery", + "materialization_attempt", + "materialization", + "materialization_check", + "revision_receipt", + "adjustment_receipt", +] MaterializationFailure = Literal[ "WRITE_FAILED", "VERIFICATION_FAILED", "CONTENT_CORRUPT", "RESOURCE_UNAVAILABLE" ] @@ -98,8 +113,8 @@ class ReportingDeliveryPrincipal(_ClosedValue): def __post_init__(self) -> None: _freeze_fields(self) - public_reference(self.account_id, maximum=255) - public_reference(self.consumer_id, maximum=255) + principal_reference(self.account_id) + principal_reference(self.consumer_id) @dataclass(frozen=True, slots=True) @@ -113,9 +128,9 @@ def __post_init__(self) -> None: if type(self.generation_key) is not ReportingConfigurationGenerationKey: raise ValueError("reporting scope requires the typed configuration generation") self.principal - public_reference(self.generation_key.delivery_config_id, maximum=64) + reporting_identifier(self.generation_key.delivery_config_id, maximum=64) _positive(self.generation_key.delivery_config_version) - public_reference(self.reporting_obligation_id, maximum=255) + reporting_identifier(self.reporting_obligation_id, maximum=255) @property def principal(self) -> ReportingDeliveryPrincipal: @@ -129,7 +144,7 @@ class ReportingMaterializationKey(_ClosedValue): def __post_init__(self) -> None: _freeze_fields(self) - public_reference(self.reporting_materialization_id, maximum=255) + reporting_identifier(self.reporting_materialization_id, maximum=255) @dataclass(frozen=True, slots=True) @@ -139,7 +154,7 @@ class ReportingReceiptKey(_ClosedValue): def __post_init__(self) -> None: _freeze_fields(self) - public_reference(self.reporting_receipt_id, maximum=255) + reporting_identifier(self.reporting_receipt_id, maximum=255) if len(self.reporting_receipt_id) < 16: raise ValueError("a reporting receipt identifier requires at least 16 characters") @@ -172,22 +187,24 @@ class ReportingDestinationBinding(_ClosedValue): def __post_init__(self) -> None: _freeze_fields(self) ReportingDeliveryScope(self.generation_key, self.consumer_id, "binding") - public_reference(self.destination_ref, maximum=255) - public_reference(self.trusted_binding_ref, maximum=255) + destination_reference(self.destination_ref) + destination_reference(self.trusted_binding_ref) if re.fullmatch(r"[a-z][a-z0-9_.-]{0,63}", self.transport) is None: raise ValueError("reporting transport requires a public protocol label") - public_reference(self.transport, maximum=64) + reporting_identifier(self.transport, maximum=64) _positive(self.resource_retention_days) object.__setattr__(self, "created_at", aware_utc(self.created_at)) object.__setattr__( self, "reader_compatibility", - tuple(public_reference(value, maximum=128) for value in self.reader_compatibility), + tuple(reader_feature_reference(value) for value in self.reader_compatibility), ) if len(set(self.reader_compatibility)) != len(self.reader_compatibility): raise ValueError("reader compatibility requirements must be unique") if self.method == "file_transfer" and self.format is None: raise ValueError("file transfer requires a declared format") + if self.verification_profile == "manifest_checksums" and self.method != "file_transfer": + raise ValueError("manifest checksum verification requires file transfer") if self.method == "warehouse_materialization" and self.success_status != "delivered": raise ValueError("warehouse materialization requires destination delivery") if self.method == "dataset_share" and self.success_status != "available": @@ -228,8 +245,8 @@ class ReportingMaterializationAttempt(_ClosedValue): def __post_init__(self) -> None: _freeze_fields(self) - public_reference(self.reporting_revision_id, maximum=255) - public_reference(self.reporting_materialization_id, maximum=255) + reporting_identifier(self.reporting_revision_id, maximum=255) + reporting_identifier(self.reporting_materialization_id, maximum=255) _positive(self.attempt) object.__setattr__(self, "created_at", aware_utc(self.created_at)) @@ -252,8 +269,8 @@ class ReportingResourceRecord(_ClosedValue): def __post_init__(self) -> None: _freeze_fields(self) - public_reference(self.resource_ref, maximum=255) - public_reference(self.location, maximum=2048, path=True) + reporting_identifier(self.resource_ref, maximum=255) + resource_location(self.location) if self.native_version_ref is not None: native_version_reference(self.native_version_ref) if self.manifest_sha256 is not None: @@ -266,12 +283,12 @@ def __post_init__(self) -> None: object.__setattr__( self, "object_refs", - tuple(public_reference(value, path=True) for value in self.object_refs), + tuple(file_object_reference(value) for value in self.object_refs), ) object.__setattr__( self, "reader_compatibility", - tuple(public_reference(value, maximum=128) for value in self.reader_compatibility), + tuple(reader_feature_reference(value) for value in self.reader_compatibility), ) if len(set(self.object_refs)) != len(self.object_refs): raise ValueError("resource object references must be unique") @@ -287,7 +304,7 @@ class ReportingPhysicalChecksum(_ClosedValue): def __post_init__(self) -> None: _freeze_fields(self) - public_reference(self.object_ref, path=True) + file_object_reference(self.object_ref) size = 64 if self.algorithm == "sha256" else 128 if ( not isinstance(self.value, str) @@ -342,8 +359,8 @@ class ReportingMaterializationRecord(_ClosedValue): def __post_init__(self) -> None: _freeze_fields(self) - public_reference(self.reporting_revision_id, maximum=255) - public_reference(self.reporting_materialization_id, maximum=255) + reporting_identifier(self.reporting_revision_id, maximum=255) + reporting_identifier(self.reporting_materialization_id, maximum=255) object.__setattr__(self, "completed_at", aware_utc(self.completed_at)) if self.status == "failed": if ( @@ -377,8 +394,8 @@ class ReportingMaterializationCheck(_ClosedValue): def __post_init__(self) -> None: _freeze_fields(self) - public_reference(self.reporting_materialization_id, maximum=255) - public_reference(self.check_id, maximum=255) + reporting_identifier(self.reporting_materialization_id, maximum=255) + reporting_identifier(self.check_id, maximum=255) object.__setattr__(self, "checked_at", aware_utc(self.checked_at)) @@ -405,8 +422,8 @@ class ReportingRevisionReceiptRecord(_ClosedValue): def __post_init__(self) -> None: _freeze_fields(self) self.key - public_reference(self.reporting_revision_id, maximum=255) - public_reference(self.reporting_materialization_id, maximum=255) + reporting_identifier(self.reporting_revision_id, maximum=255) + reporting_identifier(self.reporting_materialization_id, maximum=255) _positive(self.observed_row_count, zero=True) _unique_totals(self.observed_control_totals) if self.observed_manifest_sha256 is not None: @@ -414,7 +431,7 @@ def __post_init__(self) -> None: if self.observed_native_version_ref is not None: native_version_reference(self.observed_native_version_ref) if self.consumer_commit_ref is not None: - public_reference(self.consumer_commit_ref, maximum=512) + consumer_commit_reference(self.consumer_commit_ref) _receipt_fields(self) @property @@ -439,8 +456,8 @@ class ReportingAdjustmentReceiptRecord(_ClosedValue): def __post_init__(self) -> None: _freeze_fields(self) self.key - public_reference(self.reporting_adjustment_id, maximum=255) - public_reference(self.adjusts_reporting_revision_id, maximum=255) + reporting_identifier(self.reporting_adjustment_id, maximum=255) + reporting_identifier(self.adjusts_reporting_revision_id, maximum=255) sha256_value(self.observed_adjustment_sha256) _receipt_fields(self) diff --git a/src/adcp/reporting/ledger/delivery_pg.py b/src/adcp/reporting/ledger/delivery_pg.py index d407c1996..0e36673ce 100644 --- a/src/adcp/reporting/ledger/delivery_pg.py +++ b/src/adcp/reporting/ledger/delivery_pg.py @@ -1,7 +1,8 @@ -"""PostgreSQL reconciliation extension, sharing the Core ledger transaction/feed.""" +"""PostgreSQL reconciliation evidence with a separate, principal-qualified feed.""" from __future__ import annotations +from datetime import datetime from typing import Any, cast from adcp.reporting.ledger._delivery_state import ( @@ -13,9 +14,9 @@ fingerprint, payload, principal, - receipt_chain, record_identity, replay, + storage_identity, unavailable, validate_transition, ) @@ -23,16 +24,24 @@ ReportingReconciliationSnapshot, _ReconciliationOperations, ) +from adcp.reporting.ledger.delivery_changes import ( + ReportingReconciliationChange, + ReportingReconciliationCheckpoint, + ReportingReconciliationCursor, + ReportingReconciliationFilter, + ReportingReconciliationPage, + ReportingReconciliationSnapshotToken, + change_boundary, + change_page, + read_position, + validate_boundary, +) from adcp.reporting.ledger.delivery_models import ( ReportingAdjustmentReceiptRecord, ReportingDeliveryPrincipal, ReportingDeliveryRecord, ReportingDestinationBinding, - ReportingMaterializationAttempt, - ReportingReceiptRecord, - ReportingRevisionReceiptRecord, ) -from adcp.reporting.ledger.models import LedgerSnapshot from adcp.reporting.ledger.pg import ( _ADJUSTMENT_COLUMNS, _OBLIGATION_COLUMNS, @@ -44,6 +53,41 @@ _obligation_from_row, _revision_from_row, ) +from adcp.reporting.ledger.store import LedgerConflictError + +_IDENTITY_COLUMNS = ( + "account_id", + "consumer_id", + "namespace", + "record_id", + "record_kind", + "delivery_config_id", + "delivery_config_version", + "reporting_obligation_id", + "reporting_revision_id", + "reporting_materialization_id", + "reporting_adjustment_id", + "attempt_number", + "receipt_chain_key", + "receipt_status", + "supersedes_receipt_id", +) +_SELECT_IDENTITY = ", ".join("r." + column for column in _IDENTITY_COLUMNS) +_FEED_JOIN = ( + "r.account_id = c.account_id AND r.consumer_id = c.consumer_id" + " AND r.namespace = c.namespace AND r.record_id = c.record_id" + " AND r.record_kind = c.record_kind AND r.change_id = c.change_id" + " AND r.content_sha256 = c.content_sha256" +) +_FEED_FILTER = ( + "(%s::text[] IS NULL OR c.record_kind = ANY(%s))" + " AND (%s::text IS NULL OR r.reporting_obligation_id = %s)" +) + + +def _filter_params(filters: ReportingReconciliationFilter) -> tuple[Any, ...]: + kinds = list(filters.record_kinds) or None + return kinds, kinds, filters.reporting_obligation_id, filters.reporting_obligation_id class PgReportingReconciliationStore(PgReportingLedgerStore, _ReconciliationOperations): @@ -55,6 +99,16 @@ class PgReportingReconciliationStore(PgReportingLedgerStore, _ReconciliationOper """ async def _commit(self, record: RecordT) -> tuple[RecordT, bool]: + try: + return await self._commit_record(record) + except Exception as error: + # A concurrent/raw SQL writer must not turn a constraint detail + # (which may include an entire row) into a provider-payload echo. + if not str(getattr(error, "sqlstate", "")).startswith("23"): + raise + unavailable() + + async def _commit_record(self, record: RecordT) -> tuple[RecordT, bool]: candidate = decode_record(payload(record)) who = principal(candidate) async with self._pool.connection() as connection: @@ -75,27 +129,96 @@ async def _commit(self, record: RecordT) -> tuple[RecordT, bool]: now = time_row[0] stored = validate_transition(candidate, records, context, now) await self._insert(connection, stored) - if isinstance( - stored, (ReportingRevisionReceiptRecord, ReportingAdjustmentReceiptRecord) - ): - await self._advance_receipt(connection, stored) - await self._append_change( - connection, who.account_id, stored.kind, change_id(stored) - ) + await self._append_reconciliation_change(connection, stored) return cast(RecordT, stored), True + async def _append_reconciliation_change( + self, connection: Any, record: ReportingDeliveryRecord + ) -> None: + who = principal(record) + row = await ( + await connection.execute( + "INSERT INTO reporting_reconciliation_heads (account_id, consumer_id, max_sequence)" + " VALUES (%s, %s, 1) ON CONFLICT (account_id, consumer_id) DO UPDATE" + " SET max_sequence = reporting_reconciliation_heads.max_sequence + 1" + " RETURNING max_sequence", + (who.account_id, who.consumer_id), + ) + ).fetchone() + assert row is not None + await connection.execute( + "INSERT INTO reporting_reconciliation_changes" + " (account_id, consumer_id, seq, namespace, record_id, record_kind," + " change_id, content_sha256)" + " VALUES (%s, %s, %s, %s, %s, %s, %s, %s)", + ( + who.account_id, + who.consumer_id, + row[0], + *record_identity(record), + record.kind, + change_id(record), + fingerprint(record), + ), + ) + + async def _validate_feed( + self, connection: Any, who: ReportingDeliveryPrincipal + ) -> tuple[int, datetime]: + # Scope both sides before the join, count, or boundary. A missing/moved + # record or feed row must fail, never disappear through an inner join. + # The joined SQL fragment is constant; every caller value is parameterized. + row = await ( + await connection.execute( + "SELECT count(c.seq), COALESCE(max(c.seq), 0)," # nosec B608 + " COALESCE((SELECT max_sequence FROM reporting_reconciliation_heads" + " WHERE account_id = %s AND consumer_id = %s), 0)," + " COALESCE(bool_or(c.seq IS NULL OR r.record_id IS NULL), false), clock_timestamp()" + " FROM (SELECT * FROM reporting_reconciliation_changes" + " WHERE account_id = %s AND consumer_id = %s) c" + " FULL JOIN (SELECT * FROM reporting_reconciliation_records" + " WHERE account_id = %s AND consumer_id = %s) r ON " + _FEED_JOIN, + (who.account_id, who.consumer_id) * 3, + ) + ).fetchone() + assert row is not None + if row[0] != row[1] or row[1] != row[2] or row[3]: + fail("REPORTING_HISTORY_CORRUPT") + return row[2], row[4] + async def _records( self, connection: Any, who: ReportingDeliveryPrincipal, maximum: int | None = None ) -> tuple[ReportingDeliveryRecord, ...]: + await self._validate_feed(connection, who) + return tuple(item.record for item in await self._changes(connection, who, maximum)) + + async def _changes( + self, + connection: Any, + who: ReportingDeliveryPrincipal, + maximum: int | None = None, + *, + after: int = 0, + limit: int | None = None, + filters: ReportingReconciliationFilter = ReportingReconciliationFilter(), + ) -> tuple[ReportingReconciliationChange, ...]: rows = await ( await connection.execute( - "SELECT r.payload, r.content_sha256, c.seq, r.namespace, r.record_id," - " r.record_kind, r.change_id FROM reporting_reconciliation_records r" - " LEFT JOIN reporting_ledger_changes c ON c.account_id = r.account_id" - " AND c.record_kind = r.record_kind AND c.record_id = r.change_id" - " WHERE r.account_id = %s AND r.consumer_id = %s" - " AND (c.seq IS NULL OR %s::bigint IS NULL OR c.seq <= %s::bigint) ORDER BY c.seq", - (who.account_id, who.consumer_id, maximum, maximum), + "SELECT r.payload, r.content_sha256, c.seq, r.change_id, " + f"{_SELECT_IDENTITY} FROM reporting_reconciliation_changes c" # noqa: S608 # nosec B608 + f" JOIN reporting_reconciliation_records r ON {_FEED_JOIN}" + " WHERE c.account_id = %s AND c.consumer_id = %s" + " AND (%s::bigint IS NULL OR c.seq <= %s::bigint) AND c.seq > %s" + f" AND {_FEED_FILTER} ORDER BY c.seq LIMIT %s", + ( + who.account_id, + who.consumer_id, + maximum, + maximum, + after, + *_filter_params(filters), + limit, + ), ) ).fetchall() records = tuple(decode_record(row[0]) for row in rows) @@ -103,13 +226,34 @@ async def _records( principal(record) != who or fingerprint(record) != row[1] or row[2] is None - or record_identity(record) != (row[3], row[4]) - or record.kind != row[5] - or change_id(record) != row[6] + or change_id(record) != row[3] + or storage_identity(record) != tuple(row[4:]) for record, row in zip(records, rows) ): fail("REPORTING_HISTORY_CORRUPT") - return records + return tuple( + ReportingReconciliationChange(row[2], record) for row, record in zip(rows, records) + ) + + async def _change_count( + self, + connection: Any, + caller: ReportingDeliveryPrincipal, + after: int, + maximum: int, + filters: ReportingReconciliationFilter, + ) -> int: + row = await ( + await connection.execute( + "SELECT count(*) FROM reporting_reconciliation_changes c" + f" JOIN reporting_reconciliation_records r ON {_FEED_JOIN}" # noqa: S608 # nosec B608 + " WHERE c.account_id = %s AND c.consumer_id = %s AND c.seq > %s AND c.seq <= %s" + f" AND {_FEED_FILTER}", + (caller.account_id, caller.consumer_id, after, maximum, *_filter_params(filters)), + ) + ).fetchone() + assert row is not None + return int(row[0]) async def _delivery_context( self, connection: Any, record: ReportingDeliveryRecord @@ -152,15 +296,6 @@ async def _delivery_context( ) ).fetchone() adjustment_row = None - revision_obligation_row = None - if revision_row is not None: - revision_obligation_row = await ( - await connection.execute( - f"SELECT {_OBLIGATION_COLUMNS} FROM reporting_obligations" # noqa: S608 # nosec B608 - " WHERE account_id = %s AND reporting_obligation_id = %s", - (who.account_id, revision_row[2]), - ) - ).fetchone() if isinstance(record, ReportingAdjustmentReceiptRecord): adjustment_row = await ( await connection.execute( @@ -172,27 +307,10 @@ async def _delivery_context( return DeliveryContext( obligation=_obligation_from_row(obligation_row) if obligation_row else None, revision=_revision_from_row(revision_row) if revision_row else None, - revision_obligation=( - _obligation_from_row(revision_obligation_row) if revision_obligation_row else None - ), adjustment=_adjustment_from_row(adjustment_row) if adjustment_row else None, ) async def _insert(self, connection: Any, record: ReportingDeliveryRecord) -> None: - who = principal(record) - namespace, record_id = record_identity(record) - generation = ( - record.generation_key - if isinstance(record, ReportingDestinationBinding) - else record.scope.generation_key - ) - receipt = ( - record - if isinstance( - record, (ReportingRevisionReceiptRecord, ReportingAdjustmentReceiptRecord) - ) - else None - ) await connection.execute( "INSERT INTO reporting_reconciliation_records" " (account_id, consumer_id, namespace, record_id, record_kind, delivery_config_id," @@ -203,84 +321,85 @@ async def _insert(self, connection: Any, record: ReportingDeliveryRecord) -> Non " VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s," " %s::jsonb, %s, %s)", ( - who.account_id, - who.consumer_id, - namespace, - record_id, - record.kind, - generation.delivery_config_id, - generation.delivery_config_version, - ( - None - if isinstance(record, ReportingDestinationBinding) - else record.scope.reporting_obligation_id - ), - ( - record.adjusts_reporting_revision_id - if isinstance(record, ReportingAdjustmentReceiptRecord) - else getattr(record, "reporting_revision_id", None) - ), - getattr(record, "reporting_materialization_id", None), - ( - record.reporting_adjustment_id - if isinstance(record, ReportingAdjustmentReceiptRecord) - else None - ), - record.attempt if isinstance(record, ReportingMaterializationAttempt) else None, - receipt_chain(receipt) if receipt is not None else None, - receipt.status if receipt is not None else None, - receipt.supersedes_reporting_receipt_id if receipt is not None else None, + *storage_identity(record), _json(payload(record)), fingerprint(record), change_id(record), ), ) - async def _advance_receipt(self, connection: Any, record: ReportingReceiptRecord) -> None: - who = principal(record) - chain = receipt_chain(record) - if record.supersedes_reporting_receipt_id is None: - cursor = await connection.execute( - "INSERT INTO reporting_receipt_heads" - " (account_id, consumer_id, chain_key, receipt_id, receipt_status)" - " VALUES (%s, %s, %s, %s, %s)" - " ON CONFLICT (account_id, consumer_id, chain_key) DO NOTHING", - ( - who.account_id, - who.consumer_id, - chain, - record.reporting_receipt_id, - record.status, - ), - ) - else: - cursor = await connection.execute( - "UPDATE reporting_receipt_heads SET receipt_id = %s, receipt_status = %s," - " supersedes_receipt_id = %s" - " WHERE account_id = %s AND consumer_id = %s AND chain_key = %s" - " AND receipt_id = %s AND receipt_status = 'rejected'", - ( - record.reporting_receipt_id, - record.status, - record.supersedes_reporting_receipt_id, - who.account_id, - who.consumer_id, - chain, - record.supersedes_reporting_receipt_id, - ), - ) - if cursor.rowcount != 1: - unavailable() - async def read_reconciliation_snapshot( - self, *, caller: ReportingDeliveryPrincipal, boundary: LedgerSnapshot | None = None + self, + *, + caller: ReportingDeliveryPrincipal, + boundary: ReportingReconciliationSnapshotToken | None = None, ) -> ReportingReconciliationSnapshot: - if boundary is None: - boundary = await self.open_snapshot( - account_id=caller.account_id, filters_fingerprint=caller.consumer_id + async with self._pool.connection() as connection, connection.transaction(): + await self._lock_account(connection, caller.account_id) + maximum, now = await self._validate_feed(connection, caller) + if boundary is None: + boundary = change_boundary( + caller, maximum, self._clock() if self._clock is not None else now + ) + if ( + type(boundary) is not ReportingReconciliationSnapshotToken + or boundary.caller != caller + or boundary.min_sequence != 0 + or boundary.filters != ReportingReconciliationFilter() + ): + unavailable() + validate_boundary(caller, boundary, maximum) + records = tuple( + item.record + for item in await self._changes(connection, caller, boundary.max_sequence) ) - if boundary.account_id != caller.account_id: - unavailable() - async with self._pool.connection() as connection: - records = await self._records(connection, caller, boundary.max_sequence) + if len(records) != boundary.total_count: + fail("REPORTING_HISTORY_CORRUPT") return ReportingReconciliationSnapshot(caller, boundary, records) + + async def read_reconciliation_changes( + self, + *, + caller: ReportingDeliveryPrincipal, + changes_after: ReportingReconciliationCheckpoint | None = None, + cursor: ReportingReconciliationCursor | None = None, + limit: int = 100, + filters: ReportingReconciliationFilter = ReportingReconciliationFilter(), + ) -> ReportingReconciliationPage: + after, boundary, last_key = read_position(caller, changes_after, cursor, limit, filters) + async with self._pool.connection() as connection, connection.transaction(): + await self._lock_account(connection, caller.account_id) + maximum, now = await self._validate_feed(connection, caller) + if boundary is None: + count = await self._change_count(connection, caller, after, maximum, filters) + boundary = change_boundary( + caller, + maximum, + self._clock() if self._clock is not None else now, + after=after, + total_count=count, + filters=filters, + ) + validate_boundary(caller, boundary, maximum) + if last_key is not None: + last = await self._changes( + connection, caller, after, after=after - 1, limit=1, filters=filters + ) + if not last or change_id(last[0].record) != last_key: + raise LedgerConflictError( + "INVALID_CHECKPOINT", "reconciliation key is unavailable" + ) + count = await self._change_count( + connection, caller, boundary.min_sequence, boundary.max_sequence, filters + ) + if count != boundary.total_count: + fail("REPORTING_HISTORY_CORRUPT") + changes = await self._changes( + connection, + caller, + boundary.max_sequence, + after=after, + limit=limit + 1, + filters=filters, + ) + return change_page(caller, boundary, after, changes, limit) diff --git a/src/adcp/reporting/ledger/models.py b/src/adcp/reporting/ledger/models.py index 6a8cded52..f48cdf4e0 100644 --- a/src/adcp/reporting/ledger/models.py +++ b/src/adcp/reporting/ledger/models.py @@ -63,19 +63,7 @@ ReportingFinality = Literal["snapshot", "official"] ReportingHealth = Literal["healthy", "waiting", "delayed", "action_required", "complete"] ReportingProductionStatus = Literal["not_due", "pending", "published", "failed"] -LedgerRecordKind = Literal[ - "obligation", - "revision", - "adjustment", - "consumer_status", - "destination_binding", - "obligation_delivery", - "materialization_attempt", - "materialization", - "materialization_check", - "revision_receipt", - "adjustment_receipt", -] +LedgerRecordKind = Literal["obligation", "revision", "adjustment", "consumer_status"] #: The five values a consumer may state about one expected period. AdCP #: 3.2.0-rc.3 adds ``content_mismatch``: reporting that arrived and parsed but diff --git a/src/adcp/reporting/ledger/pg.py b/src/adcp/reporting/ledger/pg.py index dc3eaa20d..f5aa7b4f1 100644 --- a/src/adcp/reporting/ledger/pg.py +++ b/src/adcp/reporting/ledger/pg.py @@ -1061,7 +1061,8 @@ async def open_snapshot(self, *, account_id: str, filters_fingerprint: str) -> L row = await ( await connection.execute( "SELECT COALESCE(MAX(seq), 0), now() FROM reporting_ledger_changes" - " WHERE account_id = %s", + " WHERE account_id = %s AND record_kind IN" + " ('obligation', 'revision', 'adjustment', 'consumer_status')", (account_id,), ) ).fetchone() diff --git a/src/adcp/reporting/ledger/reporting_ledger_reconciliation.sql b/src/adcp/reporting/ledger/reporting_ledger_reconciliation.sql index 1c2f930af..6ab3aaabf 100644 --- a/src/adcp/reporting/ledger/reporting_ledger_reconciliation.sql +++ b/src/adcp/reporting/ledger/reporting_ledger_reconciliation.sql @@ -8,6 +8,11 @@ DECLARE evidence_nullable BOOLEAN; evidence_table TEXT; evidence_column TEXT; + graph_table TEXT; + graph_constraint TEXT; + graph_definition TEXT; + retained_record RECORD; + feed_installed BOOLEAN; BEGIN PERFORM pg_advisory_xact_lock(hashtext('adcp.reporting.schema'), hashtext(current_schema())); -- Resolve prerequisite columns/keys before installing any extension tables. @@ -165,5 +170,474 @@ BEGIN BEFORE UPDATE OR DELETE ON reporting_receipt_heads FOR EACH ROW EXECUTE FUNCTION reporting_receipt_terminal(); END IF; + + -- Exact Core identities, including the owner of each referenced publication. + CREATE UNIQUE INDEX IF NOT EXISTS reporting_obligations_generation_identity + ON reporting_obligations(account_id, delivery_config_id, delivery_config_version, + reporting_obligation_id); + CREATE UNIQUE INDEX IF NOT EXISTS reporting_revisions_obligation_identity + ON reporting_revisions(account_id, reporting_obligation_id, reporting_revision_id); + CREATE UNIQUE INDEX IF NOT EXISTS reporting_adjustments_revision_identity + ON reporting_adjustments(account_id, adjusts_reporting_revision_id, reporting_adjustment_id); + CREATE UNIQUE INDEX IF NOT EXISTS reporting_receipts_exact_identity + ON reporting_reconciliation_records(account_id, consumer_id, namespace, record_id, + receipt_chain_key, receipt_status); + CREATE UNIQUE INDEX IF NOT EXISTS reporting_receipt_one_root + ON reporting_reconciliation_records(account_id, consumer_id, receipt_chain_key) + WHERE receipt_status IS NOT NULL AND supersedes_receipt_id IS NULL; + CREATE INDEX IF NOT EXISTS reporting_reconciliation_graph + ON reporting_reconciliation_records(account_id, consumer_id, record_kind, + delivery_config_id, delivery_config_version, + reporting_obligation_id, reporting_materialization_id); + + -- ADD CONSTRAINT validates existing history as well as future SQL writes. + -- Keep existing constraint OIDs on repeated/concurrent upgrades. + FOR graph_table, graph_constraint, graph_definition IN VALUES + ('reporting_revisions', 'reporting_revision_exact_obligation', + 'FOREIGN KEY (account_id, reporting_obligation_id) REFERENCES reporting_obligations(account_id, reporting_obligation_id)'), + ('reporting_revisions', 'reporting_revision_exact_predecessor', + 'FOREIGN KEY (account_id, reporting_obligation_id, supersedes_reporting_revision_id) REFERENCES reporting_revisions(account_id, reporting_obligation_id, reporting_revision_id)'), + ('reporting_adjustments', 'reporting_adjustment_exact_revision', + 'FOREIGN KEY (account_id, adjusts_reporting_revision_id) REFERENCES reporting_revisions(account_id, reporting_revision_id)'), + ('reporting_reconciliation_records', 'reporting_record_exact_obligation', + 'FOREIGN KEY (account_id, delivery_config_id, delivery_config_version, reporting_obligation_id) REFERENCES reporting_obligations(account_id, delivery_config_id, delivery_config_version, reporting_obligation_id)'), + ('reporting_reconciliation_records', 'reporting_record_exact_revision', + 'FOREIGN KEY (account_id, reporting_obligation_id, reporting_revision_id) REFERENCES reporting_revisions(account_id, reporting_obligation_id, reporting_revision_id)'), + ('reporting_reconciliation_records', 'reporting_record_exact_adjustment', + 'FOREIGN KEY (account_id, reporting_revision_id, reporting_adjustment_id) REFERENCES reporting_adjustments(account_id, adjusts_reporting_revision_id, reporting_adjustment_id)'), + ('reporting_reconciliation_records', 'reporting_record_identity_shape', + $shape$CHECK ( + (record_kind = 'destination_binding') = (reporting_obligation_id IS NULL) + AND (record_kind IN ('materialization_attempt', 'materialization', 'revision_receipt', 'adjustment_receipt')) = (reporting_revision_id IS NOT NULL) + AND (record_kind IN ('materialization_attempt', 'materialization', 'materialization_check', 'revision_receipt')) = (reporting_materialization_id IS NOT NULL) + AND (record_kind = 'adjustment_receipt') = (reporting_adjustment_id IS NOT NULL) + AND (supersedes_receipt_id IS NULL OR receipt_status IS NOT NULL) + AND (supersedes_receipt_id IS NULL OR supersedes_receipt_id <> record_id) + )$shape$), + ('reporting_receipt_heads', 'reporting_head_exact_receipt', + 'FOREIGN KEY (account_id, consumer_id, namespace, receipt_id, chain_key, receipt_status) REFERENCES reporting_reconciliation_records(account_id, consumer_id, namespace, record_id, receipt_chain_key, receipt_status)') + LOOP + IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conrelid = graph_table::regclass + AND conname = graph_constraint) THEN + EXECUTE format('ALTER TABLE %I ADD CONSTRAINT %I %s', + graph_table, graph_constraint, graph_definition); + END IF; + END LOOP; + + CREATE OR REPLACE FUNCTION reporting_identity_sha256(value TEXT) + RETURNS TEXT LANGUAGE SQL IMMUTABLE STRICT AS $function$ + SELECT encode(sha256(convert_to(value, 'UTF8')), 'hex') + $function$; + + CREATE OR REPLACE FUNCTION reporting_reconciliation_validate(r reporting_reconciliation_records) + RETURNS VOID LANGUAGE plpgsql AS $function$ + DECLARE + generation JSONB; + identity TEXT; + chain TEXT; + expected_namespace TEXT; + parent_kind TEXT; + BEGIN + generation := jsonb_build_object('account_id', r.account_id, + 'delivery_config_id', r.delivery_config_id, + 'delivery_config_version', r.delivery_config_version); + IF NOT EXISTS (SELECT 1 FROM reporting_configurations c + WHERE (c.account_id, c.delivery_config_id, c.delivery_config_version) + = (r.account_id, r.delivery_config_id, r.delivery_config_version)) + OR (r.reporting_obligation_id IS NOT NULL AND NOT EXISTS ( + SELECT 1 FROM reporting_obligations o + WHERE (o.account_id, o.delivery_config_id, o.delivery_config_version, o.reporting_obligation_id) + = (r.account_id, r.delivery_config_id, r.delivery_config_version, r.reporting_obligation_id))) + OR (r.reporting_revision_id IS NOT NULL AND NOT EXISTS ( + SELECT 1 FROM reporting_revisions v + WHERE (v.account_id, v.reporting_obligation_id, v.reporting_revision_id) + = (r.account_id, r.reporting_obligation_id, r.reporting_revision_id))) + OR (r.reporting_adjustment_id IS NOT NULL AND NOT EXISTS ( + SELECT 1 FROM reporting_adjustments a + WHERE (a.account_id, a.adjusts_reporting_revision_id, a.reporting_adjustment_id) + = (r.account_id, r.reporting_revision_id, r.reporting_adjustment_id))) THEN + RAISE EXCEPTION 'reporting exact publication graph is unavailable' USING ERRCODE = '23514'; + END IF; + expected_namespace := CASE WHEN r.receipt_status IS NOT NULL THEN 'receipt' ELSE r.record_kind END; + identity := CASE r.record_kind + WHEN 'destination_binding' THEN reporting_identity_sha256( + '{"account_id":' || to_json(r.account_id)::text || ',"delivery_config_id":' || + to_json(r.delivery_config_id)::text || ',"delivery_config_version":' || + r.delivery_config_version::text || '}') + WHEN 'obligation_delivery' THEN r.reporting_obligation_id + WHEN 'materialization_check' THEN r.payload->>'check_id' + WHEN 'revision_receipt' THEN r.payload->>'reporting_receipt_id' + WHEN 'adjustment_receipt' THEN r.payload->>'reporting_receipt_id' + ELSE r.reporting_materialization_id END; + chain := CASE r.record_kind + WHEN 'revision_receipt' THEN reporting_identity_sha256( + '["revision",' || to_json(r.reporting_obligation_id)::text || ',' || + to_json(r.reporting_revision_id)::text || ']') + WHEN 'adjustment_receipt' THEN reporting_identity_sha256( + '["adjustment",' || to_json(r.reporting_adjustment_id)::text || ']') END; + IF r.payload->>'kind' IS DISTINCT FROM r.record_kind + OR r.namespace IS DISTINCT FROM expected_namespace + OR r.record_id IS DISTINCT FROM identity + OR r.receipt_chain_key IS DISTINCT FROM chain + OR r.change_id IS DISTINCT FROM reporting_identity_sha256( + '[' || to_json(r.account_id)::text || ',' || to_json(r.consumer_id)::text || ',' || + to_json(expected_namespace)::text || ',' || to_json(identity)::text || ']') + OR r.reporting_revision_id IS DISTINCT FROM (CASE WHEN r.record_kind = 'adjustment_receipt' + THEN r.payload->>'adjusts_reporting_revision_id' ELSE r.payload->>'reporting_revision_id' END) + OR r.reporting_materialization_id IS DISTINCT FROM r.payload->>'reporting_materialization_id' + OR r.reporting_adjustment_id IS DISTINCT FROM r.payload->>'reporting_adjustment_id' + OR coalesce(to_jsonb(r.attempt_number), 'null'::jsonb) IS DISTINCT FROM coalesce(r.payload->'attempt', 'null'::jsonb) + OR r.receipt_status IS DISTINCT FROM (CASE WHEN r.record_kind IN ('revision_receipt', 'adjustment_receipt') + THEN r.payload->>'status' END) + OR r.supersedes_receipt_id IS DISTINCT FROM r.payload->>'supersedes_reporting_receipt_id' THEN + RAISE EXCEPTION 'reporting payload identity is inconsistent' USING ERRCODE = '23514'; + END IF; + IF r.record_kind = 'destination_binding' THEN + IF r.payload->'generation_key' IS DISTINCT FROM generation + OR r.payload->>'consumer_id' IS DISTINCT FROM r.consumer_id + OR (r.payload->>'verification_profile' = 'manifest_checksums' + AND r.payload->>'method' <> 'file_transfer') THEN + RAISE EXCEPTION 'reporting destination binding is inconsistent' USING ERRCODE = '23514'; + END IF; + RETURN; + END IF; + IF r.payload->'scope' IS DISTINCT FROM jsonb_build_object('generation_key', generation, + 'consumer_id', r.consumer_id, 'reporting_obligation_id', r.reporting_obligation_id) + OR NOT EXISTS ( + SELECT 1 FROM reporting_reconciliation_records b + WHERE (b.account_id, b.consumer_id, b.delivery_config_id, b.delivery_config_version) + = (r.account_id, r.consumer_id, r.delivery_config_id, r.delivery_config_version) + AND b.record_kind = 'destination_binding') THEN + RAISE EXCEPTION 'reporting frozen binding is unavailable' USING ERRCODE = '23514'; + END IF; + IF r.record_kind = 'obligation_delivery' THEN RETURN; END IF; + IF NOT EXISTS ( + SELECT 1 FROM reporting_reconciliation_records d + WHERE (d.account_id, d.consumer_id, d.delivery_config_id, d.delivery_config_version, d.reporting_obligation_id) + = (r.account_id, r.consumer_id, r.delivery_config_id, r.delivery_config_version, r.reporting_obligation_id) + AND d.record_kind = 'obligation_delivery') THEN + RAISE EXCEPTION 'reporting obligation delivery is unavailable' USING ERRCODE = '23514'; + END IF; + parent_kind := CASE r.record_kind WHEN 'materialization' THEN 'materialization_attempt' + WHEN 'materialization_check' THEN 'materialization' WHEN 'revision_receipt' THEN 'materialization' END; + IF parent_kind IS NOT NULL AND NOT EXISTS ( + SELECT 1 FROM reporting_reconciliation_records p + WHERE (p.account_id, p.consumer_id, p.delivery_config_id, p.delivery_config_version, + p.reporting_obligation_id, p.reporting_materialization_id) + = (r.account_id, r.consumer_id, r.delivery_config_id, r.delivery_config_version, + r.reporting_obligation_id, r.reporting_materialization_id) + AND p.record_kind = parent_kind + AND (r.record_kind = 'materialization_check' OR p.reporting_revision_id = r.reporting_revision_id) + AND (parent_kind = 'materialization_attempt' OR p.payload->>'status' IN ('available', 'delivered'))) THEN + RAISE EXCEPTION 'reporting exact materialization graph is unavailable' USING ERRCODE = '23514'; + END IF; + IF r.record_kind = 'materialization' AND r.payload->>'status' IN ('available', 'delivered') THEN + IF EXISTS (SELECT 1 FROM reporting_reconciliation_records b + WHERE (b.account_id, b.consumer_id, b.delivery_config_id, b.delivery_config_version) + = (r.account_id, r.consumer_id, r.delivery_config_id, r.delivery_config_version) + AND b.record_kind = 'destination_binding' + AND (r.payload->>'status' IS DISTINCT FROM b.payload->>'success_status' + OR r.payload#>>'{verification,verification_profile}' IS DISTINCT FROM b.payload->>'verification_profile')) THEN + RAISE EXCEPTION 'reporting materialization binding is inconsistent' USING ERRCODE = '23514'; + END IF; + IF r.payload#>>'{verification,verification_profile}' = 'native_commit' AND ( + r.payload#>>'{resource,immutability}' IS DISTINCT FROM 'native_version' + OR r.payload#>>'{resource,native_version_ref}' IS NULL + OR r.payload#>>'{verification,native_version_ref}' IS DISTINCT FROM r.payload#>>'{resource,native_version_ref}' + OR r.payload#>>'{verification,native_observed_through}' IS DISTINCT FROM r.payload#>>'{verification,verification_path}' + OR coalesce(r.payload#>>'{verification,verification_path}', '') NOT IN ('representative_consumer', 'destination')) THEN + RAISE EXCEPTION 'reporting native commit is inconsistent' USING ERRCODE = '23514'; + END IF; + END IF; + IF r.record_kind = 'adjustment_receipt' AND NOT EXISTS ( + SELECT 1 FROM reporting_revisions v WHERE v.account_id = r.account_id + AND v.reporting_obligation_id = r.reporting_obligation_id + AND v.reporting_revision_id = r.reporting_revision_id + AND v.finality = 'official' AND v.finalized_at IS NOT NULL) THEN + RAISE EXCEPTION 'reporting adjustment requires its official revision' USING ERRCODE = '23514'; + END IF; + IF r.supersedes_receipt_id IS NOT NULL AND NOT EXISTS ( + SELECT 1 FROM reporting_reconciliation_records p + WHERE (p.account_id, p.consumer_id, p.record_kind, p.delivery_config_id, + p.delivery_config_version, p.reporting_obligation_id, p.reporting_revision_id, p.receipt_chain_key) + = (r.account_id, r.consumer_id, r.record_kind, r.delivery_config_id, + r.delivery_config_version, r.reporting_obligation_id, r.reporting_revision_id, r.receipt_chain_key) + AND p.record_id = r.supersedes_receipt_id AND p.receipt_status = 'rejected') THEN + RAISE EXCEPTION 'reporting receipt predecessor is unavailable' USING ERRCODE = '23514'; + END IF; + END + $function$; + + CREATE OR REPLACE FUNCTION reporting_reconciliation_guard() + RETURNS TRIGGER LANGUAGE plpgsql AS $function$ + BEGIN + PERFORM pg_advisory_xact_lock(hashtext('adcp.reporting:' || NEW.account_id)); + PERFORM reporting_reconciliation_validate(NEW); + IF NEW.record_kind = 'materialization_attempt' AND NEW.attempt_number <> ( + SELECT count(*) + 1 FROM reporting_reconciliation_records p + WHERE (p.account_id, p.consumer_id, p.reporting_obligation_id, p.reporting_revision_id) + = (NEW.account_id, NEW.consumer_id, NEW.reporting_obligation_id, NEW.reporting_revision_id) + AND p.record_kind = 'materialization_attempt') THEN + RAISE EXCEPTION 'reporting attempt ordinal is invalid' USING ERRCODE = '23514'; + END IF; + RETURN NEW; + END + $function$; + + CREATE OR REPLACE FUNCTION reporting_receipt_head_exact() + RETURNS TRIGGER LANGUAGE plpgsql AS $function$ + BEGIN + IF NOT EXISTS ( + SELECT 1 FROM reporting_reconciliation_records r + WHERE (r.account_id, r.consumer_id, r.namespace, r.record_id, r.receipt_chain_key, r.receipt_status) + = (NEW.account_id, NEW.consumer_id, NEW.namespace, NEW.receipt_id, NEW.chain_key, NEW.receipt_status) + AND r.supersedes_receipt_id IS NOT DISTINCT FROM NEW.supersedes_receipt_id + AND NOT EXISTS (SELECT 1 FROM reporting_reconciliation_records s + WHERE s.account_id = r.account_id AND s.consumer_id = r.consumer_id + AND s.supersedes_receipt_id = r.record_id)) THEN + RAISE EXCEPTION 'reporting receipt head is inconsistent' USING ERRCODE = '23514'; + END IF; + RETURN NEW; + END + $function$; + + CREATE OR REPLACE FUNCTION reporting_receipt_advance() + RETURNS TRIGGER LANGUAGE plpgsql AS $function$ + BEGIN + IF NEW.receipt_status IS NULL THEN RETURN NEW; END IF; + IF NEW.supersedes_receipt_id IS NULL THEN + INSERT INTO reporting_receipt_heads(account_id, consumer_id, chain_key, receipt_id, receipt_status) + VALUES (NEW.account_id, NEW.consumer_id, NEW.receipt_chain_key, NEW.record_id, NEW.receipt_status); + ELSE + UPDATE reporting_receipt_heads SET receipt_id = NEW.record_id, receipt_status = NEW.receipt_status, + supersedes_receipt_id = NEW.supersedes_receipt_id + WHERE account_id = NEW.account_id AND consumer_id = NEW.consumer_id + AND chain_key = NEW.receipt_chain_key AND receipt_id = NEW.supersedes_receipt_id + AND receipt_status = 'rejected'; + IF NOT FOUND THEN + RAISE EXCEPTION 'reporting receipt replacement is unavailable' USING ERRCODE = '23514'; + END IF; + END IF; + RETURN NEW; + END + $function$; + + CREATE OR REPLACE FUNCTION reporting_reconciliation_reference_immutable() + RETURNS TRIGGER LANGUAGE plpgsql AS $function$ + DECLARE + previous JSONB := to_jsonb(OLD); + proposed JSONB := to_jsonb(NEW); + referenced BOOLEAN; + BEGIN + IF proposed = previous + OR (TG_TABLE_NAME = 'reporting_configurations' AND + proposed - 'lease_worker_id' - 'lease_expires_at' = previous - 'lease_worker_id' - 'lease_expires_at') + OR (TG_TABLE_NAME = 'reporting_revisions' AND proposed - 'readable' = previous - 'readable') THEN + RETURN NEW; + END IF; + PERFORM pg_advisory_xact_lock(hashtext('adcp.reporting:' || OLD.account_id)); + IF TG_TABLE_NAME = 'reporting_configurations' THEN + previous := previous - 'lease_worker_id' - 'lease_expires_at'; + proposed := proposed - 'lease_worker_id' - 'lease_expires_at'; + SELECT EXISTS (SELECT 1 FROM reporting_reconciliation_records r + WHERE r.account_id = OLD.account_id AND r.delivery_config_id = OLD.delivery_config_id + AND r.delivery_config_version = OLD.delivery_config_version) INTO referenced; + ELSIF TG_TABLE_NAME = 'reporting_obligations' THEN + SELECT EXISTS (SELECT 1 FROM reporting_reconciliation_records r + WHERE r.account_id = OLD.account_id AND r.reporting_obligation_id = OLD.reporting_obligation_id) INTO referenced; + ELSIF TG_TABLE_NAME = 'reporting_revisions' THEN + previous := previous - 'readable'; + proposed := proposed - 'readable'; + SELECT EXISTS (SELECT 1 FROM reporting_reconciliation_records r + WHERE r.account_id = OLD.account_id AND r.reporting_revision_id = OLD.reporting_revision_id) INTO referenced; + ELSE + SELECT EXISTS (SELECT 1 FROM reporting_reconciliation_records r + WHERE r.account_id = OLD.account_id AND r.reporting_adjustment_id = OLD.reporting_adjustment_id) INTO referenced; + END IF; + IF referenced AND proposed IS DISTINCT FROM previous THEN + RAISE EXCEPTION 'reporting referenced publication is immutable' USING ERRCODE = '23514'; + END IF; + RETURN NEW; + END + $function$; + FOREACH graph_table IN ARRAY ARRAY['reporting_configurations', 'reporting_obligations', + 'reporting_revisions', 'reporting_adjustments'] LOOP + IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid = graph_table::regclass + AND tgname = 'reporting_reconciliation_reference_immutable' AND NOT tgisinternal) THEN + EXECUTE format('CREATE TRIGGER reporting_reconciliation_reference_immutable BEFORE UPDATE ON %I ' + 'FOR EACH ROW EXECUTE FUNCTION reporting_reconciliation_reference_immutable()', graph_table); + END IF; + END LOOP; + + IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid = 'reporting_reconciliation_records'::regclass + AND tgname = 'reporting_reconciliation_guard' AND NOT tgisinternal) THEN + CREATE TRIGGER reporting_reconciliation_guard BEFORE INSERT ON reporting_reconciliation_records + FOR EACH ROW EXECUTE FUNCTION reporting_reconciliation_guard(); + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid = 'reporting_reconciliation_records'::regclass + AND tgname = 'reporting_receipt_advance' AND NOT tgisinternal) THEN + CREATE TRIGGER reporting_receipt_advance AFTER INSERT ON reporting_reconciliation_records + FOR EACH ROW EXECUTE FUNCTION reporting_receipt_advance(); + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid = 'reporting_receipt_heads'::regclass + AND tgname = 'reporting_receipt_head_exact' AND NOT tgisinternal) THEN + CREATE TRIGGER reporting_receipt_head_exact BEFORE INSERT OR UPDATE ON reporting_receipt_heads + FOR EACH ROW EXECUTE FUNCTION reporting_receipt_head_exact(); + END IF; + + -- Revalidate old extension rows too; installing a trigger alone would bless + -- an already-corrupt graph. A failed upgrade leaves all prior data intact. + FOR retained_record IN SELECT r FROM reporting_reconciliation_records r LOOP + PERFORM reporting_reconciliation_validate(retained_record.r); + END LOOP; + IF EXISTS ( + SELECT 1 FROM reporting_reconciliation_records r + LEFT JOIN reporting_receipt_heads h ON (h.account_id, h.consumer_id, h.chain_key, h.receipt_id, h.receipt_status) + = (r.account_id, r.consumer_id, r.receipt_chain_key, r.record_id, r.receipt_status) + WHERE r.receipt_status IS NOT NULL + AND NOT EXISTS (SELECT 1 FROM reporting_reconciliation_records s + WHERE s.account_id = r.account_id AND s.consumer_id = r.consumer_id AND s.supersedes_receipt_id = r.record_id) + AND (h.receipt_id IS NULL OR h.supersedes_receipt_id IS DISTINCT FROM r.supersedes_receipt_id) + ) OR EXISTS ( + SELECT 1 FROM (SELECT attempt_number, row_number() OVER ( + PARTITION BY account_id, consumer_id, reporting_obligation_id, reporting_revision_id + ORDER BY attempt_number) AS ordinal FROM reporting_reconciliation_records + WHERE record_kind = 'materialization_attempt') attempts WHERE attempt_number <> ordinal + ) THEN + RAISE EXCEPTION 'reporting retained graph is inconsistent' USING ERRCODE = '23514'; + END IF; + + -- The optional feed has a dense head for each authenticated principal. It + -- never allocates Core ledger sequences or widens Core's record vocabulary. + feed_installed := to_regclass('reporting_reconciliation_changes') IS NOT NULL; + CREATE UNIQUE INDEX IF NOT EXISTS reporting_reconciliation_feed_identity + ON reporting_reconciliation_records(account_id, consumer_id, namespace, record_id, + record_kind, change_id, content_sha256); + CREATE TABLE IF NOT EXISTS reporting_reconciliation_changes ( + account_id TEXT COLLATE "C" NOT NULL, + consumer_id TEXT COLLATE "C" NOT NULL, + seq BIGINT NOT NULL CHECK (seq > 0), + namespace TEXT COLLATE "C" NOT NULL, + record_id TEXT COLLATE "C" NOT NULL, + record_kind TEXT NOT NULL, + change_id TEXT COLLATE "C" NOT NULL, + content_sha256 TEXT COLLATE "C" NOT NULL, + committed_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY (account_id, consumer_id, seq), + UNIQUE (account_id, consumer_id, namespace, record_id), + UNIQUE (account_id, consumer_id, namespace, record_id, record_kind, change_id, content_sha256), + FOREIGN KEY (account_id, consumer_id, namespace, record_id, record_kind, change_id, content_sha256) + REFERENCES reporting_reconciliation_records(account_id, consumer_id, namespace, record_id, + record_kind, change_id, content_sha256) + ); + CREATE TABLE IF NOT EXISTS reporting_reconciliation_heads ( + account_id TEXT COLLATE "C" NOT NULL, + consumer_id TEXT COLLATE "C" NOT NULL, + max_sequence BIGINT NOT NULL CHECK (max_sequence > 0), + PRIMARY KEY (account_id, consumer_id), + FOREIGN KEY (account_id, consumer_id, max_sequence) + REFERENCES reporting_reconciliation_changes(account_id, consumer_id, seq) + DEFERRABLE INITIALLY DEFERRED + ); + + IF NOT feed_installed THEN + -- Project the initial storage slice's complete change history into the + -- independent feed. Preserve every original record, head, legacy change, + -- hash and timestamp. Missing evidence is never repaired or inferred. + IF EXISTS ( + SELECT 1 FROM reporting_reconciliation_records r + LEFT JOIN reporting_ledger_changes c ON c.account_id = r.account_id + AND c.record_kind = r.record_kind AND c.record_id = r.change_id + WHERE c.seq IS NULL + ) THEN + RAISE EXCEPTION 'reporting legacy feed is incomplete' USING ERRCODE = '23514'; + END IF; + INSERT INTO reporting_reconciliation_changes + (account_id, consumer_id, seq, namespace, record_id, record_kind, + change_id, content_sha256, committed_at) + SELECT r.account_id, r.consumer_id, + row_number() OVER (PARTITION BY r.account_id, r.consumer_id ORDER BY c.seq), + r.namespace, r.record_id, r.record_kind, r.change_id, r.content_sha256, c.committed_at + FROM reporting_reconciliation_records r + JOIN reporting_ledger_changes c ON c.account_id = r.account_id + AND c.record_kind = r.record_kind AND c.record_id = r.change_id; + INSERT INTO reporting_reconciliation_heads (account_id, consumer_id, max_sequence) + SELECT account_id, consumer_id, max(seq) FROM reporting_reconciliation_changes + GROUP BY account_id, consumer_id; + END IF; + + -- Earlier audit builds tied records to Core's feed. Replace only that + -- obsolete FK; the legacy change rows themselves remain byte-for-byte. + ALTER TABLE reporting_reconciliation_records + DROP CONSTRAINT IF EXISTS reporting_record_transactional_change; + IF NOT EXISTS (SELECT 1 FROM pg_constraint + WHERE conrelid = 'reporting_reconciliation_records'::regclass + AND conname = 'reporting_record_transactional_feed') THEN + ALTER TABLE reporting_reconciliation_records + ADD CONSTRAINT reporting_record_transactional_feed + FOREIGN KEY (account_id, consumer_id, namespace, record_id, record_kind, change_id, content_sha256) + REFERENCES reporting_reconciliation_changes(account_id, consumer_id, namespace, record_id, + record_kind, change_id, content_sha256) + DEFERRABLE INITIALLY DEFERRED; + END IF; + + CREATE OR REPLACE FUNCTION reporting_reconciliation_head_guard() + RETURNS TRIGGER LANGUAGE plpgsql AS $function$ + BEGIN + IF TG_OP = 'DELETE' THEN + RAISE EXCEPTION 'reporting feed head is append-only' USING ERRCODE = '23514'; + END IF; + PERFORM pg_advisory_xact_lock(hashtext('adcp.reporting:' || NEW.account_id)); + IF (TG_OP = 'INSERT' AND NEW.max_sequence <> 1) + OR (TG_OP = 'UPDATE' AND ( + NEW.account_id <> OLD.account_id OR NEW.consumer_id <> OLD.consumer_id + OR NEW.max_sequence <> OLD.max_sequence + 1)) THEN + RAISE EXCEPTION 'reporting feed head transition is invalid' USING ERRCODE = '23514'; + END IF; + RETURN NEW; + END + $function$; + CREATE OR REPLACE FUNCTION reporting_reconciliation_change_guard() + RETURNS TRIGGER LANGUAGE plpgsql AS $function$ + BEGIN + PERFORM pg_advisory_xact_lock(hashtext('adcp.reporting:' || NEW.account_id)); + IF NOT EXISTS (SELECT 1 FROM reporting_reconciliation_heads h + WHERE h.account_id = NEW.account_id AND h.consumer_id = NEW.consumer_id + AND h.max_sequence = NEW.seq) + OR (NEW.seq > 1 AND NOT EXISTS ( + SELECT 1 FROM reporting_reconciliation_changes c + WHERE c.account_id = NEW.account_id AND c.consumer_id = NEW.consumer_id + AND c.seq = NEW.seq - 1)) THEN + RAISE EXCEPTION 'reporting feed sequence is inconsistent' USING ERRCODE = '23514'; + END IF; + RETURN NEW; + END + $function$; + IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid = 'reporting_reconciliation_heads'::regclass + AND tgname = 'reporting_reconciliation_head_guard' AND NOT tgisinternal) THEN + CREATE TRIGGER reporting_reconciliation_head_guard + BEFORE INSERT OR UPDATE OR DELETE ON reporting_reconciliation_heads + FOR EACH ROW EXECUTE FUNCTION reporting_reconciliation_head_guard(); + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid = 'reporting_reconciliation_changes'::regclass + AND tgname = 'reporting_reconciliation_change_guard' AND NOT tgisinternal) THEN + CREATE TRIGGER reporting_reconciliation_change_guard BEFORE INSERT ON reporting_reconciliation_changes + FOR EACH ROW EXECUTE FUNCTION reporting_reconciliation_change_guard(); + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid = 'reporting_reconciliation_changes'::regclass + AND tgname = 'reporting_reconciliation_change_immutable' AND NOT tgisinternal) THEN + CREATE TRIGGER reporting_reconciliation_change_immutable + BEFORE UPDATE OR DELETE ON reporting_reconciliation_changes + FOR EACH ROW EXECUTE FUNCTION reporting_reconciliation_immutable(); + END IF; + IF EXISTS ( + SELECT 1 FROM (SELECT account_id, consumer_id, count(*) AS count, max(seq) AS maximum + FROM reporting_reconciliation_changes GROUP BY account_id, consumer_id) c + FULL JOIN reporting_reconciliation_heads h USING (account_id, consumer_id) + WHERE c.count IS NULL OR h.max_sequence IS NULL OR c.count <> c.maximum + OR h.max_sequence <> c.maximum + ) THEN + RAISE EXCEPTION 'reporting retained feed is inconsistent' USING ERRCODE = '23514'; + END IF; END $reconciliation$; diff --git a/src/adcp/reporting/ledger/store.py b/src/adcp/reporting/ledger/store.py index 0ded97290..cbde8e2a6 100644 --- a/src/adcp/reporting/ledger/store.py +++ b/src/adcp/reporting/ledger/store.py @@ -1136,7 +1136,7 @@ async def read_page( ), ) - def _resolve(self, kind: LedgerRecordKind, record_id: str) -> Any: + def _resolve(self, kind: str, record_id: str) -> Any: if kind == "obligation": return self._obligations.get(record_id) if kind == "revision": diff --git a/tests/conformance/reporting/_reconciliation_support.py b/tests/conformance/reporting/_reconciliation_support.py index b6ebf5698..21c3c260e 100644 --- a/tests/conformance/reporting/_reconciliation_support.py +++ b/tests/conformance/reporting/_reconciliation_support.py @@ -88,11 +88,17 @@ async def scenario( finality: ReportingFinality = "official", reconciliation_mode: Literal["delivery_only", "consumer_receipt"] = "consumer_receipt", control_total_evidence: tuple[ReportingControlTotalRecord, ...] | None = None, + reader_compatibility: tuple[str, ...] | None = None, + revision_id: str | None = None, + obligation_id: str | None = None, + destination_ref: str = "destination-generation-1", ) -> Scenario: feed = "billing" if billing else "analytics" config = replace(configuration(account_id), feed_purpose=feed, required_finality=finality) await store.put_configuration(config) obligation = replace(obligation_for(config), currency="EUR") + if obligation_id is not None: + obligation = replace(obligation, reporting_obligation_id=obligation_id) await store.commit_obligation(obligation) scope = ReportingDeliveryScope( config.generation_key, consumer_id, obligation.reporting_obligation_id @@ -100,7 +106,7 @@ async def scenario( binding = ReportingDestinationBinding( generation_key=config.generation_key, consumer_id=consumer_id, - destination_ref="destination-generation-1", + destination_ref=destination_ref, trusted_binding_ref="trusted-binding-1", method=method, transport="test-storage", @@ -110,7 +116,11 @@ async def scenario( resource_retention_days=400, created_at=START, format="jsonl" if method == "file_transfer" else None, - reader_compatibility=("jsonl-v1",) if method == "file_transfer" else ("table-v1",), + reader_compatibility=( + reader_compatibility + if reader_compatibility is not None + else (("jsonl-v1",) if method == "file_transfer" else ("table-v1",)) + ), success_status="delivered" if method == "warehouse_materialization" else "available", ) await store.put_destination_binding(binding) @@ -131,7 +141,7 @@ async def scenario( ) if control_total_evidence is not None: total_records = control_total_evidence - revision_id = f"revision-{account_id}" + revision_id = revision_id or f"revision-{account_id}" digest = ReportingCanonicalDigest( value=hashlib.sha256(canonical_json_utf8_v1(rows)).hexdigest(), canonicalization_id="rows-v1", @@ -175,7 +185,11 @@ async def scenario( "warehouse_materialization": "warehouse_relation", }[method], location="reports/official/manifest.json", - immutability="immutable_location" if method == "file_transfer" else "native_version", + immutability=( + "immutable_location" + if method == "file_transfer" and profile != "native_commit" + else "native_version" + ), expires_at=completed + timedelta(days=400), manifest_sha256="c" * 64 if method == "file_transfer" else None, native_version_ref=( @@ -187,7 +201,7 @@ async def scenario( verification = ReportingVerificationRecord( verified_at=completed, verification_path={ - "file_transfer": "producer", + "file_transfer": "destination" if profile == "native_commit" else "producer", "dataset_share": "representative_consumer", "warehouse_materialization": "destination", }[method], diff --git a/tests/conformance/reporting/test_reporting_reconciliation_changes.py b/tests/conformance/reporting/test_reporting_reconciliation_changes.py new file mode 100644 index 000000000..8c66a64f9 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_reconciliation_changes.py @@ -0,0 +1,439 @@ +"""Independent scoped checkpoints cannot lose records to Core or to interleaving writes.""" + +from dataclasses import replace +from datetime import timedelta +from typing import get_args + +import pytest + +from adcp.reporting.ledger import ( + LedgerConflictError, + PgReportingReconciliationStore, + ReportingAdjustmentReceiptRecord, + ReportingAdjustmentRecord, + ReportingControlTotalRecord, + ReportingDeliveryPrincipal, + ReportingMaterializationCheck, + ReportingReconciliationCheckpoint, + ReportingReconciliationCursor, + ReportingReconciliationFeedStore, + ReportingReconciliationFilter, + ReportingReconciliationStore, + ReportingStatusCaller, + ReportingStatusHandler, + adjustment_to_wire, +) +from adcp.reporting.ledger.models import LedgerRecordKind +from adcp.reporting.ledger.store import decode_cursor, encode_cursor + +from ._generation_support import END +from ._reconciliation_support import Clock, Store, scenario + + +async def test_frozen_pages_final_checkpoints_and_interleavings( + reconciliation_store: tuple[Store, Clock], +) -> None: + store, clock = reconciliation_store + s = await scenario(store) + caller = s.binding.principal + assert isinstance(store, ReportingReconciliationFeedStore) + assert not hasattr(ReportingReconciliationStore, "read_reconciliation_changes") + core = await ReportingStatusHandler(store).handle( + {"view": "periods"}, caller=ReportingStatusCaller(caller.account_id, caller.consumer_id) + ) + with pytest.raises(LedgerConflictError) as invalid: + await store.read_reconciliation_changes( + caller=caller, changes_after=core["changes_checkpoint"] + ) + assert invalid.value.code == "INVALID_CHECKPOINT" + before = await store.read_reconciliation_snapshot(caller=caller) + first = await store.read_reconciliation_changes(caller=caller, limit=1) + assert first.has_more and first.cursor + assert first.changes_checkpoint is None + assert first.changes[0].record == s.binding + + # Same-account and other-account writers interleave with this frozen walk. + foreign = await scenario(store, consumer_id="another-buyer") + await store.commit_materialization(foreign.outcome) + await store.record_revision_receipt(foreign.receipt) + other = await scenario(store, account_id="acct_b") + await store.commit_materialization(other.outcome) + own_before = await store.read_reconciliation_changes(caller=caller) + foreign_only = await scenario(store, account_id="acct_c") + await store.commit_materialization(foreign_only.outcome) + unaffected = await store.read_reconciliation_changes( + caller=caller, changes_after=own_before.changes_checkpoint + ) + assert ( + unaffected.changes == () and unaffected.changes_checkpoint == own_before.changes_checkpoint + ) + await store.commit_materialization(s.outcome) + accepted, _ = await store.record_revision_receipt(s.receipt) + + # A fresh reader instance needs only the token, not an in-process snapshot map. + reader = ( + PgReportingReconciliationStore(pool=store._pool, clock=clock) + if isinstance(store, PgReportingReconciliationStore) + else store + ) + page = first + collected = list(page.changes) + while page.has_more: + page = await reader.read_reconciliation_changes(caller=caller, cursor=page.cursor, limit=1) + assert page.boundary == first.boundary + assert (page.changes_checkpoint is None) == page.has_more + collected.extend(page.changes) + assert tuple(item.record for item in collected) == before.records + assert len({item.sequence for item in collected}) == len(collected) + later = await reader.read_reconciliation_changes( + caller=caller, changes_after=page.changes_checkpoint + ) + assert tuple(item.record for item in later.changes) == (s.outcome, accepted) + empty = await reader.read_reconciliation_changes( + caller=caller, changes_after=later.changes_checkpoint + ) + assert empty.changes == () and not empty.has_more + assert empty.changes_checkpoint == later.changes_checkpoint + + # A persisted partial page resumes its frozen walk by cursor, not checkpoint. + resumed = await reader.read_reconciliation_changes(caller=caller, cursor=first.cursor) + assert tuple(item.record for item in resumed.changes) == before.records[1:] + for token_name, token in [ + ("cursor", first.cursor), + ("changes_after", later.changes_checkpoint), + ]: + for stranger in [ + ReportingDeliveryPrincipal(caller.account_id, "another-buyer"), + ReportingDeliveryPrincipal("acct_b", caller.consumer_id), + ]: + with pytest.raises(LedgerConflictError) as invalid: + await reader.read_reconciliation_changes(caller=stranger, **{token_name: token}) + assert invalid.value.code == "INVALID_CHECKPOINT" + + +async def test_repairs_and_receipts_remain_incrementally_visible_after_core_advances( + reconciliation_store: tuple[Store, Clock], +) -> None: + store, _ = reconciliation_store + s = await scenario(store) + await store.commit_materialization(s.outcome) + caller = s.binding.principal + original = await store.read_reconciliation_changes(caller=caller) + corrupt = ReportingMaterializationCheck( + s.attempt.scope, + s.attempt.reporting_materialization_id, + "incremental-corruption", + "corrupt", + END + timedelta(seconds=8), + ) + await store.record_materialization_check(corrupt) + rejected, _ = await store.record_revision_receipt( + replace( + s.receipt, + status="rejected", + rejection_codes=("CONTENT_CORRUPT",), + observed_at=END + timedelta(seconds=9), + ) + ) + repair = replace( + corrupt, + check_id="incremental-repair", + state="readable", + checked_at=END + timedelta(seconds=10), + ) + await store.record_materialization_check(repair) + accepted, _ = await store.record_revision_receipt( + replace( + s.receipt, + reporting_receipt_id="incremental-accepted-receipt", + supersedes_reporting_receipt_id=rejected.reporting_receipt_id, + observed_at=END + timedelta(seconds=11), + ) + ) + await ReportingStatusHandler(store).handle( + {"view": "periods"}, caller=ReportingStatusCaller(caller.account_id, caller.consumer_id) + ) + changes = [] + checkpoint = original.changes_checkpoint + cursor = None + # Partial pages retain only a cursor; the final checkpoint covers all repairs. + for expected in (corrupt, rejected, repair, accepted): + page = await store.read_reconciliation_changes( + caller=caller, + changes_after=checkpoint if cursor is None else None, + cursor=cursor, + limit=1, + ) + assert len(page.changes) == 1 and page.changes[0].record == expected + changes.extend(page.changes) + assert (page.changes_checkpoint is None) == page.has_more + cursor = page.cursor + if not page.has_more: + checkpoint = page.changes_checkpoint + assert len({item.sequence for item in changes}) == 4 + assert not ( + await store.read_reconciliation_changes(caller=caller, changes_after=checkpoint) + ).changes + for record, write in [ + (corrupt, store.record_materialization_check), + (repair, store.record_materialization_check), + (accepted, store.record_revision_receipt), + ]: + assert not (await write(record))[1] + assert not ( + await store.read_reconciliation_changes(caller=caller, changes_after=checkpoint) + ).changes + + +@pytest.mark.parametrize( + "mutation", + [ + "foreign_feed", + "negative", + "bool", + "extra", + "time", + "backward", + "missing", + "version", + "filter", + "bound", + "key", + "snapshot", + "count", + "count_none", + "count_bool", + "type", + "false_version", + ], +) +async def test_invalid_reconciliation_positions_fail_closed( + reconciliation_store: tuple[Store, Clock], + mutation: str, +) -> None: + store, _ = reconciliation_store + s = await scenario(store) + caller = s.binding.principal + page = await store.read_reconciliation_changes(caller=caller, limit=1) + position = decode_cursor(page.cursor) + if mutation == "foreign_feed": + position["feed"] = "core" + if mutation == "negative": + position["seq"] = -1 + if mutation == "bool": + position["seq"] = False + if mutation == "extra": + position["unknown"] = "untrusted" + if mutation == "time": + position["as_of"] = "not-a-time" + if mutation == "backward": + position["seq"] = position["through"] + 1 + if mutation == "missing": + position.pop("consumer") + if mutation == "version": + position["version"] = 2 + if mutation == "false_version": + position["version"] = True + if mutation == "filter": + position["filter"] = "f" * 64 + if mutation == "bound": + position["through"] += 1 + if mutation == "key": + position["key"] = "f" * 64 + if mutation == "snapshot": + position["snapshot"] = "rprc_wrong" + if mutation == "count": + position["count"] += 1 + if mutation == "count_none": + position["count"] = None + if mutation == "count_bool": + position["count"] = True + if mutation == "type": + position["type"] = "checkpoint" + with pytest.raises(LedgerConflictError) as error: + await store.read_reconciliation_changes(caller=caller, cursor=encode_cursor(position)) + assert error.value.code == "INVALID_CHECKPOINT" + assert "untrusted" not in str(error.value) + + +async def test_adjustment_receipts_have_their_own_incremental_change( + reconciliation_store: tuple[Store, Clock], +) -> None: + store, _ = reconciliation_store + s = await scenario(store) + before = await store.read_reconciliation_changes(caller=s.binding.principal) + adjustment = ReportingAdjustmentRecord( + "incremental-adjustment", + "acct_a", + s.revision.reporting_revision_id, + "source_correction", + END, + END + timedelta(days=30), + (("spend", "-1"),), + END + timedelta(seconds=5), + END + timedelta(seconds=6), + managed_control_total_deltas=( + ReportingControlTotalRecord("spend", "-1", "decimal", "EUR"), + ), + ) + await store.commit_adjustment(adjustment) + unchanged = await store.read_reconciliation_changes( + caller=s.binding.principal, changes_after=before.changes_checkpoint + ) + assert unchanged.changes == () and unchanged.changes_checkpoint == before.changes_checkpoint + receipt = ReportingAdjustmentReceiptRecord( + s.attempt.scope, + "incremental-adjustment-receipt", + adjustment.reporting_adjustment_id, + s.revision.reporting_revision_id, + "accepted", + adjustment_to_wire(adjustment)["canonical_adjustment_sha256"], + END + timedelta(seconds=7), + ) + accepted, _ = await store.record_adjustment_receipt(receipt) + after = await store.read_reconciliation_changes( + caller=s.binding.principal, changes_after=before.changes_checkpoint + ) + assert tuple(item.record for item in after.changes) == (accepted,) + assert not (await store.record_adjustment_receipt(receipt))[1] + assert not ( + await store.read_reconciliation_changes( + caller=s.binding.principal, changes_after=after.changes_checkpoint + ) + ).changes + + +async def test_foreign_reconciliation_writes_cannot_move_core_or_another_principal( + reconciliation_store: tuple[Store, Clock], +) -> None: + store, _ = reconciliation_store + own = await scenario(store) + foreign = await scenario(store, consumer_id="foreign-consumer") + other_account = await scenario(store, account_id="acct_other") + caller = own.binding.principal + empty_caller = ReportingDeliveryPrincipal(caller.account_id, "no-evidence") + handler = ReportingStatusHandler(store) + core_caller = ReportingStatusCaller(caller.account_id, caller.consumer_id) + core_before = await handler.handle({"view": "periods"}, caller=core_caller) + before = await store.read_reconciliation_snapshot(caller=caller) + empty = await store.read_reconciliation_snapshot(caller=empty_caller) + first = await store.read_reconciliation_changes(caller=caller, limit=1) + complete = await store.read_reconciliation_changes(caller=caller) + checkpoint = complete.changes_checkpoint + tail = await store.read_reconciliation_changes(caller=caller, changes_after=checkpoint) + assert get_args(LedgerRecordKind) == ("obligation", "revision", "adjustment", "consumer_status") + assert before.boundary.max_sequence == before.boundary.total_count == 3 + for candidate in (foreign, other_account): + await store.commit_materialization(candidate.outcome) + await store.record_revision_receipt(candidate.receipt) + assert await handler.handle({"view": "periods"}, caller=core_caller) == core_before + assert await store.read_reconciliation_snapshot(caller=caller) == before + assert await store.read_reconciliation_snapshot(caller=empty_caller) == empty + assert await store.read_reconciliation_changes(caller=caller, limit=1) == first + assert await store.read_reconciliation_changes(caller=caller) == complete + assert ( + await store.read_reconciliation_changes(caller=caller, changes_after=checkpoint) == tail + ) + await store.commit_materialization(own.outcome) + accepted, _ = await store.record_revision_receipt(own.receipt) + later = await store.read_reconciliation_changes(caller=caller, changes_after=checkpoint) + assert tuple(item.sequence for item in later.changes) == (4, 5) + assert tuple(item.record for item in later.changes) == (own.outcome, accepted) + assert await handler.handle({"view": "periods"}, caller=core_caller) == core_before + + +@pytest.mark.parametrize("by_obligation", [False, True]) +async def test_filtered_keyset_walk_freezes_count_bounds_and_final_checkpoint( + reconciliation_store: tuple[Store, Clock], by_obligation: bool +) -> None: + store, clock = reconciliation_store + s = await scenario(store) + await store.commit_materialization(s.outcome) + await store.record_revision_receipt(s.receipt) + caller = s.binding.principal + filters = ReportingReconciliationFilter( + record_kinds=("materialization_attempt", "materialization", "revision_receipt"), + reporting_obligation_id=s.obligation.reporting_obligation_id if by_obligation else None, + ) + page = await store.read_reconciliation_changes(caller=caller, filters=filters, limit=1) + assert page.total_count == 3 and page.changes_checkpoint is None + first = page + new_attempt = replace(s.attempt, attempt=2, reporting_materialization_id="later-attempt") + await store.commit_materialization_attempt(new_attempt) + collected = list(page.changes) + reader = ( + PgReportingReconciliationStore(pool=store._pool, clock=clock) + if isinstance(store, PgReportingReconciliationStore) + else store + ) + while page.has_more: + assert page.cursor is not None and page.changes_checkpoint is None + page = await reader.read_reconciliation_changes( + caller=caller, + filters=filters, + cursor=ReportingReconciliationCursor(str(page.cursor)), + limit=1, + ) + assert page.boundary == first.boundary and page.total_count == 3 + collected.extend(page.changes) + assert tuple(item.sequence for item in collected) == (3, 4, 5) + assert page.changes_checkpoint is not None + checkpoint = ReportingReconciliationCheckpoint(str(page.changes_checkpoint)) + later = await reader.read_reconciliation_changes( + caller=caller, filters=filters, changes_after=checkpoint + ) + assert later.total_count == 1 and later.changes[0].record == new_attempt + for token in ({"cursor": first.cursor}, {"changes_after": checkpoint}): + with pytest.raises(LedgerConflictError) as error: + await reader.read_reconciliation_changes(caller=caller, **token) + assert error.value.code == "INVALID_CHECKPOINT" + empty = await reader.read_reconciliation_changes( + caller=caller, + filters=ReportingReconciliationFilter(reporting_obligation_id="not-this-obligation"), + limit=1, + ) + assert empty.total_count == 0 and empty.changes == () and empty.cursor is None + assert empty.changes_checkpoint is not None + + +@pytest.mark.parametrize("limit", [0, -1, True, 1001]) +async def test_invalid_page_limits_do_not_open_a_feed( + reconciliation_store: tuple[Store, Clock], limit: int +) -> None: + store, _ = reconciliation_store + caller = ReportingDeliveryPrincipal("acct_a", "buyer") + with pytest.raises(LedgerConflictError) as error: + await store.read_reconciliation_changes(caller=caller, limit=limit) + assert error.value.code == "INVALID_CHECKPOINT" + assert not (await store.read_reconciliation_changes(caller=caller)).changes + + +async def test_position_types_and_unissued_boundaries_cannot_be_interchanged( + reconciliation_store: tuple[Store, Clock], +) -> None: + store, _ = reconciliation_store + s = await scenario(store) + caller = s.binding.principal + first = await store.read_reconciliation_changes(caller=caller, limit=1) + last = await store.read_reconciliation_changes(caller=caller) + for tokens in ( + {"cursor": last.changes_checkpoint}, + {"changes_after": first.cursor}, + {"cursor": first.cursor, "changes_after": last.changes_checkpoint}, + {"cursor": "!untrusted"}, + {"cursor": "x" * 5000}, + ): + with pytest.raises(LedgerConflictError) as error: + await store.read_reconciliation_changes(caller=caller, **tokens) + assert error.value.code == "INVALID_CHECKPOINT" + assert error.value.__cause__ is None and error.value.__context__ is None + core = await store.open_snapshot(account_id=caller.account_id, filters_fingerprint="buyer") + with pytest.raises(LedgerConflictError) as error: + await store.read_reconciliation_snapshot(caller=caller, boundary=core) + assert error.value.code == "REPORTING_RECORD_UNAVAILABLE" + from adcp.reporting.ledger.delivery_changes import change_boundary + + future = change_boundary(caller, last.boundary.max_sequence + 1, last.boundary.ledger_as_of) + with pytest.raises(LedgerConflictError) as error: + await store.read_reconciliation_snapshot(caller=caller, boundary=future) + assert error.value.code == "INVALID_CHECKPOINT" diff --git a/tests/conformance/reporting/test_reporting_reconciliation_migration.py b/tests/conformance/reporting/test_reporting_reconciliation_migration.py index 6f2e72526..efb969ea2 100644 --- a/tests/conformance/reporting/test_reporting_reconciliation_migration.py +++ b/tests/conformance/reporting/test_reporting_reconciliation_migration.py @@ -12,7 +12,14 @@ import pytest -from adcp.reporting.ledger import LedgerConflictError, PgReportingReconciliationStore +from adcp.reporting.ledger import ( + LedgerConflictError, + PgReportingReconciliationStore, + ReportingDeliveryPrincipal, + ReportingDeliveryRecord, + ReportingMaterializationCheck, +) +from adcp.reporting.ledger._delivery_state import change_id, decode_record, receipt_chain from ._generation_support import NOW, isolated_reporting_pool from ._reconciliation_support import scenario @@ -23,6 +30,36 @@ MIGRATION = RESOURCES.joinpath("reporting_ledger_reconciliation.sql") +class InitialReconciliationStore(PgReportingReconciliationStore): + """Seed the literal initial schema through its original global-feed contract.""" + + async def _records( + self, connection: Any, who: ReportingDeliveryPrincipal, maximum: int | None = None + ) -> tuple[ReportingDeliveryRecord, ...]: + rows = await ( + await connection.execute( + "SELECT r.payload FROM reporting_reconciliation_records r" + " JOIN reporting_ledger_changes c ON c.account_id = r.account_id" + " AND c.record_kind = r.record_kind AND c.record_id = r.change_id" + " WHERE r.account_id = %s AND r.consumer_id = %s" + " AND (%s::bigint IS NULL OR c.seq <= %s::bigint) ORDER BY c.seq", + (who.account_id, who.consumer_id, maximum, maximum), + ) + ).fetchall() + return tuple(decode_record(row[0]) for row in rows) + + async def _append_reconciliation_change( + self, connection: Any, record: ReportingDeliveryRecord + ) -> None: + from adcp.reporting.ledger._delivery_state import principal + + await connection.execute( + "INSERT INTO reporting_ledger_changes (account_id, record_kind, record_id)" + " VALUES (%s, %s, %s)", + (principal(record).account_id, record.kind, change_id(record)), + ) + + def test_literal_stacked_schema_fixture() -> None: # Literal #1175 bootstrap, unchanged at corrected #1171 head ff584b6f. assert hashlib.sha256((FIXTURES / "reporting_ledger_1171.sql").read_bytes()).hexdigest() == ( @@ -164,6 +201,8 @@ async def test_accepted_records_survive_all_application_connections_closing() -> s = await scenario(store) await store.commit_materialization(s.outcome) receipt, _ = await store.record_revision_receipt(s.receipt) + first_page = await store.read_reconciliation_changes(caller=s.binding.principal, limit=2) + assert first_page.has_more async with pool.connection() as connection: schema = (await (await connection.execute("SELECT current_schema()")).fetchone())[0] await pool.close() @@ -176,6 +215,29 @@ async def test_accepted_records_survive_all_application_connections_closing() -> restarted = PgReportingReconciliationStore(pool=restarted_pool, clock=lambda: NOW) assert await restarted.record_revision_receipt(s.receipt) == (receipt, False) assert await restarted.get_receipt(receipt.key) == receipt + remainder = await restarted.read_reconciliation_changes( + caller=s.binding.principal, cursor=first_page.cursor + ) + assert remainder.boundary == first_page.boundary + assert tuple(item.record for item in first_page.changes + remainder.changes) == ( + s.binding, + s.delivery, + s.attempt, + s.outcome, + receipt, + ) + check = ReportingMaterializationCheck( + s.attempt.scope, + s.attempt.reporting_materialization_id, + "after-restart-check", + "readable", + NOW, + ) + await restarted.record_materialization_check(check) + later = await restarted.read_reconciliation_changes( + caller=s.binding.principal, changes_after=remainder.changes_checkpoint + ) + assert tuple(item.record for item in later.changes) == (check,) with pytest.raises(LedgerConflictError) as error: await restarted.record_revision_receipt( replace(s.receipt, reporting_receipt_id="receipt-after-restart-0002") @@ -192,9 +254,10 @@ async def test_record_receipt_head_and_feed_rollback_together(autocommit: bool) await store.commit_materialization(s.outcome) class FailingStore(PgReportingReconciliationStore): - async def _append_change( - self, connection: Any, account_id: str, kind: Any, record_id: str + async def _append_reconciliation_change( + self, connection: Any, record: ReportingDeliveryRecord ) -> None: + await super()._append_reconciliation_change(connection, record) raise RuntimeError("injected transaction failure") failing = FailingStore(pool=pool, clock=lambda: NOW) @@ -230,12 +293,18 @@ async def test_corrupt_retained_evidence_fails_closed_without_echoing_payload( # Deliberately simulate damaged storage using the database-owner role. # The supported SDK path cannot update/delete these immutable rows. await connection.execute( - "ALTER TABLE reporting_reconciliation_records" - " DISABLE TRIGGER reporting_reconciliation_immutable" + "ALTER TABLE reporting_reconciliation_records" " DISABLE TRIGGER ALL" ) if corruption == "missing_feed": await connection.execute( - "DELETE FROM reporting_ledger_changes WHERE record_kind = 'materialization'" + "ALTER TABLE reporting_reconciliation_changes DISABLE TRIGGER ALL" + ) + await connection.execute( + "DELETE FROM reporting_reconciliation_changes" + " WHERE record_kind = 'materialization'" + ) + await connection.execute( + "ALTER TABLE reporting_reconciliation_changes ENABLE TRIGGER ALL" ) elif corruption == "fingerprint": await connection.execute( @@ -258,8 +327,7 @@ async def test_corrupt_retained_evidence_fails_closed_without_echoing_payload( (path, '"MUST_NOT_RETAIN_PROVIDER_RESPONSE"'), ) await connection.execute( - "ALTER TABLE reporting_reconciliation_records" - " ENABLE TRIGGER reporting_reconciliation_immutable" + "ALTER TABLE reporting_reconciliation_records" " ENABLE TRIGGER ALL" ) for read in [ store.get_materialization(s.attempt.key), @@ -270,3 +338,151 @@ async def test_corrupt_retained_evidence_fails_closed_without_echoing_payload( assert error.value.code in {"INVALID_REPORTING_RECORD", "REPORTING_HISTORY_CORRUPT"} assert "MUST_NOT_RETAIN" not in str(error.value) assert error.value.__cause__ is None and error.value.__context__ is None + + +@pytest.mark.parametrize("autocommit", [False, True]) +async def test_upgrade_from_initial_reconciliation_preserves_records_heads_and_changes( + autocommit: bool, +) -> None: + async with isolated_reporting_pool(autocommit=autocommit) as pool: + async with pool.connection() as connection: + await connection.execute((FIXTURES / "reporting_ledger_1171.sql").read_text()) + await connection.execute( + RESOURCES.joinpath("reporting_ledger_account_generations.sql").read_text() + ) + await connection.execute( + RESOURCES.joinpath("reporting_ledger_obligation_currency.sql").read_text() + ) + await connection.execute( + (FIXTURES / "reporting_ledger_reconciliation_initial.sql").read_text() + ) + legacy_store = InitialReconciliationStore(pool=pool, clock=lambda: NOW) + store = legacy_store + s = await scenario(store) + await store.commit_materialization(s.outcome) + # The initial schema used an application head write. Seed that old + # transaction explicitly; the upgraded schema advances heads itself. + async with pool.connection() as connection, connection.transaction(): + receipt = replace(s.receipt, received_at=NOW) + await store._insert(connection, receipt) + await store._append_reconciliation_change(connection, receipt) + await connection.execute( + "INSERT INTO reporting_receipt_heads" + " (account_id, consumer_id, chain_key, receipt_id, receipt_status)" + " VALUES (%s, %s, %s, %s, %s)", + ( + "acct_a", + "buyer", + receipt_chain(receipt), + receipt.reporting_receipt_id, + receipt.status, + ), + ) + foreign = await scenario(legacy_store, consumer_id="other-buyer") + await legacy_store.commit_materialization(foreign.outcome) + async with pool.connection() as connection: + before_records = await ( + await connection.execute( + "SELECT * FROM reporting_reconciliation_records ORDER BY change_id" + ) + ).fetchall() + before_heads = await ( + await connection.execute("SELECT * FROM reporting_receipt_heads") + ).fetchall() + before_changes = await ( + await connection.execute("SELECT * FROM reporting_ledger_changes ORDER BY seq") + ).fetchall() + store = PgReportingReconciliationStore(pool=pool, clock=lambda: NOW) + await asyncio.gather(*(store.create_schema() for _ in range(4))) + async with pool.connection() as connection: + assert ( + await ( + await connection.execute( + "SELECT * FROM reporting_reconciliation_records ORDER BY change_id" + ) + ).fetchall() + == before_records + ) + assert ( + await (await connection.execute("SELECT * FROM reporting_receipt_heads")).fetchall() + == before_heads + ) + assert ( + await ( + await connection.execute("SELECT * FROM reporting_ledger_changes ORDER BY seq") + ).fetchall() + == before_changes + ) + assert await store.record_revision_receipt(s.receipt) == (receipt, False) + assert ( + len((await store.read_reconciliation_changes(caller=s.binding.principal)).changes) == 5 + ) + assert tuple( + item.sequence + for item in ( + await store.read_reconciliation_changes(caller=foreign.binding.principal) + ).changes + ) == (1, 2, 3, 4) + core_before = await store.open_snapshot(account_id="acct_a", filters_fingerprint="buyer") + before = await store.read_reconciliation_changes(caller=s.binding.principal) + await store.record_revision_receipt(foreign.receipt) + assert ( + await store.open_snapshot(account_id="acct_a", filters_fingerprint="buyer") + == core_before + ) + assert await store.read_reconciliation_changes(caller=s.binding.principal) == before + + +@pytest.mark.parametrize("damage", ["missing_attempt", "missing_feed"]) +async def test_initial_upgrade_rejects_orphaned_outcome_without_repair(damage: str) -> None: + import psycopg + + async with isolated_reporting_pool() as pool: + async with pool.connection() as connection: + await connection.execute((FIXTURES / "reporting_ledger_1171.sql").read_text()) + await connection.execute( + RESOURCES.joinpath("reporting_ledger_account_generations.sql").read_text() + ) + await connection.execute( + RESOURCES.joinpath("reporting_ledger_obligation_currency.sql").read_text() + ) + await connection.execute( + (FIXTURES / "reporting_ledger_reconciliation_initial.sql").read_text() + ) + store = InitialReconciliationStore(pool=pool, clock=lambda: NOW) + s = await scenario(store) + from .test_reporting_reconciliation_sql import raw_insert + + await raw_insert( + pool, + ( + replace(s.outcome, reporting_materialization_id="orphaned-outcome") + if damage == "missing_attempt" + else s.outcome + ), + legacy=True, + write_change=damage != "missing_feed", + ) + async with pool.connection() as connection: + before = await ( + await connection.execute( + "SELECT * FROM reporting_reconciliation_records ORDER BY change_id" + ) + ).fetchall() + with pytest.raises(psycopg.errors.CheckViolation): + await store.create_schema() + async with pool.connection() as connection: + assert ( + await ( + await connection.execute( + "SELECT * FROM reporting_reconciliation_records ORDER BY change_id" + ) + ).fetchall() + == before + ) + assert not await ( + await connection.execute( + "SELECT 1 FROM pg_trigger WHERE tgname = 'reporting_reconciliation_guard'" + " AND tgrelid = 'reporting_reconciliation_records'::regclass" + ) + ).fetchone() diff --git a/tests/conformance/reporting/test_reporting_reconciliation_profiles.py b/tests/conformance/reporting/test_reporting_reconciliation_profiles.py new file mode 100644 index 000000000..85e1ae94d --- /dev/null +++ b/tests/conformance/reporting/test_reporting_reconciliation_profiles.py @@ -0,0 +1,172 @@ +"""Every frozen method/profile cell and provider-native operational identities.""" + +from dataclasses import replace +from typing import get_args + +import pytest + +from adcp.reporting.ledger import ( + LedgerConflictError, + ReportingDeliveryPrincipal, + ReportingMaterializationView, + ReportingPhysicalChecksum, + receipt_to_wire, +) +from adcp.reporting.ledger.delivery_models import DeliveryMethod, VerificationProfile +from adcp.types import ReportingMaterialization, ReportingReceipt + +from ._reconciliation_support import Clock, Store, scenario + + +@pytest.mark.parametrize("method", get_args(DeliveryMethod)) +@pytest.mark.parametrize("profile", get_args(VerificationProfile)) +@pytest.mark.parametrize("billing", [False, True]) +async def test_frozen_method_profile_matrix( + reconciliation_store: tuple[Store, Clock], + method: DeliveryMethod, + profile: VerificationProfile, + billing: bool, +) -> None: + store, _ = reconciliation_store + supported = (profile != "manifest_checksums" or method == "file_transfer") and ( + not billing or profile == "canonical_digest" + ) + if not supported: + with pytest.raises(ValueError, match="manifest|billing"): + await scenario(store, method=method, profile=profile, billing=billing) + assert not ( + await store.read_reconciliation_changes( + caller=ReportingDeliveryPrincipal("acct_a", "buyer") + ) + ).changes + return + s = await scenario(store, method=method, profile=profile, billing=billing) + await store.commit_materialization(s.outcome) + accepted, _ = await store.record_revision_receipt(s.receipt) + ReportingMaterialization.model_validate( + (await store.get_materialization(s.attempt.key)).to_wire() + ) + ReportingReceipt.model_validate(receipt_to_wire(accepted)) + + +@pytest.mark.parametrize("method", get_args(DeliveryMethod)) +async def test_native_commit_requires_native_immutability_even_with_matching_refs( + reconciliation_store: tuple[Store, Clock], method: DeliveryMethod +) -> None: + store, _ = reconciliation_store + s = await scenario(store, method=method, profile="native_commit", billing=False) + wrong = replace( + s.outcome, resource=replace(s.outcome.resource, immutability="immutable_location") + ) + with pytest.raises(LedgerConflictError) as error: + await store.commit_materialization(wrong) + assert error.value.code == "NATIVE_COMMIT_MISMATCH" + assert (await store.get_materialization(s.attempt.key)).outcome is None + assert (await store.commit_materialization(s.outcome))[1] + + +@pytest.mark.parametrize( + "provider,method", + [ + ("GAM", "file_transfer"), + ("FreeWheel", "file_transfer"), + ("Warehouse 欧州", "warehouse_materialization"), + ], +) +async def test_generated_provider_identifiers_round_trip_without_rewriting( + reconciliation_store: tuple[Store, Clock], provider: str, method: DeliveryMethod +) -> None: + store, _ = reconciliation_store + reader = "Parquet / 2.6 + decimal=38" + s = await scenario( + store, method=method, profile="native_commit", billing=False, reader_compatibility=(reader,) + ) + location = f"{provider} / account=123 + daily report" + native = f"/{provider} / run + version=2026-09-01==" + object_ref = f"{provider} / day=2026-09-01/report + part=000.jsonl" + commit = f"{provider} / consumer load + job=42==" + wire = ReportingMaterializationView(s.attempt, s.binding, s.outcome, ()).to_wire() + wire["resource"].update(location=location, native_version_ref=native) + wire["verification"]["native_commit_evidence"]["native_version_ref"] = native + if method == "file_transfer": + wire["verification"]["physical_checksums"][0]["object_ref"] = object_ref + generated = ReportingMaterialization.model_validate(wire) + resource = replace( + s.outcome.resource, + location=generated.resource.location, + native_version_ref=generated.resource.native_version_ref.root, + reader_compatibility=tuple(item.root for item in generated.resource.reader_compatibility), + object_refs=(object_ref,) if method == "file_transfer" else (), + ) + verification = replace( + s.outcome.verification, + native_version_ref=generated.verification.native_commit_evidence.native_version_ref.root, + physical_checksums=tuple( + ReportingPhysicalChecksum(item.object_ref.root, item.algorithm, item.value) + for item in (generated.verification.physical_checksums or []) + ), + ) + await store.commit_materialization( + replace(s.outcome, resource=resource, verification=verification) + ) + projected = ReportingMaterialization.model_validate( + (await store.get_materialization(s.attempt.key)).to_wire() + ) + assert projected.model_dump(mode="json") == generated.model_dump(mode="json") + receipt_model = ReportingReceipt.model_validate( + receipt_to_wire( + replace(s.receipt, consumer_commit_ref=commit, observed_native_version_ref=native) + ) + ) + receipt, _ = await store.record_revision_receipt( + replace( + s.receipt, + consumer_commit_ref=receipt_model.consumer_commit_ref, + observed_native_version_ref=receipt_model.observed_native_version_ref.root, + ) + ) + assert ReportingReceipt.model_validate(receipt_to_wire(receipt)).model_dump( + mode="json", exclude={"received_at"} + ) == receipt_model.model_dump(mode="json", exclude={"received_at"}) + + +@pytest.mark.parametrize( + "field", + [ + "location", + "object_refs", + "native_version_ref", + "consumer_commit_ref", + "reader_compatibility", + ], +) +@pytest.mark.parametrize( + "unsafe", + [ + "Bearer DO_NOT_RETAIN", + "password=DO_NOT_RETAIN", + "https://provider.example/path", + "reports?signature=DO_NOT_RETAIN", + "user:DO_NOT_RETAIN@provider.example", + "https%253A%252F%252Fprovider.example/path", + "Bearer%2520DO_NOT_RETAIN", + "private-key=DO_NOT_RETAIN", + "load\nDO_NOT_RETAIN", + ], +) +async def test_provider_fields_reject_credentials_and_urls_without_echo( + reconciliation_store: tuple[Store, Clock], field: str, unsafe: str +) -> None: + store, _ = reconciliation_store + s = await scenario(store) + with pytest.raises(ValueError) as error: + if field == "consumer_commit_ref": + replace(s.receipt, consumer_commit_ref=unsafe) + else: + replace( + s.outcome.resource, + **{ + field: (unsafe,) if field in {"object_refs", "reader_compatibility"} else unsafe + }, + ) + assert unsafe not in str(error.value) and "DO_NOT_RETAIN" not in str(error.value) diff --git a/tests/conformance/reporting/test_reporting_reconciliation_sql.py b/tests/conformance/reporting/test_reporting_reconciliation_sql.py new file mode 100644 index 000000000..eae0224ec --- /dev/null +++ b/tests/conformance/reporting/test_reporting_reconciliation_sql.py @@ -0,0 +1,623 @@ +"""Bypass Python transition checks: PostgreSQL must enforce the complete evidence graph.""" + +from __future__ import annotations + +import asyncio +from dataclasses import dataclass, replace +from datetime import timedelta +from typing import Any + +import pytest + +from adcp.reporting.ledger import ( + LedgerConflictError, + PgReportingReconciliationStore, + ReportingAdjustmentReceiptRecord, + ReportingAdjustmentRecord, + ReportingControlTotalRecord, + ReportingDeliveryRecord, + ReportingMaterializationCheck, + ReportingReconciliationFilter, + ReportingRevisionReceiptRecord, + adjustment_to_wire, + revision_content_sha256, +) +from adcp.reporting.ledger._delivery_state import change_id, fingerprint, payload, storage_identity +from adcp.reporting.ledger.delivery_pg import _IDENTITY_COLUMNS + +from ._generation_support import END, NOW, configuration, isolated_reporting_pool +from ._reconciliation_support import Scenario, scenario + + +async def raw_insert( + pool: Any, + record: ReportingDeliveryRecord, + *, + write_change: bool = True, + legacy: bool = False, + **overrides: Any, +) -> None: + """Literal INSERTs; never call a store validator, _insert, or receipt-head operation.""" + from psycopg import sql + from psycopg.types.json import Jsonb + + if isinstance(record, (ReportingRevisionReceiptRecord, ReportingAdjustmentReceiptRecord)): + record = replace(record, received_at=NOW) + row = dict(zip(_IDENTITY_COLUMNS, storage_identity(record))) + row.update( + payload=Jsonb(payload(record)), + content_sha256=fingerprint(record), + change_id=change_id(record), + ) + row.update(overrides) + async with pool.connection() as connection, connection.transaction(): + await connection.execute( + sql.SQL("INSERT INTO reporting_reconciliation_records ({}) VALUES ({})").format( + sql.SQL(", ").join(map(sql.Identifier, row)), + sql.SQL(", ").join(sql.Placeholder() for _ in row), + ), + tuple(row.values()), + ) + if write_change and legacy: + await connection.execute( + "INSERT INTO reporting_ledger_changes (account_id, record_kind, record_id)" + " VALUES (%s, %s, %s)", + (row["account_id"], row["record_kind"], row["change_id"]), + ) + elif write_change: + sequence = await ( + await connection.execute( + "INSERT INTO reporting_reconciliation_heads" + " (account_id, consumer_id, max_sequence)" + " VALUES (%s, %s, 1) ON CONFLICT (account_id, consumer_id) DO UPDATE" + " SET max_sequence = reporting_reconciliation_heads.max_sequence + 1" + " RETURNING max_sequence", + (row["account_id"], row["consumer_id"]), + ) + ).fetchone() + await connection.execute( + "INSERT INTO reporting_reconciliation_changes" + " (account_id, consumer_id, seq, namespace, record_id, record_kind," + " change_id, content_sha256)" + " VALUES (%s, %s, %s, %s, %s, %s, %s, %s)", + ( + row["account_id"], + row["consumer_id"], + sequence[0], + row["namespace"], + row["record_id"], + row["record_kind"], + row["change_id"], + row["content_sha256"], + ), + ) + + +@dataclass +class Graph: + pool: Any + store: PgReportingReconciliationStore + first: Scenario + second: Scenario + adjustment: ReportingAdjustmentReceiptRecord + other_adjustment: ReportingAdjustmentReceiptRecord + + +@pytest.fixture +async def graph(): + async with isolated_reporting_pool() as pool: + store = PgReportingReconciliationStore(pool=pool, clock=lambda: NOW) + await store.create_schema() + s = await scenario(store) + await store.commit_materialization(s.outcome) + config = replace( + configuration(), + delivery_config_id="second-config", + feed_purpose="billing", + required_finality="official", + ) + await store.put_configuration(config) + obligation = replace( + s.obligation, + reporting_obligation_id="second-obligation", + delivery_config_id=config.delivery_config_id, + ) + await store.commit_obligation(obligation) + binding = replace(s.binding, generation_key=config.generation_key) + await store.put_destination_binding(binding) + scope = replace( + s.attempt.scope, + generation_key=config.generation_key, + reporting_obligation_id=obligation.reporting_obligation_id, + ) + delivery = replace(s.delivery, scope=scope) + await store.bind_obligation_delivery(delivery) + rows = ( + await store.read_revision_rows( + account_id="acct_a", reporting_revision_id=s.revision.reporting_revision_id + ) + ).rows + revision = replace( + s.revision, + reporting_revision_id="second-revision", + reporting_obligation_id=obligation.reporting_obligation_id, + revision_content_sha256=revision_content_sha256( + reporting_revision_id="second-revision", + row_count=s.revision.row_count, + control_totals=s.revision.control_totals, + reporting_rows=rows, + control_total_evidence=s.revision.managed_control_totals, + ), + ) + await store.commit_revision(revision, rows) + attempt = replace( + s.attempt, + scope=scope, + reporting_revision_id=revision.reporting_revision_id, + reporting_materialization_id="second-materialization", + ) + await store.commit_materialization_attempt(attempt) + outcome = replace( + s.outcome, + scope=scope, + reporting_revision_id=revision.reporting_revision_id, + reporting_materialization_id=attempt.reporting_materialization_id, + ) + await store.commit_materialization(outcome) + receipt = replace( + s.receipt, + scope=scope, + reporting_revision_id=revision.reporting_revision_id, + reporting_materialization_id=attempt.reporting_materialization_id, + reporting_receipt_id="second-receipt-0001", + ) + second = Scenario(binding, delivery, obligation, revision, attempt, outcome, receipt) + # This principal has a real binding/delivery but no attempt or outcome. + await store.put_destination_binding(replace(s.binding, consumer_id="other-buyer")) + await store.bind_obligation_delivery( + replace(s.delivery, scope=replace(s.delivery.scope, consumer_id="other-buyer")) + ) + await store.commit_materialization_attempt( + replace(s.attempt, reporting_materialization_id="pending-attempt", attempt=2) + ) + adjustments = [] + for index, candidate in enumerate((s, second)): + adjustment = ReportingAdjustmentRecord( + f"adjustment-{index}", + "acct_a", + candidate.revision.reporting_revision_id, + "source_correction", + END, + END + timedelta(days=30), + (("spend", "-1"),), + END + timedelta(seconds=5), + END + timedelta(seconds=6), + managed_control_total_deltas=( + ReportingControlTotalRecord("spend", "-1", "decimal", "EUR"), + ), + ) + await store.commit_adjustment(adjustment) + adjustments.append( + ReportingAdjustmentReceiptRecord( + candidate.attempt.scope, + f"adjustment-receipt-{index}", + adjustment.reporting_adjustment_id, + candidate.revision.reporting_revision_id, + "accepted", + adjustment_to_wire(adjustment)["canonical_adjustment_sha256"], + END + timedelta(seconds=7), + ) + ) + yield Graph(pool, store, s, second, *adjustments) + + +@pytest.mark.parametrize( + "edge", + [ + "revision_obligation", + "obligation_generation", + "outcome_attempt", + "outcome_revision", + "outcome_principal", + "check_materialization", + "check_principal", + "receipt_materialization", + "receipt_revision", + "receipt_principal", + "adjustment_revision", + "adjustment_obligation", + "receipt_predecessor_chain", + "accepted_predecessor", + "competing_root", + "rejected_fork", + ], +) +async def test_sql_rejects_inexact_graph_edges(graph: Graph, edge: str) -> None: + from psycopg import IntegrityError + + s, t, store = graph.first, graph.second, graph.store + candidate: ReportingDeliveryRecord = replace( + s.attempt, reporting_materialization_id="raw-attempt", attempt=3 + ) + if edge == "revision_obligation": + candidate = replace(candidate, scope=t.attempt.scope) + if edge == "obligation_generation": + candidate = replace( + candidate, scope=replace(s.attempt.scope, generation_key=t.binding.generation_key) + ) + if edge.startswith("outcome_"): + candidate = replace(s.outcome, reporting_materialization_id="pending-attempt") + if edge == "outcome_attempt": + candidate = replace(candidate, reporting_materialization_id="no-attempt") + if edge == "outcome_revision": + candidate = replace(candidate, reporting_revision_id=t.revision.reporting_revision_id) + if edge == "outcome_principal": + candidate = replace( + candidate, scope=replace(candidate.scope, consumer_id="other-buyer") + ) + if edge.startswith("check_"): + candidate = ReportingMaterializationCheck( + s.attempt.scope, s.attempt.reporting_materialization_id, "sql-check", "readable", NOW + ) + if edge == "check_materialization": + candidate = replace( + candidate, reporting_materialization_id=t.attempt.reporting_materialization_id + ) + if edge == "check_principal": + candidate = replace( + candidate, scope=replace(candidate.scope, consumer_id="other-buyer") + ) + if edge in {"receipt_materialization", "receipt_revision", "receipt_principal"}: + candidate = s.receipt + if edge == "receipt_materialization": + candidate = replace( + candidate, reporting_materialization_id=t.attempt.reporting_materialization_id + ) + if edge == "receipt_revision": + candidate = replace( + candidate, + reporting_revision_id=t.revision.reporting_revision_id, + scope=t.attempt.scope, + ) + if edge == "receipt_principal": + candidate = replace( + candidate, scope=replace(candidate.scope, consumer_id="other-buyer") + ) + if edge == "adjustment_revision": + candidate = replace( + graph.adjustment, reporting_adjustment_id=graph.other_adjustment.reporting_adjustment_id + ) + if edge == "adjustment_obligation": + candidate = replace(graph.adjustment, scope=t.attempt.scope) + if edge in { + "receipt_predecessor_chain", + "accepted_predecessor", + "competing_root", + "rejected_fork", + }: + predecessor = ( + s.receipt + if edge == "accepted_predecessor" + else replace(s.receipt, status="rejected", rejection_codes=("LOAD_FAILED",)) + ) + await raw_insert(graph.pool, predecessor) + candidate = replace( + t.receipt if edge == "receipt_predecessor_chain" else s.receipt, + reporting_receipt_id="raw-successor-0001", + supersedes_reporting_receipt_id=predecessor.reporting_receipt_id, + ) + if edge == "competing_root": + candidate = replace(candidate, supersedes_reporting_receipt_id=None) + if edge == "rejected_fork": + await raw_insert( + graph.pool, + replace( + predecessor, + reporting_receipt_id="already-replaced-receipt", + supersedes_reporting_receipt_id=predecessor.reporting_receipt_id, + ), + ) + before = await store.read_reconciliation_snapshot(caller=s.binding.principal) + with pytest.raises(IntegrityError): + await raw_insert(graph.pool, candidate) + assert await store.read_reconciliation_snapshot(caller=s.binding.principal) == before + + +_COLUMN_MUTATIONS = [ + ("account_id", "other-account"), + ("consumer_id", "other-buyer"), + ("namespace", "wrong-namespace"), + ("record_id", "wrong-record"), + ("record_kind", "materialization"), + ("delivery_config_id", "second-config"), + ("delivery_config_version", 2), + ("reporting_obligation_id", "second-obligation"), + ("reporting_revision_id", "second-revision"), + ("reporting_materialization_id", "second-materialization"), + ("reporting_adjustment_id", "adjustment-1"), + ("attempt_number", 99), + ("receipt_chain_key", "f" * 64), + ("receipt_status", "rejected"), + ("supersedes_receipt_id", "nonexistent-receipt"), + ("change_id", "e" * 64), +] + + +@pytest.mark.parametrize("column,value", _COLUMN_MUTATIONS) +async def test_sql_rejects_denormalized_payload_identity_disagreement( + graph: Graph, column: str, value: Any +) -> None: + from psycopg import IntegrityError + + candidate: ReportingDeliveryRecord = graph.first.receipt + if column == "attempt_number": + candidate = replace( + graph.first.attempt, reporting_materialization_id="raw-attempt", attempt=3 + ) + if column == "reporting_adjustment_id": + candidate = graph.adjustment + with pytest.raises(IntegrityError): + await raw_insert(graph.pool, candidate, **{column: value}) + + +@pytest.mark.parametrize("column,value", _COLUMN_MUTATIONS) +async def test_records_fail_closed_after_owner_bypasses_database_guards( + graph: Graph, column: str, value: Any +) -> None: + from psycopg import sql + + candidate: ReportingDeliveryRecord = graph.first.receipt + if column == "attempt_number": + candidate = replace( + graph.first.attempt, reporting_materialization_id="raw-attempt", attempt=3 + ) + if column == "reporting_adjustment_id": + candidate = graph.adjustment + await raw_insert(graph.pool, candidate) + async with graph.pool.connection() as connection: + # A database-owner damage probe. Ordinary SQL above cannot do this. + await connection.execute("ALTER TABLE reporting_reconciliation_records DISABLE TRIGGER ALL") + constraints = await ( + await connection.execute( + "SELECT conname FROM pg_constraint" + " WHERE conrelid = 'reporting_reconciliation_records'::regclass AND contype = 'c'" + ) + ).fetchall() + for (name,) in constraints: + await connection.execute( + sql.SQL("ALTER TABLE reporting_reconciliation_records DROP CONSTRAINT {}").format( + sql.Identifier(name) + ) + ) + await connection.execute( + sql.SQL( + "UPDATE reporting_reconciliation_records SET {} = %s WHERE change_id = %s" + ).format(sql.Identifier(column)), + (value, change_id(candidate)), + ) + await connection.execute("ALTER TABLE reporting_reconciliation_records ENABLE TRIGGER ALL") + caller = graph.first.binding.principal + if column == "account_id": + caller = replace(caller, account_id=value) + if column == "consumer_id": + caller = replace(caller, consumer_id=value) + for read in [ + graph.store.read_reconciliation_snapshot(caller=caller), + graph.store.read_reconciliation_changes(caller=caller), + ]: + with pytest.raises(LedgerConflictError) as error: + await read + assert error.value.code == "REPORTING_HISTORY_CORRUPT" + assert error.value.__cause__ is None and error.value.__context__ is None + + +@pytest.mark.parametrize("has_predecessor", [False, True]) +@pytest.mark.parametrize("status", ["accepted", "rejected"]) +async def test_direct_sql_receipt_races_have_one_current_leaf( + graph: Graph, has_predecessor: bool, status: str +) -> None: + from psycopg import IntegrityError + + s = graph.first + predecessor = None + if has_predecessor: + predecessor = s.receipt.reporting_receipt_id + await raw_insert( + graph.pool, replace(s.receipt, status="rejected", rejection_codes=("LOAD_FAILED",)) + ) + results = await asyncio.gather( + *( + raw_insert( + graph.pool, + replace( + s.receipt, + reporting_receipt_id=f"direct-sql-race-{i:04}", + status=status, + rejection_codes=("LOAD_FAILED",) if status == "rejected" else (), + supersedes_reporting_receipt_id=predecessor, + ), + ) + for i in range(8) + ), + return_exceptions=True, + ) + assert sum(item is None for item in results) == 1 + assert sum(isinstance(item, IntegrityError) for item in results) == 7 + snapshot = await graph.store.read_reconciliation_snapshot(caller=s.binding.principal) + assert len(snapshot.current_receipts) == 1 + assert snapshot.current_receipts[0].status == status + + +async def test_sql_cannot_commit_evidence_without_its_change(graph: Graph) -> None: + from psycopg import IntegrityError + + before = await graph.store.read_reconciliation_snapshot(caller=graph.first.binding.principal) + with pytest.raises(IntegrityError): + await raw_insert(graph.pool, graph.first.receipt, write_change=False) + assert ( + await graph.store.read_reconciliation_snapshot(caller=graph.first.binding.principal) + == before + ) + async with graph.pool.connection() as connection: + assert not await ( + await connection.execute("SELECT 1 FROM reporting_receipt_heads") + ).fetchone() + with pytest.raises(IntegrityError): + async with graph.pool.connection() as connection, connection.transaction(): + # Also prove the deferred FK prevents removing an existing change. + await connection.execute( + "DELETE FROM reporting_reconciliation_changes WHERE record_kind = 'materialization'" + ) + + +@pytest.mark.parametrize( + "field,value", + [ + ("account_id", "foreign-account"), + ("consumer_id", "other-buyer"), + ("chain_key", "f" * 64), + ("receipt_id", "nonexistent-receipt"), + ("receipt_status", "accepted"), + ("supersedes_receipt_id", "nonexistent-predecessor"), + ], +) +async def test_sql_rejects_forged_receipt_heads(graph: Graph, field: str, value: str) -> None: + from psycopg import IntegrityError, sql + + await raw_insert( + graph.pool, + replace(graph.first.receipt, status="rejected", rejection_codes=("LOAD_FAILED",)), + ) + async with graph.pool.connection() as connection: + before = await ( + await connection.execute("SELECT * FROM reporting_receipt_heads") + ).fetchall() + with pytest.raises(IntegrityError): + async with graph.pool.connection() as connection: + await connection.execute( + sql.SQL("UPDATE reporting_receipt_heads SET {} = %s").format(sql.Identifier(field)), + (value,), + ) + async with graph.pool.connection() as connection: + assert ( + await (await connection.execute("SELECT * FROM reporting_receipt_heads")).fetchall() + == before + ) + + +@pytest.mark.parametrize( + "statement", + [ + "UPDATE reporting_configurations SET feed_purpose = 'pacing'", + "UPDATE reporting_obligations SET definition = NULL", + "UPDATE reporting_revisions SET finality = 'snapshot'", + "UPDATE reporting_revisions SET reporting_obligation_id = 'second-obligation'" + " WHERE reporting_revision_id = 'revision-acct_a'", + "UPDATE reporting_adjustments SET reason_detail = 'changed'" + " WHERE reporting_adjustment_id = 'adjustment-0'", + ], +) +async def test_sql_cannot_rewrite_roots_referenced_by_frozen_evidence( + graph: Graph, statement: str +) -> None: + from psycopg import IntegrityError + + await raw_insert(graph.pool, graph.adjustment) + with pytest.raises(IntegrityError): + async with graph.pool.connection() as connection: + await connection.execute(statement) + # Operational readability is still mutable through the supported Core API. + await graph.store.set_revision_readable( + account_id="acct_a", + reporting_revision_id=graph.first.revision.reporting_revision_id, + readable=False, + ) + + +@pytest.mark.parametrize("method", ["file_transfer", "dataset_share", "warehouse_materialization"]) +async def test_sql_native_commit_cannot_claim_immutable_location(method: str) -> None: + from psycopg import IntegrityError + + async with isolated_reporting_pool() as pool: + store = PgReportingReconciliationStore(pool=pool, clock=lambda: NOW) + await store.create_schema() + s = await scenario(store, method=method, profile="native_commit", billing=False) + with pytest.raises(IntegrityError): + await raw_insert( + pool, + replace( + s.outcome, + resource=replace(s.outcome.resource, immutability="immutable_location"), + ), + ) + await raw_insert(pool, s.outcome) + assert (await store.get_materialization(s.attempt.key)).outcome == s.outcome + + +@pytest.mark.parametrize( + "field,value", + [ + ("seq", 99), + ("account_id", "other-account"), + ("consumer_id", "other-buyer"), + ("namespace", "wrong-namespace"), + ("record_id", "wrong-record"), + ("record_kind", "materialization_attempt"), + ("change_id", "f" * 64), + ("content_sha256", "0" * 64), + ], +) +async def test_feed_identity_damage_cannot_disappear_through_a_join( + graph: Graph, field: str, value: Any +) -> None: + from psycopg import sql + + async with graph.pool.connection() as connection: + await connection.execute("ALTER TABLE reporting_reconciliation_changes DISABLE TRIGGER ALL") + await connection.execute( + sql.SQL( + "UPDATE reporting_reconciliation_changes SET {} = %s" + " WHERE account_id = 'acct_a' AND consumer_id = 'buyer'" + " AND record_kind = 'materialization' AND record_id = 'materialization-1'" + ).format(sql.Identifier(field)), + (value,), + ) + await connection.execute("ALTER TABLE reporting_reconciliation_changes ENABLE TRIGGER ALL") + caller = graph.first.binding.principal + for read in ( + graph.store.read_reconciliation_snapshot(caller=caller), + graph.store.read_reconciliation_changes(caller=caller, limit=1), + graph.store.read_reconciliation_changes( + caller=caller, filters=ReportingReconciliationFilter(record_kinds=("revision_receipt",)) + ), + ): + with pytest.raises(LedgerConflictError) as error: + await read + assert error.value.code == "REPORTING_HISTORY_CORRUPT" + assert error.value.__cause__ is None and error.value.__context__ is None + + +@pytest.mark.parametrize( + "statement", + [ + "UPDATE reporting_reconciliation_heads SET max_sequence = max_sequence + 2", + "UPDATE reporting_reconciliation_heads SET max_sequence = max_sequence + 1", + "UPDATE reporting_reconciliation_heads SET consumer_id = 'forged-principal'", + "DELETE FROM reporting_reconciliation_heads", + "DELETE FROM reporting_reconciliation_changes", + "UPDATE reporting_reconciliation_changes SET seq = seq + 100", + "INSERT INTO reporting_reconciliation_heads (account_id, consumer_id, max_sequence)" + " VALUES ('new-account', 'new-consumer', 1)", + ], +) +async def test_sql_cannot_advance_move_or_erase_the_feed_without_exact_evidence( + graph: Graph, statement: str +) -> None: + from psycopg import IntegrityError + + caller = graph.first.binding.principal + before = await graph.store.read_reconciliation_changes(caller=caller) + with pytest.raises(IntegrityError): + async with graph.pool.connection() as connection, connection.transaction(): + await connection.execute(statement) + assert await graph.store.read_reconciliation_changes(caller=caller) == before diff --git a/tests/conformance/reporting/test_reporting_reconciliation_store.py b/tests/conformance/reporting/test_reporting_reconciliation_store.py index c8e72c662..e11e8b1f6 100644 --- a/tests/conformance/reporting/test_reporting_reconciliation_store.py +++ b/tests/conformance/reporting/test_reporting_reconciliation_store.py @@ -372,19 +372,19 @@ async def test_repair_appends_evidence_and_advances_checkpoint_without_rewriting assert await store.record_materialization_check(check) == (check, False) assert await store.record_revision_receipt(accepted_input) == (accepted, False) assert await store.read_reconciliation_snapshot(caller=caller) == after - # Core consumes the shared checkpoint without counting or exposing optional records. + # Reconciliation evidence cannot move Core's independent checkpoint. core_after = await handler.handle( {"view": "periods", "changes_after": core_before["changes_checkpoint"]}, caller=status_caller, ) - assert core_after["changes_checkpoint"] != core_before["changes_checkpoint"] + assert core_after["changes_checkpoint"] == core_before["changes_checkpoint"] assert core_after["pagination"]["total_count"] == 0 assert core_after["periods"] == core_after["revisions"] == [] assert core_after["materializations"] == core_after["receipts"] == [] stranger = ReportingDeliveryPrincipal(caller.account_id, "another-consumer") - assert ( + with pytest.raises(LedgerConflictError) as error: await store.read_reconciliation_snapshot(caller=stranger, boundary=after.boundary) - ).records == () + assert error.value.code == "REPORTING_RECORD_UNAVAILABLE" with pytest.raises(LedgerConflictError) as error: await store.read_reconciliation_snapshot( caller=ReportingDeliveryPrincipal("another-account", caller.consumer_id), diff --git a/tests/conformance/reporting/test_reporting_reconciliation_transitions.py b/tests/conformance/reporting/test_reporting_reconciliation_transitions.py index 6ae2cf014..df8303428 100644 --- a/tests/conformance/reporting/test_reporting_reconciliation_transitions.py +++ b/tests/conformance/reporting/test_reporting_reconciliation_transitions.py @@ -37,6 +37,7 @@ [ ("file_transfer", "canonical_digest"), ("file_transfer", "manifest_checksums"), + ("file_transfer", "native_commit"), ("dataset_share", "canonical_digest"), ("dataset_share", "native_commit"), ("warehouse_materialization", "canonical_digest"), @@ -481,7 +482,7 @@ async def test_adjustment_ordering_and_digest_disagreement( @pytest.mark.parametrize("different_scope", [False, True]) -async def test_revision_fanout_requires_exact_frozen_logical_scope( +async def test_revision_cannot_fan_out_to_a_different_frozen_obligation( reconciliation_store: tuple[Store, Clock], different_scope: bool ) -> None: store, _ = reconciliation_store @@ -507,12 +508,11 @@ async def test_revision_fanout_requires_exact_frozen_logical_scope( scope = ReportingDeliveryScope(generation, "buyer", obligation.reporting_obligation_id) await store.bind_obligation_delivery(replace(s.delivery, scope=scope)) attempt = replace(s.attempt, scope=scope, reporting_materialization_id="fanout-materialization") - if different_scope: - with pytest.raises(LedgerConflictError) as error: - await store.commit_materialization_attempt(attempt) - assert error.value.code == "REPORTING_RECORD_UNAVAILABLE" - else: + before = await store.read_reconciliation_snapshot(caller=scope.principal) + with pytest.raises(LedgerConflictError) as error: await store.commit_materialization_attempt(attempt) + assert error.value.code == "REPORTING_RECORD_UNAVAILABLE" + with pytest.raises(LedgerConflictError) as error: await store.commit_materialization( replace( s.outcome, @@ -520,15 +520,19 @@ async def test_revision_fanout_requires_exact_frozen_logical_scope( reporting_materialization_id=attempt.reporting_materialization_id, ) ) - receipt, _ = await store.record_revision_receipt( + assert error.value.code == "REPORTING_RECORD_UNAVAILABLE" + assert await store.read_reconciliation_snapshot(caller=scope.principal) == before + await store.commit_materialization(s.outcome) + with pytest.raises(LedgerConflictError) as error: + await store.record_revision_receipt( replace( s.receipt, scope=scope, reporting_receipt_id="fanout-receipt-0001", - reporting_materialization_id=attempt.reporting_materialization_id, ) ) - assert receipt.scope == scope + assert error.value.code == "REPORTING_RECORD_UNAVAILABLE" + assert (await store.record_revision_receipt(s.receipt))[1] async def test_core_and_managed_delivery_only_do_not_require_receipt_components( diff --git a/tests/fixtures/reporting_ledger_reconciliation_initial.sql b/tests/fixtures/reporting_ledger_reconciliation_initial.sql new file mode 100644 index 000000000..1c2f930af --- /dev/null +++ b/tests/fixtures/reporting_ledger_reconciliation_initial.sql @@ -0,0 +1,169 @@ +-- Storage foundation for #1167. Apply after the account-generation and currency +-- migrations, with old writers drained. This single DO is atomic in autocommit. +-- No legacy evidence is inferred, hashed again, or backfilled. +DO $reconciliation$ +DECLARE + evidence_type OID; + evidence_default TEXT; + evidence_nullable BOOLEAN; + evidence_table TEXT; + evidence_column TEXT; +BEGIN + PERFORM pg_advisory_xact_lock(hashtext('adcp.reporting.schema'), hashtext(current_schema())); + -- Resolve prerequisite columns/keys before installing any extension tables. + PERFORM currency FROM reporting_obligations LIMIT 0; + ALTER TABLE reporting_revisions ADD COLUMN IF NOT EXISTS canonical_content_digest JSONB; + ALTER TABLE reporting_revisions ADD COLUMN IF NOT EXISTS managed_control_totals JSONB; + ALTER TABLE reporting_adjustments ADD COLUMN IF NOT EXISTS managed_control_total_deltas JSONB; + FOR evidence_table, evidence_column IN VALUES + ('reporting_revisions', 'canonical_content_digest'), + ('reporting_revisions', 'managed_control_totals'), + ('reporting_adjustments', 'managed_control_total_deltas') + LOOP + SELECT a.atttypid, pg_get_expr(d.adbin, d.adrelid), NOT a.attnotnull + INTO evidence_type, evidence_default, evidence_nullable + FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid = a.attrelid AND d.adnum = a.attnum + WHERE a.attrelid = evidence_table::regclass AND a.attname = evidence_column; + IF evidence_type <> 'jsonb'::regtype OR evidence_default IS NOT NULL OR NOT evidence_nullable THEN + RAISE EXCEPTION 'Unexpected reporting evidence column'; + END IF; + END LOOP; + + CREATE OR REPLACE FUNCTION reporting_canonical_evidence_immutable() + RETURNS TRIGGER LANGUAGE plpgsql AS $function$ + BEGIN + IF NEW.canonical_content_digest IS DISTINCT FROM OLD.canonical_content_digest + OR NEW.managed_control_totals IS DISTINCT FROM OLD.managed_control_totals THEN + RAISE EXCEPTION 'reporting canonical evidence is immutable' USING ERRCODE = '23514'; + END IF; + RETURN NEW; + END + $function$; + IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid = 'reporting_revisions'::regclass + AND tgname = 'reporting_canonical_evidence_immutable' AND NOT tgisinternal) THEN + CREATE TRIGGER reporting_canonical_evidence_immutable + BEFORE UPDATE OF canonical_content_digest, managed_control_totals ON reporting_revisions + FOR EACH ROW EXECUTE FUNCTION reporting_canonical_evidence_immutable(); + END IF; + + CREATE OR REPLACE FUNCTION reporting_adjustment_evidence_immutable() + RETURNS TRIGGER LANGUAGE plpgsql AS $function$ + BEGIN + IF NEW.managed_control_total_deltas IS DISTINCT FROM OLD.managed_control_total_deltas THEN + RAISE EXCEPTION 'reporting adjustment evidence is immutable' USING ERRCODE = '23514'; + END IF; + RETURN NEW; + END + $function$; + IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid = 'reporting_adjustments'::regclass + AND tgname = 'reporting_adjustment_evidence_immutable' AND NOT tgisinternal) THEN + CREATE TRIGGER reporting_adjustment_evidence_immutable + BEFORE UPDATE OF managed_control_total_deltas ON reporting_adjustments + FOR EACH ROW EXECUTE FUNCTION reporting_adjustment_evidence_immutable(); + END IF; + + -- The older Core identifiers remain globally unique. Every new relationship + -- nevertheless uses the account in its foreign key; a global ID is no grant. + CREATE UNIQUE INDEX IF NOT EXISTS reporting_obligations_account_identity + ON reporting_obligations(account_id, reporting_obligation_id); + CREATE UNIQUE INDEX IF NOT EXISTS reporting_revisions_account_identity + ON reporting_revisions(account_id, reporting_revision_id); + CREATE UNIQUE INDEX IF NOT EXISTS reporting_adjustments_account_identity + ON reporting_adjustments(account_id, reporting_adjustment_id); + + -- Each payload is the SDK's closed, frozen record shape, not a generated wire + -- object or provider response. Identity/join/transition fields are explicit. + CREATE TABLE IF NOT EXISTS reporting_reconciliation_records ( + account_id TEXT COLLATE "C" NOT NULL, + consumer_id TEXT COLLATE "C" NOT NULL, + namespace TEXT COLLATE "C" NOT NULL, + record_id TEXT COLLATE "C" NOT NULL, + record_kind TEXT NOT NULL CHECK (record_kind IN ( + 'destination_binding', 'obligation_delivery', 'materialization_attempt', + 'materialization', 'materialization_check', 'revision_receipt', 'adjustment_receipt')), + delivery_config_id TEXT COLLATE "C" NOT NULL, + delivery_config_version INTEGER NOT NULL, + reporting_obligation_id TEXT COLLATE "C", + reporting_revision_id TEXT COLLATE "C", + reporting_materialization_id TEXT COLLATE "C", + reporting_adjustment_id TEXT COLLATE "C", + attempt_number INTEGER CHECK (attempt_number > 0), + receipt_chain_key TEXT COLLATE "C", + receipt_status TEXT CHECK (receipt_status IN ('accepted', 'rejected')), + supersedes_receipt_id TEXT COLLATE "C", + payload JSONB NOT NULL CHECK (jsonb_typeof(payload) = 'object'), + content_sha256 TEXT COLLATE "C" NOT NULL CHECK (content_sha256 ~ '^[a-f0-9]{64}$'), + change_id TEXT COLLATE "C" NOT NULL, + PRIMARY KEY (account_id, consumer_id, namespace, record_id), + UNIQUE (account_id, record_kind, change_id), + CHECK (namespace = CASE WHEN record_kind IN ('revision_receipt', 'adjustment_receipt') + THEN 'receipt' ELSE record_kind END), + CHECK ((record_kind IN ('revision_receipt', 'adjustment_receipt')) = (receipt_status IS NOT NULL)), + CHECK ((receipt_status IS NOT NULL) = (receipt_chain_key IS NOT NULL)), + CHECK ((record_kind = 'materialization_attempt') = (attempt_number IS NOT NULL)), + CHECK (payload->>'kind' = record_kind), + FOREIGN KEY (account_id, delivery_config_id, delivery_config_version) + REFERENCES reporting_configurations(account_id, delivery_config_id, delivery_config_version), + FOREIGN KEY (account_id, reporting_obligation_id) + REFERENCES reporting_obligations(account_id, reporting_obligation_id), + FOREIGN KEY (account_id, reporting_revision_id) + REFERENCES reporting_revisions(account_id, reporting_revision_id), + FOREIGN KEY (account_id, reporting_adjustment_id) + REFERENCES reporting_adjustments(account_id, reporting_adjustment_id), + FOREIGN KEY (account_id, consumer_id, namespace, supersedes_receipt_id) + REFERENCES reporting_reconciliation_records(account_id, consumer_id, namespace, record_id) + ); + CREATE UNIQUE INDEX IF NOT EXISTS reporting_materialization_attempt_identity + ON reporting_reconciliation_records(account_id, consumer_id, reporting_obligation_id, + reporting_revision_id, attempt_number) + WHERE record_kind = 'materialization_attempt'; + CREATE UNIQUE INDEX IF NOT EXISTS reporting_receipt_one_successor + ON reporting_reconciliation_records(account_id, consumer_id, supersedes_receipt_id) + WHERE supersedes_receipt_id IS NOT NULL; + + CREATE TABLE IF NOT EXISTS reporting_receipt_heads ( + account_id TEXT COLLATE "C" NOT NULL, + consumer_id TEXT COLLATE "C" NOT NULL, + chain_key TEXT COLLATE "C" NOT NULL, + namespace TEXT NOT NULL DEFAULT 'receipt' CHECK (namespace = 'receipt'), + receipt_id TEXT COLLATE "C" NOT NULL, + receipt_status TEXT NOT NULL CHECK (receipt_status IN ('accepted', 'rejected')), + supersedes_receipt_id TEXT COLLATE "C", + PRIMARY KEY (account_id, consumer_id, chain_key), + FOREIGN KEY (account_id, consumer_id, namespace, receipt_id) + REFERENCES reporting_reconciliation_records(account_id, consumer_id, namespace, record_id) + ); + + CREATE OR REPLACE FUNCTION reporting_reconciliation_immutable() + RETURNS TRIGGER LANGUAGE plpgsql AS $function$ + BEGIN + RAISE EXCEPTION 'reporting reconciliation evidence is append-only' USING ERRCODE = '23514'; + END + $function$; + CREATE OR REPLACE FUNCTION reporting_receipt_terminal() + RETURNS TRIGGER LANGUAGE plpgsql AS $function$ + BEGIN + IF TG_OP = 'DELETE' OR OLD.receipt_status = 'accepted' + OR NEW.account_id <> OLD.account_id OR NEW.consumer_id <> OLD.consumer_id + OR NEW.chain_key <> OLD.chain_key + OR NEW.supersedes_receipt_id IS DISTINCT FROM OLD.receipt_id THEN + RAISE EXCEPTION 'reporting receipt replacement is invalid' USING ERRCODE = '23514'; + END IF; + RETURN NEW; + END + $function$; + IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid = 'reporting_reconciliation_records'::regclass + AND tgname = 'reporting_reconciliation_immutable' AND NOT tgisinternal) THEN + CREATE TRIGGER reporting_reconciliation_immutable + BEFORE UPDATE OR DELETE ON reporting_reconciliation_records + FOR EACH ROW EXECUTE FUNCTION reporting_reconciliation_immutable(); + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid = 'reporting_receipt_heads'::regclass + AND tgname = 'reporting_receipt_terminal' AND NOT tgisinternal) THEN + CREATE TRIGGER reporting_receipt_terminal + BEFORE UPDATE OR DELETE ON reporting_receipt_heads + FOR EACH ROW EXECUTE FUNCTION reporting_receipt_terminal(); + END IF; +END +$reconciliation$; diff --git a/tests/test_reporting_reconciliation.py b/tests/test_reporting_reconciliation.py index 5a2de3df8..b60e6e25a 100644 --- a/tests/test_reporting_reconciliation.py +++ b/tests/test_reporting_reconciliation.py @@ -16,6 +16,7 @@ from adcp.reporting import ( ExpectedReportingPeriod, ReportingInspectionContext, + ReportingLedger, ReportingObservation, ReportingReconciliationError, ReportingTier, @@ -1222,7 +1223,7 @@ async def get_reporting_status( @pytest.mark.asyncio -async def test_one_revision_fans_out_without_double_counting_totals() -> None: +async def test_one_revision_cannot_materialize_under_two_obligations() -> None: raw = _response() first = _obligation("obligation-a") second = _obligation("obligation-b") @@ -1259,8 +1260,106 @@ async def get_reporting_status( expected_periods=[], now=datetime.fromisoformat("2026-09-03T00:00:00+00:00"), ) - assert result.definitive, result.obligations - assert len(result.totals_by_revision) == 1 + assert not result.definitive + assert all("REVISION_SCOPE_MISMATCH" in item.reasons for item in result.obligations) + + +@pytest.mark.parametrize("finality", ["official", "snapshot"]) +@pytest.mark.parametrize("delivery", ["absent", "pending", "failed", "available"]) +def test_current_publication_is_selected_before_its_materialization( + finality: str, delivery: str +) -> None: + raw = _response() + obligation = raw["periods"][0] + obligation.update( + required_finality=finality, + revision_count=2, + reconciliation_mode="delivery_only", + reconciliation_status="not_required", + health="complete", + ) + older = deepcopy(REVISION) + older.update(reporting_revision_id="older-materialized-snapshot", finality="snapshot") + for key in ("finality_basis", "finality_policy_id", "finalized_at"): + older.pop(key) + current = deepcopy(REVISION) + if finality == "snapshot": + current.update( + finality="snapshot", supersedes_reporting_revision_id=older["reporting_revision_id"] + ) + for key in ("finality_basis", "finality_policy_id", "finalized_at"): + current.pop(key) + old_attempt = _materialization("older-materialization") + old_attempt["reporting_revision_id"] = older["reporting_revision_id"] + raw["revisions"] = [older, current] + raw["materializations"] = [old_attempt] + if delivery != "absent": + attempt = _materialization("current-materialization") + attempt["status"] = delivery + if delivery != "available": + for key in ("ready_at", "resource", "verification"): + attempt.pop(key) + if delivery == "failed": + attempt.update(failed_at="2026-09-02T00:00:05Z", failure_code="WRITE_FAILED") + raw["materializations"].append(attempt) + obligation.update( + materialization_count=len(raw["materializations"]), + successful_materialization_count=2 if delivery == "available" else 1, + ) + response = GetReportingStatusResponse.model_validate(raw) + ledger = ReportingLedger( + response.ledger_snapshot_id, + response.ledger_as_of, + response.account_id, + response.scope, + response.periods, + response.revisions, + response.materializations, + response.receipts, + ) + result = evaluate_reporting_ledger( + ledger, expected_periods=[], now=datetime.fromisoformat("2026-09-03T00:00:00+00:00") + ) + selected = result.obligations[0] + assert selected.reporting_revision_id == current["reporting_revision_id"] + assert "AMBIGUOUS_REVISION_CHAIN" not in selected.reasons + assert "ASSOCIATED_HISTORY_INCOMPLETE" not in selected.reasons + if delivery == "available": + assert selected.reporting_materialization_id == "current-materialization" + assert result.definitive, selected.reasons + else: + assert selected.reporting_materialization_id is None + assert "MISSING_VERIFIED_MATERIALIZATION" in selected.reasons + assert not result.definitive + + +@pytest.mark.parametrize("finality", ["official", "snapshot"]) +def test_publication_selection_rejects_multiple_current_revisions(finality: str) -> None: + raw = _response() + first = deepcopy(REVISION) + second = deepcopy(REVISION) + second["reporting_revision_id"] = "competing-publication" + if finality == "snapshot": + for revision in (first, second): + revision["finality"] = finality + for key in ("finality_basis", "finality_policy_id", "finalized_at"): + revision.pop(key) + raw["periods"][0].update(required_finality=finality, revision_count=2) + raw["revisions"] = [first, second] + response = GetReportingStatusResponse.model_validate(raw) + ledger = ReportingLedger( + response.ledger_snapshot_id, + response.ledger_as_of, + response.account_id, + response.scope, + response.periods, + response.revisions, + response.materializations, + response.receipts, + ) + result = evaluate_reporting_ledger(ledger, expected_periods=[]) + assert not result.definitive + assert "AMBIGUOUS_REVISION_CHAIN" in result.obligations[0].reasons @pytest.mark.asyncio diff --git a/tests/type_checks/reporting_reconciliation_records.py b/tests/type_checks/reporting_reconciliation_records.py index be0199899..577c730a5 100644 --- a/tests/type_checks/reporting_reconciliation_records.py +++ b/tests/type_checks/reporting_reconciliation_records.py @@ -16,6 +16,7 @@ ReportingCanonicalDigest, ReportingConfigurationGenerationKey, ReportingControlTotalRecord, + ReportingDeliveryPrincipal, ReportingDeliveryScope, ReportingDestinationBinding, ReportingDestinationStore, @@ -28,6 +29,11 @@ ReportingReceiptKey, ReportingReceiptRecord, ReportingReceiptStore, + ReportingReconciliationCheckpoint, + ReportingReconciliationCursor, + ReportingReconciliationFeedStore, + ReportingReconciliationPage, + ReportingReconciliationSnapshotToken, ReportingReconciliationStore, ReportingRevisionReceiptRecord, ReportingRevisionRecord, @@ -47,6 +53,36 @@ def persistent(pool: AsyncConnectionPool) -> ReportingReconciliationStore: return PgReportingReconciliationStore(pool=pool) +def incremental(pool: AsyncConnectionPool) -> ReportingReconciliationFeedStore: + return PgReportingReconciliationStore(pool=pool) + + +async def incremental_read( + store: ReportingReconciliationFeedStore, + caller: ReportingDeliveryPrincipal, + checkpoint: ReportingReconciliationCheckpoint | None, +) -> ReportingReconciliationPage: + return await store.read_reconciliation_changes( + caller=caller, changes_after=checkpoint, limit=50 + ) + + +async def continue_incremental_read( + store: ReportingReconciliationFeedStore, page: ReportingReconciliationPage +) -> ReportingReconciliationPage: + boundary: ReportingReconciliationSnapshotToken = page.boundary + if page.has_more: + assert page.cursor is not None + cursor: ReportingReconciliationCursor = page.cursor + return await store.read_reconciliation_changes( + caller=page.caller, cursor=cursor, filters=boundary.filters + ) + assert page.changes_checkpoint is not None + return await store.read_reconciliation_changes( + caller=page.caller, changes_after=page.changes_checkpoint, filters=boundary.filters + ) + + def trusted_publisher_evidence( revision: ReportingRevisionRecord, digest: ReportingCanonicalDigest, From 3c405a21f978ed9d3208611bb4a7a8434a056933 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Wed, 16 Sep 2026 05:50:27 +0000 Subject: [PATCH 4/4] fix(reporting): bind reconciliation evidence in the database and judge snapshot topology Independent adversarial review of the #1167A storage slice reproduced ten in-scope defects. Each is fixed with a regression that fails on the previous head. Buyer selector (`_reconcile._select_current`): - A unique official close masked a broken snapshot history. `current = official or leaves` never inspected snapshot topology, so two unsuperseded snapshot roots, or an A<->B supersession cycle, plus one official evaluated `definitive=True` with no reasons. Snapshot leaves are now counted on their own, and every leaf is walked back through its predecessors so a cycle's members stay unreachable and fail closed. - `native_commit` accepted a resource whose retained descriptor declared `immutability="immutable_location"` whenever the native reference and observation path matched. The seller already refused that; the buyer now mirrors the predicate. - `ReportingRevision` carries no obligation reference, so scanning semantic scope made two legitimate obligations that share a definition, profile, campaign set and period see each other's revisions and both go non-definitive. Candidates now exclude a revision only another obligation has materialized. Revisions nobody materialized -- an unmaterialized official included -- are never excluded, illegal fan-out is still reported, and an unresolvable owner still fails closed rather than guessing. Reference validators (`reporting.evidence`): - `resource_location` accepted Azure-SAS-shaped signed query material because no keyword appeared in it. Any `?name=` / `&name=` query-parameter pair is now refused in every reference field. - Unbounded credential substring matching rejected benign operational names such as `tokenized_inventory_daily`, `secretariat-report-v17` and `authorization_metrics_v2`. A keyword now condemns a value only when it stands alone as a word and introduces something after it. PostgreSQL evidence graph (`reporting_ledger_reconciliation.sql`): - `content_sha256` was only format-checked and the payload shape was open, so ordinary direct SQL -- with no trigger disabled -- retained a credential under an attacker-chosen digest and left that principal's whole feed unreadable. `reporting_canonical_json` now implements the `canonical_json_utf8_v1` profile in SQL, every insert recomputes its own fingerprint, and a closed key allowlist refuses any field outside the frozen record types. - No SQL predicate compared a successful materialization or an accepted receipt against the Core revision it named, so a coherent payload with a true fingerprint committed `row_count=999` and auto-created an accepted terminal head. `reporting_reconciliation_evidence` re-derives the predicate that decides money: revision row count, typed totals and canonical digest; binding format, readers, method, success status, verification path and retention floor; checksum-to-object binding; receipt profile, totals, profile-specific evidence and readable window; and an adjustment digest recomputed from the retained adjustment columns. - A filtered page succeeded while a payload-corrupt row sat inside the frozen boundary but outside the filter. The read integrity scan now covers the caller's whole retained graph, so any mismatch fails every page. Feed boundaries: - A caller's stale or hand-built boundary was reported as `REPORTING_HISTORY_CORRUPT`. Only a boundary this store opened can accuse itself; caller tokens now fail as `INVALID_CHECKPOINT`, indistinguishably from unknown, malformed, overlong, cross-principal, filter-mismatched and ahead tokens. Also adds the missing true concurrent-allocation probe: four order-independent records for one principal commit at once and must take dense unique sequences with a matching caller-local head. Co-Authored-By: Claude Opus 5 (1M context) --- docs/reporting-reconciliation-storage.md | 60 +++- src/adcp/reporting/_reconcile.py | 48 ++- src/adcp/reporting/evidence.py | 40 ++- src/adcp/reporting/ledger/delivery.py | 18 +- src/adcp/reporting/ledger/delivery_pg.py | 22 +- .../reporting_ledger_reconciliation.sql | 283 ++++++++++++++++ .../test_reporting_reconciliation_changes.py | 123 ++++++- .../test_reporting_reconciliation_profiles.py | 75 +++++ .../test_reporting_reconciliation_sql.py | 311 +++++++++++++++++- tests/test_reporting_reconciliation.py | 198 +++++++++++ 10 files changed, 1148 insertions(+), 30 deletions(-) diff --git a/docs/reporting-reconciliation-storage.md b/docs/reporting-reconciliation-storage.md index 26689a090..cb224f679 100644 --- a/docs/reporting-reconciliation-storage.md +++ b/docs/reporting-reconciliation-storage.md @@ -90,6 +90,16 @@ fields reject recognizable credentials and URLs, including encoded forms, withou echoing the input. Native versions remain separate from object keys and are only URI-encoded when a later adapter constructs its provider request. +Credential detection matches *shapes*, not substrings. A keyword such as `token` +or `authorization` only rejects a value when it stands alone as a word and +introduces something after it (`token=…`, `Bearer …`, `authorization: …`), so +ordinary operational names — `tokenized_inventory_daily`, `secretariat-report-v17`, +`authorization_metrics_v2` — persist unchanged. Independently, any +`?name=`/`&name=` query-parameter pair is refused in every reference field, because +presigned S3/GCS URLs and Azure SAS tokens carry their secret in parameters this +boundary cannot enumerate; `report.csv?sv=…&sig=…` never reaches storage even +though no keyword appears in it. + ## Evidence and financial ordering `ReportingRevisionRecord.canonical_content_digest` optionally holds a frozen @@ -221,9 +231,38 @@ of the scoped feed/record join before boundary/count/limit so damage cannot disa through filtering. Stored identity columns are compared against the decoded payload. Updates/deletes and terminal-head replacement are rejected; referenced Core publications stay frozen while leases and readability remain operational. + +The database does not take the retained payload or its fingerprint on trust. +`reporting_canonical_json` implements the `canonical_json_utf8_v1` profile in SQL, so +every insert recomputes `content_sha256` from the stored bytes and a closed key +allowlist refuses any field outside the frozen record types — no credential, +provider response or metadata bag can be retained even by a writer that goes +straight to the tables with correct identity columns. `reporting_reconciliation_evidence` +then re-derives the financial predicate that decides money: a successful +materialization must match its Core revision's row count, typed control totals and +canonical digest, its binding's format, readers, method and success status, and its +retention floor; an accepted revision receipt must match that materialization's +profile, totals and profile-specific evidence inside the resource's readable window; +and an accepted adjustment receipt's digest is recomputed from the retained +adjustment columns. A payload that disagrees with its own fingerprint fails every +read for that principal — including pages whose filter would have skipped the +damaged row — rather than being silently excluded. + This is the transaction in which #1168 can insert its outbox row; there is no adopter callback or after-commit webhook send in this PR. +Cursors and checkpoints bind the feed version, account, consumer, normalized filter +fingerprint, frozen bounds and last key, and a continuation restores its own frozen +boundary before the store looks at today's head, so an authorized inter-page write +is deferred to the next walk instead of invalidating this one. Like Core's cursors +they are opaque but **unsigned**: the store re-validates caller, scope, filter and +bounds on use. A principal can therefore present a boundary it built itself, but it +gains nothing it could not reach through the supported API — a checkpoint at the +current head is exactly the token a completed walk would have issued — and a +boundary whose content disagrees with the retained feed fails as +`INVALID_CHECKPOINT`, never as `REPORTING_HISTORY_CORRUPT`. Only a boundary this +store opened can report retained damage. + ## Migration and operational limits Run `create_schema()` or all four bundled SQL resources in one transaction: @@ -250,7 +289,24 @@ Table/index creation and prerequisite migrations take locks; production-sized duration is not benchmarked. The reference stores load one consumer's retained record set to validate transitions. Feed pages use indexed keyset reads, with scoped integrity/count scans; these reads share the account transaction lock with writes. +The integrity scan recomputes one canonical digest per retained record for the +calling principal, and each insert recomputes its own, so both costs grow with a +principal's history and are not benchmarked at production sizes. Large histories need benchmarks or a conforming replacement store before production rollout. Only SDK store operations -are supported writers; database owners can always circumvent application invariants -by disabling constraints. PostgreSQL connection ownership remains with the adopter. +are supported writers, but ordinary direct SQL — inserts that satisfy every column +constraint without disabling a trigger — can no longer retain extra payload +metadata, a fingerprint its payload denies, or financial evidence the Core revision +denies. A database owner who disables constraints outright can still corrupt the +tables; reads then fail closed for that principal. PostgreSQL connection ownership +remains with the adopter. + +The database canonicalizer sorts object keys by bytes, which equals the JCS +UTF-16 order for the ASCII field names these records use, and reproduces +`datetime.isoformat()` for the recomputed adjustment digest. A payload outside that +domain simply fails its digest instead of being accepted. `ReportingRevision` carries +no obligation reference on the wire, so the buyer selector associates a revision with +an obligation through materialization ownership; two Core-only obligations that share +a definition, profile, campaign set and period still fail closed as ambiguous rather +than guess. An authoritative page-local ownership projection belongs to the status +slice, not to this storage slice. diff --git a/src/adcp/reporting/_reconcile.py b/src/adcp/reporting/_reconcile.py index 69143a092..65c539730 100644 --- a/src/adcp/reporting/_reconcile.py +++ b/src/adcp/reporting/_reconcile.py @@ -437,6 +437,18 @@ def _select_current( ] revision_ids = {item.reporting_revision_id for item in attempts} managed_delivery = obligation.destination_ref is not None + # ``ReportingRevision`` carries no obligation reference, so semantic scope + # alone cannot separate two obligations that legitimately share a definition, + # profile, campaign set and period. A revision some *other* obligation has + # materialized is that obligation's publication. Anything this obligation also + # materialized stays a candidate so illegal fan-out is reported below rather + # than silently narrowed away. Revisions no obligation has materialized -- an + # unmaterialized official included -- are never excluded here. + owned_elsewhere = { + item.reporting_revision_id + for item in ledger.materializations + if item.reporting_obligation_id != obligation.reporting_obligation_id + } - revision_ids candidates = [ item for item in ledger.revisions @@ -444,6 +456,7 @@ def _select_current( _revision_matches_obligation(item, obligation) or (managed_delivery and item.reporting_revision_id in revision_ids) ) + and item.reporting_revision_id not in owned_elsewhere ] receipts = [ item @@ -488,10 +501,25 @@ def _select_current( for item in ledger.materializations ): reasons.append("REVISION_SCOPE_MISMATCH") - if any( - item.supersedes_reporting_revision_id - and item.supersedes_reporting_revision_id not in candidate_ids - for item in candidates + by_id = {item.reporting_revision_id: item for item in candidates} + leaves = [item for item in candidates if item.reporting_revision_id not in superseded] + # Walk every leaf back through its predecessors. A supersession cycle leaves + # its members unreachable, so a broken history cannot hide behind an official + # close the way a leaf-only count would let it. + reachable: set[str] = set() + for leaf in leaves: + node: ReportingRevision | None = leaf + while node is not None and node.reporting_revision_id not in reachable: + reachable.add(node.reporting_revision_id) + predecessor = node.supersedes_reporting_revision_id + node = by_id.get(predecessor) if predecessor else None + if ( + any( + item.supersedes_reporting_revision_id + and item.supersedes_reporting_revision_id not in candidate_ids + for item in candidates + ) + or reachable != candidate_ids ): reasons.append("INCOMPLETE_REVISION_CHAIN") # Publication selection precedes destination selection. An official close @@ -499,9 +527,12 @@ def _select_current( # A newer unmaterialized publication must never reveal an older snapshot as # the current deliverable merely because that snapshot has a ready resource. official = [item for item in candidates if _enum(item.finality) == "official"] - current = official or [ - item for item in candidates if item.reporting_revision_id not in superseded - ] + # Snapshot topology is judged on its own. Selecting the official close must + # never excuse a forked snapshot history the buyer cannot reconcile. + snapshot_leaves = [item for item in leaves if _enum(item.finality) != "official"] + if official and len(snapshot_leaves) > 1: + reasons.append("AMBIGUOUS_REVISION_CHAIN") + current = official or snapshot_leaves if len(current) != 1: reasons.append("MISSING_CURRENT_REVISION" if not current else "AMBIGUOUS_REVISION_CHAIN") return None, None, reasons @@ -597,6 +628,9 @@ def _select_current( evidence = materialization.verification.native_commit_evidence if ( not evidence + # A matching reference and path prove nothing unless the retained + # descriptor itself declares the resource immutable by native version. + or _enum(materialization.resource.immutability) != "native_version" or not materialization.resource.native_version_ref or evidence.native_version_ref != materialization.resource.native_version_ref or _enum(evidence.observed_through) diff --git a/src/adcp/reporting/evidence.py b/src/adcp/reporting/evidence.py index 9281025ac..c555c9c95 100644 --- a/src/adcp/reporting/evidence.py +++ b/src/adcp/reporting/evidence.py @@ -10,6 +10,32 @@ from pydantic import ConfigDict +# Credential *shapes*, never bare substrings. A keyword only condemns a value +# when it stands alone as a word and introduces something after it, so ordinary +# operational names -- ``tokenized_inventory_daily``, ``secretariat-report-v17``, +# ``authorization_metrics_v2`` -- survive while ``token=...`` and ``Bearer x`` +# do not. Narrowing every provider identity to one ASCII token grammar instead +# would reject wire-valid GAM/FreeWheel/warehouse references. +_CREDENTIAL_WORD = ( + r"bearer|password|passwd|secret|secrets|token|signature|credential|credentials|" + r"authorization|private[ _-]?key|api[ _-]?key|access[ _-]?key|access[ _-]?token|" + r"refresh[ _-]?token|client[ _-]?secret|sas[ _-]?token" +) +_CREDENTIAL_ASSIGNMENT = re.compile( + rf"(?i)(? str: """Reject recognizable credentials without normalizing a provider's identity. @@ -31,15 +57,11 @@ def _public_text(value: str, *, maximum: int) -> str: not inspected.isprintable() or "://" in inspected or inspected.startswith("//") - or re.search( - r"(?i)(?:bearer|password|secret|token|signature|credential|private.key|" - r"authorization|api[ _-]?key|access[ _-]?key|-----BEGIN|" - r"(?:^|\s)(?:https?|ftp|file|data|mailto|s3|gs):|" - r"[^\s/:]+:[^\s/]+@)", - inspected, - ) - or re.search(r"(?:^|[^A-Za-z0-9])(?:AKIA|ASIA)[A-Z0-9]{16}\b", inspected) - or re.search(r"\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+", inspected) + or _UNSAFE_SHAPE.search(inspected) + or _CREDENTIAL_ASSIGNMENT.search(inspected) + or _QUERY_PARAMETER.search(inspected) + or _AWS_KEY_ID.search(inspected) + or _JWT.search(inspected) ): raise ValueError("reporting metadata requires non-secret public text") decoded = unquote(inspected) diff --git a/src/adcp/reporting/ledger/delivery.py b/src/adcp/reporting/ledger/delivery.py index 101a06707..1bd338e99 100644 --- a/src/adcp/reporting/ledger/delivery.py +++ b/src/adcp/reporting/ledger/delivery.py @@ -9,7 +9,7 @@ from dataclasses import asdict, dataclass from datetime import datetime -from typing import Any, Protocol, cast, runtime_checkable +from typing import Any, NoReturn, Protocol, cast, runtime_checkable from adcp.reporting.ledger._delivery_state import ( DeliveryContext, @@ -289,6 +289,13 @@ def terminal_acceptances(self) -> tuple[ReportingReceiptKey, ...]: return tuple(item.key for item in self.current_receipts if item.status == "accepted") +def _boundary_unavailable(requested: bool) -> NoReturn: + """Separate a caller's stale/forged boundary from genuine retained damage.""" + if requested: + raise LedgerConflictError("INVALID_CHECKPOINT", "reconciliation boundary is unavailable") + fail("REPORTING_HISTORY_CORRUPT") + + class _ReconciliationOperations: """Typed forwarding shared by the two storage mechanisms; not an adopter hook.""" @@ -469,6 +476,7 @@ async def read_reconciliation_snapshot( caller: ReportingDeliveryPrincipal, boundary: ReportingReconciliationSnapshotToken | None = None, ) -> ReportingReconciliationSnapshot: + requested = boundary is not None async with self._lock: changes = self._caller_changes(caller) if boundary is None: @@ -482,7 +490,10 @@ async def read_reconciliation_snapshot( unavailable() validate_boundary(caller, boundary, len(changes)) if boundary.total_count != boundary.max_sequence: - fail("REPORTING_HISTORY_CORRUPT") + # A caller-presented boundary that disagrees with the retained feed + # is a stale or hand-built token, never evidence that storage is + # corrupt. Only a boundary this store opened can accuse itself. + _boundary_unavailable(requested) return ReportingReconciliationSnapshot( caller, boundary, @@ -499,6 +510,7 @@ async def read_reconciliation_changes( filters: ReportingReconciliationFilter = ReportingReconciliationFilter(), ) -> ReportingReconciliationPage: after, boundary, last_key = read_position(caller, changes_after, cursor, limit, filters) + continued = boundary is not None async with self._lock: records = self._caller_changes(caller) if boundary is None: @@ -527,7 +539,7 @@ async def read_reconciliation_changes( and filters.matches(item.record) ) if len(changes) != boundary.total_count: - fail("REPORTING_HISTORY_CORRUPT") + _boundary_unavailable(continued) return change_page( caller, boundary, diff --git a/src/adcp/reporting/ledger/delivery_pg.py b/src/adcp/reporting/ledger/delivery_pg.py index 0e36673ce..7cdb19617 100644 --- a/src/adcp/reporting/ledger/delivery_pg.py +++ b/src/adcp/reporting/ledger/delivery_pg.py @@ -22,6 +22,7 @@ ) from adcp.reporting.ledger.delivery import ( ReportingReconciliationSnapshot, + _boundary_unavailable, _ReconciliationOperations, ) from adcp.reporting.ledger.delivery_changes import ( @@ -167,24 +168,31 @@ async def _validate_feed( ) -> tuple[int, datetime]: # Scope both sides before the join, count, or boundary. A missing/moved # record or feed row must fail, never disappear through an inner join. + # The payload digest scan covers the caller's whole retained graph, not + # just the rows a requested filter happens to select: a single retained + # payload that disagrees with its fingerprint must fail every page. # The joined SQL fragment is constant; every caller value is parameterized. row = await ( await connection.execute( "SELECT count(c.seq), COALESCE(max(c.seq), 0)," # nosec B608 " COALESCE((SELECT max_sequence FROM reporting_reconciliation_heads" " WHERE account_id = %s AND consumer_id = %s), 0)," - " COALESCE(bool_or(c.seq IS NULL OR r.record_id IS NULL), false), clock_timestamp()" + " COALESCE(bool_or(c.seq IS NULL OR r.record_id IS NULL), false)," + " COALESCE((SELECT bool_or(x.content_sha256" + " <> reporting_payload_sha256(x.payload))" + " FROM reporting_reconciliation_records x" + " WHERE x.account_id = %s AND x.consumer_id = %s), false), clock_timestamp()" " FROM (SELECT * FROM reporting_reconciliation_changes" " WHERE account_id = %s AND consumer_id = %s) c" " FULL JOIN (SELECT * FROM reporting_reconciliation_records" " WHERE account_id = %s AND consumer_id = %s) r ON " + _FEED_JOIN, - (who.account_id, who.consumer_id) * 3, + (who.account_id, who.consumer_id) * 4, ) ).fetchone() assert row is not None - if row[0] != row[1] or row[1] != row[2] or row[3]: + if row[0] != row[1] or row[1] != row[2] or row[3] or row[4]: fail("REPORTING_HISTORY_CORRUPT") - return row[2], row[4] + return row[2], row[5] async def _records( self, connection: Any, who: ReportingDeliveryPrincipal, maximum: int | None = None @@ -334,6 +342,7 @@ async def read_reconciliation_snapshot( caller: ReportingDeliveryPrincipal, boundary: ReportingReconciliationSnapshotToken | None = None, ) -> ReportingReconciliationSnapshot: + requested = boundary is not None async with self._pool.connection() as connection, connection.transaction(): await self._lock_account(connection, caller.account_id) maximum, now = await self._validate_feed(connection, caller) @@ -354,7 +363,8 @@ async def read_reconciliation_snapshot( for item in await self._changes(connection, caller, boundary.max_sequence) ) if len(records) != boundary.total_count: - fail("REPORTING_HISTORY_CORRUPT") + # See _boundary_unavailable: the caller's token, not the store. + _boundary_unavailable(requested) return ReportingReconciliationSnapshot(caller, boundary, records) async def read_reconciliation_changes( @@ -393,7 +403,7 @@ async def read_reconciliation_changes( connection, caller, boundary.min_sequence, boundary.max_sequence, filters ) if count != boundary.total_count: - fail("REPORTING_HISTORY_CORRUPT") + _boundary_unavailable(True) changes = await self._changes( connection, caller, diff --git a/src/adcp/reporting/ledger/reporting_ledger_reconciliation.sql b/src/adcp/reporting/ledger/reporting_ledger_reconciliation.sql index 6ab3aaabf..1f9ecea3b 100644 --- a/src/adcp/reporting/ledger/reporting_ledger_reconciliation.sql +++ b/src/adcp/reporting/ledger/reporting_ledger_reconciliation.sql @@ -229,6 +229,262 @@ BEGIN SELECT encode(sha256(convert_to(value, 'UTF8')), 'hex') $function$; + -- canonical_json_utf8_v1 for the restricted value domain these payloads use: + -- objects, arrays, strings, safe integers, booleans and null. Keys sort by + -- bytes, which equals the JCS UTF-16 order for the ASCII field names the SDK + -- emits; anything else simply fails the digest instead of being blessed. + CREATE OR REPLACE FUNCTION reporting_canonical_json(document JSONB) + RETURNS TEXT LANGUAGE plpgsql IMMUTABLE STRICT AS $function$ + DECLARE + shape TEXT := jsonb_typeof(document); + parts TEXT; + quantity NUMERIC; + BEGIN + IF shape = 'object' THEN + SELECT coalesce(string_agg(to_json(entry.field)::text || ':' + || reporting_canonical_json(entry.nested), ',' ORDER BY entry.field COLLATE "C"), '') + INTO parts + FROM (SELECT e.key AS field, e.value AS nested FROM jsonb_each(document) AS e) AS entry; + RETURN '{' || parts || '}'; + ELSIF shape = 'array' THEN + SELECT coalesce(string_agg(reporting_canonical_json(entry.nested), ',' + ORDER BY entry.position), '') + INTO parts + FROM (SELECT e.value AS nested, e.ordinality AS position + FROM jsonb_array_elements(document) WITH ORDINALITY AS e(value, ordinality)) AS entry; + RETURN '[' || parts || ']'; + ELSIF shape = 'string' THEN + RETURN to_json(document #>> '{}')::text; + ELSIF shape = 'number' THEN + quantity := (document #>> '{}')::numeric; + -- Outside the safe-integer domain canonical_json_utf8_v1 refuses to + -- encode at all, so emit the raw text: the digest then cannot match and + -- the row is refused as inconsistent instead of raising from a read. + IF quantity = trunc(quantity) AND abs(quantity) <= 9007199254740991 THEN + RETURN trunc(quantity)::bigint::text; + END IF; + RETURN document #>> '{}'; + ELSIF shape = 'boolean' THEN + RETURN CASE WHEN (document #>> '{}')::boolean THEN 'true' ELSE 'false' END; + END IF; + RETURN 'null'; + END + $function$; + + CREATE OR REPLACE FUNCTION reporting_payload_sha256(document JSONB) + RETURNS TEXT LANGUAGE SQL IMMUTABLE STRICT AS $function$ + SELECT encode(sha256(convert_to(reporting_canonical_json(document), 'UTF8')), 'hex') + $function$; + + -- Every object key at any depth, so a closed allowlist can refuse arbitrary + -- retained metadata without enumerating one schema per record kind. + CREATE OR REPLACE FUNCTION reporting_payload_keys(document JSONB) + RETURNS SETOF TEXT LANGUAGE plpgsql IMMUTABLE STRICT AS $function$ + DECLARE + entry RECORD; + BEGIN + IF jsonb_typeof(document) = 'object' THEN + FOR entry IN SELECT e.key AS field, e.value AS nested FROM jsonb_each(document) AS e LOOP + RETURN NEXT entry.field; + RETURN QUERY SELECT reporting_payload_keys(entry.nested); + END LOOP; + ELSIF jsonb_typeof(document) = 'array' THEN + FOR entry IN SELECT e.value AS nested FROM jsonb_array_elements(document) AS e LOOP + RETURN QUERY SELECT reporting_payload_keys(entry.nested); + END LOOP; + END IF; + RETURN; + END + $function$; + + -- datetime.isoformat() with '+00:00' spelled 'Z', so a recomputed adjustment + -- digest agrees with the SDK byte-for-byte. + CREATE OR REPLACE FUNCTION reporting_iso_utc(moment TIMESTAMPTZ) + RETURNS TEXT LANGUAGE SQL IMMUTABLE STRICT AS $function$ + SELECT to_char(moment AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS') + || CASE WHEN (date_part('microsecond', moment AT TIME ZONE 'UTC')::bigint % 1000000) = 0 + THEN '' + ELSE '.' || lpad((date_part('microsecond', moment AT TIME ZONE 'UTC')::bigint + % 1000000)::text, 6, '0') + END || 'Z' + $function$; + + -- A retained column holds the wire projection (absent unit, explicit + -- algorithm); a record payload holds the dataclass projection (null unit, no + -- algorithm). Compare the facts they share rather than their spellings. + CREATE OR REPLACE FUNCTION reporting_sorted_totals(document JSONB) + RETURNS JSONB LANGUAGE SQL IMMUTABLE AS $function$ + SELECT coalesce((SELECT jsonb_agg(t ORDER BY t->>'name') + FROM jsonb_array_elements(jsonb_strip_nulls(document)) AS t), '[]'::jsonb) + $function$; + + CREATE OR REPLACE FUNCTION reporting_wire_digest(document JSONB) + RETURNS JSONB LANGUAGE SQL IMMUTABLE AS $function$ + SELECT CASE WHEN jsonb_typeof(document) = 'object' + THEN jsonb_strip_nulls(document) - 'algorithm' END + $function$; + + CREATE OR REPLACE FUNCTION reporting_sorted_strings(document JSONB) + RETURNS JSONB LANGUAGE SQL IMMUTABLE AS $function$ + SELECT coalesce((SELECT jsonb_agg(DISTINCT s ORDER BY s) + FROM jsonb_array_elements_text(document) AS s), '[]'::jsonb) + $function$; + + -- Financial acceptance cannot rest on Python pre-insert checks alone: a + -- coherent dataclass payload with a correct fingerprint would otherwise let + -- ordinary direct SQL commit a successful materialization or an accepted + -- terminal receipt whose totals, digest, format or evidence disagree with the + -- Core revision it claims. These predicates mirror _verify_materialization + -- and _verify_receipt for exactly the content that decides money. + CREATE OR REPLACE FUNCTION reporting_reconciliation_evidence(r reporting_reconciliation_records) + RETURNS VOID LANGUAGE plpgsql AS $function$ + DECLARE + binding JSONB; + delivery JSONB; + outcome JSONB; + expected_rows BIGINT; + expected_totals JSONB; + expected_digest JSONB; + observed_digest JSONB; + adjustment reporting_adjustments; + BEGIN + IF r.record_kind NOT IN ('materialization', 'revision_receipt', 'adjustment_receipt') + OR (r.record_kind = 'materialization' + AND r.payload->>'status' NOT IN ('available', 'delivered')) + OR (r.receipt_status IS NOT NULL AND r.receipt_status <> 'accepted') THEN + RETURN; + END IF; + SELECT v.row_count, v.managed_control_totals, v.canonical_content_digest + INTO expected_rows, expected_totals, expected_digest + FROM reporting_revisions v + WHERE (v.account_id, v.reporting_obligation_id, v.reporting_revision_id) + = (r.account_id, r.reporting_obligation_id, r.reporting_revision_id); + IF r.record_kind = 'adjustment_receipt' THEN + SELECT a.* INTO adjustment FROM reporting_adjustments a + WHERE (a.account_id, a.adjusts_reporting_revision_id, a.reporting_adjustment_id) + = (r.account_id, r.reporting_revision_id, r.reporting_adjustment_id); + IF adjustment.reporting_adjustment_id IS NULL + OR adjustment.managed_control_total_deltas IS NULL + OR jsonb_array_length(adjustment.managed_control_total_deltas) = 0 + OR r.payload->>'observed_adjustment_sha256' IS DISTINCT FROM reporting_payload_sha256( + jsonb_build_object( + 'reporting_adjustment_id', adjustment.reporting_adjustment_id, + 'adjusts_reporting_revision_id', adjustment.adjusts_reporting_revision_id, + 'reason_code', adjustment.reason_code, + 'accounting_period', jsonb_build_object( + 'start', reporting_iso_utc(adjustment.accounting_period_start), + 'end', reporting_iso_utc(adjustment.accounting_period_end)), + 'control_total_deltas', adjustment.managed_control_total_deltas, + 'correction_observed_at', reporting_iso_utc(adjustment.correction_observed_at), + 'created_at', reporting_iso_utc(adjustment.created_at)) + || CASE WHEN adjustment.reason_detail IS NOT NULL + THEN jsonb_build_object('reason_detail', adjustment.reason_detail) + ELSE '{}'::jsonb END) THEN + RAISE EXCEPTION 'reporting adjustment acceptance is inconsistent' USING ERRCODE = '23514'; + END IF; + RETURN; + END IF; + IF expected_totals IS NULL OR expected_rows IS NULL THEN + RAISE EXCEPTION 'reporting revision evidence is unavailable' USING ERRCODE = '23514'; + END IF; + IF r.record_kind = 'revision_receipt' THEN + SELECT m.payload INTO outcome FROM reporting_reconciliation_records m + WHERE (m.account_id, m.consumer_id, m.delivery_config_id, m.delivery_config_version, + m.reporting_obligation_id, m.reporting_materialization_id, m.reporting_revision_id) + = (r.account_id, r.consumer_id, r.delivery_config_id, r.delivery_config_version, + r.reporting_obligation_id, r.reporting_materialization_id, r.reporting_revision_id) + AND m.record_kind = 'materialization' + AND m.payload->>'status' IN ('available', 'delivered'); + observed_digest := reporting_wire_digest(r.payload->'observed_canonical_content_digest'); + expected_digest := reporting_wire_digest(expected_digest); + IF outcome IS NULL + OR r.payload->>'verification_profile' + IS DISTINCT FROM outcome#>>'{verification,verification_profile}' + OR (r.payload->>'observed_row_count')::bigint IS DISTINCT FROM expected_rows + OR reporting_sorted_totals(r.payload->'observed_control_totals') + IS DISTINCT FROM reporting_sorted_totals(outcome#>'{verification,control_totals}') + OR (observed_digest IS NOT NULL AND observed_digest IS DISTINCT FROM expected_digest) + OR (nullif(r.payload->>'observed_manifest_sha256', '') IS NOT NULL + AND r.payload->>'observed_manifest_sha256' + IS DISTINCT FROM outcome#>>'{resource,manifest_sha256}') + OR (nullif(r.payload->>'observed_native_version_ref', '') IS NOT NULL + AND r.payload->>'observed_native_version_ref' + IS DISTINCT FROM outcome#>>'{resource,native_version_ref}') + OR (r.payload->>'verification_profile' = 'canonical_digest' + AND (observed_digest IS NULL OR observed_digest IS DISTINCT FROM expected_digest)) + OR (r.payload->>'verification_profile' = 'manifest_checksums' + AND r.payload->>'observed_manifest_sha256' + IS DISTINCT FROM outcome#>>'{resource,manifest_sha256}') + OR (r.payload->>'verification_profile' = 'native_commit' + AND r.payload->>'observed_native_version_ref' + IS DISTINCT FROM outcome#>>'{resource,native_version_ref}') + OR (r.payload->>'observed_at')::timestamptz < (outcome->>'completed_at')::timestamptz + OR (r.payload->>'observed_at')::timestamptz + >= (outcome#>>'{resource,expires_at}')::timestamptz THEN + RAISE EXCEPTION 'reporting receipt acceptance is inconsistent' USING ERRCODE = '23514'; + END IF; + RETURN; + END IF; + SELECT b.payload INTO binding FROM reporting_reconciliation_records b + WHERE (b.account_id, b.consumer_id, b.delivery_config_id, b.delivery_config_version) + = (r.account_id, r.consumer_id, r.delivery_config_id, r.delivery_config_version) + AND b.record_kind = 'destination_binding'; + SELECT d.payload INTO delivery FROM reporting_reconciliation_records d + WHERE (d.account_id, d.consumer_id, d.delivery_config_id, d.delivery_config_version, + d.reporting_obligation_id) + = (r.account_id, r.consumer_id, r.delivery_config_id, r.delivery_config_version, + r.reporting_obligation_id) + AND d.record_kind = 'obligation_delivery'; + observed_digest := reporting_wire_digest(r.payload#>'{verification,canonical_content_digest}'); + expected_digest := reporting_wire_digest(expected_digest); + IF binding IS NULL OR delivery IS NULL + OR (r.payload#>>'{verification,row_count}')::bigint IS DISTINCT FROM expected_rows + OR reporting_sorted_totals(r.payload#>'{verification,control_totals}') + IS DISTINCT FROM reporting_sorted_totals(expected_totals) + OR (observed_digest IS NOT NULL AND observed_digest IS DISTINCT FROM expected_digest) + OR (r.payload#>>'{verification,verification_profile}' = 'canonical_digest' + AND (observed_digest IS NULL OR observed_digest IS DISTINCT FROM expected_digest)) + OR r.payload#>>'{verification,verified_at}' IS DISTINCT FROM r.payload->>'completed_at' + OR nullif(r.payload#>'{verification,verified_format}', 'null'::jsonb) + IS DISTINCT FROM nullif(binding->'format', 'null'::jsonb) + OR r.payload#>'{resource,reader_compatibility}' + IS DISTINCT FROM coalesce(binding->'reader_compatibility', '[]'::jsonb) + OR r.payload#>>'{resource,kind}' IS DISTINCT FROM (CASE binding->>'method' + WHEN 'file_transfer' THEN 'manifest' WHEN 'dataset_share' THEN 'dataset' + ELSE 'warehouse_relation' END) + OR (binding->>'method' = 'dataset_share' + AND r.payload#>>'{verification,verification_path}' <> 'representative_consumer') + OR (binding->>'method' = 'warehouse_materialization' + AND (r.payload#>>'{verification,verification_path}' <> 'destination' + OR r.payload->>'status' <> 'delivered')) + OR (r.payload->>'status' = 'delivered' + AND r.payload#>>'{verification,verification_path}' <> 'destination') + OR (binding->>'method' = 'file_transfer' AND ( + jsonb_array_length(coalesce(r.payload#>'{resource,object_refs}', '[]'::jsonb)) = 0 + OR jsonb_array_length( + coalesce(r.payload#>'{verification,physical_checksums}', '[]'::jsonb)) = 0)) + OR (r.payload#>>'{resource,expires_at}')::timestamptz < greatest( + (delivery->>'resource_retained_until')::timestamptz, + (r.payload->>'completed_at')::timestamptz + + ((binding->>'resource_retention_days') || ' days')::interval) THEN + RAISE EXCEPTION 'reporting materialization evidence is inconsistent' USING ERRCODE = '23514'; + END IF; + -- Every checksum names a retained object, whatever the method; file + -- transfer must additionally cover all of them. + IF EXISTS ( + SELECT 1 FROM jsonb_array_elements( + coalesce(r.payload#>'{verification,physical_checksums}', '[]'::jsonb)) AS c + WHERE NOT coalesce(r.payload#>'{resource,object_refs}', '[]'::jsonb) + @> jsonb_build_array(c->'object_ref')) + OR (binding->>'method' = 'file_transfer' AND reporting_sorted_strings( + (SELECT jsonb_agg(c->'object_ref') + FROM jsonb_array_elements(r.payload#>'{verification,physical_checksums}') AS c)) + IS DISTINCT FROM reporting_sorted_strings(r.payload#>'{resource,object_refs}')) THEN + RAISE EXCEPTION 'reporting physical checksum binding is inconsistent' USING ERRCODE = '23514'; + END IF; + END + $function$; + CREATE OR REPLACE FUNCTION reporting_reconciliation_validate(r reporting_reconciliation_records) RETURNS VOID LANGUAGE plpgsql AS $function$ DECLARE @@ -238,6 +494,32 @@ BEGIN expected_namespace TEXT; parent_kind TEXT; BEGIN + -- The retained fingerprint must be a fact about the stored bytes, and the + -- payload must be closed. Without both, ordinary direct SQL can persist a + -- provider blob or credential under an attacker-chosen digest and leave the + -- principal's whole feed unreadable. Never echo the offending value. + IF r.content_sha256 IS DISTINCT FROM reporting_payload_sha256(r.payload) + OR EXISTS (SELECT 1 FROM reporting_payload_keys(r.payload) AS field + WHERE field <> ALL (ARRAY[ + 'account_id', 'adjusts_reporting_revision_id', 'algorithm', 'attempt', + 'canonical_content_digest', 'canonicalization_id', 'canonicalization_sha256', + 'canonicalization_uri', 'check_id', 'checked_at', 'completed_at', + 'consumer_commit_ref', 'consumer_id', 'control_totals', 'created_at', 'currency', + 'delivery_config_id', 'delivery_config_version', 'destination_ref', 'expires_at', + 'failure_code', 'feed_purpose', 'format', 'generation_key', 'immutability', 'kind', + 'location', 'manifest_sha256', 'method', 'name', 'native_observed_through', + 'native_version_ref', 'object_ref', 'object_refs', 'observed_adjustment_sha256', + 'observed_at', 'observed_canonical_content_digest', 'observed_control_totals', + 'observed_manifest_sha256', 'observed_native_version_ref', 'observed_row_count', + 'physical_checksums', 'reader_compatibility', 'received_at', 'reconciliation_mode', + 'rejection_codes', 'reporting_adjustment_id', 'reporting_materialization_id', + 'reporting_obligation_id', 'reporting_receipt_id', 'reporting_revision_id', 'resource', + 'resource_ref', 'resource_retained_until', 'resource_retention_days', 'row_count', + 'scope', 'state', 'status', 'success_status', 'supersedes_reporting_receipt_id', + 'transport', 'trusted_binding_ref', 'unit', 'value', 'value_type', 'verification', + 'verification_path', 'verification_profile', 'verified_at', 'verified_format'])) THEN + RAISE EXCEPTION 'reporting payload is not closed retained evidence' USING ERRCODE = '23514'; + END IF; generation := jsonb_build_object('account_id', r.account_id, 'delivery_config_id', r.delivery_config_id, 'delivery_config_version', r.delivery_config_version); @@ -365,6 +647,7 @@ BEGIN AND p.record_id = r.supersedes_receipt_id AND p.receipt_status = 'rejected') THEN RAISE EXCEPTION 'reporting receipt predecessor is unavailable' USING ERRCODE = '23514'; END IF; + PERFORM reporting_reconciliation_evidence(r); END $function$; diff --git a/tests/conformance/reporting/test_reporting_reconciliation_changes.py b/tests/conformance/reporting/test_reporting_reconciliation_changes.py index 8c66a64f9..8a1bbc4e5 100644 --- a/tests/conformance/reporting/test_reporting_reconciliation_changes.py +++ b/tests/conformance/reporting/test_reporting_reconciliation_changes.py @@ -1,7 +1,8 @@ """Independent scoped checkpoints cannot lose records to Core or to interleaving writes.""" +import asyncio from dataclasses import replace -from datetime import timedelta +from datetime import datetime, timedelta from typing import get_args import pytest @@ -13,16 +14,20 @@ ReportingAdjustmentRecord, ReportingControlTotalRecord, ReportingDeliveryPrincipal, + ReportingMaterializationAttempt, ReportingMaterializationCheck, ReportingReconciliationCheckpoint, ReportingReconciliationCursor, ReportingReconciliationFeedStore, ReportingReconciliationFilter, + ReportingReconciliationSnapshotToken, ReportingReconciliationStore, ReportingStatusCaller, ReportingStatusHandler, adjustment_to_wire, + revision_content_sha256, ) +from adcp.reporting.ledger.delivery_changes import change_boundary from adcp.reporting.ledger.models import LedgerRecordKind from adcp.reporting.ledger.store import decode_cursor, encode_cursor @@ -437,3 +442,119 @@ async def test_position_types_and_unissued_boundaries_cannot_be_interchanged( with pytest.raises(LedgerConflictError) as error: await store.read_reconciliation_snapshot(caller=caller, boundary=future) assert error.value.code == "INVALID_CHECKPOINT" + + +@pytest.mark.parametrize("token", ["cursor", "snapshot"]) +async def test_a_callers_stale_boundary_is_a_token_error_not_retained_damage( + reconciliation_store: tuple[Store, Clock], token: str +) -> None: + """A hand-built or stale boundary must never accuse the store of corruption.""" + store, clock = reconciliation_store + s = await scenario(store) + clock.now = END + timedelta(seconds=10) + await store.commit_materialization(s.outcome) + caller = s.binding.principal + page = await store.read_reconciliation_changes(caller=caller, limit=1) + assert page.cursor is not None + decoded = decode_cursor(page.cursor) + if token == "cursor": + decoded["count"] = decoded["count"] - 1 + decoded["snapshot"] = change_boundary( + caller, + decoded["through"], + datetime.fromisoformat(decoded["as_of"]), + after=decoded["after"], + total_count=decoded["count"], + ).snapshot_id + with pytest.raises(LedgerConflictError) as error: + await store.read_reconciliation_changes( + caller=caller, cursor=ReportingReconciliationCursor(encode_cursor(decoded)), limit=1 + ) + else: + as_of = datetime.fromisoformat(decoded["as_of"]) + short = decoded["through"] - 1 + forged = ReportingReconciliationSnapshotToken( + caller, + change_boundary(caller, decoded["through"], as_of, total_count=short).snapshot_id, + as_of, + 0, + decoded["through"], + short, + ReportingReconciliationFilter(), + ) + with pytest.raises(LedgerConflictError) as error: + await store.read_reconciliation_snapshot(caller=caller, boundary=forged) + assert error.value.code == "INVALID_CHECKPOINT" + # The retained feed itself is untouched and still walks completely. + assert len((await store.read_reconciliation_changes(caller=caller)).changes) == 4 + assert len((await store.read_reconciliation_snapshot(caller=caller)).records) == 4 + + +async def test_distinct_concurrent_writes_take_dense_unique_feed_sequences( + reconciliation_store: tuple[Store, Clock], +) -> None: + """Order-independent records for one principal must not collide or lose a sequence.""" + store, clock = reconciliation_store + s = await scenario(store) + clock.now = END + timedelta(seconds=60) + await store.commit_materialization(s.outcome) + rows = ( + await store.read_revision_rows( + account_id="acct_a", reporting_revision_id=s.revision.reporting_revision_id + ) + ).rows + attempts = [] + for index in range(4): + identifier = f"concurrent-revision-{index}" + revision = replace( + s.revision, + reporting_revision_id=identifier, + finality="snapshot", + finality_basis=None, + finality_policy_id=None, + finalized_at=None, + revision_content_sha256=revision_content_sha256( + reporting_revision_id=identifier, + row_count=s.revision.row_count, + control_totals=s.revision.control_totals, + reporting_rows=rows, + control_total_evidence=s.revision.managed_control_totals, + ), + ) + await store.commit_revision(revision, rows) + # Each attempt is the first for its own revision, so validity does not + # depend on which of them the store happens to serialize first. + attempts.append( + replace( + s.attempt, + reporting_revision_id=identifier, + reporting_materialization_id=f"concurrent-materialization-{index}", + attempt=1, + ) + ) + results = await asyncio.gather( + *(store.commit_materialization_attempt(item) for item in attempts) + ) + assert all(created for _, created in results) + page = await store.read_reconciliation_changes(caller=s.binding.principal, limit=1000) + sequences = [item.sequence for item in page.changes] + assert sequences == list(range(1, 4 + len(attempts) + 1)) + assert page.total_count == len(page.changes) == 4 + len(attempts) + assert page.changes_checkpoint is not None + assert { + item.record.reporting_materialization_id + for item in page.changes + if isinstance(item.record, ReportingMaterializationAttempt) + } == {item.reporting_materialization_id for item in attempts} | { + s.attempt.reporting_materialization_id + } + if isinstance(store, PgReportingReconciliationStore): + async with store._pool.connection() as connection: + head = await ( + await connection.execute( + "SELECT max_sequence FROM reporting_reconciliation_heads" + " WHERE account_id = %s AND consumer_id = %s", + (s.binding.principal.account_id, s.binding.principal.consumer_id), + ) + ).fetchone() + assert head is not None and head[0] == len(page.changes) diff --git a/tests/conformance/reporting/test_reporting_reconciliation_profiles.py b/tests/conformance/reporting/test_reporting_reconciliation_profiles.py index 85e1ae94d..3d12a067e 100644 --- a/tests/conformance/reporting/test_reporting_reconciliation_profiles.py +++ b/tests/conformance/reporting/test_reporting_reconciliation_profiles.py @@ -170,3 +170,78 @@ async def test_provider_fields_reject_credentials_and_urls_without_echo( }, ) assert unsafe not in str(error.value) and "DO_NOT_RETAIN" not in str(error.value) + + +@pytest.mark.parametrize( + "field", + [ + "location", + "object_refs", + "native_version_ref", + "consumer_commit_ref", + "reader_compatibility", + ], +) +@pytest.mark.parametrize( + "signed", + [ + "container/report.csv?sv=2024-11-04&sp=r&se=2027-01-01&sig=DO_NOT_RETAIN", + "export.parquet?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=DO_NOT_RETAIN", + "object?GoogleAccessId=DO_NOT_RETAIN&Expires=1893456000", + "share&Signature=DO_NOT_RETAIN", + ], +) +async def test_signed_query_material_is_refused_without_a_recognizable_keyword( + reconciliation_store: tuple[Store, Clock], field: str, signed: str +) -> None: + """Presigned/SAS query material must never persist, keyword or not.""" + store, _ = reconciliation_store + s = await scenario(store) + with pytest.raises(ValueError) as error: + if field == "consumer_commit_ref": + replace(s.receipt, consumer_commit_ref=signed) + else: + replace( + s.outcome.resource, + **{ + field: ( + (signed.replace("?", "-"),) + if field in {"object_refs", "reader_compatibility"} + else signed + ) + }, + ) + assert signed not in str(error.value) and "DO_NOT_RETAIN" not in str(error.value) + + +@pytest.mark.parametrize( + "benign", + [ + "warehouse/tokenized_inventory_daily/part-000.jsonl", + "secretariat-report-v17/part-000.jsonl", + "authorization_metrics_v2/part-000.jsonl", + "credentials_review_2026/part-000.jsonl", + "Parquet & decimal=38 / secretariat.jsonl", + ], +) +async def test_operational_names_that_merely_contain_credential_words_survive( + reconciliation_store: tuple[Store, Clock], benign: str +) -> None: + """A keyword only condemns a value when it stands alone and introduces one.""" + store, _ = reconciliation_store + s = await scenario(store) + resource = replace(s.outcome.resource, location=benign, object_refs=(benign,)) + outcome = replace( + s.outcome, + resource=resource, + verification=replace( + s.outcome.verification, + physical_checksums=(ReportingPhysicalChecksum(benign, "sha256", "d" * 64),), + ), + ) + stored, created = await store.commit_materialization(outcome) + assert created and stored.resource is not None + assert stored.resource.location == benign and stored.resource.object_refs == (benign,) + view = await store.get_materialization(s.attempt.key) + assert isinstance(view, ReportingMaterializationView) + assert ReportingMaterialization.model_validate(view.to_wire()).resource.location == benign diff --git a/tests/conformance/reporting/test_reporting_reconciliation_sql.py b/tests/conformance/reporting/test_reporting_reconciliation_sql.py index eae0224ec..e465155a5 100644 --- a/tests/conformance/reporting/test_reporting_reconciliation_sql.py +++ b/tests/conformance/reporting/test_reporting_reconciliation_sql.py @@ -3,12 +3,16 @@ from __future__ import annotations import asyncio +import re +import types from dataclasses import dataclass, replace from datetime import timedelta +from importlib.resources import files from typing import Any import pytest +from adcp.reporting.canonical_json import canonical_json_utf8_v1 from adcp.reporting.ledger import ( LedgerConflictError, PgReportingReconciliationStore, @@ -22,9 +26,17 @@ adjustment_to_wire, revision_content_sha256, ) -from adcp.reporting.ledger._delivery_state import change_id, fingerprint, payload, storage_identity +from adcp.reporting.ledger._delivery_state import ( + change_id, + fingerprint, + iso, + payload, + storage_identity, +) from adcp.reporting.ledger.delivery_pg import _IDENTITY_COLUMNS +RESOURCES = files("adcp.reporting.ledger") + from ._generation_support import END, NOW, configuration, isolated_reporting_pool from ._reconciliation_support import Scenario, scenario @@ -101,6 +113,7 @@ class Graph: second: Scenario adjustment: ReportingAdjustmentReceiptRecord other_adjustment: ReportingAdjustmentReceiptRecord + adjustment_record: ReportingAdjustmentRecord @pytest.fixture @@ -181,6 +194,7 @@ async def graph(): replace(s.attempt, reporting_materialization_id="pending-attempt", attempt=2) ) adjustments = [] + records = [] for index, candidate in enumerate((s, second)): adjustment = ReportingAdjustmentRecord( f"adjustment-{index}", @@ -197,6 +211,7 @@ async def graph(): ), ) await store.commit_adjustment(adjustment) + records.append(adjustment) adjustments.append( ReportingAdjustmentReceiptRecord( candidate.attempt.scope, @@ -208,7 +223,7 @@ async def graph(): END + timedelta(seconds=7), ) ) - yield Graph(pool, store, s, second, *adjustments) + yield Graph(pool, store, s, second, *adjustments, records[0]) @pytest.mark.parametrize( @@ -621,3 +636,295 @@ async def test_sql_cannot_advance_move_or_erase_the_feed_without_exact_evidence( async with graph.pool.connection() as connection, connection.transaction(): await connection.execute(statement) assert await graph.store.read_reconciliation_changes(caller=caller) == before + + +@pytest.mark.parametrize( + "extra", + [ + {"credential": "MUST_NOT_RETAIN"}, + {"metadata": {"upstream.api_token": "MUST_NOT_RETAIN"}}, + {"provider_response": ["MUST_NOT_RETAIN"]}, + ], +) +async def test_ordinary_direct_sql_cannot_retain_extra_payload_metadata( + graph: Graph, extra: dict[str, Any] +) -> None: + """No trigger is disabled here: a closed payload is a database invariant.""" + from psycopg import IntegrityError + from psycopg.types.json import Jsonb + + record = ReportingMaterializationCheck( + graph.first.attempt.scope, + graph.first.attempt.reporting_materialization_id, + "check-poisoned", + "readable", + END + timedelta(seconds=5), + ) + poisoned = {**payload(record), **extra} + caller = graph.first.binding.principal + before = await graph.store.read_reconciliation_changes(caller=caller) + for digest in (fingerprint(record), "0" * 64): + with pytest.raises(IntegrityError) as error: + await raw_insert(graph.pool, record, payload=Jsonb(poisoned), content_sha256=digest) + assert "MUST_NOT_RETAIN" not in str(error.value) + async with graph.pool.connection() as connection: + assert ( + await ( + await connection.execute( + "SELECT count(*) FROM reporting_reconciliation_records" + " WHERE record_id = 'check-poisoned'" + ) + ).fetchone() + )[0] == 0 + assert await graph.store.read_reconciliation_changes(caller=caller) == before + + +async def test_ordinary_direct_sql_cannot_retain_a_payload_its_digest_denies( + graph: Graph, +) -> None: + from psycopg import IntegrityError + from psycopg.types.json import Jsonb + + record = ReportingMaterializationCheck( + graph.first.attempt.scope, + graph.first.attempt.reporting_materialization_id, + "check-mismatched", + "readable", + END + timedelta(seconds=5), + ) + honest = payload(record) + with pytest.raises(IntegrityError): + await raw_insert(graph.pool, record, content_sha256="0" * 64) + with pytest.raises(IntegrityError): + await raw_insert(graph.pool, record, payload=Jsonb({**honest, "state": "corrupt"})) + assert (await graph.store.record_materialization_check(record))[1] + + +@pytest.mark.parametrize( + "damage", + [ + "verification_row_count", + "verification_totals", + "verification_digest", + "verification_format", + "resource_retention", + "checksum_object_ref", + ], +) +async def test_direct_sql_cannot_commit_a_successful_outcome_the_revision_denies( + graph: Graph, damage: str +) -> None: + """A coherent dataclass payload with a true fingerprint is not enough.""" + from psycopg import IntegrityError + + from adcp.reporting.ledger import ReportingCanonicalDigest + + # 'pending-attempt' is retained with no outcome, so every case below is a + # first write for that materialization rather than a duplicate. + honest = replace(graph.first.outcome, reporting_materialization_id="pending-attempt") + verification, resource = honest.verification, honest.resource + assert verification is not None and resource is not None + if damage == "verification_row_count": + outcome = replace(honest, verification=replace(verification, row_count=999)) + elif damage == "verification_totals": + outcome = replace( + honest, + verification=replace( + verification, + control_totals=tuple( + replace(item, value="99999") if item.name == "spend" else item + for item in verification.control_totals + ), + ), + ) + elif damage == "verification_digest": + outcome = replace( + honest, + verification=replace( + verification, + canonical_content_digest=ReportingCanonicalDigest( + "a" * 64, + "rows-v1", + "https://contracts.example.test/rows-v1.json", + "b" * 64, + ), + ), + ) + elif damage == "verification_format": + outcome = replace(honest, verification=replace(verification, verified_format="csv")) + elif damage == "checksum_object_ref": + outcome = replace( + honest, + resource=replace(resource, object_refs=(*resource.object_refs, "reports/extra.jsonl")), + ) + else: + outcome = replace( + honest, + resource=replace(resource, expires_at=honest.completed_at + timedelta(days=1)), + ) + with pytest.raises(IntegrityError): + await raw_insert(graph.pool, outcome) + assert (await graph.store.commit_materialization(honest))[1] + + +@pytest.mark.parametrize( + "damage", + ["observed_row_count", "observed_totals", "observed_digest", "observed_profile"], +) +async def test_direct_sql_cannot_accept_a_receipt_its_materialization_denies( + graph: Graph, damage: str +) -> None: + from psycopg import IntegrityError + + from adcp.reporting.ledger import ReportingCanonicalDigest + + s = graph.first + if damage == "observed_row_count": + receipt = replace(s.receipt, observed_row_count=999) + elif damage == "observed_totals": + receipt = replace( + s.receipt, + observed_control_totals=tuple( + replace(item, value="99999") if item.name == "spend" else item + for item in s.receipt.observed_control_totals + ), + ) + elif damage == "observed_digest": + receipt = replace( + s.receipt, + observed_canonical_content_digest=ReportingCanonicalDigest( + "a" * 64, "rows-v1", "https://contracts.example.test/rows-v1.json", "b" * 64 + ), + ) + else: + receipt = replace(s.receipt, verification_profile="manifest_checksums") + receipt = replace(receipt, reporting_receipt_id="receipt-forged-0001") + with pytest.raises(IntegrityError): + await raw_insert(graph.pool, receipt) + async with graph.pool.connection() as connection: + assert not await ( + await connection.execute("SELECT 1 FROM reporting_receipt_heads") + ).fetchall() + assert (await graph.store.record_revision_receipt(s.receipt))[1] + + +async def test_direct_sql_cannot_accept_an_adjustment_digest_the_database_recomputes( + graph: Graph, +) -> None: + """The adjustment digest is recomputed from retained columns, not trusted.""" + from psycopg import IntegrityError + + forged = replace( + graph.adjustment, + reporting_receipt_id="receipt-forged-adjust1", + observed_adjustment_sha256="a" * 64, + ) + with pytest.raises(IntegrityError): + await raw_insert(graph.pool, forged) + assert (await graph.store.record_adjustment_receipt(graph.adjustment))[1] + + +async def test_database_canonical_digest_agrees_with_the_sdk_encoding(graph: Graph) -> None: + """Byte parity for the SQL JCS profile, including a recomputed adjustment digest.""" + from psycopg.types.json import Jsonb + + records: list[ReportingDeliveryRecord] = [ + graph.first.binding, + graph.first.delivery, + graph.first.attempt, + graph.first.outcome, + graph.first.receipt, + graph.adjustment, + ] + async with graph.pool.connection() as connection: + for record in records: + document = payload(record) + row = await ( + await connection.execute( + "SELECT reporting_canonical_json(%s::jsonb)," + " reporting_payload_sha256(%s::jsonb)", + (Jsonb(document), Jsonb(document)), + ) + ).fetchone() + assert row is not None + assert row[0].encode() == canonical_json_utf8_v1(document) + assert row[1] == fingerprint(record) + wire = adjustment_to_wire(graph.adjustment_record) + row = await ( + await connection.execute( + "SELECT reporting_payload_sha256(jsonb_build_object(" + " 'reporting_adjustment_id', a.reporting_adjustment_id," + " 'adjusts_reporting_revision_id', a.adjusts_reporting_revision_id," + " 'reason_code', a.reason_code," + " 'accounting_period', jsonb_build_object(" + " 'start', reporting_iso_utc(a.accounting_period_start)," + " 'end', reporting_iso_utc(a.accounting_period_end))," + " 'control_total_deltas', a.managed_control_total_deltas," + " 'correction_observed_at', reporting_iso_utc(a.correction_observed_at)," + " 'created_at', reporting_iso_utc(a.created_at))" + " || CASE WHEN a.reason_detail IS NOT NULL" + " THEN jsonb_build_object('reason_detail', a.reason_detail)" + " ELSE '{}'::jsonb END)" + " FROM reporting_adjustments a WHERE a.reporting_adjustment_id = %s", + (graph.adjustment_record.reporting_adjustment_id,), + ) + ).fetchone() + assert row is not None and row[0] == wire["canonical_adjustment_sha256"] + + +@pytest.mark.parametrize( + "moment", + [ + "2026-09-01T01:00:00+00:00", + "2026-09-01T01:02:03.123456+00:00", + "2026-09-01T01:02:03.000123+00:00", + "2026-09-01T01:02:03.100000+00:00", + "2026-12-31T23:59:59.999999+05:30", + ], +) +async def test_database_timestamp_encoding_matches_the_sdk_for_subsecond_evidence( + graph: Graph, moment: str +) -> None: + """The recomputed adjustment digest depends on this byte-for-byte.""" + from datetime import datetime + + parsed = datetime.fromisoformat(moment) + async with graph.pool.connection() as connection: + row = await ( + await connection.execute("SELECT reporting_iso_utc(%s::timestamptz)", (parsed,)) + ).fetchone() + assert row is not None and row[0] == iso(parsed) + + +async def test_closed_payload_allowlist_covers_every_retained_record_field() -> None: + """Drift guard: a new retained field must be added to the database allowlist.""" + import typing + from dataclasses import fields, is_dataclass + + from adcp.reporting.ledger import delivery_models + + seen: set[type] = set() + expected: set[str] = set() + + def walk(cls: type) -> None: + if cls in seen or not is_dataclass(cls): + return + seen.add(cls) + hints = typing.get_type_hints(cls) + for item in fields(cls): + expected.add(item.name) + stack = [hints[item.name]] + while stack: + annotation = stack.pop() + origin = typing.get_origin(annotation) + if origin in (types.UnionType, typing.Union) or origin is tuple: + stack.extend(a for a in typing.get_args(annotation) if a is not Ellipsis) + elif is_dataclass(annotation): + walk(annotation) + + for record in typing.get_args(delivery_models.ReportingDeliveryRecord): + walk(record) + ddl = RESOURCES.joinpath("reporting_ledger_reconciliation.sql").read_text() + body = ddl.split("WHERE field <> ALL (ARRAY[", 1)[1].split("])", 1)[0] + declared = set(re.findall(r"'([a-z0-9_]+)'", body)) + assert declared == expected diff --git a/tests/test_reporting_reconciliation.py b/tests/test_reporting_reconciliation.py index b60e6e25a..76a61fbdf 100644 --- a/tests/test_reporting_reconciliation.py +++ b/tests/test_reporting_reconciliation.py @@ -1333,6 +1333,204 @@ def test_current_publication_is_selected_before_its_materialization( assert not result.definitive +def _snapshot_revision(identifier: str, supersedes: str | None = None) -> dict[str, object]: + revision = deepcopy(REVISION) + revision.update(reporting_revision_id=identifier, finality="snapshot") + for key in ("finality_basis", "finality_policy_id", "finalized_at"): + revision.pop(key, None) + if supersedes is not None: + revision["supersedes_reporting_revision_id"] = supersedes + return revision + + +def _ledger_from(raw: dict[str, object]) -> ReportingLedger: + response = GetReportingStatusResponse.model_validate(raw) + return ReportingLedger( + response.ledger_snapshot_id, + response.ledger_as_of, + response.account_id, + response.scope, + response.periods, + response.revisions, + response.materializations, + response.receipts, + ) + + +@pytest.mark.parametrize( + "topology,reason", + [("fork", "AMBIGUOUS_REVISION_CHAIN"), ("cycle", "INCOMPLETE_REVISION_CHAIN")], +) +def test_official_close_cannot_mask_a_broken_snapshot_history(topology: str, reason: str) -> None: + """Snapshot topology is judged on its own, not skipped once an official exists.""" + raw = _response() + raw["periods"][0].update( + revision_count=3, + reconciliation_mode="delivery_only", + reconciliation_status="not_required", + health="complete", + required_finality="official", + materialization_count=1, + successful_materialization_count=1, + ) + if topology == "fork": + snapshots = [_snapshot_revision("snapshot-a"), _snapshot_revision("snapshot-b")] + else: + snapshots = [ + _snapshot_revision("snapshot-a", "snapshot-b"), + _snapshot_revision("snapshot-b", "snapshot-a"), + ] + raw["revisions"] = [*snapshots, deepcopy(REVISION)] + result = evaluate_reporting_ledger( + _ledger_from(raw), + expected_periods=[], + now=datetime.fromisoformat("2026-09-03T00:00:00+00:00"), + ) + assert not result.definitive + assert reason in result.obligations[0].reasons + + +def test_identical_scope_obligations_each_keep_their_own_materialized_publication() -> None: + """``ReportingRevision`` carries no obligation, so use materialization ownership.""" + raw = _response() + first, second = _obligation("obligation-a"), _obligation("obligation-b") + second.update(delivery_config_id="analytics-feed", feed_purpose="analytics") + for item in (first, second): + item.update( + revision_count=1, + reconciliation_mode="delivery_only", + reconciliation_status="not_required", + health="complete", + required_finality="official", + materialization_count=1, + successful_materialization_count=1, + ) + revisions = [] + materializations = [] + for suffix, obligation in (("a", first), ("b", second)): + revision = deepcopy(REVISION) + revision["reporting_revision_id"] = f"revision-{suffix}" + revisions.append(revision) + attempt = _materialization(f"materialization-{suffix}", f"obligation-{suffix}") + attempt.update( + reporting_revision_id=f"revision-{suffix}", + delivery_config_id=obligation["delivery_config_id"], + feed_purpose=obligation["feed_purpose"], + ) + materializations.append(attempt) + raw.update(periods=[first, second], revisions=revisions, materializations=materializations) + result = evaluate_reporting_ledger( + _ledger_from(raw), + expected_periods=[], + now=datetime.fromisoformat("2026-09-03T00:00:00+00:00"), + ) + assert result.definitive, [item.reasons for item in result.obligations] + assert [item.reporting_revision_id for item in result.obligations] == [ + "revision-a", + "revision-b", + ] + + +def test_unowned_publication_never_falls_back_to_an_older_materialized_snapshot() -> None: + """A newer unmaterialized official wins; an unresolvable owner fails closed.""" + raw = _response() + first, second = _obligation("obligation-a"), _obligation("obligation-b") + second.update(delivery_config_id="analytics-feed", feed_purpose="analytics") + first.update( + revision_count=2, + reconciliation_mode="delivery_only", + reconciliation_status="not_required", + health="complete", + required_finality="snapshot", + materialization_count=1, + successful_materialization_count=1, + ) + second.update( + revision_count=1, + reconciliation_mode="delivery_only", + reconciliation_status="not_required", + health="complete", + required_finality="official", + materialization_count=1, + successful_materialization_count=1, + ) + unmaterialized = deepcopy(REVISION) + unmaterialized["reporting_revision_id"] = "revision-a-official" + owned = deepcopy(REVISION) + owned["reporting_revision_id"] = "revision-b-official" + stale = _materialization("materialization-a", "obligation-a") + stale["reporting_revision_id"] = "revision-a-snapshot" + other = _materialization("materialization-b", "obligation-b") + other.update( + reporting_revision_id="revision-b-official", + delivery_config_id="analytics-feed", + feed_purpose="analytics", + ) + raw.update( + periods=[first, second], + revisions=[_snapshot_revision("revision-a-snapshot"), unmaterialized, owned], + materializations=[stale, other], + ) + result = evaluate_reporting_ledger( + _ledger_from(raw), + expected_periods=[], + now=datetime.fromisoformat("2026-09-03T00:00:00+00:00"), + ) + assert not result.definitive + selected, unresolved = result.obligations + # The older snapshot has a ready resource; the current publication does not. + assert selected.reporting_revision_id == "revision-a-official" + assert selected.reporting_materialization_id is None + assert "MISSING_VERIFIED_MATERIALIZATION" in selected.reasons + # obligation-b cannot own the unmaterialized official, so it refuses to guess. + assert unresolved.reporting_revision_id is None + assert "AMBIGUOUS_REVISION_CHAIN" in unresolved.reasons + + +def test_native_commit_requires_a_native_version_resource_descriptor() -> None: + """Matching refs and an observed path do not make a mutable location immutable.""" + raw = _response() + raw["periods"][0].update( + revision_count=1, + reconciliation_mode="delivery_only", + reconciliation_status="not_required", + health="complete", + required_finality="official", + feed_purpose="analytics", + materialization_count=1, + successful_materialization_count=1, + ) + attempt = _materialization("materialization-native") + attempt.update(feed_purpose="analytics", method="warehouse_materialization") + attempt["resource"] = { + "resource_ref": "resource-native", + "kind": "warehouse_relation", + "location": "project.dataset.table", + "immutability": "immutable_location", + "native_version_ref": "version-42", + "expires_at": "2026-12-01T00:00:00Z", + } + attempt["verification"] = { + "verified_at": "2026-09-02T00:00:05Z", + "verification_path": "destination", + "verification_profile": "native_commit", + "row_count": REVISION["row_count"], + "control_totals": TOTALS, + "native_commit_evidence": { + "native_version_ref": "version-42", + "observed_through": "destination", + }, + } + raw["materializations"] = [attempt] + result = evaluate_reporting_ledger( + _ledger_from(raw), + expected_periods=[], + now=datetime.fromisoformat("2026-09-03T00:00:00+00:00"), + ) + assert not result.definitive + assert "PRODUCER_NATIVE_EVIDENCE_MISMATCH" in result.obligations[0].reasons + + @pytest.mark.parametrize("finality", ["official", "snapshot"]) def test_publication_selection_rejects_multiple_current_revisions(finality: str) -> None: raw = _response()