From 99de6acb62ce67679c8d92aa5b64af7295af5f25 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Wed, 16 Sep 2026 10:01:05 +0000 Subject: [PATCH 1/2] feat(reporting): add transactional notification outbox Co-commit immutable ledger/readiness events and consumer-aware status-dirty history. Expand trusted subscribers atomically and dispatch encrypted prepared requests through fenced leases and the SDK-owned pinned transport. Keep wiring optional, Core readiness excluded, and status/activity emission deferred to the complete projector. Add migration, restart, adversarial and bounded separate-process PostgreSQL failure coverage. Refs #1168 --- docs/reporting-ledger-migration.md | 20 +- docs/reporting-notification-outbox.md | 205 ++++++ pyproject.toml | 2 + src/adcp/reporting/ledger/delivery.py | 21 +- src/adcp/reporting/ledger/delivery_pg.py | 32 +- .../reporting/ledger/notification_events.py | 142 ++++ .../reporting/ledger/notification_models.py | 428 +++++++++++ src/adcp/reporting/ledger/pg.py | 290 +++++++- .../ledger/reporting_notification_outbox.sql | 148 ++++ src/adcp/reporting/ledger/store.py | 226 +++++- src/adcp/reporting/outbox/__init__.py | 71 ++ src/adcp/reporting/outbox/_capabilities.py | 117 +++ src/adcp/reporting/outbox/_schema.py | 396 ++++++++++ .../reporting/outbox/_transport_logging.py | 43 ++ src/adcp/reporting/outbox/memory.py | 332 +++++++++ src/adcp/reporting/outbox/models.py | 159 ++++ src/adcp/reporting/outbox/pg.py | 532 ++++++++++++++ src/adcp/reporting/outbox/routing.py | 348 +++++++++ src/adcp/reporting/outbox/worker.py | 315 ++++++++ src/adcp/webhook_sender.py | 30 +- .../reporting/_reliable_process.py | 361 ++++++++++ .../reporting/_reliable_support.py | 470 +++++++++++- tests/conformance/reporting/conftest.py | 1 + .../test_reporting_notification_harness.py | 40 ++ .../test_reporting_notification_migration.py | 277 +++++++ .../test_reporting_notification_outbox.py | 678 ++++++++++++++++++ .../test_reporting_notification_packaging.py | 262 +++++++ ...t_reporting_notification_process_matrix.py | 275 +++++++ .../test_reporting_notification_readiness.py | 126 ++++ .../test_reporting_notification_security.py | 418 +++++++++++ ...est_reporting_notification_transactions.py | 413 +++++++++++ .../reporting_notification_outbox.py | 150 ++++ 32 files changed, 7265 insertions(+), 63 deletions(-) create mode 100644 docs/reporting-notification-outbox.md create mode 100644 src/adcp/reporting/ledger/notification_events.py create mode 100644 src/adcp/reporting/ledger/notification_models.py create mode 100644 src/adcp/reporting/ledger/reporting_notification_outbox.sql create mode 100644 src/adcp/reporting/outbox/__init__.py create mode 100644 src/adcp/reporting/outbox/_capabilities.py create mode 100644 src/adcp/reporting/outbox/_schema.py create mode 100644 src/adcp/reporting/outbox/_transport_logging.py create mode 100644 src/adcp/reporting/outbox/memory.py create mode 100644 src/adcp/reporting/outbox/models.py create mode 100644 src/adcp/reporting/outbox/pg.py create mode 100644 src/adcp/reporting/outbox/routing.py create mode 100644 src/adcp/reporting/outbox/worker.py create mode 100644 tests/conformance/reporting/_reliable_process.py create mode 100644 tests/conformance/reporting/test_reporting_notification_harness.py create mode 100644 tests/conformance/reporting/test_reporting_notification_migration.py create mode 100644 tests/conformance/reporting/test_reporting_notification_outbox.py create mode 100644 tests/conformance/reporting/test_reporting_notification_packaging.py create mode 100644 tests/conformance/reporting/test_reporting_notification_process_matrix.py create mode 100644 tests/conformance/reporting/test_reporting_notification_readiness.py create mode 100644 tests/conformance/reporting/test_reporting_notification_security.py create mode 100644 tests/conformance/reporting/test_reporting_notification_transactions.py create mode 100644 tests/type_checks/reporting_notification_outbox.py diff --git a/docs/reporting-ledger-migration.md b/docs/reporting-ledger-migration.md index 9a3fa7cf9..bc71f2b7d 100644 --- a/docs/reporting-ledger-migration.md +++ b/docs/reporting-ledger-migration.md @@ -50,21 +50,23 @@ obligation IDs, the named obligations must exist in the requested account. 2. With the upgraded SDK, run `await store.create_schema()` before starting reporting work. It creates missing tables and applies the bundled `reporting_ledger_account_generations.sql`, - `reporting_ledger_obligation_currency.sql`, and - `reporting_ledger_reconciliation.sql` migrations in one transaction. + `reporting_ledger_obligation_currency.sql`, + `reporting_ledger_reconciliation.sql`, and + `reporting_notification_outbox.sql` migrations in one transaction. 3. Restart reporting work with the upgraded SDK on every instance. For deployments managed by a migration tool, the standalone migration is [`reporting_ledger_account_generations.sql`](../src/adcp/reporting/ledger/reporting_ledger_account_generations.sql). It upgrades an existing beta.15 ledger by itself, including in autocommit mode. -For a combined bootstrap and upgrade, run all four bundled files in one transaction: +For a combined bootstrap and upgrade, run all five bundled files in one transaction: ```sh psql "$REPORTING_DATABASE_URL" --set=ON_ERROR_STOP=1 --single-transaction \ -f src/adcp/reporting/ledger/reporting_ledger.sql \ -f src/adcp/reporting/ledger/reporting_ledger_account_generations.sql \ -f src/adcp/reporting/ledger/reporting_ledger_obligation_currency.sql \ - -f src/adcp/reporting/ledger/reporting_ledger_reconciliation.sql + -f src/adcp/reporting/ledger/reporting_ledger_reconciliation.sql \ + -f src/adcp/reporting/ledger/reporting_notification_outbox.sql ``` Use the ledger's existing `search_path` and a role that owns its tables. All @@ -77,6 +79,14 @@ immutable managed digest/total evidence on revisions and adjustments. It does no evidence or enable a delivery tier. See the [storage contract](reporting-reconciliation-storage.md) for the records, migration invariants, and deferred writer/handler work. +The outbox migration adds empty event, delivery, and ordered status-dirty tables. +It preserves existing ledger rows and does not backfill historical events. +Notification enqueue remains disabled until the store is constructed with +`notifications=True`. Opted-in schema readiness checks the complete installed chain, +including constraints, indexes, and immutable-record guards. See the +[notification outbox contract](reporting-notification-outbox.md) for trusted +subscription wiring, retention, and the deferred status projector. + The migration inspects the primary-key columns under an exclusive table lock and replaces the beta.15 global primary key with the account-qualified key. It preserves the constraint's name, including a renamed beta.15 key. Existing @@ -167,7 +177,7 @@ large tables; production-sized lock time has not been benchmarked. Unexpected adopter column types/defaults fail and roll back rather than silently adapting. Stop and drain **all** beta.15 and #1169-only reporting writers before upgrading; -they do not supply frozen currency. Run `create_schema()` or the three-file SQL +they do not supply frozen currency. Run `create_schema()` or the bundled SQL command above, then start upgraded writers. If #1169 is already installed, its primary-key migration recognizes the account-qualified key without rebuilding it. The standalone currency SQL also runs atomically on an installed #1169 diff --git a/docs/reporting-notification-outbox.md b/docs/reporting-notification-outbox.md new file mode 100644 index 000000000..247540502 --- /dev/null +++ b/docs/reporting-notification-outbox.md @@ -0,0 +1,205 @@ +# Transactional reporting notification outbox + +The optional reporting outbox commits a typed logical event in the ledger's +transaction, then expands subscribers and delivers HTTP in separate phases. +This implements the ledger and Managed readiness slice of [#1168](https://github.com/adcontextprotocol/adcp-client-python/issues/1168). +It retains a durable status-dirty handoff for the later complete status projector. + +| Committed change | Retained notification work | +| --- | --- | +| New revision or adjustment | `reporting.ledger_changed` and ordered status-dirty evidence | +| Configuration, obligation, readability, consumer statement or issue transition | Ordered status-dirty evidence | +| Managed destination/reconciliation change | Consumer-scoped status-dirty evidence | +| Verified materialization with its frozen Managed binding | `reporting.delivery_ready`, scoped to the reconciliation consumer | + +There is no `reporting.status_changed` emitter. Clock sweeps, complete status +fingerprint deduplication, and webhook activity projection belong to #1168B. +The capability helper omits `status_notification` and sets +`supports_webhook_activity=false`. + +## Optional wiring + +Existing `ReportingLedgerStore` implementations and `ReportingProducer` +constructors require no new methods, callback, or argument. Concrete memory +and PostgreSQL stores accept the optional keyword `notifications=True`; the +default is off. Core producer return values and exact-content reads keep their +existing behavior. A memory outbox shares its ledger's rollback boundary but +has no process-crash durability. + +```python +from adcp.reporting.ledger import PgReportingReconciliationStore +from adcp.reporting.outbox import ( + PgReportingOutbox, + ReportingEnvelopeCipher, + ReportingNotificationWorker, +) + +ledger = PgReportingReconciliationStore(pool=pool, notifications=True) +await ledger.create_schema() +outbox = PgReportingOutbox(pool=pool) +worker = ReportingNotificationWorker( + outbox=outbox, + subscriptions=trusted_account_configurations, + signing=trusted_account_keyrings, + cipher=ReportingEnvelopeCipher(encryption_key, key_version="outbox-key-1"), +) + +# At service startup, while arranging to schedule both worker phases: +fields = await worker.advertised_notifications(ledger, account_id=account_id) +# Merge fields into the producer's reporting_delivery capability block. + +# Each call performs at most one leased unit of work and returns a bool. +await worker.expand_one(account_id=account_id) +await worker.deliver_one(account_id=account_id) +``` + +Schedule both phases for the accounts the service is authorized to process. +An empty turn is idle; use the service's normal scheduling/backoff policy. +The SDK does not start a scheduler or contact subscribers during import or +ledger construction. Supply a retained typed `ready_scope` to the capability +helper to advertise readiness. It verifies the actual configuration, +destination, and obligation binding. A profile label, capability string, or +Core subscriber request cannot create a readiness event or capability. + +The helper verifies that the opted-in ledger and outbox use the same store or +pool, that current registrations have usable authentication, and that the +entire installed PostgreSQL chain matches its column, constraint, index, +trigger, and guard-function contract. It does not infer operational readiness +from the presence of objects. Continue scheduling the worker while advertising +these fields. Custom stores can implement the additive outbox protocol; +automatic capability verification conservatively covers the SDK reference stores. + +## Domain commit and replay + +Every domain mutation and its enqueue use the exact same connection inside +an explicit `async with connection.transaction()`, including autocommit +pools. An enqueue failure rolls back the mutation. Committed logical events +are immutable and survive producer restarts. The causal key contains account, +consumer namespace, closed cause kind, non-null cause ID, and cause generation. +Readiness identifies the full `(account_id, consumer_id, materialization_id)`. +An identical mutation replay creates no event; a conflicting identity cannot +commit a ghost event. + +Each event owns one notification ID and `fired_at`. Every subscriber shares +that ID. Each subscriber/emission generation gets a cryptographically random +idempotency key and its own prepared body. Re-emitting an event with +`outbox.reemit(...)` advances the emission generation and generates new keys; +ordinary retry changes neither generation, key, nor body bytes. + +Status-dirty records form an account-ordered journal, with a trusted optional +generation/obligation/consumer/feed scope, cause generations, and immutable +record references. Readability, configuration lifecycle, and issue lifecycle +retain before-and-after evidence, so rapid reversals remain replayable. +Consumer statements and reconciliation records remain readable in their +original consumer namespace. Checkpoints use account/projector compare-and-set. +They do not delete evidence or project a wire status. The clock-dirty method +is a handoff seam, with no automatic clock sweep in this slice. + +The concrete issue methods accept optional `status_scope=ReportingStatusScope(...)`. +The store validates supplied references in the issue's account and consumer +namespace. It never parses `issue_key`. Without a typed scope, a dirty record +invalidates the account and its specified consumer broadly. An occurrence's +typed scope cannot be retargeted during its lifecycle. + +## Subscriber configuration and transport + +`ReportingSubscriptionResolver` must resolve from trusted account registration +state. A `ReportingNotificationSubscription` combines account, principal, +subscriber, normalized URL, exact event types, configuration revision, +authorization and proof-of-control references, active/authorized/proof-valid +flags, and one exclusive authentication mode. The registrar must actually +verify principal authorization and the account/subscriber/URL proof of control; +the references are evidence pointers, not substitutes for those checks. +Never construct registrations from an unauthenticated request or notification. + +Fanout uses **active-at-expansion semantics**. It resolves one complete snapshot, +validates uniqueness and fingerprints, then inserts all matching deliveries and +the completed-expansion checkpoint in one transaction. A valid empty snapshot +completes. A transient resolution error retries. A crash during insertion rolls +back every subscriber row; a restart can resolve a new snapshot without mixing +membership. An external resolver is not transactionally atomic with the ledger. + +AES-256-GCM encrypts the complete routing envelope, including the full query +URL, credential mode/material, typed event and exact prepared body. Canonical +AAD binds every routing column: account, consumer, subscriber/principal, +delivery/event/notification/idempotency IDs, cause and emission generations, +URL digest, configuration fingerprint, auth mode/signing scope, body digest, +envelope version and encryption-key version. Decryption/authentication runs +before configuration resolution, signing, DNS, or HTTP. Old encryption keys +must remain available while retained deliveries can use them; `previous_keys` +supports rotation without rewriting an immutable delivery. + +Every attempt resolves the exact account/subscriber/event configuration again, +including authorization, proof validity and the full credential fingerprint. +Removed, deactivated, or replaced registrations are terminally suppressed. +Bodies are never retargeted. RFC 9421 attempts resolve current trusted signing +material by account/principal/signing scope, producing fresh signatures during +key rotation. Legacy bearer and HMAC modes are exclusive with RFC 9421. +The worker constructs its own concrete hardened sender; injected sender +objects/subclasses, arbitrary HTTP clients and URL rewrite hooks are not accepted. + +Prepared bodies use deterministic canonical UTF-8 and the exact bundled +`3.2.0-rc.3` named schema validator, including the conditionals omitted by the +generated models. The mapping is an allowlist with recursive secret rejection; +it copies no reporting rows, resource/object lists, signed or activation URLs, +credentials, tokens, extensions, or adopter metadata. + +The SDK-owned transport permits public HTTPS on port 443, validates all DNS +answers, pins the accepted IP, verifies TLS against the original host, ignores +proxy environment settings, and refuses redirects. Rebinding to loopback, +private, link-local, metadata or IPv6 ULA addresses is rejected. +Outbox state retains only closed local error classifications. HTTP library logs +are suppressed within this transport context so URL queries, auth headers, +signatures and provider responses cannot spill into logs; unrelated traffic +retains its logging behavior. + +## Leases, retries and retention + +PostgreSQL claims use `FOR UPDATE SKIP LOCKED` in short transactions with +unguessable tokens and database-clock expiration. ACK, retry release, +suppression and quarantine are fenced by account, consumer namespace, token +and expiry. The prepared-request seam rechecks the lease after DNS/signing, +immediately before HTTP. HTTP runs outside database transactions. A crashed claim does not +consume an HTTP retry budget. Network errors, 408/425/429/5xx, and transient +signing/DNS failures retry; permanent 4xx, invalid payload/routing evidence, +permanent scopes and poison rows quarantine without blocking other subscribers. + +HTTP acceptance and worker ACK cannot be a single transaction. Receivers must +deduplicate using a trusted publisher identity that survives key rotation and +the idempotency key. A lost ACK can produce another authenticated request with +identical body/key and a fresh signature. Polling remains the recovery path. + +This slice retains events, expansion checkpoints, prepared bindings and dirty +evidence indefinitely. It has no purge API or finite advertised activity horizon. +Do not delete parent events, keys, or prepared bindings while any delivery is +nonterminal or within an adopter's promised retry/activity retention horizon. + +## Migration and conformance + +`reporting_notification_outbox.sql` follows the reviewed four-file foundation +chain. It adds notification events, expansion/delivery leases, ordered dirty +records, projector checkpoints, and typed issue scope storage. It rewrites and +backfills no ledger evidence. `create_schema()` installs all five steps atomically; +opted-in stores and `PgReportingOutbox.create_schema()` also validate the complete +installed contract before committing. Default-off Core startup preserves its +compatibility with adopter indexes. The conservative notification readiness check +requires the SDK table definitions, including their indexes and guards, to match +the bundled contract. Concurrent and repeated installations serialize on the schema +advisory lock. The standalone outbox SQL is atomic even on an autocommit +connection with the foundation already installed. See +[reporting ledger migrations](reporting-ledger-migration.md). + +The private shared harness is `tests/conformance/reporting/_reliable_support.py`. +It injects `ManualClock`, scripted sync/async adapters, barriers, failure plans, +bounded drains and deterministic destination/receiver stores into the same +memory/PostgreSQL vectors. Essential cases are not marked `integration`. + +`test_reporting_notification_process_matrix.py` runs separately pooled producer, +fanout, HTTP worker, receiver and observer services. Named IPC and SQL barriers +cover commit/fanout and real TLS HTTP acceptance/ACK. All child, pipe, receiver +and barrier waits have hard watchdogs with sanitized role/PID/checkpoint +diagnostics; no timing sleeps control an interleaving. The receiver fixture +self-check verifies both rotation keys before the full process lane is run. +The distribution tests build an sdist, build its wheel, import a real base +installation with PostgreSQL absent, then install `[pg]` and exercise migration, +commit and restart from that wheel. diff --git a/pyproject.toml b/pyproject.toml index 18092d790..f2a9fedfd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -116,6 +116,8 @@ adcp-keygen = "adcp.signing.keygen:main" [project.optional-dependencies] dev = [ + # The reporting outbox gate builds and installs the real sdist/wheel. + "build>=1.0", "pytest>=7.0.0", "pytest-asyncio>=0.21.0", "pytest-cov>=4.0.0", diff --git a/src/adcp/reporting/ledger/delivery.py b/src/adcp/reporting/ledger/delivery.py index 1bd338e99..8f854e211 100644 --- a/src/adcp/reporting/ledger/delivery.py +++ b/src/adcp/reporting/ledger/delivery.py @@ -406,7 +406,7 @@ class InMemoryReportingReconciliationStore(InMemoryReportingLedgerStore, _Reconc async def _commit(self, record: RecordT) -> tuple[RecordT, bool]: candidate = decode_record(payload(record)) who = principal(candidate) - async with self._lock: + async with self._mutation(): records = tuple(item.record for item in self._caller_changes(who)) existing = replay(candidate, records) if existing is not None: @@ -414,6 +414,24 @@ async def _commit(self, record: RecordT) -> tuple[RecordT, bool]: context = self._delivery_context(candidate) stored = validate_transition(candidate, records, context, self._clock()) self._append_reconciliation_change(stored) + if self._notification_state is not None: + from adcp.reporting.ledger.notification_events import ( + delivery_dirty, + materialization_event, + ) + + event = materialization_event( + stored, + records, + context.obligation, + context.revision, + context.configuration, + self._clock(), + ) + if event is not None: + self._record_notification(event) + scope, reason, evidence = delivery_dirty(stored, context.obligation) + self._dirty_status(scope, reason, after=evidence) return cast(RecordT, stored), True def _append_reconciliation_change(self, record: ReportingDeliveryRecord) -> None: @@ -461,6 +479,7 @@ def _delivery_context(self, record: ReportingDeliveryRecord) -> DeliveryContext: revision_id = record.adjusts_reporting_revision_id revision = self._revisions.get(revision_id) if revision_id is not None else None return DeliveryContext( + configuration=self._configurations.get(record.scope.generation_key), obligation=obligation, revision=revision, adjustment=( diff --git a/src/adcp/reporting/ledger/delivery_pg.py b/src/adcp/reporting/ledger/delivery_pg.py index 7e821a661..f1450adcb 100644 --- a/src/adcp/reporting/ledger/delivery_pg.py +++ b/src/adcp/reporting/ledger/delivery_pg.py @@ -131,6 +131,24 @@ async def _commit_record(self, record: RecordT) -> tuple[RecordT, bool]: stored = validate_transition(candidate, records, context, now) await self._insert(connection, stored) await self._append_reconciliation_change(connection, stored) + if self._notifications_enabled: + from adcp.reporting.ledger.notification_events import ( + delivery_dirty, + materialization_event, + ) + + event = materialization_event( + stored, + records, + context.obligation, + context.revision, + context.configuration, + now, + ) + if event is not None: + await self._record_notification(connection, event) + scope, reason, evidence = delivery_dirty(stored, context.obligation) + await self._dirty_status(connection, scope, reason, after=evidence) return cast(RecordT, stored), True async def _append_reconciliation_change( @@ -268,8 +286,13 @@ async def _delivery_context( self, connection: Any, record: ReportingDeliveryRecord ) -> DeliveryContext: who = principal(record) - if isinstance(record, ReportingDestinationBinding): - generation = record.generation_key + configuration = None + if isinstance(record, ReportingDestinationBinding) or self._notifications_enabled: + generation = ( + record.generation_key + if isinstance(record, ReportingDestinationBinding) + else record.scope.generation_key + ) row = await ( await connection.execute( "SELECT delivery_config_id, delivery_config_version, account_id," @@ -286,7 +309,9 @@ async def _delivery_context( ), ) ).fetchone() - return DeliveryContext(configuration=_configuration_from_row(row) if row else None) + configuration = _configuration_from_row(row) if row else None + if isinstance(record, ReportingDestinationBinding): + return DeliveryContext(configuration=configuration) obligation_row = await ( await connection.execute( f"SELECT {_OBLIGATION_COLUMNS} FROM reporting_obligations" # noqa: S608 # nosec B608 @@ -314,6 +339,7 @@ async def _delivery_context( ) ).fetchone() return DeliveryContext( + configuration=configuration, obligation=_obligation_from_row(obligation_row) if obligation_row else None, revision=_revision_from_row(revision_row) if revision_row else None, adjustment=_adjustment_from_row(adjustment_row) if adjustment_row else None, diff --git a/src/adcp/reporting/ledger/notification_events.py b/src/adcp/reporting/ledger/notification_events.py new file mode 100644 index 000000000..f080a1581 --- /dev/null +++ b/src/adcp/reporting/ledger/notification_events.py @@ -0,0 +1,142 @@ +"""Pure event mappings, invoked only by an opted-in store's transaction.""" + +from __future__ import annotations + +from datetime import datetime + +from adcp.reporting.ledger.delivery_models import ( + ReportingDeliveryRecord, + ReportingDestinationBinding, + ReportingMaterializationRecord, + ReportingObligationDeliveryRecord, +) +from adcp.reporting.ledger.models import ( + ReportingAdjustmentRecord, + ReportingConfiguration, + ReportingObligationRecord, + ReportingRevisionRecord, +) +from adcp.reporting.ledger.notification_models import ( + AdjustmentPublished, + DirtyReason, + MaterializationReady, + ReportingDomainEvent, + ReportingNotificationError, + ReportingStatusEvidence, + ReportingStatusScope, + RevisionPublished, + new_event, +) + + +def revision_event(revision: ReportingRevisionRecord, at: datetime) -> ReportingDomainEvent: + return new_event( + revision.account_id, + RevisionPublished( + revision.reporting_revision_id, + revision.finality, + revision.supersedes_reporting_revision_id, + ), + at, + ) + + +def adjustment_event(adjustment: ReportingAdjustmentRecord, at: datetime) -> ReportingDomainEvent: + return new_event( + adjustment.account_id, + AdjustmentPublished( + adjustment.reporting_adjustment_id, + adjustment.adjusts_reporting_revision_id, + ), + at, + ) + + +def materialization_event( + record: ReportingDeliveryRecord, + records: tuple[ReportingDeliveryRecord, ...], + obligation: ReportingObligationRecord | None, + revision: ReportingRevisionRecord | None, + configuration: ReportingConfiguration | None, + at: datetime, +) -> ReportingDomainEvent | None: + """Call after the shared reconciliation validator verified the exact graph. + + Core has no obligation-delivery binding or Managed destination. Neither an + advertised capability, a profile label, nor a subscriber request can create + this structural evidence. No rows/resources/provider metadata are copied. + """ + if not isinstance(record, ReportingMaterializationRecord) or record.status == "failed": + return None + binding = next( + ( + item + for item in records + if isinstance(item, ReportingDestinationBinding) + and item.generation_key == record.scope.generation_key + and item.consumer_id == record.scope.consumer_id + ), + None, + ) + frozen = next( + ( + item + for item in records + if isinstance(item, ReportingObligationDeliveryRecord) and item.scope == record.scope + ), + None, + ) + if ( + binding is None + or frozen is None + or revision is None + or obligation is None + or configuration is None + or configuration.generation_key != record.scope.generation_key + or record.verification is None + or record.resource is None + or record.scope.reporting_obligation_id != obligation.reporting_obligation_id + or record.scope.generation_key != obligation.generation_key + or revision.account_id != obligation.account_id + or revision.reporting_obligation_id != obligation.reporting_obligation_id + or revision.reporting_revision_id != record.reporting_revision_id + ): + raise ReportingNotificationError("core_delivery_ready_forbidden") + return new_event( + obligation.account_id, + MaterializationReady( + generation_key=record.scope.generation_key, + consumer_id=record.scope.consumer_id, + reporting_obligation_id=obligation.reporting_obligation_id, + destination_ref=binding.destination_ref, + method=binding.method, + reporting_revision_id=record.reporting_revision_id, + reporting_materialization_id=record.reporting_materialization_id, + readiness=record.status, + finality=revision.finality, + data_through=revision.data_through, + feed_purpose=binding.feed_purpose, + ), + at, + ) + + +def delivery_dirty( + record: ReportingDeliveryRecord, obligation: ReportingObligationRecord | None +) -> tuple[ReportingStatusScope, DirtyReason, ReportingStatusEvidence]: + from adcp.reporting.ledger._delivery_state import change_id + + if isinstance(record, ReportingDestinationBinding): + scope = ReportingStatusScope( + record.generation_key.account_id, + record.generation_key, + consumer_id=record.consumer_id, + feed_purpose=record.feed_purpose, + ) + reason: DirtyReason = "destination" + else: + if obligation is None: + raise ReportingNotificationError("invalid_status_scope") + scope = ReportingStatusScope.for_obligation(obligation, record.scope.consumer_id) + reason = "receipt" if record.kind.endswith("receipt") else "materialization" + return scope, reason, ReportingStatusEvidence(record.kind, change_id(record)) diff --git a/src/adcp/reporting/ledger/notification_models.py b/src/adcp/reporting/ledger/notification_models.py new file mode 100644 index 000000000..19f9fc311 --- /dev/null +++ b/src/adcp/reporting/ledger/notification_models.py @@ -0,0 +1,428 @@ +"""Closed logical notifications and the additive status-projector handoff. + +There is deliberately no status-changed event constructor. A dirty scope is +work for a complete projector, not evidence that projected health changed. +""" + +from __future__ import annotations + +import json +from dataclasses import asdict, dataclass, field +from datetime import datetime +from typing import Any, Literal, TypeAlias +from uuid import uuid4 + +from pydantic import TypeAdapter, ValidationError + +from adcp.reporting.evidence import aware_utc, principal_reference, reporting_identifier +from adcp.reporting.ledger.delivery_models import _ClosedValue, _freeze_fields +from adcp.reporting.ledger.models import ( + ReportingConfiguration, + ReportingConfigurationGenerationKey, + ReportingFinality, + ReportingIssueLifecycle, + ReportingObligationRecord, +) +from adcp.validation.schema_loader import get_named_validator + +NotificationType: TypeAlias = Literal["reporting.ledger_changed", "reporting.delivery_ready"] +FeedPurpose: TypeAlias = Literal["pacing", "analytics", "billing"] +DirtyReason: TypeAlias = Literal[ + "configuration", + "obligation", + "revision", + "adjustment", + "readability", + "consumer_status", + "issue", + "destination", + "materialization", + "receipt", + "clock", +] + + +class ReportingNotificationError(ValueError): + """A sanitized local classification, never an external diagnostic.""" + + def __init__(self, code: str = "invalid_notification") -> None: + self.code = code + super().__init__(code) + + +@dataclass(frozen=True, slots=True) +class RevisionPublished(_ClosedValue): + reporting_revision_id: str + finality: ReportingFinality + supersedes_reporting_revision_id: str | None = None + kind: Literal["revision_published"] = field(default="revision_published", kw_only=True) + + def __post_init__(self) -> None: + _freeze_fields(self) + reporting_identifier(self.reporting_revision_id, maximum=255) + if self.supersedes_reporting_revision_id is not None: + reporting_identifier(self.supersedes_reporting_revision_id, maximum=255) + + +@dataclass(frozen=True, slots=True) +class AdjustmentPublished(_ClosedValue): + reporting_adjustment_id: str + adjusts_reporting_revision_id: str + kind: Literal["adjustment_published"] = field(default="adjustment_published", kw_only=True) + + def __post_init__(self) -> None: + _freeze_fields(self) + reporting_identifier(self.reporting_adjustment_id, maximum=255) + reporting_identifier(self.adjusts_reporting_revision_id, maximum=255) + + +@dataclass(frozen=True, slots=True) +class MaterializationReady(_ClosedValue): + """Snapshot of a verified, frozen Managed binding; never a capability string.""" + + generation_key: ReportingConfigurationGenerationKey + consumer_id: str + reporting_obligation_id: str + destination_ref: str + method: Literal["file_transfer", "dataset_share", "warehouse_materialization"] + reporting_revision_id: str + reporting_materialization_id: str + readiness: Literal["available", "delivered"] + finality: ReportingFinality + data_through: datetime | None + feed_purpose: FeedPurpose + kind: Literal["materialization_ready"] = field(default="materialization_ready", kw_only=True) + + def __post_init__(self) -> None: + _freeze_fields(self) + principal_reference(self.consumer_id) + for value in ( + self.reporting_obligation_id, + self.reporting_revision_id, + self.reporting_materialization_id, + ): + reporting_identifier(value, maximum=255) + reporting_identifier(self.generation_key.delivery_config_id, maximum=64) + if self.data_through is not None: + object.__setattr__(self, "data_through", aware_utc(self.data_through)) + + +NotificationCause: TypeAlias = RevisionPublished | AdjustmentPublished | MaterializationReady + + +@dataclass(frozen=True, slots=True) +class ReportingDomainEvent(_ClosedValue): + account_id: str + notification_id: str + fired_at: datetime + cause: NotificationCause + cause_generation: int = 1 + + def __post_init__(self) -> None: + _freeze_fields(self) + principal_reference(self.account_id) + reporting_identifier(self.notification_id, maximum=255) + object.__setattr__(self, "fired_at", aware_utc(self.fired_at)) + if self.cause_generation < 1: + raise ReportingNotificationError() + if isinstance(self.cause, MaterializationReady): + if self.cause.generation_key.account_id != self.account_id: + raise ReportingNotificationError() + + @property + def notification_type(self) -> NotificationType: + return ( + "reporting.delivery_ready" + if isinstance(self.cause, MaterializationReady) + else "reporting.ledger_changed" + ) + + @property + def cause_id(self) -> str: + cause = self.cause + if isinstance(cause, RevisionPublished): + return cause.reporting_revision_id + if isinstance(cause, AdjustmentPublished): + return cause.reporting_adjustment_id + # Structured encoding: consumers and materializations may reuse IDs. + return json.dumps( + [cause.consumer_id, cause.reporting_materialization_id], separators=(",", ":") + ) + + @property + def consumer_namespace(self) -> str: + return self.cause.consumer_id if isinstance(self.cause, MaterializationReady) else "" + + @property + def causal_key(self) -> tuple[str, str, str, str, str, int]: + return ( + self.account_id, + self.consumer_namespace, + self.notification_type, + self.cause.kind, + self.cause_id, + self.cause_generation, + ) + + def body(self, *, subscriber_id: str, idempotency_key: str) -> bytes: + """Explicit wire allowlist, validated against all rc.3 conditionals.""" + from adcp.reporting.canonical_json import canonical_json_utf8_v1 + + value: dict[str, Any] = { + "account_id": self.account_id, + "notification_id": self.notification_id, + "notification_type": self.notification_type, + "fired_at": iso(self.fired_at), + "subscriber_id": subscriber_id, + "idempotency_key": idempotency_key, + } + cause = self.cause + if isinstance(cause, RevisionPublished): + value.update( + change_kind=cause.kind, + reporting_revision_id=cause.reporting_revision_id, + finality=cause.finality, + ) + if cause.supersedes_reporting_revision_id is not None: + value["supersedes_reporting_revision_id"] = cause.supersedes_reporting_revision_id + elif isinstance(cause, AdjustmentPublished): + value.update( + change_kind=cause.kind, + reporting_adjustment_id=cause.reporting_adjustment_id, + adjusts_reporting_revision_id=cause.adjusts_reporting_revision_id, + ) + else: + value.update( + delivery_config_id=cause.generation_key.delivery_config_id, + delivery_config_version=cause.generation_key.delivery_config_version, + reporting_revision_id=cause.reporting_revision_id, + reporting_materialization_id=cause.reporting_materialization_id, + readiness=cause.readiness, + finality=cause.finality, + data_through=iso(cause.data_through) if cause.data_through is not None else None, + feed_purpose=cause.feed_purpose, + ) + validate_notification_payload(value) + return canonical_json_utf8_v1(value) + + +@dataclass(frozen=True, slots=True) +class ReportingStatusScope(_ClosedValue): + """A typed projection target. Missing detail means invalidate the account. + + Legacy issue keys are opaque. They are never parsed to infer a generation, + obligation, consumer, or health. Adopters may supply this optional scope to + the concrete stores' issue methods without changing ReportingLedgerStore. + """ + + account_id: str + generation_key: ReportingConfigurationGenerationKey | None = None + reporting_obligation_id: str | None = None + consumer_id: str | None = None + feed_purpose: FeedPurpose | None = None + + def __post_init__(self) -> None: + _freeze_fields(self) + principal_reference(self.account_id) + if self.consumer_id is not None: + principal_reference(self.consumer_id) + if self.reporting_obligation_id is not None: + reporting_identifier(self.reporting_obligation_id, maximum=255) + if self.generation_key is not None and self.generation_key.account_id != self.account_id: + raise ReportingNotificationError("invalid_status_scope") + + @classmethod + def for_obligation( + cls, obligation: ReportingObligationRecord, consumer_id: str | None = None + ) -> ReportingStatusScope: + # Existing ledger fields are strings; validating through the closed + # adapter refuses a provider-supplied feed label rather than echoing it. + return decode_status_scope( + { + "account_id": obligation.account_id, + "generation_key": asdict(obligation.generation_key), + "reporting_obligation_id": obligation.reporting_obligation_id, + "consumer_id": consumer_id, + "feed_purpose": obligation.feed_purpose, + } + ) + + +@dataclass(frozen=True, slots=True) +class ReportingStatusEvidence(_ClosedValue): + """Replay evidence for one status mutation, containing no adopter prose. + + Immutable records are referenced in their account/consumer namespace. The + mutable inputs (readability and issue lifecycle) are snapshotted on both + sides, so rapid reversals cannot disappear before a projector runs. + """ + + record_kind: Literal[ + "configuration", + "obligation", + "revision", + "adjustment", + "consumer_status", + "issue", + "destination_binding", + "obligation_delivery", + "materialization_attempt", + "materialization", + "materialization_check", + "revision_receipt", + "adjustment_receipt", + "clock", + ] + record_id: str + record_version: int = 1 + readable: bool | None = None + issue_state: Literal["open", "acknowledged", "waived", "resolved"] | None = None + opened_at: datetime | None = None + retired_at: datetime | None = None + supersedes_id: str | None = None + activated_at: datetime | None = None + deactivated_at: datetime | None = None + automated_recovery_seconds: float | None = None + status_retention_days: int | None = None + + def __post_init__(self) -> None: + _freeze_fields(self) + reporting_identifier(self.record_id, maximum=255) + if self.record_version < 1: + raise ReportingNotificationError("invalid_status_evidence") + for name in ("opened_at", "retired_at", "activated_at", "deactivated_at"): + value = getattr(self, name) + if value is not None: + object.__setattr__(self, name, aware_utc(value)) + + +@dataclass(frozen=True, slots=True) +class ReportingStatusDirty(_ClosedValue): + sequence: int + scope: ReportingStatusScope + reason: DirtyReason + changed_at: datetime + cause_id: str + cause_generation: int + before: ReportingStatusEvidence | None = None + after: ReportingStatusEvidence | None = None + + def __post_init__(self) -> None: + _freeze_fields(self) + object.__setattr__(self, "changed_at", aware_utc(self.changed_at)) + if self.sequence < 1 or self.cause_generation < 1: + raise ReportingNotificationError("invalid_status_evidence") + + +def issue_evidence(issue: ReportingIssueLifecycle) -> ReportingStatusEvidence: + # Attribute allowlist: issue_key and external_ref are intentionally absent. + return ReportingStatusEvidence( + "issue", + issue.issue_id, + issue.generation, + issue_state=issue.issue_state, + opened_at=issue.opened_at, + retired_at=issue.retired_at, + ) + + +def configuration_evidence(configuration: ReportingConfiguration) -> ReportingStatusEvidence: + return ReportingStatusEvidence( + "configuration", + configuration.delivery_config_id, + configuration.delivery_config_version, + activated_at=configuration.activated_at, + deactivated_at=configuration.deactivated_at, + automated_recovery_seconds=configuration.automated_recovery_window.total_seconds(), + status_retention_days=configuration.status_retention_days, + ) + + +def iso(value: datetime) -> str: + return aware_utc(value).isoformat().replace("+00:00", "Z") + + +def event_storage(event: ReportingDomainEvent) -> dict[str, Any]: + return dict(json.loads(json.dumps(asdict(event), default=iso))) + + +_EVENT = TypeAdapter(ReportingDomainEvent) +_SCOPE = TypeAdapter(ReportingStatusScope) +_DIRTY = TypeAdapter(ReportingStatusDirty) + + +def dirty_storage(record: ReportingStatusDirty) -> dict[str, Any]: + return dict(json.loads(json.dumps(asdict(record), default=iso))) + + +def decode_dirty(value: object) -> ReportingStatusDirty: + try: + record = _DIRTY.validate_python(value) + if dirty_storage(record) == value: + return record + except (ValidationError, ValueError, TypeError): + pass + raise ReportingNotificationError("invalid_status_evidence") from None + + +def decode_event(value: object) -> ReportingDomainEvent: + try: + event = _EVENT.validate_python(value) + if event_storage(event) == value: + event.body(subscriber_id="validation", idempotency_key="0" * 32) + return event + except (ValidationError, ValueError, TypeError): + pass + raise ReportingNotificationError("invalid_event") from None + + +def decode_status_scope(value: object) -> ReportingStatusScope: + try: + scope = _SCOPE.validate_python(value) + if asdict(scope) == value: + return scope + except (ValidationError, ValueError, TypeError): + pass + raise ReportingNotificationError("invalid_status_scope") from None + + +def new_event(account_id: str, cause: NotificationCause, at: datetime) -> ReportingDomainEvent: + event = ReportingDomainEvent(account_id, str(uuid4()), at, cause) + event.body(subscriber_id="validation", idempotency_key="0" * 32) + return event + + +_SCHEMAS = { + "reporting.ledger_changed": "core/reporting-ledger-changed-webhook.json", + "reporting.delivery_ready": "core/reporting-delivery-ready-webhook.json", + # Validation is useful to #1168B; acceptance here does not enable emission. + "reporting.status_changed": "core/reporting-status-changed-webhook.json", +} + + +def validate_notification_payload(value: object) -> None: + """Offline full-schema gate, additionally excluding extension metadata.""" + if not isinstance(value, dict) or "ext" in value: + raise ReportingNotificationError("invalid_payload") + notification_type = value.get("notification_type") + path = _SCHEMAS.get(notification_type) if isinstance(notification_type, str) else None + if path is None: + raise ReportingNotificationError("invalid_payload") + + def scan(item: object) -> None: + if isinstance(item, str): + principal_reference(item) + elif isinstance(item, list): + for child in item: + scan(child) + elif isinstance(item, dict): + for child in item.values(): + scan(child) + + try: + scan(value) + except ValueError: + raise ReportingNotificationError("invalid_payload") from None + validator = get_named_validator(path, version="3.2.0-rc.3") + if validator is None or not validator.is_valid(value): + raise ReportingNotificationError("invalid_payload") diff --git a/src/adcp/reporting/ledger/pg.py b/src/adcp/reporting/ledger/pg.py index b39a90431..d61354ab9 100644 --- a/src/adcp/reporting/ledger/pg.py +++ b/src/adcp/reporting/ledger/pg.py @@ -27,8 +27,9 @@ the account-qualified configuration primary key for beta.15 installations. The raw DDL ships in :file:`reporting_ledger.sql` followed by :file:`reporting_ledger_account_generations.sql`, -:file:`reporting_ledger_obligation_currency.sql` and -:file:`reporting_ledger_reconciliation.sql`; run all four in one transaction +:file:`reporting_ledger_obligation_currency.sql`, +:file:`reporting_ledger_reconciliation.sql`, and +:file:`reporting_notification_outbox.sql`; run all five in one transaction when using Alembic, Flyway, or psql. See :file:`docs/reporting-ledger-migration.md` for deployment and compatibility notes. @@ -97,6 +98,16 @@ ReportingRevisionRecord, ReportingScheduleSpec, ) +from adcp.reporting.ledger.notification_models import ( + DirtyReason, + ReportingDomainEvent, + ReportingNotificationError, + ReportingStatusEvidence, + ReportingStatusScope, + configuration_evidence, + decode_status_scope, + issue_evidence, +) from adcp.reporting.ledger.store import ( LeasedConfiguration, LedgerConflictError, @@ -132,6 +143,7 @@ _ACCOUNT_GENERATIONS_DDL_PATH = Path(__file__).parent / "reporting_ledger_account_generations.sql" _CURRENCY_DDL_PATH = Path(__file__).parent / "reporting_ledger_obligation_currency.sql" _RECONCILIATION_DDL_PATH = Path(__file__).parent / "reporting_ledger_reconciliation.sql" +_NOTIFICATIONS_DDL_PATH = Path(__file__).parent / "reporting_notification_outbox.sql" __all__ = ["PG_AVAILABLE", "PgReportingLedgerStore"] @@ -158,6 +170,7 @@ def __init__( *, pool: AsyncConnectionPool, clock: Callable[[], datetime] | None = None, + notifications: bool = False, ) -> None: if not PG_AVAILABLE: raise ImportError(_INSTALL_HINT) @@ -169,6 +182,7 @@ def __init__( # tests that need to stand at a specific instant relative to seeded # evidence rather than wherever wall-clock time happens to fall. self._clock = clock + self._notifications_enabled = notifications async def create_schema(self) -> None: """Create or upgrade the ledger atomically, serializing concurrent boots. @@ -183,6 +197,117 @@ async def create_schema(self) -> None: await connection.execute(_ACCOUNT_GENERATIONS_DDL_PATH.read_text()) await connection.execute(_CURRENCY_DDL_PATH.read_text()) await connection.execute(_RECONCILIATION_DDL_PATH.read_text()) + await connection.execute(_NOTIFICATIONS_DDL_PATH.read_text()) + if self._notifications_enabled: + from adcp.reporting.outbox._schema import validate_schema + + await validate_schema(connection) + + async def _notification_now(self, connection: Any) -> datetime: + from adcp.reporting.outbox.pg import database_now + + return await database_now(connection, self._clock) + + async def _record_notification(self, connection: Any, event: ReportingDomainEvent) -> None: + if self._notifications_enabled: + from adcp.reporting.outbox.pg import enqueue_event + + await enqueue_event(connection, event) + + async def _dirty_status( + self, + connection: Any, + scope: ReportingStatusScope, + reason: DirtyReason, + before: ReportingStatusEvidence | None = None, + after: ReportingStatusEvidence | None = None, + ) -> None: + if self._notifications_enabled: + from adcp.reporting.outbox.pg import mark_dirty + + await mark_dirty( + connection, scope, reason, await self._notification_now(connection), before, after + ) + + async def _dirty_issue( + self, + connection: Any, + issue: ReportingIssueLifecycle, + status_scope: ReportingStatusScope | None, + before: ReportingIssueLifecycle | None = None, + *, + enqueue: bool = True, + ) -> None: + if not self._notifications_enabled: + return + from dataclasses import asdict + + row = await ( + await connection.execute( + "SELECT scope FROM reporting_issue_status_scopes" + " WHERE account_id = %s AND issue_id = %s", + (issue.account_id, issue.issue_id), + ) + ).fetchone() + existing = decode_status_scope(row[0]) if row else None + scope = ( + status_scope + or existing + or ReportingStatusScope(issue.account_id, consumer_id=issue.consumer_id) + ) + if ( + scope.account_id != issue.account_id + or scope.consumer_id != issue.consumer_id + or (existing is not None and scope != existing) + ): + raise ReportingNotificationError("invalid_status_scope") + if scope.generation_key is not None: + key = scope.generation_key + configuration = await ( + await connection.execute( + "SELECT 1 FROM reporting_configurations WHERE account_id = %s" + " AND delivery_config_id = %s AND delivery_config_version = %s", + (scope.account_id, key.delivery_config_id, key.delivery_config_version), + ) + ).fetchone() + if configuration is None: + raise ReportingNotificationError("invalid_status_scope") + if scope.reporting_obligation_id is not None: + obligation = await ( + await connection.execute( + "SELECT delivery_config_id, delivery_config_version, feed_purpose" + " FROM reporting_obligations WHERE account_id = %s" + " AND reporting_obligation_id = %s", + (scope.account_id, scope.reporting_obligation_id), + ) + ).fetchone() + if ( + obligation is None + or ( + scope.generation_key is not None + and obligation[:2] + != ( + scope.generation_key.delivery_config_id, + scope.generation_key.delivery_config_version, + ) + ) + or (scope.feed_purpose is not None and obligation[2] != scope.feed_purpose) + ): + raise ReportingNotificationError("invalid_status_scope") + if not enqueue: + return + await connection.execute( + "INSERT INTO reporting_issue_status_scopes (account_id, issue_id, scope)" + " VALUES (%s,%s,%s::jsonb) ON CONFLICT (account_id, issue_id) DO NOTHING", + (issue.account_id, issue.issue_id, _json(asdict(scope))), + ) + await self._dirty_status( + connection, + scope, + "issue", + issue_evidence(before) if before else None, + issue_evidence(issue), + ) # -- change feed ------------------------------------------------------ @@ -211,8 +336,9 @@ async def put_configuration(self, configuration: ReportingConfiguration) -> None key = configuration.generation_key payload = _configuration_payload(configuration) digest = _fingerprint(payload) - async with self._pool.connection() as connection: - await connection.execute( + async with self._pool.connection() as connection, connection.transaction(): + await self._lock_account(connection, key.account_id) + inserted_cursor = await connection.execute( "INSERT INTO reporting_configurations" " (delivery_config_id, delivery_config_version, account_id," " report_definition_id, reporting_profile, feed_purpose, required_finality," @@ -221,7 +347,8 @@ async def put_configuration(self, configuration: ReportingConfiguration) -> None " authoritative_party, content_sha256)" " VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s::jsonb, %s::jsonb, %s, %s, %s, %s," " %s::jsonb, %s, %s)" - " ON CONFLICT (account_id, delivery_config_id, delivery_config_version) DO NOTHING", + " ON CONFLICT (account_id, delivery_config_id, delivery_config_version) DO NOTHING" + " RETURNING delivery_config_id", ( key.delivery_config_id, key.delivery_config_version, @@ -242,6 +369,7 @@ async def put_configuration(self, configuration: ReportingConfiguration) -> None digest, ), ) + inserted = await inserted_cursor.fetchone() # Check *after* the insert. ON CONFLICT waits for a concurrent # winner; a fresh READ COMMITTED statement sees its retained # content. A pre-insert check followed by DO NOTHING could silently @@ -262,6 +390,14 @@ async def put_configuration(self, configuration: ReportingConfiguration) -> None "version instead of editing a retained generation", ) + if inserted is not None and self._notifications_enabled: + await self._dirty_status( + connection, + ReportingStatusScope(configuration.account_id, configuration.generation_key), + "configuration", + after=configuration_evidence(configuration), + ) + async def list_configurations( self, *, account_id: str, delivery_config_ids: Sequence[str] | None = None ) -> tuple[ReportingConfiguration, ...]: @@ -291,7 +427,7 @@ async def commit_obligation( self, obligation: ReportingObligationRecord ) -> ReportingObligationRecord: key = obligation.generation_key - async with self._pool.connection() as connection: + async with self._pool.connection() as connection, connection.transaction(): await self._lock_account(connection, key.account_id) existing_row = await ( await connection.execute( @@ -364,6 +500,15 @@ async def commit_obligation( "obligation", obligation.reporting_obligation_id, ) + if self._notifications_enabled: + await self._dirty_status( + connection, + ReportingStatusScope.for_obligation(obligation), + "obligation", + after=ReportingStatusEvidence( + "obligation", obligation.reporting_obligation_id + ), + ) return obligation # Another worker won the period close; converge on its obligation. existing = await self.find_obligation( @@ -520,15 +665,33 @@ async def commit_revision( await connection.cursor().executemany( "INSERT INTO reporting_revision_rows" " (reporting_revision_id, ordinal, row_payload)" - " VALUES (%s, %s, %s::jsonb)", + " SELECT reporting_revision_id, %s, %s::jsonb FROM reporting_revisions" + " WHERE account_id = %s AND reporting_revision_id = %s", [ - (revision.reporting_revision_id, ordinal, _json(row)) + (ordinal, _json(row), revision.account_id, revision.reporting_revision_id) for ordinal, row in enumerate(rows) ], ) await self._append_change( connection, revision.account_id, "revision", revision.reporting_revision_id ) + if self._notifications_enabled: + from adcp.reporting.ledger.notification_events import revision_event + + await self._record_notification( + connection, revision_event(revision, await self._notification_now(connection)) + ) + await self._dirty_status( + connection, + ReportingStatusScope.for_obligation(_obligation_from_row(obligation)), + "revision", + after=ReportingStatusEvidence( + "revision", + revision.reporting_revision_id, + readable=revision.readable, + supersedes_id=revision.supersedes_reporting_revision_id, + ), + ) return revision @staticmethod @@ -627,17 +790,42 @@ async def read_revision_rows( async def set_revision_readable( self, *, account_id: str, reporting_revision_id: str, readable: bool ) -> None: - async with self._pool.connection() as connection: - updated = await ( + async with self._pool.connection() as connection, connection.transaction(): + await self._lock_account(connection, account_id) + existing = await ( await connection.execute( - "UPDATE reporting_revisions SET readable = %s" - " WHERE account_id = %s AND reporting_revision_id = %s" - " RETURNING reporting_revision_id", - (readable, account_id, reporting_revision_id), + "SELECT readable, reporting_obligation_id FROM reporting_revisions" + " WHERE account_id = %s AND reporting_revision_id = %s FOR UPDATE", + (account_id, reporting_revision_id), ) ).fetchone() - if updated is None: - raise LedgerConflictError("REVISION_NOT_FOUND", "no such revision for this account") + if existing is None: + raise LedgerConflictError("REVISION_NOT_FOUND", "no such revision for this account") + if existing[0] == readable: + return + await connection.execute( + "UPDATE reporting_revisions SET readable = %s" + " WHERE account_id = %s AND reporting_revision_id = %s", + (readable, account_id, reporting_revision_id), + ) + if self._notifications_enabled: + obligation = await ( + await connection.execute( + f"SELECT {_OBLIGATION_COLUMNS} FROM reporting_obligations" # nosec B608 + " WHERE account_id = %s AND reporting_obligation_id = %s", + (account_id, existing[1]), + ) + ).fetchone() + assert obligation is not None + await self._dirty_status( + connection, + ReportingStatusScope.for_obligation(_obligation_from_row(obligation)), + "readability", + ReportingStatusEvidence( + "revision", reporting_revision_id, readable=existing[0] + ), + ReportingStatusEvidence("revision", reporting_revision_id, readable=readable), + ) # -- adjustments ------------------------------------------------------ @@ -728,6 +916,19 @@ async def commit_adjustment( "adjustment", adjustment.reporting_adjustment_id, ) + if self._notifications_enabled: + from adcp.reporting.ledger.notification_events import adjustment_event + + await self._record_notification( + connection, + adjustment_event(adjustment, await self._notification_now(connection)), + ) + await self._dirty_status( + connection, + ReportingStatusScope.for_obligation(_obligation_from_row(obligation)), + "adjustment", + after=ReportingStatusEvidence("adjustment", adjustment.reporting_adjustment_id), + ) return adjustment async def list_adjustments( @@ -753,12 +954,12 @@ async def record_consumer_status( ) -> tuple[ConsumerStatusRecord, bool]: key = status.generation_key digest = _fingerprint(_consumer_status_payload(status)) - async with self._pool.connection() as connection: + async with self._pool.connection() as connection, connection.transaction(): + await self._lock_account(connection, status.account_id) replay = await self._replay(connection, status, digest) if replay is not None: return replay, False - await self._lock_account(connection, status.account_id) leaf = await ( await connection.execute( "SELECT reporting_status_id FROM reporting_consumer_statuses" @@ -836,6 +1037,23 @@ async def record_consumer_status( await self._append_change( connection, status.account_id, "consumer_status", status.reporting_status_id ) + if self._notifications_enabled: + await self._dirty_status( + connection, + ReportingStatusScope( + status.account_id, + status.generation_key, + status.reporting_obligation_id, + status.consumer_id, + ), + "consumer_status", + ReportingStatusEvidence("consumer_status", leaf[0]) if leaf else None, + ReportingStatusEvidence( + "consumer_status", + status.reporting_status_id, + supersedes_id=status.supersedes_reporting_status_id, + ), + ) return status, True @staticmethod @@ -928,14 +1146,18 @@ async def ensure_issue_opened( account_id: str, consumer_id: str | None, observed_at: datetime, + status_scope: ReportingStatusScope | None = None, ) -> ReportingIssueLifecycle: - async with self._pool.connection() as connection: + async with self._pool.connection() as connection, connection.transaction(): # Serialize per account so two concurrent readers of one condition # converge on one occurrence instead of both computing # generation = N + 1 and racing the partial unique index. await self._lock_account(connection, account_id) live = await self._live_issue(connection, issue_key, account_id) if live is not None: + if self._notifications_enabled and live.consumer_id != consumer_id: + raise ReportingNotificationError("invalid_status_scope") + await self._dirty_issue(connection, live, status_scope, enqueue=False) return live row = await ( await connection.execute( @@ -953,7 +1175,7 @@ async def ensure_issue_opened( " VALUES (%s, %s, %s, %s, %s, %s, 'open')", (issue_key, account_id, generation, issue_id, consumer_id, _utc(observed_at)), ) - return ReportingIssueLifecycle( + record = ReportingIssueLifecycle( issue_key=issue_key, issue_id=issue_id, account_id=account_id, @@ -962,6 +1184,8 @@ async def ensure_issue_opened( issue_state="open", generation=generation, ) + await self._dirty_issue(connection, record, status_scope) + return record async def set_issue_state( self, @@ -971,6 +1195,7 @@ async def set_issue_state( state: Literal["acknowledged", "waived"], at: datetime, external_ref: str | None = None, + status_scope: ReportingStatusScope | None = None, ) -> ReportingIssueLifecycle: if state not in {"acknowledged", "waived"}: raise LedgerConflictError( @@ -981,7 +1206,7 @@ async def set_issue_state( "degraded projection while the statement that caused it is still the " "consumer's current leaf", ) - async with self._pool.connection() as connection: + async with self._pool.connection() as connection, connection.transaction(): await self._lock_account(connection, account_id) live = await self._live_issue(connection, issue_key, account_id) if live is None: @@ -991,22 +1216,34 @@ async def set_issue_state( "reopened, and a recurrence gets a new occurrence", ) check_issue_state_transition(live.issue_state, state) + if state == live.issue_state and ( + not external_ref or external_ref == live.external_ref + ): + await self._dirty_issue(connection, live, status_scope, enqueue=False) + return live await connection.execute( "UPDATE reporting_issue_lifecycle" " SET issue_state = %s," " external_ref = COALESCE(%s, external_ref)," - " retired_at = CASE WHEN %s = 'waived' THEN %s ELSE retired_at END" + " retired_at = CASE WHEN %s = 'waived' AND issue_state <> 'waived'" + " THEN %s ELSE retired_at END" " WHERE account_id = %s AND issue_key = %s AND generation = %s", (state, external_ref, state, _utc(at), account_id, issue_key, live.generation), ) refreshed = await self._issue_row(connection, issue_key, account_id, live.generation) assert refreshed is not None + await self._dirty_issue(connection, refreshed, status_scope, live) return refreshed async def retire_issue( - self, *, issue_key: str, account_id: str, at: datetime + self, + *, + issue_key: str, + account_id: str, + at: datetime, + status_scope: ReportingStatusScope | None = None, ) -> ReportingIssueLifecycle | None: - async with self._pool.connection() as connection: + async with self._pool.connection() as connection, connection.transaction(): await self._lock_account(connection, account_id) live = await self._live_issue(connection, issue_key, account_id) if live is None or not issue_is_retirable(live.issue_state): @@ -1022,7 +1259,10 @@ async def retire_issue( " WHERE account_id = %s AND issue_key = %s AND generation = %s", (_utc(at), account_id, issue_key, live.generation), ) - return await self._issue_row(connection, issue_key, account_id, live.generation) + retired = await self._issue_row(connection, issue_key, account_id, live.generation) + assert retired is not None + await self._dirty_issue(connection, retired, status_scope, live) + return retired async def get_issue(self, *, issue_key: str, account_id: str) -> ReportingIssueLifecycle | None: async with self._pool.connection() as connection: diff --git a/src/adcp/reporting/ledger/reporting_notification_outbox.sql b/src/adcp/reporting/ledger/reporting_notification_outbox.sql new file mode 100644 index 000000000..ce0a2d5f6 --- /dev/null +++ b/src/adcp/reporting/ledger/reporting_notification_outbox.sql @@ -0,0 +1,148 @@ +-- #1168A: additive only; never rewrite/backfill retained ledger evidence. +-- A single DO makes direct application atomic even on an autocommit connection. +DO $outbox$ +BEGIN + PERFORM pg_advisory_xact_lock(hashtext('adcp.reporting.schema'), hashtext(current_schema())); + -- Require the reviewed foundation, including the durable reconciliation feed. + PERFORM currency FROM reporting_obligations LIMIT 0; + PERFORM canonical_content_digest, managed_control_totals FROM reporting_revisions LIMIT 0; + PERFORM consumer_id, max_sequence FROM reporting_reconciliation_heads LIMIT 0; + PERFORM change_id FROM reporting_reconciliation_changes LIMIT 0; + + CREATE TABLE IF NOT EXISTS reporting_notification_events ( + account_id TEXT COLLATE "C" NOT NULL, + notification_id TEXT COLLATE "C" NOT NULL, + notification_type TEXT COLLATE "C" NOT NULL CHECK + (notification_type IN ('reporting.ledger_changed', 'reporting.delivery_ready')), + cause_kind TEXT COLLATE "C" NOT NULL CHECK + (cause_kind IN ('revision_published', 'adjustment_published', 'materialization_ready')), + cause_id TEXT COLLATE "C" NOT NULL CHECK (length(cause_id) > 0), + cause_generation BIGINT NOT NULL CHECK (cause_generation > 0), + consumer_namespace TEXT COLLATE "C" NOT NULL, + fired_at TIMESTAMPTZ NOT NULL, + snapshot JSONB NOT NULL, + PRIMARY KEY (account_id, consumer_namespace, notification_id), + UNIQUE (account_id, consumer_namespace, notification_type, + cause_kind, cause_id, cause_generation), + CHECK ((notification_type = 'reporting.delivery_ready') = + (cause_kind = 'materialization_ready')), + CHECK ((cause_kind = 'materialization_ready') = (length(consumer_namespace) > 0)), + CHECK (snapshot->>'account_id' = account_id AND + snapshot->>'notification_id' = notification_id) + ); + CREATE TABLE IF NOT EXISTS reporting_notification_expansions ( + account_id TEXT COLLATE "C" NOT NULL, + consumer_namespace TEXT COLLATE "C" NOT NULL DEFAULT '', + notification_id TEXT COLLATE "C" NOT NULL, + emission_generation BIGINT NOT NULL CHECK (emission_generation > 0), + state TEXT NOT NULL DEFAULT 'pending' CHECK + (state IN ('pending', 'leased', 'complete', 'suppressed', 'quarantined')), + due_at TIMESTAMPTZ NOT NULL, + lease_token TEXT, + lease_expires_at TIMESTAMPTZ, + claim_count BIGINT NOT NULL DEFAULT 0, + error_code TEXT CHECK (error_code IN ( + 'network', 'retryable_http', 'permanent_http', 'signing_unavailable', + 'permanent_scope', 'subscription_unavailable', 'subscription_changed', + 'invalid_configuration', 'invalid_payload', 'integrity_failure', 'lease_expired')), + PRIMARY KEY (account_id, consumer_namespace, notification_id, emission_generation), + FOREIGN KEY (account_id, consumer_namespace, notification_id) + REFERENCES reporting_notification_events (account_id, consumer_namespace, notification_id) + ); + CREATE INDEX IF NOT EXISTS reporting_notification_expansions_due + ON reporting_notification_expansions (account_id, due_at) + WHERE state IN ('pending', 'leased'); + + CREATE TABLE IF NOT EXISTS reporting_notification_deliveries ( + account_id TEXT COLLATE "C" NOT NULL, + delivery_id TEXT COLLATE "C" NOT NULL, + subscriber_id TEXT COLLATE "C" NOT NULL, + principal_id TEXT COLLATE "C" NOT NULL, + notification_id TEXT COLLATE "C" NOT NULL, + notification_type TEXT COLLATE "C" NOT NULL, + emission_generation BIGINT NOT NULL CHECK (emission_generation > 0), + idempotency_key TEXT COLLATE "C" NOT NULL, + destination_sha256 TEXT NOT NULL, + subscription_fingerprint TEXT NOT NULL, + signing_scope_id TEXT COLLATE "C", + cause_kind TEXT COLLATE "C" NOT NULL, + cause_id TEXT COLLATE "C" NOT NULL, + cause_generation BIGINT NOT NULL CHECK (cause_generation > 0), + consumer_namespace TEXT COLLATE "C" NOT NULL, + auth_mode TEXT NOT NULL, + body_sha256 TEXT NOT NULL, + envelope_version INTEGER NOT NULL, + key_version TEXT COLLATE "C" NOT NULL, + envelope BYTEA NOT NULL, + state TEXT NOT NULL DEFAULT 'pending' CHECK + (state IN ('pending', 'leased', 'complete', 'suppressed', 'quarantined')), + due_at TIMESTAMPTZ NOT NULL, + lease_token TEXT, + lease_expires_at TIMESTAMPTZ, + claim_count BIGINT NOT NULL DEFAULT 0, + error_code TEXT CHECK (error_code IN ( + 'network', 'retryable_http', 'permanent_http', 'signing_unavailable', + 'permanent_scope', 'subscription_unavailable', 'subscription_changed', + 'invalid_configuration', 'invalid_payload', 'integrity_failure', 'lease_expired')), + PRIMARY KEY (account_id, consumer_namespace, delivery_id), + UNIQUE (account_id, consumer_namespace, notification_id, emission_generation, subscriber_id), + UNIQUE (account_id, consumer_namespace, idempotency_key), + FOREIGN KEY (account_id, consumer_namespace, notification_id, emission_generation) + REFERENCES reporting_notification_expansions + (account_id, consumer_namespace, notification_id, emission_generation) + ); + CREATE INDEX IF NOT EXISTS reporting_notification_deliveries_due + ON reporting_notification_deliveries (account_id, due_at) + WHERE state IN ('pending', 'leased'); + + CREATE TABLE IF NOT EXISTS reporting_status_dirty_heads ( + account_id TEXT COLLATE "C" PRIMARY KEY, + max_sequence BIGINT NOT NULL CHECK (max_sequence >= 0) + ); + CREATE TABLE IF NOT EXISTS reporting_status_dirty ( + account_id TEXT COLLATE "C" NOT NULL, + sequence BIGINT NOT NULL, + consumer_namespace TEXT COLLATE "C" NOT NULL, + scope_sha256 TEXT NOT NULL, + reason TEXT NOT NULL, + cause_id TEXT COLLATE "C" NOT NULL, + cause_generation BIGINT NOT NULL CHECK (cause_generation > 0), + snapshot JSONB NOT NULL, + PRIMARY KEY (account_id, sequence), + UNIQUE (account_id, consumer_namespace, scope_sha256, reason, cause_id, cause_generation) + ); + CREATE TABLE IF NOT EXISTS reporting_status_checkpoints ( + account_id TEXT COLLATE "C" NOT NULL, + projector_id TEXT COLLATE "C" NOT NULL, + sequence BIGINT NOT NULL CHECK (sequence >= 0), + PRIMARY KEY (account_id, projector_id) + ); + CREATE TABLE IF NOT EXISTS reporting_issue_status_scopes ( + account_id TEXT COLLATE "C" NOT NULL, + issue_id TEXT COLLATE "C" NOT NULL, + scope JSONB NOT NULL, + PRIMARY KEY (account_id, issue_id) + ); + + CREATE OR REPLACE FUNCTION reporting_notification_immutable() + RETURNS TRIGGER LANGUAGE plpgsql AS $function$ + BEGIN + RAISE EXCEPTION 'immutable reporting notification evidence' USING ERRCODE = '23514'; + END + $function$; + IF NOT EXISTS (SELECT 1 FROM pg_trigger + WHERE tgrelid = 'reporting_notification_events'::regclass + AND tgname = 'reporting_notification_immutable' AND NOT tgisinternal) THEN + CREATE TRIGGER reporting_notification_immutable BEFORE UPDATE OR DELETE + ON reporting_notification_events FOR EACH ROW + EXECUTE FUNCTION reporting_notification_immutable(); + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_trigger + WHERE tgrelid = 'reporting_status_dirty'::regclass + AND tgname = 'reporting_status_dirty_immutable' AND NOT tgisinternal) THEN + CREATE TRIGGER reporting_status_dirty_immutable BEFORE UPDATE OR DELETE + ON reporting_status_dirty FOR EACH ROW + EXECUTE FUNCTION reporting_notification_immutable(); + END IF; +END +$outbox$; diff --git a/src/adcp/reporting/ledger/store.py b/src/adcp/reporting/ledger/store.py index accd587b1..cb6059e2f 100644 --- a/src/adcp/reporting/ledger/store.py +++ b/src/adcp/reporting/ledger/store.py @@ -35,11 +35,12 @@ import base64 import hashlib import json -from collections.abc import Callable, Sequence +from collections.abc import AsyncIterator, Callable, Sequence +from contextlib import asynccontextmanager from copy import deepcopy from dataclasses import dataclass, replace from datetime import datetime, timezone -from typing import Any, Literal, Protocol, runtime_checkable +from typing import TYPE_CHECKING, Any, Literal, Protocol, runtime_checkable from adcp.reporting.canonical_json import canonical_json_utf8_v1 from adcp.reporting.currency import ( @@ -61,6 +62,18 @@ ReportingObligationRecord, ReportingRevisionRecord, ) +from adcp.reporting.ledger.notification_models import ( + DirtyReason, + ReportingDomainEvent, + ReportingNotificationError, + ReportingStatusEvidence, + ReportingStatusScope, + configuration_evidence, + issue_evidence, +) + +if TYPE_CHECKING: + from adcp.reporting.outbox.memory import NotificationState __all__ = [ "InMemoryReportingLedgerStore", @@ -587,7 +600,9 @@ class InMemoryReportingLedgerStore: place a test's ledger boundary at a deliberate instant. """ - def __init__(self, *, clock: Callable[[], datetime] | None = None) -> None: + def __init__( + self, *, clock: Callable[[], datetime] | None = None, notifications: bool = False + ) -> None: self._clock = clock or (lambda: datetime.now(timezone.utc)) self._lock = asyncio.Lock() self._configurations: dict[ReportingConfigurationGenerationKey, ReportingConfiguration] = {} @@ -613,6 +628,97 @@ def __init__(self, *, clock: Callable[[], datetime] | None = None) -> None: # high-water mark so a recurrence never reuses an id. self._issues: dict[tuple[str, str], ReportingIssueLifecycle] = {} self._issue_generations: dict[tuple[str, str], int] = {} + self._notification_state: NotificationState | None = None + if notifications: + from adcp.reporting.outbox.memory import NotificationState + + self._notification_state = NotificationState() + + @asynccontextmanager + async def _mutation(self) -> AsyncIterator[None]: + """Publish domain changes and notifications under one rollback boundary. + + Default-off stores retain their original lock cost. The reference + in-memory transaction copies retained values only when opted in. + """ + async with self._lock: + before = ( + deepcopy( + { + key: value + for key, value in vars(self).items() + if key not in {"_lock", "_clock"} + } + ) + if self._notification_state is not None + else None + ) + try: + yield + except BaseException: + if before is not None: + vars(self).update(before) + raise + + def _record_notification(self, event: ReportingDomainEvent) -> None: + if self._notification_state is not None: + self._notification_state.enqueue(event) + + def _dirty_status( + self, + scope: ReportingStatusScope, + reason: DirtyReason, + before: ReportingStatusEvidence | None = None, + after: ReportingStatusEvidence | None = None, + ) -> None: + if self._notification_state is not None: + self._notification_state.mark_dirty(scope, reason, self._clock(), before, after) + + def _dirty_issue( + self, + issue: ReportingIssueLifecycle, + status_scope: ReportingStatusScope | None, + before: ReportingIssueLifecycle | None = None, + *, + enqueue: bool = True, + ) -> None: + if self._notification_state is None: + return + key = (issue.account_id, issue.issue_id) + existing = self._notification_state.issue_scopes.get(key) + scope = ( + status_scope + or existing + or ReportingStatusScope(issue.account_id, consumer_id=issue.consumer_id) + ) + if ( + scope.account_id != issue.account_id + or scope.consumer_id != issue.consumer_id + or (existing is not None and scope != existing) + ): + raise ReportingNotificationError("invalid_status_scope") + if scope.generation_key is not None and scope.generation_key not in self._configurations: + raise ReportingNotificationError("invalid_status_scope") + if scope.reporting_obligation_id is not None: + obligation = self._obligations.get(scope.reporting_obligation_id) + if ( + obligation is None + or obligation.account_id != scope.account_id + or ( + scope.generation_key is not None + and obligation.generation_key != scope.generation_key + ) + or ( + scope.feed_purpose is not None and obligation.feed_purpose != scope.feed_purpose + ) + ): + raise ReportingNotificationError("invalid_status_scope") + if not enqueue: + return + self._notification_state.issue_scopes[key] = scope + self._dirty_status( + scope, "issue", issue_evidence(before) if before else None, issue_evidence(issue) + ) async def create_schema(self) -> None: return None @@ -625,7 +731,7 @@ def _append(self, account_id: str, kind: LedgerRecordKind, record_id: str) -> No async def put_configuration(self, configuration: ReportingConfiguration) -> None: reject_reserved_authoritative_party(configuration) - async with self._lock: + async with self._mutation(): key = configuration.generation_key existing = self._configurations.get(key) if existing is not None and _fingerprint(_config_payload(existing)) != _fingerprint( @@ -638,6 +744,13 @@ async def put_configuration(self, configuration: ReportingConfiguration) -> None "publish a new version instead of editing a retained generation", ) self._configurations[key] = configuration + if existing != configuration and self._notification_state is not None: + self._dirty_status( + ReportingStatusScope(configuration.account_id, configuration.generation_key), + "configuration", + before=configuration_evidence(existing) if existing is not None else None, + after=configuration_evidence(configuration), + ) async def list_configurations( self, *, account_id: str, delivery_config_ids: Sequence[str] | None = None @@ -655,7 +768,7 @@ async def list_configurations( async def commit_obligation( self, obligation: ReportingObligationRecord ) -> ReportingObligationRecord: - async with self._lock: + async with self._mutation(): key = ( obligation.generation_key, _utc(obligation.period.start).isoformat(), @@ -673,6 +786,12 @@ async def commit_obligation( self._obligations[obligation.reporting_obligation_id] = obligation self._obligation_by_period[key] = obligation.reporting_obligation_id self._append(obligation.account_id, "obligation", obligation.reporting_obligation_id) + if self._notification_state is not None: + self._dirty_status( + ReportingStatusScope.for_obligation(obligation), + "obligation", + after=ReportingStatusEvidence("obligation", obligation.reporting_obligation_id), + ) return obligation async def get_obligation( @@ -721,7 +840,7 @@ async def commit_revision( f"revision declares {revision.row_count} rows but {len(rows)} were supplied", ) validate_managed_revision_rows(revision, rows) - async with self._lock: + async with self._mutation(): identity = _revision_identity(revision) existing = self._revisions.get(revision.reporting_revision_id) if existing is not None: @@ -762,6 +881,20 @@ async def commit_revision( self._revision_identity[revision.reporting_revision_id] = identity self._rows[revision.reporting_revision_id] = tuple(dict(row) for row in rows) self._append(revision.account_id, "revision", revision.reporting_revision_id) + if self._notification_state is not None: + from adcp.reporting.ledger.notification_events import revision_event + + self._record_notification(revision_event(revision, self._clock())) + self._dirty_status( + ReportingStatusScope.for_obligation(obligation), + "revision", + after=ReportingStatusEvidence( + "revision", + revision.reporting_revision_id, + readable=revision.readable, + supersedes_id=revision.supersedes_reporting_revision_id, + ), + ) return revision def _require_current_leaf( @@ -833,18 +966,30 @@ async def read_revision_rows( async def set_revision_readable( self, *, account_id: str, reporting_revision_id: str, readable: bool ) -> None: - async with self._lock: + async with self._mutation(): existing = self._revisions.get(reporting_revision_id) if existing is None or existing.account_id != account_id: raise LedgerConflictError("REVISION_NOT_FOUND", "no such revision for this account") + if existing.readable == readable: + return self._revisions[reporting_revision_id] = replace(existing, readable=readable) + if self._notification_state is not None: + obligation = self._obligations[existing.reporting_obligation_id] + self._dirty_status( + ReportingStatusScope.for_obligation(obligation), + "readability", + ReportingStatusEvidence( + "revision", reporting_revision_id, readable=existing.readable + ), + ReportingStatusEvidence("revision", reporting_revision_id, readable=readable), + ) # -- adjustments ----------------------------------------------------- async def commit_adjustment( self, adjustment: ReportingAdjustmentRecord ) -> ReportingAdjustmentRecord: - async with self._lock: + async with self._mutation(): existing = self._adjustments.get(adjustment.reporting_adjustment_id) if existing is not None: if existing.account_id != adjustment.account_id: @@ -870,6 +1015,17 @@ async def commit_adjustment( ) self._adjustments[adjustment.reporting_adjustment_id] = adjustment self._append(adjustment.account_id, "adjustment", adjustment.reporting_adjustment_id) + if self._notification_state is not None: + from adcp.reporting.ledger.notification_events import adjustment_event + + self._record_notification(adjustment_event(adjustment, self._clock())) + self._dirty_status( + ReportingStatusScope.for_obligation( + self._obligations[revision.reporting_obligation_id] + ), + "adjustment", + after=ReportingStatusEvidence("adjustment", adjustment.reporting_adjustment_id), + ) return adjustment async def list_adjustments( @@ -905,7 +1061,7 @@ def _replay(self, status: ConsumerStatusRecord) -> ConsumerStatusRecord | None: async def record_consumer_status( self, status: ConsumerStatusRecord ) -> tuple[ConsumerStatusRecord, bool]: - async with self._lock: + async with self._mutation(): identity = _consumer_status_identity(status) existing = self._replay(status) if existing is not None: @@ -931,6 +1087,26 @@ async def record_consumer_status( self._statuses[status.reporting_status_id] = status self._status_identity[status.reporting_status_id] = identity self._append(status.account_id, "consumer_status", status.reporting_status_id) + if self._notification_state is not None: + self._dirty_status( + ReportingStatusScope( + status.account_id, + status.generation_key, + status.reporting_obligation_id, + status.consumer_id, + ), + "consumer_status", + ( + ReportingStatusEvidence("consumer_status", leaf.reporting_status_id) + if leaf is not None + else None + ), + ReportingStatusEvidence( + "consumer_status", + status.reporting_status_id, + supersedes_id=status.supersedes_reporting_status_id, + ), + ) return status, True def _current_status_leaf(self, chain_key: tuple[Any, ...]) -> ConsumerStatusRecord | None: @@ -988,11 +1164,15 @@ async def ensure_issue_opened( account_id: str, consumer_id: str | None, observed_at: datetime, + status_scope: ReportingStatusScope | None = None, ) -> ReportingIssueLifecycle: - async with self._lock: + async with self._mutation(): key = (account_id, issue_key) live = self._issues.get(key) if live is not None and live.live: + if self._notification_state is not None and live.consumer_id != consumer_id: + raise ReportingNotificationError("invalid_status_scope") + self._dirty_issue(live, status_scope, enqueue=False) return live generation = self._issue_generations.get(key, 0) + 1 self._issue_generations[key] = generation @@ -1006,6 +1186,7 @@ async def ensure_issue_opened( generation=generation, ) self._issues[key] = record + self._dirty_issue(record, status_scope) return record async def set_issue_state( @@ -1016,8 +1197,9 @@ async def set_issue_state( state: Literal["acknowledged", "waived"], at: datetime, external_ref: str | None = None, + status_scope: ReportingStatusScope | None = None, ) -> ReportingIssueLifecycle: - async with self._lock: + async with self._mutation(): if state not in {"acknowledged", "waived"}: raise LedgerConflictError( "ISSUE_STATE_NOT_OPERATOR_SETTABLE", @@ -1036,21 +1218,36 @@ async def set_issue_state( "reopened, and a recurrence gets a new occurrence", ) check_issue_state_transition(live.issue_state, state) + if state == live.issue_state and ( + not external_ref or external_ref == live.external_ref + ): + self._dirty_issue(live, status_scope, enqueue=False) + return live updated = replace( live, issue_state=state, external_ref=external_ref or live.external_ref, # Set only on the way into a retired state and never cleared: # a waived issue keeps the instant it was waived. - retired_at=_utc(at) if state == "waived" else live.retired_at, + retired_at=( + _utc(at) + if state == "waived" and live.issue_state != "waived" + else live.retired_at + ), ) self._issues[key] = updated + self._dirty_issue(updated, status_scope, live) return updated async def retire_issue( - self, *, issue_key: str, account_id: str, at: datetime + self, + *, + issue_key: str, + account_id: str, + at: datetime, + status_scope: ReportingStatusScope | None = None, ) -> ReportingIssueLifecycle | None: - async with self._lock: + async with self._mutation(): key = (account_id, issue_key) live = self._issues.get(key) if live is None or not live.live or not issue_is_retirable(live.issue_state): @@ -1063,6 +1260,7 @@ async def retire_issue( check_issue_state_transition(live.issue_state, "resolved") retired = replace(live, issue_state="resolved", retired_at=_utc(at)) self._issues[key] = retired + self._dirty_issue(retired, status_scope, live) return retired async def get_issue(self, *, issue_key: str, account_id: str) -> ReportingIssueLifecycle | None: diff --git a/src/adcp/reporting/outbox/__init__.py b/src/adcp/reporting/outbox/__init__.py new file mode 100644 index 000000000..290dccef6 --- /dev/null +++ b/src/adcp/reporting/outbox/__init__.py @@ -0,0 +1,71 @@ +"""Optional transactional reporting notifications. Status projection is not enabled.""" + +from typing import TYPE_CHECKING + +from adcp.reporting.ledger.notification_models import ( + AdjustmentPublished, + MaterializationReady, + ReportingDomainEvent, + ReportingNotificationError, + ReportingStatusDirty, + ReportingStatusEvidence, + ReportingStatusScope, + RevisionPublished, + validate_notification_payload, +) +from adcp.reporting.outbox.memory import InMemoryReportingOutbox +from adcp.reporting.outbox.models import ( + DeliveryBinding, + DeliveryLease, + DeliveryStatus, + ExpansionLease, + ReportingNotificationOutbox, + StoredDelivery, +) +from adcp.reporting.outbox.routing import ( + ReportingEnvelopeCipher, + ReportingLegacyAuthentication, + ReportingNotificationSubscription, + ReportingSigningMaterial, + ReportingSigningResolver, + ReportingSubscriptionResolver, +) +from adcp.reporting.outbox.worker import ReportingNotificationWorker + +if TYPE_CHECKING: + from adcp.reporting.outbox.pg import PgReportingOutbox + +__all__ = [ + "AdjustmentPublished", + "DeliveryBinding", + "DeliveryLease", + "DeliveryStatus", + "ExpansionLease", + "InMemoryReportingOutbox", + "MaterializationReady", + "PgReportingOutbox", + "ReportingDomainEvent", + "ReportingEnvelopeCipher", + "ReportingLegacyAuthentication", + "ReportingNotificationError", + "ReportingNotificationOutbox", + "ReportingNotificationSubscription", + "ReportingNotificationWorker", + "ReportingSigningMaterial", + "ReportingSigningResolver", + "ReportingStatusDirty", + "ReportingStatusEvidence", + "ReportingStatusScope", + "ReportingSubscriptionResolver", + "RevisionPublished", + "StoredDelivery", + "validate_notification_payload", +] + + +def __getattr__(name: str) -> object: + if name == "PgReportingOutbox": + from adcp.reporting.outbox.pg import PgReportingOutbox + + return PgReportingOutbox + raise AttributeError(name) diff --git a/src/adcp/reporting/outbox/_capabilities.py b/src/adcp/reporting/outbox/_capabilities.py new file mode 100644 index 000000000..f5d972df1 --- /dev/null +++ b/src/adcp/reporting/outbox/_capabilities.py @@ -0,0 +1,117 @@ +"""Conservative startup check for the optional notification capability fields.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from adcp.reporting.ledger.delivery import InMemoryReportingReconciliationStore +from adcp.reporting.ledger.delivery_models import ReportingDeliveryScope +from adcp.reporting.ledger.notification_models import ReportingNotificationError +from adcp.reporting.ledger.store import InMemoryReportingLedgerStore, ReportingLedgerStore +from adcp.reporting.outbox.memory import InMemoryReportingOutbox +from adcp.reporting.outbox.routing import ReportingEnvelopeCipher, ReportingNotificationSubscription + +if TYPE_CHECKING: + from adcp.reporting.outbox.worker import ReportingNotificationWorker + + +async def advertised_notifications( + worker: ReportingNotificationWorker, + ledger: ReportingLedgerStore, + *, + account_id: str, + ready_scope: ReportingDeliveryScope | None, +) -> dict[str, str | bool]: + from adcp.reporting.outbox.worker import ReportingNotificationWorker + + if ( + type(worker) is not ReportingNotificationWorker + or type(worker.cipher) is not ReportingEnvelopeCipher + ): + raise ReportingNotificationError("notification_chain_unready") + if type(worker.outbox) is InMemoryReportingOutbox: + if ( + type(ledger) not in {InMemoryReportingLedgerStore, InMemoryReportingReconciliationStore} + or worker.outbox._store is not ledger + or worker.outbox._store._notification_state is None + ): + raise ReportingNotificationError("notification_chain_unready") + else: + # Deliberately lazy: the base SDK and memory implementation need no PG extra. + from adcp.reporting.ledger.delivery_pg import PgReportingReconciliationStore + from adcp.reporting.ledger.pg import PgReportingLedgerStore + from adcp.reporting.outbox._schema import validate_schema + from adcp.reporting.outbox.pg import PgReportingOutbox + + if ( + type(worker.outbox) is not PgReportingOutbox + or not isinstance(worker.outbox, PgReportingOutbox) + or type(ledger) not in {PgReportingLedgerStore, PgReportingReconciliationStore} + or not isinstance(ledger, PgReportingLedgerStore) + or ledger._pool is not worker.outbox._pool + or not ledger._notifications_enabled + ): + raise ReportingNotificationError("notification_chain_unready") + async with ledger._pool.connection() as connection, connection.transaction(): + await validate_schema(connection) + + # A successful empty resolution is a configured service, not an error. + # Every configured mode must also have usable trusted signing material. + # This is a startup check; dispatch still repeats exact resolution per attempt. + for event_type in ("reporting.ledger_changed", "reporting.delivery_ready"): + subscriptions = await worker.subscriptions.list_active( + account_id=account_id, notification_type=event_type + ) + identifiers: set[str] = set() + for subscription in subscriptions: + if ( + type(subscription) is not ReportingNotificationSubscription + or subscription.account_id != account_id + or subscription.subscriber_id in identifiers + or event_type not in subscription.event_types + or not ( + subscription.active and subscription.authorized and subscription.proof_valid + ) + ): + raise ReportingNotificationError("notification_chain_unready") + ReportingNotificationSubscription.__post_init__(subscription) + identifiers.add(subscription.subscriber_id) + sender = await worker._sender(subscription) + await sender.aclose() + + result: dict[str, str | bool] = { + "ledger_notification": "reporting.ledger_changed", + "supports_webhook_activity": False, + } + if ready_scope is not None: + if ready_scope.principal.account_id != account_id: + raise ReportingNotificationError("notification_chain_unready") + # A capability string cannot turn Core into Managed. These concrete + # stores froze and validated the configuration/destination/obligation + # relationship transactionally. All references remain immutable. + if not isinstance(ledger, InMemoryReportingReconciliationStore): + from adcp.reporting.ledger.delivery_pg import PgReportingReconciliationStore + + if not isinstance(ledger, PgReportingReconciliationStore): + raise ReportingNotificationError("core_delivery_ready_forbidden") + binding = await ledger.get_destination_binding( + caller=ready_scope.principal, generation_key=ready_scope.generation_key + ) + frozen = await ledger.get_obligation_delivery(ready_scope) + obligation = await ledger.get_obligation( + account_id=account_id, reporting_obligation_id=ready_scope.reporting_obligation_id + ) + configurations = await ledger.list_configurations(account_id=account_id) + if ( + binding is None + or frozen is None + or obligation is None + or obligation.generation_key != ready_scope.generation_key + or not any( + config.generation_key == ready_scope.generation_key for config in configurations + ) + or frozen.currency != obligation.currency + ): + raise ReportingNotificationError("notification_chain_unready") + result["readiness_notification"] = "reporting.delivery_ready" + return result diff --git a/src/adcp/reporting/outbox/_schema.py b/src/adcp/reporting/outbox/_schema.py new file mode 100644 index 000000000..4efd748fa --- /dev/null +++ b/src/adcp/reporting/outbox/_schema.py @@ -0,0 +1,396 @@ +"""Read-only verification of the installed transactional reporting chain. + +The contract covers column types/nullability/defaults, validated constraints, +usable indexes, enabled triggers, and the actual guard function definitions. +Presence of tables, a version marker, or a caller-supplied capability is not a +readiness check. Catalog reads are schema-scoped; they read no tenant records. +""" + +from __future__ import annotations + +import hashlib +import json +from typing import Any + +from adcp.reporting.ledger.notification_models import ReportingNotificationError + +# Generated from the bundled five-step chain. Deliberate schema changes must +# update this contract and exercise both fresh and populated upgrade paths. +SCHEMA_CONTRACT: dict[str, str] = { + "function:reporting_adjustment_evidence_immutable()": ( + "6435361e1fcbbf1926ee1e11f13897fb" "9a660ccc5e5f4bbe93114077c1bea636" + ), + "function:reporting_canonical_evidence_immutable()": ( + "7c62d73989b1ddcff1340e4f2108bcb8" "60bd09c54659094260aff1ee3c8af335" + ), + "function:reporting_canonical_json(document jsonb)": ( + "d50759bae48ef38808360d4a59f55978" "0eb1d1b934c2ded883cb7e918159c3e9" + ), + "function:reporting_identity_sha256(value text)": ( + "93224ff42ac26d1550e63a3726227f7e" "2ba55fea2a3b5af8f6f6879f649432f5" + ), + "function:reporting_iso_utc(moment timestamp with time zone)": ( + "af73b04da507f3d5d8fc0994aa693a7e" "621fb7e339f380e309beb5af10c60058" + ), + "function:reporting_notification_immutable()": ( + "8bfe540ace33195ad5588d04e869a870" "3fa162501aef6e955b897f1cc9413d14" + ), + "function:reporting_obligation_currency_immutable()": ( + "03e654a4967f2e8d174c1530d49de312" "540891baf061a341669bfc15112496e9" + ), + "function:reporting_payload_keys(document jsonb)": ( + "48b1d45bddd9a43affd0b63ad0eb8874" "73849985d82ded01a6a5cae6d108b476" + ), + "function:reporting_payload_sha256(document jsonb)": ( + "b08900f152a3c21c0ba84e5f89826b9f" "29967837d5277ade6ccc8de0e84eaa0d" + ), + "function:reporting_receipt_advance()": ( + "9869d9d96df91934b82f83a7c6f1f1ca" "3e6139358424646700e1f42d2c1ab6d7" + ), + "function:reporting_receipt_head_exact()": ( + "7f8887f5ea1a5a938fb8dd789398adc8" "8461927cda4b80b3b7196d5d665d88cd" + ), + "function:reporting_receipt_terminal()": ( + "bff86010f9ac57af1eee22a78a7ab461" "6ed34185a06431f3bf0b138ea2bc4386" + ), + "function:reporting_reconciliation_change_guard()": ( + "5844dc57d056f89d74ecd938f41674cd" "1994cbf90132ba567f5953d3a9aaacef" + ), + "function:reporting_reconciliation_evidence(r reporting_reconciliation_records)": ( + "60e89af5cb7a88aae39da7875aa0bb7e" "daa361cc3fbd5d2ebb9c0e0009aa48ea" + ), + "function:reporting_reconciliation_guard()": ( + "a7624ca621e3c95932cd23f5b6683af1" "16463984439e567ed3fbbe64969b7da3" + ), + "function:reporting_reconciliation_head_guard()": ( + "a92fd0d4b284d1a75aab0d2afb9e3ea7" "0829118d3d970427482ac5f5a75ce7c1" + ), + "function:reporting_reconciliation_immutable()": ( + "cb1b0e47e2848ae79292f539f4c3fe85" "8b651db28434bdd468596b36cd08d9cb" + ), + "function:reporting_reconciliation_reference_immutable()": ( + "f9048965fa08e87e4b26ecd108cce370" "f19ad939e03486f32fba66a899e7bd05" + ), + "function:reporting_reconciliation_validate(r reporting_reconciliation_records)": ( + "309164066f02cc4d0f3ff66df47cf5c4" "ddddcc69525436634f94845e0d2864ed" + ), + "function:reporting_sorted_strings(document jsonb)": ( + "9c5ec19205737071bd5f9fe79563397b" "63ee5105cb190a4d2db01dc414691736" + ), + "function:reporting_sorted_totals(document jsonb)": ( + "d29826abcc96b3de6a27851d088376e8" "f9925d59d785c05fecec0bf6dbdc721c" + ), + "function:reporting_wire_digest(document jsonb)": ( + "eba0b3444ee72fbf3d5c467c6ca2c68c" "2dd057b82e4dfd24ae10688a1e0e53af" + ), + "reporting_adjustments:columns": ( + "21f185134755dc64bf18f6650712ae1d" "06405e92e26a75913ed26ee239cfa37a" + ), + "reporting_adjustments:constraints": ( + "603d958337b456b6130c75cee2c6b9e5" "114a40fa53f1f59be86c421215a0a5af" + ), + "reporting_adjustments:indexes": ( + "bb16320d04f5cc3c1d36ee1e1a1d63ab" "e4a3901f9e9ad0505104f4b7dd8983b4" + ), + "reporting_adjustments:triggers": ( + "307588c8df362a216195f22724c5b29e" "9751c687cc36c9990bba82979a1889e7" + ), + "reporting_configurations:columns": ( + "e8f6dd2f4d6b8cc90e578dbaef272c46" "57c231db0a6a1989db3a5f2ab89a7326" + ), + "reporting_configurations:constraints": ( + "8fe2605ba4fdf546e2ae09dad3c8f506" "96be067933dfa8ef1601633693edd6a9" + ), + "reporting_configurations:indexes": ( + "0691ad10621df08ea0876d1a705c2d43" "7dc06a21350f64b481056f1e70d3d13f" + ), + "reporting_configurations:triggers": ( + "a089938618c994c2c0a6d12993bb8078" "07d3f8eda1874d540e2f8f95fcb04a5d" + ), + "reporting_consumer_statuses:columns": ( + "ec88098deb17a31cb3d84c7b4d50c07a" "c81f32f3780f20532ca9bef2845ce2e0" + ), + "reporting_consumer_statuses:constraints": ( + "45afdf7bcf4cce7974ab6b0714c036fb" "c9d1a01d0bcb33052fcf74d8d098b5eb" + ), + "reporting_consumer_statuses:indexes": ( + "d9b2b4ef81d2809cafa3da5aec6a92dc" "8e0c6fb11ada4ab59fefe9a363dbdb0b" + ), + "reporting_consumer_statuses:triggers": ( + "4f53cda18c2baa0c0354bb5f9a3ecbe5" "ed12ab4d8e11ba873c2f11161202b945" + ), + "reporting_issue_lifecycle:columns": ( + "1291ebaa7ee1fd72bfabe1a72d6a10fc" "3bc0a7429ce2efe48e3974901c550d1b" + ), + "reporting_issue_lifecycle:constraints": ( + "60f4b8b3c39bc0f413f32541c2eb3ebc" "2bd0978d97ad2dab22fa57582b537c74" + ), + "reporting_issue_lifecycle:indexes": ( + "f8869b28a542094006b97cc10ac4927c" "888e5744ba5035fa9c124f94b0cad9eb" + ), + "reporting_issue_lifecycle:triggers": ( + "4f53cda18c2baa0c0354bb5f9a3ecbe5" "ed12ab4d8e11ba873c2f11161202b945" + ), + "reporting_issue_status_scopes:columns": ( + "8af4b6a4cf02f0beb52988a124cd0d44" "dd00170bfb27ddf8840aa181f5e35379" + ), + "reporting_issue_status_scopes:constraints": ( + "7be693d1c5e228f0b61bd2aca8470328" "7478623259546a0dda0ccc2df2071590" + ), + "reporting_issue_status_scopes:indexes": ( + "239b1b0172b2335f81b32d5b1cf72fc3" "f04cf4c30a0ec40b66ae87e9f35343ed" + ), + "reporting_issue_status_scopes:triggers": ( + "4f53cda18c2baa0c0354bb5f9a3ecbe5" "ed12ab4d8e11ba873c2f11161202b945" + ), + "reporting_ledger_changes:columns": ( + "d6f6a70deebd30f0a0f3cfac3d6e9581" "1b2d60a4204d2949b2abe898e432cd1f" + ), + "reporting_ledger_changes:constraints": ( + "0f56059184f30d8e32e6323015b0ca5b" "19cb7f4865c73ca3bb2bd4459005de91" + ), + "reporting_ledger_changes:indexes": ( + "04708d175375319a7fdad3385b5fd944" "30b64cd55ef9d79b7d01dec8401faa88" + ), + "reporting_ledger_changes:triggers": ( + "4f53cda18c2baa0c0354bb5f9a3ecbe5" "ed12ab4d8e11ba873c2f11161202b945" + ), + "reporting_notification_deliveries:columns": ( + "d96c42f82f3092c8cdee3bfd5712cc49" "34efffcf0d8d4e0c5a32ef8782cc3fa8" + ), + "reporting_notification_deliveries:constraints": ( + "9934d10675a54cc3cb4bf9dfde0b083c" "9bdc2df2c71f6d94d75a8629a8ef1240" + ), + "reporting_notification_deliveries:indexes": ( + "b393111322b0cb8ab8ea5a75078411e5" "4ce87a8770ecb5509ddcc92c36289fe9" + ), + "reporting_notification_deliveries:triggers": ( + "4f53cda18c2baa0c0354bb5f9a3ecbe5" "ed12ab4d8e11ba873c2f11161202b945" + ), + "reporting_notification_events:columns": ( + "7ca4437ca676ca4abeb9ee97e17277f1" "0d7ec0f5cdab2d4d64bd7ea75042f2b7" + ), + "reporting_notification_events:constraints": ( + "043076e5170a06867999db8a1c8d1589" "56be50116ec96647be88ea1a613d03b8" + ), + "reporting_notification_events:indexes": ( + "0f014658961d9cc532c4293b63ca62f7" "2fea2d377d0614568f5eab358bfea8fa" + ), + "reporting_notification_events:triggers": ( + "2e8fabd912e2ab93f4c2f888b7a75000" "1ef85e3c5779f0cb21ce4b2a0fc583fd" + ), + "reporting_notification_expansions:columns": ( + "716e73267e27ea1122e48821d996df76" "739f47e7560bb23648dab5efb2483577" + ), + "reporting_notification_expansions:constraints": ( + "d3256a3b17badf25b73f9ff62cf7c883" "fbdaa8915f11cf5e2d6e3a875bcda7c0" + ), + "reporting_notification_expansions:indexes": ( + "d8312996f9b5dc4c060ce308bb076a8a" "8aee45dc95d55f38f7475dc677868060" + ), + "reporting_notification_expansions:triggers": ( + "4f53cda18c2baa0c0354bb5f9a3ecbe5" "ed12ab4d8e11ba873c2f11161202b945" + ), + "reporting_obligations:columns": ( + "ed51fd558b29f46b11fa84abb0125af1" "be0d8ec3020df0bee4955b85f354fc75" + ), + "reporting_obligations:constraints": ( + "38594ba5b18657441707ac036d1b3a00" "93a41d4759acf98269e9bab44bda2940" + ), + "reporting_obligations:indexes": ( + "1c4f8f60d5a93eae9a7017640d958737" "220c7d57a3e9ef7ce1ce8fcd68b22869" + ), + "reporting_obligations:triggers": ( + "497b6f5ef2514960254fc0975bfe5cb6" "36e529634eb9baae027d14ec4cb0353e" + ), + "reporting_receipt_heads:columns": ( + "df430b84134c9d75645eacf3c78d5794" "ffabfae82a291970e35f16c5ce029d53" + ), + "reporting_receipt_heads:constraints": ( + "c6913cfe5697563e9477ce428ce672fc" "ef50c9255ac950b311c6cb4220349876" + ), + "reporting_receipt_heads:indexes": ( + "7e2484d7965549de6455c430af370cca" "1d9d645f3ab725c9da140aeab8d43d1b" + ), + "reporting_receipt_heads:triggers": ( + "f15506c66bc42c0df42a30e82a3d0bdc" "d739d50e7322c60574cb6bf8bf46ab25" + ), + "reporting_reconciliation_changes:columns": ( + "1b74719a83f904f551236a99149e4cc0" "3801b13a21bc0d20eee277c9a159b642" + ), + "reporting_reconciliation_changes:constraints": ( + "796834b9cebe998250662fcfb69a8d1c" "a66e759231ba4589fe35a813c7978044" + ), + "reporting_reconciliation_changes:indexes": ( + "c16f715b3cafbd731fd2564380c2bce7" "a8800fc72fdf8792bd8f569b36d93125" + ), + "reporting_reconciliation_changes:triggers": ( + "9cfc23afa441c2cd92fb7f653d86fe42" "e6b421a125cc654fbf910ef44e860622" + ), + "reporting_reconciliation_heads:columns": ( + "2fee5528f8f85bd6d40cc4b458a78c46" "de462d460a115c7ee9856f03846e7d2f" + ), + "reporting_reconciliation_heads:constraints": ( + "e0b0d4037ec594e3ca0c533d79504ac6" "71a29a6b5e334b8df5511beaea0a8ada" + ), + "reporting_reconciliation_heads:indexes": ( + "231d8a63c10130aaf95cbd801c68335e" "060dbd38d6c4e0c465cdb7170c56f635" + ), + "reporting_reconciliation_heads:triggers": ( + "571348e9aa132f91ba2452b4d1cf8205" "952f3075ce8e6320e976445a471385e8" + ), + "reporting_reconciliation_records:columns": ( + "a7ec44cc41ad8cf17c9d1ffe8e9552f9" "dcda5fb434c95362d224a0e335db60aa" + ), + "reporting_reconciliation_records:constraints": ( + "184cda4ff4e6fd0d340c3238ad549068" "d7e07fe5628f030be0f9113a7bf694c7" + ), + "reporting_reconciliation_records:indexes": ( + "6beb345279c860a3d67bf55d00d9f723" "ab7bdd685b04c5d1f8e463f6ad29df98" + ), + "reporting_reconciliation_records:triggers": ( + "7fc917df860adedda86f1cbd68cccbf9" "f51b5d34d0eacb35c8bd645c8e230007" + ), + "reporting_revision_rows:columns": ( + "133e7beac71a52ea7c25a71b4f296297" "ecde85780eb6a28f4ff6d9847785861b" + ), + "reporting_revision_rows:constraints": ( + "829c1080b2db3344e8a67d30934a56b7" "6f934908148d662ea299c36cac1b40e2" + ), + "reporting_revision_rows:indexes": ( + "6aebb2613c50d6153b0edfe91db2a5b0" "7d5a3f8ddd90e3398c15b952c87de56b" + ), + "reporting_revision_rows:triggers": ( + "4f53cda18c2baa0c0354bb5f9a3ecbe5" "ed12ab4d8e11ba873c2f11161202b945" + ), + "reporting_revisions:columns": ( + "d861397cc8fd6986a387740a8f4a6084" "69504715e967ec438ff7d6af1aaa9540" + ), + "reporting_revisions:constraints": ( + "b6ed0e9e662bdfa3c2369fe9537b64a2" "9126e62a0f34c2c332824c51447078cb" + ), + "reporting_revisions:indexes": ( + "26df1e37323d185f21ef652224c15445" "39d03eeef282137dc8fdf86406fc4af0" + ), + "reporting_revisions:triggers": ( + "ab2267a69931c6ea860822156376d227" "28fbc7304b4ac7f9193afb3141448e16" + ), + "reporting_status_checkpoints:columns": ( + "665a64ed1e0f1546c4e15dc60cc71ec3" "16ecb701ca87ea809263c3ad1499388b" + ), + "reporting_status_checkpoints:constraints": ( + "936a423446f30aa77a1a30e2629f5da8" "4b798934f24ef855d0ec40bc50089dff" + ), + "reporting_status_checkpoints:indexes": ( + "9827d3c92209487636d64391d156414b" "9a36cef0d9971a47613b3df1bf572d01" + ), + "reporting_status_checkpoints:triggers": ( + "4f53cda18c2baa0c0354bb5f9a3ecbe5" "ed12ab4d8e11ba873c2f11161202b945" + ), + "reporting_status_dirty:columns": ( + "d93019d63daca890b024760abaf313d5" "3cce8be654f6951ac7c1d5c98be47f34" + ), + "reporting_status_dirty:constraints": ( + "a9623edb2feb4e3be913b9ee8eac1ae0" "5d301d0fedf997fc77cfb12adc76fe76" + ), + "reporting_status_dirty:indexes": ( + "b4542cf6d4be2a90969056d33ca6a61f" "9fdf2cd02be2f59a44913ec3e835bae2" + ), + "reporting_status_dirty:triggers": ( + "f8812273d8b9c664f5154737298c5837" "1e57e910b8a72a5b3b1ea73427414db9" + ), + "reporting_status_dirty_heads:columns": ( + "29981f0f8145ce9a2f07370fe19550ce" "076fcd8faf32d627453566f9998aa11b" + ), + "reporting_status_dirty_heads:constraints": ( + "a21b12facd96ab7c2031cf0b7e1a57ec" "112f60f369fa393680e87b6b6de9fa79" + ), + "reporting_status_dirty_heads:indexes": ( + "90afb31ce0eccf93a47bdcef2334dafb" "fe09a73fba9c55db977d67871a7221ae" + ), + "reporting_status_dirty_heads:triggers": ( + "4f53cda18c2baa0c0354bb5f9a3ecbe5" "ed12ab4d8e11ba873c2f11161202b945" + ), +} + + +async def schema_contract(connection: Any) -> dict[str, str]: + tables = await ( + await connection.execute( + "SELECT c.oid, c.relname FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace" + " WHERE n.nspname = current_schema() AND c.relkind = 'r'" + " AND starts_with(c.relname, 'reporting_') ORDER BY c.relname" + ) + ).fetchall() + result: dict[str, str] = {} + for oid, name in tables: + columns = await ( + await connection.execute( + "SELECT a.attname, format_type(a.atttypid, a.atttypmod), a.attnotnull, co.collname," + " pg_get_expr(d.adbin, d.adrelid) FROM pg_attribute a" + " LEFT JOIN pg_attrdef d ON d.adrelid = a.attrelid AND d.adnum = a.attnum" + " LEFT JOIN pg_collation co ON co.oid = a.attcollation" + " WHERE a.attrelid = %s AND a.attnum > 0 AND NOT a.attisdropped ORDER BY a.attname", + (oid,), + ) + ).fetchall() + constraints = await ( + await connection.execute( + "SELECT contype, pg_get_constraintdef(oid), convalidated FROM pg_constraint" + " WHERE conrelid = %s ORDER BY contype, pg_get_constraintdef(oid)", + (oid,), + ) + ).fetchall() + indexes = await ( + await connection.execute( + "SELECT i.indisunique, i.indisvalid, i.indisready," + " ARRAY(SELECT pg_get_indexdef(i.indexrelid, k, true)" + " FROM generate_series(1, i.indnatts) k), pg_get_expr(i.indpred, i.indrelid)" + " FROM pg_index i WHERE i.indrelid = %s ORDER BY 1, 4, 5", + (oid,), + ) + ).fetchall() + triggers = await ( + await connection.execute( + "SELECT tgname, tgenabled, replace(pg_get_triggerdef(oid)," + " quote_ident(current_schema()) || '.', '') FROM pg_trigger" + " WHERE tgrelid = %s AND NOT tgisinternal ORDER BY tgname", + (oid,), + ) + ).fetchall() + # Keep a separate hash for each layer so diagnosis is local and static, + # without exposing DDL, tenant data, or arbitrary database diagnostics. + for kind, value in ( + ("columns", columns), + ("constraints", constraints), + ("indexes", indexes), + ("triggers", triggers), + ): + result[f"{name}:{kind}"] = _digest(value) + functions = await ( + await connection.execute( + "SELECT p.proname, pg_get_function_identity_arguments(p.oid), p.prosrc," + " l.lanname, p.provolatile, p.proisstrict, p.prosecdef, p.proconfig," + " pg_get_function_result(p.oid) FROM pg_proc p" + " JOIN pg_namespace n ON n.oid = p.pronamespace JOIN pg_language l ON l.oid = p.prolang" + " WHERE n.nspname = current_schema() AND starts_with(p.proname, 'reporting_')" + " ORDER BY p.proname, pg_get_function_identity_arguments(p.oid)" + ) + ).fetchall() + for row in functions: + result[f"function:{row[0]}({row[1]})"] = _digest(row[2:]) + return result + + +def _digest(value: object) -> str: + return hashlib.sha256( + json.dumps(value, sort_keys=True, separators=(",", ":")).encode() + ).hexdigest() + + +async def validate_schema(connection: Any) -> None: + installed = await schema_contract(connection) + if not SCHEMA_CONTRACT or any( + installed.get(key) != value for key, value in SCHEMA_CONTRACT.items() + ): + raise ReportingNotificationError("notification_schema_unready") diff --git a/src/adcp/reporting/outbox/_transport_logging.py b/src/adcp/reporting/outbox/_transport_logging.py new file mode 100644 index 000000000..7b86be784 --- /dev/null +++ b/src/adcp/reporting/outbox/_transport_logging.py @@ -0,0 +1,43 @@ +"""Keep httpx/httpcore's URL/header diagnostics out of reporting delivery logs. + +Filters are context-local: concurrent application HTTP traffic keeps its normal +logging configuration. No handlers, levels, or global record factory are changed. +The SDK-owned sender uses HTTP/1 and the connection logger; HTTP/2 is included +defensively and the conformance test audits the installed httpcore logger set. +""" + +from __future__ import annotations + +import logging +from collections.abc import Iterator +from contextlib import contextmanager +from contextvars import ContextVar + +_PROTECTED = ContextVar("adcp_reporting_protected_transport", default=False) +_LOGGER_NAMES = ( + "httpx", + "httpcore.connection", + "httpcore.http11", + "httpcore.http2", + "httpcore.proxy", + "httpcore.socks", +) + + +class _ReportingTransportFilter(logging.Filter): + def filter(self, record: logging.LogRecord) -> bool: + return not _PROTECTED.get() + + +_FILTER = _ReportingTransportFilter() + + +@contextmanager +def protected_transport_logs() -> Iterator[None]: + for name in _LOGGER_NAMES: + logging.getLogger(name).addFilter(_FILTER) + token = _PROTECTED.set(True) + try: + yield + finally: + _PROTECTED.reset(token) diff --git a/src/adcp/reporting/outbox/memory.py b/src/adcp/reporting/outbox/memory.py new file mode 100644 index 000000000..8a667e666 --- /dev/null +++ b/src/adcp/reporting/outbox/memory.py @@ -0,0 +1,332 @@ +"""Process-local outbox sharing the ledger's lock and rollback boundary.""" + +from __future__ import annotations + +from copy import deepcopy +from dataclasses import asdict, dataclass, field +from datetime import datetime, timedelta +from secrets import token_hex +from typing import TYPE_CHECKING + +from adcp.reporting.canonical_json import canonical_json_utf8_v1 +from adcp.reporting.evidence import aware_utc +from adcp.reporting.ledger.notification_models import ( + DirtyReason, + ReportingDomainEvent, + ReportingNotificationError, + ReportingStatusDirty, + ReportingStatusEvidence, + ReportingStatusScope, + event_storage, +) +from adcp.reporting.outbox.models import ( + ERROR_CODES, + DeliveryLease, + DeliveryStatus, + ErrorCode, + ExpansionLease, + StoredDelivery, + WorkState, +) + +if TYPE_CHECKING: + from adcp.reporting.ledger.store import InMemoryReportingLedgerStore + + +@dataclass +class _Work: + due_at: datetime + state: WorkState = "pending" + token: str | None = None + expires_at: datetime | None = None + attempts: int = 0 + error_code: ErrorCode | None = None + + def available(self, now: datetime) -> bool: + return self.due_at <= now and ( + self.state == "pending" + or (self.state == "leased" and self.expires_at is not None and self.expires_at <= now) + ) + + def held(self, token: str, now: datetime) -> bool: + return ( + self.state == "leased" + and self.token == token + and self.expires_at is not None + and self.expires_at > now + ) + + def claim(self, now: datetime, seconds: float) -> tuple[str, datetime]: + if seconds <= 0: + raise ValueError("lease_seconds must be positive") + self.token = token_hex(32) + self.expires_at = now + timedelta(seconds=seconds) + self.attempts += 1 + self.state = "leased" + return self.token, self.expires_at + + +@dataclass +class NotificationState: + events: dict[tuple[str, str], ReportingDomainEvent] = field(default_factory=dict) + causes: dict[tuple[str, str, str, str, str, int], str] = field(default_factory=dict) + expansions: dict[tuple[str, str, int], _Work] = field(default_factory=dict) + deliveries: dict[tuple[str, str], tuple[StoredDelivery, _Work]] = field(default_factory=dict) + emissions: set[tuple[str, str, int, str]] = field(default_factory=set) + dirty: list[ReportingStatusDirty] = field(default_factory=list) + checkpoints: dict[tuple[str, str], int] = field(default_factory=dict) + issue_scopes: dict[tuple[str, str], ReportingStatusScope] = field(default_factory=dict) + + def enqueue(self, event: ReportingDomainEvent) -> None: + """Internal synchronous participant; caller owns the ledger transaction.""" + existing_id = self.causes.get(event.causal_key) + if existing_id is not None: + if self.events[(event.account_id, existing_id)].cause != event.cause: + raise ReportingNotificationError("event_identity_conflict") + return + self.events[(event.account_id, event.notification_id)] = event + self.causes[event.causal_key] = event.notification_id + self.expansions[(event.account_id, event.notification_id, 1)] = _Work(event.fired_at) + + def mark_dirty( + self, + scope: ReportingStatusScope, + reason: DirtyReason, + at: datetime, + before: ReportingStatusEvidence | None = None, + after: ReportingStatusEvidence | None = None, + ) -> None: + import hashlib + + sequence = sum(item.scope.account_id == scope.account_id for item in self.dirty) + 1 + evidence = after or before + cause_id = ( + evidence.record_id + if evidence is not None + else hashlib.sha256(canonical_json_utf8_v1(asdict(scope))).hexdigest() + ) + generation = ( + sum( + item.scope == scope and item.reason == reason and item.cause_id == cause_id + for item in self.dirty + ) + + 1 + ) + self.dirty.append( + ReportingStatusDirty( + sequence, scope, reason, aware_utc(at), cause_id, generation, before, after + ) + ) + + +def validate_finish(state: WorkState, error_code: ErrorCode | None) -> None: + if state not in {"pending", "complete", "suppressed", "quarantined"}: + raise ValueError("invalid work transition") + if error_code is not None and error_code not in ERROR_CODES: + raise ValueError("invalid local error classification") + + +def _finish( + work: _Work, + *, + now: datetime, + state: WorkState, + error_code: ErrorCode | None, + retry_at: datetime | None, +) -> None: + validate_finish(state, error_code) + work.state, work.error_code = state, error_code + work.due_at = aware_utc(retry_at or now) + work.token = work.expires_at = None + + +class InMemoryReportingOutbox: + """View over an opted-in ledger. Recreating this view preserves its state.""" + + def __init__(self, store: InMemoryReportingLedgerStore) -> None: + if store._notification_state is None: + raise ValueError("construct the ledger with notifications=True") + self._store = store + + @property + def _state(self) -> NotificationState: + state = self._store._notification_state + assert state is not None + return state + + async def list_events(self, *, account_id: str) -> tuple[ReportingDomainEvent, ...]: + async with self._store._lock: + return tuple( + event for (owner, _), event in self._state.events.items() if owner == account_id + ) + + async def claim_expansion( + self, *, account_id: str, now: datetime, lease_seconds: float + ) -> ExpansionLease | None: + now = aware_utc(now) + async with self._store._lock: + for (owner, notification, generation), work in self._state.expansions.items(): + if owner == account_id and work.available(now): + token, expires = work.claim(now, lease_seconds) + event = self._state.events[(owner, notification)] + return ExpansionLease( + owner, + notification, + generation, + token, + expires, + work.attempts, + event_storage(event), + event.consumer_namespace, + ) + return None + + async def complete_expansion( + self, lease: ExpansionLease, deliveries: tuple[StoredDelivery, ...], *, now: datetime + ) -> bool: + async with self._store._mutation(): + work = self._state.expansions.get( + (lease.account_id, lease.notification_id, lease.emission_generation) + ) + if work is None or not work.held(lease.token, now): + return False + for delivery in deliveries: + binding = delivery.binding + if ( + binding.account_id != lease.account_id + or binding.notification_id != lease.notification_id + or binding.emission_generation != lease.emission_generation + or binding.consumer_namespace != lease.consumer_namespace + ): + raise ReportingNotificationError("invalid_configuration") + key = ( + binding.account_id, + binding.notification_id, + binding.emission_generation, + binding.subscriber_id, + ) + if key not in self._state.emissions: + self._state.deliveries[(binding.account_id, binding.delivery_id)] = ( + delivery, + _Work(now), + ) + self._state.emissions.add(key) + _finish(work, now=now, state="complete", error_code=None, retry_at=None) + return True + + async def finish_expansion( + self, + lease: ExpansionLease, + *, + now: datetime, + state: WorkState, + error_code: ErrorCode | None = None, + retry_at: datetime | None = None, + ) -> bool: + validate_finish(state, error_code) + async with self._store._lock: + work = self._state.expansions.get( + (lease.account_id, lease.notification_id, lease.emission_generation) + ) + if work is None or not work.held(lease.token, now): + return False + _finish(work, now=now, state=state, error_code=error_code, retry_at=retry_at) + return True + + async def reemit(self, *, account_id: str, notification_id: str, now: datetime) -> int: + async with self._store._lock: + if (account_id, notification_id) not in self._state.events: + raise ReportingNotificationError("event_unavailable") + generation = ( + max( + g + for a, n, g in self._state.expansions + if a == account_id and n == notification_id + ) + + 1 + ) + self._state.expansions[(account_id, notification_id, generation)] = _Work(now) + return generation + + async def claim_delivery( + self, *, account_id: str, now: datetime, lease_seconds: float + ) -> DeliveryLease | None: + now = aware_utc(now) + async with self._store._lock: + for (owner, _), (delivery, work) in self._state.deliveries.items(): + if owner == account_id and work.available(now): + token, expires = work.claim(now, lease_seconds) + return DeliveryLease(delivery, token, expires, work.attempts) + return None + + async def delivery_lease_current(self, lease: DeliveryLease, *, now: datetime) -> bool: + async with self._store._lock: + binding = lease.delivery.binding + item = self._state.deliveries.get((binding.account_id, binding.delivery_id)) + return item is not None and item[1].held(lease.token, now) + + async def finish_delivery( + self, + lease: DeliveryLease, + *, + now: datetime, + state: WorkState, + error_code: ErrorCode | None = None, + retry_at: datetime | None = None, + ) -> bool: + validate_finish(state, error_code) + async with self._store._lock: + binding = lease.delivery.binding + item = self._state.deliveries.get((binding.account_id, binding.delivery_id)) + if item is None or not item[1].held(lease.token, now): + return False + _finish(item[1], now=now, state=state, error_code=error_code, retry_at=retry_at) + return True + + async def list_deliveries(self, *, account_id: str) -> tuple[DeliveryStatus, ...]: + async with self._store._lock: + return tuple( + DeliveryStatus(delivery, work.state, work.attempts, work.due_at, work.error_code) + for (owner, _), (delivery, work) in self._state.deliveries.items() + if owner == account_id + ) + + async def mark_status_dirty( + self, scope: ReportingStatusScope, *, reason: DirtyReason, now: datetime + ) -> None: + """Additive sweep seam; this slice schedules no clock-driven work.""" + async with self._store._lock: + self._state.mark_dirty(scope, reason, now) + + async def read_status_dirty( + self, *, account_id: str, after: int = 0, limit: int = 100 + ) -> tuple[ReportingStatusDirty, ...]: + if not 1 <= limit <= 1000 or after < 0: + raise ValueError("invalid dirty checkpoint window") + async with self._store._lock: + return tuple( + deepcopy( + [ + item + for item in self._state.dirty + if item.scope.account_id == account_id and item.sequence > after + ][:limit] + ) + ) + + async def status_checkpoint(self, *, account_id: str, projector_id: str) -> int: + async with self._store._lock: + return self._state.checkpoints.get((account_id, projector_id), 0) + + async def advance_status_checkpoint( + self, *, account_id: str, projector_id: str, expected: int, through: int + ) -> bool: + async with self._store._lock: + maximum = sum(item.scope.account_id == account_id for item in self._state.dirty) + key = (account_id, projector_id) + if not 0 <= expected <= through <= maximum: + raise ValueError("invalid status checkpoint") + if self._state.checkpoints.get(key, 0) != expected: + return False + self._state.checkpoints[key] = through + return True diff --git a/src/adcp/reporting/outbox/models.py b/src/adcp/reporting/outbox/models.py new file mode 100644 index 000000000..bda3a86dc --- /dev/null +++ b/src/adcp/reporting/outbox/models.py @@ -0,0 +1,159 @@ +"""Persistence contract for independently leased fanout and HTTP work. + +These are additive protocols. Existing ReportingLedgerStore implementations do +not acquire any new required method or producer callback. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import datetime +from typing import Any, Literal, Protocol, TypeAlias + +from adcp.reporting.ledger.notification_models import ReportingDomainEvent, ReportingStatusDirty + +WorkState: TypeAlias = Literal["pending", "leased", "complete", "suppressed", "quarantined"] +ErrorCode: TypeAlias = Literal[ + "network", + "retryable_http", + "permanent_http", + "signing_unavailable", + "permanent_scope", + "subscription_unavailable", + "subscription_changed", + "invalid_configuration", + "invalid_payload", + "integrity_failure", + "lease_expired", +] +ERROR_CODES = frozenset( + { + "network", + "retryable_http", + "permanent_http", + "signing_unavailable", + "permanent_scope", + "subscription_unavailable", + "subscription_changed", + "invalid_configuration", + "invalid_payload", + "integrity_failure", + "lease_expired", + } +) + + +@dataclass(frozen=True) +class DeliveryBinding: + """Only non-secret routing identifiers/hashes may be stored in plaintext. + + Every field participates in authenticated encryption. destination_sha256 + binds the full URL (including query); the URL itself is encrypted. + """ + + account_id: str + delivery_id: str + subscriber_id: str + principal_id: str + notification_id: str + notification_type: str + emission_generation: int + idempotency_key: str + destination_sha256: str + subscription_fingerprint: str + signing_scope_id: str | None + cause_kind: str + cause_id: str + cause_generation: int + consumer_namespace: str + auth_mode: str + body_sha256: str + envelope_version: int + key_version: str + + +@dataclass(frozen=True) +class StoredDelivery: + binding: DeliveryBinding + envelope: bytes = field(repr=False) + + +@dataclass(frozen=True) +class ExpansionLease: + account_id: str + notification_id: str + emission_generation: int + token: str = field(repr=False) + expires_at: datetime + attempt_count: int + event: dict[str, Any] = field(repr=False) + consumer_namespace: str = "" + + +@dataclass(frozen=True) +class DeliveryLease: + delivery: StoredDelivery + token: str = field(repr=False) + expires_at: datetime + attempt_count: int + + +@dataclass(frozen=True) +class DeliveryStatus: + delivery: StoredDelivery + state: WorkState + attempt_count: int + due_at: datetime + error_code: ErrorCode | None + + +class ReportingNotificationOutbox(Protocol): + async def list_events(self, *, account_id: str) -> tuple[ReportingDomainEvent, ...]: ... + + async def claim_expansion( + self, *, account_id: str, now: datetime, lease_seconds: float + ) -> ExpansionLease | None: ... + + async def complete_expansion( + self, lease: ExpansionLease, deliveries: tuple[StoredDelivery, ...], *, now: datetime + ) -> bool: ... + + async def finish_expansion( + self, + lease: ExpansionLease, + *, + now: datetime, + state: WorkState, + error_code: ErrorCode | None = None, + retry_at: datetime | None = None, + ) -> bool: ... + + async def reemit(self, *, account_id: str, notification_id: str, now: datetime) -> int: ... + + async def claim_delivery( + self, *, account_id: str, now: datetime, lease_seconds: float + ) -> DeliveryLease | None: ... + + async def delivery_lease_current(self, lease: DeliveryLease, *, now: datetime) -> bool: ... + + async def finish_delivery( + self, + lease: DeliveryLease, + *, + now: datetime, + state: WorkState, + error_code: ErrorCode | None = None, + retry_at: datetime | None = None, + ) -> bool: ... + + async def list_deliveries(self, *, account_id: str) -> tuple[DeliveryStatus, ...]: ... + + async def read_status_dirty( + self, *, account_id: str, after: int = 0, limit: int = 100 + ) -> tuple[ReportingStatusDirty, ...]: ... + + async def status_checkpoint(self, *, account_id: str, projector_id: str) -> int: ... + + async def advance_status_checkpoint( + self, *, account_id: str, projector_id: str, expected: int, through: int + ) -> bool: ... diff --git a/src/adcp/reporting/outbox/pg.py b/src/adcp/reporting/outbox/pg.py new file mode 100644 index 000000000..e522053a9 --- /dev/null +++ b/src/adcp/reporting/outbox/pg.py @@ -0,0 +1,532 @@ +"""Account-scoped PostgreSQL outbox; HTTP never holds a database transaction.""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Callable +from dataclasses import asdict, astuple, fields +from datetime import datetime, timedelta +from secrets import token_hex +from typing import TYPE_CHECKING, Any + +from adcp.reporting.canonical_json import canonical_json_utf8_v1 +from adcp.reporting.evidence import aware_utc +from adcp.reporting.ledger.notification_models import ( + DirtyReason, + ReportingDomainEvent, + ReportingNotificationError, + ReportingStatusDirty, + ReportingStatusEvidence, + ReportingStatusScope, + decode_dirty, + decode_event, + dirty_storage, + event_storage, +) +from adcp.reporting.outbox.memory import validate_finish +from adcp.reporting.outbox.models import ( + DeliveryBinding, + DeliveryLease, + DeliveryStatus, + ErrorCode, + ExpansionLease, + StoredDelivery, + WorkState, +) + +if TYPE_CHECKING: + from psycopg_pool import AsyncConnectionPool + +_BINDING_NAMES = tuple(item.name for item in fields(DeliveryBinding)) +_BINDING_COLUMNS = ", ".join(_BINDING_NAMES) +_DELIVERY_COLUMNS = _BINDING_COLUMNS + ", envelope" +_BINDING_SIZE = len(_BINDING_NAMES) + + +async def database_now(connection: Any, clock: Callable[[], datetime] | None) -> datetime: + """DB time in production; a deliberately injected clock for conformance.""" + if clock is not None: + return aware_utc(clock()) + row = await (await connection.execute("SELECT clock_timestamp()")).fetchone() + assert row is not None + at: datetime = row[0] + return at + + +async def enqueue_event(connection: Any, event: ReportingDomainEvent) -> None: + """Private transaction participant. Never acquire another connection here.""" + consumer = event.consumer_namespace + inserted = await ( + await connection.execute( + "INSERT INTO reporting_notification_events" + " (account_id, notification_id, notification_type, cause_kind, cause_id," + " cause_generation, consumer_namespace, fired_at, snapshot)" + " VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s::jsonb)" + " ON CONFLICT (account_id, consumer_namespace, notification_type," + " cause_kind, cause_id, cause_generation) DO NOTHING RETURNING notification_id", + ( + event.account_id, + event.notification_id, + event.notification_type, + event.cause.kind, + event.cause_id, + event.cause_generation, + consumer, + event.fired_at, + json.dumps(event_storage(event)), + ), + ) + ).fetchone() + if inserted is None: + row = await ( + await connection.execute( + "SELECT snapshot FROM reporting_notification_events WHERE account_id = %s" + " AND consumer_namespace = %s AND notification_type = %s AND cause_kind = %s" + " AND cause_id = %s AND cause_generation = %s", + ( + event.account_id, + consumer, + event.notification_type, + event.cause.kind, + event.cause_id, + event.cause_generation, + ), + ) + ).fetchone() + if row is None or decode_event(row[0]).cause != event.cause: + raise ReportingNotificationError("event_identity_conflict") + return + await connection.execute( + "INSERT INTO reporting_notification_expansions" + " (account_id, consumer_namespace, notification_id, emission_generation, due_at)" + " VALUES (%s,%s,%s,1,%s)", + (event.account_id, consumer, event.notification_id, event.fired_at), + ) + + +async def mark_dirty( + connection: Any, + scope: ReportingStatusScope, + reason: DirtyReason, + at: datetime, + before: ReportingStatusEvidence | None = None, + after: ReportingStatusEvidence | None = None, +) -> None: + from adcp.reporting.ledger.pg import PgReportingLedgerStore + + await PgReportingLedgerStore._lock_account(connection, scope.account_id) + scope_hash = hashlib.sha256(canonical_json_utf8_v1(asdict(scope))).hexdigest() + evidence = after or before + cause_id = evidence.record_id if evidence is not None else scope_hash + consumer = scope.consumer_id or "" + row = await ( + await connection.execute( + "SELECT COALESCE(max(cause_generation), 0) + 1 FROM reporting_status_dirty" + " WHERE account_id = %s AND consumer_namespace = %s AND scope_sha256 = %s" + " AND reason = %s AND cause_id = %s", + (scope.account_id, consumer, scope_hash, reason, cause_id), + ) + ).fetchone() + assert row is not None + generation = row[0] + row = await ( + await connection.execute( + "INSERT INTO reporting_status_dirty_heads (account_id, max_sequence) VALUES (%s,1)" + " ON CONFLICT (account_id) DO UPDATE SET max_sequence =" + " reporting_status_dirty_heads.max_sequence + 1 RETURNING max_sequence", + (scope.account_id,), + ) + ).fetchone() + assert row is not None + record = ReportingStatusDirty(row[0], scope, reason, at, cause_id, generation, before, after) + await connection.execute( + "INSERT INTO reporting_status_dirty (account_id, sequence, consumer_namespace," + " scope_sha256, reason, cause_id, cause_generation, snapshot)" + " VALUES (%s,%s,%s,%s,%s,%s,%s,%s::jsonb)", + ( + scope.account_id, + record.sequence, + consumer, + scope_hash, + reason, + cause_id, + generation, + json.dumps(dirty_storage(record)), + ), + ) + + +def _delivery(row: Any) -> StoredDelivery: + return StoredDelivery(DeliveryBinding(*row[:_BINDING_SIZE]), bytes(row[_BINDING_SIZE])) + + +class _LostLeaseError(Exception): + pass + + +class PgReportingOutbox: + """Caller-owned pool, database time, expiring random tokens, fenced writes. + + ``now`` arguments implement the shared memory protocol. In PostgreSQL they + never override the database clock; only the explicit constructor ``clock`` + seam does, for deterministic tests. Retry *durations* are applied to DB time. + Claims are counted for diagnostics, never as an HTTP retry limit. Evidence + and prepared bindings are retained indefinitely; this slice has no purge. + """ + + def __init__( + self, *, pool: AsyncConnectionPool, clock: Callable[[], datetime] | None = None + ) -> None: + self._pool, self._clock = pool, clock + + async def create_schema(self) -> None: + from adcp.reporting.ledger.pg import PgReportingLedgerStore + + await PgReportingLedgerStore(pool=self._pool, notifications=True).create_schema() + + async def list_events(self, *, account_id: str) -> tuple[ReportingDomainEvent, ...]: + async with self._pool.connection() as conn: + rows = await ( + await conn.execute( + "SELECT snapshot FROM reporting_notification_events WHERE account_id = %s" + " ORDER BY fired_at, notification_id", + (account_id,), + ) + ).fetchall() + events = tuple(decode_event(row[0]) for row in rows) + if any(event.account_id != account_id for event in events): + raise ReportingNotificationError("invalid_event") + return events + + async def claim_expansion( + self, *, account_id: str, now: datetime, lease_seconds: float + ) -> ExpansionLease | None: + if lease_seconds <= 0: + raise ValueError("lease_seconds must be positive") + async with self._pool.connection() as conn, conn.transaction(): + at = await database_now(conn, self._clock) + row = await ( + await conn.execute( + "SELECT x.notification_id, x.emission_generation, x.claim_count, e.snapshot," + " x.consumer_namespace" + " FROM reporting_notification_expansions x JOIN reporting_notification_events e" + " ON e.account_id = x.account_id AND e.notification_id = x.notification_id" + " AND e.consumer_namespace = x.consumer_namespace" + " WHERE x.account_id = %s AND x.due_at <= %s AND (x.state = 'pending' OR" + " (x.state = 'leased' AND x.lease_expires_at <= %s))" + " ORDER BY x.due_at, x.notification_id, x.emission_generation" + " FOR UPDATE OF x SKIP LOCKED LIMIT 1", + (account_id, at, at), + ) + ).fetchone() + if row is None: + return None + token, expires = token_hex(32), at + timedelta(seconds=lease_seconds) + await conn.execute( + "UPDATE reporting_notification_expansions SET state = 'leased', lease_token = %s," + " lease_expires_at = %s, claim_count = claim_count + 1" + " WHERE account_id = %s AND consumer_namespace = %s" + " AND notification_id = %s AND emission_generation = %s", + (token, expires, account_id, row[4], row[0], row[1]), + ) + return ExpansionLease( + account_id, row[0], row[1], token, expires, row[2] + 1, row[3], row[4] + ) + + async def complete_expansion( + self, lease: ExpansionLease, deliveries: tuple[StoredDelivery, ...], *, now: datetime + ) -> bool: + try: + async with self._pool.connection() as conn, conn.transaction(): + at = await database_now(conn, self._clock) + row = await ( + await conn.execute( + "SELECT 1 FROM reporting_notification_expansions WHERE account_id = %s" + " AND consumer_namespace = %s" + " AND notification_id = %s AND emission_generation = %s" + " AND state = 'leased'" + " AND lease_token = %s AND lease_expires_at > %s FOR UPDATE", + ( + lease.account_id, + lease.consumer_namespace, + lease.notification_id, + lease.emission_generation, + lease.token, + at, + ), + ) + ).fetchone() + if row is None: + return False + for delivery in deliveries: + binding = delivery.binding + if ( + binding.account_id != lease.account_id + or binding.notification_id != lease.notification_id + or binding.emission_generation != lease.emission_generation + or binding.consumer_namespace != lease.consumer_namespace + ): + raise ReportingNotificationError("invalid_configuration") + await self._insert_delivery(conn, delivery, at) + if not await self._finish_expansion( + conn, lease, state="complete", error_code=None, delay=0 + ): + # Do not commit N subscriber rows if the final fence failed. + raise _LostLeaseError + return True + except _LostLeaseError: + return False + + async def _insert_delivery(self, conn: Any, delivery: StoredDelivery, at: datetime) -> None: + placeholders = ",".join(["%s"] * (_BINDING_SIZE + 2)) + await conn.execute( + f"INSERT INTO reporting_notification_deliveries ({_DELIVERY_COLUMNS}, due_at)" # nosec B608 + f" VALUES ({placeholders}) ON CONFLICT" # nosec B608 + " (account_id, consumer_namespace, notification_id, emission_generation, subscriber_id)" + " DO NOTHING", + (*astuple(delivery.binding), delivery.envelope, at), + ) + + async def _finish_expansion( + self, + conn: Any, + lease: ExpansionLease, + *, + state: WorkState, + error_code: ErrorCode | None, + delay: float, + ) -> bool: + at = await database_now(conn, self._clock) + cursor = await conn.execute( + "UPDATE reporting_notification_expansions SET state = %s, error_code = %s," + " due_at = %s, lease_token = NULL, lease_expires_at = NULL" + " WHERE account_id = %s AND notification_id = %s AND emission_generation = %s" + " AND consumer_namespace = %s" + " AND state = 'leased' AND lease_token = %s AND lease_expires_at > %s", + ( + state, + error_code, + at + timedelta(seconds=delay), + lease.account_id, + lease.notification_id, + lease.emission_generation, + lease.consumer_namespace, + lease.token, + at, + ), + ) + return bool(cursor.rowcount) + + async def finish_expansion( + self, + lease: ExpansionLease, + *, + now: datetime, + state: WorkState, + error_code: ErrorCode | None = None, + retry_at: datetime | None = None, + ) -> bool: + validate_finish(state, error_code) + delay = max(0.0, (retry_at - now).total_seconds()) if retry_at is not None else 0.0 + async with self._pool.connection() as conn, conn.transaction(): + return await self._finish_expansion( + conn, lease, state=state, error_code=error_code, delay=delay + ) + + async def reemit(self, *, account_id: str, notification_id: str, now: datetime) -> int: + async with self._pool.connection() as conn, conn.transaction(): + row = await ( + await conn.execute( + "SELECT consumer_namespace FROM reporting_notification_events" + " WHERE account_id = %s" + " AND notification_id = %s FOR UPDATE", + (account_id, notification_id), + ) + ).fetchone() + if row is None: + raise ReportingNotificationError("event_unavailable") + consumer = row[0] + row = await ( + await conn.execute( + "SELECT max(emission_generation) + 1 FROM reporting_notification_expansions" + " WHERE account_id = %s AND notification_id = %s", + (account_id, notification_id), + ) + ).fetchone() + assert row is not None + generation = int(row[0]) + await conn.execute( + "INSERT INTO reporting_notification_expansions" + " (account_id, consumer_namespace, notification_id, emission_generation, due_at)" + " VALUES (%s,%s,%s,%s,%s)", + ( + account_id, + consumer, + notification_id, + generation, + await database_now(conn, self._clock), + ), + ) + return generation + + async def claim_delivery( + self, *, account_id: str, now: datetime, lease_seconds: float + ) -> DeliveryLease | None: + if lease_seconds <= 0: + raise ValueError("lease_seconds must be positive") + async with self._pool.connection() as conn, conn.transaction(): + at = await database_now(conn, self._clock) + row = await ( + await conn.execute( + f"SELECT {_DELIVERY_COLUMNS}, claim_count" # nosec B608 + " FROM reporting_notification_deliveries" + " WHERE account_id = %s AND due_at <= %s AND (state = 'pending' OR" + " (state = 'leased' AND lease_expires_at <= %s)) ORDER BY due_at, delivery_id" + " FOR UPDATE SKIP LOCKED LIMIT 1", + (account_id, at, at), + ) + ).fetchone() + if row is None: + return None + token, expires = token_hex(32), at + timedelta(seconds=lease_seconds) + await conn.execute( + "UPDATE reporting_notification_deliveries SET state = 'leased', lease_token = %s," + " lease_expires_at = %s, claim_count = claim_count + 1" + " WHERE account_id = %s AND consumer_namespace = %s AND delivery_id = %s", + ( + token, + expires, + account_id, + row[_BINDING_NAMES.index("consumer_namespace")], + row[1], + ), + ) + return DeliveryLease(_delivery(row), token, expires, row[-1] + 1) + + async def delivery_lease_current(self, lease: DeliveryLease, *, now: datetime) -> bool: + binding = lease.delivery.binding + async with self._pool.connection() as conn: + at = await database_now(conn, self._clock) + row = await ( + await conn.execute( + "SELECT 1 FROM reporting_notification_deliveries WHERE account_id = %s" + " AND consumer_namespace = %s" + " AND delivery_id = %s AND state = 'leased' AND lease_token = %s" + " AND lease_expires_at > %s", + ( + binding.account_id, + binding.consumer_namespace, + binding.delivery_id, + lease.token, + at, + ), + ) + ).fetchone() + return row is not None + + async def finish_delivery( + self, + lease: DeliveryLease, + *, + now: datetime, + state: WorkState, + error_code: ErrorCode | None = None, + retry_at: datetime | None = None, + ) -> bool: + validate_finish(state, error_code) + delay = max(0.0, (retry_at - now).total_seconds()) if retry_at is not None else 0.0 + binding = lease.delivery.binding + async with self._pool.connection() as conn, conn.transaction(): + at = await database_now(conn, self._clock) + cursor = await conn.execute( + "UPDATE reporting_notification_deliveries SET state = %s, error_code = %s," + " due_at = %s, lease_token = NULL, lease_expires_at = NULL" + " WHERE account_id = %s AND delivery_id = %s AND state = 'leased'" + " AND consumer_namespace = %s" + " AND lease_token = %s AND lease_expires_at > %s", + ( + state, + error_code, + at + timedelta(seconds=delay), + binding.account_id, + binding.delivery_id, + binding.consumer_namespace, + lease.token, + at, + ), + ) + return bool(cursor.rowcount) + + async def list_deliveries(self, *, account_id: str) -> tuple[DeliveryStatus, ...]: + async with self._pool.connection() as conn: + rows = await ( + await conn.execute( + f"SELECT {_DELIVERY_COLUMNS}, state, claim_count, due_at, error_code" # nosec B608 + " FROM reporting_notification_deliveries WHERE account_id = %s" + " ORDER BY due_at, delivery_id", + (account_id,), + ) + ).fetchall() + return tuple(DeliveryStatus(_delivery(row), *row[-4:]) for row in rows) + + async def mark_status_dirty( + self, scope: ReportingStatusScope, *, reason: DirtyReason, now: datetime + ) -> None: + """Additive clock-sweep handoff, not a scheduler or a status projector.""" + async with self._pool.connection() as conn, conn.transaction(): + await mark_dirty(conn, scope, reason, await database_now(conn, self._clock)) + + async def read_status_dirty( + self, *, account_id: str, after: int = 0, limit: int = 100 + ) -> tuple[ReportingStatusDirty, ...]: + if not 1 <= limit <= 1000 or after < 0: + raise ValueError("invalid dirty checkpoint window") + async with self._pool.connection() as conn: + rows = await ( + await conn.execute( + "SELECT snapshot FROM reporting_status_dirty WHERE account_id = %s" + " AND sequence > %s ORDER BY sequence LIMIT %s", + (account_id, after, limit), + ) + ).fetchall() + records = tuple(decode_dirty(row[0]) for row in rows) + if any(record.scope.account_id != account_id for record in records): + raise ReportingNotificationError("invalid_status_evidence") + return records + + async def status_checkpoint(self, *, account_id: str, projector_id: str) -> int: + async with self._pool.connection() as conn: + row = await ( + await conn.execute( + "SELECT sequence FROM reporting_status_checkpoints WHERE account_id = %s" + " AND projector_id = %s", + (account_id, projector_id), + ) + ).fetchone() + return int(row[0]) if row is not None else 0 + + async def advance_status_checkpoint( + self, *, account_id: str, projector_id: str, expected: int, through: int + ) -> bool: + async with self._pool.connection() as conn, conn.transaction(): + row = await ( + await conn.execute( + "SELECT max_sequence FROM reporting_status_dirty_heads WHERE account_id = %s", + (account_id,), + ) + ).fetchone() + maximum = int(row[0]) if row is not None else 0 + if not 0 <= expected <= through <= maximum: + raise ValueError("invalid status checkpoint") + await conn.execute( + "INSERT INTO reporting_status_checkpoints (account_id, projector_id, sequence)" + " VALUES (%s,%s,0) ON CONFLICT (account_id, projector_id) DO NOTHING", + (account_id, projector_id), + ) + cursor = await conn.execute( + "UPDATE reporting_status_checkpoints SET sequence = %s WHERE account_id = %s" + " AND projector_id = %s AND sequence = %s", + (through, account_id, projector_id, expected), + ) + return bool(cursor.rowcount) diff --git a/src/adcp/reporting/outbox/routing.py b/src/adcp/reporting/outbox/routing.py new file mode 100644 index 000000000..0a8cf29c0 --- /dev/null +++ b/src/adcp/reporting/outbox/routing.py @@ -0,0 +1,348 @@ +"""Trusted account registrations and authenticated, secret-free routing columns.""" + +from __future__ import annotations + +import base64 +import hashlib +import json +import secrets +from collections.abc import Mapping +from dataclasses import asdict, dataclass, field +from typing import Any, Literal, Protocol +from uuid import uuid4 + +import httpx +from cryptography.hazmat.primitives.asymmetric import ec, ed25519 +from cryptography.hazmat.primitives.ciphers.aead import AESGCM + +from adcp.reporting.canonical_json import canonical_json_utf8_v1 +from adcp.reporting.evidence import principal_reference, reporting_identifier +from adcp.reporting.ledger.notification_models import ( + MaterializationReady, + ReportingDomainEvent, + ReportingNotificationError, + decode_event, + event_storage, + validate_notification_payload, +) +from adcp.reporting.outbox.models import DeliveryBinding, StoredDelivery +from adcp.signing.crypto import ALG_ED25519, ALG_ES256, ALLOWED_ALGS, PrivateKey +from adcp.webhook_sender import PreparedWebhook + + +@dataclass(frozen=True) +class ReportingLegacyAuthentication: + scheme: Literal["Bearer", "HMAC-SHA256"] + credentials: str = field(repr=False) + + def __post_init__(self) -> None: + if ( + self.scheme not in {"Bearer", "HMAC-SHA256"} + or type(self.credentials) is not str + or not self.credentials + or not self.credentials.isprintable() + ): + raise ReportingNotificationError("invalid_configuration") + + +@dataclass(frozen=True) +class ReportingNotificationSubscription: + """One *trusted* normalized active account notification configuration. + + The resolver must read these fields together, from authorized principal + state and a successful account/subscriber/URL proof-of-control registration. + References are mandatory and part of the fingerprint; they are not a + substitute for performing those checks in the registration service. + Never construct this from a webhook body or an unauthenticated sync request. + """ + + account_id: str + subscriber_id: str + principal_id: str + url: str = field(repr=False) + event_types: tuple[str, ...] + configuration_revision: str + authorization_ref: str + proof_of_control_ref: str + signing_scope_id: str | None = None + authentication: ReportingLegacyAuthentication | None = field(default=None, repr=False) + active: bool = field(kw_only=True) + authorized: bool = field(kw_only=True) + proof_valid: bool = field(kw_only=True) + + def __post_init__(self) -> None: + try: + principal_reference(self.account_id) + principal_reference(self.principal_id) + reporting_identifier(self.subscriber_id, maximum=64) + for value in ( + self.configuration_revision, + self.authorization_ref, + self.proof_of_control_ref, + ): + reporting_identifier(value, maximum=255) + if self.signing_scope_id is not None: + principal_reference(self.signing_scope_id) + if any( + type(value) is not bool + for value in (self.active, self.authorized, self.proof_valid) + ): + raise ValueError + events = tuple(sorted(self.event_types)) + if not events or len(set(events)) != len(events): + raise ValueError + for event in events: + reporting_identifier(event, maximum=255) + object.__setattr__(self, "event_types", events) + if (self.authentication is None) == (self.signing_scope_id is None): + raise ValueError + if ( + self.authentication is not None + and type(self.authentication) is not ReportingLegacyAuthentication + ): + raise ValueError + if type(self.url) is not str or not self.url.isprintable(): + raise ValueError + parsed = httpx.URL(self.url) + if ( + parsed.scheme != "https" + or not parsed.host + or parsed.userinfo + or parsed.fragment + or parsed.port not in (None, 443) + ): + raise ValueError + object.__setattr__(self, "url", str(parsed)) + except (ValueError, TypeError, httpx.InvalidURL): + raise ReportingNotificationError("invalid_configuration") from None + + @property + def auth_mode(self) -> str: + return self.authentication.scheme if self.authentication is not None else "rfc9421" + + @property + def fingerprint(self) -> str: + return hashlib.sha256(canonical_json_utf8_v1(asdict(self))).hexdigest() + + def matches(self, event: ReportingDomainEvent) -> bool: + return ( + self.active + and self.authorized + and self.proof_valid + and self.account_id == event.account_id + and event.notification_type in self.event_types + and ( + not isinstance(event.cause, MaterializationReady) + or self.principal_id == event.cause.consumer_id + ) + ) + + +class ReportingSubscriptionResolver(Protocol): + """External state: active-at-expansion, never claimed transactionally atomic. + + Each result must come from one normalized trusted account configuration, + including current authorization and proof validity. Fanout reads once; the + worker repeats exact resolution immediately before *every* HTTP attempt. + """ + + async def list_active( + self, *, account_id: str, notification_type: str + ) -> tuple[ReportingNotificationSubscription, ...]: ... + + async def get_active( + self, *, account_id: str, subscriber_id: str, notification_type: str + ) -> ReportingNotificationSubscription | None: ... + + +@dataclass(frozen=True) +class ReportingSigningMaterial: + """Trusted current key material, not an adopter-supplied HTTP sender.""" + + private_key: PrivateKey = field(repr=False) + key_id: str + algorithm: str + advertised_algorithms: frozenset[str] + + def __post_init__(self) -> None: + object.__setattr__(self, "advertised_algorithms", frozenset(self.advertised_algorithms)) + key_matches = ( + self.algorithm == ALG_ED25519 + and isinstance(self.private_key, ed25519.Ed25519PrivateKey) + ) or ( + self.algorithm == ALG_ES256 + and isinstance(self.private_key, ec.EllipticCurvePrivateKey) + and isinstance(self.private_key.curve, ec.SECP256R1) + ) + if ( + self.algorithm not in ALLOWED_ALGS + or self.algorithm not in self.advertised_algorithms + or not self.advertised_algorithms.issubset(ALLOWED_ALGS) + or not key_matches + or not self.key_id + ): + raise ReportingNotificationError("permanent_scope") + + +class ReportingSigningResolver(Protocol): + async def resolve( + self, *, account_id: str, principal_id: str, signing_scope_id: str + ) -> ReportingSigningMaterial: ... + + +@dataclass(frozen=True) +class OpenedReportingDelivery: + subscription: ReportingNotificationSubscription = field(repr=False) + event: ReportingDomainEvent + prepared: PreparedWebhook = field(repr=False) + + +def _decode_subscription(value: Any) -> ReportingNotificationSubscription: + if not isinstance(value, dict): + raise ReportingNotificationError("integrity_failure") + candidate = dict(value) + auth = candidate.get("authentication") + if auth is not None: + candidate["authentication"] = ReportingLegacyAuthentication(**auth) + subscription = ReportingNotificationSubscription(**candidate) + normalized = json.loads(canonical_json_utf8_v1(asdict(subscription))) + if normalized != value: + raise ReportingNotificationError("integrity_failure") + return subscription + + +class ReportingEnvelopeCipher: + """AES-256-GCM with all routing columns and body identity in canonical AAD. + + Key versions enable rolling encryption-key rotation while retaining old + decryptors. Signing-key rotation is independent and resolved per attempt. + Neither ciphertext nor plaintext routing is ever logged by this module. + """ + + def __init__( + self, + key: bytes, + *, + key_version: str = "v1", + previous_keys: Mapping[str, bytes] | None = None, + ) -> None: + keys = dict(previous_keys or {}) + keys[key_version] = key + if any(type(value) is not bytes or len(value) != 32 for value in keys.values()): + raise ValueError("reporting outbox encryption keys must contain 32 bytes") + for version in keys: + reporting_identifier(version, maximum=64) + self._keys, self.key_version = keys, key_version + + @staticmethod + def _aad(binding: DeliveryBinding) -> bytes: + return canonical_json_utf8_v1({"purpose": "adcp.reporting.delivery", **asdict(binding)}) + + def prepare( + self, + event: ReportingDomainEvent, + subscription: ReportingNotificationSubscription, + emission_generation: int, + ) -> StoredDelivery: + if type(subscription) is not ReportingNotificationSubscription or not subscription.matches( + event + ): + raise ReportingNotificationError("invalid_configuration") + if type(emission_generation) is not int or emission_generation < 1: + raise ReportingNotificationError("invalid_configuration") + idempotency_key = str(uuid4()) + body = event.body(subscriber_id=subscription.subscriber_id, idempotency_key=idempotency_key) + binding = DeliveryBinding( + account_id=event.account_id, + delivery_id=str(uuid4()), + subscriber_id=subscription.subscriber_id, + principal_id=subscription.principal_id, + notification_id=event.notification_id, + notification_type=event.notification_type, + emission_generation=emission_generation, + idempotency_key=idempotency_key, + destination_sha256=hashlib.sha256(subscription.url.encode("utf-8")).hexdigest(), + subscription_fingerprint=subscription.fingerprint, + signing_scope_id=subscription.signing_scope_id, + cause_kind=event.cause.kind, + cause_id=event.cause_id, + cause_generation=event.cause_generation, + consumer_namespace=( + event.cause.consumer_id if isinstance(event.cause, MaterializationReady) else "" + ), + auth_mode=subscription.auth_mode, + body_sha256=hashlib.sha256(body).hexdigest(), + envelope_version=1, + key_version=self.key_version, + ) + plaintext = canonical_json_utf8_v1( + { + "subscription": asdict(subscription), + "event": event_storage(event), + "body": base64.b64encode(body).decode("ascii"), + } + ) + nonce = secrets.token_bytes(12) + encrypted = AESGCM(self._keys[self.key_version]).encrypt( + nonce, plaintext, self._aad(binding) + ) + return StoredDelivery(binding, nonce + encrypted) + + def open(self, delivery: StoredDelivery) -> OpenedReportingDelivery: + """Authenticate all bound columns before any resolver/DNS/signing/HTTP.""" + try: + binding = delivery.binding + if ( + type(delivery.envelope) is not bytes + or not 28 <= len(delivery.envelope) <= 1024 * 1024 + ): + raise ValueError + key = self._keys[binding.key_version] + plaintext = AESGCM(key).decrypt( + delivery.envelope[:12], delivery.envelope[12:], self._aad(binding) + ) + if binding.envelope_version != 1: + raise ValueError + value = json.loads(plaintext) + if set(value) != {"subscription", "event", "body"}: + raise ValueError + subscription = _decode_subscription(value["subscription"]) + event = decode_event(value["event"]) + body = base64.b64decode(value["body"], validate=True) + payload = json.loads(body) + validate_notification_payload(payload) + if ( + not subscription.matches(event) + or subscription.account_id != binding.account_id + or subscription.subscriber_id != binding.subscriber_id + or subscription.principal_id != binding.principal_id + or subscription.signing_scope_id != binding.signing_scope_id + or subscription.auth_mode != binding.auth_mode + or subscription.fingerprint != binding.subscription_fingerprint + or hashlib.sha256(subscription.url.encode("utf-8")).hexdigest() + != binding.destination_sha256 + or event.notification_id != binding.notification_id + or event.notification_type != binding.notification_type + or event.cause.kind != binding.cause_kind + or event.cause_id != binding.cause_id + or event.cause_generation != binding.cause_generation + or ( + event.cause.consumer_id if isinstance(event.cause, MaterializationReady) else "" + ) + != binding.consumer_namespace + or hashlib.sha256(body).hexdigest() != binding.body_sha256 + or body + != event.body( + subscriber_id=binding.subscriber_id, idempotency_key=binding.idempotency_key + ) + ): + raise ValueError + return OpenedReportingDelivery( + subscription, + event, + PreparedWebhook(subscription.url, binding.idempotency_key, body), + ) + except Exception: + # This phase is entirely local. Deeply malformed/oversized poison + # is terminal too; it must not strand a worker before later rows. + raise ReportingNotificationError("integrity_failure") from None diff --git a/src/adcp/reporting/outbox/worker.py b/src/adcp/reporting/outbox/worker.py new file mode 100644 index 000000000..2cacbbca4 --- /dev/null +++ b/src/adcp/reporting/outbox/worker.py @@ -0,0 +1,315 @@ +"""Stateless turns over separately durable expansion and HTTP delivery phases.""" + +from __future__ import annotations + +import asyncio +from collections.abc import Callable +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from typing import TYPE_CHECKING + +import httpx + +from adcp.reporting.ledger.notification_models import ( + ReportingNotificationError, + decode_event, +) +from adcp.reporting.outbox._transport_logging import protected_transport_logs +from adcp.reporting.outbox.models import ( + DeliveryLease, + ErrorCode, + ReportingNotificationOutbox, + WorkState, +) +from adcp.reporting.outbox.routing import ( + OpenedReportingDelivery, + ReportingEnvelopeCipher, + ReportingNotificationSubscription, + ReportingSigningMaterial, + ReportingSigningResolver, + ReportingSubscriptionResolver, +) +from adcp.signing.jwks import SSRFValidationError +from adcp.webhook_sender import ( + PreparedWebhookAttemptExpiredError, + ScopePermanentlyUnknown, + ScopeTransientlyUnavailable, + WebhookSender, +) + +if TYPE_CHECKING: + from adcp.reporting.ledger.delivery_models import ReportingDeliveryScope + from adcp.reporting.ledger.store import ReportingLedgerStore + + +@dataclass(frozen=True) +class _Outcome: + state: WorkState + error: ErrorCode | None = None + + +class ReportingNotificationWorker: + """At-least-once delivery with immutable bytes and per-attempt signing. + + A worker has no general-purpose sender injection. It constructs the exact + SDK sender from trusted current key material, with the SDK-owned IP-pinned + transport, public HTTPS/443 only, no rewrite hooks, and no redirects. + + Cancellation/process death intentionally leaves the lease to expire. Claims + never exhaust a retry budget. A receiver must dedupe by authenticated sender + and idempotency key: HTTP acceptance and our ACK cannot be one transaction. + """ + + def __init__( + self, + *, + outbox: ReportingNotificationOutbox, + subscriptions: ReportingSubscriptionResolver, + cipher: ReportingEnvelopeCipher, + signing: ReportingSigningResolver | None = None, + clock: Callable[[], datetime] | None = None, + lease_seconds: float = 60, + retry_seconds: float = 5, + ) -> None: + if lease_seconds < 1 or retry_seconds <= 0: + raise ValueError("positive retry and at least one second of lease are required") + self.outbox, self.subscriptions, self.cipher, self.signing = ( + outbox, + subscriptions, + cipher, + signing, + ) + self._clock = clock or (lambda: datetime.now(timezone.utc)) + self.lease_seconds, self.retry_seconds = lease_seconds, retry_seconds + + async def advertised_notifications( + self, + ledger: ReportingLedgerStore, + *, + account_id: str, + ready_scope: ReportingDeliveryScope | None = None, + ) -> dict[str, str | bool]: + """Check the installed chain at startup before publishing these fields. + + Merge the result into the producer's capability block while this worker + is scheduled. Ready support additionally requires a retained Managed + scope. No status or activity projection is claimed by this slice. + """ + from adcp.reporting.outbox._capabilities import advertised_notifications + + return await advertised_notifications( + self, ledger, account_id=account_id, ready_scope=ready_scope + ) + + async def expand_one(self, *, account_id: str) -> bool: + lease = await self.outbox.claim_expansion( + account_id=account_id, now=self._clock(), lease_seconds=self.lease_seconds + ) + if lease is None: + return False + try: + event = decode_event(lease.event) + if ( + event.account_id != lease.account_id + or event.notification_id != lease.notification_id + or event.consumer_namespace != lease.consumer_namespace + ): + raise ReportingNotificationError("invalid_event") + except Exception: + await self.outbox.finish_expansion( + lease, now=self._clock(), state="quarantined", error_code="invalid_payload" + ) + return True + + try: + # One external snapshot. Its membership is fixed only when the + # subsequent all-N insertion + complete checkpoint commits. + resolved = await asyncio.wait_for( + self.subscriptions.list_active( + account_id=account_id, notification_type=event.notification_type + ), + timeout=self.lease_seconds * 0.8, + ) + except Exception: + now = self._clock() + await self.outbox.finish_expansion( + lease, + now=now, + state="pending", + error_code="subscription_unavailable", + retry_at=now + timedelta(seconds=self.retry_seconds), + ) + return True + + try: + if not isinstance(resolved, (list, tuple)): + raise ReportingNotificationError() + snapshot = tuple(resolved) + subscribers: set[str] = set() + for subscription in snapshot: + if ( + type(subscription) is not ReportingNotificationSubscription + or subscription.account_id != account_id + or subscription.subscriber_id in subscribers + ): + raise ReportingNotificationError() + # Reconstruct the closed normalized type rather than trusting + # an overridden fingerprint/matches property on a subclass. + ReportingNotificationSubscription.__post_init__(subscription) + subscribers.add(subscription.subscriber_id) + deliveries = tuple( + self.cipher.prepare(event, subscription, lease.emission_generation) + for subscription in snapshot + if subscription.matches(event) + ) + except (ValueError, TypeError, AttributeError): + await self.outbox.finish_expansion( + lease, now=self._clock(), state="quarantined", error_code="invalid_configuration" + ) + return True + # Empty valid membership completes too. Failure here rolls back the + # entire snapshot. A restarted worker may then observe new membership; + # it can never combine committed rows from two snapshots. + await self.outbox.complete_expansion(lease, deliveries, now=self._clock()) + return True + + async def deliver_one(self, *, account_id: str) -> bool: + lease = await self.outbox.claim_delivery( + account_id=account_id, now=self._clock(), lease_seconds=self.lease_seconds + ) + if lease is None: + return False + try: + opened = self.cipher.open(lease.delivery) + except (ReportingNotificationError, ValueError, TypeError): + outcome = _Outcome("quarantined", "integrity_failure") + else: + try: + outcome = await asyncio.wait_for( + self._attempt(lease, opened), timeout=self.lease_seconds * 0.8 + ) + except (TimeoutError, asyncio.TimeoutError): + outcome = _Outcome("pending", "network") + now = self._clock() + # A DB failure after HTTP acceptance is intentionally not converted to + # success. Expiry/restart retries these exact protected body bytes/key. + await self.outbox.finish_delivery( + lease, + now=now, + state=outcome.state, + error_code=outcome.error, + retry_at=( + now + timedelta(seconds=self.retry_seconds) if outcome.state == "pending" else None + ), + ) + return True + + async def _attempt(self, lease: DeliveryLease, opened: OpenedReportingDelivery) -> _Outcome: + binding = lease.delivery.binding + try: + current = await self.subscriptions.get_active( + account_id=binding.account_id, + subscriber_id=binding.subscriber_id, + notification_type=binding.notification_type, + ) + except Exception: + return _Outcome("pending", "subscription_unavailable") + if type(current) is ReportingNotificationSubscription: + try: + ReportingNotificationSubscription.__post_init__(current) + except (ValueError, TypeError, AttributeError): + return _Outcome("suppressed", "subscription_changed") + if ( + type(current) is not ReportingNotificationSubscription + or not current.matches(opened.event) + or current.subscriber_id != binding.subscriber_id + or current.fingerprint != binding.subscription_fingerprint + ): + return _Outcome("suppressed", "subscription_changed") + try: + sender = await self._sender(opened.subscription) + except ScopePermanentlyUnknown: + return _Outcome("quarantined", "permanent_scope") + except ScopeTransientlyUnavailable: + return _Outcome("pending", "signing_unavailable") + try: + if not await self.outbox.delivery_lease_current(lease, now=self._clock()): + return _Outcome("pending", "lease_expired") + try: + # Invoke the concrete SDK seam. Adopter-provided sender objects, + # subclasses, overridden send methods, clients and hooks never + # cross this boundary. URL/DNS validation is inside this seam. + async def current_fence() -> bool: + return await self.outbox.delivery_lease_current(lease, now=self._clock()) + + with protected_transport_logs(): + result = await WebhookSender.send_prepared( + sender, opened.prepared, before_attempt=current_fence + ) + except PreparedWebhookAttemptExpiredError: + return _Outcome("pending", "lease_expired") + except SSRFValidationError as error: + return ( + _Outcome("pending", "network") + if error.transient + else (_Outcome("quarantined", "invalid_configuration")) + ) + except (httpx.TransportError, OSError, TimeoutError): + return _Outcome("pending", "network") + except (ValueError, TypeError): + return _Outcome("quarantined", "invalid_payload") + except Exception: + # Signing backends can fail transiently; retain no exception + # prose, traceback, headers, URL, or response/provider body. + return _Outcome("pending", "signing_unavailable") + if result.ok: + return _Outcome("complete") + if result.status_code in {408, 425, 429} or 500 <= result.status_code < 600: + return _Outcome("pending", "retryable_http") + return _Outcome("quarantined", "permanent_http") + finally: + await sender.aclose() + + async def _sender(self, subscription: ReportingNotificationSubscription) -> WebhookSender: + timeout = min(10.0, self.lease_seconds * 0.5) + if subscription.authentication is not None: + auth = subscription.authentication + if subscription.signing_scope_id is not None: + raise ScopePermanentlyUnknown from None + if auth.scheme == "Bearer": + return WebhookSender.from_bearer_token( + auth.credentials, + timeout_seconds=timeout, + allowed_destination_ports=frozenset({443}), + ) + return WebhookSender.from_adcp_legacy_hmac( + auth.credentials.encode("utf-8"), + key_id="reporting-registration", + timeout_seconds=timeout, + allowed_destination_ports=frozenset({443}), + ) + if self.signing is None or subscription.signing_scope_id is None: + raise ScopePermanentlyUnknown from None + try: + material = await self.signing.resolve( + account_id=subscription.account_id, + principal_id=subscription.principal_id, + signing_scope_id=subscription.signing_scope_id, + ) + except (ScopePermanentlyUnknown, ScopeTransientlyUnavailable): + raise + except Exception: + raise ScopeTransientlyUnavailable from None + if type(material) is not ReportingSigningMaterial: + raise ScopePermanentlyUnknown from None + try: + ReportingSigningMaterial.__post_init__(material) + return WebhookSender( + private_key=material.private_key, + key_id=material.key_id, + alg=material.algorithm, + timeout_seconds=timeout, + allowed_destination_ports=frozenset({443}), + ) + except (ValueError, TypeError, AttributeError): + raise ScopePermanentlyUnknown from None diff --git a/src/adcp/webhook_sender.py b/src/adcp/webhook_sender.py index 3cae3bb08..20583f8a0 100644 --- a/src/adcp/webhook_sender.py +++ b/src/adcp/webhook_sender.py @@ -30,7 +30,7 @@ import json import warnings -from collections.abc import Mapping, Sequence +from collections.abc import Awaitable, Callable, Mapping, Sequence from dataclasses import dataclass, field from datetime import datetime, timezone from pathlib import Path @@ -243,6 +243,10 @@ class ScopeTransientlyUnavailable(RuntimeError): # noqa: N818 - public issue co """ +class PreparedWebhookAttemptExpiredError(RuntimeError): + """A prepared request lost its durable fence before the HTTP attempt.""" + + @dataclass(frozen=True, slots=True) class WebhookSenderResolution: """A sender bound to the algorithms advertised for its trusted scope. @@ -767,8 +771,17 @@ def prepare_mcp( extra_headers=dict(extra_headers) if extra_headers else {}, ) - async def send_prepared(self, prepared: PreparedWebhook) -> WebhookDeliveryResult: - """Sign and post a previously prepared immutable webhook request.""" + async def send_prepared( + self, + prepared: PreparedWebhook, + *, + before_attempt: Callable[[], Awaitable[bool]] | None = None, + ) -> WebhookDeliveryResult: + """Sign and post a previously prepared immutable webhook request. + + Durable publishers can recheck an expiring fence after DNS/signing and + immediately before HTTP. A false result aborts without sending bytes. + """ if not prepared.idempotency_key: raise ValueError("prepared webhook idempotency_key must be non-empty") if not prepared.body: @@ -793,6 +806,7 @@ async def send_prepared(self, prepared: PreparedWebhook) -> WebhookDeliveryResul body=prepared.body, idempotency_key=prepared.idempotency_key, extra_headers=prepared.extra_headers or None, + before_attempt=before_attempt, ) async def send_revocation_notification( @@ -1124,6 +1138,7 @@ async def _send_bytes( body: bytes, idempotency_key: str, extra_headers: Mapping[str, str] | None, + before_attempt: Callable[[], Awaitable[bool]] | None = None, ) -> WebhookDeliveryResult: """Sign + POST a pre-serialized body through an SSRF-validated transport. @@ -1181,6 +1196,15 @@ async def _send_bytes( reserved=self._auth.reserved_headers(), ) + if before_attempt is not None: + try: + if not await before_attempt(): + raise PreparedWebhookAttemptExpiredError("prepared_attempt_expired") + except BaseException: + if transport is not None: + await transport.aclose() + raise + if transport is not None: # Owned-client path. ``trust_env=False`` prevents httpx from # routing the request through ``HTTPS_PROXY`` / ``HTTP_PROXY`` diff --git a/tests/conformance/reporting/_reliable_process.py b/tests/conformance/reporting/_reliable_process.py new file mode 100644 index 000000000..1f6e0b74c --- /dev/null +++ b/tests/conformance/reporting/_reliable_process.py @@ -0,0 +1,361 @@ +"""Private service executable for the extensible cross-layer PG failure matrix. + +All storage, transactions, pinning, HTTP framing, TLS, signatures and receiver +verification are real. Only the test socket address mapping and certificate +trust root differ from deployment. No DNS/SSRF or sender policy is bypassed. +""" + +from __future__ import annotations + +import asyncio +import base64 +import json +import os +import socket +import ssl +import sys +from datetime import timedelta +from types import SimpleNamespace + +from psycopg import AsyncConnection +from psycopg_pool import AsyncConnectionPool + +from adcp.reporting.ledger import PgReportingReconciliationStore +from adcp.reporting.outbox import ( + PgReportingOutbox, + ReportingEnvelopeCipher, + ReportingNotificationWorker, +) + +from ._generation_support import configuration, obligation_for, revision_for +from ._reliable_support import ( + DeterministicReceiverStore, + FailurePlan, + ManualClock, + ScriptedSigning, + ScriptedSubscriptions, + _BytesStore, + notification_subscription, + notification_verification_keys, +) + +_INPUT = None +_INPUT_TRANSPORT = None +_STAGE = "startup" +_DEADLINES = {} + + +class ServiceDeadlineError(TimeoutError): + def __init__(self, stage): + self.stage = stage + super().__init__("service_deadline") + + +async def bounded(awaitable, *, stage, seconds=10): + global _STAGE + _STAGE = stage + try: + return await asyncio.wait_for(awaitable, _DEADLINES.get(stage, seconds)) + except ServiceDeadlineError: + # Preserve the innermost named barrier/receiver deadline. The outer + # service watchdog must not relabel an already bounded failure. + raise + except (TimeoutError, asyncio.TimeoutError): + raise ServiceDeadlineError(stage) from None + + +def emit(point, **data): + print(json.dumps({"point": point, **data}), flush=True) + + +async def command(*, stage="command"): + global _INPUT, _INPUT_TRANSPORT + if _INPUT is None: + _INPUT = asyncio.StreamReader(limit=65536) + _INPUT_TRANSPORT, _ = await bounded( + asyncio.get_running_loop().connect_read_pipe( + lambda: asyncio.StreamReaderProtocol(_INPUT), sys.stdin.buffer + ), + stage="stdin_open", + ) + # A cancellable pipe reader, not a to_thread(readline) that strands the + # executor during asyncio.run shutdown after a barrier deadline. + line = await bounded(_INPUT.readline(), stage=stage, seconds=40) + if not line: + raise EOFError + return json.loads(line) + + +async def barrier(point, **data): + emit(point, **data) + assert (await command(stage=f"barrier:{point}"))["continue"] == point + + +def install_test_socket(settings, clock): + from httpcore._backends.anyio import AnyIOBackend + + from adcp.signing import ip_pinned_transport, signer + + resolve = socket.getaddrinfo + + def addresses(host, port, *args, **kwargs): + if host == "receiver.example.test": + return [(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", ("8.8.8.8", port))] + return resolve(host, port, *args, **kwargs) + + connect = AnyIOBackend.connect_tcp + + async def connect_pinned(self, host, port, **kwargs): + # The SDK already resolved, rejected unsafe addresses and pinned this + # exact public endpoint before this test-only socket map runs. + assert host == "8.8.8.8" and port == 443 + return await bounded( + connect(self, "127.0.0.1", settings["receiver_port"], **kwargs), + stage="pinned_socket_connect", + seconds=5, + ) + + def context(): + value = ssl.create_default_context(cafile=settings["certificate"]) + assert value.check_hostname and value.verify_mode == ssl.CERT_REQUIRED + return value + + socket.getaddrinfo = addresses + AnyIOBackend.connect_tcp = connect_pinned + ip_pinned_transport._build_ssl_context = context + signer.time = SimpleNamespace(time=lambda: clock().timestamp()) + + +async def receiver(pool, settings, clock): + from adcp.signing.jwks import StaticJwksResolver + from adcp.signing.webhook_verifier import WebhookVerifyOptions, verify_webhook_signature + + blobs = _BytesStore(pool) + await blobs.create_schema() + store = DeterministicReceiverStore(blobs, FailurePlan()) + options = WebhookVerifyOptions( + jwks_resolver=StaticJwksResolver({"keys": notification_verification_keys()}), + clock=lambda: clock().timestamp(), + ) + attempts = 0 + responses = {} + + async def accept(reader, writer): + nonlocal attempts + try: + head = await bounded(reader.readuntil(b"\r\n\r\n"), stage="receiver_headers", seconds=5) + lines = head.decode("ascii").split("\r\n") + headers = dict(line.split(": ", 1) for line in lines[1:] if line) + headers = {name.lower(): value for name, value in headers.items()} + size = int(headers["content-length"]) + assert 0 < size < 65536 + body = await bounded(reader.readexactly(size), stage="receiver_body", seconds=5) + target = lines[0].split(" ")[1] + identity = verify_webhook_signature( + method="POST", + url="https://receiver.example.test" + target, + headers=headers, + body=body, + options=options, + ) + value = json.loads(body) + assert value["account_id"] == "acct_a" + await bounded( + store.write(value["account_id"], value["idempotency_key"], body), + stage="receiver_accept_commit", + ) + attempts += 1 + response_gate = responses[attempts] = asyncio.Event() + await bounded( + blobs.put( + "http-attempt", + "acct_a", + value["idempotency_key"], + json.dumps( + { + "body": base64.b64encode(body).decode(), + "headers": headers, + "verified_key": identity.key_id, + } + ).encode(), + str(attempts), + ), + stage="receiver_attempt_commit", + ) + emit("http_accepted", attempt=attempts) + if settings.get("pause_responses"): + await bounded(response_gate.wait(), stage="receiver_response_gate", seconds=30) + writer.write(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n") + await bounded(writer.drain(), stage="receiver_response_write", seconds=5) + except Exception as exc: + emit( + "receiver_failed", + classification=failure_code(exc), + stage=getattr(exc, "stage", _STAGE), + ) + finally: + writer.close() + await bounded(writer.wait_closed(), stage="receiver_socket_close", seconds=5) + + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.load_cert_chain(settings["certificate"], settings["certificate_key"]) + async with await bounded( + asyncio.start_server(accept, "127.0.0.1", 0, ssl=context, ssl_handshake_timeout=5), + stage="receiver_listen", + ) as server: + emit("listening", port=server.sockets[0].getsockname()[1]) + while True: + value = await command(stage="receiver_control") + if value.get("stop"): + return + if "release_http" in value: + responses[value["release_http"]].set() + emit("http_released") + continue + clock.advance(timedelta(seconds=value["advance_seconds"])) + emit("clock_advanced") + + +async def main(): + global _DEADLINES + settings, role = await command(), sys.argv[1] + _DEADLINES = settings.get("deadlines", {}) + clock = ManualClock() + clock.advance(timedelta(seconds=settings.get("advance_seconds", 0))) + pause = settings.get("pause") + + class CheckpointConnection(AsyncConnection): + async def execute(self, query, params=None, **kwargs): + cursor = await bounded( + super().execute(query, params, **kwargs), stage="database_statement", seconds=18 + ) + if ( + pause == "ack_written" + and isinstance(query, str) + and query.startswith("UPDATE reporting_notification_deliveries SET state = %s") + and params[0] == "complete" + ): + await barrier("ack_written", backend_pid=self.info.backend_pid) + return cursor + + kwargs = { + **settings["pool_kwargs"], + "autocommit": True, + "application_name": f"reporting-matrix-{role}-{os.getpid()}", + } + async with AsyncConnectionPool( + settings["conninfo"], + kwargs=kwargs, + min_size=2, + max_size=3, + connection_class=CheckpointConnection, + open=False, + ) as pool: + await bounded(pool.wait(timeout=10), stage="pool_open", seconds=12) + if role == "barrier_probe": + await barrier("held") + return + if role == "receiver": + await receiver(pool, settings, clock) + return + if role == "producer": + store = PgReportingReconciliationStore(pool=pool, clock=clock, notifications=True) + config = configuration() + await store.put_configuration(config) + obligation = await store.commit_obligation(obligation_for(config)) + original = store._record_notification + + async def enqueue(conn, event): + await original(conn, event) + if pause == "event_inserted": + await barrier(pause) + + store._record_notification = enqueue + revision, rows = revision_for(obligation, suffix="process") + await store.commit_revision(revision, rows) + if pause == "revision_committed": + await barrier(pause) + emit("done") + return + outbox = PgReportingOutbox(pool=pool, clock=clock) + failures = FailurePlan() + subscriptions = ScriptedSubscriptions(failures) + for subscriber in settings.get("subscribers", ["buyer"]): + subscriptions.put(notification_subscription(subscriber=subscriber)) + signing = ScriptedSigning(failures) + signing.generation = settings.get("signing_generation", 1) + worker = ReportingNotificationWorker( + outbox=outbox, + subscriptions=subscriptions, + signing=signing, + cipher=ReportingEnvelopeCipher(b"e" * 32), + clock=clock, + ) + if role == "fanout": + original_insert = outbox._insert_delivery + inserted = 0 + + async def insert(conn, delivery, at): + nonlocal inserted + await original_insert(conn, delivery, at) + inserted += 1 + if inserted == 1 and pause == "fanout_partial": + await barrier(pause) + + outbox._insert_delivery = insert + result = await worker.expand_one(account_id="acct_a") + else: + assert role == "worker" + install_test_socket(settings, clock) + if settings.get("start_paused"): + await barrier("worker_ready") + finish = outbox.finish_delivery + + async def ack(lease, **kwargs): + if pause in {"before_ack", "ack_written"} and kwargs["state"] != "complete": + emit("attempt_released", classification=kwargs["error_code"] or "unknown_state") + if pause == "before_ack" and kwargs["state"] == "complete": + await barrier(pause) + return await finish(lease, **kwargs) + + outbox.finish_delivery = ack + result = await worker.deliver_one(account_id="acct_a") + emit("done", did_work=result) + + +def failure_code(error): + from psycopg import Error + + from adcp.signing.errors import SignatureVerificationError + + if isinstance(error, (TimeoutError, asyncio.TimeoutError)): + return "deadline" + if isinstance(error, SignatureVerificationError): + return error.code + if isinstance(error, Error): + return "database_failure" + if isinstance(error, AssertionError): + return "assertion_failure" + if isinstance(error, (EOFError, ValueError, TypeError, KeyError)): + return "protocol_failure" + return "service_failure" + + +async def run_service(): + try: + await bounded(main(), stage="service_lifetime", seconds=80) + finally: + if _INPUT_TRANSPORT is not None: + _INPUT_TRANSPORT.close() + + +if __name__ == "__main__": + try: + asyncio.run(run_service()) + except Exception as error: + emit( + "service_failed", + classification=failure_code(error), + stage=getattr(error, "stage", _STAGE), + ) + sys.exit(1) diff --git a/tests/conformance/reporting/_reliable_support.py b/tests/conformance/reporting/_reliable_support.py index 4399c628d..b0953cb51 100644 --- a/tests/conformance/reporting/_reliable_support.py +++ b/tests/conformance/reporting/_reliable_support.py @@ -19,7 +19,7 @@ from copy import deepcopy from dataclasses import dataclass, field, replace from datetime import datetime, timedelta -from typing import TYPE_CHECKING, Any, Literal +from typing import TYPE_CHECKING, Any, Literal, Protocol, TypeVar import pytest @@ -59,9 +59,16 @@ ReportingRevisionRecord, ReportingScheduleSpec, ReportingVerificationRecord, - WorkerTurn, revision_content_sha256, ) +from adcp.reporting.outbox import ( + InMemoryReportingOutbox, + PgReportingOutbox, + ReportingEnvelopeCipher, + ReportingNotificationSubscription, + ReportingNotificationWorker, + ReportingSigningMaterial, +) from adcp.reporting.source import ( MetricOfferingV1, ReportingSourceCapabilitiesV1, @@ -121,10 +128,10 @@ class FailurePlan: """Finite, named fault/barrier scripts; no probabilistic failure or sleeps.""" def __init__(self) -> None: - self.steps: dict[str, deque[Exception | Barrier]] = defaultdict(deque) + self.steps: dict[str, deque[BaseException | Barrier]] = defaultdict(deque) self.hits: list[str] = [] - def at(self, point: str, *steps: Exception | Barrier) -> None: + def at(self, point: str, *steps: BaseException | Barrier) -> None: self.steps[point].extend(steps) async def hit(self, point: str) -> None: @@ -174,13 +181,25 @@ async def async_fetch(self, request: ReportingSourceSliceRequestV1) -> InlineFet return result +class _DidWork(Protocol): + @property + def did_work(self) -> bool: ... + + +_TurnT = TypeVar("_TurnT", bound=_DidWork, covariant=True) + + +class _Runnable(Protocol[_TurnT]): + async def run_worker(self) -> _TurnT: ... + + async def drain_until_idle( - producer: ReportingProducer, + producer: _Runnable[_TurnT], clock: ManualClock, *, idle_turns: int = 1, max_turns: int = 32, -) -> tuple[WorkerTurn, ...]: +) -> tuple[_TurnT, ...]: """Drain a known number of configurations with a finite retry/lease budget. Set idle_turns to the number of configurations: one idle account must not @@ -188,7 +207,7 @@ async def drain_until_idle( """ if not 1 <= idle_turns <= max_turns: raise ValueError("idle_turns must fit the positive turn budget") - turns: list[WorkerTurn] = [] + turns: list[_TurnT] = [] idle = 0 for _ in range(max_turns): turn = await asyncio.wait_for(producer.run_worker(), timeout=10) @@ -516,6 +535,7 @@ class ReliableHarness: clock: ManualClock blobs: _BytesStore failures: FailurePlan = field(default_factory=FailurePlan) + notifications: bool = False currencies: dict[str, str] = field(default_factory=lambda: {"eur": "EUR", "usd": "USD"}) manifests: dict[tuple[str, str], SourceBatchManifestV1] = field(default_factory=dict) staging: _Staging = field(init=False) @@ -584,7 +604,9 @@ async def restart(self) -> None: ) await pool.open(wait=True) self.blobs = _BytesStore(pool) - self.store = PgReportingReconciliationStore(pool=pool, clock=self.clock) + self.store = PgReportingReconciliationStore( + pool=pool, clock=self.clock, notifications=self.notifications + ) self.__post_init__() else: # This is an explicit test fixture image, not an SDK persistence API. @@ -609,18 +631,28 @@ async def restart(self) -> None: @asynccontextmanager async def reliable_factory( - backend: Backend, *, initialize: bool = True + backend: Backend, + *, + initialize: bool = True, + notifications: bool = False, + autocommit: bool = False, ) -> AsyncIterator[ReliableHarness]: clock = ManualClock() if backend == "memory": harness = ReliableHarness( - InMemoryReportingReconciliationStore(clock=clock), clock, _BytesStore() + InMemoryReportingReconciliationStore(clock=clock, notifications=notifications), + clock, + _BytesStore(), + notifications=notifications, ) yield harness else: - async with isolated_reporting_pool() as pool: + async with isolated_reporting_pool(autocommit=autocommit) as pool: harness = ReliableHarness( - PgReportingReconciliationStore(pool=pool, clock=clock), clock, _BytesStore(pool) + PgReportingReconciliationStore(pool=pool, clock=clock, notifications=notifications), + clock, + _BytesStore(pool), + notifications=notifications, ) if initialize: await harness.store.create_schema() @@ -743,3 +775,417 @@ async def publication_records( consumer_commit_ref=f"load-{suffix}", ) return PublishedRecords(binding, delivery, attempt, outcome, receipt) + + +class SimulatedCrash(BaseException): + """Abrupt service death, deliberately outside ordinary retry catches.""" + + +def notification_subscription( + account: str = "acct_a", + subscriber: str = "buyer", + *, + principal: str = "buyer", + events: tuple[str, ...] = ("reporting.ledger_changed", "reporting.delivery_ready"), + url: str = "https://receiver.example.test/reporting?token=URL_SECRET", + **changes: Any, +) -> ReportingNotificationSubscription: + values: dict[str, Any] = dict( + account_id=account, + subscriber_id=subscriber, + principal_id=principal, + url=url, + event_types=events, + configuration_revision="registration-1", + authorization_ref="authorized-principal-1", + proof_of_control_ref="account-challenge-1", + signing_scope_id="seller-signing-scope-1", + active=True, + authorized=True, + proof_valid=True, + ) + values.update(changes) + return ReportingNotificationSubscription(**values) + + +class ScriptedSubscriptions: + """Normalized trusted account configurations, with one atomic list read.""" + + def __init__(self, failures: FailurePlan) -> None: + self.failures = failures + self.values: dict[tuple[str, str], ReportingNotificationSubscription] = {} + self.lists: list[tuple[str, str]] = [] + self.gets: list[tuple[str, str, str]] = [] + + def put(self, value: ReportingNotificationSubscription) -> None: + self.values[(value.account_id, value.subscriber_id)] = value + + async def list_active(self, *, account_id: str, notification_type: str): + self.lists.append((account_id, notification_type)) + await self.failures.hit("subscriptions.list.before") + snapshot = tuple( + value + for (account, _), value in self.values.items() + if account == account_id and value.active and notification_type in value.event_types + ) + await self.failures.hit("subscriptions.list.after") + return snapshot + + async def get_active(self, *, account_id: str, subscriber_id: str, notification_type: str): + self.gets.append((account_id, subscriber_id, notification_type)) + await self.failures.hit("subscriptions.get") + return self.values.get((account_id, subscriber_id)) + + +class ScriptedSigning: + def __init__(self, failures: FailurePlan) -> None: + self.failures = failures + self.generation = 1 + self.calls: list[tuple[str, str, str]] = [] + + async def resolve(self, *, account_id: str, principal_id: str, signing_scope_id: str): + from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey + + self.calls.append((account_id, principal_id, signing_scope_id)) + await self.failures.hit("signing.resolve") + return ReportingSigningMaterial( + Ed25519PrivateKey.from_private_bytes(bytes([self.generation]) * 32), + f"https://seller.example.test/keys#key-{self.generation}", + "ed25519", + frozenset({"ed25519"}), + ) + + +def notification_verification_keys() -> list[dict[str, Any]]: + """The exact public-key fixture used by the separate receiver process.""" + import base64 + + from cryptography.hazmat.primitives import serialization + from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey + + result = [] + for generation in (1, 2): + public = Ed25519PrivateKey.from_private_bytes(bytes([generation]) * 32).public_key() + result.append( + { + "kty": "OKP", + "crv": "Ed25519", + "alg": "EdDSA", + "use": "sig", + "adcp_use": "request-signing", + "key_ops": ["verify"], + "kid": f"https://seller.example.test/keys#key-{generation}", + "x": base64.urlsafe_b64encode( + public.public_bytes(serialization.Encoding.Raw, serialization.PublicFormat.Raw) + ) + .rstrip(b"=") + .decode(), + } + ) + return result + + +@dataclass(frozen=True) +class ReceivedNotification: + account_id: str + subscriber_id: str + idempotency_key: str + body: bytes = field(repr=False) + headers: dict[str, str] = field(repr=False) + target: str = field(repr=False) + + +class ScriptedNotificationReceiver: + """HTTP/1 byte peer below the SDK's actual pinning and signing paths. + + We replace only the network socket. The production resolver, SSRF policy, + pinning backend, sender, httpx/httpcore HTTP encoding, and signature code all + run. This receiver persists accepted bytes in the same deterministic private + receiver store used by the preceding reporting slice. + """ + + def __init__(self, harness: ReliableHarness) -> None: + self.harness = harness + self.received: list[ReceivedNotification] = [] + self.connections: list[tuple[str, int]] = [] + self.responses: dict[str, deque[int | Exception]] = defaultdict(deque) + self.dns_addresses: dict[str, list[str]] = {} + self.dns_calls: list[str] = [] + + def install(self, monkeypatch: pytest.MonkeyPatch) -> None: + import socket + from types import SimpleNamespace + + from httpcore._backends.anyio import AnyIOBackend + + from adcp import webhook_auth + from adcp.signing import signer + + # The signing library has its own crypto clock. Keep even that private + # test dependency deterministic without replacing global time.time(). + crypto_time = SimpleNamespace(time=lambda: self.harness.clock().timestamp()) + monkeypatch.setattr(signer, "time", crypto_time) + monkeypatch.setattr(webhook_auth, "time", crypto_time) + + original_resolve = socket.getaddrinfo + receiver = self + + def resolve(host, port, *args, **kwargs): + if isinstance(host, bytes): + host = host.decode("ascii") + if str(host).endswith(".example.test"): + receiver.dns_calls.append(host) + addresses = receiver.dns_addresses.get(host, ["8.8.8.8"]) + address = addresses.pop(0) if len(addresses) > 1 else addresses[0] + family = socket.AF_INET6 if ":" in address else socket.AF_INET + return [(family, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", (address, port))] + return original_resolve(host, port, *args, **kwargs) + + async def connect(backend, host, port, **kwargs): + receiver.connections.append((host, port)) + # The parent of the SDK pinning backend receives an IP, never an + # attacker-controlled hostname to resolve for a second time. + assert host in {"8.8.8.8", "1.1.1.1"} + assert port == 443 + return _NotificationStream(receiver) + + monkeypatch.setattr(socket, "getaddrinfo", resolve) + monkeypatch.setattr(AnyIOBackend, "connect_tcp", connect) + + async def respond(self, raw: bytes) -> bytes: + head, body = raw.split(b"\r\n\r\n", 1) + lines = head.decode("ascii").split("\r\n") + headers = dict(line.split(": ", 1) for line in lines[1:]) + headers = {key.lower(): value for key, value in headers.items()} + value = json.loads(body) + subscriber = value["subscriber_id"] + await self.harness.failures.hit("http.before") + await self.harness.failures.hit(f"http.before:{subscriber}") + script = self.responses[subscriber] + step = script.popleft() if script else 200 + if isinstance(step, Exception): + raise step + self.received.append( + ReceivedNotification( + value["account_id"], + subscriber, + value["idempotency_key"], + body, + headers, + lines[0].split(" ")[1], + ) + ) + if 200 <= step < 300: + await self.harness.receiver.write(value["account_id"], value["idempotency_key"], body) + await self.harness.failures.hit("http.accepted") + response_body = b"provider token=DO_NOT_PERSIST" + redirect = b"Location: https://169.254.169.254/secret\r\n" if step == 302 else b"" + return ( + f"HTTP/1.1 {step} Test\r\nContent-Length: {len(response_body)}\r\n".encode() + + redirect + + b"\r\n" + + response_body + ) + + +class _NotificationStream: + def __init__(self, receiver: ScriptedNotificationReceiver) -> None: + self.receiver, self.request, self.response = receiver, bytearray(), None + + async def write(self, buffer, timeout=None): + self.request.extend(buffer) + + async def read(self, max_bytes, timeout=None): + if self.response is None: + self.response = await self.receiver.respond(bytes(self.request)) + chunk, self.response = self.response[:max_bytes], self.response[max_bytes:] + return chunk + + async def start_tls(self, ssl_context, server_hostname=None, timeout=None): + return self + + async def aclose(self): + pass + + def get_extra_info(self, info): + return None + + +@dataclass(frozen=True) +class NotificationTurn: + did_work: bool + + +class NotificationRunner: + """Adapter for the shared bounded drain_until_idle state-machine driver.""" + + def __init__(self, worker: ReportingNotificationWorker, accounts: tuple[str, ...]) -> None: + self.worker, self.accounts, self.turn = worker, accounts, 0 + + async def run_worker(self) -> NotificationTurn: + account = self.accounts[self.turn % len(self.accounts)] + self.turn += 1 + expanded = await self.worker.expand_one(account_id=account) + delivered = await self.worker.deliver_one(account_id=account) + return NotificationTurn(expanded or delivered) + + +@dataclass +class NotificationHarness: + reliable: ReliableHarness + subscriptions: ScriptedSubscriptions = field(init=False) + signing: ScriptedSigning = field(init=False) + receiver: ScriptedNotificationReceiver = field(init=False) + cipher: ReportingEnvelopeCipher = field( + default_factory=lambda: ReportingEnvelopeCipher(b"e" * 32) + ) + + def __post_init__(self) -> None: + self.subscriptions = ScriptedSubscriptions(self.reliable.failures) + self.signing = ScriptedSigning(self.reliable.failures) + self.receiver = ScriptedNotificationReceiver(self.reliable) + self.subscriptions.put(notification_subscription()) + + @property + def outbox(self): + if self.reliable.blobs.pool is not None: + return PgReportingOutbox(pool=self.reliable.blobs.pool, clock=self.reliable.clock) + return InMemoryReportingOutbox(self.reliable.store) + + def worker(self, **kwargs: Any) -> ReportingNotificationWorker: + return ReportingNotificationWorker( + outbox=self.outbox, + subscriptions=self.subscriptions, + signing=self.signing, + cipher=self.cipher, + clock=self.reliable.clock, + **kwargs, + ) + + async def drain(self, *accounts: str): + selected = accounts or ("acct_a",) + return await drain_until_idle( + NotificationRunner(self.worker(), selected), + self.reliable.clock, + idle_turns=len(selected), + ) + + +@pytest.fixture(params=["memory", "postgres"]) +async def notification_harness(request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch): + async with reliable_factory(request.param, notifications=True, autocommit=True) as reliable: + harness = NotificationHarness(reliable) + harness.receiver.install(monkeypatch) + yield harness + + +@dataclass +class ServiceProcess: + """A separately pooled service controlled by newline-JSON barriers. + + Waits have watchdog bounds. Neither process orchestration nor convergence + uses timing sleeps. Configuration travels over stdin, never command args. + """ + + process: asyncio.subprocess.Process + role: str + last_point: str = "spawned" + lifetime_expired: bool = False + + def diagnostic(self, action: str) -> str: + return ( + f"reporting_matrix role={self.role} pid={self.process.pid} action={action}" + f" last_point={self.last_point} exit={self.process.returncode}" + f" lifetime_expired={self.lifetime_expired}" + ) + + def trace(self, action: str) -> None: + # Visible under pytest -s; no routing data, secrets, or child prose. + print(self.diagnostic(action), flush=True) + + async def send(self, **value: Any) -> None: + assert self.process.stdin is not None + self.process.stdin.write(json.dumps(value).encode() + b"\n") + try: + await asyncio.wait_for(self.process.stdin.drain(), 5) + except (TimeoutError, asyncio.TimeoutError, BrokenPipeError, ConnectionResetError): + raise AssertionError(self.diagnostic("stdin_failed")) from None + + async def event(self, point: str, *, timeout_seconds: float = 30) -> dict[str, Any]: + assert self.process.stdout is not None + self.trace(f"waiting:{point}") + try: + raw = await asyncio.wait_for(self.process.stdout.readline(), timeout_seconds) + except (TimeoutError, asyncio.TimeoutError): + raise AssertionError(self.diagnostic(f"deadline:{point}")) from None + if not raw: + # Reading stderr to EOF here can itself hang on a living child; + # arbitrary tracebacks are also inappropriate diagnostics. + raise AssertionError(self.diagnostic(f"eof_before:{point}")) + value: dict[str, Any] = json.loads(raw) + allowed = {"point", "classification", "stage", "attempt", "backend_pid", "port", "did_work"} + assert set(value).issubset(allowed), self.diagnostic("invalid_child_protocol") + self.last_point = value["point"] + self.trace(f"received:{self.last_point}") + assert value["point"] == point, (self.diagnostic(f"expected:{point}"), value) + return value + + async def finish(self, *, code: int = 0) -> None: + try: + actual = await asyncio.wait_for(self.process.wait(), 15) + except (TimeoutError, asyncio.TimeoutError): + raise AssertionError(self.diagnostic("exit_deadline")) from None + assert actual == code, self.diagnostic(f"expected_exit:{code}") + self.trace("exited") + + async def kill(self) -> None: + if self.process.returncode is None: + self.trace("kill_owned_process") + try: + self.process.kill() + except ProcessLookupError: + pass + if self.process.stdin is not None: + self.process.stdin.close() + try: + await asyncio.wait_for(self.process.wait(), 15) + except (TimeoutError, asyncio.TimeoutError): + raise AssertionError(self.diagnostic("kill_deadline")) from None + + async def watchdog(self) -> None: + try: + await asyncio.wait_for(self.process.wait(), 90) + except (TimeoutError, asyncio.TimeoutError): + self.lifetime_expired = True + self.trace("lifetime_deadline") + await self.kill() + + +@asynccontextmanager +async def service_process( + pool: AsyncConnectionPool, role: str, **settings: Any +) -> AsyncIterator[ServiceProcess]: + import sys + + process = await asyncio.wait_for( + asyncio.create_subprocess_exec( + sys.executable, + "-m", + "tests.conformance.reporting._reliable_process", + role, + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ), + 10, + ) + service = ServiceProcess(process, role) + service.trace("spawned") + watchdog = asyncio.create_task(service.watchdog()) + try: + await service.send(conninfo=pool.conninfo, pool_kwargs=pool.kwargs, **settings) + yield service + finally: + await service.kill() + watchdog.cancel() + await asyncio.gather(watchdog, return_exceptions=True) diff --git a/tests/conformance/reporting/conftest.py b/tests/conformance/reporting/conftest.py index c7bf97e2d..f29c44241 100644 --- a/tests/conformance/reporting/conftest.py +++ b/tests/conformance/reporting/conftest.py @@ -1,4 +1,5 @@ """Shared storage state-machine fixture, registered for reporting conformance.""" from ._reconciliation_support import reconciliation_store as reconciliation_store +from ._reliable_support import notification_harness as notification_harness from ._reliable_support import reliable as reliable diff --git a/tests/conformance/reporting/test_reporting_notification_harness.py b/tests/conformance/reporting/test_reporting_notification_harness.py new file mode 100644 index 000000000..58613e65b --- /dev/null +++ b/tests/conformance/reporting/test_reporting_notification_harness.py @@ -0,0 +1,40 @@ +"""A stuck child is a failing bounded contract, never successful evidence.""" + +import asyncio +import sys + +import pytest + +from ._reliable_support import ServiceProcess + + +async def test_parent_deadline_reports_exact_child_and_only_kills_its_owned_process(): + async def child(): + return await asyncio.wait_for( + asyncio.create_subprocess_exec( + sys.executable, + "-u", + "-c", + 'import sys; print(\'{"point":"blocked"}\', flush=True); sys.stdin.read()', + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ), + 10, + ) + + blocked = ServiceProcess(await child(), "deadline-probe") + independent = ServiceProcess(await child(), "independent-probe") + try: + await blocked.event("blocked") + await independent.event("blocked") + with pytest.raises( + AssertionError, match=r"role=deadline-probe .*deadline:never .*last_point=blocked" + ): + await blocked.event("never", timeout_seconds=0) + await blocked.kill() + assert blocked.process.returncode is not None + assert independent.process.returncode is None + finally: + await blocked.kill() + await independent.kill() diff --git a/tests/conformance/reporting/test_reporting_notification_migration.py b/tests/conformance/reporting/test_reporting_notification_migration.py new file mode 100644 index 000000000..0ee08d9f5 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_notification_migration.py @@ -0,0 +1,277 @@ +"""Atomic direct/hopwise installs, historical upgrades, and complete readiness.""" + +from __future__ import annotations + +import asyncio +from importlib.resources import files + +import pytest + +from adcp.reporting.ledger import PgReportingReconciliationStore +from adcp.reporting.outbox import PgReportingOutbox, ReportingNotificationError +from adcp.reporting.outbox._schema import SCHEMA_CONTRACT, schema_contract, validate_schema + +from ._generation_support import NOW, isolated_reporting_pool, revision_for +from ._reconciliation_support import scenario +from ._reliable_support import Barrier +from .test_reporting_reconciliation_migration import FIXTURES + +RESOURCES = files("adcp.reporting.ledger") +CHAIN = ( + "reporting_ledger.sql", + "reporting_ledger_account_generations.sql", + "reporting_ledger_obligation_currency.sql", + "reporting_ledger_reconciliation.sql", + "reporting_notification_outbox.sql", +) + + +async def foundation(pool): + async with pool.connection() as conn, conn.transaction(): + for name in CHAIN[:-1]: + await conn.execute(RESOURCES.joinpath(name).read_text()) + + +async def retained_physical_rows(pool): + """Include MVCC identity: no migration rewrite/backfill of retained evidence.""" + from psycopg import sql + + tables = ( + "reporting_configurations", + "reporting_obligations", + "reporting_revisions", + "reporting_revision_rows", + "reporting_adjustments", + "reporting_consumer_statuses", + "reporting_issue_lifecycle", + "reporting_ledger_changes", + "reporting_reconciliation_records", + "reporting_reconciliation_changes", + "reporting_reconciliation_heads", + "reporting_receipt_heads", + ) + result = {} + async with pool.connection() as conn: + for table in tables: + result[table] = await ( + await conn.execute( + sql.SQL( + "SELECT t.ctid::text, t.xmin::text, to_jsonb(t) FROM {} t ORDER BY t.ctid" + ).format(sql.Identifier(table)) + ) + ).fetchall() + return result + + +@pytest.mark.parametrize("autocommit", [False, True]) +@pytest.mark.parametrize("installation", ["direct", "full_chain"]) +async def test_populated_pre_outbox_upgrade_never_rewrites_or_backfills(autocommit, installation): + async with isolated_reporting_pool(autocommit=autocommit) as pool: + await foundation(pool) + old = PgReportingReconciliationStore(pool=pool, clock=lambda: NOW) + s = await scenario(old) + await old.commit_materialization(s.outcome) + await old.record_revision_receipt(s.receipt) + await old.ensure_issue_opened( + account_id="acct_a", consumer_id="buyer", issue_key="opaque-legacy", observed_at=NOW + ) + before = await retained_physical_rows(pool) + store = PgReportingReconciliationStore(pool=pool, clock=lambda: NOW, notifications=True) + if installation == "direct": + async with pool.connection() as conn: + await conn.execute(RESOURCES.joinpath(CHAIN[-1]).read_text()) + else: + await store.create_schema() + assert await retained_physical_rows(pool) == before + outbox = PgReportingOutbox(pool=pool, clock=lambda: NOW) + assert await outbox.list_events(account_id="acct_a") == () + assert await outbox.read_status_dirty(account_id="acct_a") == () + async with pool.connection() as conn: + await validate_schema(conn) + # Repeated installation neither rebuilds constraints nor changes MVCC rows. + await store.create_schema() + assert await retained_physical_rows(pool) == before + revision, rows = revision_for(s.obligation, suffix="post-upgrade") + await store.commit_revision(revision, rows) + restarted = PgReportingOutbox(pool=pool, clock=lambda: NOW) + assert len(await restarted.list_events(account_id="acct_a")) == 1 + + +@pytest.mark.parametrize( + "source", + ["reporting_ledger_beta15.sql", "reporting_ledger_1169.sql", "reporting_ledger_1171.sql"], +) +@pytest.mark.parametrize("hopwise", [False, True]) +async def test_direct_and_hopwise_historical_schema_chain(source, hopwise): + async with isolated_reporting_pool(autocommit=True) as pool: + async with pool.connection() as conn: + await conn.execute((FIXTURES / source).read_text()) + await conn.execute((FIXTURES / "reporting_ledger_beta15_data.sql").read_text()) + if hopwise: + for name in CHAIN[1:]: + await conn.execute(RESOURCES.joinpath(name).read_text()) + await PgReportingReconciliationStore(pool=pool).create_schema() + async with pool.connection() as conn: + assert await schema_contract(conn) == SCHEMA_CONTRACT + assert await PgReportingOutbox(pool=pool).list_events(account_id="acct_a") == () + + +@pytest.mark.parametrize("autocommit", [False, True]) +async def test_concurrent_repeated_install_from_independent_pools(autocommit): + from psycopg_pool import AsyncConnectionPool + + async with isolated_reporting_pool(autocommit=autocommit) as pool: + async with AsyncConnectionPool( + pool.conninfo, kwargs=pool.kwargs, min_size=2, max_size=6, open=False + ) as other: + await other.wait(timeout=10) + gate = asyncio.Event() + + async def install(selected): + await gate.wait() + await PgReportingReconciliationStore(pool=selected).create_schema() + + tasks = [asyncio.create_task(install(selected)) for selected in (pool, other) * 3] + gate.set() + await asyncio.wait_for(asyncio.gather(*tasks), 20) + async with other.connection() as conn: + await validate_schema(conn) + await PgReportingReconciliationStore(pool=other).create_schema() + + +async def test_interrupted_autocommit_install_is_invisible_and_restart_converges(): + from psycopg_pool import AsyncConnectionPool + + async with isolated_reporting_pool(autocommit=True) as pool: + await foundation(pool) + gate = Barrier() + async with AsyncConnectionPool( + pool.conninfo, kwargs=pool.kwargs, min_size=2, max_size=2, open=False + ) as observer: + await observer.wait(timeout=10) + + async def install_then_rollback(): + async with pool.connection() as conn, conn.transaction(): + await conn.execute(RESOURCES.joinpath(CHAIN[-1]).read_text()) + await gate.pause() + raise OSError("injected precommit interruption") + + task = asyncio.create_task(install_then_rollback()) + await gate.wait() + async with observer.connection() as conn: + assert ( + await ( + await conn.execute("SELECT to_regclass('reporting_notification_events')") + ).fetchone() + )[0] is None + gate.release() + with pytest.raises(OSError): + await task + async with observer.connection() as conn: + assert ( + await ( + await conn.execute("SELECT to_regclass('reporting_notification_events')") + ).fetchone() + )[0] is None + await PgReportingReconciliationStore(pool=observer).create_schema() + async with observer.connection() as conn: + await validate_schema(conn) + + +@pytest.mark.parametrize( + "damage", + [ + "ALTER TABLE reporting_notification_events" + " DISABLE TRIGGER reporting_notification_immutable", + "ALTER TABLE reporting_status_dirty ALTER COLUMN cause_generation DROP NOT NULL", + "DROP INDEX reporting_notification_deliveries_due", + "ALTER TABLE reporting_notification_deliveries DROP COLUMN body_sha256", + "ALTER TABLE reporting_notification_expansions" + " DROP CONSTRAINT reporting_notification_expansions_account_id_consumer_namespa_fkey", + "ALTER TABLE reporting_reconciliation_records" + " DISABLE TRIGGER reporting_reconciliation_guard", + "ALTER TABLE reporting_configurations" + " DROP CONSTRAINT reporting_configurations_pkey CASCADE", + "CREATE OR REPLACE FUNCTION reporting_notification_immutable() RETURNS TRIGGER" + " LANGUAGE plpgsql AS $$ BEGIN RETURN NEW; END $$", + ], +) +async def test_readiness_validates_the_installed_chain_not_table_presence(damage): + async with isolated_reporting_pool(autocommit=True) as pool: + await PgReportingReconciliationStore(pool=pool).create_schema() + async with pool.connection() as conn: + # PostgreSQL truncates generated FK names. Resolve the named table's + # FK in the one vector concerned with composite namespace binding. + if "DROP CONSTRAINT reporting_notification_expansions_" in damage: + from psycopg import sql + + name = ( + await ( + await conn.execute( + "SELECT conname FROM pg_constraint WHERE contype = 'f'" + " AND conrelid = 'reporting_notification_expansions'::regclass" + ) + ).fetchone() + )[0] + await conn.execute( + sql.SQL( + "ALTER TABLE reporting_notification_expansions DROP CONSTRAINT {}" + ).format(sql.Identifier(name)) + ) + else: + await conn.execute(damage) + before = await schema_contract(conn) + with pytest.raises(ReportingNotificationError, match="notification_schema_unready"): + await validate_schema(conn) + assert await schema_contract(conn) == before + + +async def test_malformed_outbox_upgrade_rolls_back_entire_chain(): + import psycopg + + async with isolated_reporting_pool(autocommit=True) as pool: + async with pool.connection() as conn: + await conn.execute((FIXTURES / "reporting_ledger_beta15.sql").read_text()) + await conn.execute("CREATE TABLE reporting_notification_events (adopter_marker text)") + with pytest.raises(psycopg.Error): + await PgReportingReconciliationStore(pool=pool).create_schema() + async with pool.connection() as conn: + assert ( + await ( + await conn.execute( + "SELECT count(*) FROM pg_attribute" + " WHERE attrelid = 'reporting_obligations'::regclass" + " AND attname = 'currency' AND NOT attisdropped" + ) + ).fetchone() + )[0] == 0 + + +async def test_default_off_upgrade_preserves_adopter_index_and_opt_in_checks_readiness(): + async with isolated_reporting_pool(autocommit=True) as pool: + store = PgReportingReconciliationStore(pool=pool) + await store.create_schema() + async with pool.connection() as conn: + await conn.execute( + "CREATE INDEX adopter_configuration_lookup" + " ON reporting_configurations (account_id, delivery_config_id)" + ) + original = ( + await ( + await conn.execute("SELECT 'adopter_configuration_lookup'::regclass::oid") + ).fetchone() + )[0] + # Existing Core startup retains its compatibility with adopter indexes. + await store.create_schema() + async with pool.connection() as conn: + retained = ( + await ( + await conn.execute("SELECT 'adopter_configuration_lookup'::regclass::oid") + ).fetchone() + )[0] + assert retained == original + # Enabling the outbox invokes the full conservative SDK chain check. + from adcp.reporting.outbox import PgReportingOutbox + + with pytest.raises(ReportingNotificationError, match="notification_schema_unready"): + await PgReportingOutbox(pool=pool).create_schema() diff --git a/tests/conformance/reporting/test_reporting_notification_outbox.py b/tests/conformance/reporting/test_reporting_notification_outbox.py new file mode 100644 index 000000000..08eba7f5c --- /dev/null +++ b/tests/conformance/reporting/test_reporting_notification_outbox.py @@ -0,0 +1,678 @@ +"""Shared deterministic notification state-machine vectors: memory and real PG.""" + +from __future__ import annotations + +import asyncio +import inspect +import json +from dataclasses import asdict, replace +from datetime import timedelta + +import pytest + +from adcp.reporting.ledger import ( + ConsumerStatusRecord, + InMemoryReportingLedgerStore, + LedgerConflictError, + ReportingAdjustmentRecord, + ReportingProducer, +) +from adcp.reporting.outbox import ( + ReportingEnvelopeCipher, + ReportingLegacyAuthentication, + ReportingNotificationError, + ReportingStatusScope, + validate_notification_payload, +) +from adcp.webhook_sender import ScopePermanentlyUnknown, ScopeTransientlyUnavailable, WebhookSender + +from ._generation_support import END, NOW, START, configuration, obligation_for, revision_for +from ._reconciliation_support import scenario +from ._reliable_support import ( + Barrier, + NotificationHarness, + SimulatedCrash, + notification_subscription, +) + + +async def seed(h: NotificationHarness, *, account: str = "acct_a", official: bool = False): + store = h.reliable.store + config = configuration(account) + await store.put_configuration(config) + obligation = await store.commit_obligation(obligation_for(config)) + revision, rows = revision_for(obligation) + if official: + revision = replace( + revision, + finality="official", + finality_basis="source_final", + finality_policy_id="policy-1", + finalized_at=END, + ) + await store.commit_revision(revision, rows) + return obligation, revision, rows + + +def statement(obligation, consumer="buyer", suffix="one"): + return ConsumerStatusRecord( + reporting_status_id=f"consumer-status-{consumer}-{suffix}", + account_id=obligation.account_id, + consumer_id=consumer, + delivery_config_id=obligation.delivery_config_id, + delivery_config_version=obligation.delivery_config_version, + report_definition_id=obligation.report_definition_id, + period_start=START, + period_end=END, + period_source_timezone="UTC", + consumer_status="unavailable", + status_as_of=NOW, + recorded_at=NOW, + reporting_obligation_id=obligation.reporting_obligation_id, + ) + + +async def test_commit_replay_restart_fanout_and_random_reemission(notification_harness): + h = notification_harness + _, revision, rows = await seed(h) + store = h.reliable.store + before_dirty = await h.outbox.read_status_dirty(account_id="acct_a") + await store.commit_revision(revision, rows) + assert await h.outbox.read_status_dirty(account_id="acct_a") == before_dirty + (event,) = await h.outbox.list_events(account_id="acct_a") + assert event.fired_at == h.reliable.clock() + h.subscriptions.put(notification_subscription(subscriber="second")) + await h.reliable.restart() + assert await h.outbox.list_events(account_id="acct_a") == (event,) + await h.drain() + statuses = await h.outbox.list_deliveries(account_id="acct_a") + assert len(statuses) == 2 and {item.state for item in statuses} == {"complete"} + received = h.receiver.received + assert len({json.loads(item.body)["notification_id"] for item in received}) == 1 + assert len({item.idempotency_key for item in received}) == 2 + for item in received: + validate_notification_payload(json.loads(item.body)) + assert json.loads(item.body)["fired_at"] == event.fired_at.isoformat().replace( + "+00:00", "Z" + ) + assert ( + await h.outbox.reemit( + account_id="acct_a", notification_id=event.notification_id, now=h.reliable.clock() + ) + == 2 + ) + await h.drain() + assert len({item.idempotency_key for item in h.receiver.received}) == 4 + assert {json.loads(item.body)["notification_id"] for item in h.receiver.received} == { + event.notification_id + } + assert await h.outbox.list_events(account_id="other") == () + assert await h.outbox.list_deliveries(account_id="other") == () + assert await h.outbox.read_status_dirty(account_id="other") == () + + +async def test_two_consumers_and_rapid_mutable_transitions_remain_replayable(notification_harness): + h = notification_harness + obligation, revision, _ = await seed(h) + store = h.reliable.store + for consumer in ("buyer", "auditor"): + status = statement(obligation, consumer) + assert (await store.record_consumer_status(status))[1] + assert not (await store.record_consumer_status(status))[1] + scope = ReportingStatusScope.for_obligation(obligation, consumer) + issue = await store.ensure_issue_opened( + issue_key=f"opaque|not/a/scope?token={consumer}", + account_id="acct_a", + consumer_id=consumer, + observed_at=h.reliable.clock(), + status_scope=scope, + ) + h.reliable.clock.advance() + await store.set_issue_state( + issue_key=issue.issue_key, + account_id="acct_a", + state="acknowledged", + at=h.reliable.clock(), + ) + await store.set_issue_state( + issue_key=issue.issue_key, + account_id="acct_a", + state="acknowledged", + at=h.reliable.clock(), + ) + h.reliable.clock.advance() + await store.retire_issue( + issue_key=issue.issue_key, account_id="acct_a", at=h.reliable.clock() + ) + for readable in (False, False, True, False, True): + h.reliable.clock.advance() + await store.set_revision_readable( + account_id="acct_a", + reporting_revision_id=revision.reporting_revision_id, + readable=readable, + ) + dirty = await h.outbox.read_status_dirty(account_id="acct_a") + assert [record.sequence for record in dirty] == list(range(1, len(dirty) + 1)) + readability = [record for record in dirty if record.reason == "readability"] + assert [(record.before.readable, record.after.readable) for record in readability] == [ + (True, False), + (False, True), + (True, False), + (False, True), + ] + assert [record.cause_generation for record in readability] == [1, 2, 3, 4] + for consumer in ("buyer", "auditor"): + issues = [ + record + for record in dirty + if record.reason == "issue" and record.scope.consumer_id == consumer + ] + assert [record.after.issue_state for record in issues] == [ + "open", + "acknowledged", + "resolved", + ] + assert [record.before.issue_state if record.before else None for record in issues] == [ + None, + "open", + "acknowledged", + ] + assert [record.cause_generation for record in issues] == [1, 2, 3] + assert len({record.cause_id for record in issues}) == 1 + assert {record.scope.reporting_obligation_id for record in issues} == { + obligation.reporting_obligation_id + } + assert len(await h.outbox.list_events(account_id="acct_a")) == 1 + assert "opaque" not in json.dumps([asdict(record) for record in dirty], default=str) + await h.reliable.restart() + assert await h.outbox.read_status_dirty(account_id="acct_a") == dirty + assert await h.outbox.advance_status_checkpoint( + account_id="acct_a", projector_id="future-B", expected=0, through=dirty[3].sequence + ) + assert not await h.outbox.advance_status_checkpoint( + account_id="acct_a", projector_id="future-B", expected=0, through=dirty[-1].sequence + ) + assert await h.outbox.status_checkpoint(account_id="other", projector_id="future-B") == 0 + with pytest.raises(ValueError): + await h.outbox.advance_status_checkpoint( + account_id="other", projector_id="future-B", expected=0, through=1 + ) + + +async def test_issue_scope_is_optional_and_cannot_cross_consumer_or_account(notification_harness): + h = notification_harness + obligation, _, _ = await seed(h) + with pytest.raises(ReportingNotificationError, match="invalid_status_scope"): + await h.reliable.store.ensure_issue_opened( + issue_key="opaque", + account_id="acct_a", + consumer_id="buyer", + observed_at=NOW, + status_scope=ReportingStatusScope.for_obligation(obligation, "auditor"), + ) + assert await h.reliable.store.get_issue(issue_key="opaque", account_id="acct_a") is None + await h.reliable.store.ensure_issue_opened( + issue_key="legacy:unknown/scope", account_id="acct_a", consumer_id="buyer", observed_at=NOW + ) + latest = (await h.outbox.read_status_dirty(account_id="acct_a"))[-1] + assert latest.scope == ReportingStatusScope("acct_a", consumer_id="buyer") + + +async def test_verified_ready_identity_includes_consumer_namespace(notification_harness): + h = notification_harness + h.subscriptions.put(notification_subscription(subscriber="audit", principal="auditor")) + for consumer in ("buyer", "auditor"): + s = await scenario(h.reliable.store, consumer_id=consumer) + await h.reliable.store.commit_materialization(s.outcome) + assert not (await h.reliable.store.commit_materialization(s.outcome))[1] + events = await h.outbox.list_events(account_id="acct_a") + ready = [event for event in events if event.notification_type == "reporting.delivery_ready"] + assert len(ready) == 2 and len({event.notification_id for event in ready}) == 2 + assert {event.cause.consumer_id for event in ready} == {"buyer", "auditor"} + assert len({event.causal_key for event in ready}) == 2 + await h.drain() + deliveries = [ + json.loads(item.body) + for item in h.receiver.received + if json.loads(item.body)["notification_type"] == "reporting.delivery_ready" + ] + assert len(deliveries) == 2 + expected = {event.notification_id: event.cause.consumer_id for event in ready} + actual_principals = {"buyer": "buyer", "audit": "auditor"} + assert all( + actual_principals[item["subscriber_id"]] == expected[item["notification_id"]] + for item in deliveries + ) + + +async def test_core_explicit_ready_subscription_never_enqueues_ready(notification_harness): + h = notification_harness + h.subscriptions.values.clear() + h.subscriptions.put( + notification_subscription(events=("reporting.delivery_ready", "reporting.status_changed")) + ) + obligation, revision, rows = await seed(h) + # Neither a profile label nor a subscriber's capability request creates + # a frozen Managed destination/obligation/materialization binding. + assert obligation.reporting_profile == "paid_media_delivery" + await h.reliable.store.commit_revision(revision, rows) + await h.drain() + assert h.receiver.received == [] + assert { + event.notification_type for event in await h.outbox.list_events(account_id="acct_a") + } == {"reporting.ledger_changed"} + assert "notifications" not in inspect.signature(ReportingProducer).parameters + assert not hasattr(InMemoryReportingLedgerStore(), "commit_materialization") + + +async def test_adjustments_use_exclusive_allowlisted_payload(notification_harness): + h = notification_harness + _, revision, _ = await seed(h, official=True) + adjustment = ReportingAdjustmentRecord( + "adjustment-1", + "acct_a", + revision.reporting_revision_id, + "source_correction", + START, + END, + (("impressions", "-1"),), + NOW, + NOW, + reason_detail="provider token=NEVER_COPY https://secret.example.test/path?key=SECRET", + ) + await h.reliable.store.commit_adjustment(adjustment) + await h.reliable.store.commit_adjustment(adjustment) + await h.drain() + values = [json.loads(item.body) for item in h.receiver.received] + assert len(values) == 2 + changed = next(item for item in values if item["change_kind"] == "adjustment_published") + assert not ( + {"finality", "reporting_revision_id", "supersedes_reporting_revision_id"} & changed.keys() + ) + assert "NEVER_COPY" not in json.dumps(values) + assert "SECRET" not in json.dumps(values) + bad = {**changed, "reporting_revision_id": "revision-extra"} + with pytest.raises(ReportingNotificationError): + validate_notification_payload(bad) + + +async def test_concurrent_identical_commit_and_conflict_have_no_ghost(notification_harness): + h = notification_harness + config = configuration() + await h.reliable.store.put_configuration(config) + obligation = await h.reliable.store.commit_obligation(obligation_for(config)) + revision, rows = revision_for(obligation) + result = await asyncio.gather( + *(h.reliable.store.commit_revision(revision, rows) for _ in range(5)) + ) + assert result == [revision] * 5 + conflicting = replace(revision, revision_content_sha256="f" * 64) + result = await asyncio.gather( + h.reliable.store.commit_revision(revision, rows), + h.reliable.store.commit_revision(conflicting, rows), + return_exceptions=True, + ) + assert isinstance(result[1], LedgerConflictError) + assert len(await h.outbox.list_events(account_id="acct_a")) == 1 + assert ( + len( + [ + d + for d in await h.outbox.read_status_dirty(account_id="acct_a") + if d.reason == "revision" + ] + ) + == 1 + ) + + +async def test_concurrent_fanout_and_http_claims_have_one_active_attempt(notification_harness): + h = notification_harness + await seed(h) + assert ( + sum(await asyncio.gather(*(h.worker().expand_one(account_id="acct_a") for _ in range(5)))) + == 1 + ) + assert len(await h.outbox.list_deliveries(account_id="acct_a")) == 1 + barrier = Barrier() + h.reliable.failures.at("http.accepted", barrier) + first = asyncio.create_task(h.worker().deliver_one(account_id="acct_a")) + try: + await barrier.wait() + assert not await h.worker().deliver_one(account_id="acct_a") + assert len(h.receiver.received) == 1 + finally: + barrier.release() + await first + + +async def test_expired_lease_reclaims_and_all_stale_updates_are_fenced(notification_harness): + h = notification_harness + await seed(h) + await h.worker().expand_one(account_id="acct_a") + original = await h.outbox.claim_delivery( + account_id="acct_a", now=h.reliable.clock(), lease_seconds=60 + ) + assert original is not None + h.reliable.clock.advance(timedelta(seconds=61)) + reclaimed = await h.outbox.claim_delivery( + account_id="acct_a", now=h.reliable.clock(), lease_seconds=60 + ) + assert reclaimed is not None and original.token != reclaimed.token + assert original.delivery == reclaimed.delivery + for state in ("complete", "pending", "quarantined", "suppressed"): + assert not await h.outbox.finish_delivery(original, now=h.reliable.clock(), state=state) + assert await h.outbox.finish_delivery(reclaimed, now=h.reliable.clock(), state="complete") + assert (await h.outbox.list_deliveries(account_id="acct_a"))[0].state == "complete" + + +@pytest.mark.parametrize("stage", ["dns", "signing"]) +async def test_expiry_during_preparation_is_fenced_before_http( + notification_harness, monkeypatch, stage +): + import socket + + from adcp.webhook_auth import JwkSignerStrategy + + h = notification_harness + await seed(h) + await h.worker().expand_one(account_id="acct_a") + (original,) = await h.outbox.list_deliveries(account_id="acct_a") + target, name = ( + (socket, "getaddrinfo") if stage == "dns" else (JwkSignerStrategy, "build_auth_headers") + ) + prepare = getattr(target, name) + expired = False + + def expire_after_preparation(*args, **kwargs): + nonlocal expired + result = prepare(*args, **kwargs) + if not expired: + expired = True + h.reliable.clock.advance(timedelta(seconds=61)) + return result + + monkeypatch.setattr(target, name, expire_after_preparation) + assert await h.worker().deliver_one(account_id="acct_a") + assert expired and not h.receiver.connections and not h.receiver.received + (retained,) = await h.outbox.list_deliveries(account_id="acct_a") + assert retained.state == "leased" and retained.delivery == original.delivery + await h.drain() + (completed,) = await h.outbox.list_deliveries(account_id="acct_a") + assert completed.state == "complete" and completed.delivery == original.delivery + assert len(h.receiver.received) == 1 + + +async def test_reclaimed_expansion_rejects_stale_snapshot_and_all_finishes(notification_harness): + h = notification_harness + await seed(h) + (event,) = await h.outbox.list_events(account_id="acct_a") + stale = await h.outbox.claim_expansion( + account_id="acct_a", now=h.reliable.clock(), lease_seconds=60 + ) + assert stale is not None + h.reliable.clock.advance(timedelta(seconds=61)) + current = await h.outbox.claim_expansion( + account_id="acct_a", now=h.reliable.clock(), lease_seconds=60 + ) + assert current is not None and current.token != stale.token + old = h.cipher.prepare(event, notification_subscription(subscriber="old"), 1) + assert not await h.outbox.complete_expansion(stale, (old,), now=h.reliable.clock()) + for state in ("complete", "pending", "quarantined", "suppressed"): + assert not await h.outbox.finish_expansion(stale, now=h.reliable.clock(), state=state) + assert await h.outbox.list_deliveries(account_id="acct_a") == () + new = h.cipher.prepare(event, notification_subscription(subscriber="new"), 1) + assert await h.outbox.complete_expansion(current, (new,), now=h.reliable.clock()) + (row,) = await h.outbox.list_deliveries(account_id="acct_a") + assert row.delivery == new + + +async def test_acceptance_before_ack_retries_exact_body_key_with_rotated_signature( + notification_harness, +): + h = notification_harness + await seed(h) + await h.worker().expand_one(account_id="acct_a") + h.reliable.failures.at("http.accepted", SimulatedCrash()) + with pytest.raises(SimulatedCrash): + await h.worker().deliver_one(account_id="acct_a") + first = h.receiver.received[0] + h.reliable.clock.advance(timedelta(seconds=61)) + h.signing.generation = 2 + await h.reliable.restart() + await h.drain() + second = h.receiver.received[1] + assert first.body == second.body and first.idempotency_key == second.idempotency_key + assert first.headers["signature"] != second.headers["signature"] + assert ( + "key-1" in first.headers["signature-input"] and "key-2" in second.headers["signature-input"] + ) + assert len(h.signing.calls) == 2 + assert await h.reliable.receiver.read("acct_a", first.idempotency_key) == first.body + assert (await h.outbox.list_deliveries(account_id="acct_a"))[0].state == "complete" + + +@pytest.mark.parametrize("status", [408, 425, 429, 500, 503, 599]) +async def test_retryable_http_and_retry_state_survive_restart(notification_harness, status): + h = notification_harness + await seed(h) + h.receiver.responses["buyer"].append(status) + await h.drain() + first = h.receiver.received[0] + assert (await h.outbox.list_deliveries(account_id="acct_a"))[0].error_code == "retryable_http" + await h.reliable.restart() + h.reliable.clock.advance(timedelta(seconds=6)) + await h.drain() + assert len(h.receiver.received) == 2 + assert first.body == h.receiver.received[1].body + + +@pytest.mark.parametrize("status", [302, 400, 401, 403, 404, 410, 422]) +async def test_permanent_http_is_quarantined_and_provider_text_is_never_retained( + notification_harness, status +): + h = notification_harness + await seed(h) + h.receiver.responses["buyer"].append(status) + await h.drain() + (row,) = await h.outbox.list_deliveries(account_id="acct_a") + assert row.state == "quarantined" and row.error_code == "permanent_http" + assert "DO_NOT_PERSIST" not in repr(row) + assert "URL_SECRET" not in repr(row) + assert len(h.receiver.received) == 1 + + +@pytest.mark.parametrize( + "change", + ["removed", "url", "credentials", "principal", "inactive", "authz", "proof", "events", "scope"], +) +async def test_removed_or_replaced_subscription_is_suppressed_without_external_leakage( + notification_harness, change +): + h = notification_harness + await seed(h) + await h.worker().expand_one(account_id="acct_a") + original = h.subscriptions.values[("acct_a", "buyer")] + replacements = { + "url": dict(url="https://replacement.example.test/new?token=NEW_SECRET"), + "credentials": dict( + signing_scope_id=None, + authentication=ReportingLegacyAuthentication("Bearer", "NEW_SECRET"), + ), + "principal": dict(principal_id="other"), + "inactive": dict(active=False), + "authz": dict(authorized=False), + "proof": dict(proof_valid=False), + "events": dict(event_types=("reporting.delivery_ready",)), + "scope": dict(signing_scope_id="scope-two"), + } + if change == "removed": + del h.subscriptions.values[("acct_a", "buyer")] + else: + h.subscriptions.put(replace(original, **replacements[change])) + await h.worker().deliver_one(account_id="acct_a") + (row,) = await h.outbox.list_deliveries(account_id="acct_a") + assert row.state == "suppressed" + assert not h.signing.calls and not h.receiver.connections and not h.receiver.dns_calls + assert h.cipher.open(row.delivery).subscription == original + + +async def test_subscriber_failure_does_not_block_an_independent_subscriber(notification_harness): + h = notification_harness + await seed(h) + h.subscriptions.put(notification_subscription(subscriber="healthy")) + h.receiver.responses["buyer"].append(503) + await h.drain() + rows = await h.outbox.list_deliveries(account_id="acct_a") + assert {row.delivery.binding.subscriber_id: row.state for row in rows} == { + "buyer": "pending", + "healthy": "complete", + } + + +@pytest.mark.parametrize( + "error,expected", + [ + (ScopePermanentlyUnknown("secret"), "quarantined"), + (ScopeTransientlyUnavailable("secret"), "pending"), + (OSError("provider secret"), "pending"), + ], +) +async def test_signing_failures_are_closed_classifications(notification_harness, error, expected): + h = notification_harness + await seed(h) + h.reliable.failures.at("signing.resolve", error) + await h.drain() + (row,) = await h.outbox.list_deliveries(account_id="acct_a") + assert row.state == expected + assert "secret" not in repr(row) + assert not h.receiver.connections + + +@pytest.mark.parametrize("fake_type", ["object", "subclass"]) +async def test_fake_or_subclass_sender_never_crosses_owned_transport_boundary( + notification_harness, fake_type +): + h = notification_harness + await seed(h) + calls = [] + + class MaliciousSender(WebhookSender): + async def send_prepared(self, prepared): + calls.append(prepared) + raise AssertionError("secret leaked") + + fake = object.__new__(MaliciousSender) if fake_type == "subclass" else type("Fake", (), {})() + fake._owns_client = True + fake._allow_private_destinations = False + if fake_type == "object": + fake.signs_with_rfc9421 = True + + async def malicious(**kwargs): + return fake + + h.signing.resolve = malicious + await h.drain() + assert not calls and not h.receiver.dns_calls and not h.receiver.connections + assert (await h.outbox.list_deliveries(account_id="acct_a"))[0].state == "quarantined" + + +async def test_envelope_key_rotation_preserves_old_prepared_bytes(notification_harness): + h = notification_harness + await seed(h) + await h.worker().expand_one(account_id="acct_a") + (row,) = await h.outbox.list_deliveries(account_id="acct_a") + previous = h.cipher.open(row.delivery).prepared + h.cipher = ReportingEnvelopeCipher(b"n" * 32, key_version="v2", previous_keys={"v1": b"e" * 32}) + assert h.cipher.open(row.delivery).prepared == previous + await h.drain() + assert h.receiver.received[0].body == previous.body + + +@pytest.mark.parametrize( + "issue_ids", [None, [], [""], ["ok"] * 2, [f"issue-{i}" for i in range(17)]] +) +def test_full_rc3_status_conditional_and_bounded_issue_ids(issue_ids): + value = { + "idempotency_key": "0" * 32, + "notification_id": "status-1", + "subscriber_id": "buyer", + "account_id": "acct_a", + "notification_type": "reporting.status_changed", + "fired_at": NOW.isoformat(), + "delivery_config_id": "daily", + "delivery_config_version": 1, + "feed_purpose": "analytics", + "health": "delayed", + } + if issue_ids is not None: + value["issue_ids"] = issue_ids + with pytest.raises(ReportingNotificationError): + validate_notification_payload(value) + value["issue_ids"] = ["issue-1"] + validate_notification_payload(value) + + +async def test_claim_crashes_do_not_exhaust_delivery_retry_budget(notification_harness): + h = notification_harness + await seed(h) + await h.worker().expand_one(account_id="acct_a") + for _ in range(12): + lease = await h.outbox.claim_delivery( + account_id="acct_a", now=h.reliable.clock(), lease_seconds=1 + ) + assert lease is not None + h.reliable.clock.advance(timedelta(seconds=2)) + assert h.receiver.received == [] + await h.reliable.restart() + await h.drain() + assert len(h.receiver.received) == 1 + assert (await h.outbox.list_deliveries(account_id="acct_a"))[0].state == "complete" + + +async def test_empty_fanout_completes_but_transient_resolution_does_not(notification_harness): + h = notification_harness + await seed(h) + h.subscriptions.values.clear() + h.reliable.failures.at("subscriptions.list.before", OSError("provider token=SECRET")) + assert await h.worker().expand_one(account_id="acct_a") + assert not await h.worker().expand_one(account_id="acct_a") + h.reliable.clock.advance(timedelta(seconds=6)) + assert await h.worker().expand_one(account_id="acct_a") + h.subscriptions.put(notification_subscription()) + h.reliable.clock.advance(timedelta(hours=1)) + assert not await h.worker().expand_one(account_id="acct_a") + assert len(h.subscriptions.lists) == 2 + assert await h.outbox.list_deliveries(account_id="acct_a") == () + + +async def test_duplicate_subscriber_snapshot_is_rejected_atomically(notification_harness): + h = notification_harness + await seed(h) + subscription = notification_subscription() + + async def duplicate(**kwargs): + return (subscription, replace(subscription, configuration_revision="different")) + + h.subscriptions.list_active = duplicate + assert await h.worker().expand_one(account_id="acct_a") + assert not await h.worker().expand_one(account_id="acct_a") + assert await h.outbox.list_deliveries(account_id="acct_a") == () + + +async def test_one_blocked_subscriber_does_not_hold_other_http_work(notification_harness): + h = notification_harness + await seed(h) + h.subscriptions.put(notification_subscription(subscriber="second")) + await h.worker().expand_one(account_id="acct_a") + barrier = Barrier() + h.reliable.failures.at("http.accepted", barrier) + first = asyncio.create_task(h.worker().deliver_one(account_id="acct_a")) + try: + await barrier.wait() + assert await h.worker().deliver_one(account_id="acct_a") + assert len(h.receiver.received) == 2 + assert {item.subscriber_id for item in h.receiver.received} == {"buyer", "second"} + finally: + barrier.release() + await first diff --git a/tests/conformance/reporting/test_reporting_notification_packaging.py b/tests/conformance/reporting/test_reporting_notification_packaging.py new file mode 100644 index 000000000..9fb362859 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_notification_packaging.py @@ -0,0 +1,262 @@ +"""Real wheel/sdist installation, lazy base imports, and installed [pg] restart.""" + +from __future__ import annotations + +import asyncio +import json +import os +import shutil +import signal +import subprocess +import sys +import tarfile +import zipfile +from pathlib import Path + +import pytest +from pydantic import TypeAdapter + +from ._generation_support import ( + NOW, + configuration, + isolated_reporting_pool, + obligation_for, + revision_for, +) +from .test_reporting_notification_migration import CHAIN + +ROOT = Path(__file__).resolve().parents[3] + + +def run_step(command, *, label, cwd, value=None, timeout=120): + process = subprocess.Popen( + command, + cwd=cwd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + start_new_session=True, + env={key: item for key, item in os.environ.items() if key != "PYTHONPATH"}, + ) + print(f"notification_distribution stage={label} pid={process.pid} started", flush=True) + try: + stdout, _ = process.communicate( + json.dumps(value) if value is not None else None, + timeout=timeout, + ) + except subprocess.TimeoutExpired: + # The new session contains only this test's build/install descendants. + # Kill the owned group too, so a package-manager child cannot survive a + # timed-out build or keep an inherited pipe open indefinitely. + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + try: + process.communicate(timeout=10) + except subprocess.TimeoutExpired: + raise AssertionError( + f"notification distribution {label}: cleanup_deadline pid={process.pid}" + ) from None + raise AssertionError( + f"notification distribution {label}: deadline" + f" pid={process.pid} exit={process.returncode}" + ) from None + # Do not turn package-manager/provider output into test diagnostics. + assert process.returncode == 0, f"notification distribution {label}: exit {process.returncode}" + print(f"notification_distribution stage={label} passed", flush=True) + return stdout + + +def test_distribution_subprocess_deadline_is_bounded_and_sanitized(tmp_path): + with pytest.raises(AssertionError, match=r"deadline_probe: deadline pid=\d+ exit=-9"): + run_step( + [sys.executable, "-c", "import threading; threading.Event().wait()"], + label="deadline_probe", + cwd=tmp_path, + timeout=0, + ) + + +@pytest.fixture(scope="module") +def installed_distribution(tmp_path_factory): + path = tmp_path_factory.mktemp("reporting-outbox-distribution") + project = path / "project" + project.mkdir() + for name in ("pyproject.toml", "setup.py", "MANIFEST.in", "README.md", "LICENSE"): + shutil.copy2(ROOT / name, project / name) + for source in ROOT.glob("MIGRATION*.md"): + shutil.copy2(source, project / source.name) + shutil.copytree( + ROOT / "src", + project / "src", + ignore=shutil.ignore_patterns("__pycache__", "*.egg-info", "_schemas"), + ) + for version in ("2.5", "3.0", "3.1", "3.2.0-rc.3"): + shutil.copytree( + ROOT / "schemas" / "cache" / version, project / "schemas" / "cache" / version + ) + dist = path / "dist" + # build's default path makes an sdist, then builds the wheel FROM that sdist. + run_step( + [sys.executable, "-m", "build", "--outdir", str(dist), str(project)], + label="build-sdist-and-wheel", + cwd=path, + ) + wheel, source = next(dist.glob("*.whl")), next(dist.glob("*.tar.gz")) + environment = path / "installed" + run_step( + [sys.executable, "-m", "venv", str(environment)], + label="isolated-environment", + cwd=path, + timeout=30, + ) + python = environment / "bin" / "python" + # UV is an acceleration only; normal pip-based CI exercises the same wheel. + installer = ( + [shutil.which("uv"), "pip", "install", "--python", str(python)] + if shutil.which("uv") + else [str(python), "-m", "pip", "install"] + ) + run_step([*installer, str(wheel)], label="base-install", cwd=path) + base_check = r""" +import importlib.util +from importlib.resources import files +from pathlib import Path +import adcp +from adcp.reporting.outbox import ( + InMemoryReportingOutbox, + ReportingEnvelopeCipher, + ReportingNotificationWorker, +) +from adcp.reporting.ledger import InMemoryReportingLedgerStore, ReportingProducer +from adcp.validation.schema_loader import get_named_validator +import inspect + +assert importlib.util.find_spec("psycopg") is None +assert importlib.util.find_spec("psycopg_pool") is None +assert "installed" in str(Path(adcp.__file__)) +assert "notifications" not in inspect.signature(ReportingProducer).parameters +assert not hasattr(InMemoryReportingLedgerStore(), "commit_materialization") +outbox = InMemoryReportingOutbox(InMemoryReportingLedgerStore(notifications=True)) +assert files("adcp.reporting.ledger").joinpath("reporting_notification_outbox.sql").is_file() +assert ( + get_named_validator("core/reporting-ledger-changed-webhook.json", version="3.2.0-rc.3") + is not None +) +print("base-import-without-pg-ok") +""" + assert ( + run_step([str(python), "-c", base_check], label="base-import-without-pg", cwd=path).strip() + == "base-import-without-pg-ok" + ) + return path, python, installer, wheel, source + + +def test_wheel_and_sdist_contain_exact_complete_sql_chain(installed_distribution): + _, _, _, wheel, source = installed_distribution + with zipfile.ZipFile(wheel) as archive, tarfile.open(source) as tar: + prefix = tar.getnames()[0].split("/")[0] + for name in CHAIN: + expected = (ROOT / "src" / "adcp" / "reporting" / "ledger" / name).read_bytes() + assert archive.read(f"adcp/reporting/ledger/{name}") == expected + member = tar.extractfile(f"{prefix}/src/adcp/reporting/ledger/{name}") + assert member is not None and member.read() == expected + assert ( + archive.read("adcp/reporting/outbox/_schema.py") + == (ROOT / "src" / "adcp" / "reporting" / "outbox" / "_schema.py").read_bytes() + ) + + +def test_installed_base_import_needs_no_pg_extra(installed_distribution): + # The fixture runs the import with PG genuinely absent, before any extra + # installation, so collection/test order cannot accidentally fake this gate. + _, python, _, _, _ = installed_distribution + assert python.is_file() + + +async def test_installed_pg_extra_migrates_commits_and_restarts(installed_distribution): + path, python, installer, wheel, _ = installed_distribution + async with isolated_reporting_pool(autocommit=True) as pool: + await asyncio.to_thread( + run_step, [*installer, str(wheel) + "[pg]"], label="pg-install", cwd=path + ) + config = configuration() + obligation = obligation_for(config) + revision, rows = revision_for(obligation) + values = { + "conninfo": pool.conninfo, + "kwargs": pool.kwargs, + "now": NOW.isoformat(), + "rows": rows, + } + for name, record in ( + ("config", config), + ("obligation", obligation), + ("revision", revision), + ): + values[name] = TypeAdapter(type(record)).dump_python(record, mode="json") + script = r""" +import asyncio, json, sys +from datetime import datetime +from pydantic import TypeAdapter +from psycopg_pool import AsyncConnectionPool +from adcp.reporting.ledger import ( + PgReportingReconciliationStore, + ReportingConfiguration, + ReportingObligationRecord, + ReportingRevisionRecord, +) +from adcp.reporting.outbox import PgReportingOutbox + +values = json.load(sys.stdin) + + +async def main(): + clock = lambda: datetime.fromisoformat(values["now"]) + async with AsyncConnectionPool( + values["conninfo"], kwargs=values["kwargs"], open=False + ) as pool: + await pool.wait(timeout=10) + store = PgReportingReconciliationStore(pool=pool, clock=clock, notifications=True) + await store.create_schema() + await store.put_configuration( + TypeAdapter(ReportingConfiguration).validate_python(values["config"]) + ) + await store.commit_obligation( + TypeAdapter(ReportingObligationRecord).validate_python(values["obligation"]) + ) + await store.commit_revision( + TypeAdapter(ReportingRevisionRecord).validate_python(values["revision"]), + values["rows"], + ) + events = await PgReportingOutbox(pool=pool, clock=clock).list_events( + account_id="acct_a" + ) + async with AsyncConnectionPool( + values["conninfo"], kwargs=values["kwargs"], open=False + ) as fresh: + await fresh.wait(timeout=10) + await PgReportingReconciliationStore(pool=fresh).create_schema() + outbox = PgReportingOutbox(pool=fresh, clock=clock) + assert len(events) == 1 and await outbox.list_events(account_id="acct_a") == events + assert ( + await outbox.claim_expansion(account_id="acct_a", now=clock(), lease_seconds=60) + is not None + ) + assert await outbox.list_events(account_id="other") == () + print("installed-pg-restart-ok") + + +asyncio.run(asyncio.wait_for(main(), 35)) +""" + result = await asyncio.to_thread( + run_step, + [str(python), "-c", script], + label="pg-migration-restart", + cwd=path, + value=values, + timeout=45, + ) + assert result.strip() == "installed-pg-restart-ok" diff --git a/tests/conformance/reporting/test_reporting_notification_process_matrix.py b/tests/conformance/reporting/test_reporting_notification_process_matrix.py new file mode 100644 index 000000000..c918bbe4f --- /dev/null +++ b/tests/conformance/reporting/test_reporting_notification_process_matrix.py @@ -0,0 +1,275 @@ +"""Cross-layer real PostgreSQL failure matrix, extended by later slices. + +Producer, fanout worker, HTTP worker, receiver and observer have independent +processes/pools. Named IPC/SQL barriers control interleavings; no sleeps. +""" + +from __future__ import annotations + +import asyncio +import base64 +import json +from datetime import datetime, timedelta, timezone +from functools import wraps + +import pytest + +from adcp.reporting.ledger import PgReportingReconciliationStore +from adcp.reporting.outbox import DeliveryLease, PgReportingOutbox + +from ._generation_support import isolated_reporting_pool +from ._reliable_support import ManualClock, _BytesStore, service_process + + +def case_deadline(function): + @wraps(function) + async def run(*args, **kwargs): + try: + return await asyncio.wait_for(function(*args, **kwargs), 150) + except (TimeoutError, asyncio.TimeoutError): + raise AssertionError(f"reporting_matrix case={function.__name__} deadline") from None + + return run + + +async def settle(pool): + """A SQL barrier ensures disconnected transactions released their fences.""" + async with pool.connection() as conn, conn.transaction(): + await PgReportingReconciliationStore._lock_account(conn, "acct_a") + await conn.execute( + "SELECT 1 FROM reporting_notification_expansions WHERE account_id = %s FOR UPDATE", + ("acct_a",), + ) + await conn.execute( + "SELECT 1 FROM reporting_notification_deliveries WHERE account_id = %s FOR UPDATE", + ("acct_a",), + ) + + +async def produce_and_expand(pool): + async with service_process(pool, "producer") as producer: + await producer.event("done") + await producer.finish() + async with service_process(pool, "fanout") as fanout: + assert (await fanout.event("done"))["did_work"] + await fanout.finish() + + +@pytest.mark.parametrize("crash", ["event_inserted", "revision_committed"]) +@case_deadline +async def test_revision_commit_to_fanout_process_crash_restart(crash): + async with isolated_reporting_pool(autocommit=True) as pool: + await PgReportingReconciliationStore(pool=pool).create_schema() + clock = ManualClock() + observer = PgReportingReconciliationStore(pool=pool, clock=clock, notifications=True) + outbox = PgReportingOutbox(pool=pool, clock=clock) + async with service_process(pool, "producer", pause=crash) as producer: + await producer.event(crash) + revision = await observer.get_revision( + account_id="acct_a", reporting_revision_id="rpr_acct_a_process" + ) + events = await outbox.list_events(account_id="acct_a") + assert (revision is not None) == (crash == "revision_committed") + assert len(events) == (1 if crash == "revision_committed" else 0) + await producer.kill() + await settle(pool) + await produce_and_expand(pool) + after = await outbox.list_events(account_id="acct_a") + assert len(after) == 1 + if events: + assert after == events + assert (await outbox.list_deliveries(account_id="acct_a"))[0].state == "pending" + assert await outbox.list_events(account_id="other") == () + assert await outbox.list_deliveries(account_id="other") == () + + +@case_deadline +async def test_process_death_partway_through_fanout_cannot_mix_membership(): + async with isolated_reporting_pool(autocommit=True) as pool: + await PgReportingReconciliationStore(pool=pool).create_schema() + async with service_process(pool, "producer") as producer: + await producer.event("done") + await producer.finish() + clock = ManualClock() + outbox = PgReportingOutbox(pool=pool, clock=clock) + async with service_process( + pool, "fanout", subscribers=["old-a", "old-b"], pause="fanout_partial" + ) as fanout: + await fanout.event("fanout_partial") + assert await outbox.list_deliveries(account_id="acct_a") == () + await fanout.kill() + await settle(pool) + async with service_process( + pool, "fanout", subscribers=["replacement"], advance_seconds=61 + ) as restarted: + assert (await restarted.event("done"))["did_work"] + await restarted.finish() + rows = await outbox.list_deliveries(account_id="acct_a") + assert [row.delivery.binding.subscriber_id for row in rows] == ["replacement"] + assert len(await outbox.list_events(account_id="acct_a")) == 1 + async with service_process( + pool, "fanout", subscribers=["yet-another"], advance_seconds=62 + ) as replay: + assert not (await replay.event("done"))["did_work"] + assert await outbox.list_deliveries(account_id="acct_a") == rows + + +@pytest.fixture +def certificate(tmp_path): + from cryptography import x509 + from cryptography.hazmat.primitives import hashes, serialization + from cryptography.hazmat.primitives.asymmetric import ec + from cryptography.x509.oid import ExtendedKeyUsageOID, NameOID + + key = ec.derive_private_key(3, ec.SECP256R1()) + subject = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "receiver.example.test")]) + cert = ( + x509.CertificateBuilder() + .subject_name(subject) + .issuer_name(subject) + .public_key(key.public_key()) + .serial_number(1) + .not_valid_before(datetime(2020, 1, 1, tzinfo=timezone.utc)) + .not_valid_after(datetime(2040, 1, 1, tzinfo=timezone.utc)) + .add_extension(x509.BasicConstraints(ca=True, path_length=None), critical=True) + .add_extension( + x509.SubjectAlternativeName([x509.DNSName("receiver.example.test")]), critical=False + ) + .add_extension(x509.ExtendedKeyUsage([ExtendedKeyUsageOID.SERVER_AUTH]), critical=False) + .sign(key, hashes.SHA256()) + ) + cert_file, key_file = tmp_path / "receiver.pem", tmp_path / "receiver-key.pem" + cert_file.write_bytes(cert.public_bytes(serialization.Encoding.PEM)) + key_file.write_bytes( + key.private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.PKCS8, + serialization.NoEncryption(), + ) + ) + key_file.chmod(0o600) + return {"certificate": str(cert_file), "certificate_key": str(key_file)} + + +@pytest.mark.parametrize("crash", ["before_ack", "ack_written", "ack_connection_loss"]) +@case_deadline +async def test_http_acceptance_to_ack_process_failure_converges_with_immutable_retry( + crash, certificate +): + async with isolated_reporting_pool(autocommit=True) as pool: + await PgReportingReconciliationStore(pool=pool).create_schema() + await produce_and_expand(pool) + clock = ManualClock() + outbox = PgReportingOutbox(pool=pool, clock=clock) + (original,) = await outbox.list_deliveries(account_id="acct_a") + body_store = _BytesStore(pool) + async with service_process( + pool, "receiver", pause_responses=True, **certificate + ) as receiver: + port = (await receiver.event("listening"))["port"] + network = {**certificate, "receiver_port": port} + pause = "before_ack" if crash == "before_ack" else "ack_written" + # Import and open the competitor's pool before holding an HTTP + # response. Process startup must never race the sender's timeout. + async with service_process(pool, "worker", start_paused=True, **network) as competitor: + await competitor.event("worker_ready") + async with service_process(pool, "worker", pause=pause, **network) as worker: + stale = await acceptance_to_ack( + pool, outbox, original, receiver, worker, competitor, pause, crash + ) + await settle(pool) + clock.advance(timedelta(seconds=61)) + await receiver.send(advance_seconds=61) + await receiver.event("clock_advanced") + async with service_process( + pool, + "worker", + pause="before_ack", + advance_seconds=61, + signing_generation=2, + **network, + ) as restarted: + assert (await receiver.event("http_accepted"))["attempt"] == 2 + for state in ("complete", "pending", "suppressed", "quarantined"): + assert not await outbox.finish_delivery(stale, now=clock(), state=state) + await receiver.send(release_http=2) + await receiver.event("http_released") + await restarted.event("before_ack") + await restarted.send(**{"continue": "before_ack"}) + assert (await restarted.event("done"))["did_work"] + await restarted.finish() + await receiver.send(stop=True) + await receiver.finish() + key = original.delivery.binding.idempotency_key + attempts = [ + json.loads(await body_store.get("http-attempt", "acct_a", key, str(n))) for n in (1, 2) + ] + assert attempts[0]["body"] == attempts[1]["body"] + assert base64.b64decode(attempts[0]["body"]) == await body_store.get( + "receiver", "acct_a", key + ) + assert attempts[0]["headers"]["signature"] != attempts[1]["headers"]["signature"] + assert attempts[0]["verified_key"].endswith("key-1") + assert attempts[1]["verified_key"].endswith("key-2") + (retained,) = await outbox.list_deliveries(account_id="acct_a") + assert retained.delivery == original.delivery + assert retained.state == "complete" and retained.attempt_count == 2 + assert len(await outbox.list_events(account_id="acct_a")) == 1 + + +async def acceptance_to_ack(pool, outbox, original, receiver, worker, competitor, pause, crash): + assert (await receiver.event("http_accepted"))["attempt"] == 1 + # HTTP is waiting for its response. No worker connection holds + # a transaction, and the other worker cannot claim this lease. + async with pool.connection() as conn: + states = await ( + await conn.execute( + "SELECT state FROM pg_stat_activity WHERE application_name = %s", + (f"reporting-matrix-worker-{worker.process.pid}",), + ) + ).fetchall() + assert states and all(state == "idle" for (state,) in states) + lease_row = await ( + await conn.execute( + "SELECT lease_token, lease_expires_at, claim_count" + " FROM reporting_notification_deliveries" + " WHERE account_id = %s AND delivery_id = %s", + ("acct_a", original.delivery.binding.delivery_id), + ) + ).fetchone() + stale = DeliveryLease(original.delivery, *lease_row) + await competitor.send(**{"continue": "worker_ready"}) + assert not (await competitor.event("done"))["did_work"] + await competitor.finish() + await receiver.send(release_http=1) + await receiver.event("http_released") + stopped = await worker.event(pause) + assert (await outbox.list_deliveries(account_id="acct_a"))[0].state == "leased" + if crash == "ack_connection_loss": + async with pool.connection() as conn: + assert ( + await ( + await conn.execute("SELECT pg_terminate_backend(%s)", (stopped["backend_pid"],)) + ).fetchone() + )[0] + await worker.send(**{"continue": pause}) + failure = await worker.event("service_failed") + assert failure["classification"] == "database_failure" + await worker.finish(code=1) + else: + await worker.kill() + return stale + + +@case_deadline +async def test_child_barrier_deadline_exits_with_named_sanitized_diagnostic(): + async with isolated_reporting_pool(autocommit=True) as pool: + async with service_process(pool, "barrier_probe", deadlines={"barrier:held": 0}) as probe: + await probe.event("held") + error = await probe.event("service_failed") + assert error == { + "point": "service_failed", + "classification": "deadline", + "stage": "barrier:held", + } + await probe.finish(code=1) diff --git a/tests/conformance/reporting/test_reporting_notification_readiness.py b/tests/conformance/reporting/test_reporting_notification_readiness.py new file mode 100644 index 000000000..23221733f --- /dev/null +++ b/tests/conformance/reporting/test_reporting_notification_readiness.py @@ -0,0 +1,126 @@ +"""Optional wiring, truthful capabilities, and complete dirty scope evidence.""" + +from dataclasses import replace +from datetime import timedelta + +import pytest + +from adcp.reporting.ledger import InMemoryReportingLedgerStore, ReportingDeliveryScope +from adcp.reporting.outbox import ( + InMemoryReportingOutbox, + ReportingNotificationError, + ReportingStatusScope, +) + +from ._generation_support import NOW, configuration +from ._reconciliation_support import scenario +from .test_reporting_notification_outbox import seed + + +async def test_capability_fragment_exposes_only_complete_notifications(notification_harness): + h = notification_harness + await seed(h) + fields = await h.worker().advertised_notifications(h.reliable.store, account_id="acct_a") + assert fields == { + "ledger_notification": "reporting.ledger_changed", + "supports_webhook_activity": False, + } + assert "status_notification" not in fields + # A Core obligation alone cannot authorize a readiness capability. + core_scope = ReportingDeliveryScope(configuration().generation_key, "buyer", "rpo_acct_a") + with pytest.raises(ReportingNotificationError): + await h.worker().advertised_notifications( + h.reliable.store, account_id="acct_a", ready_scope=core_scope + ) + + +async def test_managed_capability_requires_retained_configuration_and_frozen_scope( + notification_harness, +): + h = notification_harness + s = await scenario(h.reliable.store) + fields = await h.worker().advertised_notifications( + h.reliable.store, account_id="acct_a", ready_scope=s.delivery.scope + ) + assert fields == { + "ledger_notification": "reporting.ledger_changed", + "readiness_notification": "reporting.delivery_ready", + "supports_webhook_activity": False, + } + with pytest.raises(ReportingNotificationError): + await h.worker().advertised_notifications( + h.reliable.store, account_id="other", ready_scope=s.delivery.scope + ) + with pytest.raises(ReportingNotificationError): + await h.worker().advertised_notifications( + h.reliable.store, + account_id="acct_a", + ready_scope=replace(s.delivery.scope, consumer_id="unconfigured"), + ) + + +async def test_capability_rejects_unwired_store_and_unavailable_signing(notification_harness): + h = notification_harness + with pytest.raises(ReportingNotificationError, match="notification_chain_unready"): + await h.worker().advertised_notifications( + InMemoryReportingLedgerStore(), account_id="acct_a" + ) + from adcp.webhook_sender import ScopePermanentlyUnknown + + worker = h.worker() + worker.signing = None + with pytest.raises(ScopePermanentlyUnknown): + await worker.advertised_notifications(h.reliable.store, account_id="acct_a") + + +@pytest.mark.parametrize("damage", ["generation", "obligation", "feed", "consumer_replay"]) +async def test_optional_issue_scope_must_resolve_inside_trusted_namespace( + notification_harness, damage +): + h = notification_harness + obligation, _, _ = await seed(h) + scope = ReportingStatusScope.for_obligation(obligation, "buyer") + if damage == "generation": + scope = replace( + scope, generation_key=replace(scope.generation_key, delivery_config_version=77) + ) + elif damage == "obligation": + scope = replace(scope, reporting_obligation_id="missing-or-foreign") + elif damage == "feed": + scope = replace(scope, feed_purpose="billing") + else: + await h.reliable.store.ensure_issue_opened( + account_id="acct_a", consumer_id="auditor", issue_key="same-opaque-key", observed_at=NOW + ) + before = await h.outbox.read_status_dirty(account_id="acct_a") + with pytest.raises(ReportingNotificationError, match="invalid_status_scope"): + await h.reliable.store.ensure_issue_opened( + account_id="acct_a", + consumer_id="buyer", + issue_key="same-opaque-key", + observed_at=NOW, + status_scope=scope, + ) + assert await h.outbox.read_status_dirty(account_id="acct_a") == before + + +async def test_mutable_memory_configuration_lifecycle_keeps_core_semantics(): + store = InMemoryReportingLedgerStore(notifications=True, clock=lambda: NOW) + outbox = InMemoryReportingOutbox(store) + first = configuration() + second = replace( + first, + deactivated_at=first.deactivated_at + timedelta(hours=1), + automated_recovery_window=timedelta(hours=2), + ) + await store.put_configuration(first) + await store.put_configuration(second) + await store.put_configuration(second) + await store.put_configuration(first) + dirty = await outbox.read_status_dirty(account_id="acct_a") + assert [record.cause_generation for record in dirty] == [1, 2, 3] + assert dirty[1].before == dirty[0].after and dirty[2].before == dirty[1].after + assert dirty[1].after.deactivated_at == second.deactivated_at + assert dirty[1].after.automated_recovery_seconds == 7200 + assert dirty[2].after == dirty[0].after + assert await outbox.list_events(account_id="acct_a") == () diff --git a/tests/conformance/reporting/test_reporting_notification_security.py b/tests/conformance/reporting/test_reporting_notification_security.py new file mode 100644 index 000000000..5c0aaf89c --- /dev/null +++ b/tests/conformance/reporting/test_reporting_notification_security.py @@ -0,0 +1,418 @@ +"""Authenticated routing, secret rejection, and the real SDK-owned transport.""" + +from __future__ import annotations + +import logging +import socket +from dataclasses import asdict, fields, replace +from datetime import timedelta + +import pytest + +from adcp.reporting.ledger import InMemoryReportingLedgerStore +from adcp.reporting.outbox import ( + DeliveryBinding, + ReportingLegacyAuthentication, + ReportingNotificationError, + validate_notification_payload, +) +from adcp.signing.jwks import SSRFValidationError +from adcp.webhook_auth import JwkSignerStrategy +from adcp.webhook_sender import PreparedWebhook, WebhookSender + +from ._generation_support import revision_for +from ._reliable_support import notification_subscription +from .test_reporting_notification_outbox import seed + + +async def test_process_receiver_fixture_verifies_both_rotation_keys(): + from adcp.signing.jwks import StaticJwksResolver + from adcp.signing.webhook_signer import sign_webhook + from adcp.signing.webhook_verifier import WebhookVerifyOptions, verify_webhook_signature + + from ._reliable_support import ( + FailurePlan, + ManualClock, + ScriptedSigning, + notification_verification_keys, + ) + + clock, signing = ManualClock(), ScriptedSigning(FailurePlan()) + options = WebhookVerifyOptions( + jwks_resolver=StaticJwksResolver({"keys": notification_verification_keys()}), + clock=lambda: clock().timestamp(), + ) + url = notification_subscription().url + body = b'{"idempotency_key":"receiver-fixture-self-check"}' + for generation in (1, 2): + signing.generation = generation + material = await signing.resolve( + account_id="acct_a", principal_id="buyer", signing_scope_id="scope" + ) + headers = {"Content-Type": "application/json"} + headers.update( + sign_webhook( + method="POST", + url=url, + headers=headers, + body=body, + private_key=material.private_key, + key_id=material.key_id, + alg=material.algorithm, + created=int(clock().timestamp()), + ).as_dict() + ) + verified = verify_webhook_signature( + method="POST", url=url, headers=headers, body=body, options=options + ) + assert verified.key_id == material.key_id and verified.alg == "ed25519" + clock.advance(timedelta(seconds=61)) + + +async def replace_stored(h, old, changed): + if isinstance(h.reliable.store, InMemoryReportingLedgerStore): + state = h.reliable.store._notification_state + _, work = state.deliveries.pop((old.binding.account_id, old.binding.delivery_id)) + state.deliveries[(changed.binding.account_id, changed.binding.delivery_id)] = ( + changed, + work, + ) + return + from psycopg import sql + + previous, updated = asdict(old.binding), asdict(changed.binding) + differences = {name: value for name, value in updated.items() if previous[name] != value} + if old.envelope != changed.envelope: + differences["envelope"] = changed.envelope + async with h.reliable.blobs.pool.connection() as conn: + if changed.binding.account_id != old.binding.account_id: + # Supply a valid destination-side parent so the account AAD check, + # rather than an FK alone, is forced to defend the swapped row. + await conn.execute( + "INSERT INTO reporting_notification_events" + " SELECT %s, notification_id, notification_type, cause_kind, cause_id," + " cause_generation, consumer_namespace, fired_at," + " jsonb_set(snapshot, '{account_id}', to_jsonb(%s::text))" + " FROM reporting_notification_events" + " WHERE account_id = %s AND notification_id = %s", + ( + changed.binding.account_id, + changed.binding.account_id, + old.binding.account_id, + old.binding.notification_id, + ), + ) + await conn.execute( + "INSERT INTO reporting_notification_expansions" + " (account_id, notification_id, emission_generation, due_at)" + " SELECT %s, notification_id, emission_generation, due_at" + " FROM reporting_notification_expansions" + " WHERE account_id = %s AND notification_id = %s", + (changed.binding.account_id, old.binding.account_id, old.binding.notification_id), + ) + if changed.binding.consumer_namespace != old.binding.consumer_namespace: + # A malicious database writer can also supply a namespace parent. + # Authentication must still reject the single-column delivery swap. + await conn.execute( + "INSERT INTO reporting_notification_events" + " SELECT account_id, notification_id," + " 'reporting.delivery_ready', 'materialization_ready'," + " cause_id, cause_generation, %s, fired_at, snapshot" + " FROM reporting_notification_events WHERE account_id = %s" + " AND consumer_namespace = %s AND notification_id = %s", + ( + changed.binding.consumer_namespace, + old.binding.account_id, + old.binding.consumer_namespace, + old.binding.notification_id, + ), + ) + await conn.execute( + "INSERT INTO reporting_notification_expansions" + " (account_id, consumer_namespace, notification_id, emission_generation, due_at)" + " SELECT account_id, %s, notification_id, emission_generation, due_at" + " FROM reporting_notification_expansions WHERE account_id = %s" + " AND consumer_namespace = %s AND notification_id = %s", + ( + changed.binding.consumer_namespace, + old.binding.account_id, + old.binding.consumer_namespace, + old.binding.notification_id, + ), + ) + assignments = sql.SQL(", ").join( + sql.SQL("{} = %s").format(sql.Identifier(name)) for name in differences + ) + await conn.execute( + sql.SQL( + "UPDATE reporting_notification_deliveries SET {}" + " WHERE account_id = %s AND delivery_id = %s" + ).format(assignments), + (*differences.values(), old.binding.account_id, old.binding.delivery_id), + ) + + +@pytest.mark.parametrize("column", [item.name for item in fields(DeliveryBinding)]) +async def test_every_bound_column_swap_fails_before_any_external_effect( + notification_harness, column +): + h = notification_harness + obligation, _, _ = await seed(h) + await h.worker().expand_one(account_id="acct_a") + (row,) = await h.outbox.list_deliveries(account_id="acct_a") + original = row.delivery + current = getattr(original.binding, column) + value = current + 1 if isinstance(current, int) else "swapped-value" + if column == "notification_id": + revision, rows = revision_for(obligation, suffix="another") + await h.reliable.store.commit_revision(revision, rows) + value = next( + event.notification_id + for event in await h.outbox.list_events(account_id="acct_a") + if event.notification_id != original.binding.notification_id + ) + if column == "emission_generation": + value = await h.outbox.reemit( + account_id="acct_a", + notification_id=original.binding.notification_id, + now=h.reliable.clock(), + ) + changed = replace(original, binding=replace(original.binding, **{column: value})) + await replace_stored(h, original, changed) + assert await h.worker().deliver_one(account_id=changed.binding.account_id) + (retained,) = await h.outbox.list_deliveries(account_id=changed.binding.account_id) + assert retained.state == "quarantined" and retained.error_code == "integrity_failure" + assert not h.subscriptions.gets and not h.signing.calls + assert not h.receiver.connections and not h.receiver.dns_calls + + +@pytest.mark.parametrize("attack", ["ciphertext", "nonce", "truncated", "poison_json"]) +async def test_poison_and_envelope_transplants_quarantine_without_starving_later_work( + notification_harness, attack +): + h = notification_harness + await seed(h) + h.subscriptions.put(notification_subscription(subscriber="healthy")) + await h.worker().expand_one(account_id="acct_a") + rows = await h.outbox.list_deliveries(account_id="acct_a") + target = next(row.delivery for row in rows if row.delivery.binding.subscriber_id == "buyer") + donor = next(row.delivery for row in rows if row.delivery.binding.subscriber_id == "healthy") + envelope = { + "ciphertext": donor.envelope, + "nonce": donor.envelope[:12] + target.envelope[12:], + "truncated": b"broken", + "poison_json": b"not-an-encrypted-envelope", + }[attack] + await replace_stored(h, target, replace(target, envelope=envelope)) + await h.drain() + states = { + row.delivery.binding.subscriber_id: row.state + for row in await h.outbox.list_deliveries(account_id="acct_a") + } + assert states == {"buyer": "quarantined", "healthy": "complete"} + assert [item.subscriber_id for item in h.receiver.received] == ["healthy"] + + +@pytest.mark.parametrize( + "address", + [ + "127.0.0.1", + "0.0.0.0", + "10.0.0.1", + "172.16.0.1", + "192.168.1.1", + "169.254.0.1", + "169.254.169.254", + "100.64.0.1", + "::1", + "fd00::1", + "fc00::1", + "fe80::1", + "::ffff:127.0.0.1", + ], +) +async def test_ssrf_matrix_on_worker_owned_pinned_transport( + notification_harness, monkeypatch, address +): + h = notification_harness + await seed(h) + h.receiver.dns_addresses["receiver.example.test"] = [address] + signed = [] + original = JwkSignerStrategy.build_auth_headers + + def sign(self, **kwargs): + signed.append(True) + return original(self, **kwargs) + + monkeypatch.setattr(JwkSignerStrategy, "build_auth_headers", sign) + await h.drain() + assert not signed and not h.receiver.connections + assert (await h.outbox.list_deliveries(account_id="acct_a"))[0].state == "quarantined" + + +async def test_dns_rebinding_is_pinned_and_revalidation_blocks_the_next_attempt( + notification_harness, +): + h = notification_harness + await seed(h) + h.receiver.dns_addresses["receiver.example.test"] = ["8.8.8.8", "169.254.169.254"] + h.receiver.responses["buyer"].append(503) + await h.drain() + assert h.receiver.dns_calls == ["receiver.example.test"] + assert h.receiver.connections == [("8.8.8.8", 443)] + h.reliable.clock.advance(timedelta(seconds=6)) + await h.drain() + assert len(h.receiver.dns_calls) == 2 + assert len(h.receiver.connections) == 1 + assert (await h.outbox.list_deliveries(account_id="acct_a"))[0].state == "quarantined" + + +async def test_mixed_public_private_dns_answers_fail_closed(notification_harness, monkeypatch): + h = notification_harness + await seed(h) + + def mixed(host, port, *args, **kwargs): + return [ + (socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", (ip, 443)) + for ip in ("8.8.8.8", "10.0.0.1") + ] + + monkeypatch.setattr(socket, "getaddrinfo", mixed) + await h.drain() + assert not h.receiver.connections + assert (await h.outbox.list_deliveries(account_id="acct_a"))[0].state == "quarantined" + + +async def test_transient_dns_failure_retries_without_secret_diagnostics( + notification_harness, monkeypatch +): + h = notification_harness + await seed(h) + resolve = socket.getaddrinfo + + def fail(host, port, *args, **kwargs): + if str(host).endswith(".example.test"): + raise socket.gaierror("dns provider token=SECRET") + return resolve(host, port, *args, **kwargs) + + with monkeypatch.context() as context: + context.setattr(socket, "getaddrinfo", fail) + await h.drain() + (row,) = await h.outbox.list_deliveries(account_id="acct_a") + assert row.state == "pending" and row.error_code == "network" + assert "SECRET" not in repr(row) + h.reliable.clock.advance(timedelta(seconds=6)) + await h.drain() + assert len(h.receiver.received) == 1 + + +@pytest.mark.parametrize( + "url", + [ + "https://receiver.example.test:80/", + "https://receiver.example.test:8443/", + "http://receiver.example.test/", + "https://user:secret@receiver.example.test/", + "https://receiver.example.test/#fragment", + ], +) +def test_unsafe_registration_ports_schemes_userinfo_and_fragments_are_rejected(url): + with pytest.raises(ReportingNotificationError): + notification_subscription(url=url) + + +async def test_disallowed_port_is_also_enforced_by_owned_prepared_transport(notification_harness): + h = notification_harness + await seed(h) + (event,) = await h.outbox.list_events(account_id="acct_a") + material = await h.signing.resolve( + account_id="acct_a", principal_id="buyer", signing_scope_id="scope" + ) + sender = WebhookSender( + private_key=material.private_key, + key_id=material.key_id, + alg=material.algorithm, + allowed_destination_ports=frozenset({443}), + ) + try: + prepared = PreparedWebhook( + "https://receiver.example.test:8443/", + "0" * 32, + event.body(subscriber_id="buyer", idempotency_key="0" * 32), + ) + with pytest.raises(SSRFValidationError): + await sender.send_prepared(prepared) + finally: + await sender.aclose() + assert h.receiver.connections == [] + + +@pytest.mark.parametrize("scheme", ["Bearer", "HMAC-SHA256"]) +async def test_legacy_credentials_encrypted_modes_exclusive_and_logs_sanitized( + notification_harness, caplog, scheme +): + h = notification_harness + await seed(h) + h.subscriptions.put( + notification_subscription( + signing_scope_id=None, + authentication=ReportingLegacyAuthentication(scheme, "LEGACY_CREDENTIAL_SECRET"), + ) + ) + caplog.set_level(logging.DEBUG) + await h.drain() + (row,) = await h.outbox.list_deliveries(account_id="acct_a") + assert row.state == "complete" + assert not h.signing.calls + assert "signature" not in h.receiver.received[0].headers + assert ("authorization" in h.receiver.received[0].headers) == (scheme == "Bearer") + rendered = caplog.text + repr(row) + assert all( + secret not in rendered + for secret in ("LEGACY_CREDENTIAL_SECRET", "URL_SECRET", "DO_NOT_PERSIST") + ) + assert b"URL_SECRET" not in row.delivery.envelope + assert b"LEGACY_CREDENTIAL_SECRET" not in row.delivery.envelope + with pytest.raises(ReportingNotificationError): + notification_subscription(authentication=ReportingLegacyAuthentication(scheme, "secret")) + + +async def test_rfc_logs_never_retain_url_headers_signatures_or_provider_text( + notification_harness, caplog +): + h = notification_harness + await seed(h) + caplog.set_level(logging.DEBUG) + await h.drain() + assert len(h.receiver.received) == 1 + receipt = h.receiver.received[0] + assert "URL_SECRET" not in caplog.text and "DO_NOT_PERSIST" not in caplog.text + assert receipt.headers["signature"] not in caplog.text + assert receipt.headers["signature-input"] not in caplog.text + # Other concurrent traffic retains its normal logging behavior. + logging.getLogger("httpx").info("unrelated-safe-application-log") + assert "unrelated-safe-application-log" in caplog.text + + +@pytest.mark.parametrize( + "value", + [ + "https://object.example.test/?token=secret", + "token=secret", + "Bearer secret", + "eyJhbGciOiJ9.eyJzdWIiOiJ9.signature", + "https%3A%2F%2Fsecret.example", + ], +) +async def test_recursive_payload_secret_scan_rejects_even_schema_legal_account_strings( + notification_harness, value +): + h = notification_harness + await seed(h) + (event,) = await h.outbox.list_events(account_id="acct_a") + import json + + body = json.loads(event.body(subscriber_id="buyer", idempotency_key="0" * 32)) + body["account_id"] = value + with pytest.raises(ReportingNotificationError): + validate_notification_payload(body) diff --git a/tests/conformance/reporting/test_reporting_notification_transactions.py b/tests/conformance/reporting/test_reporting_notification_transactions.py new file mode 100644 index 000000000..3318596fc --- /dev/null +++ b/tests/conformance/reporting/test_reporting_notification_transactions.py @@ -0,0 +1,413 @@ +"""Failure injection at every domain/dirty boundary, including autocommit PG.""" + +from __future__ import annotations + +import asyncio +from copy import deepcopy +from dataclasses import replace +from datetime import timedelta + +import pytest + +from adcp.reporting.ledger import ( + InMemoryReportingLedgerStore, + PgReportingReconciliationStore, + ReportingAdjustmentRecord, + ReportingMaterializationCheck, + derive_period, +) +from adcp.reporting.outbox import PgReportingOutbox + +from ._generation_support import END, NOW, START, configuration, revision_for +from ._reconciliation_support import scenario +from ._reliable_support import ( + Barrier, + NotificationHarness, + notification_subscription, + reliable_factory, +) +from .test_reporting_notification_outbox import seed, statement + + +async def test_postgres_lease_and_retry_use_database_time_despite_caller_clock(): + async with reliable_factory("postgres", notifications=True, autocommit=True) as reliable: + h = NotificationHarness(reliable) + await seed(h) + pool = reliable.blobs.pool + assert pool is not None + outbox = PgReportingOutbox(pool=pool) + caller_time = NOW + timedelta(days=100000) + + async def database_time(): + async with pool.connection() as conn: + return (await (await conn.execute("SELECT clock_timestamp()")).fetchone())[0] + + before = await database_time() + lease = await outbox.claim_expansion( + account_id="acct_a", now=caller_time, lease_seconds=600 + ) + after = await database_time() + assert lease is not None + assert before <= lease.expires_at - timedelta(seconds=600) <= after + assert ( + await outbox.claim_expansion(account_id="acct_a", now=caller_time, lease_seconds=600) + is None + ) + before = await database_time() + assert await outbox.finish_expansion( + lease, now=caller_time, state="pending", retry_at=caller_time + timedelta(seconds=600) + ) + after = await database_time() + async with pool.connection() as conn: + due_at = ( + await ( + await conn.execute( + "SELECT due_at FROM reporting_notification_expansions" + " WHERE account_id = %s AND consumer_namespace = %s" + " AND notification_id = %s", + ("acct_a", lease.consumer_namespace, lease.notification_id), + ) + ).fetchone() + )[0] + assert before <= due_at - timedelta(seconds=600) <= after + # Explicit database state, never a timing sleep, makes this retry due. + await conn.execute( + "UPDATE reporting_notification_expansions SET due_at = clock_timestamp()" + " WHERE account_id = %s AND consumer_namespace = %s AND notification_id = %s", + ("acct_a", lease.consumer_namespace, lease.notification_id), + ) + reclaimed = await outbox.claim_expansion(account_id="acct_a", now=NOW, lease_seconds=600) + assert reclaimed is not None and reclaimed.token != lease.token + assert not await outbox.finish_expansion(lease, now=NOW, state="complete") + assert await outbox.finish_expansion(reclaimed, now=caller_time, state="complete") + + +async def test_postgres_expansion_expiry_before_final_fence_rolls_back_all_members(): + async with reliable_factory("postgres", notifications=True, autocommit=True) as reliable: + h = NotificationHarness(reliable) + await seed(h) + h.subscriptions.put(notification_subscription(subscriber="second")) + outbox = h.outbox + insert = outbox._insert_delivery + + async def expire_during_insert(conn, delivery, at): + await insert(conn, delivery, at) + reliable.clock.advance(timedelta(seconds=61)) + + outbox._insert_delivery = expire_during_insert + worker = h.worker() + worker.outbox = outbox + assert await worker.expand_one(account_id="acct_a") + assert await outbox.list_deliveries(account_id="acct_a") == () + assert len(await outbox.list_events(account_id="acct_a")) == 1 + # A new worker snapshots the current complete membership after expiry. + assert await h.worker().expand_one(account_id="acct_a") + assert { + row.delivery.binding.subscriber_id + for row in await outbox.list_deliveries(account_id="acct_a") + } == {"buyer", "second"} + + +async def prepare(h: NotificationHarness, operation: str): + store = h.reliable.store + if operation.startswith("managed_"): + s = await scenario(store) + if operation in {"managed_check", "managed_receipt"}: + await store.commit_materialization(s.outcome) + if operation == "managed_destination": + return lambda: store.put_destination_binding(replace(s.binding, consumer_id="auditor")) + if operation == "managed_obligation": + await store.put_destination_binding(replace(s.binding, consumer_id="auditor")) + return lambda: store.bind_obligation_delivery( + replace(s.delivery, scope=replace(s.delivery.scope, consumer_id="auditor")) + ) + if operation == "managed_attempt": + return lambda: store.commit_materialization_attempt( + replace( + s.attempt, + reporting_materialization_id="attempt-two", + attempt=2, + created_at=s.attempt.created_at + timedelta(seconds=1), + ) + ) + if operation == "managed_materialization": + return lambda: store.commit_materialization(s.outcome) + if operation == "managed_check": + return lambda: store.record_materialization_check( + ReportingMaterializationCheck( + s.attempt.scope, + s.attempt.reporting_materialization_id, + "check-one", + "unavailable", + NOW, + ) + ) + return lambda: store.record_revision_receipt(s.receipt) + obligation, revision, rows = await seed(h, official=operation == "adjustment") + if operation == "configuration": + return lambda: store.put_configuration(replace(configuration(), delivery_config_version=2)) + if operation == "obligation": + other = replace( + obligation, + reporting_obligation_id="other-period", + period=derive_period(configuration().schedule, account_timezone="UTC", ordinal=1), + scope_resolved_at=END + timedelta(hours=1), + ) + return lambda: store.commit_obligation(other) + if operation == "revision": + changed, rows = revision_for(obligation, suffix="second") + return lambda: store.commit_revision(changed, rows) + if operation == "readability": + return lambda: store.set_revision_readable( + account_id="acct_a", + reporting_revision_id=revision.reporting_revision_id, + readable=False, + ) + if operation == "adjustment": + value = ReportingAdjustmentRecord( + "adj-new", + "acct_a", + revision.reporting_revision_id, + "source_correction", + START, + END, + (("impressions", "-1"),), + NOW, + NOW, + ) + return lambda: store.commit_adjustment(value) + if operation == "consumer_status": + return lambda: store.record_consumer_status(statement(obligation)) + if operation in {"issue_state", "issue_retire"}: + await store.ensure_issue_opened( + issue_key="opaque", account_id="acct_a", consumer_id="buyer", observed_at=NOW + ) + if operation == "issue_state": + return lambda: store.set_issue_state( + issue_key="opaque", account_id="acct_a", state="acknowledged", at=NOW + ) + if operation == "issue_retire": + return lambda: store.retire_issue(issue_key="opaque", account_id="acct_a", at=NOW) + return lambda: store.ensure_issue_opened( + issue_key="opaque", account_id="acct_a", consumer_id="buyer", observed_at=NOW + ) + + +async def image(h: NotificationHarness): + """Read actual retained domain rows; never infer rollback from event count.""" + if isinstance(h.reliable.store, InMemoryReportingLedgerStore): + return deepcopy( + { + key: value + for key, value in vars(h.reliable.store).items() + if key not in {"_clock", "_lock"} + } + ) + pool = h.reliable.blobs.pool + tables = ( + "reporting_configurations", + "reporting_obligations", + "reporting_revisions", + "reporting_adjustments", + "reporting_consumer_statuses", + "reporting_issue_lifecycle", + "reporting_ledger_changes", + "reporting_reconciliation_records", + "reporting_reconciliation_heads", + "reporting_reconciliation_changes", + "reporting_notification_events", + "reporting_notification_expansions", + "reporting_status_dirty", + "reporting_status_dirty_heads", + "reporting_issue_status_scopes", + ) + from psycopg import sql + + result = {} + async with pool.connection() as conn: + for table in tables: + rows = await ( + await conn.execute( + sql.SQL( + "SELECT to_jsonb(t) FROM {} t WHERE account_id = %s" + " ORDER BY to_jsonb(t)::text" + ).format(sql.Identifier(table)), + ("acct_a",), + ) + ).fetchall() + result[table] = rows + result["rows"] = await ( + await conn.execute( + "SELECT r.reporting_revision_id, r.ordinal, r.row_payload" + " FROM reporting_revision_rows r" + " JOIN reporting_revisions v ON v.reporting_revision_id = r.reporting_revision_id" + " WHERE v.account_id = %s ORDER BY r.reporting_revision_id, r.ordinal", + ("acct_a",), + ) + ).fetchall() + return result + + +@pytest.mark.parametrize( + "operation", + [ + "configuration", + "obligation", + "revision", + "readability", + "adjustment", + "consumer_status", + "issue_open", + "issue_state", + "issue_retire", + "managed_destination", + "managed_obligation", + "managed_attempt", + "managed_materialization", + "managed_check", + "managed_receipt", + ], +) +@pytest.mark.parametrize("failure_position", ["before_dirty", "after_dirty"]) +async def test_every_domain_mutation_rolls_back_if_dirty_enqueue_fails( + notification_harness, monkeypatch, operation, failure_position +): + h = notification_harness + mutation = await prepare(h, operation) + before = await image(h) + cls = type(h.reliable.store) + original = cls._dirty_status + if isinstance(h.reliable.store, InMemoryReportingLedgerStore): + + def fail(self, *args, **kwargs): + if failure_position == "after_dirty": + original(self, *args, **kwargs) + raise OSError("injected enqueue failure") + + else: + + async def fail(self, *args, **kwargs): + if failure_position == "after_dirty": + await original(self, *args, **kwargs) + raise OSError("injected enqueue failure") + + with monkeypatch.context() as context: + context.setattr(cls, "_dirty_status", fail) + with pytest.raises(OSError, match="injected"): + await mutation() + assert await image(h) == before + # Rollback does not leave a ghost immutable ID or a consumed feed generation. + await mutation() + assert await image(h) != before + + +@pytest.mark.parametrize("position", ["before_event", "after_event"]) +async def test_event_enqueue_failure_rolls_back_revision_and_rows( + notification_harness, monkeypatch, position +): + h = notification_harness + mutation = await prepare(h, "revision") + before = await image(h) + cls = type(h.reliable.store) + original = cls._record_notification + if isinstance(h.reliable.store, InMemoryReportingLedgerStore): + + def fail(self, *args): + if position == "after_event": + original(self, *args) + raise OSError("event failure") + + else: + + async def fail(self, *args): + if position == "after_event": + await original(self, *args) + raise OSError("event failure") + + with monkeypatch.context() as context: + context.setattr(cls, "_record_notification", fail) + with pytest.raises(OSError, match="event failure"): + await mutation() + assert await image(h) == before + + +async def test_postgres_precommit_invisibility_from_distinct_autocommit_pool(monkeypatch): + from psycopg_pool import AsyncConnectionPool + + async with reliable_factory("postgres", notifications=True, autocommit=True) as reliable: + h = NotificationHarness(reliable) + mutation = await prepare(h, "revision") + before = await h.outbox.list_events(account_id="acct_a") + barrier = Barrier() + original = PgReportingReconciliationStore._record_notification + + async def pause_after_event(self, conn, event): + await original(self, conn, event) + await barrier.pause() + raise OSError("rollback after insert") + + monkeypatch.setattr( + PgReportingReconciliationStore, "_record_notification", pause_after_event + ) + parent = reliable.blobs.pool + async with AsyncConnectionPool(parent.conninfo, kwargs=parent.kwargs, open=False) as other: + await other.wait() + observer = PgReportingReconciliationStore(pool=other) + observer_outbox = PgReportingOutbox(pool=other) + task = asyncio.create_task(mutation()) + try: + await barrier.wait() + assert ( + await observer.get_revision( + account_id="acct_a", reporting_revision_id="rpr_acct_a_second" + ) + is None + ) + assert await observer_outbox.list_events(account_id="acct_a") == before + finally: + barrier.release() + with pytest.raises(OSError): + await task + assert ( + await observer.get_revision( + account_id="acct_a", reporting_revision_id="rpr_acct_a_second" + ) + is None + ) + assert await observer_outbox.list_events(account_id="acct_a") == before + + +async def test_postgres_partial_fanout_crash_and_changed_membership_never_mix(monkeypatch): + async with reliable_factory("postgres", notifications=True, autocommit=True) as reliable: + h = NotificationHarness(reliable) + h.receiver.install(monkeypatch) + await seed(h) + h.subscriptions.put(notification_subscription(subscriber="old-second")) + original = PgReportingOutbox._insert_delivery + barrier = Barrier() + + async def insert_then_crash(self, conn, delivery, at): + await original(self, conn, delivery, at) + await barrier.pause() + raise OSError("partial fanout crash") + + with monkeypatch.context() as patch: + patch.setattr(PgReportingOutbox, "_insert_delivery", insert_then_crash) + task = asyncio.create_task(h.worker().expand_one(account_id="acct_a")) + try: + await barrier.wait() + assert await h.outbox.list_deliveries(account_id="acct_a") == () + h.subscriptions.values.clear() + h.subscriptions.put(notification_subscription(subscriber="new-only")) + finally: + barrier.release() + with pytest.raises(OSError): + await task + assert await h.outbox.list_deliveries(account_id="acct_a") == () + reliable.clock.advance(timedelta(seconds=61)) + await reliable.restart() + await h.drain() + rows = await h.outbox.list_deliveries(account_id="acct_a") + assert [(row.delivery.binding.subscriber_id, row.state) for row in rows] == [ + ("new-only", "complete") + ] diff --git a/tests/type_checks/reporting_notification_outbox.py b/tests/type_checks/reporting_notification_outbox.py new file mode 100644 index 000000000..7267972ae --- /dev/null +++ b/tests/type_checks/reporting_notification_outbox.py @@ -0,0 +1,150 @@ +"""Optional outbox adoption leaves the existing ledger/producer contracts intact.""" + +from collections.abc import Callable +from datetime import datetime + +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey +from psycopg_pool import AsyncConnectionPool + +from adcp.reporting.ledger import ( + InMemoryReportingLedgerStore, + InMemoryReportingReconciliationStore, + PgReportingReconciliationStore, + ReportingLedgerStore, + ReportingObligationRecord, +) +from adcp.reporting.outbox import ( + InMemoryReportingOutbox, + PgReportingOutbox, + ReportingEnvelopeCipher, + ReportingNotificationOutbox, + ReportingNotificationSubscription, + ReportingNotificationWorker, + ReportingSigningMaterial, + ReportingSigningResolver, + ReportingStatusDirty, + ReportingStatusScope, + ReportingSubscriptionResolver, +) + + +def unchanged_core() -> ReportingLedgerStore: + return InMemoryReportingLedgerStore() + + +def reference( + clock: Callable[[], datetime], +) -> tuple[ReportingLedgerStore, ReportingNotificationOutbox]: + store = InMemoryReportingReconciliationStore(clock=clock, notifications=True) + return store, InMemoryReportingOutbox(store) + + +def durable(pool: AsyncConnectionPool) -> tuple[ReportingLedgerStore, ReportingNotificationOutbox]: + return PgReportingReconciliationStore(pool=pool, notifications=True), PgReportingOutbox( + pool=pool + ) + + +class TrustedConfigurations: + def __init__(self, subscription: ReportingNotificationSubscription) -> None: + self.subscription = subscription + + async def list_active( + self, *, account_id: str, notification_type: str + ) -> tuple[ReportingNotificationSubscription, ...]: + value = self.subscription + return ( + (value,) + if value.account_id == account_id and notification_type in value.event_types + else () + ) + + async def get_active( + self, *, account_id: str, subscriber_id: str, notification_type: str + ) -> ReportingNotificationSubscription | None: + return next( + ( + value + for value in await self.list_active( + account_id=account_id, notification_type=notification_type + ) + if value.subscriber_id == subscriber_id + ), + None, + ) + + +class TrustedKeys: + def __init__(self, keys: dict[tuple[str, str, str], ReportingSigningMaterial]) -> None: + self.keys = keys + + async def resolve( + self, *, account_id: str, principal_id: str, signing_scope_id: str + ) -> ReportingSigningMaterial: + return self.keys[(account_id, principal_id, signing_scope_id)] + + +def worker( + outbox: ReportingNotificationOutbox, key: Ed25519PrivateKey, clock: Callable[[], datetime] +) -> ReportingNotificationWorker: + configurations: ReportingSubscriptionResolver = TrustedConfigurations( + ReportingNotificationSubscription( + account_id="account-a", + subscriber_id="subscriber-a", + principal_id="consumer-a", + url="https://receiver.example.test/notifications", + event_types=("reporting.ledger_changed",), + configuration_revision="registration-1", + authorization_ref="principal-grant-1", + proof_of_control_ref="account-challenge-1", + signing_scope_id="seller-keyring", + active=True, + authorized=True, + proof_valid=True, + ) + ) + keys: ReportingSigningResolver = TrustedKeys( + { + ("account-a", "consumer-a", "seller-keyring"): ReportingSigningMaterial( + key, "key-1", "ed25519", frozenset({"ed25519"}) + ), + } + ) + return ReportingNotificationWorker( + outbox=outbox, + subscriptions=configurations, + signing=keys, + cipher=ReportingEnvelopeCipher(b"t" * 32), + clock=clock, + ) + + +async def typed_issue_handoff( + store: PgReportingReconciliationStore | InMemoryReportingReconciliationStore, + obligation: ReportingObligationRecord, + at: datetime, +) -> None: + await store.ensure_issue_opened( + issue_key="opaque-condition-reference", + account_id=obligation.account_id, + consumer_id="consumer-a", + observed_at=at, + status_scope=ReportingStatusScope.for_obligation(obligation, "consumer-a"), + ) + + +async def projector_checkpoint( + outbox: ReportingNotificationOutbox, +) -> tuple[ReportingStatusDirty, ...]: + previous = await outbox.status_checkpoint( + account_id="account-a", projector_id="later-projector" + ) + records = await outbox.read_status_dirty(account_id="account-a", after=previous, limit=100) + if records: + await outbox.advance_status_checkpoint( + account_id="account-a", + projector_id="later-projector", + expected=previous, + through=records[-1].sequence, + ) + return records From 21bf443e7d850d1800ec8a6f2e4abec1c8f85541 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Wed, 16 Sep 2026 11:01:59 +0000 Subject: [PATCH 2/2] fix(reporting): apply rc.3 configuration lifecycle and keep issue returns PostgreSQL dropped a lifecycle-only put_configuration through ON CONFLICT DO NOTHING, so a deactivated feed kept minting obligations and never produced its status-dirty generation, while the memory store applied both. rc.3's reporting-delivery-config-state.json walks one immutable generation from ready to inactive and requires deactivated_at there, which is why activation, deactivation and the recovery/retention windows are excluded from content_sha256. Apply them to the retained generation and co-commit the dirty record on that exact connection under the account lock. An unchanged re-put stays a no-op that enqueues nothing; changed content is still a CONFIGURATION_GENERATION_IMMUTABLE conflict that writes nothing. Exempt exactly those lifecycle columns from the reconciliation reference guard, so a Managed generation bound by reconciliation records keeps its published content immutable and can still be deactivated, and refresh the one schema-contract digest that moves. Restore the parent's set_issue_state return values: retired_at advances on every waive again, since no rc.3 field backs it and default-off Core behavior must not change. Derive the dirty no-op from the resulting record instead of predicting it, so an idempotent re-acknowledge still enqueues nothing while a repeated waive stays reconstructable. Raise the standard actionable [pg] install hint from PgReportingOutbox construction, matching the sibling PostgreSQL stores. Refs #1168 Co-Authored-By: Claude Opus 5 (1M context) --- docs/reporting-ledger-migration.md | 7 +- docs/reporting-notification-outbox.md | 16 ++ src/adcp/reporting/ledger/pg.py | 69 +++++-- .../reporting_ledger_reconciliation.sql | 15 +- src/adcp/reporting/ledger/store.py | 43 +++-- src/adcp/reporting/outbox/_schema.py | 2 +- src/adcp/reporting/outbox/pg.py | 7 + .../test_reporting_notification_packaging.py | 9 + .../test_reporting_notification_readiness.py | 176 +++++++++++++++++- ...est_reporting_notification_transactions.py | 9 + 10 files changed, 321 insertions(+), 32 deletions(-) diff --git a/docs/reporting-ledger-migration.md b/docs/reporting-ledger-migration.md index bc71f2b7d..ab7e8eb19 100644 --- a/docs/reporting-ledger-migration.md +++ b/docs/reporting-ledger-migration.md @@ -76,7 +76,12 @@ leaves the old primary key in place and does not perform this upgrade. The reconciliation migration adds empty optional evidence tables and nullable, immutable managed digest/total evidence on revisions and adjustments. It does not infer historical -evidence or enable a delivery tier. See the [storage contract](reporting-reconciliation-storage.md) +evidence or enable a delivery tier. Re-running it also replaces the +`reporting_reconciliation_reference_immutable()` guard function in place, so +that a referenced configuration generation keeps its published content +immutable while still accepting the rc.3 lifecycle columns (`activated_at`, +`deactivated_at`, `automated_recovery_seconds`, `status_retention_days`). It +rewrites no rows. See the [storage contract](reporting-reconciliation-storage.md) for the records, migration invariants, and deferred writer/handler work. The outbox migration adds empty event, delivery, and ordered status-dirty tables. diff --git a/docs/reporting-notification-outbox.md b/docs/reporting-notification-outbox.md index 247540502..a361bb804 100644 --- a/docs/reporting-notification-outbox.md +++ b/docs/reporting-notification-outbox.md @@ -86,6 +86,22 @@ idempotency key and its own prepared body. Re-emitting an event with `outbox.reemit(...)` advances the emission generation and generates new keys; ordinary retry changes neither generation, key, nor body bytes. +A configuration generation's *content* stays immutable, while the rc.3 +lifecycle state it carries -- activation, deactivation, and the recovery and +retention windows -- keeps evolving on that same generation +(`reporting-delivery-config-state.json` walks one generation from `ready` to +`inactive`). A re-put that changes only those fields therefore applies to the +retained generation and co-commits one status-dirty generation; an unchanged +re-put stays a no-op and enqueues nothing; changed content is still a +`CONFIGURATION_GENERATION_IMMUTABLE` conflict that writes nothing. The +reconciliation reference guard exempts exactly this lifecycle state, so a +Managed generation bound by reconciliation records can still be deactivated. + +A dirty record follows the retained evidence rather than a predicted +transition: an idempotent re-acknowledge of an issue changes nothing and +enqueues nothing, while a repeated waive does move `retired_at` and stays +reconstructable. Existing issue return values are unchanged by opting in. + Status-dirty records form an account-ordered journal, with a trusted optional generation/obligation/consumer/feed scope, cause generations, and immutable record references. Readability, configuration lifecycle, and issue lifecycle diff --git a/src/adcp/reporting/ledger/pg.py b/src/adcp/reporting/ledger/pg.py index d61354ab9..378298be8 100644 --- a/src/adcp/reporting/ledger/pg.py +++ b/src/adcp/reporting/ledger/pg.py @@ -76,6 +76,7 @@ import json from collections.abc import Callable, Sequence from copy import deepcopy +from dataclasses import replace from datetime import datetime, timedelta, timezone from pathlib import Path from typing import TYPE_CHECKING, Any, ClassVar, Literal @@ -114,6 +115,7 @@ LedgerPage, ReportingRowPage, check_issue_state_transition, + configuration_lifecycle, decode_cursor, encode_cursor, issue_is_retirable, @@ -376,9 +378,11 @@ async def put_configuration(self, configuration: ReportingConfiguration) -> None # accept a different immutable generation from a losing writer. row = await ( await connection.execute( - "SELECT content_sha256 FROM reporting_configurations" + "SELECT content_sha256, activated_at, deactivated_at," + " automated_recovery_seconds, status_retention_days" + " FROM reporting_configurations" " WHERE account_id = %s AND delivery_config_id = %s" - " AND delivery_config_version = %s", + " AND delivery_config_version = %s FOR UPDATE", (key.account_id, key.delivery_config_id, key.delivery_config_version), ) ).fetchone() @@ -390,11 +394,52 @@ async def put_configuration(self, configuration: ReportingConfiguration) -> None "version instead of editing a retained generation", ) - if inserted is not None and self._notifications_enabled: + scope = ReportingStatusScope(configuration.account_id, configuration.generation_key) + if inserted is not None: + if self._notifications_enabled: + await self._dirty_status( + connection, + scope, + "configuration", + after=configuration_evidence(configuration), + ) + return + # rc.3 carries activation/deactivation and the recovery/retention + # windows as lifecycle state over one immutable generation, so a + # re-put that changes only those must apply -- otherwise a + # deactivated feed keeps minting obligations -- and must co-commit + # its status-dirty generation on this exact connection. An + # unchanged re-put stays a no-op and enqueues nothing. + retained = replace( + configuration, + activated_at=_utc(row[1]) if row[1] else None, + deactivated_at=_utc(row[2]) if row[2] else None, + automated_recovery_window=timedelta(seconds=float(row[3])), + status_retention_days=row[4], + ) + if configuration_lifecycle(retained) == configuration_lifecycle(configuration): + return + await connection.execute( + "UPDATE reporting_configurations SET activated_at = %s, deactivated_at = %s," + " automated_recovery_seconds = %s, status_retention_days = %s" + " WHERE account_id = %s AND delivery_config_id = %s" + " AND delivery_config_version = %s", + ( + configuration.activated_at, + configuration.deactivated_at, + configuration.automated_recovery_window.total_seconds(), + configuration.status_retention_days, + key.account_id, + key.delivery_config_id, + key.delivery_config_version, + ), + ) + if self._notifications_enabled: await self._dirty_status( connection, - ReportingStatusScope(configuration.account_id, configuration.generation_key), + scope, "configuration", + before=configuration_evidence(retained), after=configuration_evidence(configuration), ) @@ -1216,23 +1261,23 @@ async def set_issue_state( "reopened, and a recurrence gets a new occurrence", ) check_issue_state_transition(live.issue_state, state) - if state == live.issue_state and ( - not external_ref or external_ref == live.external_ref - ): - await self._dirty_issue(connection, live, status_scope, enqueue=False) - return live await connection.execute( "UPDATE reporting_issue_lifecycle" " SET issue_state = %s," " external_ref = COALESCE(%s, external_ref)," - " retired_at = CASE WHEN %s = 'waived' AND issue_state <> 'waived'" - " THEN %s ELSE retired_at END" + " retired_at = CASE WHEN %s = 'waived' THEN %s ELSE retired_at END" " WHERE account_id = %s AND issue_key = %s AND generation = %s", (state, external_ref, state, _utc(at), account_id, issue_key, live.generation), ) refreshed = await self._issue_row(connection, issue_key, account_id, live.generation) assert refreshed is not None - await self._dirty_issue(connection, refreshed, status_scope, live) + # Derive the no-op from the resulting row rather than predicting + # it: an idempotent re-acknowledge changes nothing and enqueues + # nothing, while anything that does move retained evidence stays + # reconstructable for the projector. + await self._dirty_issue( + connection, refreshed, status_scope, live, enqueue=refreshed != live + ) return refreshed async def retire_issue( diff --git a/src/adcp/reporting/ledger/reporting_ledger_reconciliation.sql b/src/adcp/reporting/ledger/reporting_ledger_reconciliation.sql index 1f9ecea3b..1d9bb1635 100644 --- a/src/adcp/reporting/ledger/reporting_ledger_reconciliation.sql +++ b/src/adcp/reporting/ledger/reporting_ledger_reconciliation.sql @@ -711,17 +711,26 @@ BEGIN previous JSONB := to_jsonb(OLD); proposed JSONB := to_jsonb(NEW); referenced BOOLEAN; + -- Period-close leasing plus the rc.3 lifecycle state that evolves over + -- one immutable generation: reporting-delivery-config-state.json moves + -- the same generation through ready -> inactive, requiring + -- deactivated_at, and its operational recovery/retention windows are + -- likewise state rather than published content. Reconciliation records + -- bind the generation key and its content, none of which these carry. + mutable TEXT[] := ARRAY['lease_worker_id', 'lease_expires_at', 'activated_at', + 'deactivated_at', 'automated_recovery_seconds', + 'status_retention_days']; BEGIN IF proposed = previous OR (TG_TABLE_NAME = 'reporting_configurations' AND - proposed - 'lease_worker_id' - 'lease_expires_at' = previous - 'lease_worker_id' - 'lease_expires_at') + proposed - mutable = previous - mutable) OR (TG_TABLE_NAME = 'reporting_revisions' AND proposed - 'readable' = previous - 'readable') THEN RETURN NEW; END IF; PERFORM pg_advisory_xact_lock(hashtext('adcp.reporting:' || OLD.account_id)); IF TG_TABLE_NAME = 'reporting_configurations' THEN - previous := previous - 'lease_worker_id' - 'lease_expires_at'; - proposed := proposed - 'lease_worker_id' - 'lease_expires_at'; + previous := previous - mutable; + proposed := proposed - mutable; SELECT EXISTS (SELECT 1 FROM reporting_reconciliation_records r WHERE r.account_id = OLD.account_id AND r.delivery_config_id = OLD.delivery_config_id AND r.delivery_config_version = OLD.delivery_config_version) INTO referenced; diff --git a/src/adcp/reporting/ledger/store.py b/src/adcp/reporting/ledger/store.py index cb6059e2f..ff6dfe2ce 100644 --- a/src/adcp/reporting/ledger/store.py +++ b/src/adcp/reporting/ledger/store.py @@ -744,7 +744,10 @@ async def put_configuration(self, configuration: ReportingConfiguration) -> None "publish a new version instead of editing a retained generation", ) self._configurations[key] = configuration - if existing != configuration and self._notification_state is not None: + changed = existing is None or configuration_lifecycle( + existing + ) != configuration_lifecycle(configuration) + if changed and self._notification_state is not None: self._dirty_status( ReportingStatusScope(configuration.account_id, configuration.generation_key), "configuration", @@ -1218,25 +1221,19 @@ async def set_issue_state( "reopened, and a recurrence gets a new occurrence", ) check_issue_state_transition(live.issue_state, state) - if state == live.issue_state and ( - not external_ref or external_ref == live.external_ref - ): - self._dirty_issue(live, status_scope, enqueue=False) - return live updated = replace( live, issue_state=state, external_ref=external_ref or live.external_ref, - # Set only on the way into a retired state and never cleared: - # a waived issue keeps the instant it was waived. - retired_at=( - _utc(at) - if state == "waived" and live.issue_state != "waived" - else live.retired_at - ), + # Set on the way into a retired state and never cleared. + retired_at=_utc(at) if state == "waived" else live.retired_at, ) self._issues[key] = updated - self._dirty_issue(updated, status_scope, live) + # Derive the no-op from the resulting record rather than predicting + # it: an idempotent re-acknowledge changes nothing and enqueues + # nothing, while anything that does move retained evidence stays + # reconstructable for the projector. + self._dirty_issue(updated, status_scope, live, enqueue=updated != live) return updated async def retire_issue( @@ -1453,6 +1450,24 @@ def reject_reserved_authoritative_party(configuration: ReportingConfiguration) - ) +def configuration_lifecycle(configuration: ReportingConfiguration) -> tuple[Any, ...]: + """The rc.3 lifecycle state carried over one immutable content generation. + + ``reporting-delivery-config-state.json`` walks a single generation from + ``ready`` to ``inactive``, requiring ``deactivated_at`` on the way, and the + recovery/retention windows are operational state too. That is why none of + these fields feed ``content_sha256``: a re-put changing only them applies + to the retained generation instead of conflicting with it. Both stores + compare exactly this tuple so their status-dirty journals agree. + """ + return ( + _utc(configuration.activated_at) if configuration.activated_at else None, + _utc(configuration.deactivated_at) if configuration.deactivated_at else None, + configuration.automated_recovery_window, + configuration.status_retention_days, + ) + + def _config_payload(configuration: ReportingConfiguration) -> dict[str, Any]: schedule = configuration.schedule return { diff --git a/src/adcp/reporting/outbox/_schema.py b/src/adcp/reporting/outbox/_schema.py index 4efd748fa..d0a6d5e46 100644 --- a/src/adcp/reporting/outbox/_schema.py +++ b/src/adcp/reporting/outbox/_schema.py @@ -69,7 +69,7 @@ "cb1b0e47e2848ae79292f539f4c3fe85" "8b651db28434bdd468596b36cd08d9cb" ), "function:reporting_reconciliation_reference_immutable()": ( - "f9048965fa08e87e4b26ecd108cce370" "f19ad939e03486f32fba66a899e7bd05" + "6461682d3ebaceb7ccd795f1d87c524a" "02f8d1f1487dd182e23779b44dae4ddf" ), "function:reporting_reconciliation_validate(r reporting_reconciliation_records)": ( "309164066f02cc4d0f3ff66df47cf5c4" "ddddcc69525436634f94845e0d2864ed" diff --git a/src/adcp/reporting/outbox/pg.py b/src/adcp/reporting/outbox/pg.py index e522053a9..dfd7b69df 100644 --- a/src/adcp/reporting/outbox/pg.py +++ b/src/adcp/reporting/outbox/pg.py @@ -178,6 +178,13 @@ class PgReportingOutbox: def __init__( self, *, pool: AsyncConnectionPool, clock: Callable[[], datetime] | None = None ) -> None: + # Same actionable hint the sibling PG stores raise, so a base-install + # adopter is told to add the extra instead of meeting a driver error + # from inside the first query. + from adcp.reporting.ledger.pg import _INSTALL_HINT, PG_AVAILABLE + + if not PG_AVAILABLE: + raise ImportError(_INSTALL_HINT) self._pool, self._clock = pool, clock async def create_schema(self) -> None: diff --git a/tests/conformance/reporting/test_reporting_notification_packaging.py b/tests/conformance/reporting/test_reporting_notification_packaging.py index 9fb362859..edfada1a7 100644 --- a/tests/conformance/reporting/test_reporting_notification_packaging.py +++ b/tests/conformance/reporting/test_reporting_notification_packaging.py @@ -127,6 +127,7 @@ def installed_distribution(tmp_path_factory): import adcp from adcp.reporting.outbox import ( InMemoryReportingOutbox, + PgReportingOutbox, ReportingEnvelopeCipher, ReportingNotificationWorker, ) @@ -140,6 +141,14 @@ def installed_distribution(tmp_path_factory): assert "notifications" not in inspect.signature(ReportingProducer).parameters assert not hasattr(InMemoryReportingLedgerStore(), "commit_materialization") outbox = InMemoryReportingOutbox(InMemoryReportingLedgerStore(notifications=True)) +# The PG outbox must refuse construction with the same actionable extra hint +# the sibling PG stores raise, not a driver error from inside a later query. +try: + PgReportingOutbox(pool=None) +except ImportError as error: + assert "adcp[pg]" in str(error), str(error) +else: + raise AssertionError("PgReportingOutbox must raise the [pg] install hint") assert files("adcp.reporting.ledger").joinpath("reporting_notification_outbox.sql").is_file() assert ( get_named_validator("core/reporting-ledger-changed-webhook.json", version="3.2.0-rc.3") diff --git a/tests/conformance/reporting/test_reporting_notification_readiness.py b/tests/conformance/reporting/test_reporting_notification_readiness.py index 23221733f..ce88a85cd 100644 --- a/tests/conformance/reporting/test_reporting_notification_readiness.py +++ b/tests/conformance/reporting/test_reporting_notification_readiness.py @@ -5,7 +5,11 @@ import pytest -from adcp.reporting.ledger import InMemoryReportingLedgerStore, ReportingDeliveryScope +from adcp.reporting.ledger import ( + InMemoryReportingLedgerStore, + LedgerConflictError, + ReportingDeliveryScope, +) from adcp.reporting.outbox import ( InMemoryReportingOutbox, ReportingNotificationError, @@ -14,6 +18,7 @@ from ._generation_support import NOW, configuration from ._reconciliation_support import scenario +from ._reliable_support import reliable_factory from .test_reporting_notification_outbox import seed @@ -124,3 +129,172 @@ async def test_mutable_memory_configuration_lifecycle_keeps_core_semantics(): assert dirty[1].after.automated_recovery_seconds == 7200 assert dirty[2].after == dirty[0].after assert await outbox.list_events(account_id="acct_a") == () + + +async def test_configuration_lifecycle_state_is_shared_by_memory_and_postgres( + notification_harness, +): + """rc.3 walks one immutable generation through ready -> inactive. + + Both stores must apply a lifecycle-only re-put to the retained generation + and co-commit exactly one status-dirty generation for it, so #1168B's + projector sees the same journal on either backend. A PostgreSQL + ``ON CONFLICT DO NOTHING`` that dropped the update would leave a + deactivated feed minting obligations forever and never mark it dirty. + """ + h = notification_harness + store = h.reliable.store + first = configuration() + assert first.deactivated_at is not None + inactive = replace( + first, + deactivated_at=first.deactivated_at + timedelta(hours=1), + automated_recovery_window=timedelta(hours=2), + status_retention_days=30, + ) + await store.put_configuration(first) + # An unchanged re-put stays a no-op on both stores. + await store.put_configuration(first) + await store.put_configuration(inactive) + await store.put_configuration(inactive) + # Reverting the lifecycle is a further transition, not a rollback. + await store.put_configuration(first) + + retained = await store.list_configurations(account_id="acct_a") + assert [item.deactivated_at for item in retained] == [first.deactivated_at] + assert retained[0].automated_recovery_window == first.automated_recovery_window + assert retained[0].status_retention_days == first.status_retention_days + + dirty = [ + record + for record in await h.outbox.read_status_dirty(account_id="acct_a") + if record.reason == "configuration" + ] + assert [record.cause_generation for record in dirty] == [1, 2, 3] + assert [record.scope for record in dirty] == [ + ReportingStatusScope("acct_a", first.generation_key) + ] * 3 + assert dirty[0].before is None + assert dirty[1].before == dirty[0].after and dirty[2].before == dirty[1].after + assert dirty[1].after.deactivated_at == inactive.deactivated_at + assert dirty[1].after.automated_recovery_seconds == 7200 + assert dirty[1].after.status_retention_days == 30 + assert dirty[2].after == dirty[0].after + # Lifecycle state is not a ledger change: no logical event is emitted. + assert await h.outbox.list_events(account_id="acct_a") == () + + # Changed content is still a conflict that mutates nothing and enqueues + # nothing, on the same generation the lifecycle re-put just touched. + with pytest.raises(LedgerConflictError) as conflict: + await store.put_configuration(replace(first, report_definition_id="rpd_other")) + assert conflict.value.code == "CONFIGURATION_GENERATION_IMMUTABLE" + assert await store.list_configurations(account_id="acct_a") == retained + assert [ + record + for record in await h.outbox.read_status_dirty(account_id="acct_a") + if record.reason == "configuration" + ] == dirty + + +async def test_reconciled_managed_generation_can_still_be_deactivated(notification_harness): + """The parent reference-immutability guard must not freeze lifecycle state. + + A Managed generation referenced by reconciliation records keeps its + published content immutable while still reaching rc.3 ``inactive``. + """ + h = notification_harness + s = await scenario(h.reliable.store) + await h.reliable.store.commit_materialization(s.outcome) + (config,) = await h.reliable.store.list_configurations(account_id="acct_a") + assert config.generation_key == s.binding.generation_key + assert config.deactivated_at is not None + stopped = replace(config, deactivated_at=config.deactivated_at + timedelta(hours=3)) + await h.reliable.store.put_configuration(stopped) + (retained,) = await h.reliable.store.list_configurations(account_id="acct_a") + assert retained.deactivated_at == stopped.deactivated_at + assert retained.report_definition_id == config.report_definition_id + latest = [ + record + for record in await h.outbox.read_status_dirty(account_id="acct_a") + if record.reason == "configuration" + ][-1] + assert latest.before.deactivated_at == config.deactivated_at + assert latest.after.deactivated_at == stopped.deactivated_at + with pytest.raises(LedgerConflictError) as conflict: + await h.reliable.store.put_configuration(replace(config, report_definition_id="rpd_other")) + assert conflict.value.code == "CONFIGURATION_GENERATION_IMMUTABLE" + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +async def test_default_off_issue_waive_keeps_parent_return_values(backend): + """Opting out must not change any existing Core return value. + + ``retired_at`` is SDK bookkeeping with no rc.3 field behind it, so the A + slice may not silently freeze it. A repeated waive keeps advancing it and a + later ``external_ref`` still lands, exactly as before the outbox existed. + """ + async with reliable_factory(backend, notifications=False) as reliable: + store = reliable.store + await store.ensure_issue_opened( + issue_key="k", account_id="acct_a", consumer_id=None, observed_at=NOW + ) + await store.set_issue_state( + issue_key="k", account_id="acct_a", state="acknowledged", at=NOW + ) + first = await store.set_issue_state( + issue_key="k", account_id="acct_a", state="waived", at=NOW + timedelta(hours=1) + ) + assert first.retired_at == NOW + timedelta(hours=1) + again = await store.set_issue_state( + issue_key="k", account_id="acct_a", state="waived", at=NOW + timedelta(hours=9) + ) + assert again.retired_at == NOW + timedelta(hours=9) + tagged = await store.set_issue_state( + issue_key="k", + account_id="acct_a", + state="waived", + at=NOW + timedelta(hours=20), + external_ref="ticket-2", + ) + assert tagged.retired_at == NOW + timedelta(hours=20) + assert tagged.external_ref == "ticket-2" + + +async def test_issue_dirty_records_track_actual_retained_evidence(notification_harness): + """The journal follows the record, so a projector can trust either store. + + An idempotent re-acknowledge moves nothing and enqueues nothing. A repeated + waive does move ``retired_at``, so it must stay reconstructable. + """ + h = notification_harness + store = h.reliable.store + await store.ensure_issue_opened( + issue_key="k", account_id="acct_a", consumer_id=None, observed_at=NOW + ) + await store.set_issue_state(issue_key="k", account_id="acct_a", state="acknowledged", at=NOW) + await store.set_issue_state( + issue_key="k", account_id="acct_a", state="acknowledged", at=NOW + timedelta(hours=1) + ) + waived = await store.set_issue_state( + issue_key="k", account_id="acct_a", state="waived", at=NOW + timedelta(hours=2) + ) + rewaived = await store.set_issue_state( + issue_key="k", account_id="acct_a", state="waived", at=NOW + timedelta(hours=3) + ) + issues = [ + record + for record in await h.outbox.read_status_dirty(account_id="acct_a") + if record.reason == "issue" + ] + assert [record.after.issue_state for record in issues] == [ + "open", + "acknowledged", + "waived", + "waived", + ] + assert [record.cause_generation for record in issues] == [1, 2, 3, 4] + assert issues[2].before.issue_state == "acknowledged" + assert issues[2].after.retired_at == waived.retired_at + assert issues[3].before.retired_at == waived.retired_at + assert issues[3].after.retired_at == rewaived.retired_at + assert await h.outbox.list_events(account_id="acct_a") == () diff --git a/tests/conformance/reporting/test_reporting_notification_transactions.py b/tests/conformance/reporting/test_reporting_notification_transactions.py index 3318596fc..b4d410024 100644 --- a/tests/conformance/reporting/test_reporting_notification_transactions.py +++ b/tests/conformance/reporting/test_reporting_notification_transactions.py @@ -146,6 +146,14 @@ async def prepare(h: NotificationHarness, operation: str): obligation, revision, rows = await seed(h, official=operation == "adjustment") if operation == "configuration": return lambda: store.put_configuration(replace(configuration(), delivery_config_version=2)) + if operation == "configuration_lifecycle": + # The retained generation already exists; only rc.3 lifecycle state + # moves, so this exercises the UPDATE path rather than the INSERT. + base = configuration() + assert base.deactivated_at is not None + return lambda: store.put_configuration( + replace(base, deactivated_at=base.deactivated_at + timedelta(hours=1)) + ) if operation == "obligation": other = replace( obligation, @@ -252,6 +260,7 @@ async def image(h: NotificationHarness): "operation", [ "configuration", + "configuration_lifecycle", "obligation", "revision", "readability",