diff --git a/.github/workflows/longhaul-smoke.yml b/.github/workflows/longhaul-smoke.yml index 8d9e08df6..d17259543 100644 --- a/.github/workflows/longhaul-smoke.yml +++ b/.github/workflows/longhaul-smoke.yml @@ -16,16 +16,19 @@ # The operator + sidecar are built from THIS PR's source (via the shared # test-build-and-package.yml reusable workflow) so a code change that breaks # the operator is caught. The documentdb + gateway (database) images are NOT -# rebuilt: test-build-and-package.yml probes the public registry for the -# pinned documentDbVersion and, when present, runs in "registry mode" and -# simply re-tags the public images into the build artifact. The long-haul -# driver image is the only extra image this workflow builds. +# built at all: the smoke exercises an upgrade between two published database +# releases, pulling both the 0.110.0 base and the 0.113.0 target straight +# from the public registry. The long-haul driver image is the only extra +# image this workflow builds. # # Environment bring-up is delegated to the shared composite action # .github/actions/setup-test-environment -# (use-external-images: false — load operator/sidecar/documentdb/gateway from -# the build artifacts into kind), exactly like test-e2e.yml. No operator-install -# logic is duplicated here. +# (use-external-images: false — load the operator/sidecar built from this PR +# into kind), exactly like test-e2e.yml. No operator-install logic is +# duplicated here. The action also loads the artifact's documentdb/gateway +# images, but the smoke does not use them: the database base and target are +# pulled from the public registry (see the setup-step overrides and the +# prepare step). # # Why reuse test/longhaul/deploy/{deployment,rbac}.yaml as-is # The whole point of the gate is to exercise the SAME manifests the long-haul @@ -42,23 +45,38 @@ # longhaul-report ConfigMap's result field. # # CI budget -# Scale/upgrade disruption ops are disabled for the smoke run -# (LONGHAUL_MIN_INSTANCES == LONGHAUL_MAX_INSTANCES) so the gate is a fast, -# deterministic data-durability check (writers + verifier) that fits a -# GitHub-hosted runner and finishes in a few minutes. +# Scale ops run for real (MIN=2, MAX=3). The gate exercises every registered +# operation once and finishes within a GitHub-hosted runner's budget. +# +# Operations exercised (sequence mode) +# The smoke runs the operation scheduler in sequence mode +# (LONGHAUL_OPERATION_MODE=sequence) with an explicit ordered list +# (LONGHAUL_OPERATION_SEQUENCE). Each operation — scale-up, scale-down, +# upgrade-documentdb, kill-operator-pod, kill-primary-pod — runs exactly once, +# in order, gated by the same steady-state / precondition / recovery logic the +# long-haul run uses. This is the deterministic path that builds confidence the +# operations execute end-to-end; the gate asserts every sequenced operation +# reached PASSED and the run reached COMPLETE. The upgrade is a real +# cross-version rolling upgrade between two published releases: the cluster +# starts on DB_BASE_VERSION (0.110.0) and the upgrade-documentdb operation +# moves documentDBVersion forward to DB_TARGET_VERSION (0.113.0), exercising +# the operator's rolling-update path with clean semver versions the +# image-rollback webhook accepts. Both endpoints are hardcoded in the job env +# (keep DB_TARGET_VERSION in sync with the chart's documentDbVersion). +# MAX_DURATION is only the completion watchdog. # # Data-protection gate # The backup verifier is exercised for real, not just compiled in. The kind # cluster already has CSI VolumeSnapshot support (setup-test-environment runs # deploy-csi-driver.sh: external-snapshotter + a default csi-hostpath -# VolumeSnapshotClass), so a single-instance cluster can complete snapshot -# backups — exactly as the e2e scheduled-backup test proves. The smoke run -# sets a per-minute backup schedule and a 30s verify interval (vs the 5m -# default, via LONGHAUL_BACKUP_VERIFY_INTERVAL) so the verifier's periodic -# loop fires many times within the window and reliably observes a -# scheduled+completed backup. It then asserts scheduled AND completed >= 1 -# (with no retention leak or completion stall), so a broken backup path fails -# the PR rather than passing silently as a no-op. +# VolumeSnapshotClass), so the cluster can complete snapshot backups — +# exactly as the e2e scheduled-backup test proves. The smoke run sets a +# per-minute backup schedule and a 30s verify interval (vs the 5m default, +# via LONGHAUL_BACKUP_VERIFY_INTERVAL) so the verifier's periodic loop fires +# many times within the window and reliably observes a scheduled+completed +# backup. It then asserts scheduled AND completed >= 1 (with no retention leak +# or completion stall), so a broken backup path fails the PR rather than +# passing silently as a no-op. name: Long-Haul Smoke Gate @@ -74,9 +92,9 @@ on: workflow_dispatch: inputs: max_duration: - description: "Bounded driver run length (Go duration). Keep >= 6m so at least one per-minute backup is scheduled and completed within the window." + description: "Completion watchdog duration (Go duration, e.g. 20m). Keep >= 6m so at least one per-minute backup is scheduled and completed within the window." required: false - default: "6m" + default: "20m" permissions: contents: read @@ -102,8 +120,9 @@ env: jobs: # --------------------------------------------------------------------------- - # Build operator + sidecar from this PR's source; reuse public documentdb / - # gateway images (registry mode) so the database images are NOT rebuilt. + # Build operator + sidecar from this PR's source; the database (documentdb / + # gateway) images are NOT built or used from the artifact — the smoke pulls + # the released base and upgrade-target database images directly. # Produces the platform-images tar + helm chart artifacts the setup action # loads into kind. # --------------------------------------------------------------------------- @@ -119,7 +138,7 @@ jobs: needs: build if: always() && needs.build.result == 'success' runs-on: ubuntu-22.04 - timeout-minutes: 40 + timeout-minutes: 50 env: IMAGE_TAG: ${{ needs.build.outputs.image_tag }} EXT_IMAGE_TAG: ${{ needs.build.outputs.ext_image_tag }} @@ -128,8 +147,20 @@ jobs: # Must match the cluster name the composite action derives: # documentdb--- KIND_CLUSTER: documentdb-longhaul-amd64-smoke - # The pruner's first tick is at 5m; the default must run beyond it. - MAX_DURATION: ${{ github.event.inputs.max_duration || '6m' }} + # Completion watchdog; also kept beyond the 5m pruner tick and long enough + # for at least one per-minute backup to schedule and complete. + MAX_DURATION: ${{ github.event.inputs.max_duration || '20m' }} + # Two published database releases the smoke upgrades between. The cluster + # STARTS at DB_BASE_VERSION and the upgrade operation moves it forward to + # DB_TARGET_VERSION. Both are hardcoded here and pulled from the public + # registry — the smoke builds no database image. Keep DB_TARGET_VERSION in + # sync with the chart's documentDbVersion (nothing enforces it). 0.110.0 is + # the most recent published release before 0.113.0 (0.111.0 / 0.112.0 were + # never published). + DB_BASE_VERSION: "0.110.0" + DB_TARGET_VERSION: "0.113.0" + DOCUMENTDB_REPO: ghcr.io/documentdb/documentdb-kubernetes-operator/documentdb + GATEWAY_REPO: ghcr.io/documentdb/documentdb-kubernetes-operator/gateway steps: - name: Checkout uses: actions/checkout@v4 @@ -167,7 +198,7 @@ jobs: runner: "ubuntu-22.04" test-scenario-name: "smoke" node-count: "1" - instances-per-node: "1" + instances-per-node: "2" cert-manager-namespace: ${{ env.CERT_MANAGER_NS }} operator-namespace: ${{ env.OPERATOR_NS }} db-namespace: ${{ env.DB_NS }} @@ -176,7 +207,18 @@ jobs: db-password: ${{ env.DB_PASSWORD }} db-port: ${{ env.GATEWAY_PORT }} image-tag: ${{ env.IMAGE_TAG }} + # Required only to satisfy the shared action's mandatory local-build + # DB-image load (it kind-loads the artifact's documentdb/gateway images + # tagged with this value). The smoke does not otherwise use those + # images — the upgrade's base and target are pulled from the public + # registry (see the overrides below and the prepare step). documentdb-image-tag: ${{ env.EXT_IMAGE_TAG }} + # Start the cluster at the released base version so the upgrade + # operation performs a real cross-version rolling upgrade. These + # overrides make the composite action pull the released images and + # deploy the CR with explicit spec.image fields pinned to the base. + documentdb-image: ${{ env.DOCUMENTDB_REPO }}:${{ env.DB_BASE_VERSION }} + gateway-image: ${{ env.GATEWAY_REPO }}:${{ env.DB_BASE_VERSION }} chart-version: ${{ env.CHART_VERSION }} use-external-images: "false" github-token: ${{ secrets.GITHUB_TOKEN }} @@ -198,6 +240,50 @@ jobs: --from-literal=uri="${URI}" \ --dry-run=client -o yaml | kubectl apply -f - + - name: Prepare cross-version upgrade target (0.110.0 -> 0.113.0) + run: | + set -euo pipefail + target_version="${DB_TARGET_VERSION}" + echo "Upgrade: ${DB_BASE_VERSION} -> ${target_version}" + + # Pull the RELEASED target database images straight from the public + # registry and load them into kind under their canonical repo + semver + # tag, so documentDBVersion= resolves to the real release. The + # base (0.110.0) was already pulled and deployed by the setup step via + # the spec.image overrides; the smoke builds no database image. + for component in documentdb gateway; do + img="ghcr.io/documentdb/documentdb-kubernetes-operator/${component}:${target_version}" + docker pull "${img}" + kind load docker-image "${img}" --name "${KIND_CLUSTER}" + done + + # setup-test-environment deployed the CR with explicit spec.image + # fields pinned to the base version, which take precedence over + # documentDBVersion. Convert to the equivalent version-based reference + # (same base payload, image cleared) so the driver's upgrade operation + # can move documentDBVersion from the base to the target. Because the + # resolved version is unchanged (base == running) the image-rollback + # admission webhook accepts this patch. + version_patch=$(BASE_VERSION="${DB_BASE_VERSION}" jq -nc '{ + spec: { + documentDBVersion: env.BASE_VERSION, + image: { + documentDB: null, + gateway: null + } + } + }') + kubectl patch documentdb "${DB_NAME}" -n "${DB_NS}" \ + --type merge -p "${version_patch}" + + # The upgrade operation reads this desired version from the ConfigMap + # and patches documentDBVersion to it; both base and target are clean + # semvers so the forward move (0.110.0 -> target) is permitted. + kubectl create configmap longhaul-versions \ + -n "${DB_NS}" \ + --from-literal="desired-documentdb-version=${target_version}" \ + --dry-run=client -o yaml | kubectl apply -f - + - name: Deploy long-haul driver (real manifests, bounded override) run: | # RBAC applies unmodified (namespace matches DB_NS). @@ -215,11 +301,21 @@ jobs: # Bounded, deterministic smoke override — patch ONLY runtime knobs on # the shipped ConfigMap; the manifest structure is unchanged. - # - MAX_DURATION: finite run + # - MAX_DURATION: finite run (completion watchdog) # - RESET_DATA: fresh collection each CI run - # - RETAIN_PER_WRITER: low enough to force a real prune at 5m - # - MIN==MAX instances: disable disruptive scale ops (fast + stable) + # - RETAIN_PER_WRITER + PRUNE_INTERVAL: retain few docs and prune + # every 30s (vs the 5m default) so a real prune fires within the + # bounded window instead of relying on the run happening to outlast + # the default first tick. + # - OPERATION_MODE=sequence + OPERATION_SEQUENCE: run every operation + # (scale-up, scale-down, upgrade-documentdb, kill-operator-pod, + # kill-primary-pod) exactly once, in a fixed order. This is the + # deterministic path that builds confidence the operations execute + # end-to-end; MAX_DURATION becomes the completion watchdog. # - short cadences so the verifier gets several cycles in the window + # - short steady-state gate with a bounded recovery budget. Sequence + # mode paces ops purely by the steady-state/recovery gates, so + # LONGHAUL_OP_COOLDOWN (a random-mode-only rate limiter) is omitted. # - BACKUP_*: exercise the data-protection verifier for real — a # per-minute schedule so at least one backup is scheduled and # completed within the bounded window, plus a 30s verify interval @@ -233,12 +329,14 @@ jobs: LONGHAUL_RESET_DATA: "true", LONGHAUL_NUM_WRITERS: "2", LONGHAUL_RETAIN_PER_WRITER: "100", - LONGHAUL_OP_COOLDOWN: "30s", - LONGHAUL_STEADY_STATE_WAIT: "10s", - LONGHAUL_RECOVERY_TIMEOUT: "2m", + LONGHAUL_PRUNE_INTERVAL: "30s", + LONGHAUL_OPERATION_MODE: "sequence", + LONGHAUL_OPERATION_SEQUENCE: "scale-up,scale-down,upgrade-documentdb,kill-operator-pod,kill-primary-pod", + LONGHAUL_STEADY_STATE_WAIT: "5s", + LONGHAUL_RECOVERY_TIMEOUT: "5m", LONGHAUL_REPORT_INTERVAL: "30s", - LONGHAUL_MIN_INSTANCES: "1", - LONGHAUL_MAX_INSTANCES: "1", + LONGHAUL_MIN_INSTANCES: "2", + LONGHAUL_MAX_INSTANCES: "3", LONGHAUL_BACKUP_ENABLED: "true", LONGHAUL_BACKUP_SCHEDULE: "*/1 * * * *", LONGHAUL_BACKUP_RETENTION_DAYS: "1", @@ -255,7 +353,7 @@ jobs: id: wait run: | set -euo pipefail - deadline=$(( $(date +%s) + 900 )) # 15 min hard cap + deadline=$(( $(date +%s) + 1800 )) # 30 min hard cap exit_code="" while [[ $(date +%s) -lt ${deadline} ]]; do pod=$(kubectl get pods -n "${DB_NS}" \ @@ -301,8 +399,14 @@ jobs: -o jsonpath='{.data.result}' 2>/dev/null || echo "MISSING") report=$(kubectl get configmap longhaul-report -n "${DB_NS}" \ -o jsonpath='{.data.latest-report}' 2>/dev/null || echo "MISSING") + operation_status=$(kubectl get configmap longhaul-report -n "${DB_NS}" \ + -o json | jq -r '.data["operation-status"] // "MISSING"') + operation_results=$(kubectl get configmap longhaul-report -n "${DB_NS}" \ + -o json | jq -r '.data["operation-results"] // "MISSING"') echo "Driver exit code : ${exit_code}" echo "Report result : ${result}" + echo "Operation status : ${operation_status}" + echo "Operation results: ${operation_results}" if [[ "${exit_code}" != "0" ]]; then echo "::error::Driver exited non-zero (${exit_code})." @@ -312,11 +416,24 @@ jobs: echo "::error::longhaul-report result is '${result}', expected PASS." exit 1 fi + if [[ "${operation_status}" != "COMPLETE" ]]; then + echo "::error::operation-status is '${operation_status}', expected COMPLETE." + exit 1 + fi + # Sequence mode: assert every operation in the configured sequence ran + # and passed. Order is fixed by LONGHAUL_OPERATION_SEQUENCE above. + if ! jq -e ' + (map(.name)) == ["scale-up","scale-down","upgrade-documentdb","kill-operator-pod","kill-primary-pod"] and + all(.[]; .status == "PASSED") + ' <<<"${operation_results}" >/dev/null; then + echo "::error::operation-results did not show every sequenced operation PASSED in order." + exit 1 + fi if ! grep -Eq 'pruner: pruned [1-9][0-9]* docs' <<<"${report}"; then echo "::error::Retention pruner did not report deleting any documents." exit 1 fi - echo "✅ Long-haul smoke gate passed (exit 0, report PASS, retention pruned documents)." + echo "✅ Long-haul smoke gate passed (sequence COMPLETE, all operations PASSED, report PASS, retention pruned documents)." - name: Assert data-protection verifier ran run: | diff --git a/docs/designs/long-haul-test-design.md b/docs/designs/long-haul-test-design.md index 0fc0aea82..7319201ae 100644 --- a/docs/designs/long-haul-test-design.md +++ b/docs/designs/long-haul-test-design.md @@ -44,7 +44,7 @@ flowchart LR | Component | Role | Output | |---|---|---| | **Writer/Verifier** | Data-plane workload. Connects via `mongodb://` only — no k8s imports. Writers insert monotonic sequences with checksums under majority write concern; verifiers scan for gaps and bad checksums. | Counters (acked, failed, verify passes, gaps, checksum errors); errors to journal. | -| **Operation Scheduler** | Control plane. Applies weighted-random ops (scale, kill, failover, backup, upgrade) with preconditions and cooldowns. | Operation start/end events to journal. | +| **Operation Runner** | Control plane. Applies weighted-random ops for production long-haul runs, a deterministic named sequence for smoke/reproduction, or no ops when disabled. | Bounded per-operation results/aggregates plus operation events to journal. | | **Monitor** | Polls pod RSS/CPU and checks readiness of operator + DB pods. | Periodic samples + readiness events to journal. | | **Journal** | In-process append-only event log shared by all components. | Reproducible event stream for the report. | | **Report** | Aggregates the journal into a markdown summary at a configurable interval; raises alerts on threshold breaches. | Markdown report; alert lines. | @@ -77,7 +77,11 @@ The test runs **continuously** — no cycles, no scheduled resets. Workload, met ## Operations -The scheduler picks operations from these categories with weighted randomization: +Production runs use weighted randomization. Deterministic smoke and reproduction +runs can instead request a comma-separated sequence of stable operation names; +each operation runs exactly once in order and the driver exits as soon as the +sequence completes or fails. A disabled mode leaves the workload running without +management operations. | Category | Examples | |---|---| @@ -87,16 +91,47 @@ The scheduler picks operations from these categories with weighted randomization | **Chaos** | kill primary pod, drain node, kill operator pod | | **Data protection** | trigger backup, verify backup | -**Sequencing invariants** (enforced by the scheduler — exact values live in code): +**Operation invariants** (exact values live in code): -- One disruptive op at a time. Overlapping disruptions are non-diagnosable. -- Per-category cooldown between ops. Lets the cluster stabilize. -- Steady-state gate — health check must pass before the next op fires. +- One disruptive op at a time in every mode. Overlapping disruptions are + non-diagnosable. +- Random mode applies the global cooldown between attempts. +- The steady-state gate must pass before each operation. Sequence mode also + requires each named precondition to become true within the recovery timeout. **Backup is not isolated.** It runs concurrently with topology changes and chaos so that backup-vs-topology serialization bugs surface here rather than in production — that serialization is the backup feature's job, not the harness's. Each operation declares an **outage policy**: tolerated write failures during its disruption window and a max recovery time. Breaching the policy is recorded as a Tier-1 failure (see Failure Tiers). +### Outage budgets + +Budgets are wall-clock write-outage durations, independent of the writer count; the longer whole-topology restart is bounded separately by the recovery timeout. + +- **Scale up / down and `kill-operator-pod`** keep the primary write path up throughout, so they are held to a **near-zero** write-outage budget — a regression that unexpectedly disrupts writes during a "safe" operation is caught. +- **`kill-primary-pod`** tolerates a short outage for a single automatic failover (~30s). +- **`upgrade-documentdb`** tolerates a larger one (~90s): a cross-version rolling upgrade's primary switchover coincides with the extension migration under live write load. + +Exact values live in code (`test/longhaul/journal/policy.go`). + +### HA preconditions + +`upgrade-documentdb` and `kill-primary-pod` require an HA topology (`spec.instancesPerNode >= 2`): with no standby to absorb writes, the disruption would produce real (true-positive) downtime that no operator change can prevent. The two runners handle an unmet precondition differently: **random** mode **auto-skips** (the skip consumes no cooldown and is re-evaluated on the next scheduler tick, so scaling up makes the operation eligible again), whereas **sequence** mode does not skip — it waits for the precondition up to the recovery timeout and then fails the sequence, so an HA-dependent op must be preceded by a `scale-up` (or start at `instancesPerNode >= 2`). + +Operation state is intentionally bounded for multi-day runs. Random mode keeps +only passed/failed counters per registered operation type; sequence mode keeps +one mutable `PENDING`/`RUNNING`/`PASSED`/`FAILED` result per requested item. +Execution errors, precondition timeouts, outage-policy violations, and an +incomplete sequence at shutdown all produce a failing final verdict. + +**Sequence mode** (used by the PR smoke gate) runs the registered operations in +an explicit, fixed order (`LONGHAUL_OPERATION_SEQUENCE`), executing each exactly +once behind the same steady-state / precondition / recovery gates as random +mode, rather than selecting operations by weight for the full duration. This +gives the smoke gate a deterministic PASS/FAIL verdict — every sequenced +operation must reach `PASSED` and the run must reach `COMPLETE`; `MAX_DURATION` +becomes the completion watchdog, and a sequence that has not finished at shutdown +is a failing `INCOMPLETE` verdict. + --- ## Data Plane Workload @@ -115,6 +150,20 @@ Losing an acknowledged write or observing a checksum mismatch is a Tier-1 failur --- +## Backup Verification + +When enabled, the driver maintains a canary `ScheduledBackup` named `-longhaul` (reconciled in place, never recreated, so history survives restarts and parameter changes) and runs a verifier concurrently with the operation scheduler — backup is deliberately not isolated from topology/chaos. + +The verifier only checks properties a **multi-day** run can establish, which unit and e2e tests cannot: + +- **Scheduling liveness** — `status.lastScheduledTime` keeps advancing; a stalled scheduler (past `nextScheduledTime` + grace) raises a warning. +- **Completion** — child `Backup` CRs keep reaching `completed`. Only terminal `failed` backups count as failures; a `skipped` backup (e.g. the operator declines to back up a standby) is an intentional no-op. If backups are scheduled but stop completing for **3 consecutive schedules**, the run FAILs; a completed or skipped backup resets the gap, so transient chaos-induced failures and normal standby/failover intervals are tolerated. +- **Retention leak** — no completed backup outlives its retention window (`stoppedAt + spec.retentionDays*24h` + grace), taken from each backup's own stamped `retentionDays` (so the check stays correct even if a later run uses a different retention). A lingering backup is a FAIL: expired backups (and their PVCs / VolumeSnapshots) would otherwise grow unbounded. + +The oracle is black-box: expired backups disappear. It deliberately does **not** re-verify the operator's retention *arithmetic* (`expiredAt == stoppedAt + retentionDays*24h`) — that is a pure function already covered by operator unit tests and needs no accumulation. Because the minimum meaningful retention is 1 day, the leak check only fires on multi-day runs — exactly the accumulation window long-haul exists to cover. + +--- + ## Observability **Per-component attribution.** Metrics are tagged by component (operator pod RSS, DB pod RSS, goroutine count, reconcile rate, API-call rate). Without separate series, a memory climb at hour 30 is undiagnosable. @@ -152,6 +201,32 @@ A Fatal failure does **not** auto-recreate the cluster — the preserved state i --- +## Relationship to `test/e2e/` + +The `test/e2e/` Ginkgo suite (added in PR #346) and this long-haul harness are **separate modules with intentionally different shapes**. They share a problem domain (exercising a DocumentDB cluster) but answer different questions: + +| | `test/e2e/` | `test/longhaul/` | +|---|---|---| +| Shape | Go test binary (Ginkgo specs) | Standalone long-running daemon | +| Lifetime | Minutes per spec | Days–weeks per run | +| Asserts | One behavior per spec, then exits | Continuous invariants over time | +| Failure mode | `t.Fail` per spec | Journal entry + alert + auto-restart | +| Cluster | Created + torn down per run | Long-lived dedicated AKS cluster | +| Operator API | Typed (`previewv1.DocumentDB` via controller-runtime) | Typed (`previewv1.DocumentDB` via controller-runtime + `test/shared/documentdb` helpers) | + +**Shared code today.** The harness consumes the `test/shared/` module (extracted in PR #401): + +- `test/shared/documentdb` — typed `DocumentDB` CR helpers (`Get`, `IsHealthy`, `PatchInstances`, `PatchSpec`). The monitor's `K8sClusterClient` uses these as the single source of truth for the readiness predicate so longhaul and e2e can't drift on what "healthy" means. +- `test/shared/mongo` — `NewFromURI` for the data-plane connection. + +**Future opportunities.** The e2e suite has additional helpers in `test/e2e/pkg/e2eutils/` that this harness will likely consume as it grows: + +- `e2eutils/mongo` — `BuildURI` (URL-escapes username/password), TLS-from-CA-bundle, `Handle` with port-forward + secret-backed credentials. +- `e2eutils/operatorhealth` — pod-ready / CRD-ready gating used during e2e setup. +- `e2eutils/clusterprobe` — CRD presence checks. + +--- + ## Future Scope - **Multi-region canary** — extend the Primary/Baseline pattern across regions via AKS Fleet to catch issues that only appear with cross-region replication / failover. diff --git a/test/longhaul/README.md b/test/longhaul/README.md index 61ea8478e..3e6b4ddb0 100644 --- a/test/longhaul/README.md +++ b/test/longhaul/README.md @@ -14,14 +14,16 @@ See the [design document](../../docs/designs/long-haul-test-design.md) for archi - `kubectl` configured to access the cluster - Go 1.26+ -> **HA topology required for upgrade tests.** The `upgrade-documentdb` operation -> auto-skips when `spec.instancesPerNode < 2` because a single-instance cluster -> has no standby to absorb writes during the rolling restart — the upgrade -> would produce real (true-positive) downtime that no operator change can -> prevent. Run with `instancesPerNode: 2` (or `3`) to exercise the HA upgrade -> path. The skip is "free": no cooldown is consumed, and the next 10s scheduler -> tick re-evaluates eligibility, so scaling up at any point makes the upgrade -> immediately schedulable. +> **HA topology required for upgrade / failover ops.** `upgrade-documentdb` and +> `kill-primary-pod` require `spec.instancesPerNode >= 2` (a standby to absorb +> writes). Behaviour when the precondition is unmet differs by mode: in +> **random** mode they auto-skip (no cooldown consumed; re-evaluated on the next +> tick), but in **sequence** mode there is *no* skip — the runner waits for the +> precondition up to `LONGHAUL_RECOVERY_TIMEOUT` and then fails the sequence. So +> run with `instancesPerNode: 2` (or `3`), or place a `scale-up` earlier in the +> sequence, to exercise them. (See the +> [design doc](../../docs/designs/long-haul-test-design.md#ha-preconditions) for +> the rationale.) ### Run the Config Unit Tests @@ -125,10 +127,14 @@ All configuration is via environment variables. | `LONGHAUL_DOCUMENTDB_URI` | Yes | — | Connection string to the DocumentDB gateway. | | `LONGHAUL_CLUSTER_NAME` | Yes | — | Name of the target DocumentDB cluster CR. | | `LONGHAUL_NAMESPACE` | No | `default` | Kubernetes namespace of the target cluster. | +| `LONGHAUL_OPERATOR_NAMESPACE` | No | `documentdb-operator` | Namespace of the DocumentDB operator Deployment (target of the `kill-operator-pod` chaos op). | | `LONGHAUL_MAX_DURATION` | No | `30m` | Max test duration. Use `0s` for run-until-failure. | | `LONGHAUL_NUM_WRITERS` | No | `5` | Number of concurrent writers. | -| `LONGHAUL_OP_COOLDOWN` | No | `5m` | Cooldown between management operations. | +| `LONGHAUL_OPERATION_MODE` | No | `random` | Operation runner: `random`, `sequence`, or `disabled`. | +| `LONGHAUL_OPERATION_SEQUENCE` | No | empty | Comma-separated stable operation names. Required and used only in `sequence` mode; rejected in `random`/`disabled` mode. Whitespace is trimmed, and duplicate or unknown names are rejected. | +| `LONGHAUL_OP_COOLDOWN` | No | `5m` | Minimum spacing between operations. Random mode only — `sequence` mode paces ops by the steady-state/recovery gates. | | `LONGHAUL_RECOVERY_TIMEOUT` | No | `5m` | Max wait for cluster recovery after an operation. | +| `LONGHAUL_STEADY_STATE_WAIT` | No | `60s` | Continuous healthy duration required by the steady-state gate. | | `LONGHAUL_MIN_INSTANCES` | No | `1` | Minimum `spec.instancesPerNode` for scale-down operations (CRD lower bound: 1). | | `LONGHAUL_MAX_INSTANCES` | No | `3` | Maximum `spec.instancesPerNode` for scale-up operations (CRD upper bound: 3). | | `LONGHAUL_REPORT_INTERVAL` | No | `1h` | How often to write checkpoint reports to ConfigMap. | @@ -138,94 +144,88 @@ All configuration is via environment variables. | `LONGHAUL_BACKUP_VERIFY_INTERVAL` | No | `5m` | How often the backup verifier samples the `ScheduledBackup` and its children. Lower it for short bounded runs (e.g. the smoke gate uses `30s`) so the periodic loop fires several times within the window. | | `LONGHAUL_RESET_DATA` | No | `false` | If `true`, drop the workload collection on startup. Off by default so a Deployment pod restart preserves durability history. | | `LONGHAUL_RETAIN_PER_WRITER` | No | `2000000` | Retention window: most-recent verified documents kept per writer before the pruner deletes older ones, bounding disk usage. `0` disables pruning (unbounded growth). | +| `LONGHAUL_PRUNE_INTERVAL` | No | `5m` | How often the pruner trims old documents. Lower it for short bounded runs (e.g. the smoke gate uses `30s`) so a prune fires within the window. | ### Data Protection (ScheduledBackup + retention) -When `LONGHAUL_BACKUP_ENABLED` is true, the driver ensures a `ScheduledBackup` -named `-longhaul` exists and matches the run's schedule/retention -(an existing CR is reconciled in place, never recreated, so backup history is -preserved across restarts and parameter changes) and runs a verifier -concurrently with the operation scheduler (backup is deliberately **not** -isolated from topology/chaos, per the design). - -The verifier only checks the properties a **multi-day** run can establish — -things unit and e2e tests cannot: - -- **Scheduling liveness** — `status.lastScheduledTime` keeps advancing; a stalled - scheduler (past `status.nextScheduledTime` + grace) raises a warning. -- **Completion** — child `Backup` CRs keep reaching `completed`; only terminal - `failed` backups are counted as failures. A `skipped` backup is an intentional - no-op (e.g. the operator declines to back up a non-primary/standby) and is - **not** counted as a failure. If backups keep being scheduled but stop - completing for 3 consecutive schedules (a dead completion path — every backup - failing or hanging), the run is a **FAIL**. A completed **or** skipped backup - resets this gap, so transient chaos-induced failures and normal standby / - failover intervals (where several consecutive schedules are skipped) are - tolerated. -- **Retention leak** — no completed backup outlives its retention window - (`stoppedAt + spec.retentionDays*24h` + grace). The window is taken from each - backup's **own** `spec.retentionDays` (stamped at creation), so the check - stays correct even if a later run uses a different retention. A lingering - backup is a **FAIL**: expired backups aren't garbage-collected and the - population (and its PVCs / VolumeSnapshots) grows unbounded. - -It deliberately does **not** re-verify the operator's retention *arithmetic* -(`expiredAt == stoppedAt + retentionDays*24h`) — that is a pure function already -covered by the operator's unit tests and needs no accumulation. The oracle here -is black-box: expired backups disappear. Because the minimum meaningful -retention is 1 day, the leak check only fires on multi-day runs — exactly the -accumulation window long-haul exists to cover. +When `LONGHAUL_BACKUP_ENABLED` is true, the driver maintains a `ScheduledBackup` +named `-longhaul` (matching the run's schedule/retention; an existing CR +is reconciled in place, never recreated, so backup history is preserved across +restarts and parameter changes) and runs a verifier alongside the workload. The +verifier FAILs the run if backups stop completing for 3 consecutive schedules, or +if an expired backup is not garbage-collected (a retention leak); `skipped` +backups (e.g. on a standby) are tolerated. See the +[design document](../../docs/designs/long-haul-test-design.md#backup-verification) +for exactly what it checks and why. > **RBAC.** The driver ServiceAccount needs `create`/`get`/`list`/`update` on > `scheduledbackups.documentdb.io` and `list` on `backups.documentdb.io`. These > verbs are granted by the `longhaul-test` Role in `deploy/rbac.yaml`; without > them the backup verifier logs an error and the rest of the run continues. +## Operations + +`random` mode preserves the production long-haul behavior: the scheduler picks +weighted eligible operations every 10 seconds, runs one disruptive operation at +a time, and applies the global cooldown. `sequence` mode runs each configured +operation exactly once and in order, stopping on the first execution, +precondition, recovery, or policy failure; a successful or failed sequence +emits its final report and exits immediately instead of waiting for +`LONGHAUL_MAX_DURATION`. `disabled` mode runs no operations. All modes keep the +continuous writer/verifier workload active. + +Current stable operation names: + +| Operation | Kind | Notes | +|-----------|------|-------| +| `scale-up` / `scale-down` | Topology | Adjusts `spec.instancesPerNode` within `[MIN, MAX]`. Only adds/removes a standby, so the primary write path is untouched (near-zero outage budget). | +| `upgrade-documentdb` | Topology | In-place version upgrade; requires HA (`instancesPerNode>=2`). | +| `kill-operator-pod` | Chaos | Deletes the operator pod; asserts the data plane keeps serving (near-zero outage budget). | +| `kill-primary-pod` | Chaos | Deletes the CNPG primary pod to exercise automatic failover; requires HA (`instancesPerNode>=2`). | + +Each operation has a write-outage budget: the scale ops and `kill-operator-pod` +keep writes up (near-zero budget), `kill-primary-pod` tolerates a single +failover, and `upgrade-documentdb` a cross-version switchover. See the +[design document](../../docs/designs/long-haul-test-design.md#outage-budgets) +for the exact budgets and rationale. + +Operation execution failures are terminal verdict failures in both `random` and +`sequence` modes. The `longhaul-report` ConfigMap exposes `operation-status`, +`operation-results` JSON (one result per sequenced operation), and, in random +mode, `operation-aggregates` JSON (passed/failed counts per operation), +alongside the `result` and `latest-report` fields. + +### RBAC for chaos operations + +`deploy/rbac.yaml` already grants everything the driver ServiceAccount needs, so +`kubectl apply -f deploy/rbac.yaml` is all that's required. The one non-obvious +part: the chaos operations delete pods, and `kill-operator-pod` deletes the +operator pod in the **operator's** namespace — not the driver's. That +cross-namespace access is granted by a separate Role/RoleBinding scoped to +`LONGHAUL_OPERATOR_NAMESPACE` (default `documentdb-operator`). If your operator +runs in a different namespace, set that variable and update the binding to +match. (`kill-primary-pod` stays within the cluster namespace and needs no extra +setup.) + ## CI Safety -The long haul test binary is deployed as a Kubernetes Deployment on a dedicated AKS -cluster. It does **not** run in any PR-gated CI workflow. Because a Deployment -auto-restarts crashed pods, the source of truth for "did the test pass?" is the -`longhaul-report` ConfigMap and the GitHub Actions annotations, not the pod -status. +The production long-haul binary runs as a Kubernetes Deployment on a dedicated +AKS cluster. A short PR smoke workflow (`.github/workflows/longhaul-smoke.yml`) +runs the same driver and manifests against kind in **sequence mode**, exercising +every operation once (scale up, scale down, upgrade DocumentDB, kill the operator +pod, kill the primary pod — including a real cross-version upgrade) and asserting +the `longhaul-report` ConfigMap reaches a `PASS` / `COMPLETE` verdict. Because a +Deployment auto-restarts exited pods, that report (and the GitHub Actions +annotations) — not the pod status — is the source of truth for "did the test +pass?". The config unit tests (`test/longhaul/config/`) run unconditionally and are included in normal CI test runs — they are fast (~0.002s) and require no cluster. ## Relationship to `test/e2e/` -The `test/e2e/` Ginkgo suite (added in PR #346) and this long haul harness are **separate -modules with intentionally different shapes**. They share a problem domain (exercising a -DocumentDB cluster) but answer different questions: - -| | `test/e2e/` | `test/longhaul/` | -|---|---|---| -| Shape | Go test binary (Ginkgo specs) | Standalone long-running daemon | -| Lifetime | Minutes per spec | Days–weeks per run | -| Asserts | One behavior per spec, then exits | Continuous invariants over time | -| Failure mode | `t.Fail` per spec | Journal entry + alert + auto-restart | -| Cluster | Created + torn down per run | Long-lived dedicated AKS cluster | -| Operator API | Typed (`previewv1.DocumentDB` via controller-runtime) | Typed (`previewv1.DocumentDB` via controller-runtime + `test/shared/documentdb` helpers) | - -### Code that is shared today - -The harness consumes the `test/shared/` module (extracted in PR #401): - -- `test/shared/documentdb` — typed `DocumentDB` CR helpers (`Get`, `IsHealthy`, - `PatchInstances`, `PatchSpec`). The monitor's `K8sClusterClient` uses these - as the single source of truth for the readiness predicate so longhaul and - e2e can't drift on what "healthy" means. -- `test/shared/mongo` — `NewFromURI` for the data-plane connection. - -### Future opportunities - -The e2e suite has additional helpers in `test/e2e/pkg/e2eutils/` that this -harness will likely consume as it grows: - -- `e2eutils/mongo` — `BuildURI` (URL-escapes username/password), TLS-from-CA-bundle, - `Handle` with port-forward + secret-backed credentials. The long haul driver - currently takes a raw `LONGHAUL_DOCUMENTDB_URI` string; when it moves to per-secret - credentials or in-cluster TLS, these helpers become directly applicable. -- `e2eutils/operatorhealth` — pod-ready / CRD-ready gating used during e2e setup. - The monitor's `isPodReady` could delegate to this. -- `e2eutils/clusterprobe` — CRD presence checks. +The `test/e2e/` Ginkgo suite and this long-haul harness are **separate modules +with intentionally different shapes** that share the `test/shared/` helpers +(`test/shared/documentdb` CR helpers and `test/shared/mongo`). See the +[design document](../../docs/designs/long-haul-test-design.md#relationship-to-teste2e) +for the full comparison, the shared code today, and future opportunities. diff --git a/test/longhaul/cmd/longhaul/main.go b/test/longhaul/cmd/longhaul/main.go index 47733402c..4345c8474 100644 --- a/test/longhaul/cmd/longhaul/main.go +++ b/test/longhaul/cmd/longhaul/main.go @@ -141,22 +141,22 @@ func run(cfg config.Config) int { // below the verifier's confirmed floor, so it never affects the durability // verdict. Disabled when RetainPerWriter == 0. if cfg.RetainPerWriter > 0 { - workload.StartPruner(ctx, db.Collection(workload.CollectionName), writers, verifier, cfg.RetainPerWriter, metrics, j) + workload.StartPruner(ctx, db.Collection(workload.CollectionName), writers, verifier, cfg.RetainPerWriter, cfg.PruneInterval, metrics, j) j.Info("main", "retention pruner started") } else { j.Info("main", "retention pruning disabled (LONGHAUL_RETAIN_PER_WRITER=0)") } - // Configure operations. - ops := []operations.Operation{ - operations.NewScaleUp(clusterClient, healthMon, cfg.MaxInstances, cfg.RecoveryTimeout), - operations.NewScaleDown(clusterClient, healthMon, cfg.MinInstances, cfg.RecoveryTimeout), - operations.NewUpgradeDocumentDB(clusterClient, k8sClientset, healthMon, j, cfg.Namespace, cfg.RecoveryTimeout), + // Build the operation registry once, then select the configured runner. + registry, err := operations.NewDefaultRegistry(cfg, clusterClient, k8sClientset, healthMon, j) + if err != nil { + log.Fatalf("failed to build operation registry: %v", err) } - - // Start operation scheduler. - scheduler := operations.NewScheduler(ops, healthMon, j, cfg.OpCooldown) - go scheduler.Run(ctx) + opRunner, err := newOperationRunner(cfg, registry, healthMon, j) + if err != nil { + log.Fatalf("failed to configure operation runner: %v", err) + } + go opRunner.Run(ctx) // Start data-protection verifier (ScheduledBackup + retention). Runs // concurrently with the scheduler by design — backup is deliberately not @@ -184,32 +184,54 @@ func run(cfg config.Config) int { go runMetricsSampling(ctx, clusterClient, leakDetector, j) // Start periodic checkpoint reporter. - summaryFunc := func() report.Summary { - return buildSummary(metrics, backupMetrics, leakDetector, scheduler, j) + summaryFunc := func(final bool) report.Summary { + return buildSummary(metrics, backupMetrics, leakDetector, opRunner, j, final) } reporter := report.NewCheckpointReporter(k8sClientset, cfg.Namespace, cfg.ReportInterval, summaryFunc) go reporter.Run(ctx) j.Info("main", "all components started, entering main loop") - // Main loop: wait for context expiry. - <-ctx.Done() - j.Info("main", fmt.Sprintf("test ending: %v", ctx.Err())) + // Sequence mode is completion-driven: it exits as soon as its operations + // have finished (or a failure occurs); MaxDuration is only its watchdog. + // Random and disabled modes are duration-driven. + completionDriven := cfg.OperationMode == config.OperationModeSequence + if completionDriven { + select { + case <-opRunner.Done(): + j.Info("main", "operations finished") + case <-ctx.Done(): + j.Info("main", fmt.Sprintf("operations watchdog fired: %v", ctx.Err())) + if sr, ok := opRunner.(*operations.SequenceRunner); ok { + sr.MarkIncomplete( + fmt.Sprintf("operation sequence incomplete: watchdog fired: %v", ctx.Err()), + ) + } + <-opRunner.Done() + } + } else { + // Random/disabled modes are duration-driven, but a terminal operation + // failure also halts the runner early (Scheduler.Run returns and closes + // Done). React to that too so the FAIL verdict is emitted promptly + // instead of waiting out MaxDuration, which is unbounded in production. + select { + case <-ctx.Done(): + j.Info("main", fmt.Sprintf("test ending: %v", ctx.Err())) + case <-opRunner.Done(): + j.Info("main", "operations halted early (failure)") + } + cancel() + <-opRunner.Done() + } + cancel() // Allow goroutines to flush. time.Sleep(500 * time.Millisecond) - // Generate final report. Persist to the report ConfigMap synchronously - // here (before os.Exit) so the authoritative verdict reaches the source - // of truth that operators consult — the Run() goroutine cannot do this - // reliably because os.Exit can kill it mid-Update. - summary := buildSummary(metrics, backupMetrics, leakDetector, scheduler, j) - markdown := report.GenerateMarkdown(summary) - fmt.Println("\n" + markdown) - reporter.EmitFinal() - - // Emit final GitHub Actions annotation. - report.EmitAnnotation(summary) + // Emit exactly one terminal report synchronously before os.Exit. EmitFinal + // prints the markdown, emits the GitHub Actions annotation, and persists the + // authoritative verdict to the report ConfigMap. + summary := reporter.EmitFinal() if summary.Result == report.ResultFail { log.Printf("TEST FAILED: %s", summary.FailReason) @@ -220,36 +242,77 @@ func run(cfg config.Config) int { return 0 } +func newOperationRunner( + cfg config.Config, + registry *operations.Registry, + health *monitor.HealthMonitor, + j *journal.Journal, +) (operations.Runner, error) { + switch cfg.OperationMode { + case config.OperationModeRandom: + return operations.NewScheduler(registry.All(), health, j, cfg.OpCooldown), nil + case config.OperationModeSequence: + ops, err := registry.Resolve(cfg.OperationSequence) + if err != nil { + return nil, err + } + return operations.NewSequenceRunner(ops, health, j, cfg.RecoveryTimeout), nil + case config.OperationModeDisabled: + return operations.NewDisabledRunner(), nil + default: + return nil, fmt.Errorf("unsupported operation mode %q", cfg.OperationMode) + } +} + // buildSummary constructs a report.Summary from current state. -func buildSummary(metrics *workload.Metrics, backupMetrics *backup.Metrics, leakDetector *monitor.LeakDetector, scheduler *operations.Scheduler, j *journal.Journal) report.Summary { +func buildSummary( + metrics *workload.Metrics, + backupMetrics *backup.Metrics, + leakDetector *monitor.LeakDetector, + opRunner operations.Runner, + j *journal.Journal, + final bool, +) report.Summary { snap := metrics.Snapshot() backupSnap := backupMetrics.Snapshot() leakAnalysis := leakDetector.Analyze() + operationRun := opRunner.Snapshot() result := report.ResultPass failReason := "" - appendReason := func(msg string) { + if snap.HasDataLoss() { + result = report.ResultFail + failReason = appendFailReason(failReason, fmt.Sprintf("data loss: %d gaps, %d checksum errors", + snap.GapsDetected, snap.ChecksumErrors)) + } + if operationRun.HasFailure() { result = report.ResultFail - if failReason != "" { - failReason += "; " + failReason = appendFailReason(failReason, operationRun.FailureReason) + if operationRun.FailureReason == "" { + failReason = appendFailReason(failReason, "operation execution failed") } - failReason += msg } - - if snap.HasDataLoss() { - appendReason(fmt.Sprintf("data loss: %d gaps, %d checksum errors", - snap.GapsDetected, snap.ChecksumErrors)) + if final && + operationRun.Mode == config.OperationModeSequence && + operationRun.Status != operations.RunStatusComplete && + !operationRun.HasFailure() { + result = report.ResultFail + failReason = appendFailReason(failReason, + fmt.Sprintf("operation sequence incomplete (status %s)", operationRun.Status)) } if j.HasPolicyViolation() { - appendReason("outage policy violated") + result = report.ResultFail + failReason = appendFailReason(failReason, "outage policy violated") } if backupSnap.HasRetentionLeak() { - appendReason(fmt.Sprintf("backup retention leak: %d expired backups not collected", + result = report.ResultFail + failReason = appendFailReason(failReason, fmt.Sprintf("backup retention leak: %d expired backups not collected", backupSnap.RetentionLeaks)) } if backupSnap.HasCompletionStall() { - appendReason(fmt.Sprintf("backup completion stalled: %d backups scheduled with no completion", + result = report.ResultFail + failReason = appendFailReason(failReason, fmt.Sprintf("backup completion stalled: %d backups scheduled with no completion", backupSnap.MaxScheduledWithoutCompletion)) } @@ -259,13 +322,27 @@ func buildSummary(metrics *workload.Metrics, backupMetrics *backup.Metrics, leak Metrics: snap, Backup: backupSnap, LeakAnalysis: leakAnalysis, - OpsExecuted: scheduler.OpsExecuted(), + OpsExecuted: operationRun.OpsExecuted(), + OperationRun: operationRun, Windows: j.DisruptionWindows(), Events: j.Events(), FailReason: failReason, } } +func appendFailReason(existing, reason string) string { + if reason == "" { + return existing + } + if existing == "" { + return reason + } + if existing == reason { + return existing + } + return existing + "; " + reason +} + // runMetricsSampling periodically collects pod resource metrics and feeds the leak detector. func runMetricsSampling(ctx context.Context, client *monitor.K8sClusterClient, ld *monitor.LeakDetector, j *journal.Journal) { if !client.MetricsAvailable() { diff --git a/test/longhaul/cmd/longhaul/main_test.go b/test/longhaul/cmd/longhaul/main_test.go new file mode 100644 index 000000000..921efacb2 --- /dev/null +++ b/test/longhaul/cmd/longhaul/main_test.go @@ -0,0 +1,89 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package main + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/documentdb/documentdb-operator/test/longhaul/backup" + "github.com/documentdb/documentdb-operator/test/longhaul/config" + "github.com/documentdb/documentdb-operator/test/longhaul/journal" + "github.com/documentdb/documentdb-operator/test/longhaul/monitor" + "github.com/documentdb/documentdb-operator/test/longhaul/operations" + "github.com/documentdb/documentdb-operator/test/longhaul/report" + "github.com/documentdb/documentdb-operator/test/longhaul/workload" +) + +type snapshotRunner struct { + snapshot operations.RunSnapshot + done chan struct{} +} + +func (r *snapshotRunner) Run(context.Context) {} +func (r *snapshotRunner) Snapshot() operations.RunSnapshot { return r.snapshot } +func (r *snapshotRunner) Done() <-chan struct{} { return r.done } + +func summaryFor(snapshot operations.RunSnapshot, final bool) report.Summary { + j := journal.New() + return buildSummary( + workload.NewMetrics(), + backup.NewMetrics(), + monitor.NewLeakDetector(j, 10, 10), + &snapshotRunner{snapshot: snapshot, done: make(chan struct{})}, + j, + final, + ) +} + +var _ = Describe("buildSummary operation verdicts", func() { + It("fails random mode when any execution failed", func() { + summary := summaryFor(operations.RunSnapshot{ + Mode: config.OperationModeRandom, + Status: operations.RunStatusFailed, + FailureReason: "operation scale-up execute failed: boom", + Aggregates: []operations.OperationAggregate{ + {Name: "scale-up", Failed: 1}, + }, + }, true) + + Expect(summary.Result).To(Equal(report.ResultFail)) + Expect(summary.FailReason).To(ContainSubstring("execute failed")) + }) + + It("allows an in-progress sequence at a checkpoint", func() { + summary := summaryFor(operations.RunSnapshot{ + Mode: config.OperationModeSequence, + Status: operations.RunStatusRunning, + Results: []operations.OperationResult{ + {Name: "kill-operator-pod", Status: operations.OperationPending}, + }, + }, false) + Expect(summary.Result).To(Equal(report.ResultPass)) + }) + + It("fails an incomplete requested sequence at final shutdown", func() { + summary := summaryFor(operations.RunSnapshot{ + Mode: config.OperationModeSequence, + Status: operations.RunStatusRunning, + Results: []operations.OperationResult{ + {Name: "kill-operator-pod", Status: operations.OperationRunning}, + {Name: "kill-primary-pod", Status: operations.OperationPending}, + }, + }, true) + + Expect(summary.Result).To(Equal(report.ResultFail)) + Expect(summary.FailReason).To(ContainSubstring("operation sequence incomplete")) + }) + + It("does not impose an operation completion requirement in disabled mode", func() { + summary := summaryFor(operations.RunSnapshot{ + Mode: config.OperationModeDisabled, + Status: operations.RunStatusDisabled, + }, true) + Expect(summary.Result).To(Equal(report.ResultPass)) + }) +}) diff --git a/test/longhaul/cmd/longhaul/suite_test.go b/test/longhaul/cmd/longhaul/suite_test.go new file mode 100644 index 000000000..27e6e2bdf --- /dev/null +++ b/test/longhaul/cmd/longhaul/suite_test.go @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package main + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestLonghaulMain(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Long Haul Main Suite") +} diff --git a/test/longhaul/config/config.go b/test/longhaul/config/config.go index d2e640fd8..d300b675a 100644 --- a/test/longhaul/config/config.go +++ b/test/longhaul/config/config.go @@ -18,12 +18,18 @@ const ( EnvNamespace = "LONGHAUL_NAMESPACE" EnvClusterName = "LONGHAUL_CLUSTER_NAME" + // EnvOperatorNamespace is the namespace where the DocumentDB operator + // Deployment runs (target of the kill-operator-pod chaos op). + EnvOperatorNamespace = "LONGHAUL_OPERATOR_NAMESPACE" + // Workload and operation tuning. EnvDocumentDBURI = "LONGHAUL_DOCUMENTDB_URI" EnvNumWriters = "LONGHAUL_NUM_WRITERS" EnvOpCooldown = "LONGHAUL_OP_COOLDOWN" EnvRecoveryTimeout = "LONGHAUL_RECOVERY_TIMEOUT" EnvSteadyStateWait = "LONGHAUL_STEADY_STATE_WAIT" + EnvOperationMode = "LONGHAUL_OPERATION_MODE" + EnvOperationSeq = "LONGHAUL_OPERATION_SEQUENCE" // Scale operation bounds. The DocumentDB CRD hard-caps spec.nodeCount=1, // so the scale dimension actually exercised is spec.instancesPerNode (1-3). EnvMinInstances = "LONGHAUL_MIN_INSTANCES" @@ -44,6 +50,9 @@ const ( // Data retention. Bounds the workload collection so an unbounded write // test does not eventually exhaust the PVC. EnvRetainPerWriter = "LONGHAUL_RETAIN_PER_WRITER" + + // EnvPruneInterval overrides how often the pruner trims old documents. + EnvPruneInterval = "LONGHAUL_PRUNE_INTERVAL" ) // DefaultRetainPerWriter is the default number of most-recent documents kept @@ -51,6 +60,15 @@ const ( // roughly 55 hours of history per writer while bounding steady-state disk use. const DefaultRetainPerWriter = 2_000_000 +// OperationMode controls how disruptive operations are run. +type OperationMode string + +const ( + OperationModeRandom OperationMode = "random" + OperationModeSequence OperationMode = "sequence" + OperationModeDisabled OperationMode = "disabled" +) + // Config holds all configuration for a long haul test run. type Config struct { // MaxDuration is the maximum test duration. Zero means run until failure. @@ -62,6 +80,10 @@ type Config struct { // ClusterName is the name of the target DocumentDB cluster CR. ClusterName string + // OperatorNamespace is the namespace of the DocumentDB operator Deployment, + // targeted by the kill-operator-pod chaos operation. + OperatorNamespace string + // DocumentDBURI is the DocumentDB connection string for data-plane workload. DocumentDBURI string @@ -77,6 +99,12 @@ type Config struct { // SteadyStateWait is how long the cluster must be healthy before an operation fires. SteadyStateWait time.Duration + // OperationMode selects weighted-random, deterministic sequence, or no operations. + OperationMode OperationMode + + // OperationSequence is the ordered list used only in sequence mode. + OperationSequence []string + // MinInstances is the minimum spec.instancesPerNode for scale-down. // CRD lower bound is 1. MinInstances int @@ -113,22 +141,29 @@ type Config struct { // Older, already-verified documents are pruned to bound disk usage. Zero // disables pruning (unbounded growth — the pre-retention behavior). RetainPerWriter int64 + + // PruneInterval is how often the pruner trims old documents. The 5m default + // suits a multi-day run; short bounded runs (e.g. the smoke gate) lower it + // so the pruner fires within the window. + PruneInterval time.Duration } // DefaultConfig returns a Config with safe defaults for local development. func DefaultConfig() Config { return Config{ - MaxDuration: 30 * time.Minute, - Namespace: "default", - ClusterName: "", - DocumentDBURI: "", - NumWriters: 5, - OpCooldown: 5 * time.Minute, - RecoveryTimeout: 5 * time.Minute, - SteadyStateWait: 60 * time.Second, - MinInstances: 1, - MaxInstances: 3, - ReportInterval: 1 * time.Hour, + MaxDuration: 30 * time.Minute, + Namespace: "default", + ClusterName: "", + OperatorNamespace: "documentdb-operator", + DocumentDBURI: "", + NumWriters: 5, + OpCooldown: 5 * time.Minute, + RecoveryTimeout: 5 * time.Minute, + SteadyStateWait: 60 * time.Second, + OperationMode: OperationModeRandom, + MinInstances: 1, + MaxInstances: 3, + ReportInterval: 1 * time.Hour, BackupEnabled: true, BackupSchedule: "0 */6 * * *", @@ -136,6 +171,7 @@ func DefaultConfig() Config { BackupVerifyInterval: 5 * time.Minute, RetainPerWriter: DefaultRetainPerWriter, + PruneInterval: 5 * time.Minute, } } @@ -160,6 +196,10 @@ func LoadFromEnv() (Config, error) { cfg.ClusterName = v } + if v := os.Getenv(EnvOperatorNamespace); v != "" { + cfg.OperatorNamespace = v + } + if v := os.Getenv(EnvDocumentDBURI); v != "" { cfg.DocumentDBURI = v } @@ -196,6 +236,18 @@ func LoadFromEnv() (Config, error) { cfg.SteadyStateWait = d } + if v := strings.TrimSpace(os.Getenv(EnvOperationMode)); v != "" { + cfg.OperationMode = OperationMode(strings.ToLower(v)) + } + + if v := strings.TrimSpace(os.Getenv(EnvOperationSeq)); v != "" { + sequence, err := parseOperationSequence(v) + if err != nil { + return cfg, fmt.Errorf("invalid %s=%q: %w", EnvOperationSeq, v, err) + } + cfg.OperationSequence = sequence + } + if v := os.Getenv(EnvMinInstances); v != "" { n, err := strconv.Atoi(v) if err != nil { @@ -256,6 +308,14 @@ func LoadFromEnv() (Config, error) { cfg.RetainPerWriter = n } + if v := os.Getenv(EnvPruneInterval); v != "" { + d, err := time.ParseDuration(v) + if err != nil { + return cfg, fmt.Errorf("invalid %s=%q: %w", EnvPruneInterval, v, err) + } + cfg.PruneInterval = d + } + return cfg, nil } @@ -270,6 +330,9 @@ func (c *Config) Validate() error { if c.ClusterName == "" { return fmt.Errorf("cluster name must not be empty") } + if c.OperatorNamespace == "" { + return fmt.Errorf("operator namespace must not be empty") + } if c.NumWriters < 1 { return fmt.Errorf("num writers must be at least 1, got %d", c.NumWriters) } @@ -279,6 +342,26 @@ func (c *Config) Validate() error { if c.RecoveryTimeout <= 0 { return fmt.Errorf("recovery timeout must be positive, got %s", c.RecoveryTimeout) } + switch c.OperationMode { + case OperationModeRandom, OperationModeDisabled: + if len(c.OperationSequence) > 0 { + return fmt.Errorf("operation sequence must be empty when operation mode is %q", c.OperationMode) + } + case OperationModeSequence: + if len(c.OperationSequence) == 0 { + return fmt.Errorf("operation sequence must not be empty when operation mode is %q", c.OperationMode) + } + seen := make(map[string]struct{}, len(c.OperationSequence)) + for _, name := range c.OperationSequence { + if _, ok := seen[name]; ok { + return fmt.Errorf("operation sequence contains duplicate name %q", name) + } + seen[name] = struct{}{} + } + default: + return fmt.Errorf("operation mode must be one of %q, %q, or %q, got %q", + OperationModeRandom, OperationModeSequence, OperationModeDisabled, c.OperationMode) + } if c.MinInstances < 1 { return fmt.Errorf("min instances must be at least 1, got %d", c.MinInstances) } @@ -305,6 +388,19 @@ func (c *Config) Validate() error { return nil } +func parseOperationSequence(value string) ([]string, error) { + parts := strings.Split(value, ",") + sequence := make([]string, 0, len(parts)) + for _, part := range parts { + name := strings.TrimSpace(part) + if name == "" { + return nil, fmt.Errorf("operation names must not be empty") + } + sequence = append(sequence, name) + } + return sequence, nil +} + // IsEnabled returns true if the long haul test is explicitly enabled // via the LONGHAUL_ENABLED environment variable. func IsEnabled() bool { diff --git a/test/longhaul/config/config_test.go b/test/longhaul/config/config_test.go index 69054b73f..f8d9bd551 100644 --- a/test/longhaul/config/config_test.go +++ b/test/longhaul/config/config_test.go @@ -17,10 +17,13 @@ var _ = Describe("Config", func() { Expect(cfg.MaxDuration).To(Equal(30 * time.Minute)) Expect(cfg.Namespace).To(Equal("default")) Expect(cfg.ClusterName).To(BeEmpty()) + Expect(cfg.OperatorNamespace).To(Equal("documentdb-operator")) Expect(cfg.NumWriters).To(Equal(5)) Expect(cfg.OpCooldown).To(Equal(5 * time.Minute)) Expect(cfg.RecoveryTimeout).To(Equal(5 * time.Minute)) Expect(cfg.SteadyStateWait).To(Equal(60 * time.Second)) + Expect(cfg.OperationMode).To(Equal(OperationModeRandom)) + Expect(cfg.OperationSequence).To(BeEmpty()) Expect(cfg.MinInstances).To(Equal(1)) Expect(cfg.MaxInstances).To(Equal(3)) Expect(cfg.RetainPerWriter).To(Equal(int64(DefaultRetainPerWriter))) @@ -33,8 +36,10 @@ var _ = Describe("Config", func() { BeforeEach(func() { for _, k := range []string{ EnvEnabled, EnvMaxDuration, EnvNamespace, EnvClusterName, + EnvOperatorNamespace, EnvDocumentDBURI, EnvNumWriters, EnvOpCooldown, EnvRecoveryTimeout, EnvSteadyStateWait, + EnvOperationMode, EnvOperationSeq, EnvMinInstances, EnvMaxInstances, EnvReportInterval, EnvBackupEnabled, EnvBackupSchedule, EnvBackupRetentionDays, EnvBackupVerifyInterval, @@ -74,6 +79,13 @@ var _ = Describe("Config", func() { Expect(cfg.ClusterName).To(Equal("my-cluster")) }) + It("parses OperatorNamespace from env", func() { + GinkgoT().Setenv(EnvOperatorNamespace, "custom-operator-ns") + cfg, err := LoadFromEnv() + Expect(err).NotTo(HaveOccurred()) + Expect(cfg.OperatorNamespace).To(Equal("custom-operator-ns")) + }) + It("returns error for invalid MaxDuration", func() { GinkgoT().Setenv(EnvMaxDuration, "not-a-duration") _, err := LoadFromEnv() @@ -109,51 +121,25 @@ var _ = Describe("Config", func() { Expect(cfg.DocumentDBURI).To(Equal("mongodb://localhost:27017")) }) - It("parses the backup env knobs", func() { - GinkgoT().Setenv(EnvBackupEnabled, "true") - GinkgoT().Setenv(EnvBackupSchedule, "0 */6 * * *") - GinkgoT().Setenv(EnvBackupRetentionDays, "7") - GinkgoT().Setenv(EnvBackupVerifyInterval, "30s") - cfg, err := LoadFromEnv() - Expect(err).NotTo(HaveOccurred()) - Expect(cfg.BackupEnabled).To(BeTrue()) - Expect(cfg.BackupSchedule).To(Equal("0 */6 * * *")) - Expect(cfg.BackupRetentionDays).To(Equal(7)) - Expect(cfg.BackupVerifyInterval).To(Equal(30 * time.Second)) - }) - - It("returns error for invalid BackupRetentionDays", func() { - GinkgoT().Setenv(EnvBackupRetentionDays, "abc") - _, err := LoadFromEnv() - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring(EnvBackupRetentionDays)) - }) - - It("returns error for invalid BackupVerifyInterval", func() { - GinkgoT().Setenv(EnvBackupVerifyInterval, "not-a-duration") + It("returns error for invalid RetainPerWriter", func() { + GinkgoT().Setenv(EnvRetainPerWriter, "not-a-number") _, err := LoadFromEnv() - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring(EnvBackupVerifyInterval)) - }) - - It("parses RetainPerWriter from env", func() { - GinkgoT().Setenv(EnvRetainPerWriter, "500000") - cfg, err := LoadFromEnv() - Expect(err).NotTo(HaveOccurred()) - Expect(cfg.RetainPerWriter).To(Equal(int64(500_000))) + Expect(err).To(MatchError(ContainSubstring(EnvRetainPerWriter))) }) - It("parses RetainPerWriter=0 to disable pruning", func() { - GinkgoT().Setenv(EnvRetainPerWriter, "0") + It("normalizes operation mode and trims sequence names", func() { + GinkgoT().Setenv(EnvOperationMode, " Sequence ") + GinkgoT().Setenv(EnvOperationSeq, " kill-operator-pod, kill-primary-pod ") cfg, err := LoadFromEnv() Expect(err).NotTo(HaveOccurred()) - Expect(cfg.RetainPerWriter).To(BeZero()) + Expect(cfg.OperationMode).To(Equal(OperationModeSequence)) + Expect(cfg.OperationSequence).To(Equal([]string{"kill-operator-pod", "kill-primary-pod"})) }) - It("returns error for invalid RetainPerWriter", func() { - GinkgoT().Setenv(EnvRetainPerWriter, "not-a-number") + It("rejects empty names in a non-empty sequence", func() { + GinkgoT().Setenv(EnvOperationSeq, "scale-up, ,scale-down") _, err := LoadFromEnv() - Expect(err).To(MatchError(ContainSubstring(EnvRetainPerWriter))) + Expect(err).To(MatchError(ContainSubstring("operation names must not be empty"))) }) }) @@ -176,6 +162,13 @@ var _ = Describe("Config", func() { Expect(cfg.Validate()).To(MatchError(ContainSubstring("cluster name"))) }) + It("fails when OperatorNamespace is empty", func() { + cfg := DefaultConfig() + cfg.ClusterName = "test" + cfg.OperatorNamespace = "" + Expect(cfg.Validate()).To(MatchError(ContainSubstring("operator namespace"))) + }) + It("fails when MaxDuration is negative", func() { cfg := DefaultConfig() cfg.ClusterName = "test" @@ -197,6 +190,48 @@ var _ = Describe("Config", func() { Expect(cfg.Validate()).To(MatchError(ContainSubstring("recovery timeout"))) }) + It("fails for an unknown operation mode", func() { + cfg := DefaultConfig() + cfg.ClusterName = "test" + cfg.OperationMode = "roulette" + Expect(cfg.Validate()).To(MatchError(ContainSubstring("operation mode must be one of"))) + }) + + It("requires a non-empty sequence in sequence mode", func() { + cfg := DefaultConfig() + cfg.ClusterName = "test" + cfg.OperationMode = OperationModeSequence + Expect(cfg.Validate()).To(MatchError(ContainSubstring("operation sequence must not be empty"))) + }) + + It("rejects duplicate sequence names", func() { + cfg := DefaultConfig() + cfg.ClusterName = "test" + cfg.OperationMode = OperationModeSequence + cfg.OperationSequence = []string{"scale-up", "scale-up"} + Expect(cfg.Validate()).To(MatchError(ContainSubstring(`duplicate name "scale-up"`))) + }) + + DescribeTable("rejects a sequence outside sequence mode", + func(mode OperationMode) { + cfg := DefaultConfig() + cfg.ClusterName = "test" + cfg.OperationMode = mode + cfg.OperationSequence = []string{"scale-up"} + Expect(cfg.Validate()).To(MatchError(ContainSubstring("operation sequence must be empty"))) + }, + Entry("random", OperationModeRandom), + Entry("disabled", OperationModeDisabled), + ) + + It("accepts a valid sequence configuration", func() { + cfg := DefaultConfig() + cfg.ClusterName = "test" + cfg.OperationMode = OperationModeSequence + cfg.OperationSequence = []string{"kill-operator-pod", "kill-primary-pod"} + Expect(cfg.Validate()).To(Succeed()) + }) + It("fails when MaxInstances < MinInstances", func() { cfg := DefaultConfig() cfg.ClusterName = "test" diff --git a/test/longhaul/deploy/deployment.yaml b/test/longhaul/deploy/deployment.yaml index 835e577fc..fecc7cdf2 100644 --- a/test/longhaul/deploy/deployment.yaml +++ b/test/longhaul/deploy/deployment.yaml @@ -15,10 +15,10 @@ # driver pods concurrently against the same DocumentDB cluster / # workload collection). # -# Failure semantics: on critical failure (data loss, policy violation) -# the driver exits non-zero. The Deployment auto-restarts the pod, which -# gives MTBF data; the alert workflow polls the report ConfigMap and -# pages on incident-count thresholds. +# Failure semantics: on critical failure (data loss, operation failure, or +# policy violation) the driver exits non-zero. The Deployment auto-restarts +# the pod, which gives MTBF data; the alert workflow polls the report ConfigMap +# and pages on incident-count thresholds. # # Image refs are templated; the longhaul-deploy workflow substitutes: # __OWNER__ -> lowercased ${{ github.repository_owner }} @@ -42,6 +42,10 @@ data: # Writer/verifier counts. LONGHAUL_NUM_WRITERS: "5" # Operation scheduling. + # random preserves the production long-haul behavior. sequence executes + # LONGHAUL_OPERATION_SEQUENCE exactly once in order; disabled runs no ops. + LONGHAUL_OPERATION_MODE: "random" + LONGHAUL_OPERATION_SEQUENCE: "" LONGHAUL_OP_COOLDOWN: "10m" LONGHAUL_RECOVERY_TIMEOUT: "5m" # How long the cluster must be observed healthy before the next diff --git a/test/longhaul/deploy/rbac.yaml b/test/longhaul/deploy/rbac.yaml index 0f4395546..6029fbea0 100644 --- a/test/longhaul/deploy/rbac.yaml +++ b/test/longhaul/deploy/rbac.yaml @@ -21,14 +21,19 @@ metadata: app.kubernetes.io/name: longhaul-test app.kubernetes.io/component: testing rules: - # Read pod status for health monitoring. + # Read pod status for health monitoring; delete pods for the kill-primary-pod + # chaos op (deletes the CNPG primary to exercise automatic failover). - apiGroups: [""] resources: ["pods"] - verbs: ["get", "list", "watch"] + verbs: ["get", "list", "watch", "delete"] # Read and patch DocumentDB CRs for health check and scale operations. - apiGroups: ["documentdb.io"] resources: ["dbs"] verbs: ["get", "list", "patch"] + # Read the CNPG Cluster to resolve the current primary pod (kill-primary-pod). + - apiGroups: ["postgresql.cnpg.io"] + resources: ["clusters"] + verbs: ["get", "list"] # Manage ScheduledBackups and read their child Backups for the data-protection # verifier (ensure a ScheduledBackup, then watch child Backup CRs). - apiGroups: ["documentdb.io"] @@ -59,6 +64,49 @@ subjects: name: longhaul-test namespace: documentdb-test-ns --- +# Role in the operator namespace for the kill-operator-pod chaos op: read the +# operator Deployment (to build its pod selector and check availability) and +# delete its pod. Namespaced separately because the operator runs outside the +# driver's own namespace (LONGHAUL_OPERATOR_NAMESPACE, default documentdb-operator). +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: longhaul-test-operator + # NOTE: keep in sync with LONGHAUL_OPERATOR_NAMESPACE. If the driver overrides + # that env var, this namespace must be edited to match or kill-operator-pod + # fails with RBAC errors. + namespace: documentdb-operator + labels: + app.kubernetes.io/name: longhaul-test + app.kubernetes.io/component: testing +rules: + - apiGroups: ["apps"] + resources: ["deployments"] + verbs: ["get"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "delete"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: longhaul-test-operator + # NOTE: must match the Role namespace above (and LONGHAUL_OPERATOR_NAMESPACE). + # Update in lockstep or the binding won't grant permissions in the right + # namespace. + namespace: documentdb-operator + labels: + app.kubernetes.io/name: longhaul-test + app.kubernetes.io/component: testing +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: longhaul-test-operator +subjects: + - kind: ServiceAccount + name: longhaul-test + namespace: documentdb-test-ns +--- # ClusterRole for metrics-server access (metrics API is cluster-scoped). apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole diff --git a/test/longhaul/go.mod b/test/longhaul/go.mod index 5d3fa4687..ebde56ba3 100644 --- a/test/longhaul/go.mod +++ b/test/longhaul/go.mod @@ -3,6 +3,7 @@ module github.com/documentdb/documentdb-operator/test/longhaul go 1.26.6 require ( + github.com/cloudnative-pg/cloudnative-pg v1.29.2 github.com/documentdb/documentdb-operator v0.0.0-00010101000000-000000000000 github.com/documentdb/documentdb-operator/test/shared v0.0.0-00010101000000-000000000000 github.com/onsi/ginkgo/v2 v2.32.0 @@ -25,7 +26,6 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudnative-pg/barman-cloud v0.5.1 // indirect - github.com/cloudnative-pg/cloudnative-pg v1.29.2 // indirect github.com/cloudnative-pg/cnpg-i v0.5.0 // indirect github.com/cloudnative-pg/machinery v0.5.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect diff --git a/test/longhaul/journal/journal.go b/test/longhaul/journal/journal.go index bfddebc76..fdb698d4e 100644 --- a/test/longhaul/journal/journal.go +++ b/test/longhaul/journal/journal.go @@ -24,8 +24,9 @@ const ( // trim cost is amortized over many appends (one copy every trimHeadroom // events), not paid on every append once we hit the cap. const ( - maxEvents = 10000 - trimHeadroom = 1000 + maxEvents = 10000 + trimHeadroom = 1000 + maxDisruptionWindows = 1000 ) // Event represents a single journal entry. @@ -108,7 +109,7 @@ func (j *Journal) OpenDisruptionWindow(operationName string, policy OutagePolicy // Close any existing window first. if j.activeWindow != nil { j.activeWindow.EndTime = time.Now() - j.closedWindows = append(j.closedWindows, *j.activeWindow) + j.appendClosedWindow(*j.activeWindow) } j.activeWindow = &DisruptionWindow{ @@ -125,17 +126,18 @@ func (j *Journal) OpenDisruptionWindow(operationName string, policy OutagePolicy }) } -// CloseDisruptionWindow ends the active disruption period. -func (j *Journal) CloseDisruptionWindow() { +// CloseDisruptionWindow ends the active disruption period and returns a copy. +func (j *Journal) CloseDisruptionWindow() *DisruptionWindow { j.mu.Lock() defer j.mu.Unlock() if j.activeWindow == nil { - return + return nil } j.activeWindow.EndTime = time.Now() - j.closedWindows = append(j.closedWindows, *j.activeWindow) + j.appendClosedWindow(*j.activeWindow) + closed := *j.activeWindow j.events = append(j.events, Event{ Timestamp: time.Now(), @@ -146,14 +148,43 @@ func (j *Journal) CloseDisruptionWindow() { }) j.activeWindow = nil + return &closed +} + +func (j *Journal) appendClosedWindow(window DisruptionWindow) { + j.closedWindows = append(j.closedWindows, window) + if len(j.closedWindows) > maxDisruptionWindows { + copy(j.closedWindows, j.closedWindows[len(j.closedWindows)-maxDisruptionWindows:]) + j.closedWindows = j.closedWindows[:maxDisruptionWindows] + } } -// RecordWriteFailure increments the failure count for the active disruption window. -func (j *Journal) RecordWriteFailure() { +// RecordWriteOutcome reports the result of a single write attempt to the active +// disruption window so it can measure the real write-outage duration from +// timestamps. attemptStart is when the write attempt began (before the driver +// call, which may block for the full server-selection timeout during an +// outage). A failure opens or extends the current outage; a success closes it, +// recording the first-failure -> first-success span as a candidate for the +// window's longest observed outage. No-op when no window is active. +func (j *Journal) RecordWriteOutcome(attemptStart time.Time, failed bool) { j.mu.Lock() defer j.mu.Unlock() - if j.activeWindow != nil { - j.activeWindow.WriteFailures++ + w := j.activeWindow + if w == nil { + return + } + if failed { + w.WriteFailures++ + if w.WriteOutageStart.IsZero() { + w.WriteOutageStart = attemptStart + } + return + } + if !w.WriteOutageStart.IsZero() { + if gap := attemptStart.Sub(w.WriteOutageStart); gap > w.MaxWriteOutageObserved { + w.MaxWriteOutageObserved = gap + } + w.WriteOutageStart = time.Time{} } } diff --git a/test/longhaul/journal/journal_test.go b/test/longhaul/journal/journal_test.go index 02e2b3c6c..32b7f7ded 100644 --- a/test/longhaul/journal/journal_test.go +++ b/test/longhaul/journal/journal_test.go @@ -43,7 +43,7 @@ var _ = Describe("Journal", func() { Describe("DisruptionWindow lifecycle", func() { It("opens, records failures, and closes correctly", func() { j := New() - policy := OutagePolicy{MustRecoverWithin: time.Minute, AllowedWriteFailures: 10} + policy := OutagePolicy{MustRecoverWithin: time.Minute, MaxWriteOutage: time.Second} Expect(j.ActiveWindow()).To(BeNil()) @@ -53,9 +53,9 @@ var _ = Describe("Journal", func() { Expect(w.OperationName).To(Equal("scale-up")) Expect(w.IsActive()).To(BeTrue()) - j.RecordWriteFailure() - j.RecordWriteFailure() - j.RecordWriteFailure() + j.RecordWriteOutcome(time.Now(), true) + j.RecordWriteOutcome(time.Now(), true) + j.RecordWriteOutcome(time.Now(), true) Expect(j.ActiveWindow().WriteFailures).To(Equal(int64(3))) j.CloseDisruptionWindow() @@ -66,6 +66,29 @@ var _ = Describe("Journal", func() { Expect(closed[0].IsActive()).To(BeFalse()) }) + It("measures the write outage as first-failure to first-success", func() { + j := New() + j.OpenDisruptionWindow("kill-primary", OutagePolicy{MustRecoverWithin: time.Minute, MaxWriteOutage: time.Minute}) + + start := time.Now() + // Writes fail across a ~200ms span, then recover. + j.RecordWriteOutcome(start, true) + j.RecordWriteOutcome(start.Add(100*time.Millisecond), true) + j.RecordWriteOutcome(start.Add(200*time.Millisecond), false) // recovery + + w := j.ActiveWindow() + Expect(w.WriteFailures).To(Equal(int64(2))) + Expect(w.WriteOutageStart.IsZero()).To(BeTrue(), "outage should be closed after a success") + Expect(w.EstimatedWriteOutage()).To(BeNumerically("~", 200*time.Millisecond, 5*time.Millisecond)) + + // A single lost write that blocks for the full server-selection + // timeout must not undercount: one failure whose next success is + // 30s later still measures the real 30s span (the bug this fixes). + j.RecordWriteOutcome(start.Add(1*time.Second), true) + j.RecordWriteOutcome(start.Add(31*time.Second), false) + Expect(j.ActiveWindow().EstimatedWriteOutage()).To(BeNumerically("~", 30*time.Second, 5*time.Millisecond)) + }) + It("opening a new window closes the previous active window", func() { j := New() j.OpenDisruptionWindow("op1", DefaultOutagePolicy()) @@ -76,9 +99,23 @@ var _ = Describe("Journal", func() { Expect(closed[0].OperationName).To(Equal("op1")) }) - It("RecordWriteFailure without an active window is a no-op", func() { + It("RecordWriteOutcome without an active window is a no-op", func() { + j := New() + Expect(func() { j.RecordWriteOutcome(time.Now(), true) }).NotTo(Panic()) + }) + + It("bounds closed disruption-window diagnostics to the newest entries", func() { j := New() - Expect(func() { j.RecordWriteFailure() }).NotTo(Panic()) + total := maxDisruptionWindows + 5 + for i := 0; i < total; i++ { + j.OpenDisruptionWindow(fmt.Sprintf("op-%d", i), DefaultOutagePolicy()) + j.CloseDisruptionWindow() + } + + windows := j.DisruptionWindows() + Expect(windows).To(HaveLen(maxDisruptionWindows)) + Expect(windows[0].OperationName).To(Equal("op-5")) + Expect(windows[len(windows)-1].OperationName).To(Equal(fmt.Sprintf("op-%d", total-1))) }) }) @@ -89,23 +126,24 @@ var _ = Describe("Journal", func() { It("returns false on a closed window within budget", func() { j := New() - j.OpenDisruptionWindow("op", OutagePolicy{MustRecoverWithin: time.Minute, AllowedWriteFailures: 10}) + j.OpenDisruptionWindow("op", OutagePolicy{MustRecoverWithin: time.Minute, MaxWriteOutage: time.Second}) j.CloseDisruptionWindow() Expect(j.HasPolicyViolation()).To(BeFalse()) }) - It("returns true on a closed window over write-failure budget", func() { + It("returns true on a closed window over write-outage budget", func() { j := New() - j.OpenDisruptionWindow("op", OutagePolicy{MustRecoverWithin: time.Minute, AllowedWriteFailures: 1}) - j.RecordWriteFailure() - j.RecordWriteFailure() + j.OpenDisruptionWindow("op", OutagePolicy{MustRecoverWithin: time.Minute, MaxWriteOutage: 10 * time.Millisecond}) + // Writes started failing 50ms ago and never recovered, so the + // outage spans start->close (~50ms), exceeding the 10ms budget. + j.RecordWriteOutcome(time.Now().Add(-50*time.Millisecond), true) j.CloseDisruptionWindow() Expect(j.HasPolicyViolation()).To(BeTrue()) }) It("returns true on an active window over time budget", func() { j := New() - j.OpenDisruptionWindow("op", OutagePolicy{MustRecoverWithin: time.Nanosecond, AllowedWriteFailures: 10}) + j.OpenDisruptionWindow("op", OutagePolicy{MustRecoverWithin: time.Nanosecond, MaxWriteOutage: time.Second}) time.Sleep(1 * time.Millisecond) Expect(j.HasPolicyViolation()).To(BeTrue()) }) diff --git a/test/longhaul/journal/policy.go b/test/longhaul/journal/policy.go index 25d28e826..f0cb0d41c 100644 --- a/test/longhaul/journal/policy.go +++ b/test/longhaul/journal/policy.go @@ -5,20 +5,105 @@ package journal import "time" -// OutagePolicy defines acceptable disruption bounds for an operation. +// OutagePolicy defines acceptable disruption bounds for an operation. Its two +// fields assert on different properties of the managed cluster and fail +// independently (ExceededPolicy trips if either is exceeded): MaxWriteOutage +// bounds client-visible write availability, while MustRecoverWithin bounds the +// cluster's return to its full declared topology (all pods Ready, CR Ready). +// Operation execution errors also fail the run independently of this policy. +// Each can be violated while the other is fine — e.g. after a failover writes +// resume quickly (MaxWriteOutage happy) yet the cluster stays degraded until a +// replacement standby rejoins, which only MustRecoverWithin catches. type OutagePolicy struct { - // AllowedWriteFailures is the maximum number of write failures during the window. - AllowedWriteFailures int64 + // MaxWriteOutage bounds how long the write path (client -> gateway -> + // primary) may be unavailable during the window. It is measured from write + // timestamps as the longest span from the first failing write attempt to + // the first subsequent successful write (see + // DisruptionWindow.EstimatedWriteOutage), so it reflects real wall-clock + // unavailability regardless of how many writer goroutines + // (LONGHAUL_NUM_WRITERS) are configured or how the driver batches its + // retries. + MaxWriteOutage time.Duration - // MustRecoverWithin is the maximum time from operation start to full recovery. + // MustRecoverWithin is the maximum time from operation start to full cluster + // recovery (steady state). MustRecoverWithin time.Duration } // DefaultOutagePolicy returns a conservative policy suitable for most operations. func DefaultOutagePolicy() OutagePolicy { return OutagePolicy{ - AllowedWriteFailures: 50, - MustRecoverWithin: 5 * time.Minute, + MaxWriteOutage: 5 * time.Second, + MustRecoverWithin: 5 * time.Minute, + } +} + +// NoOutageWriteOutageCushion is the tiny write-outage budget granted to +// operations that are expected NOT to disrupt the data plane. It is not a +// tolerance for real outages: the write-outage is measured as the span from the +// first failing write to the next successful one, so a lone transient failure +// (one writer, recovered on the next ~100ms tick) maps to roughly one +// writeInterval of outage. This ~3-tick cushion absorbs unrelated background +// noise (a client reconnect, service-endpoint churn) without tolerating a +// genuine primary outage. Centralized so it can be recalibrated against real +// long-haul runs in one place. +const NoOutageWriteOutageCushion = 300 * time.Millisecond + +// NoOutagePolicy is the outage budget for operations that keep the write path +// up throughout and therefore must not cause a write outage. It is shared by +// every "no data-plane impact" operation: +// - control-plane faults, e.g. an operator pod restart, and +// - scaling that only adds or removes a standby replica (the primary, and +// thus the write path, is never touched). +// +// recovery bounds how long the cluster may take to return to steady state. +func NoOutagePolicy(recovery time.Duration) OutagePolicy { + return OutagePolicy{ + MaxWriteOutage: NoOutageWriteOutageCushion, + MustRecoverWithin: recovery, + } +} + +// PrimaryHandoverWriteOutage is the write-outage budget for kill-primary-pod: +// an *ungraceful* failover that detects the lost pod, then promotes a standby. +// The write path is interrupted for exactly one primary handover. +// +// Sized to comfortably cover a healthy single CNPG failover; heuristic pending +// calibration against real long-haul runs. +const PrimaryHandoverWriteOutage = 30 * time.Second + +// PrimaryHandoverPolicy is the outage budget for operations whose write path is +// interrupted for a single primary handover (see PrimaryHandoverWriteOutage). +// recovery bounds how long the cluster may take to return to full topology, +// which can legitimately differ per operation (a rolling upgrade restarts every +// pod and takes longer than a single failover). +func PrimaryHandoverPolicy(recovery time.Duration) OutagePolicy { + return OutagePolicy{ + MaxWriteOutage: PrimaryHandoverWriteOutage, + MustRecoverWithin: recovery, + } +} + +// UpgradeWriteOutage is the write-outage budget for upgrade-documentdb. A +// cross-version rolling upgrade still interrupts writes for a single primary +// switchover (the standby restarts do NOT interrupt writes), but that +// switchover is heavier than a plain failover: it coincides with the extension +// version migration running under live write load, and the newly promoted +// primary must come up on the new image before it accepts writes. Calibrated +// against a real 0.110.0 -> 0.113.0 upgrade on a resource-constrained kind +// runner, which measured a ~33s switchover outage; the budget carries headroom +// over that so genuine cross-version upgrades are not flagged while a gross +// regression (a multi-minute write stall) still is. The upgrade's longer, +// whole-topology restart is bounded separately by MustRecoverWithin. +const UpgradeWriteOutage = 90 * time.Second + +// UpgradeOutagePolicy is the outage budget for a cross-version DocumentDB +// upgrade (see UpgradeWriteOutage). recovery bounds how long the whole-topology +// rolling restart may take to return to full topology. +func UpgradeOutagePolicy(recovery time.Duration) OutagePolicy { + return OutagePolicy{ + MaxWriteOutage: UpgradeWriteOutage, + MustRecoverWithin: recovery, } } @@ -36,8 +121,41 @@ type DisruptionWindow struct { // Policy is the outage budget for this window. Policy OutagePolicy - // WriteFailures counts failures observed during this window. + // WriteFailures counts individual failed write attempts observed during + // this window. Retained for reporting only; the outage budget is evaluated + // from timestamps (see EstimatedWriteOutage), not this count. WriteFailures int64 + + // WriteOutageStart is the attempt-start time of the first failing write of + // the currently-open outage, or zero when writes are not currently failing. + // Set on the first failure after writes were healthy and cleared once a + // write succeeds again. + WriteOutageStart time.Time + + // MaxWriteOutageObserved is the longest completed outage seen so far in this + // window: the span from a first failing write to the first subsequent + // success. EstimatedWriteOutage combines it with any still-open outage. + MaxWriteOutageObserved time.Duration +} + +// EstimatedWriteOutage returns the longest span during this window for which +// the write path was unavailable, measured from write timestamps as +// first-failing-attempt -> first-subsequent-success. If writes are still +// failing when this is evaluated, the currently-open outage is measured up to +// the window end (or now, for an active window). Returns 0 when no write ever +// failed during the window. +func (w *DisruptionWindow) EstimatedWriteOutage() time.Duration { + outage := w.MaxWriteOutageObserved + if !w.WriteOutageStart.IsZero() { + end := w.EndTime + if end.IsZero() { + end = time.Now() + } + if open := end.Sub(w.WriteOutageStart); open > outage { + outage = open + } + } + return outage } // IsActive returns true if the disruption window has not been closed. @@ -59,7 +177,7 @@ func (w *DisruptionWindow) ExceededPolicy() bool { if w.Duration() > w.Policy.MustRecoverWithin { return true } - if w.WriteFailures > w.Policy.AllowedWriteFailures { + if w.EstimatedWriteOutage() > w.Policy.MaxWriteOutage { return true } return false diff --git a/test/longhaul/journal/policy_test.go b/test/longhaul/journal/policy_test.go index 4fe26b1a4..99e6265f0 100644 --- a/test/longhaul/journal/policy_test.go +++ b/test/longhaul/journal/policy_test.go @@ -43,42 +43,71 @@ var _ = Describe("DisruptionWindow", func() { }, Entry("within all budgets", DisruptionWindow{ - StartTime: time.Now().Add(-10 * time.Second), - EndTime: time.Now(), - WriteFailures: 5, - Policy: OutagePolicy{MustRecoverWithin: time.Minute, AllowedWriteFailures: 50}, + StartTime: time.Now().Add(-10 * time.Second), + EndTime: time.Now(), + MaxWriteOutageObserved: 100 * time.Millisecond, + Policy: OutagePolicy{MustRecoverWithin: time.Minute, MaxWriteOutage: time.Second}, }, false), Entry("exceeds MustRecoverWithin", DisruptionWindow{ - StartTime: time.Now().Add(-2 * time.Minute), - EndTime: time.Now(), - WriteFailures: 1, - Policy: OutagePolicy{MustRecoverWithin: time.Minute, AllowedWriteFailures: 50}, + StartTime: time.Now().Add(-2 * time.Minute), + EndTime: time.Now(), + MaxWriteOutageObserved: 10 * time.Millisecond, + Policy: OutagePolicy{MustRecoverWithin: time.Minute, MaxWriteOutage: time.Second}, }, true), - Entry("exceeds AllowedWriteFailures", + Entry("exceeds MaxWriteOutage", DisruptionWindow{ - StartTime: time.Now().Add(-10 * time.Second), - EndTime: time.Now(), - WriteFailures: 100, - Policy: OutagePolicy{MustRecoverWithin: time.Minute, AllowedWriteFailures: 50}, + StartTime: time.Now().Add(-10 * time.Second), + EndTime: time.Now(), + MaxWriteOutageObserved: 2 * time.Second, + Policy: OutagePolicy{MustRecoverWithin: time.Minute, MaxWriteOutage: time.Second}, }, true), - Entry("boundary: equal to write-failure budget is allowed", + Entry("boundary: observed outage equal to budget is allowed", DisruptionWindow{ - StartTime: time.Now().Add(-10 * time.Second), - EndTime: time.Now(), - WriteFailures: 50, - Policy: OutagePolicy{MustRecoverWithin: time.Minute, AllowedWriteFailures: 50}, + StartTime: time.Now().Add(-10 * time.Second), + EndTime: time.Now(), + MaxWriteOutageObserved: time.Second, + Policy: OutagePolicy{MustRecoverWithin: time.Minute, MaxWriteOutage: time.Second}, }, false), + Entry("no write failure means no outage", + DisruptionWindow{ + StartTime: time.Now().Add(-10 * time.Second), + EndTime: time.Now(), + Policy: OutagePolicy{MustRecoverWithin: time.Minute, MaxWriteOutage: time.Second}, + }, false), + Entry("still-open outage on a closed window is measured to EndTime", + DisruptionWindow{ + StartTime: time.Now().Add(-10 * time.Second), + EndTime: time.Now(), + WriteOutageStart: time.Now().Add(-3 * time.Second), + Policy: OutagePolicy{MustRecoverWithin: time.Minute, MaxWriteOutage: time.Second}, + }, true), Entry("active window also evaluated against MustRecoverWithin", DisruptionWindow{ StartTime: time.Now().Add(-2 * time.Minute), - Policy: OutagePolicy{MustRecoverWithin: time.Minute, AllowedWriteFailures: 50}, + Policy: OutagePolicy{MustRecoverWithin: time.Minute, MaxWriteOutage: time.Second}, + }, true), + Entry("active window with an open outage measured to now", + DisruptionWindow{ + StartTime: time.Now().Add(-3 * time.Second), + WriteOutageStart: time.Now().Add(-2 * time.Second), + Policy: OutagePolicy{MustRecoverWithin: time.Minute, MaxWriteOutage: time.Second}, }, true), ) It("DefaultOutagePolicy returns no zero-valued field", func() { p := DefaultOutagePolicy() Expect(p.MustRecoverWithin).NotTo(BeZero()) - Expect(p.AllowedWriteFailures).NotTo(BeZero()) + Expect(p.MaxWriteOutage).NotTo(BeZero()) + }) + + It("NoOutagePolicy grants the near-zero cushion and echoes recovery", func() { + p := NoOutagePolicy(3 * time.Minute) + Expect(p.MaxWriteOutage).To(Equal(NoOutageWriteOutageCushion)) + Expect(p.MustRecoverWithin).To(Equal(3 * time.Minute)) + }) + + It("NoOutagePolicy is far tighter than DefaultOutagePolicy", func() { + Expect(NoOutageWriteOutageCushion).To(BeNumerically("<", DefaultOutagePolicy().MaxWriteOutage)) }) }) diff --git a/test/longhaul/monitor/health.go b/test/longhaul/monitor/health.go index 0c5e95163..f2b6749a9 100644 --- a/test/longhaul/monitor/health.go +++ b/test/longhaul/monitor/health.go @@ -49,6 +49,15 @@ type ClusterClient interface { // UpgradeDocumentDB patches spec.documentDBVersion and spec.schemaVersion="auto". UpgradeDocumentDB(ctx context.Context, version string) error + + // GetPrimaryInstance returns the name of the pod currently serving as the + // CNPG primary (from Cluster.status.currentPrimary). The pod name equals + // the CNPG instance name. Returns an error if no primary is known yet. + GetPrimaryInstance(ctx context.Context) (string, error) + + // DeletePod deletes the named pod in the cluster namespace. Used by chaos + // operations to inject pod-loss faults. + DeletePod(ctx context.Context, name string) error } // HealthMonitor continuously monitors cluster health and tracks steady-state. @@ -144,6 +153,21 @@ func (h *HealthMonitor) IsSteadyState() bool { return time.Since(h.steadySince) >= h.steadyStateWait } +// InvalidateSteadyState resets the steady-state epoch so the next successful +// WaitForSteadyState / IsSteadyState must observe a *fresh* continuous-healthy +// interval: at least one health sample taken after this call, then +// steadyStateWait of continuous health. Callers invoke it when they open a +// disruption (e.g. patch the topology or delete a pod) so a stale +// pre-operation steadySince — the monitor polls on its own, slower cadence — +// cannot satisfy the post-operation recovery gate before the monitor has +// actually observed the change. +func (h *HealthMonitor) InvalidateSteadyState() { + h.mu.Lock() + defer h.mu.Unlock() + h.steadySince = time.Time{} + h.healthySamples = 0 +} + // LastHealth returns the most recent health observation. func (h *HealthMonitor) LastHealth() ClusterHealth { h.mu.RLock() diff --git a/test/longhaul/monitor/health_test.go b/test/longhaul/monitor/health_test.go index 1ad903fd9..4d5385e1e 100644 --- a/test/longhaul/monitor/health_test.go +++ b/test/longhaul/monitor/health_test.go @@ -41,9 +41,11 @@ func (f *fakeClusterClient) GetClusterHealth(_ context.Context) (ClusterHealth, func (f *fakeClusterClient) GetCurrentDocumentDBImageTag(_ context.Context) (string, error) { return "", nil } -func (f *fakeClusterClient) GetInstancesPerNode(_ context.Context) (int, error) { return 1, nil } -func (f *fakeClusterClient) ScaleCluster(_ context.Context, _ int) error { return nil } -func (f *fakeClusterClient) UpgradeDocumentDB(_ context.Context, _ string) error { return nil } +func (f *fakeClusterClient) GetInstancesPerNode(_ context.Context) (int, error) { return 1, nil } +func (f *fakeClusterClient) ScaleCluster(_ context.Context, _ int) error { return nil } +func (f *fakeClusterClient) UpgradeDocumentDB(_ context.Context, _ string) error { return nil } +func (f *fakeClusterClient) GetPrimaryInstance(_ context.Context) (string, error) { return "", nil } +func (f *fakeClusterClient) DeletePod(_ context.Context, _ string) error { return nil } var _ = Describe("HealthMonitor", func() { Describe("IsSteadyState", func() { diff --git a/test/longhaul/monitor/k8sclient.go b/test/longhaul/monitor/k8sclient.go index f843f960b..f1bc91b13 100644 --- a/test/longhaul/monitor/k8sclient.go +++ b/test/longhaul/monitor/k8sclient.go @@ -63,7 +63,7 @@ func NewK8sClusterClient(cfg K8sClientConfig) (*K8sClusterClient, error) { return nil, fmt.Errorf("failed to create clientset: %w", err) } - scheme, err := shareddb.NewScheme() + scheme, err := shareddb.NewScheme(cnpgv1.AddToScheme) if err != nil { return nil, fmt.Errorf("failed to build scheme: %w", err) } @@ -229,6 +229,29 @@ func (k *K8sClusterClient) UpgradeDocumentDB(ctx context.Context, version string return nil } +// GetPrimaryInstance reads status.currentPrimary from the CNPG Cluster that +// backs this DocumentDB. The CNPG Cluster name equals the DocumentDB CR name, +// and the returned instance name equals the primary pod name. +func (k *K8sClusterClient) GetPrimaryInstance(ctx context.Context) (string, error) { + var cluster cnpgv1.Cluster + key := types.NamespacedName{Namespace: k.namespace, Name: k.clusterName} + if err := k.crClient.Get(ctx, key, &cluster); err != nil { + return "", fmt.Errorf("failed to get CNPG Cluster: %w", err) + } + if cluster.Status.CurrentPrimary == "" { + return "", fmt.Errorf("CNPG Cluster %s has no current primary yet", k.clusterName) + } + return cluster.Status.CurrentPrimary, nil +} + +// DeletePod deletes the named pod in the cluster namespace. +func (k *K8sClusterClient) DeletePod(ctx context.Context, name string) error { + if err := k.clientset.CoreV1().Pods(k.namespace).Delete(ctx, name, metav1.DeleteOptions{}); err != nil { + return fmt.Errorf("failed to delete pod %s/%s: %w", k.namespace, name, err) + } + return nil +} + // GetPodMetrics queries metrics-server for pod resource usage. // Returns nil, nil if metrics-server is not available. func (k *K8sClusterClient) GetPodMetrics(ctx context.Context) ([]PodMetrics, error) { diff --git a/test/longhaul/operations/kill_operator.go b/test/longhaul/operations/kill_operator.go new file mode 100644 index 000000000..6b268a50c --- /dev/null +++ b/test/longhaul/operations/kill_operator.go @@ -0,0 +1,242 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package operations + +import ( + "context" + "fmt" + "time" + + "github.com/documentdb/documentdb-operator/test/longhaul/journal" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" +) + +// OperatorDeploymentName is the fixed name of the operator Deployment. The +// operator is a cluster singleton, so this name is stable across installs. +const OperatorDeploymentName = "documentdb-operator" + +// KillOperatorPod deletes the running operator pod to verify that an operator +// restart does not disrupt the data plane. The CNPG-managed database keeps +// serving reads and writes while the Deployment reschedules the control plane, +// so the workload verifier should observe (near) zero write failures. Recovery +// is asserted by the Deployment returning to Available. +type KillOperatorPod struct { + clientset kubernetes.Interface + namespace string + deployment string + recovery time.Duration +} + +// NewKillOperatorPod creates a KillOperatorPod operation targeting the operator +// Deployment in the given namespace. +func NewKillOperatorPod(clientset kubernetes.Interface, namespace string, recovery time.Duration) *KillOperatorPod { + return &KillOperatorPod{ + clientset: clientset, + namespace: namespace, + deployment: OperatorDeploymentName, + recovery: recovery, + } +} + +func (k *KillOperatorPod) Name() string { return "kill-operator-pod" } + +func (k *KillOperatorPod) Weight() int { return 2 } + +// Precondition requires the operator Deployment to exist and currently be +// Available, so the fault isn't stacked on an already-restarting operator. +func (k *KillOperatorPod) Precondition(ctx context.Context) (bool, string) { + dep, err := k.getDeployment(ctx) + if err != nil { + return false, fmt.Sprintf("cannot get operator deployment: %v", err) + } + if !isDeploymentAvailable(dep) { + return false, "operator deployment not currently available" + } + return true, "" +} + +func (k *KillOperatorPod) Execute(ctx context.Context) error { + dep, err := k.getDeployment(ctx) + if err != nil { + return fmt.Errorf("get operator deployment: %w", err) + } + + // Fail fast if the Deployment has no label selector: SelectorFromSet on an + // empty map yields an "everything" selector, so the List below would match + // (and the delete could target) every pod in the namespace. + if dep.Spec.Selector == nil || len(dep.Spec.Selector.MatchLabels) == 0 { + return fmt.Errorf("operator deployment %s has no matchLabels selector; refusing to list all pods", k.deployment) + } + + // Resolve the pod set from the Deployment's own selector so we don't + // depend on the release-name-derived "app" label value. + selector := labels.SelectorFromSet(dep.Spec.Selector.MatchLabels).String() + pods, err := k.clientset.CoreV1().Pods(k.namespace).List(ctx, metav1.ListOptions{LabelSelector: selector}) + if err != nil { + return fmt.Errorf("list operator pods: %w", err) + } + + target, targetUID := oldestRunningPod(pods.Items) + if target == "" { + return fmt.Errorf("no running operator pod found for selector %q", selector) + } + if err := k.clientset.CoreV1().Pods(k.namespace).Delete(ctx, target, metav1.DeleteOptions{}); err != nil { + return fmt.Errorf("delete operator pod %s: %w", target, err) + } + + recoveryCtx, cancel := context.WithTimeout(ctx, k.recovery) + defer cancel() + + // Confirm the targeted pod is actually gone before checking Deployment + // availability. Deleting a pod does not bump the Deployment's + // ObservedGeneration, so its status can still read "Available" from the + // pre-deletion state and let waitForDeploymentAvailable return immediately + // without ever observing the restart. + if err := k.waitForPodGone(recoveryCtx, target, targetUID); err != nil { + return err + } + + // The old pod being gone does not yet mean recovery: the Deployment's + // status counters may still momentarily reflect the pre-deletion replica + // as Ready. Require a replacement pod (different UID) to actually reach + // Ready before trusting the Deployment-level availability check. + if err := k.waitForReplacementReady(recoveryCtx, selector, targetUID); err != nil { + return err + } + + // Wait for the Deployment to reschedule and become Available again. + return k.waitForDeploymentAvailable(recoveryCtx) +} + +// waitForReplacementReady blocks until a pod matching selector, with a UID +// different from the deleted pod, is Running and Ready. This guarantees the +// operator has genuinely rescheduled rather than letting a stale Deployment +// status (still counting the pre-deletion replica) report a false recovery. +func (k *KillOperatorPod) waitForReplacementReady(ctx context.Context, selector string, deletedUID types.UID) error { + ticker := time.NewTicker(2 * time.Second) + defer ticker.Stop() + for { + pods, err := k.clientset.CoreV1().Pods(k.namespace).List(ctx, metav1.ListOptions{LabelSelector: selector}) + if err == nil { + for i := range pods.Items { + p := &pods.Items[i] + if p.UID != deletedUID && p.DeletionTimestamp == nil && isPodReady(p) { + return nil + } + } + } + select { + case <-ctx.Done(): + return fmt.Errorf("timed out waiting for a replacement operator pod to become ready: %w", ctx.Err()) + case <-ticker.C: + } + } +} + +// waitForPodGone blocks until the pod identified by name/uid is deleted +// (NotFound) or replaced by a new pod with a different UID, guaranteeing the +// disruption has actually landed before we assert recovery. +func (k *KillOperatorPod) waitForPodGone(ctx context.Context, name string, uid types.UID) error { + ticker := time.NewTicker(2 * time.Second) + defer ticker.Stop() + for { + pod, err := k.clientset.CoreV1().Pods(k.namespace).Get(ctx, name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + return nil + } + if err == nil && pod.UID != uid { + return nil + } + select { + case <-ctx.Done(): + return fmt.Errorf("timed out waiting for operator pod %s to be deleted: %w", name, ctx.Err()) + case <-ticker.C: + } + } +} + +// OutagePolicy: an operator restart is a control-plane fault that must not take +// down the data plane, so it shares the near-zero NoOutagePolicy budget. +func (k *KillOperatorPod) OutagePolicy() journal.OutagePolicy { + return journal.NoOutagePolicy(k.recovery) +} + +func (k *KillOperatorPod) getDeployment(ctx context.Context) (*appsv1.Deployment, error) { + return k.clientset.AppsV1().Deployments(k.namespace).Get(ctx, k.deployment, metav1.GetOptions{}) +} + +func (k *KillOperatorPod) waitForDeploymentAvailable(ctx context.Context) error { + ticker := time.NewTicker(5 * time.Second) + defer ticker.Stop() + for { + if dep, err := k.getDeployment(ctx); err == nil && isDeploymentAvailable(dep) { + return nil + } + select { + case <-ctx.Done(): + return fmt.Errorf("timed out waiting for operator deployment to become available: %w", ctx.Err()) + case <-ticker.C: + } + } +} + +// isDeploymentAvailable reports whether the Deployment has its full desired +// replica count ready with none unavailable and the observed generation caught +// up to the latest spec. +func isDeploymentAvailable(dep *appsv1.Deployment) bool { + if dep == nil { + return false + } + desired := int32(1) + if dep.Spec.Replicas != nil { + desired = *dep.Spec.Replicas + } + if dep.Status.ObservedGeneration < dep.Generation { + return false + } + return dep.Status.ReadyReplicas >= desired && dep.Status.UnavailableReplicas == 0 +} + +// isPodReady reports whether the pod is in the Running phase with a Ready +// condition set to True. +func isPodReady(p *corev1.Pod) bool { + if p.Status.Phase != corev1.PodRunning { + return false + } + for _, c := range p.Status.Conditions { + if c.Type == corev1.PodReady { + return c.Status == corev1.ConditionTrue + } + } + return false +} + +// oldestRunningPod returns the name and UID of the oldest pod in the Running +// phase, or ("", "") if none are running. Targeting the oldest makes the choice +// deterministic; the UID lets callers confirm that specific pod is later gone. +func oldestRunningPod(pods []corev1.Pod) (string, types.UID) { + name := "" + var uid types.UID + var oldest time.Time + for i := range pods { + p := &pods[i] + if p.Status.Phase != corev1.PodRunning || p.DeletionTimestamp != nil { + continue + } + ts := p.CreationTimestamp.Time + if name == "" || ts.Before(oldest) { + name = p.Name + uid = p.UID + oldest = ts + } + } + return name, uid +} diff --git a/test/longhaul/operations/kill_operator_test.go b/test/longhaul/operations/kill_operator_test.go new file mode 100644 index 000000000..0ea46abe5 --- /dev/null +++ b/test/longhaul/operations/kill_operator_test.go @@ -0,0 +1,143 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package operations + +import ( + "context" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/documentdb/documentdb-operator/test/longhaul/journal" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/fake" +) + +const opNS = "documentdb-operator" + +func operatorDeployment(desired, ready, unavailable int32, gen, observed int64) *appsv1.Deployment { + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: OperatorDeploymentName, + Namespace: opNS, + Generation: gen, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: &desired, + Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "documentdb-operator"}}, + }, + Status: appsv1.DeploymentStatus{ + ReadyReplicas: ready, + UnavailableReplicas: unavailable, + ObservedGeneration: observed, + }, + } +} + +func operatorPod(name string, phase corev1.PodPhase, ageSeconds int) *corev1.Pod { + status := corev1.PodStatus{Phase: phase} + if phase == corev1.PodRunning { + status.Conditions = []corev1.PodCondition{ + {Type: corev1.PodReady, Status: corev1.ConditionTrue}, + } + } + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: opNS, + UID: types.UID(name), + Labels: map[string]string{"app": "documentdb-operator"}, + CreationTimestamp: metav1.NewTime(time.Now().Add(-time.Duration(ageSeconds) * time.Second)), + }, + Status: status, + } +} + +var _ = Describe("KillOperatorPod", func() { + It("Name is kill-operator-pod and Weight is 2", func() { + k := NewKillOperatorPod(fake.NewSimpleClientset(), opNS, time.Minute) + Expect(k.Name()).To(Equal("kill-operator-pod")) + Expect(k.Weight()).To(Equal(2)) + }) + + It("OutagePolicy uses the near-zero NoOutagePolicy budget", func() { + k := NewKillOperatorPod(fake.NewSimpleClientset(), opNS, 2*time.Minute) + p := k.OutagePolicy() + Expect(p.MaxWriteOutage).To(Equal(journal.NoOutageWriteOutageCushion)) + Expect(p.MustRecoverWithin).To(Equal(2 * time.Minute)) + }) + + Describe("Precondition", func() { + It("skips when the deployment is missing", func() { + k := NewKillOperatorPod(fake.NewSimpleClientset(), opNS, time.Minute) + ok, reason := k.Precondition(context.Background()) + Expect(ok).To(BeFalse()) + Expect(reason).To(ContainSubstring("cannot get operator deployment")) + }) + + It("skips when the deployment is not available", func() { + dep := operatorDeployment(1, 0, 1, 1, 1) + k := NewKillOperatorPod(fake.NewSimpleClientset(dep), opNS, time.Minute) + ok, reason := k.Precondition(context.Background()) + Expect(ok).To(BeFalse()) + Expect(reason).To(ContainSubstring("not currently available")) + }) + + It("is eligible when the deployment is available", func() { + dep := operatorDeployment(1, 1, 0, 1, 1) + k := NewKillOperatorPod(fake.NewSimpleClientset(dep), opNS, time.Minute) + ok, _ := k.Precondition(context.Background()) + Expect(ok).To(BeTrue()) + }) + }) + + Describe("Execute", func() { + It("deletes the oldest running operator pod and returns once available", func() { + dep := operatorDeployment(1, 1, 0, 1, 1) + newer := operatorPod("op-new", corev1.PodRunning, 10) + older := operatorPod("op-old", corev1.PodRunning, 100) + cs := fake.NewSimpleClientset(dep, newer, older) + k := NewKillOperatorPod(cs, opNS, time.Minute) + + err := k.Execute(context.Background()) + Expect(err).NotTo(HaveOccurred()) + + _, getErr := cs.CoreV1().Pods(opNS).Get(context.Background(), "op-old", metav1.GetOptions{}) + Expect(getErr).To(HaveOccurred(), "oldest pod should have been deleted") + _, getErr = cs.CoreV1().Pods(opNS).Get(context.Background(), "op-new", metav1.GetOptions{}) + Expect(getErr).NotTo(HaveOccurred(), "newer pod should be untouched") + }) + + It("fails when no running pod matches the selector", func() { + dep := operatorDeployment(1, 1, 0, 1, 1) + pending := operatorPod("op-pending", corev1.PodPending, 10) + cs := fake.NewSimpleClientset(dep, pending) + k := NewKillOperatorPod(cs, opNS, time.Minute) + + err := k.Execute(context.Background()) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("no running operator pod")) + }) + + It("refuses to run when the deployment has no matchLabels selector", func() { + dep := operatorDeployment(1, 1, 0, 1, 1) + dep.Spec.Selector = &metav1.LabelSelector{} + running := operatorPod("op-run", corev1.PodRunning, 10) + cs := fake.NewSimpleClientset(dep, running) + k := NewKillOperatorPod(cs, opNS, time.Minute) + + err := k.Execute(context.Background()) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("no matchLabels selector")) + + _, getErr := cs.CoreV1().Pods(opNS).Get(context.Background(), "op-run", metav1.GetOptions{}) + Expect(getErr).NotTo(HaveOccurred(), "no pod should be deleted when the selector is empty") + }) + }) +}) diff --git a/test/longhaul/operations/kill_primary.go b/test/longhaul/operations/kill_primary.go new file mode 100644 index 000000000..d22d0e17a --- /dev/null +++ b/test/longhaul/operations/kill_primary.go @@ -0,0 +1,134 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package operations + +import ( + "context" + "fmt" + "time" + + "github.com/documentdb/documentdb-operator/test/longhaul/journal" + "github.com/documentdb/documentdb-operator/test/longhaul/monitor" +) + +// KillPrimaryPod deletes the CNPG primary pod to exercise the automatic +// failover path: CNPG must promote a standby, and the cluster must return to +// steady state within the recovery budget. The continuous workload verifier +// independently catches any data loss caused by the failover. +type KillPrimaryPod struct { + client monitor.ClusterClient + healthMon SteadyStateGate + recovery time.Duration + primaryPollInterval time.Duration +} + +// NewKillPrimaryPod creates a KillPrimaryPod operation. +func NewKillPrimaryPod(client monitor.ClusterClient, health SteadyStateGate, recovery time.Duration) *KillPrimaryPod { + return &KillPrimaryPod{ + client: client, + healthMon: health, + recovery: recovery, + primaryPollInterval: time.Second, + } +} + +func (k *KillPrimaryPod) Name() string { return "kill-primary-pod" } + +func (k *KillPrimaryPod) Weight() int { return 2 } + +// Precondition requires at least one standby (instancesPerNode>=2). Killing the +// sole instance of a single-instance cluster would cause guaranteed downtime +// with no failover target — a true-but-useless policy violation. The same guard +// (and rationale) is used by UpgradeDocumentDB; skips don't consume the +// scheduler cooldown, so this is free to re-evaluate on the next tick. +func (k *KillPrimaryPod) Precondition(ctx context.Context) (bool, string) { + ipn, err := k.client.GetInstancesPerNode(ctx) + if err != nil { + return false, fmt.Sprintf("cannot read instancesPerNode: %v", err) + } + if ipn < 2 { + return false, fmt.Sprintf("instancesPerNode=%d (no HA standby) — killing primary would cause real downtime; skipping", ipn) + } + return true, "" +} + +func (k *KillPrimaryPod) Execute(ctx context.Context) error { + primary, err := k.client.GetPrimaryInstance(ctx) + if err != nil { + return fmt.Errorf("get primary instance: %w", err) + } + if primary == "" { + return fmt.Errorf("get primary instance: cluster returned an empty primary pod name") + } + if k.healthMon == nil { + return fmt.Errorf("kill-primary-pod: health monitor is nil") + } + + recoveryCtx, cancel := context.WithTimeout(ctx, k.recovery) + defer cancel() + + if err := k.client.DeletePod(recoveryCtx, primary); err != nil { + return fmt.Errorf("delete primary pod %s: %w", primary, err) + } + + if err := k.waitForPrimaryChange(recoveryCtx, primary); err != nil { + return err + } + + // A changed primary proves CNPG promoted a standby rather than merely + // recreating the deleted pod and reporting the old primary again. + if err := k.healthMon.WaitForSteadyState(recoveryCtx); err != nil { + return fmt.Errorf("wait for steady-state recovery: %w", err) + } + + current, err := k.client.GetPrimaryInstance(recoveryCtx) + if err != nil { + return fmt.Errorf("verify primary after steady-state recovery: %w", err) + } + if current == "" || current == primary { + return fmt.Errorf("verify primary after steady-state recovery: expected a non-empty primary different from %q, got %q", + primary, current) + } + return nil +} + +func (k *KillPrimaryPod) waitForPrimaryChange(ctx context.Context, original string) error { + ticker := time.NewTicker(k.primaryPollInterval) + defer ticker.Stop() + + lastObserved := original + var lastErr error + for { + current, err := k.client.GetPrimaryInstance(ctx) + if err == nil { + lastObserved = current + if current != "" && current != original { + return nil + } + } else { + lastErr = err + } + + select { + case <-ctx.Done(): + if lastErr != nil { + return fmt.Errorf("primary did not change from %q before recovery timeout (last read error: %v): %w", + original, lastErr, ctx.Err()) + } + return fmt.Errorf("primary did not change from %q before recovery timeout (last observed %q): %w", + original, lastObserved, ctx.Err()) + case <-ticker.C: + } + } +} + +// OutagePolicy bounds the write outage of an automatic failover. Killing the +// primary interrupts writes until CNPG detects the loss and promotes a standby, +// so it uses the single-primary-handover budget (journal.PrimaryHandoverPolicy, +// ~30s). upgrade-documentdb has its own, larger budget +// (journal.UpgradeOutagePolicy, ~90s): its graceful switchover coincides with +// the extension migration under live write load. +func (k *KillPrimaryPod) OutagePolicy() journal.OutagePolicy { + return journal.PrimaryHandoverPolicy(k.recovery) +} diff --git a/test/longhaul/operations/kill_primary_test.go b/test/longhaul/operations/kill_primary_test.go new file mode 100644 index 000000000..b7553d985 --- /dev/null +++ b/test/longhaul/operations/kill_primary_test.go @@ -0,0 +1,131 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package operations + +import ( + "context" + "errors" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/documentdb/documentdb-operator/test/longhaul/journal" +) + +type successfulSteadyGate struct { + calls int + onWait func() +} + +func (g *successfulSteadyGate) WaitForSteadyState(context.Context) error { + g.calls++ + if g.onWait != nil { + g.onWait() + } + return nil +} + +func (g *successfulSteadyGate) InvalidateSteadyState() {} + +var _ = Describe("KillPrimaryPod", func() { + It("Name is kill-primary-pod and Weight is 2", func() { + k := NewKillPrimaryPod(&fakeClient{}, nil, time.Minute) + Expect(k.Name()).To(Equal("kill-primary-pod")) + Expect(k.Weight()).To(Equal(2)) + }) + + It("OutagePolicy shares the single-primary-handover budget with upgrade", func() { + k := NewKillPrimaryPod(&fakeClient{}, nil, 3*time.Minute) + p := k.OutagePolicy() + Expect(p.MaxWriteOutage).To(Equal(journal.PrimaryHandoverWriteOutage)) + Expect(p.MustRecoverWithin).To(Equal(3 * time.Minute)) + }) + + DescribeTable("Precondition", + func(ipn int, ipnErr error, wantOK bool, wantReasonHas string) { + c := &fakeClient{instancesPerNode: ipn, ipnErr: ipnErr} + k := NewKillPrimaryPod(c, nil, time.Minute) + + ok, reason := k.Precondition(context.Background()) + Expect(ok).To(Equal(wantOK), "reason=%q", reason) + if wantReasonHas != "" { + Expect(reason).To(ContainSubstring(wantReasonHas)) + } + }, + Entry("single-instance: ipn=1 -> skip", 1, nil, false, "no HA standby"), + Entry("read error -> skip", 0, errors.New("boom"), false, "cannot read instancesPerNode"), + Entry("HA: ipn=2 -> eligible", 2, nil, true, ""), + Entry("HA: ipn=3 -> eligible", 3, nil, true, ""), + ) + + It("Execute deletes the original primary and verifies a different primary", func() { + c := &fakeClient{ + instancesPerNode: 2, + primary: "cluster-1", + replacementPrimary: "cluster-2", + } + gate := &successfulSteadyGate{} + k := NewKillPrimaryPod(c, gate, time.Second) + + Expect(k.Execute(context.Background())).To(Succeed()) + c.mu.Lock() + defer c.mu.Unlock() + Expect(c.deletedPods).To(ConsistOf("cluster-1")) + Expect(c.primary).To(Equal("cluster-2")) + Expect(gate.calls).To(Equal(1)) + }) + + It("fails when CNPG keeps reporting the deleted primary", func() { + c := &fakeClient{instancesPerNode: 2, primary: "cluster-1"} + gate := &successfulSteadyGate{} + k := NewKillPrimaryPod(c, gate, 20*time.Millisecond) + k.primaryPollInterval = time.Millisecond + + err := k.Execute(context.Background()) + Expect(err).To(MatchError(ContainSubstring(`primary did not change from "cluster-1"`))) + Expect(gate.calls).To(Equal(0), "steady-state recovery must wait until primary change is proven") + }) + + It("fails if the recovered cluster reports the original primary again", func() { + c := &fakeClient{ + instancesPerNode: 2, + primary: "cluster-1", + replacementPrimary: "cluster-2", + } + gate := &successfulSteadyGate{onWait: func() { + c.mu.Lock() + defer c.mu.Unlock() + c.primary = "cluster-1" + }} + k := NewKillPrimaryPod(c, gate, time.Second) + + err := k.Execute(context.Background()) + Expect(err).To(MatchError(ContainSubstring("expected a non-empty primary different"))) + }) + + It("Execute fails without deleting when the primary is unknown", func() { + c := &fakeClient{instancesPerNode: 2, primaryErr: errors.New("no primary")} + k := NewKillPrimaryPod(c, nil, time.Second) + + err := k.Execute(context.Background()) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("get primary instance")) + c.mu.Lock() + defer c.mu.Unlock() + Expect(c.deletedPods).To(BeEmpty()) + }) + + It("Execute fails without deleting when the primary name is empty", func() { + c := &fakeClient{instancesPerNode: 2, primary: ""} + k := NewKillPrimaryPod(c, nil, time.Second) + + err := k.Execute(context.Background()) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("empty primary pod name")) + c.mu.Lock() + defer c.mu.Unlock() + Expect(c.deletedPods).To(BeEmpty()) + }) +}) diff --git a/test/longhaul/operations/registry.go b/test/longhaul/operations/registry.go new file mode 100644 index 000000000..f85c3f843 --- /dev/null +++ b/test/longhaul/operations/registry.go @@ -0,0 +1,88 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package operations + +import ( + "fmt" + + "k8s.io/client-go/kubernetes" + + "github.com/documentdb/documentdb-operator/test/longhaul/config" + "github.com/documentdb/documentdb-operator/test/longhaul/journal" + "github.com/documentdb/documentdb-operator/test/longhaul/monitor" +) + +// Registry stores operations by their stable Name() values while preserving +// registration order for deterministic snapshots and random-mode summaries. +type Registry struct { + order []string + operations map[string]Operation +} + +// NewRegistry builds a validated operation registry. +func NewRegistry(ops ...Operation) (*Registry, error) { + registry := &Registry{ + order: make([]string, 0, len(ops)), + operations: make(map[string]Operation, len(ops)), + } + for _, op := range ops { + if op == nil { + return nil, fmt.Errorf("operation registry contains a nil operation") + } + name := op.Name() + if name == "" { + return nil, fmt.Errorf("operation registry contains an operation with an empty name") + } + if _, exists := registry.operations[name]; exists { + return nil, fmt.Errorf("operation registry contains duplicate name %q", name) + } + registry.order = append(registry.order, name) + registry.operations[name] = op + } + return registry, nil +} + +// NewDefaultRegistry centralizes construction of every supported operation. +func NewDefaultRegistry( + cfg config.Config, + clusterClient monitor.ClusterClient, + clientset kubernetes.Interface, + health *monitor.HealthMonitor, + j *journal.Journal, +) (*Registry, error) { + return NewRegistry( + NewScaleUp(clusterClient, health, cfg.MaxInstances, cfg.RecoveryTimeout), + NewScaleDown(clusterClient, health, cfg.MinInstances, cfg.RecoveryTimeout), + NewUpgradeDocumentDB(clusterClient, clientset, health, j, cfg.Namespace, cfg.RecoveryTimeout), + NewKillOperatorPod(clientset, cfg.OperatorNamespace, cfg.RecoveryTimeout), + NewKillPrimaryPod(clusterClient, health, cfg.RecoveryTimeout), + ) +} + +// All returns all registered operations in stable registration order. +func (r *Registry) All() []Operation { + ops := make([]Operation, 0, len(r.order)) + for _, name := range r.order { + ops = append(ops, r.operations[name]) + } + return ops +} + +// Resolve returns the named operations in exactly the requested order. +func (r *Registry) Resolve(names []string) ([]Operation, error) { + resolved := make([]Operation, 0, len(names)) + seen := make(map[string]struct{}, len(names)) + for _, name := range names { + if _, duplicate := seen[name]; duplicate { + return nil, fmt.Errorf("operation sequence contains duplicate name %q", name) + } + op, ok := r.operations[name] + if !ok { + return nil, fmt.Errorf("operation sequence contains unknown name %q", name) + } + seen[name] = struct{}{} + resolved = append(resolved, op) + } + return resolved, nil +} diff --git a/test/longhaul/operations/registry_test.go b/test/longhaul/operations/registry_test.go new file mode 100644 index 000000000..03493269a --- /dev/null +++ b/test/longhaul/operations/registry_test.go @@ -0,0 +1,62 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package operations + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/documentdb/documentdb-operator/test/longhaul/config" + "github.com/documentdb/documentdb-operator/test/longhaul/journal" +) + +var _ = Describe("Registry", func() { + It("resolves exact stable names in requested order", func() { + a := &fakeOp{name: "a"} + b := &fakeOp{name: "b"} + registry, err := NewRegistry(a, b) + Expect(err).NotTo(HaveOccurred()) + + resolved, err := registry.Resolve([]string{"b", "a"}) + Expect(err).NotTo(HaveOccurred()) + Expect(resolved).To(Equal([]Operation{b, a})) + Expect(registry.All()).To(Equal([]Operation{a, b})) + }) + + It("rejects unknown requested names", func() { + registry, err := NewRegistry(&fakeOp{name: "known"}) + Expect(err).NotTo(HaveOccurred()) + _, err = registry.Resolve([]string{"unknown"}) + Expect(err).To(MatchError(ContainSubstring(`unknown name "unknown"`))) + }) + + It("rejects duplicate requested names", func() { + registry, err := NewRegistry(&fakeOp{name: "known"}) + Expect(err).NotTo(HaveOccurred()) + _, err = registry.Resolve([]string{"known", "known"}) + Expect(err).To(MatchError(ContainSubstring(`duplicate name "known"`))) + }) + + It("rejects duplicate registered operation names", func() { + _, err := NewRegistry(&fakeOp{name: "same"}, &fakeOp{name: "same"}) + Expect(err).To(MatchError(ContainSubstring(`duplicate name "same"`))) + }) + + It("constructs the default registry with the stable operation names", func() { + registry, err := NewDefaultRegistry(config.DefaultConfig(), nil, nil, nil, journal.New()) + Expect(err).NotTo(HaveOccurred()) + + names := make([]string, 0) + for _, op := range registry.All() { + names = append(names, op.Name()) + } + Expect(names).To(Equal([]string{ + "scale-up", + "scale-down", + "upgrade-documentdb", + "kill-operator-pod", + "kill-primary-pod", + })) + }) +}) diff --git a/test/longhaul/operations/runner.go b/test/longhaul/operations/runner.go new file mode 100644 index 000000000..1884290a4 --- /dev/null +++ b/test/longhaul/operations/runner.go @@ -0,0 +1,152 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package operations + +import ( + "context" + "sync" + + "github.com/documentdb/documentdb-operator/test/longhaul/config" +) + +// RunStatus is the bounded lifecycle state of the operation runner. +type RunStatus string + +const ( + RunStatusPending RunStatus = "PENDING" + RunStatusRunning RunStatus = "RUNNING" + RunStatusComplete RunStatus = "COMPLETE" + RunStatusFailed RunStatus = "FAILED" + RunStatusIncomplete RunStatus = "INCOMPLETE" + RunStatusDisabled RunStatus = "DISABLED" +) + +// OperationResultStatus is the state of one requested sequence operation. +type OperationResultStatus string + +const ( + OperationPending OperationResultStatus = "PENDING" + OperationRunning OperationResultStatus = "RUNNING" + OperationPassed OperationResultStatus = "PASSED" + OperationFailed OperationResultStatus = "FAILED" +) + +// OperationResult is the single mutable result for one requested sequence item. +type OperationResult struct { + Name string `json:"name"` + Status OperationResultStatus `json:"status"` + Error string `json:"error,omitempty"` +} + +// OperationAggregate bounds random-mode history to counters per operation type. +type OperationAggregate struct { + Name string `json:"name"` + Passed int `json:"passed"` + Failed int `json:"failed"` +} + +// RunSnapshot is a concurrency-safe value snapshot of operation execution. +type RunSnapshot struct { + Mode config.OperationMode `json:"mode"` + Status RunStatus `json:"status"` + Results []OperationResult `json:"results,omitempty"` + Aggregates []OperationAggregate `json:"aggregates,omitempty"` + FailureReason string `json:"failureReason,omitempty"` +} + +// OpsExecuted returns the number of terminal operation attempts. +func (s RunSnapshot) OpsExecuted() int { + if s.Mode == config.OperationModeSequence { + count := 0 + for _, result := range s.Results { + if result.Status == OperationPassed || result.Status == OperationFailed { + count++ + } + } + return count + } + + count := 0 + for _, aggregate := range s.Aggregates { + count += aggregate.Passed + aggregate.Failed + } + return count +} + +// HasFailure reports whether an operation attempt or sequence lifecycle failed. +func (s RunSnapshot) HasFailure() bool { + if s.Status == RunStatusFailed || s.Status == RunStatusIncomplete { + return true + } + for _, aggregate := range s.Aggregates { + if aggregate.Failed > 0 { + return true + } + } + return false +} + +// Runner is the common reporting and lifecycle surface for every operation mode. +type Runner interface { + Run(ctx context.Context) + Snapshot() RunSnapshot + Done() <-chan struct{} +} + +type runnerState struct { + mu sync.RWMutex + snapshot RunSnapshot + done chan struct{} + doneOnce sync.Once +} + +func newRunnerState(snapshot RunSnapshot) runnerState { + return runnerState{snapshot: snapshot, done: make(chan struct{})} +} + +func (s *runnerState) Snapshot() RunSnapshot { + s.mu.RLock() + defer s.mu.RUnlock() + snapshot := s.snapshot + snapshot.Results = append([]OperationResult(nil), s.snapshot.Results...) + snapshot.Aggregates = append([]OperationAggregate(nil), s.snapshot.Aggregates...) + return snapshot +} + +func (s *runnerState) Done() <-chan struct{} { + return s.done +} + +func (s *runnerState) closeDone() { + s.doneOnce.Do(func() { close(s.done) }) +} + +// DisabledRunner performs no operations and has no completion requirement. +type DisabledRunner struct { + state runnerState +} + +// NewDisabledRunner creates a runner for disabled operation mode. +func NewDisabledRunner() *DisabledRunner { + return &DisabledRunner{state: newRunnerState(RunSnapshot{ + Mode: config.OperationModeDisabled, + Status: RunStatusDisabled, + })} +} + +// Run waits for shutdown without scheduling operations. +func (r *DisabledRunner) Run(ctx context.Context) { + <-ctx.Done() + r.state.closeDone() +} + +// Snapshot returns the disabled runner state. +func (r *DisabledRunner) Snapshot() RunSnapshot { + return r.state.Snapshot() +} + +// Done closes when Run returns after cancellation. +func (r *DisabledRunner) Done() <-chan struct{} { + return r.state.Done() +} diff --git a/test/longhaul/operations/scale.go b/test/longhaul/operations/scale.go index 9a1672959..f002fdc61 100644 --- a/test/longhaul/operations/scale.go +++ b/test/longhaul/operations/scale.go @@ -77,6 +77,9 @@ type ScaleUp struct{ scaleOp } // NewScaleUp creates a ScaleUp operation. maxInstances is clamped to the // CRD upper bound (3) to avoid admission rejections. +// +// Scaling up only adds a standby replica (the primary and thus the write path +// is untouched), so it uses the near-zero NoOutagePolicy budget. func NewScaleUp(client monitor.ClusterClient, health *monitor.HealthMonitor, maxInstances int, recovery time.Duration) *ScaleUp { if maxInstances > 3 { maxInstances = 3 @@ -90,10 +93,7 @@ func NewScaleUp(client monitor.ClusterClient, health *monitor.HealthMonitor, max bound: maxInstances, boundKind: "max", recovery: recovery, - policy: journal.OutagePolicy{ - AllowedWriteFailures: 20, - MustRecoverWithin: recovery, - }, + policy: journal.NoOutagePolicy(recovery), }} } @@ -105,6 +105,10 @@ type ScaleDown struct{ scaleOp } // NewScaleDown creates a ScaleDown operation. minInstances is clamped to the // CRD lower bound (1) to avoid admission rejections. +// +// Scaling down removes the highest-ordinal standby (CNPG never removes the +// primary), so the write path stays up and it uses the same near-zero +// NoOutagePolicy budget as scale-up. func NewScaleDown(client monitor.ClusterClient, health *monitor.HealthMonitor, minInstances int, recovery time.Duration) *ScaleDown { if minInstances < 1 { minInstances = 1 @@ -118,10 +122,7 @@ func NewScaleDown(client monitor.ClusterClient, health *monitor.HealthMonitor, m bound: minInstances, boundKind: "min", recovery: recovery, - policy: journal.OutagePolicy{ - AllowedWriteFailures: 50, - MustRecoverWithin: recovery, - }, + policy: journal.NoOutagePolicy(recovery), }} } diff --git a/test/longhaul/operations/scale_test.go b/test/longhaul/operations/scale_test.go index 3df032c05..e929033d6 100644 --- a/test/longhaul/operations/scale_test.go +++ b/test/longhaul/operations/scale_test.go @@ -12,17 +12,23 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/documentdb/documentdb-operator/test/longhaul/journal" "github.com/documentdb/documentdb-operator/test/longhaul/monitor" ) // fakeClient is a minimal monitor.ClusterClient stub for unit tests. type fakeClient struct { - mu sync.Mutex - instancesPerNode int - ipnErr error - imageTag string - scaleCalls []int - upgradeCalls []string + mu sync.Mutex + instancesPerNode int + ipnErr error + imageTag string + scaleCalls []int + upgradeCalls []string + primary string + primaryErr error + replacementPrimary string + deleteErr error + deletedPods []string } func (f *fakeClient) GetClusterHealth(_ context.Context) (monitor.ClusterHealth, error) { @@ -51,6 +57,23 @@ func (f *fakeClient) UpgradeDocumentDB(_ context.Context, v string) error { f.upgradeCalls = append(f.upgradeCalls, v) return nil } +func (f *fakeClient) GetPrimaryInstance(_ context.Context) (string, error) { + f.mu.Lock() + defer f.mu.Unlock() + return f.primary, f.primaryErr +} +func (f *fakeClient) DeletePod(_ context.Context, name string) error { + f.mu.Lock() + defer f.mu.Unlock() + if f.deleteErr != nil { + return f.deleteErr + } + f.deletedPods = append(f.deletedPods, name) + if f.replacementPrimary != "" { + f.primary = f.replacementPrimary + } + return nil +} var _ = Describe("ScaleUp", func() { DescribeTable("clamps maxInstances to the CRD upper bound", @@ -87,10 +110,10 @@ var _ = Describe("ScaleUp", func() { Entry("blocked: ipn read error", 0, errors.New("apiserver down"), 3, false, "cannot get instancesPerNode"), ) - It("OutagePolicy uses tighter budgets and echoes MustRecoverWithin", func() { + It("OutagePolicy uses the near-zero NoOutagePolicy budget and echoes MustRecoverWithin", func() { s := NewScaleUp(&fakeClient{}, nil, 3, 5*time.Minute) p := s.OutagePolicy() - Expect(p.AllowedWriteFailures).To(Equal(int64(20))) + Expect(p.MaxWriteOutage).To(Equal(journal.NoOutageWriteOutageCushion)) Expect(p.MustRecoverWithin).To(Equal(5 * time.Minute)) }) }) @@ -130,9 +153,9 @@ var _ = Describe("ScaleDown", func() { Entry("blocked: ipn read error", 0, errors.New("apiserver down"), 1, false, "cannot get instancesPerNode"), ) - It("OutagePolicy is more lenient than scale-up", func() { + It("OutagePolicy shares the near-zero NoOutagePolicy budget with scale-up", func() { s := NewScaleDown(&fakeClient{}, nil, 1, 5*time.Minute) p := s.OutagePolicy() - Expect(p.AllowedWriteFailures).To(Equal(int64(50))) + Expect(p.MaxWriteOutage).To(Equal(journal.NoOutageWriteOutageCushion)) }) }) diff --git a/test/longhaul/operations/scheduler.go b/test/longhaul/operations/scheduler.go index a9351d18d..464f3afe4 100644 --- a/test/longhaul/operations/scheduler.go +++ b/test/longhaul/operations/scheduler.go @@ -1,8 +1,8 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -// Package operations implements the operation scheduler and individual -// disruptive operations for long haul tests. +// Package operations implements operation runners and individual disruptive +// operations for long haul tests. package operations import ( @@ -12,6 +12,7 @@ import ( "sync" "time" + "github.com/documentdb/documentdb-operator/test/longhaul/config" "github.com/documentdb/documentdb-operator/test/longhaul/journal" "github.com/documentdb/documentdb-operator/test/longhaul/monitor" ) @@ -46,6 +47,9 @@ type Scheduler struct { lastOpTime time.Time opsExecuted int inProgress bool + + state runnerState + aggregateIndex map[string]int } // NewScheduler creates an operation scheduler. @@ -55,18 +59,44 @@ func NewScheduler( j *journal.Journal, cooldown time.Duration, ) *Scheduler { + aggregates := make([]OperationAggregate, 0, len(ops)) + aggregateIndex := make(map[string]int, len(ops)) + for _, op := range ops { + if _, exists := aggregateIndex[op.Name()]; exists { + continue + } + aggregateIndex[op.Name()] = len(aggregates) + aggregates = append(aggregates, OperationAggregate{Name: op.Name()}) + } return &Scheduler{ operations: ops, healthMonitor: health, journal: j, cooldown: cooldown, + state: newRunnerState(RunSnapshot{ + Mode: config.OperationModeRandom, + Status: RunStatusPending, + Aggregates: aggregates, + }), + aggregateIndex: aggregateIndex, } } // Run starts the scheduler loop. It blocks until context is cancelled. func (s *Scheduler) Run(ctx context.Context) { s.journal.Info("scheduler", "operation scheduler started") - defer s.journal.Info("scheduler", "operation scheduler stopped") + s.state.mu.Lock() + s.state.snapshot.Status = RunStatusRunning + s.state.mu.Unlock() + defer func() { + s.state.mu.Lock() + if s.state.snapshot.Status == RunStatusRunning { + s.state.snapshot.Status = RunStatusComplete + } + s.state.mu.Unlock() + s.state.closeDone() + s.journal.Info("scheduler", "operation scheduler stopped") + }() ticker := time.NewTicker(10 * time.Second) defer ticker.Stop() @@ -76,34 +106,41 @@ func (s *Scheduler) Run(ctx context.Context) { case <-ctx.Done(): return case <-ticker.C: - s.tryExecute(ctx) + if err := s.tryExecute(ctx); err != nil { + // A terminal operation failure ends the run immediately so the + // FAIL verdict is emitted promptly. In production MaxDuration is + // unbounded, so without this the loop would run forever and the + // failure would never surface. + s.journal.Error("scheduler", fmt.Sprintf("halting run after operation failure: %v", err)) + return + } } } } -func (s *Scheduler) tryExecute(ctx context.Context) { +func (s *Scheduler) tryExecute(ctx context.Context) error { s.mu.Lock() if s.inProgress { s.mu.Unlock() - return + return nil } // Check cooldown. if !s.lastOpTime.IsZero() && time.Since(s.lastOpTime) < s.cooldown { s.mu.Unlock() - return + return nil } s.mu.Unlock() // Check steady-state gate. if !s.healthMonitor.IsSteadyState() { - return + return nil } // Select an operation. op := s.selectOperation(ctx) if op == nil { - return + return nil } // Execute. @@ -111,13 +148,16 @@ func (s *Scheduler) tryExecute(ctx context.Context) { s.inProgress = true s.mu.Unlock() - s.executeOp(ctx, op) + err := s.executeOp(ctx, op) s.mu.Lock() s.inProgress = false s.lastOpTime = time.Now() s.opsExecuted++ s.mu.Unlock() + + s.recordExecution(op.Name(), err) + return err } func (s *Scheduler) selectOperation(ctx context.Context) Operation { @@ -153,19 +193,54 @@ func (s *Scheduler) selectOperation(ctx context.Context) Operation { return candidates[len(candidates)-1].op } -func (s *Scheduler) executeOp(ctx context.Context, op Operation) { +func (s *Scheduler) executeOp(ctx context.Context, op Operation) error { s.journal.Info("scheduler", fmt.Sprintf("executing operation: %s", op.Name())) s.journal.OpenDisruptionWindow(op.Name(), op.OutagePolicy()) + // Reset the steady-state epoch so the operation's internal recovery wait + // and the next scheduler-tick steady-state gate must observe a health + // sample taken after this disruption, not a stale pre-operation one. + if s.healthMonitor != nil { + s.healthMonitor.InvalidateSteadyState() + } err := op.Execute(ctx) - - s.journal.CloseDisruptionWindow() + window := s.journal.CloseDisruptionWindow() if err != nil { s.journal.Error("scheduler", fmt.Sprintf("operation %s failed: %v", op.Name(), err)) - } else { - s.journal.Info("scheduler", fmt.Sprintf("operation %s completed successfully", op.Name())) + return fmt.Errorf("operation %s execute failed: %w", op.Name(), err) } + if window == nil { + err = fmt.Errorf("operation %s closed without a disruption window", op.Name()) + s.journal.Error("scheduler", err.Error()) + return err + } + if window.ExceededPolicy() { + err = fmt.Errorf("operation %s exceeded its outage policy", op.Name()) + s.journal.Error("scheduler", err.Error()) + return err + } + + s.journal.Info("scheduler", fmt.Sprintf("operation %s completed successfully", op.Name())) + return nil +} + +func (s *Scheduler) recordExecution(name string, err error) { + s.state.mu.Lock() + defer s.state.mu.Unlock() + index, ok := s.aggregateIndex[name] + if !ok { + return + } + if err != nil { + s.state.snapshot.Aggregates[index].Failed++ + s.state.snapshot.Status = RunStatusFailed + if s.state.snapshot.FailureReason == "" { + s.state.snapshot.FailureReason = err.Error() + } + return + } + s.state.snapshot.Aggregates[index].Passed++ } // OpsExecuted returns the number of operations completed. @@ -174,3 +249,13 @@ func (s *Scheduler) OpsExecuted() int { defer s.mu.Unlock() return s.opsExecuted } + +// Snapshot returns bounded aggregate counters in registration order. +func (s *Scheduler) Snapshot() RunSnapshot { + return s.state.Snapshot() +} + +// Done closes when the scheduler stops after context cancellation. +func (s *Scheduler) Done() <-chan struct{} { + return s.state.Done() +} diff --git a/test/longhaul/operations/scheduler_test.go b/test/longhaul/operations/scheduler_test.go index cd351eb9c..e1df7b3d4 100644 --- a/test/longhaul/operations/scheduler_test.go +++ b/test/longhaul/operations/scheduler_test.go @@ -102,7 +102,8 @@ var _ = Describe("Scheduler", func() { It("records an ERROR event when Execute fails", func() { op := &fakeOp{name: "boom", weight: 1, available: true, err: errors.New("kaboom")} s := newSchedulerForTest(op) - s.executeOp(context.Background(), op) + err := s.executeOp(context.Background(), op) + Expect(err).To(HaveOccurred(), "executeOp must surface the failure so Run can halt the run") var sawError bool for _, e := range s.journal.Events() { @@ -120,4 +121,25 @@ var _ = Describe("Scheduler", func() { s.opsExecuted = 7 Expect(s.OpsExecuted()).To(Equal(7)) }) + + It("keeps bounded aggregate counters and exposes failures", func() { + a := &fakeOp{name: "a"} + b := &fakeOp{name: "b"} + s := NewScheduler([]Operation{a, b}, nil, journal.New(), time.Hour) + + for i := 0; i < 1000; i++ { + s.recordExecution("a", nil) + } + s.recordExecution("b", errors.New("first failure")) + s.recordExecution("b", errors.New("second failure")) + + snapshot := s.Snapshot() + Expect(snapshot.Aggregates).To(Equal([]OperationAggregate{ + {Name: "a", Passed: 1000}, + {Name: "b", Failed: 2}, + })) + Expect(snapshot.Status).To(Equal(RunStatusFailed)) + Expect(snapshot.HasFailure()).To(BeTrue()) + Expect(snapshot.FailureReason).To(ContainSubstring("first failure")) + }) }) diff --git a/test/longhaul/operations/sequence.go b/test/longhaul/operations/sequence.go new file mode 100644 index 000000000..5bbbaade6 --- /dev/null +++ b/test/longhaul/operations/sequence.go @@ -0,0 +1,253 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package operations + +import ( + "context" + "fmt" + "time" + + "github.com/documentdb/documentdb-operator/test/longhaul/config" + "github.com/documentdb/documentdb-operator/test/longhaul/journal" +) + +const defaultPreconditionPollInterval = time.Second + +// SteadyStateGate is the health-monitor surface needed by sequence mode. +type SteadyStateGate interface { + WaitForSteadyState(ctx context.Context) error + InvalidateSteadyState() +} + +type preconditionWaitFunc func(context.Context, Operation) error + +// SequenceRunner executes each requested operation exactly once and in order. +type SequenceRunner struct { + operations []Operation + steadyStateGate SteadyStateGate + journal *journal.Journal + recoveryTimeout time.Duration + state runnerState + + waitForPrecondition preconditionWaitFunc + terminal bool +} + +// NewSequenceRunner creates a deterministic sequential operation runner. +func NewSequenceRunner( + ops []Operation, + gate SteadyStateGate, + j *journal.Journal, + recoveryTimeout time.Duration, +) *SequenceRunner { + results := make([]OperationResult, len(ops)) + for i, op := range ops { + results[i] = OperationResult{Name: op.Name(), Status: OperationPending} + } + runner := &SequenceRunner{ + operations: append([]Operation(nil), ops...), + steadyStateGate: gate, + journal: j, + recoveryTimeout: recoveryTimeout, + state: newRunnerState(RunSnapshot{ + Mode: config.OperationModeSequence, + Status: RunStatusPending, + Results: results, + }), + } + runner.waitForPrecondition = runner.pollPrecondition + return runner +} + +// Run executes the configured sequence and stops on the first failure. +func (r *SequenceRunner) Run(ctx context.Context) { + defer r.state.closeDone() + + r.state.mu.Lock() + if r.terminal { + r.state.mu.Unlock() + return + } + r.state.snapshot.Status = RunStatusRunning + r.state.mu.Unlock() + + for i, op := range r.operations { + if !r.setResult(i, OperationRunning, "") { + return + } + if err := r.runOne(ctx, op); err != nil { + status := RunStatusFailed + reason := err.Error() + if ctx.Err() != nil { + status = RunStatusIncomplete + reason = fmt.Sprintf("operation sequence incomplete during %s: %v", op.Name(), ctx.Err()) + } + r.setFailure(i, status, reason) + return + } + if !r.setResult(i, OperationPassed, "") { + return + } + } + + r.state.mu.Lock() + if !r.terminal { + r.terminal = true + r.state.snapshot.Status = RunStatusComplete + } + r.state.mu.Unlock() +} + +func (r *SequenceRunner) runOne(ctx context.Context, op Operation) error { + if r.steadyStateGate == nil { + return fmt.Errorf("operation %s steady-state gate is nil", op.Name()) + } + + steadyCtx, cancelSteady := context.WithTimeout(ctx, r.recoveryTimeout) + err := r.steadyStateGate.WaitForSteadyState(steadyCtx) + cancelSteady() + if err != nil { + return fmt.Errorf("operation %s initial steady-state gate failed: %w", op.Name(), err) + } + + preconditionCtx, cancelPrecondition := context.WithTimeout(ctx, r.recoveryTimeout) + err = r.waitForPrecondition(preconditionCtx, op) + cancelPrecondition() + if err != nil { + return fmt.Errorf("operation %s precondition timeout: %w", op.Name(), err) + } + + r.journal.Info("sequence", fmt.Sprintf("executing operation: %s", op.Name())) + r.journal.OpenDisruptionWindow(op.Name(), op.OutagePolicy()) + // Reset the steady-state epoch so the post-recovery gate (and the + // operation's own internal steady-state wait) must observe a health + // sample taken after the disruption is opened, rather than being + // satisfied instantly by the pre-operation steadySince. + r.steadyStateGate.InvalidateSteadyState() + + executeCtx, cancelExecute := context.WithTimeout(ctx, r.recoveryTimeout) + executeErr := op.Execute(executeCtx) + cancelExecute() + window := r.journal.CloseDisruptionWindow() + + if executeErr != nil { + r.journal.Error("sequence", fmt.Sprintf("operation %s failed: %v", op.Name(), executeErr)) + return fmt.Errorf("operation %s execute failed: %w", op.Name(), executeErr) + } + if window == nil { + return fmt.Errorf("operation %s closed without a disruption window", op.Name()) + } + if window.ExceededPolicy() { + err := fmt.Errorf("operation %s exceeded its outage policy", op.Name()) + r.journal.Error("sequence", err.Error()) + return err + } + + recoveryCtx, cancelRecovery := context.WithTimeout(ctx, r.recoveryTimeout) + err = r.steadyStateGate.WaitForSteadyState(recoveryCtx) + cancelRecovery() + if err != nil { + return fmt.Errorf("operation %s post-recovery steady-state gate failed: %w", op.Name(), err) + } + + r.journal.Info("sequence", fmt.Sprintf("operation %s completed successfully", op.Name())) + return nil +} + +func (r *SequenceRunner) pollPrecondition(ctx context.Context, op Operation) error { + ticker := time.NewTicker(defaultPreconditionPollInterval) + defer ticker.Stop() + + lastReason := "precondition not met" + for { + ok, reason := op.Precondition(ctx) + if ok { + return nil + } + if reason != "" { + lastReason = reason + } + + select { + case <-ctx.Done(): + return fmt.Errorf("%s: %w", lastReason, ctx.Err()) + case <-ticker.C: + } + } +} + +func (r *SequenceRunner) setResult(index int, status OperationResultStatus, reason string) bool { + r.state.mu.Lock() + defer r.state.mu.Unlock() + if r.terminal { + return false + } + r.state.snapshot.Results[index].Status = status + r.state.snapshot.Results[index].Error = reason + return true +} + +func (r *SequenceRunner) setFailure(index int, status RunStatus, reason string) { + r.state.mu.Lock() + defer r.state.mu.Unlock() + if r.terminal { + return + } + r.terminal = true + r.state.snapshot.Status = status + r.state.snapshot.FailureReason = reason + r.state.snapshot.Results[index].Status = OperationFailed + r.state.snapshot.Results[index].Error = reason +} + +// MarkIncomplete terminally fails a sequence whose watchdog or shutdown +// cancellation fired before Run could publish its own terminal snapshot. +func (r *SequenceRunner) MarkIncomplete(reason string) { + r.state.mu.Lock() + if r.terminal { + r.state.mu.Unlock() + return + } + allPassed := len(r.state.snapshot.Results) > 0 + for _, result := range r.state.snapshot.Results { + if result.Status != OperationPassed { + allPassed = false + break + } + } + if allPassed { + r.terminal = true + r.state.snapshot.Status = RunStatusComplete + r.state.mu.Unlock() + r.state.closeDone() + return + } + r.terminal = true + r.state.snapshot.Status = RunStatusIncomplete + r.state.snapshot.FailureReason = reason + for i := range r.state.snapshot.Results { + if r.state.snapshot.Results[i].Status == OperationRunning { + r.state.snapshot.Results[i].Status = OperationFailed + r.state.snapshot.Results[i].Error = reason + break + } + if r.state.snapshot.Results[i].Status == OperationPending { + r.state.snapshot.Results[i].Status = OperationFailed + r.state.snapshot.Results[i].Error = reason + break + } + } + r.state.mu.Unlock() + r.state.closeDone() +} + +// Snapshot returns a deterministic copy ordered by the configured sequence. +func (r *SequenceRunner) Snapshot() RunSnapshot { + return r.state.Snapshot() +} + +// Done closes when the sequence completes or stops on failure/cancellation. +func (r *SequenceRunner) Done() <-chan struct{} { + return r.state.Done() +} diff --git a/test/longhaul/operations/sequence_test.go b/test/longhaul/operations/sequence_test.go new file mode 100644 index 000000000..f08ede0b7 --- /dev/null +++ b/test/longhaul/operations/sequence_test.go @@ -0,0 +1,219 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package operations + +import ( + "context" + "errors" + "sync" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/documentdb/documentdb-operator/test/longhaul/journal" +) + +type sequenceTestOp struct { + name string + execute func(context.Context) error + precondition func(context.Context) (bool, string) + policy journal.OutagePolicy +} + +func (o *sequenceTestOp) Name() string { return o.name } +func (o *sequenceTestOp) Weight() int { return 1 } +func (o *sequenceTestOp) Precondition(ctx context.Context) (bool, string) { + if o.precondition != nil { + return o.precondition(ctx) + } + return true, "" +} +func (o *sequenceTestOp) Execute(ctx context.Context) error { + if o.execute != nil { + return o.execute(ctx) + } + return nil +} +func (o *sequenceTestOp) OutagePolicy() journal.OutagePolicy { + if o.policy.MustRecoverWithin != 0 || o.policy.MaxWriteOutage != 0 { + return o.policy + } + return journal.DefaultOutagePolicy() +} + +type sequenceTestGate struct { + mu sync.Mutex + calls int + err error +} + +func (g *sequenceTestGate) WaitForSteadyState(context.Context) error { + g.mu.Lock() + defer g.mu.Unlock() + g.calls++ + return g.err +} + +func (g *sequenceTestGate) InvalidateSteadyState() {} + +func runSequence(runner *SequenceRunner, ctx context.Context) RunSnapshot { + go runner.Run(ctx) + Eventually(runner.Done()).Should(BeClosed()) + return runner.Snapshot() +} + +var _ = Describe("SequenceRunner", func() { + It("executes each operation exactly once in exact order", func() { + var order []string + ops := []Operation{ + &sequenceTestOp{name: "first", execute: func(context.Context) error { + order = append(order, "first") + return nil + }}, + &sequenceTestOp{name: "second", execute: func(context.Context) error { + order = append(order, "second") + return nil + }}, + } + gate := &sequenceTestGate{} + snapshot := runSequence(NewSequenceRunner(ops, gate, journal.New(), time.Second), context.Background()) + + Expect(order).To(Equal([]string{"first", "second"})) + Expect(snapshot.Status).To(Equal(RunStatusComplete)) + Expect(snapshot.Results).To(Equal([]OperationResult{ + {Name: "first", Status: OperationPassed}, + {Name: "second", Status: OperationPassed}, + })) + Expect(gate.calls).To(Equal(4), "initial and post-recovery gate for each operation") + }) + + It("records an execute error and stops before later operations", func() { + var order []string + ops := []Operation{ + &sequenceTestOp{name: "first", execute: func(context.Context) error { + order = append(order, "first") + return nil + }}, + &sequenceTestOp{name: "broken", execute: func(context.Context) error { + order = append(order, "broken") + return errors.New("kaboom") + }}, + &sequenceTestOp{name: "never", execute: func(context.Context) error { + order = append(order, "never") + return nil + }}, + } + snapshot := runSequence( + NewSequenceRunner(ops, &sequenceTestGate{}, journal.New(), time.Second), + context.Background(), + ) + + Expect(order).To(Equal([]string{"first", "broken"})) + Expect(snapshot.Status).To(Equal(RunStatusFailed)) + Expect(snapshot.FailureReason).To(ContainSubstring("kaboom")) + Expect(snapshot.Results).To(Equal([]OperationResult{ + {Name: "first", Status: OperationPassed}, + {Name: "broken", Status: OperationFailed, Error: snapshot.FailureReason}, + {Name: "never", Status: OperationPending}, + })) + }) + + It("fails deterministically when a precondition times out", func() { + op := &sequenceTestOp{name: "blocked"} + runner := NewSequenceRunner([]Operation{op}, &sequenceTestGate{}, journal.New(), time.Second) + runner.waitForPrecondition = func(context.Context, Operation) error { + return context.DeadlineExceeded + } + + snapshot := runSequence(runner, context.Background()) + Expect(snapshot.Status).To(Equal(RunStatusFailed)) + Expect(snapshot.Results[0].Status).To(Equal(OperationFailed)) + Expect(snapshot.FailureReason).To(ContainSubstring("precondition timeout")) + }) + + It("marks cancellation as incomplete and leaves later operations pending", func() { + started := make(chan struct{}) + op := &sequenceTestOp{name: "cancelled", execute: func(ctx context.Context) error { + close(started) + <-ctx.Done() + return ctx.Err() + }} + runner := NewSequenceRunner( + []Operation{op, &sequenceTestOp{name: "never"}}, + &sequenceTestGate{}, + journal.New(), + time.Minute, + ) + ctx, cancel := context.WithCancel(context.Background()) + go runner.Run(ctx) + Eventually(started).Should(BeClosed()) + cancel() + Eventually(runner.Done()).Should(BeClosed()) + + snapshot := runner.Snapshot() + Expect(snapshot.Status).To(Equal(RunStatusIncomplete)) + Expect(snapshot.Results[0].Status).To(Equal(OperationFailed)) + Expect(snapshot.Results[1].Status).To(Equal(OperationPending)) + Expect(snapshot.FailureReason).To(ContainSubstring("incomplete")) + }) + + It("publishes a terminal incomplete snapshot when the watchdog wins", func() { + runner := NewSequenceRunner( + []Operation{ + &sequenceTestOp{name: "first"}, + &sequenceTestOp{name: "second"}, + }, + &sequenceTestGate{}, + journal.New(), + time.Minute, + ) + + runner.MarkIncomplete("watchdog fired") + Expect(runner.Done()).To(BeClosed()) + snapshot := runner.Snapshot() + Expect(snapshot.Status).To(Equal(RunStatusIncomplete)) + Expect(snapshot.FailureReason).To(Equal("watchdog fired")) + Expect(snapshot.Results).To(Equal([]OperationResult{ + {Name: "first", Status: OperationFailed, Error: "watchdog fired"}, + {Name: "second", Status: OperationPending}, + })) + }) + + It("preserves completion when the watchdog races after every operation passed", func() { + runner := NewSequenceRunner( + []Operation{&sequenceTestOp{name: "done"}}, + &sequenceTestGate{}, + journal.New(), + time.Second, + ) + runner.state.snapshot.Results[0].Status = OperationPassed + + runner.MarkIncomplete("watchdog fired") + + snapshot := runner.Snapshot() + Expect(snapshot.Status).To(Equal(RunStatusComplete)) + Expect(snapshot.Results).To(Equal([]OperationResult{{ + Name: "done", + Status: OperationPassed, + }})) + }) + + It("fails when the closed disruption window exceeds policy", func() { + op := &sequenceTestOp{ + name: "policy", + policy: journal.OutagePolicy{ + MaxWriteOutage: time.Hour, + MustRecoverWithin: -time.Nanosecond, + }, + } + snapshot := runSequence( + NewSequenceRunner([]Operation{op}, &sequenceTestGate{}, journal.New(), time.Second), + context.Background(), + ) + + Expect(snapshot.Status).To(Equal(RunStatusFailed)) + Expect(snapshot.FailureReason).To(ContainSubstring("exceeded its outage policy")) + }) +}) diff --git a/test/longhaul/operations/upgrade.go b/test/longhaul/operations/upgrade.go index f9a586b6e..4bb531626 100644 --- a/test/longhaul/operations/upgrade.go +++ b/test/longhaul/operations/upgrade.go @@ -170,11 +170,14 @@ func (u *UpgradeDocumentDB) readDesiredVersion(ctx context.Context) (string, err return cm.Data[VersionConfigMapKey], nil } -// OutagePolicy allows for a longer disruption window during an upgrade -// because rolling restarts touch every pod sequentially. +// OutagePolicy bounds the write outage of a rolling upgrade. Standby restarts +// do not block writes; the write path is only interrupted during the single +// primary switchover. That switchover is heavier than a plain failover because +// it coincides with the extension version migration under live write load and +// the new primary must come up on the new image before accepting writes, so the +// upgrade uses its own (larger) write-outage budget rather than sharing the +// kill-primary-pod one (see journal.UpgradeOutagePolicy). The upgrade's longer +// whole-topology restart is bounded separately by MustRecoverWithin. func (u *UpgradeDocumentDB) OutagePolicy() journal.OutagePolicy { - return journal.OutagePolicy{ - AllowedWriteFailures: 200, - MustRecoverWithin: u.recovery, - } + return journal.UpgradeOutagePolicy(u.recovery) } diff --git a/test/longhaul/operations/upgrade_test.go b/test/longhaul/operations/upgrade_test.go index f442e5b58..ec5763242 100644 --- a/test/longhaul/operations/upgrade_test.go +++ b/test/longhaul/operations/upgrade_test.go @@ -10,6 +10,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/documentdb/documentdb-operator/test/longhaul/journal" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/fake" @@ -22,10 +23,10 @@ var _ = Describe("UpgradeDocumentDB", func() { Expect(u.Weight()).To(Equal(1)) }) - It("OutagePolicy gives upgrades a more lenient failure budget", func() { + It("OutagePolicy uses the dedicated cross-version upgrade write-outage budget", func() { u := NewUpgradeDocumentDB(&fakeClient{}, fake.NewSimpleClientset(), nil, nil, "ns", 10*time.Minute) p := u.OutagePolicy() - Expect(p.AllowedWriteFailures).To(Equal(int64(200))) + Expect(p.MaxWriteOutage).To(Equal(journal.UpgradeWriteOutage)) Expect(p.MustRecoverWithin).To(Equal(10 * time.Minute)) }) diff --git a/test/longhaul/report/checkpoint.go b/test/longhaul/report/checkpoint.go index 5875412c7..d6d28130f 100644 --- a/test/longhaul/report/checkpoint.go +++ b/test/longhaul/report/checkpoint.go @@ -8,8 +8,11 @@ import ( "encoding/json" "fmt" "log" + "sync" "time" + "github.com/documentdb/documentdb-operator/test/longhaul/operations" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -21,8 +24,9 @@ const ( ConfigMapName = "longhaul-report" ) -// SummaryFunc is called to generate the current test summary. -type SummaryFunc func() Summary +// SummaryFunc is called to generate the current test summary. final is true +// only for the terminal emit, when incomplete sequence execution must fail. +type SummaryFunc func(final bool) Summary // CheckpointReporter periodically generates and persists reports. type CheckpointReporter struct { @@ -30,6 +34,10 @@ type CheckpointReporter struct { namespace string interval time.Duration summaryFunc SummaryFunc + + emitMu sync.Mutex + finalEmitted bool + finalSummary Summary } // NewCheckpointReporter creates a periodic reporter that writes to stdout and ConfigMap. @@ -67,18 +75,31 @@ func (r *CheckpointReporter) Run(ctx context.Context) { // not as RUNNING) using a bounded context. Safe to call after the main // context has been cancelled. Intended to be called synchronously from main // just before exit so the verdict is durable in the ConfigMap. -func (r *CheckpointReporter) EmitFinal() { +func (r *CheckpointReporter) EmitFinal() Summary { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() - r.emit(ctx, true) + return r.emit(ctx, true) } // emit writes the current summary to stdout, GH Actions annotations, and the // status ConfigMap. final=true means this is the shutdown emit, in which case // PASS is persisted as "PASS" (not "RUNNING") so consumers can distinguish a // finished clean run from an in-flight checkpoint. -func (r *CheckpointReporter) emit(ctx context.Context, final bool) { - summary := r.summaryFunc() +func (r *CheckpointReporter) emit(ctx context.Context, final bool) Summary { + r.emitMu.Lock() + defer r.emitMu.Unlock() + if final && r.finalEmitted { + return r.finalSummary + } + if !final && r.finalEmitted { + return Summary{} + } + + summary := r.summaryFunc(final) + if final { + r.finalEmitted = true + r.finalSummary = summary + } // Intermediate PASS checkpoints surface as RUNNING; the final emit // preserves the true PASS/FAIL outcome. @@ -99,13 +120,18 @@ func (r *CheckpointReporter) emit(ctx context.Context, final bool) { // Persist to ConfigMap. if r.clientset == nil { - return + return summary } data := map[string]string{ - "latest-report": markdown, - "last-updated": time.Now().UTC().Format(time.RFC3339), - "result": resultStr, + "latest-report": markdown, + "last-updated": time.Now().UTC().Format(time.RFC3339), + "result": resultStr, + "operation-status": string(summary.OperationRun.Status), + "operation-results": marshalOperationResults(summary.OperationRun.Results), + } + if len(summary.OperationRun.Aggregates) > 0 { + data["operation-aggregates"] = marshalOperationAggregates(summary.OperationRun.Aggregates) } cm := &corev1.ConfigMap{ @@ -142,14 +168,32 @@ func (r *CheckpointReporter) emit(ctx context.Context, final bool) { // Also log the summary as JSON for structured log consumers. summaryJSON, _ := json.Marshal(map[string]any{ - "result": resultStr, - "elapsed": summary.Duration.String(), - "writes": summary.Metrics.WriteAttempted, - "gaps": summary.Metrics.GapsDetected, - "ops": summary.OpsExecuted, - "memory_leak": summary.LeakAnalysis.HasLeak, - "memory_slope": fmt.Sprintf("%.2f MB/h", summary.LeakAnalysis.MemorySlopeMB), - "checkpoint_time": time.Now().UTC().Format(time.RFC3339), + "result": resultStr, + "elapsed": summary.Duration.String(), + "writes": summary.Metrics.WriteAttempted, + "gaps": summary.Metrics.GapsDetected, + "ops": summary.OpsExecuted, + "memory_leak": summary.LeakAnalysis.HasLeak, + "memory_slope": fmt.Sprintf("%.2f MB/h", summary.LeakAnalysis.MemorySlopeMB), + "operation_status": summary.OperationRun.Status, + "checkpoint_time": time.Now().UTC().Format(time.RFC3339), }) log.Printf("[checkpoint] %s", string(summaryJSON)) + return summary +} + +func marshalOperationResults(results []operations.OperationResult) string { + if results == nil { + results = []operations.OperationResult{} + } + data, _ := json.Marshal(results) + return string(data) +} + +func marshalOperationAggregates(aggregates []operations.OperationAggregate) string { + if aggregates == nil { + aggregates = []operations.OperationAggregate{} + } + data, _ := json.Marshal(aggregates) + return string(data) } diff --git a/test/longhaul/report/checkpoint_test.go b/test/longhaul/report/checkpoint_test.go index 891e90b4d..d9c80f7ef 100644 --- a/test/longhaul/report/checkpoint_test.go +++ b/test/longhaul/report/checkpoint_test.go @@ -5,6 +5,7 @@ package report import ( "context" + "encoding/json" "time" . "github.com/onsi/ginkgo/v2" @@ -12,11 +13,14 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes/fake" + + "github.com/documentdb/documentdb-operator/test/longhaul/config" + "github.com/documentdb/documentdb-operator/test/longhaul/operations" ) var _ = Describe("CheckpointReporter", func() { It("emit() is safe with a nil clientset (logs to stdout, does not panic)", func() { - r := NewCheckpointReporter(nil, "ns", time.Second, func() Summary { + r := NewCheckpointReporter(nil, "ns", time.Second, func(bool) Summary { return Summary{Result: ResultPass, Duration: time.Minute} }) Expect(func() { r.emit(context.Background(), false) }).NotTo(Panic()) @@ -24,7 +28,7 @@ var _ = Describe("CheckpointReporter", func() { It("creates the ConfigMap on first emit and labels it identifiably", func() { cs := fake.NewSimpleClientset() - r := NewCheckpointReporter(cs, "ns", time.Second, func() Summary { + r := NewCheckpointReporter(cs, "ns", time.Second, func(bool) Summary { return Summary{Result: ResultPass, Duration: 2 * time.Hour, OpsExecuted: 5} }) @@ -35,6 +39,8 @@ var _ = Describe("CheckpointReporter", func() { Expect(cm.Data).To(HaveKey("latest-report")) Expect(cm.Data).To(HaveKey("last-updated")) Expect(cm.Data).To(HaveKey("result")) + Expect(cm.Data).To(HaveKeyWithValue("operation-status", "")) + Expect(cm.Data).To(HaveKeyWithValue("operation-results", "[]")) // PASS at intermediate checkpoint is persisted as RUNNING so consumers // can distinguish in-flight from final state. Expect(cm.Data["result"]).To(Equal("RUNNING")) @@ -43,7 +49,7 @@ var _ = Describe("CheckpointReporter", func() { It("persists FAIL results as FAIL", func() { cs := fake.NewSimpleClientset() - r := NewCheckpointReporter(cs, "ns", time.Second, func() Summary { + r := NewCheckpointReporter(cs, "ns", time.Second, func(bool) Summary { return Summary{Result: ResultFail, FailReason: "data loss"} }) @@ -58,7 +64,7 @@ var _ = Describe("CheckpointReporter", func() { cs := fake.NewSimpleClientset() calls := 0 - r := NewCheckpointReporter(cs, "ns", time.Second, func() Summary { + r := NewCheckpointReporter(cs, "ns", time.Second, func(bool) Summary { calls++ return Summary{Result: ResultPass, Duration: time.Duration(calls) * time.Hour, OpsExecuted: calls * 10} }) @@ -76,4 +82,87 @@ var _ = Describe("CheckpointReporter", func() { Expect(cm2.Data["latest-report"]).NotTo(Equal(report1)) Expect(calls).To(Equal(2)) }) + + It("persists ordered sequence results as bounded JSON", func() { + cs := fake.NewSimpleClientset() + results := []operations.OperationResult{ + {Name: "kill-operator-pod", Status: operations.OperationPassed}, + {Name: "kill-primary-pod", Status: operations.OperationPassed}, + } + r := NewCheckpointReporter(cs, "ns", time.Second, func(bool) Summary { + return Summary{ + Result: ResultPass, + OperationRun: operations.RunSnapshot{ + Mode: config.OperationModeSequence, + Status: operations.RunStatusComplete, + Results: results, + }, + } + }) + + r.emit(context.Background(), true) + cm, err := cs.CoreV1().ConfigMaps("ns").Get(context.Background(), ConfigMapName, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(cm.Data["operation-status"]).To(Equal("COMPLETE")) + + var persisted []operations.OperationResult + Expect(json.Unmarshal([]byte(cm.Data["operation-results"]), &persisted)).To(Succeed()) + Expect(persisted).To(Equal(results)) + Expect(cm.Data).NotTo(HaveKey("operation-aggregates")) + }) + + It("overwrites mode-specific fields instead of retaining stale aggregates", func() { + cs := fake.NewSimpleClientset() + random := true + r := NewCheckpointReporter(cs, "ns", time.Second, func(bool) Summary { + if random { + return Summary{ + Result: ResultPass, + OperationRun: operations.RunSnapshot{ + Mode: config.OperationModeRandom, + Status: operations.RunStatusRunning, + Aggregates: []operations.OperationAggregate{{Name: "scale-up", Passed: 3}}, + }, + } + } + return Summary{ + Result: ResultPass, + OperationRun: operations.RunSnapshot{ + Mode: config.OperationModeSequence, + Status: operations.RunStatusComplete, + Results: []operations.OperationResult{{Name: "scale-up", Status: operations.OperationPassed}}, + }, + } + }) + + r.emit(context.Background(), false) + random = false + r.emit(context.Background(), true) + + cm, err := cs.CoreV1().ConfigMaps("ns").Get(context.Background(), ConfigMapName, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(cm.Data).NotTo(HaveKey("operation-aggregates")) + Expect(cm.Data["operation-results"]).To(MatchJSON(`[{"name":"scale-up","status":"PASSED"}]`)) + }) + + It("emits the final report exactly once and rejects later checkpoints", func() { + cs := fake.NewSimpleClientset() + calls := 0 + r := NewCheckpointReporter(cs, "ns", time.Second, func(final bool) Summary { + calls++ + Expect(final).To(BeTrue()) + return Summary{Result: ResultPass, Duration: time.Duration(calls) * time.Minute} + }) + + first := r.EmitFinal() + second := r.EmitFinal() + r.emit(context.Background(), false) + + Expect(calls).To(Equal(1)) + Expect(second).To(Equal(first)) + cm, err := cs.CoreV1().ConfigMaps("ns").Get(context.Background(), ConfigMapName, metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(cm.Data["result"]).To(Equal("PASS")) + Expect(cm.Data["latest-report"]).To(ContainSubstring("**Duration:** 1m0s")) + }) }) diff --git a/test/longhaul/report/report.go b/test/longhaul/report/report.go index 37cd17142..f31abf87a 100644 --- a/test/longhaul/report/report.go +++ b/test/longhaul/report/report.go @@ -9,8 +9,10 @@ import ( "time" "github.com/documentdb/documentdb-operator/test/longhaul/backup" + "github.com/documentdb/documentdb-operator/test/longhaul/config" "github.com/documentdb/documentdb-operator/test/longhaul/journal" "github.com/documentdb/documentdb-operator/test/longhaul/monitor" + "github.com/documentdb/documentdb-operator/test/longhaul/operations" "github.com/documentdb/documentdb-operator/test/longhaul/workload" ) @@ -26,9 +28,8 @@ const ( // It is a pure value snapshot — no live counters, no channels — so it can be // passed across goroutines and re-rendered offline. type Summary struct { - // Result is the current verdict. PASS while data-loss counters stay zero, - // flipped to FAIL when the durability oracle detects gaps/checksum errors - // or a disruption window blows its policy budget. + // Result is the current verdict. It flips to FAIL for durability errors, + // operation failures/incomplete sequences, or outage-policy violations. Result Result // Duration is wall-clock time since the run started (process StartTime), @@ -49,13 +50,14 @@ type Summary struct { // only emits a warning annotation. LeakAnalysis monitor.LeakAnalysis - // OpsExecuted is the count of operations (scale up/down, restart, etc.) - // the operations scheduler has run since startup. + // OpsExecuted is the count of terminal operation attempts since startup. OpsExecuted int - // Windows is every disruption window opened during the run, in start - // order. Each window records its op, duration, write-failure count, and - // whether it exceeded its policy budget. + // OperationRun is the bounded sequence result or random aggregate snapshot. + OperationRun operations.RunSnapshot + + // Windows is the journal's bounded set of recent closed disruption windows, + // in start order. Windows []journal.DisruptionWindow // Events is the journal's full event ring (info/warn/error log lines). @@ -83,6 +85,27 @@ func GenerateMarkdown(s Summary) string { } b.WriteString("\n") + switch s.OperationRun.Mode { + case config.OperationModeSequence: + b.WriteString("## Operation Results\n\n") + b.WriteString("| # | Operation | Status | Error |\n") + b.WriteString("|---|-----------|--------|-------|\n") + for i, result := range s.OperationRun.Results { + fmt.Fprintf(&b, "| %d | %s | %s | %s |\n", + i+1, result.Name, result.Status, markdownCell(result.Error)) + } + b.WriteString("\n") + case config.OperationModeRandom: + b.WriteString("## Operation Summary\n\n") + b.WriteString("| Operation | Passed | Failed |\n") + b.WriteString("|-----------|--------|--------|\n") + for _, aggregate := range s.OperationRun.Aggregates { + fmt.Fprintf(&b, "| %s | %d | %d |\n", + aggregate.Name, aggregate.Passed, aggregate.Failed) + } + b.WriteString("\n") + } + // Data Plane Metrics b.WriteString("## Data Plane Metrics\n\n") b.WriteString("| Metric | Value |\n") @@ -115,15 +138,16 @@ func GenerateMarkdown(s Summary) string { // Disruption Windows if len(s.Windows) > 0 { b.WriteString("## Disruption Windows\n\n") - b.WriteString("| Operation | Duration | Write Failures | Policy Exceeded |\n") - b.WriteString("|-----------|----------|----------------|------------------|\n") + b.WriteString("| Operation | Duration | Write Failures | Est. Write Outage | Policy Exceeded |\n") + b.WriteString("|-----------|----------|----------------|-------------------|------------------|\n") for _, w := range s.Windows { exceeded := "No" if w.ExceededPolicy() { exceeded = "**YES**" } - fmt.Fprintf(&b, "| %s | %s | %d | %s |\n", - w.OperationName, w.Duration().Round(time.Second), w.WriteFailures, exceeded) + fmt.Fprintf(&b, "| %s | %s | %d | %s | %s |\n", + w.OperationName, w.Duration().Round(time.Second), w.WriteFailures, + w.EstimatedWriteOutage().Round(time.Millisecond), exceeded) } b.WriteString("\n") } @@ -156,3 +180,11 @@ func GenerateMarkdown(s Summary) string { return b.String() } + +func markdownCell(value string) string { + if value == "" { + return "—" + } + value = strings.ReplaceAll(value, "|", "\\|") + return strings.ReplaceAll(value, "\n", " ") +} diff --git a/test/longhaul/report/report_test.go b/test/longhaul/report/report_test.go index f8b77297e..ccdb907cc 100644 --- a/test/longhaul/report/report_test.go +++ b/test/longhaul/report/report_test.go @@ -10,8 +10,10 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/documentdb/documentdb-operator/test/longhaul/config" "github.com/documentdb/documentdb-operator/test/longhaul/journal" "github.com/documentdb/documentdb-operator/test/longhaul/monitor" + "github.com/documentdb/documentdb-operator/test/longhaul/operations" "github.com/documentdb/documentdb-operator/test/longhaul/workload" ) @@ -69,16 +71,52 @@ var _ = Describe("GenerateMarkdown", func() { Expect(md).NotTo(ContainSubstring("Disruption Windows")) }) + It("renders ordered sequence operation results", func() { + md := GenerateMarkdown(Summary{ + Result: ResultPass, + OperationRun: operations.RunSnapshot{ + Mode: config.OperationModeSequence, + Status: operations.RunStatusComplete, + Results: []operations.OperationResult{ + {Name: "kill-operator-pod", Status: operations.OperationPassed}, + {Name: "kill-primary-pod", Status: operations.OperationFailed, Error: "primary unchanged"}, + }, + }, + }) + Expect(md).To(ContainSubstring("## Operation Results")) + Expect(md).To(ContainSubstring("| 1 | kill-operator-pod | PASSED |")) + Expect(md).To(ContainSubstring("| 2 | kill-primary-pod | FAILED | primary unchanged |")) + Expect(strings.Index(md, "kill-operator-pod")).To(BeNumerically("<", strings.Index(md, "kill-primary-pod"))) + }) + + It("renders bounded random aggregate counters", func() { + md := GenerateMarkdown(Summary{ + Result: ResultPass, + OperationRun: operations.RunSnapshot{ + Mode: config.OperationModeRandom, + Status: operations.RunStatusRunning, + Aggregates: []operations.OperationAggregate{ + {Name: "scale-up", Passed: 12, Failed: 1}, + {Name: "scale-down", Passed: 9}, + }, + }, + }) + Expect(md).To(ContainSubstring("## Operation Summary")) + Expect(md).To(ContainSubstring("| scale-up | 12 | 1 |")) + Expect(md).To(ContainSubstring("| scale-down | 9 | 0 |")) + }) + It("appears with the operation name when at least one window exists", func() { now := time.Now() md := GenerateMarkdown(Summary{ Result: ResultPass, Windows: []journal.DisruptionWindow{{ - OperationName: "scale-up", - StartTime: now.Add(-30 * time.Second), - EndTime: now, - WriteFailures: 3, - Policy: journal.OutagePolicy{MustRecoverWithin: time.Minute, AllowedWriteFailures: 50}, + OperationName: "scale-up", + StartTime: now.Add(-30 * time.Second), + EndTime: now, + WriteFailures: 3, + MaxWriteOutageObserved: 100 * time.Millisecond, + Policy: journal.OutagePolicy{MustRecoverWithin: time.Minute, MaxWriteOutage: time.Second}, }}, }) Expect(md).To(ContainSubstring("Disruption Windows")) diff --git a/test/longhaul/workload/metrics.go b/test/longhaul/workload/metrics.go index 64c1a416e..dc8e1acbe 100644 --- a/test/longhaul/workload/metrics.go +++ b/test/longhaul/workload/metrics.go @@ -26,8 +26,9 @@ type Metrics struct { WriteAcknowledged atomic.Int64 // WriteFailed counts non-DupKey insert errors. Does not advance seq, so - // the next tick retries the same seq; charged against the disruption-window - // budget via journal.RecordWriteFailure. + // the next tick retries the same seq; reported to the disruption window via + // journal.RecordWriteOutcome, which measures the outage duration from + // timestamps. WriteFailed atomic.Int64 // VerifyPasses is the number of completed verifier scan cycles. diff --git a/test/longhaul/workload/pruner.go b/test/longhaul/workload/pruner.go index 71fee7d0b..df10c3069 100644 --- a/test/longhaul/workload/pruner.go +++ b/test/longhaul/workload/pruner.go @@ -15,10 +15,12 @@ import ( ) const ( - // pruneInterval is how often the pruner trims old documents. A long-haul - // run writes ~10 docs/sec/writer, so a few thousand rows accumulate per - // writer between cycles — a small, index-backed DeleteMany each time. - pruneInterval = 5 * time.Minute + // defaultPruneInterval is how often the pruner trims old documents. A + // long-haul run writes ~10 docs/sec/writer, so a few thousand rows + // accumulate per writer between cycles — a small, index-backed DeleteMany + // each time. Overridable via the Pruner's interval (see StartPruner) so the + // smoke gate can prune within a short bounded run. + defaultPruneInterval = 5 * time.Minute ) // floorProvider reports the highest fully-verified seq per writer. *Verifier @@ -69,6 +71,7 @@ type Pruner struct { floor floorProvider backend pruneBackend retainPerWriter int64 + interval time.Duration metrics *Metrics journal *journal.Journal @@ -79,13 +82,18 @@ type Pruner struct { } // NewPruner constructs a Pruner. retainPerWriter must be > 0; callers gate on -// that (0 disables pruning entirely) before constructing. -func NewPruner(coll *mongo.Collection, writers []*Writer, floor floorProvider, retainPerWriter int64, metrics *Metrics, j *journal.Journal) *Pruner { +// that (0 disables pruning entirely) before constructing. A non-positive +// interval falls back to defaultPruneInterval. +func NewPruner(coll *mongo.Collection, writers []*Writer, floor floorProvider, retainPerWriter int64, interval time.Duration, metrics *Metrics, j *journal.Journal) *Pruner { + if interval <= 0 { + interval = defaultPruneInterval + } return &Pruner{ writers: writers, floor: floor, backend: docdbPruneBackend{coll: coll}, retainPerWriter: retainPerWriter, + interval: interval, metrics: metrics, journal: j, } @@ -96,7 +104,7 @@ func (p *Pruner) Run(ctx context.Context) { p.journal.Info("pruner", fmt.Sprintf("pruner started (retain %d docs/writer)", p.retainPerWriter)) defer p.journal.Info("pruner", "pruner stopped") - ticker := time.NewTicker(pruneInterval) + ticker := time.NewTicker(p.interval) defer ticker.Stop() for { @@ -151,8 +159,8 @@ func (p *Pruner) pruneWriter(ctx context.Context, writerID string) { } // StartPruner launches a single pruner goroutine and returns it. -func StartPruner(ctx context.Context, coll *mongo.Collection, writers []*Writer, floor floorProvider, retainPerWriter int64, metrics *Metrics, j *journal.Journal) *Pruner { - p := NewPruner(coll, writers, floor, retainPerWriter, metrics, j) +func StartPruner(ctx context.Context, coll *mongo.Collection, writers []*Writer, floor floorProvider, retainPerWriter int64, interval time.Duration, metrics *Metrics, j *journal.Journal) *Pruner { + p := NewPruner(coll, writers, floor, retainPerWriter, interval, metrics, j) go p.Run(ctx) return p } diff --git a/test/longhaul/workload/writer.go b/test/longhaul/workload/writer.go index c2fdcc3e2..be2d51660 100644 --- a/test/longhaul/workload/writer.go +++ b/test/longhaul/workload/writer.go @@ -139,6 +139,7 @@ func (w *Writer) writeOne(ctx context.Context) { w.metrics.WriteAttempted.Add(1) + attemptStart := time.Now() err := w.backend.insert(ctx, doc) if err != nil { // Retryable writes are on by default in the v2 driver, so a network @@ -151,17 +152,19 @@ func (w *Writer) writeOne(ctx context.Context) { if w.backend.isDuplicate(err) { w.seq.Store(seq) w.metrics.WriteAcknowledged.Add(1) + w.journal.RecordWriteOutcome(attemptStart, false) return } // For any other error the document was NOT committed. Do NOT advance // seq, otherwise the verifier will see a permanent gap and report // false-positive data loss. The next tick will retry the same seq. w.metrics.WriteFailed.Add(1) - w.journal.RecordWriteFailure() + w.journal.RecordWriteOutcome(attemptStart, true) return } w.seq.Store(seq) w.metrics.WriteAcknowledged.Add(1) + w.journal.RecordWriteOutcome(attemptStart, false) } // Resume seeds the writer's seq counter from the highest seq already persisted