diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5aff9863f..e39363970 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,5 +1,10 @@ name: ci +# Gate-only workflow: fmt / clippy / test / deny / xtask. It touches no +# infrastructure. The DigitalOcean staging soak was retired (owner decision, +# 2026-09-10) — main pushes now go straight to prod through +# images.yml (GHCR digests → prod pins) and deploy-prod.yml. + on: push: branches: [main] @@ -63,111 +68,3 @@ jobs: - name: compose matrix assertions run: bash deploy/scripts/assert-compose-matrix.sh - - # --------------------------------------------------------------------------- - # Auto-deploy staging. Lives here rather than in deploy-staging.yml so that - # `needs: ci` keeps the CI-green ordering on every push to main. - # deploy-staging.yml remains for manual re-deploys. - # --------------------------------------------------------------------------- - deploy-staging: - name: deploy staging ${{ matrix.role }} - needs: [ci] - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - runs-on: ubuntu-latest - timeout-minutes: 120 - concurrency: - group: deploy-staging-${{ matrix.role }} - cancel-in-progress: false - strategy: - fail-fast: false - matrix: - role: [master, validator] - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Install SSH key - run: | - set -euo pipefail - test -n "${{ secrets.STAGING_SSH_KEY }}" || { echo "missing STAGING_SSH_KEY"; exit 1; } - mkdir -p ~/.ssh - echo "${{ secrets.STAGING_SSH_KEY }}" > ~/.ssh/staging_ed25519 - chmod 600 ~/.ssh/staging_ed25519 - echo "StrictHostKeyChecking accept-new" > ~/.ssh/config - - - name: Resolve host - id: host - run: | - set -euo pipefail - if [[ "${{ matrix.role }}" == "master" ]]; then - h="${{ secrets.STAGING_MASTER_HOST }}" - else - h="${{ secrets.STAGING_VALIDATOR_HOST }}" - fi - test -n "$h" || { echo "missing staging host secret for ${{ matrix.role }}"; exit 1; } - echo "value=$h" >> "$GITHUB_OUTPUT" - - # Port 22 is closed to everything but the operator IP, and runners get - # ephemeral Azure addresses. Open a hole for this runner's own /32 and - # close it again in the always() step below. - - name: Open firewall for this runner - id: fw - uses: ./.github/actions/do-firewall - with: - action: open - token: ${{ secrets.DIGITALOCEAN_TOKEN }} - - - name: Deploy - run: | - set -euo pipefail - chmod +x deploy/scripts/remote-deploy.sh - export BASE_SSH_IDENTITY="$HOME/.ssh/staging_ed25519" - EXTRA=() - if [[ "${{ matrix.role }}" == "validator" && -n "${{ secrets.STAGING_MASTER_GATEWAY_URL }}" ]]; then - EXTRA+=(--gateway-endpoint "${{ secrets.STAGING_MASTER_GATEWAY_URL }}") - fi - ./deploy/scripts/remote-deploy.sh \ - --host "root@${{ steps.host.outputs.value }}" \ - --role "${{ matrix.role }}" \ - --env staging \ - --build-from source \ - "${EXTRA[@]}" - - # Relies on Docker's own health state, which the compose healthchecks now - # report honestly (they used to `|| exit 0` when curl was absent). - - name: Smoke health (fail-closed) - run: | - set -euo pipefail - ssh -i "$HOME/.ssh/staging_ed25519" -o BatchMode=yes \ - "root@${{ steps.host.outputs.value }}" \ - 'cd /opt/base && docker compose ps --format "{{.Service}}\t{{.Status}}"; \ - for i in $(seq 1 24); do \ - bad=$(for c in $(docker compose ps -q); do \ - st=$(docker inspect -f "{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}" "$c"); \ - case "$st" in healthy|none) ;; *) echo "$(docker inspect -f "{{.Name}}" "$c")=$st" ;; esac; \ - done); \ - if [ -z "$bad" ]; then echo "all containers healthy"; exit 0; fi; \ - sleep 5; \ - done; echo "UNHEALTHY: $bad"; exit 1' - - - name: Verify live chain (no fake backend) - if: matrix.role == 'validator' - run: | - set -euo pipefail - ssh -i "$HOME/.ssh/staging_ed25519" -o BatchMode=yes \ - "root@${{ steps.host.outputs.value }}" \ - 'cd /opt/base && logs=$(docker compose logs validator --no-log-prefix --tail 200 2>&1); \ - echo "$logs" | grep -q "live chain reachable" \ - || { echo "validator never reached the live chain"; exit 1; }; \ - echo "$logs" | grep -qi "fake" \ - && { echo "fake backend present in validator logs"; exit 1; }; \ - echo "live chain confirmed"' - - - name: Close firewall for this runner - if: always() && steps.fw.outputs.ip != '' - uses: ./.github/actions/do-firewall - with: - action: close - token: ${{ secrets.DIGITALOCEAN_TOKEN }} - ip: ${{ steps.fw.outputs.ip }} - firewall-id: ${{ steps.fw.outputs.firewall-id }} diff --git a/.github/workflows/deploy-prod.yml b/.github/workflows/deploy-prod.yml index 22bd238f9..026d6ed7b 100644 --- a/.github/workflows/deploy-prod.yml +++ b/.github/workflows/deploy-prod.yml @@ -1,32 +1,48 @@ name: deploy-prod -# Promote digest-pinned images to prod droplets on signed version tags. +# Promote digest-pinned images to prod droplets. # -# Trigger: push tag v*.*.* (cut from main) or manual workflow_dispatch. -# Triggers on version tags only — never on branch pushes. +# The DigitalOcean staging soak was retired (owner decision, 2026-09-10): there +# is no staging deploy lane and no staging pin file to match against. Every +# green `images` run on main deploys prod, and version tags / manual dispatch +# remain available for re-deploys and rollbacks. +# +# Triggers: +# - workflow_run: `images` completed successfully on main (the normal path) +# - push tag v*.*.* (cut from main) +# - workflow_dispatch with an explicit commit_sha (rollback = an older SHA) # # Steps: -# 1. Preflight — CI green for the tag SHA; origin/main staging pins carry -# that commit_sha (ladder tip may be a later pin-commit on main). -# 2. Fail-closed Postgres backup to DO Spaces, then promote.sh --env prod -# --confirm-prod for each pin service (staging.json → prod.json). +# 1. Preflight — the SHA is on origin/main, CI is green for it, and the +# `images` run for that SHA published a `prod-pins-` artifact. +# 2. Fail-closed Postgres backup to DO Spaces. # 3. Deploy master + validator with remote-deploy.sh --build-from registry # (GHCR digest pull + retag; no Rust compile on the droplet). # 4. Smoke /healthz (fail-closed). +# +# Prod pins are never committed: main is protected (PR + Greptile review), so a +# pin commit from CI is rejected with GH013. deploy/pins/prod.json in git is a +# template — the deployed pins are the `prod-pins-` artifact built by +# images.yml from the GHCR digests of that exact commit. on: + workflow_run: + workflows: [images] + types: [completed] + branches: [main] push: tags: - "v*.*.*" workflow_dispatch: inputs: commit_sha: - description: "Commit SHA to deploy (must have passed staging)" + description: "Commit SHA on main to deploy (defaults to the ref's SHA)" required: false permissions: - contents: write + contents: read packages: read + actions: read concurrency: group: deploy-prod @@ -35,9 +51,14 @@ concurrency: jobs: preflight: name: preflight checks + if: >- + github.event_name != 'workflow_run' + || github.event.workflow_run.conclusion == 'success' runs-on: ubuntu-latest + timeout-minutes: 60 outputs: commit_sha: ${{ steps.resolve.outputs.commit_sha }} + images_run_id: ${{ steps.images.outputs.run_id }} steps: - name: Checkout uses: actions/checkout@v4 @@ -46,65 +67,102 @@ jobs: - name: Resolve commit SHA id: resolve + env: + INPUT_SHA: ${{ github.event.inputs.commit_sha }} + RUN_SHA: ${{ github.event.workflow_run.head_sha }} run: | set -euo pipefail - if [[ -n "${{ github.event.inputs.commit_sha }}" ]]; then - SHA="${{ github.event.inputs.commit_sha }}" + if [[ -n "${INPUT_SHA:-}" ]]; then + SHA="$INPUT_SHA" + elif [[ -n "${RUN_SHA:-}" ]]; then + SHA="$RUN_SHA" elif [[ "$GITHUB_REF" == refs/tags/* ]]; then SHA=$(git rev-list -n 1 "$GITHUB_REF") else SHA="$GITHUB_SHA" fi + [[ "$SHA" =~ ^[0-9a-f]{40}$ ]] || { echo "not a full commit SHA: $SHA"; exit 1; } echo "commit_sha=$SHA" >> "$GITHUB_OUTPUT" echo "Resolved commit: $SHA" - - name: Verify CI succeeded for this SHA - env: - GH_TOKEN: ${{ github.token }} + - name: Verify the commit is on origin/main run: | set -euo pipefail SHA="${{ steps.resolve.outputs.commit_sha }}" - # Check that the ci workflow passed for this commit on main - conclusion=$(gh run list \ - --workflow ci.yml \ - --branch main \ - --commit "$SHA" \ - --status completed \ - --json conclusion \ - --jq '.[0].conclusion // empty' 2>/dev/null || true) - if [[ "$conclusion" != "success" ]]; then - echo "CI has not passed for commit $SHA on main (conclusion: ${conclusion:-none})" - echo "Failing — prod deploy requires a green CI run for this exact commit." + git fetch --quiet origin main + # Prod only ever runs code that is merged: a tag cut off main, a stale + # branch SHA, or a hand-typed dispatch input is refused here. + if ! git merge-base --is-ancestor "$SHA" origin/main; then + echo "commit $SHA is not an ancestor of origin/main — refusing to deploy prod" exit 1 fi - echo "CI passed for $SHA" + echo "$SHA is on origin/main" - - name: Verify staging pins exist for this SHA + - name: Wait for CI green on this SHA + env: + GH_TOKEN: ${{ github.token }} run: | set -euo pipefail SHA="${{ steps.resolve.outputs.commit_sha }}" - # Pin commits land on origin/main after images.yml; the tag may point at - # the image SHA while staging.json lives on a later pin commit. - git fetch origin main - if ! git show origin/main:deploy/pins/staging.json > /tmp/staging-pins.json 2>/dev/null; then - echo "deploy/pins/staging.json not found on origin/main — staging pins must be committed first" + # images.yml and ci.yml run in parallel on a main push, so the CI run + # for this SHA may still be going when the images run completes. + for i in $(seq 1 60); do + conclusion=$(gh run list \ + --workflow ci.yml \ + --commit "$SHA" \ + --status completed \ + --json conclusion \ + --jq '.[0].conclusion // empty' 2>/dev/null || true) + case "$conclusion" in + success) echo "CI passed for $SHA"; exit 0 ;; + "") echo "waiting for a completed ci run on $SHA ($i/60)"; sleep 30 ;; + *) echo "CI concluded '$conclusion' for $SHA — refusing to deploy prod"; exit 1 ;; + esac + done + echo "no completed ci run for $SHA within 30 minutes — refusing to deploy prod" + exit 1 + + - name: Locate the images run that published prod pins + id: images + env: + GH_TOKEN: ${{ github.token }} + EVENT_RUN_ID: ${{ github.event.workflow_run.id }} + run: | + set -euo pipefail + SHA="${{ steps.resolve.outputs.commit_sha }}" + if [[ -n "${EVENT_RUN_ID:-}" ]]; then + RUN_ID="$EVENT_RUN_ID" + else + RUN_ID=$(gh run list \ + --workflow images.yml \ + --commit "$SHA" \ + --status success \ + --json databaseId \ + --jq '.[0].databaseId // empty' 2>/dev/null || true) + fi + if [[ -z "$RUN_ID" ]]; then + echo "no successful images run for $SHA" + echo "(re-run that commit's images run, or dispatch images.yml on a ref pointing at it)" exit 1 fi - pin_sha=$(python3 -c \ - "import json; print(json.load(open('/tmp/staging-pins.json')).get('commit_sha',''))") - if [[ "$pin_sha" != "$SHA" ]]; then - echo "Staging pins commit_sha=$pin_sha, expected $SHA" - echo "Staging must record digests for this commit before promoting to prod." + # Fail here rather than in the deploy job: without the artifact there + # are no digests to pin, and prod must never fall back to a source build. + found=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${RUN_ID}/artifacts?per_page=100" \ + --jq "[.artifacts[] | select(.name == \"prod-pins-${SHA}\" and .expired == false)] | length") + if [[ "$found" -lt 1 ]]; then + echo "images run $RUN_ID has no live prod-pins-${SHA} artifact" + echo "(artifacts expire — re-run that images run, or dispatch images.yml" + echo " on a ref pointing at this commit, to rebuild the pins)" exit 1 fi - echo "Staging pins on origin/main match commit $SHA" + echo "run_id=$RUN_ID" >> "$GITHUB_OUTPUT" + echo "prod pins come from images run $RUN_ID" - promote: - name: promote staging → prod pins + backup: + name: prod pins + fail-closed backup needs: preflight runs-on: ubuntu-latest - outputs: - pins_artifact: prod-pins-${{ needs.preflight.outputs.commit_sha }} + timeout-minutes: 60 steps: - name: Checkout uses: actions/checkout@v4 @@ -112,18 +170,42 @@ jobs: ref: ${{ needs.preflight.outputs.commit_sha }} fetch-depth: 0 - - name: Load staging pins from origin/main + - name: Download prod pins from the images run + uses: actions/download-artifact@v4 + with: + name: prod-pins-${{ needs.preflight.outputs.commit_sha }} + path: /tmp/prod-pins-in + run-id: ${{ needs.preflight.outputs.images_run_id }} + github-token: ${{ github.token }} + + - name: Validate pins against this commit run: | set -euo pipefail - git fetch origin main - git show origin/main:deploy/pins/staging.json > deploy/pins/staging.json - # Digests manifest (optional pulls for prism/attest-helper) when present. SHA="${{ needs.preflight.outputs.commit_sha }}" - if git show "origin/main:deploy/digests/${SHA}.json" > "deploy/digests/${SHA}.json" 2>/dev/null; then - echo "Loaded deploy/digests/${SHA}.json from origin/main" + if [[ -f /tmp/prod-pins-in/deploy/pins/prod.json ]]; then + BASE=/tmp/prod-pins-in/deploy + elif [[ -f /tmp/prod-pins-in/pins/prod.json ]]; then + BASE=/tmp/prod-pins-in else - echo "WARNING: deploy/digests/${SHA}.json missing on origin/main" + echo "unexpected artifact layout:"; find /tmp/prod-pins-in -type f | head -50 + exit 1 fi + test -f "$BASE/digests/${SHA}.json" || { echo "artifact has no digests/${SHA}.json"; exit 1; } + mkdir -p deploy/pins deploy/digests + cp "$BASE/pins/prod.json" deploy/pins/prod.json + cp "$BASE/digests/${SHA}.json" "deploy/digests/${SHA}.json" + python3 - "$SHA" <<'PY' + import json, re, sys + sha = sys.argv[1] + pin = json.load(open("deploy/pins/prod.json", encoding="utf-8")) + assert pin["commit_sha"] == sha, (pin["commit_sha"], sha) + for svc in ("validator", "gateway", "updater", "bounty-challenge", "proof-challenge"): + digest = pin["services"][svc]["digest"] + assert re.fullmatch(r"sha256:[0-9a-f]{64}", digest), (svc, digest) + hexpart = digest.split(":", 1)[1] + assert not re.fullmatch(r"0+1?", hexpart), (svc, digest) + print("prod pins validated for", sha) + PY - name: Fail-closed — require Spaces backup credentials env: @@ -137,8 +219,8 @@ jobs: [[ -n "${AWS_ACCESS_KEY_ID:-}" ]] || missing+=("SPACES_ACCESS_KEY_ID (or AWS_ACCESS_KEY_ID)") [[ -n "${AWS_SECRET_ACCESS_KEY:-}" ]] || missing+=("SPACES_SECRET_ACCESS_KEY (or AWS_SECRET_ACCESS_KEY)") if [[ ${#missing[@]} -gt 0 ]]; then - echo "Prod promote is fail-closed: missing GitHub secrets: ${missing[*]}" - echo "Set DO Spaces credentials before cutting a prod tag." + echo "Prod deploy is fail-closed: missing GitHub secrets: ${missing[*]}" + echo "Set DO Spaces credentials before deploying prod." echo "See deploy/README.md § Promotion pipeline (BASE_BACKUP_ENDPOINT + Spaces keys)." echo "Postgres dump runs on the prod master host via SSH; Spaces upload runs here." exit 1 @@ -210,7 +292,7 @@ jobs: echo "backup_uri=$URI" >> "$GITHUB_OUTPUT" echo "Backup ok: $URI ($BYTES bytes, sha256=$SUM)" - - name: Close firewall for promote runner + - name: Close firewall for this runner if: always() && steps.fw.outputs.ip != '' uses: ./.github/actions/do-firewall with: @@ -219,41 +301,6 @@ jobs: ip: ${{ steps.fw.outputs.ip }} firewall-id: ${{ steps.fw.outputs.firewall-id }} - - name: Promote each service staging → prod - run: | - set -euo pipefail - SHA="${{ needs.preflight.outputs.commit_sha }}" - chmod +x deploy/scripts/promote.sh - # Backup already succeeded above (fail-closed). promote.sh --skip-backup - # is used only because Postgres is not reachable as PGHOST from the runner; - # we never skip the Spaces backup step itself. - mapfile -t SERVICES < <(python3 -c ' - import json - for s in sorted(json.load(open("deploy/pins/staging.json"))["services"]): - print(s) - ') - for svc in "${SERVICES[@]}"; do - IMAGE=$(python3 -c ' - import json, sys - meta = json.load(open("deploy/pins/staging.json"))["services"][sys.argv[1]] - img = meta.get("image") or "" - digest = meta["digest"] - if "/" in img and "@sha256:" in img: - print(img) - else: - print(f"ghcr.io/baseintelligence/base/{sys.argv[1]}@{digest}") - ' "$svc") - echo "Promoting $svc → $IMAGE" - ./deploy/scripts/promote.sh \ - --env prod \ - --service "$svc" \ - --image "$IMAGE" \ - --confirm-prod \ - --commit "$SHA" \ - --skip-backup - done - python3 -c 'import json; p=json.load(open("deploy/pins/prod.json")); assert p["commit_sha"]; print(p["commit_sha"], list(p["services"]))' - - name: Stage pins artifact run: | set -euo pipefail @@ -261,55 +308,18 @@ jobs: STAGE="/tmp/prod-pins-artifact" mkdir -p "$STAGE/deploy/pins" "$STAGE/deploy/digests" cp deploy/pins/prod.json "$STAGE/deploy/pins/prod.json" - cp deploy/pins/staging.json "$STAGE/deploy/pins/staging.json" - if [[ -f "deploy/digests/${SHA}.json" ]]; then - cp "deploy/digests/${SHA}.json" "$STAGE/deploy/digests/${SHA}.json" - fi + cp "deploy/digests/${SHA}.json" "$STAGE/deploy/digests/${SHA}.json" - - name: Upload promoted pins artifact + - name: Upload deployed pins artifact uses: actions/upload-artifact@v4 with: - name: prod-pins-${{ needs.preflight.outputs.commit_sha }} + name: deployed-prod-pins-${{ needs.preflight.outputs.commit_sha }} path: /tmp/prod-pins-artifact if-no-files-found: error - - name: Commit prod pins to origin/main - run: | - set -euo pipefail - SHA="${{ needs.preflight.outputs.commit_sha }}" - mkdir -p /tmp/prod-pin-commit - cp deploy/pins/prod.json /tmp/prod-pin-commit/prod.json - if [[ -f "deploy/digests/${SHA}.json" ]]; then - cp "deploy/digests/${SHA}.json" "/tmp/prod-pin-commit/${SHA}.json" - fi - git fetch origin main - # Earlier steps rewrite deploy/pins/* and drop deploy/digests/.json - # into the worktree (all preserved under /tmp/prod-pin-commit). Reset so - # the checkout to origin/main cannot fail on a dirty tree. - git reset --hard HEAD - git clean -fd deploy/pins deploy/digests - git checkout -B main origin/main - cp /tmp/prod-pin-commit/prod.json deploy/pins/prod.json - if [[ -f "/tmp/prod-pin-commit/${SHA}.json" ]]; then - mkdir -p deploy/digests - cp "/tmp/prod-pin-commit/${SHA}.json" "deploy/digests/${SHA}.json" - fi - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add deploy/pins/prod.json - if [[ -f "deploy/digests/${SHA}.json" ]]; then - git add "deploy/digests/${SHA}.json" - fi - if git diff --cached --quiet; then - echo "No pin changes to commit" - exit 0 - fi - git commit -m "deploy: promote prod pins for ${SHA}" - git push origin main - deploy: name: deploy prod ${{ matrix.role }} - needs: [preflight, promote] + needs: [preflight, backup] runs-on: ubuntu-latest timeout-minutes: 60 strategy: @@ -326,13 +336,13 @@ jobs: with: ref: ${{ needs.preflight.outputs.commit_sha }} - - name: Download promoted pins + - name: Download deployed pins uses: actions/download-artifact@v4 with: - name: prod-pins-${{ needs.preflight.outputs.commit_sha }} + name: deployed-prod-pins-${{ needs.preflight.outputs.commit_sha }} path: /tmp/prod-pins - - name: Install promoted pins into tree + - name: Install pins into tree run: | set -euo pipefail SHA="${{ needs.preflight.outputs.commit_sha }}" @@ -346,10 +356,7 @@ jobs: fi mkdir -p deploy/pins deploy/digests cp "$BASE/pins/prod.json" deploy/pins/prod.json - cp "$BASE/pins/staging.json" deploy/pins/staging.json - if [[ -f "$BASE/digests/${SHA}.json" ]]; then - cp "$BASE/digests/${SHA}.json" "deploy/digests/${SHA}.json" - fi + cp "$BASE/digests/${SHA}.json" "deploy/digests/${SHA}.json" python3 -c ' import json, re p = json.load(open("deploy/pins/prod.json")) diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml deleted file mode 100644 index 0bc4e6486..000000000 --- a/.github/workflows/deploy-staging.yml +++ /dev/null @@ -1,180 +0,0 @@ -name: deploy-staging - -# Auto-deploy control plane to DigitalOcean staging after CI is green on main. -# Roles: -# - staging master = validator + gateway + hotkey (owner) -# - staging validator = validator only (no hotkey / no gateway) -# -# Deploys both roles in parallel (fail-fast: false). Master must be up -# before validator can reach the gateway — the validator retry loop -# tolerates a brief gateway outage, so parallel is safe. - -# Manual re-deploy only. The automatic post-CI deploy is the `deploy-staging` -# job in ci.yml, which runs on every green push to main. -on: - workflow_dispatch: - inputs: - role: - description: "Which staging role(s) to deploy" - type: choice - options: - - both - - master - - validator - default: both - -permissions: - contents: read - packages: read - -concurrency: - group: deploy-staging - cancel-in-progress: false - -jobs: - deploy: - name: deploy staging ${{ matrix.role }} - runs-on: ubuntu-latest - timeout-minutes: 120 - strategy: - fail-fast: false - matrix: - include: - - role: master - host_secret: STAGING_MASTER_HOST - - role: validator - host_secret: STAGING_VALIDATOR_HOST - steps: - - name: Skip unwanted roles on manual dispatch - id: gate - run: | - set -euo pipefail - want="${{ github.event.inputs.role || 'both' }}" - role="${{ matrix.role }}" - if [[ "$want" == "both" || "$want" == "$role" ]]; then - echo "run=true" >> "$GITHUB_OUTPUT" - else - echo "run=false" >> "$GITHUB_OUTPUT" - fi - - - name: Checkout - if: steps.gate.outputs.run == 'true' - uses: actions/checkout@v4 - with: - ref: ${{ github.sha }} - - - name: Install SSH key - if: steps.gate.outputs.run == 'true' - run: | - set -euo pipefail - test -n "${{ secrets.STAGING_SSH_KEY }}" || { echo "missing STAGING_SSH_KEY"; exit 1; } - mkdir -p ~/.ssh - echo "${{ secrets.STAGING_SSH_KEY }}" > ~/.ssh/staging_ed25519 - chmod 600 ~/.ssh/staging_ed25519 - echo "StrictHostKeyChecking accept-new" > ~/.ssh/config - - - name: Resolve host - if: steps.gate.outputs.run == 'true' - id: host - run: | - set -euo pipefail - if [[ "${{ matrix.role }}" == "master" ]]; then - h="${{ secrets.STAGING_MASTER_HOST }}" - else - h="${{ secrets.STAGING_VALIDATOR_HOST }}" - fi - test -n "$h" || { echo "missing host secret ${{ matrix.host_secret }}"; exit 1; } - echo "value=$h" >> "$GITHUB_OUTPUT" - - - name: Open firewall for this runner - id: fw - uses: ./.github/actions/do-firewall - with: - action: open - token: ${{ secrets.DIGITALOCEAN_TOKEN }} - - - name: Deploy - if: steps.gate.outputs.run == 'true' - run: | - set -euo pipefail - chmod +x deploy/scripts/remote-deploy.sh - export BASE_SSH_IDENTITY="$HOME/.ssh/staging_ed25519" - EXTRA=() - if [[ "${{ matrix.role }}" == "validator" ]]; then - if [[ -n "${{ secrets.STAGING_MASTER_GATEWAY_URL }}" ]]; then - EXTRA+=(--gateway-endpoint "${{ secrets.STAGING_MASTER_GATEWAY_URL }}") - fi - fi - ./deploy/scripts/remote-deploy.sh \ - --host "root@${{ steps.host.outputs.value }}" \ - --role "${{ matrix.role }}" \ - --env staging \ - --build-from source \ - "${EXTRA[@]}" - - - name: Smoke health (fail-closed) - if: steps.gate.outputs.run == 'true' - run: | - set -euo pipefail - export BASE_SSH_IDENTITY="$HOME/.ssh/staging_ed25519" - HOST="root@${{ steps.host.outputs.value }}" - ROLE="${{ matrix.role }}" - COMPOSE="docker compose -f docker-compose.yml -f deploy/compose/role-${ROLE}.yml -f deploy/compose/env-staging.yml" - # Verify containers are up - ssh -i "$BASE_SSH_IDENTITY" -o BatchMode=yes -o StrictHostKeyChecking=accept-new \ - "$HOST" \ - "cd /opt/base && ${COMPOSE} ps --format 'table {{.Service}}\t{{.Status}}'" - # Master has no on-chain validator (sole submitter = validator host). - # Probe gateway on master; probe validator only on validator role. - if [[ "$ROLE" == "master" ]]; then - ssh -i "$BASE_SSH_IDENTITY" -o BatchMode=yes -o StrictHostKeyChecking=accept-new \ - "$HOST" \ - "cd /opt/base && \ - if ${COMPOSE} ps --status running --services 2>/dev/null | grep -qx validator; then \ - echo 'master smoke: ERROR validator running on master (dual submitter)'; exit 1; \ - fi; \ - for i in \$(seq 1 12); do \ - if ${COMPOSE} exec -T gateway curl -fsS -m 5 http://127.0.0.1:8080/healthz 2>/dev/null; then \ - echo 'gateway health: ok'; exit 0; \ - fi; \ - sleep 5; \ - done; \ - echo 'gateway health: FAILED (no /healthz within 60s)'; exit 1" - else - ssh -i "$BASE_SSH_IDENTITY" -o BatchMode=yes -o StrictHostKeyChecking=accept-new \ - "$HOST" \ - "cd /opt/base && \ - for i in \$(seq 1 12); do \ - if ${COMPOSE} exec -T validator curl -fsS -m 5 http://127.0.0.1:8080/healthz 2>/dev/null; then \ - echo 'validator health: ok'; exit 0; \ - fi; \ - sleep 5; \ - done; \ - echo 'validator health: FAILED (no /healthz within 60s)'; exit 1" - fi - - - name: Post-deploy Match gate (validator only) - if: steps.gate.outputs.run == 'true' && matrix.role == 'validator' - run: | - set -euo pipefail - export BASE_SSH_IDENTITY="$HOME/.ssh/staging_ed25519" - HOST="root@${{ steps.host.outputs.value }}" - # Wait up to 180s for a Match epoch= line in validator logs - ssh -i "$BASE_SSH_IDENTITY" -o BatchMode=yes -o StrictHostKeyChecking=accept-new \ - "$HOST" \ - 'for i in $(seq 1 36); do \ - if docker logs $(docker ps -q --filter name=validator) 2>&1 | grep -q "Match epoch="; then \ - echo "bundle Match confirmed"; exit 0; \ - fi; \ - sleep 5; \ - done; \ - echo "WARNING: no Match epoch= in validator logs within 180s (may still be syncing)"; exit 0' - - - name: Close firewall for this runner - if: always() && steps.fw.outputs.ip != '' - uses: ./.github/actions/do-firewall - with: - action: close - token: ${{ secrets.DIGITALOCEAN_TOKEN }} - ip: ${{ steps.fw.outputs.ip }} - firewall-id: ${{ steps.fw.outputs.firewall-id }} diff --git a/.github/workflows/ghcr-public.yml b/.github/workflows/ghcr-public.yml index f97b82481..2ea397214 100644 --- a/.github/workflows/ghcr-public.yml +++ b/.github/workflows/ghcr-public.yml @@ -1,5 +1,7 @@ -# Make GHCR container packages public so paid Phala / unauth pulls work. -# Packages live under the *base* repo path: ghcr.io/baseintelligence/base/ +# Make GHCR container packages public so unauthenticated droplet pulls work. +# Packages live under the repo path: ghcr.io///. +# Prod deploys pull digests anonymously (remote-deploy.sh --build-from registry +# never logs in), so a private package is a broken prod deploy. name: ghcr-public # Manual re-run only. The automatic pass is the `ghcr-public` job in @@ -21,14 +23,19 @@ jobs: OWNER: ${{ github.repository_owner }} run: | set -euo pipefail - # Nested image names: ghcr.io//base/ → package name "base/" + # Nested image names: ghcr.io/// → package "/". + # Derive the repo segment: the hardcoded "base/" prefix went stale when + # the repo was renamed, so every visibility call 404'd and the droplets + # were left pulling private packages. + repo="${GITHUB_REPOSITORY##*/}" + repo="${repo,,}" pkgs=( - "base/base-attest-helper" - "base/gateway" - "base/validator" - "base/updater" - "base/bounty-challenge" - "base/proof-challenge" + "${repo}/base-attest-helper" + "${repo}/gateway" + "${repo}/validator" + "${repo}/updater" + "${repo}/bounty-challenge" + "${repo}/proof-challenge" ) ok=0 # First list packages to Learn exact names (debug) diff --git a/.github/workflows/images.yml b/.github/workflows/images.yml index 8fc78eb19..440f85c14 100644 --- a/.github/workflows/images.yml +++ b/.github/workflows/images.yml @@ -2,9 +2,10 @@ name: images # Build service images, push by commit SHA + branch tag, record digests as an artifact. # Digests-only pins are consumed by deploy/scripts/promote.sh. -# After merge: promote pin services into deploy/pins/staging.json and commit so -# deploy-prod preflight can match staging.json.commit_sha to the image SHA. -# Service images use GHCR. +# After merge: promote pin services into deploy/pins/prod.json and publish the +# result as the `prod-pins-` artifact that deploy-prod.yml consumes. +# Nothing is pushed to main: main is protected (PR + Greptile review required), +# so a pin commit from CI is rejected with GH013. Service images use GHCR. on: push: @@ -15,7 +16,7 @@ on: - "deploy/Dockerfile" - "deploy/attest-helper/**" # Ops scripts (promote/remote-deploy/burn-seal) — keep the - # digest → staging-pin ladder reachable for ops-only commits. + # digest → prod-pin ladder reachable for ops-only commits. - "deploy/scripts/**" - "Cargo.toml" - "Cargo.lock" @@ -185,25 +186,26 @@ jobs: if-no-files-found: error # --------------------------------------------------------------------------- - # Close the staging → prod ladder: write deploy/digests/.json and promote - # pin services into deploy/pins/staging.json, then commit/push to origin/main. - # Staging droplet deploy remains --build-from source (deploy-staging / ci.yml). + # Digest → prod pin ladder. Writes deploy/digests/.json and promotes the + # pin services into deploy/pins/prod.json, then publishes both as the + # `prod-pins-` artifact. deploy-prod.yml downloads that artifact instead + # of reading pins from git: main is protected, so CI cannot push a pin commit. # --------------------------------------------------------------------------- - update-staging-pins: - name: update · commit staging pins + prod-pins: + name: record prod pins needs: merge-digests - if: github.event_name == 'push' && github.ref == 'refs/heads/main' + # Also on workflow_dispatch: re-running images for a commit whose pin + # artifact expired is the documented prod rollback path, so a manual run + # must rebuild the artifact too. Nothing reaches prod from a stray ref — + # deploy-prod refuses any SHA that is not an ancestor of origin/main. + if: github.event_name != 'pull_request' runs-on: ubuntu-latest permissions: - contents: write + contents: read packages: read steps: - name: Checkout uses: actions/checkout@v4 - with: - ref: main - fetch-depth: 0 - token: ${{ secrets.GITHUB_TOKEN }} - name: Download merged digests uses: actions/download-artifact@v4 @@ -211,7 +213,7 @@ jobs: name: base-image-digests-${{ github.sha }} path: /tmp/digests-in - - name: Record digests + promote staging pins + - name: Record digests + promote prod pins run: | set -euo pipefail SHA="${{ github.sha }}" @@ -229,72 +231,62 @@ jobs: cp "$SRC" "deploy/digests/${SHA}.json" echo "Recorded digest manifest: deploy/digests/${SHA}.json" chmod +x deploy/scripts/promote.sh - # Snapshot the full current staging release once. Sequential - # promote.sh calls overwrite `previous` per service and would - # otherwise leave a mixed rollback (new images + one old). - python3 -c ' - import json - p = json.load(open("deploy/pins/staging.json")) - prev = { - "commit_sha": p.get("commit_sha"), - "services": p.get("services") or {}, - "updated_at": p.get("updated_at"), - } - with open("/tmp/staging-previous.json", "w", encoding="utf-8") as f: - json.dump(prev, f, indent=2, sort_keys=True) - f.write("\n") - print("captured previous staging release", prev.get("commit_sha")) - ' - # Pin services only (validator/gateway/updater/live challenges). - # --skip-backup: this job records CI digests; Spaces/PG backup runs at - # prod promote (deploy-prod.yml, fail-closed). + # --force-prod skips the staging-digest ladder check: the staging soak + # was retired, so deploy/pins/staging.json is no longer written by any + # workflow and comparing against it would pin prod to a dead lane. + # --skip-backup: this job only records pins into an artifact and never + # deploys; the fail-closed Postgres backup runs in deploy-prod.yml. for svc in validator gateway updater bounty-challenge proof-challenge; do IMAGE=$(python3 -c ' import json, sys d = json.load(open(sys.argv[1])) - meta = d["images"][sys.argv[2]] - print(meta["image"]) + print(d["images"][sys.argv[2]]["image"]) ' "deploy/digests/${SHA}.json" "$svc") - echo "staging promote $svc → $IMAGE" + echo "prod pin $svc → $IMAGE" ./deploy/scripts/promote.sh \ - --env staging \ + --env prod \ --service "$svc" \ --image "$IMAGE" \ + --confirm-prod \ + --force-prod \ --commit "$SHA" \ --skip-backup done python3 -c ' import json, sys - p = json.load(open("deploy/pins/staging.json")) - prev = json.load(open("/tmp/staging-previous.json")) - p["previous"] = prev - with open("deploy/pins/staging.json", "w", encoding="utf-8") as f: - json.dump(p, f, indent=2, sort_keys=True) - f.write("\n") + p = json.load(open("deploy/pins/prod.json")) assert p["commit_sha"] == sys.argv[1], (p["commit_sha"], sys.argv[1]) for s in ("validator", "gateway", "updater", "bounty-challenge", "proof-challenge"): assert s in p["services"], s assert p["services"][s]["digest"].startswith("sha256:") - assert s in prev["services"], s - assert prev["services"][s]["digest"].startswith("sha256:") - print("staging pins ok", p["commit_sha"], "previous", prev.get("commit_sha")) + print("prod pins ok", p["commit_sha"]) ' "$SHA" - - name: Commit and push staging pins + - name: Stage pins artifact run: | set -euo pipefail SHA="${{ github.sha }}" - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add "deploy/digests/${SHA}.json" deploy/pins/staging.json - if git diff --cached --quiet; then - echo "No pin/digest changes to commit" - exit 0 - fi - git commit -m "deploy: staging pins for ${SHA}" - # Rebase onto latest main in case another pin commit landed. - git pull --rebase origin main - git push origin HEAD:main + STAGE="/tmp/prod-pins-artifact" + mkdir -p "$STAGE/deploy/pins" "$STAGE/deploy/digests" + cp deploy/pins/prod.json "$STAGE/deploy/pins/prod.json" + cp "deploy/digests/${SHA}.json" "$STAGE/deploy/digests/${SHA}.json" + python3 - >> "$GITHUB_STEP_SUMMARY" <<'PY' + import json + pin = json.load(open("deploy/pins/prod.json", encoding="utf-8")) + sha = pin["commit_sha"] + out = [f"### prod pins for `{sha}`", "", "| service | digest |", "| --- | --- |"] + for svc in sorted(pin["services"]): + out.append(f"| {svc} | `{pin['services'][svc]['digest']}` |") + out += ["", f"deploy-prod.yml consumes these as artifact `prod-pins-{sha}`."] + print("\n".join(out)) + PY + + - name: Upload prod pins artifact + uses: actions/upload-artifact@v4 + with: + name: prod-pins-${{ github.sha }} + path: /tmp/prod-pins-artifact + if-no-files-found: error # --------------------------------------------------------------------------- # Make GHCR packages public so unauthenticated / Phala pulls work. Lives here @@ -303,7 +295,7 @@ jobs: # --------------------------------------------------------------------------- ghcr-public: needs: merge-digests - if: github.event_name == 'push' && github.ref == 'refs/heads/main' + if: github.event_name != 'pull_request' runs-on: ubuntu-latest permissions: contents: read @@ -315,14 +307,19 @@ jobs: OWNER: ${{ github.repository_owner }} run: | set -euo pipefail - # Nested image names: ghcr.io//base/ → package name "base/" + # Nested image names: ghcr.io/// → package "/". + # Derive the repo segment: the hardcoded "base/" prefix went stale when + # the repo was renamed, so every visibility call 404'd and the droplets + # were left pulling private packages. + repo="${GITHUB_REPOSITORY##*/}" + repo="${repo,,}" pkgs=( - "base/base-attest-helper" - "base/gateway" - "base/validator" - "base/updater" - "base/bounty-challenge" - "base/proof-challenge" + "${repo}/base-attest-helper" + "${repo}/gateway" + "${repo}/validator" + "${repo}/updater" + "${repo}/bounty-challenge" + "${repo}/proof-challenge" ) ok=0 # First list packages to Learn exact names (debug) diff --git a/deploy/AGENTS.md b/deploy/AGENTS.md index 5520dfe7a..2a57c1d86 100644 --- a/deploy/AGENTS.md +++ b/deploy/AGENTS.md @@ -213,15 +213,26 @@ Validator logs should show `Match epoch=` then `Match → submit_intent` / `subm Tunnel writes gitignored `deploy/env/local-tunnel.env` (`BASE_GATEWAY_PUBLIC_URL`). Co-located validator stays on `http://gateway:8080`; external clients use the tunnel URL. Host probe ports default to `2808x` (avoid staging SSH on `1808x`). -## CI: staging vs prod +## CI: prod only + +**No CI staging lane.** The DigitalOcean staging soak was retired (owner +decision, 2026-09-10): `deploy-staging.yml` is deleted and `ci.yml` deploys +nothing. The staging droplets and `deploy/compose/env-staging.yml` remain for +manual `remote-deploy.sh` work and for `local-e2e.sh`. | Lane | Trigger | Build stance | |------|---------|--------------| -| Staging | CI green on `main` (`deploy-staging.yml`) | `--build-from source` on droplet OK for iteration | -| Images | Push to `main` (`images.yml`) | Build/push GHCR digests; promote + **commit** `deploy/pins/staging.json` + `deploy/digests/.json` | -| Prod | Tag `v*.*.*` (`deploy-prod.yml`) | **`--build-from registry` only** — promote staging→prod pins, pull GHCR digests; no Rust source build on prod hosts | - -Ladder: CI → GHCR digests → `deploy/pins/staging.json` (committed by `images.yml`) → tag → preflight (CI + staging pins match tag SHA) → `promote.sh` → `remote-deploy.sh --build-from registry`. Details: [`README.md`](README.md) § Auto CI deploy and § Promotion pipeline. +| CI | Push / PR on `main` (`ci.yml`) | fmt · clippy · test · deny · xtask — no droplet is touched | +| Images | Push to `main` (`images.yml`) | Build/push GHCR digests; `promote.sh --env prod` over those digests; publish `deploy/pins/prod.json` + `deploy/digests/.json` as artifact `prod-pins-` | +| Prod | Green `images` run on `main`, `v*.*.*` tag, or dispatch (`deploy-prod.yml`) | **`--build-from registry` only** — pull GHCR digests; no Rust source build on prod hosts | + +Ladder: CI green + GHCR digests → prod pins artifact → preflight (SHA on +`origin/main`, CI green, artifact live) → fail-closed Spaces backup → +`remote-deploy.sh --build-from registry`. **CI never pushes pins to `main`** — +branch protection (PR + Greptile review) rejects it with GH013, which is why +pins travel as a run artifact. Rollback = dispatch `deploy-prod` with the +previous good commit SHA. Details: [`README.md`](README.md) § Auto CI deploy and +§ Promotion pipeline. ## Secrets / age @@ -230,13 +241,13 @@ Ladder: CI → GHCR digests → `deploy/pins/staging.json` (committed by `images - Runtime secret files (wallets, keys): mode **0400**, owner **uid 65532**. - Helpers: `age-encrypt-env.sh`, `age-push-env.sh`. Checklist: [`docs/OPERATOR_SECURITY.md`](../docs/OPERATOR_SECURITY.md). -## First prod tag checklist +## Prod deploy checklist -1. Staging healthy on the exact commit you will tag; `deploy/pins/staging.json` `commit_sha` matches that SHA. +1. `ci` and `images` both green on the commit; the `images` run published `prod-pins-`. 2. Digests recorded / promoted for services you will ship (`promote.sh`, `verify-task-43.sh` locally if needed). 3. Age identity + env ages present on both prod hosts; wallets hotkeys under `deploy/secrets/wallets/` (0400 / 65532). 4. Mainnet owner wallet on disk matches SubnetOwnerHotkey; `env-prod.yml` sets `BASE_GATEWAY_REQUIRE_OWNER=1` (`gateway_admin_token` required). Recreate the gateway on droplets after compose changes. -5. Cut `vX.Y.Z` on `main`, push tag; pass `deploy-prod` preflight + `environment: production` reviewers. +5. Merging to `main` deploys prod once `images` is green; a `vX.Y.Z` tag or a `deploy-prod` dispatch re-deploys the same digests. 6. Smoke `/healthz` on both prod hosts; confirm `evil-gateway` absent. ## Out of scope for agents (ops) diff --git a/deploy/README.md b/deploy/README.md index 9a07c0170..4567aeb82 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -113,29 +113,42 @@ combination. Verify locally: `./deploy/scripts/assert-compose-matrix.sh`. ### Auto CI deploy -- `.github/workflows/ci.yml` — auto-deploy staging after its `ci` job succeeds on `main`; `deploy-staging.yml` is the manual lane -- `.github/workflows/deploy-prod.yml` — on push of `v*.*.*` tags from `main` (and manual dispatch with SHA) - -**Prod release flow (tag-based):** -1. CI passes on `main` for commit X. -2. `images.yml` builds/pushes GHCR digests for X, promotes pin services into `deploy/pins/staging.json`, commits digests + pins to `main`. -3. Staging droplets may still deploy with `--build-from source` (iteration); the pin ladder is what authorizes prod. -4. Operator cuts `git tag vX.Y.Z` on commit X and pushes the tag. -5. `deploy-prod.yml` preflight: CI green for X; `origin/main` staging pins `commit_sha == X`. -6. Fail-closed Postgres backup (SSH dump on prod master → DO Spaces), then `promote.sh --env prod --confirm-prod` per service. -7. Both prod hosts: `remote-deploy.sh --build-from registry` (pull GHCR `@sha256`, retag to Compose tags, `up --no-build`). -8. Smoke `/healthz`. `environment: production` (enable required reviewers in GitHub UI). +**The DigitalOcean staging soak is retired** (owner decision, 2026-09-10). CI no +longer deploys staging droplets: `ci.yml` is fmt/clippy/test/deny/xtask only and +`deploy-staging.yml` is deleted. `deploy/compose/env-staging.yml` stays — it is +the testnet overlay `local-e2e.sh` builds on, not a CI deploy lane. + +- `.github/workflows/images.yml` — on push to `main`: build/push GHCR digests, then record prod pins as the `prod-pins-` artifact +- `.github/workflows/deploy-prod.yml` — on a successful `images` run on `main`, on `v*.*.*` tags, or manual dispatch with a SHA + +**Prod deploy flow (every main update):** +1. CI passes on `main` for commit X; `images.yml` builds/pushes GHCR digests for X. +2. `images.yml` job `prod-pins` runs `promote.sh --env prod` over those digests and uploads `deploy/pins/prod.json` + `deploy/digests/X.json` as artifact `prod-pins-X`. **Nothing is pushed to `main`** — branch protection (PR + Greptile review) rejects a CI pin commit with GH013. +3. `deploy-prod.yml` preflight: X is an ancestor of `origin/main`, CI is green for X (it polls, since `ci` and `images` run in parallel), and the `images` run for X has a live `prod-pins-X` artifact. +4. Fail-closed Postgres backup (SSH dump on prod master → DO Spaces). +5. Both prod hosts: `remote-deploy.sh --build-from registry` (pull GHCR `@sha256`, retag to Compose tags, `up --no-build`). +6. Smoke `/healthz` (fail-closed). + +`deploy/pins/prod.json` in git is a **template**, not the deployed state: CI +derives the deployed pins per commit from the GHCR digests and keeps them in the +run artifact. **Rollback = dispatch `deploy-prod` with the previous good commit +SHA** (its `images` run artifact is still the pin set for that commit); the +in-tree `promote.sh --rollback` path stays for local/manual pin work. + +Artifacts expire. If the pin artifact for the commit you want is gone, +`deploy-prod` preflight refuses rather than deploying something unpinned — re-run +that commit's `images` run, or dispatch `images.yml` on a ref pointing at it +(`prod-pins` runs on manual dispatch too, exactly for this recovery). + +Prod hosts pull GHCR anonymously (`remote-deploy.sh` never logs in), so the +`ghcr-public` job must keep the packages public. Required GitHub secrets: | Secret | Purpose | |--------|---------| -| `STAGING_SSH_KEY` | private key for droplet SSH | -| `STAGING_MASTER_HOST` | public IPv4 of `base-staging` | -| `STAGING_VALIDATOR_HOST` | public IPv4 of `base-staging-validator` | -| `STAGING_MASTER_GATEWAY_URL` | optional, default `http://10.116.0.2:8080` | | `PROD_HOST` | public IPv4 of `base-prod` | -| `PROD_SSH_KEY` | optional override of staging key | +| `PROD_SSH_KEY` | private key for prod droplet SSH (falls back to `STAGING_SSH_KEY`, which is the same operator key) | | `PROD_VALIDATOR_HOST` | public IPv4 of `base-prod-validator` | | `PROD_MASTER_GATEWAY_URL` | optional, default `http://10.116.0.3:8080` | | `BASE_BACKUP_ENDPOINT` | DO Spaces endpoint (e.g. `https://nyc3.digitaloceanspaces.com`) — **required for prod promote (fail-closed)** | @@ -213,9 +226,11 @@ export BASE_BACKUP_BUCKET=base-backups --env staging --service validator \ --image ghcr.io/org/validator@sha256:<64-hex> -# 3) After staging is healthy, promote same digest to prod +# 3) Promote a digest to prod +# --force-prod skips the staging-digest ladder, which no longer exists: +# nothing writes deploy/pins/staging.json since the staging soak was retired. ./deploy/scripts/promote.sh \ - --env prod --service validator --confirm-prod \ + --env prod --service validator --confirm-prod --force-prod \ --image ghcr.io/org/validator@sha256:<64-hex> # 4) Rollback = re-promote previous snapshot @@ -226,7 +241,11 @@ export BASE_BACKUP_BUCKET=base-backups ``` Pin files: `deploy/pins/staging.json`, `deploy/pins/prod.json`. -Staging promote **never** writes the prod pin. Prod promote requires staging ladder + `--confirm-prod`. -Updater consumes `BASE_UPDATER_DESIRED_IMAGE` (also written to `deploy/pins/.desired.env`). +Staging promote **never** writes the prod pin; the staging pin file is now only a +local/manual scratch env (`verify-task-43.sh` exercises it) and no workflow +writes it. Prod promote still requires `--confirm-prod`. +Updater consumes `BASE_UPDATER_DESIRED_IMAGE` (also written to `deploy/pins/.desired.env`). +In CI the prod rollback is a `deploy-prod` dispatch on the previous good commit +SHA, not a pin-file edit — pins are rebuilt from that commit's GHCR digests. Verify locally: `./deploy/scripts/verify-task-43.sh` diff --git a/docs/COMPLETENESS.md b/docs/COMPLETENESS.md index a6814ed27..a3159b6ba 100644 --- a/docs/COMPLETENESS.md +++ b/docs/COMPLETENESS.md @@ -103,9 +103,9 @@ Agent/operator contracts: root [`AGENTS.md`](../AGENTS.md), [`deploy/AGENTS.md`] | Staging master | done | Migrated to `/opt/base` CI-managed; old `/opt/gbase` stack torn down. | | Staging validator | done | Redeployed from same commit; `bundle gateway signature invalid` resolved. | | Prod master | done | Droplet up. Mainnet owner wallet on disk matches SubnetOwnerHotkey; `env-prod.yml` sets `BASE_GATEWAY_REQUIRE_OWNER=1` (`gateway_admin_token` required). Recreate the gateway on droplets after that compose change. | -| `deploy-staging.yml` | done | Auto on CI green; `--build-from source` for fast iteration; fail-closed health gate. | -| `deploy-prod.yml` | done | Tag-based (`v*.*.*`); preflight (CI green + `origin/main` staging pins `commit_sha`); fail-closed Spaces backup; `promote.sh --confirm-prod`; `--build-from registry` (GHCR digest pull, no Rust compile on droplet). | -| `images.yml` pin ladder | done | After GHCR push: write `deploy/digests/.json`, `promote.sh --env staging` for pin services, commit/push so prod preflight can match. | +| `deploy-staging.yml` | removed | DigitalOcean staging soak retired (owner decision, 2026-09-10). The staging droplets stay for manual `remote-deploy.sh`; no workflow deploys them. | +| `deploy-prod.yml` | done | Runs on a green `images` run on `main`, on `v*.*.*` tags, or on dispatch; preflight (SHA on `origin/main` + CI green + live `prod-pins-` artifact); fail-closed Spaces backup; `--build-from registry` (GHCR digest pull, no Rust compile on droplet). | +| `images.yml` pin ladder | done | After GHCR push: write `deploy/digests/.json`, `promote.sh --env prod` for pin services, upload `prod-pins-`. No push to `main` — branch protection rejects a CI pin commit (GH013). | | GitHub secrets | done | Host/SSH/gateway secrets set. Prod promote also needs Spaces: `BASE_BACKUP_ENDPOINT`, `SPACES_ACCESS_KEY_ID` / `SPACES_SECRET_ACCESS_KEY` (fail-closed if absent). | ## Keys and identity diff --git a/docs/runbooks/promote-rollback-restore.md b/docs/runbooks/promote-rollback-restore.md index 101741b21..ac15ba74c 100644 --- a/docs/runbooks/promote-rollback-restore.md +++ b/docs/runbooks/promote-rollback-restore.md @@ -1,6 +1,10 @@ # Runbook: promote, rollback, restore -Digest-pinned rollouts for staging → prod. Updater behaviour: crate `updater` (D14). Compose layout: [`../../deploy/README.md`](../../deploy/README.md). +Digest-pinned rollouts to prod. Updater behaviour: crate `updater` (D14). Compose layout: [`../../deploy/README.md`](../../deploy/README.md). + +The DigitalOcean staging soak was retired on 2026-09-10: nothing gates prod on a +prior staging deploy. CI green on the commit plus the fail-closed prod smoke is +the gate, and rollback is a `deploy-prod` dispatch on the previous good SHA. **Self-update of the updater is an operator one-shot, never automatic in prod.** @@ -10,7 +14,7 @@ Digest-pinned rollouts for staging → prod. Updater behaviour: crate `updater` - [ ] CI green on the commit you are promoting. - [ ] Images built and pushed as `repo@sha256:<64 hex>` only. -- [ ] Staging has run the new digest long enough to pass `/readyz` and smoke checks. +- [ ] The digest you are promoting was built by `images.yml` for that exact commit. - [ ] You have SSH to the target host and age identity already on the box (R11). - [ ] Postgres volume is healthy. @@ -103,7 +107,7 @@ docker compose stop validator gateway updater docker compose --profile master up -d # on master; omit profile on pure validators ``` -Prod must stay on the last good digest until staging proves the fix. +Prod must stay on the last good digest until a fix lands on `main` and its `images` run is green. --- diff --git a/docs/runbooks/staging-testnet-e2e.md b/docs/runbooks/staging-testnet-e2e.md index 3114731b1..f84b06742 100644 --- a/docs/runbooks/staging-testnet-e2e.md +++ b/docs/runbooks/staging-testnet-e2e.md @@ -6,7 +6,7 @@ End-to-end testnet 541 procedure on the 2-host staging pair. - Staging master (`base-staging`, 68.183.23.51 / 10.116.0.2) running master role - Staging validator (`base-staging-validator`, 142.93.197.253 / 10.116.0.4) running validator role -- Both deployed from the same `main` commit via `deploy-staging.yml` or manual `remote-deploy.sh` +- Both deployed from the same `main` commit via manual `remote-deploy.sh` (the `deploy-staging.yml` lane was retired on 2026-09-10 — staging is operator-driven now) - `deploy/secrets/bounty_sk`, `deploy/secrets/proof_sk`, and `deploy/secrets/gateway_sk` present on master (mode 0400, uid 65532) - `deploy/env/*.env` materialized on both hosts (mode 0600) @@ -96,13 +96,15 @@ cargo run -p xtask -- metadata-snapshot --check ## Deploying a new commit to staging -1. Push to `main` — `ci.yml` runs, then `deploy-staging.yml` auto-deploys both hosts. -2. Or manual: - ```bash - ./deploy/scripts/remote-deploy.sh --host root@68.183.23.51 --role master --env staging --build-from source - ./deploy/scripts/remote-deploy.sh --host root@142.93.197.253 --role validator --env staging --build-from source - ``` -3. Post-deploy: CI checks validator `/healthz` (fail-closed) and greps for `Match epoch=` within 180s. +No workflow deploys staging any more — run it by hand from the commit you want: + +```bash +./deploy/scripts/remote-deploy.sh --host root@68.183.23.51 --role master --env staging --build-from source +./deploy/scripts/remote-deploy.sh --host root@142.93.197.253 --role validator --env staging --build-from source +``` + +Then check the validator yourself: `/healthz` up and a `Match epoch=` line in +`docker compose logs validator` (CI used to assert both after its auto-deploy). ## Rollback diff --git a/metadata/testnet.lock b/metadata/testnet.lock index 937278e6d..f61a50eec 100644 --- a/metadata/testnet.lock +++ b/metadata/testnet.lock @@ -5,11 +5,11 @@ "snapshot_netuid": 1, "chain": { "spec_name": "node-subtensor", - "spec_version": 445, + "spec_version": 455, "transaction_version": 1, "ss58_prefix": 42 }, - "metadata_digest": "0x25b0d497f71c915c622efe3a55a4d3b5d6aa405856df193bca7df0791cae5497", + "metadata_digest": "0x9980e6bf18c2a801597592599de95eed229838a395c18f07431867073a87e570", "call_indices": { "set_weights": { "pallet": "SubtensorModule",