ci: releases/symlink deploy with health-check rollback and smoke-test gate

.github/workflows/deploy-Dataspace.yml

@@ -6,10 +6,18 @@ on:
       - dev
       - main
 
+# Prevent two deploys to the same environment from racing each other and
+# corrupting the releases/ directory or the .last_good / .rollback_target markers.
+concurrency:
+  group: deploy-dataspace-${{ github.ref_name }}
+  cancel-in-progress: false
+
 jobs:
-  deploy:
+  build-and-deploy:
     runs-on: ubuntu-latest
     environment: ${{ github.ref_name == 'main' && 'production' || 'development' }}
+    outputs:
+      release: ${{ steps.meta.outputs.release }}
     env:
       KEYCLOAK_CLIENT_ID: ${{ secrets.KEYCLOAK_CLIENT_ID }}
       KEYCLOAK_CLIENT_SECRET: ${{ secrets.KEYCLOAK_CLIENT_SECRET }}
@@ -39,14 +47,31 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v4
 
+      - name: Compute release name
+        id: meta
+        run: echo "release=$(date -u +%Y%m%d%H%M%S)-${GITHUB_SHA::7}" >> "$GITHUB_OUTPUT"
+
       - name: Setup Node.js
         uses: actions/setup-node@v4
         with:
           node-version: '24'
           cache: 'npm'
 
-      - name: Install dependencies
-        run: npm install
+      # npm ci, not npm install: installs exactly what package-lock.json pins.
+      # npm install can silently resolve a different version than the lockfile
+      # (this is how a canary/newer `next` ended up in a previous build).
+      - name: Install dependencies (from lockfile only)
+        run: npm ci --legacy-peer-deps
+
+      - name: Guard - installed Next.js must match package.json
+        run: |
+          DECLARED=$(node -p "require('./package.json').dependencies.next")
+          INSTALLED=$(node -p "require('./node_modules/next/package.json').version")
+          echo "declared next: $DECLARED / installed next: $INSTALLED"
+          if [ "$DECLARED" != "$INSTALLED" ]; then
+            echo "::error::Installed next ($INSTALLED) does not match package.json ($DECLARED). Aborting before build."
+            exit 1
+          fi
 
       - name: Generate
         run: npm run generate:ci
@@ -60,41 +85,98 @@ jobs:
       - name: Build
         run: npm run build
 
-      - name: Rename .next to .next2
-        run: mv .next .next2
-
-      - name: Rename public to public2
-        run: mv public public2
+      - name: Package release artifact
+        run: tar czf release.tar.gz .next public package.json package-lock.json next.config.mjs
 
-      - name: Send .next2 to EC2
+      - name: Send release artifact to EC2
         uses: appleboy/scp-action@v0.1.7
         with:
           host: ${{ vars.EC2_HOST }}
           username: ${{ secrets.EC2_USERNAME }}
           key: ${{ secrets.EC2_PRIVATE_KEY }}
-          source: .next2
-          target: DataExchange/DataExFrontend
+          source: release.tar.gz
+          target: DataExchange/incoming/${{ steps.meta.outputs.release }}
 
-      - name: Send public2 to EC2
-        uses: appleboy/scp-action@v0.1.7
+      # Extract into a brand-new releases/<release>/ directory, install deps
+      # THERE (never touching the currently-live release), verify, and only
+      # then flip the DataExFrontend symlink. If anything up to and including
+      # the health check fails, the previous release is still on disk untouched
+      # and this step rolls the symlink back itself before exiting non-zero.
+      - name: Extract, install, verify, and activate release on EC2
+        uses: appleboy/ssh-action@v1.0.3
         with:
           host: ${{ vars.EC2_HOST }}
           username: ${{ secrets.EC2_USERNAME }}
           key: ${{ secrets.EC2_PRIVATE_KEY }}
-          source: public2
-          target: DataExchange/DataExFrontend
+          script: |
+            set -euo pipefail
+            BASE=/home/ubuntu/DataExchange
+            RELEASE=${{ steps.meta.outputs.release }}
+            RELEASE_DIR="$BASE/releases/$RELEASE"
+            NODE_BIN=/home/ubuntu/.nvm/versions/node/v24.13.0/bin
+            PM2="$NODE_BIN/pm2"
+            NPM="$NODE_BIN/npm"
+            NODE="$NODE_BIN/node"
+
+            mkdir -p "$RELEASE_DIR" "$BASE/releases" "$BASE/shared"
+            tar xzf "$BASE/incoming/$RELEASE/release.tar.gz" -C "$RELEASE_DIR"
+            rm -rf "$BASE/incoming/$RELEASE"
+
+            # Runtime secrets live only in shared/.env.local (NOT the docker-compose .env one level up), never shipped by CI.
+            ln -sfn "$BASE/shared/.env.local" "$RELEASE_DIR/.env.local"
 
-      - name: Update with new Build
+            cd "$RELEASE_DIR"
+            "$NPM" ci --omit=dev
+
+            INSTALLED=$("$NODE" -p "require('./node_modules/next/package.json').version")
+            DECLARED=$("$NODE" -p "require('./package.json').dependencies.next")
+            if [ "$INSTALLED" != "$DECLARED" ]; then
+              echo "Next version mismatch on server ($INSTALLED vs $DECLARED). Not activating." >&2
+              exit 1
+            fi
+
+            if [ -f "$BASE/releases/.last_good" ]; then
+              PREVIOUS=$(cat "$BASE/releases/.last_good")
+            else
+              PREVIOUS=$(basename "$(readlink -f "$BASE/DataExFrontend")")
+            fi
+            echo "$PREVIOUS" > "$BASE/releases/.rollback_target"
+
+            ln -sfn "$RELEASE_DIR" "$BASE/DataExFrontend"
+            "$PM2" restart dataspace
+
+            ATTEMPTS=0
+            until curl -f -s -o /dev/null http://127.0.0.1:3000; do
+              ATTEMPTS=$((ATTEMPTS+1))
+              if [ "$ATTEMPTS" -ge 10 ]; then
+                echo "Health check failed after $ATTEMPTS attempts. Rolling back to $PREVIOUS." >&2
+                ln -sfn "$BASE/releases/$PREVIOUS" "$BASE/DataExFrontend"
+                "$PM2" restart dataspace
+                exit 1
+              fi
+              sleep 3
+            done
+
+            echo "Release $RELEASE is live and passed the boot health check."
+
+            # Keep the 5 most recent releases plus whatever the rollback target is.
+            cd "$BASE/releases"
+            ls -1dt */ 2>/dev/null | tail -n +6 | grep -v "^${PREVIOUS}/$" | xargs -r rm -rf
+
+      # Production has no smoke-test job today, so its only quality gate is the
+      # boot health check above. Mark this release good immediately so a FUTURE
+      # deploy knows what to roll back to if it fails.
+      - name: Mark release as last-known-good (prod only)
+        if: github.ref_name == 'main'
         uses: appleboy/ssh-action@v1.0.3
-        continue-on-error: false
         with:
           host: ${{ vars.EC2_HOST }}
           username: ${{ secrets.EC2_USERNAME }}
           key: ${{ secrets.EC2_PRIVATE_KEY }}
-          script: rm -rf DataExchange/DataExFrontend/.next; rm -rf DataExchange/DataExFrontend/public; mv DataExchange/DataExFrontend/.next2 DataExchange/DataExFrontend/.next; mv DataExchange/DataExFrontend/public2 DataExchange/DataExFrontend/public; /home/ubuntu/.nvm/versions/node/v20.11.1/bin/pm2 restart dataspace
+          script: echo "${{ steps.meta.outputs.release }}" > /home/ubuntu/DataExchange/releases/.last_good
 
   smoke-tests:
-    needs: deploy
+    needs: build-and-deploy
     if: github.ref_name == 'dev'
     uses: CivicDataLab/CivicDataSpace-test/.github/workflows/run-smoke.yml@CI
     secrets:
@@ -103,3 +185,40 @@ jobs:
       TEST_PASSWORD_1: ${{ secrets.TEST_PASSWORD_1 }}
       TEST_EMAIL_2: ${{ secrets.TEST_EMAIL_2 }}
       TEST_PASSWORD_2: ${{ secrets.TEST_PASSWORD_2 }}
+
+  # dev only: smoke tests are the real quality gate here. Only once they pass
+  # does this release become the thing a future rollback would target.
+  promote-dev:
+    needs: [build-and-deploy, smoke-tests]
+    if: github.ref_name == 'dev' && needs.smoke-tests.result == 'success'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Mark this release as last-known-good
+        uses: appleboy/ssh-action@v1.0.3
+        with:
+          host: ${{ vars.EC2_HOST }}
+          username: ${{ secrets.EC2_USERNAME }}
+          key: ${{ secrets.EC2_PRIVATE_KEY }}
+          script: echo "${{ needs.build-and-deploy.outputs.release }}" > /home/ubuntu/DataExchange/releases/.last_good
+
+  rollback-dev:
+    needs: [build-and-deploy, smoke-tests]
+    if: github.ref_name == 'dev' && needs.smoke-tests.result == 'failure'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Revert to last known-good release
+        uses: appleboy/ssh-action@v1.0.3
+        with:
+          host: ${{ vars.EC2_HOST }}
+          username: ${{ secrets.EC2_USERNAME }}
+          key: ${{ secrets.EC2_PRIVATE_KEY }}
+          script: |
+            set -euo pipefail
+            BASE=/home/ubuntu/DataExchange
+            PM2=/home/ubuntu/.nvm/versions/node/v24.13.0/bin/pm2
+            TARGET=$(cat "$BASE/releases/.rollback_target")
+            echo "Smoke tests failed. Reverting DataExFrontend -> releases/$TARGET"
+            ln -sfn "$BASE/releases/$TARGET" "$BASE/DataExFrontend"
+            "$PM2" restart dataspace
+            sleep 3
+            curl -f http://127.0.0.1:3000 -o /dev/null -s -w "post-rollback HTTP %{http_code}\n"

deploy/ec2-migrate-to-releases.sh

@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+# ONE-TIME migration: converts DataExFrontend from a plain directory into a
+# symlink pointing at releases/<name>/, with .env.local moved to a shared/
+# folder outside of any release. Run this by hand over SSH, once, before the
+# new GitHub Actions workflow's first deploy.
+#
+# NOTE: the runtime secrets file is .env.local (inside DataExFrontend itself),
+# NOT the .env one level up in /home/ubuntu/DataExchange/ - that one belongs
+# to the docker-compose stack (DataExAuth/DataExBackend/DataExKeycloak), not
+# this Next.js app. Confirmed via `pm2 env` showing no secrets in PM2's own
+# captured environment, and .env.local's size/presence matching
+# .env.local.example.
+#
+# Safe to re-run: it no-ops if DataExFrontend is already a symlink.
+#
+# Usage: bash ec2-migrate-to-releases.sh
+
+set -euo pipefail
+
+BASE="/home/ubuntu/DataExchange"
+APP_DIR="$BASE/DataExFrontend"
+RELEASES_DIR="$BASE/releases"
+SHARED_DIR="$BASE/shared"
+
+if [ -L "$APP_DIR" ]; then
+  echo "DataExFrontend is already a symlink -> $(readlink -f "$APP_DIR"). Nothing to do."
+  exit 0
+fi
+
+if [ ! -d "$APP_DIR" ]; then
+  echo "ERROR: $APP_DIR does not exist or is not a plain directory. Aborting." >&2
+  exit 1
+fi
+
+mkdir -p "$RELEASES_DIR" "$SHARED_DIR"
+
+LEGACY_NAME="legacy-$(date -u +%Y%m%d%H%M%S)"
+LEGACY_PATH="$RELEASES_DIR/$LEGACY_NAME"
+
+echo "Moving current $APP_DIR -> $LEGACY_PATH"
+mv "$APP_DIR" "$LEGACY_PATH"
+
+if [ -f "$LEGACY_PATH/.env.local" ]; then
+  echo "Moving .env.local -> $SHARED_DIR/.env.local"
+  mv "$LEGACY_PATH/.env.local" "$SHARED_DIR/.env.local"
+else
+  echo "WARNING: no .env.local found in the old app directory. If runtime env comes" >&2
+  echo "from somewhere else, create $SHARED_DIR/.env.local yourself with the right" >&2
+  echo "contents before deploying, or the app will start with no secrets configured." >&2
+fi
+
+ln -s "$SHARED_DIR/.env.local" "$LEGACY_PATH/.env.local"
+
+echo "Creating symlink $APP_DIR -> $LEGACY_PATH"
+ln -s "$LEGACY_PATH" "$APP_DIR"
+
+echo "Marking this as the last-known-good release"
+echo "$LEGACY_NAME" > "$RELEASES_DIR/.last_good"
+
+echo "Restarting PM2 to confirm nothing broke (cwd resolves through the new symlink transparently)"
+/home/ubuntu/.nvm/versions/node/v24.13.0/bin/pm2 restart dataspace
+
+sleep 3
+echo "Health check:"
+curl -f http://127.0.0.1:3000 -o /dev/null -s -w "HTTP %{http_code}\n"
+
+echo "Done. DataExFrontend now -> $(readlink -f "$APP_DIR")"
+echo "Verify the site AND login (Keycloak auth) manually before pushing the new workflow."