From 481381da34ffef04e023ccb1b9b3cb95197b0078 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Sat, 12 Sep 2026 16:40:37 +0800 Subject: [PATCH] fix: expose sanitized keepalive failure stages Co-Authored-By: Codex --- .github/workflows/main.yml | 31 ++++++- tests/test_workflow_shared_config.sh | 125 +++++++++++++++++++++++++++ 2 files changed, 155 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2f8dc1e..666ba0f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -635,6 +635,18 @@ jobs: if [ "${DEPLOY_MODE}" = "keepalive" ]; then REMOTE_DEPLOY_COMMAND=$(cat <&2 + ;; + *) + echo "GATEWAY_REMOTE_FAILURE_STAGE=UNKNOWN" >&2 + ;; + esac + } + trap 'status=\$?; if [ "\${status}" -ne 0 ]; then emit_remote_failure_stage; fi; exit "\${status}"' EXIT cd '${DEPLOY_PATH}' echo "[\$(date -u +%FT%TZ)] Scheduled keepalive mode: skip docker build" set -a @@ -673,23 +685,32 @@ jobs: set -e return "\${restore_status}" } + remote_failure_stage="ARM_WATCHER_FAILSAFE" arm_gateway_watchers_failsafe - trap 'status=\$?; if [ "\${watchers_restored}" != "true" ]; then restore_gateway_watchers || true; fi; exit "\${status}"' EXIT + trap 'status=\$?; if [ "\${status}" -ne 0 ]; then emit_remote_failure_stage; fi; if [ "\${watchers_restored}" != "true" ]; then restore_gateway_watchers || true; fi; exit "\${status}"' EXIT + remote_failure_stage="STOP_WATCHERS" sudo systemctl stop "\${IBKR_GATEWAY_HEALTHCHECK_TIMER}" "\${IBKR_GATEWAY_HEALTHCHECK_SERVICE}" 2>/dev/null || true sudo systemctl stop "\${IBKR_GATEWAY_DAILY_RESTART_TIMER}" "\${IBKR_GATEWAY_DAILY_RESTART_SERVICE}" 2>/dev/null || true + remote_failure_stage="ENSURE_HOST_SWAP" sudo bash ./scripts/ensure_host_swap.sh + remote_failure_stage="CHECK_RUNTIME_IMAGE" if ! sudo docker image inspect "\${compose_image}" >/dev/null 2>&1; then echo "[\$(date -u +%FT%TZ)] Keepalive fallback: image \${compose_image} is missing, building only \${compose_service_name}" sudo docker compose build "\${compose_service_name}" fi + remote_failure_stage="CHECK_2FA_BIND_MOUNT" host_2fa_bot_sha256="\$(sha256sum ./2fa_bot.py | awk '{print \$1}')" container_2fa_bot_sha256="\$(sudo timeout 15 docker exec "\${container_name}" sha256sum /home/ibgateway/2fa_bot.py 2>/dev/null | awk '{print \$1}' || true)" if [ "\${container_2fa_bot_sha256}" != "\${host_2fa_bot_sha256}" ]; then + remote_failure_stage="REFRESH_2FA_BIND_MOUNT" echo "[\$(date -u +%FT%TZ)] Refreshing stale 2FA bot bind mount for \${container_name}" sudo docker compose up -d --force-recreate --no-build "\${compose_service_name}" fi + remote_failure_stage="RECOVER_GATEWAY" sudo env IB_GATEWAY_CONTAINER_NAME="\${container_name}" IB_GATEWAY_COMPOSE_SERVICE_NAME="\${compose_service_name}" bash ./scripts/recover_ib_gateway_ready.sh '${IB_GATEWAY_MODE}' + remote_failure_stage="RESTORE_WATCHERS" restore_gateway_watchers + remote_failure_stage="REPORT_RUNTIME_STATUS" sudo docker compose ps sudo systemctl status "\${IBKR_2FA_BOT_TIMER}" --no-pager sudo systemctl status "\${IBKR_GATEWAY_HEALTHCHECK_TIMER}" --no-pager @@ -763,10 +784,18 @@ jobs: emit_sanitized_remote_failure_stage() { local command_log="$1" local recovery_stage + local remote_stage recovery_stage="$(grep -Eo 'GATEWAY_RECOVERY_FAILURE_STAGE=[A-Z0-9_]+' "${command_log}" | tail -n 1 || true)" case "${recovery_stage}" in GATEWAY_RECOVERY_FAILURE_STAGE=RECOVERY_CONFIGURATION_INVALID| GATEWAY_RECOVERY_FAILURE_STAGE=RECOVERY_LOCK_TIMEOUT| GATEWAY_RECOVERY_FAILURE_STAGE=GATEWAY_UI_BLOCKER| GATEWAY_RECOVERY_FAILURE_STAGE=DIALOG_RECOVERY_NOT_READY| GATEWAY_RECOVERY_FAILURE_STAGE=CONTAINER_START_FAILED| GATEWAY_RECOVERY_FAILURE_STAGE=TWOFA_WATCHER_FAILED| GATEWAY_RECOVERY_FAILURE_STAGE=CONTAINER_RESTART_FAILED| GATEWAY_RECOVERY_FAILURE_STAGE=CONTAINER_RECREATE_FAILED| GATEWAY_RECOVERY_FAILURE_STAGE=GATEWAY_NOT_READY_AFTER_RECREATE) log_step "${recovery_stage}" + return + ;; + esac + remote_stage="$(grep -Eo 'GATEWAY_REMOTE_FAILURE_STAGE=[A-Z0-9_]+' "${command_log}" | tail -n 1 || true)" + case "${remote_stage}" in + GATEWAY_REMOTE_FAILURE_STAGE=LOAD_RUNTIME_ENV| GATEWAY_REMOTE_FAILURE_STAGE=ARM_WATCHER_FAILSAFE| GATEWAY_REMOTE_FAILURE_STAGE=STOP_WATCHERS| GATEWAY_REMOTE_FAILURE_STAGE=ENSURE_HOST_SWAP| GATEWAY_REMOTE_FAILURE_STAGE=CHECK_RUNTIME_IMAGE| GATEWAY_REMOTE_FAILURE_STAGE=CHECK_2FA_BIND_MOUNT| GATEWAY_REMOTE_FAILURE_STAGE=REFRESH_2FA_BIND_MOUNT| GATEWAY_REMOTE_FAILURE_STAGE=RECOVER_GATEWAY| GATEWAY_REMOTE_FAILURE_STAGE=RESTORE_WATCHERS| GATEWAY_REMOTE_FAILURE_STAGE=REPORT_RUNTIME_STATUS) + log_step "${remote_stage}" ;; *) log_step "GATEWAY_DEPLOY_FAILURE_STAGE=REMOTE_COMMAND_OR_TRANSPORT_FAILED" diff --git a/tests/test_workflow_shared_config.sh b/tests/test_workflow_shared_config.sh index 9630d64..6de9375 100644 --- a/tests/test_workflow_shared_config.sh +++ b/tests/test_workflow_shared_config.sh @@ -278,12 +278,137 @@ grep -Fq 'name: Maintain gateway target' "$repo_dir/.github/workflows/remote-mai python3 - "$workflow_file" <<'PY' from pathlib import Path +import subprocess import sys +import tempfile workflow = Path(sys.argv[1]).read_text(encoding="utf-8") assert 'emit_sanitized_remote_failure_stage()' in workflow assert "GATEWAY_DEPLOY_FAILURE_STAGE=REMOTE_COMMAND_OR_TRANSPORT_FAILED" in workflow assert "GATEWAY_RECOVERY_FAILURE_STAGE=GATEWAY_NOT_READY_AFTER_RECREATE" in workflow +assert 'GATEWAY_REMOTE_FAILURE_STAGE=LOAD_RUNTIME_ENV' in workflow +assert 'GATEWAY_REMOTE_FAILURE_STAGE=ARM_WATCHER_FAILSAFE' in workflow +assert 'GATEWAY_REMOTE_FAILURE_STAGE=STOP_WATCHERS' in workflow +assert 'GATEWAY_REMOTE_FAILURE_STAGE=ENSURE_HOST_SWAP' in workflow +assert 'GATEWAY_REMOTE_FAILURE_STAGE=CHECK_RUNTIME_IMAGE' in workflow +assert 'GATEWAY_REMOTE_FAILURE_STAGE=CHECK_2FA_BIND_MOUNT' in workflow +assert 'GATEWAY_REMOTE_FAILURE_STAGE=REFRESH_2FA_BIND_MOUNT' in workflow +assert 'GATEWAY_REMOTE_FAILURE_STAGE=RECOVER_GATEWAY' in workflow +assert 'GATEWAY_REMOTE_FAILURE_STAGE=RESTORE_WATCHERS' in workflow +assert 'GATEWAY_REMOTE_FAILURE_STAGE=REPORT_RUNTIME_STATUS' in workflow assert 'gcloud compute ssh "${REMOTE_TARGET}" "${SSH_FLAGS[@]}" --command "${REMOTE_DEPLOY_COMMAND}" >"${command_log}" 2>&1' in workflow assert 'run_sanitized_remote_deploy' in workflow + +function_start = " emit_sanitized_remote_failure_stage() {\n" +function_end = "\n }\n\n run_sanitized_remote_deploy()" +classifier = workflow.split(function_start, 1)[1].split(function_end, 1)[0] +classifier = "emit_sanitized_remote_failure_stage() {\n" + "\n".join( + line.removeprefix(" ") for line in classifier.splitlines() +) + "\n}" + +def classify(log_text: str) -> subprocess.CompletedProcess[str]: + with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8") as command_log: + command_log.write(log_text) + command_log.flush() + script = "log_step() { printf '%s\\n' \"$1\"; }\n" + classifier + '\nemit_sanitized_remote_failure_stage "$1"\n' + return subprocess.run( + ["bash", "-c", script, "classifier", command_log.name], + capture_output=True, + check=True, + text=True, + ) + +sensitive = "TOTP_SECRET=do-not-print account=U12345678" +recovery = classify(f"{sensitive}\nGATEWAY_RECOVERY_FAILURE_STAGE=GATEWAY_UI_BLOCKER\n") +assert recovery.stdout.strip() == "GATEWAY_RECOVERY_FAILURE_STAGE=GATEWAY_UI_BLOCKER" +assert sensitive not in recovery.stdout + +remote = classify(f"{sensitive}\nGATEWAY_REMOTE_FAILURE_STAGE=ENSURE_HOST_SWAP\n") +assert remote.stdout.strip() == "GATEWAY_REMOTE_FAILURE_STAGE=ENSURE_HOST_SWAP" +assert sensitive not in remote.stdout + +unknown = classify(f"{sensitive}\nGATEWAY_REMOTE_FAILURE_STAGE=UNTRUSTED_STAGE\n") +assert unknown.stdout.strip().endswith("GATEWAY_DEPLOY_FAILURE_STAGE=REMOTE_COMMAND_OR_TRANSPORT_FAILED") +assert sensitive not in unknown.stdout + +transport = classify(f"{sensitive}\nssh: connect to host failed\n") +assert transport.stdout.strip().endswith("GATEWAY_DEPLOY_FAILURE_STAGE=REMOTE_COMMAND_OR_TRANSPORT_FAILED") +assert sensitive not in transport.stdout + +keepalive_start = ' if [ "${DEPLOY_MODE}" = "keepalive" ]; then\n' +remote_start = " REMOTE_DEPLOY_COMMAND=$(cat <>"$restore_log_path"; }\n' + + 'remote_failure_stage="ARM_WATCHER_FAILSAFE"\nexit 19\n', + "arm-failure", + restore_log.name, + ], + capture_output=True, + text=True, + ) + restore_log.seek(0) + assert arm_failure.returncode == 19 + assert restore_log.read() == "" + assert arm_failure.stderr.strip() == "GATEWAY_REMOTE_FAILURE_STAGE=ARM_WATCHER_FAILSAFE" + +with tempfile.NamedTemporaryFile(mode="r+", encoding="utf-8") as restore_log: + post_arm_failure = subprocess.run( + [ + "bash", + "-c", + "set -euo pipefail\n" + + 'remote_failure_stage="ENSURE_HOST_SWAP"\n' + + 'watchers_restored=false\n' + + 'restore_log_path="$1"\n' + + 'emit_remote_failure_stage() { printf "GATEWAY_REMOTE_FAILURE_STAGE=%s\\n" "$remote_failure_stage" >&2; }\n' + + 'restore_gateway_watchers() { printf restore >>"$restore_log_path"; }\n' + + trap_lines[1] + + "\nexit 23\n", + "post-arm-failure", + restore_log.name, + ], + capture_output=True, + text=True, + ) + restore_log.seek(0) + assert post_arm_failure.returncode == 23 + assert restore_log.read() == "restore" + assert post_arm_failure.stderr.strip() == "GATEWAY_REMOTE_FAILURE_STAGE=ENSURE_HOST_SWAP" + +failed_pipe = subprocess.run( + ["bash", "-c", remote_prelude + '\nremote_failure_stage="CHECK_RUNTIME_IMAGE"\nprintf x | false\n'], + capture_output=True, + text=True, +) +assert failed_pipe.returncode != 0 +assert failed_pipe.stderr.strip() == "GATEWAY_REMOTE_FAILURE_STAGE=CHECK_RUNTIME_IMAGE" + +successful_remote = subprocess.run( + ["bash", "-c", remote_prelude + "\ntrue\n"], + capture_output=True, + text=True, +) +assert successful_remote.returncode == 0 +assert "GATEWAY_REMOTE_FAILURE_STAGE" not in successful_remote.stderr PY