diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 666ba0f..7365ced 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -21,6 +21,7 @@ on: options: - keepalive - full + - restore-env approved_ref: description: Approved source ref for a full deployment; only refs/heads/main is accepted. required: false @@ -109,9 +110,9 @@ jobs: selected_name = os.environ.get("SELECTED_TARGET", "") or "all" selected_deploy_mode = os.environ.get("SELECTED_DEPLOY_MODE", "") or "keepalive" - if selected_deploy_mode == "full" and selected_name == "all": + if selected_deploy_mode in {"full", "restore-env"} and selected_name == "all": raise SystemExit( - "A full gateway deployment requires one explicit target; " + "A full or restore-env gateway operation requires one explicit target; " "use keepalive for all-target maintenance." ) if selected_name == "all": @@ -190,7 +191,7 @@ jobs: uses: actions/checkout@v6 - name: Verify full-deploy source provenance - if: ${{ github.event_name == 'workflow_dispatch' && inputs.deploy_mode == 'full' }} + if: ${{ github.event_name == 'workflow_dispatch' && (inputs.deploy_mode == 'full' || inputs.deploy_mode == 'restore-env') }} run: | set -euo pipefail @@ -252,7 +253,7 @@ jobs: require_secret_source SSH_PRIVATE_KEY_SECRET_NAME SSH_PRIVATE_KEY SSH_PRIVATE_KEY - name: Check full-deploy credential config - if: ${{ github.event_name == 'workflow_dispatch' && inputs.deploy_mode == 'full' }} + if: ${{ github.event_name == 'workflow_dispatch' && (inputs.deploy_mode == 'full' || inputs.deploy_mode == 'restore-env') }} env: TWS_USERID: ${{ secrets.TWS_USERID }} TWS_PASSWORD: ${{ secrets.TWS_PASSWORD }} @@ -277,9 +278,21 @@ jobs: require_secret_source TWS_USERID_SECRET_NAME TWS_USERID TWS_USERID require_secret_source TWS_PASSWORD_SECRET_NAME TWS_PASSWORD TWS_PASSWORD require_secret_source VNC_SERVER_PASSWORD_SECRET_NAME VNC_SERVER_PASSWORD VNC_SERVER_PASSWORD + if [ "${WORKFLOW_DISPATCH_MODE:-}" = "restore-env" ]; then + for required_secret_name in TWS_USERID_SECRET_NAME TWS_PASSWORD_SECRET_NAME VNC_SERVER_PASSWORD_SECRET_NAME; do + if [ -z "${!required_secret_name:-}" ]; then + echo "${required_secret_name} is required for restore-env; refusing cross-account fallback." >&2 + exit 1 + fi + done + fi twofa_autofill="$(printf '%s' "${IBKR_2FA_AUTOFILL:-}" | tr '[:upper:]' '[:lower:]')" if [ "${twofa_autofill}" = "yes" ] || [ "${twofa_autofill}" = "true" ] || [ "${twofa_autofill}" = "1" ]; then require_secret_source TOTP_SECRET_SECRET_NAME TOTP_SECRET TOTP_SECRET + if [ "${WORKFLOW_DISPATCH_MODE:-}" = "restore-env" ] && [ -z "${TOTP_SECRET_SECRET_NAME:-}" ]; then + echo "TOTP_SECRET_SECRET_NAME is required for restore-env; refusing cross-account fallback." >&2 + exit 1 + fi fi - name: Authenticate to Google Cloud @@ -390,6 +403,10 @@ jobs: if [ -n "${secret_name:-}" ]; then gcloud secrets versions access latest --project "${GCP_PROJECT_ID}" --secret "${secret_name}" else + if [ "${WORKFLOW_DISPATCH_MODE:-}" = "restore-env" ]; then + echo "restore-env requires target-specific Secret Manager names." >&2 + return 1 + fi printf '%s' "${fallback_value}" fi } @@ -412,7 +429,7 @@ jobs: echo "SSH_KEY_FILE=$SSH_KEY_FILE" >> "$GITHUB_ENV" - name: Prepare full-deploy runtime environment - if: ${{ github.event_name == 'workflow_dispatch' && inputs.deploy_mode == 'full' }} + if: ${{ github.event_name == 'workflow_dispatch' && (inputs.deploy_mode == 'full' || inputs.deploy_mode == 'restore-env') }} env: TWS_USERID: ${{ secrets.TWS_USERID }} TWS_PASSWORD: ${{ secrets.TWS_PASSWORD }} @@ -435,6 +452,11 @@ jobs: tws_userid="$(resolve_secret "${TWS_USERID_SECRET_NAME:-}" "${TWS_USERID:-}")" tws_password="$(resolve_secret "${TWS_PASSWORD_SECRET_NAME:-}" "${TWS_PASSWORD:-}")" totp_secret="" + if [ "${WORKFLOW_DISPATCH_MODE:-}" = "restore-env" ] && [ -z "${TOTP_SECRET_SECRET_NAME:-}" ] \ + && [ -n "${TOTP_SECRET:-}" ]; then + echo "TOTP_SECRET_SECRET_NAME is required for restore-env; refusing global TOTP fallback." >&2 + exit 1 + fi if [ -n "${TOTP_SECRET_SECRET_NAME:-}" ] || [ -n "${TOTP_SECRET:-}" ]; then totp_secret="$(resolve_secret "${TOTP_SECRET_SECRET_NAME:-}" "${TOTP_SECRET:-}")" fi @@ -505,13 +527,184 @@ jobs: def quote(value: str) -> str: return "'" + value.replace("'", "'\"'\"'") + "'" + if os.environ.get("WORKFLOW_DISPATCH_MODE") == "restore-env": + os.umask(0o077) with open(env_file, "w", encoding="utf-8") as fh: for key, value in values.items(): fh.write(f"{key}={quote(value)}\n") + os.chmod(env_file, 0o600) PY + if [ "${WORKFLOW_DISPATCH_MODE:-}" = "restore-env" ]; then + syntax_error_file="$RUNNER_TEMP/restore-env-bash-syntax.err" + if ! env -i bash -n "$ENV_FILE" 2>"$syntax_error_file"; then + echo "Generated restore-env file failed shell syntax validation." >&2 + rm -f "$syntax_error_file" + exit 1 + fi + rm -f "$syntax_error_file" + fi echo "ENV_FILE=$ENV_FILE" >> "$GITHUB_ENV" + - name: Restore missing gateway runtime environment + if: ${{ github.event_name == 'workflow_dispatch' && inputs.deploy_mode == 'restore-env' }} + run: | + set -euo pipefail + if [ "${WORKFLOW_DISPATCH_TARGET:-all}" = "all" ]; then + echo "restore-env requires one explicit target." >&2 + exit 1 + fi + REMOTE_TARGET="${GCE_USER}@${GCE_INSTANCE_NAME}" + REMOTE_COMMON_FLAGS=(--project "${GCP_PROJECT_ID}" --zone "${GCE_ZONE}" --quiet --tunnel-through-iap --ssh-key-file "${SSH_KEY_FILE}") + SSH_FLAGS=("${REMOTE_COMMON_FLAGS[@]}" --ssh-flag="-o ServerAliveInterval=30" --ssh-flag="-o ServerAliveCountMax=10" --ssh-flag="-o TCPKeepAlive=yes") + SCP_FLAGS=("${REMOTE_COMMON_FLAGS[@]}" --scp-flag="-o ServerAliveInterval=30" --scp-flag="-o ServerAliveCountMax=10" --scp-flag="-o TCPKeepAlive=yes") + remote_env="/tmp/ibkr-restore-${TARGET_NAME}-${GITHUB_RUN_ID}.env" + remote_command=$(cat <&2 + exit 1 + fi + inspect_file="\$(mktemp)" + compose_file="\$(mktemp)" + cleanup() { rm -f "\${inspect_file}" "\${compose_file}" "\${candidate}"; } + trap cleanup EXIT + if ! sudo docker inspect "\${container}" >"\${inspect_file}" 2>/dev/null; then + echo 'GATEWAY_RESTORE_FAILURE_STAGE=RUNNING_CONTAINER_NOT_FOUND' >&2 + exit 1 + fi + if ! sudo docker inspect --format '{{json .State.Running}}' "\${container}" | grep -Fxq true; then + echo 'GATEWAY_RESTORE_FAILURE_STAGE=CONTAINER_NOT_READY' >&2 + exit 1 + fi + if ! sudo docker compose --project-name "\${project}" --env-file "\${candidate}" config --format json >"\${compose_file}" 2>/dev/null; then + echo 'GATEWAY_RESTORE_FAILURE_STAGE=COMPOSE_CONFIG_FAILED' >&2 + exit 1 + fi + if ! python3 - "\${inspect_file}" "\${compose_file}" "\${container}" "\${service}" "\${project}" '${DEPLOY_PATH}' <<'PY' + import json + import sys + + inspect = json.load(open(sys.argv[1], encoding="utf-8"))[0] + compose = json.load(open(sys.argv[2], encoding="utf-8")) + container, service, project, deploy_path = sys.argv[3:] + labels = inspect.get("Config", {}).get("Labels") or {} + running_env = {} + for item in inspect.get("Config", {}).get("Env") or []: + if "=" in item: + key, value = item.split("=", 1) + running_env[key] = value + spec = (compose.get("services") or {}).get(service) + identity_checks = { + "container_name": inspect.get("Name") == "/" + container, + "compose_project": labels.get("com.docker.compose.project") == project, + "compose_service": labels.get("com.docker.compose.service") == service, + "compose_working_dir": labels.get("com.docker.compose.project.working_dir") == deploy_path, + } + identity = all(identity_checks.values()) + expected = spec.get("environment") if isinstance(spec, dict) else None + if isinstance(expected, dict): + expected_env = {key: str(value) for key, value in expected.items() if value is not None} + elif isinstance(expected, list): + expected_env = {item.split("=", 1)[0]: item.split("=", 1)[1] for item in expected if "=" in item} + else: + expected_env = {} + security_checks = { + key: running_env.get(key) == value for key, value in expected_env.items() + } + security = bool(expected_env) and all(security_checks.values()) + working_dir = (spec or {}).get("working_dir") if isinstance(spec, dict) else None + if working_dir: + security_checks["working_dir"] = inspect.get("Config", {}).get("WorkingDir") == working_dir + security = security and security_checks["working_dir"] + running_ports = inspect.get("NetworkSettings", {}).get("Ports") or {} + for port in (spec or {}).get("ports", []): + if not isinstance(port, dict): + security = False + security_checks["ports"] = False + continue + key = f"{port.get('target')}/{port.get('protocol', 'tcp')}" + bindings = running_ports.get(key) + published = str(port.get("published", "")) + host_ip = str(port.get("host_ip") or "") + expected_host_ips = {host_ip} if host_ip else {"0.0.0.0", "::"} + if not bindings or published and not any( + item.get("HostPort") == published + and item.get("HostIp", "") in expected_host_ips + for item in bindings + ): + security = False + security_checks["ports"] = False + mismatches = [key for key, matched in identity_checks.items() if not matched] + mismatches.extend(key for key, matched in security_checks.items() if not matched) + print( + f"container_identity_match={'true' if identity else 'false'} " + f"security_config_match={'true' if security else 'false'} " + f"mismatch_fields={','.join(sorted(set(mismatches))) or 'none'}" + ) + raise SystemExit(0 if identity and security else 1) + PY + then + echo 'GATEWAY_RESTORE_FAILURE_STAGE=CONTAINER_IDENTITY_OR_SECURITY_MISMATCH' >&2 + exit 1 + fi + sudo python3 - "\${candidate}" "\${destination}" '${GCE_USER}' <<'PY' + import os + import pwd + import shutil + import sys + import tempfile + + source, destination, owner = sys.argv[1:] + linked = False + if os.path.lexists(destination): + raise SystemExit("runtime environment appeared during preflight") + directory = os.path.dirname(destination) + uid = pwd.getpwnam(owner).pw_uid + gid = pwd.getpwnam(owner).pw_gid + fd, staged = tempfile.mkstemp(prefix=".env.restore-", dir=directory) + try: + os.fchmod(fd, 0o600) + with os.fdopen(fd, "wb") as output, open(source, "rb") as input_file: + shutil.copyfileobj(input_file, output) + os.chown(staged, uid, gid) + os.link(staged, destination) + linked = True + os.unlink(staged) + if (os.stat(destination).st_mode & 0o777) != 0o600: + raise SystemExit("runtime environment mode verification failed") + with open(source, "rb") as input_file, open(destination, "rb") as output: + if input_file.read() != output.read(): + raise SystemExit("runtime environment content verification failed") + except BaseException: + if linked: + try: + os.unlink(destination) + except FileNotFoundError: + pass + try: + os.unlink(staged) + except FileNotFoundError: + pass + raise + PY + echo 'GATEWAY_RESTORE_RESULT=CREATED_MISSING_ENV_ONLY' + EOF + ) + cleanup_remote() { + gcloud compute ssh "${REMOTE_TARGET}" "${SSH_FLAGS[@]}" --command "rm -f '${remote_env}'" >/dev/null 2>&1 || true + } + trap cleanup_remote EXIT + gcloud compute scp "${ENV_FILE}" "${REMOTE_TARGET}:${remote_env}" "${SCP_FLAGS[@]}" >/dev/null + gcloud compute ssh "${REMOTE_TARGET}" "${SSH_FLAGS[@]}" --command "${remote_command}" + - name: Deploy and Log Setup + if: ${{ github.event_name != 'workflow_dispatch' || inputs.deploy_mode != 'restore-env' }} run: | set -euo pipefail diff --git a/tests/test_restore_env_workflow_mock.py b/tests/test_restore_env_workflow_mock.py new file mode 100644 index 0000000..f3c886f --- /dev/null +++ b/tests/test_restore_env_workflow_mock.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +"""Offline safety checks for the restore-env workflow step.""" + +import os +import subprocess +import tempfile +from pathlib import Path + +WORKFLOW = Path(__file__).parents[1] / ".github/workflows/main.yml" +workflow_text = WORKFLOW.read_text(encoding="utf-8") +step_text = workflow_text.split(" - name: Restore missing gateway runtime environment\n", 1)[1] +step_text = step_text.split(" - name: Deploy and Log Setup\n", 1)[0] +run = "\n".join(line[10:] for line in step_text.splitlines()[1:] if line.startswith(" ")) + + +def exercise(kind: str) -> subprocess.CompletedProcess[str]: + root = Path(tempfile.mkdtemp(prefix="restore-env-workflow-")) + deploy = root / "deploy" + deploy.mkdir() + candidate = root / "candidate.env" + candidate.write_text("TRADING_MODE='paper'\nREAD_ONLY_API='yes'\n", encoding="utf-8") + if kind == "existing": + (deploy / ".env").write_bytes(b"keep-me") + elif kind == "symlink": + sentinel = root / "sentinel" + sentinel.write_bytes(b"keep-me") + (deploy / ".env").symlink_to(sentinel) + running_env = 'TRADING_MODE=paper","READ_ONLY_API=yes' + host_port = "4001" + host_ip = "0.0.0.0" + if kind == "env-mismatch": + running_env = 'TRADING_MODE=live","READ_ONLY_API=yes' + elif kind == "port-mismatch": + host_port = "9999" + elif kind == "host-ip-mismatch": + host_ip = "127.0.0.1" + fake = root / "fake" + fake.mkdir() + (fake / "gcloud").write_text( + '#!/usr/bin/env bash\nset -e\n' + 'if [ "$1" = compute ] && [ "$2" = scp ]; then ' + 'src="$3"; dest="${4#*:}"; cp "$src" "$dest"; exit 0; fi\n' + 'if [ "$1" = compute ] && [ "$2" = ssh ]; then shift 2; ' + 'while [ "$#" -gt 0 ]; do if [ "$1" = --command ]; then shift; ' + 'bash -c "$1"; exit $?; fi; shift; done; fi\n', + encoding="utf-8", + ) + (fake / "sudo").write_text('#!/usr/bin/env bash\nexec "$@"\n', encoding="utf-8") + (fake / "docker").write_text( + f'''#!/usr/bin/env bash +set -e +if [ "$1" = inspect ] && [ "$2" = --format ]; then + if [[ "$3" == *State.Running* ]]; then echo true; exit 0; fi + exit 1 +fi +if [ "$1" = inspect ]; then + cat <<'JSON' +[{{"Name":"/ib-gateway","Config":{{"Labels":{{"com.docker.compose.project":"app","com.docker.compose.service":"ib-gateway","com.docker.compose.project.working_dir":"{deploy}"}},"Env":["{running_env}"],"WorkingDir":"/"}},"NetworkSettings":{{"Ports":{{"4001/tcp":[{{"HostPort":"{host_port}","HostIp":"{host_ip}"}}]}}}}}}] +JSON + exit 0 +fi +if [ "$1" = compose ]; then + cat <<'JSON' +{{"services":{{"ib-gateway":{{"environment":{{"TRADING_MODE":"paper","READ_ONLY_API":"yes"}},"ports":[{{"target":4001,"published":4001,"protocol":"tcp"}}]}}}}}} +JSON + exit 0 +fi +exit 1 +''', + encoding="utf-8", + ) + for executable in fake.iterdir(): + executable.chmod(0o755) + script = root / "run.sh" + script.write_text("#!/usr/bin/env bash\n" + run + "\n", encoding="utf-8") + script.chmod(0o755) + env = os.environ | { + "PATH": f"{fake}:{os.environ['PATH']}", + "WORKFLOW_DISPATCH_TARGET": "gateway-a", + "GCE_USER": os.environ.get("USER", "root"), + "GCE_INSTANCE_NAME": "vm", + "GCP_PROJECT_ID": "project", + "GCE_ZONE": "zone", + "SSH_KEY_FILE": "/tmp/key", + "TARGET_NAME": "gateway-a", + "GITHUB_RUN_ID": f"{kind}-{root.name}", + "DEPLOY_PATH": str(deploy), + "IB_GATEWAY_CONTAINER_NAME": "ib-gateway", + "IB_GATEWAY_COMPOSE_SERVICE_NAME": "ib-gateway", + "IB_GATEWAY_COMPOSE_PROJECT_NAME": "app", + "ENV_FILE": str(candidate), + } + result = subprocess.run(["bash", str(script)], env=env, text=True, capture_output=True) + destination = deploy / ".env" + if kind == "positive": + assert destination.read_bytes() == candidate.read_bytes() + assert destination.stat().st_mode & 0o777 == 0o600 + elif kind in {"env-mismatch", "port-mismatch", "host-ip-mismatch"}: + assert not destination.exists() + elif kind == "existing": + assert destination.read_bytes() == b"keep-me" + elif kind == "symlink": + assert destination.is_symlink() + return result + + +for case in ("positive", "existing", "symlink", "env-mismatch", "port-mismatch", "host-ip-mismatch"): + result = exercise(case) + assert (result.returncode == 0) == (case == "positive"), (case, result.stderr) + print(f"{case}: ok") diff --git a/tests/test_workflow_shared_config.sh b/tests/test_workflow_shared_config.sh index 6de9375..7cd5a33 100644 --- a/tests/test_workflow_shared_config.sh +++ b/tests/test_workflow_shared_config.sh @@ -28,12 +28,12 @@ start = " python3 - <<'PY'\n" code = workflow.split(start, 1)[1].split(" PY\n", 1)[0] code = "\n".join(line.removeprefix(" ") for line in code.splitlines()) -def select(targets_json: str) -> subprocess.CompletedProcess[str]: +def select(targets_json: str, mode: str = "keepalive") -> subprocess.CompletedProcess[str]: with tempfile.NamedTemporaryFile() as output: env = os.environ | { "TARGETS_JSON": targets_json, "SELECTED_TARGET": "all", - "SELECTED_DEPLOY_MODE": "keepalive", + "SELECTED_DEPLOY_MODE": mode, "GITHUB_OUTPUT": output.name, } return subprocess.run([sys.executable, "-c", code], env=env, text=True, capture_output=True) @@ -44,6 +44,9 @@ assert "IB_GATEWAY_TARGETS_JSON is required" in missing.stderr configured = select('{"gateway-a": {}}') assert configured.returncode == 0, configured.stderr +restore_all = select('{"gateway-a": {}}', "restore-env") +assert restore_all.returncode != 0 +assert "one explicit target" in restore_all.stderr PY grep -Fq 'id-token: write' "$workflow_file" grep -Fq 'timeout-minutes: 75' "$workflow_file" @@ -77,7 +80,18 @@ grep -Fq 'tar -xzf' "$workflow_file" grep -Fq 'gcloud compute instances reset "${GCE_INSTANCE_NAME}"' "$workflow_file" grep -Fq 'run_remote_ssh "Repository sync" "${REMOTE_SYNC_COMMAND}"' "$workflow_file" grep -Fq 'copy_remote_file "${ENV_FILE}" "${DEPLOY_PATH}/.env"' "$workflow_file" -grep -Fq 'A full gateway deployment requires one explicit target' "$workflow_file" +grep -Fq 'A full or restore-env gateway operation requires one explicit target' "$workflow_file" +grep -Fq ' - restore-env' "$workflow_file" +grep -Fq 'Restore missing gateway runtime environment' "$workflow_file" +grep -Fq 'RUNTIME_ENV_ALREADY_EXISTS' "$workflow_file" +grep -Fq 'CONTAINER_IDENTITY_OR_SECURITY_MISMATCH' "$workflow_file" +grep -Fq 'docker compose --project-name "\${project}" --env-file "\${candidate}" config --format json' "$workflow_file" +grep -Fq 'os.link(staged, destination)' "$workflow_file" +grep -Fq 'os.fchmod(fd, 0o600)' "$workflow_file" +grep -Fq 'restore-env requires target-specific Secret Manager names' "$workflow_file" +grep -Fq 'TOTP_SECRET_SECRET_NAME is required for restore-env' "$workflow_file" +grep -Fq "inputs.deploy_mode != 'restore-env'" "$workflow_file" +python3 "$repo_dir/tests/test_restore_env_workflow_mock.py" grep -Fq 'Runtime keepalive uses the already released source and environment' "$workflow_file" grep -Fq 'sudo bash ./scripts/ensure_host_swap.sh' "$workflow_file" grep -Fq 'host_2fa_bot_sha256="\$(sha256sum ./2fa_bot.py' "$workflow_file"