Skip to content

fix: restore missing Gateway env without restart #145

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Pigbibi merged 1 commit into from
Sep 15, 2026
Merged

fix: restore missing Gateway env without restart #145

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
203 changes: 198 additions & 5 deletions .github/workflows/main.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ on:
options:
- keepalive
- full
- restore-env
approved_ref:
description: Approved source ref for a full deployment; only refs/heads/main is accepted.
required: false
Expand Down Expand Up @@ -109,9 +110,9 @@ jobs:

selected_name = os.environ.get("SELECTED_TARGET", "") or "all"
selected_deploy_mode = os.environ.get("SELECTED_DEPLOY_MODE", "") or "keepalive"
if selected_deploy_mode == "full" and selected_name == "all":
if selected_deploy_mode in {"full", "restore-env"} and selected_name == "all":
raise SystemExit(
"A full gateway deployment requires one explicit target; "
"A full or restore-env gateway operation requires one explicit target; "
"use keepalive for all-target maintenance."
)
if selected_name == "all":
Expand Down Expand Up @@ -190,7 +191,7 @@ jobs:
uses: actions/checkout@v6

- name: Verify full-deploy source provenance
if: ${{ github.event_name == 'workflow_dispatch' && inputs.deploy_mode == 'full' }}
if: ${{ github.event_name == 'workflow_dispatch' && (inputs.deploy_mode == 'full' || inputs.deploy_mode == 'restore-env') }}
run: |
set -euo pipefail

Expand Down Expand Up @@ -252,7 +253,7 @@ jobs:
require_secret_source SSH_PRIVATE_KEY_SECRET_NAME SSH_PRIVATE_KEY SSH_PRIVATE_KEY

- name: Check full-deploy credential config
if: ${{ github.event_name == 'workflow_dispatch' && inputs.deploy_mode == 'full' }}
if: ${{ github.event_name == 'workflow_dispatch' && (inputs.deploy_mode == 'full' || inputs.deploy_mode == 'restore-env') }}
env:
TWS_USERID: ${{ secrets.TWS_USERID }}
TWS_PASSWORD: ${{ secrets.TWS_PASSWORD }}
Expand All @@ -277,9 +278,21 @@ jobs:
require_secret_source TWS_USERID_SECRET_NAME TWS_USERID TWS_USERID
require_secret_source TWS_PASSWORD_SECRET_NAME TWS_PASSWORD TWS_PASSWORD
require_secret_source VNC_SERVER_PASSWORD_SECRET_NAME VNC_SERVER_PASSWORD VNC_SERVER_PASSWORD
if [ "${WORKFLOW_DISPATCH_MODE:-}" = "restore-env" ]; then
for required_secret_name in TWS_USERID_SECRET_NAME TWS_PASSWORD_SECRET_NAME VNC_SERVER_PASSWORD_SECRET_NAME; do
if [ -z "${!required_secret_name:-}" ]; then
echo "${required_secret_name} is required for restore-env; refusing cross-account fallback." >&2
exit 1
fi
done
fi
twofa_autofill="$(printf '%s' "${IBKR_2FA_AUTOFILL:-}" | tr '[:upper:]' '[:lower:]')"
if [ "${twofa_autofill}" = "yes" ] || [ "${twofa_autofill}" = "true" ] || [ "${twofa_autofill}" = "1" ]; then
require_secret_source TOTP_SECRET_SECRET_NAME TOTP_SECRET TOTP_SECRET
if [ "${WORKFLOW_DISPATCH_MODE:-}" = "restore-env" ] && [ -z "${TOTP_SECRET_SECRET_NAME:-}" ]; then
echo "TOTP_SECRET_SECRET_NAME is required for restore-env; refusing cross-account fallback." >&2
exit 1
fi
fi

- name: Authenticate to Google Cloud
Expand Down Expand Up @@ -390,6 +403,10 @@ jobs:
if [ -n "${secret_name:-}" ]; then
gcloud secrets versions access latest --project "${GCP_PROJECT_ID}" --secret "${secret_name}"
else
if [ "${WORKFLOW_DISPATCH_MODE:-}" = "restore-env" ]; then
echo "restore-env requires target-specific Secret Manager names." >&2
return 1
fi
printf '%s' "${fallback_value}"
fi
}
Expand All @@ -412,7 +429,7 @@ jobs:
echo "SSH_KEY_FILE=$SSH_KEY_FILE" >> "$GITHUB_ENV"

- name: Prepare full-deploy runtime environment
if: ${{ github.event_name == 'workflow_dispatch' && inputs.deploy_mode == 'full' }}
if: ${{ github.event_name == 'workflow_dispatch' && (inputs.deploy_mode == 'full' || inputs.deploy_mode == 'restore-env') }}
env:
TWS_USERID: ${{ secrets.TWS_USERID }}
TWS_PASSWORD: ${{ secrets.TWS_PASSWORD }}
Expand All @@ -435,6 +452,11 @@ jobs:
tws_userid="$(resolve_secret "${TWS_USERID_SECRET_NAME:-}" "${TWS_USERID:-}")"
tws_password="$(resolve_secret "${TWS_PASSWORD_SECRET_NAME:-}" "${TWS_PASSWORD:-}")"
totp_secret=""
if [ "${WORKFLOW_DISPATCH_MODE:-}" = "restore-env" ] && [ -z "${TOTP_SECRET_SECRET_NAME:-}" ] \
&& [ -n "${TOTP_SECRET:-}" ]; then
echo "TOTP_SECRET_SECRET_NAME is required for restore-env; refusing global TOTP fallback." >&2
exit 1
fi
if [ -n "${TOTP_SECRET_SECRET_NAME:-}" ] || [ -n "${TOTP_SECRET:-}" ]; then
totp_secret="$(resolve_secret "${TOTP_SECRET_SECRET_NAME:-}" "${TOTP_SECRET:-}")"
fi
Expand Down Expand Up @@ -505,13 +527,184 @@ jobs:
def quote(value: str) -> str:
return "'" + value.replace("'", "'\"'\"'") + "'"

if os.environ.get("WORKFLOW_DISPATCH_MODE") == "restore-env":
os.umask(0o077)
with open(env_file, "w", encoding="utf-8") as fh:
for key, value in values.items():
fh.write(f"{key}={quote(value)}\n")
os.chmod(env_file, 0o600)
PY
if [ "${WORKFLOW_DISPATCH_MODE:-}" = "restore-env" ]; then
syntax_error_file="$RUNNER_TEMP/restore-env-bash-syntax.err"
if ! env -i bash -n "$ENV_FILE" 2>"$syntax_error_file"; then
echo "Generated restore-env file failed shell syntax validation." >&2
rm -f "$syntax_error_file"
exit 1
fi
rm -f "$syntax_error_file"
fi
echo "ENV_FILE=$ENV_FILE" >> "$GITHUB_ENV"

- name: Restore missing gateway runtime environment
if: ${{ github.event_name == 'workflow_dispatch' && inputs.deploy_mode == 'restore-env' }}
run: |
set -euo pipefail
if [ "${WORKFLOW_DISPATCH_TARGET:-all}" = "all" ]; then
echo "restore-env requires one explicit target." >&2
exit 1
fi
REMOTE_TARGET="${GCE_USER}@${GCE_INSTANCE_NAME}"
REMOTE_COMMON_FLAGS=(--project "${GCP_PROJECT_ID}" --zone "${GCE_ZONE}" --quiet --tunnel-through-iap --ssh-key-file "${SSH_KEY_FILE}")
SSH_FLAGS=("${REMOTE_COMMON_FLAGS[@]}" --ssh-flag="-o ServerAliveInterval=30" --ssh-flag="-o ServerAliveCountMax=10" --ssh-flag="-o TCPKeepAlive=yes")
SCP_FLAGS=("${REMOTE_COMMON_FLAGS[@]}" --scp-flag="-o ServerAliveInterval=30" --scp-flag="-o ServerAliveCountMax=10" --scp-flag="-o TCPKeepAlive=yes")
remote_env="/tmp/ibkr-restore-${TARGET_NAME}-${GITHUB_RUN_ID}.env"
remote_command=$(cat <<EOF
set -euo pipefail
candidate='${remote_env}'
destination='${DEPLOY_PATH}/.env'
container='${IB_GATEWAY_CONTAINER_NAME}'
service='${IB_GATEWAY_COMPOSE_SERVICE_NAME}'
project='${IB_GATEWAY_COMPOSE_PROJECT_NAME}'
cd '${DEPLOY_PATH}'
if [ -e "\${destination}" ] || [ -L "\${destination}" ]; then
echo 'GATEWAY_RESTORE_FAILURE_STAGE=RUNTIME_ENV_ALREADY_EXISTS' >&2
exit 1
fi
inspect_file="\$(mktemp)"
compose_file="\$(mktemp)"
cleanup() { rm -f "\${inspect_file}" "\${compose_file}" "\${candidate}"; }
trap cleanup EXIT
if ! sudo docker inspect "\${container}" >"\${inspect_file}" 2>/dev/null; then
echo 'GATEWAY_RESTORE_FAILURE_STAGE=RUNNING_CONTAINER_NOT_FOUND' >&2
exit 1
fi
if ! sudo docker inspect --format '{{json .State.Running}}' "\${container}" | grep -Fxq true; then
echo 'GATEWAY_RESTORE_FAILURE_STAGE=CONTAINER_NOT_READY' >&2
exit 1
fi
if ! sudo docker compose --project-name "\${project}" --env-file "\${candidate}" config --format json >"\${compose_file}" 2>/dev/null; then
echo 'GATEWAY_RESTORE_FAILURE_STAGE=COMPOSE_CONFIG_FAILED' >&2
exit 1
fi
if ! python3 - "\${inspect_file}" "\${compose_file}" "\${container}" "\${service}" "\${project}" '${DEPLOY_PATH}' <<'PY'
import json
import sys

inspect = json.load(open(sys.argv[1], encoding="utf-8"))[0]
compose = json.load(open(sys.argv[2], encoding="utf-8"))
container, service, project, deploy_path = sys.argv[3:]
labels = inspect.get("Config", {}).get("Labels") or {}
running_env = {}
for item in inspect.get("Config", {}).get("Env") or []:
if "=" in item:
key, value = item.split("=", 1)
running_env[key] = value
spec = (compose.get("services") or {}).get(service)
identity_checks = {
"container_name": inspect.get("Name") == "/" + container,
"compose_project": labels.get("com.docker.compose.project") == project,
"compose_service": labels.get("com.docker.compose.service") == service,
"compose_working_dir": labels.get("com.docker.compose.project.working_dir") == deploy_path,
}
identity = all(identity_checks.values())
expected = spec.get("environment") if isinstance(spec, dict) else None
if isinstance(expected, dict):
expected_env = {key: str(value) for key, value in expected.items() if value is not None}
elif isinstance(expected, list):
expected_env = {item.split("=", 1)[0]: item.split("=", 1)[1] for item in expected if "=" in item}
else:
expected_env = {}
security_checks = {
key: running_env.get(key) == value for key, value in expected_env.items()
}
security = bool(expected_env) and all(security_checks.values())
working_dir = (spec or {}).get("working_dir") if isinstance(spec, dict) else None
if working_dir:
security_checks["working_dir"] = inspect.get("Config", {}).get("WorkingDir") == working_dir
security = security and security_checks["working_dir"]
running_ports = inspect.get("NetworkSettings", {}).get("Ports") or {}
for port in (spec or {}).get("ports", []):
if not isinstance(port, dict):
security = False
security_checks["ports"] = False
continue
key = f"{port.get('target')}/{port.get('protocol', 'tcp')}"
bindings = running_ports.get(key)
published = str(port.get("published", ""))
host_ip = str(port.get("host_ip") or "")
expected_host_ips = {host_ip} if host_ip else {"0.0.0.0", "::"}
if not bindings or published and not any(
item.get("HostPort") == published
and item.get("HostIp", "") in expected_host_ips
for item in bindings
):
security = False
security_checks["ports"] = False
mismatches = [key for key, matched in identity_checks.items() if not matched]
mismatches.extend(key for key, matched in security_checks.items() if not matched)
print(
f"container_identity_match={'true' if identity else 'false'} "
f"security_config_match={'true' if security else 'false'} "
f"mismatch_fields={','.join(sorted(set(mismatches))) or 'none'}"
)
raise SystemExit(0 if identity and security else 1)
PY
then
echo 'GATEWAY_RESTORE_FAILURE_STAGE=CONTAINER_IDENTITY_OR_SECURITY_MISMATCH' >&2
exit 1
fi
sudo python3 - "\${candidate}" "\${destination}" '${GCE_USER}' <<'PY'
import os
import pwd
import shutil
import sys
import tempfile

source, destination, owner = sys.argv[1:]
linked = False
if os.path.lexists(destination):
raise SystemExit("runtime environment appeared during preflight")
directory = os.path.dirname(destination)
uid = pwd.getpwnam(owner).pw_uid
gid = pwd.getpwnam(owner).pw_gid
fd, staged = tempfile.mkstemp(prefix=".env.restore-", dir=directory)
try:
os.fchmod(fd, 0o600)
with os.fdopen(fd, "wb") as output, open(source, "rb") as input_file:
shutil.copyfileobj(input_file, output)
os.chown(staged, uid, gid)
os.link(staged, destination)
linked = True
os.unlink(staged)
if (os.stat(destination).st_mode & 0o777) != 0o600:
raise SystemExit("runtime environment mode verification failed")
with open(source, "rb") as input_file, open(destination, "rb") as output:
if input_file.read() != output.read():
raise SystemExit("runtime environment content verification failed")
except BaseException:
if linked:
try:
os.unlink(destination)
except FileNotFoundError:
pass
try:
os.unlink(staged)
except FileNotFoundError:
pass
raise
PY
echo 'GATEWAY_RESTORE_RESULT=CREATED_MISSING_ENV_ONLY'
EOF
)
cleanup_remote() {
gcloud compute ssh "${REMOTE_TARGET}" "${SSH_FLAGS[@]}" --command "rm -f '${remote_env}'" >/dev/null 2>&1 || true
}
trap cleanup_remote EXIT
gcloud compute scp "${ENV_FILE}" "${REMOTE_TARGET}:${remote_env}" "${SCP_FLAGS[@]}" >/dev/null
gcloud compute ssh "${REMOTE_TARGET}" "${SSH_FLAGS[@]}" --command "${remote_command}"

- name: Deploy and Log Setup
if: ${{ github.event_name != 'workflow_dispatch' || inputs.deploy_mode != 'restore-env' }}
run: |
set -euo pipefail

Expand Down
110 changes: 110 additions & 0 deletions tests/test_restore_env_workflow_mock.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
#!/usr/bin/env python3
"""Offline safety checks for the restore-env workflow step."""

import os
import subprocess
import tempfile
from pathlib import Path

WORKFLOW = Path(__file__).parents[1] / ".github/workflows/main.yml"
workflow_text = WORKFLOW.read_text(encoding="utf-8")
step_text = workflow_text.split(" - name: Restore missing gateway runtime environment\n", 1)[1]
step_text = step_text.split(" - name: Deploy and Log Setup\n", 1)[0]
run = "\n".join(line[10:] for line in step_text.splitlines()[1:] if line.startswith(" "))


def exercise(kind: str) -> subprocess.CompletedProcess[str]:
root = Path(tempfile.mkdtemp(prefix="restore-env-workflow-"))
deploy = root / "deploy"
deploy.mkdir()
candidate = root / "candidate.env"
candidate.write_text("TRADING_MODE='paper'\nREAD_ONLY_API='yes'\n", encoding="utf-8")
if kind == "existing":
(deploy / ".env").write_bytes(b"keep-me")
elif kind == "symlink":
sentinel = root / "sentinel"
sentinel.write_bytes(b"keep-me")
(deploy / ".env").symlink_to(sentinel)
running_env = 'TRADING_MODE=paper","READ_ONLY_API=yes'
host_port = "4001"
host_ip = "0.0.0.0"
if kind == "env-mismatch":
running_env = 'TRADING_MODE=live","READ_ONLY_API=yes'
elif kind == "port-mismatch":
host_port = "9999"
elif kind == "host-ip-mismatch":
host_ip = "127.0.0.1"
fake = root / "fake"
fake.mkdir()
(fake / "gcloud").write_text(
'#!/usr/bin/env bash\nset -e\n'
'if [ "$1" = compute ] && [ "$2" = scp ]; then '
'src="$3"; dest="${4#*:}"; cp "$src" "$dest"; exit 0; fi\n'
'if [ "$1" = compute ] && [ "$2" = ssh ]; then shift 2; '
'while [ "$#" -gt 0 ]; do if [ "$1" = --command ]; then shift; '
'bash -c "$1"; exit $?; fi; shift; done; fi\n',
encoding="utf-8",
)
(fake / "sudo").write_text('#!/usr/bin/env bash\nexec "$@"\n', encoding="utf-8")
(fake / "docker").write_text(
f'''#!/usr/bin/env bash
set -e
if [ "$1" = inspect ] && [ "$2" = --format ]; then
if [[ "$3" == *State.Running* ]]; then echo true; exit 0; fi
exit 1
fi
if [ "$1" = inspect ]; then
cat <<'JSON'
[{{"Name":"/ib-gateway","Config":{{"Labels":{{"com.docker.compose.project":"app","com.docker.compose.service":"ib-gateway","com.docker.compose.project.working_dir":"{deploy}"}},"Env":["{running_env}"],"WorkingDir":"/"}},"NetworkSettings":{{"Ports":{{"4001/tcp":[{{"HostPort":"{host_port}","HostIp":"{host_ip}"}}]}}}}}}]
JSON
exit 0
fi
if [ "$1" = compose ]; then
cat <<'JSON'
{{"services":{{"ib-gateway":{{"environment":{{"TRADING_MODE":"paper","READ_ONLY_API":"yes"}},"ports":[{{"target":4001,"published":4001,"protocol":"tcp"}}]}}}}}}
JSON
exit 0
fi
exit 1
''',
encoding="utf-8",
)
for executable in fake.iterdir():
executable.chmod(0o755)
script = root / "run.sh"
script.write_text("#!/usr/bin/env bash\n" + run + "\n", encoding="utf-8")
script.chmod(0o755)
env = os.environ | {
"PATH": f"{fake}:{os.environ['PATH']}",
"WORKFLOW_DISPATCH_TARGET": "gateway-a",
"GCE_USER": os.environ.get("USER", "root"),
"GCE_INSTANCE_NAME": "vm",
"GCP_PROJECT_ID": "project",
"GCE_ZONE": "zone",
"SSH_KEY_FILE": "/tmp/key",
"TARGET_NAME": "gateway-a",
"GITHUB_RUN_ID": f"{kind}-{root.name}",
"DEPLOY_PATH": str(deploy),
"IB_GATEWAY_CONTAINER_NAME": "ib-gateway",
"IB_GATEWAY_COMPOSE_SERVICE_NAME": "ib-gateway",
"IB_GATEWAY_COMPOSE_PROJECT_NAME": "app",
"ENV_FILE": str(candidate),
}
result = subprocess.run(["bash", str(script)], env=env, text=True, capture_output=True)
destination = deploy / ".env"
if kind == "positive":
assert destination.read_bytes() == candidate.read_bytes()
assert destination.stat().st_mode & 0o777 == 0o600
elif kind in {"env-mismatch", "port-mismatch", "host-ip-mismatch"}:
assert not destination.exists()
elif kind == "existing":
assert destination.read_bytes() == b"keep-me"
elif kind == "symlink":
assert destination.is_symlink()
return result


for case in ("positive", "existing", "symlink", "env-mismatch", "port-mismatch", "host-ip-mismatch"):
result = exercise(case)
assert (result.returncode == 0) == (case == "positive"), (case, result.stderr)
print(f"{case}: ok")
Loading