From 3164ddfea71a990283584230da8508d0b0bc2a42 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Wed, 16 Sep 2026 01:07:01 +0800 Subject: [PATCH] fix(binance): expose safe startup diagnostics Co-Authored-By: Codex --- application/reconciliation_recovery.py | 19 ++++++- docs/operator_runbook.md | 1 + main.py | 28 +++++++++- notify_i18n_support.py | 10 ++++ scripts/validate_runtime_startup.py | 20 +++++-- tests/test_main_runtime_error_notification.py | 33 ++++++++++++ tests/test_reconciliation_recovery.py | 52 +++++++++++++++++++ tests/test_validate_runtime_startup.py | 19 +++++++ 8 files changed, 174 insertions(+), 8 deletions(-) diff --git a/application/reconciliation_recovery.py b/application/reconciliation_recovery.py index 0afdd4a0..4baccc7b 100644 --- a/application/reconciliation_recovery.py +++ b/application/reconciliation_recovery.py @@ -31,6 +31,15 @@ ) +class RecoveryControlReadError(RuntimeError): + """A recovery-control read failed before its payload could be validated.""" + + reason_code = "recovery_control_read_failed" + + def __init__(self): + super().__init__(self.reason_code) + + def _reader(expected): return lambda *_: json.dumps(expected) @@ -199,5 +208,11 @@ def load_activated_target(target): if os.getenv("BINANCE_RECOVERY_CONTROL_ENABLED", "false").lower() != "true": return target from live_services import get_firestore_client - snapshot = get_firestore_client().collection("strategy").document("MULTI_ASSET_STATE__recovery").get(retry=None) - return activated_target(target, snapshot.to_dict() if snapshot.exists else None, expected=_expected_digests()) + try: + snapshot = get_firestore_client().collection("strategy").document("MULTI_ASSET_STATE__recovery").get(retry=None) + control = snapshot.to_dict() if snapshot.exists else None + except Exception: + # Keep provider and credential details out of runtime logs. Validation + # failures after a successful read retain their existing reason codes. + raise RecoveryControlReadError() from None + return activated_target(target, control, expected=_expected_digests()) diff --git a/docs/operator_runbook.md b/docs/operator_runbook.md index c6deb211..90308e24 100644 --- a/docs/operator_runbook.md +++ b/docs/operator_runbook.md @@ -670,6 +670,7 @@ python3 -m unittest discover -s tests -v - The runtime workflow now authenticates to Google Cloud with GitHub OIDC + Workload Identity Federation. - For safe runner-side verification, dispatch `main.yml` with `validate_only=true`; that loads the actual configuration, committed recovery control and strategy entrypoint without broker credentials or live execution. It also works while the runtime is paused. - A failed startup check reports only an exact allowlisted `reason_code`. In particular, `runtime_recovery_not_active` means the legacy strategy has no active recovery grant; validation still fails and does not activate it. Profile/dry-run conflicts and invalid recovery controls have distinct safe codes; unknown exceptions remain `runtime_startup_validation_failed`, without raw provider details. Investigate the stated prerequisite before another authorized validation attempt. +- A live startup failure while reading the committed recovery-control document is reported as `recovery_control_read_failed`. This code covers only the read boundary; payload validation and authorization failures retain their existing reason codes. It does not retry the read or change recovery state. - Local manual runs can still use `GOOGLE_APPLICATION_CREDENTIALS=/path/to/gcp-sa.json` when needed. ## Escalation Guidelines diff --git a/main.py b/main.py index 89d78961..f6d894a6 100644 --- a/main.py +++ b/main.py @@ -133,6 +133,24 @@ ExecutionRuntime = _ExecutionRuntime +_SAFE_RUNTIME_SETUP_REASONS = frozenset({ + "recovery_control_read_failed", + "runtime_recovery_not_active", + "recovery_control_state_invalid", + "recovery_active_binding_invalid", +}) +_RUNTIME_ERROR_NOTIFICATION_REASONS = { + "recovery_control_read_failed": "runtime_error_reason_recovery_control_read_failed", + "runtime_recovery_not_active": "runtime_error_reason_recovery_not_active", + "recovery_control_state_invalid": "runtime_error_reason_recovery_control_state_invalid", + "recovery_active_binding_invalid": "runtime_error_reason_recovery_active_binding_invalid", +} + + +def _runtime_setup_reason(exc): + reason = str(exc) + return reason if reason in _SAFE_RUNTIME_SETUP_REASONS else "runtime_startup_failed" + def _load_import_safe_strategy_runtime(): """Keep pure replay/helpers importable when no profile is execution-enabled.""" @@ -471,10 +489,15 @@ def send_tg_msg(token, chat_id, text): def _runtime_error_notification_message(_exc): strategy_name = build_strategy_display_name(t)(os.getenv("STRATEGY_PROFILE", "")) + reason = _runtime_setup_reason(_exc) + reason_key = _RUNTIME_ERROR_NOTIFICATION_REASONS.get( + reason, "runtime_error_reason_generic" + ) return "\n".join( ( t("runtime_error_title"), t("strategy_label", name=strategy_name) if strategy_name else "", + t(reason_key), t("runtime_error_result"), t("runtime_error_action"), ) @@ -1287,6 +1310,7 @@ def main(): exit_fn=sys.exit, ) except Exception as exc: + reason = _runtime_setup_reason(exc) print(t("runtime_setup_failed")) frames = traceback.extract_tb(exc.__traceback__) stage = "runtime_setup" @@ -1296,8 +1320,8 @@ def main(): elif frame.filename.endswith(("strategy_loader.py", "strategy_runtime.py", "strategy_registry.py")): stage = "strategy_load" error_type = type(exc).__name__ if type(exc) in {ValueError, KeyError, TypeError, OSError, RuntimeError} else "RuntimeError" - print(f"runtime_setup_failed stage={stage} error_type={error_type}") - if _notify_runtime_error(RuntimeError("runtime_setup_failed")): + print(f"runtime_setup_failed stage={stage} error_type={error_type} reason_code={reason}") + if _notify_runtime_error(RuntimeError(reason)): output_path = os.getenv("GITHUB_OUTPUT") if output_path: with open(output_path, "a", encoding="utf-8") as output: diff --git a/notify_i18n_support.py b/notify_i18n_support.py index 4c37b490..a4f89891 100644 --- a/notify_i18n_support.py +++ b/notify_i18n_support.py @@ -17,6 +17,11 @@ "firestore_write_failed": "Firestore write failed: {error}", "telegram_send_failed": "Telegram send failed", "runtime_error_title": "⚠️ Binance strategy run failed", + "runtime_error_reason_recovery_control_read_failed": "Temporary recovery configuration read failure; trading did not start.", + "runtime_error_reason_recovery_not_active": "Recovery authorization is not active; trading did not start.", + "runtime_error_reason_recovery_control_state_invalid": "Recovery configuration is invalid; trading did not start.", + "runtime_error_reason_recovery_active_binding_invalid": "Recovery authorization binding is invalid; trading did not start.", + "runtime_error_reason_generic": "The runtime could not start safely; inspect the startup diagnostics.", "runtime_health_monitor_started": "Health monitor started", "runtime_setup_failed": "Strategy startup failed: runtime_setup_failed", "runtime_error_result": "The run did not finish successfully; check the latest execution report.", @@ -153,6 +158,11 @@ "firestore_write_failed": "Firestore 写入状态失败: {error}", "telegram_send_failed": "Telegram 发送失败", "runtime_error_title": "⚠️ Binance 策略运行失败", + "runtime_error_reason_recovery_control_read_failed": "暂时无法读取恢复配置，本次未启动交易。", + "runtime_error_reason_recovery_not_active": "恢复授权尚未激活，本次未启动交易。", + "runtime_error_reason_recovery_control_state_invalid": "恢复配置状态无效，本次未启动交易。", + "runtime_error_reason_recovery_active_binding_invalid": "恢复授权绑定无效，本次未启动交易。", + "runtime_error_reason_generic": "运行时无法安全启动，请查看启动诊断。", "runtime_health_monitor_started": "运行监测已启动", "runtime_setup_failed": "策略启动失败：runtime_setup_failed", "runtime_error_result": "本次运行未正常结束，请查看最新执行报告。", diff --git a/scripts/validate_runtime_startup.py b/scripts/validate_runtime_startup.py index 9e9e3bb5..9dd19a1f 100644 --- a/scripts/validate_runtime_startup.py +++ b/scripts/validate_runtime_startup.py @@ -13,6 +13,7 @@ "runtime_recovery_not_active": "runtime_recovery_not_active", "recovery_control_state_invalid": "recovery_control_state_invalid", "recovery_active_binding_invalid": "recovery_active_binding_invalid", + "recovery_control_read_failed": "recovery_control_read_failed", "STRATEGY_PROFILE does not match RUNTIME_TARGET_JSON.strategy_profile": "startup_strategy_profile_conflict", "BINANCE_DRY_RUN does not match RUNTIME_TARGET_JSON.dry_run_only": "startup_dry_run_conflict", "full_cycle_requires_disabled_runtime": "full_cycle_requires_disabled_runtime", @@ -143,12 +144,22 @@ def _error_type_name(exc): return "RuntimeError" +def _safe_startup_reason(exc): + """Return an exact allowlisted reason carried by a startup exception.""" + reason = getattr(exc, "reason_code", None) + if type(exc) is ValueError: + reason = str(exc) + if type(reason) is not str: + return None + return _SAFE_STARTUP_REASONS.get(reason) + + def _wrap_full_cycle_error(exc, *, failure_stage, fallback_reason): if isinstance(exc, FullCycleValidationError): return exc reason_code = _authority_reason_code(exc) - if reason_code is None and type(exc) is ValueError: - reason_code = _SAFE_STARTUP_REASONS.get(str(exc)) + if reason_code is None: + reason_code = _safe_startup_reason(exc) return FullCycleValidationError( reason_code or fallback_reason, failure_stage=failure_stage, @@ -474,8 +485,9 @@ def validate_startup(): reason = exc.reason_code failure_stage = exc.failure_stage kind = exc.error_type - if type(exc) is ValueError: - reason = _SAFE_STARTUP_REASONS.get(str(exc), reason) + safe_reason = _safe_startup_reason(exc) + if safe_reason is not None: + reason = safe_reason output = { "status": "failed", "stage": "full_cycle_validation" if full_cycle else "runtime_startup_validation", diff --git a/tests/test_main_runtime_error_notification.py b/tests/test_main_runtime_error_notification.py index 13a8999f..5d87397c 100644 --- a/tests/test_main_runtime_error_notification.py +++ b/tests/test_main_runtime_error_notification.py @@ -69,6 +69,12 @@ class BinanceAPIException(Exception): strategy_registry_module.resolve_strategy_metadata = lambda *_args, **_kwargs: types.SimpleNamespace( display_name="Crypto Live Pool Rotation", ) + strategy_registry_module.resolve_research_strategy_metadata = strategy_registry_module.resolve_strategy_metadata + strategy_registry_module.resolve_research_strategy_definition = strategy_registry_module.resolve_strategy_definition + strategy_registry_module.resolve_runtime_target_strategy = lambda target: ( + target, + strategy_registry_module.resolve_strategy_definition(getattr(target, "strategy_profile", None)), + ) sys.modules["strategy_registry"] = strategy_registry_module if "strategy_runtime" not in sys.modules: @@ -123,6 +129,33 @@ def test_runtime_error_message_uses_chinese_without_exposing_exception(self): self.assertNotIn("PRIVATE_SENTINEL", message) self.assertNotIn("未提交订单", message) + def test_runtime_error_notification_uses_fixed_human_reason_text(self): + with patch.dict(os.environ, {"NOTIFY_LANG": "zh-CN", "STRATEGY_PROFILE": "crypto_live_pool_rotation"}): + message = main._runtime_error_notification_message(RuntimeError("recovery_control_read_failed")) + self.assertIn("暂时无法读取恢复配置，本次未启动交易。", message) + self.assertNotIn("recovery_control_read_failed", message) + + with patch.dict(os.environ, {"NOTIFY_LANG": "en", "STRATEGY_PROFILE": "crypto_live_pool_rotation"}): + message = main._runtime_error_notification_message(RuntimeError("untrusted-provider-detail")) + self.assertIn("The runtime could not start safely", message) + self.assertNotIn("untrusted-provider-detail", message) + + def test_main_reports_only_fixed_recovery_read_reason(self): + observed = [] + + def fake_run_cli_entrypoint(**_kwargs): + raise RuntimeError("recovery_control_read_failed") + + with patch.object(main, "run_cli_entrypoint", fake_run_cli_entrypoint), \ + patch.object(main, "_notify_runtime_error", return_value=False), \ + patch("builtins.print", lambda *args, **_kwargs: observed.append(" ".join(map(str, args)))): + with self.assertRaisesRegex(RuntimeError, "runtime_setup_failed"): + main.main() + + rendered = " ".join(observed) + self.assertIn("reason_code=recovery_control_read_failed", rendered) + self.assertNotIn("provider", rendered) + def test_main_wires_cli_entrypoint_with_runtime_builder_and_cycle_runner(self): observed = {} diff --git a/tests/test_reconciliation_recovery.py b/tests/test_reconciliation_recovery.py index 8799d217..6e104bc7 100644 --- a/tests/test_reconciliation_recovery.py +++ b/tests/test_reconciliation_recovery.py @@ -99,6 +99,58 @@ def test_source_digest_is_not_a_substitute_for_verifying_frozen_config(): validate_source(source, runtime_target=args["runtime_target"], expected=args["expected"], now=NOW + timedelta(minutes=31)) +def test_recovery_control_read_failure_is_fixed_and_redacted(monkeypatch): + import application.reconciliation_recovery as recovery + import live_services + + class Ref: + def get(self, **_kwargs): + raise RuntimeError("provider secret and credential details") + + class Client: + def collection(self, name): + assert name == "strategy" + return self + + def document(self, name): + assert name == "MULTI_ASSET_STATE__recovery" + return Ref() + + monkeypatch.setenv("BINANCE_RECOVERY_CONTROL_ENABLED", "true") + monkeypatch.setattr(live_services, "get_firestore_client", lambda: Client()) + with pytest.raises(recovery.RecoveryControlReadError) as error: + recovery.load_activated_target(_target()) + assert str(error.value) == "recovery_control_read_failed" + assert "provider secret" not in str(error.value) + + +def test_recovery_control_validation_failure_keeps_existing_reason(monkeypatch): + import application.reconciliation_recovery as recovery + import live_services + + class Ref: + exists = True + + def get(self, **_kwargs): + return self + + @staticmethod + def to_dict(): + return {"state": "UNSUPPORTED"} + + class Client: + def collection(self, _name): + return self + + def document(self, _name): + return Ref() + + monkeypatch.setenv("BINANCE_RECOVERY_CONTROL_ENABLED", "true") + monkeypatch.setattr(live_services, "get_firestore_client", lambda: Client()) + with pytest.raises(ValueError, match="recovery_control_state_invalid"): + recovery.load_activated_target(_target()) + + def test_candidate_payload_tampering_rejected(): args = inputs() source = collect_recovery_source(**args) diff --git a/tests/test_validate_runtime_startup.py b/tests/test_validate_runtime_startup.py index eaa971ba..94848347 100644 --- a/tests/test_validate_runtime_startup.py +++ b/tests/test_validate_runtime_startup.py @@ -335,6 +335,7 @@ def test_full_cycle_failure_projection_preserves_safe_daily_state_reason_code(): ('runtime_recovery_not_active', 'runtime_recovery_not_active'), ('recovery_control_state_invalid', 'recovery_control_state_invalid'), ('recovery_active_binding_invalid', 'recovery_active_binding_invalid'), + ('recovery_control_read_failed', 'recovery_control_read_failed'), ('STRATEGY_PROFILE does not match RUNTIME_TARGET_JSON.strategy_profile', 'startup_strategy_profile_conflict'), ('BINANCE_DRY_RUN does not match RUNTIME_TARGET_JSON.dry_run_only', 'startup_dry_run_conflict'), ('runtime_recovery_not_active: DO_NOT_LOG_THIS_SYNTHETIC_TOKEN', 'runtime_startup_validation_failed'), @@ -362,6 +363,24 @@ def build(): 'error_type': 'ValueError', 'reason_code': expected, } + +def test_startup_cli_reports_reason_from_safe_runtime_exception(monkeypatch, capsys): + from application.reconciliation_recovery import RecoveryControlReadError + + monkeypatch.setenv('RUNTIME_TARGET_ENABLED', 'false') + monkeypatch.delenv('BINANCE_API_KEY', raising=False) + monkeypatch.delenv('BINANCE_API_SECRET', raising=False) + + def build(): + raise RecoveryControlReadError() + + monkeypatch.setitem(sys.modules, 'main', SimpleNamespace(build_live_runtime=build)) + script = Path(__file__).resolve().parents[1] / 'scripts/validate_runtime_startup.py' + with pytest.raises(SystemExit) as error: + runpy.run_path(str(script), run_name='__main__') + assert error.value.code == 1 + assert json.loads(capsys.readouterr().out)['reason_code'] == 'recovery_control_read_failed' + @pytest.mark.parametrize( "authority_message, expected_reason", [