Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 17 additions & 2 deletions application/reconciliation_recovery.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,15 @@
)


class RecoveryControlReadError(RuntimeError):
"""A recovery-control read failed before its payload could be validated."""

reason_code = "recovery_control_read_failed"

def __init__(self):
super().__init__(self.reason_code)


def _reader(expected):
return lambda *_: json.dumps(expected)

Expand Down Expand Up @@ -199,5 +208,11 @@ def load_activated_target(target):
if os.getenv("BINANCE_RECOVERY_CONTROL_ENABLED", "false").lower() != "true":
return target
from live_services import get_firestore_client
snapshot = get_firestore_client().collection("strategy").document("MULTI_ASSET_STATE__recovery").get(retry=None)
return activated_target(target, snapshot.to_dict() if snapshot.exists else None, expected=_expected_digests())
try:
snapshot = get_firestore_client().collection("strategy").document("MULTI_ASSET_STATE__recovery").get(retry=None)
control = snapshot.to_dict() if snapshot.exists else None
except Exception:
# Keep provider and credential details out of runtime logs. Validation
# failures after a successful read retain their existing reason codes.
raise RecoveryControlReadError() from None
return activated_target(target, control, expected=_expected_digests())
1 change: 1 addition & 0 deletions docs/operator_runbook.md
Original file line number Diff line number Diff line change
Expand Up @@ -670,6 +670,7 @@ python3 -m unittest discover -s tests -v
- The runtime workflow now authenticates to Google Cloud with GitHub OIDC + Workload Identity Federation.
- For safe runner-side verification, dispatch `main.yml` with `validate_only=true`; that loads the actual configuration, committed recovery control and strategy entrypoint without broker credentials or live execution. It also works while the runtime is paused.
- A failed startup check reports only an exact allowlisted `reason_code`. In particular, `runtime_recovery_not_active` means the legacy strategy has no active recovery grant; validation still fails and does not activate it. Profile/dry-run conflicts and invalid recovery controls have distinct safe codes; unknown exceptions remain `runtime_startup_validation_failed`, without raw provider details. Investigate the stated prerequisite before another authorized validation attempt.
- A live startup failure while reading the committed recovery-control document is reported as `recovery_control_read_failed`. This code covers only the read boundary; payload validation and authorization failures retain their existing reason codes. It does not retry the read or change recovery state.
- Local manual runs can still use `GOOGLE_APPLICATION_CREDENTIALS=/path/to/gcp-sa.json` when needed.

## Escalation Guidelines
Expand Down
28 changes: 26 additions & 2 deletions main.py
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,24 @@

ExecutionRuntime = _ExecutionRuntime

_SAFE_RUNTIME_SETUP_REASONS = frozenset({
"recovery_control_read_failed",
"runtime_recovery_not_active",
"recovery_control_state_invalid",
"recovery_active_binding_invalid",
})
_RUNTIME_ERROR_NOTIFICATION_REASONS = {
"recovery_control_read_failed": "runtime_error_reason_recovery_control_read_failed",
"runtime_recovery_not_active": "runtime_error_reason_recovery_not_active",
"recovery_control_state_invalid": "runtime_error_reason_recovery_control_state_invalid",
"recovery_active_binding_invalid": "runtime_error_reason_recovery_active_binding_invalid",
}


def _runtime_setup_reason(exc):
reason = str(exc)
return reason if reason in _SAFE_RUNTIME_SETUP_REASONS else "runtime_startup_failed"


def _load_import_safe_strategy_runtime():
"""Keep pure replay/helpers importable when no profile is execution-enabled."""
Expand Down Expand Up @@ -471,10 +489,15 @@ def send_tg_msg(token, chat_id, text):

def _runtime_error_notification_message(_exc):
strategy_name = build_strategy_display_name(t)(os.getenv("STRATEGY_PROFILE", ""))
reason = _runtime_setup_reason(_exc)
reason_key = _RUNTIME_ERROR_NOTIFICATION_REASONS.get(
reason, "runtime_error_reason_generic"
)
return "\n".join(
(
t("runtime_error_title"),
t("strategy_label", name=strategy_name) if strategy_name else "",
t(reason_key),
t("runtime_error_result"),
t("runtime_error_action"),
)
Expand Down Expand Up @@ -1287,6 +1310,7 @@ def main():
exit_fn=sys.exit,
)
except Exception as exc:
reason = _runtime_setup_reason(exc)
print(t("runtime_setup_failed"))
frames = traceback.extract_tb(exc.__traceback__)
stage = "runtime_setup"
Expand All @@ -1296,8 +1320,8 @@ def main():
elif frame.filename.endswith(("strategy_loader.py", "strategy_runtime.py", "strategy_registry.py")):
stage = "strategy_load"
error_type = type(exc).__name__ if type(exc) in {ValueError, KeyError, TypeError, OSError, RuntimeError} else "RuntimeError"
print(f"runtime_setup_failed stage={stage} error_type={error_type}")
if _notify_runtime_error(RuntimeError("runtime_setup_failed")):
print(f"runtime_setup_failed stage={stage} error_type={error_type} reason_code={reason}")
if _notify_runtime_error(RuntimeError(reason)):
output_path = os.getenv("GITHUB_OUTPUT")
if output_path:
with open(output_path, "a", encoding="utf-8") as output:
Expand Down
10 changes: 10 additions & 0 deletions notify_i18n_support.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,11 @@
"firestore_write_failed": "Firestore write failed: {error}",
"telegram_send_failed": "Telegram send failed",
"runtime_error_title": "⚠️ Binance strategy run failed",
"runtime_error_reason_recovery_control_read_failed": "Temporary recovery configuration read failure; trading did not start.",
"runtime_error_reason_recovery_not_active": "Recovery authorization is not active; trading did not start.",
"runtime_error_reason_recovery_control_state_invalid": "Recovery configuration is invalid; trading did not start.",
"runtime_error_reason_recovery_active_binding_invalid": "Recovery authorization binding is invalid; trading did not start.",
"runtime_error_reason_generic": "The runtime could not start safely; inspect the startup diagnostics.",
"runtime_health_monitor_started": "Health monitor started",
"runtime_setup_failed": "Strategy startup failed: runtime_setup_failed",
"runtime_error_result": "The run did not finish successfully; check the latest execution report.",
Expand Down Expand Up @@ -153,6 +158,11 @@
"firestore_write_failed": "Firestore 写入状态失败: {error}",
"telegram_send_failed": "Telegram 发送失败",
"runtime_error_title": "⚠️ Binance 策略运行失败",
"runtime_error_reason_recovery_control_read_failed": "暂时无法读取恢复配置,本次未启动交易。",
"runtime_error_reason_recovery_not_active": "恢复授权尚未激活,本次未启动交易。",
"runtime_error_reason_recovery_control_state_invalid": "恢复配置状态无效,本次未启动交易。",
"runtime_error_reason_recovery_active_binding_invalid": "恢复授权绑定无效,本次未启动交易。",
"runtime_error_reason_generic": "运行时无法安全启动,请查看启动诊断。",
"runtime_health_monitor_started": "运行监测已启动",
"runtime_setup_failed": "策略启动失败:runtime_setup_failed",
"runtime_error_result": "本次运行未正常结束,请查看最新执行报告。",
Expand Down
20 changes: 16 additions & 4 deletions scripts/validate_runtime_startup.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
"runtime_recovery_not_active": "runtime_recovery_not_active",
"recovery_control_state_invalid": "recovery_control_state_invalid",
"recovery_active_binding_invalid": "recovery_active_binding_invalid",
"recovery_control_read_failed": "recovery_control_read_failed",
"STRATEGY_PROFILE does not match RUNTIME_TARGET_JSON.strategy_profile": "startup_strategy_profile_conflict",
"BINANCE_DRY_RUN does not match RUNTIME_TARGET_JSON.dry_run_only": "startup_dry_run_conflict",
"full_cycle_requires_disabled_runtime": "full_cycle_requires_disabled_runtime",
Expand Down Expand Up @@ -143,12 +144,22 @@ def _error_type_name(exc):
return "RuntimeError"


def _safe_startup_reason(exc):
"""Return an exact allowlisted reason carried by a startup exception."""
reason = getattr(exc, "reason_code", None)
if type(exc) is ValueError:
reason = str(exc)
if type(reason) is not str:
return None
return _SAFE_STARTUP_REASONS.get(reason)


def _wrap_full_cycle_error(exc, *, failure_stage, fallback_reason):
if isinstance(exc, FullCycleValidationError):
return exc
reason_code = _authority_reason_code(exc)
if reason_code is None and type(exc) is ValueError:
reason_code = _SAFE_STARTUP_REASONS.get(str(exc))
if reason_code is None:
reason_code = _safe_startup_reason(exc)
return FullCycleValidationError(
reason_code or fallback_reason,
failure_stage=failure_stage,
Expand Down Expand Up @@ -474,8 +485,9 @@ def validate_startup():
reason = exc.reason_code
failure_stage = exc.failure_stage
kind = exc.error_type
if type(exc) is ValueError:
reason = _SAFE_STARTUP_REASONS.get(str(exc), reason)
safe_reason = _safe_startup_reason(exc)
if safe_reason is not None:
reason = safe_reason
output = {
"status": "failed",
"stage": "full_cycle_validation" if full_cycle else "runtime_startup_validation",
Expand Down
33 changes: 33 additions & 0 deletions tests/test_main_runtime_error_notification.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,12 @@ class BinanceAPIException(Exception):
strategy_registry_module.resolve_strategy_metadata = lambda *_args, **_kwargs: types.SimpleNamespace(
display_name="Crypto Live Pool Rotation",
)
strategy_registry_module.resolve_research_strategy_metadata = strategy_registry_module.resolve_strategy_metadata
strategy_registry_module.resolve_research_strategy_definition = strategy_registry_module.resolve_strategy_definition
strategy_registry_module.resolve_runtime_target_strategy = lambda target: (
target,
strategy_registry_module.resolve_strategy_definition(getattr(target, "strategy_profile", None)),
)
sys.modules["strategy_registry"] = strategy_registry_module

if "strategy_runtime" not in sys.modules:
Expand Down Expand Up @@ -123,6 +129,33 @@ def test_runtime_error_message_uses_chinese_without_exposing_exception(self):
self.assertNotIn("PRIVATE_SENTINEL", message)
self.assertNotIn("未提交订单", message)

def test_runtime_error_notification_uses_fixed_human_reason_text(self):
with patch.dict(os.environ, {"NOTIFY_LANG": "zh-CN", "STRATEGY_PROFILE": "crypto_live_pool_rotation"}):
message = main._runtime_error_notification_message(RuntimeError("recovery_control_read_failed"))
self.assertIn("暂时无法读取恢复配置,本次未启动交易。", message)
self.assertNotIn("recovery_control_read_failed", message)

with patch.dict(os.environ, {"NOTIFY_LANG": "en", "STRATEGY_PROFILE": "crypto_live_pool_rotation"}):
message = main._runtime_error_notification_message(RuntimeError("untrusted-provider-detail"))
self.assertIn("The runtime could not start safely", message)
self.assertNotIn("untrusted-provider-detail", message)

def test_main_reports_only_fixed_recovery_read_reason(self):
observed = []

def fake_run_cli_entrypoint(**_kwargs):
raise RuntimeError("recovery_control_read_failed")

with patch.object(main, "run_cli_entrypoint", fake_run_cli_entrypoint), \
patch.object(main, "_notify_runtime_error", return_value=False), \
patch("builtins.print", lambda *args, **_kwargs: observed.append(" ".join(map(str, args)))):
with self.assertRaisesRegex(RuntimeError, "runtime_setup_failed"):
main.main()

rendered = " ".join(observed)
self.assertIn("reason_code=recovery_control_read_failed", rendered)
self.assertNotIn("provider", rendered)

def test_main_wires_cli_entrypoint_with_runtime_builder_and_cycle_runner(self):
observed = {}

Expand Down
52 changes: 52 additions & 0 deletions tests/test_reconciliation_recovery.py
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,58 @@ def test_source_digest_is_not_a_substitute_for_verifying_frozen_config():
validate_source(source, runtime_target=args["runtime_target"], expected=args["expected"], now=NOW + timedelta(minutes=31))


def test_recovery_control_read_failure_is_fixed_and_redacted(monkeypatch):
import application.reconciliation_recovery as recovery
import live_services

class Ref:
def get(self, **_kwargs):
raise RuntimeError("provider secret and credential details")

class Client:
def collection(self, name):
assert name == "strategy"
return self

def document(self, name):
assert name == "MULTI_ASSET_STATE__recovery"
return Ref()

monkeypatch.setenv("BINANCE_RECOVERY_CONTROL_ENABLED", "true")
monkeypatch.setattr(live_services, "get_firestore_client", lambda: Client())
with pytest.raises(recovery.RecoveryControlReadError) as error:
recovery.load_activated_target(_target())
assert str(error.value) == "recovery_control_read_failed"
assert "provider secret" not in str(error.value)


def test_recovery_control_validation_failure_keeps_existing_reason(monkeypatch):
import application.reconciliation_recovery as recovery
import live_services

class Ref:
exists = True

def get(self, **_kwargs):
return self

@staticmethod
def to_dict():
return {"state": "UNSUPPORTED"}

class Client:
def collection(self, _name):
return self

def document(self, _name):
return Ref()

monkeypatch.setenv("BINANCE_RECOVERY_CONTROL_ENABLED", "true")
monkeypatch.setattr(live_services, "get_firestore_client", lambda: Client())
with pytest.raises(ValueError, match="recovery_control_state_invalid"):
recovery.load_activated_target(_target())


def test_candidate_payload_tampering_rejected():
args = inputs()
source = collect_recovery_source(**args)
Expand Down
19 changes: 19 additions & 0 deletions tests/test_validate_runtime_startup.py
Original file line number Diff line number Diff line change
Expand Up @@ -335,6 +335,7 @@ def test_full_cycle_failure_projection_preserves_safe_daily_state_reason_code():
('runtime_recovery_not_active', 'runtime_recovery_not_active'),
('recovery_control_state_invalid', 'recovery_control_state_invalid'),
('recovery_active_binding_invalid', 'recovery_active_binding_invalid'),
('recovery_control_read_failed', 'recovery_control_read_failed'),
('STRATEGY_PROFILE does not match RUNTIME_TARGET_JSON.strategy_profile', 'startup_strategy_profile_conflict'),
('BINANCE_DRY_RUN does not match RUNTIME_TARGET_JSON.dry_run_only', 'startup_dry_run_conflict'),
('runtime_recovery_not_active: DO_NOT_LOG_THIS_SYNTHETIC_TOKEN', 'runtime_startup_validation_failed'),
Expand Down Expand Up @@ -362,6 +363,24 @@ def build():
'error_type': 'ValueError', 'reason_code': expected,
}


def test_startup_cli_reports_reason_from_safe_runtime_exception(monkeypatch, capsys):
from application.reconciliation_recovery import RecoveryControlReadError

monkeypatch.setenv('RUNTIME_TARGET_ENABLED', 'false')
monkeypatch.delenv('BINANCE_API_KEY', raising=False)
monkeypatch.delenv('BINANCE_API_SECRET', raising=False)

def build():
raise RecoveryControlReadError()

monkeypatch.setitem(sys.modules, 'main', SimpleNamespace(build_live_runtime=build))
script = Path(__file__).resolve().parents[1] / 'scripts/validate_runtime_startup.py'
with pytest.raises(SystemExit) as error:
runpy.run_path(str(script), run_name='__main__')
assert error.value.code == 1
assert json.loads(capsys.readouterr().out)['reason_code'] == 'recovery_control_read_failed'

@pytest.mark.parametrize(
"authority_message, expected_reason",
[
Expand Down