From 984ccd94e82080c458c753665cab59538fe80d16 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Sun, 6 Sep 2026 18:17:33 +0800 Subject: [PATCH] fix(overlay): ignore signals not yet available at decision time Enforce generated/available time <= decision time and reject non-finite 0-1 confidence values during shadow replay validation. Co-Authored-By: Claude --- .../overlay_backtest.py | 50 ++++++++++++-- .../schema.py | 4 ++ tests/test_overlay_backtest.py | 69 +++++++++++++++++++ tests/test_signal_validation.py | 22 ++++++ 4 files changed, 141 insertions(+), 4 deletions(-) diff --git a/src/research_signal_context_pipelines/overlay_backtest.py b/src/research_signal_context_pipelines/overlay_backtest.py index 4f9edfa..9cb273e 100644 --- a/src/research_signal_context_pipelines/overlay_backtest.py +++ b/src/research_signal_context_pipelines/overlay_backtest.py @@ -36,6 +36,27 @@ def parse_date(value: str) -> dt.date: return parse_price_date(value) +def parse_datetime(value: str) -> dt.datetime: + text = str(value).strip() + if text.endswith("Z"): + text = f"{text[:-1]}+00:00" + parsed = dt.datetime.fromisoformat(text) + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=dt.timezone.utc) + return parsed.astimezone(dt.timezone.utc) + + +def decision_datetime_for_date(date: dt.date) -> dt.datetime: + # Date-only replay decisions are treated as start-of-day UTC so same-day + # after-hours generation cannot affect earlier same-day decisions. + return dt.datetime(date.year, date.month, date.day, tzinfo=dt.timezone.utc) + + +def signal_available_at(signal: dict[str, Any]) -> dt.datetime: + raw = signal.get("available_at", signal["generated_at"]) + return parse_datetime(str(raw)) + + def load_price_history(path: Path, *, symbol: str) -> list[PricePoint]: rows = [ PricePoint(date=row.date, close=row.close) @@ -57,14 +78,35 @@ def load_signals(path: Path) -> list[dict[str, Any]]: return signals -def signal_active_on(signal: dict[str, Any], date: dt.date) -> bool: +def signal_active_on( + signal: dict[str, Any], + date: dt.date, + *, + decision_time: dt.datetime | None = None, +) -> bool: as_of = parse_date(str(signal["as_of"])) expires_at = parse_date(str(signal["expires_at"])) - return as_of <= date <= expires_at + if not (as_of <= date <= expires_at): + return False + decision = decision_time if decision_time is not None else decision_datetime_for_date(date) + if decision.tzinfo is None: + decision = decision.replace(tzinfo=dt.timezone.utc) + else: + decision = decision.astimezone(dt.timezone.utc) + return signal_available_at(signal) <= decision -def signal_for_date(signals: list[dict[str, Any]], date: dt.date) -> dict[str, Any] | None: - active = [signal for signal in signals if signal_active_on(signal, date)] +def signal_for_date( + signals: list[dict[str, Any]], + date: dt.date, + *, + decision_time: dt.datetime | None = None, +) -> dict[str, Any] | None: + active = [ + signal + for signal in signals + if signal_active_on(signal, date, decision_time=decision_time) + ] return active[-1] if active else None diff --git a/src/research_signal_context_pipelines/schema.py b/src/research_signal_context_pipelines/schema.py index 4076cdc..dbad249 100644 --- a/src/research_signal_context_pipelines/schema.py +++ b/src/research_signal_context_pipelines/schema.py @@ -1,5 +1,7 @@ from __future__ import annotations +import math + import datetime as dt from collections.abc import Mapping, Sequence from typing import Any @@ -74,6 +76,8 @@ def _require_string_list(value: Any, name: str, *, allow_empty: bool = False) -> def _require_number_0_1(value: Any, name: str) -> None: if not isinstance(value, (int, float)) or isinstance(value, bool): raise SignalValidationError(f"{name} must be numeric") + if not math.isfinite(value): + raise SignalValidationError(f"{name} must be a finite number between 0 and 1") if value < 0 or value > 1: raise SignalValidationError(f"{name} must be between 0 and 1") diff --git a/tests/test_overlay_backtest.py b/tests/test_overlay_backtest.py index 16a0f45..c996f0a 100644 --- a/tests/test_overlay_backtest.py +++ b/tests/test_overlay_backtest.py @@ -1,14 +1,18 @@ from __future__ import annotations +import copy import datetime as dt +import json from pathlib import Path from research_signal_context_pipelines.overlay_backtest import ( OverlayPolicy, + PricePoint, backtest_overlay, exposure_for_signal, load_price_history, load_signals, + signal_active_on, signal_for_date, ) @@ -16,6 +20,12 @@ ROOT = Path(__file__).resolve().parents[1] +def _example_signal(**overrides: object) -> dict: + payload = json.loads((ROOT / "examples" / "signal_history" / "2026-02-06.json").read_text(encoding="utf-8")) + payload.update(overrides) + return payload + + def test_exposure_for_signal_is_risk_reducing_only() -> None: policy = OverlayPolicy() assert exposure_for_signal(None, policy) == 1.0 @@ -34,6 +44,65 @@ def test_signal_for_date_uses_latest_active_signal() -> None: assert signal_for_date(signals, dt.date(2026, 6, 1)) is None +def test_signal_active_on_requires_generated_at_not_after_decision_time() -> None: + signal = _example_signal( + as_of="2026-02-06", + generated_at="2026-02-06T22:00:00Z", + expires_at="2026-03-19", + ) + morning = dt.datetime(2026, 2, 6, 15, 0, tzinfo=dt.timezone.utc) + after_hours = dt.datetime(2026, 2, 6, 22, 0, tzinfo=dt.timezone.utc) + + assert signal_active_on(signal, dt.date(2026, 2, 6), decision_time=morning) is False + assert signal_active_on(signal, dt.date(2026, 2, 6), decision_time=after_hours) is True + assert signal_active_on(signal, dt.date(2026, 2, 6)) is False + assert signal_for_date([signal], dt.date(2026, 2, 9))["regime"] == "risk_off" + + +def test_signal_active_on_uses_available_at_when_present() -> None: + signal = _example_signal( + as_of="2026-02-06", + generated_at="2026-02-05T12:00:00Z", + available_at="2026-02-06T22:00:00Z", + expires_at="2026-03-19", + ) + morning = dt.datetime(2026, 2, 6, 15, 0, tzinfo=dt.timezone.utc) + + assert signal_active_on(signal, dt.date(2026, 2, 6), decision_time=morning) is False + assert signal_for_date([signal], dt.date(2026, 2, 9)) is not None + + +def test_future_generated_at_does_not_rewrite_earlier_overlay_path() -> None: + prices = [ + PricePoint(date=dt.date(2026, 2, 6), close=100.0), + PricePoint(date=dt.date(2026, 2, 13), close=90.0), + PricePoint(date=dt.date(2026, 2, 20), close=80.0), + PricePoint(date=dt.date(2026, 2, 27), close=70.0), + ] + known = _example_signal( + as_of="2026-02-06", + generated_at="2026-02-06T22:00:00Z", + expires_at="2026-03-19", + regime="risk_off", + confidence=0.9, + risk_flags=["liquidity_stress"], + ) + future = copy.deepcopy(known) + future["generated_at"] = "2026-02-20T22:00:00Z" + + without_signal = backtest_overlay(prices, []) + with_future = backtest_overlay(prices, [future]) + with_known = backtest_overlay(prices, [known]) + + # Periods before the future generated_at must match the no-signal path. + assert with_future["overlay"]["final_equity"] == without_signal["overlay"]["final_equity"] + assert with_future["overlay"]["total_return"] == without_signal["overlay"]["total_return"] + assert with_future["overlay"]["avg_exposure"] == without_signal["overlay"]["avg_exposure"] + # Once available, an unexpired risk-off signal still reduces exposure vs baseline. + assert with_known["overlay"]["avg_exposure"] < without_signal["overlay"]["avg_exposure"] + assert with_known["overlay"]["final_equity"] > without_signal["overlay"]["final_equity"] + + def test_backtest_overlay_reduces_drawdown_on_synthetic_path() -> None: prices = load_price_history(ROOT / "examples" / "price_history.example.csv", symbol="QQQ") signals = load_signals(ROOT / "examples" / "signal_history") diff --git a/tests/test_signal_validation.py b/tests/test_signal_validation.py index d1e16c1..001b075 100644 --- a/tests/test_signal_validation.py +++ b/tests/test_signal_validation.py @@ -1,5 +1,6 @@ from __future__ import annotations +import datetime as dt import hashlib import json from pathlib import Path @@ -7,6 +8,7 @@ import pytest from research_signal_context_pipelines import SignalValidationError, validate_signal +from research_signal_context_pipelines.overlay_backtest import signal_active_on from scripts import post_shadow_signal_request as shadow_issue from scripts import validate_latest_signal as signal_validator @@ -24,6 +26,18 @@ def test_example_signal_is_valid() -> None: assert payload["horizon"] == "1-3 years" +def test_validated_signal_is_inactive_before_generated_at() -> None: + payload = load_example() + payload["as_of"] = "2026-05-28" + payload["generated_at"] = "2026-05-28T22:00:00Z" + payload["expires_at"] = "2026-06-30" + validate_signal(payload) + + morning = dt.datetime(2026, 5, 28, 14, 30, tzinfo=dt.timezone.utc) + assert signal_active_on(payload, dt.date(2026, 5, 28), decision_time=morning) is False + assert signal_active_on(payload, dt.date(2026, 5, 29)) is True + + def test_v2_signal_requires_versioned_model_metadata() -> None: payload = load_example() payload.update( @@ -209,3 +223,11 @@ def test_shadow_request_binds_context_digest_to_immutable_commit(tmp_path: Path) "producer_commit_sha": "c" * 40, "input_digest": "sha256:" + hashlib.sha256(context_path.read_bytes()).hexdigest(), } + + +def test_confidence_rejects_nan() -> None: + payload = load_example() + payload["confidence"] = float("nan") + + with pytest.raises(SignalValidationError, match="finite|between 0 and 1"): + validate_signal(payload)