fix: preserve observed price gaps in ordinary HK backtests

src/hk_equity_strategies/backtest/combo_simulator.py

@@ -12,11 +12,11 @@
     HkRotationBacktestConfig,
     HkRotationBacktestResult,
     StrategySignalFn,
+    _build_observed_close_matrix,
     build_rebalance_dates,
     compute_backtest_metrics,
     rebalance_holdings,
 )
-from hk_equity_strategies.strategies.etf_rotation_core import build_close_matrix
 from hk_equity_strategies.strategies.hk_equity_combo import (
     DEFAULT_DIVIDEND_WEIGHT,
     DEFAULT_ETF_WEIGHT,
@@ -151,6 +151,10 @@ def _combo_strategy_returns(
     if DIVIDEND_SYMBOL not in prices.columns:
         prices[DIVIDEND_SYMBOL] = 1.0
     prices[DIVIDEND_SYMBOL] = (1.0 + dividend_returns.reindex(prices.index).fillna(0.0)).cumprod()
+    # A modeled dividend price is not evidence of an observed source quote.
+    dividend_sources = close[[DIVIDEND_SYMBOL]] if DIVIDEND_SYMBOL in close.columns else close
+    observed = (dividend_sources.gt(0.0) & dividend_sources.lt(math.inf)).all(axis=1)
+    prices[DIVIDEND_SYMBOL] = prices[DIVIDEND_SYMBOL].where(observed)
 
     cost_rate = float(combo_config.cost_bps) / 10_000.0
     if not math.isfinite(cost_rate) or not 0.0 <= cost_rate < 1.0:
@@ -203,7 +207,7 @@ def run_combo_backtest(
         cost_bps=combo.cost_bps,
         rebalance_frequency=combo.rebalance_frequency,
     )
-    close = build_close_matrix(market_history, universe_symbols=universe_symbols)
+    close = _build_observed_close_matrix(market_history, universe_symbols=universe_symbols)
     if len(close) < int(combo.min_history_days):
         raise ValueError(
             f"market_history requires at least {int(combo.min_history_days)} overlapping trading days"

src/hk_equity_strategies/backtest/etf_rotation_simulator.py

@@ -8,7 +8,7 @@
 
 import pandas as pd
 
-from hk_equity_strategies.strategies.etf_rotation_core import build_close_matrix, normalize_symbol
+from hk_equity_strategies.strategies.etf_rotation_core import normalize_symbol, normalize_universe_symbols
 
 StrategySignalFn = Callable[[Any], tuple[Mapping[str, float], Mapping[str, object]]]
 
@@ -26,6 +26,27 @@ class HkRotationBacktestResult:
     metrics: dict[str, float | int] = field(default_factory=dict)
 
 
+def _build_observed_close_matrix(market_history: pd.DataFrame, *, universe_symbols=None) -> pd.DataFrame:
+    """Keep missing observations visible to fill/mark checks, unlike signal history."""
+    frame = market_history.copy()
+    missing_columns = {"date", "symbol", "close"} - set(frame.columns)
+    if missing_columns:
+        raise ValueError(f"market_history missing required columns: {', '.join(sorted(missing_columns))}")
+    frame["date"] = pd.to_datetime(frame["date"], utc=False).dt.tz_localize(None).dt.normalize()
+    frame["symbol"] = frame["symbol"].map(normalize_symbol)
+    frame["close"] = pd.to_numeric(frame["close"], errors="coerce")
+    symbols = normalize_universe_symbols(universe_symbols)
+    missing = set(symbols) - set(frame.loc[frame["close"].notna(), "symbol"])
+    if missing:
+        raise ValueError(f"market_history missing required strategy symbols: {', '.join(sorted(missing))}")
+    close = (
+        frame.loc[frame["symbol"].isin(symbols)]
+        .pivot_table(index="date", columns="symbol", values="close", aggfunc="last", dropna=False)
+        .sort_index()
+    )
+    return close.loc[:, list(symbols)]
+
+
 def _rebalance_dates(index: pd.DatetimeIndex, *, frequency: str) -> pd.DatetimeIndex:
     if frequency == "monthly":
         return index.to_series().resample("ME").last().dropna().index
@@ -133,7 +154,7 @@ def run_etf_rotation_backtest(
 ) -> HkRotationBacktestResult:
     settings = config or HkRotationBacktestConfig()
     kwargs = dict(strategy_kwargs or {})
-    close = build_close_matrix(market_history, universe_symbols=universe_symbols)
+    close = _build_observed_close_matrix(market_history, universe_symbols=universe_symbols)
     if len(close) < int(settings.min_history_days):
         raise ValueError(
             f"market_history requires at least {int(settings.min_history_days)} overlapping trading days"

tests/test_orchestrator_runner.py

@@ -12,6 +12,7 @@
 from pathlib import Path
 
 import pandas as pd
+import pytest
 
 from hk_equity_strategies.backtest.orchestrator_runner import (
     SUPPORTED_PROFILES,
@@ -29,6 +30,82 @@
 )
 
 
+def _run_observed_price_case(history, runner, weights=None):
+    from hk_equity_strategies.backtest.etf_rotation_simulator import HkRotationBacktestConfig, run_etf_rotation_backtest
+    from hk_equity_strategies.backtest.combo_simulator import HkComboBacktestConfig, run_combo_backtest
+
+    def signal(_):
+        return weights if weights is not None else {"A": 1.0}, {}
+    config = HkRotationBacktestConfig(min_history_days=1, cost_bps=0.0)
+    if runner == "rotation":
+        return run_etf_rotation_backtest(history, signal, config=config, universe_symbols=["A", "B"])
+    return run_combo_backtest(
+        history, signal, rotation_config=config, universe_symbols=["A", "B"],
+        combo_config=HkComboBacktestConfig(combo_mode="static", etf_weight=1.0,
+                                          dividend_weight=0.0, min_history_days=1, cost_bps=0.0),
+    )
+
+
+def _observed_price_history():
+    return pd.DataFrame({
+        "date": pd.to_datetime(["2024-01-31", "2024-02-01", "2024-02-02"] * 2),
+        "symbol": ["A"] * 3 + ["B"] * 3, "close": [100.0] * 6,
+    })
+
+
+@pytest.mark.parametrize("runner", ["rotation", "combo"])
+@pytest.mark.parametrize("invalid", [float("nan"), 0.0, -1.0, float("inf"), "omitted"])
+def test_held_asset_missing_or_invalid_marks_are_not_filled_or_dropped(runner, invalid):
+    history = _observed_price_history()
+    if invalid == "omitted":
+        history = history.drop(index=[1, 2])
+    else:
+        history.loc[[1, 2], "close"] = invalid
+    with pytest.raises(ValueError, match="positive finite.*prices"):
+        _run_observed_price_case(history, runner)
+
+
+@pytest.mark.parametrize("runner", ["rotation", "combo"])
+def test_all_missing_observed_day_is_not_removed(runner):
+    history = _observed_price_history()
+    history.loc[[1, 4], "close"] = float("nan")
+    with pytest.raises(ValueError, match="positive finite.*prices"):
+        _run_observed_price_case(history, runner)
+
+
+@pytest.mark.parametrize("runner", ["rotation", "combo"])
+def test_missing_execution_price_is_not_borrowed_from_earlier_day(runner):
+    history = _observed_price_history()
+    history["date"] = pd.to_datetime(["2024-01-30", "2024-01-31", "2024-02-01"] * 2)
+    history.loc[1, "close"] = float("nan")
+    with pytest.raises(ValueError, match="positive finite fill prices"):
+        _run_observed_price_case(history, runner)
+
+
+@pytest.mark.parametrize("runner", ["rotation", "combo"])
+def test_unused_missing_asset_and_explicit_cash_do_not_require_a_price(runner):
+    history = _observed_price_history()
+    history.loc[4, "close"] = float("nan")
+    assert _run_observed_price_case(history, runner).daily_returns.tolist() == [0.0] * 3
+    assert _run_observed_price_case(history, runner, weights={}).daily_returns.tolist() == [0.0] * 3
+
+
+@pytest.mark.parametrize("has_dividend_source", [True, False])
+def test_held_dividend_proxy_does_not_hide_missing_source_quote(has_dividend_source):
+    from hk_equity_strategies.backtest.combo_simulator import HkComboBacktestConfig, run_combo_backtest
+
+    history = _observed_price_history()
+    if has_dividend_source:
+        history = history.replace({"B": "03110"})
+    history.loc[4, "close"] = float("nan")
+    with pytest.raises(ValueError, match="positive finite mark prices"):
+        run_combo_backtest(
+            history, lambda _: ({}, {}), universe_symbols=["A", "03110" if has_dividend_source else "B"],
+            combo_config=HkComboBacktestConfig(combo_mode="static", etf_weight=0.0,
+                                              dividend_weight=1.0, min_history_days=1, cost_bps=0.0),
+        )
+
+
 def _synthetic_history_digest(history: pd.DataFrame) -> str:
     return hashlib.sha256(pd.util.hash_pandas_object(history, index=True).values.tobytes()).hexdigest()