diff --git a/scripts/run_walk_forward_backtest.py b/scripts/run_walk_forward_backtest.py index c18a000..ff69b60 100644 --- a/scripts/run_walk_forward_backtest.py +++ b/scripts/run_walk_forward_backtest.py @@ -95,7 +95,7 @@ def _normalize_panel(panel: pd.DataFrame) -> pd.DataFrame: frame["open"] = pd.to_numeric(frame["open"], errors="coerce") frame["final_score"] = pd.to_numeric(frame["final_score"], errors="coerce") frame["in_universe"] = frame["in_universe"].astype(str).str.lower().isin({"true", "1"}) - frame = frame.dropna(subset=["date", "symbol", "open"]) + frame = frame.dropna(subset=["date", "symbol"]) if frame.duplicated(["date", "symbol"]).any(): raise ValueError("research panel contains duplicate date/symbol rows") return frame.set_index(["date", "symbol"]).sort_index() diff --git a/src/crypto_strategies/backtest/live_pool_simulator.py b/src/crypto_strategies/backtest/live_pool_simulator.py index eb54764..bec0555 100644 --- a/src/crypto_strategies/backtest/live_pool_simulator.py +++ b/src/crypto_strategies/backtest/live_pool_simulator.py @@ -84,6 +84,8 @@ def run_live_pool_rotation_backtest( A score observed on ``signal_date`` is tradable at ``effective_date`` after ``signal_lag`` rows; returns are measured from that effective open to the next open. Costs are charged on half-L1 turnover at each rebalance. + Required execution and valuation opens must be finite and positive; missing + prices on unexposed assets do not invalidate a cash or invested period. """ if int(top_n) <= 0: raise ValueError("top_n must be positive") @@ -123,7 +125,6 @@ def run_live_pool_rotation_backtest( .reindex(index=dates, columns=symbols) .astype(float) ) - open_returns = open_matrix.shift(-1).div(open_matrix).sub(1.0).fillna(0.0) portfolio_weights = pd.Series(0.0, index=symbols, dtype=float) daily_returns: list[float] = [] @@ -137,6 +138,7 @@ def run_live_pool_rotation_backtest( ): signal_idx = effective_idx - signal_lag signal_date = dates[signal_idx] + held = portfolio_weights.ne(0.0) turnover = 0.0 fee = 0.0 slippage = 0.0 @@ -172,7 +174,18 @@ def run_live_pool_rotation_backtest( } ) - gross_return = float((portfolio_weights * open_returns.loc[effective_date]).sum()) + exposed = portfolio_weights.ne(0.0) + # Exiting assets need this open, but only retained/new assets need the next. + current_prices = open_matrix.loc[effective_date, held | exposed] + next_prices = open_matrix.iloc[effective_idx + 1].loc[exposed] + required_prices = pd.concat([current_prices, next_prices]) + if not (np.isfinite(required_prices) & required_prices.gt(0.0)).all(): + raise ValueError("required open prices must be finite and positive") + open_returns = ( + next_prices.div(open_matrix.loc[effective_date, exposed]).sub(1.0) + .reindex(symbols, fill_value=0.0) + ) + gross_return = float((portfolio_weights * open_returns).sum()) cost = fee + slippage if cost >= 1.0: raise ValueError("transaction cost must be less than 1.0") @@ -185,7 +198,7 @@ def run_live_pool_rotation_backtest( daily_slippage.append(slippage) gross_growth = 1.0 + gross_return portfolio_weights = ( - portfolio_weights.mul(1.0 + open_returns.loc[effective_date]) + portfolio_weights.mul(1.0 + open_returns) .div(gross_growth) .fillna(0.0) if gross_growth > 0.0 diff --git a/tests/test_live_pool_simulator_prices.py b/tests/test_live_pool_simulator_prices.py new file mode 100644 index 0000000..fcdab71 --- /dev/null +++ b/tests/test_live_pool_simulator_prices.py @@ -0,0 +1,138 @@ +from __future__ import annotations + +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest +from quant_platform_kit.strategy_lifecycle.backtest_orchestrator import ( + BacktestOrchestrator, +) +from quant_platform_kit.strategy_lifecycle.performance_store import PerformanceStore + +from crypto_strategies.backtest.live_pool_simulator import ( + run_live_pool_rotation_backtest, +) +from crypto_strategies.backtest.orchestrator_runner import ( + PROFILE_NAME, + CryptoLivePoolBacktestRunner, +) + + +def _panel() -> pd.DataFrame: + index = pd.MultiIndex.from_product( + [pd.date_range("2024-01-01", periods=5), ["A", "B"]], names=["date", "symbol"] + ) + panel = pd.DataFrame({"open": 100.0, "in_universe": True}, index=index) + panel["final_score"] = [1.0, 0.0] * 5 + return panel + + +@pytest.mark.parametrize("bad_open", [np.nan, np.inf, -np.inf, 0.0, -1.0]) +@pytest.mark.parametrize("day_index", [1, 2, 4], ids=["entry", "held", "terminal"]) +def test_required_open_must_be_finite_and_positive(bad_open: float, day_index: int) -> None: + panel = _panel() + day = panel.index.get_level_values("date").unique()[day_index] + panel.loc[(day, "A"), "open"] = bad_open + + with pytest.raises(ValueError, match="required open prices must be finite and positive"): + run_live_pool_rotation_backtest(panel, top_n=1, rebalance_every=7) + + +@pytest.mark.parametrize("day_index", [1, 2, 4]) +def test_missing_required_symbol_row_is_not_a_zero_return(day_index: int) -> None: + panel = _panel() + day = panel.index.get_level_values("date").unique()[day_index] + panel = panel.drop(index=(day, "A")) + + with pytest.raises(ValueError, match="required open prices must be finite and positive"): + run_live_pool_rotation_backtest(panel, top_n=1, rebalance_every=7) + + +def test_missing_open_cannot_be_hidden_by_exiting_to_cash() -> None: + panel = _panel() + dates = panel.index.get_level_values("date").unique() + panel.loc[(dates[1:], slice(None)), "in_universe"] = False + panel.loc[(dates[2], "A"), "open"] = np.nan + + with pytest.raises(ValueError, match="required open prices must be finite and positive"): + run_live_pool_rotation_backtest(panel, top_n=1, rebalance_every=1) + + +@pytest.mark.parametrize("bad_open", [np.nan, np.inf, -np.inf, 0.0, -1.0]) +def test_unselected_prices_do_not_affect_exposed_returns(bad_open: float) -> None: + panel = _panel() + dates = panel.index.get_level_values("date").unique() + panel.loc[(slice(None), "B"), "open"] = bad_open + panel.loc[(dates[2:], "A"), "open"] = 101.0 + + result = run_live_pool_rotation_backtest(panel, top_n=1, rebalance_every=7) + + assert result.returns.tolist() == pytest.approx([0.01, 0.0, 0.0]) + + +def test_cash_after_exit_does_not_require_future_asset_prices() -> None: + panel = _panel() + dates = panel.index.get_level_values("date").unique() + panel.loc[(dates[1:], slice(None)), "in_universe"] = False + panel.loc[(dates[3:], slice(None)), "open"] = np.nan + + result = run_live_pool_rotation_backtest(panel, top_n=1, rebalance_every=1, fee_bps=100) + + assert result.returns.tolist() == pytest.approx([-0.01, -0.01, 0.0]) + assert result.trade_log["turnover"].tolist() == pytest.approx([1.0, 1.0, 0.0]) + + +def test_cash_before_late_selection_does_not_require_asset_prices() -> None: + panel = _panel() + dates = panel.index.get_level_values("date").unique() + panel.loc[(dates[:2], slice(None)), "in_universe"] = False + panel.loc[(dates[:3], slice(None)), "open"] = np.nan + + result = run_live_pool_rotation_backtest(panel, top_n=1, rebalance_every=1) + + assert result.returns.tolist() == [0.0, 0.0, 0.0] + assert result.trade_log["turnover"].tolist() == [0.0, 0.0, 1.0] + + +@pytest.mark.parametrize("signal_lag", [0, 1, 5]) +def test_terminal_signal_does_not_require_an_execution_beyond_window(signal_lag: int) -> None: + panel = _panel() + dates = panel.index.get_level_values("date").unique() + panel.loc[:, "in_universe"] = False + panel.loc[(dates[-1], "A"), "in_universe"] = True + panel.loc[:, "open"] = np.nan + + result = run_live_pool_rotation_backtest(panel, top_n=1, signal_lag=signal_lag) + + assert result.returns.tolist() == [0.0] * max(len(dates) - signal_lag - 1, 0) + + +def test_orchestrator_propagates_invalid_input_without_persisting(tmp_path: Path) -> None: + panel = _panel() + dates = panel.index.get_level_values("date").unique() + panel.loc[(dates[-1], "A"), "open"] = np.nan + runner = CryptoLivePoolBacktestRunner(panel=panel) + orchestrator = BacktestOrchestrator(store=PerformanceStore(local_root=tmp_path)) + orchestrator.register_runner("crypto", runner) + + with pytest.raises(ValueError, match="required open prices must be finite and positive"): + orchestrator.run(PROFILE_NAME, domain="crypto", params={"top_n": 1}) + + assert runner.last_daily_returns.empty + assert not runner.run_return_history + assert not list(tmp_path.rglob("*.json")) + + +def test_runner_uses_only_prices_inside_requested_window() -> None: + panel = _panel() + dates = panel.index.get_level_values("date").unique() + panel.loc[(dates[-1], slice(None)), "open"] = np.nan + runner = CryptoLivePoolBacktestRunner(panel=panel) + + result = runner.run( + PROFILE_NAME, {"top_n": 1}, start_date=dates[0].date(), end_date=dates[-2].date() + ) + + assert result.observation_count == 2 + assert runner.last_daily_returns.tolist() == [0.0, 0.0] diff --git a/tests/test_run_walk_forward_backtest.py b/tests/test_run_walk_forward_backtest.py index 780132d..ee15cc3 100644 --- a/tests/test_run_walk_forward_backtest.py +++ b/tests/test_run_walk_forward_backtest.py @@ -188,3 +188,72 @@ def test_normalized_panel_preserves_unscored_open_rows() -> None: assert len(normalized) == 1 assert pd.isna(normalized.iloc[0]["final_score"]) + + +@pytest.mark.parametrize("opening", [None, "unavailable"]) +def test_normalized_panel_preserves_missing_prices_but_still_drops_invalid_dates(opening) -> None: + panel = pd.DataFrame([ + {"date": "2024-01-01", "symbol": " a ", "in_universe": True, "open": opening, "final_score": 1}, + {"date": "invalid", "symbol": "B", "in_universe": True, "open": opening, "final_score": 1}, + ]) + + normalized = _normalize_panel(panel) + + assert normalized.index.tolist() == [(pd.Timestamp("2024-01-01"), "A")] + assert pd.isna(normalized.iloc[0]["open"]) + + +@pytest.mark.parametrize("opening", [None, "unavailable"]) +def test_cli_rejects_whole_missing_price_day_without_success_output( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture, opening, +) -> None: + dates = pd.date_range(walk_forward.DEFAULT_WINDOWS[0][0], walk_forward.DEFAULT_WINDOWS[-1][1]) + panel = pd.DataFrame([ + {"date": day, "symbol": symbol, "in_universe": True, "open": 100.0, "final_score": score} + for day in dates + for symbol, score in [("BTCUSDT", 3), ("ETHUSDT", 2), ("SOLUSDT", 1)] + ]) + history = panel.loc[panel["symbol"].isin(["BTCUSDT", "ETHUSDT"]), ["date", "symbol", "open"]] + history = history.rename(columns={"open": "close"}) + panel["open"] = panel["open"].astype(object) + panel.loc[panel["date"] == dates[40], "open"] = opening + panel_path = tmp_path / "panel.csv" + history_path = tmp_path / "history.csv" + output_path = tmp_path / "result.json" + returns_path = tmp_path / "returns.csv" + store = tmp_path / "store" + panel.to_csv(panel_path, index=False) + history.to_csv(history_path, index=False) + monkeypatch.setattr(sys, "argv", [ + "run_walk_forward_backtest.py", "--panel", str(panel_path), + "--market-history", str(history_path), "--store-root", str(store), + "--json-output", str(output_path), "--returns-output", str(returns_path), + ]) + + with pytest.raises(ValueError, match="required open prices must be finite and positive"): + walk_forward.main() + + assert not output_path.exists() + assert not returns_path.exists() + assert not list(store.rglob("*.json")) + assert not capsys.readouterr().out + + +@pytest.mark.parametrize("cash_first", [False, True]) +def test_normalized_missing_unexposed_prices_preserve_runner_periods(cash_first: bool) -> None: + dates = pd.date_range("2024-01-01", periods=5) + panel = pd.DataFrame([ + {"date": day, "symbol": symbol, "in_universe": True, "open": 100.0, "final_score": score} + for day in dates for symbol, score in [("A", 1), ("B", 0)] + ]) + panel.loc[panel["symbol"] == "B", "open"] = float("nan") + if cash_first: + panel.loc[panel["date"] < dates[2], "in_universe"] = False + panel.loc[panel["date"] < dates[3], "open"] = float("nan") + runner = orchestrator_runner.CryptoLivePoolBacktestRunner(panel=_normalize_panel(panel)) + + result = runner.run("crypto_live_pool_rotation", {"top_n": 1, "rebalance_every": 1}) + + assert result.observation_count == 3 + assert runner.last_daily_returns.index.tolist() == dates[1:-1].tolist() + assert runner.last_daily_returns.tolist() == [0.0, 0.0, 0.0]