From ae1aeb3c84b6c91c3b83542e03009f6cc44a9a18 Mon Sep 17 00:00:00 2001 From: Dennis Thiessen Date: Thu, 9 Jul 2026 14:28:50 +0200 Subject: [PATCH] Align backtest production sim with live runtime config The portfolio monitor's Production row now replays the live qualification flag and the Admin exit policy (mode/ATR multiplier/hold days) instead of a frozen research-variant gate, so Admin tuning is reflected in the next run. Single-source the 80/20 strategy_rank weights in momentum_service and pin every dual-defined constant with a parity test. Behavior-preserving today: the production sim reproduces the README baseline exactly. Co-Authored-By: Claude Fable 5 --- README.md | 2 + app/services/backtest_service.py | 77 +++++++++++++++++++--- app/services/momentum_service.py | 12 +++- tests/unit/test_prod_strategy_parity.py | 88 +++++++++++++++++++++++++ 4 files changed, 169 insertions(+), 10 deletions(-) create mode 100644 tests/unit/test_prod_strategy_parity.py diff --git a/README.md b/README.md index 694c7fa..96d820b 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,8 @@ The conclusion is not "trade high volatility alone." Keep residual momentum as t Live-ranking note: the backtest ranks residual momentum and volatility inside each weekly setup-candidate cross-section. The live scanner computes the same 80/20 formula across the current ticker universe before scanning so every generated setup carries a stable ticker-level rank. That is the production approximation; reconcile it later only if candidate-only post-scan ranking proves materially different. +Parity guard (July 2026): the portfolio monitor's **Production** row replays the *runtime* configuration — the live activation gate (`qualified` flag) and the Admin exit policy (mode / ATR multiplier / hold days) — so tuning the strategy in Admin is reflected in the next backtest run instead of silently diverging. Constants defined on both sides (exit defaults, trail width, the 80/20 ordering weights, the promoted cutoff) are pinned by `tests/unit/test_prod_strategy_parity.py`, and the ordering weights are single-sourced from `momentum_service`. + ### The iron rule for strategy changes A signal earns its way into selection **only** through the factor harness: diff --git a/app/services/backtest_service.py b/app/services/backtest_service.py index bdd3f5e..a7b6e49 100644 --- a/app/services/backtest_service.py +++ b/app/services/backtest_service.py @@ -44,7 +44,10 @@ from app.models.ticker import Ticker from app.services import settings_store from app.services.admin_service import get_activation_config, update_setting from app.services.indicator_service import _extract_ohlcv, compute_atr -from app.services.momentum_service import compute_realized_vol_6m +from app.services.momentum_service import ( + STRATEGY_RANK_MOMENTUM_WEIGHT, + compute_realized_vol_6m, +) from app.services.outcome_service import ( OUTCOME_AMBIGUOUS, OUTCOME_STOP_HIT, @@ -941,7 +944,9 @@ def _assign_residual_high_vol_blend(candidates: list[dict]) -> None: """Research ranks: residual momentum blended with higher-vol preference.""" for output_key, residual_weight in ( (RESIDUAL_HIGH_VOL_BLEND_90_10_KEY, 0.9), - (RESIDUAL_HIGH_VOL_BLEND_80_20_KEY, 0.8), + # The production ordering weight comes from momentum_service so the + # simulated production rank cannot drift from the live strategy_rank. + (RESIDUAL_HIGH_VOL_BLEND_80_20_KEY, STRATEGY_RANK_MOMENTUM_WEIGHT), (RESIDUAL_HIGH_VOL_BLEND_KEY, 0.7), (RESIDUAL_HIGH_VOL_BLEND_60_40_KEY, 0.6), ): @@ -1061,6 +1066,20 @@ SIM_STARTING_CAPITAL = 10_000.0 SIM_MAX_POSITIONS = 10 SIM_RISK_PER_TRADE = 0.01 # fraction of equity risked per position (entry→stop) SIM_NOTIONAL_CAP = 0.20 # max fraction of equity per position (no margin) +# The "atr_trail3" research policy's trail width. Must equal the live default +# (paper_trade_service.DEFAULT_ATR_MULTIPLIER) — enforced by the parity test. +# The production portfolio-monitor row additionally follows the *runtime* Admin +# exit policy, so tuning it live is reflected in the next backtest run. +ATR_TRAIL_MULTIPLIER = 3.0 +# How live Admin exit modes map onto simulator exit policies. "trailing" +# (percent trail) has no simulator counterpart and falls back to the plain +# hold-to-horizon book; the row's live_exit_mode field keeps that visible. +LIVE_EXIT_MODE_TO_SIM = { + "atr_trailing": "atr_trail3", + "time": "hold", + "target": "target", + "trailing": "hold", +} def _simulate_portfolio( @@ -1074,6 +1093,7 @@ def _simulate_portfolio( ranking_key: str = PRODUCTION_PERCENTILE_KEY, max_positions: int = SIM_MAX_POSITIONS, risk_per_trade: float = SIM_RISK_PER_TRADE, + atr_trail_multiplier: float = ATR_TRAIL_MULTIPLIER, start_date: date | None = None, include_curve: bool = False, ) -> dict | None: @@ -1249,7 +1269,7 @@ def _simulate_portfolio( pos["highest_close"] = max(pos["highest_close"], bar.close) atr = _atr(sym, bar.idx) if atr is not None: - next_stop = pos["highest_close"] - 3.0 * atr + next_stop = pos["highest_close"] - atr_trail_multiplier * atr if next_stop < bar.close: pos["stop"] = max(pos["stop"], next_stop) @@ -1756,9 +1776,16 @@ PORTFOLIO_MONITOR_STRATEGIES: tuple[dict, ...] = ( { "strategy": PRODUCTION_PORTFOLIO_STRATEGY, "label": "Production: residual/high-vol 80/20 + 3x ATR trail", - "description": "Residual gate, 80/20 residual/high-vol rank, 3x ATR trailing stop.", + "description": ( + "The live strategy: production activation gate and Admin exit policy " + "as currently configured, 80/20 residual/high-vol rank." + ), "entry_variant": "residual80_highvol_blend80_20_fixed10", "exit_policy": "atr_trail3", + # The production row replays what the platform actually does right now: + # the live qualification flag (runtime Admin activation settings) and the + # live Admin exit policy, instead of the frozen research-variant gate. + "use_live_config": True, "is_production": True, }, ) @@ -1817,6 +1844,7 @@ def _portfolio_monitor( prices: dict[str, tuple], _spy_closes: dict[date, float] | None, hold_days: int, + live_exit_policy: dict | None = None, ) -> dict: latest_ord = max((max(cols[0]) for cols in prices.values() if cols[0]), default=None) rows: list[dict] = [] @@ -1825,18 +1853,40 @@ def _portfolio_monitor( if entry_cfg is None: continue ranking_key = str(entry_cfg.get("ranking_key") or entry_cfg["percentile_key"]) + # The production row must replay the LIVE configuration: the runtime + # qualification flag (Admin activation settings) instead of the frozen + # research-variant gate, and the Admin exit policy instead of the + # hardcoded 3x-trail/30d defaults. Research rows stay frozen so they + # remain comparable across runs. + use_live = bool(strategy.get("use_live_config")) + exit_policy = str(strategy["exit_policy"]) + row_hold_days = hold_days + trail_multiplier = ATR_TRAIL_MULTIPLIER + live_exit_mode: str | None = None + if use_live and live_exit_policy is not None: + live_exit_mode = str(live_exit_policy.get("mode", "atr_trailing")) + exit_policy = LIVE_EXIT_MODE_TO_SIM.get(live_exit_mode, "atr_trail3") + row_hold_days = int(live_exit_policy.get("hold_days", hold_days)) + trail_multiplier = float( + live_exit_policy.get("atr_multiplier", ATR_TRAIL_MULTIPLIER) + ) + qualified_fn = ( + None if use_live + else lambda c, config=entry_cfg: _qualifies_strategy_variant(c, config) + ) for lookback in PORTFOLIO_MONITOR_LOOKBACKS: start = _lookback_start(latest_ord, lookback["days"]) sim = _simulate_portfolio( candidates, prices, _spy_closes, - str(strategy["exit_policy"]), - hold_days, - qualified_fn=lambda c, config=entry_cfg: _qualifies_strategy_variant(c, config), + exit_policy, + row_hold_days, + qualified_fn=qualified_fn, ranking_key=ranking_key, max_positions=int(entry_cfg["max_positions"]), risk_per_trade=float(entry_cfg["risk_per_trade"]), + atr_trail_multiplier=trail_multiplier, start_date=start, include_curve=True, ) @@ -1848,7 +1898,8 @@ def _portfolio_monitor( "description": strategy["description"], "is_production": bool(strategy.get("is_production")), "entry_variant": strategy["entry_variant"], - "exit_policy": strategy["exit_policy"], + "exit_policy": exit_policy, + "live_exit_mode": live_exit_mode, "lookback": lookback["lookback"], "lookback_label": lookback["label"], **sim, @@ -2415,8 +2466,16 @@ async def run_backtest( exit_policy_rows = _exit_policy_sims( candidates, price_columns, spy_closes, hold_horizon ) + live_exit_policy: dict | None = None + try: + from app.services.paper_trade_service import get_exit_policy + + live_exit_policy = await get_exit_policy(db) + except Exception: + logger.exception("Live exit policy load failed; monitor uses defaults") portfolio_monitor_report = _portfolio_monitor( - candidates, price_columns, spy_closes, hold_horizon + candidates, price_columns, spy_closes, hold_horizon, + live_exit_policy=live_exit_policy, ) except Exception: logger.exception("Portfolio simulation failed") diff --git a/app/services/momentum_service.py b/app/services/momentum_service.py index 727a91a..534f637 100644 --- a/app/services/momentum_service.py +++ b/app/services/momentum_service.py @@ -27,6 +27,12 @@ logger = logging.getLogger(__name__) _MOM_LOOKBACK = 252 _MOM_SKIP = 21 +# Promoted production ordering: strategy_rank blends the momentum and realized- +# volatility percentiles. Single source of truth — the backtest's production +# ranking key imports these so live and simulated ordering cannot drift. +STRATEGY_RANK_MOMENTUM_WEIGHT = 0.8 +STRATEGY_RANK_VOL_WEIGHT = 1.0 - STRATEGY_RANK_MOMENTUM_WEIGHT + def compute_12_1_momentum(closes: list[float]) -> float | None: """Return over the window ending ~1 month ago, starting ~12 months ago. @@ -199,7 +205,11 @@ async def compute_activation_ranks(db: AsyncSession) -> dict[str, dict[str, floa momentum_pct = momentum_percentiles.get(sym) vol_pct = vol_percentiles.get(sym) strategy_rank = ( - round(momentum_pct * 0.8 + vol_pct * 0.2, 2) + round( + momentum_pct * STRATEGY_RANK_MOMENTUM_WEIGHT + + vol_pct * STRATEGY_RANK_VOL_WEIGHT, + 2, + ) if momentum_pct is not None and vol_pct is not None else momentum_pct ) diff --git a/tests/unit/test_prod_strategy_parity.py b/tests/unit/test_prod_strategy_parity.py new file mode 100644 index 0000000..7d2e028 --- /dev/null +++ b/tests/unit/test_prod_strategy_parity.py @@ -0,0 +1,88 @@ +"""Parity guards: the backtest's production strategy must equal the live setup. + +The portfolio monitor's production row replays the live qualification flag and +the runtime Admin exit policy, but several constants are still defined on both +sides (defaults, trail width, ordering weights). These tests fail if the two +sides drift, so a change to the live strategy forces the backtest — and vice +versa — to move with it. +""" + +import pytest + +from app.services import paper_trade_service +from app.services.admin_service import ACTIVATION_DEFAULTS +from app.services.backtest_service import ( + ATR_TRAIL_MULTIPLIER, + LIVE_EXIT_MODE_TO_SIM, + PORTFOLIO_MONITOR_STRATEGIES, + PRODUCTION_PERCENTILE_KEY, + RESIDUAL_HIGH_VOL_BLEND_80_20_KEY, + TIME_EXIT_DAYS, + _entry_variant_config, + _momentum_qualifies, + _qualifies_strategy_variant, +) +from app.services.momentum_service import ( + STRATEGY_RANK_MOMENTUM_WEIGHT, + STRATEGY_RANK_VOL_WEIGHT, +) + + +def _production_monitor_row() -> dict: + return next(s for s in PORTFOLIO_MONITOR_STRATEGIES if s.get("is_production")) + + +def test_exit_defaults_match_the_simulated_exit() -> None: + assert paper_trade_service.DEFAULT_EXIT_MODE == "atr_trailing" + assert LIVE_EXIT_MODE_TO_SIM[paper_trade_service.DEFAULT_EXIT_MODE] == "atr_trail3" + assert paper_trade_service.DEFAULT_ATR_MULTIPLIER == ATR_TRAIL_MULTIPLIER + assert paper_trade_service.DEFAULT_HOLD_DAYS == max(TIME_EXIT_DAYS) + + +def test_every_live_exit_mode_has_a_sim_mapping() -> None: + assert set(paper_trade_service._VALID_EXIT_MODES) == set(LIVE_EXIT_MODE_TO_SIM) + + +def test_gate_default_matches_the_promoted_cutoff() -> None: + prod = _production_monitor_row() + entry_cfg = _entry_variant_config(str(prod["entry_variant"])) + assert entry_cfg is not None + assert float(entry_cfg["cutoff"]) == float(ACTIVATION_DEFAULTS["min_momentum_percentile"]) + + +def test_production_ordering_weights_are_single_sourced() -> None: + # The promoted ordering is 80/20 momentum/vol; the backtest imports the + # weight, so equality here pins the *value* the promotion was validated at. + assert STRATEGY_RANK_MOMENTUM_WEIGHT == 0.8 + assert STRATEGY_RANK_VOL_WEIGHT == pytest.approx(0.2) + prod = _production_monitor_row() + entry_cfg = _entry_variant_config(str(prod["entry_variant"])) + assert entry_cfg is not None + assert entry_cfg["ranking_key"] == RESIDUAL_HIGH_VOL_BLEND_80_20_KEY + + +def test_production_monitor_row_replays_the_live_config() -> None: + prod = _production_monitor_row() + assert prod.get("use_live_config") is True + assert prod["exit_policy"] == "atr_trail3" + + +def test_live_gate_equals_the_production_variant_gate() -> None: + """The monitor's live-gate switch relies on the runtime `qualified` flag + (_momentum_qualifies) selecting exactly what the frozen production variant + gate selects at the default cutoff.""" + prod = _production_monitor_row() + entry_cfg = _entry_variant_config(str(prod["entry_variant"])) + assert entry_cfg is not None + cutoff = float(ACTIVATION_DEFAULTS["min_momentum_percentile"]) + for cand in ( + {"meets_core": True, "direction": "long", PRODUCTION_PERCENTILE_KEY: 92.0}, + {"meets_core": True, "direction": "long", PRODUCTION_PERCENTILE_KEY: 80.0}, + {"meets_core": True, "direction": "long", PRODUCTION_PERCENTILE_KEY: 79.9}, + {"meets_core": True, "direction": "long", PRODUCTION_PERCENTILE_KEY: None}, + {"meets_core": True, "direction": "short", PRODUCTION_PERCENTILE_KEY: 95.0}, + {"meets_core": False, "direction": "long", PRODUCTION_PERCENTILE_KEY: 95.0}, + ): + assert _momentum_qualifies(cand, cutoff) == _qualifies_strategy_variant( + cand, entry_cfg + ), cand