Align backtest production sim with live runtime config
The portfolio monitor's Production row now replays the live qualification flag and the Admin exit policy (mode/ATR multiplier/hold days) instead of a frozen research-variant gate, so Admin tuning is reflected in the next run. Single-source the 80/20 strategy_rank weights in momentum_service and pin every dual-defined constant with a parity test. Behavior-preserving today: the production sim reproduces the README baseline exactly. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -83,6 +83,8 @@ The conclusion is not "trade high volatility alone." Keep residual momentum as t
|
|||||||
|
|
||||||
Live-ranking note: the backtest ranks residual momentum and volatility inside each weekly setup-candidate cross-section. The live scanner computes the same 80/20 formula across the current ticker universe before scanning so every generated setup carries a stable ticker-level rank. That is the production approximation; reconcile it later only if candidate-only post-scan ranking proves materially different.
|
Live-ranking note: the backtest ranks residual momentum and volatility inside each weekly setup-candidate cross-section. The live scanner computes the same 80/20 formula across the current ticker universe before scanning so every generated setup carries a stable ticker-level rank. That is the production approximation; reconcile it later only if candidate-only post-scan ranking proves materially different.
|
||||||
|
|
||||||
|
Parity guard (July 2026): the portfolio monitor's **Production** row replays the *runtime* configuration — the live activation gate (`qualified` flag) and the Admin exit policy (mode / ATR multiplier / hold days) — so tuning the strategy in Admin is reflected in the next backtest run instead of silently diverging. Constants defined on both sides (exit defaults, trail width, the 80/20 ordering weights, the promoted cutoff) are pinned by `tests/unit/test_prod_strategy_parity.py`, and the ordering weights are single-sourced from `momentum_service`.
|
||||||
|
|
||||||
### The iron rule for strategy changes
|
### The iron rule for strategy changes
|
||||||
|
|
||||||
A signal earns its way into selection **only** through the factor harness:
|
A signal earns its way into selection **only** through the factor harness:
|
||||||
|
|||||||
@@ -44,7 +44,10 @@ from app.models.ticker import Ticker
|
|||||||
from app.services import settings_store
|
from app.services import settings_store
|
||||||
from app.services.admin_service import get_activation_config, update_setting
|
from app.services.admin_service import get_activation_config, update_setting
|
||||||
from app.services.indicator_service import _extract_ohlcv, compute_atr
|
from app.services.indicator_service import _extract_ohlcv, compute_atr
|
||||||
from app.services.momentum_service import compute_realized_vol_6m
|
from app.services.momentum_service import (
|
||||||
|
STRATEGY_RANK_MOMENTUM_WEIGHT,
|
||||||
|
compute_realized_vol_6m,
|
||||||
|
)
|
||||||
from app.services.outcome_service import (
|
from app.services.outcome_service import (
|
||||||
OUTCOME_AMBIGUOUS,
|
OUTCOME_AMBIGUOUS,
|
||||||
OUTCOME_STOP_HIT,
|
OUTCOME_STOP_HIT,
|
||||||
@@ -941,7 +944,9 @@ def _assign_residual_high_vol_blend(candidates: list[dict]) -> None:
|
|||||||
"""Research ranks: residual momentum blended with higher-vol preference."""
|
"""Research ranks: residual momentum blended with higher-vol preference."""
|
||||||
for output_key, residual_weight in (
|
for output_key, residual_weight in (
|
||||||
(RESIDUAL_HIGH_VOL_BLEND_90_10_KEY, 0.9),
|
(RESIDUAL_HIGH_VOL_BLEND_90_10_KEY, 0.9),
|
||||||
(RESIDUAL_HIGH_VOL_BLEND_80_20_KEY, 0.8),
|
# The production ordering weight comes from momentum_service so the
|
||||||
|
# simulated production rank cannot drift from the live strategy_rank.
|
||||||
|
(RESIDUAL_HIGH_VOL_BLEND_80_20_KEY, STRATEGY_RANK_MOMENTUM_WEIGHT),
|
||||||
(RESIDUAL_HIGH_VOL_BLEND_KEY, 0.7),
|
(RESIDUAL_HIGH_VOL_BLEND_KEY, 0.7),
|
||||||
(RESIDUAL_HIGH_VOL_BLEND_60_40_KEY, 0.6),
|
(RESIDUAL_HIGH_VOL_BLEND_60_40_KEY, 0.6),
|
||||||
):
|
):
|
||||||
@@ -1061,6 +1066,20 @@ SIM_STARTING_CAPITAL = 10_000.0
|
|||||||
SIM_MAX_POSITIONS = 10
|
SIM_MAX_POSITIONS = 10
|
||||||
SIM_RISK_PER_TRADE = 0.01 # fraction of equity risked per position (entry→stop)
|
SIM_RISK_PER_TRADE = 0.01 # fraction of equity risked per position (entry→stop)
|
||||||
SIM_NOTIONAL_CAP = 0.20 # max fraction of equity per position (no margin)
|
SIM_NOTIONAL_CAP = 0.20 # max fraction of equity per position (no margin)
|
||||||
|
# The "atr_trail3" research policy's trail width. Must equal the live default
|
||||||
|
# (paper_trade_service.DEFAULT_ATR_MULTIPLIER) — enforced by the parity test.
|
||||||
|
# The production portfolio-monitor row additionally follows the *runtime* Admin
|
||||||
|
# exit policy, so tuning it live is reflected in the next backtest run.
|
||||||
|
ATR_TRAIL_MULTIPLIER = 3.0
|
||||||
|
# How live Admin exit modes map onto simulator exit policies. "trailing"
|
||||||
|
# (percent trail) has no simulator counterpart and falls back to the plain
|
||||||
|
# hold-to-horizon book; the row's live_exit_mode field keeps that visible.
|
||||||
|
LIVE_EXIT_MODE_TO_SIM = {
|
||||||
|
"atr_trailing": "atr_trail3",
|
||||||
|
"time": "hold",
|
||||||
|
"target": "target",
|
||||||
|
"trailing": "hold",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def _simulate_portfolio(
|
def _simulate_portfolio(
|
||||||
@@ -1074,6 +1093,7 @@ def _simulate_portfolio(
|
|||||||
ranking_key: str = PRODUCTION_PERCENTILE_KEY,
|
ranking_key: str = PRODUCTION_PERCENTILE_KEY,
|
||||||
max_positions: int = SIM_MAX_POSITIONS,
|
max_positions: int = SIM_MAX_POSITIONS,
|
||||||
risk_per_trade: float = SIM_RISK_PER_TRADE,
|
risk_per_trade: float = SIM_RISK_PER_TRADE,
|
||||||
|
atr_trail_multiplier: float = ATR_TRAIL_MULTIPLIER,
|
||||||
start_date: date | None = None,
|
start_date: date | None = None,
|
||||||
include_curve: bool = False,
|
include_curve: bool = False,
|
||||||
) -> dict | None:
|
) -> dict | None:
|
||||||
@@ -1249,7 +1269,7 @@ def _simulate_portfolio(
|
|||||||
pos["highest_close"] = max(pos["highest_close"], bar.close)
|
pos["highest_close"] = max(pos["highest_close"], bar.close)
|
||||||
atr = _atr(sym, bar.idx)
|
atr = _atr(sym, bar.idx)
|
||||||
if atr is not None:
|
if atr is not None:
|
||||||
next_stop = pos["highest_close"] - 3.0 * atr
|
next_stop = pos["highest_close"] - atr_trail_multiplier * atr
|
||||||
if next_stop < bar.close:
|
if next_stop < bar.close:
|
||||||
pos["stop"] = max(pos["stop"], next_stop)
|
pos["stop"] = max(pos["stop"], next_stop)
|
||||||
|
|
||||||
@@ -1756,9 +1776,16 @@ PORTFOLIO_MONITOR_STRATEGIES: tuple[dict, ...] = (
|
|||||||
{
|
{
|
||||||
"strategy": PRODUCTION_PORTFOLIO_STRATEGY,
|
"strategy": PRODUCTION_PORTFOLIO_STRATEGY,
|
||||||
"label": "Production: residual/high-vol 80/20 + 3x ATR trail",
|
"label": "Production: residual/high-vol 80/20 + 3x ATR trail",
|
||||||
"description": "Residual gate, 80/20 residual/high-vol rank, 3x ATR trailing stop.",
|
"description": (
|
||||||
|
"The live strategy: production activation gate and Admin exit policy "
|
||||||
|
"as currently configured, 80/20 residual/high-vol rank."
|
||||||
|
),
|
||||||
"entry_variant": "residual80_highvol_blend80_20_fixed10",
|
"entry_variant": "residual80_highvol_blend80_20_fixed10",
|
||||||
"exit_policy": "atr_trail3",
|
"exit_policy": "atr_trail3",
|
||||||
|
# The production row replays what the platform actually does right now:
|
||||||
|
# the live qualification flag (runtime Admin activation settings) and the
|
||||||
|
# live Admin exit policy, instead of the frozen research-variant gate.
|
||||||
|
"use_live_config": True,
|
||||||
"is_production": True,
|
"is_production": True,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
@@ -1817,6 +1844,7 @@ def _portfolio_monitor(
|
|||||||
prices: dict[str, tuple],
|
prices: dict[str, tuple],
|
||||||
_spy_closes: dict[date, float] | None,
|
_spy_closes: dict[date, float] | None,
|
||||||
hold_days: int,
|
hold_days: int,
|
||||||
|
live_exit_policy: dict | None = None,
|
||||||
) -> dict:
|
) -> dict:
|
||||||
latest_ord = max((max(cols[0]) for cols in prices.values() if cols[0]), default=None)
|
latest_ord = max((max(cols[0]) for cols in prices.values() if cols[0]), default=None)
|
||||||
rows: list[dict] = []
|
rows: list[dict] = []
|
||||||
@@ -1825,18 +1853,40 @@ def _portfolio_monitor(
|
|||||||
if entry_cfg is None:
|
if entry_cfg is None:
|
||||||
continue
|
continue
|
||||||
ranking_key = str(entry_cfg.get("ranking_key") or entry_cfg["percentile_key"])
|
ranking_key = str(entry_cfg.get("ranking_key") or entry_cfg["percentile_key"])
|
||||||
|
# The production row must replay the LIVE configuration: the runtime
|
||||||
|
# qualification flag (Admin activation settings) instead of the frozen
|
||||||
|
# research-variant gate, and the Admin exit policy instead of the
|
||||||
|
# hardcoded 3x-trail/30d defaults. Research rows stay frozen so they
|
||||||
|
# remain comparable across runs.
|
||||||
|
use_live = bool(strategy.get("use_live_config"))
|
||||||
|
exit_policy = str(strategy["exit_policy"])
|
||||||
|
row_hold_days = hold_days
|
||||||
|
trail_multiplier = ATR_TRAIL_MULTIPLIER
|
||||||
|
live_exit_mode: str | None = None
|
||||||
|
if use_live and live_exit_policy is not None:
|
||||||
|
live_exit_mode = str(live_exit_policy.get("mode", "atr_trailing"))
|
||||||
|
exit_policy = LIVE_EXIT_MODE_TO_SIM.get(live_exit_mode, "atr_trail3")
|
||||||
|
row_hold_days = int(live_exit_policy.get("hold_days", hold_days))
|
||||||
|
trail_multiplier = float(
|
||||||
|
live_exit_policy.get("atr_multiplier", ATR_TRAIL_MULTIPLIER)
|
||||||
|
)
|
||||||
|
qualified_fn = (
|
||||||
|
None if use_live
|
||||||
|
else lambda c, config=entry_cfg: _qualifies_strategy_variant(c, config)
|
||||||
|
)
|
||||||
for lookback in PORTFOLIO_MONITOR_LOOKBACKS:
|
for lookback in PORTFOLIO_MONITOR_LOOKBACKS:
|
||||||
start = _lookback_start(latest_ord, lookback["days"])
|
start = _lookback_start(latest_ord, lookback["days"])
|
||||||
sim = _simulate_portfolio(
|
sim = _simulate_portfolio(
|
||||||
candidates,
|
candidates,
|
||||||
prices,
|
prices,
|
||||||
_spy_closes,
|
_spy_closes,
|
||||||
str(strategy["exit_policy"]),
|
exit_policy,
|
||||||
hold_days,
|
row_hold_days,
|
||||||
qualified_fn=lambda c, config=entry_cfg: _qualifies_strategy_variant(c, config),
|
qualified_fn=qualified_fn,
|
||||||
ranking_key=ranking_key,
|
ranking_key=ranking_key,
|
||||||
max_positions=int(entry_cfg["max_positions"]),
|
max_positions=int(entry_cfg["max_positions"]),
|
||||||
risk_per_trade=float(entry_cfg["risk_per_trade"]),
|
risk_per_trade=float(entry_cfg["risk_per_trade"]),
|
||||||
|
atr_trail_multiplier=trail_multiplier,
|
||||||
start_date=start,
|
start_date=start,
|
||||||
include_curve=True,
|
include_curve=True,
|
||||||
)
|
)
|
||||||
@@ -1848,7 +1898,8 @@ def _portfolio_monitor(
|
|||||||
"description": strategy["description"],
|
"description": strategy["description"],
|
||||||
"is_production": bool(strategy.get("is_production")),
|
"is_production": bool(strategy.get("is_production")),
|
||||||
"entry_variant": strategy["entry_variant"],
|
"entry_variant": strategy["entry_variant"],
|
||||||
"exit_policy": strategy["exit_policy"],
|
"exit_policy": exit_policy,
|
||||||
|
"live_exit_mode": live_exit_mode,
|
||||||
"lookback": lookback["lookback"],
|
"lookback": lookback["lookback"],
|
||||||
"lookback_label": lookback["label"],
|
"lookback_label": lookback["label"],
|
||||||
**sim,
|
**sim,
|
||||||
@@ -2415,8 +2466,16 @@ async def run_backtest(
|
|||||||
exit_policy_rows = _exit_policy_sims(
|
exit_policy_rows = _exit_policy_sims(
|
||||||
candidates, price_columns, spy_closes, hold_horizon
|
candidates, price_columns, spy_closes, hold_horizon
|
||||||
)
|
)
|
||||||
|
live_exit_policy: dict | None = None
|
||||||
|
try:
|
||||||
|
from app.services.paper_trade_service import get_exit_policy
|
||||||
|
|
||||||
|
live_exit_policy = await get_exit_policy(db)
|
||||||
|
except Exception:
|
||||||
|
logger.exception("Live exit policy load failed; monitor uses defaults")
|
||||||
portfolio_monitor_report = _portfolio_monitor(
|
portfolio_monitor_report = _portfolio_monitor(
|
||||||
candidates, price_columns, spy_closes, hold_horizon
|
candidates, price_columns, spy_closes, hold_horizon,
|
||||||
|
live_exit_policy=live_exit_policy,
|
||||||
)
|
)
|
||||||
except Exception:
|
except Exception:
|
||||||
logger.exception("Portfolio simulation failed")
|
logger.exception("Portfolio simulation failed")
|
||||||
|
|||||||
@@ -27,6 +27,12 @@ logger = logging.getLogger(__name__)
|
|||||||
_MOM_LOOKBACK = 252
|
_MOM_LOOKBACK = 252
|
||||||
_MOM_SKIP = 21
|
_MOM_SKIP = 21
|
||||||
|
|
||||||
|
# Promoted production ordering: strategy_rank blends the momentum and realized-
|
||||||
|
# volatility percentiles. Single source of truth — the backtest's production
|
||||||
|
# ranking key imports these so live and simulated ordering cannot drift.
|
||||||
|
STRATEGY_RANK_MOMENTUM_WEIGHT = 0.8
|
||||||
|
STRATEGY_RANK_VOL_WEIGHT = 1.0 - STRATEGY_RANK_MOMENTUM_WEIGHT
|
||||||
|
|
||||||
|
|
||||||
def compute_12_1_momentum(closes: list[float]) -> float | None:
|
def compute_12_1_momentum(closes: list[float]) -> float | None:
|
||||||
"""Return over the window ending ~1 month ago, starting ~12 months ago.
|
"""Return over the window ending ~1 month ago, starting ~12 months ago.
|
||||||
@@ -199,7 +205,11 @@ async def compute_activation_ranks(db: AsyncSession) -> dict[str, dict[str, floa
|
|||||||
momentum_pct = momentum_percentiles.get(sym)
|
momentum_pct = momentum_percentiles.get(sym)
|
||||||
vol_pct = vol_percentiles.get(sym)
|
vol_pct = vol_percentiles.get(sym)
|
||||||
strategy_rank = (
|
strategy_rank = (
|
||||||
round(momentum_pct * 0.8 + vol_pct * 0.2, 2)
|
round(
|
||||||
|
momentum_pct * STRATEGY_RANK_MOMENTUM_WEIGHT
|
||||||
|
+ vol_pct * STRATEGY_RANK_VOL_WEIGHT,
|
||||||
|
2,
|
||||||
|
)
|
||||||
if momentum_pct is not None and vol_pct is not None
|
if momentum_pct is not None and vol_pct is not None
|
||||||
else momentum_pct
|
else momentum_pct
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -0,0 +1,88 @@
|
|||||||
|
"""Parity guards: the backtest's production strategy must equal the live setup.
|
||||||
|
|
||||||
|
The portfolio monitor's production row replays the live qualification flag and
|
||||||
|
the runtime Admin exit policy, but several constants are still defined on both
|
||||||
|
sides (defaults, trail width, ordering weights). These tests fail if the two
|
||||||
|
sides drift, so a change to the live strategy forces the backtest — and vice
|
||||||
|
versa — to move with it.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from app.services import paper_trade_service
|
||||||
|
from app.services.admin_service import ACTIVATION_DEFAULTS
|
||||||
|
from app.services.backtest_service import (
|
||||||
|
ATR_TRAIL_MULTIPLIER,
|
||||||
|
LIVE_EXIT_MODE_TO_SIM,
|
||||||
|
PORTFOLIO_MONITOR_STRATEGIES,
|
||||||
|
PRODUCTION_PERCENTILE_KEY,
|
||||||
|
RESIDUAL_HIGH_VOL_BLEND_80_20_KEY,
|
||||||
|
TIME_EXIT_DAYS,
|
||||||
|
_entry_variant_config,
|
||||||
|
_momentum_qualifies,
|
||||||
|
_qualifies_strategy_variant,
|
||||||
|
)
|
||||||
|
from app.services.momentum_service import (
|
||||||
|
STRATEGY_RANK_MOMENTUM_WEIGHT,
|
||||||
|
STRATEGY_RANK_VOL_WEIGHT,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _production_monitor_row() -> dict:
|
||||||
|
return next(s for s in PORTFOLIO_MONITOR_STRATEGIES if s.get("is_production"))
|
||||||
|
|
||||||
|
|
||||||
|
def test_exit_defaults_match_the_simulated_exit() -> None:
|
||||||
|
assert paper_trade_service.DEFAULT_EXIT_MODE == "atr_trailing"
|
||||||
|
assert LIVE_EXIT_MODE_TO_SIM[paper_trade_service.DEFAULT_EXIT_MODE] == "atr_trail3"
|
||||||
|
assert paper_trade_service.DEFAULT_ATR_MULTIPLIER == ATR_TRAIL_MULTIPLIER
|
||||||
|
assert paper_trade_service.DEFAULT_HOLD_DAYS == max(TIME_EXIT_DAYS)
|
||||||
|
|
||||||
|
|
||||||
|
def test_every_live_exit_mode_has_a_sim_mapping() -> None:
|
||||||
|
assert set(paper_trade_service._VALID_EXIT_MODES) == set(LIVE_EXIT_MODE_TO_SIM)
|
||||||
|
|
||||||
|
|
||||||
|
def test_gate_default_matches_the_promoted_cutoff() -> None:
|
||||||
|
prod = _production_monitor_row()
|
||||||
|
entry_cfg = _entry_variant_config(str(prod["entry_variant"]))
|
||||||
|
assert entry_cfg is not None
|
||||||
|
assert float(entry_cfg["cutoff"]) == float(ACTIVATION_DEFAULTS["min_momentum_percentile"])
|
||||||
|
|
||||||
|
|
||||||
|
def test_production_ordering_weights_are_single_sourced() -> None:
|
||||||
|
# The promoted ordering is 80/20 momentum/vol; the backtest imports the
|
||||||
|
# weight, so equality here pins the *value* the promotion was validated at.
|
||||||
|
assert STRATEGY_RANK_MOMENTUM_WEIGHT == 0.8
|
||||||
|
assert STRATEGY_RANK_VOL_WEIGHT == pytest.approx(0.2)
|
||||||
|
prod = _production_monitor_row()
|
||||||
|
entry_cfg = _entry_variant_config(str(prod["entry_variant"]))
|
||||||
|
assert entry_cfg is not None
|
||||||
|
assert entry_cfg["ranking_key"] == RESIDUAL_HIGH_VOL_BLEND_80_20_KEY
|
||||||
|
|
||||||
|
|
||||||
|
def test_production_monitor_row_replays_the_live_config() -> None:
|
||||||
|
prod = _production_monitor_row()
|
||||||
|
assert prod.get("use_live_config") is True
|
||||||
|
assert prod["exit_policy"] == "atr_trail3"
|
||||||
|
|
||||||
|
|
||||||
|
def test_live_gate_equals_the_production_variant_gate() -> None:
|
||||||
|
"""The monitor's live-gate switch relies on the runtime `qualified` flag
|
||||||
|
(_momentum_qualifies) selecting exactly what the frozen production variant
|
||||||
|
gate selects at the default cutoff."""
|
||||||
|
prod = _production_monitor_row()
|
||||||
|
entry_cfg = _entry_variant_config(str(prod["entry_variant"]))
|
||||||
|
assert entry_cfg is not None
|
||||||
|
cutoff = float(ACTIVATION_DEFAULTS["min_momentum_percentile"])
|
||||||
|
for cand in (
|
||||||
|
{"meets_core": True, "direction": "long", PRODUCTION_PERCENTILE_KEY: 92.0},
|
||||||
|
{"meets_core": True, "direction": "long", PRODUCTION_PERCENTILE_KEY: 80.0},
|
||||||
|
{"meets_core": True, "direction": "long", PRODUCTION_PERCENTILE_KEY: 79.9},
|
||||||
|
{"meets_core": True, "direction": "long", PRODUCTION_PERCENTILE_KEY: None},
|
||||||
|
{"meets_core": True, "direction": "short", PRODUCTION_PERCENTILE_KEY: 95.0},
|
||||||
|
{"meets_core": False, "direction": "long", PRODUCTION_PERCENTILE_KEY: 95.0},
|
||||||
|
):
|
||||||
|
assert _momentum_qualifies(cand, cutoff) == _qualifies_strategy_variant(
|
||||||
|
cand, entry_cfg
|
||||||
|
), cand
|
||||||
Reference in New Issue
Block a user