dennisthiessen/signal-platform
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(backtest): address the second review of the breakeven and book-config work
Deploy / lint (push) Successful in 14s
Details
Deploy / test (push) Successful in 2m0s
Details
Deploy / deploy (push) Successful in 48s
Details

Browse Source 
- The breakeven study grades at the shadow book's capacity AND risk per
  trade, and labels each with where it came from. A missing setting row
  now reads "shadow_book default" instead of crediting the setting; the
  committed report said shadow_book_capacity though the snapshot has no
  such row.
- _production_book_config reads strategies through the same
  _portfolio_monitor_strategies() hook as the monitor.
- breakeven_gate_reset is rejected when there is no post-stop episode or
  cooldown for it to start, and the study skips its diagnostic arm for a
  book without one, so a production config change cannot abort the run.
- identical_to_control is renamed ties_control_on_graded_metrics and needs
  real values: be_2r ties on every graded metric but still changed 2 trades.
- _BookConfig drops an unused field and hashes by identity; the min-R:R
  sweep swaps its gate in with dataclasses.replace instead of passing it.
- New tests cover the parity guard's success path, the no-config fallback
  and the gate-reset check.

The breakeven report is regenerated: every row, diagnostic row and the
parity check are identical; only the labels, the risk fields and the
renamed flag changed.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dennisthiessenandClaude Opus 5.5 committed 2026-10-10 21:41:44 +02:00
1 parent 7a9b3ad38b
commit 231372fb3a
5 files changed
+168 -88
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

app/services/backtest_service.py
+79 -49
View File
@@ -51,7 +51,7 @@ import statistics
from collections import defaultdict
from collections.abc import Callable
from concurrent.futures import ProcessPoolExecutor
from dataclasses import dataclass, field
from dataclasses import dataclass, field, replace
from datetime import date, datetime, timezone
from types import SimpleNamespace
from typing import Any
@@ -2365,6 +2365,10 @@ def _simulate_portfolio(
raise ValueError("breakeven_r only applies to the ATR-trail exit policies")
if breakeven_gate_reset and breakeven_r is None:
raise ValueError("breakeven_gate_reset needs breakeven_r")
if breakeven_gate_reset and post_stop_reentry_fn is None and reentry_cooldown_sessions <= 0:
raise ValueError(
"breakeven_gate_reset needs post_stop_reentry_fn or reentry_cooldown_sessions"
)
clamp_lo, clamp_hi = float(vol_clamp[0]), float(vol_clamp[1])
if clamp_lo <= 0 or clamp_hi < clamp_lo:
raise ValueError("vol_clamp must satisfy 0 < lo <= hi")
@@ -3535,13 +3539,13 @@ def _entry_variant_config(variant: str) -> dict | None:
return next((cfg for cfg in STRATEGY_VARIANTS if cfg["variant"] == variant), None)
@dataclass(frozen=True)
# eq=False keeps identity hashing: the fields hold dicts and callables.
@dataclass(frozen=True, eq=False)
class _BookConfig:
"""How one ``PORTFOLIO_MONITOR_STRATEGIES`` entry is replayed. The monitor and
every production-book study resolve it here, so they cannot drift apart."""
strategy: dict
entry_cfg: dict
exit_policy: str
live_exit_mode: str | None
hold_days: int
@@ -3557,17 +3561,16 @@ class _BookConfig:
candidates: list[dict],
prices: dict[str, tuple],
cadence: str,
qualified_fn: Callable[[dict], bool] | None,
) -> Callable[[str, int, dict, Any], dict | None] | None:
"""The gate-reset callback for this book, built on ``qualified_fn`` (a
sweep passes its own gate), or None for immediate re-entry."""
"""The gate-reset callback on this book's own gate, or None for immediate
re-entry. A sweep that changes the gate replaces ``qualified_fn`` first."""
if self.reentry_policy != "gate_reset":
return None
return _make_gate_reset_reentry_fn(
candidates,
prices,
cadence=cadence,
qualified_fn=qualified_fn,
qualified_fn=self.qualified_fn,
ranking_key=self.ranking_key,
)
@@ -3594,7 +3597,6 @@ def _strategy_book_config(
)
return _BookConfig(
strategy=strategy,
entry_cfg=entry_cfg,
exit_policy=exit_policy,
live_exit_mode=live_exit_mode,
hold_days=row_hold_days,
@@ -3618,7 +3620,7 @@ def _production_book_config(
hold_days: int, live_exit_policy: dict | None
) -> _BookConfig | None:
strategy = next(
(s for s in PORTFOLIO_MONITOR_STRATEGIES if s.get("is_production")), None
(s for s in _portfolio_monitor_strategies() if s.get("is_production")), None
)
if strategy is None:
return None
@@ -3758,9 +3760,9 @@ def _min_rr_sweep(
max_positions=book.max_positions,
risk_per_trade=book.risk_per_trade,
atr_trail_multiplier=book.trail_multiplier,
post_stop_reentry_fn=book.post_stop_reentry_fn(
candidates, prices, cadence, qualified_fn
),
post_stop_reentry_fn=replace(
book, qualified_fn=qualified_fn
).post_stop_reentry_fn(candidates, prices, cadence),
start_date=sweep_start,
)
if sim is None:
@@ -3832,9 +3834,7 @@ def _holdout_evaluation(
book = _production_book_config(hold_days, live_exit_policy)
if book is None:
return {}
post_stop_reentry_fn = book.post_stop_reentry_fn(
candidates, prices, cadence, book.qualified_fn
)
post_stop_reentry_fn = book.post_stop_reentry_fn(candidates, prices, cadence)
rows: list[dict] = []
for window, start, end in (
@@ -3875,7 +3875,7 @@ def _holdout_evaluation(
# Pre-registered 2026-10-09 — docs/research/breakeven-stop.md. Two trigger levels,
# no sweep; graded on both halves of a fixed split at the live book capacity.
# no sweep; graded on both halves of a fixed split at the live book's capacity and risk.
BREAKEVEN_STUDY_ARMS: tuple[tuple[str, float | None], ...] = (
("control", None),
("be_1r", 1.0),
@@ -3918,15 +3918,16 @@ def _breakeven_exit_study(
live_exit_policy: dict | None = None,
cadence: str = DEFAULT_BACKTEST_CADENCE,
monitor_report: dict | None = None,
live_capacity: int | None = None,
live_book: dict | None = None,
) -> dict:
"""Production book with a breakeven stop added to the ATR trail, vs control.
Mirrors ``_holdout_evaluation``'s production configuration and changes only
``breakeven_r`` and, for every graded row, capacity: ``live_capacity`` (the
shadow book's setting) when the caller could read it, else
``SIM_MAX_POSITIONS``, rather than the monitor's 10. A capacity-10 control on
full history must reproduce the monitor's production row, or the study is not
``breakeven_r`` and, for every graded row, the book's size: capacity and risk
per trade from ``live_book`` (the shadow book's config, with where each value
came from) when the caller could read it, else ``SIM_MAX_POSITIONS`` and the
entry variant's risk. A control at the monitor's capacity and risk on full
history must reproduce the monitor's production row, or the study is not
grading the production book.
"""
book = _production_book_config(hold_days, live_exit_policy)
@@ -3936,13 +3937,12 @@ def _breakeven_exit_study(
return {
"error": f"live exit maps to {book.exit_policy!r}; the study needs atr_trail3"
}
post_stop_reentry_fn = book.post_stop_reentry_fn(
candidates, prices, cadence, book.qualified_fn
)
post_stop_reentry_fn = book.post_stop_reentry_fn(candidates, prices, cadence)
def run(
breakeven_r: float | None,
max_positions: int,
risk_per_trade: float,
start: date | None = None,
end: date | None = None,
gate_reset: bool = False,
@@ -3956,7 +3956,7 @@ def _breakeven_exit_study(
qualified_fn=book.qualified_fn,
ranking_key=book.ranking_key,
max_positions=max_positions,
risk_per_trade=book.risk_per_trade,
risk_per_trade=risk_per_trade,
atr_trail_multiplier=book.trail_multiplier,
post_stop_reentry_fn=post_stop_reentry_fn,
start_date=start,
@@ -3973,23 +3973,31 @@ def _breakeven_exit_study(
sim["trades_5r_plus"] = sum(1 for r in rs if r >= 5.0)
return sim
capacity = int(live_capacity) if live_capacity is not None else SIM_MAX_POSITIONS
if live_book is not None:
capacity, risk = int(live_book["capacity"]), float(live_book["risk_per_trade"])
capacity_source, risk_source = live_book["capacity_source"], live_book["risk_source"]
else:
capacity, risk = SIM_MAX_POSITIONS, book.risk_per_trade
capacity_source, risk_source = "SIM_MAX_POSITIONS", "entry variant"
split = BREAKEVEN_STUDY_SPLIT
windows = (("full", None, None), ("train", None, split), ("test", split, None))
rows: list[dict] = []
for arm, breakeven_r in BREAKEVEN_STUDY_ARMS:
for window, start, end in windows:
sim = run(breakeven_r, capacity, start, end)
sim = run(breakeven_r, capacity, risk, start, end)
if sim is not None:
rows.append({"arm": arm, "window": window, **sim})
diagnostic_rows: list[dict] = []
for arm, breakeven_r in BREAKEVEN_STUDY_DIAGNOSTIC_ARMS:
# Gating needs a post-stop episode to start; a book without one has nothing
# for the diagnostic to measure.
diagnostic_arms = BREAKEVEN_STUDY_DIAGNOSTIC_ARMS if post_stop_reentry_fn else ()
for arm, breakeven_r in diagnostic_arms:
for window, start, end in windows:
sim = run(breakeven_r, capacity, start, end, gate_reset=True)
sim = run(breakeven_r, capacity, risk, start, end, gate_reset=True)
if sim is not None:
diagnostic_rows.append({"arm": arm, "window": window, **sim})
parity_row = run(None, book.max_positions)
parity_row = run(None, book.max_positions, book.risk_per_trade)
monitor_row = next(
(
r for r in (monitor_report or {}).get("runs") or []
@@ -4006,12 +4014,15 @@ def _breakeven_exit_study(
def row(arm: str, window: str) -> dict | None:
return next((r for r in rows if r["arm"] == arm and r["window"] == window), None)
def same_as_control(arm: str, window: str) -> bool:
def ties_control(arm: str, window: str) -> bool:
treated, control = row(arm, window), row("control", window)
return (
treated is not None
and control is not None
and all(treated.get(k) == control.get(k) for k in _BREAKEVEN_GRADED_KEYS)
and all(
treated.get(k) is not None and treated.get(k) == control.get(k)
for k in _BREAKEVEN_GRADED_KEYS
)
)
verdicts: dict[str, dict] = {}
@@ -4028,9 +4039,10 @@ def _breakeven_exit_study(
all(verdict[w].values()) for w in ("train", "test")
)
# The frozen rule passes ties. An arm that matches control on every
# graded metric in both windows is the control exit, not a finding.
verdict["identical_to_control"] = all(
same_as_control(arm, w) for w in ("train", "test")
# graded metric in both windows has found nothing, even if a few of its
# trades differ.
verdict["ties_control_on_graded_metrics"] = all(
ties_control(arm, w) for w in ("train", "test")
)
verdicts[arm] = verdict
@@ -4038,9 +4050,9 @@ def _breakeven_exit_study(
"protocol": "docs/research/breakeven-stop.md",
"split_date": split.isoformat(),
"capacity": capacity,
"capacity_source": (
"shadow_book_capacity" if live_capacity is not None else "SIM_MAX_POSITIONS"
),
"capacity_source": capacity_source,
"risk_per_trade": risk,
"risk_source": risk_source,
"cadence": cadence,
"exit_policy": book.exit_policy,
"atr_trail_multiplier": book.trail_multiplier,
@@ -4049,6 +4061,7 @@ def _breakeven_exit_study(
"parity": {
"ok": parity_ok,
"capacity": book.max_positions,
"risk_per_trade": book.risk_per_trade,
"study_control": (
{k: parity_row.get(k) for k in _BREAKEVEN_PARITY_KEYS} if parity_row else None
),
@@ -4062,7 +4075,7 @@ def _breakeven_exit_study(
"note": (
"Graded on train and test only: an arm passes when Sharpe and CAGR are "
"no lower and max drawdown no higher than control in both windows; "
"identical_to_control marks a pass that is only ties. Win rate and avg R "
"ties_control_on_graded_metrics marks a pass that is only ties. Win rate and avg R "
"are reported, not graded. diagnostic_rows were added after the first run "
"and are never graded. A pass earns forward paper validation, not a "
"production change."
@@ -4085,9 +4098,7 @@ def _portfolio_monitor(
book = _strategy_book_config(strategy, hold_days, live_exit_policy)
if book is None:
continue
post_stop_reentry_fn = book.post_stop_reentry_fn(
candidates, prices, cadence, book.qualified_fn
)
post_stop_reentry_fn = book.post_stop_reentry_fn(candidates, prices, cadence)
for lookback in PORTFOLIO_MONITOR_LOOKBACKS:
start = _lookback_start(latest_ord, lookback["days"])
sim = _simulate_portfolio(
@@ -4805,23 +4816,42 @@ async def run_backtest(
hold_horizon, live_exit_policy=live_exit_policy, cadence=cadence,
)
if _breakeven_study_enabled():
live_capacity: int | None = None
live_book: dict | None = None
try:
from app.services.shadow_book_service import get_config
from app.services import shadow_book_service as shadow_book
live_capacity = int((await get_config(db))["capacity"])
shadow_config = await shadow_book.get_config(db)
stored = await settings_store.get_map(
db, [shadow_book.KEY_CAPACITY, shadow_book.KEY_RISK_PCT]
)
def _source(key: str) -> str:
# get_config falls back to its default for a missing or
# unreadable row; say so rather than credit the setting.
try:
float(stored[key])
except (KeyError, TypeError, ValueError):
return "shadow_book default"
return key
live_book = {
"capacity": int(shadow_config["capacity"]),
"risk_per_trade": float(shadow_config["risk_pct"]) / 100.0,
"capacity_source": _source(shadow_book.KEY_CAPACITY),
"risk_source": _source(shadow_book.KEY_RISK_PCT),
}
except Exception:
logger.exception(
"Shadow book capacity load failed; breakeven study uses "
"SIM_MAX_POSITIONS"
"Shadow book config load failed; breakeven study uses "
"SIM_MAX_POSITIONS and the entry variant's risk"
)
await _rollback_quietly(db, "shadow book capacity load")
await _rollback_quietly(db, "shadow book config load")
breakeven_study_report = _breakeven_exit_study(
candidates, price_columns, spy_closes, hold_horizon,
live_exit_policy=live_exit_policy,
cadence=cadence,
monitor_report=portfolio_monitor_report,
live_capacity=live_capacity,
live_book=live_book,
)
except Exception:
logger.exception("Portfolio simulation failed")