dennisthiessen/signal-platform
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(backtest): address the second review of the breakeven and book-config work
Deploy / lint (push) Successful in 14s
Details
Deploy / test (push) Successful in 2m0s
Details
Deploy / deploy (push) Successful in 48s
Details

Browse Source 
- The breakeven study grades at the shadow book's capacity AND risk per
  trade, and labels each with where it came from. A missing setting row
  now reads "shadow_book default" instead of crediting the setting; the
  committed report said shadow_book_capacity though the snapshot has no
  such row.
- _production_book_config reads strategies through the same
  _portfolio_monitor_strategies() hook as the monitor.
- breakeven_gate_reset is rejected when there is no post-stop episode or
  cooldown for it to start, and the study skips its diagnostic arm for a
  book without one, so a production config change cannot abort the run.
- identical_to_control is renamed ties_control_on_graded_metrics and needs
  real values: be_2r ties on every graded metric but still changed 2 trades.
- _BookConfig drops an unused field and hashes by identity; the min-R:R
  sweep swaps its gate in with dataclasses.replace instead of passing it.
- New tests cover the parity guard's success path, the no-config fallback
  and the gate-reset check.

The breakeven report is regenerated: every row, diagnostic row and the
parity check are identical; only the labels, the risk fields and the
renamed flag changed.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dennisthiessenandClaude Opus 5.5 committed 2026-10-10 21:41:44 +02:00
1 parent 7a9b3ad38b
commit 231372fb3a
5 files changed
+168 -88
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

app/services/backtest_service.py
+79 -49
View File
@@ -51,7 +51,7 @@ import statistics
from collections import defaultdict from collections import defaultdict
from collections.abc import Callable from collections.abc import Callable
from concurrent.futures import ProcessPoolExecutor from concurrent.futures import ProcessPoolExecutor
from dataclasses import dataclass, field from dataclasses import dataclass, field, replace
from datetime import date, datetime, timezone from datetime import date, datetime, timezone
from types import SimpleNamespace from types import SimpleNamespace
from typing import Any from typing import Any
@@ -2365,6 +2365,10 @@ def _simulate_portfolio(
raise ValueError("breakeven_r only applies to the ATR-trail exit policies") raise ValueError("breakeven_r only applies to the ATR-trail exit policies")
if breakeven_gate_reset and breakeven_r is None: if breakeven_gate_reset and breakeven_r is None:
raise ValueError("breakeven_gate_reset needs breakeven_r") raise ValueError("breakeven_gate_reset needs breakeven_r")
if breakeven_gate_reset and post_stop_reentry_fn is None and reentry_cooldown_sessions <= 0:
raise ValueError(
"breakeven_gate_reset needs post_stop_reentry_fn or reentry_cooldown_sessions"
)
clamp_lo, clamp_hi = float(vol_clamp[0]), float(vol_clamp[1]) clamp_lo, clamp_hi = float(vol_clamp[0]), float(vol_clamp[1])
if clamp_lo <= 0 or clamp_hi < clamp_lo: if clamp_lo <= 0 or clamp_hi < clamp_lo:
raise ValueError("vol_clamp must satisfy 0 < lo <= hi") raise ValueError("vol_clamp must satisfy 0 < lo <= hi")
@@ -3535,13 +3539,13 @@ def _entry_variant_config(variant: str) -> dict | None:
return next((cfg for cfg in STRATEGY_VARIANTS if cfg["variant"] == variant), None) return next((cfg for cfg in STRATEGY_VARIANTS if cfg["variant"] == variant), None)
@dataclass(frozen=True) # eq=False keeps identity hashing: the fields hold dicts and callables.
@dataclass(frozen=True, eq=False)
class _BookConfig: class _BookConfig:
"""How one ``PORTFOLIO_MONITOR_STRATEGIES`` entry is replayed. The monitor and """How one ``PORTFOLIO_MONITOR_STRATEGIES`` entry is replayed. The monitor and
every production-book study resolve it here, so they cannot drift apart.""" every production-book study resolve it here, so they cannot drift apart."""
strategy: dict strategy: dict
entry_cfg: dict
exit_policy: str exit_policy: str
live_exit_mode: str | None live_exit_mode: str | None
hold_days: int hold_days: int
@@ -3557,17 +3561,16 @@ class _BookConfig:
candidates: list[dict], candidates: list[dict],
prices: dict[str, tuple], prices: dict[str, tuple],
cadence: str, cadence: str,
qualified_fn: Callable[[dict], bool] | None,
) -> Callable[[str, int, dict, Any], dict | None] | None: ) -> Callable[[str, int, dict, Any], dict | None] | None:
"""The gate-reset callback for this book, built on ``qualified_fn`` (a """The gate-reset callback on this book's own gate, or None for immediate
sweep passes its own gate), or None for immediate re-entry.""" re-entry. A sweep that changes the gate replaces ``qualified_fn`` first."""
if self.reentry_policy != "gate_reset": if self.reentry_policy != "gate_reset":
return None return None
return _make_gate_reset_reentry_fn( return _make_gate_reset_reentry_fn(
candidates, candidates,
prices, prices,
cadence=cadence, cadence=cadence,
qualified_fn=qualified_fn, qualified_fn=self.qualified_fn,
ranking_key=self.ranking_key, ranking_key=self.ranking_key,
) )
@@ -3594,7 +3597,6 @@ def _strategy_book_config(
) )
return _BookConfig( return _BookConfig(
strategy=strategy, strategy=strategy,
entry_cfg=entry_cfg,
exit_policy=exit_policy, exit_policy=exit_policy,
live_exit_mode=live_exit_mode, live_exit_mode=live_exit_mode,
hold_days=row_hold_days, hold_days=row_hold_days,
@@ -3618,7 +3620,7 @@ def _production_book_config(
hold_days: int, live_exit_policy: dict | None hold_days: int, live_exit_policy: dict | None
) -> _BookConfig | None: ) -> _BookConfig | None:
strategy = next( strategy = next(
(s for s in PORTFOLIO_MONITOR_STRATEGIES if s.get("is_production")), None (s for s in _portfolio_monitor_strategies() if s.get("is_production")), None
) )
if strategy is None: if strategy is None:
return None return None
@@ -3758,9 +3760,9 @@ def _min_rr_sweep(
max_positions=book.max_positions, max_positions=book.max_positions,
risk_per_trade=book.risk_per_trade, risk_per_trade=book.risk_per_trade,
atr_trail_multiplier=book.trail_multiplier, atr_trail_multiplier=book.trail_multiplier,
post_stop_reentry_fn=book.post_stop_reentry_fn( post_stop_reentry_fn=replace(
candidates, prices, cadence, qualified_fn book, qualified_fn=qualified_fn
), ).post_stop_reentry_fn(candidates, prices, cadence),
start_date=sweep_start, start_date=sweep_start,
) )
if sim is None: if sim is None:
@@ -3832,9 +3834,7 @@ def _holdout_evaluation(
book = _production_book_config(hold_days, live_exit_policy) book = _production_book_config(hold_days, live_exit_policy)
if book is None: if book is None:
return {} return {}
post_stop_reentry_fn = book.post_stop_reentry_fn( post_stop_reentry_fn = book.post_stop_reentry_fn(candidates, prices, cadence)
candidates, prices, cadence, book.qualified_fn
)
rows: list[dict] = [] rows: list[dict] = []
for window, start, end in ( for window, start, end in (
@@ -3875,7 +3875,7 @@ def _holdout_evaluation(
# Pre-registered 2026-10-09 — docs/research/breakeven-stop.md. Two trigger levels, # Pre-registered 2026-10-09 — docs/research/breakeven-stop.md. Two trigger levels,
# no sweep; graded on both halves of a fixed split at the live book capacity. # no sweep; graded on both halves of a fixed split at the live book's capacity and risk.
BREAKEVEN_STUDY_ARMS: tuple[tuple[str, float | None], ...] = ( BREAKEVEN_STUDY_ARMS: tuple[tuple[str, float | None], ...] = (
("control", None), ("control", None),
("be_1r", 1.0), ("be_1r", 1.0),
@@ -3918,15 +3918,16 @@ def _breakeven_exit_study(
live_exit_policy: dict | None = None, live_exit_policy: dict | None = None,
cadence: str = DEFAULT_BACKTEST_CADENCE, cadence: str = DEFAULT_BACKTEST_CADENCE,
monitor_report: dict | None = None, monitor_report: dict | None = None,
live_capacity: int | None = None, live_book: dict | None = None,
) -> dict: ) -> dict:
"""Production book with a breakeven stop added to the ATR trail, vs control. """Production book with a breakeven stop added to the ATR trail, vs control.
Mirrors ``_holdout_evaluation``'s production configuration and changes only Mirrors ``_holdout_evaluation``'s production configuration and changes only
``breakeven_r`` and, for every graded row, capacity: ``live_capacity`` (the ``breakeven_r`` and, for every graded row, the book's size: capacity and risk
shadow book's setting) when the caller could read it, else per trade from ``live_book`` (the shadow book's config, with where each value
``SIM_MAX_POSITIONS``, rather than the monitor's 10. A capacity-10 control on came from) when the caller could read it, else ``SIM_MAX_POSITIONS`` and the
full history must reproduce the monitor's production row, or the study is not entry variant's risk. A control at the monitor's capacity and risk on full
history must reproduce the monitor's production row, or the study is not
grading the production book. grading the production book.
""" """
book = _production_book_config(hold_days, live_exit_policy) book = _production_book_config(hold_days, live_exit_policy)
@@ -3936,13 +3937,12 @@ def _breakeven_exit_study(
return { return {
"error": f"live exit maps to {book.exit_policy!r}; the study needs atr_trail3" "error": f"live exit maps to {book.exit_policy!r}; the study needs atr_trail3"
} }
post_stop_reentry_fn = book.post_stop_reentry_fn( post_stop_reentry_fn = book.post_stop_reentry_fn(candidates, prices, cadence)
candidates, prices, cadence, book.qualified_fn
)
def run( def run(
breakeven_r: float | None, breakeven_r: float | None,
max_positions: int, max_positions: int,
risk_per_trade: float,
start: date | None = None, start: date | None = None,
end: date | None = None, end: date | None = None,
gate_reset: bool = False, gate_reset: bool = False,
@@ -3956,7 +3956,7 @@ def _breakeven_exit_study(
qualified_fn=book.qualified_fn, qualified_fn=book.qualified_fn,
ranking_key=book.ranking_key, ranking_key=book.ranking_key,
max_positions=max_positions, max_positions=max_positions,
risk_per_trade=book.risk_per_trade, risk_per_trade=risk_per_trade,
atr_trail_multiplier=book.trail_multiplier, atr_trail_multiplier=book.trail_multiplier,
post_stop_reentry_fn=post_stop_reentry_fn, post_stop_reentry_fn=post_stop_reentry_fn,
start_date=start, start_date=start,
@@ -3973,23 +3973,31 @@ def _breakeven_exit_study(
sim["trades_5r_plus"] = sum(1 for r in rs if r >= 5.0) sim["trades_5r_plus"] = sum(1 for r in rs if r >= 5.0)
return sim return sim
capacity = int(live_capacity) if live_capacity is not None else SIM_MAX_POSITIONS if live_book is not None:
capacity, risk = int(live_book["capacity"]), float(live_book["risk_per_trade"])
capacity_source, risk_source = live_book["capacity_source"], live_book["risk_source"]
else:
capacity, risk = SIM_MAX_POSITIONS, book.risk_per_trade
capacity_source, risk_source = "SIM_MAX_POSITIONS", "entry variant"
split = BREAKEVEN_STUDY_SPLIT split = BREAKEVEN_STUDY_SPLIT
windows = (("full", None, None), ("train", None, split), ("test", split, None)) windows = (("full", None, None), ("train", None, split), ("test", split, None))
rows: list[dict] = [] rows: list[dict] = []
for arm, breakeven_r in BREAKEVEN_STUDY_ARMS: for arm, breakeven_r in BREAKEVEN_STUDY_ARMS:
for window, start, end in windows: for window, start, end in windows:
sim = run(breakeven_r, capacity, start, end) sim = run(breakeven_r, capacity, risk, start, end)
if sim is not None: if sim is not None:
rows.append({"arm": arm, "window": window, **sim}) rows.append({"arm": arm, "window": window, **sim})
diagnostic_rows: list[dict] = [] diagnostic_rows: list[dict] = []
for arm, breakeven_r in BREAKEVEN_STUDY_DIAGNOSTIC_ARMS: # Gating needs a post-stop episode to start; a book without one has nothing
# for the diagnostic to measure.
diagnostic_arms = BREAKEVEN_STUDY_DIAGNOSTIC_ARMS if post_stop_reentry_fn else ()
for arm, breakeven_r in diagnostic_arms:
for window, start, end in windows: for window, start, end in windows:
sim = run(breakeven_r, capacity, start, end, gate_reset=True) sim = run(breakeven_r, capacity, risk, start, end, gate_reset=True)
if sim is not None: if sim is not None:
diagnostic_rows.append({"arm": arm, "window": window, **sim}) diagnostic_rows.append({"arm": arm, "window": window, **sim})
parity_row = run(None, book.max_positions) parity_row = run(None, book.max_positions, book.risk_per_trade)
monitor_row = next( monitor_row = next(
( (
r for r in (monitor_report or {}).get("runs") or [] r for r in (monitor_report or {}).get("runs") or []
@@ -4006,12 +4014,15 @@ def _breakeven_exit_study(
def row(arm: str, window: str) -> dict | None: def row(arm: str, window: str) -> dict | None:
return next((r for r in rows if r["arm"] == arm and r["window"] == window), None) return next((r for r in rows if r["arm"] == arm and r["window"] == window), None)
def same_as_control(arm: str, window: str) -> bool: def ties_control(arm: str, window: str) -> bool:
treated, control = row(arm, window), row("control", window) treated, control = row(arm, window), row("control", window)
return ( return (
treated is not None treated is not None
and control is not None and control is not None
and all(treated.get(k) == control.get(k) for k in _BREAKEVEN_GRADED_KEYS) and all(
treated.get(k) is not None and treated.get(k) == control.get(k)
for k in _BREAKEVEN_GRADED_KEYS
)
) )
verdicts: dict[str, dict] = {} verdicts: dict[str, dict] = {}
@@ -4028,9 +4039,10 @@ def _breakeven_exit_study(
all(verdict[w].values()) for w in ("train", "test") all(verdict[w].values()) for w in ("train", "test")
) )
# The frozen rule passes ties. An arm that matches control on every # The frozen rule passes ties. An arm that matches control on every
# graded metric in both windows is the control exit, not a finding. # graded metric in both windows has found nothing, even if a few of its
verdict["identical_to_control"] = all( # trades differ.
same_as_control(arm, w) for w in ("train", "test") verdict["ties_control_on_graded_metrics"] = all(
ties_control(arm, w) for w in ("train", "test")
) )
verdicts[arm] = verdict verdicts[arm] = verdict
@@ -4038,9 +4050,9 @@ def _breakeven_exit_study(
"protocol": "docs/research/breakeven-stop.md", "protocol": "docs/research/breakeven-stop.md",
"split_date": split.isoformat(), "split_date": split.isoformat(),
"capacity": capacity, "capacity": capacity,
"capacity_source": ( "capacity_source": capacity_source,
"shadow_book_capacity" if live_capacity is not None else "SIM_MAX_POSITIONS" "risk_per_trade": risk,
), "risk_source": risk_source,
"cadence": cadence, "cadence": cadence,
"exit_policy": book.exit_policy, "exit_policy": book.exit_policy,
"atr_trail_multiplier": book.trail_multiplier, "atr_trail_multiplier": book.trail_multiplier,
@@ -4049,6 +4061,7 @@ def _breakeven_exit_study(
"parity": { "parity": {
"ok": parity_ok, "ok": parity_ok,
"capacity": book.max_positions, "capacity": book.max_positions,
"risk_per_trade": book.risk_per_trade,
"study_control": ( "study_control": (
{k: parity_row.get(k) for k in _BREAKEVEN_PARITY_KEYS} if parity_row else None {k: parity_row.get(k) for k in _BREAKEVEN_PARITY_KEYS} if parity_row else None
), ),
@@ -4062,7 +4075,7 @@ def _breakeven_exit_study(
"note": ( "note": (
"Graded on train and test only: an arm passes when Sharpe and CAGR are " "Graded on train and test only: an arm passes when Sharpe and CAGR are "
"no lower and max drawdown no higher than control in both windows; " "no lower and max drawdown no higher than control in both windows; "
"identical_to_control marks a pass that is only ties. Win rate and avg R " "ties_control_on_graded_metrics marks a pass that is only ties. Win rate and avg R "
"are reported, not graded. diagnostic_rows were added after the first run " "are reported, not graded. diagnostic_rows were added after the first run "
"and are never graded. A pass earns forward paper validation, not a " "and are never graded. A pass earns forward paper validation, not a "
"production change." "production change."
@@ -4085,9 +4098,7 @@ def _portfolio_monitor(
book = _strategy_book_config(strategy, hold_days, live_exit_policy) book = _strategy_book_config(strategy, hold_days, live_exit_policy)
if book is None: if book is None:
continue continue
post_stop_reentry_fn = book.post_stop_reentry_fn( post_stop_reentry_fn = book.post_stop_reentry_fn(candidates, prices, cadence)
candidates, prices, cadence, book.qualified_fn
)
for lookback in PORTFOLIO_MONITOR_LOOKBACKS: for lookback in PORTFOLIO_MONITOR_LOOKBACKS:
start = _lookback_start(latest_ord, lookback["days"]) start = _lookback_start(latest_ord, lookback["days"])
sim = _simulate_portfolio( sim = _simulate_portfolio(
@@ -4805,23 +4816,42 @@ async def run_backtest(
hold_horizon, live_exit_policy=live_exit_policy, cadence=cadence, hold_horizon, live_exit_policy=live_exit_policy, cadence=cadence,
) )
if _breakeven_study_enabled(): if _breakeven_study_enabled():
live_capacity: int | None = None live_book: dict | None = None
try: try:
from app.services.shadow_book_service import get_config from app.services import shadow_book_service as shadow_book
live_capacity = int((await get_config(db))["capacity"]) shadow_config = await shadow_book.get_config(db)
stored = await settings_store.get_map(
db, [shadow_book.KEY_CAPACITY, shadow_book.KEY_RISK_PCT]
)
def _source(key: str) -> str:
# get_config falls back to its default for a missing or
# unreadable row; say so rather than credit the setting.
try:
float(stored[key])
except (KeyError, TypeError, ValueError):
return "shadow_book default"
return key
live_book = {
"capacity": int(shadow_config["capacity"]),
"risk_per_trade": float(shadow_config["risk_pct"]) / 100.0,
"capacity_source": _source(shadow_book.KEY_CAPACITY),
"risk_source": _source(shadow_book.KEY_RISK_PCT),
}
except Exception: except Exception:
logger.exception( logger.exception(
"Shadow book capacity load failed; breakeven study uses " "Shadow book config load failed; breakeven study uses "
"SIM_MAX_POSITIONS" "SIM_MAX_POSITIONS and the entry variant's risk"
) )
await _rollback_quietly(db, "shadow book capacity load") await _rollback_quietly(db, "shadow book config load")
breakeven_study_report = _breakeven_exit_study( breakeven_study_report = _breakeven_exit_study(
candidates, price_columns, spy_closes, hold_horizon, candidates, price_columns, spy_closes, hold_horizon,
live_exit_policy=live_exit_policy, live_exit_policy=live_exit_policy,
cadence=cadence, cadence=cadence,
monitor_report=portfolio_monitor_report, monitor_report=portfolio_monitor_report,
live_capacity=live_capacity, live_book=live_book,
) )
except Exception: except Exception:
logger.exception("Portfolio simulation failed") logger.exception("Portfolio simulation failed")
docs/research/breakeven-stop.md
+14 -11
View File
@@ -75,9 +75,11 @@ BACKTEST_BREAKEVEN_STUDY=1 python scripts/run_backtest_snapshot.py \
--out reports/backtest-20261009-breakeven-stop.json --out reports/backtest-20261009-breakeven-stop.json
``` ```
The graded capacity is read from the snapshot's `shadow_book_capacity` setting. The graded capacity and risk per trade come from the shadow book's config in the
`prod.sqlite` has no such row, so the shadow book's default of 15 applies; the database the run reads (`shadow_book_capacity`, `shadow_book_risk_pct`).
report records it as `capacity` / `capacity_source`. `prod.sqlite` has neither row, so the shadow book's defaults apply: 15
positions and 1% risk. The report records each value with its source
(`capacity_source`, `risk_source`), which reads `shadow_book default` here.
## Result — no change; `be_1r` fails, `be_2r` passes only by being the production exit ## Result — no change; `be_1r` fails, `be_2r` passes only by being the production exit
@@ -201,11 +203,12 @@ and live still agree.
- `breakeven_r` must be positive and finite, and is rejected for exit policies - `breakeven_r` must be positive and finite, and is rejected for exit policies
without an ATR trail. Before, a non-positive value moved the stop to entry on without an ATR trail. Before, a non-positive value moved the stop to entry on
the first bar, and other policies ignored the setting silently. the first bar, and other policies ignored the setting silently.
- Each verdict now records `identical_to_control`, which is true for `be_2r`. A - Each verdict now records `ties_control_on_graded_metrics`, which is true for
pass that is only ties can no longer read as a finding. `be_2r`. A pass that is only ties can no longer read as a finding. The flag
- The graded capacity comes from the shadow book's capacity config (the compares the graded metrics only: `be_2r` still changed 2 trades.
`shadow_book_capacity` setting in the database the run reads, or its default - The graded capacity and risk per trade come from the shadow book's config
of 15 when the row is missing) instead of the `SIM_MAX_POSITIONS` constant. On instead of the `SIM_MAX_POSITIONS` constant and the entry variant's fixed 1%.
a snapshot that is the snapshot's copy of the setting, not the current live On a snapshot that is the snapshot's copy of the settings, not the current
value ([Reproducing](#reproducing)). `capacity_source` reads live values, and a missing row is labeled as the default it fell back to
`shadow_book_capacity` in both cases. ([Reproducing](#reproducing)). In this run both are defaults, equal to the
values the first run used.
docs/research/residual-normalization-findings.md
+4
View File
@@ -55,6 +55,10 @@ long-only, gated, capital-constrained production book.
calendar overlap, and an end-to-end screen check. Ruff passed for the runner calendar overlap, and an end-to-end screen check. Ruff passed for the runner
and its tests. The existing Python runtime was usable outside the filesystem and its tests. The existing Python runtime was usable outside the filesystem
sandbox; no environment replacement was needed. sandbox; no environment replacement was needed.
- A 2026-10-10 code review added four tests (missing prices, file release,
paired-coverage guard, source ids) with the runner fixes they cover. The
report was regenerated with the fixed runner; its screen results are
byte-identical to the original run.
**Interpretation and decision** **Interpretation and decision**
reports/backtest-20261009-breakeven-stop.json
+8 -5
View File
@@ -1,5 +1,5 @@
{ {
"generated_at": "2026-10-10T07:30:27.484621+00:00", "generated_at": "2026-10-10T19:39:30.155792+00:00",
"tickers": 506, "tickers": 506,
"rank_only_tickers": 0, "rank_only_tickers": 0,
"candidates": 1011248, "candidates": 1011248,
@@ -143382,7 +143382,9 @@
"protocol": "docs/research/breakeven-stop.md", "protocol": "docs/research/breakeven-stop.md",
"split_date": "2024-07-01", "split_date": "2024-07-01",
"capacity": 15, "capacity": 15,
"capacity_source": "shadow_book_capacity", "capacity_source": "shadow_book default",
"risk_per_trade": 0.01,
"risk_source": "shadow_book default",
"cadence": "daily", "cadence": "daily",
"exit_policy": "atr_trail3", "exit_policy": "atr_trail3",
"atr_trail_multiplier": 3.0, "atr_trail_multiplier": 3.0,
@@ -143391,6 +143393,7 @@
"parity": { "parity": {
"ok": true, "ok": true,
"capacity": 10, "capacity": 10,
"risk_per_trade": 0.01,
"study_control": { "study_control": {
"sharpe": 1.84, "sharpe": 1.84,
"cagr_pct": 49.8, "cagr_pct": 49.8,
@@ -143986,7 +143989,7 @@
"max_drawdown": false "max_drawdown": false
}, },
"passes": false, "passes": false,
"identical_to_control": false "ties_control_on_graded_metrics": false
}, },
"be_2r": { "be_2r": {
"train": { "train": {
@@ -144000,7 +144003,7 @@
"max_drawdown": true "max_drawdown": true
}, },
"passes": true, "passes": true,
"identical_to_control": true "ties_control_on_graded_metrics": true
} }
}, },
"diagnostic_rows": [ "diagnostic_rows": [
@@ -144199,7 +144202,7 @@
"trades_5r_plus": 10 "trades_5r_plus": 10
} }
], ],
"note": "Graded on train and test only: an arm passes when Sharpe and CAGR are no lower and max drawdown no higher than control in both windows; identical_to_control marks a pass that is only ties. Win rate and avg R are reported, not graded. diagnostic_rows were added after the first run and are never graded. A pass earns forward paper validation, not a production change." "note": "Graded on train and test only: an arm passes when Sharpe and CAGR are no lower and max drawdown no higher than control in both windows; ties_control_on_graded_metrics marks a pass that is only ties. Win rate and avg R are reported, not graded. diagnostic_rows were added after the first run and are never graded. A pass earns forward paper validation, not a production change."
}, },
"recommendation": { "recommendation": {
"headline": "Production baseline: residual/high-vol 80/20 entry rank with a 3x ATR trailing exit, 30-trading-day max hold, and re-entry only after the gate fails and later qualifies again.", "headline": "Production baseline: residual/high-vol 80/20 entry rank with a 3x ATR trailing exit, 30-trading-day max hold, and re-entry only after the gate fails and later qualifies again.",
tests/unit/test_backtest_service.py
+63 -23
View File
@@ -1005,6 +1005,8 @@ class TestSimulatePortfolio:
("atr_trail3", {"breakeven_r": float("nan")}), ("atr_trail3", {"breakeven_r": float("nan")}),
("hold", {"breakeven_r": 1.0}), ("hold", {"breakeven_r": 1.0}),
("atr_trail3", {"breakeven_gate_reset": True}), ("atr_trail3", {"breakeven_gate_reset": True}),
# Gated, but nothing to gate: no post-stop episode and no cooldown.
("atr_trail3", {"breakeven_r": 1.0, "breakeven_gate_reset": True}),
], ],
) )
def test_breakeven_rejects_settings_it_cannot_honor(self, policy, kwargs): def test_breakeven_rejects_settings_it_cannot_honor(self, policy, kwargs):
@@ -1013,47 +1015,85 @@ class TestSimulatePortfolio:
with pytest.raises(ValueError): with pytest.raises(ValueError):
bt._simulate_portfolio([cand], prices, None, policy, 30, **kwargs) bt._simulate_portfolio([cand], prices, None, policy, 30, **kwargs)
def test_breakeven_study_uses_live_capacity_and_flags_ties(self, monkeypatch): LIVE_EXIT = {"mode": "atr_trailing", "hold_days": 30, "atr_multiplier": 3.0}
# One trade per window: AAA before the split, BBB after it. be_2r never
# triggers (+2R needs a 110 close), so it ties control in both windows. def _split_book(self, monkeypatch, per_window: int = 1) -> tuple[list[dict], dict]:
"""``per_window`` trades on each side of the split, one symbol each, 50
days apart. be_2r never triggers (+2R needs a 110 close), so it ties
control in both windows. Three per window spans the >0.25y CAGR needs."""
closes = [100.0, 103.0, 106.0, 104.0, 101.0, 99.0, 97.0, 96.0] closes = [100.0, 103.0, 106.0, 104.0, 101.0, 99.0, 97.0, 96.0]
prices = { split = self.ORD + 50 * per_window
"AAA": _sim_prices(self.ORD, closes), prices, cands = {}, []
"BBB": _sim_prices(self.ORD + 10, closes), for side, first in (("A", self.ORD), ("B", split)):
} for k in range(per_window):
cands = [ sym, day = f"{side}{k}", first + 50 * k
_sim_cand("AAA", self.ORD, entry=100.0, stop=95.0, target=130.0), prices[sym] = _sim_prices(day, closes)
_sim_cand("BBB", self.ORD + 10, entry=100.0, stop=95.0, target=130.0), cands.append(_sim_cand(sym, day, entry=100.0, stop=95.0, target=130.0))
] monkeypatch.setattr(bt, "BREAKEVEN_STUDY_SPLIT", date.fromordinal(split))
monkeypatch.setattr(bt, "BREAKEVEN_STUDY_SPLIT", date.fromordinal(self.ORD + 10)) return cands, prices
capacities: list[int] = []
def test_breakeven_study_grades_the_live_book_and_flags_ties(self, monkeypatch):
cands, prices = self._split_book(monkeypatch, per_window=3)
sizes: list[tuple[int, float]] = []
real_simulate = bt._simulate_portfolio real_simulate = bt._simulate_portfolio
def spy(*args, **kwargs): def spy(*args, **kwargs):
capacities.append(kwargs["max_positions"]) sizes.append((kwargs["max_positions"], kwargs["risk_per_trade"]))
return real_simulate(*args, **kwargs) return real_simulate(*args, **kwargs)
monkeypatch.setattr(bt, "_simulate_portfolio", spy) monkeypatch.setattr(bt, "_simulate_portfolio", spy)
report = bt._breakeven_exit_study( report = bt._breakeven_exit_study(
cands, prices, None, 30, cands, prices, None, 30,
live_exit_policy={"mode": "atr_trailing", "hold_days": 30, "atr_multiplier": 3.0}, live_exit_policy=self.LIVE_EXIT,
live_capacity=3, live_book={
"capacity": 3, "risk_per_trade": 0.02,
"capacity_source": "shadow_book_capacity",
"risk_source": "shadow_book default",
},
) )
assert report["capacity"] == 3 assert (report["capacity"], report["risk_per_trade"]) == (3, 0.02)
assert report["capacity_source"] == "shadow_book_capacity" assert report["capacity_source"] == "shadow_book_capacity"
assert report["verdicts"]["be_2r"]["identical_to_control"] is True assert report["risk_source"] == "shadow_book default"
assert report["verdicts"]["be_1r"]["identical_to_control"] is False assert report["verdicts"]["be_2r"]["ties_control_on_graded_metrics"] is True
assert report["verdicts"]["be_1r"]["ties_control_on_graded_metrics"] is False
# No monitor row to match, so nothing can pass. # No monitor row to match, so nothing can pass.
assert report["parity"]["ok"] is False assert report["parity"]["ok"] is False
assert not any(v["passes"] for v in report["verdicts"].values()) assert not any(v["passes"] for v in report["verdicts"].values())
assert {r["arm"] for r in report["diagnostic_rows"]} == {"be_1r_gated"} assert {r["arm"] for r in report["diagnostic_rows"]} == {"be_1r_gated"}
assert all(r["breakeven_gate_reset"] for r in report["diagnostic_rows"]) assert all(r["breakeven_gate_reset"] for r in report["diagnostic_rows"])
# 3 graded arms + 1 diagnostic arm x 3 windows at capacity 3, then the # 3 graded arms + 1 diagnostic arm x 3 windows on the live book, then the
# parity row at the monitor's capacity. # parity row on the monitor's.
assert capacities[:12] == [3] * 12 assert sizes[:12] == [(3, 0.02)] * 12
assert capacities[12:] == [report["parity"]["capacity"]] assert sizes[12:] == [
(report["parity"]["capacity"], report["parity"]["risk_per_trade"])
]
def test_breakeven_study_without_live_config_says_so(self, monkeypatch):
cands, prices = self._split_book(monkeypatch)
report = bt._breakeven_exit_study(cands, prices, None, 30, live_exit_policy=self.LIVE_EXIT)
assert (report["capacity"], report["capacity_source"]) == (
bt.SIM_MAX_POSITIONS, "SIM_MAX_POSITIONS",
)
assert report["risk_source"] == "entry variant"
def test_breakeven_parity_finds_the_monitor_production_row(self, monkeypatch):
# The guard's success path: the study's control at the monitor's capacity
# must match the monitor's production "all" row, and then a tie can pass.
cands, prices = self._split_book(monkeypatch, per_window=3)
monitor = bt._portfolio_monitor(cands, prices, None, 30, live_exit_policy=self.LIVE_EXIT)
report = bt._breakeven_exit_study(
cands, prices, None, 30, live_exit_policy=self.LIVE_EXIT, monitor_report=monitor,
)
parity = report["parity"]
assert parity["monitor_production_all"] is not None
assert parity["study_control"] == parity["monitor_production_all"]
assert parity["ok"] is True
assert report["verdicts"]["be_2r"]["passes"] is True
assert report["verdicts"]["be_2r"]["ties_control_on_graded_metrics"] is True
def test_breakeven_checks_fail_on_missing_metrics(self): def test_breakeven_checks_fail_on_missing_metrics(self):
control = {"sharpe": 1.5, "cagr_pct": 40.0, "max_drawdown_pct": 20.0} control = {"sharpe": 1.5, "cagr_pct": 40.0, "max_drawdown_pct": 20.0}