fix(backtest): quote one book, not two
The recommendation's "Book vs SPY" line read from portfolio_sim — the hold/target policy book — while the tiles directly above read from portfolio_monitor, the production ATR-trail book. Same SPY figure, different portfolio return, on one screen. It now reads the same production row the tiles do. Also removed, for the same reason: the "legacy exit diagnostic" comparing hold against the S/R target. Both are exits the production book replaced, so a recommendation between them could not lead to an action. And the fallback headline, which advised the fixed-hold exit whenever a report had no production row — a report that cannot describe the production baseline now states none. portfolio_sim stays in the report payload: scripts/run_backtest_snapshot.py and reports/compare_reports.py read it, and it is no longer surfaced in the UI. The test fixture now carries a production monitor whose numbers differ from its policy sim, so re-sourcing that line from the old place fails rather than passes unnoticed. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
@@ -1272,20 +1272,36 @@ def test_build_recommendation_reads_the_report():
|
||||
{"min_momentum_percentile": 60.0, "net_avg_r": 0.05, "total": 300},
|
||||
{"min_momentum_percentile": 0.0, "net_avg_r": -0.12, "total": 1000},
|
||||
],
|
||||
# Legacy policy book. Its numbers are deliberately DIFFERENT from the
|
||||
# production monitor's below, so sourcing the benchmark line from here
|
||||
# again would fail the assertion rather than pass unnoticed.
|
||||
"portfolio_sim": {"policies": [
|
||||
{"policy": "target", "cagr_pct": 23.7, "total_return_pct": 134.8,
|
||||
"spy_return_pct": 95.9, "max_drawdown_pct": 20.7},
|
||||
{"policy": "hold", "cagr_pct": 31.9, "total_return_pct": 203.6,
|
||||
"spy_return_pct": 95.9, "max_drawdown_pct": 21.2},
|
||||
]},
|
||||
"portfolio_monitor": {
|
||||
"production_strategy": "prod",
|
||||
"runs": [{
|
||||
"strategy": "prod", "lookback": "all", "lookback_label": "All history",
|
||||
"cagr_pct": 40.0, "sharpe": 1.72, "max_drawdown_pct": 17.7,
|
||||
"total_return_pct": 297.8, "spy_return_pct": 101.9,
|
||||
}],
|
||||
},
|
||||
}
|
||||
rec = bt._build_recommendation(report)
|
||||
by_topic: dict[str, list[str]] = {}
|
||||
for item in rec["items"]:
|
||||
by_topic.setdefault(item["topic"], []).append(item["text"])
|
||||
|
||||
assert rec["headline"] is not None and "hold 30" in rec["headline"]
|
||||
assert any("hold 30 trading days" in t for t in by_topic["exit"])
|
||||
assert rec["headline"] is not None and "Production baseline" in rec["headline"]
|
||||
# The hold-vs-target comparison is gone: both are exits the production book
|
||||
# replaced, so a recommendation between them cannot lead to an action.
|
||||
assert "exit" not in by_topic
|
||||
# Benchmark must quote the SAME row the page's tiles show, not the policy sim.
|
||||
assert "+297.8%" in by_topic["benchmark"][0]
|
||||
assert "203.6" not in by_topic["benchmark"][0]
|
||||
gate_texts = " | ".join(by_topic["gate"])
|
||||
assert "confidence floor adds nothing" in gate_texts
|
||||
assert "keep the R:R floor" in gate_texts
|
||||
@@ -1810,3 +1826,24 @@ class TestPortfolioQualityMetrics:
|
||||
for key in ("sortino", "gain_to_pain", "profit_factor"):
|
||||
assert key in sim
|
||||
assert sim["sortino"] is None
|
||||
|
||||
def test_build_recommendation_states_no_baseline_without_a_production_row():
|
||||
"""A report with no portfolio monitor cannot describe the production book.
|
||||
It used to fall back to recommending the fixed-hold exit — advice for a model
|
||||
the production book had already replaced."""
|
||||
report = {
|
||||
"overall_qualified": {"net_avg_r": 0.13, "net_avg_r_ex_top5": 0.05},
|
||||
"time_exit_sweep": [{"hold_days": 30, "net_avg_r": 0.50, "net_avg_r_ex_top5": 0.21}],
|
||||
"portfolio_sim": {"policies": [
|
||||
{"policy": "hold", "cagr_pct": 31.9, "total_return_pct": 203.6,
|
||||
"spy_return_pct": 95.9, "max_drawdown_pct": 21.2},
|
||||
]},
|
||||
}
|
||||
rec = bt._build_recommendation(report)
|
||||
topics = {item["topic"] for item in rec["items"]}
|
||||
|
||||
assert rec["headline"] is None
|
||||
# Nothing may be sourced from the legacy policy book.
|
||||
assert "benchmark" not in topics
|
||||
assert "exit" not in topics
|
||||
|
||||
|
||||
Reference in New Issue
Block a user