dennisthiessenandClaude Opus 5
f5d4b516ab
Brings the durable artifacts of research/portfolio-capacity-rebalancing onto main so the rationale for raising the count cap lives with the code that cites it. The matrix runner, the research simulator hooks and the study's unit tests are deliberately left behind; they remain at tag research/portfolio-capacity-final. Corrects conclusions that were reached on EV per trade and are now superseded: the findings doc's decisions 1 (keep cap 10) and 4 (run the risk-floor A/B) are struck through and answered in a new correction section, and the research README and phase-A matrix entries are updated to match. The frozen specification itself is untouched -- its recorded SHA-256 f1e37783 still verifies. effective-risk-floor-ab.md is retained but marked CLOSED/NEGATIVE: the study it proposes is already answered by cap15 vs cash_unbounded (-0.753pp CAGR while EV/trade rises), and its EV-based pass rule would have shipped it. scripts/research_rankings.py replaces a fourth copy of the historical rank-map helper; run_research_matrix, run_execution_recovery_matrix and run_daily_reentry_matrix now share it. The shared version adds a duplicate observation guard and a deterministic symbol tie-break the copies lacked. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
760 lines
29 KiB
Python
760 lines
29 KiB
Python
"""Phase-A research matrix: max-hold, vol targeting, next-open fill, corr caps.
|
||
|
||
Promotion rule (pre-registered — do not edit after a run starts)
|
||
----------------------------------------------------------------
|
||
An arm may be promoted over the close-fill production **control** only if ALL of:
|
||
|
||
1. Validation-window (entries ≥ ``--validation-split``, default 2024-07-01)
|
||
Sharpe ≥ control validation Sharpe.
|
||
2. Validation max drawdown is not worse than control by more than 2 percentage
|
||
points (higher DD is worse).
|
||
3. Train-window Sharpe is not worse than control train Sharpe (both-windows
|
||
consistency — same standard as the min_rr sweep).
|
||
4. Report whether the validation Sharpe delta exceeds 1 × SE (control or arm);
|
||
most arms will fail this distinguishability check — that is expected and is
|
||
the reason SE/PSR ship on every row. Failing the 1-SE bar does **not** alone
|
||
veto promotion under (1)–(3), but it must be stated.
|
||
|
||
Naming: the post-split window is called **validation**, not "holdout". It has
|
||
been opened by prior experiments; treat it as a disciplined check, not a
|
||
pristine sample.
|
||
|
||
Arms (pre-registered; N used for Deflated Sharpe)
|
||
-------------------------------------------------
|
||
- A0 control: production gate/rank/trail, hold=30, close fill, no vol target, no corr cap
|
||
- A2 max-hold: hold ∈ {30, 45, 60, 90} (30 is the control row; listed once)
|
||
- A3 vol-target: target ∈ {15%, 20%, 25%} × clamp {[0.5,1.5], [0.25,2.0]} at lookback 60;
|
||
plus sensitivity lookbacks {20, 126} at target 20% / clamp [0.5,1.5] only
|
||
- A4 next-open fill (measurement + portfolio consequence vs control)
|
||
- A5 corr cap: threshold ∈ {0.6, 0.7, 0.8} × action ∈ {skip, half-size}
|
||
|
||
DSR uses N = number of pre-registered strategy arms in this matrix (see
|
||
``PRE_REGISTERED_ARM_IDS``). Standalone backtests do not invent a DSR.
|
||
|
||
Calendar truncation
|
||
-------------------
|
||
The simulator always cuts the equity calendar at last_signal + hold_days
|
||
(+1 for next-open). The runner asserts validation end_date ≤ last price date and
|
||
that the sim end is within hold_days+pad of the last admitted signal so a 90d
|
||
arm cannot sit in trailing flat cash.
|
||
|
||
Usage
|
||
-----
|
||
python scripts/run_research_matrix.py backtest_snapshots/prod.sqlite \\
|
||
--workers 7 --allow-spawn --candidate-cache reports/.cache/research-cands.pkl
|
||
|
||
python scripts/run_research_matrix.py ... --only a2,a3
|
||
python scripts/run_research_matrix.py ... --skip a4
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import asyncio
|
||
import json
|
||
import multiprocessing
|
||
import os
|
||
import pickle
|
||
import sys
|
||
from concurrent.futures import ProcessPoolExecutor, as_completed
|
||
from datetime import date, datetime
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
from sqlalchemy import select
|
||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
if str(ROOT) not in sys.path:
|
||
sys.path.insert(0, str(ROOT))
|
||
|
||
from scripts.research_rankings import ( # noqa: E402
|
||
_live_universe_rank_map,
|
||
_period_percentiles,
|
||
)
|
||
|
||
CACHE_VERSION = "research-matrix-v1-daily-prod"
|
||
|
||
# Pre-registered arm catalogue (order is report order). Control is a0.
|
||
# Count N for DSR excludes pure measurement-only rows if any; every arm below
|
||
# is a portfolio book and counts.
|
||
PRE_REGISTERED_ARMS: tuple[dict[str, Any], ...] = (
|
||
{
|
||
"id": "a0_control",
|
||
"group": "a0",
|
||
"label": "Control: close fill, hold 30, risk 1%, no corr/vol",
|
||
"hold_days": 30,
|
||
"fill_mode": "close",
|
||
},
|
||
# A2 — max hold (30 is control; still emitted as a2 for the sweep table)
|
||
{"id": "a2_hold_30", "group": "a2", "label": "Max hold 30", "hold_days": 30},
|
||
{"id": "a2_hold_45", "group": "a2", "label": "Max hold 45", "hold_days": 45},
|
||
{"id": "a2_hold_60", "group": "a2", "label": "Max hold 60", "hold_days": 60},
|
||
{"id": "a2_hold_90", "group": "a2", "label": "Max hold 90", "hold_days": 90},
|
||
# A3 — vol targeting
|
||
{
|
||
"id": "a3_vt15_c05_15_lb60",
|
||
"group": "a3",
|
||
"label": "Vol target 15% clamp[0.5,1.5] lb60",
|
||
"vol_target": 0.15,
|
||
"vol_clamp": (0.5, 1.5),
|
||
"vol_lookback": 60,
|
||
},
|
||
{
|
||
"id": "a3_vt20_c05_15_lb60",
|
||
"group": "a3",
|
||
"label": "Vol target 20% clamp[0.5,1.5] lb60",
|
||
"vol_target": 0.20,
|
||
"vol_clamp": (0.5, 1.5),
|
||
"vol_lookback": 60,
|
||
},
|
||
{
|
||
"id": "a3_vt25_c05_15_lb60",
|
||
"group": "a3",
|
||
"label": "Vol target 25% clamp[0.5,1.5] lb60",
|
||
"vol_target": 0.25,
|
||
"vol_clamp": (0.5, 1.5),
|
||
"vol_lookback": 60,
|
||
},
|
||
{
|
||
"id": "a3_vt15_c025_20_lb60",
|
||
"group": "a3",
|
||
"label": "Vol target 15% clamp[0.25,2.0] lb60",
|
||
"vol_target": 0.15,
|
||
"vol_clamp": (0.25, 2.0),
|
||
"vol_lookback": 60,
|
||
},
|
||
{
|
||
"id": "a3_vt20_c025_20_lb60",
|
||
"group": "a3",
|
||
"label": "Vol target 20% clamp[0.25,2.0] lb60",
|
||
"vol_target": 0.20,
|
||
"vol_clamp": (0.25, 2.0),
|
||
"vol_lookback": 60,
|
||
},
|
||
{
|
||
"id": "a3_vt25_c025_20_lb60",
|
||
"group": "a3",
|
||
"label": "Vol target 25% clamp[0.25,2.0] lb60",
|
||
"vol_target": 0.25,
|
||
"vol_clamp": (0.25, 2.0),
|
||
"vol_lookback": 60,
|
||
},
|
||
{
|
||
"id": "a3_vt20_c05_15_lb20",
|
||
"group": "a3",
|
||
"label": "Vol target 20% clamp[0.5,1.5] lb20 (sensitivity)",
|
||
"vol_target": 0.20,
|
||
"vol_clamp": (0.5, 1.5),
|
||
"vol_lookback": 20,
|
||
},
|
||
{
|
||
"id": "a3_vt20_c05_15_lb126",
|
||
"group": "a3",
|
||
"label": "Vol target 20% clamp[0.5,1.5] lb126 (sensitivity)",
|
||
"vol_target": 0.20,
|
||
"vol_clamp": (0.5, 1.5),
|
||
"vol_lookback": 126,
|
||
},
|
||
# A4 — next-open fill
|
||
{
|
||
"id": "a4_next_open",
|
||
"group": "a4",
|
||
"label": "Next-open fill (t+1 open, stop from fill−1.5 ATR)",
|
||
"fill_mode": "next_open",
|
||
},
|
||
# A5 — correlation caps
|
||
{
|
||
"id": "a5_corr06_skip",
|
||
"group": "a5",
|
||
"label": "Corr max 0.6 skip",
|
||
"corr_max": 0.6,
|
||
"corr_action": "skip",
|
||
},
|
||
{
|
||
"id": "a5_corr07_skip",
|
||
"group": "a5",
|
||
"label": "Corr max 0.7 skip",
|
||
"corr_max": 0.7,
|
||
"corr_action": "skip",
|
||
},
|
||
{
|
||
"id": "a5_corr08_skip",
|
||
"group": "a5",
|
||
"label": "Corr max 0.8 skip",
|
||
"corr_max": 0.8,
|
||
"corr_action": "skip",
|
||
},
|
||
{
|
||
"id": "a5_corr06_half",
|
||
"group": "a5",
|
||
"label": "Corr max 0.6 half-size",
|
||
"corr_max": 0.6,
|
||
"corr_action": "half_size",
|
||
},
|
||
{
|
||
"id": "a5_corr07_half",
|
||
"group": "a5",
|
||
"label": "Corr max 0.7 half-size",
|
||
"corr_max": 0.7,
|
||
"corr_action": "half_size",
|
||
},
|
||
{
|
||
"id": "a5_corr08_half",
|
||
"group": "a5",
|
||
"label": "Corr max 0.8 half-size",
|
||
"corr_max": 0.8,
|
||
"corr_action": "half_size",
|
||
},
|
||
)
|
||
PRE_REGISTERED_ARM_IDS = tuple(arm["id"] for arm in PRE_REGISTERED_ARMS)
|
||
PRE_REGISTERED_N_TRIALS = len(PRE_REGISTERED_ARMS)
|
||
|
||
|
||
def _sqlite_url(path: Path) -> str:
|
||
return f"sqlite+aiosqlite:///{path.resolve().as_posix()}"
|
||
|
||
|
||
def _parse_args() -> argparse.Namespace:
|
||
parser = argparse.ArgumentParser(
|
||
description=__doc__,
|
||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||
)
|
||
parser.add_argument("snapshot", help="SQLite backtest snapshot.")
|
||
parser.add_argument("--workers", type=int, default=6)
|
||
parser.add_argument(
|
||
"--allow-spawn",
|
||
action="store_true",
|
||
help="Allow spawn multiprocessing (needed on Windows).",
|
||
)
|
||
parser.add_argument("--out", default=None, help="JSON report path.")
|
||
parser.add_argument(
|
||
"--candidate-cache",
|
||
default=None,
|
||
help="Optional pickle cache for the daily qualified candidate set.",
|
||
)
|
||
parser.add_argument(
|
||
"--validation-split",
|
||
default="2024-07-01",
|
||
help="Train/validation entry split (YYYY-MM-DD). Validation = entries on/after.",
|
||
)
|
||
parser.add_argument(
|
||
"--only",
|
||
default=None,
|
||
help="Comma-separated arm groups or ids to run (e.g. a2,a3 or a0_control,a4_next_open).",
|
||
)
|
||
parser.add_argument(
|
||
"--skip",
|
||
default=None,
|
||
help="Comma-separated arm groups or ids to skip.",
|
||
)
|
||
parser.add_argument("--quiet", action="store_true")
|
||
parser.add_argument(
|
||
"--cadence",
|
||
choices=("daily", "weekly"),
|
||
default="daily",
|
||
help="Candidate replay cadence. Daily matches the re-entry matrix production arm.",
|
||
)
|
||
return parser.parse_args()
|
||
|
||
|
||
def _default_output_path() -> Path:
|
||
stamp = datetime.now().strftime("%Y%m%d-%H%M%S")
|
||
return Path("reports") / f"research-matrix-{stamp}.json"
|
||
|
||
|
||
def _parse_selector(raw: str | None) -> set[str] | None:
|
||
if raw is None or not raw.strip():
|
||
return None
|
||
return {part.strip().lower() for part in raw.split(",") if part.strip()}
|
||
|
||
|
||
def _arm_selected(arm: dict[str, Any], only: set[str] | None, skip: set[str] | None) -> bool:
|
||
arm_id = str(arm["id"]).lower()
|
||
group = str(arm["group"]).lower()
|
||
if skip and (arm_id in skip or group in skip):
|
||
return False
|
||
if only is None:
|
||
return True
|
||
return arm_id in only or group in only
|
||
|
||
|
||
def _write_checkpoint(path: Path, report: dict) -> None:
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
tmp = path.with_suffix(path.suffix + ".tmp")
|
||
tmp.write_text(json.dumps(report, indent=2, default=str), encoding="utf-8")
|
||
tmp.replace(path)
|
||
md_path = path.with_suffix(".md")
|
||
md_path.write_text(_markdown_table(report), encoding="utf-8")
|
||
|
||
|
||
def _markdown_table(report: dict) -> str:
|
||
lines = [
|
||
f"# Research matrix — {report.get('generated_at', '')}",
|
||
"",
|
||
f"Validation split: **{report.get('validation_split')}**. "
|
||
f"Pre-registered N for DSR: **{report.get('n_trials')}**.",
|
||
"",
|
||
"## Promotion rule",
|
||
"",
|
||
report.get("promotion_rule", ""),
|
||
"",
|
||
"## Arms",
|
||
"",
|
||
"| arm | window | Sharpe | SE | PSR | DSR | CAGR | MaxDD | Calmar | trades | avg scalar |",
|
||
"|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|",
|
||
]
|
||
for arm in report.get("arms") or []:
|
||
for window_row in arm.get("windows") or []:
|
||
lines.append(
|
||
"| {arm} | {window} | {sharpe} | {se} | {psr} | {dsr} | {cagr} | {dd} | {calmar} | {trades} | {scalar} |".format(
|
||
arm=arm.get("id"),
|
||
window=window_row.get("window"),
|
||
sharpe=_fmt(window_row.get("sharpe")),
|
||
se=_fmt(window_row.get("sharpe_se")),
|
||
psr=_fmt(window_row.get("psr")),
|
||
dsr=_fmt(window_row.get("dsr")),
|
||
cagr=_fmt(window_row.get("cagr_pct")),
|
||
dd=_fmt(window_row.get("max_drawdown_pct")),
|
||
calmar=_fmt(window_row.get("calmar")),
|
||
trades=_fmt(window_row.get("trades")),
|
||
scalar=_fmt(window_row.get("avg_vol_scalar")),
|
||
)
|
||
)
|
||
promo = report.get("promotion") or {}
|
||
lines.extend(["", "## Promotion decisions", ""])
|
||
if not promo:
|
||
lines.append("_No arms graded yet._")
|
||
else:
|
||
for arm_id, decision in promo.items():
|
||
lines.append(
|
||
f"- **{arm_id}**: {'PROMOTE' if decision.get('promote') else 'reject'} — "
|
||
f"{decision.get('reason')}"
|
||
)
|
||
lines.append("")
|
||
return "\n".join(lines)
|
||
|
||
|
||
def _fmt(value: Any) -> str:
|
||
if value is None:
|
||
return "—"
|
||
if isinstance(value, float):
|
||
return f"{value:.3g}"
|
||
return str(value)
|
||
|
||
|
||
def _grade_promotion(control: dict, arm: dict, se_ref: float | None) -> dict:
|
||
"""Apply the pre-registered promotion rule. control/arm are arm result dicts."""
|
||
c_val = _window(control, "validation")
|
||
a_val = _window(arm, "validation")
|
||
c_train = _window(control, "train")
|
||
a_train = _window(arm, "train")
|
||
if not c_val or not a_val or not c_train or not a_train:
|
||
return {"promote": False, "reason": "missing train/validation rows"}
|
||
|
||
c_s = c_val.get("sharpe")
|
||
a_s = a_val.get("sharpe")
|
||
c_dd = c_val.get("max_drawdown_pct")
|
||
a_dd = a_val.get("max_drawdown_pct")
|
||
c_ts = c_train.get("sharpe")
|
||
a_ts = a_train.get("sharpe")
|
||
if None in (c_s, a_s, c_dd, a_dd, c_ts, a_ts):
|
||
return {"promote": False, "reason": "missing Sharpe/DD on a required window"}
|
||
|
||
delta = float(a_s) - float(c_s)
|
||
se = se_ref
|
||
if se is None:
|
||
se = a_val.get("sharpe_se") or c_val.get("sharpe_se")
|
||
exceeds_1se = se is not None and abs(delta) > float(se)
|
||
|
||
checks = {
|
||
"validation_sharpe_ge_control": float(a_s) >= float(c_s),
|
||
"validation_dd_not_worse_by_2pp": float(a_dd) <= float(c_dd) + 2.0,
|
||
"train_sharpe_not_worse": float(a_ts) >= float(c_ts),
|
||
"delta_exceeds_1se": exceeds_1se,
|
||
"validation_sharpe_delta": round(delta, 4),
|
||
"se_used": se,
|
||
}
|
||
promote = (
|
||
checks["validation_sharpe_ge_control"]
|
||
and checks["validation_dd_not_worse_by_2pp"]
|
||
and checks["train_sharpe_not_worse"]
|
||
)
|
||
if promote:
|
||
reason = (
|
||
f"validation Sharpe {a_s} ≥ control {c_s}; "
|
||
f"DD {a_dd} within +2pp of {c_dd}; train Sharpe {a_ts} ≥ {c_ts}"
|
||
)
|
||
if not exceeds_1se:
|
||
reason += " (delta ≤ 1 SE — distinguishable noise bar not cleared)"
|
||
else:
|
||
reason += " (delta > 1 SE)"
|
||
else:
|
||
failed = [k for k, v in checks.items() if k.startswith(("validation", "train")) and v is False]
|
||
reason = "failed: " + ", ".join(failed) if failed else "failed promotion checks"
|
||
return {"promote": promote, "reason": reason, "checks": checks}
|
||
|
||
|
||
def _window(arm_result: dict, name: str) -> dict | None:
|
||
for row in arm_result.get("windows") or []:
|
||
if row.get("window") == name:
|
||
return row
|
||
return None
|
||
|
||
|
||
def _assert_calendar_truncation(sim: dict, hold_days: int, fill_mode: str) -> None:
|
||
"""Guard against trailing flat-cash after the last resolvable signal."""
|
||
start = sim.get("start_date")
|
||
end = sim.get("end_date")
|
||
if not start or not end:
|
||
return
|
||
# Soft check: book span should not massively exceed hold window beyond data needs.
|
||
# Hard assert lives on entry-end vs sim end when trade_details present.
|
||
details = sim.get("trade_details") or []
|
||
if not details:
|
||
return
|
||
last_entry = max(date.fromisoformat(t["entry_date"]) for t in details)
|
||
sim_end = date.fromisoformat(str(end))
|
||
pad = hold_days + (1 if fill_mode == "next_open" else 0)
|
||
# Allow calendar days ≈ trading-day pad with weekend slack (2×).
|
||
max_slack_days = pad * 2 + 5
|
||
if (sim_end - last_entry).days > max_slack_days:
|
||
raise AssertionError(
|
||
f"calendar truncation failed: last entry {last_entry} but sim end "
|
||
f"{sim_end} (hold_days={hold_days}, fill_mode={fill_mode})"
|
||
)
|
||
|
||
|
||
async def _main() -> None:
|
||
args = _parse_args()
|
||
snapshot = Path(args.snapshot)
|
||
if not snapshot.exists():
|
||
raise SystemExit(f"Snapshot not found: {snapshot}")
|
||
if args.workers < 1:
|
||
raise SystemExit("--workers must be positive")
|
||
|
||
only = _parse_selector(args.only)
|
||
skip = _parse_selector(args.skip)
|
||
validation_split = date.fromisoformat(args.validation_split)
|
||
out_path = Path(args.out) if args.out else _default_output_path()
|
||
|
||
os.environ["BACKTEST_SNAPSHOT_OFFLINE"] = "1"
|
||
if args.allow_spawn:
|
||
os.environ["BACKTEST_ALLOW_SPAWN"] = "1"
|
||
|
||
from app.models.ticker import Ticker
|
||
from app.services import backtest_service as bt
|
||
from app.services.admin_service import get_activation_config
|
||
from app.services.paper_trade_service import get_exit_policy
|
||
from app.services.recommendation_service import get_recommendation_config
|
||
|
||
db_engine = create_async_engine(_sqlite_url(snapshot), pool_pre_ping=True)
|
||
Session = async_sessionmaker(db_engine, class_=AsyncSession, expire_on_commit=False)
|
||
try:
|
||
async with Session() as db:
|
||
recommendation_config = await get_recommendation_config(db)
|
||
activation = await get_activation_config(db)
|
||
exit_config = await get_exit_policy(db)
|
||
benchmark_closes = await bt._load_benchmark_closes_for_backtest(
|
||
db, days=None, refresh=False
|
||
)
|
||
ticker_result = await db.execute(select(Ticker).order_by(Ticker.symbol))
|
||
symbols = [t.symbol for t in ticker_result.scalars().all()]
|
||
prices: dict[str, tuple] = {}
|
||
for index, symbol in enumerate(symbols, 1):
|
||
columns = await bt._fetch_columns(db, symbol)
|
||
if columns is not None:
|
||
prices[symbol] = columns
|
||
if not args.quiet and index % 50 == 0:
|
||
print(f"loaded prices: {index}/{len(symbols)}", flush=True)
|
||
finally:
|
||
await db_engine.dispose()
|
||
|
||
if not prices:
|
||
raise SystemExit("No price columns loaded from snapshot")
|
||
|
||
snapshot_stat = snapshot.stat()
|
||
cache_key = {
|
||
"version": CACHE_VERSION,
|
||
"snapshot": str(snapshot.resolve()),
|
||
"snapshot_size": snapshot_stat.st_size,
|
||
"snapshot_mtime_ns": snapshot_stat.st_mtime_ns,
|
||
"cadence": args.cadence,
|
||
"target_model": "production_gtl",
|
||
}
|
||
cache_path = Path(args.candidate_cache) if args.candidate_cache else None
|
||
qualified: list[dict] | None = None
|
||
entry_candidate_count = 0
|
||
fip_signal_eval: list[dict] | None = None
|
||
|
||
if cache_path is not None and cache_path.exists():
|
||
with cache_path.open("rb") as handle:
|
||
cached = pickle.load(handle) # noqa: S301 - trusted local cache
|
||
if cached.get("key") == cache_key:
|
||
qualified = list(cached["qualified_candidates"])
|
||
entry_candidate_count = int(cached["entry_candidate_count"])
|
||
fip_signal_eval = cached.get("fip_signal_eval")
|
||
if not args.quiet:
|
||
print(f"loaded candidate cache: {cache_path}", flush=True)
|
||
elif not args.quiet:
|
||
print(f"candidate cache mismatch; rebuilding: {cache_path}", flush=True)
|
||
|
||
if qualified is None:
|
||
replay_start = date(1900, 1, 1)
|
||
workers = max(1, min(int(args.workers), max(1, multiprocessing.cpu_count() - 1)))
|
||
context = bt._mp_context() or multiprocessing.get_context("spawn")
|
||
replay_rows: list[dict] = []
|
||
with ProcessPoolExecutor(max_workers=workers, mp_context=context) as pool:
|
||
futures = {
|
||
pool.submit(
|
||
bt._replay_candidates_for_period,
|
||
symbol,
|
||
columns,
|
||
recommendation_config,
|
||
activation,
|
||
benchmark_closes,
|
||
replay_start,
|
||
args.cadence,
|
||
True,
|
||
True,
|
||
): symbol
|
||
for symbol, columns in prices.items()
|
||
}
|
||
for index, future in enumerate(as_completed(futures), 1):
|
||
replay_rows.extend(future.result())
|
||
if not args.quiet and index % 25 == 0:
|
||
print(f"replay: {index}/{len(futures)} tickers", flush=True)
|
||
|
||
setup_candidates = [row for row in replay_rows if not row.get("_rank_only")]
|
||
rank_observations = [
|
||
row for row in replay_rows if row.get("_universe_rank_observation")
|
||
]
|
||
entry_candidate_count = len(setup_candidates)
|
||
|
||
# Live-universe ranking (same semantics as run_daily_reentry_matrix).
|
||
live_ranks = _live_universe_rank_map(
|
||
rank_observations,
|
||
benchmark_closes,
|
||
bt.STRATEGY_RANK_MOMENTUM_WEIGHT,
|
||
)
|
||
threshold = float(activation.get("min_momentum_percentile", 80.0))
|
||
qualified = []
|
||
for setup in setup_candidates:
|
||
if setup.get("direction") != "long":
|
||
continue
|
||
candidate = {
|
||
key: value
|
||
for key, value in setup.items()
|
||
if not key.startswith("_universe_")
|
||
}
|
||
identity = (str(setup["symbol"]), str(setup["date"]))
|
||
rank = live_ranks.get(identity)
|
||
if rank is None:
|
||
continue
|
||
candidate[bt.PRODUCTION_PERCENTILE_KEY] = rank["momentum_percentile"]
|
||
candidate[bt.VOL_PERCENTILE_KEY] = rank["volatility_percentile"]
|
||
candidate[bt.RESIDUAL_HIGH_VOL_BLEND_80_20_KEY] = rank["strategy_rank"]
|
||
candidate["qualified"] = bt._momentum_qualifies(candidate, threshold)
|
||
if candidate["qualified"]:
|
||
qualified.append(candidate)
|
||
|
||
# fip_id fingerprint via the shared weekly signal harness.
|
||
collected: dict = {}
|
||
for symbol, columns in prices.items():
|
||
# Rebuild minimal records for signal eval from column arrays.
|
||
ords, _o, highs, _l, closes, _v = columns
|
||
records = [
|
||
type("R", (), {"date": date.fromordinal(int(ords[i])), "close": closes[i], "high": highs[i]})()
|
||
for i in range(len(ords))
|
||
]
|
||
series = bt._signal_series(records, benchmark_closes)
|
||
for name, weeks in series.items():
|
||
bucket = collected.setdefault(name, {})
|
||
for week_key, pairs in weeks.items():
|
||
bucket.setdefault(week_key, []).extend(pairs)
|
||
fip_signal_eval = [
|
||
row for row in bt._signal_evaluation(collected) if row.get("signal") == "fip_id"
|
||
]
|
||
|
||
if cache_path is not None:
|
||
cache_path.parent.mkdir(parents=True, exist_ok=True)
|
||
with cache_path.open("wb") as handle:
|
||
pickle.dump(
|
||
{
|
||
"key": cache_key,
|
||
"entry_candidate_count": entry_candidate_count,
|
||
"qualified_candidates": qualified,
|
||
"fip_signal_eval": fip_signal_eval,
|
||
},
|
||
handle,
|
||
protocol=pickle.HIGHEST_PROTOCOL,
|
||
)
|
||
if not args.quiet:
|
||
print(f"wrote candidate cache: {cache_path}", flush=True)
|
||
|
||
if not qualified:
|
||
raise SystemExit("No qualified long candidates after replay")
|
||
|
||
strategy = next(s for s in bt.PORTFOLIO_MONITOR_STRATEGIES if s.get("is_production"))
|
||
entry_config = bt._entry_variant_config(str(strategy["entry_variant"]))
|
||
if entry_config is None:
|
||
raise RuntimeError("Production entry configuration missing")
|
||
ranking_key = str(entry_config.get("ranking_key") or entry_config["percentile_key"])
|
||
exit_policy = bt.LIVE_EXIT_MODE_TO_SIM.get(
|
||
str(exit_config.get("mode", "atr_trailing")), "atr_trail3"
|
||
)
|
||
default_hold = int(exit_config.get("hold_days", 30))
|
||
trail_multiplier = float(exit_config.get("atr_multiplier", bt.ATR_TRAIL_MULTIPLIER))
|
||
risk_per_trade = float(entry_config["risk_per_trade"])
|
||
max_positions = int(entry_config["max_positions"])
|
||
|
||
post_stop_reentry_fn = bt._make_gate_reset_reentry_fn(
|
||
qualified,
|
||
prices,
|
||
cadence=args.cadence,
|
||
ranking_key=ranking_key,
|
||
)
|
||
|
||
selected_arms = [
|
||
arm for arm in PRE_REGISTERED_ARMS if _arm_selected(arm, only, skip)
|
||
]
|
||
# Always include control when grading promotions for non-control arms.
|
||
if selected_arms and not any(a["id"] == "a0_control" for a in selected_arms):
|
||
if only is None or "a0" in (only or set()) or "a0_control" in (only or set()):
|
||
pass
|
||
else:
|
||
# Force control into the run for comparison baselines.
|
||
control_arm = next(a for a in PRE_REGISTERED_ARMS if a["id"] == "a0_control")
|
||
selected_arms = [control_arm, *selected_arms]
|
||
|
||
if not selected_arms:
|
||
raise SystemExit("No arms selected — check --only / --skip")
|
||
|
||
report: dict[str, Any] = {
|
||
"generated_at": datetime.now().isoformat(),
|
||
"snapshot": str(snapshot.resolve()),
|
||
"cadence": args.cadence,
|
||
"validation_split": validation_split.isoformat(),
|
||
"n_trials": PRE_REGISTERED_N_TRIALS,
|
||
"pre_registered_arm_ids": list(PRE_REGISTERED_ARM_IDS),
|
||
"selected_arm_ids": [a["id"] for a in selected_arms],
|
||
"entry_candidate_count": entry_candidate_count,
|
||
"qualified_longs": len(qualified),
|
||
"promotion_rule": (
|
||
"Promote only if validation Sharpe ≥ control, validation DD not worse "
|
||
"by >2pp, and train Sharpe not worse. Always report whether validation "
|
||
"Sharpe delta exceeds 1 SE (expect most will not)."
|
||
),
|
||
"fip_id_fingerprint": fip_signal_eval,
|
||
"arms": [],
|
||
"promotion": {},
|
||
}
|
||
_write_checkpoint(out_path, report)
|
||
|
||
control_result: dict | None = None
|
||
|
||
def run_arm(arm: dict[str, Any]) -> dict:
|
||
hold_days = int(arm.get("hold_days", default_hold))
|
||
fill_mode = str(arm.get("fill_mode", bt.FILL_MODE_CLOSE))
|
||
windows: list[dict] = []
|
||
for window_name, start, end in (
|
||
("train", None, validation_split),
|
||
("validation", validation_split, None),
|
||
("full", None, None),
|
||
):
|
||
sim = bt._simulate_portfolio(
|
||
qualified,
|
||
prices,
|
||
benchmark_closes,
|
||
exit_policy,
|
||
hold_days,
|
||
ranking_key=ranking_key,
|
||
max_positions=max_positions,
|
||
risk_per_trade=risk_per_trade,
|
||
atr_trail_multiplier=trail_multiplier,
|
||
post_stop_reentry_fn=post_stop_reentry_fn,
|
||
start_date=start,
|
||
end_date=end,
|
||
fill_mode=fill_mode,
|
||
vol_target=arm.get("vol_target"),
|
||
vol_lookback=int(arm.get("vol_lookback", bt.VOL_TARGET_LOOKBACK_HEADLINE)),
|
||
vol_clamp=tuple(arm.get("vol_clamp", bt.VOL_TARGET_CLAMP_HEADLINE)),
|
||
corr_max=arm.get("corr_max"),
|
||
corr_action=str(arm.get("corr_action", "skip")),
|
||
include_trades=True,
|
||
)
|
||
if sim is None:
|
||
windows.append({"window": window_name, "error": "no_trades"})
|
||
continue
|
||
_assert_calendar_truncation(sim, hold_days, fill_mode)
|
||
dsr = bt.deflated_sharpe_ratio(
|
||
sim.get("sharpe"),
|
||
sim.get("sharpe_se"),
|
||
PRE_REGISTERED_N_TRIALS,
|
||
n_returns=sim.get("n_returns"),
|
||
return_skew=sim.get("return_skew"),
|
||
return_kurtosis=sim.get("return_kurtosis"),
|
||
)
|
||
# Drop heavy trade lists from the checkpointed JSON.
|
||
sim.pop("trade_details", None)
|
||
sim.pop("equity_curve", None)
|
||
sim.pop("benchmark_curve", None)
|
||
sim.pop("reentry_events", None)
|
||
windows.append({"window": window_name, "dsr": dsr, **sim})
|
||
return {
|
||
"id": arm["id"],
|
||
"group": arm["group"],
|
||
"label": arm["label"],
|
||
"config": {
|
||
key: arm[key]
|
||
for key in arm
|
||
if key not in {"id", "group", "label"}
|
||
},
|
||
"windows": windows,
|
||
}
|
||
|
||
for arm in selected_arms:
|
||
if not args.quiet:
|
||
print(f"running arm {arm['id']} ...", flush=True)
|
||
result = run_arm(arm)
|
||
report["arms"].append(result)
|
||
if arm["id"] == "a0_control":
|
||
control_result = result
|
||
elif control_result is not None:
|
||
report["promotion"][arm["id"]] = _grade_promotion(
|
||
control_result, result, None
|
||
)
|
||
_write_checkpoint(out_path, report)
|
||
if not args.quiet:
|
||
val = _window(result, "validation") or {}
|
||
print(
|
||
f" done {arm['id']}: validation Sharpe={val.get('sharpe')} "
|
||
f"DD={val.get('max_drawdown_pct')} trades={val.get('trades')}",
|
||
flush=True,
|
||
)
|
||
|
||
# Re-grade all arms once control is known (handles --only without ordering issues).
|
||
if control_result is not None:
|
||
for result in report["arms"]:
|
||
if result["id"] == "a0_control":
|
||
continue
|
||
report["promotion"][result["id"]] = _grade_promotion(
|
||
control_result, result, None
|
||
)
|
||
_write_checkpoint(out_path, report)
|
||
|
||
if not args.quiet:
|
||
print(f"wrote {out_path}", flush=True)
|
||
print(f"wrote {out_path.with_suffix('.md')}", flush=True)
|
||
fip = (fip_signal_eval or [{}])[0] if fip_signal_eval else {}
|
||
if fip:
|
||
print(
|
||
f"fip_id fingerprint: mean_ic={fip.get('mean_ic')} "
|
||
f"t={fip.get('ic_t_stat')} (target ≈ -0.045 / -2.9)",
|
||
flush=True,
|
||
)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
asyncio.run(_main())
|