feat: add Phase A research matrix (vol target, fill, corr, SE/DSR)
Ship shared Sharpe SE/PSR diagnostics, next-open fill and equity-curve vol targeting in the portfolio simulator, re-derived fip_id, and a checkpointed offline matrix runner for Mac-side validation sweeps.
This commit is contained in:
@@ -0,0 +1,814 @@
|
||||
"""Phase-A research matrix: max-hold, vol targeting, next-open fill, corr caps.
|
||||
|
||||
Promotion rule (pre-registered — do not edit after a run starts)
|
||||
----------------------------------------------------------------
|
||||
An arm may be promoted over the close-fill production **control** only if ALL of:
|
||||
|
||||
1. Validation-window (entries ≥ ``--validation-split``, default 2024-07-01)
|
||||
Sharpe ≥ control validation Sharpe.
|
||||
2. Validation max drawdown is not worse than control by more than 2 percentage
|
||||
points (higher DD is worse).
|
||||
3. Train-window Sharpe is not worse than control train Sharpe (both-windows
|
||||
consistency — same standard as the min_rr sweep).
|
||||
4. Report whether the validation Sharpe delta exceeds 1 × SE (control or arm);
|
||||
most arms will fail this distinguishability check — that is expected and is
|
||||
the reason SE/PSR ship on every row. Failing the 1-SE bar does **not** alone
|
||||
veto promotion under (1)–(3), but it must be stated.
|
||||
|
||||
Naming: the post-split window is called **validation**, not "holdout". It has
|
||||
been opened by prior experiments; treat it as a disciplined check, not a
|
||||
pristine sample.
|
||||
|
||||
Arms (pre-registered; N used for Deflated Sharpe)
|
||||
-------------------------------------------------
|
||||
- A0 control: production gate/rank/trail, hold=30, close fill, no vol target, no corr cap
|
||||
- A2 max-hold: hold ∈ {30, 45, 60, 90} (30 is the control row; listed once)
|
||||
- A3 vol-target: target ∈ {15%, 20%, 25%} × clamp {[0.5,1.5], [0.25,2.0]} at lookback 60;
|
||||
plus sensitivity lookbacks {20, 126} at target 20% / clamp [0.5,1.5] only
|
||||
- A4 next-open fill (measurement + portfolio consequence vs control)
|
||||
- A5 corr cap: threshold ∈ {0.6, 0.7, 0.8} × action ∈ {skip, half-size}
|
||||
|
||||
DSR uses N = number of pre-registered strategy arms in this matrix (see
|
||||
``PRE_REGISTERED_ARM_IDS``). Standalone backtests do not invent a DSR.
|
||||
|
||||
Calendar truncation
|
||||
-------------------
|
||||
The simulator always cuts the equity calendar at last_signal + hold_days
|
||||
(+1 for next-open). The runner asserts validation end_date ≤ last price date and
|
||||
that the sim end is within hold_days+pad of the last admitted signal so a 90d
|
||||
arm cannot sit in trailing flat cash.
|
||||
|
||||
Usage
|
||||
-----
|
||||
python scripts/run_research_matrix.py backtest_snapshots/prod.sqlite \\
|
||||
--workers 7 --allow-spawn --candidate-cache reports/.cache/research-cands.pkl
|
||||
|
||||
python scripts/run_research_matrix.py ... --only a2,a3
|
||||
python scripts/run_research_matrix.py ... --skip a4
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import multiprocessing
|
||||
import os
|
||||
import pickle
|
||||
import sys
|
||||
from concurrent.futures import ProcessPoolExecutor, as_completed
|
||||
from datetime import date, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
CACHE_VERSION = "research-matrix-v1-daily-prod"
|
||||
|
||||
# Pre-registered arm catalogue (order is report order). Control is a0.
|
||||
# Count N for DSR excludes pure measurement-only rows if any; every arm below
|
||||
# is a portfolio book and counts.
|
||||
PRE_REGISTERED_ARMS: tuple[dict[str, Any], ...] = (
|
||||
{
|
||||
"id": "a0_control",
|
||||
"group": "a0",
|
||||
"label": "Control: close fill, hold 30, risk 1%, no corr/vol",
|
||||
"hold_days": 30,
|
||||
"fill_mode": "close",
|
||||
},
|
||||
# A2 — max hold (30 is control; still emitted as a2 for the sweep table)
|
||||
{"id": "a2_hold_30", "group": "a2", "label": "Max hold 30", "hold_days": 30},
|
||||
{"id": "a2_hold_45", "group": "a2", "label": "Max hold 45", "hold_days": 45},
|
||||
{"id": "a2_hold_60", "group": "a2", "label": "Max hold 60", "hold_days": 60},
|
||||
{"id": "a2_hold_90", "group": "a2", "label": "Max hold 90", "hold_days": 90},
|
||||
# A3 — vol targeting
|
||||
{
|
||||
"id": "a3_vt15_c05_15_lb60",
|
||||
"group": "a3",
|
||||
"label": "Vol target 15% clamp[0.5,1.5] lb60",
|
||||
"vol_target": 0.15,
|
||||
"vol_clamp": (0.5, 1.5),
|
||||
"vol_lookback": 60,
|
||||
},
|
||||
{
|
||||
"id": "a3_vt20_c05_15_lb60",
|
||||
"group": "a3",
|
||||
"label": "Vol target 20% clamp[0.5,1.5] lb60",
|
||||
"vol_target": 0.20,
|
||||
"vol_clamp": (0.5, 1.5),
|
||||
"vol_lookback": 60,
|
||||
},
|
||||
{
|
||||
"id": "a3_vt25_c05_15_lb60",
|
||||
"group": "a3",
|
||||
"label": "Vol target 25% clamp[0.5,1.5] lb60",
|
||||
"vol_target": 0.25,
|
||||
"vol_clamp": (0.5, 1.5),
|
||||
"vol_lookback": 60,
|
||||
},
|
||||
{
|
||||
"id": "a3_vt15_c025_20_lb60",
|
||||
"group": "a3",
|
||||
"label": "Vol target 15% clamp[0.25,2.0] lb60",
|
||||
"vol_target": 0.15,
|
||||
"vol_clamp": (0.25, 2.0),
|
||||
"vol_lookback": 60,
|
||||
},
|
||||
{
|
||||
"id": "a3_vt20_c025_20_lb60",
|
||||
"group": "a3",
|
||||
"label": "Vol target 20% clamp[0.25,2.0] lb60",
|
||||
"vol_target": 0.20,
|
||||
"vol_clamp": (0.25, 2.0),
|
||||
"vol_lookback": 60,
|
||||
},
|
||||
{
|
||||
"id": "a3_vt25_c025_20_lb60",
|
||||
"group": "a3",
|
||||
"label": "Vol target 25% clamp[0.25,2.0] lb60",
|
||||
"vol_target": 0.25,
|
||||
"vol_clamp": (0.25, 2.0),
|
||||
"vol_lookback": 60,
|
||||
},
|
||||
{
|
||||
"id": "a3_vt20_c05_15_lb20",
|
||||
"group": "a3",
|
||||
"label": "Vol target 20% clamp[0.5,1.5] lb20 (sensitivity)",
|
||||
"vol_target": 0.20,
|
||||
"vol_clamp": (0.5, 1.5),
|
||||
"vol_lookback": 20,
|
||||
},
|
||||
{
|
||||
"id": "a3_vt20_c05_15_lb126",
|
||||
"group": "a3",
|
||||
"label": "Vol target 20% clamp[0.5,1.5] lb126 (sensitivity)",
|
||||
"vol_target": 0.20,
|
||||
"vol_clamp": (0.5, 1.5),
|
||||
"vol_lookback": 126,
|
||||
},
|
||||
# A4 — next-open fill
|
||||
{
|
||||
"id": "a4_next_open",
|
||||
"group": "a4",
|
||||
"label": "Next-open fill (t+1 open, stop from fill−1.5 ATR)",
|
||||
"fill_mode": "next_open",
|
||||
},
|
||||
# A5 — correlation caps
|
||||
{
|
||||
"id": "a5_corr06_skip",
|
||||
"group": "a5",
|
||||
"label": "Corr max 0.6 skip",
|
||||
"corr_max": 0.6,
|
||||
"corr_action": "skip",
|
||||
},
|
||||
{
|
||||
"id": "a5_corr07_skip",
|
||||
"group": "a5",
|
||||
"label": "Corr max 0.7 skip",
|
||||
"corr_max": 0.7,
|
||||
"corr_action": "skip",
|
||||
},
|
||||
{
|
||||
"id": "a5_corr08_skip",
|
||||
"group": "a5",
|
||||
"label": "Corr max 0.8 skip",
|
||||
"corr_max": 0.8,
|
||||
"corr_action": "skip",
|
||||
},
|
||||
{
|
||||
"id": "a5_corr06_half",
|
||||
"group": "a5",
|
||||
"label": "Corr max 0.6 half-size",
|
||||
"corr_max": 0.6,
|
||||
"corr_action": "half_size",
|
||||
},
|
||||
{
|
||||
"id": "a5_corr07_half",
|
||||
"group": "a5",
|
||||
"label": "Corr max 0.7 half-size",
|
||||
"corr_max": 0.7,
|
||||
"corr_action": "half_size",
|
||||
},
|
||||
{
|
||||
"id": "a5_corr08_half",
|
||||
"group": "a5",
|
||||
"label": "Corr max 0.8 half-size",
|
||||
"corr_max": 0.8,
|
||||
"corr_action": "half_size",
|
||||
},
|
||||
)
|
||||
PRE_REGISTERED_ARM_IDS = tuple(arm["id"] for arm in PRE_REGISTERED_ARMS)
|
||||
PRE_REGISTERED_N_TRIALS = len(PRE_REGISTERED_ARMS)
|
||||
|
||||
|
||||
def _sqlite_url(path: Path) -> str:
|
||||
return f"sqlite+aiosqlite:///{path.resolve().as_posix()}"
|
||||
|
||||
|
||||
def _period_percentiles(
|
||||
observations: list[dict], value_key: str
|
||||
) -> dict[tuple[str, str], float]:
|
||||
by_period: dict[tuple, list[dict]] = {}
|
||||
for row in observations:
|
||||
if row.get(value_key) is None:
|
||||
continue
|
||||
period = tuple(row["ranking_period"])
|
||||
by_period.setdefault(period, []).append(row)
|
||||
result: dict[tuple[str, str], float] = {}
|
||||
for group in by_period.values():
|
||||
ordered = sorted(
|
||||
group, key=lambda row: (float(row[value_key]), str(row["symbol"]))
|
||||
)
|
||||
denominator = len(ordered) - 1
|
||||
for rank, row in enumerate(ordered):
|
||||
result[(str(row["symbol"]), str(row["date"]))] = round(
|
||||
rank / denominator * 100.0 if denominator > 0 else 100.0,
|
||||
2,
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def _live_universe_rank_map(
|
||||
observations: list[dict],
|
||||
benchmark_closes: dict[date, float],
|
||||
momentum_weight: float,
|
||||
) -> dict[tuple[str, str], dict[str, float | None]]:
|
||||
raw_pct = _period_percentiles(observations, "momentum")
|
||||
residual_pct = _period_percentiles(observations, "residual_momentum")
|
||||
vol_pct = _period_percentiles(observations, "vol_6m")
|
||||
benchmark_ords = sorted(value.toordinal() for value in benchmark_closes)
|
||||
residual_start_ord = benchmark_ords[251] if len(benchmark_ords) >= 252 else None
|
||||
ranks: dict[tuple[str, str], dict[str, float | None]] = {}
|
||||
for row in observations:
|
||||
identity = (str(row["symbol"]), str(row["date"]))
|
||||
asof_ord = date.fromisoformat(identity[1]).toordinal()
|
||||
momentum_pct = (
|
||||
residual_pct.get(identity)
|
||||
if residual_start_ord is not None and asof_ord >= residual_start_ord
|
||||
else raw_pct.get(identity)
|
||||
)
|
||||
volatility_pct = vol_pct.get(identity)
|
||||
strategy_rank = (
|
||||
round(
|
||||
momentum_pct * momentum_weight
|
||||
+ volatility_pct * (1.0 - momentum_weight),
|
||||
2,
|
||||
)
|
||||
if momentum_pct is not None and volatility_pct is not None
|
||||
else momentum_pct
|
||||
)
|
||||
ranks[identity] = {
|
||||
"momentum_percentile": momentum_pct,
|
||||
"volatility_percentile": volatility_pct,
|
||||
"strategy_rank": strategy_rank,
|
||||
}
|
||||
return ranks
|
||||
|
||||
|
||||
def _parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description=__doc__,
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
)
|
||||
parser.add_argument("snapshot", help="SQLite backtest snapshot.")
|
||||
parser.add_argument("--workers", type=int, default=6)
|
||||
parser.add_argument(
|
||||
"--allow-spawn",
|
||||
action="store_true",
|
||||
help="Allow spawn multiprocessing (needed on Windows).",
|
||||
)
|
||||
parser.add_argument("--out", default=None, help="JSON report path.")
|
||||
parser.add_argument(
|
||||
"--candidate-cache",
|
||||
default=None,
|
||||
help="Optional pickle cache for the daily qualified candidate set.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--validation-split",
|
||||
default="2024-07-01",
|
||||
help="Train/validation entry split (YYYY-MM-DD). Validation = entries on/after.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--only",
|
||||
default=None,
|
||||
help="Comma-separated arm groups or ids to run (e.g. a2,a3 or a0_control,a4_next_open).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--skip",
|
||||
default=None,
|
||||
help="Comma-separated arm groups or ids to skip.",
|
||||
)
|
||||
parser.add_argument("--quiet", action="store_true")
|
||||
parser.add_argument(
|
||||
"--cadence",
|
||||
choices=("daily", "weekly"),
|
||||
default="daily",
|
||||
help="Candidate replay cadence. Daily matches the re-entry matrix production arm.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def _default_output_path() -> Path:
|
||||
stamp = datetime.now().strftime("%Y%m%d-%H%M%S")
|
||||
return Path("reports") / f"research-matrix-{stamp}.json"
|
||||
|
||||
|
||||
def _parse_selector(raw: str | None) -> set[str] | None:
|
||||
if raw is None or not raw.strip():
|
||||
return None
|
||||
return {part.strip().lower() for part in raw.split(",") if part.strip()}
|
||||
|
||||
|
||||
def _arm_selected(arm: dict[str, Any], only: set[str] | None, skip: set[str] | None) -> bool:
|
||||
arm_id = str(arm["id"]).lower()
|
||||
group = str(arm["group"]).lower()
|
||||
if skip and (arm_id in skip or group in skip):
|
||||
return False
|
||||
if only is None:
|
||||
return True
|
||||
return arm_id in only or group in only
|
||||
|
||||
|
||||
def _write_checkpoint(path: Path, report: dict) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = path.with_suffix(path.suffix + ".tmp")
|
||||
tmp.write_text(json.dumps(report, indent=2, default=str), encoding="utf-8")
|
||||
tmp.replace(path)
|
||||
md_path = path.with_suffix(".md")
|
||||
md_path.write_text(_markdown_table(report), encoding="utf-8")
|
||||
|
||||
|
||||
def _markdown_table(report: dict) -> str:
|
||||
lines = [
|
||||
f"# Research matrix — {report.get('generated_at', '')}",
|
||||
"",
|
||||
f"Validation split: **{report.get('validation_split')}**. "
|
||||
f"Pre-registered N for DSR: **{report.get('n_trials')}**.",
|
||||
"",
|
||||
"## Promotion rule",
|
||||
"",
|
||||
report.get("promotion_rule", ""),
|
||||
"",
|
||||
"## Arms",
|
||||
"",
|
||||
"| arm | window | Sharpe | SE | PSR | DSR | CAGR | MaxDD | Calmar | trades | avg scalar |",
|
||||
"|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|",
|
||||
]
|
||||
for arm in report.get("arms") or []:
|
||||
for window_row in arm.get("windows") or []:
|
||||
lines.append(
|
||||
"| {arm} | {window} | {sharpe} | {se} | {psr} | {dsr} | {cagr} | {dd} | {calmar} | {trades} | {scalar} |".format(
|
||||
arm=arm.get("id"),
|
||||
window=window_row.get("window"),
|
||||
sharpe=_fmt(window_row.get("sharpe")),
|
||||
se=_fmt(window_row.get("sharpe_se")),
|
||||
psr=_fmt(window_row.get("psr")),
|
||||
dsr=_fmt(window_row.get("dsr")),
|
||||
cagr=_fmt(window_row.get("cagr_pct")),
|
||||
dd=_fmt(window_row.get("max_drawdown_pct")),
|
||||
calmar=_fmt(window_row.get("calmar")),
|
||||
trades=_fmt(window_row.get("trades")),
|
||||
scalar=_fmt(window_row.get("avg_vol_scalar")),
|
||||
)
|
||||
)
|
||||
promo = report.get("promotion") or {}
|
||||
lines.extend(["", "## Promotion decisions", ""])
|
||||
if not promo:
|
||||
lines.append("_No arms graded yet._")
|
||||
else:
|
||||
for arm_id, decision in promo.items():
|
||||
lines.append(
|
||||
f"- **{arm_id}**: {'PROMOTE' if decision.get('promote') else 'reject'} — "
|
||||
f"{decision.get('reason')}"
|
||||
)
|
||||
lines.append("")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def _fmt(value: Any) -> str:
|
||||
if value is None:
|
||||
return "—"
|
||||
if isinstance(value, float):
|
||||
return f"{value:.3g}"
|
||||
return str(value)
|
||||
|
||||
|
||||
def _grade_promotion(control: dict, arm: dict, se_ref: float | None) -> dict:
|
||||
"""Apply the pre-registered promotion rule. control/arm are arm result dicts."""
|
||||
c_val = _window(control, "validation")
|
||||
a_val = _window(arm, "validation")
|
||||
c_train = _window(control, "train")
|
||||
a_train = _window(arm, "train")
|
||||
if not c_val or not a_val or not c_train or not a_train:
|
||||
return {"promote": False, "reason": "missing train/validation rows"}
|
||||
|
||||
c_s = c_val.get("sharpe")
|
||||
a_s = a_val.get("sharpe")
|
||||
c_dd = c_val.get("max_drawdown_pct")
|
||||
a_dd = a_val.get("max_drawdown_pct")
|
||||
c_ts = c_train.get("sharpe")
|
||||
a_ts = a_train.get("sharpe")
|
||||
if None in (c_s, a_s, c_dd, a_dd, c_ts, a_ts):
|
||||
return {"promote": False, "reason": "missing Sharpe/DD on a required window"}
|
||||
|
||||
delta = float(a_s) - float(c_s)
|
||||
se = se_ref
|
||||
if se is None:
|
||||
se = a_val.get("sharpe_se") or c_val.get("sharpe_se")
|
||||
exceeds_1se = se is not None and abs(delta) > float(se)
|
||||
|
||||
checks = {
|
||||
"validation_sharpe_ge_control": float(a_s) >= float(c_s),
|
||||
"validation_dd_not_worse_by_2pp": float(a_dd) <= float(c_dd) + 2.0,
|
||||
"train_sharpe_not_worse": float(a_ts) >= float(c_ts),
|
||||
"delta_exceeds_1se": exceeds_1se,
|
||||
"validation_sharpe_delta": round(delta, 4),
|
||||
"se_used": se,
|
||||
}
|
||||
promote = (
|
||||
checks["validation_sharpe_ge_control"]
|
||||
and checks["validation_dd_not_worse_by_2pp"]
|
||||
and checks["train_sharpe_not_worse"]
|
||||
)
|
||||
if promote:
|
||||
reason = (
|
||||
f"validation Sharpe {a_s} ≥ control {c_s}; "
|
||||
f"DD {a_dd} within +2pp of {c_dd}; train Sharpe {a_ts} ≥ {c_ts}"
|
||||
)
|
||||
if not exceeds_1se:
|
||||
reason += " (delta ≤ 1 SE — distinguishable noise bar not cleared)"
|
||||
else:
|
||||
reason += " (delta > 1 SE)"
|
||||
else:
|
||||
failed = [k for k, v in checks.items() if k.startswith(("validation", "train")) and v is False]
|
||||
reason = "failed: " + ", ".join(failed) if failed else "failed promotion checks"
|
||||
return {"promote": promote, "reason": reason, "checks": checks}
|
||||
|
||||
|
||||
def _window(arm_result: dict, name: str) -> dict | None:
|
||||
for row in arm_result.get("windows") or []:
|
||||
if row.get("window") == name:
|
||||
return row
|
||||
return None
|
||||
|
||||
|
||||
def _assert_calendar_truncation(sim: dict, hold_days: int, fill_mode: str) -> None:
|
||||
"""Guard against trailing flat-cash after the last resolvable signal."""
|
||||
start = sim.get("start_date")
|
||||
end = sim.get("end_date")
|
||||
if not start or not end:
|
||||
return
|
||||
# Soft check: book span should not massively exceed hold window beyond data needs.
|
||||
# Hard assert lives on entry-end vs sim end when trade_details present.
|
||||
details = sim.get("trade_details") or []
|
||||
if not details:
|
||||
return
|
||||
last_entry = max(date.fromisoformat(t["entry_date"]) for t in details)
|
||||
sim_end = date.fromisoformat(str(end))
|
||||
pad = hold_days + (1 if fill_mode == "next_open" else 0)
|
||||
# Allow calendar days ≈ trading-day pad with weekend slack (2×).
|
||||
max_slack_days = pad * 2 + 5
|
||||
if (sim_end - last_entry).days > max_slack_days:
|
||||
raise AssertionError(
|
||||
f"calendar truncation failed: last entry {last_entry} but sim end "
|
||||
f"{sim_end} (hold_days={hold_days}, fill_mode={fill_mode})"
|
||||
)
|
||||
|
||||
|
||||
async def _main() -> None:
|
||||
args = _parse_args()
|
||||
snapshot = Path(args.snapshot)
|
||||
if not snapshot.exists():
|
||||
raise SystemExit(f"Snapshot not found: {snapshot}")
|
||||
if args.workers < 1:
|
||||
raise SystemExit("--workers must be positive")
|
||||
|
||||
only = _parse_selector(args.only)
|
||||
skip = _parse_selector(args.skip)
|
||||
validation_split = date.fromisoformat(args.validation_split)
|
||||
out_path = Path(args.out) if args.out else _default_output_path()
|
||||
|
||||
os.environ["BACKTEST_SNAPSHOT_OFFLINE"] = "1"
|
||||
if args.allow_spawn:
|
||||
os.environ["BACKTEST_ALLOW_SPAWN"] = "1"
|
||||
|
||||
from app.models.ticker import Ticker
|
||||
from app.services import backtest_service as bt
|
||||
from app.services.admin_service import get_activation_config
|
||||
from app.services.paper_trade_service import get_exit_policy
|
||||
from app.services.recommendation_service import get_recommendation_config
|
||||
|
||||
db_engine = create_async_engine(_sqlite_url(snapshot), pool_pre_ping=True)
|
||||
Session = async_sessionmaker(db_engine, class_=AsyncSession, expire_on_commit=False)
|
||||
try:
|
||||
async with Session() as db:
|
||||
recommendation_config = await get_recommendation_config(db)
|
||||
activation = await get_activation_config(db)
|
||||
exit_config = await get_exit_policy(db)
|
||||
benchmark_closes = await bt._load_benchmark_closes_for_backtest(
|
||||
db, days=None, refresh=False
|
||||
)
|
||||
ticker_result = await db.execute(select(Ticker).order_by(Ticker.symbol))
|
||||
symbols = [t.symbol for t in ticker_result.scalars().all()]
|
||||
prices: dict[str, tuple] = {}
|
||||
for index, symbol in enumerate(symbols, 1):
|
||||
columns = await bt._fetch_columns(db, symbol)
|
||||
if columns is not None:
|
||||
prices[symbol] = columns
|
||||
if not args.quiet and index % 50 == 0:
|
||||
print(f"loaded prices: {index}/{len(symbols)}", flush=True)
|
||||
finally:
|
||||
await db_engine.dispose()
|
||||
|
||||
if not prices:
|
||||
raise SystemExit("No price columns loaded from snapshot")
|
||||
|
||||
snapshot_stat = snapshot.stat()
|
||||
cache_key = {
|
||||
"version": CACHE_VERSION,
|
||||
"snapshot": str(snapshot.resolve()),
|
||||
"snapshot_size": snapshot_stat.st_size,
|
||||
"snapshot_mtime_ns": snapshot_stat.st_mtime_ns,
|
||||
"cadence": args.cadence,
|
||||
"target_model": "production_gtl",
|
||||
}
|
||||
cache_path = Path(args.candidate_cache) if args.candidate_cache else None
|
||||
qualified: list[dict] | None = None
|
||||
entry_candidate_count = 0
|
||||
fip_signal_eval: list[dict] | None = None
|
||||
|
||||
if cache_path is not None and cache_path.exists():
|
||||
with cache_path.open("rb") as handle:
|
||||
cached = pickle.load(handle) # noqa: S301 - trusted local cache
|
||||
if cached.get("key") == cache_key:
|
||||
qualified = list(cached["qualified_candidates"])
|
||||
entry_candidate_count = int(cached["entry_candidate_count"])
|
||||
fip_signal_eval = cached.get("fip_signal_eval")
|
||||
if not args.quiet:
|
||||
print(f"loaded candidate cache: {cache_path}", flush=True)
|
||||
elif not args.quiet:
|
||||
print(f"candidate cache mismatch; rebuilding: {cache_path}", flush=True)
|
||||
|
||||
if qualified is None:
|
||||
replay_start = date(1900, 1, 1)
|
||||
workers = max(1, min(int(args.workers), max(1, multiprocessing.cpu_count() - 1)))
|
||||
context = bt._mp_context() or multiprocessing.get_context("spawn")
|
||||
replay_rows: list[dict] = []
|
||||
with ProcessPoolExecutor(max_workers=workers, mp_context=context) as pool:
|
||||
futures = {
|
||||
pool.submit(
|
||||
bt._replay_candidates_for_period,
|
||||
symbol,
|
||||
columns,
|
||||
recommendation_config,
|
||||
activation,
|
||||
benchmark_closes,
|
||||
replay_start,
|
||||
args.cadence,
|
||||
True,
|
||||
True,
|
||||
): symbol
|
||||
for symbol, columns in prices.items()
|
||||
}
|
||||
for index, future in enumerate(as_completed(futures), 1):
|
||||
replay_rows.extend(future.result())
|
||||
if not args.quiet and index % 25 == 0:
|
||||
print(f"replay: {index}/{len(futures)} tickers", flush=True)
|
||||
|
||||
setup_candidates = [row for row in replay_rows if not row.get("_rank_only")]
|
||||
rank_observations = [
|
||||
row for row in replay_rows if row.get("_universe_rank_observation")
|
||||
]
|
||||
entry_candidate_count = len(setup_candidates)
|
||||
|
||||
# Live-universe ranking (same semantics as run_daily_reentry_matrix).
|
||||
live_ranks = _live_universe_rank_map(
|
||||
rank_observations,
|
||||
benchmark_closes,
|
||||
bt.STRATEGY_RANK_MOMENTUM_WEIGHT,
|
||||
)
|
||||
threshold = float(activation.get("min_momentum_percentile", 80.0))
|
||||
qualified = []
|
||||
for setup in setup_candidates:
|
||||
if setup.get("direction") != "long":
|
||||
continue
|
||||
candidate = {
|
||||
key: value
|
||||
for key, value in setup.items()
|
||||
if not key.startswith("_universe_")
|
||||
}
|
||||
identity = (str(setup["symbol"]), str(setup["date"]))
|
||||
rank = live_ranks.get(identity)
|
||||
if rank is None:
|
||||
continue
|
||||
candidate[bt.PRODUCTION_PERCENTILE_KEY] = rank["momentum_percentile"]
|
||||
candidate[bt.VOL_PERCENTILE_KEY] = rank["volatility_percentile"]
|
||||
candidate[bt.RESIDUAL_HIGH_VOL_BLEND_80_20_KEY] = rank["strategy_rank"]
|
||||
candidate["qualified"] = bt._momentum_qualifies(candidate, threshold)
|
||||
if candidate["qualified"]:
|
||||
qualified.append(candidate)
|
||||
|
||||
# fip_id fingerprint via the shared weekly signal harness.
|
||||
collected: dict = {}
|
||||
for symbol, columns in prices.items():
|
||||
# Rebuild minimal records for signal eval from column arrays.
|
||||
ords, _o, highs, _l, closes, _v = columns
|
||||
records = [
|
||||
type("R", (), {"date": date.fromordinal(int(ords[i])), "close": closes[i], "high": highs[i]})()
|
||||
for i in range(len(ords))
|
||||
]
|
||||
series = bt._signal_series(records, benchmark_closes)
|
||||
for name, weeks in series.items():
|
||||
bucket = collected.setdefault(name, {})
|
||||
for week_key, pairs in weeks.items():
|
||||
bucket.setdefault(week_key, []).extend(pairs)
|
||||
fip_signal_eval = [
|
||||
row for row in bt._signal_evaluation(collected) if row.get("signal") == "fip_id"
|
||||
]
|
||||
|
||||
if cache_path is not None:
|
||||
cache_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with cache_path.open("wb") as handle:
|
||||
pickle.dump(
|
||||
{
|
||||
"key": cache_key,
|
||||
"entry_candidate_count": entry_candidate_count,
|
||||
"qualified_candidates": qualified,
|
||||
"fip_signal_eval": fip_signal_eval,
|
||||
},
|
||||
handle,
|
||||
protocol=pickle.HIGHEST_PROTOCOL,
|
||||
)
|
||||
if not args.quiet:
|
||||
print(f"wrote candidate cache: {cache_path}", flush=True)
|
||||
|
||||
if not qualified:
|
||||
raise SystemExit("No qualified long candidates after replay")
|
||||
|
||||
strategy = next(s for s in bt.PORTFOLIO_MONITOR_STRATEGIES if s.get("is_production"))
|
||||
entry_config = bt._entry_variant_config(str(strategy["entry_variant"]))
|
||||
if entry_config is None:
|
||||
raise RuntimeError("Production entry configuration missing")
|
||||
ranking_key = str(entry_config.get("ranking_key") or entry_config["percentile_key"])
|
||||
exit_policy = bt.LIVE_EXIT_MODE_TO_SIM.get(
|
||||
str(exit_config.get("mode", "atr_trailing")), "atr_trail3"
|
||||
)
|
||||
default_hold = int(exit_config.get("hold_days", 30))
|
||||
trail_multiplier = float(exit_config.get("atr_multiplier", bt.ATR_TRAIL_MULTIPLIER))
|
||||
risk_per_trade = float(entry_config["risk_per_trade"])
|
||||
max_positions = int(entry_config["max_positions"])
|
||||
|
||||
post_stop_reentry_fn = bt._make_gate_reset_reentry_fn(
|
||||
qualified,
|
||||
prices,
|
||||
cadence=args.cadence,
|
||||
ranking_key=ranking_key,
|
||||
)
|
||||
|
||||
selected_arms = [
|
||||
arm for arm in PRE_REGISTERED_ARMS if _arm_selected(arm, only, skip)
|
||||
]
|
||||
# Always include control when grading promotions for non-control arms.
|
||||
if selected_arms and not any(a["id"] == "a0_control" for a in selected_arms):
|
||||
if only is None or "a0" in (only or set()) or "a0_control" in (only or set()):
|
||||
pass
|
||||
else:
|
||||
# Force control into the run for comparison baselines.
|
||||
control_arm = next(a for a in PRE_REGISTERED_ARMS if a["id"] == "a0_control")
|
||||
selected_arms = [control_arm, *selected_arms]
|
||||
|
||||
if not selected_arms:
|
||||
raise SystemExit("No arms selected — check --only / --skip")
|
||||
|
||||
report: dict[str, Any] = {
|
||||
"generated_at": datetime.now().isoformat(),
|
||||
"snapshot": str(snapshot.resolve()),
|
||||
"cadence": args.cadence,
|
||||
"validation_split": validation_split.isoformat(),
|
||||
"n_trials": PRE_REGISTERED_N_TRIALS,
|
||||
"pre_registered_arm_ids": list(PRE_REGISTERED_ARM_IDS),
|
||||
"selected_arm_ids": [a["id"] for a in selected_arms],
|
||||
"entry_candidate_count": entry_candidate_count,
|
||||
"qualified_longs": len(qualified),
|
||||
"promotion_rule": (
|
||||
"Promote only if validation Sharpe ≥ control, validation DD not worse "
|
||||
"by >2pp, and train Sharpe not worse. Always report whether validation "
|
||||
"Sharpe delta exceeds 1 SE (expect most will not)."
|
||||
),
|
||||
"fip_id_fingerprint": fip_signal_eval,
|
||||
"arms": [],
|
||||
"promotion": {},
|
||||
}
|
||||
_write_checkpoint(out_path, report)
|
||||
|
||||
control_result: dict | None = None
|
||||
|
||||
def run_arm(arm: dict[str, Any]) -> dict:
|
||||
hold_days = int(arm.get("hold_days", default_hold))
|
||||
fill_mode = str(arm.get("fill_mode", bt.FILL_MODE_CLOSE))
|
||||
windows: list[dict] = []
|
||||
for window_name, start, end in (
|
||||
("train", None, validation_split),
|
||||
("validation", validation_split, None),
|
||||
("full", None, None),
|
||||
):
|
||||
sim = bt._simulate_portfolio(
|
||||
qualified,
|
||||
prices,
|
||||
benchmark_closes,
|
||||
exit_policy,
|
||||
hold_days,
|
||||
ranking_key=ranking_key,
|
||||
max_positions=max_positions,
|
||||
risk_per_trade=risk_per_trade,
|
||||
atr_trail_multiplier=trail_multiplier,
|
||||
post_stop_reentry_fn=post_stop_reentry_fn,
|
||||
start_date=start,
|
||||
end_date=end,
|
||||
fill_mode=fill_mode,
|
||||
vol_target=arm.get("vol_target"),
|
||||
vol_lookback=int(arm.get("vol_lookback", bt.VOL_TARGET_LOOKBACK_HEADLINE)),
|
||||
vol_clamp=tuple(arm.get("vol_clamp", bt.VOL_TARGET_CLAMP_HEADLINE)),
|
||||
corr_max=arm.get("corr_max"),
|
||||
corr_action=str(arm.get("corr_action", "skip")),
|
||||
include_trades=True,
|
||||
)
|
||||
if sim is None:
|
||||
windows.append({"window": window_name, "error": "no_trades"})
|
||||
continue
|
||||
_assert_calendar_truncation(sim, hold_days, fill_mode)
|
||||
dsr = bt.deflated_sharpe_ratio(
|
||||
sim.get("sharpe"),
|
||||
sim.get("sharpe_se"),
|
||||
PRE_REGISTERED_N_TRIALS,
|
||||
n_returns=sim.get("n_returns"),
|
||||
return_skew=sim.get("return_skew"),
|
||||
return_kurtosis=sim.get("return_kurtosis"),
|
||||
)
|
||||
# Drop heavy trade lists from the checkpointed JSON.
|
||||
sim.pop("trade_details", None)
|
||||
sim.pop("equity_curve", None)
|
||||
sim.pop("benchmark_curve", None)
|
||||
sim.pop("reentry_events", None)
|
||||
windows.append({"window": window_name, "dsr": dsr, **sim})
|
||||
return {
|
||||
"id": arm["id"],
|
||||
"group": arm["group"],
|
||||
"label": arm["label"],
|
||||
"config": {
|
||||
key: arm[key]
|
||||
for key in arm
|
||||
if key not in {"id", "group", "label"}
|
||||
},
|
||||
"windows": windows,
|
||||
}
|
||||
|
||||
for arm in selected_arms:
|
||||
if not args.quiet:
|
||||
print(f"running arm {arm['id']} ...", flush=True)
|
||||
result = run_arm(arm)
|
||||
report["arms"].append(result)
|
||||
if arm["id"] == "a0_control":
|
||||
control_result = result
|
||||
elif control_result is not None:
|
||||
report["promotion"][arm["id"]] = _grade_promotion(
|
||||
control_result, result, None
|
||||
)
|
||||
_write_checkpoint(out_path, report)
|
||||
if not args.quiet:
|
||||
val = _window(result, "validation") or {}
|
||||
print(
|
||||
f" done {arm['id']}: validation Sharpe={val.get('sharpe')} "
|
||||
f"DD={val.get('max_drawdown_pct')} trades={val.get('trades')}",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
# Re-grade all arms once control is known (handles --only without ordering issues).
|
||||
if control_result is not None:
|
||||
for result in report["arms"]:
|
||||
if result["id"] == "a0_control":
|
||||
continue
|
||||
report["promotion"][result["id"]] = _grade_promotion(
|
||||
control_result, result, None
|
||||
)
|
||||
_write_checkpoint(out_path, report)
|
||||
|
||||
if not args.quiet:
|
||||
print(f"wrote {out_path}", flush=True)
|
||||
print(f"wrote {out_path.with_suffix('.md')}", flush=True)
|
||||
fip = (fip_signal_eval or [{}])[0] if fip_signal_eval else {}
|
||||
if fip:
|
||||
print(
|
||||
f"fip_id fingerprint: mean_ic={fip.get('mean_ic')} "
|
||||
f"t={fip.get('ic_t_stat')} (target ≈ -0.045 / -2.9)",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(_main())
|
||||
Reference in New Issue
Block a user