dennisthiessen
06cf054f60
Extract app/ssl_bootstrap.py (shared with FastAPI main), wire it into research scripts, and teach run_tier1_macbook.sh to locate combined-ca-bundle.pem, certifi, optional USE_CORP_PROXY, plus --ssl-check diagnostics.
481 lines
17 KiB
Python
481 lines
17 KiB
Python
"""History-depth extension research (local / MacBook).
|
|
|
|
Phases
|
|
------
|
|
coverage — bars per calendar year; no rebuild
|
|
harness — race-guard snapshot, full signal_eval, era split pre/post-2021
|
|
|
|
Does not retune production knobs. Does not modify scheduler/gates.
|
|
|
|
Example
|
|
-------
|
|
python scripts/run_history_depth_research.py --phase coverage \\
|
|
--snapshot backtest_snapshots/prod.sqlite
|
|
|
|
python scripts/run_history_depth_research.py --phase harness \\
|
|
--snapshot backtest_snapshots/research.sqlite --workers 8 --allow-spawn
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import asyncio
|
|
import json
|
|
import os
|
|
import sys
|
|
from collections import defaultdict
|
|
from datetime import date, datetime
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from sqlalchemy import create_engine, text
|
|
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
if str(ROOT) not in sys.path:
|
|
sys.path.insert(0, str(ROOT))
|
|
|
|
from app.ssl_bootstrap import bootstrap_ssl # noqa: E402
|
|
|
|
bootstrap_ssl()
|
|
|
|
ERA_SPLIT = date(2021, 1, 1)
|
|
SURVIVORSHIP_BANNER = (
|
|
"SURVIVORSHIP BIAS: today's constituents backfilled historically. "
|
|
"Absolute Sharpe/CAGR levels on deep history are optimistic. "
|
|
"Use RELATIVE signal IC comparisons and era stability only — not levels."
|
|
)
|
|
|
|
|
|
def _sqlite_url(path: Path) -> str:
|
|
return f"sqlite+aiosqlite:///{path.resolve().as_posix()}"
|
|
|
|
|
|
def _parse_args() -> argparse.Namespace:
|
|
p = argparse.ArgumentParser(description=__doc__)
|
|
p.add_argument("--phase", choices=("coverage", "harness", "all"), default="all")
|
|
p.add_argument("--snapshot", default="backtest_snapshots/research.sqlite")
|
|
p.add_argument("--workers", type=int, default=8)
|
|
p.add_argument("--allow-spawn", action="store_true")
|
|
p.add_argument("--quiet", action="store_true")
|
|
p.add_argument("--out", default=None)
|
|
return p.parse_args()
|
|
|
|
|
|
def _coverage_report(snapshot: Path) -> dict[str, Any]:
|
|
engine = create_engine(
|
|
f"sqlite:///{snapshot.resolve().as_posix()}",
|
|
future=True,
|
|
)
|
|
try:
|
|
with engine.connect() as conn:
|
|
ticker_n = int(conn.execute(text("SELECT COUNT(*) FROM tickers")).scalar_one())
|
|
ohlcv_n = int(
|
|
conn.execute(text("SELECT COUNT(*) FROM ohlcv_records")).scalar_one()
|
|
)
|
|
d_range = conn.execute(
|
|
text("SELECT MIN(date), MAX(date) FROM ohlcv_records")
|
|
).fetchone()
|
|
# Bars per calendar year (global).
|
|
by_year = conn.execute(
|
|
text(
|
|
"""
|
|
SELECT substr(date, 1, 4) AS y, COUNT(*) AS n,
|
|
COUNT(DISTINCT ticker_id) AS tickers
|
|
FROM ohlcv_records
|
|
GROUP BY substr(date, 1, 4)
|
|
ORDER BY y
|
|
"""
|
|
)
|
|
).fetchall()
|
|
# Per-symbol min/max date + bar count (summary percentiles).
|
|
per_sym = conn.execute(
|
|
text(
|
|
"""
|
|
SELECT t.symbol, COUNT(*) AS n, MIN(o.date), MAX(o.date)
|
|
FROM ohlcv_records o
|
|
JOIN tickers t ON t.id = o.ticker_id
|
|
GROUP BY t.symbol
|
|
"""
|
|
)
|
|
).fetchall()
|
|
finally:
|
|
engine.dispose()
|
|
|
|
ns = sorted(int(r[1]) for r in per_sym)
|
|
def pct(p: float) -> int | None:
|
|
if not ns:
|
|
return None
|
|
i = int(round(p * (len(ns) - 1)))
|
|
return ns[i]
|
|
|
|
starts = sorted(str(r[2]) for r in per_sym if r[2])
|
|
start_hist: dict[str, int] = defaultdict(int)
|
|
for s in starts:
|
|
start_hist[s[:4]] += 1
|
|
|
|
return {
|
|
"snapshot": str(snapshot.resolve()),
|
|
"ticker_count": ticker_n,
|
|
"ohlcv_row_count": ohlcv_n,
|
|
"date_range": {"min": d_range[0], "max": d_range[1]},
|
|
"bars_per_year": [
|
|
{"year": y, "bars": n, "tickers_with_bars": t} for y, n, t in by_year
|
|
],
|
|
"bars_per_symbol": {
|
|
"min": ns[0] if ns else None,
|
|
"p10": pct(0.10),
|
|
"p50": pct(0.50),
|
|
"p90": pct(0.90),
|
|
"max": ns[-1] if ns else None,
|
|
},
|
|
"symbols_by_start_year": dict(sorted(start_hist.items())),
|
|
"note": (
|
|
"Where ticker counts drop in early years, the feed (or listing history) "
|
|
"thins — do not treat those years as a full 505-name cross-section."
|
|
),
|
|
"survivorship_banner": SURVIVORSHIP_BANNER,
|
|
}
|
|
|
|
|
|
def _assert_complete(snapshot: Path) -> dict[str, Any]:
|
|
from scripts.research_snapshot_manifest import ( # type: ignore
|
|
assert_research_snapshot_complete,
|
|
load_manifest,
|
|
)
|
|
|
|
m = load_manifest(snapshot)
|
|
if m is None:
|
|
# Prod snapshot may lack manifest; still require healthy bar depth.
|
|
eng = create_engine(
|
|
f"sqlite:///{snapshot.resolve().as_posix()}",
|
|
future=True,
|
|
)
|
|
try:
|
|
with eng.connect() as conn:
|
|
avg = conn.execute(
|
|
text(
|
|
"""
|
|
SELECT AVG(c) FROM (
|
|
SELECT COUNT(*) AS c FROM ohlcv_records GROUP BY ticker_id
|
|
)
|
|
"""
|
|
)
|
|
).scalar_one()
|
|
finally:
|
|
eng.dispose()
|
|
if avg is None or float(avg) < 400:
|
|
raise SystemExit(
|
|
f"No completion manifest and avg bars={avg} look short. "
|
|
"Rebuild research.sqlite via extend_snapshot_universe.py"
|
|
)
|
|
return {"manifest": None, "avg_bars": float(avg), "ok": True}
|
|
return {"manifest": assert_research_snapshot_complete(snapshot), "ok": True}
|
|
|
|
|
|
async def _harness(snapshot: Path, *, workers: int, quiet: bool) -> dict[str, Any]:
|
|
from app.config import settings
|
|
from app.services.backtest_service import run_backtest
|
|
|
|
os.environ["BACKTEST_SNAPSHOT_OFFLINE"] = "1"
|
|
os.environ["BACKTEST_SIGNAL_EVAL_ONLY"] = "1"
|
|
if Path("data/research/ticker_sector_map.json").exists():
|
|
os.environ["BACKTEST_SECTOR_MAP_PATH"] = str(
|
|
Path("data/research/ticker_sector_map.json").resolve()
|
|
)
|
|
settings.backtest_workers = workers
|
|
|
|
engine = create_async_engine(_sqlite_url(snapshot), pool_pre_ping=True)
|
|
Session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
|
|
|
|
def progress(done: int, total: int, symbol: str) -> None:
|
|
if quiet:
|
|
return
|
|
print(f" progress {done}/{total} {symbol}", end="\r", flush=True)
|
|
|
|
try:
|
|
async with Session() as db:
|
|
report = await run_backtest(db, progress_cb=progress, cadence="weekly")
|
|
finally:
|
|
await engine.dispose()
|
|
if not quiet:
|
|
print()
|
|
|
|
signal_eval = report.get("signal_eval") or []
|
|
|
|
# Era-split IC: recompute from collected is not available post-run.
|
|
# Approximate via second pass is expensive; instead document that era split
|
|
# requires collecting weekly ICs. We re-run evaluation if the report embeds
|
|
# nothing — for v1, call internal collection is too heavy to duplicate.
|
|
# Lightweight approach: mark era_split as requiring BACKTEST with custom
|
|
# filter — implemented below by re-scoring from a dedicated collection pass.
|
|
era = await _era_split_ics(snapshot, workers=workers, quiet=quiet)
|
|
|
|
return {
|
|
"survivorship_banner": SURVIVORSHIP_BANNER,
|
|
"signal_eval": signal_eval,
|
|
"era_split": era,
|
|
"params": report.get("params"),
|
|
"tickers": report.get("tickers"),
|
|
"generated_at_run": report.get("generated_at"),
|
|
}
|
|
|
|
|
|
async def _era_split_ics(
|
|
snapshot: Path, *, workers: int, quiet: bool
|
|
) -> dict[str, Any]:
|
|
"""Collect weekly signal series and evaluate pre/post ERA_SPLIT separately."""
|
|
from app.config import settings
|
|
from app.services import backtest_service as bt
|
|
from app.services.benchmark_service import load_benchmark_closes
|
|
from app.models.ticker import Ticker
|
|
from sqlalchemy import select
|
|
from collections import defaultdict as dd
|
|
|
|
os.environ["BACKTEST_SNAPSHOT_OFFLINE"] = "1"
|
|
settings.backtest_workers = max(1, workers)
|
|
|
|
engine = create_async_engine(_sqlite_url(snapshot), pool_pre_ping=True)
|
|
Session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
|
|
|
|
collected: dict = dd(lambda: dd(list))
|
|
try:
|
|
async with Session() as db:
|
|
tickers = list(
|
|
(await db.execute(select(Ticker).order_by(Ticker.symbol))).scalars()
|
|
)
|
|
spy = await load_benchmark_closes(db, "SPY")
|
|
symbol_to_sector = {}
|
|
sector_etf: dict = {}
|
|
try:
|
|
from app.services.sector_map import (
|
|
SECTOR_ETFS,
|
|
load_ticker_sector_map,
|
|
)
|
|
|
|
symbol_to_sector = load_ticker_sector_map()
|
|
for etf in SECTOR_ETFS:
|
|
series = await load_benchmark_closes(db, etf)
|
|
if series:
|
|
sector_etf[etf] = series
|
|
except Exception:
|
|
pass
|
|
|
|
for idx, t in enumerate(tickers):
|
|
if not quiet and idx % 100 == 0:
|
|
print(f" era-collect {idx}/{len(tickers)}", end="\r", flush=True)
|
|
cols = await bt._fetch_columns(db, t.symbol)
|
|
if cols is None:
|
|
continue
|
|
records = [
|
|
type(
|
|
"R",
|
|
(),
|
|
{
|
|
"date": date.fromordinal(int(cols[0][i])),
|
|
"close": cols[4][i],
|
|
"high": cols[2][i],
|
|
"volume": cols[5][i] if len(cols) > 5 else 0,
|
|
},
|
|
)()
|
|
for i in range(len(cols[0]))
|
|
]
|
|
series = bt._signal_series(
|
|
records,
|
|
spy,
|
|
symbol=t.symbol,
|
|
sector_etf_closes=bt._sector_etf_closes_for_symbol(
|
|
t.symbol, symbol_to_sector, sector_etf
|
|
),
|
|
)
|
|
for name, weeks in series.items():
|
|
for wk, pairs in weeks.items():
|
|
collected[name][wk].extend(pairs)
|
|
if symbol_to_sector:
|
|
bt._inject_sector_demeaned_momentum(collected, symbol_to_sector)
|
|
finally:
|
|
await engine.dispose()
|
|
if not quiet:
|
|
print()
|
|
|
|
def _filter_era(coll: dict, *, pre: bool) -> dict:
|
|
out: dict = dd(lambda: dd(list))
|
|
for name, weeks in coll.items():
|
|
for wk, recs in weeks.items():
|
|
# ISO week key (year, week) — approximate era by ISO year.
|
|
year = int(wk[0]) if isinstance(wk, tuple) else int(str(wk)[:4])
|
|
if pre and year >= ERA_SPLIT.year:
|
|
continue
|
|
if not pre and year < ERA_SPLIT.year:
|
|
continue
|
|
out[name][wk].extend(recs)
|
|
return out
|
|
|
|
pre_eval = bt._signal_evaluation(_filter_era(collected, pre=True))
|
|
post_eval = bt._signal_evaluation(_filter_era(collected, pre=False))
|
|
full_eval = bt._signal_evaluation(collected)
|
|
|
|
def _index(rows: list[dict]) -> dict[str, dict]:
|
|
return {r["signal"]: r for r in rows}
|
|
|
|
return {
|
|
"era_split_date": ERA_SPLIT.isoformat(),
|
|
"note": "Diagnostic only — not a tuning input. Nested lookbacks are not OOS.",
|
|
"full": _index(full_eval),
|
|
"pre_2021": _index(pre_eval),
|
|
"post_2021": _index(post_eval),
|
|
}
|
|
|
|
|
|
def _write_md(path: Path, payload: dict) -> None:
|
|
pre = path.read_text(encoding="utf-8") if path.exists() else ""
|
|
marker = "## Results"
|
|
idx = pre.find(marker)
|
|
header = pre[:idx] if idx >= 0 else pre.split("## Verdict")[0]
|
|
|
|
lines = [
|
|
header.rstrip(),
|
|
"",
|
|
"## Results",
|
|
"",
|
|
f"Generated: `{payload.get('generated_at')}`",
|
|
"",
|
|
f"> **{SURVIVORSHIP_BANNER}**",
|
|
"",
|
|
"### Coverage",
|
|
"",
|
|
f"```json\n{json.dumps(payload.get('coverage') or {}, indent=2, default=str)}\n```",
|
|
"",
|
|
"### Race guard",
|
|
"",
|
|
f"```json\n{json.dumps(payload.get('race_guard') or {}, indent=2, default=str)}\n```",
|
|
"",
|
|
"### Signal IC (full extended window)",
|
|
"",
|
|
]
|
|
harness = payload.get("harness") or {}
|
|
rows = harness.get("signal_eval") or []
|
|
if rows:
|
|
lines.extend([
|
|
"| signal | mean_ic | ic_t_stat | weeks | avg_N | reliable |",
|
|
"|---|---:|---:|---:|---:|---|",
|
|
])
|
|
for r in rows:
|
|
lines.append(
|
|
f"| {r.get('signal')} | {r.get('mean_ic')} | {r.get('ic_t_stat')} | "
|
|
f"{r.get('weeks')} | {r.get('avg_cross_section')} | {r.get('reliable')} |"
|
|
)
|
|
else:
|
|
lines.append("_Harness not run this pass._")
|
|
|
|
era = (harness.get("era_split") or {})
|
|
lines.extend(["", "### Era split (diagnostic only)", ""])
|
|
if era:
|
|
for label in ("full", "pre_2021", "post_2021"):
|
|
block = era.get(label) or {}
|
|
lines.append(f"#### {label}")
|
|
lines.append("")
|
|
lines.append("| signal | mean_ic | t | weeks | N |")
|
|
lines.append("|---|---:|---:|---:|---:|")
|
|
for name in sorted(block):
|
|
r = block[name]
|
|
lines.append(
|
|
f"| {name} | {r.get('mean_ic')} | {r.get('ic_t_stat')} | "
|
|
f"{r.get('weeks')} | {r.get('avg_cross_section')} |"
|
|
)
|
|
lines.append("")
|
|
else:
|
|
lines.append("_No era split._")
|
|
|
|
lines.extend([
|
|
"",
|
|
"## Verdict",
|
|
"",
|
|
f"**{payload.get('verdict')}**",
|
|
"",
|
|
payload.get("verdict_detail") or "",
|
|
"",
|
|
"## What a human must decide next",
|
|
"",
|
|
payload.get("human_next")
|
|
or "- Do not retune production knobs from this report without review.",
|
|
"",
|
|
f"Artifacts: `{payload.get('report_path')}`",
|
|
"",
|
|
])
|
|
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
|
|
|
|
|
async def _main() -> None:
|
|
args = _parse_args()
|
|
snapshot = Path(args.snapshot)
|
|
if not snapshot.exists():
|
|
raise SystemExit(f"Missing snapshot: {snapshot}")
|
|
if args.allow_spawn:
|
|
os.environ["BACKTEST_ALLOW_SPAWN"] = "1"
|
|
|
|
coverage = None
|
|
race = None
|
|
harness = None
|
|
|
|
if args.phase in ("coverage", "all"):
|
|
print("Coverage probe…")
|
|
coverage = _coverage_report(snapshot)
|
|
print(
|
|
f" tickers={coverage['ticker_count']} ohlcv={coverage['ohlcv_row_count']} "
|
|
f"range={coverage['date_range']}"
|
|
)
|
|
for row in coverage["bars_per_year"]:
|
|
print(
|
|
f" year {row['year']}: bars={row['bars']} "
|
|
f"tickers={row['tickers_with_bars']}"
|
|
)
|
|
|
|
if args.phase in ("harness", "all"):
|
|
print("Race guard…")
|
|
race = _assert_complete(snapshot)
|
|
print(f" ok={race.get('ok')}")
|
|
print("Full harness + era split (LONG)…")
|
|
print(f" {SURVIVORSHIP_BANNER}")
|
|
harness = await _harness(
|
|
snapshot, workers=args.workers, quiet=args.quiet
|
|
)
|
|
|
|
stamp = datetime.now().strftime("%Y%m%d-%H%M%S")
|
|
out = (
|
|
Path(args.out)
|
|
if args.out
|
|
else Path("reports") / f"history-depth-{stamp}.json"
|
|
)
|
|
payload = {
|
|
"generated_at": datetime.now().isoformat(),
|
|
"survivorship_banner": SURVIVORSHIP_BANNER,
|
|
"coverage": coverage,
|
|
"race_guard": race,
|
|
"harness": harness,
|
|
"verdict": "PENDING_HUMAN" if harness else "COVERAGE_ONLY",
|
|
"verdict_detail": (
|
|
"Harness complete — human interprets relative IC / era stability. "
|
|
"No production retune from this artifact."
|
|
if harness
|
|
else "Coverage probe only; run --phase harness after deep rebuild."
|
|
),
|
|
"human_next": (
|
|
"- Compare sector residual vs market residual across eras.\n"
|
|
"- If pre-2021 IC collapses, park Task 1 wire-in.\n"
|
|
"- Do not retune production knobs on deep history levels."
|
|
),
|
|
"report_path": str(out.as_posix()),
|
|
}
|
|
out.parent.mkdir(parents=True, exist_ok=True)
|
|
out.write_text(json.dumps(payload, indent=2, default=str) + "\n", encoding="utf-8")
|
|
md = Path("docs/research/history-depth-extension.md")
|
|
_write_md(md, payload)
|
|
out.with_suffix(".md").write_text(md.read_text(encoding="utf-8"), encoding="utf-8")
|
|
print(f"Wrote {out}")
|
|
print(f"Wrote {md}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(_main())
|