dennisthiessen/signal-platform
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

research: sector residual, earnings gap/SUE, history-depth scaffolding

Browse Source 
Tier-1 alpha research (local only, no production deploy):

Sector residual momentum: two-factor SPY+sector residual and sector demean signals, IC harness + A/B. Sector resid clears pre-registered bars narrowly (PROMOTE for human wire design only). Sector demean fails t vs market resid.

Earnings: earnings_events backfill (FMP bulk paid; FMP/AV per-symbol), 2a gap diagnostic report-only, 2b SUE IC (PARK; incomplete 48/506 coverage).

History-depth: pre-registered doc + runner for MacBook deep rebuild/harness.

Do not ship production residual or filters from this branch.
This commit is contained in:
dennisthiessen
2026-07-19 09:33:34 +02:00
parent 8f285acb00
commit fa25b6ee68
18 changed files with 6093 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/backfill_earnings_events.py
+528
View File
@@ -0,0 +1,528 @@
"""Backfill historical earnings into a snapshot ``earnings_events`` table.
Prefers FMP bulk date-range ``earnings-calendar`` (one request per window).
On free-tier 402/403, falls back to per-symbol ``/stable/earnings`` with
resume support and request counting (≈250 req/day free tier).
Research only — writes to the local snapshot SQLite, never production Postgres.
Example
-------
python scripts/backfill_earnings_events.py \\
--snapshot backtest_snapshots/prod.sqlite --limit 250
"""
from __future__ import annotations
import argparse
import asyncio
import json
import sys
import time
from datetime import date, datetime, timedelta, timezone
from pathlib import Path
import httpx
from sqlalchemy import create_engine, text
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
FMP_STABLE = "https://financialmodelingprep.com/stable"
DDL = """
CREATE TABLE IF NOT EXISTS earnings_events (
id INTEGER PRIMARY KEY,
symbol TEXT NOT NULL,
announce_date TEXT NOT NULL,
announce_time TEXT,
eps_estimate REAL,
eps_actual REAL,
revenue_estimate REAL,
revenue_actual REAL,
source TEXT NOT NULL,
fetched_at TEXT NOT NULL,
UNIQUE(symbol, announce_date)
)
"""
# Side table tracks which symbols have been fully pulled (resume).
META_DDL = """
CREATE TABLE IF NOT EXISTS earnings_backfill_meta (
symbol TEXT PRIMARY KEY,
status TEXT NOT NULL,
n_events INTEGER NOT NULL DEFAULT 0,
updated_at TEXT NOT NULL,
note TEXT
)
"""
def _parse_args() -> argparse.Namespace:
p = argparse.ArgumentParser(description=__doc__)
p.add_argument("--snapshot", default="backtest_snapshots/prod.sqlite")
p.add_argument(
"--from-date",
default="2020-01-01",
help="Bulk calendar window start (also filters per-symbol rows).",
)
p.add_argument(
"--to-date",
default=None,
help="Bulk calendar window end (default: today).",
)
p.add_argument(
"--limit",
type=int,
default=250,
help="Max FMP requests this run (free-tier cushion).",
)
p.add_argument("--sleep", type=float, default=0.35)
p.add_argument(
"--force-symbol",
action="store_true",
help="Skip bulk attempt; go straight to per-symbol.",
)
p.add_argument(
"--refetch-done",
action="store_true",
help="Re-fetch symbols already marked done.",
)
p.add_argument(
"--provider",
choices=("fmp", "alpha_vantage", "auto"),
default="auto",
help="Earnings provider. auto tries FMP bulk then FMP/AV per-symbol.",
)
return p.parse_args()
def _ensure_tables(engine) -> None:
with engine.begin() as conn:
conn.execute(text(DDL))
conn.execute(text(META_DDL))
def _upsert_events(conn, rows: list[dict], source: str) -> int:
if not rows:
return 0
now = datetime.now(timezone.utc).isoformat()
written = 0
for r in rows:
conn.execute(
text(
"""
INSERT INTO earnings_events (
symbol, announce_date, announce_time,
eps_estimate, eps_actual, revenue_estimate, revenue_actual,
source, fetched_at
) VALUES (
:symbol, :announce_date, :announce_time,
:eps_estimate, :eps_actual, :revenue_estimate, :revenue_actual,
:source, :fetched_at
)
ON CONFLICT(symbol, announce_date) DO UPDATE SET
announce_time=excluded.announce_time,
eps_estimate=excluded.eps_estimate,
eps_actual=excluded.eps_actual,
revenue_estimate=excluded.revenue_estimate,
revenue_actual=excluded.revenue_actual,
source=excluded.source,
fetched_at=excluded.fetched_at
"""
),
{
"symbol": r["symbol"],
"announce_date": r["announce_date"],
"announce_time": r.get("announce_time"),
"eps_estimate": r.get("eps_estimate"),
"eps_actual": r.get("eps_actual"),
"revenue_estimate": r.get("revenue_estimate"),
"revenue_actual": r.get("revenue_actual"),
"source": source,
"fetched_at": now,
},
)
written += 1
return written
def _parse_bulk_item(item: dict) -> dict | None:
sym = (item.get("symbol") or "").strip().upper()
d = item.get("date") or item.get("earningsDate")
if not sym or not d:
return None
return {
"symbol": sym.replace(".", "-"),
"announce_date": str(d)[:10],
"announce_time": item.get("time") or item.get("announceTime"),
"eps_estimate": _f(item.get("epsEstimated") or item.get("estimatedEarning")),
"eps_actual": _f(item.get("epsActual") or item.get("eps")),
"revenue_estimate": _f(item.get("revenueEstimated")),
"revenue_actual": _f(item.get("revenueActual")),
}
def _parse_symbol_item(item: dict, symbol: str) -> dict | None:
d = item.get("date")
if not d:
return None
return {
"symbol": symbol.replace(".", "-").upper(),
"announce_date": str(d)[:10],
"announce_time": item.get("time"),
"eps_estimate": _f(item.get("epsEstimated")),
"eps_actual": _f(item.get("epsActual")),
"revenue_estimate": _f(item.get("revenueEstimated")),
"revenue_actual": _f(item.get("revenueActual")),
}
def _f(v) -> float | None:
if v is None or v == "":
return None
try:
return float(v)
except (TypeError, ValueError):
return None
async def _try_bulk(
client: httpx.AsyncClient,
api_key: str,
start: date,
end: date,
*,
window_days: int = 30,
) -> tuple[list[dict], int, str | None]:
"""Return (rows, requests_used, error_note)."""
rows: list[dict] = []
reqs = 0
cur = start
while cur <= end:
win_end = min(end, cur + timedelta(days=window_days - 1))
resp = await client.get(
f"{FMP_STABLE}/earnings-calendar",
params={
"from": cur.isoformat(),
"to": win_end.isoformat(),
"apikey": api_key,
},
)
reqs += 1
if resp.status_code in (402, 403):
return [], reqs, f"bulk_unavailable status={resp.status_code}"
if resp.status_code == 429:
return rows, reqs, "rate_limited"
resp.raise_for_status()
data = resp.json()
if not isinstance(data, list):
return [], reqs, f"unexpected bulk payload type={type(data)}"
for item in data:
if isinstance(item, dict):
parsed = _parse_bulk_item(item)
if parsed:
rows.append(parsed)
cur = win_end + timedelta(days=1)
return rows, reqs, None
async def _fetch_symbol(
client: httpx.AsyncClient, api_key: str, symbol: str
) -> list[dict]:
resp = await client.get(
f"{FMP_STABLE}/earnings",
params={"symbol": symbol, "apikey": api_key},
)
if resp.status_code == 429:
raise RuntimeError("rate_limited")
if resp.status_code == 402:
return []
resp.raise_for_status()
data = resp.json()
if not isinstance(data, list):
return []
out: list[dict] = []
for item in data:
if isinstance(item, dict):
parsed = _parse_symbol_item(item, symbol)
if parsed:
out.append(parsed)
return out
async def _fetch_symbol_alpha_vantage(
client: httpx.AsyncClient, api_key: str, symbol: str
) -> list[dict]:
"""Alpha Vantage EARNINGS — includes reportedDate (announce) + estimate/actual."""
resp = await client.get(
"https://www.alphavantage.co/query",
params={"function": "EARNINGS", "symbol": symbol, "apikey": api_key},
)
if resp.status_code == 429:
raise RuntimeError("rate_limited")
resp.raise_for_status()
data = resp.json()
if not isinstance(data, dict):
return []
note = str(data.get("Note") or data.get("Information") or "")
if "rate limit" in note.lower() or "Thank you for using Alpha Vantage" in note:
raise RuntimeError("rate_limited")
if data.get("Error Message"):
return []
quarterly = data.get("quarterlyEarnings") or []
out: list[dict] = []
for item in quarterly:
if not isinstance(item, dict):
continue
# Prefer announce (reportedDate); fall back to fiscal end (worse PIT).
ad = item.get("reportedDate") or item.get("fiscalDateEnding")
if not ad:
continue
out.append({
"symbol": symbol.replace(".", "-").upper(),
"announce_date": str(ad)[:10],
"announce_time": item.get("reportTime"),
"eps_estimate": _f(item.get("estimatedEPS")),
"eps_actual": _f(item.get("reportedEPS")),
"revenue_estimate": None,
"revenue_actual": None,
})
return out
async def _main() -> None:
args = _parse_args()
snapshot = Path(args.snapshot)
if not snapshot.exists():
raise SystemExit(f"Snapshot not found: {snapshot}")
from app.config import settings
if not settings.fmp_api_key:
raise SystemExit("FMP_API_KEY required")
start = date.fromisoformat(args.from_date)
end = date.fromisoformat(args.to_date) if args.to_date else date.today()
engine = create_engine(
f"sqlite:///{snapshot.resolve().as_posix()}",
future=True,
)
_ensure_tables(engine)
with engine.connect() as conn:
symbols = [
str(r[0]).upper().replace(".", "-")
for r in conn.execute(text("SELECT symbol FROM tickers ORDER BY symbol"))
]
done = set()
if not args.refetch_done:
done = {
str(r[0])
for r in conn.execute(
text(
"SELECT symbol FROM earnings_backfill_meta "
"WHERE status='done' AND n_events > 0"
)
)
}
pending = [s for s in symbols if s not in done]
print(f"Snapshot: {snapshot}")
print(f"Universe: {len(symbols)}; pending: {len(pending)}; done: {len(done)}")
print(f"Window filter: {start} → {end}")
print(f"Provider: {args.provider}")
req_budget = int(args.limit)
reqs_used = 0
events_written = 0
mode = "per_symbol"
use_av = args.provider in ("alpha_vantage", "auto") and bool(
getattr(settings, "alpha_vantage_api_key", "")
)
use_fmp = args.provider in ("fmp", "auto") and bool(settings.fmp_api_key)
async with httpx.AsyncClient(timeout=60.0) as client:
if (
not args.force_symbol
and req_budget > 0
and use_fmp
and args.provider != "alpha_vantage"
):
print("Attempting bulk earnings-calendar…")
bulk_rows, bulk_reqs, err = await _try_bulk(
client, settings.fmp_api_key, start, end
)
reqs_used += bulk_reqs
if err:
print(f" Bulk unavailable: {err} (requests={bulk_reqs})")
else:
# Filter to universe.
uni = set(symbols)
bulk_rows = [r for r in bulk_rows if r["symbol"] in uni]
with engine.begin() as conn:
events_written += _upsert_events(conn, bulk_rows, "fmp_earnings_calendar")
for sym in symbols:
n = conn.execute(
text(
"SELECT COUNT(*) FROM earnings_events WHERE symbol=:s"
),
{"s": sym},
).scalar_one()
conn.execute(
text(
"""
INSERT INTO earnings_backfill_meta(symbol, status, n_events, updated_at, note)
VALUES (:s, 'done', :n, :t, 'bulk')
ON CONFLICT(symbol) DO UPDATE SET
status='done', n_events=excluded.n_events,
updated_at=excluded.updated_at, note=excluded.note
"""
),
{
"s": sym,
"n": int(n),
"t": datetime.now(timezone.utc).isoformat(),
},
)
mode = "bulk"
print(f" Bulk wrote {events_written} events; requests={bulk_reqs}")
pending = []
# Per-symbol fallback / completion.
fmp_limited = False
for sym in pending:
if reqs_used >= req_budget:
print(f"Request budget exhausted ({req_budget}). Resume later.")
break
items: list[dict] = []
source = "fmp_earnings"
note = "per_symbol"
try:
if use_fmp and not fmp_limited and args.provider != "alpha_vantage":
items = await _fetch_symbol(client, settings.fmp_api_key, sym)
source = "fmp_earnings"
note = "fmp_per_symbol"
# Empty list may mean soft-limit or no data — try AV if available.
if not items and use_av:
items = await _fetch_symbol_alpha_vantage(
client, settings.alpha_vantage_api_key, sym
)
source = "alpha_vantage_earnings"
note = "av_after_fmp_empty"
reqs_used += 1 # count AV call separately below too
elif use_av:
items = await _fetch_symbol_alpha_vantage(
client, settings.alpha_vantage_api_key, sym
)
source = "alpha_vantage_earnings"
note = "av_per_symbol"
else:
raise RuntimeError("no provider available")
except Exception as exc:
msg = str(exc)
print(f" FAIL {sym}: {msg}")
reqs_used += 1
if "rate_limited" in msg and note.startswith("fmp"):
fmp_limited = True
with engine.begin() as conn:
conn.execute(
text(
"""
INSERT INTO earnings_backfill_meta(symbol, status, n_events, updated_at, note)
VALUES (:s, 'error', 0, :t, :n)
ON CONFLICT(symbol) DO UPDATE SET
status='error', updated_at=excluded.updated_at, note=excluded.note
"""
),
{
"s": sym,
"t": datetime.now(timezone.utc).isoformat(),
"n": msg[:200],
},
)
if args.sleep > 0:
await asyncio.sleep(args.sleep)
continue
reqs_used += 1
# Keep all rows with dates on/before end — SUE needs trailing history.
filtered = [
r for r in items if r["announce_date"] <= end.isoformat()
]
# Do NOT mark empty as done — leave pending for another provider/day.
status = "done" if filtered else "empty"
with engine.begin() as conn:
n_w = _upsert_events(conn, filtered, source) if filtered else 0
events_written += n_w
conn.execute(
text(
"""
INSERT INTO earnings_backfill_meta(symbol, status, n_events, updated_at, note)
VALUES (:s, :st, :n, :t, :note)
ON CONFLICT(symbol) DO UPDATE SET
status=excluded.status, n_events=excluded.n_events,
updated_at=excluded.updated_at, note=excluded.note
"""
),
{
"s": sym,
"st": status,
"n": len(filtered),
"t": datetime.now(timezone.utc).isoformat(),
"note": note,
},
)
if reqs_used % 10 == 0 or reqs_used == 1:
print(
f" progress reqs={reqs_used}/{req_budget} last={sym} "
f"events_batch={len(filtered)} src={source}"
)
# AV free tier is ~5/min or 25/day — be polite when using it.
sleep_s = float(args.sleep)
if source.startswith("alpha_vantage"):
sleep_s = max(sleep_s, 12.0)
if sleep_s > 0:
await asyncio.sleep(sleep_s)
with engine.connect() as conn:
total_events = int(
conn.execute(text("SELECT COUNT(*) FROM earnings_events")).scalar_one()
)
done_n = int(
conn.execute(
text("SELECT COUNT(*) FROM earnings_backfill_meta WHERE status='done'")
).scalar_one()
)
d_range = conn.execute(
text("SELECT MIN(announce_date), MAX(announce_date) FROM earnings_events")
).fetchone()
with_actual = int(
conn.execute(
text(
"SELECT COUNT(*) FROM earnings_events "
"WHERE eps_actual IS NOT NULL AND eps_estimate IS NOT NULL"
)
).scalar_one()
)
summary = {
"mode": mode,
"fmp_requests": reqs_used,
"events_written_this_run": events_written,
"total_events": total_events,
"symbols_done": done_n,
"symbols_universe": len(symbols),
"announce_date_range": {"min": d_range[0], "max": d_range[1]},
"events_with_actual_and_estimate": with_actual,
"budget": req_budget,
"complete": done_n >= len(symbols),
}
print(json.dumps(summary, indent=2))
out = Path("reports") / "earnings-backfill-status.json"
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8")
print(f"Wrote {out}")
if __name__ == "__main__":
asyncio.run(_main())
scripts/build_ticker_sector_map.py
+229
View File
@@ -0,0 +1,229 @@
"""Build a local ticker → GICS sector map for research residualization.
Sources (in order):
1. Public S&P 500 constituents CSV (datasets/s-and-p-500-companies) — bulk, free.
2. Existing map file (resume).
3. FMP stable ``profile`` for still-missing symbols (budget ~250 req/day).
Writes ``data/research/ticker_sector_map.json``. Never touches production Postgres.
Example
-------
python scripts/build_ticker_sector_map.py \\
--snapshot backtest_snapshots/prod.sqlite
python scripts/build_ticker_sector_map.py --fmp-limit 50
"""
from __future__ import annotations
import argparse
import asyncio
import csv
import io
import json
import sys
import time
from datetime import datetime, timezone
from pathlib import Path
import httpx
from sqlalchemy import create_engine, text
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from app.services.sector_map import ( # noqa: E402
DEFAULT_SECTOR_MAP_PATH,
coverage_stats,
load_ticker_sector_map,
normalise_symbol,
save_ticker_sector_map,
sector_to_etf,
)
SP500_CSV_URL = (
"https://raw.githubusercontent.com/datasets/s-and-p-500-companies/"
"master/data/constituents.csv"
)
FMP_STABLE = "https://financialmodelingprep.com/stable"
def _parse_args() -> argparse.Namespace:
p = argparse.ArgumentParser(description=__doc__)
p.add_argument(
"--snapshot",
default="backtest_snapshots/prod.sqlite",
help="Snapshot whose tickers define the universe.",
)
p.add_argument(
"--out",
default=str(DEFAULT_SECTOR_MAP_PATH),
help="Output JSON path.",
)
p.add_argument(
"--fmp-limit",
type=int,
default=200,
help="Max FMP profile requests this run (free-tier cushion).",
)
p.add_argument(
"--skip-fmp",
action="store_true",
help="Only use public SP500 CSV + existing map.",
)
p.add_argument("--sleep", type=float, default=0.35, help="Pause between FMP calls.")
return p.parse_args()
def _snapshot_symbols(snapshot: Path) -> list[str]:
engine = create_engine(f"sqlite:///{snapshot.resolve().as_posix()}", future=True)
try:
with engine.connect() as conn:
rows = conn.execute(text("SELECT symbol FROM tickers ORDER BY symbol")).fetchall()
finally:
engine.dispose()
return [normalise_symbol(r[0]) for r in rows if r[0]]
def _fetch_sp500_map() -> dict[str, str]:
with httpx.Client(timeout=60.0, follow_redirects=True) as client:
resp = client.get(SP500_CSV_URL)
resp.raise_for_status()
reader = csv.DictReader(io.StringIO(resp.text))
out: dict[str, str] = {}
for row in reader:
sym = normalise_symbol(row.get("Symbol") or "")
sector = (row.get("GICS Sector") or "").strip()
if sym and sector:
out[sym] = sector
return out
async def _fmp_profile_sector(client: httpx.AsyncClient, api_key: str, symbol: str) -> str | None:
resp = await client.get(
f"{FMP_STABLE}/profile",
params={"symbol": symbol, "apikey": api_key},
)
if resp.status_code == 429:
raise RuntimeError(f"FMP rate limited on {symbol}")
if resp.status_code == 402:
return None
resp.raise_for_status()
data = resp.json()
if isinstance(data, list):
data = data[0] if data else {}
if not isinstance(data, dict):
return None
sector = (data.get("sector") or data.get("industry") or "").strip()
# industry alone is not a GICS sector — only accept if we can map to an ETF
if sector and sector_to_etf(sector):
return sector
# FMP sometimes returns industry under sector when sector missing; try sector field only
sec = (data.get("sector") or "").strip()
return sec or None
async def _fill_from_fmp(
missing: list[str],
*,
api_key: str,
limit: int,
sleep_s: float,
) -> tuple[dict[str, str], int]:
filled: dict[str, str] = {}
used = 0
async with httpx.AsyncClient(timeout=30.0) as client:
for sym in missing:
if used >= limit:
break
try:
sector = await _fmp_profile_sector(client, api_key, sym)
except Exception as exc:
print(f" FMP fail {sym}: {exc}")
used += 1
await asyncio.sleep(sleep_s)
continue
used += 1
if sector:
filled[sym] = sector
print(f" FMP {sym} → {sector}")
else:
print(f" FMP {sym} → (no sector)")
if sleep_s > 0:
await asyncio.sleep(sleep_s)
return filled, used
async def _main() -> None:
args = _parse_args()
snapshot = Path(args.snapshot)
if not snapshot.exists():
raise SystemExit(f"Snapshot not found: {snapshot}")
symbols = _snapshot_symbols(snapshot)
print(f"Universe: {len(symbols)} symbols from {snapshot}")
existing = load_ticker_sector_map(args.out)
print(f"Existing map entries: {len(existing)}")
print("Fetching public S&P 500 sector CSV…")
sp500 = _fetch_sp500_map()
print(f" SP500 CSV rows: {len(sp500)}")
mapping = dict(existing)
from_sp500 = 0
for sym in symbols:
if sym in mapping:
continue
if sym in sp500:
mapping[sym] = sp500[sym]
from_sp500 += 1
print(f" Newly filled from SP500 CSV: {from_sp500}")
missing = [s for s in symbols if s not in mapping]
fmp_used = 0
from_fmp = 0
if missing and not args.skip_fmp:
from app.config import settings
if not settings.fmp_api_key:
print("WARNING: FMP key missing; leaving gaps unfilled")
else:
print(f"FMP fill for {len(missing)} missing (limit={args.fmp_limit})…")
filled, fmp_used = await _fill_from_fmp(
missing,
api_key=settings.fmp_api_key,
limit=int(args.fmp_limit),
sleep_s=float(args.sleep),
)
mapping.update(filled)
from_fmp = len(filled)
still_missing = [s for s in symbols if s not in mapping]
stats = coverage_stats(symbols, mapping)
meta = {
"built_at": datetime.now(timezone.utc).isoformat(),
"snapshot": str(snapshot.resolve()),
"from_existing": len(existing),
"from_sp500_csv": from_sp500,
"from_fmp": from_fmp,
"fmp_requests": fmp_used,
"still_missing": still_missing,
"coverage": {
k: stats[k]
for k in ("universe", "mapped", "mapped_pct", "with_etf", "by_sector")
},
}
out_path = save_ticker_sector_map(mapping, args.out, meta=meta)
print(f"Wrote {out_path}")
print(json.dumps(meta["coverage"], indent=2))
if still_missing:
print(f"Still missing ({len(still_missing)}): {still_missing[:40]}")
if len(still_missing) > 40:
print(f" … +{len(still_missing) - 40} more")
if __name__ == "__main__":
asyncio.run(_main())
scripts/fetch_sector_etfs_to_snapshot.py
+183
View File
@@ -0,0 +1,183 @@
"""Fetch the 11 SPDR sector ETFs into a snapshot's ``benchmark_prices``.
Research-only. Sector ETFs are auxiliary series (like SPY) — they must not
enter the tradable ticker universe or candidate replay. Storing them in
``benchmark_prices`` keeps that invariant.
Also refreshes SPY on the same window so residual factors share a calendar.
Example
-------
python scripts/fetch_sector_etfs_to_snapshot.py \\
--snapshot backtest_snapshots/prod.sqlite --history-days 2200
"""
from __future__ import annotations
import argparse
import asyncio
import sys
import time
from datetime import date, timedelta
from pathlib import Path
from sqlalchemy import create_engine, text
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from app.services.sector_map import SECTOR_ETFS # noqa: E402
def _parse_args() -> argparse.Namespace:
p = argparse.ArgumentParser(description=__doc__)
p.add_argument("--snapshot", default="backtest_snapshots/prod.sqlite")
p.add_argument(
"--history-days",
type=int,
default=2200,
help="Lookback calendar days (default ~6y; covers 5y snapshot + cushion).",
)
p.add_argument("--sleep", type=float, default=0.25)
p.add_argument(
"--symbols",
default=None,
help="Comma-separated override (default: SPY + 11 sector ETFs).",
)
return p.parse_args()
async def _fetch_and_upsert(
engine,
provider,
symbol: str,
start: date,
end: date,
*,
sleep_s: float,
) -> int:
from app.exceptions import ProviderError, RateLimitError
for attempt in range(5):
try:
bars = await provider.fetch_ohlcv(symbol, start, end)
break
except RateLimitError:
wait = min(60.0, 2.0 ** attempt)
print(f" rate limited {symbol}; sleep {wait:.0f}s")
await asyncio.sleep(wait)
bars = []
except ProviderError as exc:
if attempt + 1 >= 5:
raise
await asyncio.sleep(1.0)
print(f" retry {symbol}: {exc}")
bars = []
else:
bars = []
if sleep_s > 0:
await asyncio.sleep(sleep_s)
if not bars:
print(f" {symbol}: empty")
return 0
written = 0
with engine.begin() as conn:
for bar in bars:
d = bar.date.isoformat() if hasattr(bar.date, "isoformat") else str(bar.date)
close = float(bar.close)
existing = conn.execute(
text(
"SELECT id, close FROM benchmark_prices "
"WHERE symbol = :sym AND date = :d"
),
{"sym": symbol, "d": d},
).fetchone()
if existing is None:
# id is INTEGER PK — let sqlite autoincrement if possible
conn.execute(
text(
"INSERT INTO benchmark_prices (symbol, date, close) "
"VALUES (:sym, :d, :c)"
),
{"sym": symbol, "d": d, "c": close},
)
written += 1
elif abs(float(existing[1]) - close) > 1e-9:
conn.execute(
text(
"UPDATE benchmark_prices SET close = :c WHERE id = :id"
),
{"c": close, "id": int(existing[0])},
)
written += 1
print(f" {symbol}: {len(bars)} bars, {written} rows written/updated")
return written
async def _main() -> None:
args = _parse_args()
snapshot = Path(args.snapshot)
if not snapshot.exists():
raise SystemExit(f"Snapshot not found: {snapshot}")
from app.config import settings
from app.providers.alpaca import AlpacaOHLCVProvider
if not settings.alpaca_api_key or not settings.alpaca_api_secret:
raise SystemExit("ALPACA_API_KEY / ALPACA_API_SECRET required")
if args.symbols:
symbols = [s.strip().upper() for s in args.symbols.split(",") if s.strip()]
else:
symbols = ["SPY", *SECTOR_ETFS]
end = date.today()
start = end - timedelta(days=int(args.history_days))
provider = AlpacaOHLCVProvider(settings.alpaca_api_key, settings.alpaca_api_secret)
engine = create_engine(
f"sqlite:///{snapshot.resolve().as_posix()}",
future=True,
)
print(f"Snapshot: {snapshot}")
print(f"Window: {start} → {end}")
print(f"Symbols: {symbols}")
t0 = time.monotonic()
total = 0
try:
for sym in symbols:
n = await _fetch_and_upsert(
engine, provider, sym, start, end, sleep_s=float(args.sleep)
)
total += n
finally:
engine.dispose()
# Summary counts
engine = create_engine(
f"sqlite:///{snapshot.resolve().as_posix()}",
future=True,
)
try:
with engine.connect() as conn:
rows = conn.execute(
text(
"SELECT symbol, COUNT(*), MIN(date), MAX(date) "
"FROM benchmark_prices GROUP BY symbol ORDER BY symbol"
)
).fetchall()
finally:
engine.dispose()
print(f"Done in {(time.monotonic() - t0) / 60:.1f}m; rows touched={total}")
for sym, n, d0, d1 in rows:
print(f" {sym}: n={n} {d0}→{d1}")
if __name__ == "__main__":
asyncio.run(_main())
scripts/run_earnings_research.py
+927
View File
@@ -0,0 +1,927 @@
"""Earnings gap diagnostic (2a) + SUE IC (2b). Local research only.
Requires ``earnings_events`` on the snapshot (see backfill_earnings_events.py).
Example
-------
python scripts/run_earnings_research.py \\
--snapshot backtest_snapshots/prod.sqlite --workers 6 --allow-spawn
"""
from __future__ import annotations
import argparse
import asyncio
import json
import math
import os
import sys
from collections import defaultdict
from datetime import date, datetime, timedelta
from pathlib import Path
from typing import Any
from sqlalchemy import create_engine, text
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
IRON_IC_BAR = 0.03
MIN_RELIABLE = 12
SUE_CARRY_DAYS = 63
SUE_TRAIL = 8
def _sqlite_url(path: Path) -> str:
return f"sqlite+aiosqlite:///{path.resolve().as_posix()}"
def _parse_args() -> argparse.Namespace:
p = argparse.ArgumentParser(description=__doc__)
p.add_argument("--snapshot", default="backtest_snapshots/prod.sqlite")
p.add_argument("--workers", type=int, default=6)
p.add_argument("--allow-spawn", action="store_true")
p.add_argument("--skip-2a", action="store_true")
p.add_argument("--skip-2b", action="store_true")
p.add_argument("--quiet", action="store_true")
p.add_argument("--out", default=None)
return p.parse_args()
def _load_earnings(snapshot: Path) -> list[dict]:
engine = create_engine(
f"sqlite:///{snapshot.resolve().as_posix()}",
future=True,
)
try:
with engine.connect() as conn:
# Table must exist.
tables = {
r[0]
for r in conn.execute(
text("SELECT name FROM sqlite_master WHERE type='table'")
)
}
if "earnings_events" not in tables:
raise SystemExit(
"earnings_events table missing — run scripts/backfill_earnings_events.py"
)
rows = conn.execute(
text(
"""
SELECT symbol, announce_date, announce_time,
eps_estimate, eps_actual, revenue_estimate, revenue_actual
FROM earnings_events
ORDER BY symbol, announce_date
"""
)
).fetchall()
meta = {}
if "earnings_backfill_meta" in tables:
meta = {
"done": int(
conn.execute(
text(
"SELECT COUNT(*) FROM earnings_backfill_meta "
"WHERE status='done'"
)
).scalar_one()
),
"universe_tickers": int(
conn.execute(text("SELECT COUNT(*) FROM tickers")).scalar_one()
),
}
finally:
engine.dispose()
events = [
{
"symbol": str(r[0]).upper(),
"announce_date": date.fromisoformat(str(r[1])[:10]),
"announce_time": r[2],
"eps_estimate": r[3],
"eps_actual": r[4],
"revenue_estimate": r[5],
"revenue_actual": r[6],
}
for r in rows
]
return events, meta
def _percentile(xs: list[float], q: float) -> float | None:
if not xs:
return None
s = sorted(xs)
if len(s) == 1:
return s[0]
idx = q * (len(s) - 1)
lo = int(math.floor(idx))
hi = int(math.ceil(idx))
if lo == hi:
return s[lo]
w = idx - lo
return s[lo] * (1 - w) + s[hi] * w
def _r_dist(rs: list[float]) -> dict[str, Any]:
if not rs:
return {"n": 0}
return {
"n": len(rs),
"mean": round(sum(rs) / len(rs), 4),
"win_rate": round(sum(1 for r in rs if r > 0) / len(rs), 4),
"p05": round(_percentile(rs, 0.05), 4),
"p25": round(_percentile(rs, 0.25), 4),
"p50": round(_percentile(rs, 0.50), 4),
"p75": round(_percentile(rs, 0.75), 4),
"p95": round(_percentile(rs, 0.95), 4),
"min": round(min(rs), 4),
"max": round(max(rs), 4),
}
def _trading_days_between(
entry: date, exit_: date, calendar: set[date]
) -> list[date]:
"""Inclusive trading dates in [entry, exit_] present on the union calendar."""
out = []
d = entry
while d <= exit_:
if d in calendar:
out.append(d)
d += timedelta(days=1)
return out
def _nth_trading_day_after(
start: date, n: int, ordered_calendar: list[date]
) -> date | None:
"""First calendar date strictly after ``start``, then + (n-1) more sessions.
announce+1 trading day: n=1 → first session after announce date
(if announce is a trading day, still use the *next* session for PIT).
"""
# Sessions strictly after start.
after = [d for d in ordered_calendar if d > start]
if len(after) < n:
return None
return after[n - 1]
def _build_sue_series(
events_by_symbol: dict[str, list[dict]],
prices: dict[str, tuple],
) -> dict[str, dict[date, float]]:
"""symbol → {asof_date: sue_value} for days when SUE is live (announce+1 .. +63)."""
out: dict[str, dict[date, float]] = {}
for sym, cols in prices.items():
ords = cols[0]
closes = cols[4]
dates = [date.fromordinal(int(o)) for o in ords]
if not dates:
continue
ordered = dates # already chronological
cal_set = set(ordered)
events = events_by_symbol.get(sym.upper(), [])
# Chronological surprises with actual+estimate.
surprises: list[tuple[date, float, float]] = [] # announce, surprise, close_for_scale
for ev in events:
act, est = ev.get("eps_actual"), ev.get("eps_estimate")
if act is None or est is None:
continue
ad = ev["announce_date"]
# Close on/before announce for price fallback scale.
close_px = None
for d, c in zip(reversed(dates), reversed(closes)):
if d <= ad and float(c) > 0:
close_px = float(c)
break
surprises.append((ad, float(act) - float(est), close_px or 1.0))
surprises.sort(key=lambda x: x[0])
sue_on_day: dict[date, float] = {}
for i, (ad, surprise, px) in enumerate(surprises):
trail = [surprises[j][1] for j in range(max(0, i - SUE_TRAIL), i)]
# Need history of surprises; include current only for value, stdev from prior 8.
if len(trail) >= 3:
mean_t = sum(trail) / len(trail)
var = sum((x - mean_t) ** 2 for x in trail) / (len(trail) - 1)
sd = math.sqrt(var) if var > 0 else None
else:
sd = None
if sd is not None and sd > 1e-9:
sue = surprise / sd
else:
# Fallback: scale by price (EPS surprise / price).
sue = surprise / px if px > 0 else None
if sue is None or not math.isfinite(sue):
continue
usable_from = _nth_trading_day_after(ad, 1, ordered)
if usable_from is None:
continue
# Carry for SUE_CARRY_DAYS trading sessions starting at usable_from.
try:
start_idx = ordered.index(usable_from)
except ValueError:
# usable_from not in this symbol's calendar (halted etc.)
start_idx = next(
(k for k, d in enumerate(ordered) if d >= usable_from), None
)
if start_idx is None:
continue
end_idx = min(len(ordered) - 1, start_idx + SUE_CARRY_DAYS - 1)
for k in range(start_idx, end_idx + 1):
# Later announcements overwrite earlier carry (latest SUE wins).
sue_on_day[ordered[k]] = sue
if sue_on_day:
out[sym.upper()] = sue_on_day
return out
async def _run_2a(
snapshot: Path,
events: list[dict],
*,
quiet: bool,
workers: int,
) -> dict[str, Any]:
from app.config import settings
from app.services import backtest_service as bt
from app.services.admin_service import get_activation_config
from app.services.recommendation_service import get_recommendation_config
from app.services.paper_trade_service import get_exit_policy
from app.services.benchmark_service import load_benchmark_closes
from app.models.ticker import Ticker
from sqlalchemy import select
os.environ["BACKTEST_SNAPSHOT_OFFLINE"] = "1"
settings.backtest_workers = workers
engine = create_async_engine(_sqlite_url(snapshot), pool_pre_ping=True)
Session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
try:
async with Session() as db:
config = await get_recommendation_config(db)
activation = await get_activation_config(db)
exit_config = await get_exit_policy(db)
tickers = list(
(await db.execute(select(Ticker).order_by(Ticker.symbol))).scalars()
)
spy = await load_benchmark_closes(db, "SPY")
prices: dict[str, tuple] = {}
candidates: list[dict] = []
for idx, t in enumerate(tickers):
if not quiet and idx % 50 == 0:
print(f" 2a fetch {idx}/{len(tickers)}", end="\r", flush=True)
cols = await bt._fetch_columns(db, t.symbol)
if cols is None:
continue
prices[t.symbol] = cols
cands, _ = bt._replay_and_signals(
t.symbol,
cols,
config,
activation,
spy,
bt.PRODUCTION_GTL_TARGET_MODEL,
"weekly",
False,
)
candidates.extend(cands)
finally:
await engine.dispose()
if not quiet:
print()
# Production ranks + qualify.
bt._assign_momentum_percentiles(candidates)
bt._assign_residual_momentum_percentiles(candidates)
bt._assign_low_volatility_percentiles(candidates)
bt._assign_activation_momentum_percentiles(candidates)
bt._assign_residual_high_vol_blend(candidates)
for c in candidates:
c["qualified"] = bt._momentum_qualifies(c, 80.0)
longs = [
c for c in candidates if c.get("qualified") and c.get("direction") == "long"
]
strategy = next(s for s in bt.PORTFOLIO_MONITOR_STRATEGIES if s.get("is_production"))
entry_cfg = bt._entry_variant_config(str(strategy["entry_variant"]))
assert entry_cfg is not None
ranking_key = str(entry_cfg.get("ranking_key") or entry_cfg["percentile_key"])
exit_policy = bt.LIVE_EXIT_MODE_TO_SIM.get(
str(exit_config.get("mode", "atr_trailing")), "atr_trail3"
)
hold_days = int(exit_config.get("hold_days", 30))
trail = float(exit_config.get("atr_multiplier", bt.ATR_TRAIL_MULTIPLIER))
reentry = bt._make_gate_reset_reentry_fn(
longs, prices, cadence="weekly", ranking_key=ranking_key
)
sim = bt._simulate_portfolio(
longs,
prices,
spy,
exit_policy,
hold_days,
ranking_key=ranking_key,
max_positions=int(entry_cfg["max_positions"]),
risk_per_trade=float(entry_cfg["risk_per_trade"]),
atr_trail_multiplier=trail,
post_stop_reentry_fn=reentry,
fill_mode=bt.FILL_MODE_CLOSE,
include_trades=True,
)
if sim is None:
return {"error": "no_trades"}
details = sim.get("trade_details") or []
# Build per-symbol earnings announce dates.
earns_by_sym: dict[str, list[date]] = defaultdict(list)
for ev in events:
earns_by_sym[ev["symbol"]].append(ev["announce_date"])
for sym in earns_by_sym:
earns_by_sym[sym].sort()
# Union trading calendar from prices.
cal: set[date] = set()
for cols in prices.values():
for o in cols[0]:
cal.add(date.fromordinal(int(o)))
ordered_cal = sorted(cal)
# Map entry date → list of announce dates for symbol (for pre-entry lookback).
trades_parsed: list[dict] = []
for t in details:
sym = str(t.get("symbol") or "").upper()
# Field names from simulator.
entry_s = t.get("entry_date") or t.get("open_date") or t.get("date")
exit_s = t.get("exit_date") or t.get("close_date")
r = t.get("realized_r")
if r is None:
r = t.get("r")
if entry_s is None or exit_s is None or r is None:
continue
entry_d = date.fromisoformat(str(entry_s)[:10])
exit_d = date.fromisoformat(str(exit_s)[:10])
announces = earns_by_sym.get(sym, [])
# Earnings between entry and exit (exclusive of entry day? inclusive hold).
# "between entry and exit" — any announce with entry < announce <= exit
# (gap often overnight after entry). Also count announce on entry day.
in_hold = [
a for a in announces if entry_d <= a <= exit_d
]
# Entries within 3 trading days BEFORE an announcement:
# exists announce such that entry is in the 3 sessions immediately before announce.
pre_earn = False
for a in announces:
# trading sessions in (a-lookback, a)
sessions_before = [d for d in ordered_cal if d < a]
last3 = sessions_before[-3:] if len(sessions_before) >= 3 else sessions_before
if entry_d in last3:
pre_earn = True
break
trades_parsed.append({
"symbol": sym,
"entry": entry_d.isoformat(),
"exit": exit_d.isoformat(),
"r": float(r),
"earnings_in_hold": len(in_hold) > 0,
"n_earnings_in_hold": len(in_hold),
"entry_within_3d_before_earn": pre_earn,
})
all_r = [t["r"] for t in trades_parsed]
loss_lt_1r = [t for t in trades_parsed if t["r"] < -1.0]
loss_with_earn = [t for t in loss_lt_1r if t["earnings_in_hold"]]
pre = [t["r"] for t in trades_parsed if t["entry_within_3d_before_earn"]]
other = [t["r"] for t in trades_parsed if not t["entry_within_3d_before_earn"]]
return {
"sim_summary": {
k: sim.get(k)
for k in (
"sharpe",
"sharpe_se",
"cagr_pct",
"max_drawdown_pct",
"trades",
"total_return_pct",
)
},
"n_trades_parsed": len(trades_parsed),
"q1_losses_worse_than_minus_1r": {
"n_losses_lt_minus_1r": len(loss_lt_1r),
"n_with_earnings_in_hold": len(loss_with_earn),
"fraction_with_earnings": (
round(len(loss_with_earn) / len(loss_lt_1r), 4) if loss_lt_1r else None
),
"all_trades_with_earnings_in_hold": sum(
1 for t in trades_parsed if t["earnings_in_hold"]
),
"fraction_all_trades_with_earnings": (
round(
sum(1 for t in trades_parsed if t["earnings_in_hold"])
/ len(trades_parsed),
4,
)
if trades_parsed
else None
),
},
"q2_entry_within_3d_before_announce": {
"pre_earn_entries": _r_dist(pre),
"other_entries": _r_dist(other),
"all_entries": _r_dist(all_r),
"tail_trim_note": (
"Compare p95/max and mean of pre_earn vs other. "
"Rising win_rate with falling mean/p95 = right-tail trim red flag."
),
},
"note": "REPORT-ONLY — no filter shipped.",
}
async def _run_2b_ic(
snapshot: Path,
events: list[dict],
*,
quiet: bool,
workers: int,
) -> dict[str, Any]:
"""SUE IC via harness on identical cross-sections as momentum baselines."""
from app.config import settings
from app.services import backtest_service as bt
from app.services.benchmark_service import load_benchmark_closes
from app.models.ticker import Ticker
from sqlalchemy import select
from collections import defaultdict as dd
os.environ["BACKTEST_SNAPSHOT_OFFLINE"] = "1"
os.environ["BACKTEST_SIGNAL_EVAL_ONLY"] = "1"
# Load sector map if present so sector signals also appear (side-by-side optional).
if Path("data/research/ticker_sector_map.json").exists():
os.environ["BACKTEST_SECTOR_MAP_PATH"] = str(
Path("data/research/ticker_sector_map.json").resolve()
)
settings.backtest_workers = workers
engine = create_async_engine(_sqlite_url(snapshot), pool_pre_ping=True)
Session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
# Collect base signals + attach SUE.
collected: dict = dd(lambda: dd(list))
try:
async with Session() as db:
tickers = list(
(await db.execute(select(Ticker).order_by(Ticker.symbol))).scalars()
)
spy = await load_benchmark_closes(db, "SPY")
sector_etf: dict[str, dict] = {}
try:
from app.services.sector_map import SECTOR_ETFS, load_ticker_sector_map
symbol_to_sector = load_ticker_sector_map()
for etf in SECTOR_ETFS:
series = await load_benchmark_closes(db, etf)
if series:
sector_etf[etf] = series
except Exception:
symbol_to_sector = {}
sector_etf = {}
prices: dict[str, tuple] = {}
for idx, t in enumerate(tickers):
if not quiet and idx % 50 == 0:
print(f" 2b fetch {idx}/{len(tickers)}", end="\r", flush=True)
cols = await bt._fetch_columns(db, t.symbol)
if cols is None:
continue
prices[t.symbol] = cols
series = bt._signal_series(
[
type(
"R",
(),
{
"date": date.fromordinal(int(cols[0][i])),
"close": cols[4][i],
"high": cols[2][i],
"volume": cols[5][i] if len(cols) > 5 else 0,
},
)()
for i in range(len(cols[0]))
],
spy,
symbol=t.symbol,
sector_etf_closes=bt._sector_etf_closes_for_symbol(
t.symbol, symbol_to_sector, sector_etf
),
)
for name, weeks in series.items():
for wk, pairs in weeks.items():
collected[name][wk].extend(pairs)
finally:
await engine.dispose()
if not quiet:
print()
if symbol_to_sector:
bt._inject_sector_demeaned_momentum(collected, symbol_to_sector)
# SUE series.
events_by_sym: dict[str, list[dict]] = defaultdict(list)
for ev in events:
events_by_sym[ev["symbol"]].append(ev)
sue_map = _build_sue_series(events_by_sym, prices)
# Inject sue_latest into collected using mom_12_1 observations as the
# weekly as-of skeleton (same weeks / symbols).
sue_collected: dict = dd(list)
mom_weeks = collected.get("mom_12_1") or {}
for week_key, recs in mom_weeks.items():
for rec in recs:
pair = bt._obs_val_fwd(rec)
if pair is None:
continue
_val, fwd = pair
sym = None
if isinstance(rec, dict):
sym = rec.get("symbol")
if not sym:
continue
# Need as-of date: recover from week — use Friday of ISO week as proxy
# is weak. Better: re-derive from prices weekly indices.
# Store asof on rich recs? Current rich rows lack asof date.
# Fall back: compute SUE observations directly from prices weekly as-ofs.
pass
# Direct weekly as-of SUE + forward return (authoritative).
for sym, cols in prices.items():
ords, _o, highs, _l, closes, _v = cols
dates = [date.fromordinal(int(o)) for o in ords]
sue_days = sue_map.get(sym.upper()) or {}
if not sue_days:
continue
n = len(dates)
# weekly as-of indices: reuse harness helper via fake records.
records = [
type("R", (), {"date": dates[i], "close": closes[i], "high": highs[i]})()
for i in range(n)
]
for i in bt._weekly_asof_indices(records):
j = i + bt.HORIZON
if j >= n or closes[i] <= 0:
continue
asof = dates[i]
sue = sue_days.get(asof)
if sue is None:
continue
fwd = float(closes[j]) / float(closes[i]) - 1.0
iso = asof.isocalendar()
week_key = (iso[0], iso[1])
# Also grab mom for conditional.
mom = None
if i >= 252 and closes[i - 252] > 0:
mom = float(closes[i - 21]) / float(closes[i - 252]) - 1.0
sue_collected[week_key].append({
"val": float(sue),
"fwd": fwd,
"symbol": sym,
"mom_12_1": mom,
})
collected["sue_latest"] = sue_collected
signal_eval = bt._signal_evaluation(collected)
# Fair side-by-side: re-evaluate mom baselines on the *same* (symbol, week)
# observations where SUE is present (incomplete backfill otherwise inflates
# mom N relative to SUE).
sue_pairs_by_week = sue_collected
restricted: dict = dd(lambda: dd(list))
for week_key, recs in sue_pairs_by_week.items():
syms = {str(r.get("symbol")).upper() for r in recs if r.get("symbol")}
for base_name in ("mom_12_1", "mom_12_1_resid"):
base_recs = (collected.get(base_name) or {}).get(week_key) or []
for rec in base_recs:
pair = bt._obs_val_fwd(rec)
if pair is None:
continue
sym = None
if isinstance(rec, dict):
sym = rec.get("symbol")
if not sym or str(sym).upper() not in syms:
continue
restricted[base_name][week_key].append(rec)
restricted["sue_latest"][week_key].extend(recs)
restricted_eval = bt._signal_evaluation(restricted)
# Momentum-conditional: IC of SUE within top mom quintile each week.
cond_ics: list[float] = []
stride = max(1, round(bt.HORIZON / 5))
usable = [wk for wk, recs in sue_collected.items() if len(recs) >= bt.MIN_CROSS_SECTION]
kept = bt._nonoverlapping_weeks(usable, stride)
for wk in kept:
recs = sue_collected[wk]
with_mom = [r for r in recs if r.get("mom_12_1") is not None]
if len(with_mom) < bt.MIN_CROSS_SECTION:
continue
ordered = sorted(with_mom, key=lambda r: float(r["mom_12_1"]))
k = max(1, len(ordered) // 5)
top = ordered[-k:]
if len(top) < 5:
continue
ic = bt._spearman(
[float(r["val"]) for r in top],
[float(r["fwd"]) for r in top],
)
if ic is not None:
cond_ics.append(ic)
if cond_ics:
mean_c = sum(cond_ics) / len(cond_ics)
if len(cond_ics) > 1:
std = math.sqrt(
sum((x - mean_c) ** 2 for x in cond_ics) / (len(cond_ics) - 1)
)
t_c = mean_c / std * math.sqrt(len(cond_ics)) if std > 0 else None
else:
t_c = None
mom_cond = {
"mean_ic": round(mean_c, 4),
"ic_t_stat": round(t_c, 2) if t_c is not None else None,
"weeks": len(cond_ics),
"note": "IC of sue_latest within top mom_12_1 quintile (non-overlapping weeks)",
}
else:
mom_cond = {"mean_ic": None, "weeks": 0}
def _find(name: str) -> dict | None:
for row in signal_eval:
if row.get("signal") == name:
return row
return None
sue = _find("sue_latest")
grade = {
"green": False,
"reason": "sue_latest missing",
}
if sue:
mean_ic = sue.get("mean_ic")
t = sue.get("ic_t_stat")
reliable = bool(sue.get("reliable"))
sign_ok = mean_ic is not None and float(mean_ic) > 0
mag_ok = mean_ic is not None and abs(float(mean_ic)) >= IRON_IC_BAR
grade = {
"green": bool(sign_ok and mag_ok and reliable),
"checks": {
"mean_ic": mean_ic,
"sign_positive": sign_ok,
"abs_ge_0_03": mag_ok,
"reliable": reliable,
"ic_t_stat": t,
"weeks": sue.get("weeks"),
},
"reason": (
"iron rule cleared — STOP; book-integration is a separate human step"
if (sign_ok and mag_ok and reliable)
else "iron rule not met"
),
"row": sue,
}
def _find_r(name: str) -> dict | None:
for row in restricted_eval:
if row.get("signal") == name:
return row
return None
# Side-by-side baselines from same evaluation.
side = {
name: _find(name)
for name in (
"mom_12_1",
"mom_12_1_resid",
"mom_12_1_sector_resid",
"mom_12_1_sector_demeaned",
"sue_latest",
"fip_id",
)
}
side_restricted = {
name: _find_r(name)
for name in ("mom_12_1", "mom_12_1_resid", "sue_latest")
}
return {
"signal_eval_side_by_side": side,
"signal_eval_identical_sue_subset": side_restricted,
"identical_subset_note": (
"Mom baselines re-scored only on (week, symbol) cells where SUE exists. "
"Use this table when backfill is incomplete — full-universe mom N is not comparable."
),
"full_signal_eval": signal_eval,
"sue_grade": grade,
"momentum_conditional_sue": mom_cond,
"sue_coverage": {
"symbols_with_sue": len(sue_map),
"avg_weeks_with_sue": (
round(
sum(len(v) for v in sue_collected.values())
/ max(1, len(sue_collected)),
1,
)
if sue_collected
else 0
),
"weeks_with_min_cross_section": len(usable),
},
}
def _write_md(path: Path, payload: dict) -> None:
pre = path.read_text(encoding="utf-8") if path.exists() else ""
marker = "## Results"
idx = pre.find(marker)
header = pre[:idx] if idx >= 0 else pre.split("## Verdict")[0]
lines = [
header.rstrip(),
"",
"## Results",
"",
f"Generated: `{payload.get('generated_at')}`",
"",
"### Data provenance",
"",
f"```json\n{json.dumps(payload.get('data_provenance') or {}, indent=2, default=str)}\n```",
"",
"### 2a — Earnings-gap risk (report-only)",
"",
]
a = payload.get("experiment_2a")
if not a:
lines.append("_Skipped or unavailable._")
else:
lines.append(f"```json\n{json.dumps(a, indent=2, default=str)}\n```")
lines.extend(["", "### 2b — SUE / PEAD IC", ""])
b = payload.get("experiment_2b")
if not b:
lines.append("_Skipped or unavailable._")
else:
side = b.get("signal_eval_side_by_side") or {}
lines.extend([
"| signal | mean_ic | ic_t_stat | weeks | avg_N | reliable |",
"|---|---:|---:|---:|---:|---|",
])
for name in (
"mom_12_1",
"mom_12_1_resid",
"sue_latest",
"mom_12_1_sector_resid",
"fip_id",
):
r = side.get(name) or {}
lines.append(
f"| {name} | {r.get('mean_ic', '')} | {r.get('ic_t_stat', '')} | "
f"{r.get('weeks', '')} | {r.get('avg_cross_section', '')} | "
f"{r.get('reliable', '')} |"
)
lines.extend([
"",
f"**SUE grade:** `{json.dumps(b.get('sue_grade') or {}, default=str)}`",
"",
f"**Momentum-conditional SUE:** `{json.dumps(b.get('momentum_conditional_sue') or {}, default=str)}`",
"",
])
lines.extend([
"",
"## Verdict",
"",
f"**{payload.get('verdict')}**",
"",
payload.get("verdict_detail") or "",
"",
"## What a human must decide next",
"",
payload.get("human_next") or "- Review; no auto-ship.",
"",
f"Artifacts: `{payload.get('report_path')}`",
"",
])
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
async def _main() -> None:
args = _parse_args()
snapshot = Path(args.snapshot)
if not snapshot.exists():
raise SystemExit(f"Missing snapshot {snapshot}")
if args.allow_spawn:
os.environ["BACKTEST_ALLOW_SPAWN"] = "1"
events, meta = _load_earnings(snapshot)
# Race guard lite on earnings completeness.
provenance = {
"snapshot": str(snapshot.resolve()),
"n_earnings_events": len(events),
"backfill_meta": meta,
"announce_range": {
"min": min((e["announce_date"] for e in events), default=None),
"max": max((e["announce_date"] for e in events), default=None),
},
"with_actual_and_estimate": sum(
1
for e in events
if e.get("eps_actual") is not None and e.get("eps_estimate") is not None
),
}
print(
f"Earnings events: {provenance['n_earnings_events']} "
f"(with act+est={provenance['with_actual_and_estimate']}) meta={meta}"
)
if meta and meta.get("done", 0) < 0.9 * (meta.get("universe_tickers") or 1):
print(
"WARNING: earnings backfill incomplete "
f"({meta.get('done')}/{meta.get('universe_tickers')}). "
"Results may be biased; resume backfill."
)
exp_2a = None
exp_2b = None
if not args.skip_2a:
print("Running 2a earnings-gap diagnostic…")
exp_2a = await _run_2a(
snapshot, events, quiet=args.quiet, workers=args.workers
)
print(
" 2a losses<-1R with earnings:",
(exp_2a.get("q1_losses_worse_than_minus_1r") or {}),
)
if not args.skip_2b:
print("Running 2b SUE IC harness…")
exp_2b = await _run_2b_ic(
snapshot, events, quiet=args.quiet, workers=args.workers
)
g = exp_2b.get("sue_grade") or {}
print(f" 2b SUE green={g.get('green')} {g.get('reason')}")
# Verdict
if exp_2b and (exp_2b.get("sue_grade") or {}).get("green"):
verdict = "PROMOTE (2b SUE) — STOP for human wire design"
detail = (
"SUE cleared iron rule. No book integration without human approval. "
"2a remains report-only."
)
human = (
"- Design tilt vs second gate if desired.\n"
"- Do not auto-filter from 2a without separate approval + tail review."
)
else:
sue_ic = None
if exp_2b:
sue_ic = ((exp_2b.get("sue_grade") or {}).get("row") or {}).get("mean_ic")
if sue_ic is not None and abs(float(sue_ic)) >= 0.015:
verdict = "PARK"
detail = f"SUE IC={sue_ic} below iron bar or unreliable; keep data, no wire."
else:
verdict = "DEAD (2b) / REPORT-ONLY (2a)"
detail = (
"SUE does not clear iron rule on this window. "
"2a distributions for human risk review only — no filter."
)
human = (
"- No SUE book change.\n"
"- Read 2a tails before considering any earnings-avoid filter."
)
stamp = datetime.now().strftime("%Y%m%d-%H%M%S")
out = Path(args.out) if args.out else Path("reports") / f"earnings-gap-sue-{stamp}.json"
payload = {
"generated_at": datetime.now().isoformat(),
"data_provenance": provenance,
"experiment_2a": exp_2a,
"experiment_2b": exp_2b,
"verdict": verdict,
"verdict_detail": detail,
"human_next": human,
"report_path": str(out.as_posix()),
"fmp_note": (
"Bulk earnings-calendar is paid (402 on free tier). "
"Backfill used per-symbol /stable/earnings; see earnings-backfill-status.json."
),
}
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text(json.dumps(payload, indent=2, default=str) + "\n", encoding="utf-8")
md = Path("docs/research/earnings-gap-and-sue.md")
_write_md(md, payload)
out.with_suffix(".md").write_text(md.read_text(encoding="utf-8"), encoding="utf-8")
print(f"Verdict: {verdict}")
print(f"Wrote {out}")
if __name__ == "__main__":
asyncio.run(_main())
scripts/run_history_depth_research.py
+476
View File
@@ -0,0 +1,476 @@
"""History-depth extension research (local / MacBook).
Phases
------
coverage — bars per calendar year; no rebuild
harness — race-guard snapshot, full signal_eval, era split pre/post-2021
Does not retune production knobs. Does not modify scheduler/gates.
Example
-------
python scripts/run_history_depth_research.py --phase coverage \\
--snapshot backtest_snapshots/prod.sqlite
python scripts/run_history_depth_research.py --phase harness \\
--snapshot backtest_snapshots/research.sqlite --workers 8 --allow-spawn
"""
from __future__ import annotations
import argparse
import asyncio
import json
import os
import sys
from collections import defaultdict
from datetime import date, datetime
from pathlib import Path
from typing import Any
from sqlalchemy import create_engine, text
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
ERA_SPLIT = date(2021, 1, 1)
SURVIVORSHIP_BANNER = (
"SURVIVORSHIP BIAS: today's constituents backfilled historically. "
"Absolute Sharpe/CAGR levels on deep history are optimistic. "
"Use RELATIVE signal IC comparisons and era stability only — not levels."
)
def _sqlite_url(path: Path) -> str:
return f"sqlite+aiosqlite:///{path.resolve().as_posix()}"
def _parse_args() -> argparse.Namespace:
p = argparse.ArgumentParser(description=__doc__)
p.add_argument("--phase", choices=("coverage", "harness", "all"), default="all")
p.add_argument("--snapshot", default="backtest_snapshots/research.sqlite")
p.add_argument("--workers", type=int, default=8)
p.add_argument("--allow-spawn", action="store_true")
p.add_argument("--quiet", action="store_true")
p.add_argument("--out", default=None)
return p.parse_args()
def _coverage_report(snapshot: Path) -> dict[str, Any]:
engine = create_engine(
f"sqlite:///{snapshot.resolve().as_posix()}",
future=True,
)
try:
with engine.connect() as conn:
ticker_n = int(conn.execute(text("SELECT COUNT(*) FROM tickers")).scalar_one())
ohlcv_n = int(
conn.execute(text("SELECT COUNT(*) FROM ohlcv_records")).scalar_one()
)
d_range = conn.execute(
text("SELECT MIN(date), MAX(date) FROM ohlcv_records")
).fetchone()
# Bars per calendar year (global).
by_year = conn.execute(
text(
"""
SELECT substr(date, 1, 4) AS y, COUNT(*) AS n,
COUNT(DISTINCT ticker_id) AS tickers
FROM ohlcv_records
GROUP BY substr(date, 1, 4)
ORDER BY y
"""
)
).fetchall()
# Per-symbol min/max date + bar count (summary percentiles).
per_sym = conn.execute(
text(
"""
SELECT t.symbol, COUNT(*) AS n, MIN(o.date), MAX(o.date)
FROM ohlcv_records o
JOIN tickers t ON t.id = o.ticker_id
GROUP BY t.symbol
"""
)
).fetchall()
finally:
engine.dispose()
ns = sorted(int(r[1]) for r in per_sym)
def pct(p: float) -> int | None:
if not ns:
return None
i = int(round(p * (len(ns) - 1)))
return ns[i]
starts = sorted(str(r[2]) for r in per_sym if r[2])
start_hist: dict[str, int] = defaultdict(int)
for s in starts:
start_hist[s[:4]] += 1
return {
"snapshot": str(snapshot.resolve()),
"ticker_count": ticker_n,
"ohlcv_row_count": ohlcv_n,
"date_range": {"min": d_range[0], "max": d_range[1]},
"bars_per_year": [
{"year": y, "bars": n, "tickers_with_bars": t} for y, n, t in by_year
],
"bars_per_symbol": {
"min": ns[0] if ns else None,
"p10": pct(0.10),
"p50": pct(0.50),
"p90": pct(0.90),
"max": ns[-1] if ns else None,
},
"symbols_by_start_year": dict(sorted(start_hist.items())),
"note": (
"Where ticker counts drop in early years, the feed (or listing history) "
"thins — do not treat those years as a full 505-name cross-section."
),
"survivorship_banner": SURVIVORSHIP_BANNER,
}
def _assert_complete(snapshot: Path) -> dict[str, Any]:
from scripts.research_snapshot_manifest import ( # type: ignore
assert_research_snapshot_complete,
load_manifest,
)
m = load_manifest(snapshot)
if m is None:
# Prod snapshot may lack manifest; still require healthy bar depth.
eng = create_engine(
f"sqlite:///{snapshot.resolve().as_posix()}",
future=True,
)
try:
with eng.connect() as conn:
avg = conn.execute(
text(
"""
SELECT AVG(c) FROM (
SELECT COUNT(*) AS c FROM ohlcv_records GROUP BY ticker_id
)
"""
)
).scalar_one()
finally:
eng.dispose()
if avg is None or float(avg) < 400:
raise SystemExit(
f"No completion manifest and avg bars={avg} look short. "
"Rebuild research.sqlite via extend_snapshot_universe.py"
)
return {"manifest": None, "avg_bars": float(avg), "ok": True}
return {"manifest": assert_research_snapshot_complete(snapshot), "ok": True}
async def _harness(snapshot: Path, *, workers: int, quiet: bool) -> dict[str, Any]:
from app.config import settings
from app.services.backtest_service import run_backtest
os.environ["BACKTEST_SNAPSHOT_OFFLINE"] = "1"
os.environ["BACKTEST_SIGNAL_EVAL_ONLY"] = "1"
if Path("data/research/ticker_sector_map.json").exists():
os.environ["BACKTEST_SECTOR_MAP_PATH"] = str(
Path("data/research/ticker_sector_map.json").resolve()
)
settings.backtest_workers = workers
engine = create_async_engine(_sqlite_url(snapshot), pool_pre_ping=True)
Session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
def progress(done: int, total: int, symbol: str) -> None:
if quiet:
return
print(f" progress {done}/{total} {symbol}", end="\r", flush=True)
try:
async with Session() as db:
report = await run_backtest(db, progress_cb=progress, cadence="weekly")
finally:
await engine.dispose()
if not quiet:
print()
signal_eval = report.get("signal_eval") or []
# Era-split IC: recompute from collected is not available post-run.
# Approximate via second pass is expensive; instead document that era split
# requires collecting weekly ICs. We re-run evaluation if the report embeds
# nothing — for v1, call internal collection is too heavy to duplicate.
# Lightweight approach: mark era_split as requiring BACKTEST with custom
# filter — implemented below by re-scoring from a dedicated collection pass.
era = await _era_split_ics(snapshot, workers=workers, quiet=quiet)
return {
"survivorship_banner": SURVIVORSHIP_BANNER,
"signal_eval": signal_eval,
"era_split": era,
"params": report.get("params"),
"tickers": report.get("tickers"),
"generated_at_run": report.get("generated_at"),
}
async def _era_split_ics(
snapshot: Path, *, workers: int, quiet: bool
) -> dict[str, Any]:
"""Collect weekly signal series and evaluate pre/post ERA_SPLIT separately."""
from app.config import settings
from app.services import backtest_service as bt
from app.services.benchmark_service import load_benchmark_closes
from app.models.ticker import Ticker
from sqlalchemy import select
from collections import defaultdict as dd
os.environ["BACKTEST_SNAPSHOT_OFFLINE"] = "1"
settings.backtest_workers = max(1, workers)
engine = create_async_engine(_sqlite_url(snapshot), pool_pre_ping=True)
Session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
collected: dict = dd(lambda: dd(list))
try:
async with Session() as db:
tickers = list(
(await db.execute(select(Ticker).order_by(Ticker.symbol))).scalars()
)
spy = await load_benchmark_closes(db, "SPY")
symbol_to_sector = {}
sector_etf: dict = {}
try:
from app.services.sector_map import (
SECTOR_ETFS,
load_ticker_sector_map,
)
symbol_to_sector = load_ticker_sector_map()
for etf in SECTOR_ETFS:
series = await load_benchmark_closes(db, etf)
if series:
sector_etf[etf] = series
except Exception:
pass
for idx, t in enumerate(tickers):
if not quiet and idx % 100 == 0:
print(f" era-collect {idx}/{len(tickers)}", end="\r", flush=True)
cols = await bt._fetch_columns(db, t.symbol)
if cols is None:
continue
records = [
type(
"R",
(),
{
"date": date.fromordinal(int(cols[0][i])),
"close": cols[4][i],
"high": cols[2][i],
"volume": cols[5][i] if len(cols) > 5 else 0,
},
)()
for i in range(len(cols[0]))
]
series = bt._signal_series(
records,
spy,
symbol=t.symbol,
sector_etf_closes=bt._sector_etf_closes_for_symbol(
t.symbol, symbol_to_sector, sector_etf
),
)
for name, weeks in series.items():
for wk, pairs in weeks.items():
collected[name][wk].extend(pairs)
if symbol_to_sector:
bt._inject_sector_demeaned_momentum(collected, symbol_to_sector)
finally:
await engine.dispose()
if not quiet:
print()
def _filter_era(coll: dict, *, pre: bool) -> dict:
out: dict = dd(lambda: dd(list))
for name, weeks in coll.items():
for wk, recs in weeks.items():
# ISO week key (year, week) — approximate era by ISO year.
year = int(wk[0]) if isinstance(wk, tuple) else int(str(wk)[:4])
if pre and year >= ERA_SPLIT.year:
continue
if not pre and year < ERA_SPLIT.year:
continue
out[name][wk].extend(recs)
return out
pre_eval = bt._signal_evaluation(_filter_era(collected, pre=True))
post_eval = bt._signal_evaluation(_filter_era(collected, pre=False))
full_eval = bt._signal_evaluation(collected)
def _index(rows: list[dict]) -> dict[str, dict]:
return {r["signal"]: r for r in rows}
return {
"era_split_date": ERA_SPLIT.isoformat(),
"note": "Diagnostic only — not a tuning input. Nested lookbacks are not OOS.",
"full": _index(full_eval),
"pre_2021": _index(pre_eval),
"post_2021": _index(post_eval),
}
def _write_md(path: Path, payload: dict) -> None:
pre = path.read_text(encoding="utf-8") if path.exists() else ""
marker = "## Results"
idx = pre.find(marker)
header = pre[:idx] if idx >= 0 else pre.split("## Verdict")[0]
lines = [
header.rstrip(),
"",
"## Results",
"",
f"Generated: `{payload.get('generated_at')}`",
"",
f"> **{SURVIVORSHIP_BANNER}**",
"",
"### Coverage",
"",
f"```json\n{json.dumps(payload.get('coverage') or {}, indent=2, default=str)}\n```",
"",
"### Race guard",
"",
f"```json\n{json.dumps(payload.get('race_guard') or {}, indent=2, default=str)}\n```",
"",
"### Signal IC (full extended window)",
"",
]
harness = payload.get("harness") or {}
rows = harness.get("signal_eval") or []
if rows:
lines.extend([
"| signal | mean_ic | ic_t_stat | weeks | avg_N | reliable |",
"|---|---:|---:|---:|---:|---|",
])
for r in rows:
lines.append(
f"| {r.get('signal')} | {r.get('mean_ic')} | {r.get('ic_t_stat')} | "
f"{r.get('weeks')} | {r.get('avg_cross_section')} | {r.get('reliable')} |"
)
else:
lines.append("_Harness not run this pass._")
era = (harness.get("era_split") or {})
lines.extend(["", "### Era split (diagnostic only)", ""])
if era:
for label in ("full", "pre_2021", "post_2021"):
block = era.get(label) or {}
lines.append(f"#### {label}")
lines.append("")
lines.append("| signal | mean_ic | t | weeks | N |")
lines.append("|---|---:|---:|---:|---:|")
for name in sorted(block):
r = block[name]
lines.append(
f"| {name} | {r.get('mean_ic')} | {r.get('ic_t_stat')} | "
f"{r.get('weeks')} | {r.get('avg_cross_section')} |"
)
lines.append("")
else:
lines.append("_No era split._")
lines.extend([
"",
"## Verdict",
"",
f"**{payload.get('verdict')}**",
"",
payload.get("verdict_detail") or "",
"",
"## What a human must decide next",
"",
payload.get("human_next")
or "- Do not retune production knobs from this report without review.",
"",
f"Artifacts: `{payload.get('report_path')}`",
"",
])
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
async def _main() -> None:
args = _parse_args()
snapshot = Path(args.snapshot)
if not snapshot.exists():
raise SystemExit(f"Missing snapshot: {snapshot}")
if args.allow_spawn:
os.environ["BACKTEST_ALLOW_SPAWN"] = "1"
coverage = None
race = None
harness = None
if args.phase in ("coverage", "all"):
print("Coverage probe…")
coverage = _coverage_report(snapshot)
print(
f" tickers={coverage['ticker_count']} ohlcv={coverage['ohlcv_row_count']} "
f"range={coverage['date_range']}"
)
for row in coverage["bars_per_year"]:
print(
f" year {row['year']}: bars={row['bars']} "
f"tickers={row['tickers_with_bars']}"
)
if args.phase in ("harness", "all"):
print("Race guard…")
race = _assert_complete(snapshot)
print(f" ok={race.get('ok')}")
print("Full harness + era split (LONG)…")
print(f" {SURVIVORSHIP_BANNER}")
harness = await _harness(
snapshot, workers=args.workers, quiet=args.quiet
)
stamp = datetime.now().strftime("%Y%m%d-%H%M%S")
out = (
Path(args.out)
if args.out
else Path("reports") / f"history-depth-{stamp}.json"
)
payload = {
"generated_at": datetime.now().isoformat(),
"survivorship_banner": SURVIVORSHIP_BANNER,
"coverage": coverage,
"race_guard": race,
"harness": harness,
"verdict": "PENDING_HUMAN" if harness else "COVERAGE_ONLY",
"verdict_detail": (
"Harness complete — human interprets relative IC / era stability. "
"No production retune from this artifact."
if harness
else "Coverage probe only; run --phase harness after deep rebuild."
),
"human_next": (
"- Compare sector residual vs market residual across eras.\n"
"- If pre-2021 IC collapses, park Task 1 wire-in.\n"
"- Do not retune production knobs on deep history levels."
),
"report_path": str(out.as_posix()),
}
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text(json.dumps(payload, indent=2, default=str) + "\n", encoding="utf-8")
md = Path("docs/research/history-depth-extension.md")
_write_md(md, payload)
out.with_suffix(".md").write_text(md.read_text(encoding="utf-8"), encoding="utf-8")
print(f"Wrote {out}")
print(f"Wrote {md}")
if __name__ == "__main__":
asyncio.run(_main())
scripts/run_sector_residual_research.py
+1014
View File
File diff suppressed because it is too large Load Diff