dennisthiessen
9704e0d85a
Add research-only snapshot extender, PIT dollar-volume mask for signal IC, rank-only harness path, fingerprint+breadth runner, and docs. Fingerprint reproduced IC -0.045 / t -2.91 on prod.sqlite. No production gate/schedule changes.
330 lines
11 KiB
Python
330 lines
11 KiB
Python
"""Extend a *copy* of the production backtest snapshot with broad-universe OHLCV.
|
||
|
||
Research only — never writes to production Postgres.
|
||
|
||
Pipeline
|
||
--------
|
||
1. Copy ``--source`` snapshot (default ``backtest_snapshots/prod.sqlite``) to
|
||
``--output`` (default ``backtest_snapshots/research.sqlite``).
|
||
2. Resolve symbol pool = nasdaq_all ∪ sp500 via ``ticker_universe_service``.
|
||
3. Fetch ~5y daily bars from Alpaca for symbols missing (or short) in the copy.
|
||
4. Insert new tickers + OHLCV; mark them in side table ``research_rank_only``
|
||
so the harness can feed signal IC without GTL/candidate replay.
|
||
|
||
Resume-friendly: re-running skips symbols that already have ≥ ``--min-bars``.
|
||
|
||
Example
|
||
-------
|
||
python scripts/extend_snapshot_universe.py \\
|
||
--source backtest_snapshots/prod.sqlite \\
|
||
--output backtest_snapshots/research.sqlite \\
|
||
--force-copy
|
||
|
||
# smoke: first 50 missing symbols only
|
||
python scripts/extend_snapshot_universe.py --limit 50
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import asyncio
|
||
import shutil
|
||
import sys
|
||
import time
|
||
from datetime import date, datetime, timedelta, timezone
|
||
from pathlib import Path
|
||
|
||
from sqlalchemy import create_engine, select, text
|
||
from sqlalchemy.orm import Session
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
if str(ROOT) not in sys.path:
|
||
sys.path.insert(0, str(ROOT))
|
||
|
||
|
||
def _parse_args() -> argparse.Namespace:
|
||
p = argparse.ArgumentParser(description=__doc__)
|
||
p.add_argument(
|
||
"--source",
|
||
default="backtest_snapshots/prod.sqlite",
|
||
help="Existing prod snapshot to copy (read-only after copy).",
|
||
)
|
||
p.add_argument(
|
||
"--output",
|
||
default="backtest_snapshots/research.sqlite",
|
||
help="Research snapshot path (created/updated).",
|
||
)
|
||
p.add_argument(
|
||
"--force-copy",
|
||
action="store_true",
|
||
help="Overwrite output by re-copying from source first.",
|
||
)
|
||
p.add_argument(
|
||
"--history-days",
|
||
type=int,
|
||
default=1825,
|
||
help="OHLCV lookback days (~5y). Default 1825.",
|
||
)
|
||
p.add_argument(
|
||
"--min-bars",
|
||
type=int,
|
||
default=260,
|
||
help="Skip re-fetch when a symbol already has this many bars.",
|
||
)
|
||
p.add_argument(
|
||
"--limit",
|
||
type=int,
|
||
default=None,
|
||
help="Max *new* symbols to fetch (smoke tests).",
|
||
)
|
||
p.add_argument(
|
||
"--sleep",
|
||
type=float,
|
||
default=0.15,
|
||
help="Seconds between Alpaca symbol requests (rate-limit cushion).",
|
||
)
|
||
p.add_argument(
|
||
"--max-retries",
|
||
type=int,
|
||
default=5,
|
||
help="Retries per symbol on RateLimitError.",
|
||
)
|
||
p.add_argument("--quiet", action="store_true")
|
||
return p.parse_args()
|
||
|
||
|
||
def _ensure_rank_only_table(conn) -> None:
|
||
conn.execute(
|
||
text(
|
||
"""
|
||
CREATE TABLE IF NOT EXISTS research_rank_only (
|
||
ticker_id INTEGER PRIMARY KEY,
|
||
symbol TEXT NOT NULL UNIQUE
|
||
)
|
||
"""
|
||
)
|
||
)
|
||
conn.commit()
|
||
|
||
|
||
async def _resolve_pool() -> tuple[list[str], dict[str, str]]:
|
||
"""Return sorted unique symbols and source labels."""
|
||
from app.database import async_session_factory
|
||
from app.services.ticker_universe_service import fetch_universe_symbols
|
||
|
||
sources: dict[str, str] = {}
|
||
symbols: set[str] = set()
|
||
# Need a DB session for cache writes; use local async engine if configured,
|
||
# but public/FMP fetch works with any session. Prefer a throwaway sqlite.
|
||
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
|
||
from sqlalchemy.ext.asyncio import AsyncSession
|
||
|
||
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
|
||
Session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
|
||
try:
|
||
async with Session() as db:
|
||
for universe in ("nasdaq_all", "sp500"):
|
||
try:
|
||
syms, src = await fetch_universe_symbols(db, universe)
|
||
except Exception as exc:
|
||
print(f"WARNING: universe {universe} failed: {exc}")
|
||
continue
|
||
sources[universe] = src
|
||
symbols.update(syms)
|
||
print(f" {universe}: {len(syms)} symbols (source={src})")
|
||
finally:
|
||
await engine.dispose()
|
||
return sorted(symbols), sources
|
||
|
||
|
||
async def _fetch_symbol_bars(
|
||
provider,
|
||
symbol: str,
|
||
start: date,
|
||
end: date,
|
||
*,
|
||
max_retries: int,
|
||
sleep_s: float,
|
||
) -> list:
|
||
from app.exceptions import ProviderError, RateLimitError
|
||
|
||
for attempt in range(max_retries):
|
||
try:
|
||
bars = await provider.fetch_ohlcv(symbol, start, end)
|
||
if sleep_s > 0:
|
||
await asyncio.sleep(sleep_s)
|
||
return bars
|
||
except RateLimitError:
|
||
wait = min(60.0, 2.0 ** attempt)
|
||
print(f" rate limited on {symbol}; sleep {wait:.0f}s")
|
||
await asyncio.sleep(wait)
|
||
except ProviderError as exc:
|
||
if attempt + 1 >= max_retries:
|
||
raise
|
||
await asyncio.sleep(1.0)
|
||
_ = exc
|
||
return []
|
||
|
||
|
||
async def _main() -> None:
|
||
args = _parse_args()
|
||
source = Path(args.source)
|
||
output = Path(args.output)
|
||
if not source.exists():
|
||
raise SystemExit(f"Source snapshot not found: {source}")
|
||
|
||
if args.force_copy or not output.exists():
|
||
output.parent.mkdir(parents=True, exist_ok=True)
|
||
if output.exists():
|
||
output.unlink()
|
||
print(f"Copying {source} → {output}")
|
||
shutil.copy2(source, output)
|
||
else:
|
||
print(f"Updating existing research snapshot: {output}")
|
||
|
||
from app.config import settings
|
||
from app.models.ohlcv import OHLCVRecord
|
||
from app.models.ticker import Ticker
|
||
from app.providers.alpaca import AlpacaOHLCVProvider
|
||
|
||
if not settings.alpaca_api_key or not settings.alpaca_api_secret:
|
||
raise SystemExit("ALPACA_API_KEY / ALPACA_API_SECRET required in .env")
|
||
|
||
provider = AlpacaOHLCVProvider(settings.alpaca_api_key, settings.alpaca_api_secret)
|
||
end = date.today()
|
||
start = end - timedelta(days=int(args.history_days))
|
||
|
||
print("Resolving universe pool (nasdaq_all ∪ sp500)…")
|
||
pool, sources = await _resolve_pool()
|
||
print(f"Pool size: {len(pool)} (sources={sources})")
|
||
|
||
# Sync sqlite via sqlalchemy core (simpler than async for bulk insert)
|
||
engine = create_engine(f"sqlite:///{output.resolve().as_posix()}")
|
||
with Session(engine) as session:
|
||
_ensure_rank_only_table(session.connection())
|
||
existing = {
|
||
row.symbol: row
|
||
for row in session.execute(select(Ticker)).scalars().all()
|
||
}
|
||
prod_symbols = set(existing)
|
||
|
||
# Bar counts
|
||
bar_counts: dict[str, int] = {}
|
||
for sym, ticker in existing.items():
|
||
n = session.execute(
|
||
text("SELECT COUNT(*) FROM ohlcv_records WHERE ticker_id = :tid"),
|
||
{"tid": ticker.id},
|
||
).scalar_one()
|
||
bar_counts[sym] = int(n)
|
||
|
||
to_fetch: list[str] = []
|
||
for sym in pool:
|
||
if sym in existing and bar_counts.get(sym, 0) >= args.min_bars:
|
||
# Existing production or previously extended — keep rank_only
|
||
# only for *new* research names, not original prod universe.
|
||
continue
|
||
to_fetch.append(sym)
|
||
|
||
if args.limit is not None:
|
||
to_fetch = to_fetch[: max(0, int(args.limit))]
|
||
|
||
print(f"Symbols to fetch/extend: {len(to_fetch)}")
|
||
ok = 0
|
||
fail = 0
|
||
t0 = time.monotonic()
|
||
for index, sym in enumerate(to_fetch, 1):
|
||
try:
|
||
bars = await _fetch_symbol_bars(
|
||
provider,
|
||
sym,
|
||
start,
|
||
end,
|
||
max_retries=args.max_retries,
|
||
sleep_s=args.sleep,
|
||
)
|
||
except Exception as exc:
|
||
fail += 1
|
||
if not args.quiet:
|
||
print(f" [{index}/{len(to_fetch)}] {sym} FAIL {exc}")
|
||
continue
|
||
|
||
if not bars:
|
||
fail += 1
|
||
if not args.quiet:
|
||
print(f" [{index}/{len(to_fetch)}] {sym} empty")
|
||
continue
|
||
|
||
ticker = existing.get(sym)
|
||
is_new = ticker is None
|
||
if ticker is None:
|
||
ticker = Ticker(symbol=sym, name=None, created_at=datetime.now(timezone.utc))
|
||
session.add(ticker)
|
||
session.flush()
|
||
existing[sym] = ticker
|
||
|
||
# Upsert bars (delete+insert range for simplicity on research path)
|
||
session.execute(
|
||
text(
|
||
"DELETE FROM ohlcv_records WHERE ticker_id = :tid "
|
||
"AND date >= :start AND date <= :end"
|
||
),
|
||
{"tid": ticker.id, "start": start.isoformat(), "end": end.isoformat()},
|
||
)
|
||
now = datetime.utcnow()
|
||
session.bulk_insert_mappings(
|
||
OHLCVRecord,
|
||
[
|
||
{
|
||
"ticker_id": ticker.id,
|
||
"date": b.date,
|
||
"open": b.open,
|
||
"high": b.high,
|
||
"low": b.low,
|
||
"close": b.close,
|
||
"volume": b.volume,
|
||
"created_at": now,
|
||
}
|
||
for b in bars
|
||
],
|
||
)
|
||
|
||
# rank_only only for names that were NOT in the original production
|
||
# snapshot at copy time (or are newly introduced to this research DB).
|
||
if is_new or sym not in prod_symbols:
|
||
# Re-evaluate: if source copy already had the symbol, don't flag.
|
||
# Only new inserts get rank_only.
|
||
if is_new:
|
||
session.execute(
|
||
text(
|
||
"INSERT OR REPLACE INTO research_rank_only "
|
||
"(ticker_id, symbol) VALUES (:tid, :sym)"
|
||
),
|
||
{"tid": ticker.id, "sym": sym},
|
||
)
|
||
|
||
session.commit()
|
||
ok += 1
|
||
if not args.quiet and (index % 25 == 0 or index == len(to_fetch)):
|
||
elapsed = time.monotonic() - t0
|
||
print(
|
||
f" progress {index}/{len(to_fetch)} ok={ok} fail={fail} "
|
||
f"elapsed={elapsed/60:.1f}m last={sym} bars={len(bars)}"
|
||
)
|
||
|
||
rank_only_n = session.execute(
|
||
text("SELECT COUNT(*) FROM research_rank_only")
|
||
).scalar_one()
|
||
ticker_n = session.execute(text("SELECT COUNT(*) FROM tickers")).scalar_one()
|
||
ohlcv_n = session.execute(text("SELECT COUNT(*) FROM ohlcv_records")).scalar_one()
|
||
|
||
print("Done.")
|
||
print(f" output: {output}")
|
||
print(f" tickers: {ticker_n}")
|
||
print(f" ohlcv rows: {ohlcv_n}")
|
||
print(f" research_rank_only: {rank_only_n}")
|
||
print(f" fetched ok/fail: {ok}/{fail}")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
asyncio.run(_main())
|