feat: Phase B fip_id liquid-breadth research tooling

Add research-only snapshot extender, PIT dollar-volume mask for signal IC, rank-only harness path, fingerprint+breadth runner, and docs. Fingerprint reproduced IC -0.045 / t -2.91 on prod.sqlite. No production gate/schedule changes.
2026-07-18 20:22:11 +02:00
parent c2d29184dd
commit 9704e0d85a
8 changed files with 1578 additions and 73 deletions
@@ -0,0 +1,329 @@
+"""Extend a *copy* of the production backtest snapshot with broad-universe OHLCV.
+
+Research only — never writes to production Postgres.
+
+Pipeline
+--------
+1. Copy ``--source`` snapshot (default ``backtest_snapshots/prod.sqlite``) to
+   ``--output`` (default ``backtest_snapshots/research.sqlite``).
+2. Resolve symbol pool = nasdaq_all ∪ sp500 via ``ticker_universe_service``.
+3. Fetch ~5y daily bars from Alpaca for symbols missing (or short) in the copy.
+4. Insert new tickers + OHLCV; mark them in side table ``research_rank_only``
+   so the harness can feed signal IC without GTL/candidate replay.
+
+Resume-friendly: re-running skips symbols that already have ≥ ``--min-bars``.
+
+Example
+-------
+  python scripts/extend_snapshot_universe.py \\
+      --source backtest_snapshots/prod.sqlite \\
+      --output backtest_snapshots/research.sqlite \\
+      --force-copy
+
+  # smoke: first 50 missing symbols only
+  python scripts/extend_snapshot_universe.py --limit 50
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import shutil
+import sys
+import time
+from datetime import date, datetime, timedelta, timezone
+from pathlib import Path
+
+from sqlalchemy import create_engine, select, text
+from sqlalchemy.orm import Session
+
+ROOT = Path(__file__).resolve().parents[1]
+if str(ROOT) not in sys.path:
+    sys.path.insert(0, str(ROOT))
+
+
+def _parse_args() -> argparse.Namespace:
+    p = argparse.ArgumentParser(description=__doc__)
+    p.add_argument(
+        "--source",
+        default="backtest_snapshots/prod.sqlite",
+        help="Existing prod snapshot to copy (read-only after copy).",
+    )
+    p.add_argument(
+        "--output",
+        default="backtest_snapshots/research.sqlite",
+        help="Research snapshot path (created/updated).",
+    )
+    p.add_argument(
+        "--force-copy",
+        action="store_true",
+        help="Overwrite output by re-copying from source first.",
+    )
+    p.add_argument(
+        "--history-days",
+        type=int,
+        default=1825,
+        help="OHLCV lookback days (~5y). Default 1825.",
+    )
+    p.add_argument(
+        "--min-bars",
+        type=int,
+        default=260,
+        help="Skip re-fetch when a symbol already has this many bars.",
+    )
+    p.add_argument(
+        "--limit",
+        type=int,
+        default=None,
+        help="Max *new* symbols to fetch (smoke tests).",
+    )
+    p.add_argument(
+        "--sleep",
+        type=float,
+        default=0.15,
+        help="Seconds between Alpaca symbol requests (rate-limit cushion).",
+    )
+    p.add_argument(
+        "--max-retries",
+        type=int,
+        default=5,
+        help="Retries per symbol on RateLimitError.",
+    )
+    p.add_argument("--quiet", action="store_true")
+    return p.parse_args()
+
+
+def _ensure_rank_only_table(conn) -> None:
+    conn.execute(
+        text(
+            """
+            CREATE TABLE IF NOT EXISTS research_rank_only (
+                ticker_id INTEGER PRIMARY KEY,
+                symbol TEXT NOT NULL UNIQUE
+            )
+            """
+        )
+    )
+    conn.commit()
+
+
+async def _resolve_pool() -> tuple[list[str], dict[str, str]]:
+    """Return sorted unique symbols and source labels."""
+    from app.database import async_session_factory
+    from app.services.ticker_universe_service import fetch_universe_symbols
+
+    sources: dict[str, str] = {}
+    symbols: set[str] = set()
+    # Need a DB session for cache writes; use local async engine if configured,
+    # but public/FMP fetch works with any session. Prefer a throwaway sqlite.
+    from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
+    from sqlalchemy.ext.asyncio import AsyncSession
+
+    engine = create_async_engine("sqlite+aiosqlite:///:memory:")
+    Session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
+    try:
+        async with Session() as db:
+            for universe in ("nasdaq_all", "sp500"):
+                try:
+                    syms, src = await fetch_universe_symbols(db, universe)
+                except Exception as exc:
+                    print(f"WARNING: universe {universe} failed: {exc}")
+                    continue
+                sources[universe] = src
+                symbols.update(syms)
+                print(f"  {universe}: {len(syms)} symbols (source={src})")
+    finally:
+        await engine.dispose()
+    return sorted(symbols), sources
+
+
+async def _fetch_symbol_bars(
+    provider,
+    symbol: str,
+    start: date,
+    end: date,
+    *,
+    max_retries: int,
+    sleep_s: float,
+) -> list:
+    from app.exceptions import ProviderError, RateLimitError
+
+    for attempt in range(max_retries):
+        try:
+            bars = await provider.fetch_ohlcv(symbol, start, end)
+            if sleep_s > 0:
+                await asyncio.sleep(sleep_s)
+            return bars
+        except RateLimitError:
+            wait = min(60.0, 2.0 ** attempt)
+            print(f"  rate limited on {symbol}; sleep {wait:.0f}s")
+            await asyncio.sleep(wait)
+        except ProviderError as exc:
+            if attempt + 1 >= max_retries:
+                raise
+            await asyncio.sleep(1.0)
+            _ = exc
+    return []
+
+
+async def _main() -> None:
+    args = _parse_args()
+    source = Path(args.source)
+    output = Path(args.output)
+    if not source.exists():
+        raise SystemExit(f"Source snapshot not found: {source}")
+
+    if args.force_copy or not output.exists():
+        output.parent.mkdir(parents=True, exist_ok=True)
+        if output.exists():
+            output.unlink()
+        print(f"Copying {source} → {output}")
+        shutil.copy2(source, output)
+    else:
+        print(f"Updating existing research snapshot: {output}")
+
+    from app.config import settings
+    from app.models.ohlcv import OHLCVRecord
+    from app.models.ticker import Ticker
+    from app.providers.alpaca import AlpacaOHLCVProvider
+
+    if not settings.alpaca_api_key or not settings.alpaca_api_secret:
+        raise SystemExit("ALPACA_API_KEY / ALPACA_API_SECRET required in .env")
+
+    provider = AlpacaOHLCVProvider(settings.alpaca_api_key, settings.alpaca_api_secret)
+    end = date.today()
+    start = end - timedelta(days=int(args.history_days))
+
+    print("Resolving universe pool (nasdaq_all ∪ sp500)…")
+    pool, sources = await _resolve_pool()
+    print(f"Pool size: {len(pool)} (sources={sources})")
+
+    # Sync sqlite via sqlalchemy core (simpler than async for bulk insert)
+    engine = create_engine(f"sqlite:///{output.resolve().as_posix()}")
+    with Session(engine) as session:
+        _ensure_rank_only_table(session.connection())
+        existing = {
+            row.symbol: row
+            for row in session.execute(select(Ticker)).scalars().all()
+        }
+        prod_symbols = set(existing)
+
+        # Bar counts
+        bar_counts: dict[str, int] = {}
+        for sym, ticker in existing.items():
+            n = session.execute(
+                text("SELECT COUNT(*) FROM ohlcv_records WHERE ticker_id = :tid"),
+                {"tid": ticker.id},
+            ).scalar_one()
+            bar_counts[sym] = int(n)
+
+        to_fetch: list[str] = []
+        for sym in pool:
+            if sym in existing and bar_counts.get(sym, 0) >= args.min_bars:
+                # Existing production or previously extended — keep rank_only
+                # only for *new* research names, not original prod universe.
+                continue
+            to_fetch.append(sym)
+
+        if args.limit is not None:
+            to_fetch = to_fetch[: max(0, int(args.limit))]
+
+        print(f"Symbols to fetch/extend: {len(to_fetch)}")
+        ok = 0
+        fail = 0
+        t0 = time.monotonic()
+        for index, sym in enumerate(to_fetch, 1):
+            try:
+                bars = await _fetch_symbol_bars(
+                    provider,
+                    sym,
+                    start,
+                    end,
+                    max_retries=args.max_retries,
+                    sleep_s=args.sleep,
+                )
+            except Exception as exc:
+                fail += 1
+                if not args.quiet:
+                    print(f"  [{index}/{len(to_fetch)}] {sym} FAIL {exc}")
+                continue
+
+            if not bars:
+                fail += 1
+                if not args.quiet:
+                    print(f"  [{index}/{len(to_fetch)}] {sym} empty")
+                continue
+
+            ticker = existing.get(sym)
+            is_new = ticker is None
+            if ticker is None:
+                ticker = Ticker(symbol=sym, name=None, created_at=datetime.now(timezone.utc))
+                session.add(ticker)
+                session.flush()
+                existing[sym] = ticker
+
+            # Upsert bars (delete+insert range for simplicity on research path)
+            session.execute(
+                text(
+                    "DELETE FROM ohlcv_records WHERE ticker_id = :tid "
+                    "AND date >= :start AND date <= :end"
+                ),
+                {"tid": ticker.id, "start": start.isoformat(), "end": end.isoformat()},
+            )
+            now = datetime.utcnow()
+            session.bulk_insert_mappings(
+                OHLCVRecord,
+                [
+                    {
+                        "ticker_id": ticker.id,
+                        "date": b.date,
+                        "open": b.open,
+                        "high": b.high,
+                        "low": b.low,
+                        "close": b.close,
+                        "volume": b.volume,
+                        "created_at": now,
+                    }
+                    for b in bars
+                ],
+            )
+
+            # rank_only only for names that were NOT in the original production
+            # snapshot at copy time (or are newly introduced to this research DB).
+            if is_new or sym not in prod_symbols:
+                # Re-evaluate: if source copy already had the symbol, don't flag.
+                # Only new inserts get rank_only.
+                if is_new:
+                    session.execute(
+                        text(
+                            "INSERT OR REPLACE INTO research_rank_only "
+                            "(ticker_id, symbol) VALUES (:tid, :sym)"
+                        ),
+                        {"tid": ticker.id, "sym": sym},
+                    )
+
+            session.commit()
+            ok += 1
+            if not args.quiet and (index % 25 == 0 or index == len(to_fetch)):
+                elapsed = time.monotonic() - t0
+                print(
+                    f"  progress {index}/{len(to_fetch)} ok={ok} fail={fail} "
+                    f"elapsed={elapsed/60:.1f}m last={sym} bars={len(bars)}"
+                )
+
+        rank_only_n = session.execute(
+            text("SELECT COUNT(*) FROM research_rank_only")
+        ).scalar_one()
+        ticker_n = session.execute(text("SELECT COUNT(*) FROM tickers")).scalar_one()
+        ohlcv_n = session.execute(text("SELECT COUNT(*) FROM ohlcv_records")).scalar_one()
+
+    print("Done.")
+    print(f"  output: {output}")
+    print(f"  tickers: {ticker_n}")
+    print(f"  ohlcv rows: {ohlcv_n}")
+    print(f"  research_rank_only: {rank_only_n}")
+    print(f"  fetched ok/fail: {ok}/{fail}")
+
+
+if __name__ == "__main__":
+    asyncio.run(_main())