From 9704e0d85a1d1803641f84bb5d8a971bedd58c0c Mon Sep 17 00:00:00 2001 From: Dennis Thiessen Date: Sat, 18 Jul 2026 20:22:11 +0200 Subject: [PATCH] feat: Phase B fip_id liquid-breadth research tooling Add research-only snapshot extender, PIT dollar-volume mask for signal IC, rank-only harness path, fingerprint+breadth runner, and docs. Fingerprint reproduced IC -0.045 / t -2.91 on prod.sqlite. No production gate/schedule changes. --- app/services/backtest_service.py | 320 +++++++--- docs/research/README.md | 4 +- docs/research/fip-breadth-ic.md | 54 ++ ...p-breadth-20260718-194828-fingerprint.json | 578 ++++++++++++++++++ reports/fip-breadth-20260718-194828.json | 21 + scripts/extend_snapshot_universe.py | 329 ++++++++++ scripts/run_fip_breadth_research.py | 299 +++++++++ tests/unit/test_backtest_service.py | 46 ++ 8 files changed, 1578 insertions(+), 73 deletions(-) create mode 100644 docs/research/fip-breadth-ic.md create mode 100644 reports/fip-breadth-20260718-194828-fingerprint.json create mode 100644 reports/fip-breadth-20260718-194828.json create mode 100644 scripts/extend_snapshot_universe.py create mode 100644 scripts/run_fip_breadth_research.py diff --git a/app/services/backtest_service.py b/app/services/backtest_service.py index 01bf89e..a9503eb 100644 --- a/app/services/backtest_service.py +++ b/app/services/backtest_service.py @@ -30,6 +30,11 @@ Environment variables (see also run_backtest_snapshot.py): BACKTEST_FALLBACK_CLEAR_AIR_ONLY=1 BACKTEST_RESEARCH_EXITS=1 BACKTEST_MIN_RR_SWEEP=1 + + Broad-universe signal research (local snapshots only; inert when unset): + BACKTEST_LIQUID_BREADTH=1500 # PIT top-N by 63d median $vol, price floor + BACKTEST_LIQUID_MIN_PRICE=5 # USD close floor at as-of (default 5) + BACKTEST_SIGNAL_EVAL_ONLY=1 # skip portfolio_sim / monitor (signal IC only) """ from __future__ import annotations @@ -876,6 +881,63 @@ def _signal_values( return out +def _liquid_breadth_top_n() -> int: + """0 = off (production path). N > 0 enables PIT top-N $vol mask for signal IC.""" + raw = os.getenv("BACKTEST_LIQUID_BREADTH", "").strip() + if not raw: + return 0 + try: + return max(0, int(raw)) + except ValueError: + return 0 + + +def _liquid_min_price() -> float: + raw = os.getenv("BACKTEST_LIQUID_MIN_PRICE", "5").strip() or "5" + try: + return max(0.0, float(raw)) + except ValueError: + return 5.0 + + +def _signal_eval_only() -> bool: + return os.getenv("BACKTEST_SIGNAL_EVAL_ONLY", "").strip() in ("1", "true", "yes") + + +async def _load_research_rank_only_symbols(db: AsyncSession) -> set[str]: + """Symbols that feed signal IC only (no GTL/candidate replay). + + Optional side table ``research_rank_only`` on research snapshots. Missing + table → empty set (production path unchanged). + """ + from sqlalchemy import text + + try: + result = await db.execute(text("SELECT symbol FROM research_rank_only")) + return {str(row[0]).upper() for row in result.fetchall() if row[0]} + except Exception: + return set() + + +def _median_dollar_vol_63( + closes: list[float], volumes: list[float], i: int, lookback: int = 63 +) -> float | None: + """Rolling median of close×volume over ``lookback`` bars ending at ``i`` (inclusive).""" + if i + 1 < lookback or lookback < 2: + return None + dvs: list[float] = [] + for k in range(i - lookback + 1, i + 1): + if closes[k] > 0 and volumes[k] >= 0: + dvs.append(closes[k] * float(volumes[k])) + if len(dvs) < max(20, lookback // 2): + return None + dvs_sorted = sorted(dvs) + mid = len(dvs_sorted) // 2 + if len(dvs_sorted) % 2: + return dvs_sorted[mid] + return 0.5 * (dvs_sorted[mid - 1] + dvs_sorted[mid]) + + def _accumulate_signal_series( records: list, collected: dict, @@ -883,13 +945,20 @@ def _accumulate_signal_series( ) -> None: """For each weekly as-of bar, emit (signal, forward-return) pairs keyed by ISO week into ``collected[name][week_key]``. Forward return is close-to-close over - HORIZON trading days. Mutates ``collected`` (a dict of dict of list).""" + HORIZON trading days. Mutates ``collected`` (a dict of dict of list). + + When ``BACKTEST_LIQUID_BREADTH`` is set, observations are dicts with PIT + liquidity fields for the mask; otherwise plain ``(val, fwd)`` tuples so the + production signal path stays unchanged. + """ n = len(records) if n < HORIZON + 21: return closes = [float(r.close) for r in records] highs = [float(r.high) for r in records] + volumes = [float(getattr(r, "volume", 0) or 0) for r in records] dates = [r.date for r in records] + liquid_mode = _liquid_breadth_top_n() > 0 for i in _weekly_asof_indices(records): j = i + HORIZON if j >= n or closes[i] <= 0: @@ -897,8 +966,17 @@ def _accumulate_signal_series( fwd = closes[j] / closes[i] - 1.0 iso = records[i].date.isocalendar() week_key = (iso[0], iso[1]) + dvol = _median_dollar_vol_63(closes, volumes, i) if liquid_mode else None for name, val in _signal_values(dates, closes, highs, i, benchmark_closes).items(): - collected[name][week_key].append((val, fwd)) + if liquid_mode: + collected[name][week_key].append({ + "val": val, + "fwd": fwd, + "close": closes[i], + "median_dvol_63": dvol, + }) + else: + collected[name][week_key].append((val, fwd)) def _rank(xs: list[float]) -> list[float]: @@ -937,6 +1015,54 @@ def _spearman(xs: list[float], ys: list[float]) -> float | None: return _pearson(_rank(xs), _rank(ys)) +def _obs_val_fwd(rec: object) -> tuple[float, float] | None: + """Unpack a signal observation: ``(val, fwd)`` or research dict form.""" + if isinstance(rec, dict): + try: + return float(rec["val"]), float(rec["fwd"]) + except (KeyError, TypeError, ValueError): + return None + if isinstance(rec, (tuple, list)) and len(rec) >= 2: + try: + return float(rec[0]), float(rec[1]) + except (TypeError, ValueError): + return None + return None + + +def _filter_liquid_breadth_week( + recs: list, + *, + top_n: int, + min_price: float, +) -> list[tuple[float, float]]: + """Point-in-time top-N by median $vol among names with price ≥ floor. + + Ranking is relative (IEX volume undercount is OK for order stats). Membership + is recomputed every week from as-of bars — never frozen from today's liquidity. + """ + ranked: list[tuple[float, float, float]] = [] # (-dvol, val, fwd) + for rec in recs: + if not isinstance(rec, dict): + pair = _obs_val_fwd(rec) + if pair is not None: + ranked.append((0.0, pair[0], pair[1])) + continue + close = rec.get("close") + dvol = rec.get("median_dvol_63") + if close is None or float(close) < min_price: + continue + if dvol is None or float(dvol) <= 0: + continue + pair = _obs_val_fwd(rec) + if pair is None: + continue + ranked.append((-float(dvol), pair[0], pair[1])) + ranked.sort(key=lambda row: row[0]) + kept = ranked[:top_n] + return [(val, fwd) for _, val, fwd in kept] + + def _quintile_spread(pairs: list[tuple[float, float]]) -> float | None: """Mean forward return of the top signal-quintile minus the bottom quintile.""" n = len(pairs) @@ -982,10 +1108,16 @@ def _signal_evaluation(collected: dict) -> list[dict]: IC is measured on NON-OVERLAPPING forward windows (weeks thinned to ~HORIZON apart) so the t-stat isn't inflated by autocorrelation. A signal with no edge - lands near IC 0 / spread 0; one with too few independent windows is flagged + lands near IC 0 / score 0; one with too few independent windows is flagged unreliable rather than trusted on a lucky handful. + + When ``BACKTEST_LIQUID_BREADTH=N`` is set, each week's cross-section is first + restricted to the top-N names by point-in-time 63d median dollar volume + (price ≥ BACKTEST_LIQUID_MIN_PRICE). Production path (flag unset) is unchanged. """ stride = max(1, round(HORIZON / 5)) # ISO weeks spanned by the forward window + top_n = _liquid_breadth_top_n() + min_price = _liquid_min_price() rows: list[dict] = [] for name in sorted(collected): weeks_map = collected[name] @@ -996,13 +1128,25 @@ def _signal_evaluation(collected: dict) -> list[dict]: sizes: list[int] = [] for wk in kept: recs = weeks_map[wk] - ic = _spearman([r[0] for r in recs], [r[1] for r in recs]) + if top_n > 0: + pairs = _filter_liquid_breadth_week( + recs, top_n=top_n, min_price=min_price + ) + else: + pairs = [] + for rec in recs: + pair = _obs_val_fwd(rec) + if pair is not None: + pairs.append(pair) + if len(pairs) < MIN_CROSS_SECTION: + continue + ic = _spearman([p[0] for p in pairs], [p[1] for p in pairs]) if ic is not None: ics.append(ic) - spread = _quintile_spread(recs) + spread = _quintile_spread(pairs) if spread is not None: spreads.append(spread) - sizes.append(len(recs)) + sizes.append(len(pairs)) if not ics: continue mean_ic = sum(ics) / len(ics) @@ -1011,7 +1155,7 @@ def _signal_evaluation(collected: dict) -> list[dict]: else: std = 0.0 t_stat = mean_ic / std * math.sqrt(len(ics)) if std > 0 else None - rows.append({ + row = { "signal": name, "weeks": len(ics), "avg_cross_section": round(sum(sizes) / len(sizes), 1) if sizes else None, @@ -1020,7 +1164,11 @@ def _signal_evaluation(collected: dict) -> list[dict]: "ic_positive_pct": round(sum(1 for x in ics if x > 0) / len(ics) * 100, 1), "mean_quintile_spread": round(sum(spreads) / len(spreads), 4) if spreads else None, "reliable": len(ics) >= MIN_RELIABLE_PERIODS, - }) + } + if top_n > 0: + row["liquid_breadth_top_n"] = top_n + row["liquid_min_price"] = min_price + rows.append(row) rows.sort(key=lambda r: r["mean_ic"], reverse=True) return rows @@ -1041,10 +1189,15 @@ def _replay_and_signals( benchmark_closes: dict[date, float] | None = None, target_model: str = PRODUCTION_GTL_TARGET_MODEL, cadence: str = DEFAULT_BACKTEST_CADENCE, + signal_only: bool = False, ) -> tuple[list[dict], dict]: """The CPU-bound per-ticker work, as a top-level (picklable) function so it can run in a worker process. Takes primitive column arrays (cheap to pickle), - rebuilds bar objects, and returns (candidates, signal_series).""" + rebuilds bar objects, and returns (candidates, signal_series). + + ``signal_only=True`` (research rank-only names): skip GTL/candidate replay so + the production portfolio book is never polluted by broad-universe tickers. + """ date_ords, opens, highs, lows, closes, volumes = columns bars = [ SimpleNamespace( @@ -1052,8 +1205,9 @@ def _replay_and_signals( ) for o, op, hi, lo, cl, vo in zip(date_ords, opens, highs, lows, closes, volumes) ] - return ( - _replay_ticker( + candidates: list[dict] = [] + if not signal_only: + candidates = _replay_ticker( symbol, bars, config, @@ -1061,7 +1215,9 @@ def _replay_and_signals( benchmark_closes, target_model, cadence, - ), + ) + return ( + candidates, _signal_series(bars, benchmark_closes), ) @@ -3789,6 +3945,12 @@ async def run_backtest( result = await db.execute(select(Ticker).order_by(Ticker.symbol)) tickers = list(result.scalars().all()) total = len(tickers) + rank_only_symbols = await _load_research_rank_only_symbols(db) + if rank_only_symbols: + logger.info(json.dumps({ + "event": "backtest_rank_only_loaded", + "count": len(rank_only_symbols), + })) candidates: list[dict] = [] # Signal IC remains a weekly, non-overlapping diagnostic regardless of the @@ -3847,10 +4009,16 @@ async def run_backtest( continue if columns is not None: futures.append(loop.run_in_executor( - pool, _replay_and_signals, ticker.symbol, columns, config, activation, + pool, + _replay_and_signals, + ticker.symbol, + columns, + config, + activation, benchmark_closes, target_model, cadence, + ticker.symbol in rank_only_symbols, )) for result in await asyncio.gather(*futures, return_exceptions=True): if isinstance(result, Exception): @@ -3870,10 +4038,15 @@ async def run_backtest( columns = await _fetch_columns(db, ticker.symbol) if columns is not None: _merge(await asyncio.to_thread( - _replay_and_signals, ticker.symbol, columns, config, activation, + _replay_and_signals, + ticker.symbol, + columns, + config, + activation, benchmark_closes, target_model, cadence, + ticker.symbol in rank_only_symbols, )) except Exception: logger.exception("Backtest replay failed for %s", ticker.symbol) @@ -3916,73 +4089,75 @@ async def run_backtest( portfolio_monitor_report: dict | None = None holdout_report: dict | None = None min_rr_sweep_report: dict | None = None - try: - qual_symbols = sorted({ - c["symbol"] - for c in candidates - if c.get("qualified") - or any(_qualifies_strategy_variant(c, cfg) for cfg in STRATEGY_VARIANTS) - }) - price_columns: dict[str, tuple] = {} - for sym in qual_symbols: - cols = await _fetch_columns(db, sym) - if cols is not None: - price_columns[sym] = cols - - spy_closes: dict | None = None + if not _signal_eval_only(): try: - oldest = min((cols[0][0] for cols in price_columns.values()), default=None) - days_needed = None - if oldest is not None and not _offline_snapshot_mode(): - days_needed = (date.today() - date.fromordinal(oldest)).days + 30 - spy_closes = await _load_benchmark_closes_for_backtest( - db, days=days_needed, refresh=oldest is not None - ) - except Exception: - logger.exception("Benchmark load for the portfolio sim failed") + qual_symbols = sorted({ + c["symbol"] + for c in candidates + if c.get("qualified") + or any(_qualifies_strategy_variant(c, cfg) for cfg in STRATEGY_VARIANTS) + }) + price_columns: dict[str, tuple] = {} + for sym in qual_symbols: + cols = await _fetch_columns(db, sym) + if cols is not None: + price_columns[sym] = cols - for policy in ("target", "hold"): - sim = _simulate_portfolio( - candidates, price_columns, spy_closes, policy, hold_horizon - ) - if sim is not None: - sim_policies.append({"policy": policy, **sim}) - strategy_variant_rows = _strategy_variant_sims( - candidates, price_columns, spy_closes, hold_horizon - ) - exit_policy_rows = _exit_policy_sims( - candidates, price_columns, spy_closes, hold_horizon - ) - live_exit_policy: dict | None = None - try: - from app.services.paper_trade_service import get_exit_policy + spy_closes: dict | None = None + try: + oldest = min((cols[0][0] for cols in price_columns.values()), default=None) + days_needed = None + if oldest is not None and not _offline_snapshot_mode(): + days_needed = (date.today() - date.fromordinal(oldest)).days + 30 + spy_closes = await _load_benchmark_closes_for_backtest( + db, days=days_needed, refresh=oldest is not None + ) + except Exception: + logger.exception("Benchmark load for the portfolio sim failed") - live_exit_policy = await get_exit_policy(db) - except Exception: - logger.exception("Live exit policy load failed; monitor uses defaults") - portfolio_monitor_report = _portfolio_monitor( - candidates, price_columns, spy_closes, hold_horizon, - live_exit_policy=live_exit_policy, - cadence=cadence, - ) - split = _holdout_split() - if split is not None: - holdout_report = _holdout_evaluation( - candidates, price_columns, spy_closes, hold_horizon, split, + for policy in ("target", "hold"): + sim = _simulate_portfolio( + candidates, price_columns, spy_closes, policy, hold_horizon + ) + if sim is not None: + sim_policies.append({"policy": policy, **sim}) + strategy_variant_rows = _strategy_variant_sims( + candidates, price_columns, spy_closes, hold_horizon + ) + exit_policy_rows = _exit_policy_sims( + candidates, price_columns, spy_closes, hold_horizon + ) + live_exit_policy: dict | None = None + try: + from app.services.paper_trade_service import get_exit_policy + + live_exit_policy = await get_exit_policy(db) + except Exception: + logger.exception("Live exit policy load failed; monitor uses defaults") + portfolio_monitor_report = _portfolio_monitor( + candidates, price_columns, spy_closes, hold_horizon, live_exit_policy=live_exit_policy, cadence=cadence, ) - if _min_rr_sweep_enabled(): - min_rr_sweep_report = _min_rr_sweep( - candidates, price_columns, spy_closes, activation, current_min_pct, - hold_horizon, live_exit_policy=live_exit_policy, cadence=cadence, - ) - except Exception: - logger.exception("Portfolio simulation failed") + split = _holdout_split() + if split is not None: + holdout_report = _holdout_evaluation( + candidates, price_columns, spy_closes, hold_horizon, split, + live_exit_policy=live_exit_policy, + cadence=cadence, + ) + if _min_rr_sweep_enabled(): + min_rr_sweep_report = _min_rr_sweep( + candidates, price_columns, spy_closes, activation, current_min_pct, + hold_horizon, live_exit_policy=live_exit_policy, cadence=cadence, + ) + except Exception: + logger.exception("Portfolio simulation failed") report = { "generated_at": datetime.now(timezone.utc).isoformat(), "tickers": total, + "rank_only_tickers": len(rank_only_symbols), "candidates": len(candidates), "qualified": len(qualified), "params": { @@ -3999,6 +4174,9 @@ async def run_backtest( "target_model_label": BACKTEST_TARGET_MODELS[target_model], "is_production_target_model": target_model == PRODUCTION_GTL_TARGET_MODEL, "production_reentry_policy": PRODUCTION_REENTRY_POLICY, + "liquid_breadth_top_n": _liquid_breadth_top_n() or None, + "liquid_min_price": _liquid_min_price() if _liquid_breadth_top_n() else None, + "signal_eval_only": _signal_eval_only(), }, "activation": activation, "overall_qualified": _bucket_stats(qualified), diff --git a/docs/research/README.md b/docs/research/README.md index 396decb..7be97f3 100644 --- a/docs/research/README.md +++ b/docs/research/README.md @@ -141,8 +141,8 @@ knobs. | Lead | Why it's interesting | Blocker | |---|---|---| | **Near-close / MOC execution (ops)** | Recovers overnight momentum drift left on the table by a morning EU scan; evidence closed | Implement schedule + partial-bar scan path; one qualifying scan/day only | -| **`fip_id`** (information discreteness over the 12-1 window) | **Strongest cross-sectional signal measured on this universe** — IC −0.045, t = −2.91, correct sign; re-derived fingerprint matched Phase A | Doesn't improve *this* book. Revisit when the universe broadens — **after** execution path is decided | -| **Broader universe** (`nasdaq_all`) | Strengthens every week's cross-section and the IC t-stat | Grade under the fill mode you will trade | +| **`fip_id`** (information discreteness over the 12-1 window) | **Strongest cross-sectional signal measured on this universe** — IC −0.045, t = −2.91, correct sign; re-derived fingerprint matched Phase A; ticker technicals show it display-only | Doesn't improve *this* book as a filter. **Phase B tooling ready:** liquid-breadth IC on research.sqlite — see [fip-breadth-ic.md](fip-breadth-ic.md) | +| **Broader universe** (`nasdaq_all` / liquid top-N) | Strengthens cross-sections; where `fip_id` may become tradeable | Offline research only first (`extend_snapshot_universe.py`); not prod scan | | **Forward paper-trade record** | The only true out-of-sample evidence the snapshot cannot give | Time; mark entries at actual near-close fill once ops ships | | **Better target model for clear-air names** | The return is demonstrably there (#2 wins on raw CAGR in *both* train and test); it's the *flat* 3× ATR target that makes it too expensive in risk | Needs a per-name model, not a constant k×ATR | diff --git a/docs/research/fip-breadth-ic.md b/docs/research/fip-breadth-ic.md new file mode 100644 index 0000000..d02e879 --- /dev/null +++ b/docs/research/fip-breadth-ic.md @@ -0,0 +1,54 @@ +# Broad-universe fip_id IC research (Phase B) + +Generated: 2026-07-18T19:48:28.127710 + +## Scope + +- **Research only** — production universe, gate, scanner, schedule unchanged. +- Price-only signal harness; no sentiment/fundamentals on the broad tier. +- Point-in-time liquidity mask: top **1500** by 63d median $vol, price ≥ **$5.0** at as-of. + +## Caveats + +- **Survivorship bias**: today's constituents backfilled historically (worse in small caps). +- **IEX volume undercount**: relative $vol rank only, not absolute floors. +- **Pool skew**: nasdaq_all ∪ sp500 tilts tech/biotech; missing pure NYSE mid-caps. + +## Fingerprint (505-name prod snapshot) + +- Expected: IC ≈ -0.045, t ≈ -2.9 +- Observed: IC = -0.045, t = -2.91, weeks = 35, reliable = True +- Pass: **True** + +## Liquid-breadth signal_eval (fip_id) + +**Not run yet in this environment** (no Alpaca credentials to build +`backtest_snapshots/research.sqlite`). Local machine with keys: + +```bash +python scripts/extend_snapshot_universe.py --force-copy +python scripts/run_fip_breadth_research.py --skip-fingerprint --allow-spawn --workers 6 +``` + +(`--skip-fingerprint` only after a green fingerprint on this machine.) + +| metric | value | +|---|---| +| mean_ic | _pending_ | +| ic_t_stat | _pending_ | +| weeks | _pending_ | +| avg_cross_section | _pending_ | +| reliable | _pending_ | + +## Verdict (iron rule) + +- **Fingerprint:** green (IC −0.045 / t −2.91 / reliable / 35 weeks). +- **Breadth iron rule:** **pending** until research.sqlite run completes. +- Green breadth would authorize a **follow-up proposal** only (two-tier universe / + gate revalidation) — **not** production wire-in. + +## Artifacts + +- Fingerprint report: `reports/fip-breadth-20260718-194828-fingerprint.json` + (and summary `reports/fip-breadth-20260718-194828.json`) +- Breadth report: _pending_ diff --git a/reports/fip-breadth-20260718-194828-fingerprint.json b/reports/fip-breadth-20260718-194828-fingerprint.json new file mode 100644 index 0000000..aeae3d0 --- /dev/null +++ b/reports/fip-breadth-20260718-194828-fingerprint.json @@ -0,0 +1,578 @@ +{ + "generated_at": "2026-07-18T18:21:28.359793+00:00", + "tickers": 506, + "rank_only_tickers": 0, + "candidates": 202765, + "qualified": 1086, + "params": { + "step_days": 5, + "step_sessions": 5, + "entry_cadence": "weekly", + "signal_eval_cadence": "weekly", + "horizon_days": 30, + "min_lookback": 60, + "cost_per_side_pct": 0.1, + "target_model": "production_gtl", + "target_model_label": "Live GTL (production)", + "is_production_target_model": true, + "production_reentry_policy": "gate_reset", + "liquid_breadth_top_n": null, + "liquid_min_price": null, + "signal_eval_only": true + }, + "activation": { + "min_momentum_percentile": 80.0, + "min_rr": 2.0, + "min_confidence": 0.0, + "require_high_conviction": false, + "exclude_conflicts": false, + "exclude_neutral": true + }, + "overall_qualified": { + "total": 1086, + "wins": 379, + "losses": 591, + "expired": 116, + "hit_rate": 39.1, + "avg_r": 0.255, + "total_r": 276.76, + "net_avg_r": 0.209, + "net_total_r": 226.56, + "best_r": 8.85, + "worst_r": -3.38, + "avg_hold_days": 12.0, + "net_r_per_day": 0.0174, + "median_net_r": -1.031, + "profit_factor": 1.34, + "net_avg_r_ex_top5": 0.049 + }, + "overall_all": { + "total": 202765, + "wins": 82220, + "losses": 113809, + "expired": 6736, + "hit_rate": 41.9, + "avg_r": -0.04, + "total_r": -8186.18, + "net_avg_r": -0.095, + "net_total_r": -19184.55, + "best_r": 9.24, + "worst_r": -16.42, + "avg_hold_days": 8.2, + "net_r_per_day": -0.0115, + "median_net_r": -1.035, + "profit_factor": 0.85, + "net_avg_r_ex_top5": -0.22 + }, + "by_direction": { + "long": { + "total": 1086, + "wins": 379, + "losses": 591, + "expired": 116, + "hit_rate": 39.1, + "avg_r": 0.255, + "total_r": 276.76, + "net_avg_r": 0.209, + "net_total_r": 226.56, + "best_r": 8.85, + "worst_r": -3.38, + "avg_hold_days": 12.0, + "net_r_per_day": 0.0174, + "median_net_r": -1.031, + "profit_factor": 1.34, + "net_avg_r_ex_top5": 0.049 + }, + "short": { + "total": 0, + "wins": 0, + "losses": 0, + "expired": 0, + "hit_rate": null, + "avg_r": null, + "total_r": null, + "net_avg_r": null, + "net_total_r": null, + "best_r": null, + "worst_r": null, + "avg_hold_days": null, + "net_r_per_day": null, + "median_net_r": null, + "profit_factor": null, + "net_avg_r_ex_top5": null + } + }, + "min_momentum_percentile": 80.0, + "sweep": [ + { + "min_momentum_percentile": 90.0, + "total": 497, + "wins": 177, + "losses": 269, + "expired": 51, + "hit_rate": 39.7, + "avg_r": 0.276, + "total_r": 137.05, + "net_avg_r": 0.235, + "net_total_r": 116.55, + "best_r": 8.85, + "worst_r": -3.38, + "avg_hold_days": 11.8, + "net_r_per_day": 0.0199, + "median_net_r": -1.026, + "profit_factor": 1.39, + "net_avg_r_ex_top5": 0.071 + }, + { + "min_momentum_percentile": 80.0, + "total": 1086, + "wins": 379, + "losses": 591, + "expired": 116, + "hit_rate": 39.1, + "avg_r": 0.255, + "total_r": 276.76, + "net_avg_r": 0.209, + "net_total_r": 226.56, + "best_r": 8.85, + "worst_r": -3.38, + "avg_hold_days": 12.0, + "net_r_per_day": 0.0174, + "median_net_r": -1.031, + "profit_factor": 1.34, + "net_avg_r_ex_top5": 0.049 + }, + { + "min_momentum_percentile": 70.0, + "total": 1841, + "wins": 597, + "losses": 1062, + "expired": 182, + "hit_rate": 36.0, + "avg_r": 0.152, + "total_r": 280.26, + "net_avg_r": 0.104, + "net_total_r": 190.75, + "best_r": 8.85, + "worst_r": -4.21, + "avg_hold_days": 11.8, + "net_r_per_day": 0.0088, + "median_net_r": -1.037, + "profit_factor": 1.16, + "net_avg_r_ex_top5": -0.055 + }, + { + "min_momentum_percentile": 60.0, + "total": 2772, + "wins": 873, + "losses": 1611, + "expired": 288, + "hit_rate": 35.1, + "avg_r": 0.126, + "total_r": 348.07, + "net_avg_r": 0.075, + "net_total_r": 209.25, + "best_r": 8.85, + "worst_r": -4.21, + "avg_hold_days": 12.0, + "net_r_per_day": 0.0063, + "median_net_r": -1.04, + "profit_factor": 1.12, + "net_avg_r_ex_top5": -0.077 + }, + { + "min_momentum_percentile": 50.0, + "total": 3901, + "wins": 1182, + "losses": 2295, + "expired": 424, + "hit_rate": 34.0, + "avg_r": 0.089, + "total_r": 345.37, + "net_avg_r": 0.038, + "net_total_r": 146.96, + "best_r": 8.85, + "worst_r": -4.86, + "avg_hold_days": 12.1, + "net_r_per_day": 0.0031, + "median_net_r": -1.042, + "profit_factor": 1.06, + "net_avg_r_ex_top5": -0.114 + }, + { + "min_momentum_percentile": 0.0, + "total": 14588, + "wins": 3719, + "losses": 9271, + "expired": 1598, + "hit_rate": 28.6, + "avg_r": -0.065, + "total_r": -952.08, + "net_avg_r": -0.115, + "net_total_r": -1676.86, + "best_r": 9.24, + "worst_r": -15.94, + "avg_hold_days": 12.1, + "net_r_per_day": -0.0095, + "median_net_r": -1.043, + "profit_factor": 0.84, + "net_avg_r_ex_top5": -0.275 + } + ], + "gate_ablation": [ + { + "variant": "all_floors", + "total": 1086, + "wins": 379, + "losses": 591, + "expired": 116, + "hit_rate": 39.1, + "avg_r": 0.255, + "total_r": 276.76, + "net_avg_r": 0.209, + "net_total_r": 226.56, + "best_r": 8.85, + "worst_r": -3.38, + "avg_hold_days": 12.0, + "net_r_per_day": 0.0174, + "median_net_r": -1.031, + "profit_factor": 1.34, + "net_avg_r_ex_top5": 0.049, + "hold_days": 30, + "hold_avg_r": 0.631, + "hold_net_avg_r": 0.585, + "hold_total_r": 684.97 + }, + { + "variant": "no_confidence_floor", + "total": 1093, + "wins": 380, + "losses": 596, + "expired": 117, + "hit_rate": 38.9, + "avg_r": 0.25, + "total_r": 273.55, + "net_avg_r": 0.204, + "net_total_r": 222.99, + "best_r": 8.85, + "worst_r": -3.38, + "avg_hold_days": 11.9, + "net_r_per_day": 0.0171, + "median_net_r": -1.031, + "profit_factor": 1.33, + "net_avg_r_ex_top5": 0.045, + "hold_days": 30, + "hold_avg_r": 0.626, + "hold_net_avg_r": 0.58, + "hold_total_r": 684.08 + }, + { + "variant": "no_rr_floor", + "total": 6849, + "wins": 3235, + "losses": 3409, + "expired": 205, + "hit_rate": 48.7, + "avg_r": 0.112, + "total_r": 770.17, + "net_avg_r": 0.061, + "net_total_r": 418.96, + "best_r": 8.85, + "worst_r": -5.16, + "avg_hold_days": 7.9, + "net_r_per_day": 0.0078, + "median_net_r": -0.061, + "profit_factor": 1.11, + "net_avg_r_ex_top5": -0.061, + "hold_days": 30, + "hold_avg_r": 0.354, + "hold_net_avg_r": 0.303, + "hold_total_r": 2425.86 + }, + { + "variant": "no_neutral_exclusion", + "total": 2313, + "wins": 770, + "losses": 1279, + "expired": 264, + "hit_rate": 37.6, + "avg_r": 0.2, + "total_r": 462.35, + "net_avg_r": 0.154, + "net_total_r": 355.89, + "best_r": 8.85, + "worst_r": -3.38, + "avg_hold_days": 12.6, + "net_r_per_day": 0.0122, + "median_net_r": -1.031, + "profit_factor": 1.25, + "net_avg_r_ex_top5": 0.004, + "hold_days": 30, + "hold_avg_r": 0.583, + "hold_net_avg_r": 0.537, + "hold_total_r": 1348.89 + }, + { + "variant": "momentum_only", + "total": 14696, + "wins": 6827, + "losses": 7359, + "expired": 510, + "hit_rate": 48.1, + "avg_r": 0.114, + "total_r": 1669.64, + "net_avg_r": 0.064, + "net_total_r": 936.93, + "best_r": 8.85, + "worst_r": -5.71, + "avg_hold_days": 8.4, + "net_r_per_day": 0.0076, + "median_net_r": -1.013, + "profit_factor": 1.11, + "net_avg_r_ex_top5": -0.055, + "hold_days": 30, + "hold_avg_r": 0.395, + "hold_net_avg_r": 0.345, + "hold_total_r": 5798.82 + } + ], + "gate_ablation_note": "Each row re-qualifies the same candidates at the current momentum cutoff (80) with one floor removed (long-only while the momentum gate is active). If dropping a floor doesn't hurt net expectancy, that floor isn't pulling its weight. The Hold columns grade the same variants under the hold-to-horizon time exit instead of the S/R target \u2014 the view that matters if the exit policy moves to a fixed hold.", + "time_exit_sweep": [ + { + "hold_days": 5, + "total": 1086, + "wins": 603, + "win_rate": 55.5, + "avg_r": 0.175, + "total_r": 190.16, + "net_avg_r": 0.129, + "net_total_r": 139.97, + "best_r": 5.09, + "worst_r": -2.51, + "avg_hold_days": 4.5, + "net_r_per_day": 0.0285, + "median_net_r": 0.115, + "profit_factor": 1.36, + "net_avg_r_ex_top5": -0.002 + }, + { + "hold_days": 10, + "total": 1086, + "wins": 559, + "win_rate": 51.5, + "avg_r": 0.357, + "total_r": 387.9, + "net_avg_r": 0.311, + "net_total_r": 337.7, + "best_r": 6.73, + "worst_r": -2.51, + "avg_hold_days": 7.9, + "net_r_per_day": 0.0395, + "median_net_r": 0.031, + "profit_factor": 1.67, + "net_avg_r_ex_top5": 0.112 + }, + { + "hold_days": 21, + "total": 1086, + "wins": 487, + "win_rate": 44.8, + "avg_r": 0.525, + "total_r": 570.33, + "net_avg_r": 0.479, + "net_total_r": 520.14, + "best_r": 9.86, + "worst_r": -3.38, + "avg_hold_days": 13.7, + "net_r_per_day": 0.0349, + "median_net_r": -1.027, + "profit_factor": 1.81, + "net_avg_r_ex_top5": 0.191 + }, + { + "hold_days": 30, + "total": 1086, + "wins": 434, + "win_rate": 40.0, + "avg_r": 0.631, + "total_r": 684.97, + "net_avg_r": 0.585, + "net_total_r": 634.78, + "best_r": 12.87, + "worst_r": -3.38, + "avg_hold_days": 17.8, + "net_r_per_day": 0.0329, + "median_net_r": -1.033, + "profit_factor": 1.9, + "net_avg_r_ex_top5": 0.212 + } + ], + "portfolio_sim": { + "params": { + "starting_capital": 10000.0, + "max_positions": 10, + "risk_per_trade_pct": 1.0, + "notional_cap_pct": 20.0, + "cost_per_side_pct": 0.1, + "hold_days": 30 + }, + "policies": [], + "note": "One capital-constrained book over the same qualified setups the tables above grade per-setup: at most 10 concurrent positions (one per ticker), best momentum first, fixed-fractional risk sizing with a no-leverage cap, entries at the detection close, stops filled at the worse of stop or open. 'target' races the S/R target against the stop (timeout at the horizon); 'hold' keeps the initial stop and exits at the horizon close. SPY return is price-only over the same window. In-sample; no dividends." + }, + "strategy_variants": { + "variants": [], + "note": "Research-only hold-to-horizon portfolio variants. Production now uses residual 12-1 momentum at cutoff 80; the remaining rows compare the legacy raw rank, raw cutoff 90, one max-15 capacity check, and volatility overlays." + }, + "exit_policy_variants": { + "variants": [], + "note": "Research-only exit policies over the residual/high-vol 80/20 entry candidate. Every row uses the same entry qualification/ranking and changes only the exit discipline." + }, + "portfolio_monitor": null, + "production_cadence_comparison": null, + "holdout": null, + "min_rr_sweep": null, + "target_model_diagnostics": { + "target_model": "production_gtl", + "target_model_label": "Live GTL (production)", + "candidate_count": 202765, + "primary_source_counts": { + "pivot_point": 196290, + "range_grid": 180036 + }, + "primary_round_only": 0, + "primary_strength_100": 138596, + "avg_primary_strength": 80.109, + "avg_primary_distance_atr": 2.293, + "avg_primary_rejection_count": 41.908, + "avg_raw_level_count": 53.204, + "avg_gate_level_count": 53.204 + }, + "signal_eval": [ + { + "signal": "vol_6m", + "weeks": 39, + "avg_cross_section": 498.2, + "mean_ic": 0.0609, + "ic_t_stat": 1.48, + "ic_positive_pct": 64.1, + "mean_quintile_spread": 0.0337, + "reliable": true + }, + { + "signal": "mom_12_1_resid", + "weeks": 35, + "avg_cross_section": 497.7, + "mean_ic": 0.0552, + "ic_t_stat": 1.98, + "ic_positive_pct": 60.0, + "mean_quintile_spread": 0.0207, + "reliable": true + }, + { + "signal": "mom_12_1", + "weeks": 35, + "avg_cross_section": 497.7, + "mean_ic": 0.0531, + "ic_t_stat": 1.61, + "ic_positive_pct": 65.7, + "mean_quintile_spread": 0.0206, + "reliable": true + }, + { + "signal": "trend_200", + "weeks": 37, + "avg_cross_section": 497.9, + "mean_ic": 0.0161, + "ic_t_stat": 0.44, + "ic_positive_pct": 59.5, + "mean_quintile_spread": 0.006, + "reliable": true + }, + { + "signal": "reversal_1m", + "weeks": 43, + "avg_cross_section": 498.7, + "mean_ic": 0.0059, + "ic_t_stat": 0.22, + "ic_positive_pct": 53.5, + "mean_quintile_spread": 0.0053, + "reliable": true + }, + { + "signal": "mom_6_1", + "weeks": 39, + "avg_cross_section": 498.2, + "mean_ic": 0.0051, + "ic_t_stat": 0.21, + "ic_positive_pct": 56.4, + "mean_quintile_spread": 0.0087, + "reliable": true + }, + { + "signal": "mom_3_1", + "weeks": 42, + "avg_cross_section": 498.5, + "mean_ic": -0.0064, + "ic_t_stat": -0.25, + "ic_positive_pct": 50.0, + "mean_quintile_spread": 0.0046, + "reliable": true + }, + { + "signal": "high_52w", + "weeks": 35, + "avg_cross_section": 497.7, + "mean_ic": -0.0086, + "ic_t_stat": -0.26, + "ic_positive_pct": 54.3, + "mean_quintile_spread": -0.0088, + "reliable": true + }, + { + "signal": "fip_id", + "weeks": 35, + "avg_cross_section": 497.7, + "mean_ic": -0.045, + "ic_t_stat": -2.91, + "ic_positive_pct": 25.7, + "mean_quintile_spread": -0.0168, + "reliable": true + } + ], + "signal_eval_note": "Cross-sectional rank-IC of price-only signals vs the forward 30-day return (min 20 names/window). |IC| \u2273 0.03 with a consistent sign is a real (if small) edge; near 0 means ranking on it sorts nothing. Momentum factors and high_52w are expected positive; reversal_1m and vol_6m expected negative (mean-reversion / low-vol anomaly). IC is measured on non-overlapping windows; signals with fewer than 12 independent windows are flagged unreliable (too few regimes \u2014 deepen history with the Data Backfill job).", + "note": "Sentiment & fundamentals held neutral (no point-in-time history). Stops fill at the worse of the stop or the bar's open (gaps through the stop are modeled, so a loss can exceed \u22121R); targets never fill better than their level. ~6 months \u2248 one market regime \u2014 treat as directional, not gospel.", + "recommendation": { + "headline": "Trade the qualified list long-only; hold 30 trading days with the initial ATR stop.", + "items": [ + { + "topic": "exit", + "text": "Legacy exit diagnostic: hold 30 trading days with the initial stop (+0.58R net/trade vs +0.21R for the S/R target exit)." + }, + { + "topic": "gate", + "text": "Gate: the confidence floor adds nothing \u2014 dropping it costs +0.01R/trade and adds 7 trades." + }, + { + "topic": "gate", + "text": "Gate: keep the R:R floor (worth +0.28R/trade under the hold exit)." + }, + { + "topic": "gate", + "text": "Gate: keep the NEUTRAL exclusion (worth +0.05R/trade under the hold exit)." + }, + { + "topic": "cutoff", + "text": "Residual-momentum cutoff: 90 has the best per-trade net (+0.23R over 497 setups)." + }, + { + "topic": "robustness", + "text": "Robustness: expectancy survives removing the top 5% of winners (+0.21R net/trade under the recommended 30d hold) \u2014 the edge is not a handful of outliers." + } + ], + "note": "Derived from this report's numbers on every run \u2014 the advice flips if the data does." + }, + "research_recommendation": { + "items": [], + "note": "Strategy variants unavailable; re-run the backtest after benchmark data is present." + } +} \ No newline at end of file diff --git a/reports/fip-breadth-20260718-194828.json b/reports/fip-breadth-20260718-194828.json new file mode 100644 index 0000000..c6c1163 --- /dev/null +++ b/reports/fip-breadth-20260718-194828.json @@ -0,0 +1,21 @@ +{ + "generated_at": "2026-07-18T19:48:28.127710", + "liquid_breadth_top_n": 1500, + "liquid_min_price": 5.0, + "fingerprint": { + "signal": "fip_id", + "weeks": 35, + "avg_cross_section": 497.7, + "mean_ic": -0.045, + "ic_t_stat": -2.91, + "ic_positive_pct": 25.7, + "mean_quintile_spread": -0.0168, + "reliable": true, + "pass": true, + "expected_ic": -0.045, + "expected_t": -2.9 + }, + "breadth": null, + "verdict": null, + "fingerprint_report_path": "reports\\fip-breadth-20260718-194828-fingerprint.json" +} \ No newline at end of file diff --git a/scripts/extend_snapshot_universe.py b/scripts/extend_snapshot_universe.py new file mode 100644 index 0000000..e3585b8 --- /dev/null +++ b/scripts/extend_snapshot_universe.py @@ -0,0 +1,329 @@ +"""Extend a *copy* of the production backtest snapshot with broad-universe OHLCV. + +Research only — never writes to production Postgres. + +Pipeline +-------- +1. Copy ``--source`` snapshot (default ``backtest_snapshots/prod.sqlite``) to + ``--output`` (default ``backtest_snapshots/research.sqlite``). +2. Resolve symbol pool = nasdaq_all ∪ sp500 via ``ticker_universe_service``. +3. Fetch ~5y daily bars from Alpaca for symbols missing (or short) in the copy. +4. Insert new tickers + OHLCV; mark them in side table ``research_rank_only`` + so the harness can feed signal IC without GTL/candidate replay. + +Resume-friendly: re-running skips symbols that already have ≥ ``--min-bars``. + +Example +------- + python scripts/extend_snapshot_universe.py \\ + --source backtest_snapshots/prod.sqlite \\ + --output backtest_snapshots/research.sqlite \\ + --force-copy + + # smoke: first 50 missing symbols only + python scripts/extend_snapshot_universe.py --limit 50 +""" + +from __future__ import annotations + +import argparse +import asyncio +import shutil +import sys +import time +from datetime import date, datetime, timedelta, timezone +from pathlib import Path + +from sqlalchemy import create_engine, select, text +from sqlalchemy.orm import Session + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + + +def _parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument( + "--source", + default="backtest_snapshots/prod.sqlite", + help="Existing prod snapshot to copy (read-only after copy).", + ) + p.add_argument( + "--output", + default="backtest_snapshots/research.sqlite", + help="Research snapshot path (created/updated).", + ) + p.add_argument( + "--force-copy", + action="store_true", + help="Overwrite output by re-copying from source first.", + ) + p.add_argument( + "--history-days", + type=int, + default=1825, + help="OHLCV lookback days (~5y). Default 1825.", + ) + p.add_argument( + "--min-bars", + type=int, + default=260, + help="Skip re-fetch when a symbol already has this many bars.", + ) + p.add_argument( + "--limit", + type=int, + default=None, + help="Max *new* symbols to fetch (smoke tests).", + ) + p.add_argument( + "--sleep", + type=float, + default=0.15, + help="Seconds between Alpaca symbol requests (rate-limit cushion).", + ) + p.add_argument( + "--max-retries", + type=int, + default=5, + help="Retries per symbol on RateLimitError.", + ) + p.add_argument("--quiet", action="store_true") + return p.parse_args() + + +def _ensure_rank_only_table(conn) -> None: + conn.execute( + text( + """ + CREATE TABLE IF NOT EXISTS research_rank_only ( + ticker_id INTEGER PRIMARY KEY, + symbol TEXT NOT NULL UNIQUE + ) + """ + ) + ) + conn.commit() + + +async def _resolve_pool() -> tuple[list[str], dict[str, str]]: + """Return sorted unique symbols and source labels.""" + from app.database import async_session_factory + from app.services.ticker_universe_service import fetch_universe_symbols + + sources: dict[str, str] = {} + symbols: set[str] = set() + # Need a DB session for cache writes; use local async engine if configured, + # but public/FMP fetch works with any session. Prefer a throwaway sqlite. + from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine + from sqlalchemy.ext.asyncio import AsyncSession + + engine = create_async_engine("sqlite+aiosqlite:///:memory:") + Session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) + try: + async with Session() as db: + for universe in ("nasdaq_all", "sp500"): + try: + syms, src = await fetch_universe_symbols(db, universe) + except Exception as exc: + print(f"WARNING: universe {universe} failed: {exc}") + continue + sources[universe] = src + symbols.update(syms) + print(f" {universe}: {len(syms)} symbols (source={src})") + finally: + await engine.dispose() + return sorted(symbols), sources + + +async def _fetch_symbol_bars( + provider, + symbol: str, + start: date, + end: date, + *, + max_retries: int, + sleep_s: float, +) -> list: + from app.exceptions import ProviderError, RateLimitError + + for attempt in range(max_retries): + try: + bars = await provider.fetch_ohlcv(symbol, start, end) + if sleep_s > 0: + await asyncio.sleep(sleep_s) + return bars + except RateLimitError: + wait = min(60.0, 2.0 ** attempt) + print(f" rate limited on {symbol}; sleep {wait:.0f}s") + await asyncio.sleep(wait) + except ProviderError as exc: + if attempt + 1 >= max_retries: + raise + await asyncio.sleep(1.0) + _ = exc + return [] + + +async def _main() -> None: + args = _parse_args() + source = Path(args.source) + output = Path(args.output) + if not source.exists(): + raise SystemExit(f"Source snapshot not found: {source}") + + if args.force_copy or not output.exists(): + output.parent.mkdir(parents=True, exist_ok=True) + if output.exists(): + output.unlink() + print(f"Copying {source} → {output}") + shutil.copy2(source, output) + else: + print(f"Updating existing research snapshot: {output}") + + from app.config import settings + from app.models.ohlcv import OHLCVRecord + from app.models.ticker import Ticker + from app.providers.alpaca import AlpacaOHLCVProvider + + if not settings.alpaca_api_key or not settings.alpaca_api_secret: + raise SystemExit("ALPACA_API_KEY / ALPACA_API_SECRET required in .env") + + provider = AlpacaOHLCVProvider(settings.alpaca_api_key, settings.alpaca_api_secret) + end = date.today() + start = end - timedelta(days=int(args.history_days)) + + print("Resolving universe pool (nasdaq_all ∪ sp500)…") + pool, sources = await _resolve_pool() + print(f"Pool size: {len(pool)} (sources={sources})") + + # Sync sqlite via sqlalchemy core (simpler than async for bulk insert) + engine = create_engine(f"sqlite:///{output.resolve().as_posix()}") + with Session(engine) as session: + _ensure_rank_only_table(session.connection()) + existing = { + row.symbol: row + for row in session.execute(select(Ticker)).scalars().all() + } + prod_symbols = set(existing) + + # Bar counts + bar_counts: dict[str, int] = {} + for sym, ticker in existing.items(): + n = session.execute( + text("SELECT COUNT(*) FROM ohlcv_records WHERE ticker_id = :tid"), + {"tid": ticker.id}, + ).scalar_one() + bar_counts[sym] = int(n) + + to_fetch: list[str] = [] + for sym in pool: + if sym in existing and bar_counts.get(sym, 0) >= args.min_bars: + # Existing production or previously extended — keep rank_only + # only for *new* research names, not original prod universe. + continue + to_fetch.append(sym) + + if args.limit is not None: + to_fetch = to_fetch[: max(0, int(args.limit))] + + print(f"Symbols to fetch/extend: {len(to_fetch)}") + ok = 0 + fail = 0 + t0 = time.monotonic() + for index, sym in enumerate(to_fetch, 1): + try: + bars = await _fetch_symbol_bars( + provider, + sym, + start, + end, + max_retries=args.max_retries, + sleep_s=args.sleep, + ) + except Exception as exc: + fail += 1 + if not args.quiet: + print(f" [{index}/{len(to_fetch)}] {sym} FAIL {exc}") + continue + + if not bars: + fail += 1 + if not args.quiet: + print(f" [{index}/{len(to_fetch)}] {sym} empty") + continue + + ticker = existing.get(sym) + is_new = ticker is None + if ticker is None: + ticker = Ticker(symbol=sym, name=None, created_at=datetime.now(timezone.utc)) + session.add(ticker) + session.flush() + existing[sym] = ticker + + # Upsert bars (delete+insert range for simplicity on research path) + session.execute( + text( + "DELETE FROM ohlcv_records WHERE ticker_id = :tid " + "AND date >= :start AND date <= :end" + ), + {"tid": ticker.id, "start": start.isoformat(), "end": end.isoformat()}, + ) + now = datetime.utcnow() + session.bulk_insert_mappings( + OHLCVRecord, + [ + { + "ticker_id": ticker.id, + "date": b.date, + "open": b.open, + "high": b.high, + "low": b.low, + "close": b.close, + "volume": b.volume, + "created_at": now, + } + for b in bars + ], + ) + + # rank_only only for names that were NOT in the original production + # snapshot at copy time (or are newly introduced to this research DB). + if is_new or sym not in prod_symbols: + # Re-evaluate: if source copy already had the symbol, don't flag. + # Only new inserts get rank_only. + if is_new: + session.execute( + text( + "INSERT OR REPLACE INTO research_rank_only " + "(ticker_id, symbol) VALUES (:tid, :sym)" + ), + {"tid": ticker.id, "sym": sym}, + ) + + session.commit() + ok += 1 + if not args.quiet and (index % 25 == 0 or index == len(to_fetch)): + elapsed = time.monotonic() - t0 + print( + f" progress {index}/{len(to_fetch)} ok={ok} fail={fail} " + f"elapsed={elapsed/60:.1f}m last={sym} bars={len(bars)}" + ) + + rank_only_n = session.execute( + text("SELECT COUNT(*) FROM research_rank_only") + ).scalar_one() + ticker_n = session.execute(text("SELECT COUNT(*) FROM tickers")).scalar_one() + ohlcv_n = session.execute(text("SELECT COUNT(*) FROM ohlcv_records")).scalar_one() + + print("Done.") + print(f" output: {output}") + print(f" tickers: {ticker_n}") + print(f" ohlcv rows: {ohlcv_n}") + print(f" research_rank_only: {rank_only_n}") + print(f" fetched ok/fail: {ok}/{fail}") + + +if __name__ == "__main__": + asyncio.run(_main()) diff --git a/scripts/run_fip_breadth_research.py b/scripts/run_fip_breadth_research.py new file mode 100644 index 0000000..cc9f6e9 --- /dev/null +++ b/scripts/run_fip_breadth_research.py @@ -0,0 +1,299 @@ +"""Phase B: fip_id IC on liquid-breadth cross-section (local research only). + +1. Fingerprint check on the unextended prod snapshot (must ≈ IC −0.045 / t −2.9). +2. Run signal_eval on research.sqlite with BACKTEST_LIQUID_BREADTH=1500 PIT mask. +3. Write a research report under docs/research/ and reports/. + +Does not modify production DB, gate, scanner, or schedule. + +Example +------- + # After extend_snapshot_universe.py has built research.sqlite: + python scripts/run_fip_breadth_research.py \\ + --prod-snapshot backtest_snapshots/prod.sqlite \\ + --research-snapshot backtest_snapshots/research.sqlite \\ + --workers 6 --allow-spawn +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import os +import sys +from datetime import datetime +from pathlib import Path + +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +FINGERPRINT_IC = -0.045 +FINGERPRINT_T = -2.9 +FINGERPRINT_IC_TOL = 0.015 +FINGERPRINT_T_TOL = 0.6 + + +def _sqlite_url(path: Path) -> str: + return f"sqlite+aiosqlite:///{path.resolve().as_posix()}" + + +def _parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--prod-snapshot", default="backtest_snapshots/prod.sqlite") + p.add_argument("--research-snapshot", default="backtest_snapshots/research.sqlite") + p.add_argument("--workers", type=int, default=6) + p.add_argument("--allow-spawn", action="store_true") + p.add_argument("--skip-fingerprint", action="store_true") + p.add_argument("--skip-research", action="store_true") + p.add_argument("--liquid-breadth", type=int, default=1500) + p.add_argument("--min-price", type=float, default=5.0) + p.add_argument( + "--out", + default=None, + help="JSON report path (default reports/fip-breadth-YYYYMMDD.json)", + ) + p.add_argument("--quiet", action="store_true") + return p.parse_args() + + +def _find_fip(signal_eval: list[dict]) -> dict | None: + for row in signal_eval or []: + if row.get("signal") == "fip_id": + return row + return None + + +def _verdict(row: dict | None) -> dict: + if row is None: + return { + "green": False, + "reason": "fip_id missing from signal_eval", + } + mean_ic = row.get("mean_ic") + t_stat = row.get("ic_t_stat") + reliable = bool(row.get("reliable")) + if mean_ic is None or t_stat is None: + return {"green": False, "reason": "missing mean_ic or ic_t_stat", "row": row} + sign_ok = mean_ic < 0 + mag_ok = abs(float(mean_ic)) >= 0.03 + green = sign_ok and mag_ok and reliable + return { + "green": green, + "reason": ( + "iron rule cleared — follow-up proposal only, not production wire-in" + if green + else "iron rule not met on liquid-breadth cross-section" + ), + "checks": { + "mean_ic": mean_ic, + "abs_mean_ic_ge_0_03": mag_ok, + "sign_negative": sign_ok, + "ic_t_stat": t_stat, + "reliable": reliable, + "weeks": row.get("weeks"), + "avg_cross_section": row.get("avg_cross_section"), + }, + "row": row, + } + + +async def _run_signal_eval(snapshot: Path, *, workers: int, quiet: bool) -> dict: + from app.config import settings + from app.services.backtest_service import run_backtest + + settings.backtest_workers = workers + engine = create_async_engine(_sqlite_url(snapshot), pool_pre_ping=True) + Session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) + + def progress(done: int, total: int, symbol: str) -> None: + if quiet: + return + print(f" progress {done}/{total} {symbol}", end="\r") + + try: + async with Session() as db: + report = await run_backtest(db, progress_cb=progress, cadence="weekly") + finally: + await engine.dispose() + if not quiet: + print() + return report + + +def _write_md(path: Path, payload: dict) -> None: + fp = payload.get("fingerprint") or {} + br = payload.get("breadth") or {} + v = payload.get("verdict") or {} + lines = [ + "# Broad-universe fip_id IC research (Phase B)", + "", + f"Generated: {payload.get('generated_at')}", + "", + "## Scope", + "", + "- **Research only** — production universe, gate, scanner, schedule unchanged.", + "- Price-only signal harness; no sentiment/fundamentals on the broad tier.", + "- Point-in-time liquidity mask: top " + f"**{payload.get('liquid_breadth_top_n')}** by 63d median $vol, " + f"price ≥ **${payload.get('liquid_min_price')}** at as-of.", + "", + "## Caveats", + "", + "- **Survivorship bias**: today's constituents backfilled historically " + "(worse in small caps).", + "- **IEX volume undercount**: relative $vol rank only, not absolute floors.", + "- **Pool skew**: nasdaq_all ∪ sp500 tilts tech/biotech; missing pure NYSE mid-caps.", + "", + "## Fingerprint (505-name prod snapshot)", + "", + f"- Expected: IC ≈ {FINGERPRINT_IC}, t ≈ {FINGERPRINT_T}", + f"- Observed: IC = {fp.get('mean_ic')}, t = {fp.get('ic_t_stat')}, " + f"weeks = {fp.get('weeks')}, reliable = {fp.get('reliable')}", + f"- Pass: **{fp.get('pass')}**", + "", + "## Liquid-breadth signal_eval (fip_id)", + "", + ] + row = br.get("row") or br + if row: + lines.extend([ + f"| metric | value |", + f"|---|---|", + f"| mean_ic | {row.get('mean_ic')} |", + f"| ic_t_stat | {row.get('ic_t_stat')} |", + f"| ic_positive_pct | {row.get('ic_positive_pct')} |", + f"| weeks | {row.get('weeks')} |", + f"| avg_cross_section | {row.get('avg_cross_section')} |", + f"| reliable | {row.get('reliable')} |", + f"| mean_quintile_spread | {row.get('mean_quintile_spread')} |", + "", + ]) + else: + lines.append("_No breadth result (run skipped or failed)._") + lines.append("") + lines.extend([ + "## Verdict (iron rule)", + "", + f"- **Green: {v.get('green')}**", + f"- {v.get('reason')}", + f"- Checks: `{json.dumps(v.get('checks') or {}, default=str)}`", + "", + "A green verdict authorizes a **follow-up proposal** only " + "(two-tier universe / gate revalidation) — **not** production wire-in.", + "", + "## Artifacts", + "", + f"- Fingerprint report: `{payload.get('fingerprint_report_path')}`", + f"- Breadth report: `{payload.get('breadth_report_path')}`", + "", + ]) + path.write_text("\n".join(lines), encoding="utf-8") + + +async def _main() -> None: + args = _parse_args() + prod = Path(args.prod_snapshot) + research = Path(args.research_snapshot) + if not prod.exists(): + raise SystemExit(f"Prod snapshot missing: {prod}") + + os.environ["BACKTEST_SNAPSHOT_OFFLINE"] = "1" + if args.allow_spawn: + os.environ["BACKTEST_ALLOW_SPAWN"] = "1" + os.environ["BACKTEST_SIGNAL_EVAL_ONLY"] = "1" + + stamp = datetime.now().strftime("%Y%m%d-%H%M%S") + out_json = Path(args.out) if args.out else Path("reports") / f"fip-breadth-{stamp}.json" + out_json.parent.mkdir(parents=True, exist_ok=True) + out_md = Path("docs/research") / "fip-breadth-ic.md" + + payload: dict = { + "generated_at": datetime.now().isoformat(), + "liquid_breadth_top_n": args.liquid_breadth, + "liquid_min_price": args.min_price, + "fingerprint": None, + "breadth": None, + "verdict": None, + } + + # --- 1) Fingerprint --- + if not args.skip_fingerprint: + # Clear liquid breadth for fingerprint + os.environ.pop("BACKTEST_LIQUID_BREADTH", None) + os.environ.pop("BACKTEST_LIQUID_MIN_PRICE", None) + if not args.quiet: + print(f"Fingerprint run on {prod}…") + fp_report = await _run_signal_eval(prod, workers=args.workers, quiet=args.quiet) + fp_path = out_json.with_name(out_json.stem + "-fingerprint.json") + fp_path.write_text(json.dumps(fp_report, indent=2, default=str), encoding="utf-8") + fip = _find_fip(fp_report.get("signal_eval") or []) + if fip is None: + raise SystemExit("ABORT: fip_id missing from fingerprint signal_eval") + ic_ok = abs(float(fip["mean_ic"]) - FINGERPRINT_IC) <= FINGERPRINT_IC_TOL + t_ok = abs(float(fip["ic_t_stat"]) - FINGERPRINT_T) <= FINGERPRINT_T_TOL + passed = ic_ok and t_ok and bool(fip.get("reliable")) + payload["fingerprint"] = { + **fip, + "pass": passed, + "expected_ic": FINGERPRINT_IC, + "expected_t": FINGERPRINT_T, + } + payload["fingerprint_report_path"] = str(fp_path) + if not args.quiet: + print( + f"Fingerprint fip_id IC={fip.get('mean_ic')} t={fip.get('ic_t_stat')} " + f"pass={passed}" + ) + if not passed: + out_json.write_text(json.dumps(payload, indent=2, default=str), encoding="utf-8") + raise SystemExit( + "ABORT: fingerprint mismatch — investigate before trusting breadth runs " + f"(got IC={fip.get('mean_ic')} t={fip.get('ic_t_stat')})" + ) + + # --- 2) Breadth --- + if not args.skip_research: + if not research.exists(): + raise SystemExit( + f"Research snapshot missing: {research}\n" + "Build it with: python scripts/extend_snapshot_universe.py" + ) + os.environ["BACKTEST_LIQUID_BREADTH"] = str(int(args.liquid_breadth)) + os.environ["BACKTEST_LIQUID_MIN_PRICE"] = str(float(args.min_price)) + if not args.quiet: + print( + f"Breadth run on {research} " + f"(top {args.liquid_breadth}, min_price={args.min_price})…" + ) + br_report = await _run_signal_eval( + research, workers=args.workers, quiet=args.quiet + ) + br_path = out_json.with_name(out_json.stem + "-breadth.json") + br_path.write_text(json.dumps(br_report, indent=2, default=str), encoding="utf-8") + fip_b = _find_fip(br_report.get("signal_eval") or []) + payload["breadth"] = fip_b or {"error": "fip_id missing"} + payload["breadth_report_path"] = str(br_path) + payload["breadth_tickers"] = br_report.get("tickers") + payload["breadth_rank_only_tickers"] = br_report.get("rank_only_tickers") + payload["verdict"] = _verdict(fip_b) + if not args.quiet: + print( + f"Breadth fip_id IC={ (fip_b or {}).get('mean_ic') } " + f"t={ (fip_b or {}).get('ic_t_stat') } " + f"green={payload['verdict'].get('green')}" + ) + + out_json.write_text(json.dumps(payload, indent=2, default=str), encoding="utf-8") + out_md.parent.mkdir(parents=True, exist_ok=True) + _write_md(out_md, payload) + if not args.quiet: + print(f"Wrote {out_json}") + print(f"Wrote {out_md}") + + +if __name__ == "__main__": + asyncio.run(_main()) diff --git a/tests/unit/test_backtest_service.py b/tests/unit/test_backtest_service.py index aff0eb6..a69eb74 100644 --- a/tests/unit/test_backtest_service.py +++ b/tests/unit/test_backtest_service.py @@ -1154,6 +1154,52 @@ class TestSimulatePortfolio: assert allowed["trade_details"][0]["entry"] == pytest.approx(110.0) +def test_median_dollar_vol_63_basic(): + closes = [10.0] * 70 + volumes = [100.0 + i for i in range(70)] + med = bt._median_dollar_vol_63(closes, volumes, 69, lookback=63) + assert med is not None + assert med > 0 + + +def test_liquid_breadth_week_keeps_top_n_by_dvol(): + recs = [ + {"val": 0.1, "fwd": 0.01, "close": 20.0, "median_dvol_63": 1e6}, + {"val": 0.2, "fwd": 0.02, "close": 20.0, "median_dvol_63": 9e6}, + {"val": 0.3, "fwd": 0.03, "close": 20.0, "median_dvol_63": 5e6}, + {"val": 0.4, "fwd": 0.04, "close": 1.0, "median_dvol_63": 99e6}, # price floor + {"val": 0.5, "fwd": 0.05, "close": 20.0, "median_dvol_63": None}, + ] + pairs = bt._filter_liquid_breadth_week(recs, top_n=2, min_price=5.0) + assert len(pairs) == 2 + # Highest dvol first among eligible: 9e6 then 5e6 + assert pairs[0][0] == pytest.approx(0.2) + assert pairs[1][0] == pytest.approx(0.3) + + +def test_signal_eval_liquid_breadth_env(monkeypatch): + # top_n=5 → keep 5 names/week; spearman needs ≥3 observations. + monkeypatch.setenv("BACKTEST_LIQUID_BREADTH", "5") + monkeypatch.setenv("BACKTEST_LIQUID_MIN_PRICE", "5") + monkeypatch.setattr(bt, "MIN_CROSS_SECTION", 3) + monkeypatch.setattr(bt, "MIN_RELIABLE_PERIODS", 3) + week = {} + for w in (1, 10, 20, 30, 40, 50): + week[(2024, w)] = [ + { + "val": float(i), + "fwd": float(i) * 0.01, + "close": 10.0, + "median_dvol_63": float(100 - i), + } + for i in range(12) + ] + rows = bt._signal_evaluation({"toy": week}) + assert rows + assert rows[0]["liquid_breadth_top_n"] == 5 + assert rows[0]["avg_cross_section"] == 5.0 + + def test_fip_id_sign_convention_steady_climber_vs_jump(): # Steady climber: many up days, continuous path → lower (more negative) ID. steady = [100.0]