diff --git a/app/services/backtest_service.py b/app/services/backtest_service.py index a9503eb..01bf89e 100644 --- a/app/services/backtest_service.py +++ b/app/services/backtest_service.py @@ -30,11 +30,6 @@ Environment variables (see also run_backtest_snapshot.py): BACKTEST_FALLBACK_CLEAR_AIR_ONLY=1 BACKTEST_RESEARCH_EXITS=1 BACKTEST_MIN_RR_SWEEP=1 - - Broad-universe signal research (local snapshots only; inert when unset): - BACKTEST_LIQUID_BREADTH=1500 # PIT top-N by 63d median $vol, price floor - BACKTEST_LIQUID_MIN_PRICE=5 # USD close floor at as-of (default 5) - BACKTEST_SIGNAL_EVAL_ONLY=1 # skip portfolio_sim / monitor (signal IC only) """ from __future__ import annotations @@ -881,63 +876,6 @@ def _signal_values( return out -def _liquid_breadth_top_n() -> int: - """0 = off (production path). N > 0 enables PIT top-N $vol mask for signal IC.""" - raw = os.getenv("BACKTEST_LIQUID_BREADTH", "").strip() - if not raw: - return 0 - try: - return max(0, int(raw)) - except ValueError: - return 0 - - -def _liquid_min_price() -> float: - raw = os.getenv("BACKTEST_LIQUID_MIN_PRICE", "5").strip() or "5" - try: - return max(0.0, float(raw)) - except ValueError: - return 5.0 - - -def _signal_eval_only() -> bool: - return os.getenv("BACKTEST_SIGNAL_EVAL_ONLY", "").strip() in ("1", "true", "yes") - - -async def _load_research_rank_only_symbols(db: AsyncSession) -> set[str]: - """Symbols that feed signal IC only (no GTL/candidate replay). - - Optional side table ``research_rank_only`` on research snapshots. Missing - table → empty set (production path unchanged). - """ - from sqlalchemy import text - - try: - result = await db.execute(text("SELECT symbol FROM research_rank_only")) - return {str(row[0]).upper() for row in result.fetchall() if row[0]} - except Exception: - return set() - - -def _median_dollar_vol_63( - closes: list[float], volumes: list[float], i: int, lookback: int = 63 -) -> float | None: - """Rolling median of close×volume over ``lookback`` bars ending at ``i`` (inclusive).""" - if i + 1 < lookback or lookback < 2: - return None - dvs: list[float] = [] - for k in range(i - lookback + 1, i + 1): - if closes[k] > 0 and volumes[k] >= 0: - dvs.append(closes[k] * float(volumes[k])) - if len(dvs) < max(20, lookback // 2): - return None - dvs_sorted = sorted(dvs) - mid = len(dvs_sorted) // 2 - if len(dvs_sorted) % 2: - return dvs_sorted[mid] - return 0.5 * (dvs_sorted[mid - 1] + dvs_sorted[mid]) - - def _accumulate_signal_series( records: list, collected: dict, @@ -945,20 +883,13 @@ def _accumulate_signal_series( ) -> None: """For each weekly as-of bar, emit (signal, forward-return) pairs keyed by ISO week into ``collected[name][week_key]``. Forward return is close-to-close over - HORIZON trading days. Mutates ``collected`` (a dict of dict of list). - - When ``BACKTEST_LIQUID_BREADTH`` is set, observations are dicts with PIT - liquidity fields for the mask; otherwise plain ``(val, fwd)`` tuples so the - production signal path stays unchanged. - """ + HORIZON trading days. Mutates ``collected`` (a dict of dict of list).""" n = len(records) if n < HORIZON + 21: return closes = [float(r.close) for r in records] highs = [float(r.high) for r in records] - volumes = [float(getattr(r, "volume", 0) or 0) for r in records] dates = [r.date for r in records] - liquid_mode = _liquid_breadth_top_n() > 0 for i in _weekly_asof_indices(records): j = i + HORIZON if j >= n or closes[i] <= 0: @@ -966,17 +897,8 @@ def _accumulate_signal_series( fwd = closes[j] / closes[i] - 1.0 iso = records[i].date.isocalendar() week_key = (iso[0], iso[1]) - dvol = _median_dollar_vol_63(closes, volumes, i) if liquid_mode else None for name, val in _signal_values(dates, closes, highs, i, benchmark_closes).items(): - if liquid_mode: - collected[name][week_key].append({ - "val": val, - "fwd": fwd, - "close": closes[i], - "median_dvol_63": dvol, - }) - else: - collected[name][week_key].append((val, fwd)) + collected[name][week_key].append((val, fwd)) def _rank(xs: list[float]) -> list[float]: @@ -1015,54 +937,6 @@ def _spearman(xs: list[float], ys: list[float]) -> float | None: return _pearson(_rank(xs), _rank(ys)) -def _obs_val_fwd(rec: object) -> tuple[float, float] | None: - """Unpack a signal observation: ``(val, fwd)`` or research dict form.""" - if isinstance(rec, dict): - try: - return float(rec["val"]), float(rec["fwd"]) - except (KeyError, TypeError, ValueError): - return None - if isinstance(rec, (tuple, list)) and len(rec) >= 2: - try: - return float(rec[0]), float(rec[1]) - except (TypeError, ValueError): - return None - return None - - -def _filter_liquid_breadth_week( - recs: list, - *, - top_n: int, - min_price: float, -) -> list[tuple[float, float]]: - """Point-in-time top-N by median $vol among names with price ≥ floor. - - Ranking is relative (IEX volume undercount is OK for order stats). Membership - is recomputed every week from as-of bars — never frozen from today's liquidity. - """ - ranked: list[tuple[float, float, float]] = [] # (-dvol, val, fwd) - for rec in recs: - if not isinstance(rec, dict): - pair = _obs_val_fwd(rec) - if pair is not None: - ranked.append((0.0, pair[0], pair[1])) - continue - close = rec.get("close") - dvol = rec.get("median_dvol_63") - if close is None or float(close) < min_price: - continue - if dvol is None or float(dvol) <= 0: - continue - pair = _obs_val_fwd(rec) - if pair is None: - continue - ranked.append((-float(dvol), pair[0], pair[1])) - ranked.sort(key=lambda row: row[0]) - kept = ranked[:top_n] - return [(val, fwd) for _, val, fwd in kept] - - def _quintile_spread(pairs: list[tuple[float, float]]) -> float | None: """Mean forward return of the top signal-quintile minus the bottom quintile.""" n = len(pairs) @@ -1108,16 +982,10 @@ def _signal_evaluation(collected: dict) -> list[dict]: IC is measured on NON-OVERLAPPING forward windows (weeks thinned to ~HORIZON apart) so the t-stat isn't inflated by autocorrelation. A signal with no edge - lands near IC 0 / score 0; one with too few independent windows is flagged + lands near IC 0 / spread 0; one with too few independent windows is flagged unreliable rather than trusted on a lucky handful. - - When ``BACKTEST_LIQUID_BREADTH=N`` is set, each week's cross-section is first - restricted to the top-N names by point-in-time 63d median dollar volume - (price ≥ BACKTEST_LIQUID_MIN_PRICE). Production path (flag unset) is unchanged. """ stride = max(1, round(HORIZON / 5)) # ISO weeks spanned by the forward window - top_n = _liquid_breadth_top_n() - min_price = _liquid_min_price() rows: list[dict] = [] for name in sorted(collected): weeks_map = collected[name] @@ -1128,25 +996,13 @@ def _signal_evaluation(collected: dict) -> list[dict]: sizes: list[int] = [] for wk in kept: recs = weeks_map[wk] - if top_n > 0: - pairs = _filter_liquid_breadth_week( - recs, top_n=top_n, min_price=min_price - ) - else: - pairs = [] - for rec in recs: - pair = _obs_val_fwd(rec) - if pair is not None: - pairs.append(pair) - if len(pairs) < MIN_CROSS_SECTION: - continue - ic = _spearman([p[0] for p in pairs], [p[1] for p in pairs]) + ic = _spearman([r[0] for r in recs], [r[1] for r in recs]) if ic is not None: ics.append(ic) - spread = _quintile_spread(pairs) + spread = _quintile_spread(recs) if spread is not None: spreads.append(spread) - sizes.append(len(pairs)) + sizes.append(len(recs)) if not ics: continue mean_ic = sum(ics) / len(ics) @@ -1155,7 +1011,7 @@ def _signal_evaluation(collected: dict) -> list[dict]: else: std = 0.0 t_stat = mean_ic / std * math.sqrt(len(ics)) if std > 0 else None - row = { + rows.append({ "signal": name, "weeks": len(ics), "avg_cross_section": round(sum(sizes) / len(sizes), 1) if sizes else None, @@ -1164,11 +1020,7 @@ def _signal_evaluation(collected: dict) -> list[dict]: "ic_positive_pct": round(sum(1 for x in ics if x > 0) / len(ics) * 100, 1), "mean_quintile_spread": round(sum(spreads) / len(spreads), 4) if spreads else None, "reliable": len(ics) >= MIN_RELIABLE_PERIODS, - } - if top_n > 0: - row["liquid_breadth_top_n"] = top_n - row["liquid_min_price"] = min_price - rows.append(row) + }) rows.sort(key=lambda r: r["mean_ic"], reverse=True) return rows @@ -1189,15 +1041,10 @@ def _replay_and_signals( benchmark_closes: dict[date, float] | None = None, target_model: str = PRODUCTION_GTL_TARGET_MODEL, cadence: str = DEFAULT_BACKTEST_CADENCE, - signal_only: bool = False, ) -> tuple[list[dict], dict]: """The CPU-bound per-ticker work, as a top-level (picklable) function so it can run in a worker process. Takes primitive column arrays (cheap to pickle), - rebuilds bar objects, and returns (candidates, signal_series). - - ``signal_only=True`` (research rank-only names): skip GTL/candidate replay so - the production portfolio book is never polluted by broad-universe tickers. - """ + rebuilds bar objects, and returns (candidates, signal_series).""" date_ords, opens, highs, lows, closes, volumes = columns bars = [ SimpleNamespace( @@ -1205,9 +1052,8 @@ def _replay_and_signals( ) for o, op, hi, lo, cl, vo in zip(date_ords, opens, highs, lows, closes, volumes) ] - candidates: list[dict] = [] - if not signal_only: - candidates = _replay_ticker( + return ( + _replay_ticker( symbol, bars, config, @@ -1215,9 +1061,7 @@ def _replay_and_signals( benchmark_closes, target_model, cadence, - ) - return ( - candidates, + ), _signal_series(bars, benchmark_closes), ) @@ -3945,12 +3789,6 @@ async def run_backtest( result = await db.execute(select(Ticker).order_by(Ticker.symbol)) tickers = list(result.scalars().all()) total = len(tickers) - rank_only_symbols = await _load_research_rank_only_symbols(db) - if rank_only_symbols: - logger.info(json.dumps({ - "event": "backtest_rank_only_loaded", - "count": len(rank_only_symbols), - })) candidates: list[dict] = [] # Signal IC remains a weekly, non-overlapping diagnostic regardless of the @@ -4009,16 +3847,10 @@ async def run_backtest( continue if columns is not None: futures.append(loop.run_in_executor( - pool, - _replay_and_signals, - ticker.symbol, - columns, - config, - activation, + pool, _replay_and_signals, ticker.symbol, columns, config, activation, benchmark_closes, target_model, cadence, - ticker.symbol in rank_only_symbols, )) for result in await asyncio.gather(*futures, return_exceptions=True): if isinstance(result, Exception): @@ -4038,15 +3870,10 @@ async def run_backtest( columns = await _fetch_columns(db, ticker.symbol) if columns is not None: _merge(await asyncio.to_thread( - _replay_and_signals, - ticker.symbol, - columns, - config, - activation, + _replay_and_signals, ticker.symbol, columns, config, activation, benchmark_closes, target_model, cadence, - ticker.symbol in rank_only_symbols, )) except Exception: logger.exception("Backtest replay failed for %s", ticker.symbol) @@ -4089,75 +3916,73 @@ async def run_backtest( portfolio_monitor_report: dict | None = None holdout_report: dict | None = None min_rr_sweep_report: dict | None = None - if not _signal_eval_only(): + try: + qual_symbols = sorted({ + c["symbol"] + for c in candidates + if c.get("qualified") + or any(_qualifies_strategy_variant(c, cfg) for cfg in STRATEGY_VARIANTS) + }) + price_columns: dict[str, tuple] = {} + for sym in qual_symbols: + cols = await _fetch_columns(db, sym) + if cols is not None: + price_columns[sym] = cols + + spy_closes: dict | None = None try: - qual_symbols = sorted({ - c["symbol"] - for c in candidates - if c.get("qualified") - or any(_qualifies_strategy_variant(c, cfg) for cfg in STRATEGY_VARIANTS) - }) - price_columns: dict[str, tuple] = {} - for sym in qual_symbols: - cols = await _fetch_columns(db, sym) - if cols is not None: - price_columns[sym] = cols - - spy_closes: dict | None = None - try: - oldest = min((cols[0][0] for cols in price_columns.values()), default=None) - days_needed = None - if oldest is not None and not _offline_snapshot_mode(): - days_needed = (date.today() - date.fromordinal(oldest)).days + 30 - spy_closes = await _load_benchmark_closes_for_backtest( - db, days=days_needed, refresh=oldest is not None - ) - except Exception: - logger.exception("Benchmark load for the portfolio sim failed") - - for policy in ("target", "hold"): - sim = _simulate_portfolio( - candidates, price_columns, spy_closes, policy, hold_horizon - ) - if sim is not None: - sim_policies.append({"policy": policy, **sim}) - strategy_variant_rows = _strategy_variant_sims( - candidates, price_columns, spy_closes, hold_horizon + oldest = min((cols[0][0] for cols in price_columns.values()), default=None) + days_needed = None + if oldest is not None and not _offline_snapshot_mode(): + days_needed = (date.today() - date.fromordinal(oldest)).days + 30 + spy_closes = await _load_benchmark_closes_for_backtest( + db, days=days_needed, refresh=oldest is not None ) - exit_policy_rows = _exit_policy_sims( - candidates, price_columns, spy_closes, hold_horizon - ) - live_exit_policy: dict | None = None - try: - from app.services.paper_trade_service import get_exit_policy + except Exception: + logger.exception("Benchmark load for the portfolio sim failed") - live_exit_policy = await get_exit_policy(db) - except Exception: - logger.exception("Live exit policy load failed; monitor uses defaults") - portfolio_monitor_report = _portfolio_monitor( - candidates, price_columns, spy_closes, hold_horizon, + for policy in ("target", "hold"): + sim = _simulate_portfolio( + candidates, price_columns, spy_closes, policy, hold_horizon + ) + if sim is not None: + sim_policies.append({"policy": policy, **sim}) + strategy_variant_rows = _strategy_variant_sims( + candidates, price_columns, spy_closes, hold_horizon + ) + exit_policy_rows = _exit_policy_sims( + candidates, price_columns, spy_closes, hold_horizon + ) + live_exit_policy: dict | None = None + try: + from app.services.paper_trade_service import get_exit_policy + + live_exit_policy = await get_exit_policy(db) + except Exception: + logger.exception("Live exit policy load failed; monitor uses defaults") + portfolio_monitor_report = _portfolio_monitor( + candidates, price_columns, spy_closes, hold_horizon, + live_exit_policy=live_exit_policy, + cadence=cadence, + ) + split = _holdout_split() + if split is not None: + holdout_report = _holdout_evaluation( + candidates, price_columns, spy_closes, hold_horizon, split, live_exit_policy=live_exit_policy, cadence=cadence, ) - split = _holdout_split() - if split is not None: - holdout_report = _holdout_evaluation( - candidates, price_columns, spy_closes, hold_horizon, split, - live_exit_policy=live_exit_policy, - cadence=cadence, - ) - if _min_rr_sweep_enabled(): - min_rr_sweep_report = _min_rr_sweep( - candidates, price_columns, spy_closes, activation, current_min_pct, - hold_horizon, live_exit_policy=live_exit_policy, cadence=cadence, - ) - except Exception: - logger.exception("Portfolio simulation failed") + if _min_rr_sweep_enabled(): + min_rr_sweep_report = _min_rr_sweep( + candidates, price_columns, spy_closes, activation, current_min_pct, + hold_horizon, live_exit_policy=live_exit_policy, cadence=cadence, + ) + except Exception: + logger.exception("Portfolio simulation failed") report = { "generated_at": datetime.now(timezone.utc).isoformat(), "tickers": total, - "rank_only_tickers": len(rank_only_symbols), "candidates": len(candidates), "qualified": len(qualified), "params": { @@ -4174,9 +3999,6 @@ async def run_backtest( "target_model_label": BACKTEST_TARGET_MODELS[target_model], "is_production_target_model": target_model == PRODUCTION_GTL_TARGET_MODEL, "production_reentry_policy": PRODUCTION_REENTRY_POLICY, - "liquid_breadth_top_n": _liquid_breadth_top_n() or None, - "liquid_min_price": _liquid_min_price() if _liquid_breadth_top_n() else None, - "signal_eval_only": _signal_eval_only(), }, "activation": activation, "overall_qualified": _bucket_stats(qualified), diff --git a/docs/research/README.md b/docs/research/README.md index 7be97f3..396decb 100644 --- a/docs/research/README.md +++ b/docs/research/README.md @@ -141,8 +141,8 @@ knobs. | Lead | Why it's interesting | Blocker | |---|---|---| | **Near-close / MOC execution (ops)** | Recovers overnight momentum drift left on the table by a morning EU scan; evidence closed | Implement schedule + partial-bar scan path; one qualifying scan/day only | -| **`fip_id`** (information discreteness over the 12-1 window) | **Strongest cross-sectional signal measured on this universe** — IC −0.045, t = −2.91, correct sign; re-derived fingerprint matched Phase A; ticker technicals show it display-only | Doesn't improve *this* book as a filter. **Phase B tooling ready:** liquid-breadth IC on research.sqlite — see [fip-breadth-ic.md](fip-breadth-ic.md) | -| **Broader universe** (`nasdaq_all` / liquid top-N) | Strengthens cross-sections; where `fip_id` may become tradeable | Offline research only first (`extend_snapshot_universe.py`); not prod scan | +| **`fip_id`** (information discreteness over the 12-1 window) | **Strongest cross-sectional signal measured on this universe** — IC −0.045, t = −2.91, correct sign; re-derived fingerprint matched Phase A | Doesn't improve *this* book. Revisit when the universe broadens — **after** execution path is decided | +| **Broader universe** (`nasdaq_all`) | Strengthens every week's cross-section and the IC t-stat | Grade under the fill mode you will trade | | **Forward paper-trade record** | The only true out-of-sample evidence the snapshot cannot give | Time; mark entries at actual near-close fill once ops ships | | **Better target model for clear-air names** | The return is demonstrably there (#2 wins on raw CAGR in *both* train and test); it's the *flat* 3× ATR target that makes it too expensive in risk | Needs a per-name model, not a constant k×ATR | diff --git a/docs/research/fip-breadth-ic.md b/docs/research/fip-breadth-ic.md deleted file mode 100644 index d02e879..0000000 --- a/docs/research/fip-breadth-ic.md +++ /dev/null @@ -1,54 +0,0 @@ -# Broad-universe fip_id IC research (Phase B) - -Generated: 2026-07-18T19:48:28.127710 - -## Scope - -- **Research only** — production universe, gate, scanner, schedule unchanged. -- Price-only signal harness; no sentiment/fundamentals on the broad tier. -- Point-in-time liquidity mask: top **1500** by 63d median $vol, price ≥ **$5.0** at as-of. - -## Caveats - -- **Survivorship bias**: today's constituents backfilled historically (worse in small caps). -- **IEX volume undercount**: relative $vol rank only, not absolute floors. -- **Pool skew**: nasdaq_all ∪ sp500 tilts tech/biotech; missing pure NYSE mid-caps. - -## Fingerprint (505-name prod snapshot) - -- Expected: IC ≈ -0.045, t ≈ -2.9 -- Observed: IC = -0.045, t = -2.91, weeks = 35, reliable = True -- Pass: **True** - -## Liquid-breadth signal_eval (fip_id) - -**Not run yet in this environment** (no Alpaca credentials to build -`backtest_snapshots/research.sqlite`). Local machine with keys: - -```bash -python scripts/extend_snapshot_universe.py --force-copy -python scripts/run_fip_breadth_research.py --skip-fingerprint --allow-spawn --workers 6 -``` - -(`--skip-fingerprint` only after a green fingerprint on this machine.) - -| metric | value | -|---|---| -| mean_ic | _pending_ | -| ic_t_stat | _pending_ | -| weeks | _pending_ | -| avg_cross_section | _pending_ | -| reliable | _pending_ | - -## Verdict (iron rule) - -- **Fingerprint:** green (IC −0.045 / t −2.91 / reliable / 35 weeks). -- **Breadth iron rule:** **pending** until research.sqlite run completes. -- Green breadth would authorize a **follow-up proposal** only (two-tier universe / - gate revalidation) — **not** production wire-in. - -## Artifacts - -- Fingerprint report: `reports/fip-breadth-20260718-194828-fingerprint.json` - (and summary `reports/fip-breadth-20260718-194828.json`) -- Breadth report: _pending_ diff --git a/reports/fip-breadth-20260718-194828-fingerprint.json b/reports/fip-breadth-20260718-194828-fingerprint.json deleted file mode 100644 index aeae3d0..0000000 --- a/reports/fip-breadth-20260718-194828-fingerprint.json +++ /dev/null @@ -1,578 +0,0 @@ -{ - "generated_at": "2026-07-18T18:21:28.359793+00:00", - "tickers": 506, - "rank_only_tickers": 0, - "candidates": 202765, - "qualified": 1086, - "params": { - "step_days": 5, - "step_sessions": 5, - "entry_cadence": "weekly", - "signal_eval_cadence": "weekly", - "horizon_days": 30, - "min_lookback": 60, - "cost_per_side_pct": 0.1, - "target_model": "production_gtl", - "target_model_label": "Live GTL (production)", - "is_production_target_model": true, - "production_reentry_policy": "gate_reset", - "liquid_breadth_top_n": null, - "liquid_min_price": null, - "signal_eval_only": true - }, - "activation": { - "min_momentum_percentile": 80.0, - "min_rr": 2.0, - "min_confidence": 0.0, - "require_high_conviction": false, - "exclude_conflicts": false, - "exclude_neutral": true - }, - "overall_qualified": { - "total": 1086, - "wins": 379, - "losses": 591, - "expired": 116, - "hit_rate": 39.1, - "avg_r": 0.255, - "total_r": 276.76, - "net_avg_r": 0.209, - "net_total_r": 226.56, - "best_r": 8.85, - "worst_r": -3.38, - "avg_hold_days": 12.0, - "net_r_per_day": 0.0174, - "median_net_r": -1.031, - "profit_factor": 1.34, - "net_avg_r_ex_top5": 0.049 - }, - "overall_all": { - "total": 202765, - "wins": 82220, - "losses": 113809, - "expired": 6736, - "hit_rate": 41.9, - "avg_r": -0.04, - "total_r": -8186.18, - "net_avg_r": -0.095, - "net_total_r": -19184.55, - "best_r": 9.24, - "worst_r": -16.42, - "avg_hold_days": 8.2, - "net_r_per_day": -0.0115, - "median_net_r": -1.035, - "profit_factor": 0.85, - "net_avg_r_ex_top5": -0.22 - }, - "by_direction": { - "long": { - "total": 1086, - "wins": 379, - "losses": 591, - "expired": 116, - "hit_rate": 39.1, - "avg_r": 0.255, - "total_r": 276.76, - "net_avg_r": 0.209, - "net_total_r": 226.56, - "best_r": 8.85, - "worst_r": -3.38, - "avg_hold_days": 12.0, - "net_r_per_day": 0.0174, - "median_net_r": -1.031, - "profit_factor": 1.34, - "net_avg_r_ex_top5": 0.049 - }, - "short": { - "total": 0, - "wins": 0, - "losses": 0, - "expired": 0, - "hit_rate": null, - "avg_r": null, - "total_r": null, - "net_avg_r": null, - "net_total_r": null, - "best_r": null, - "worst_r": null, - "avg_hold_days": null, - "net_r_per_day": null, - "median_net_r": null, - "profit_factor": null, - "net_avg_r_ex_top5": null - } - }, - "min_momentum_percentile": 80.0, - "sweep": [ - { - "min_momentum_percentile": 90.0, - "total": 497, - "wins": 177, - "losses": 269, - "expired": 51, - "hit_rate": 39.7, - "avg_r": 0.276, - "total_r": 137.05, - "net_avg_r": 0.235, - "net_total_r": 116.55, - "best_r": 8.85, - "worst_r": -3.38, - "avg_hold_days": 11.8, - "net_r_per_day": 0.0199, - "median_net_r": -1.026, - "profit_factor": 1.39, - "net_avg_r_ex_top5": 0.071 - }, - { - "min_momentum_percentile": 80.0, - "total": 1086, - "wins": 379, - "losses": 591, - "expired": 116, - "hit_rate": 39.1, - "avg_r": 0.255, - "total_r": 276.76, - "net_avg_r": 0.209, - "net_total_r": 226.56, - "best_r": 8.85, - "worst_r": -3.38, - "avg_hold_days": 12.0, - "net_r_per_day": 0.0174, - "median_net_r": -1.031, - "profit_factor": 1.34, - "net_avg_r_ex_top5": 0.049 - }, - { - "min_momentum_percentile": 70.0, - "total": 1841, - "wins": 597, - "losses": 1062, - "expired": 182, - "hit_rate": 36.0, - "avg_r": 0.152, - "total_r": 280.26, - "net_avg_r": 0.104, - "net_total_r": 190.75, - "best_r": 8.85, - "worst_r": -4.21, - "avg_hold_days": 11.8, - "net_r_per_day": 0.0088, - "median_net_r": -1.037, - "profit_factor": 1.16, - "net_avg_r_ex_top5": -0.055 - }, - { - "min_momentum_percentile": 60.0, - "total": 2772, - "wins": 873, - "losses": 1611, - "expired": 288, - "hit_rate": 35.1, - "avg_r": 0.126, - "total_r": 348.07, - "net_avg_r": 0.075, - "net_total_r": 209.25, - "best_r": 8.85, - "worst_r": -4.21, - "avg_hold_days": 12.0, - "net_r_per_day": 0.0063, - "median_net_r": -1.04, - "profit_factor": 1.12, - "net_avg_r_ex_top5": -0.077 - }, - { - "min_momentum_percentile": 50.0, - "total": 3901, - "wins": 1182, - "losses": 2295, - "expired": 424, - "hit_rate": 34.0, - "avg_r": 0.089, - "total_r": 345.37, - "net_avg_r": 0.038, - "net_total_r": 146.96, - "best_r": 8.85, - "worst_r": -4.86, - "avg_hold_days": 12.1, - "net_r_per_day": 0.0031, - "median_net_r": -1.042, - "profit_factor": 1.06, - "net_avg_r_ex_top5": -0.114 - }, - { - "min_momentum_percentile": 0.0, - "total": 14588, - "wins": 3719, - "losses": 9271, - "expired": 1598, - "hit_rate": 28.6, - "avg_r": -0.065, - "total_r": -952.08, - "net_avg_r": -0.115, - "net_total_r": -1676.86, - "best_r": 9.24, - "worst_r": -15.94, - "avg_hold_days": 12.1, - "net_r_per_day": -0.0095, - "median_net_r": -1.043, - "profit_factor": 0.84, - "net_avg_r_ex_top5": -0.275 - } - ], - "gate_ablation": [ - { - "variant": "all_floors", - "total": 1086, - "wins": 379, - "losses": 591, - "expired": 116, - "hit_rate": 39.1, - "avg_r": 0.255, - "total_r": 276.76, - "net_avg_r": 0.209, - "net_total_r": 226.56, - "best_r": 8.85, - "worst_r": -3.38, - "avg_hold_days": 12.0, - "net_r_per_day": 0.0174, - "median_net_r": -1.031, - "profit_factor": 1.34, - "net_avg_r_ex_top5": 0.049, - "hold_days": 30, - "hold_avg_r": 0.631, - "hold_net_avg_r": 0.585, - "hold_total_r": 684.97 - }, - { - "variant": "no_confidence_floor", - "total": 1093, - "wins": 380, - "losses": 596, - "expired": 117, - "hit_rate": 38.9, - "avg_r": 0.25, - "total_r": 273.55, - "net_avg_r": 0.204, - "net_total_r": 222.99, - "best_r": 8.85, - "worst_r": -3.38, - "avg_hold_days": 11.9, - "net_r_per_day": 0.0171, - "median_net_r": -1.031, - "profit_factor": 1.33, - "net_avg_r_ex_top5": 0.045, - "hold_days": 30, - "hold_avg_r": 0.626, - "hold_net_avg_r": 0.58, - "hold_total_r": 684.08 - }, - { - "variant": "no_rr_floor", - "total": 6849, - "wins": 3235, - "losses": 3409, - "expired": 205, - "hit_rate": 48.7, - "avg_r": 0.112, - "total_r": 770.17, - "net_avg_r": 0.061, - "net_total_r": 418.96, - "best_r": 8.85, - "worst_r": -5.16, - "avg_hold_days": 7.9, - "net_r_per_day": 0.0078, - "median_net_r": -0.061, - "profit_factor": 1.11, - "net_avg_r_ex_top5": -0.061, - "hold_days": 30, - "hold_avg_r": 0.354, - "hold_net_avg_r": 0.303, - "hold_total_r": 2425.86 - }, - { - "variant": "no_neutral_exclusion", - "total": 2313, - "wins": 770, - "losses": 1279, - "expired": 264, - "hit_rate": 37.6, - "avg_r": 0.2, - "total_r": 462.35, - "net_avg_r": 0.154, - "net_total_r": 355.89, - "best_r": 8.85, - "worst_r": -3.38, - "avg_hold_days": 12.6, - "net_r_per_day": 0.0122, - "median_net_r": -1.031, - "profit_factor": 1.25, - "net_avg_r_ex_top5": 0.004, - "hold_days": 30, - "hold_avg_r": 0.583, - "hold_net_avg_r": 0.537, - "hold_total_r": 1348.89 - }, - { - "variant": "momentum_only", - "total": 14696, - "wins": 6827, - "losses": 7359, - "expired": 510, - "hit_rate": 48.1, - "avg_r": 0.114, - "total_r": 1669.64, - "net_avg_r": 0.064, - "net_total_r": 936.93, - "best_r": 8.85, - "worst_r": -5.71, - "avg_hold_days": 8.4, - "net_r_per_day": 0.0076, - "median_net_r": -1.013, - "profit_factor": 1.11, - "net_avg_r_ex_top5": -0.055, - "hold_days": 30, - "hold_avg_r": 0.395, - "hold_net_avg_r": 0.345, - "hold_total_r": 5798.82 - } - ], - "gate_ablation_note": "Each row re-qualifies the same candidates at the current momentum cutoff (80) with one floor removed (long-only while the momentum gate is active). If dropping a floor doesn't hurt net expectancy, that floor isn't pulling its weight. The Hold columns grade the same variants under the hold-to-horizon time exit instead of the S/R target \u2014 the view that matters if the exit policy moves to a fixed hold.", - "time_exit_sweep": [ - { - "hold_days": 5, - "total": 1086, - "wins": 603, - "win_rate": 55.5, - "avg_r": 0.175, - "total_r": 190.16, - "net_avg_r": 0.129, - "net_total_r": 139.97, - "best_r": 5.09, - "worst_r": -2.51, - "avg_hold_days": 4.5, - "net_r_per_day": 0.0285, - "median_net_r": 0.115, - "profit_factor": 1.36, - "net_avg_r_ex_top5": -0.002 - }, - { - "hold_days": 10, - "total": 1086, - "wins": 559, - "win_rate": 51.5, - "avg_r": 0.357, - "total_r": 387.9, - "net_avg_r": 0.311, - "net_total_r": 337.7, - "best_r": 6.73, - "worst_r": -2.51, - "avg_hold_days": 7.9, - "net_r_per_day": 0.0395, - "median_net_r": 0.031, - "profit_factor": 1.67, - "net_avg_r_ex_top5": 0.112 - }, - { - "hold_days": 21, - "total": 1086, - "wins": 487, - "win_rate": 44.8, - "avg_r": 0.525, - "total_r": 570.33, - "net_avg_r": 0.479, - "net_total_r": 520.14, - "best_r": 9.86, - "worst_r": -3.38, - "avg_hold_days": 13.7, - "net_r_per_day": 0.0349, - "median_net_r": -1.027, - "profit_factor": 1.81, - "net_avg_r_ex_top5": 0.191 - }, - { - "hold_days": 30, - "total": 1086, - "wins": 434, - "win_rate": 40.0, - "avg_r": 0.631, - "total_r": 684.97, - "net_avg_r": 0.585, - "net_total_r": 634.78, - "best_r": 12.87, - "worst_r": -3.38, - "avg_hold_days": 17.8, - "net_r_per_day": 0.0329, - "median_net_r": -1.033, - "profit_factor": 1.9, - "net_avg_r_ex_top5": 0.212 - } - ], - "portfolio_sim": { - "params": { - "starting_capital": 10000.0, - "max_positions": 10, - "risk_per_trade_pct": 1.0, - "notional_cap_pct": 20.0, - "cost_per_side_pct": 0.1, - "hold_days": 30 - }, - "policies": [], - "note": "One capital-constrained book over the same qualified setups the tables above grade per-setup: at most 10 concurrent positions (one per ticker), best momentum first, fixed-fractional risk sizing with a no-leverage cap, entries at the detection close, stops filled at the worse of stop or open. 'target' races the S/R target against the stop (timeout at the horizon); 'hold' keeps the initial stop and exits at the horizon close. SPY return is price-only over the same window. In-sample; no dividends." - }, - "strategy_variants": { - "variants": [], - "note": "Research-only hold-to-horizon portfolio variants. Production now uses residual 12-1 momentum at cutoff 80; the remaining rows compare the legacy raw rank, raw cutoff 90, one max-15 capacity check, and volatility overlays." - }, - "exit_policy_variants": { - "variants": [], - "note": "Research-only exit policies over the residual/high-vol 80/20 entry candidate. Every row uses the same entry qualification/ranking and changes only the exit discipline." - }, - "portfolio_monitor": null, - "production_cadence_comparison": null, - "holdout": null, - "min_rr_sweep": null, - "target_model_diagnostics": { - "target_model": "production_gtl", - "target_model_label": "Live GTL (production)", - "candidate_count": 202765, - "primary_source_counts": { - "pivot_point": 196290, - "range_grid": 180036 - }, - "primary_round_only": 0, - "primary_strength_100": 138596, - "avg_primary_strength": 80.109, - "avg_primary_distance_atr": 2.293, - "avg_primary_rejection_count": 41.908, - "avg_raw_level_count": 53.204, - "avg_gate_level_count": 53.204 - }, - "signal_eval": [ - { - "signal": "vol_6m", - "weeks": 39, - "avg_cross_section": 498.2, - "mean_ic": 0.0609, - "ic_t_stat": 1.48, - "ic_positive_pct": 64.1, - "mean_quintile_spread": 0.0337, - "reliable": true - }, - { - "signal": "mom_12_1_resid", - "weeks": 35, - "avg_cross_section": 497.7, - "mean_ic": 0.0552, - "ic_t_stat": 1.98, - "ic_positive_pct": 60.0, - "mean_quintile_spread": 0.0207, - "reliable": true - }, - { - "signal": "mom_12_1", - "weeks": 35, - "avg_cross_section": 497.7, - "mean_ic": 0.0531, - "ic_t_stat": 1.61, - "ic_positive_pct": 65.7, - "mean_quintile_spread": 0.0206, - "reliable": true - }, - { - "signal": "trend_200", - "weeks": 37, - "avg_cross_section": 497.9, - "mean_ic": 0.0161, - "ic_t_stat": 0.44, - "ic_positive_pct": 59.5, - "mean_quintile_spread": 0.006, - "reliable": true - }, - { - "signal": "reversal_1m", - "weeks": 43, - "avg_cross_section": 498.7, - "mean_ic": 0.0059, - "ic_t_stat": 0.22, - "ic_positive_pct": 53.5, - "mean_quintile_spread": 0.0053, - "reliable": true - }, - { - "signal": "mom_6_1", - "weeks": 39, - "avg_cross_section": 498.2, - "mean_ic": 0.0051, - "ic_t_stat": 0.21, - "ic_positive_pct": 56.4, - "mean_quintile_spread": 0.0087, - "reliable": true - }, - { - "signal": "mom_3_1", - "weeks": 42, - "avg_cross_section": 498.5, - "mean_ic": -0.0064, - "ic_t_stat": -0.25, - "ic_positive_pct": 50.0, - "mean_quintile_spread": 0.0046, - "reliable": true - }, - { - "signal": "high_52w", - "weeks": 35, - "avg_cross_section": 497.7, - "mean_ic": -0.0086, - "ic_t_stat": -0.26, - "ic_positive_pct": 54.3, - "mean_quintile_spread": -0.0088, - "reliable": true - }, - { - "signal": "fip_id", - "weeks": 35, - "avg_cross_section": 497.7, - "mean_ic": -0.045, - "ic_t_stat": -2.91, - "ic_positive_pct": 25.7, - "mean_quintile_spread": -0.0168, - "reliable": true - } - ], - "signal_eval_note": "Cross-sectional rank-IC of price-only signals vs the forward 30-day return (min 20 names/window). |IC| \u2273 0.03 with a consistent sign is a real (if small) edge; near 0 means ranking on it sorts nothing. Momentum factors and high_52w are expected positive; reversal_1m and vol_6m expected negative (mean-reversion / low-vol anomaly). IC is measured on non-overlapping windows; signals with fewer than 12 independent windows are flagged unreliable (too few regimes \u2014 deepen history with the Data Backfill job).", - "note": "Sentiment & fundamentals held neutral (no point-in-time history). Stops fill at the worse of the stop or the bar's open (gaps through the stop are modeled, so a loss can exceed \u22121R); targets never fill better than their level. ~6 months \u2248 one market regime \u2014 treat as directional, not gospel.", - "recommendation": { - "headline": "Trade the qualified list long-only; hold 30 trading days with the initial ATR stop.", - "items": [ - { - "topic": "exit", - "text": "Legacy exit diagnostic: hold 30 trading days with the initial stop (+0.58R net/trade vs +0.21R for the S/R target exit)." - }, - { - "topic": "gate", - "text": "Gate: the confidence floor adds nothing \u2014 dropping it costs +0.01R/trade and adds 7 trades." - }, - { - "topic": "gate", - "text": "Gate: keep the R:R floor (worth +0.28R/trade under the hold exit)." - }, - { - "topic": "gate", - "text": "Gate: keep the NEUTRAL exclusion (worth +0.05R/trade under the hold exit)." - }, - { - "topic": "cutoff", - "text": "Residual-momentum cutoff: 90 has the best per-trade net (+0.23R over 497 setups)." - }, - { - "topic": "robustness", - "text": "Robustness: expectancy survives removing the top 5% of winners (+0.21R net/trade under the recommended 30d hold) \u2014 the edge is not a handful of outliers." - } - ], - "note": "Derived from this report's numbers on every run \u2014 the advice flips if the data does." - }, - "research_recommendation": { - "items": [], - "note": "Strategy variants unavailable; re-run the backtest after benchmark data is present." - } -} \ No newline at end of file diff --git a/reports/fip-breadth-20260718-194828.json b/reports/fip-breadth-20260718-194828.json deleted file mode 100644 index c6c1163..0000000 --- a/reports/fip-breadth-20260718-194828.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "generated_at": "2026-07-18T19:48:28.127710", - "liquid_breadth_top_n": 1500, - "liquid_min_price": 5.0, - "fingerprint": { - "signal": "fip_id", - "weeks": 35, - "avg_cross_section": 497.7, - "mean_ic": -0.045, - "ic_t_stat": -2.91, - "ic_positive_pct": 25.7, - "mean_quintile_spread": -0.0168, - "reliable": true, - "pass": true, - "expected_ic": -0.045, - "expected_t": -2.9 - }, - "breadth": null, - "verdict": null, - "fingerprint_report_path": "reports\\fip-breadth-20260718-194828-fingerprint.json" -} \ No newline at end of file diff --git a/scripts/extend_snapshot_universe.py b/scripts/extend_snapshot_universe.py deleted file mode 100644 index e3585b8..0000000 --- a/scripts/extend_snapshot_universe.py +++ /dev/null @@ -1,329 +0,0 @@ -"""Extend a *copy* of the production backtest snapshot with broad-universe OHLCV. - -Research only — never writes to production Postgres. - -Pipeline --------- -1. Copy ``--source`` snapshot (default ``backtest_snapshots/prod.sqlite``) to - ``--output`` (default ``backtest_snapshots/research.sqlite``). -2. Resolve symbol pool = nasdaq_all ∪ sp500 via ``ticker_universe_service``. -3. Fetch ~5y daily bars from Alpaca for symbols missing (or short) in the copy. -4. Insert new tickers + OHLCV; mark them in side table ``research_rank_only`` - so the harness can feed signal IC without GTL/candidate replay. - -Resume-friendly: re-running skips symbols that already have ≥ ``--min-bars``. - -Example -------- - python scripts/extend_snapshot_universe.py \\ - --source backtest_snapshots/prod.sqlite \\ - --output backtest_snapshots/research.sqlite \\ - --force-copy - - # smoke: first 50 missing symbols only - python scripts/extend_snapshot_universe.py --limit 50 -""" - -from __future__ import annotations - -import argparse -import asyncio -import shutil -import sys -import time -from datetime import date, datetime, timedelta, timezone -from pathlib import Path - -from sqlalchemy import create_engine, select, text -from sqlalchemy.orm import Session - -ROOT = Path(__file__).resolve().parents[1] -if str(ROOT) not in sys.path: - sys.path.insert(0, str(ROOT)) - - -def _parse_args() -> argparse.Namespace: - p = argparse.ArgumentParser(description=__doc__) - p.add_argument( - "--source", - default="backtest_snapshots/prod.sqlite", - help="Existing prod snapshot to copy (read-only after copy).", - ) - p.add_argument( - "--output", - default="backtest_snapshots/research.sqlite", - help="Research snapshot path (created/updated).", - ) - p.add_argument( - "--force-copy", - action="store_true", - help="Overwrite output by re-copying from source first.", - ) - p.add_argument( - "--history-days", - type=int, - default=1825, - help="OHLCV lookback days (~5y). Default 1825.", - ) - p.add_argument( - "--min-bars", - type=int, - default=260, - help="Skip re-fetch when a symbol already has this many bars.", - ) - p.add_argument( - "--limit", - type=int, - default=None, - help="Max *new* symbols to fetch (smoke tests).", - ) - p.add_argument( - "--sleep", - type=float, - default=0.15, - help="Seconds between Alpaca symbol requests (rate-limit cushion).", - ) - p.add_argument( - "--max-retries", - type=int, - default=5, - help="Retries per symbol on RateLimitError.", - ) - p.add_argument("--quiet", action="store_true") - return p.parse_args() - - -def _ensure_rank_only_table(conn) -> None: - conn.execute( - text( - """ - CREATE TABLE IF NOT EXISTS research_rank_only ( - ticker_id INTEGER PRIMARY KEY, - symbol TEXT NOT NULL UNIQUE - ) - """ - ) - ) - conn.commit() - - -async def _resolve_pool() -> tuple[list[str], dict[str, str]]: - """Return sorted unique symbols and source labels.""" - from app.database import async_session_factory - from app.services.ticker_universe_service import fetch_universe_symbols - - sources: dict[str, str] = {} - symbols: set[str] = set() - # Need a DB session for cache writes; use local async engine if configured, - # but public/FMP fetch works with any session. Prefer a throwaway sqlite. - from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine - from sqlalchemy.ext.asyncio import AsyncSession - - engine = create_async_engine("sqlite+aiosqlite:///:memory:") - Session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) - try: - async with Session() as db: - for universe in ("nasdaq_all", "sp500"): - try: - syms, src = await fetch_universe_symbols(db, universe) - except Exception as exc: - print(f"WARNING: universe {universe} failed: {exc}") - continue - sources[universe] = src - symbols.update(syms) - print(f" {universe}: {len(syms)} symbols (source={src})") - finally: - await engine.dispose() - return sorted(symbols), sources - - -async def _fetch_symbol_bars( - provider, - symbol: str, - start: date, - end: date, - *, - max_retries: int, - sleep_s: float, -) -> list: - from app.exceptions import ProviderError, RateLimitError - - for attempt in range(max_retries): - try: - bars = await provider.fetch_ohlcv(symbol, start, end) - if sleep_s > 0: - await asyncio.sleep(sleep_s) - return bars - except RateLimitError: - wait = min(60.0, 2.0 ** attempt) - print(f" rate limited on {symbol}; sleep {wait:.0f}s") - await asyncio.sleep(wait) - except ProviderError as exc: - if attempt + 1 >= max_retries: - raise - await asyncio.sleep(1.0) - _ = exc - return [] - - -async def _main() -> None: - args = _parse_args() - source = Path(args.source) - output = Path(args.output) - if not source.exists(): - raise SystemExit(f"Source snapshot not found: {source}") - - if args.force_copy or not output.exists(): - output.parent.mkdir(parents=True, exist_ok=True) - if output.exists(): - output.unlink() - print(f"Copying {source} → {output}") - shutil.copy2(source, output) - else: - print(f"Updating existing research snapshot: {output}") - - from app.config import settings - from app.models.ohlcv import OHLCVRecord - from app.models.ticker import Ticker - from app.providers.alpaca import AlpacaOHLCVProvider - - if not settings.alpaca_api_key or not settings.alpaca_api_secret: - raise SystemExit("ALPACA_API_KEY / ALPACA_API_SECRET required in .env") - - provider = AlpacaOHLCVProvider(settings.alpaca_api_key, settings.alpaca_api_secret) - end = date.today() - start = end - timedelta(days=int(args.history_days)) - - print("Resolving universe pool (nasdaq_all ∪ sp500)…") - pool, sources = await _resolve_pool() - print(f"Pool size: {len(pool)} (sources={sources})") - - # Sync sqlite via sqlalchemy core (simpler than async for bulk insert) - engine = create_engine(f"sqlite:///{output.resolve().as_posix()}") - with Session(engine) as session: - _ensure_rank_only_table(session.connection()) - existing = { - row.symbol: row - for row in session.execute(select(Ticker)).scalars().all() - } - prod_symbols = set(existing) - - # Bar counts - bar_counts: dict[str, int] = {} - for sym, ticker in existing.items(): - n = session.execute( - text("SELECT COUNT(*) FROM ohlcv_records WHERE ticker_id = :tid"), - {"tid": ticker.id}, - ).scalar_one() - bar_counts[sym] = int(n) - - to_fetch: list[str] = [] - for sym in pool: - if sym in existing and bar_counts.get(sym, 0) >= args.min_bars: - # Existing production or previously extended — keep rank_only - # only for *new* research names, not original prod universe. - continue - to_fetch.append(sym) - - if args.limit is not None: - to_fetch = to_fetch[: max(0, int(args.limit))] - - print(f"Symbols to fetch/extend: {len(to_fetch)}") - ok = 0 - fail = 0 - t0 = time.monotonic() - for index, sym in enumerate(to_fetch, 1): - try: - bars = await _fetch_symbol_bars( - provider, - sym, - start, - end, - max_retries=args.max_retries, - sleep_s=args.sleep, - ) - except Exception as exc: - fail += 1 - if not args.quiet: - print(f" [{index}/{len(to_fetch)}] {sym} FAIL {exc}") - continue - - if not bars: - fail += 1 - if not args.quiet: - print(f" [{index}/{len(to_fetch)}] {sym} empty") - continue - - ticker = existing.get(sym) - is_new = ticker is None - if ticker is None: - ticker = Ticker(symbol=sym, name=None, created_at=datetime.now(timezone.utc)) - session.add(ticker) - session.flush() - existing[sym] = ticker - - # Upsert bars (delete+insert range for simplicity on research path) - session.execute( - text( - "DELETE FROM ohlcv_records WHERE ticker_id = :tid " - "AND date >= :start AND date <= :end" - ), - {"tid": ticker.id, "start": start.isoformat(), "end": end.isoformat()}, - ) - now = datetime.utcnow() - session.bulk_insert_mappings( - OHLCVRecord, - [ - { - "ticker_id": ticker.id, - "date": b.date, - "open": b.open, - "high": b.high, - "low": b.low, - "close": b.close, - "volume": b.volume, - "created_at": now, - } - for b in bars - ], - ) - - # rank_only only for names that were NOT in the original production - # snapshot at copy time (or are newly introduced to this research DB). - if is_new or sym not in prod_symbols: - # Re-evaluate: if source copy already had the symbol, don't flag. - # Only new inserts get rank_only. - if is_new: - session.execute( - text( - "INSERT OR REPLACE INTO research_rank_only " - "(ticker_id, symbol) VALUES (:tid, :sym)" - ), - {"tid": ticker.id, "sym": sym}, - ) - - session.commit() - ok += 1 - if not args.quiet and (index % 25 == 0 or index == len(to_fetch)): - elapsed = time.monotonic() - t0 - print( - f" progress {index}/{len(to_fetch)} ok={ok} fail={fail} " - f"elapsed={elapsed/60:.1f}m last={sym} bars={len(bars)}" - ) - - rank_only_n = session.execute( - text("SELECT COUNT(*) FROM research_rank_only") - ).scalar_one() - ticker_n = session.execute(text("SELECT COUNT(*) FROM tickers")).scalar_one() - ohlcv_n = session.execute(text("SELECT COUNT(*) FROM ohlcv_records")).scalar_one() - - print("Done.") - print(f" output: {output}") - print(f" tickers: {ticker_n}") - print(f" ohlcv rows: {ohlcv_n}") - print(f" research_rank_only: {rank_only_n}") - print(f" fetched ok/fail: {ok}/{fail}") - - -if __name__ == "__main__": - asyncio.run(_main()) diff --git a/scripts/run_fip_breadth_research.py b/scripts/run_fip_breadth_research.py deleted file mode 100644 index cc9f6e9..0000000 --- a/scripts/run_fip_breadth_research.py +++ /dev/null @@ -1,299 +0,0 @@ -"""Phase B: fip_id IC on liquid-breadth cross-section (local research only). - -1. Fingerprint check on the unextended prod snapshot (must ≈ IC −0.045 / t −2.9). -2. Run signal_eval on research.sqlite with BACKTEST_LIQUID_BREADTH=1500 PIT mask. -3. Write a research report under docs/research/ and reports/. - -Does not modify production DB, gate, scanner, or schedule. - -Example -------- - # After extend_snapshot_universe.py has built research.sqlite: - python scripts/run_fip_breadth_research.py \\ - --prod-snapshot backtest_snapshots/prod.sqlite \\ - --research-snapshot backtest_snapshots/research.sqlite \\ - --workers 6 --allow-spawn -""" - -from __future__ import annotations - -import argparse -import asyncio -import json -import os -import sys -from datetime import datetime -from pathlib import Path - -from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine - -ROOT = Path(__file__).resolve().parents[1] -if str(ROOT) not in sys.path: - sys.path.insert(0, str(ROOT)) - -FINGERPRINT_IC = -0.045 -FINGERPRINT_T = -2.9 -FINGERPRINT_IC_TOL = 0.015 -FINGERPRINT_T_TOL = 0.6 - - -def _sqlite_url(path: Path) -> str: - return f"sqlite+aiosqlite:///{path.resolve().as_posix()}" - - -def _parse_args() -> argparse.Namespace: - p = argparse.ArgumentParser(description=__doc__) - p.add_argument("--prod-snapshot", default="backtest_snapshots/prod.sqlite") - p.add_argument("--research-snapshot", default="backtest_snapshots/research.sqlite") - p.add_argument("--workers", type=int, default=6) - p.add_argument("--allow-spawn", action="store_true") - p.add_argument("--skip-fingerprint", action="store_true") - p.add_argument("--skip-research", action="store_true") - p.add_argument("--liquid-breadth", type=int, default=1500) - p.add_argument("--min-price", type=float, default=5.0) - p.add_argument( - "--out", - default=None, - help="JSON report path (default reports/fip-breadth-YYYYMMDD.json)", - ) - p.add_argument("--quiet", action="store_true") - return p.parse_args() - - -def _find_fip(signal_eval: list[dict]) -> dict | None: - for row in signal_eval or []: - if row.get("signal") == "fip_id": - return row - return None - - -def _verdict(row: dict | None) -> dict: - if row is None: - return { - "green": False, - "reason": "fip_id missing from signal_eval", - } - mean_ic = row.get("mean_ic") - t_stat = row.get("ic_t_stat") - reliable = bool(row.get("reliable")) - if mean_ic is None or t_stat is None: - return {"green": False, "reason": "missing mean_ic or ic_t_stat", "row": row} - sign_ok = mean_ic < 0 - mag_ok = abs(float(mean_ic)) >= 0.03 - green = sign_ok and mag_ok and reliable - return { - "green": green, - "reason": ( - "iron rule cleared — follow-up proposal only, not production wire-in" - if green - else "iron rule not met on liquid-breadth cross-section" - ), - "checks": { - "mean_ic": mean_ic, - "abs_mean_ic_ge_0_03": mag_ok, - "sign_negative": sign_ok, - "ic_t_stat": t_stat, - "reliable": reliable, - "weeks": row.get("weeks"), - "avg_cross_section": row.get("avg_cross_section"), - }, - "row": row, - } - - -async def _run_signal_eval(snapshot: Path, *, workers: int, quiet: bool) -> dict: - from app.config import settings - from app.services.backtest_service import run_backtest - - settings.backtest_workers = workers - engine = create_async_engine(_sqlite_url(snapshot), pool_pre_ping=True) - Session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) - - def progress(done: int, total: int, symbol: str) -> None: - if quiet: - return - print(f" progress {done}/{total} {symbol}", end="\r") - - try: - async with Session() as db: - report = await run_backtest(db, progress_cb=progress, cadence="weekly") - finally: - await engine.dispose() - if not quiet: - print() - return report - - -def _write_md(path: Path, payload: dict) -> None: - fp = payload.get("fingerprint") or {} - br = payload.get("breadth") or {} - v = payload.get("verdict") or {} - lines = [ - "# Broad-universe fip_id IC research (Phase B)", - "", - f"Generated: {payload.get('generated_at')}", - "", - "## Scope", - "", - "- **Research only** — production universe, gate, scanner, schedule unchanged.", - "- Price-only signal harness; no sentiment/fundamentals on the broad tier.", - "- Point-in-time liquidity mask: top " - f"**{payload.get('liquid_breadth_top_n')}** by 63d median $vol, " - f"price ≥ **${payload.get('liquid_min_price')}** at as-of.", - "", - "## Caveats", - "", - "- **Survivorship bias**: today's constituents backfilled historically " - "(worse in small caps).", - "- **IEX volume undercount**: relative $vol rank only, not absolute floors.", - "- **Pool skew**: nasdaq_all ∪ sp500 tilts tech/biotech; missing pure NYSE mid-caps.", - "", - "## Fingerprint (505-name prod snapshot)", - "", - f"- Expected: IC ≈ {FINGERPRINT_IC}, t ≈ {FINGERPRINT_T}", - f"- Observed: IC = {fp.get('mean_ic')}, t = {fp.get('ic_t_stat')}, " - f"weeks = {fp.get('weeks')}, reliable = {fp.get('reliable')}", - f"- Pass: **{fp.get('pass')}**", - "", - "## Liquid-breadth signal_eval (fip_id)", - "", - ] - row = br.get("row") or br - if row: - lines.extend([ - f"| metric | value |", - f"|---|---|", - f"| mean_ic | {row.get('mean_ic')} |", - f"| ic_t_stat | {row.get('ic_t_stat')} |", - f"| ic_positive_pct | {row.get('ic_positive_pct')} |", - f"| weeks | {row.get('weeks')} |", - f"| avg_cross_section | {row.get('avg_cross_section')} |", - f"| reliable | {row.get('reliable')} |", - f"| mean_quintile_spread | {row.get('mean_quintile_spread')} |", - "", - ]) - else: - lines.append("_No breadth result (run skipped or failed)._") - lines.append("") - lines.extend([ - "## Verdict (iron rule)", - "", - f"- **Green: {v.get('green')}**", - f"- {v.get('reason')}", - f"- Checks: `{json.dumps(v.get('checks') or {}, default=str)}`", - "", - "A green verdict authorizes a **follow-up proposal** only " - "(two-tier universe / gate revalidation) — **not** production wire-in.", - "", - "## Artifacts", - "", - f"- Fingerprint report: `{payload.get('fingerprint_report_path')}`", - f"- Breadth report: `{payload.get('breadth_report_path')}`", - "", - ]) - path.write_text("\n".join(lines), encoding="utf-8") - - -async def _main() -> None: - args = _parse_args() - prod = Path(args.prod_snapshot) - research = Path(args.research_snapshot) - if not prod.exists(): - raise SystemExit(f"Prod snapshot missing: {prod}") - - os.environ["BACKTEST_SNAPSHOT_OFFLINE"] = "1" - if args.allow_spawn: - os.environ["BACKTEST_ALLOW_SPAWN"] = "1" - os.environ["BACKTEST_SIGNAL_EVAL_ONLY"] = "1" - - stamp = datetime.now().strftime("%Y%m%d-%H%M%S") - out_json = Path(args.out) if args.out else Path("reports") / f"fip-breadth-{stamp}.json" - out_json.parent.mkdir(parents=True, exist_ok=True) - out_md = Path("docs/research") / "fip-breadth-ic.md" - - payload: dict = { - "generated_at": datetime.now().isoformat(), - "liquid_breadth_top_n": args.liquid_breadth, - "liquid_min_price": args.min_price, - "fingerprint": None, - "breadth": None, - "verdict": None, - } - - # --- 1) Fingerprint --- - if not args.skip_fingerprint: - # Clear liquid breadth for fingerprint - os.environ.pop("BACKTEST_LIQUID_BREADTH", None) - os.environ.pop("BACKTEST_LIQUID_MIN_PRICE", None) - if not args.quiet: - print(f"Fingerprint run on {prod}…") - fp_report = await _run_signal_eval(prod, workers=args.workers, quiet=args.quiet) - fp_path = out_json.with_name(out_json.stem + "-fingerprint.json") - fp_path.write_text(json.dumps(fp_report, indent=2, default=str), encoding="utf-8") - fip = _find_fip(fp_report.get("signal_eval") or []) - if fip is None: - raise SystemExit("ABORT: fip_id missing from fingerprint signal_eval") - ic_ok = abs(float(fip["mean_ic"]) - FINGERPRINT_IC) <= FINGERPRINT_IC_TOL - t_ok = abs(float(fip["ic_t_stat"]) - FINGERPRINT_T) <= FINGERPRINT_T_TOL - passed = ic_ok and t_ok and bool(fip.get("reliable")) - payload["fingerprint"] = { - **fip, - "pass": passed, - "expected_ic": FINGERPRINT_IC, - "expected_t": FINGERPRINT_T, - } - payload["fingerprint_report_path"] = str(fp_path) - if not args.quiet: - print( - f"Fingerprint fip_id IC={fip.get('mean_ic')} t={fip.get('ic_t_stat')} " - f"pass={passed}" - ) - if not passed: - out_json.write_text(json.dumps(payload, indent=2, default=str), encoding="utf-8") - raise SystemExit( - "ABORT: fingerprint mismatch — investigate before trusting breadth runs " - f"(got IC={fip.get('mean_ic')} t={fip.get('ic_t_stat')})" - ) - - # --- 2) Breadth --- - if not args.skip_research: - if not research.exists(): - raise SystemExit( - f"Research snapshot missing: {research}\n" - "Build it with: python scripts/extend_snapshot_universe.py" - ) - os.environ["BACKTEST_LIQUID_BREADTH"] = str(int(args.liquid_breadth)) - os.environ["BACKTEST_LIQUID_MIN_PRICE"] = str(float(args.min_price)) - if not args.quiet: - print( - f"Breadth run on {research} " - f"(top {args.liquid_breadth}, min_price={args.min_price})…" - ) - br_report = await _run_signal_eval( - research, workers=args.workers, quiet=args.quiet - ) - br_path = out_json.with_name(out_json.stem + "-breadth.json") - br_path.write_text(json.dumps(br_report, indent=2, default=str), encoding="utf-8") - fip_b = _find_fip(br_report.get("signal_eval") or []) - payload["breadth"] = fip_b or {"error": "fip_id missing"} - payload["breadth_report_path"] = str(br_path) - payload["breadth_tickers"] = br_report.get("tickers") - payload["breadth_rank_only_tickers"] = br_report.get("rank_only_tickers") - payload["verdict"] = _verdict(fip_b) - if not args.quiet: - print( - f"Breadth fip_id IC={ (fip_b or {}).get('mean_ic') } " - f"t={ (fip_b or {}).get('ic_t_stat') } " - f"green={payload['verdict'].get('green')}" - ) - - out_json.write_text(json.dumps(payload, indent=2, default=str), encoding="utf-8") - out_md.parent.mkdir(parents=True, exist_ok=True) - _write_md(out_md, payload) - if not args.quiet: - print(f"Wrote {out_json}") - print(f"Wrote {out_md}") - - -if __name__ == "__main__": - asyncio.run(_main()) diff --git a/tests/unit/test_backtest_service.py b/tests/unit/test_backtest_service.py index a69eb74..aff0eb6 100644 --- a/tests/unit/test_backtest_service.py +++ b/tests/unit/test_backtest_service.py @@ -1154,52 +1154,6 @@ class TestSimulatePortfolio: assert allowed["trade_details"][0]["entry"] == pytest.approx(110.0) -def test_median_dollar_vol_63_basic(): - closes = [10.0] * 70 - volumes = [100.0 + i for i in range(70)] - med = bt._median_dollar_vol_63(closes, volumes, 69, lookback=63) - assert med is not None - assert med > 0 - - -def test_liquid_breadth_week_keeps_top_n_by_dvol(): - recs = [ - {"val": 0.1, "fwd": 0.01, "close": 20.0, "median_dvol_63": 1e6}, - {"val": 0.2, "fwd": 0.02, "close": 20.0, "median_dvol_63": 9e6}, - {"val": 0.3, "fwd": 0.03, "close": 20.0, "median_dvol_63": 5e6}, - {"val": 0.4, "fwd": 0.04, "close": 1.0, "median_dvol_63": 99e6}, # price floor - {"val": 0.5, "fwd": 0.05, "close": 20.0, "median_dvol_63": None}, - ] - pairs = bt._filter_liquid_breadth_week(recs, top_n=2, min_price=5.0) - assert len(pairs) == 2 - # Highest dvol first among eligible: 9e6 then 5e6 - assert pairs[0][0] == pytest.approx(0.2) - assert pairs[1][0] == pytest.approx(0.3) - - -def test_signal_eval_liquid_breadth_env(monkeypatch): - # top_n=5 → keep 5 names/week; spearman needs ≥3 observations. - monkeypatch.setenv("BACKTEST_LIQUID_BREADTH", "5") - monkeypatch.setenv("BACKTEST_LIQUID_MIN_PRICE", "5") - monkeypatch.setattr(bt, "MIN_CROSS_SECTION", 3) - monkeypatch.setattr(bt, "MIN_RELIABLE_PERIODS", 3) - week = {} - for w in (1, 10, 20, 30, 40, 50): - week[(2024, w)] = [ - { - "val": float(i), - "fwd": float(i) * 0.01, - "close": 10.0, - "median_dvol_63": float(100 - i), - } - for i in range(12) - ] - rows = bt._signal_evaluation({"toy": week}) - assert rows - assert rows[0]["liquid_breadth_top_n"] == 5 - assert rows[0]["avg_cross_section"] == 5.0 - - def test_fip_id_sign_convention_steady_climber_vs_jump(): # Steady climber: many up days, continuous path → lower (more negative) ID. steady = [100.0]