dennisthiessen/signal-platform
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Revert "feat: Phase B fip_id liquid-breadth research tooling"
Deploy / lint (push) Successful in 9s
Details
Deploy / test (push) Successful in 1m23s
Details
Deploy / deploy (push) Successful in 41s
Details

Browse Source 
This reverts commit 9704e0d85a.
This commit is contained in:
dennisthiessen
2026-07-18 20:24:17 +02:00
parent 9704e0d85a
commit c2c7244d1a
8 changed files with 73 additions and 1578 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/services/backtest_service.py
+71 -249
View File
@@ -30,11 +30,6 @@ Environment variables (see also run_backtest_snapshot.py):
BACKTEST_FALLBACK_CLEAR_AIR_ONLY=1 BACKTEST_FALLBACK_CLEAR_AIR_ONLY=1
BACKTEST_RESEARCH_EXITS=1 BACKTEST_RESEARCH_EXITS=1
BACKTEST_MIN_RR_SWEEP=1 BACKTEST_MIN_RR_SWEEP=1
Broad-universe signal research (local snapshots only; inert when unset):
BACKTEST_LIQUID_BREADTH=1500 # PIT top-N by 63d median $vol, price floor
BACKTEST_LIQUID_MIN_PRICE=5 # USD close floor at as-of (default 5)
BACKTEST_SIGNAL_EVAL_ONLY=1 # skip portfolio_sim / monitor (signal IC only)
""" """
from __future__ import annotations from __future__ import annotations
@@ -881,63 +876,6 @@ def _signal_values(
return out return out
def _liquid_breadth_top_n() -> int:
"""0 = off (production path). N > 0 enables PIT top-N $vol mask for signal IC."""
raw = os.getenv("BACKTEST_LIQUID_BREADTH", "").strip()
if not raw:
return 0
try:
return max(0, int(raw))
except ValueError:
return 0
def _liquid_min_price() -> float:
raw = os.getenv("BACKTEST_LIQUID_MIN_PRICE", "5").strip() or "5"
try:
return max(0.0, float(raw))
except ValueError:
return 5.0
def _signal_eval_only() -> bool:
return os.getenv("BACKTEST_SIGNAL_EVAL_ONLY", "").strip() in ("1", "true", "yes")
async def _load_research_rank_only_symbols(db: AsyncSession) -> set[str]:
"""Symbols that feed signal IC only (no GTL/candidate replay).
Optional side table ``research_rank_only`` on research snapshots. Missing
table → empty set (production path unchanged).
"""
from sqlalchemy import text
try:
result = await db.execute(text("SELECT symbol FROM research_rank_only"))
return {str(row[0]).upper() for row in result.fetchall() if row[0]}
except Exception:
return set()
def _median_dollar_vol_63(
closes: list[float], volumes: list[float], i: int, lookback: int = 63
) -> float | None:
"""Rolling median of close×volume over ``lookback`` bars ending at ``i`` (inclusive)."""
if i + 1 < lookback or lookback < 2:
return None
dvs: list[float] = []
for k in range(i - lookback + 1, i + 1):
if closes[k] > 0 and volumes[k] >= 0:
dvs.append(closes[k] * float(volumes[k]))
if len(dvs) < max(20, lookback // 2):
return None
dvs_sorted = sorted(dvs)
mid = len(dvs_sorted) // 2
if len(dvs_sorted) % 2:
return dvs_sorted[mid]
return 0.5 * (dvs_sorted[mid - 1] + dvs_sorted[mid])
def _accumulate_signal_series( def _accumulate_signal_series(
records: list, records: list,
collected: dict, collected: dict,
@@ -945,20 +883,13 @@ def _accumulate_signal_series(
) -> None: ) -> None:
"""For each weekly as-of bar, emit (signal, forward-return) pairs keyed by ISO """For each weekly as-of bar, emit (signal, forward-return) pairs keyed by ISO
week into ``collected[name][week_key]``. Forward return is close-to-close over week into ``collected[name][week_key]``. Forward return is close-to-close over
HORIZON trading days. Mutates ``collected`` (a dict of dict of list). HORIZON trading days. Mutates ``collected`` (a dict of dict of list)."""
When ``BACKTEST_LIQUID_BREADTH`` is set, observations are dicts with PIT
liquidity fields for the mask; otherwise plain ``(val, fwd)`` tuples so the
production signal path stays unchanged.
"""
n = len(records) n = len(records)
if n < HORIZON + 21: if n < HORIZON + 21:
return return
closes = [float(r.close) for r in records] closes = [float(r.close) for r in records]
highs = [float(r.high) for r in records] highs = [float(r.high) for r in records]
volumes = [float(getattr(r, "volume", 0) or 0) for r in records]
dates = [r.date for r in records] dates = [r.date for r in records]
liquid_mode = _liquid_breadth_top_n() > 0
for i in _weekly_asof_indices(records): for i in _weekly_asof_indices(records):
j = i + HORIZON j = i + HORIZON
if j >= n or closes[i] <= 0: if j >= n or closes[i] <= 0:
@@ -966,17 +897,8 @@ def _accumulate_signal_series(
fwd = closes[j] / closes[i] - 1.0 fwd = closes[j] / closes[i] - 1.0
iso = records[i].date.isocalendar() iso = records[i].date.isocalendar()
week_key = (iso[0], iso[1]) week_key = (iso[0], iso[1])
dvol = _median_dollar_vol_63(closes, volumes, i) if liquid_mode else None
for name, val in _signal_values(dates, closes, highs, i, benchmark_closes).items(): for name, val in _signal_values(dates, closes, highs, i, benchmark_closes).items():
if liquid_mode: collected[name][week_key].append((val, fwd))
collected[name][week_key].append({
"val": val,
"fwd": fwd,
"close": closes[i],
"median_dvol_63": dvol,
})
else:
collected[name][week_key].append((val, fwd))
def _rank(xs: list[float]) -> list[float]: def _rank(xs: list[float]) -> list[float]:
@@ -1015,54 +937,6 @@ def _spearman(xs: list[float], ys: list[float]) -> float | None:
return _pearson(_rank(xs), _rank(ys)) return _pearson(_rank(xs), _rank(ys))
def _obs_val_fwd(rec: object) -> tuple[float, float] | None:
"""Unpack a signal observation: ``(val, fwd)`` or research dict form."""
if isinstance(rec, dict):
try:
return float(rec["val"]), float(rec["fwd"])
except (KeyError, TypeError, ValueError):
return None
if isinstance(rec, (tuple, list)) and len(rec) >= 2:
try:
return float(rec[0]), float(rec[1])
except (TypeError, ValueError):
return None
return None
def _filter_liquid_breadth_week(
recs: list,
*,
top_n: int,
min_price: float,
) -> list[tuple[float, float]]:
"""Point-in-time top-N by median $vol among names with price ≥ floor.
Ranking is relative (IEX volume undercount is OK for order stats). Membership
is recomputed every week from as-of bars — never frozen from today's liquidity.
"""
ranked: list[tuple[float, float, float]] = [] # (-dvol, val, fwd)
for rec in recs:
if not isinstance(rec, dict):
pair = _obs_val_fwd(rec)
if pair is not None:
ranked.append((0.0, pair[0], pair[1]))
continue
close = rec.get("close")
dvol = rec.get("median_dvol_63")
if close is None or float(close) < min_price:
continue
if dvol is None or float(dvol) <= 0:
continue
pair = _obs_val_fwd(rec)
if pair is None:
continue
ranked.append((-float(dvol), pair[0], pair[1]))
ranked.sort(key=lambda row: row[0])
kept = ranked[:top_n]
return [(val, fwd) for _, val, fwd in kept]
def _quintile_spread(pairs: list[tuple[float, float]]) -> float | None: def _quintile_spread(pairs: list[tuple[float, float]]) -> float | None:
"""Mean forward return of the top signal-quintile minus the bottom quintile.""" """Mean forward return of the top signal-quintile minus the bottom quintile."""
n = len(pairs) n = len(pairs)
@@ -1108,16 +982,10 @@ def _signal_evaluation(collected: dict) -> list[dict]:
IC is measured on NON-OVERLAPPING forward windows (weeks thinned to ~HORIZON IC is measured on NON-OVERLAPPING forward windows (weeks thinned to ~HORIZON
apart) so the t-stat isn't inflated by autocorrelation. A signal with no edge apart) so the t-stat isn't inflated by autocorrelation. A signal with no edge
lands near IC 0 / score 0; one with too few independent windows is flagged lands near IC 0 / spread 0; one with too few independent windows is flagged
unreliable rather than trusted on a lucky handful. unreliable rather than trusted on a lucky handful.
When ``BACKTEST_LIQUID_BREADTH=N`` is set, each week's cross-section is first
restricted to the top-N names by point-in-time 63d median dollar volume
(price ≥ BACKTEST_LIQUID_MIN_PRICE). Production path (flag unset) is unchanged.
""" """
stride = max(1, round(HORIZON / 5)) # ISO weeks spanned by the forward window stride = max(1, round(HORIZON / 5)) # ISO weeks spanned by the forward window
top_n = _liquid_breadth_top_n()
min_price = _liquid_min_price()
rows: list[dict] = [] rows: list[dict] = []
for name in sorted(collected): for name in sorted(collected):
weeks_map = collected[name] weeks_map = collected[name]
@@ -1128,25 +996,13 @@ def _signal_evaluation(collected: dict) -> list[dict]:
sizes: list[int] = [] sizes: list[int] = []
for wk in kept: for wk in kept:
recs = weeks_map[wk] recs = weeks_map[wk]
if top_n > 0: ic = _spearman([r[0] for r in recs], [r[1] for r in recs])
pairs = _filter_liquid_breadth_week(
recs, top_n=top_n, min_price=min_price
)
else:
pairs = []
for rec in recs:
pair = _obs_val_fwd(rec)
if pair is not None:
pairs.append(pair)
if len(pairs) < MIN_CROSS_SECTION:
continue
ic = _spearman([p[0] for p in pairs], [p[1] for p in pairs])
if ic is not None: if ic is not None:
ics.append(ic) ics.append(ic)
spread = _quintile_spread(pairs) spread = _quintile_spread(recs)
if spread is not None: if spread is not None:
spreads.append(spread) spreads.append(spread)
sizes.append(len(pairs)) sizes.append(len(recs))
if not ics: if not ics:
continue continue
mean_ic = sum(ics) / len(ics) mean_ic = sum(ics) / len(ics)
@@ -1155,7 +1011,7 @@ def _signal_evaluation(collected: dict) -> list[dict]:
else: else:
std = 0.0 std = 0.0
t_stat = mean_ic / std * math.sqrt(len(ics)) if std > 0 else None t_stat = mean_ic / std * math.sqrt(len(ics)) if std > 0 else None
row = { rows.append({
"signal": name, "signal": name,
"weeks": len(ics), "weeks": len(ics),
"avg_cross_section": round(sum(sizes) / len(sizes), 1) if sizes else None, "avg_cross_section": round(sum(sizes) / len(sizes), 1) if sizes else None,
@@ -1164,11 +1020,7 @@ def _signal_evaluation(collected: dict) -> list[dict]:
"ic_positive_pct": round(sum(1 for x in ics if x > 0) / len(ics) * 100, 1), "ic_positive_pct": round(sum(1 for x in ics if x > 0) / len(ics) * 100, 1),
"mean_quintile_spread": round(sum(spreads) / len(spreads), 4) if spreads else None, "mean_quintile_spread": round(sum(spreads) / len(spreads), 4) if spreads else None,
"reliable": len(ics) >= MIN_RELIABLE_PERIODS, "reliable": len(ics) >= MIN_RELIABLE_PERIODS,
} })
if top_n > 0:
row["liquid_breadth_top_n"] = top_n
row["liquid_min_price"] = min_price
rows.append(row)
rows.sort(key=lambda r: r["mean_ic"], reverse=True) rows.sort(key=lambda r: r["mean_ic"], reverse=True)
return rows return rows
@@ -1189,15 +1041,10 @@ def _replay_and_signals(
benchmark_closes: dict[date, float] | None = None, benchmark_closes: dict[date, float] | None = None,
target_model: str = PRODUCTION_GTL_TARGET_MODEL, target_model: str = PRODUCTION_GTL_TARGET_MODEL,
cadence: str = DEFAULT_BACKTEST_CADENCE, cadence: str = DEFAULT_BACKTEST_CADENCE,
signal_only: bool = False,
) -> tuple[list[dict], dict]: ) -> tuple[list[dict], dict]:
"""The CPU-bound per-ticker work, as a top-level (picklable) function so it can """The CPU-bound per-ticker work, as a top-level (picklable) function so it can
run in a worker process. Takes primitive column arrays (cheap to pickle), run in a worker process. Takes primitive column arrays (cheap to pickle),
rebuilds bar objects, and returns (candidates, signal_series). rebuilds bar objects, and returns (candidates, signal_series)."""
``signal_only=True`` (research rank-only names): skip GTL/candidate replay so
the production portfolio book is never polluted by broad-universe tickers.
"""
date_ords, opens, highs, lows, closes, volumes = columns date_ords, opens, highs, lows, closes, volumes = columns
bars = [ bars = [
SimpleNamespace( SimpleNamespace(
@@ -1205,9 +1052,8 @@ def _replay_and_signals(
) )
for o, op, hi, lo, cl, vo in zip(date_ords, opens, highs, lows, closes, volumes) for o, op, hi, lo, cl, vo in zip(date_ords, opens, highs, lows, closes, volumes)
] ]
candidates: list[dict] = [] return (
if not signal_only: _replay_ticker(
candidates = _replay_ticker(
symbol, symbol,
bars, bars,
config, config,
@@ -1215,9 +1061,7 @@ def _replay_and_signals(
benchmark_closes, benchmark_closes,
target_model, target_model,
cadence, cadence,
) ),
return (
candidates,
_signal_series(bars, benchmark_closes), _signal_series(bars, benchmark_closes),
) )
@@ -3945,12 +3789,6 @@ async def run_backtest(
result = await db.execute(select(Ticker).order_by(Ticker.symbol)) result = await db.execute(select(Ticker).order_by(Ticker.symbol))
tickers = list(result.scalars().all()) tickers = list(result.scalars().all())
total = len(tickers) total = len(tickers)
rank_only_symbols = await _load_research_rank_only_symbols(db)
if rank_only_symbols:
logger.info(json.dumps({
"event": "backtest_rank_only_loaded",
"count": len(rank_only_symbols),
}))
candidates: list[dict] = [] candidates: list[dict] = []
# Signal IC remains a weekly, non-overlapping diagnostic regardless of the # Signal IC remains a weekly, non-overlapping diagnostic regardless of the
@@ -4009,16 +3847,10 @@ async def run_backtest(
continue continue
if columns is not None: if columns is not None:
futures.append(loop.run_in_executor( futures.append(loop.run_in_executor(
pool, pool, _replay_and_signals, ticker.symbol, columns, config, activation,
_replay_and_signals,
ticker.symbol,
columns,
config,
activation,
benchmark_closes, benchmark_closes,
target_model, target_model,
cadence, cadence,
ticker.symbol in rank_only_symbols,
)) ))
for result in await asyncio.gather(*futures, return_exceptions=True): for result in await asyncio.gather(*futures, return_exceptions=True):
if isinstance(result, Exception): if isinstance(result, Exception):
@@ -4038,15 +3870,10 @@ async def run_backtest(
columns = await _fetch_columns(db, ticker.symbol) columns = await _fetch_columns(db, ticker.symbol)
if columns is not None: if columns is not None:
_merge(await asyncio.to_thread( _merge(await asyncio.to_thread(
_replay_and_signals, _replay_and_signals, ticker.symbol, columns, config, activation,
ticker.symbol,
columns,
config,
activation,
benchmark_closes, benchmark_closes,
target_model, target_model,
cadence, cadence,
ticker.symbol in rank_only_symbols,
)) ))
except Exception: except Exception:
logger.exception("Backtest replay failed for %s", ticker.symbol) logger.exception("Backtest replay failed for %s", ticker.symbol)
@@ -4089,75 +3916,73 @@ async def run_backtest(
portfolio_monitor_report: dict | None = None portfolio_monitor_report: dict | None = None
holdout_report: dict | None = None holdout_report: dict | None = None
min_rr_sweep_report: dict | None = None min_rr_sweep_report: dict | None = None
if not _signal_eval_only(): try:
qual_symbols = sorted({
c["symbol"]
for c in candidates
if c.get("qualified")
or any(_qualifies_strategy_variant(c, cfg) for cfg in STRATEGY_VARIANTS)
})
price_columns: dict[str, tuple] = {}
for sym in qual_symbols:
cols = await _fetch_columns(db, sym)
if cols is not None:
price_columns[sym] = cols
spy_closes: dict | None = None
try: try:
qual_symbols = sorted({ oldest = min((cols[0][0] for cols in price_columns.values()), default=None)
c["symbol"] days_needed = None
for c in candidates if oldest is not None and not _offline_snapshot_mode():
if c.get("qualified") days_needed = (date.today() - date.fromordinal(oldest)).days + 30
or any(_qualifies_strategy_variant(c, cfg) for cfg in STRATEGY_VARIANTS) spy_closes = await _load_benchmark_closes_for_backtest(
}) db, days=days_needed, refresh=oldest is not None
price_columns: dict[str, tuple] = {}
for sym in qual_symbols:
cols = await _fetch_columns(db, sym)
if cols is not None:
price_columns[sym] = cols
spy_closes: dict | None = None
try:
oldest = min((cols[0][0] for cols in price_columns.values()), default=None)
days_needed = None
if oldest is not None and not _offline_snapshot_mode():
days_needed = (date.today() - date.fromordinal(oldest)).days + 30
spy_closes = await _load_benchmark_closes_for_backtest(
db, days=days_needed, refresh=oldest is not None
)
except Exception:
logger.exception("Benchmark load for the portfolio sim failed")
for policy in ("target", "hold"):
sim = _simulate_portfolio(
candidates, price_columns, spy_closes, policy, hold_horizon
)
if sim is not None:
sim_policies.append({"policy": policy, **sim})
strategy_variant_rows = _strategy_variant_sims(
candidates, price_columns, spy_closes, hold_horizon
) )
exit_policy_rows = _exit_policy_sims( except Exception:
candidates, price_columns, spy_closes, hold_horizon logger.exception("Benchmark load for the portfolio sim failed")
)
live_exit_policy: dict | None = None
try:
from app.services.paper_trade_service import get_exit_policy
live_exit_policy = await get_exit_policy(db) for policy in ("target", "hold"):
except Exception: sim = _simulate_portfolio(
logger.exception("Live exit policy load failed; monitor uses defaults") candidates, price_columns, spy_closes, policy, hold_horizon
portfolio_monitor_report = _portfolio_monitor( )
candidates, price_columns, spy_closes, hold_horizon, if sim is not None:
sim_policies.append({"policy": policy, **sim})
strategy_variant_rows = _strategy_variant_sims(
candidates, price_columns, spy_closes, hold_horizon
)
exit_policy_rows = _exit_policy_sims(
candidates, price_columns, spy_closes, hold_horizon
)
live_exit_policy: dict | None = None
try:
from app.services.paper_trade_service import get_exit_policy
live_exit_policy = await get_exit_policy(db)
except Exception:
logger.exception("Live exit policy load failed; monitor uses defaults")
portfolio_monitor_report = _portfolio_monitor(
candidates, price_columns, spy_closes, hold_horizon,
live_exit_policy=live_exit_policy,
cadence=cadence,
)
split = _holdout_split()
if split is not None:
holdout_report = _holdout_evaluation(
candidates, price_columns, spy_closes, hold_horizon, split,
live_exit_policy=live_exit_policy, live_exit_policy=live_exit_policy,
cadence=cadence, cadence=cadence,
) )
split = _holdout_split() if _min_rr_sweep_enabled():
if split is not None: min_rr_sweep_report = _min_rr_sweep(
holdout_report = _holdout_evaluation( candidates, price_columns, spy_closes, activation, current_min_pct,
candidates, price_columns, spy_closes, hold_horizon, split, hold_horizon, live_exit_policy=live_exit_policy, cadence=cadence,
live_exit_policy=live_exit_policy, )
cadence=cadence, except Exception:
) logger.exception("Portfolio simulation failed")
if _min_rr_sweep_enabled():
min_rr_sweep_report = _min_rr_sweep(
candidates, price_columns, spy_closes, activation, current_min_pct,
hold_horizon, live_exit_policy=live_exit_policy, cadence=cadence,
)
except Exception:
logger.exception("Portfolio simulation failed")
report = { report = {
"generated_at": datetime.now(timezone.utc).isoformat(), "generated_at": datetime.now(timezone.utc).isoformat(),
"tickers": total, "tickers": total,
"rank_only_tickers": len(rank_only_symbols),
"candidates": len(candidates), "candidates": len(candidates),
"qualified": len(qualified), "qualified": len(qualified),
"params": { "params": {
@@ -4174,9 +3999,6 @@ async def run_backtest(
"target_model_label": BACKTEST_TARGET_MODELS[target_model], "target_model_label": BACKTEST_TARGET_MODELS[target_model],
"is_production_target_model": target_model == PRODUCTION_GTL_TARGET_MODEL, "is_production_target_model": target_model == PRODUCTION_GTL_TARGET_MODEL,
"production_reentry_policy": PRODUCTION_REENTRY_POLICY, "production_reentry_policy": PRODUCTION_REENTRY_POLICY,
"liquid_breadth_top_n": _liquid_breadth_top_n() or None,
"liquid_min_price": _liquid_min_price() if _liquid_breadth_top_n() else None,
"signal_eval_only": _signal_eval_only(),
}, },
"activation": activation, "activation": activation,
"overall_qualified": _bucket_stats(qualified), "overall_qualified": _bucket_stats(qualified),
docs/research/README.md
+2 -2
View File
@@ -141,8 +141,8 @@ knobs.
| Lead | Why it's interesting | Blocker | | Lead | Why it's interesting | Blocker |
|---|---|---| |---|---|---|
| **Near-close / MOC execution (ops)** | Recovers overnight momentum drift left on the table by a morning EU scan; evidence closed | Implement schedule + partial-bar scan path; one qualifying scan/day only | | **Near-close / MOC execution (ops)** | Recovers overnight momentum drift left on the table by a morning EU scan; evidence closed | Implement schedule + partial-bar scan path; one qualifying scan/day only |
| **`fip_id`** (information discreteness over the 12-1 window) | **Strongest cross-sectional signal measured on this universe** — IC −0.045, t = −2.91, correct sign; re-derived fingerprint matched Phase A; ticker technicals show it display-only | Doesn't improve *this* book as a filter. **Phase B tooling ready:** liquid-breadth IC on research.sqlite — see [fip-breadth-ic.md](fip-breadth-ic.md) | | **`fip_id`** (information discreteness over the 12-1 window) | **Strongest cross-sectional signal measured on this universe** — IC −0.045, t = −2.91, correct sign; re-derived fingerprint matched Phase A | Doesn't improve *this* book. Revisit when the universe broadens — **after** execution path is decided |
| **Broader universe** (`nasdaq_all` / liquid top-N) | Strengthens cross-sections; where `fip_id` may become tradeable | Offline research only first (`extend_snapshot_universe.py`); not prod scan | | **Broader universe** (`nasdaq_all`) | Strengthens every week's cross-section and the IC t-stat | Grade under the fill mode you will trade |
| **Forward paper-trade record** | The only true out-of-sample evidence the snapshot cannot give | Time; mark entries at actual near-close fill once ops ships | | **Forward paper-trade record** | The only true out-of-sample evidence the snapshot cannot give | Time; mark entries at actual near-close fill once ops ships |
| **Better target model for clear-air names** | The return is demonstrably there (#2 wins on raw CAGR in *both* train and test); it's the *flat* 3× ATR target that makes it too expensive in risk | Needs a per-name model, not a constant k×ATR | | **Better target model for clear-air names** | The return is demonstrably there (#2 wins on raw CAGR in *both* train and test); it's the *flat* 3× ATR target that makes it too expensive in risk | Needs a per-name model, not a constant k×ATR |
docs/research/fip-breadth-ic.md
-54
View File
@@ -1,54 +0,0 @@
# Broad-universe fip_id IC research (Phase B)
Generated: 2026-07-18T19:48:28.127710
## Scope
- **Research only** — production universe, gate, scanner, schedule unchanged.
- Price-only signal harness; no sentiment/fundamentals on the broad tier.
- Point-in-time liquidity mask: top **1500** by 63d median $vol, price ≥ **$5.0** at as-of.
## Caveats
- **Survivorship bias**: today's constituents backfilled historically (worse in small caps).
- **IEX volume undercount**: relative $vol rank only, not absolute floors.
- **Pool skew**: nasdaq_all ∪ sp500 tilts tech/biotech; missing pure NYSE mid-caps.
## Fingerprint (505-name prod snapshot)
- Expected: IC ≈ -0.045, t ≈ -2.9
- Observed: IC = -0.045, t = -2.91, weeks = 35, reliable = True
- Pass: **True**
## Liquid-breadth signal_eval (fip_id)
**Not run yet in this environment** (no Alpaca credentials to build
`backtest_snapshots/research.sqlite`). Local machine with keys:
```bash
python scripts/extend_snapshot_universe.py --force-copy
python scripts/run_fip_breadth_research.py --skip-fingerprint --allow-spawn --workers 6
```
(`--skip-fingerprint` only after a green fingerprint on this machine.)
| metric | value |
|---|---|
| mean_ic | _pending_ |
| ic_t_stat | _pending_ |
| weeks | _pending_ |
| avg_cross_section | _pending_ |
| reliable | _pending_ |
## Verdict (iron rule)
- **Fingerprint:** green (IC −0.045 / t −2.91 / reliable / 35 weeks).
- **Breadth iron rule:** **pending** until research.sqlite run completes.
- Green breadth would authorize a **follow-up proposal** only (two-tier universe /
gate revalidation) — **not** production wire-in.
## Artifacts
- Fingerprint report: `reports/fip-breadth-20260718-194828-fingerprint.json`
(and summary `reports/fip-breadth-20260718-194828.json`)
- Breadth report: _pending_
reports/fip-breadth-20260718-194828-fingerprint.json
-578
View File
@@ -1,578 +0,0 @@
{
"generated_at": "2026-07-18T18:21:28.359793+00:00",
"tickers": 506,
"rank_only_tickers": 0,
"candidates": 202765,
"qualified": 1086,
"params": {
"step_days": 5,
"step_sessions": 5,
"entry_cadence": "weekly",
"signal_eval_cadence": "weekly",
"horizon_days": 30,
"min_lookback": 60,
"cost_per_side_pct": 0.1,
"target_model": "production_gtl",
"target_model_label": "Live GTL (production)",
"is_production_target_model": true,
"production_reentry_policy": "gate_reset",
"liquid_breadth_top_n": null,
"liquid_min_price": null,
"signal_eval_only": true
},
"activation": {
"min_momentum_percentile": 80.0,
"min_rr": 2.0,
"min_confidence": 0.0,
"require_high_conviction": false,
"exclude_conflicts": false,
"exclude_neutral": true
},
"overall_qualified": {
"total": 1086,
"wins": 379,
"losses": 591,
"expired": 116,
"hit_rate": 39.1,
"avg_r": 0.255,
"total_r": 276.76,
"net_avg_r": 0.209,
"net_total_r": 226.56,
"best_r": 8.85,
"worst_r": -3.38,
"avg_hold_days": 12.0,
"net_r_per_day": 0.0174,
"median_net_r": -1.031,
"profit_factor": 1.34,
"net_avg_r_ex_top5": 0.049
},
"overall_all": {
"total": 202765,
"wins": 82220,
"losses": 113809,
"expired": 6736,
"hit_rate": 41.9,
"avg_r": -0.04,
"total_r": -8186.18,
"net_avg_r": -0.095,
"net_total_r": -19184.55,
"best_r": 9.24,
"worst_r": -16.42,
"avg_hold_days": 8.2,
"net_r_per_day": -0.0115,
"median_net_r": -1.035,
"profit_factor": 0.85,
"net_avg_r_ex_top5": -0.22
},
"by_direction": {
"long": {
"total": 1086,
"wins": 379,
"losses": 591,
"expired": 116,
"hit_rate": 39.1,
"avg_r": 0.255,
"total_r": 276.76,
"net_avg_r": 0.209,
"net_total_r": 226.56,
"best_r": 8.85,
"worst_r": -3.38,
"avg_hold_days": 12.0,
"net_r_per_day": 0.0174,
"median_net_r": -1.031,
"profit_factor": 1.34,
"net_avg_r_ex_top5": 0.049
},
"short": {
"total": 0,
"wins": 0,
"losses": 0,
"expired": 0,
"hit_rate": null,
"avg_r": null,
"total_r": null,
"net_avg_r": null,
"net_total_r": null,
"best_r": null,
"worst_r": null,
"avg_hold_days": null,
"net_r_per_day": null,
"median_net_r": null,
"profit_factor": null,
"net_avg_r_ex_top5": null
}
},
"min_momentum_percentile": 80.0,
"sweep": [
{
"min_momentum_percentile": 90.0,
"total": 497,
"wins": 177,
"losses": 269,
"expired": 51,
"hit_rate": 39.7,
"avg_r": 0.276,
"total_r": 137.05,
"net_avg_r": 0.235,
"net_total_r": 116.55,
"best_r": 8.85,
"worst_r": -3.38,
"avg_hold_days": 11.8,
"net_r_per_day": 0.0199,
"median_net_r": -1.026,
"profit_factor": 1.39,
"net_avg_r_ex_top5": 0.071
},
{
"min_momentum_percentile": 80.0,
"total": 1086,
"wins": 379,
"losses": 591,
"expired": 116,
"hit_rate": 39.1,
"avg_r": 0.255,
"total_r": 276.76,
"net_avg_r": 0.209,
"net_total_r": 226.56,
"best_r": 8.85,
"worst_r": -3.38,
"avg_hold_days": 12.0,
"net_r_per_day": 0.0174,
"median_net_r": -1.031,
"profit_factor": 1.34,
"net_avg_r_ex_top5": 0.049
},
{
"min_momentum_percentile": 70.0,
"total": 1841,
"wins": 597,
"losses": 1062,
"expired": 182,
"hit_rate": 36.0,
"avg_r": 0.152,
"total_r": 280.26,
"net_avg_r": 0.104,
"net_total_r": 190.75,
"best_r": 8.85,
"worst_r": -4.21,
"avg_hold_days": 11.8,
"net_r_per_day": 0.0088,
"median_net_r": -1.037,
"profit_factor": 1.16,
"net_avg_r_ex_top5": -0.055
},
{
"min_momentum_percentile": 60.0,
"total": 2772,
"wins": 873,
"losses": 1611,
"expired": 288,
"hit_rate": 35.1,
"avg_r": 0.126,
"total_r": 348.07,
"net_avg_r": 0.075,
"net_total_r": 209.25,
"best_r": 8.85,
"worst_r": -4.21,
"avg_hold_days": 12.0,
"net_r_per_day": 0.0063,
"median_net_r": -1.04,
"profit_factor": 1.12,
"net_avg_r_ex_top5": -0.077
},
{
"min_momentum_percentile": 50.0,
"total": 3901,
"wins": 1182,
"losses": 2295,
"expired": 424,
"hit_rate": 34.0,
"avg_r": 0.089,
"total_r": 345.37,
"net_avg_r": 0.038,
"net_total_r": 146.96,
"best_r": 8.85,
"worst_r": -4.86,
"avg_hold_days": 12.1,
"net_r_per_day": 0.0031,
"median_net_r": -1.042,
"profit_factor": 1.06,
"net_avg_r_ex_top5": -0.114
},
{
"min_momentum_percentile": 0.0,
"total": 14588,
"wins": 3719,
"losses": 9271,
"expired": 1598,
"hit_rate": 28.6,
"avg_r": -0.065,
"total_r": -952.08,
"net_avg_r": -0.115,
"net_total_r": -1676.86,
"best_r": 9.24,
"worst_r": -15.94,
"avg_hold_days": 12.1,
"net_r_per_day": -0.0095,
"median_net_r": -1.043,
"profit_factor": 0.84,
"net_avg_r_ex_top5": -0.275
}
],
"gate_ablation": [
{
"variant": "all_floors",
"total": 1086,
"wins": 379,
"losses": 591,
"expired": 116,
"hit_rate": 39.1,
"avg_r": 0.255,
"total_r": 276.76,
"net_avg_r": 0.209,
"net_total_r": 226.56,
"best_r": 8.85,
"worst_r": -3.38,
"avg_hold_days": 12.0,
"net_r_per_day": 0.0174,
"median_net_r": -1.031,
"profit_factor": 1.34,
"net_avg_r_ex_top5": 0.049,
"hold_days": 30,
"hold_avg_r": 0.631,
"hold_net_avg_r": 0.585,
"hold_total_r": 684.97
},
{
"variant": "no_confidence_floor",
"total": 1093,
"wins": 380,
"losses": 596,
"expired": 117,
"hit_rate": 38.9,
"avg_r": 0.25,
"total_r": 273.55,
"net_avg_r": 0.204,
"net_total_r": 222.99,
"best_r": 8.85,
"worst_r": -3.38,
"avg_hold_days": 11.9,
"net_r_per_day": 0.0171,
"median_net_r": -1.031,
"profit_factor": 1.33,
"net_avg_r_ex_top5": 0.045,
"hold_days": 30,
"hold_avg_r": 0.626,
"hold_net_avg_r": 0.58,
"hold_total_r": 684.08
},
{
"variant": "no_rr_floor",
"total": 6849,
"wins": 3235,
"losses": 3409,
"expired": 205,
"hit_rate": 48.7,
"avg_r": 0.112,
"total_r": 770.17,
"net_avg_r": 0.061,
"net_total_r": 418.96,
"best_r": 8.85,
"worst_r": -5.16,
"avg_hold_days": 7.9,
"net_r_per_day": 0.0078,
"median_net_r": -0.061,
"profit_factor": 1.11,
"net_avg_r_ex_top5": -0.061,
"hold_days": 30,
"hold_avg_r": 0.354,
"hold_net_avg_r": 0.303,
"hold_total_r": 2425.86
},
{
"variant": "no_neutral_exclusion",
"total": 2313,
"wins": 770,
"losses": 1279,
"expired": 264,
"hit_rate": 37.6,
"avg_r": 0.2,
"total_r": 462.35,
"net_avg_r": 0.154,
"net_total_r": 355.89,
"best_r": 8.85,
"worst_r": -3.38,
"avg_hold_days": 12.6,
"net_r_per_day": 0.0122,
"median_net_r": -1.031,
"profit_factor": 1.25,
"net_avg_r_ex_top5": 0.004,
"hold_days": 30,
"hold_avg_r": 0.583,
"hold_net_avg_r": 0.537,
"hold_total_r": 1348.89
},
{
"variant": "momentum_only",
"total": 14696,
"wins": 6827,
"losses": 7359,
"expired": 510,
"hit_rate": 48.1,
"avg_r": 0.114,
"total_r": 1669.64,
"net_avg_r": 0.064,
"net_total_r": 936.93,
"best_r": 8.85,
"worst_r": -5.71,
"avg_hold_days": 8.4,
"net_r_per_day": 0.0076,
"median_net_r": -1.013,
"profit_factor": 1.11,
"net_avg_r_ex_top5": -0.055,
"hold_days": 30,
"hold_avg_r": 0.395,
"hold_net_avg_r": 0.345,
"hold_total_r": 5798.82
}
],
"gate_ablation_note": "Each row re-qualifies the same candidates at the current momentum cutoff (80) with one floor removed (long-only while the momentum gate is active). If dropping a floor doesn't hurt net expectancy, that floor isn't pulling its weight. The Hold columns grade the same variants under the hold-to-horizon time exit instead of the S/R target \u2014 the view that matters if the exit policy moves to a fixed hold.",
"time_exit_sweep": [
{
"hold_days": 5,
"total": 1086,
"wins": 603,
"win_rate": 55.5,
"avg_r": 0.175,
"total_r": 190.16,
"net_avg_r": 0.129,
"net_total_r": 139.97,
"best_r": 5.09,
"worst_r": -2.51,
"avg_hold_days": 4.5,
"net_r_per_day": 0.0285,
"median_net_r": 0.115,
"profit_factor": 1.36,
"net_avg_r_ex_top5": -0.002
},
{
"hold_days": 10,
"total": 1086,
"wins": 559,
"win_rate": 51.5,
"avg_r": 0.357,
"total_r": 387.9,
"net_avg_r": 0.311,
"net_total_r": 337.7,
"best_r": 6.73,
"worst_r": -2.51,
"avg_hold_days": 7.9,
"net_r_per_day": 0.0395,
"median_net_r": 0.031,
"profit_factor": 1.67,
"net_avg_r_ex_top5": 0.112
},
{
"hold_days": 21,
"total": 1086,
"wins": 487,
"win_rate": 44.8,
"avg_r": 0.525,
"total_r": 570.33,
"net_avg_r": 0.479,
"net_total_r": 520.14,
"best_r": 9.86,
"worst_r": -3.38,
"avg_hold_days": 13.7,
"net_r_per_day": 0.0349,
"median_net_r": -1.027,
"profit_factor": 1.81,
"net_avg_r_ex_top5": 0.191
},
{
"hold_days": 30,
"total": 1086,
"wins": 434,
"win_rate": 40.0,
"avg_r": 0.631,
"total_r": 684.97,
"net_avg_r": 0.585,
"net_total_r": 634.78,
"best_r": 12.87,
"worst_r": -3.38,
"avg_hold_days": 17.8,
"net_r_per_day": 0.0329,
"median_net_r": -1.033,
"profit_factor": 1.9,
"net_avg_r_ex_top5": 0.212
}
],
"portfolio_sim": {
"params": {
"starting_capital": 10000.0,
"max_positions": 10,
"risk_per_trade_pct": 1.0,
"notional_cap_pct": 20.0,
"cost_per_side_pct": 0.1,
"hold_days": 30
},
"policies": [],
"note": "One capital-constrained book over the same qualified setups the tables above grade per-setup: at most 10 concurrent positions (one per ticker), best momentum first, fixed-fractional risk sizing with a no-leverage cap, entries at the detection close, stops filled at the worse of stop or open. 'target' races the S/R target against the stop (timeout at the horizon); 'hold' keeps the initial stop and exits at the horizon close. SPY return is price-only over the same window. In-sample; no dividends."
},
"strategy_variants": {
"variants": [],
"note": "Research-only hold-to-horizon portfolio variants. Production now uses residual 12-1 momentum at cutoff 80; the remaining rows compare the legacy raw rank, raw cutoff 90, one max-15 capacity check, and volatility overlays."
},
"exit_policy_variants": {
"variants": [],
"note": "Research-only exit policies over the residual/high-vol 80/20 entry candidate. Every row uses the same entry qualification/ranking and changes only the exit discipline."
},
"portfolio_monitor": null,
"production_cadence_comparison": null,
"holdout": null,
"min_rr_sweep": null,
"target_model_diagnostics": {
"target_model": "production_gtl",
"target_model_label": "Live GTL (production)",
"candidate_count": 202765,
"primary_source_counts": {
"pivot_point": 196290,
"range_grid": 180036
},
"primary_round_only": 0,
"primary_strength_100": 138596,
"avg_primary_strength": 80.109,
"avg_primary_distance_atr": 2.293,
"avg_primary_rejection_count": 41.908,
"avg_raw_level_count": 53.204,
"avg_gate_level_count": 53.204
},
"signal_eval": [
{
"signal": "vol_6m",
"weeks": 39,
"avg_cross_section": 498.2,
"mean_ic": 0.0609,
"ic_t_stat": 1.48,
"ic_positive_pct": 64.1,
"mean_quintile_spread": 0.0337,
"reliable": true
},
{
"signal": "mom_12_1_resid",
"weeks": 35,
"avg_cross_section": 497.7,
"mean_ic": 0.0552,
"ic_t_stat": 1.98,
"ic_positive_pct": 60.0,
"mean_quintile_spread": 0.0207,
"reliable": true
},
{
"signal": "mom_12_1",
"weeks": 35,
"avg_cross_section": 497.7,
"mean_ic": 0.0531,
"ic_t_stat": 1.61,
"ic_positive_pct": 65.7,
"mean_quintile_spread": 0.0206,
"reliable": true
},
{
"signal": "trend_200",
"weeks": 37,
"avg_cross_section": 497.9,
"mean_ic": 0.0161,
"ic_t_stat": 0.44,
"ic_positive_pct": 59.5,
"mean_quintile_spread": 0.006,
"reliable": true
},
{
"signal": "reversal_1m",
"weeks": 43,
"avg_cross_section": 498.7,
"mean_ic": 0.0059,
"ic_t_stat": 0.22,
"ic_positive_pct": 53.5,
"mean_quintile_spread": 0.0053,
"reliable": true
},
{
"signal": "mom_6_1",
"weeks": 39,
"avg_cross_section": 498.2,
"mean_ic": 0.0051,
"ic_t_stat": 0.21,
"ic_positive_pct": 56.4,
"mean_quintile_spread": 0.0087,
"reliable": true
},
{
"signal": "mom_3_1",
"weeks": 42,
"avg_cross_section": 498.5,
"mean_ic": -0.0064,
"ic_t_stat": -0.25,
"ic_positive_pct": 50.0,
"mean_quintile_spread": 0.0046,
"reliable": true
},
{
"signal": "high_52w",
"weeks": 35,
"avg_cross_section": 497.7,
"mean_ic": -0.0086,
"ic_t_stat": -0.26,
"ic_positive_pct": 54.3,
"mean_quintile_spread": -0.0088,
"reliable": true
},
{
"signal": "fip_id",
"weeks": 35,
"avg_cross_section": 497.7,
"mean_ic": -0.045,
"ic_t_stat": -2.91,
"ic_positive_pct": 25.7,
"mean_quintile_spread": -0.0168,
"reliable": true
}
],
"signal_eval_note": "Cross-sectional rank-IC of price-only signals vs the forward 30-day return (min 20 names/window). |IC| \u2273 0.03 with a consistent sign is a real (if small) edge; near 0 means ranking on it sorts nothing. Momentum factors and high_52w are expected positive; reversal_1m and vol_6m expected negative (mean-reversion / low-vol anomaly). IC is measured on non-overlapping windows; signals with fewer than 12 independent windows are flagged unreliable (too few regimes \u2014 deepen history with the Data Backfill job).",
"note": "Sentiment & fundamentals held neutral (no point-in-time history). Stops fill at the worse of the stop or the bar's open (gaps through the stop are modeled, so a loss can exceed \u22121R); targets never fill better than their level. ~6 months \u2248 one market regime \u2014 treat as directional, not gospel.",
"recommendation": {
"headline": "Trade the qualified list long-only; hold 30 trading days with the initial ATR stop.",
"items": [
{
"topic": "exit",
"text": "Legacy exit diagnostic: hold 30 trading days with the initial stop (+0.58R net/trade vs +0.21R for the S/R target exit)."
},
{
"topic": "gate",
"text": "Gate: the confidence floor adds nothing \u2014 dropping it costs +0.01R/trade and adds 7 trades."
},
{
"topic": "gate",
"text": "Gate: keep the R:R floor (worth +0.28R/trade under the hold exit)."
},
{
"topic": "gate",
"text": "Gate: keep the NEUTRAL exclusion (worth +0.05R/trade under the hold exit)."
},
{
"topic": "cutoff",
"text": "Residual-momentum cutoff: 90 has the best per-trade net (+0.23R over 497 setups)."
},
{
"topic": "robustness",
"text": "Robustness: expectancy survives removing the top 5% of winners (+0.21R net/trade under the recommended 30d hold) \u2014 the edge is not a handful of outliers."
}
],
"note": "Derived from this report's numbers on every run \u2014 the advice flips if the data does."
},
"research_recommendation": {
"items": [],
"note": "Strategy variants unavailable; re-run the backtest after benchmark data is present."
}
}
reports/fip-breadth-20260718-194828.json
-21
View File
@@ -1,21 +0,0 @@
{
"generated_at": "2026-07-18T19:48:28.127710",
"liquid_breadth_top_n": 1500,
"liquid_min_price": 5.0,
"fingerprint": {
"signal": "fip_id",
"weeks": 35,
"avg_cross_section": 497.7,
"mean_ic": -0.045,
"ic_t_stat": -2.91,
"ic_positive_pct": 25.7,
"mean_quintile_spread": -0.0168,
"reliable": true,
"pass": true,
"expected_ic": -0.045,
"expected_t": -2.9
},
"breadth": null,
"verdict": null,
"fingerprint_report_path": "reports\\fip-breadth-20260718-194828-fingerprint.json"
}
scripts/extend_snapshot_universe.py
-329
View File
@@ -1,329 +0,0 @@
"""Extend a *copy* of the production backtest snapshot with broad-universe OHLCV.
Research only — never writes to production Postgres.
Pipeline
--------
1. Copy ``--source`` snapshot (default ``backtest_snapshots/prod.sqlite``) to
``--output`` (default ``backtest_snapshots/research.sqlite``).
2. Resolve symbol pool = nasdaq_all ∪ sp500 via ``ticker_universe_service``.
3. Fetch ~5y daily bars from Alpaca for symbols missing (or short) in the copy.
4. Insert new tickers + OHLCV; mark them in side table ``research_rank_only``
so the harness can feed signal IC without GTL/candidate replay.
Resume-friendly: re-running skips symbols that already have ≥ ``--min-bars``.
Example
-------
python scripts/extend_snapshot_universe.py \\
--source backtest_snapshots/prod.sqlite \\
--output backtest_snapshots/research.sqlite \\
--force-copy
# smoke: first 50 missing symbols only
python scripts/extend_snapshot_universe.py --limit 50
"""
from __future__ import annotations
import argparse
import asyncio
import shutil
import sys
import time
from datetime import date, datetime, timedelta, timezone
from pathlib import Path
from sqlalchemy import create_engine, select, text
from sqlalchemy.orm import Session
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
def _parse_args() -> argparse.Namespace:
p = argparse.ArgumentParser(description=__doc__)
p.add_argument(
"--source",
default="backtest_snapshots/prod.sqlite",
help="Existing prod snapshot to copy (read-only after copy).",
)
p.add_argument(
"--output",
default="backtest_snapshots/research.sqlite",
help="Research snapshot path (created/updated).",
)
p.add_argument(
"--force-copy",
action="store_true",
help="Overwrite output by re-copying from source first.",
)
p.add_argument(
"--history-days",
type=int,
default=1825,
help="OHLCV lookback days (~5y). Default 1825.",
)
p.add_argument(
"--min-bars",
type=int,
default=260,
help="Skip re-fetch when a symbol already has this many bars.",
)
p.add_argument(
"--limit",
type=int,
default=None,
help="Max *new* symbols to fetch (smoke tests).",
)
p.add_argument(
"--sleep",
type=float,
default=0.15,
help="Seconds between Alpaca symbol requests (rate-limit cushion).",
)
p.add_argument(
"--max-retries",
type=int,
default=5,
help="Retries per symbol on RateLimitError.",
)
p.add_argument("--quiet", action="store_true")
return p.parse_args()
def _ensure_rank_only_table(conn) -> None:
conn.execute(
text(
"""
CREATE TABLE IF NOT EXISTS research_rank_only (
ticker_id INTEGER PRIMARY KEY,
symbol TEXT NOT NULL UNIQUE
)
"""
)
)
conn.commit()
async def _resolve_pool() -> tuple[list[str], dict[str, str]]:
"""Return sorted unique symbols and source labels."""
from app.database import async_session_factory
from app.services.ticker_universe_service import fetch_universe_symbols
sources: dict[str, str] = {}
symbols: set[str] = set()
# Need a DB session for cache writes; use local async engine if configured,
# but public/FMP fetch works with any session. Prefer a throwaway sqlite.
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from sqlalchemy.ext.asyncio import AsyncSession
engine = create_async_engine("sqlite+aiosqlite:///:memory:")
Session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
try:
async with Session() as db:
for universe in ("nasdaq_all", "sp500"):
try:
syms, src = await fetch_universe_symbols(db, universe)
except Exception as exc:
print(f"WARNING: universe {universe} failed: {exc}")
continue
sources[universe] = src
symbols.update(syms)
print(f" {universe}: {len(syms)} symbols (source={src})")
finally:
await engine.dispose()
return sorted(symbols), sources
async def _fetch_symbol_bars(
provider,
symbol: str,
start: date,
end: date,
*,
max_retries: int,
sleep_s: float,
) -> list:
from app.exceptions import ProviderError, RateLimitError
for attempt in range(max_retries):
try:
bars = await provider.fetch_ohlcv(symbol, start, end)
if sleep_s > 0:
await asyncio.sleep(sleep_s)
return bars
except RateLimitError:
wait = min(60.0, 2.0 ** attempt)
print(f" rate limited on {symbol}; sleep {wait:.0f}s")
await asyncio.sleep(wait)
except ProviderError as exc:
if attempt + 1 >= max_retries:
raise
await asyncio.sleep(1.0)
_ = exc
return []
async def _main() -> None:
args = _parse_args()
source = Path(args.source)
output = Path(args.output)
if not source.exists():
raise SystemExit(f"Source snapshot not found: {source}")
if args.force_copy or not output.exists():
output.parent.mkdir(parents=True, exist_ok=True)
if output.exists():
output.unlink()
print(f"Copying {source} → {output}")
shutil.copy2(source, output)
else:
print(f"Updating existing research snapshot: {output}")
from app.config import settings
from app.models.ohlcv import OHLCVRecord
from app.models.ticker import Ticker
from app.providers.alpaca import AlpacaOHLCVProvider
if not settings.alpaca_api_key or not settings.alpaca_api_secret:
raise SystemExit("ALPACA_API_KEY / ALPACA_API_SECRET required in .env")
provider = AlpacaOHLCVProvider(settings.alpaca_api_key, settings.alpaca_api_secret)
end = date.today()
start = end - timedelta(days=int(args.history_days))
print("Resolving universe pool (nasdaq_all ∪ sp500)…")
pool, sources = await _resolve_pool()
print(f"Pool size: {len(pool)} (sources={sources})")
# Sync sqlite via sqlalchemy core (simpler than async for bulk insert)
engine = create_engine(f"sqlite:///{output.resolve().as_posix()}")
with Session(engine) as session:
_ensure_rank_only_table(session.connection())
existing = {
row.symbol: row
for row in session.execute(select(Ticker)).scalars().all()
}
prod_symbols = set(existing)
# Bar counts
bar_counts: dict[str, int] = {}
for sym, ticker in existing.items():
n = session.execute(
text("SELECT COUNT(*) FROM ohlcv_records WHERE ticker_id = :tid"),
{"tid": ticker.id},
).scalar_one()
bar_counts[sym] = int(n)
to_fetch: list[str] = []
for sym in pool:
if sym in existing and bar_counts.get(sym, 0) >= args.min_bars:
# Existing production or previously extended — keep rank_only
# only for *new* research names, not original prod universe.
continue
to_fetch.append(sym)
if args.limit is not None:
to_fetch = to_fetch[: max(0, int(args.limit))]
print(f"Symbols to fetch/extend: {len(to_fetch)}")
ok = 0
fail = 0
t0 = time.monotonic()
for index, sym in enumerate(to_fetch, 1):
try:
bars = await _fetch_symbol_bars(
provider,
sym,
start,
end,
max_retries=args.max_retries,
sleep_s=args.sleep,
)
except Exception as exc:
fail += 1
if not args.quiet:
print(f" [{index}/{len(to_fetch)}] {sym} FAIL {exc}")
continue
if not bars:
fail += 1
if not args.quiet:
print(f" [{index}/{len(to_fetch)}] {sym} empty")
continue
ticker = existing.get(sym)
is_new = ticker is None
if ticker is None:
ticker = Ticker(symbol=sym, name=None, created_at=datetime.now(timezone.utc))
session.add(ticker)
session.flush()
existing[sym] = ticker
# Upsert bars (delete+insert range for simplicity on research path)
session.execute(
text(
"DELETE FROM ohlcv_records WHERE ticker_id = :tid "
"AND date >= :start AND date <= :end"
),
{"tid": ticker.id, "start": start.isoformat(), "end": end.isoformat()},
)
now = datetime.utcnow()
session.bulk_insert_mappings(
OHLCVRecord,
[
{
"ticker_id": ticker.id,
"date": b.date,
"open": b.open,
"high": b.high,
"low": b.low,
"close": b.close,
"volume": b.volume,
"created_at": now,
}
for b in bars
],
)
# rank_only only for names that were NOT in the original production
# snapshot at copy time (or are newly introduced to this research DB).
if is_new or sym not in prod_symbols:
# Re-evaluate: if source copy already had the symbol, don't flag.
# Only new inserts get rank_only.
if is_new:
session.execute(
text(
"INSERT OR REPLACE INTO research_rank_only "
"(ticker_id, symbol) VALUES (:tid, :sym)"
),
{"tid": ticker.id, "sym": sym},
)
session.commit()
ok += 1
if not args.quiet and (index % 25 == 0 or index == len(to_fetch)):
elapsed = time.monotonic() - t0
print(
f" progress {index}/{len(to_fetch)} ok={ok} fail={fail} "
f"elapsed={elapsed/60:.1f}m last={sym} bars={len(bars)}"
)
rank_only_n = session.execute(
text("SELECT COUNT(*) FROM research_rank_only")
).scalar_one()
ticker_n = session.execute(text("SELECT COUNT(*) FROM tickers")).scalar_one()
ohlcv_n = session.execute(text("SELECT COUNT(*) FROM ohlcv_records")).scalar_one()
print("Done.")
print(f" output: {output}")
print(f" tickers: {ticker_n}")
print(f" ohlcv rows: {ohlcv_n}")
print(f" research_rank_only: {rank_only_n}")
print(f" fetched ok/fail: {ok}/{fail}")
if __name__ == "__main__":
asyncio.run(_main())
scripts/run_fip_breadth_research.py
-299
View File
@@ -1,299 +0,0 @@
"""Phase B: fip_id IC on liquid-breadth cross-section (local research only).
1. Fingerprint check on the unextended prod snapshot (must ≈ IC −0.045 / t −2.9).
2. Run signal_eval on research.sqlite with BACKTEST_LIQUID_BREADTH=1500 PIT mask.
3. Write a research report under docs/research/ and reports/.
Does not modify production DB, gate, scanner, or schedule.
Example
-------
# After extend_snapshot_universe.py has built research.sqlite:
python scripts/run_fip_breadth_research.py \\
--prod-snapshot backtest_snapshots/prod.sqlite \\
--research-snapshot backtest_snapshots/research.sqlite \\
--workers 6 --allow-spawn
"""
from __future__ import annotations
import argparse
import asyncio
import json
import os
import sys
from datetime import datetime
from pathlib import Path
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
FINGERPRINT_IC = -0.045
FINGERPRINT_T = -2.9
FINGERPRINT_IC_TOL = 0.015
FINGERPRINT_T_TOL = 0.6
def _sqlite_url(path: Path) -> str:
return f"sqlite+aiosqlite:///{path.resolve().as_posix()}"
def _parse_args() -> argparse.Namespace:
p = argparse.ArgumentParser(description=__doc__)
p.add_argument("--prod-snapshot", default="backtest_snapshots/prod.sqlite")
p.add_argument("--research-snapshot", default="backtest_snapshots/research.sqlite")
p.add_argument("--workers", type=int, default=6)
p.add_argument("--allow-spawn", action="store_true")
p.add_argument("--skip-fingerprint", action="store_true")
p.add_argument("--skip-research", action="store_true")
p.add_argument("--liquid-breadth", type=int, default=1500)
p.add_argument("--min-price", type=float, default=5.0)
p.add_argument(
"--out",
default=None,
help="JSON report path (default reports/fip-breadth-YYYYMMDD.json)",
)
p.add_argument("--quiet", action="store_true")
return p.parse_args()
def _find_fip(signal_eval: list[dict]) -> dict | None:
for row in signal_eval or []:
if row.get("signal") == "fip_id":
return row
return None
def _verdict(row: dict | None) -> dict:
if row is None:
return {
"green": False,
"reason": "fip_id missing from signal_eval",
}
mean_ic = row.get("mean_ic")
t_stat = row.get("ic_t_stat")
reliable = bool(row.get("reliable"))
if mean_ic is None or t_stat is None:
return {"green": False, "reason": "missing mean_ic or ic_t_stat", "row": row}
sign_ok = mean_ic < 0
mag_ok = abs(float(mean_ic)) >= 0.03
green = sign_ok and mag_ok and reliable
return {
"green": green,
"reason": (
"iron rule cleared — follow-up proposal only, not production wire-in"
if green
else "iron rule not met on liquid-breadth cross-section"
),
"checks": {
"mean_ic": mean_ic,
"abs_mean_ic_ge_0_03": mag_ok,
"sign_negative": sign_ok,
"ic_t_stat": t_stat,
"reliable": reliable,
"weeks": row.get("weeks"),
"avg_cross_section": row.get("avg_cross_section"),
},
"row": row,
}
async def _run_signal_eval(snapshot: Path, *, workers: int, quiet: bool) -> dict:
from app.config import settings
from app.services.backtest_service import run_backtest
settings.backtest_workers = workers
engine = create_async_engine(_sqlite_url(snapshot), pool_pre_ping=True)
Session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
def progress(done: int, total: int, symbol: str) -> None:
if quiet:
return
print(f" progress {done}/{total} {symbol}", end="\r")
try:
async with Session() as db:
report = await run_backtest(db, progress_cb=progress, cadence="weekly")
finally:
await engine.dispose()
if not quiet:
print()
return report
def _write_md(path: Path, payload: dict) -> None:
fp = payload.get("fingerprint") or {}
br = payload.get("breadth") or {}
v = payload.get("verdict") or {}
lines = [
"# Broad-universe fip_id IC research (Phase B)",
"",
f"Generated: {payload.get('generated_at')}",
"",
"## Scope",
"",
"- **Research only** — production universe, gate, scanner, schedule unchanged.",
"- Price-only signal harness; no sentiment/fundamentals on the broad tier.",
"- Point-in-time liquidity mask: top "
f"**{payload.get('liquid_breadth_top_n')}** by 63d median $vol, "
f"price ≥ **${payload.get('liquid_min_price')}** at as-of.",
"",
"## Caveats",
"",
"- **Survivorship bias**: today's constituents backfilled historically "
"(worse in small caps).",
"- **IEX volume undercount**: relative $vol rank only, not absolute floors.",
"- **Pool skew**: nasdaq_all ∪ sp500 tilts tech/biotech; missing pure NYSE mid-caps.",
"",
"## Fingerprint (505-name prod snapshot)",
"",
f"- Expected: IC ≈ {FINGERPRINT_IC}, t ≈ {FINGERPRINT_T}",
f"- Observed: IC = {fp.get('mean_ic')}, t = {fp.get('ic_t_stat')}, "
f"weeks = {fp.get('weeks')}, reliable = {fp.get('reliable')}",
f"- Pass: **{fp.get('pass')}**",
"",
"## Liquid-breadth signal_eval (fip_id)",
"",
]
row = br.get("row") or br
if row:
lines.extend([
f"| metric | value |",
f"|---|---|",
f"| mean_ic | {row.get('mean_ic')} |",
f"| ic_t_stat | {row.get('ic_t_stat')} |",
f"| ic_positive_pct | {row.get('ic_positive_pct')} |",
f"| weeks | {row.get('weeks')} |",
f"| avg_cross_section | {row.get('avg_cross_section')} |",
f"| reliable | {row.get('reliable')} |",
f"| mean_quintile_spread | {row.get('mean_quintile_spread')} |",
"",
])
else:
lines.append("_No breadth result (run skipped or failed)._")
lines.append("")
lines.extend([
"## Verdict (iron rule)",
"",
f"- **Green: {v.get('green')}**",
f"- {v.get('reason')}",
f"- Checks: `{json.dumps(v.get('checks') or {}, default=str)}`",
"",
"A green verdict authorizes a **follow-up proposal** only "
"(two-tier universe / gate revalidation) — **not** production wire-in.",
"",
"## Artifacts",
"",
f"- Fingerprint report: `{payload.get('fingerprint_report_path')}`",
f"- Breadth report: `{payload.get('breadth_report_path')}`",
"",
])
path.write_text("\n".join(lines), encoding="utf-8")
async def _main() -> None:
args = _parse_args()
prod = Path(args.prod_snapshot)
research = Path(args.research_snapshot)
if not prod.exists():
raise SystemExit(f"Prod snapshot missing: {prod}")
os.environ["BACKTEST_SNAPSHOT_OFFLINE"] = "1"
if args.allow_spawn:
os.environ["BACKTEST_ALLOW_SPAWN"] = "1"
os.environ["BACKTEST_SIGNAL_EVAL_ONLY"] = "1"
stamp = datetime.now().strftime("%Y%m%d-%H%M%S")
out_json = Path(args.out) if args.out else Path("reports") / f"fip-breadth-{stamp}.json"
out_json.parent.mkdir(parents=True, exist_ok=True)
out_md = Path("docs/research") / "fip-breadth-ic.md"
payload: dict = {
"generated_at": datetime.now().isoformat(),
"liquid_breadth_top_n": args.liquid_breadth,
"liquid_min_price": args.min_price,
"fingerprint": None,
"breadth": None,
"verdict": None,
}
# --- 1) Fingerprint ---
if not args.skip_fingerprint:
# Clear liquid breadth for fingerprint
os.environ.pop("BACKTEST_LIQUID_BREADTH", None)
os.environ.pop("BACKTEST_LIQUID_MIN_PRICE", None)
if not args.quiet:
print(f"Fingerprint run on {prod}…")
fp_report = await _run_signal_eval(prod, workers=args.workers, quiet=args.quiet)
fp_path = out_json.with_name(out_json.stem + "-fingerprint.json")
fp_path.write_text(json.dumps(fp_report, indent=2, default=str), encoding="utf-8")
fip = _find_fip(fp_report.get("signal_eval") or [])
if fip is None:
raise SystemExit("ABORT: fip_id missing from fingerprint signal_eval")
ic_ok = abs(float(fip["mean_ic"]) - FINGERPRINT_IC) <= FINGERPRINT_IC_TOL
t_ok = abs(float(fip["ic_t_stat"]) - FINGERPRINT_T) <= FINGERPRINT_T_TOL
passed = ic_ok and t_ok and bool(fip.get("reliable"))
payload["fingerprint"] = {
**fip,
"pass": passed,
"expected_ic": FINGERPRINT_IC,
"expected_t": FINGERPRINT_T,
}
payload["fingerprint_report_path"] = str(fp_path)
if not args.quiet:
print(
f"Fingerprint fip_id IC={fip.get('mean_ic')} t={fip.get('ic_t_stat')} "
f"pass={passed}"
)
if not passed:
out_json.write_text(json.dumps(payload, indent=2, default=str), encoding="utf-8")
raise SystemExit(
"ABORT: fingerprint mismatch — investigate before trusting breadth runs "
f"(got IC={fip.get('mean_ic')} t={fip.get('ic_t_stat')})"
)
# --- 2) Breadth ---
if not args.skip_research:
if not research.exists():
raise SystemExit(
f"Research snapshot missing: {research}\n"
"Build it with: python scripts/extend_snapshot_universe.py"
)
os.environ["BACKTEST_LIQUID_BREADTH"] = str(int(args.liquid_breadth))
os.environ["BACKTEST_LIQUID_MIN_PRICE"] = str(float(args.min_price))
if not args.quiet:
print(
f"Breadth run on {research} "
f"(top {args.liquid_breadth}, min_price={args.min_price})…"
)
br_report = await _run_signal_eval(
research, workers=args.workers, quiet=args.quiet
)
br_path = out_json.with_name(out_json.stem + "-breadth.json")
br_path.write_text(json.dumps(br_report, indent=2, default=str), encoding="utf-8")
fip_b = _find_fip(br_report.get("signal_eval") or [])
payload["breadth"] = fip_b or {"error": "fip_id missing"}
payload["breadth_report_path"] = str(br_path)
payload["breadth_tickers"] = br_report.get("tickers")
payload["breadth_rank_only_tickers"] = br_report.get("rank_only_tickers")
payload["verdict"] = _verdict(fip_b)
if not args.quiet:
print(
f"Breadth fip_id IC={ (fip_b or {}).get('mean_ic') } "
f"t={ (fip_b or {}).get('ic_t_stat') } "
f"green={payload['verdict'].get('green')}"
)
out_json.write_text(json.dumps(payload, indent=2, default=str), encoding="utf-8")
out_md.parent.mkdir(parents=True, exist_ok=True)
_write_md(out_md, payload)
if not args.quiet:
print(f"Wrote {out_json}")
print(f"Wrote {out_md}")
if __name__ == "__main__":
asyncio.run(_main())
tests/unit/test_backtest_service.py
-46
View File
@@ -1154,52 +1154,6 @@ class TestSimulatePortfolio:
assert allowed["trade_details"][0]["entry"] == pytest.approx(110.0) assert allowed["trade_details"][0]["entry"] == pytest.approx(110.0)
def test_median_dollar_vol_63_basic():
closes = [10.0] * 70
volumes = [100.0 + i for i in range(70)]
med = bt._median_dollar_vol_63(closes, volumes, 69, lookback=63)
assert med is not None
assert med > 0
def test_liquid_breadth_week_keeps_top_n_by_dvol():
recs = [
{"val": 0.1, "fwd": 0.01, "close": 20.0, "median_dvol_63": 1e6},
{"val": 0.2, "fwd": 0.02, "close": 20.0, "median_dvol_63": 9e6},
{"val": 0.3, "fwd": 0.03, "close": 20.0, "median_dvol_63": 5e6},
{"val": 0.4, "fwd": 0.04, "close": 1.0, "median_dvol_63": 99e6}, # price floor
{"val": 0.5, "fwd": 0.05, "close": 20.0, "median_dvol_63": None},
]
pairs = bt._filter_liquid_breadth_week(recs, top_n=2, min_price=5.0)
assert len(pairs) == 2
# Highest dvol first among eligible: 9e6 then 5e6
assert pairs[0][0] == pytest.approx(0.2)
assert pairs[1][0] == pytest.approx(0.3)
def test_signal_eval_liquid_breadth_env(monkeypatch):
# top_n=5 → keep 5 names/week; spearman needs ≥3 observations.
monkeypatch.setenv("BACKTEST_LIQUID_BREADTH", "5")
monkeypatch.setenv("BACKTEST_LIQUID_MIN_PRICE", "5")
monkeypatch.setattr(bt, "MIN_CROSS_SECTION", 3)
monkeypatch.setattr(bt, "MIN_RELIABLE_PERIODS", 3)
week = {}
for w in (1, 10, 20, 30, 40, 50):
week[(2024, w)] = [
{
"val": float(i),
"fwd": float(i) * 0.01,
"close": 10.0,
"median_dvol_63": float(100 - i),
}
for i in range(12)
]
rows = bt._signal_evaluation({"toy": week})
assert rows
assert rows[0]["liquid_breadth_top_n"] == 5
assert rows[0]["avg_cross_section"] == 5.0
def test_fip_id_sign_convention_steady_climber_vs_jump(): def test_fip_id_sign_convention_steady_climber_vs_jump():
# Steady climber: many up days, continuous path → lower (more negative) ID. # Steady climber: many up days, continuous path → lower (more negative) ID.
steady = [100.0] steady = [100.0]