From fa25b6ee68b225b6ee89b62c80ab8546aef08271 Mon Sep 17 00:00:00 2001 From: Dennis Thiessen Date: Sun, 19 Jul 2026 09:33:34 +0200 Subject: [PATCH 01/14] research: sector residual, earnings gap/SUE, history-depth scaffolding Tier-1 alpha research (local only, no production deploy): Sector residual momentum: two-factor SPY+sector residual and sector demean signals, IC harness + A/B. Sector resid clears pre-registered bars narrowly (PROMOTE for human wire design only). Sector demean fails t vs market resid. Earnings: earnings_events backfill (FMP bulk paid; FMP/AV per-symbol), 2a gap diagnostic report-only, 2b SUE IC (PARK; incomplete 48/506 coverage). History-depth: pre-registered doc + runner for MacBook deep rebuild/harness. Do not ship production residual or filters from this branch. --- app/services/backtest_service.py | 260 ++++- app/services/sector_map.py | 145 +++ data/research/ticker_sector_map.json | 543 +++++++++ docs/research/earnings-gap-and-sue.md | 202 ++++ docs/research/history-depth-extension.md | 108 ++ docs/research/sector-residual-momentum.md | 237 ++++ reports/earnings-backfill-status.json | 15 + reports/earnings-gap-sue-20260719-093129.json | 331 ++++++ reports/earnings-gap-sue-20260719-093129.md | 202 ++++ reports/sector-residual-20260719-083356.json | 412 +++++++ reports/sector-residual-20260719-083356.md | 237 ++++ scripts/backfill_earnings_events.py | 528 +++++++++ scripts/build_ticker_sector_map.py | 229 ++++ scripts/fetch_sector_etfs_to_snapshot.py | 183 +++ scripts/run_earnings_research.py | 927 +++++++++++++++ scripts/run_history_depth_research.py | 476 ++++++++ scripts/run_sector_residual_research.py | 1014 +++++++++++++++++ tests/unit/test_backtest_service.py | 69 ++ 18 files changed, 6093 insertions(+), 25 deletions(-) create mode 100644 app/services/sector_map.py create mode 100644 data/research/ticker_sector_map.json create mode 100644 docs/research/earnings-gap-and-sue.md create mode 100644 docs/research/history-depth-extension.md create mode 100644 docs/research/sector-residual-momentum.md create mode 100644 reports/earnings-backfill-status.json create mode 100644 reports/earnings-gap-sue-20260719-093129.json create mode 100644 reports/earnings-gap-sue-20260719-093129.md create mode 100644 reports/sector-residual-20260719-083356.json create mode 100644 reports/sector-residual-20260719-083356.md create mode 100644 scripts/backfill_earnings_events.py create mode 100644 scripts/build_ticker_sector_map.py create mode 100644 scripts/fetch_sector_etfs_to_snapshot.py create mode 100644 scripts/run_earnings_research.py create mode 100644 scripts/run_history_depth_research.py create mode 100644 scripts/run_sector_residual_research.py diff --git a/app/services/backtest_service.py b/app/services/backtest_service.py index 6b29be4..55280db 100644 --- a/app/services/backtest_service.py +++ b/app/services/backtest_service.py @@ -791,31 +791,86 @@ def _residual_momentum_12_1( with an intercept estimated over the same window, the arithmetic residuals sum to ~zero by construction, which would destroy the signal. """ - if not benchmark_closes or i - 252 < 0: + return _multi_factor_residual_momentum_12_1( + dates, closes, i, [benchmark_closes] if benchmark_closes else None + ) + + +def _multi_factor_residual_momentum_12_1( + dates: list[date], + closes: list[float], + i: int, + factor_closes: list[dict[date, float]] | None, +) -> float | None: + """12-1 residual momentum vs one or more factors (OLS, no intercept). + + Same formation window as raw / single-factor residual momentum: + daily returns from close[i-252] → close[i-21], require ≥100 paired obs. + Factors are stacked as columns; betas are OLS without intercept so the + cumulative residual is not forced to zero. + """ + if not factor_closes or i - 252 < 0: + return None + n_factors = len(factor_closes) + if n_factors < 1: return None stock_rets: list[float] = [] - market_rets: list[float] = [] - # Same daily intervals as mom_12_1: close[i-252] -> close[i-21]. + factor_rets: list[list[float]] = [[] for _ in range(n_factors)] for k in range(i - 251, i - 20): prev_close = closes[k - 1] - bench_prev = benchmark_closes.get(dates[k - 1]) - bench_cur = benchmark_closes.get(dates[k]) - if prev_close <= 0 or bench_prev is None or bench_cur is None or bench_prev <= 0: + if prev_close <= 0: + continue + f_day: list[float] = [] + ok = True + for fc in factor_closes: + f_prev = fc.get(dates[k - 1]) + f_cur = fc.get(dates[k]) + if f_prev is None or f_cur is None or f_prev <= 0: + ok = False + break + f_day.append(f_cur / f_prev - 1.0) + if not ok: continue stock_rets.append(closes[k] / prev_close - 1.0) - market_rets.append(bench_cur / bench_prev - 1.0) + for j, r in enumerate(f_day): + factor_rets[j].append(r) - if len(stock_rets) < 100: + n = len(stock_rets) + if n < 100: return None - mean_market = sum(market_rets) / len(market_rets) - mean_stock = sum(stock_rets) / len(stock_rets) - var_market = sum((x - mean_market) ** 2 for x in market_rets) - if var_market <= 0: + + if n_factors == 1: + # Fast path: identical algebra to the historical single-factor form. + market_rets = factor_rets[0] + mean_market = sum(market_rets) / n + mean_stock = sum(stock_rets) / n + var_market = sum((x - mean_market) ** 2 for x in market_rets) + if var_market <= 0: + return None + cov = sum( + (stock_rets[k] - mean_stock) * (market_rets[k] - mean_market) + for k in range(n) + ) + beta = cov / var_market + return sum(stock_rets[k] - beta * market_rets[k] for k in range(n)) + + # OLS without intercept: β = (X'X)^{-1} X'y for X columns = factor returns. + # Implemented for exactly two factors (market + sector); refuse larger. + if n_factors != 2: return None - cov = sum((stock_rets[k] - mean_stock) * (market_rets[k] - mean_market) for k in range(len(stock_rets))) - beta = cov / var_market - return sum(stock_rets[k] - beta * market_rets[k] for k in range(len(stock_rets))) + f1, f2 = factor_rets[0], factor_rets[1] + s11 = sum(a * a for a in f1) + s22 = sum(a * a for a in f2) + s12 = sum(f1[k] * f2[k] for k in range(n)) + sy1 = sum(stock_rets[k] * f1[k] for k in range(n)) + sy2 = sum(stock_rets[k] * f2[k] for k in range(n)) + det = s11 * s22 - s12 * s12 + if abs(det) < 1e-18: + return None + b1 = (s22 * sy1 - s12 * sy2) / det + b2 = (s11 * sy2 - s12 * sy1) / det + return sum(stock_rets[k] - b1 * f1[k] - b2 * f2[k] for k in range(n)) def _realized_vol_6m(closes: list[float], i: int) -> float | None: @@ -840,6 +895,7 @@ def _signal_values( highs: list[float], i: int, benchmark_closes: dict[date, float] | None = None, + sector_etf_closes: dict[date, float] | None = None, ) -> dict[str, float]: """Point-in-time candidate signals at as-of index ``i`` (price-only). @@ -851,6 +907,11 @@ def _signal_values( higher = nearer the high, expect positive IC). ``vol_6m`` is 126-day realized volatility (expect negative IC if the low-volatility anomaly holds). ``fip_id`` is Da/Gurun/Warachka information discreteness (expect negative IC). + + When ``sector_etf_closes`` is supplied (research path), also emit + ``mom_12_1_sector_resid``: two-factor residual vs SPY + sector ETF. + Cross-sectional ``mom_12_1_sector_demeaned`` is injected later from the + full weekly cross-section (cannot be computed per-ticker alone). """ out: dict[str, float] = {} if i - 252 >= 0 and closes[i - 252] > 0: @@ -858,6 +919,12 @@ def _signal_values( residual = _residual_momentum_12_1(dates, closes, i, benchmark_closes) if residual is not None: out["mom_12_1_resid"] = residual + if benchmark_closes and sector_etf_closes: + sector_resid = _multi_factor_residual_momentum_12_1( + dates, closes, i, [benchmark_closes, sector_etf_closes] + ) + if sector_resid is not None: + out["mom_12_1_sector_resid"] = sector_resid fip = _fip_id(closes, i) if fip is not None: out["fip_id"] = fip @@ -944,14 +1011,16 @@ def _accumulate_signal_series( benchmark_closes: dict[date, float] | None = None, *, symbol: str | None = None, + sector_etf_closes: dict[date, float] | None = None, ) -> None: """For each weekly as-of bar, emit (signal, forward-return) pairs keyed by ISO week into ``collected[name][week_key]``. Forward return is close-to-close over HORIZON trading days. Mutates ``collected`` (a dict of dict of list). When ``BACKTEST_LIQUID_BREADTH`` is set, observations are dicts with PIT - liquidity fields for the mask; otherwise plain ``(val, fwd)`` tuples so the - production signal path stays unchanged. + liquidity fields for the mask. When ``symbol`` is provided, observations are + also dicts (so sector demeaning can group by name); otherwise plain + ``(val, fwd)`` tuples keep the production path unchanged. """ n = len(records) if n < HORIZON + 21: @@ -961,6 +1030,7 @@ def _accumulate_signal_series( volumes = [float(getattr(r, "volume", 0) or 0) for r in records] dates = [r.date for r in records] liquid_mode = _liquid_breadth_top_n() > 0 + rich = liquid_mode or symbol is not None for i in _weekly_asof_indices(records): j = i + HORIZON if j >= n or closes[i] <= 0: @@ -969,19 +1039,79 @@ def _accumulate_signal_series( iso = records[i].date.isocalendar() week_key = (iso[0], iso[1]) dvol = _median_dollar_vol_63(closes, volumes, i) if liquid_mode else None - for name, val in _signal_values(dates, closes, highs, i, benchmark_closes).items(): - if liquid_mode: - collected[name][week_key].append({ + for name, val in _signal_values( + dates, closes, highs, i, benchmark_closes, sector_etf_closes + ).items(): + if rich: + row = { "val": val, "fwd": fwd, - "close": closes[i], - "median_dvol_63": dvol, "symbol": symbol, - }) + } + if liquid_mode: + row["close"] = closes[i] + row["median_dvol_63"] = dvol + collected[name][week_key].append(row) else: collected[name][week_key].append((val, fwd)) +def _inject_sector_demeaned_momentum( + collected: dict, + symbol_to_sector: dict[str, str], + *, + min_sector_names: int = 2, +) -> None: + """Cross-sectional demean of ``mom_12_1`` within GICS sector per week. + + ``mom_12_1_sector_demeaned[i] = mom_12_1[i] − mean(mom_12_1 | sector_i)``. + Requires rich observations with a ``symbol`` field (research path). Names + without a sector label, or sectors with fewer than ``min_sector_names`` + members that week, are dropped from the demeaned series. + """ + if not symbol_to_sector or "mom_12_1" not in collected: + return + from app.services.sector_map import normalise_symbol + + demeaned: dict = defaultdict(list) + for week_key, recs in collected["mom_12_1"].items(): + parsed: list[tuple[str, float, float, object]] = [] + by_sector: dict[str, list[float]] = defaultdict(list) + for rec in recs: + pair = _obs_val_fwd(rec) + if pair is None: + continue + val, fwd = pair + if isinstance(rec, dict): + sym = rec.get("symbol") + else: + sym = None + if not sym: + continue + sector = symbol_to_sector.get(normalise_symbol(str(sym))) + if not sector: + continue + parsed.append((sector, val, fwd, rec)) + by_sector[sector].append(val) + means = { + sec: sum(vs) / len(vs) + for sec, vs in by_sector.items() + if len(vs) >= min_sector_names + } + for sector, val, fwd, rec in parsed: + if sector not in means: + continue + dval = val - means[sector] + if isinstance(rec, dict): + row = dict(rec) + row["val"] = dval + demeaned[week_key].append(row) + else: + demeaned[week_key].append((dval, fwd)) + if demeaned: + collected["mom_12_1_sector_demeaned"] = demeaned + + def _rank(xs: list[float]) -> list[float]: """Average (tie-corrected) ranks, 1-based.""" order = sorted(range(len(xs)), key=lambda k: xs[k]) @@ -1258,14 +1388,38 @@ def _signal_series( benchmark_closes: dict[date, float] | None = None, *, symbol: str | None = None, + sector_etf_closes: dict[date, float] | None = None, ) -> dict: """Per-ticker signal/forward-return series as a PLAIN (picklable) nested dict — no defaultdict/lambda — so it can cross a process boundary.""" tmp: dict = defaultdict(lambda: defaultdict(list)) - _accumulate_signal_series(records, tmp, benchmark_closes, symbol=symbol) + _accumulate_signal_series( + records, + tmp, + benchmark_closes, + symbol=symbol, + sector_etf_closes=sector_etf_closes, + ) return {name: dict(weeks) for name, weeks in tmp.items()} +def _sector_etf_closes_for_symbol( + symbol: str, + symbol_to_sector: dict[str, str] | None, + sector_etf_closes: dict[str, dict[date, float]] | None, +) -> dict[date, float] | None: + """Resolve the sector-ETF close series for one ticker, or None.""" + if not symbol_to_sector or not sector_etf_closes: + return None + from app.services.sector_map import etf_for_symbol + + etf = etf_for_symbol(symbol, symbol_to_sector) + if not etf: + return None + series = sector_etf_closes.get(etf) + return series or None + + def _replay_and_signals( symbol: str, columns: tuple, @@ -1275,6 +1429,8 @@ def _replay_and_signals( target_model: str = PRODUCTION_GTL_TARGET_MODEL, cadence: str = DEFAULT_BACKTEST_CADENCE, signal_only: bool = False, + sector_etf_closes: dict[str, dict[date, float]] | None = None, + symbol_to_sector: dict[str, str] | None = None, ) -> tuple[list[dict], dict]: """The CPU-bound per-ticker work, as a top-level (picklable) function so it can run in a worker process. Takes primitive column arrays (cheap to pickle), @@ -1301,9 +1457,17 @@ def _replay_and_signals( target_model, cadence, ) + etf_closes = _sector_etf_closes_for_symbol( + symbol, symbol_to_sector, sector_etf_closes + ) return ( candidates, - _signal_series(bars, benchmark_closes, symbol=symbol), + _signal_series( + bars, + benchmark_closes, + symbol=symbol, + sector_etf_closes=etf_closes, + ), ) @@ -4054,6 +4218,41 @@ async def run_backtest( except Exception: logger.exception("Benchmark load for residual momentum failed") + # Optional sector residualisation (research): local ticker→sector map + sector + # ETF closes stored in benchmark_prices. Absent map/series → no sector signals. + symbol_to_sector: dict[str, str] = {} + sector_etf_closes: dict[str, dict[date, float]] = {} + try: + from app.services.sector_map import ( + SECTOR_ETFS, + load_ticker_sector_map, + normalise_symbol, + ) + from app.services.benchmark_service import load_benchmark_closes + + map_path = os.getenv("BACKTEST_SECTOR_MAP_PATH", "").strip() or None + symbol_to_sector = { + normalise_symbol(k): v + for k, v in load_ticker_sector_map(map_path).items() + } + if symbol_to_sector: + for etf in SECTOR_ETFS: + try: + series = await load_benchmark_closes(db, etf) + except Exception: + series = {} + if series: + sector_etf_closes[etf] = series + logger.info(json.dumps({ + "event": "backtest_sector_context_loaded", + "sector_map_size": len(symbol_to_sector), + "sector_etfs_loaded": sorted(sector_etf_closes), + })) + except Exception: + logger.exception("Sector residual context load failed; continuing without") + symbol_to_sector = {} + sector_etf_closes = {} + def _merge(result: tuple[list[dict], dict]) -> None: cands, series = result candidates.extend(cands) @@ -4104,6 +4303,8 @@ async def run_backtest( target_model, cadence, ticker.symbol in rank_only_symbols, + sector_etf_closes or None, + symbol_to_sector or None, )) for result in await asyncio.gather(*futures, return_exceptions=True): if isinstance(result, Exception): @@ -4132,6 +4333,8 @@ async def run_backtest( target_model, cadence, ticker.symbol in rank_only_symbols, + sector_etf_closes or None, + symbol_to_sector or None, )) except Exception: logger.exception("Backtest replay failed for %s", ticker.symbol) @@ -4139,6 +4342,13 @@ async def run_backtest( if progress_cb is not None and total: progress_cb(total, total, "") + # Cross-sectional sector demean needs the full weekly universe. + if symbol_to_sector: + try: + _inject_sector_demeaned_momentum(collected, symbol_to_sector) + except Exception: + logger.exception("Sector demeaned momentum injection failed") + # Cross-sectional momentum: rank every week's universe, then "qualified" means # floors + top ``min_momentum_percentile`` by promoted residual 12-1 momentum # (raw 12-1 fallback only when benchmark data is unavailable). diff --git a/app/services/sector_map.py b/app/services/sector_map.py new file mode 100644 index 0000000..f72230c --- /dev/null +++ b/app/services/sector_map.py @@ -0,0 +1,145 @@ +"""Ticker → GICS sector → SPDR sector ETF mapping (research only). + +Sector residual momentum residualizes 12-1 momentum against SPY and the name's +sector ETF. Labels are persisted under ``data/research/ticker_sector_map.json`` +so research runs do not depend on live FMP calls. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +# Eleven SPDR sector ETFs. Auxiliary series only — never tradable book members. +SECTOR_ETFS: tuple[str, ...] = ( + "XLB", + "XLC", + "XLE", + "XLF", + "XLI", + "XLK", + "XLP", + "XLRE", + "XLU", + "XLV", + "XLY", +) + +# GICS sector name (and common aliases) → SPDR ETF. +# Keys are lower-case for matching. +GICS_SECTOR_TO_ETF: dict[str, str] = { + "materials": "XLB", + "basic materials": "XLB", + "communication services": "XLC", + "communications": "XLC", + "telecommunication services": "XLC", + "energy": "XLE", + "financials": "XLF", + "financial services": "XLF", + "financial": "XLF", + "industrials": "XLI", + "industrial goods": "XLI", + "information technology": "XLK", + "technology": "XLK", + "consumer staples": "XLP", + "consumer defensive": "XLP", + "real estate": "XLRE", + "utilities": "XLU", + "health care": "XLV", + "healthcare": "XLV", + "consumer discretionary": "XLY", + "consumer cyclical": "XLY", +} + +DEFAULT_SECTOR_MAP_PATH = Path("data/research/ticker_sector_map.json") + + +def normalise_symbol(symbol: str) -> str: + """Alpaca-style symbols: BRK.B / BRK/B → BRK-B.""" + s = str(symbol or "").strip().upper() + s = s.replace(".", "-").replace("/", "-") + return s + + +def sector_to_etf(sector: str | None) -> str | None: + if not sector: + return None + return GICS_SECTOR_TO_ETF.get(str(sector).strip().lower()) + + +def etf_for_symbol(symbol: str, symbol_to_sector: dict[str, str]) -> str | None: + sector = symbol_to_sector.get(normalise_symbol(symbol)) + return sector_to_etf(sector) + + +def load_ticker_sector_map(path: Path | str | None = None) -> dict[str, str]: + """Load ``{symbol: gics_sector}`` from JSON. Empty dict if missing.""" + p = Path(path) if path is not None else DEFAULT_SECTOR_MAP_PATH + if not p.exists(): + return {} + raw = json.loads(p.read_text(encoding="utf-8")) + if not isinstance(raw, dict): + return {} + out: dict[str, str] = {} + # Accept either flat map or {"map": {...}, "meta": ...} + payload = raw.get("map") if "map" in raw and isinstance(raw.get("map"), dict) else raw + if not isinstance(payload, dict): + return {} + for sym, sector in payload.items(): + if sym in ("meta", "schema_version", "map"): + continue + if sector is None: + continue + ns = normalise_symbol(str(sym)) + if ns: + out[ns] = str(sector).strip() + return out + + +def save_ticker_sector_map( + mapping: dict[str, str], + path: Path | str | None = None, + *, + meta: dict[str, Any] | None = None, +) -> Path: + p = Path(path) if path is not None else DEFAULT_SECTOR_MAP_PATH + p.parent.mkdir(parents=True, exist_ok=True) + # Normalise keys on write. + clean = { + normalise_symbol(k): str(v).strip() + for k, v in mapping.items() + if k and v and normalise_symbol(k) + } + payload: dict[str, Any] = { + "schema_version": 1, + "map": clean, + "meta": meta or {}, + } + p.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") + return p + + +def coverage_stats( + symbols: list[str], mapping: dict[str, str] +) -> dict[str, Any]: + total = len(symbols) + mapped = [s for s in symbols if normalise_symbol(s) in mapping] + with_etf = [ + s + for s in mapped + if sector_to_etf(mapping[normalise_symbol(s)]) is not None + ] + missing = [s for s in symbols if normalise_symbol(s) not in mapping] + by_sector: dict[str, int] = {} + for s in mapped: + sec = mapping[normalise_symbol(s)] + by_sector[sec] = by_sector.get(sec, 0) + 1 + return { + "universe": total, + "mapped": len(mapped), + "mapped_pct": round(100.0 * len(mapped) / total, 1) if total else 0.0, + "with_etf": len(with_etf), + "missing": missing, + "by_sector": dict(sorted(by_sector.items(), key=lambda kv: (-kv[1], kv[0]))), + } diff --git a/data/research/ticker_sector_map.json b/data/research/ticker_sector_map.json new file mode 100644 index 0000000..31cc0db --- /dev/null +++ b/data/research/ticker_sector_map.json @@ -0,0 +1,543 @@ +{ + "map": { + "A": "Health Care", + "AAPL": "Information Technology", + "ABBV": "Health Care", + "ABNB": "Consumer Discretionary", + "ABT": "Health Care", + "ACGL": "Financials", + "ACN": "Information Technology", + "ADBE": "Information Technology", + "ADI": "Information Technology", + "ADM": "Consumer Staples", + "ADP": "Industrials", + "ADSK": "Information Technology", + "AEE": "Utilities", + "AEP": "Utilities", + "AES": "Utilities", + "AFL": "Financials", + "AIG": "Financials", + "AIZ": "Financials", + "AJG": "Financials", + "AKAM": "Information Technology", + "ALB": "Materials", + "ALGN": "Health Care", + "ALL": "Financials", + "ALLE": "Industrials", + "AMAT": "Information Technology", + "AMCR": "Materials", + "AMD": "Information Technology", + "AME": "Industrials", + "AMGN": "Health Care", + "AMP": "Financials", + "AMT": "Real Estate", + "AMZN": "Consumer Discretionary", + "ANET": "Information Technology", + "AON": "Financials", + "AOS": "Industrials", + "APA": "Energy", + "APD": "Materials", + "APH": "Information Technology", + "APO": "Financials", + "APP": "Information Technology", + "APTV": "Consumer Discretionary", + "ARE": "Real Estate", + "ARES": "Financials", + "ATO": "Utilities", + "AVB": "Real Estate", + "AVGO": "Information Technology", + "AVY": "Materials", + "AWK": "Utilities", + "AXON": "Industrials", + "AXP": "Financials", + "AZO": "Consumer Discretionary", + "BA": "Industrials", + "BAC": "Financials", + "BALL": "Materials", + "BAX": "Health Care", + "BBY": "Consumer Discretionary", + "BDX": "Health Care", + "BEN": "Financials", + "BF-B": "Consumer Staples", + "BG": "Consumer Staples", + "BIIB": "Health Care", + "BK": "Financial Services", + "BKNG": "Consumer Discretionary", + "BKR": "Energy", + "BLDR": "Industrials", + "BLK": "Financials", + "BMY": "Health Care", + "BR": "Industrials", + "BRK-B": "Financials", + "BRO": "Financials", + "BSX": "Health Care", + "BX": "Financials", + "BXP": "Real Estate", + "C": "Financials", + "CAG": "Consumer Defensive", + "CAH": "Health Care", + "CARR": "Industrials", + "CASY": "Consumer Staples", + "CAT": "Industrials", + "CB": "Financials", + "CBOE": "Financials", + "CBRE": "Real Estate", + "CCI": "Real Estate", + "CCL": "Consumer Discretionary", + "CDNS": "Information Technology", + "CDW": "Information Technology", + "CEG": "Utilities", + "CF": "Materials", + "CFG": "Financials", + "CHD": "Consumer Staples", + "CHRW": "Industrials", + "CHTR": "Communication Services", + "CI": "Health Care", + "CIEN": "Information Technology", + "CINF": "Financials", + "CL": "Consumer Staples", + "CLX": "Consumer Staples", + "CMCSA": "Communication Services", + "CME": "Financials", + "CMG": "Consumer Discretionary", + "CMI": "Industrials", + "CMS": "Utilities", + "CNC": "Health Care", + "CNP": "Utilities", + "COF": "Financials", + "COHR": "Information Technology", + "COIN": "Financials", + "COO": "Health Care", + "COP": "Energy", + "COR": "Health Care", + "COST": "Consumer Staples", + "CPAY": "Financials", + "CPB": "Consumer Defensive", + "CPRT": "Industrials", + "CPT": "Real Estate", + "CRH": "Materials", + "CRL": "Health Care", + "CRM": "Information Technology", + "CRWD": "Information Technology", + "CSCO": "Information Technology", + "CSGP": "Real Estate", + "CSX": "Industrials", + "CTAS": "Industrials", + "CTRA": "Energy", + "CTSH": "Information Technology", + "CTVA": "Materials", + "CVNA": "Consumer Discretionary", + "CVS": "Health Care", + "CVX": "Energy", + "D": "Utilities", + "DAL": "Industrials", + "DASH": "Consumer Discretionary", + "DD": "Materials", + "DDOG": "Information Technology", + "DE": "Industrials", + "DECK": "Consumer Discretionary", + "DELL": "Information Technology", + "DG": "Consumer Staples", + "DGX": "Health Care", + "DHI": "Consumer Discretionary", + "DHR": "Health Care", + "DIS": "Communication Services", + "DLR": "Real Estate", + "DLTR": "Consumer Staples", + "DOC": "Real Estate", + "DOV": "Industrials", + "DOW": "Materials", + "DPZ": "Consumer Discretionary", + "DRI": "Consumer Discretionary", + "DTE": "Utilities", + "DUK": "Utilities", + "DVA": "Health Care", + "DVN": "Energy", + "DXCM": "Health Care", + "EA": "Communication Services", + "EBAY": "Consumer Discretionary", + "ECL": "Materials", + "ED": "Utilities", + "EFX": "Industrials", + "EG": "Financials", + "EIX": "Utilities", + "EL": "Consumer Staples", + "ELV": "Health Care", + "EME": "Industrials", + "EMR": "Industrials", + "EOG": "Energy", + "EPAM": "Technology", + "EQIX": "Real Estate", + "EQR": "Real Estate", + "EQT": "Energy", + "ERIE": "Financials", + "ES": "Utilities", + "ESS": "Real Estate", + "ETN": "Industrials", + "ETR": "Utilities", + "EVRG": "Utilities", + "EW": "Health Care", + "EXC": "Utilities", + "EXE": "Energy", + "EXPD": "Industrials", + "EXPE": "Consumer Discretionary", + "EXR": "Real Estate", + "F": "Consumer Discretionary", + "FANG": "Energy", + "FAST": "Industrials", + "FCX": "Materials", + "FDS": "Financials", + "FDX": "Industrials", + "FE": "Utilities", + "FFIV": "Information Technology", + "FICO": "Information Technology", + "FIS": "Financials", + "FISV": "Financials", + "FITB": "Financials", + "FIX": "Industrials", + "FOX": "Communication Services", + "FOXA": "Communication Services", + "FRT": "Real Estate", + "FSLR": "Information Technology", + "FTNT": "Information Technology", + "FTV": "Industrials", + "GD": "Industrials", + "GDDY": "Information Technology", + "GE": "Industrials", + "GEHC": "Health Care", + "GEN": "Information Technology", + "GEV": "Industrials", + "GILD": "Health Care", + "GIS": "Consumer Staples", + "GL": "Financials", + "GLW": "Information Technology", + "GM": "Consumer Discretionary", + "GNRC": "Industrials", + "GOOG": "Communication Services", + "GOOGL": "Communication Services", + "GPC": "Consumer Discretionary", + "GPN": "Financials", + "GRMN": "Consumer Discretionary", + "GS": "Financials", + "GWW": "Industrials", + "HAL": "Energy", + "HAS": "Consumer Discretionary", + "HBAN": "Financials", + "HCA": "Health Care", + "HD": "Consumer Discretionary", + "HIG": "Financials", + "HII": "Industrials", + "HLT": "Consumer Discretionary", + "HON": "Industrials", + "HOOD": "Financials", + "HPE": "Information Technology", + "HPQ": "Information Technology", + "HRL": "Consumer Staples", + "HSIC": "Health Care", + "HST": "Real Estate", + "HSY": "Consumer Staples", + "HUBB": "Industrials", + "HUM": "Health Care", + "HWM": "Industrials", + "IBKR": "Financials", + "IBM": "Information Technology", + "ICE": "Financials", + "IDXX": "Health Care", + "IEX": "Industrials", + "IFF": "Materials", + "INCY": "Health Care", + "INTC": "Information Technology", + "INTU": "Information Technology", + "INVH": "Real Estate", + "IP": "Materials", + "IQV": "Health Care", + "IR": "Industrials", + "IRM": "Real Estate", + "ISRG": "Health Care", + "IT": "Information Technology", + "ITW": "Industrials", + "IVZ": "Financials", + "J": "Industrials", + "JBHT": "Industrials", + "JBL": "Information Technology", + "JCI": "Industrials", + "JKHY": "Financials", + "JNJ": "Health Care", + "JPM": "Financials", + "KDP": "Consumer Staples", + "KEY": "Financials", + "KEYS": "Information Technology", + "KHC": "Consumer Staples", + "KIM": "Real Estate", + "KKR": "Financials", + "KLAC": "Information Technology", + "KMB": "Consumer Staples", + "KMI": "Energy", + "KO": "Consumer Staples", + "KR": "Consumer Staples", + "KVUE": "Consumer Staples", + "L": "Financials", + "LDOS": "Industrials", + "LEN": "Consumer Discretionary", + "LH": "Health Care", + "LHX": "Industrials", + "LII": "Industrials", + "LIN": "Materials", + "LITE": "Information Technology", + "LLY": "Health Care", + "LMT": "Industrials", + "LNT": "Utilities", + "LOW": "Consumer Discretionary", + "LRCX": "Information Technology", + "LULU": "Consumer Discretionary", + "LUV": "Industrials", + "LVS": "Consumer Discretionary", + "LYB": "Materials", + "LYV": "Communication Services", + "MA": "Financials", + "MAA": "Real Estate", + "MAR": "Consumer Discretionary", + "MAS": "Industrials", + "MCD": "Consumer Discretionary", + "MCHP": "Information Technology", + "MCK": "Health Care", + "MCO": "Financials", + "MDLZ": "Consumer Staples", + "MDT": "Health Care", + "MET": "Financials", + "META": "Communication Services", + "MGM": "Consumer Discretionary", + "MKC": "Consumer Staples", + "MLM": "Materials", + "MMM": "Industrials", + "MNST": "Consumer Staples", + "MO": "Consumer Staples", + "MOS": "Materials", + "MPC": "Energy", + "MPWR": "Information Technology", + "MRK": "Health Care", + "MRNA": "Health Care", + "MRSH": "Financials", + "MS": "Financials", + "MSCI": "Financials", + "MSFT": "Information Technology", + "MSI": "Information Technology", + "MSTR": "Technology", + "MTB": "Financials", + "MTD": "Health Care", + "MU": "Information Technology", + "NCLH": "Consumer Discretionary", + "NDAQ": "Financials", + "NDSN": "Industrials", + "NEE": "Utilities", + "NEM": "Materials", + "NFLX": "Communication Services", + "NI": "Utilities", + "NKE": "Consumer Discretionary", + "NOC": "Industrials", + "NOW": "Information Technology", + "NRG": "Utilities", + "NSC": "Industrials", + "NTAP": "Information Technology", + "NTRS": "Financials", + "NUE": "Materials", + "NVDA": "Information Technology", + "NVR": "Consumer Discretionary", + "NWS": "Communication Services", + "NWSA": "Communication Services", + "NXPI": "Information Technology", + "O": "Real Estate", + "ODFL": "Industrials", + "OKE": "Energy", + "OMC": "Communication Services", + "ON": "Information Technology", + "ORCL": "Information Technology", + "ORLY": "Consumer Discretionary", + "OTIS": "Industrials", + "OXY": "Energy", + "PANW": "Information Technology", + "PAYX": "Industrials", + "PCAR": "Industrials", + "PCG": "Utilities", + "PEG": "Utilities", + "PEP": "Consumer Staples", + "PFE": "Health Care", + "PFG": "Financials", + "PG": "Consumer Staples", + "PGR": "Financials", + "PH": "Industrials", + "PHM": "Consumer Discretionary", + "PKG": "Materials", + "PLD": "Real Estate", + "PLTR": "Information Technology", + "PM": "Consumer Staples", + "PNC": "Financials", + "PNR": "Industrials", + "PNW": "Utilities", + "PODD": "Health Care", + "POOL": "Industrials", + "PPG": "Materials", + "PPL": "Utilities", + "PRU": "Financials", + "PSA": "Real Estate", + "PSKY": "Communication Services", + "PSX": "Energy", + "PTC": "Information Technology", + "PWR": "Industrials", + "PYPL": "Financials", + "Q": "Information Technology", + "QCOM": "Information Technology", + "RCL": "Consumer Discretionary", + "REG": "Real Estate", + "REGN": "Health Care", + "RF": "Financials", + "RJF": "Financials", + "RL": "Consumer Discretionary", + "RMD": "Health Care", + "ROK": "Industrials", + "ROL": "Industrials", + "ROP": "Information Technology", + "ROST": "Consumer Discretionary", + "RSG": "Industrials", + "RTX": "Industrials", + "RVTY": "Health Care", + "SATS": "Communication Services", + "SBAC": "Real Estate", + "SBUX": "Consumer Discretionary", + "SCHW": "Financials", + "SHW": "Materials", + "SJM": "Consumer Staples", + "SLB": "Energy", + "SMCI": "Information Technology", + "SNA": "Industrials", + "SNDK": "Information Technology", + "SNPS": "Information Technology", + "SO": "Utilities", + "SOLV": "Health Care", + "SPCX": "Industrials", + "SPG": "Real Estate", + "SPGI": "Financials", + "SRE": "Utilities", + "STE": "Health Care", + "STLD": "Materials", + "STT": "Financials", + "STX": "Information Technology", + "STZ": "Consumer Staples", + "SW": "Materials", + "SWK": "Industrials", + "SWKS": "Information Technology", + "SYF": "Financials", + "SYK": "Health Care", + "SYY": "Consumer Staples", + "T": "Communication Services", + "TAP": "Consumer Staples", + "TDG": "Industrials", + "TDY": "Information Technology", + "TECH": "Health Care", + "TEL": "Information Technology", + "TER": "Information Technology", + "TFC": "Financials", + "TGT": "Consumer Staples", + "TJX": "Consumer Discretionary", + "TKO": "Communication Services", + "TMO": "Health Care", + "TMUS": "Communication Services", + "TPL": "Energy", + "TPR": "Consumer Discretionary", + "TRGP": "Energy", + "TRMB": "Information Technology", + "TROW": "Financials", + "TRV": "Financials", + "TSCO": "Consumer Discretionary", + "TSLA": "Consumer Discretionary", + "TSN": "Consumer Staples", + "TT": "Industrials", + "TTD": "Communication Services", + "TTWO": "Communication Services", + "TXN": "Information Technology", + "TXT": "Industrials", + "TYL": "Information Technology", + "UAL": "Industrials", + "UBER": "Industrials", + "UDR": "Real Estate", + "UHS": "Health Care", + "ULTA": "Consumer Discretionary", + "UNH": "Health Care", + "UNP": "Industrials", + "UPS": "Industrials", + "URI": "Industrials", + "USB": "Financials", + "V": "Financials", + "VICI": "Real Estate", + "VLO": "Energy", + "VLTO": "Industrials", + "VMC": "Materials", + "VRSK": "Industrials", + "VRSN": "Information Technology", + "VRT": "Industrials", + "VRTX": "Health Care", + "VST": "Utilities", + "VTR": "Real Estate", + "VTRS": "Health Care", + "VZ": "Communication Services", + "WAB": "Industrials", + "WAT": "Health Care", + "WBD": "Communication Services", + "WDAY": "Information Technology", + "WDC": "Information Technology", + "WEC": "Utilities", + "WELL": "Real Estate", + "WFC": "Financials", + "WM": "Industrials", + "WMB": "Energy", + "WMT": "Consumer Staples", + "WRB": "Financials", + "WSM": "Consumer Discretionary", + "WST": "Health Care", + "WTW": "Financials", + "WY": "Real Estate", + "WYNN": "Consumer Discretionary", + "XEL": "Utilities", + "XOM": "Energy", + "XYL": "Industrials", + "XYZ": "Financials", + "YUM": "Consumer Discretionary", + "ZBH": "Health Care", + "ZBRA": "Information Technology", + "ZTS": "Health Care" + }, + "meta": { + "built_at": "2026-07-19T05:35:41.184460+00:00", + "coverage": { + "by_sector": { + "Communication Services": 23, + "Consumer Defensive": 2, + "Consumer Discretionary": 47, + "Consumer Staples": 34, + "Energy": 22, + "Financial Services": 1, + "Financials": 75, + "Health Care": 58, + "Industrials": 81, + "Information Technology": 72, + "Materials": 26, + "Real Estate": 31, + "Technology": 2, + "Utilities": 31 + }, + "mapped": 505, + "mapped_pct": 99.8, + "universe": 506, + "with_etf": 505 + }, + "fmp_requests": 10, + "from_existing": 0, + "from_fmp": 9, + "from_sp500_csv": 496, + "snapshot": "C:\\Workspace\\signal-platform\\backtest_snapshots\\prod.sqlite", + "still_missing": [ + "RHM" + ] + }, + "schema_version": 1 +} diff --git a/docs/research/earnings-gap-and-sue.md b/docs/research/earnings-gap-and-sue.md new file mode 100644 index 0000000..4211c84 --- /dev/null +++ b/docs/research/earnings-gap-and-sue.md @@ -0,0 +1,202 @@ +# Earnings gap diagnostic + SUE / PEAD (Tier-1 alpha research) + +**Status:** **PARK** (incomplete earnings coverage; SUE fails iron rule on available sample). +**Branch:** `research/earnings-gap-and-sue` +**Production impact:** none. Local research only. **No filters shipped from 2a.** +**Artifacts:** `reports/earnings-gap-sue-20260719-093129.json` (+ companion `.md`) + +--- + +## Pre-registration (locked before first research run) + +### Data + +- Historical earnings calendar for the production universe over the full snapshot + window (and deeper if the feed provides it). +- Preferred source: FMP **date-range earnings-calendar** (bulk). If unavailable on + free tier, fall back to per-symbol `/stable/earnings` with request accounting. +- Store in a real local table `earnings_events` (symbol + announce_date key). +- Point-in-time: a surprise is usable only from **announce date + 1 trading day** + onward. + +### Experiment 2a — earnings-gap risk (defense, report-only) + +Join simulated production-config trades (`fill_mode=close`) with earnings dates. + +**Pre-registered questions:** + +1. What fraction of losses worse than **−1R** occur with an earnings announcement + **between entry and exit** (inclusive of the holding window)? +2. What is the mean R of entries taken within **3 trading days BEFORE** an + announcement vs all other entries — report **both tails** of the R + distribution (rule 4: any earnings-avoid entry filter is presumed guilty of + right-tail trimming until the win distribution shows otherwise)? + +**Output:** distributions and counts only. +**No filter is shipped.** If numbers argue for a filter → report and stop. + +### Experiment 2b — SUE / PEAD (offense) + +Signal `sue_latest`: + +\[ +\text{SUE} = \frac{\text{actual} - \text{estimate}}{\sigma(\text{trailing 8 surprises})} +\] + +Fallback if estimate history is thin: scale surprise by price. +Carry forward from announce+1 for **63 trading days**, else NaN (name drops out +of that cross-section). + +**Iron rule (IC harness):** mean weekly Spearman IC on non-overlapping weeks; +\|mean IC\| ≥ ~0.03, **positive** sign (drift), `reliable: true` (≥12 windows). + +Always side-by-side with `mom_12_1` and `mom_12_1_resid` on **identical** +cross-sections. + +Also report **momentum-conditional** IC (within top momentum quintile). + +**If it passes iron rule:** STOP and report. Book-integration design is a +separate human-approved step — do not wire. + +### Verdict labels + +| label | meaning | +|---|---| +| **PROMOTE** | (2b only) iron rule cleared → human designs tilt/gate | +| **PARK** | Interesting but incomplete / weak | +| **DEAD** | No edge / diagnostic argues against action | +| **REPORT-ONLY** | (2a) always — never auto-filter | + +--- + +## Data provenance + +| item | result | +|---|---| +| Snapshot | `backtest_snapshots/prod.sqlite` (506 names) | +| FMP bulk `earnings-calendar` | **402 Premium** — not available on free tier | +| FMP per-symbol `/stable/earnings` | used; hit daily rate limit ~225 reqs | +| Alpha Vantage `EARNINGS` | used for +24 symbols (announce = `reportedDate`) | +| Symbols with events | **48 / 506 (9.5%)** | +| Total events | 5,612 (5,018 with actual+estimate) | +| Announce range | 1985-08-31 → 2026-07-16 | +| FMP requests (first day) | 260 FMP + 25 AV (see `reports/earnings-backfill-status.json`) | + +**Incomplete backfill is first-class.** 2a under-detects earnings overlaps; 2b SUE +cross-section averages **~47 names**, not ~500. Resume: + +```bash +# Day N (FMP free ~250/day; AV free ~25/day — prefer FMP after reset) +python scripts/backfill_earnings_events.py \ + --snapshot backtest_snapshots/prod.sqlite \ + --provider fmp --force-symbol --limit 250 --sleep 0.4 + +# When done==506: +python scripts/run_earnings_research.py \ + --snapshot backtest_snapshots/prod.sqlite \ + --workers 6 --allow-spawn +``` + +--- + +## Results + +Generated: `2026-07-19T09:31:29` + +### 2a — Earnings-gap risk (report-only) + +Production book sim: Sharpe 2.09 (SE 0.497), CAGR 51.6%, max DD 21.4%, **322 trades**, +`fill_mode=close`. + +#### Q1 — Losses worse than −1R with earnings in hold + +| metric | value | +|---|---:| +| n losses < −1R | 28 | +| of which earnings in hold | **1** | +| fraction | **3.6%** | +| all trades with earnings in hold | 14 / 322 (4.4%) | + +**Read:** On incomplete earnings labels this is a **lower bound** on earnings +overlap, not a clean “earnings rarely hurt.” Do **not** conclude earnings risk is +immaterial until coverage ≥ ~95% of the book’s names. + +#### Q2 — Entry within 3 trading days before announce (both tails) + +| cohort | n | mean R | win rate | p05 | p50 | p95 | max | +|---|---:|---:|---:|---:|---:|---:|---:| +| pre-earn (≤3d before) | **4** | 1.94 | 50% | −1.24 | 1.12 | 6.26 | 6.84 | +| other | 318 | 0.70 | 37% | −1.11 | −0.83 | 6.08 | **12.87** | +| all | 322 | 0.71 | 37% | −1.12 | −0.83 | 6.22 | 12.87 | + +**Tail-trim presumption:** n=4 is not a sample. Point estimate does **not** show +right-tail destruction of pre-earn entries (p95 similar; max actually higher in +“other”). **No earnings-avoid filter is supported.** Re-run after full backfill. + +--- + +### 2b — SUE / PEAD IC + +#### Full-universe harness (mom on ~500; SUE only where labeled) + +| signal | mean_ic | ic_t_stat | weeks | avg_N | reliable | +|---|---:|---:|---:|---:|---| +| mom_12_1_sector_resid | 0.0578 | 2.34 | 35 | 497.7 | true | +| mom_12_1_resid | 0.0552 | 1.98 | 35 | 497.7 | true | +| mom_12_1 | 0.0531 | 1.61 | 35 | 497.7 | true | +| **sue_latest** | **0.0172** | **0.6** | 44 | **47.4** | true | +| fip_id | −0.045 | −2.91 | 35 | 497.7 | true | + +#### Identical SUE subset (fair side-by-side — use this while coverage is thin) + +| signal | mean_ic | ic_t_stat | weeks | avg_N | +|---|---:|---:|---:|---:| +| sue_latest | 0.0172 | 0.6 | 44 | 47.4 | +| mom_12_1 | −0.0174 | −0.42 | 35 | 47.3 | +| mom_12_1_resid | −0.0104 | −0.27 | 35 | 47.3 | + +On the thin labeled subset, momentum itself is noise — so the subset is not yet +a meaningful PEAD test. + +#### Momentum-conditional SUE (top mom quintile) + +| metric | value | +|---|---:| +| mean IC | **−0.0065** | +| t | −0.1 | +| weeks | 35 | + +Wrong sign vs “ride positive surprises inside the momentum gate.” + +**Iron rule:** fail (\|IC\| 0.017 < 0.03; t 0.6). **No promote.** + +--- + +## Verdict + +| piece | verdict | +|---|---| +| **2a earnings-gap** | **REPORT-ONLY** — no filter. Coverage too thin for risk claims; tails do not argue for an avoid-filter on n=4. | +| **2b SUE** | **PARK** (effectively not green). Mild positive IC on ~48 names; fails iron bar; mom-conditional flat/negative. Re-score after full backfill before DEAD. | +| **Production** | **no change** | + +--- + +## What a human must decide next + +1. Resume multi-day earnings backfill to **506/506**, then re-run + `run_earnings_research.py` (heavy — MacBook OK). +2. Do **not** ship an earnings-avoid entry filter from 2a. +3. Do **not** wire SUE until a full-coverage IC clears the iron rule (and + preferably mom-conditional > 0). +4. Do not merge into main strategy docs without review. + +--- + +## Implementation notes + +| piece | role | +|---|---| +| `scripts/backfill_earnings_events.py` | bulk attempt → FMP/AV per-symbol; `earnings_events` + meta on snapshot | +| `scripts/run_earnings_research.py` | 2a trade join + 2b SUE IC / mom-conditional | +| Snapshot table `earnings_events` | real table (not SystemSetting JSON) | diff --git a/docs/research/history-depth-extension.md b/docs/research/history-depth-extension.md new file mode 100644 index 0000000..6ffcc16 --- /dev/null +++ b/docs/research/history-depth-extension.md @@ -0,0 +1,108 @@ +# History-depth extension (Tier-1 alpha research) + +**Status:** PRE-REGISTERED — run on MacBook (heavy I/O + full harness). +**Branch:** `research/history-depth-extension` (create from latest research stack). +**Production impact:** none. **Do not retune any production knob on deep history.** + +--- + +## Pre-registration (locked before rebuild) + +### Motivation + +All current conclusions rest on ~35 non-overlapping weekly windows in essentially +one post-2021 regime. Extending history toward max Alpaca daily-bar depth adds +the 2018 vol shock and full 2020 crash (where the feed allows). + +### Protocol + +1. **Empirical coverage first** — bars per calendar year per symbol; document + where the feed thins out. Do **not** assume a uniform start date. +2. **Rebuild the research snapshot completely** from prod source + max history + per symbol (`Adjustment.SPLIT`, ~200 req/min pacing via existing extender). +3. **Race guard (rule 6)** — refuse analysis until completion manifest is + `complete=true` and live counts match. +4. **Re-run full signal harness** (all existing signals incl. sector residual / + SUE if present) on the extended window. +5. **Report per signal:** mean IC, t, window count, and **era split** + (pre-/post-2021) — diagnostic only, **not a tuning input**. +6. **Log prominently:** survivorship bias grows with depth (today’s constituents + backfilled). Absolute Sharpe/CAGR on deep history is optimistic; payload is + **relative** signal comparisons and IC stability, not levels. +7. **Do not retune** production knobs. If a knob’s confirmation looks + overturned on deep history → report only; human decides. + +### Success / interpretation (not promotion of a new signal) + +| outcome | meaning | +|---|---| +| Sector residual still ≥ market residual on deep IC + stable sign | strengthens Task 1 PROMOTE case | +| Sector residual collapses pre-2021 | **PARK** Task 1 wire-in | +| SUE remains weak after full earnings + depth | **DEAD** SUE for this stack | +| Any production knob looks worse deep | report; no auto-retune | + +--- + +## MacBook runbook + +```bash +# 0. Repo + env +git fetch origin +git checkout research/earnings-gap-and-sue # or history-depth branch once pushed +# ensure .env has ALPACA_* (and FMP if resuming earnings) + +# 1. (Optional) finish earnings backfill first — multi-day free tier +python scripts/backfill_earnings_events.py \ + --snapshot backtest_snapshots/prod.sqlite \ + --provider fmp --force-symbol --limit 250 --sleep 0.35 + +# 2. Coverage probe (before long rebuild) +python scripts/run_history_depth_research.py --phase coverage \ + --snapshot backtest_snapshots/prod.sqlite + +# 3. Full deep rebuild of research.sqlite (LONG — Alpaca per symbol) +# Clears prior completion manifest; writes complete=true only at end. +python scripts/extend_snapshot_universe.py \ + --source backtest_snapshots/prod.sqlite \ + --output backtest_snapshots/research.sqlite \ + --force-copy \ + --history-days 5000 \ + --min-bars 260 \ + --sleep 0.15 + +# 4. Also refresh SPY + sector ETFs to the same depth on BOTH snapshots +python scripts/fetch_sector_etfs_to_snapshot.py \ + --snapshot backtest_snapshots/research.sqlite --history-days 5000 +python scripts/fetch_sector_etfs_to_snapshot.py \ + --snapshot backtest_snapshots/prod.sqlite --history-days 5000 + +# 5. Harness + era split (after race guard passes) +python scripts/run_history_depth_research.py --phase harness \ + --snapshot backtest_snapshots/research.sqlite \ + --workers 8 --allow-spawn + +# 6. Copy reports/ + docs/research/history-depth-extension.md results back +``` + +--- + +## Data provenance + +*(filled at run time)* + +--- + +## Results + +*(filled at run time)* + +--- + +## Verdict + +**Pending MacBook run.** + +## What a human must decide next + +- Do not retune production from deep history without explicit review. +- Use relative IC stability to accept/reject Task 1 sector residual wire-in. diff --git a/docs/research/sector-residual-momentum.md b/docs/research/sector-residual-momentum.md new file mode 100644 index 0000000..90a15ee --- /dev/null +++ b/docs/research/sector-residual-momentum.md @@ -0,0 +1,237 @@ +# Sector-residual momentum (Tier-1 alpha research) + +**Status:** **PROMOTE (to human design decision only)** — IC + A/B bars cleared; **do not ship**. +**Branch:** `research/sector-residual-momentum` +**Production impact:** none. Local research only. No scheduler / gate / prod-config changes. +**Artifacts:** `reports/sector-residual-20260719-083356.json` (+ companion `.md`) + +--- + +## Pre-registration (locked before first research run) + +### Hypothesis + +Residualizing 12–1 momentum against the sector, not only the market, reduces +factor volatility at similar return (Blitz / Huij / Martens-style) → higher +Sharpe on the production book when the residual replaces market-only residual +as the momentum leg. + +### Signals (candidates) + +| signal | construction | +|---|---| +| `mom_12_1_sector_resid` | Two-factor residual vs SPY + ticker’s sector ETF. Same window as `mom_12_1_resid`: ≥100 daily obs, 252-bar lookback, 21-bar skip; two-factor OLS betas **without intercept**; cumulate residual returns over the formation window. | +| `mom_12_1_sector_demeaned` | Plain `mom_12_1` minus the **cross-sectional** mean of `mom_12_1` within the same GICS sector that week (≥2 names in sector). No regression. | + +### Baselines (same run, same cross-sections — iron rule) + +Always report side-by-side with: + +- `mom_12_1` +- `mom_12_1_resid` + +Computed on the **identical** weekly non-overlapping cross-sections in this run. +Never compare against IC numbers from another report. + +### Iron rule (IC harness) + +Source of truth: `_signal_evaluation` in `app/services/backtest_service.py`. + +- Mean weekly Spearman IC on **non-overlapping** weekly windows +- Bar: \|mean IC\| ≥ ~0.03, **consistent positive sign**, `reliable: true` (≥ 12 windows) + +### Promotion to portfolio A/B (candidate → book) + +A candidate promotes to A/B **only if**: + +1. It clears the iron-rule bar **and** +2. Its IC **t-stat ≥** that of `mom_12_1_resid` on the same cross-sections. + +### Portfolio A/B grading (if and only if IC promotion fires) + +- Swap candidate in as the **momentum leg** of the production 80/20 momentum/vol + rank **and** as the gate-percentile signal. +- `fill_mode=close`, `COST_PER_SIDE = 0.001`, full config otherwise unchanged. +- Validation window = entries ≥ **2024-07-01** (call it **validation**, not + holdout — contaminated by prior experiments). +- Pre-registered promotion bar: + - validation Sharpe ≥ control − 0.5·SE + - full-period Sharpe and max-DD **not worse** than control +- Report Lo / Mertens-adjusted SEs. + +### Optional sector-cap sub-experiment + +Only if labels are in **and** A/B ran: max **3** positions per sector in the +10-slot book. Same A/B grading. **Tail-trim presumption of guilt** (rule 4): +report entry counts and both tails of the R distribution. Rising win rate with +falling Sharpe/CAGR = red flag → do not promote. + +**This run:** sector-cap arm **not executed** (optional; A/B unconstrained book +only). Can be a human-approved follow-up. + +### Verdict labels + +| label | meaning | +|---|---| +| **PROMOTE** | Clears pre-registered bar; human decides next (wire design separate) | +| **PARK** | Inconclusive / weak; keep machinery, no book change | +| **DEAD** | Failed iron rule or worse than residual baseline with clear sign | + +### Explicit non-goals + +- No production deploy from this doc +- Do not resurrect: take-profit exits, EV gate, regime entry-blocking, + inverse-vol sizing, gap-caps, unconditional FIP filter + +--- + +## Data provenance + +### Snapshot race guard + +| check | result | +|---|---| +| Snapshot path | `backtest_snapshots/prod.sqlite` | +| Manifest | none (expected for prod snapshot); bar-count sanity applied | +| Tickers / OHLCV | **506** / **629,263** | +| Bars min / avg / max | 14 / 1246.1 / 1261 | +| OHLCV range | 2021-06-24 → 2026-07-02 | +| Partial-build red flags | none (avg bars healthy) | + +Integrity fingerprint on same run: `fip_id` mean IC **−0.045** / t **−2.91** +(35 weeks, N≈498) — matches the established prod fingerprint. + +### Sector labels + +| source | count | +|---|---:| +| Public S&P 500 GICS CSV | 496 newly filled | +| FMP profile requests | 10 (all missing after CSV) | +| Mapped / universe | **505 / 506 (99.8%)** | +| With mappable ETF | 505 | +| Still missing | **RHM** only | + +Persist path: `data/research/ticker_sector_map.json`. + +FMP aliases (`Technology`, `Consumer Defensive`, `Financial Services`) map to +SPDRs via the alias table in `app/services/sector_map.py`. + +### Sector ETFs in `benchmark_prices` (auxiliary only — not tradable) + +| symbol | bars | min date | max date | +|---|---:|---|---| +| SPY | 1516 | 2020-07-06 | 2026-07-17 | +| XLB…XLY (11) | 1512 each | 2020-07-10 | 2026-07-17 | + +Fetched via Alpaca `Adjustment.SPLIT` into **`benchmark_prices`** (same table as +SPY) so they never enter the ticker universe or candidate replay. + +--- + +## Results + +Generated: `2026-07-19T08:33:56` + +### IC harness (identical cross-sections, production 506-name universe) + +| signal | mean_ic | ic_t_stat | weeks | avg_N | reliable | ic+_pct | quintile spread | +|---|---:|---:|---:|---:|---|---:|---:| +| **mom_12_1_sector_resid** | **0.0578** | **2.34** | 35 | 497.7 | true | 65.7 | 0.0245 | +| mom_12_1_resid | 0.0552 | 1.98 | 35 | 497.7 | true | 60.0 | 0.0207 | +| mom_12_1 | 0.0531 | 1.61 | 35 | 497.7 | true | 65.7 | 0.0206 | +| mom_12_1_sector_demeaned | 0.0340 | 1.32 | 35 | 496.7 | true | 62.9 | 0.0154 | + +### IC promotion grades + +| candidate | iron rule | t ≥ resid | promote_to_ab | +|---|---|---|---| +| `mom_12_1_sector_resid` | pass (IC 0.058, +sign, reliable) | **yes** (2.34 ≥ 1.98) | **yes** | +| `mom_12_1_sector_demeaned` | pass (IC 0.034, +sign, reliable) | **no** (1.32 < 1.98) | **no** | + +### Portfolio A/B — `mom_12_1_sector_resid` as residual leg + +Config: production 80/20 residual/high-vol rank + gate percentile, `fill_mode=close`, +cost 10 bps/side, ATR trail / gate-reset re-entry as live. Validation split +2024-07-01. + +| window | arm | Sharpe | Sharpe SE (Mertens) | CAGR % | max DD % | trades | n_days | +|---|---|---:|---:|---:|---:|---:|---:| +| train | control (resid) | 1.30 | 0.685 | 29.2 | 21.4 | 176 | 525 | +| train | treatment (sector resid) | **1.57** | 0.677 | **35.5** | **19.8** | 176 | 530 | +| validation | control | **2.92** | 0.709 | **76.3** | **11.7** | 150 | 501 | +| validation | treatment | 2.57 | 0.701 | 66.3 | 14.8 | 163 | 501 | +| full | control | 2.09 | 0.497 | 51.6 | 21.4 | 322 | 1000 | +| full | treatment | 2.09 | 0.491 | 51.0 | **19.8** | 337 | 1005 | + +**Pre-registered A/B checks** + +| check | result | +|---|---| +| val Sharpe ≥ control − 0.5·SE | **pass** (2.57 ≥ 2.92 − 0.5×0.701 = 2.5695) — **knife-edge** | +| full Sharpe not worse | **pass** (2.09 = 2.09) | +| full max DD not worse | **pass** (19.8 < 21.4) | + +Qualified long candidates: control 1086 vs treatment 1210 (sector residual +gates a slightly larger set). + +--- + +## Verdict + +| signal | verdict | note | +|---|---|---| +| **`mom_12_1_sector_resid`** | **PROMOTE → human wire-in decision** | IC modestly beats market residual; A/B clears pre-reg bar narrowly. **Do not ship from this branch.** | +| **`mom_12_1_sector_demeaned`** | **DEAD** (for promotion) | Iron-rule IC magnitude ok, but t-stat loses to `mom_12_1_resid`. Cheap variant not competitive. | + +### Read carefully (for the human) + +1. **IC edge is real but small.** Sector residual IC 0.0578 / t 2.34 vs market + residual 0.0552 / t 1.98 on the **same** 35 windows — better consistency + (ic+ 65.7% vs 60%) and slightly higher mean, not a different factor class. +2. **A/B is not a clear Sharpe win.** Full-period Sharpe is flat (2.09). + Validation Sharpe is **lower** than control (2.57 vs 2.92) and only clears + the pre-registered “within 0.5 SE” cushion by ~0.001. Train improves; + validation worsens — classic regime-split noise on ~2 years. +3. **Risk side is friendly.** Full max DD improves (19.8% vs 21.4%); train DD + also better. Matches the “lower factor vol” half of the hypothesis more than + the “higher Sharpe” half on this window. +4. **Survivorship / short history.** Same caveats as all current research: + today’s constituents, ~35 independent weekly windows, one post-2021 regime + dominant. Task 3 (history depth) should re-check IC stability before any + wire-in. +5. **Not shipped.** Machinery lives on the research branch; production residual + path is untouched. + +--- + +## What a human must decide next + +1. **Accept or reject** replacing `mom_12_1_resid` with `mom_12_1_sector_resid` + as the production residual (gate + 80/20 mom leg), **or** keep market residual + and treat sector residual as research-only. +2. If leaning accept: require **Task 3 history-depth** confirmation (IC era split + pre/post-2021) before any production PR. +3. Optional: run **sector-cap ≤3** A/B with full tail diagnostics (not run here). +4. **Do not** merge this verdict into main strategy docs without review. +5. Wire-in design (live sector map refresh, ETF series ops, fallback when sector + missing) is a **separate** approved engineering step. + +--- + +## Implementation notes (research machinery) + +| piece | role | +|---|---| +| `app/services/sector_map.py` | GICS→ETF map, symbol normalise, JSON load/save | +| `app/services/backtest_service.py` | multi-factor residual; `mom_12_1_sector_resid` in `_signal_values`; demean inject | +| `scripts/build_ticker_sector_map.py` | SP500 CSV + FMP gap fill | +| `scripts/fetch_sector_etfs_to_snapshot.py` | Alpaca → snapshot `benchmark_prices` | +| `scripts/run_sector_residual_research.py` | race guard, IC, optional A/B, reports | +| `data/research/ticker_sector_map.json` | persisted labels (research only) | + +--- + +## Artifacts + +- JSON: `reports/sector-residual-20260719-083356.json` +- MD copy: `reports/sector-residual-20260719-083356.md` diff --git a/reports/earnings-backfill-status.json b/reports/earnings-backfill-status.json new file mode 100644 index 0000000..7b22f25 --- /dev/null +++ b/reports/earnings-backfill-status.json @@ -0,0 +1,15 @@ +{ + "mode": "per_symbol", + "fmp_requests": 25, + "events_written_this_run": 2541, + "total_events": 5612, + "symbols_done": 48, + "symbols_universe": 506, + "announce_date_range": { + "min": "1985-08-31", + "max": "2026-07-16" + }, + "events_with_actual_and_estimate": 5018, + "budget": 25, + "complete": false +} diff --git a/reports/earnings-gap-sue-20260719-093129.json b/reports/earnings-gap-sue-20260719-093129.json new file mode 100644 index 0000000..57dc35f --- /dev/null +++ b/reports/earnings-gap-sue-20260719-093129.json @@ -0,0 +1,331 @@ +{ + "generated_at": "2026-07-19T09:31:29.078611", + "data_provenance": { + "snapshot": "C:\\Workspace\\signal-platform\\backtest_snapshots\\prod.sqlite", + "n_earnings_events": 5612, + "backfill_meta": { + "done": 48, + "universe_tickers": 506 + }, + "announce_range": { + "min": "1985-08-31", + "max": "2026-07-16" + }, + "with_actual_and_estimate": 5018 + }, + "experiment_2a": { + "sim_summary": { + "sharpe": 2.09, + "sharpe_se": 0.497, + "cagr_pct": 51.6, + "max_drawdown_pct": 21.4, + "trades": 322, + "total_return_pct": 424.6 + }, + "n_trades_parsed": 322, + "q1_losses_worse_than_minus_1r": { + "n_losses_lt_minus_1r": 28, + "n_with_earnings_in_hold": 1, + "fraction_with_earnings": 0.0357, + "all_trades_with_earnings_in_hold": 14, + "fraction_all_trades_with_earnings": 0.0435 + }, + "q2_entry_within_3d_before_announce": { + "pre_earn_entries": { + "n": 4, + "mean": 1.9379, + "win_rate": 0.5, + "p05": -1.2428, + "p25": -0.8833, + "p50": 1.1209, + "p75": 3.942, + "p95": 6.2623, + "min": -1.3327, + "max": 6.8424 + }, + "other_entries": { + "n": 318, + "mean": 0.6965, + "win_rate": 0.3711, + "p05": -1.1052, + "p25": -1.0, + "p50": -0.8259, + "p75": 2.1053, + "p95": 6.077, + "min": -3.2587, + "max": 12.8654 + }, + "all_entries": { + "n": 322, + "mean": 0.7119, + "win_rate": 0.3727, + "p05": -1.1209, + "p25": -1.0, + "p50": -0.8251, + "p75": 2.1595, + "p95": 6.2246, + "min": -3.2587, + "max": 12.8654 + }, + "tail_trim_note": "Compare p95/max and mean of pre_earn vs other. Rising win_rate with falling mean/p95 = right-tail trim red flag." + }, + "note": "REPORT-ONLY \u2014 no filter shipped." + }, + "experiment_2b": { + "signal_eval_side_by_side": { + "mom_12_1": { + "signal": "mom_12_1", + "weeks": 35, + "avg_cross_section": 497.7, + "mean_ic": 0.0531, + "ic_t_stat": 1.61, + "ic_positive_pct": 65.7, + "mean_quintile_spread": 0.0206, + "reliable": true + }, + "mom_12_1_resid": { + "signal": "mom_12_1_resid", + "weeks": 35, + "avg_cross_section": 497.7, + "mean_ic": 0.0552, + "ic_t_stat": 1.98, + "ic_positive_pct": 60.0, + "mean_quintile_spread": 0.0207, + "reliable": true + }, + "mom_12_1_sector_resid": { + "signal": "mom_12_1_sector_resid", + "weeks": 35, + "avg_cross_section": 497.7, + "mean_ic": 0.0578, + "ic_t_stat": 2.34, + "ic_positive_pct": 65.7, + "mean_quintile_spread": 0.0245, + "reliable": true + }, + "mom_12_1_sector_demeaned": { + "signal": "mom_12_1_sector_demeaned", + "weeks": 35, + "avg_cross_section": 496.7, + "mean_ic": 0.034, + "ic_t_stat": 1.32, + "ic_positive_pct": 62.9, + "mean_quintile_spread": 0.0154, + "reliable": true + }, + "sue_latest": { + "signal": "sue_latest", + "weeks": 44, + "avg_cross_section": 47.4, + "mean_ic": 0.0172, + "ic_t_stat": 0.6, + "ic_positive_pct": 47.7, + "mean_quintile_spread": 0.0064, + "reliable": true + }, + "fip_id": { + "signal": "fip_id", + "weeks": 35, + "avg_cross_section": 497.7, + "mean_ic": -0.045, + "ic_t_stat": -2.91, + "ic_positive_pct": 25.7, + "mean_quintile_spread": -0.0168, + "reliable": true + } + }, + "signal_eval_identical_sue_subset": { + "mom_12_1": { + "signal": "mom_12_1", + "weeks": 35, + "avg_cross_section": 47.3, + "mean_ic": -0.0174, + "ic_t_stat": -0.42, + "ic_positive_pct": 45.7, + "mean_quintile_spread": 0.0077, + "reliable": true + }, + "mom_12_1_resid": { + "signal": "mom_12_1_resid", + "weeks": 35, + "avg_cross_section": 47.3, + "mean_ic": -0.0104, + "ic_t_stat": -0.27, + "ic_positive_pct": 51.4, + "mean_quintile_spread": 0.0075, + "reliable": true + }, + "sue_latest": { + "signal": "sue_latest", + "weeks": 44, + "avg_cross_section": 47.4, + "mean_ic": 0.0172, + "ic_t_stat": 0.6, + "ic_positive_pct": 47.7, + "mean_quintile_spread": 0.0064, + "reliable": true + } + }, + "identical_subset_note": "Mom baselines re-scored only on (week, symbol) cells where SUE exists. Use this table when backfill is incomplete \u2014 full-universe mom N is not comparable.", + "full_signal_eval": [ + { + "signal": "vol_6m", + "weeks": 39, + "avg_cross_section": 498.2, + "mean_ic": 0.0609, + "ic_t_stat": 1.48, + "ic_positive_pct": 64.1, + "mean_quintile_spread": 0.0337, + "reliable": true + }, + { + "signal": "mom_12_1_sector_resid", + "weeks": 35, + "avg_cross_section": 497.7, + "mean_ic": 0.0578, + "ic_t_stat": 2.34, + "ic_positive_pct": 65.7, + "mean_quintile_spread": 0.0245, + "reliable": true + }, + { + "signal": "mom_12_1_resid", + "weeks": 35, + "avg_cross_section": 497.7, + "mean_ic": 0.0552, + "ic_t_stat": 1.98, + "ic_positive_pct": 60.0, + "mean_quintile_spread": 0.0207, + "reliable": true + }, + { + "signal": "mom_12_1", + "weeks": 35, + "avg_cross_section": 497.7, + "mean_ic": 0.0531, + "ic_t_stat": 1.61, + "ic_positive_pct": 65.7, + "mean_quintile_spread": 0.0206, + "reliable": true + }, + { + "signal": "mom_12_1_sector_demeaned", + "weeks": 35, + "avg_cross_section": 496.7, + "mean_ic": 0.034, + "ic_t_stat": 1.32, + "ic_positive_pct": 62.9, + "mean_quintile_spread": 0.0154, + "reliable": true + }, + { + "signal": "sue_latest", + "weeks": 44, + "avg_cross_section": 47.4, + "mean_ic": 0.0172, + "ic_t_stat": 0.6, + "ic_positive_pct": 47.7, + "mean_quintile_spread": 0.0064, + "reliable": true + }, + { + "signal": "trend_200", + "weeks": 37, + "avg_cross_section": 497.9, + "mean_ic": 0.0161, + "ic_t_stat": 0.44, + "ic_positive_pct": 59.5, + "mean_quintile_spread": 0.006, + "reliable": true + }, + { + "signal": "reversal_1m", + "weeks": 43, + "avg_cross_section": 498.7, + "mean_ic": 0.0059, + "ic_t_stat": 0.22, + "ic_positive_pct": 53.5, + "mean_quintile_spread": 0.0053, + "reliable": true + }, + { + "signal": "mom_6_1", + "weeks": 39, + "avg_cross_section": 498.2, + "mean_ic": 0.0051, + "ic_t_stat": 0.21, + "ic_positive_pct": 56.4, + "mean_quintile_spread": 0.0087, + "reliable": true + }, + { + "signal": "mom_3_1", + "weeks": 42, + "avg_cross_section": 498.5, + "mean_ic": -0.0064, + "ic_t_stat": -0.25, + "ic_positive_pct": 50.0, + "mean_quintile_spread": 0.0046, + "reliable": true + }, + { + "signal": "high_52w", + "weeks": 35, + "avg_cross_section": 497.7, + "mean_ic": -0.0086, + "ic_t_stat": -0.26, + "ic_positive_pct": 54.3, + "mean_quintile_spread": -0.0088, + "reliable": true + }, + { + "signal": "fip_id", + "weeks": 35, + "avg_cross_section": 497.7, + "mean_ic": -0.045, + "ic_t_stat": -2.91, + "ic_positive_pct": 25.7, + "mean_quintile_spread": -0.0168, + "reliable": true + } + ], + "sue_grade": { + "green": false, + "checks": { + "mean_ic": 0.0172, + "sign_positive": true, + "abs_ge_0_03": false, + "reliable": true, + "ic_t_stat": 0.6, + "weeks": 44 + }, + "reason": "iron rule not met", + "row": { + "signal": "sue_latest", + "weeks": 44, + "avg_cross_section": 47.4, + "mean_ic": 0.0172, + "ic_t_stat": 0.6, + "ic_positive_pct": 47.7, + "mean_quintile_spread": 0.0064, + "reliable": true + } + }, + "momentum_conditional_sue": { + "mean_ic": -0.0065, + "ic_t_stat": -0.1, + "weeks": 35, + "note": "IC of sue_latest within top mom_12_1 quintile (non-overlapping weeks)" + }, + "sue_coverage": { + "symbols_with_sue": 48, + "avg_weeks_with_sue": 47.1, + "weeks_with_min_cross_section": 256 + } + }, + "verdict": "PARK", + "verdict_detail": "SUE IC=0.0172 below iron bar or unreliable; keep data, no wire.", + "human_next": "- No SUE book change.\n- Read 2a tails before considering any earnings-avoid filter.", + "report_path": "reports/earnings-gap-sue-20260719-093129.json", + "fmp_note": "Bulk earnings-calendar is paid (402 on free tier). Backfill used per-symbol /stable/earnings; see earnings-backfill-status.json." +} diff --git a/reports/earnings-gap-sue-20260719-093129.md b/reports/earnings-gap-sue-20260719-093129.md new file mode 100644 index 0000000..4211c84 --- /dev/null +++ b/reports/earnings-gap-sue-20260719-093129.md @@ -0,0 +1,202 @@ +# Earnings gap diagnostic + SUE / PEAD (Tier-1 alpha research) + +**Status:** **PARK** (incomplete earnings coverage; SUE fails iron rule on available sample). +**Branch:** `research/earnings-gap-and-sue` +**Production impact:** none. Local research only. **No filters shipped from 2a.** +**Artifacts:** `reports/earnings-gap-sue-20260719-093129.json` (+ companion `.md`) + +--- + +## Pre-registration (locked before first research run) + +### Data + +- Historical earnings calendar for the production universe over the full snapshot + window (and deeper if the feed provides it). +- Preferred source: FMP **date-range earnings-calendar** (bulk). If unavailable on + free tier, fall back to per-symbol `/stable/earnings` with request accounting. +- Store in a real local table `earnings_events` (symbol + announce_date key). +- Point-in-time: a surprise is usable only from **announce date + 1 trading day** + onward. + +### Experiment 2a — earnings-gap risk (defense, report-only) + +Join simulated production-config trades (`fill_mode=close`) with earnings dates. + +**Pre-registered questions:** + +1. What fraction of losses worse than **−1R** occur with an earnings announcement + **between entry and exit** (inclusive of the holding window)? +2. What is the mean R of entries taken within **3 trading days BEFORE** an + announcement vs all other entries — report **both tails** of the R + distribution (rule 4: any earnings-avoid entry filter is presumed guilty of + right-tail trimming until the win distribution shows otherwise)? + +**Output:** distributions and counts only. +**No filter is shipped.** If numbers argue for a filter → report and stop. + +### Experiment 2b — SUE / PEAD (offense) + +Signal `sue_latest`: + +\[ +\text{SUE} = \frac{\text{actual} - \text{estimate}}{\sigma(\text{trailing 8 surprises})} +\] + +Fallback if estimate history is thin: scale surprise by price. +Carry forward from announce+1 for **63 trading days**, else NaN (name drops out +of that cross-section). + +**Iron rule (IC harness):** mean weekly Spearman IC on non-overlapping weeks; +\|mean IC\| ≥ ~0.03, **positive** sign (drift), `reliable: true` (≥12 windows). + +Always side-by-side with `mom_12_1` and `mom_12_1_resid` on **identical** +cross-sections. + +Also report **momentum-conditional** IC (within top momentum quintile). + +**If it passes iron rule:** STOP and report. Book-integration design is a +separate human-approved step — do not wire. + +### Verdict labels + +| label | meaning | +|---|---| +| **PROMOTE** | (2b only) iron rule cleared → human designs tilt/gate | +| **PARK** | Interesting but incomplete / weak | +| **DEAD** | No edge / diagnostic argues against action | +| **REPORT-ONLY** | (2a) always — never auto-filter | + +--- + +## Data provenance + +| item | result | +|---|---| +| Snapshot | `backtest_snapshots/prod.sqlite` (506 names) | +| FMP bulk `earnings-calendar` | **402 Premium** — not available on free tier | +| FMP per-symbol `/stable/earnings` | used; hit daily rate limit ~225 reqs | +| Alpha Vantage `EARNINGS` | used for +24 symbols (announce = `reportedDate`) | +| Symbols with events | **48 / 506 (9.5%)** | +| Total events | 5,612 (5,018 with actual+estimate) | +| Announce range | 1985-08-31 → 2026-07-16 | +| FMP requests (first day) | 260 FMP + 25 AV (see `reports/earnings-backfill-status.json`) | + +**Incomplete backfill is first-class.** 2a under-detects earnings overlaps; 2b SUE +cross-section averages **~47 names**, not ~500. Resume: + +```bash +# Day N (FMP free ~250/day; AV free ~25/day — prefer FMP after reset) +python scripts/backfill_earnings_events.py \ + --snapshot backtest_snapshots/prod.sqlite \ + --provider fmp --force-symbol --limit 250 --sleep 0.4 + +# When done==506: +python scripts/run_earnings_research.py \ + --snapshot backtest_snapshots/prod.sqlite \ + --workers 6 --allow-spawn +``` + +--- + +## Results + +Generated: `2026-07-19T09:31:29` + +### 2a — Earnings-gap risk (report-only) + +Production book sim: Sharpe 2.09 (SE 0.497), CAGR 51.6%, max DD 21.4%, **322 trades**, +`fill_mode=close`. + +#### Q1 — Losses worse than −1R with earnings in hold + +| metric | value | +|---|---:| +| n losses < −1R | 28 | +| of which earnings in hold | **1** | +| fraction | **3.6%** | +| all trades with earnings in hold | 14 / 322 (4.4%) | + +**Read:** On incomplete earnings labels this is a **lower bound** on earnings +overlap, not a clean “earnings rarely hurt.” Do **not** conclude earnings risk is +immaterial until coverage ≥ ~95% of the book’s names. + +#### Q2 — Entry within 3 trading days before announce (both tails) + +| cohort | n | mean R | win rate | p05 | p50 | p95 | max | +|---|---:|---:|---:|---:|---:|---:|---:| +| pre-earn (≤3d before) | **4** | 1.94 | 50% | −1.24 | 1.12 | 6.26 | 6.84 | +| other | 318 | 0.70 | 37% | −1.11 | −0.83 | 6.08 | **12.87** | +| all | 322 | 0.71 | 37% | −1.12 | −0.83 | 6.22 | 12.87 | + +**Tail-trim presumption:** n=4 is not a sample. Point estimate does **not** show +right-tail destruction of pre-earn entries (p95 similar; max actually higher in +“other”). **No earnings-avoid filter is supported.** Re-run after full backfill. + +--- + +### 2b — SUE / PEAD IC + +#### Full-universe harness (mom on ~500; SUE only where labeled) + +| signal | mean_ic | ic_t_stat | weeks | avg_N | reliable | +|---|---:|---:|---:|---:|---| +| mom_12_1_sector_resid | 0.0578 | 2.34 | 35 | 497.7 | true | +| mom_12_1_resid | 0.0552 | 1.98 | 35 | 497.7 | true | +| mom_12_1 | 0.0531 | 1.61 | 35 | 497.7 | true | +| **sue_latest** | **0.0172** | **0.6** | 44 | **47.4** | true | +| fip_id | −0.045 | −2.91 | 35 | 497.7 | true | + +#### Identical SUE subset (fair side-by-side — use this while coverage is thin) + +| signal | mean_ic | ic_t_stat | weeks | avg_N | +|---|---:|---:|---:|---:| +| sue_latest | 0.0172 | 0.6 | 44 | 47.4 | +| mom_12_1 | −0.0174 | −0.42 | 35 | 47.3 | +| mom_12_1_resid | −0.0104 | −0.27 | 35 | 47.3 | + +On the thin labeled subset, momentum itself is noise — so the subset is not yet +a meaningful PEAD test. + +#### Momentum-conditional SUE (top mom quintile) + +| metric | value | +|---|---:| +| mean IC | **−0.0065** | +| t | −0.1 | +| weeks | 35 | + +Wrong sign vs “ride positive surprises inside the momentum gate.” + +**Iron rule:** fail (\|IC\| 0.017 < 0.03; t 0.6). **No promote.** + +--- + +## Verdict + +| piece | verdict | +|---|---| +| **2a earnings-gap** | **REPORT-ONLY** — no filter. Coverage too thin for risk claims; tails do not argue for an avoid-filter on n=4. | +| **2b SUE** | **PARK** (effectively not green). Mild positive IC on ~48 names; fails iron bar; mom-conditional flat/negative. Re-score after full backfill before DEAD. | +| **Production** | **no change** | + +--- + +## What a human must decide next + +1. Resume multi-day earnings backfill to **506/506**, then re-run + `run_earnings_research.py` (heavy — MacBook OK). +2. Do **not** ship an earnings-avoid entry filter from 2a. +3. Do **not** wire SUE until a full-coverage IC clears the iron rule (and + preferably mom-conditional > 0). +4. Do not merge into main strategy docs without review. + +--- + +## Implementation notes + +| piece | role | +|---|---| +| `scripts/backfill_earnings_events.py` | bulk attempt → FMP/AV per-symbol; `earnings_events` + meta on snapshot | +| `scripts/run_earnings_research.py` | 2a trade join + 2b SUE IC / mom-conditional | +| Snapshot table `earnings_events` | real table (not SystemSetting JSON) | diff --git a/reports/sector-residual-20260719-083356.json b/reports/sector-residual-20260719-083356.json new file mode 100644 index 0000000..eecfc98 --- /dev/null +++ b/reports/sector-residual-20260719-083356.json @@ -0,0 +1,412 @@ +{ + "generated_at": "2026-07-19T08:33:56.651229", + "snapshot_guard": { + "snapshot": "C:\\Workspace\\signal-platform\\backtest_snapshots\\prod.sqlite", + "manifest": null, + "manifest_ok": null, + "note": "No completion manifest (prod.sqlite is expected without one). Bar-count sanity still applied.", + "ticker_count": 506, + "ohlcv_row_count": 629263, + "bars_min_avg_max": { + "min": 14, + "avg": 1246.1, + "max": 1261 + }, + "ohlcv_date_range": { + "min": "2021-06-24", + "max": "2026-07-02" + }, + "benchmark_prices": [ + { + "symbol": "SPY", + "n": 1516, + "min": "2020-07-06", + "max": "2026-07-17" + }, + { + "symbol": "XLB", + "n": 1512, + "min": "2020-07-10", + "max": "2026-07-17" + }, + { + "symbol": "XLC", + "n": 1512, + "min": "2020-07-10", + "max": "2026-07-17" + }, + { + "symbol": "XLE", + "n": 1512, + "min": "2020-07-10", + "max": "2026-07-17" + }, + { + "symbol": "XLF", + "n": 1512, + "min": "2020-07-10", + "max": "2026-07-17" + }, + { + "symbol": "XLI", + "n": 1512, + "min": "2020-07-10", + "max": "2026-07-17" + }, + { + "symbol": "XLK", + "n": 1512, + "min": "2020-07-10", + "max": "2026-07-17" + }, + { + "symbol": "XLP", + "n": 1512, + "min": "2020-07-10", + "max": "2026-07-17" + }, + { + "symbol": "XLRE", + "n": 1512, + "min": "2020-07-10", + "max": "2026-07-17" + }, + { + "symbol": "XLU", + "n": 1512, + "min": "2020-07-10", + "max": "2026-07-17" + }, + { + "symbol": "XLV", + "n": 1512, + "min": "2020-07-10", + "max": "2026-07-17" + }, + { + "symbol": "XLY", + "n": 1512, + "min": "2020-07-10", + "max": "2026-07-17" + } + ], + "missing_sector_etfs": [] + }, + "sector_coverage": { + "universe": 506, + "mapped": 505, + "mapped_pct": 99.8, + "with_etf": 505, + "missing": [ + "RHM" + ], + "by_sector": { + "Industrials": 81, + "Financials": 75, + "Information Technology": 72, + "Health Care": 58, + "Consumer Discretionary": 47, + "Consumer Staples": 34, + "Real Estate": 31, + "Utilities": 31, + "Materials": 26, + "Communication Services": 23, + "Energy": 22, + "Consumer Defensive": 2, + "Technology": 2, + "Financial Services": 1 + } + }, + "sector_map_path": "C:\\Workspace\\signal-platform\\data\\research\\ticker_sector_map.json", + "signal_eval": [ + { + "signal": "vol_6m", + "weeks": 39, + "avg_cross_section": 498.2, + "mean_ic": 0.0609, + "ic_t_stat": 1.48, + "ic_positive_pct": 64.1, + "mean_quintile_spread": 0.0337, + "reliable": true + }, + { + "signal": "mom_12_1_sector_resid", + "weeks": 35, + "avg_cross_section": 497.7, + "mean_ic": 0.0578, + "ic_t_stat": 2.34, + "ic_positive_pct": 65.7, + "mean_quintile_spread": 0.0245, + "reliable": true + }, + { + "signal": "mom_12_1_resid", + "weeks": 35, + "avg_cross_section": 497.7, + "mean_ic": 0.0552, + "ic_t_stat": 1.98, + "ic_positive_pct": 60.0, + "mean_quintile_spread": 0.0207, + "reliable": true + }, + { + "signal": "mom_12_1", + "weeks": 35, + "avg_cross_section": 497.7, + "mean_ic": 0.0531, + "ic_t_stat": 1.61, + "ic_positive_pct": 65.7, + "mean_quintile_spread": 0.0206, + "reliable": true + }, + { + "signal": "mom_12_1_sector_demeaned", + "weeks": 35, + "avg_cross_section": 496.7, + "mean_ic": 0.034, + "ic_t_stat": 1.32, + "ic_positive_pct": 62.9, + "mean_quintile_spread": 0.0154, + "reliable": true + }, + { + "signal": "trend_200", + "weeks": 37, + "avg_cross_section": 497.9, + "mean_ic": 0.0161, + "ic_t_stat": 0.44, + "ic_positive_pct": 59.5, + "mean_quintile_spread": 0.006, + "reliable": true + }, + { + "signal": "reversal_1m", + "weeks": 43, + "avg_cross_section": 498.7, + "mean_ic": 0.0059, + "ic_t_stat": 0.22, + "ic_positive_pct": 53.5, + "mean_quintile_spread": 0.0053, + "reliable": true + }, + { + "signal": "mom_6_1", + "weeks": 39, + "avg_cross_section": 498.2, + "mean_ic": 0.0051, + "ic_t_stat": 0.21, + "ic_positive_pct": 56.4, + "mean_quintile_spread": 0.0087, + "reliable": true + }, + { + "signal": "mom_3_1", + "weeks": 42, + "avg_cross_section": 498.5, + "mean_ic": -0.0064, + "ic_t_stat": -0.25, + "ic_positive_pct": 50.0, + "mean_quintile_spread": 0.0046, + "reliable": true + }, + { + "signal": "high_52w", + "weeks": 35, + "avg_cross_section": 497.7, + "mean_ic": -0.0086, + "ic_t_stat": -0.26, + "ic_positive_pct": 54.3, + "mean_quintile_spread": -0.0088, + "reliable": true + }, + { + "signal": "fip_id", + "weeks": 35, + "avg_cross_section": 497.7, + "mean_ic": -0.045, + "ic_t_stat": -2.91, + "ic_positive_pct": 25.7, + "mean_quintile_spread": -0.0168, + "reliable": true + } + ], + "ic_grades": { + "mom_12_1_sector_resid": { + "promote_to_ab": true, + "checks": { + "sign_ok": true, + "abs_mean_ic_ge_0_03": true, + "reliable": true, + "t_ge_resid": true, + "mean_ic": 0.0578, + "ic_t_stat": 2.34, + "resid_ic_t_stat": 1.98, + "weeks": 35 + }, + "reason": "clears iron rule and t \u2265 mom_12_1_resid \u2014 authorized for A/B only", + "row": { + "signal": "mom_12_1_sector_resid", + "weeks": 35, + "avg_cross_section": 497.7, + "mean_ic": 0.0578, + "ic_t_stat": 2.34, + "ic_positive_pct": 65.7, + "mean_quintile_spread": 0.0245, + "reliable": true + } + }, + "mom_12_1_sector_demeaned": { + "promote_to_ab": false, + "checks": { + "sign_ok": true, + "abs_mean_ic_ge_0_03": true, + "reliable": true, + "t_ge_resid": false, + "mean_ic": 0.034, + "ic_t_stat": 1.32, + "resid_ic_t_stat": 1.98, + "weeks": 35 + }, + "reason": "does not clear pre-registered IC promotion bar", + "row": { + "signal": "mom_12_1_sector_demeaned", + "weeks": 35, + "avg_cross_section": 496.7, + "mean_ic": 0.034, + "ic_t_stat": 1.32, + "ic_positive_pct": 62.9, + "mean_quintile_spread": 0.0154, + "reliable": true + } + } + }, + "portfolio_ab": { + "signal": "mom_12_1_sector_resid", + "ranking_key": "residual_high_vol_blend_80_20_score", + "fill_mode": "close", + "validation_split": "2024-07-01", + "control": { + "label": "control_mom_12_1_resid", + "n_qualified_longs": 1086, + "windows": { + "train": { + "sharpe": 1.3, + "sharpe_se": 0.685, + "cagr_pct": 29.2, + "max_drawdown_pct": 21.4, + "total_return_pct": 70.9, + "trades": 176, + "win_rate_pct": null, + "avg_r": null, + "n_returns": 525, + "return_skew": 0.3722, + "return_kurtosis": 4.6208, + "psr": 0.971 + }, + "validation": { + "sharpe": 2.92, + "sharpe_se": 0.709, + "cagr_pct": 76.3, + "max_drawdown_pct": 11.7, + "total_return_pct": 210.7, + "trades": 150, + "win_rate_pct": null, + "avg_r": null, + "n_returns": 501, + "return_skew": 0.1734, + "return_kurtosis": 4.4625, + "psr": 1.0 + }, + "full": { + "sharpe": 2.09, + "sharpe_se": 0.497, + "cagr_pct": 51.6, + "max_drawdown_pct": 21.4, + "total_return_pct": 424.6, + "trades": 322, + "win_rate_pct": null, + "avg_r": null, + "n_returns": 1000, + "return_skew": 0.2686, + "return_kurtosis": 4.5653, + "psr": 1.0 + } + } + }, + "treatment": { + "label": "treatment_mom_12_1_sector_resid", + "n_qualified_longs": 1210, + "windows": { + "train": { + "sharpe": 1.57, + "sharpe_se": 0.677, + "cagr_pct": 35.5, + "max_drawdown_pct": 19.8, + "total_return_pct": 90.0, + "trades": 176, + "win_rate_pct": null, + "avg_r": null, + "n_returns": 530, + "return_skew": 0.466, + "return_kurtosis": 4.4413, + "psr": 0.99 + }, + "validation": { + "sharpe": 2.57, + "sharpe_se": 0.701, + "cagr_pct": 66.3, + "max_drawdown_pct": 14.8, + "total_return_pct": 176.4, + "trades": 163, + "win_rate_pct": null, + "avg_r": null, + "n_returns": 501, + "return_skew": 0.3003, + "return_kurtosis": 4.4305, + "psr": 0.9999 + }, + "full": { + "sharpe": 2.09, + "sharpe_se": 0.491, + "cagr_pct": 51.0, + "max_drawdown_pct": 19.8, + "total_return_pct": 421.3, + "trades": 337, + "win_rate_pct": null, + "avg_r": null, + "n_returns": 1005, + "return_skew": 0.4066, + "return_kurtosis": 4.3627, + "psr": 1.0 + } + } + }, + "promotion": { + "promote": true, + "checks": { + "validation_sharpe_ge_control_minus_half_se": true, + "full_sharpe_not_worse": true, + "full_maxdd_not_worse": true, + "control_validation_sharpe": 2.92, + "treatment_validation_sharpe": 2.57, + "se_used": 0.701, + "control_full_sharpe": 2.09, + "treatment_full_sharpe": 2.09, + "control_full_maxdd": 21.4, + "treatment_full_maxdd": 19.8 + }, + "reason": "clears pre-registered A/B bar \u2014 human decides wire-in" + } + }, + "verdict": "PROMOTE", + "verdict_detail": "mom_12_1_sector_resid cleared IC + A/B bars. Human must design wire-in; do not ship from this branch.", + "human_next": "- Approve or reject production residual swap vs dual-signal design.\n- If sector-cap arm ran, review tail-trim diagnostics before any cap.", + "report_path": "reports/sector-residual-20260719-083356.json", + "pre_registration": { + "iron_ic_bar": 0.03, + "validation_split": "2024-07-01", + "fill_mode": "close", + "cost_per_side": 0.001, + "ab_rule": "val Sharpe >= control - 0.5*SE; full Sharpe & maxDD not worse" + } +} diff --git a/reports/sector-residual-20260719-083356.md b/reports/sector-residual-20260719-083356.md new file mode 100644 index 0000000..90a15ee --- /dev/null +++ b/reports/sector-residual-20260719-083356.md @@ -0,0 +1,237 @@ +# Sector-residual momentum (Tier-1 alpha research) + +**Status:** **PROMOTE (to human design decision only)** — IC + A/B bars cleared; **do not ship**. +**Branch:** `research/sector-residual-momentum` +**Production impact:** none. Local research only. No scheduler / gate / prod-config changes. +**Artifacts:** `reports/sector-residual-20260719-083356.json` (+ companion `.md`) + +--- + +## Pre-registration (locked before first research run) + +### Hypothesis + +Residualizing 12–1 momentum against the sector, not only the market, reduces +factor volatility at similar return (Blitz / Huij / Martens-style) → higher +Sharpe on the production book when the residual replaces market-only residual +as the momentum leg. + +### Signals (candidates) + +| signal | construction | +|---|---| +| `mom_12_1_sector_resid` | Two-factor residual vs SPY + ticker’s sector ETF. Same window as `mom_12_1_resid`: ≥100 daily obs, 252-bar lookback, 21-bar skip; two-factor OLS betas **without intercept**; cumulate residual returns over the formation window. | +| `mom_12_1_sector_demeaned` | Plain `mom_12_1` minus the **cross-sectional** mean of `mom_12_1` within the same GICS sector that week (≥2 names in sector). No regression. | + +### Baselines (same run, same cross-sections — iron rule) + +Always report side-by-side with: + +- `mom_12_1` +- `mom_12_1_resid` + +Computed on the **identical** weekly non-overlapping cross-sections in this run. +Never compare against IC numbers from another report. + +### Iron rule (IC harness) + +Source of truth: `_signal_evaluation` in `app/services/backtest_service.py`. + +- Mean weekly Spearman IC on **non-overlapping** weekly windows +- Bar: \|mean IC\| ≥ ~0.03, **consistent positive sign**, `reliable: true` (≥ 12 windows) + +### Promotion to portfolio A/B (candidate → book) + +A candidate promotes to A/B **only if**: + +1. It clears the iron-rule bar **and** +2. Its IC **t-stat ≥** that of `mom_12_1_resid` on the same cross-sections. + +### Portfolio A/B grading (if and only if IC promotion fires) + +- Swap candidate in as the **momentum leg** of the production 80/20 momentum/vol + rank **and** as the gate-percentile signal. +- `fill_mode=close`, `COST_PER_SIDE = 0.001`, full config otherwise unchanged. +- Validation window = entries ≥ **2024-07-01** (call it **validation**, not + holdout — contaminated by prior experiments). +- Pre-registered promotion bar: + - validation Sharpe ≥ control − 0.5·SE + - full-period Sharpe and max-DD **not worse** than control +- Report Lo / Mertens-adjusted SEs. + +### Optional sector-cap sub-experiment + +Only if labels are in **and** A/B ran: max **3** positions per sector in the +10-slot book. Same A/B grading. **Tail-trim presumption of guilt** (rule 4): +report entry counts and both tails of the R distribution. Rising win rate with +falling Sharpe/CAGR = red flag → do not promote. + +**This run:** sector-cap arm **not executed** (optional; A/B unconstrained book +only). Can be a human-approved follow-up. + +### Verdict labels + +| label | meaning | +|---|---| +| **PROMOTE** | Clears pre-registered bar; human decides next (wire design separate) | +| **PARK** | Inconclusive / weak; keep machinery, no book change | +| **DEAD** | Failed iron rule or worse than residual baseline with clear sign | + +### Explicit non-goals + +- No production deploy from this doc +- Do not resurrect: take-profit exits, EV gate, regime entry-blocking, + inverse-vol sizing, gap-caps, unconditional FIP filter + +--- + +## Data provenance + +### Snapshot race guard + +| check | result | +|---|---| +| Snapshot path | `backtest_snapshots/prod.sqlite` | +| Manifest | none (expected for prod snapshot); bar-count sanity applied | +| Tickers / OHLCV | **506** / **629,263** | +| Bars min / avg / max | 14 / 1246.1 / 1261 | +| OHLCV range | 2021-06-24 → 2026-07-02 | +| Partial-build red flags | none (avg bars healthy) | + +Integrity fingerprint on same run: `fip_id` mean IC **−0.045** / t **−2.91** +(35 weeks, N≈498) — matches the established prod fingerprint. + +### Sector labels + +| source | count | +|---|---:| +| Public S&P 500 GICS CSV | 496 newly filled | +| FMP profile requests | 10 (all missing after CSV) | +| Mapped / universe | **505 / 506 (99.8%)** | +| With mappable ETF | 505 | +| Still missing | **RHM** only | + +Persist path: `data/research/ticker_sector_map.json`. + +FMP aliases (`Technology`, `Consumer Defensive`, `Financial Services`) map to +SPDRs via the alias table in `app/services/sector_map.py`. + +### Sector ETFs in `benchmark_prices` (auxiliary only — not tradable) + +| symbol | bars | min date | max date | +|---|---:|---|---| +| SPY | 1516 | 2020-07-06 | 2026-07-17 | +| XLB…XLY (11) | 1512 each | 2020-07-10 | 2026-07-17 | + +Fetched via Alpaca `Adjustment.SPLIT` into **`benchmark_prices`** (same table as +SPY) so they never enter the ticker universe or candidate replay. + +--- + +## Results + +Generated: `2026-07-19T08:33:56` + +### IC harness (identical cross-sections, production 506-name universe) + +| signal | mean_ic | ic_t_stat | weeks | avg_N | reliable | ic+_pct | quintile spread | +|---|---:|---:|---:|---:|---|---:|---:| +| **mom_12_1_sector_resid** | **0.0578** | **2.34** | 35 | 497.7 | true | 65.7 | 0.0245 | +| mom_12_1_resid | 0.0552 | 1.98 | 35 | 497.7 | true | 60.0 | 0.0207 | +| mom_12_1 | 0.0531 | 1.61 | 35 | 497.7 | true | 65.7 | 0.0206 | +| mom_12_1_sector_demeaned | 0.0340 | 1.32 | 35 | 496.7 | true | 62.9 | 0.0154 | + +### IC promotion grades + +| candidate | iron rule | t ≥ resid | promote_to_ab | +|---|---|---|---| +| `mom_12_1_sector_resid` | pass (IC 0.058, +sign, reliable) | **yes** (2.34 ≥ 1.98) | **yes** | +| `mom_12_1_sector_demeaned` | pass (IC 0.034, +sign, reliable) | **no** (1.32 < 1.98) | **no** | + +### Portfolio A/B — `mom_12_1_sector_resid` as residual leg + +Config: production 80/20 residual/high-vol rank + gate percentile, `fill_mode=close`, +cost 10 bps/side, ATR trail / gate-reset re-entry as live. Validation split +2024-07-01. + +| window | arm | Sharpe | Sharpe SE (Mertens) | CAGR % | max DD % | trades | n_days | +|---|---|---:|---:|---:|---:|---:|---:| +| train | control (resid) | 1.30 | 0.685 | 29.2 | 21.4 | 176 | 525 | +| train | treatment (sector resid) | **1.57** | 0.677 | **35.5** | **19.8** | 176 | 530 | +| validation | control | **2.92** | 0.709 | **76.3** | **11.7** | 150 | 501 | +| validation | treatment | 2.57 | 0.701 | 66.3 | 14.8 | 163 | 501 | +| full | control | 2.09 | 0.497 | 51.6 | 21.4 | 322 | 1000 | +| full | treatment | 2.09 | 0.491 | 51.0 | **19.8** | 337 | 1005 | + +**Pre-registered A/B checks** + +| check | result | +|---|---| +| val Sharpe ≥ control − 0.5·SE | **pass** (2.57 ≥ 2.92 − 0.5×0.701 = 2.5695) — **knife-edge** | +| full Sharpe not worse | **pass** (2.09 = 2.09) | +| full max DD not worse | **pass** (19.8 < 21.4) | + +Qualified long candidates: control 1086 vs treatment 1210 (sector residual +gates a slightly larger set). + +--- + +## Verdict + +| signal | verdict | note | +|---|---|---| +| **`mom_12_1_sector_resid`** | **PROMOTE → human wire-in decision** | IC modestly beats market residual; A/B clears pre-reg bar narrowly. **Do not ship from this branch.** | +| **`mom_12_1_sector_demeaned`** | **DEAD** (for promotion) | Iron-rule IC magnitude ok, but t-stat loses to `mom_12_1_resid`. Cheap variant not competitive. | + +### Read carefully (for the human) + +1. **IC edge is real but small.** Sector residual IC 0.0578 / t 2.34 vs market + residual 0.0552 / t 1.98 on the **same** 35 windows — better consistency + (ic+ 65.7% vs 60%) and slightly higher mean, not a different factor class. +2. **A/B is not a clear Sharpe win.** Full-period Sharpe is flat (2.09). + Validation Sharpe is **lower** than control (2.57 vs 2.92) and only clears + the pre-registered “within 0.5 SE” cushion by ~0.001. Train improves; + validation worsens — classic regime-split noise on ~2 years. +3. **Risk side is friendly.** Full max DD improves (19.8% vs 21.4%); train DD + also better. Matches the “lower factor vol” half of the hypothesis more than + the “higher Sharpe” half on this window. +4. **Survivorship / short history.** Same caveats as all current research: + today’s constituents, ~35 independent weekly windows, one post-2021 regime + dominant. Task 3 (history depth) should re-check IC stability before any + wire-in. +5. **Not shipped.** Machinery lives on the research branch; production residual + path is untouched. + +--- + +## What a human must decide next + +1. **Accept or reject** replacing `mom_12_1_resid` with `mom_12_1_sector_resid` + as the production residual (gate + 80/20 mom leg), **or** keep market residual + and treat sector residual as research-only. +2. If leaning accept: require **Task 3 history-depth** confirmation (IC era split + pre/post-2021) before any production PR. +3. Optional: run **sector-cap ≤3** A/B with full tail diagnostics (not run here). +4. **Do not** merge this verdict into main strategy docs without review. +5. Wire-in design (live sector map refresh, ETF series ops, fallback when sector + missing) is a **separate** approved engineering step. + +--- + +## Implementation notes (research machinery) + +| piece | role | +|---|---| +| `app/services/sector_map.py` | GICS→ETF map, symbol normalise, JSON load/save | +| `app/services/backtest_service.py` | multi-factor residual; `mom_12_1_sector_resid` in `_signal_values`; demean inject | +| `scripts/build_ticker_sector_map.py` | SP500 CSV + FMP gap fill | +| `scripts/fetch_sector_etfs_to_snapshot.py` | Alpaca → snapshot `benchmark_prices` | +| `scripts/run_sector_residual_research.py` | race guard, IC, optional A/B, reports | +| `data/research/ticker_sector_map.json` | persisted labels (research only) | + +--- + +## Artifacts + +- JSON: `reports/sector-residual-20260719-083356.json` +- MD copy: `reports/sector-residual-20260719-083356.md` diff --git a/scripts/backfill_earnings_events.py b/scripts/backfill_earnings_events.py new file mode 100644 index 0000000..971d555 --- /dev/null +++ b/scripts/backfill_earnings_events.py @@ -0,0 +1,528 @@ +"""Backfill historical earnings into a snapshot ``earnings_events`` table. + +Prefers FMP bulk date-range ``earnings-calendar`` (one request per window). +On free-tier 402/403, falls back to per-symbol ``/stable/earnings`` with +resume support and request counting (≈250 req/day free tier). + +Research only — writes to the local snapshot SQLite, never production Postgres. + +Example +------- + python scripts/backfill_earnings_events.py \\ + --snapshot backtest_snapshots/prod.sqlite --limit 250 +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import sys +import time +from datetime import date, datetime, timedelta, timezone +from pathlib import Path + +import httpx +from sqlalchemy import create_engine, text + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +FMP_STABLE = "https://financialmodelingprep.com/stable" +DDL = """ +CREATE TABLE IF NOT EXISTS earnings_events ( + id INTEGER PRIMARY KEY, + symbol TEXT NOT NULL, + announce_date TEXT NOT NULL, + announce_time TEXT, + eps_estimate REAL, + eps_actual REAL, + revenue_estimate REAL, + revenue_actual REAL, + source TEXT NOT NULL, + fetched_at TEXT NOT NULL, + UNIQUE(symbol, announce_date) +) +""" +# Side table tracks which symbols have been fully pulled (resume). +META_DDL = """ +CREATE TABLE IF NOT EXISTS earnings_backfill_meta ( + symbol TEXT PRIMARY KEY, + status TEXT NOT NULL, + n_events INTEGER NOT NULL DEFAULT 0, + updated_at TEXT NOT NULL, + note TEXT +) +""" + + +def _parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--snapshot", default="backtest_snapshots/prod.sqlite") + p.add_argument( + "--from-date", + default="2020-01-01", + help="Bulk calendar window start (also filters per-symbol rows).", + ) + p.add_argument( + "--to-date", + default=None, + help="Bulk calendar window end (default: today).", + ) + p.add_argument( + "--limit", + type=int, + default=250, + help="Max FMP requests this run (free-tier cushion).", + ) + p.add_argument("--sleep", type=float, default=0.35) + p.add_argument( + "--force-symbol", + action="store_true", + help="Skip bulk attempt; go straight to per-symbol.", + ) + p.add_argument( + "--refetch-done", + action="store_true", + help="Re-fetch symbols already marked done.", + ) + p.add_argument( + "--provider", + choices=("fmp", "alpha_vantage", "auto"), + default="auto", + help="Earnings provider. auto tries FMP bulk then FMP/AV per-symbol.", + ) + return p.parse_args() + + +def _ensure_tables(engine) -> None: + with engine.begin() as conn: + conn.execute(text(DDL)) + conn.execute(text(META_DDL)) + + +def _upsert_events(conn, rows: list[dict], source: str) -> int: + if not rows: + return 0 + now = datetime.now(timezone.utc).isoformat() + written = 0 + for r in rows: + conn.execute( + text( + """ + INSERT INTO earnings_events ( + symbol, announce_date, announce_time, + eps_estimate, eps_actual, revenue_estimate, revenue_actual, + source, fetched_at + ) VALUES ( + :symbol, :announce_date, :announce_time, + :eps_estimate, :eps_actual, :revenue_estimate, :revenue_actual, + :source, :fetched_at + ) + ON CONFLICT(symbol, announce_date) DO UPDATE SET + announce_time=excluded.announce_time, + eps_estimate=excluded.eps_estimate, + eps_actual=excluded.eps_actual, + revenue_estimate=excluded.revenue_estimate, + revenue_actual=excluded.revenue_actual, + source=excluded.source, + fetched_at=excluded.fetched_at + """ + ), + { + "symbol": r["symbol"], + "announce_date": r["announce_date"], + "announce_time": r.get("announce_time"), + "eps_estimate": r.get("eps_estimate"), + "eps_actual": r.get("eps_actual"), + "revenue_estimate": r.get("revenue_estimate"), + "revenue_actual": r.get("revenue_actual"), + "source": source, + "fetched_at": now, + }, + ) + written += 1 + return written + + +def _parse_bulk_item(item: dict) -> dict | None: + sym = (item.get("symbol") or "").strip().upper() + d = item.get("date") or item.get("earningsDate") + if not sym or not d: + return None + return { + "symbol": sym.replace(".", "-"), + "announce_date": str(d)[:10], + "announce_time": item.get("time") or item.get("announceTime"), + "eps_estimate": _f(item.get("epsEstimated") or item.get("estimatedEarning")), + "eps_actual": _f(item.get("epsActual") or item.get("eps")), + "revenue_estimate": _f(item.get("revenueEstimated")), + "revenue_actual": _f(item.get("revenueActual")), + } + + +def _parse_symbol_item(item: dict, symbol: str) -> dict | None: + d = item.get("date") + if not d: + return None + return { + "symbol": symbol.replace(".", "-").upper(), + "announce_date": str(d)[:10], + "announce_time": item.get("time"), + "eps_estimate": _f(item.get("epsEstimated")), + "eps_actual": _f(item.get("epsActual")), + "revenue_estimate": _f(item.get("revenueEstimated")), + "revenue_actual": _f(item.get("revenueActual")), + } + + +def _f(v) -> float | None: + if v is None or v == "": + return None + try: + return float(v) + except (TypeError, ValueError): + return None + + +async def _try_bulk( + client: httpx.AsyncClient, + api_key: str, + start: date, + end: date, + *, + window_days: int = 30, +) -> tuple[list[dict], int, str | None]: + """Return (rows, requests_used, error_note).""" + rows: list[dict] = [] + reqs = 0 + cur = start + while cur <= end: + win_end = min(end, cur + timedelta(days=window_days - 1)) + resp = await client.get( + f"{FMP_STABLE}/earnings-calendar", + params={ + "from": cur.isoformat(), + "to": win_end.isoformat(), + "apikey": api_key, + }, + ) + reqs += 1 + if resp.status_code in (402, 403): + return [], reqs, f"bulk_unavailable status={resp.status_code}" + if resp.status_code == 429: + return rows, reqs, "rate_limited" + resp.raise_for_status() + data = resp.json() + if not isinstance(data, list): + return [], reqs, f"unexpected bulk payload type={type(data)}" + for item in data: + if isinstance(item, dict): + parsed = _parse_bulk_item(item) + if parsed: + rows.append(parsed) + cur = win_end + timedelta(days=1) + return rows, reqs, None + + +async def _fetch_symbol( + client: httpx.AsyncClient, api_key: str, symbol: str +) -> list[dict]: + resp = await client.get( + f"{FMP_STABLE}/earnings", + params={"symbol": symbol, "apikey": api_key}, + ) + if resp.status_code == 429: + raise RuntimeError("rate_limited") + if resp.status_code == 402: + return [] + resp.raise_for_status() + data = resp.json() + if not isinstance(data, list): + return [] + out: list[dict] = [] + for item in data: + if isinstance(item, dict): + parsed = _parse_symbol_item(item, symbol) + if parsed: + out.append(parsed) + return out + + +async def _fetch_symbol_alpha_vantage( + client: httpx.AsyncClient, api_key: str, symbol: str +) -> list[dict]: + """Alpha Vantage EARNINGS — includes reportedDate (announce) + estimate/actual.""" + resp = await client.get( + "https://www.alphavantage.co/query", + params={"function": "EARNINGS", "symbol": symbol, "apikey": api_key}, + ) + if resp.status_code == 429: + raise RuntimeError("rate_limited") + resp.raise_for_status() + data = resp.json() + if not isinstance(data, dict): + return [] + note = str(data.get("Note") or data.get("Information") or "") + if "rate limit" in note.lower() or "Thank you for using Alpha Vantage" in note: + raise RuntimeError("rate_limited") + if data.get("Error Message"): + return [] + quarterly = data.get("quarterlyEarnings") or [] + out: list[dict] = [] + for item in quarterly: + if not isinstance(item, dict): + continue + # Prefer announce (reportedDate); fall back to fiscal end (worse PIT). + ad = item.get("reportedDate") or item.get("fiscalDateEnding") + if not ad: + continue + out.append({ + "symbol": symbol.replace(".", "-").upper(), + "announce_date": str(ad)[:10], + "announce_time": item.get("reportTime"), + "eps_estimate": _f(item.get("estimatedEPS")), + "eps_actual": _f(item.get("reportedEPS")), + "revenue_estimate": None, + "revenue_actual": None, + }) + return out + + +async def _main() -> None: + args = _parse_args() + snapshot = Path(args.snapshot) + if not snapshot.exists(): + raise SystemExit(f"Snapshot not found: {snapshot}") + + from app.config import settings + + if not settings.fmp_api_key: + raise SystemExit("FMP_API_KEY required") + + start = date.fromisoformat(args.from_date) + end = date.fromisoformat(args.to_date) if args.to_date else date.today() + engine = create_engine( + f"sqlite:///{snapshot.resolve().as_posix()}", + future=True, + ) + _ensure_tables(engine) + + with engine.connect() as conn: + symbols = [ + str(r[0]).upper().replace(".", "-") + for r in conn.execute(text("SELECT symbol FROM tickers ORDER BY symbol")) + ] + done = set() + if not args.refetch_done: + done = { + str(r[0]) + for r in conn.execute( + text( + "SELECT symbol FROM earnings_backfill_meta " + "WHERE status='done' AND n_events > 0" + ) + ) + } + + pending = [s for s in symbols if s not in done] + print(f"Snapshot: {snapshot}") + print(f"Universe: {len(symbols)}; pending: {len(pending)}; done: {len(done)}") + print(f"Window filter: {start} → {end}") + print(f"Provider: {args.provider}") + + req_budget = int(args.limit) + reqs_used = 0 + events_written = 0 + mode = "per_symbol" + use_av = args.provider in ("alpha_vantage", "auto") and bool( + getattr(settings, "alpha_vantage_api_key", "") + ) + use_fmp = args.provider in ("fmp", "auto") and bool(settings.fmp_api_key) + + async with httpx.AsyncClient(timeout=60.0) as client: + if ( + not args.force_symbol + and req_budget > 0 + and use_fmp + and args.provider != "alpha_vantage" + ): + print("Attempting bulk earnings-calendar…") + bulk_rows, bulk_reqs, err = await _try_bulk( + client, settings.fmp_api_key, start, end + ) + reqs_used += bulk_reqs + if err: + print(f" Bulk unavailable: {err} (requests={bulk_reqs})") + else: + # Filter to universe. + uni = set(symbols) + bulk_rows = [r for r in bulk_rows if r["symbol"] in uni] + with engine.begin() as conn: + events_written += _upsert_events(conn, bulk_rows, "fmp_earnings_calendar") + for sym in symbols: + n = conn.execute( + text( + "SELECT COUNT(*) FROM earnings_events WHERE symbol=:s" + ), + {"s": sym}, + ).scalar_one() + conn.execute( + text( + """ + INSERT INTO earnings_backfill_meta(symbol, status, n_events, updated_at, note) + VALUES (:s, 'done', :n, :t, 'bulk') + ON CONFLICT(symbol) DO UPDATE SET + status='done', n_events=excluded.n_events, + updated_at=excluded.updated_at, note=excluded.note + """ + ), + { + "s": sym, + "n": int(n), + "t": datetime.now(timezone.utc).isoformat(), + }, + ) + mode = "bulk" + print(f" Bulk wrote {events_written} events; requests={bulk_reqs}") + pending = [] + + # Per-symbol fallback / completion. + fmp_limited = False + for sym in pending: + if reqs_used >= req_budget: + print(f"Request budget exhausted ({req_budget}). Resume later.") + break + items: list[dict] = [] + source = "fmp_earnings" + note = "per_symbol" + try: + if use_fmp and not fmp_limited and args.provider != "alpha_vantage": + items = await _fetch_symbol(client, settings.fmp_api_key, sym) + source = "fmp_earnings" + note = "fmp_per_symbol" + # Empty list may mean soft-limit or no data — try AV if available. + if not items and use_av: + items = await _fetch_symbol_alpha_vantage( + client, settings.alpha_vantage_api_key, sym + ) + source = "alpha_vantage_earnings" + note = "av_after_fmp_empty" + reqs_used += 1 # count AV call separately below too + elif use_av: + items = await _fetch_symbol_alpha_vantage( + client, settings.alpha_vantage_api_key, sym + ) + source = "alpha_vantage_earnings" + note = "av_per_symbol" + else: + raise RuntimeError("no provider available") + except Exception as exc: + msg = str(exc) + print(f" FAIL {sym}: {msg}") + reqs_used += 1 + if "rate_limited" in msg and note.startswith("fmp"): + fmp_limited = True + with engine.begin() as conn: + conn.execute( + text( + """ + INSERT INTO earnings_backfill_meta(symbol, status, n_events, updated_at, note) + VALUES (:s, 'error', 0, :t, :n) + ON CONFLICT(symbol) DO UPDATE SET + status='error', updated_at=excluded.updated_at, note=excluded.note + """ + ), + { + "s": sym, + "t": datetime.now(timezone.utc).isoformat(), + "n": msg[:200], + }, + ) + if args.sleep > 0: + await asyncio.sleep(args.sleep) + continue + + reqs_used += 1 + # Keep all rows with dates on/before end — SUE needs trailing history. + filtered = [ + r for r in items if r["announce_date"] <= end.isoformat() + ] + # Do NOT mark empty as done — leave pending for another provider/day. + status = "done" if filtered else "empty" + with engine.begin() as conn: + n_w = _upsert_events(conn, filtered, source) if filtered else 0 + events_written += n_w + conn.execute( + text( + """ + INSERT INTO earnings_backfill_meta(symbol, status, n_events, updated_at, note) + VALUES (:s, :st, :n, :t, :note) + ON CONFLICT(symbol) DO UPDATE SET + status=excluded.status, n_events=excluded.n_events, + updated_at=excluded.updated_at, note=excluded.note + """ + ), + { + "s": sym, + "st": status, + "n": len(filtered), + "t": datetime.now(timezone.utc).isoformat(), + "note": note, + }, + ) + if reqs_used % 10 == 0 or reqs_used == 1: + print( + f" progress reqs={reqs_used}/{req_budget} last={sym} " + f"events_batch={len(filtered)} src={source}" + ) + # AV free tier is ~5/min or 25/day — be polite when using it. + sleep_s = float(args.sleep) + if source.startswith("alpha_vantage"): + sleep_s = max(sleep_s, 12.0) + if sleep_s > 0: + await asyncio.sleep(sleep_s) + + with engine.connect() as conn: + total_events = int( + conn.execute(text("SELECT COUNT(*) FROM earnings_events")).scalar_one() + ) + done_n = int( + conn.execute( + text("SELECT COUNT(*) FROM earnings_backfill_meta WHERE status='done'") + ).scalar_one() + ) + d_range = conn.execute( + text("SELECT MIN(announce_date), MAX(announce_date) FROM earnings_events") + ).fetchone() + with_actual = int( + conn.execute( + text( + "SELECT COUNT(*) FROM earnings_events " + "WHERE eps_actual IS NOT NULL AND eps_estimate IS NOT NULL" + ) + ).scalar_one() + ) + + summary = { + "mode": mode, + "fmp_requests": reqs_used, + "events_written_this_run": events_written, + "total_events": total_events, + "symbols_done": done_n, + "symbols_universe": len(symbols), + "announce_date_range": {"min": d_range[0], "max": d_range[1]}, + "events_with_actual_and_estimate": with_actual, + "budget": req_budget, + "complete": done_n >= len(symbols), + } + print(json.dumps(summary, indent=2)) + out = Path("reports") / "earnings-backfill-status.json" + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8") + print(f"Wrote {out}") + + +if __name__ == "__main__": + asyncio.run(_main()) diff --git a/scripts/build_ticker_sector_map.py b/scripts/build_ticker_sector_map.py new file mode 100644 index 0000000..f4a9ae9 --- /dev/null +++ b/scripts/build_ticker_sector_map.py @@ -0,0 +1,229 @@ +"""Build a local ticker → GICS sector map for research residualization. + +Sources (in order): +1. Public S&P 500 constituents CSV (datasets/s-and-p-500-companies) — bulk, free. +2. Existing map file (resume). +3. FMP stable ``profile`` for still-missing symbols (budget ~250 req/day). + +Writes ``data/research/ticker_sector_map.json``. Never touches production Postgres. + +Example +------- + python scripts/build_ticker_sector_map.py \\ + --snapshot backtest_snapshots/prod.sqlite + + python scripts/build_ticker_sector_map.py --fmp-limit 50 +""" + +from __future__ import annotations + +import argparse +import asyncio +import csv +import io +import json +import sys +import time +from datetime import datetime, timezone +from pathlib import Path + +import httpx +from sqlalchemy import create_engine, text + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from app.services.sector_map import ( # noqa: E402 + DEFAULT_SECTOR_MAP_PATH, + coverage_stats, + load_ticker_sector_map, + normalise_symbol, + save_ticker_sector_map, + sector_to_etf, +) + +SP500_CSV_URL = ( + "https://raw.githubusercontent.com/datasets/s-and-p-500-companies/" + "master/data/constituents.csv" +) +FMP_STABLE = "https://financialmodelingprep.com/stable" + + +def _parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument( + "--snapshot", + default="backtest_snapshots/prod.sqlite", + help="Snapshot whose tickers define the universe.", + ) + p.add_argument( + "--out", + default=str(DEFAULT_SECTOR_MAP_PATH), + help="Output JSON path.", + ) + p.add_argument( + "--fmp-limit", + type=int, + default=200, + help="Max FMP profile requests this run (free-tier cushion).", + ) + p.add_argument( + "--skip-fmp", + action="store_true", + help="Only use public SP500 CSV + existing map.", + ) + p.add_argument("--sleep", type=float, default=0.35, help="Pause between FMP calls.") + return p.parse_args() + + +def _snapshot_symbols(snapshot: Path) -> list[str]: + engine = create_engine(f"sqlite:///{snapshot.resolve().as_posix()}", future=True) + try: + with engine.connect() as conn: + rows = conn.execute(text("SELECT symbol FROM tickers ORDER BY symbol")).fetchall() + finally: + engine.dispose() + return [normalise_symbol(r[0]) for r in rows if r[0]] + + +def _fetch_sp500_map() -> dict[str, str]: + with httpx.Client(timeout=60.0, follow_redirects=True) as client: + resp = client.get(SP500_CSV_URL) + resp.raise_for_status() + reader = csv.DictReader(io.StringIO(resp.text)) + out: dict[str, str] = {} + for row in reader: + sym = normalise_symbol(row.get("Symbol") or "") + sector = (row.get("GICS Sector") or "").strip() + if sym and sector: + out[sym] = sector + return out + + +async def _fmp_profile_sector(client: httpx.AsyncClient, api_key: str, symbol: str) -> str | None: + resp = await client.get( + f"{FMP_STABLE}/profile", + params={"symbol": symbol, "apikey": api_key}, + ) + if resp.status_code == 429: + raise RuntimeError(f"FMP rate limited on {symbol}") + if resp.status_code == 402: + return None + resp.raise_for_status() + data = resp.json() + if isinstance(data, list): + data = data[0] if data else {} + if not isinstance(data, dict): + return None + sector = (data.get("sector") or data.get("industry") or "").strip() + # industry alone is not a GICS sector — only accept if we can map to an ETF + if sector and sector_to_etf(sector): + return sector + # FMP sometimes returns industry under sector when sector missing; try sector field only + sec = (data.get("sector") or "").strip() + return sec or None + + +async def _fill_from_fmp( + missing: list[str], + *, + api_key: str, + limit: int, + sleep_s: float, +) -> tuple[dict[str, str], int]: + filled: dict[str, str] = {} + used = 0 + async with httpx.AsyncClient(timeout=30.0) as client: + for sym in missing: + if used >= limit: + break + try: + sector = await _fmp_profile_sector(client, api_key, sym) + except Exception as exc: + print(f" FMP fail {sym}: {exc}") + used += 1 + await asyncio.sleep(sleep_s) + continue + used += 1 + if sector: + filled[sym] = sector + print(f" FMP {sym} → {sector}") + else: + print(f" FMP {sym} → (no sector)") + if sleep_s > 0: + await asyncio.sleep(sleep_s) + return filled, used + + +async def _main() -> None: + args = _parse_args() + snapshot = Path(args.snapshot) + if not snapshot.exists(): + raise SystemExit(f"Snapshot not found: {snapshot}") + + symbols = _snapshot_symbols(snapshot) + print(f"Universe: {len(symbols)} symbols from {snapshot}") + + existing = load_ticker_sector_map(args.out) + print(f"Existing map entries: {len(existing)}") + + print("Fetching public S&P 500 sector CSV…") + sp500 = _fetch_sp500_map() + print(f" SP500 CSV rows: {len(sp500)}") + + mapping = dict(existing) + from_sp500 = 0 + for sym in symbols: + if sym in mapping: + continue + if sym in sp500: + mapping[sym] = sp500[sym] + from_sp500 += 1 + print(f" Newly filled from SP500 CSV: {from_sp500}") + + missing = [s for s in symbols if s not in mapping] + fmp_used = 0 + from_fmp = 0 + if missing and not args.skip_fmp: + from app.config import settings + + if not settings.fmp_api_key: + print("WARNING: FMP key missing; leaving gaps unfilled") + else: + print(f"FMP fill for {len(missing)} missing (limit={args.fmp_limit})…") + filled, fmp_used = await _fill_from_fmp( + missing, + api_key=settings.fmp_api_key, + limit=int(args.fmp_limit), + sleep_s=float(args.sleep), + ) + mapping.update(filled) + from_fmp = len(filled) + + still_missing = [s for s in symbols if s not in mapping] + stats = coverage_stats(symbols, mapping) + meta = { + "built_at": datetime.now(timezone.utc).isoformat(), + "snapshot": str(snapshot.resolve()), + "from_existing": len(existing), + "from_sp500_csv": from_sp500, + "from_fmp": from_fmp, + "fmp_requests": fmp_used, + "still_missing": still_missing, + "coverage": { + k: stats[k] + for k in ("universe", "mapped", "mapped_pct", "with_etf", "by_sector") + }, + } + out_path = save_ticker_sector_map(mapping, args.out, meta=meta) + print(f"Wrote {out_path}") + print(json.dumps(meta["coverage"], indent=2)) + if still_missing: + print(f"Still missing ({len(still_missing)}): {still_missing[:40]}") + if len(still_missing) > 40: + print(f" … +{len(still_missing) - 40} more") + + +if __name__ == "__main__": + asyncio.run(_main()) diff --git a/scripts/fetch_sector_etfs_to_snapshot.py b/scripts/fetch_sector_etfs_to_snapshot.py new file mode 100644 index 0000000..db558e0 --- /dev/null +++ b/scripts/fetch_sector_etfs_to_snapshot.py @@ -0,0 +1,183 @@ +"""Fetch the 11 SPDR sector ETFs into a snapshot's ``benchmark_prices``. + +Research-only. Sector ETFs are auxiliary series (like SPY) — they must not +enter the tradable ticker universe or candidate replay. Storing them in +``benchmark_prices`` keeps that invariant. + +Also refreshes SPY on the same window so residual factors share a calendar. + +Example +------- + python scripts/fetch_sector_etfs_to_snapshot.py \\ + --snapshot backtest_snapshots/prod.sqlite --history-days 2200 +""" + +from __future__ import annotations + +import argparse +import asyncio +import sys +import time +from datetime import date, timedelta +from pathlib import Path + +from sqlalchemy import create_engine, text + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from app.services.sector_map import SECTOR_ETFS # noqa: E402 + + +def _parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--snapshot", default="backtest_snapshots/prod.sqlite") + p.add_argument( + "--history-days", + type=int, + default=2200, + help="Lookback calendar days (default ~6y; covers 5y snapshot + cushion).", + ) + p.add_argument("--sleep", type=float, default=0.25) + p.add_argument( + "--symbols", + default=None, + help="Comma-separated override (default: SPY + 11 sector ETFs).", + ) + return p.parse_args() + + +async def _fetch_and_upsert( + engine, + provider, + symbol: str, + start: date, + end: date, + *, + sleep_s: float, +) -> int: + from app.exceptions import ProviderError, RateLimitError + + for attempt in range(5): + try: + bars = await provider.fetch_ohlcv(symbol, start, end) + break + except RateLimitError: + wait = min(60.0, 2.0 ** attempt) + print(f" rate limited {symbol}; sleep {wait:.0f}s") + await asyncio.sleep(wait) + bars = [] + except ProviderError as exc: + if attempt + 1 >= 5: + raise + await asyncio.sleep(1.0) + print(f" retry {symbol}: {exc}") + bars = [] + else: + bars = [] + + if sleep_s > 0: + await asyncio.sleep(sleep_s) + + if not bars: + print(f" {symbol}: empty") + return 0 + + written = 0 + with engine.begin() as conn: + for bar in bars: + d = bar.date.isoformat() if hasattr(bar.date, "isoformat") else str(bar.date) + close = float(bar.close) + existing = conn.execute( + text( + "SELECT id, close FROM benchmark_prices " + "WHERE symbol = :sym AND date = :d" + ), + {"sym": symbol, "d": d}, + ).fetchone() + if existing is None: + # id is INTEGER PK — let sqlite autoincrement if possible + conn.execute( + text( + "INSERT INTO benchmark_prices (symbol, date, close) " + "VALUES (:sym, :d, :c)" + ), + {"sym": symbol, "d": d, "c": close}, + ) + written += 1 + elif abs(float(existing[1]) - close) > 1e-9: + conn.execute( + text( + "UPDATE benchmark_prices SET close = :c WHERE id = :id" + ), + {"c": close, "id": int(existing[0])}, + ) + written += 1 + print(f" {symbol}: {len(bars)} bars, {written} rows written/updated") + return written + + +async def _main() -> None: + args = _parse_args() + snapshot = Path(args.snapshot) + if not snapshot.exists(): + raise SystemExit(f"Snapshot not found: {snapshot}") + + from app.config import settings + from app.providers.alpaca import AlpacaOHLCVProvider + + if not settings.alpaca_api_key or not settings.alpaca_api_secret: + raise SystemExit("ALPACA_API_KEY / ALPACA_API_SECRET required") + + if args.symbols: + symbols = [s.strip().upper() for s in args.symbols.split(",") if s.strip()] + else: + symbols = ["SPY", *SECTOR_ETFS] + + end = date.today() + start = end - timedelta(days=int(args.history_days)) + provider = AlpacaOHLCVProvider(settings.alpaca_api_key, settings.alpaca_api_secret) + engine = create_engine( + f"sqlite:///{snapshot.resolve().as_posix()}", + future=True, + ) + + print(f"Snapshot: {snapshot}") + print(f"Window: {start} → {end}") + print(f"Symbols: {symbols}") + + t0 = time.monotonic() + total = 0 + try: + for sym in symbols: + n = await _fetch_and_upsert( + engine, provider, sym, start, end, sleep_s=float(args.sleep) + ) + total += n + finally: + engine.dispose() + + # Summary counts + engine = create_engine( + f"sqlite:///{snapshot.resolve().as_posix()}", + future=True, + ) + try: + with engine.connect() as conn: + rows = conn.execute( + text( + "SELECT symbol, COUNT(*), MIN(date), MAX(date) " + "FROM benchmark_prices GROUP BY symbol ORDER BY symbol" + ) + ).fetchall() + finally: + engine.dispose() + + print(f"Done in {(time.monotonic() - t0) / 60:.1f}m; rows touched={total}") + for sym, n, d0, d1 in rows: + print(f" {sym}: n={n} {d0}→{d1}") + + +if __name__ == "__main__": + asyncio.run(_main()) diff --git a/scripts/run_earnings_research.py b/scripts/run_earnings_research.py new file mode 100644 index 0000000..20c695f --- /dev/null +++ b/scripts/run_earnings_research.py @@ -0,0 +1,927 @@ +"""Earnings gap diagnostic (2a) + SUE IC (2b). Local research only. + +Requires ``earnings_events`` on the snapshot (see backfill_earnings_events.py). + +Example +------- + python scripts/run_earnings_research.py \\ + --snapshot backtest_snapshots/prod.sqlite --workers 6 --allow-spawn +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import math +import os +import sys +from collections import defaultdict +from datetime import date, datetime, timedelta +from pathlib import Path +from typing import Any + +from sqlalchemy import create_engine, text +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +IRON_IC_BAR = 0.03 +MIN_RELIABLE = 12 +SUE_CARRY_DAYS = 63 +SUE_TRAIL = 8 + + +def _sqlite_url(path: Path) -> str: + return f"sqlite+aiosqlite:///{path.resolve().as_posix()}" + + +def _parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--snapshot", default="backtest_snapshots/prod.sqlite") + p.add_argument("--workers", type=int, default=6) + p.add_argument("--allow-spawn", action="store_true") + p.add_argument("--skip-2a", action="store_true") + p.add_argument("--skip-2b", action="store_true") + p.add_argument("--quiet", action="store_true") + p.add_argument("--out", default=None) + return p.parse_args() + + +def _load_earnings(snapshot: Path) -> list[dict]: + engine = create_engine( + f"sqlite:///{snapshot.resolve().as_posix()}", + future=True, + ) + try: + with engine.connect() as conn: + # Table must exist. + tables = { + r[0] + for r in conn.execute( + text("SELECT name FROM sqlite_master WHERE type='table'") + ) + } + if "earnings_events" not in tables: + raise SystemExit( + "earnings_events table missing — run scripts/backfill_earnings_events.py" + ) + rows = conn.execute( + text( + """ + SELECT symbol, announce_date, announce_time, + eps_estimate, eps_actual, revenue_estimate, revenue_actual + FROM earnings_events + ORDER BY symbol, announce_date + """ + ) + ).fetchall() + meta = {} + if "earnings_backfill_meta" in tables: + meta = { + "done": int( + conn.execute( + text( + "SELECT COUNT(*) FROM earnings_backfill_meta " + "WHERE status='done'" + ) + ).scalar_one() + ), + "universe_tickers": int( + conn.execute(text("SELECT COUNT(*) FROM tickers")).scalar_one() + ), + } + finally: + engine.dispose() + + events = [ + { + "symbol": str(r[0]).upper(), + "announce_date": date.fromisoformat(str(r[1])[:10]), + "announce_time": r[2], + "eps_estimate": r[3], + "eps_actual": r[4], + "revenue_estimate": r[5], + "revenue_actual": r[6], + } + for r in rows + ] + return events, meta + + +def _percentile(xs: list[float], q: float) -> float | None: + if not xs: + return None + s = sorted(xs) + if len(s) == 1: + return s[0] + idx = q * (len(s) - 1) + lo = int(math.floor(idx)) + hi = int(math.ceil(idx)) + if lo == hi: + return s[lo] + w = idx - lo + return s[lo] * (1 - w) + s[hi] * w + + +def _r_dist(rs: list[float]) -> dict[str, Any]: + if not rs: + return {"n": 0} + return { + "n": len(rs), + "mean": round(sum(rs) / len(rs), 4), + "win_rate": round(sum(1 for r in rs if r > 0) / len(rs), 4), + "p05": round(_percentile(rs, 0.05), 4), + "p25": round(_percentile(rs, 0.25), 4), + "p50": round(_percentile(rs, 0.50), 4), + "p75": round(_percentile(rs, 0.75), 4), + "p95": round(_percentile(rs, 0.95), 4), + "min": round(min(rs), 4), + "max": round(max(rs), 4), + } + + +def _trading_days_between( + entry: date, exit_: date, calendar: set[date] +) -> list[date]: + """Inclusive trading dates in [entry, exit_] present on the union calendar.""" + out = [] + d = entry + while d <= exit_: + if d in calendar: + out.append(d) + d += timedelta(days=1) + return out + + +def _nth_trading_day_after( + start: date, n: int, ordered_calendar: list[date] +) -> date | None: + """First calendar date strictly after ``start``, then + (n-1) more sessions. + + announce+1 trading day: n=1 → first session after announce date + (if announce is a trading day, still use the *next* session for PIT). + """ + # Sessions strictly after start. + after = [d for d in ordered_calendar if d > start] + if len(after) < n: + return None + return after[n - 1] + + +def _build_sue_series( + events_by_symbol: dict[str, list[dict]], + prices: dict[str, tuple], +) -> dict[str, dict[date, float]]: + """symbol → {asof_date: sue_value} for days when SUE is live (announce+1 .. +63).""" + out: dict[str, dict[date, float]] = {} + for sym, cols in prices.items(): + ords = cols[0] + closes = cols[4] + dates = [date.fromordinal(int(o)) for o in ords] + if not dates: + continue + ordered = dates # already chronological + cal_set = set(ordered) + events = events_by_symbol.get(sym.upper(), []) + # Chronological surprises with actual+estimate. + surprises: list[tuple[date, float, float]] = [] # announce, surprise, close_for_scale + for ev in events: + act, est = ev.get("eps_actual"), ev.get("eps_estimate") + if act is None or est is None: + continue + ad = ev["announce_date"] + # Close on/before announce for price fallback scale. + close_px = None + for d, c in zip(reversed(dates), reversed(closes)): + if d <= ad and float(c) > 0: + close_px = float(c) + break + surprises.append((ad, float(act) - float(est), close_px or 1.0)) + surprises.sort(key=lambda x: x[0]) + + sue_on_day: dict[date, float] = {} + for i, (ad, surprise, px) in enumerate(surprises): + trail = [surprises[j][1] for j in range(max(0, i - SUE_TRAIL), i)] + # Need history of surprises; include current only for value, stdev from prior 8. + if len(trail) >= 3: + mean_t = sum(trail) / len(trail) + var = sum((x - mean_t) ** 2 for x in trail) / (len(trail) - 1) + sd = math.sqrt(var) if var > 0 else None + else: + sd = None + if sd is not None and sd > 1e-9: + sue = surprise / sd + else: + # Fallback: scale by price (EPS surprise / price). + sue = surprise / px if px > 0 else None + if sue is None or not math.isfinite(sue): + continue + usable_from = _nth_trading_day_after(ad, 1, ordered) + if usable_from is None: + continue + # Carry for SUE_CARRY_DAYS trading sessions starting at usable_from. + try: + start_idx = ordered.index(usable_from) + except ValueError: + # usable_from not in this symbol's calendar (halted etc.) + start_idx = next( + (k for k, d in enumerate(ordered) if d >= usable_from), None + ) + if start_idx is None: + continue + end_idx = min(len(ordered) - 1, start_idx + SUE_CARRY_DAYS - 1) + for k in range(start_idx, end_idx + 1): + # Later announcements overwrite earlier carry (latest SUE wins). + sue_on_day[ordered[k]] = sue + if sue_on_day: + out[sym.upper()] = sue_on_day + return out + + +async def _run_2a( + snapshot: Path, + events: list[dict], + *, + quiet: bool, + workers: int, +) -> dict[str, Any]: + from app.config import settings + from app.services import backtest_service as bt + from app.services.admin_service import get_activation_config + from app.services.recommendation_service import get_recommendation_config + from app.services.paper_trade_service import get_exit_policy + from app.services.benchmark_service import load_benchmark_closes + from app.models.ticker import Ticker + from sqlalchemy import select + + os.environ["BACKTEST_SNAPSHOT_OFFLINE"] = "1" + settings.backtest_workers = workers + + engine = create_async_engine(_sqlite_url(snapshot), pool_pre_ping=True) + Session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) + + try: + async with Session() as db: + config = await get_recommendation_config(db) + activation = await get_activation_config(db) + exit_config = await get_exit_policy(db) + tickers = list( + (await db.execute(select(Ticker).order_by(Ticker.symbol))).scalars() + ) + spy = await load_benchmark_closes(db, "SPY") + prices: dict[str, tuple] = {} + candidates: list[dict] = [] + for idx, t in enumerate(tickers): + if not quiet and idx % 50 == 0: + print(f" 2a fetch {idx}/{len(tickers)}", end="\r", flush=True) + cols = await bt._fetch_columns(db, t.symbol) + if cols is None: + continue + prices[t.symbol] = cols + cands, _ = bt._replay_and_signals( + t.symbol, + cols, + config, + activation, + spy, + bt.PRODUCTION_GTL_TARGET_MODEL, + "weekly", + False, + ) + candidates.extend(cands) + finally: + await engine.dispose() + if not quiet: + print() + + # Production ranks + qualify. + bt._assign_momentum_percentiles(candidates) + bt._assign_residual_momentum_percentiles(candidates) + bt._assign_low_volatility_percentiles(candidates) + bt._assign_activation_momentum_percentiles(candidates) + bt._assign_residual_high_vol_blend(candidates) + for c in candidates: + c["qualified"] = bt._momentum_qualifies(c, 80.0) + longs = [ + c for c in candidates if c.get("qualified") and c.get("direction") == "long" + ] + + strategy = next(s for s in bt.PORTFOLIO_MONITOR_STRATEGIES if s.get("is_production")) + entry_cfg = bt._entry_variant_config(str(strategy["entry_variant"])) + assert entry_cfg is not None + ranking_key = str(entry_cfg.get("ranking_key") or entry_cfg["percentile_key"]) + exit_policy = bt.LIVE_EXIT_MODE_TO_SIM.get( + str(exit_config.get("mode", "atr_trailing")), "atr_trail3" + ) + hold_days = int(exit_config.get("hold_days", 30)) + trail = float(exit_config.get("atr_multiplier", bt.ATR_TRAIL_MULTIPLIER)) + reentry = bt._make_gate_reset_reentry_fn( + longs, prices, cadence="weekly", ranking_key=ranking_key + ) + sim = bt._simulate_portfolio( + longs, + prices, + spy, + exit_policy, + hold_days, + ranking_key=ranking_key, + max_positions=int(entry_cfg["max_positions"]), + risk_per_trade=float(entry_cfg["risk_per_trade"]), + atr_trail_multiplier=trail, + post_stop_reentry_fn=reentry, + fill_mode=bt.FILL_MODE_CLOSE, + include_trades=True, + ) + if sim is None: + return {"error": "no_trades"} + + details = sim.get("trade_details") or [] + # Build per-symbol earnings announce dates. + earns_by_sym: dict[str, list[date]] = defaultdict(list) + for ev in events: + earns_by_sym[ev["symbol"]].append(ev["announce_date"]) + for sym in earns_by_sym: + earns_by_sym[sym].sort() + + # Union trading calendar from prices. + cal: set[date] = set() + for cols in prices.values(): + for o in cols[0]: + cal.add(date.fromordinal(int(o))) + ordered_cal = sorted(cal) + + # Map entry date → list of announce dates for symbol (for pre-entry lookback). + trades_parsed: list[dict] = [] + for t in details: + sym = str(t.get("symbol") or "").upper() + # Field names from simulator. + entry_s = t.get("entry_date") or t.get("open_date") or t.get("date") + exit_s = t.get("exit_date") or t.get("close_date") + r = t.get("realized_r") + if r is None: + r = t.get("r") + if entry_s is None or exit_s is None or r is None: + continue + entry_d = date.fromisoformat(str(entry_s)[:10]) + exit_d = date.fromisoformat(str(exit_s)[:10]) + announces = earns_by_sym.get(sym, []) + # Earnings between entry and exit (exclusive of entry day? inclusive hold). + # "between entry and exit" — any announce with entry < announce <= exit + # (gap often overnight after entry). Also count announce on entry day. + in_hold = [ + a for a in announces if entry_d <= a <= exit_d + ] + # Entries within 3 trading days BEFORE an announcement: + # exists announce such that entry is in the 3 sessions immediately before announce. + pre_earn = False + for a in announces: + # trading sessions in (a-lookback, a) + sessions_before = [d for d in ordered_cal if d < a] + last3 = sessions_before[-3:] if len(sessions_before) >= 3 else sessions_before + if entry_d in last3: + pre_earn = True + break + trades_parsed.append({ + "symbol": sym, + "entry": entry_d.isoformat(), + "exit": exit_d.isoformat(), + "r": float(r), + "earnings_in_hold": len(in_hold) > 0, + "n_earnings_in_hold": len(in_hold), + "entry_within_3d_before_earn": pre_earn, + }) + + all_r = [t["r"] for t in trades_parsed] + loss_lt_1r = [t for t in trades_parsed if t["r"] < -1.0] + loss_with_earn = [t for t in loss_lt_1r if t["earnings_in_hold"]] + pre = [t["r"] for t in trades_parsed if t["entry_within_3d_before_earn"]] + other = [t["r"] for t in trades_parsed if not t["entry_within_3d_before_earn"]] + + return { + "sim_summary": { + k: sim.get(k) + for k in ( + "sharpe", + "sharpe_se", + "cagr_pct", + "max_drawdown_pct", + "trades", + "total_return_pct", + ) + }, + "n_trades_parsed": len(trades_parsed), + "q1_losses_worse_than_minus_1r": { + "n_losses_lt_minus_1r": len(loss_lt_1r), + "n_with_earnings_in_hold": len(loss_with_earn), + "fraction_with_earnings": ( + round(len(loss_with_earn) / len(loss_lt_1r), 4) if loss_lt_1r else None + ), + "all_trades_with_earnings_in_hold": sum( + 1 for t in trades_parsed if t["earnings_in_hold"] + ), + "fraction_all_trades_with_earnings": ( + round( + sum(1 for t in trades_parsed if t["earnings_in_hold"]) + / len(trades_parsed), + 4, + ) + if trades_parsed + else None + ), + }, + "q2_entry_within_3d_before_announce": { + "pre_earn_entries": _r_dist(pre), + "other_entries": _r_dist(other), + "all_entries": _r_dist(all_r), + "tail_trim_note": ( + "Compare p95/max and mean of pre_earn vs other. " + "Rising win_rate with falling mean/p95 = right-tail trim red flag." + ), + }, + "note": "REPORT-ONLY — no filter shipped.", + } + + +async def _run_2b_ic( + snapshot: Path, + events: list[dict], + *, + quiet: bool, + workers: int, +) -> dict[str, Any]: + """SUE IC via harness on identical cross-sections as momentum baselines.""" + from app.config import settings + from app.services import backtest_service as bt + from app.services.benchmark_service import load_benchmark_closes + from app.models.ticker import Ticker + from sqlalchemy import select + from collections import defaultdict as dd + + os.environ["BACKTEST_SNAPSHOT_OFFLINE"] = "1" + os.environ["BACKTEST_SIGNAL_EVAL_ONLY"] = "1" + # Load sector map if present so sector signals also appear (side-by-side optional). + if Path("data/research/ticker_sector_map.json").exists(): + os.environ["BACKTEST_SECTOR_MAP_PATH"] = str( + Path("data/research/ticker_sector_map.json").resolve() + ) + settings.backtest_workers = workers + + engine = create_async_engine(_sqlite_url(snapshot), pool_pre_ping=True) + Session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) + + # Collect base signals + attach SUE. + collected: dict = dd(lambda: dd(list)) + try: + async with Session() as db: + tickers = list( + (await db.execute(select(Ticker).order_by(Ticker.symbol))).scalars() + ) + spy = await load_benchmark_closes(db, "SPY") + sector_etf: dict[str, dict] = {} + try: + from app.services.sector_map import SECTOR_ETFS, load_ticker_sector_map + + symbol_to_sector = load_ticker_sector_map() + for etf in SECTOR_ETFS: + series = await load_benchmark_closes(db, etf) + if series: + sector_etf[etf] = series + except Exception: + symbol_to_sector = {} + sector_etf = {} + + prices: dict[str, tuple] = {} + for idx, t in enumerate(tickers): + if not quiet and idx % 50 == 0: + print(f" 2b fetch {idx}/{len(tickers)}", end="\r", flush=True) + cols = await bt._fetch_columns(db, t.symbol) + if cols is None: + continue + prices[t.symbol] = cols + series = bt._signal_series( + [ + type( + "R", + (), + { + "date": date.fromordinal(int(cols[0][i])), + "close": cols[4][i], + "high": cols[2][i], + "volume": cols[5][i] if len(cols) > 5 else 0, + }, + )() + for i in range(len(cols[0])) + ], + spy, + symbol=t.symbol, + sector_etf_closes=bt._sector_etf_closes_for_symbol( + t.symbol, symbol_to_sector, sector_etf + ), + ) + for name, weeks in series.items(): + for wk, pairs in weeks.items(): + collected[name][wk].extend(pairs) + finally: + await engine.dispose() + if not quiet: + print() + + if symbol_to_sector: + bt._inject_sector_demeaned_momentum(collected, symbol_to_sector) + + # SUE series. + events_by_sym: dict[str, list[dict]] = defaultdict(list) + for ev in events: + events_by_sym[ev["symbol"]].append(ev) + sue_map = _build_sue_series(events_by_sym, prices) + + # Inject sue_latest into collected using mom_12_1 observations as the + # weekly as-of skeleton (same weeks / symbols). + sue_collected: dict = dd(list) + mom_weeks = collected.get("mom_12_1") or {} + for week_key, recs in mom_weeks.items(): + for rec in recs: + pair = bt._obs_val_fwd(rec) + if pair is None: + continue + _val, fwd = pair + sym = None + if isinstance(rec, dict): + sym = rec.get("symbol") + if not sym: + continue + # Need as-of date: recover from week — use Friday of ISO week as proxy + # is weak. Better: re-derive from prices weekly indices. + # Store asof on rich recs? Current rich rows lack asof date. + # Fall back: compute SUE observations directly from prices weekly as-ofs. + pass + + # Direct weekly as-of SUE + forward return (authoritative). + for sym, cols in prices.items(): + ords, _o, highs, _l, closes, _v = cols + dates = [date.fromordinal(int(o)) for o in ords] + sue_days = sue_map.get(sym.upper()) or {} + if not sue_days: + continue + n = len(dates) + # weekly as-of indices: reuse harness helper via fake records. + records = [ + type("R", (), {"date": dates[i], "close": closes[i], "high": highs[i]})() + for i in range(n) + ] + for i in bt._weekly_asof_indices(records): + j = i + bt.HORIZON + if j >= n or closes[i] <= 0: + continue + asof = dates[i] + sue = sue_days.get(asof) + if sue is None: + continue + fwd = float(closes[j]) / float(closes[i]) - 1.0 + iso = asof.isocalendar() + week_key = (iso[0], iso[1]) + # Also grab mom for conditional. + mom = None + if i >= 252 and closes[i - 252] > 0: + mom = float(closes[i - 21]) / float(closes[i - 252]) - 1.0 + sue_collected[week_key].append({ + "val": float(sue), + "fwd": fwd, + "symbol": sym, + "mom_12_1": mom, + }) + collected["sue_latest"] = sue_collected + + signal_eval = bt._signal_evaluation(collected) + + # Fair side-by-side: re-evaluate mom baselines on the *same* (symbol, week) + # observations where SUE is present (incomplete backfill otherwise inflates + # mom N relative to SUE). + sue_pairs_by_week = sue_collected + restricted: dict = dd(lambda: dd(list)) + for week_key, recs in sue_pairs_by_week.items(): + syms = {str(r.get("symbol")).upper() for r in recs if r.get("symbol")} + for base_name in ("mom_12_1", "mom_12_1_resid"): + base_recs = (collected.get(base_name) or {}).get(week_key) or [] + for rec in base_recs: + pair = bt._obs_val_fwd(rec) + if pair is None: + continue + sym = None + if isinstance(rec, dict): + sym = rec.get("symbol") + if not sym or str(sym).upper() not in syms: + continue + restricted[base_name][week_key].append(rec) + restricted["sue_latest"][week_key].extend(recs) + restricted_eval = bt._signal_evaluation(restricted) + + # Momentum-conditional: IC of SUE within top mom quintile each week. + cond_ics: list[float] = [] + stride = max(1, round(bt.HORIZON / 5)) + usable = [wk for wk, recs in sue_collected.items() if len(recs) >= bt.MIN_CROSS_SECTION] + kept = bt._nonoverlapping_weeks(usable, stride) + for wk in kept: + recs = sue_collected[wk] + with_mom = [r for r in recs if r.get("mom_12_1") is not None] + if len(with_mom) < bt.MIN_CROSS_SECTION: + continue + ordered = sorted(with_mom, key=lambda r: float(r["mom_12_1"])) + k = max(1, len(ordered) // 5) + top = ordered[-k:] + if len(top) < 5: + continue + ic = bt._spearman( + [float(r["val"]) for r in top], + [float(r["fwd"]) for r in top], + ) + if ic is not None: + cond_ics.append(ic) + if cond_ics: + mean_c = sum(cond_ics) / len(cond_ics) + if len(cond_ics) > 1: + std = math.sqrt( + sum((x - mean_c) ** 2 for x in cond_ics) / (len(cond_ics) - 1) + ) + t_c = mean_c / std * math.sqrt(len(cond_ics)) if std > 0 else None + else: + t_c = None + mom_cond = { + "mean_ic": round(mean_c, 4), + "ic_t_stat": round(t_c, 2) if t_c is not None else None, + "weeks": len(cond_ics), + "note": "IC of sue_latest within top mom_12_1 quintile (non-overlapping weeks)", + } + else: + mom_cond = {"mean_ic": None, "weeks": 0} + + def _find(name: str) -> dict | None: + for row in signal_eval: + if row.get("signal") == name: + return row + return None + + sue = _find("sue_latest") + grade = { + "green": False, + "reason": "sue_latest missing", + } + if sue: + mean_ic = sue.get("mean_ic") + t = sue.get("ic_t_stat") + reliable = bool(sue.get("reliable")) + sign_ok = mean_ic is not None and float(mean_ic) > 0 + mag_ok = mean_ic is not None and abs(float(mean_ic)) >= IRON_IC_BAR + grade = { + "green": bool(sign_ok and mag_ok and reliable), + "checks": { + "mean_ic": mean_ic, + "sign_positive": sign_ok, + "abs_ge_0_03": mag_ok, + "reliable": reliable, + "ic_t_stat": t, + "weeks": sue.get("weeks"), + }, + "reason": ( + "iron rule cleared — STOP; book-integration is a separate human step" + if (sign_ok and mag_ok and reliable) + else "iron rule not met" + ), + "row": sue, + } + + def _find_r(name: str) -> dict | None: + for row in restricted_eval: + if row.get("signal") == name: + return row + return None + + # Side-by-side baselines from same evaluation. + side = { + name: _find(name) + for name in ( + "mom_12_1", + "mom_12_1_resid", + "mom_12_1_sector_resid", + "mom_12_1_sector_demeaned", + "sue_latest", + "fip_id", + ) + } + side_restricted = { + name: _find_r(name) + for name in ("mom_12_1", "mom_12_1_resid", "sue_latest") + } + return { + "signal_eval_side_by_side": side, + "signal_eval_identical_sue_subset": side_restricted, + "identical_subset_note": ( + "Mom baselines re-scored only on (week, symbol) cells where SUE exists. " + "Use this table when backfill is incomplete — full-universe mom N is not comparable." + ), + "full_signal_eval": signal_eval, + "sue_grade": grade, + "momentum_conditional_sue": mom_cond, + "sue_coverage": { + "symbols_with_sue": len(sue_map), + "avg_weeks_with_sue": ( + round( + sum(len(v) for v in sue_collected.values()) + / max(1, len(sue_collected)), + 1, + ) + if sue_collected + else 0 + ), + "weeks_with_min_cross_section": len(usable), + }, + } + + +def _write_md(path: Path, payload: dict) -> None: + pre = path.read_text(encoding="utf-8") if path.exists() else "" + marker = "## Results" + idx = pre.find(marker) + header = pre[:idx] if idx >= 0 else pre.split("## Verdict")[0] + + lines = [ + header.rstrip(), + "", + "## Results", + "", + f"Generated: `{payload.get('generated_at')}`", + "", + "### Data provenance", + "", + f"```json\n{json.dumps(payload.get('data_provenance') or {}, indent=2, default=str)}\n```", + "", + "### 2a — Earnings-gap risk (report-only)", + "", + ] + a = payload.get("experiment_2a") + if not a: + lines.append("_Skipped or unavailable._") + else: + lines.append(f"```json\n{json.dumps(a, indent=2, default=str)}\n```") + lines.extend(["", "### 2b — SUE / PEAD IC", ""]) + b = payload.get("experiment_2b") + if not b: + lines.append("_Skipped or unavailable._") + else: + side = b.get("signal_eval_side_by_side") or {} + lines.extend([ + "| signal | mean_ic | ic_t_stat | weeks | avg_N | reliable |", + "|---|---:|---:|---:|---:|---|", + ]) + for name in ( + "mom_12_1", + "mom_12_1_resid", + "sue_latest", + "mom_12_1_sector_resid", + "fip_id", + ): + r = side.get(name) or {} + lines.append( + f"| {name} | {r.get('mean_ic', '')} | {r.get('ic_t_stat', '')} | " + f"{r.get('weeks', '')} | {r.get('avg_cross_section', '')} | " + f"{r.get('reliable', '')} |" + ) + lines.extend([ + "", + f"**SUE grade:** `{json.dumps(b.get('sue_grade') or {}, default=str)}`", + "", + f"**Momentum-conditional SUE:** `{json.dumps(b.get('momentum_conditional_sue') or {}, default=str)}`", + "", + ]) + + lines.extend([ + "", + "## Verdict", + "", + f"**{payload.get('verdict')}**", + "", + payload.get("verdict_detail") or "", + "", + "## What a human must decide next", + "", + payload.get("human_next") or "- Review; no auto-ship.", + "", + f"Artifacts: `{payload.get('report_path')}`", + "", + ]) + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + + +async def _main() -> None: + args = _parse_args() + snapshot = Path(args.snapshot) + if not snapshot.exists(): + raise SystemExit(f"Missing snapshot {snapshot}") + if args.allow_spawn: + os.environ["BACKTEST_ALLOW_SPAWN"] = "1" + + events, meta = _load_earnings(snapshot) + # Race guard lite on earnings completeness. + provenance = { + "snapshot": str(snapshot.resolve()), + "n_earnings_events": len(events), + "backfill_meta": meta, + "announce_range": { + "min": min((e["announce_date"] for e in events), default=None), + "max": max((e["announce_date"] for e in events), default=None), + }, + "with_actual_and_estimate": sum( + 1 + for e in events + if e.get("eps_actual") is not None and e.get("eps_estimate") is not None + ), + } + print( + f"Earnings events: {provenance['n_earnings_events']} " + f"(with act+est={provenance['with_actual_and_estimate']}) meta={meta}" + ) + if meta and meta.get("done", 0) < 0.9 * (meta.get("universe_tickers") or 1): + print( + "WARNING: earnings backfill incomplete " + f"({meta.get('done')}/{meta.get('universe_tickers')}). " + "Results may be biased; resume backfill." + ) + + exp_2a = None + exp_2b = None + if not args.skip_2a: + print("Running 2a earnings-gap diagnostic…") + exp_2a = await _run_2a( + snapshot, events, quiet=args.quiet, workers=args.workers + ) + print( + " 2a losses<-1R with earnings:", + (exp_2a.get("q1_losses_worse_than_minus_1r") or {}), + ) + if not args.skip_2b: + print("Running 2b SUE IC harness…") + exp_2b = await _run_2b_ic( + snapshot, events, quiet=args.quiet, workers=args.workers + ) + g = exp_2b.get("sue_grade") or {} + print(f" 2b SUE green={g.get('green')} {g.get('reason')}") + + # Verdict + if exp_2b and (exp_2b.get("sue_grade") or {}).get("green"): + verdict = "PROMOTE (2b SUE) — STOP for human wire design" + detail = ( + "SUE cleared iron rule. No book integration without human approval. " + "2a remains report-only." + ) + human = ( + "- Design tilt vs second gate if desired.\n" + "- Do not auto-filter from 2a without separate approval + tail review." + ) + else: + sue_ic = None + if exp_2b: + sue_ic = ((exp_2b.get("sue_grade") or {}).get("row") or {}).get("mean_ic") + if sue_ic is not None and abs(float(sue_ic)) >= 0.015: + verdict = "PARK" + detail = f"SUE IC={sue_ic} below iron bar or unreliable; keep data, no wire." + else: + verdict = "DEAD (2b) / REPORT-ONLY (2a)" + detail = ( + "SUE does not clear iron rule on this window. " + "2a distributions for human risk review only — no filter." + ) + human = ( + "- No SUE book change.\n" + "- Read 2a tails before considering any earnings-avoid filter." + ) + + stamp = datetime.now().strftime("%Y%m%d-%H%M%S") + out = Path(args.out) if args.out else Path("reports") / f"earnings-gap-sue-{stamp}.json" + payload = { + "generated_at": datetime.now().isoformat(), + "data_provenance": provenance, + "experiment_2a": exp_2a, + "experiment_2b": exp_2b, + "verdict": verdict, + "verdict_detail": detail, + "human_next": human, + "report_path": str(out.as_posix()), + "fmp_note": ( + "Bulk earnings-calendar is paid (402 on free tier). " + "Backfill used per-symbol /stable/earnings; see earnings-backfill-status.json." + ), + } + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(payload, indent=2, default=str) + "\n", encoding="utf-8") + md = Path("docs/research/earnings-gap-and-sue.md") + _write_md(md, payload) + out.with_suffix(".md").write_text(md.read_text(encoding="utf-8"), encoding="utf-8") + print(f"Verdict: {verdict}") + print(f"Wrote {out}") + + +if __name__ == "__main__": + asyncio.run(_main()) diff --git a/scripts/run_history_depth_research.py b/scripts/run_history_depth_research.py new file mode 100644 index 0000000..02bb229 --- /dev/null +++ b/scripts/run_history_depth_research.py @@ -0,0 +1,476 @@ +"""History-depth extension research (local / MacBook). + +Phases +------ + coverage — bars per calendar year; no rebuild + harness — race-guard snapshot, full signal_eval, era split pre/post-2021 + +Does not retune production knobs. Does not modify scheduler/gates. + +Example +------- + python scripts/run_history_depth_research.py --phase coverage \\ + --snapshot backtest_snapshots/prod.sqlite + + python scripts/run_history_depth_research.py --phase harness \\ + --snapshot backtest_snapshots/research.sqlite --workers 8 --allow-spawn +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import os +import sys +from collections import defaultdict +from datetime import date, datetime +from pathlib import Path +from typing import Any + +from sqlalchemy import create_engine, text +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +ERA_SPLIT = date(2021, 1, 1) +SURVIVORSHIP_BANNER = ( + "SURVIVORSHIP BIAS: today's constituents backfilled historically. " + "Absolute Sharpe/CAGR levels on deep history are optimistic. " + "Use RELATIVE signal IC comparisons and era stability only — not levels." +) + + +def _sqlite_url(path: Path) -> str: + return f"sqlite+aiosqlite:///{path.resolve().as_posix()}" + + +def _parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--phase", choices=("coverage", "harness", "all"), default="all") + p.add_argument("--snapshot", default="backtest_snapshots/research.sqlite") + p.add_argument("--workers", type=int, default=8) + p.add_argument("--allow-spawn", action="store_true") + p.add_argument("--quiet", action="store_true") + p.add_argument("--out", default=None) + return p.parse_args() + + +def _coverage_report(snapshot: Path) -> dict[str, Any]: + engine = create_engine( + f"sqlite:///{snapshot.resolve().as_posix()}", + future=True, + ) + try: + with engine.connect() as conn: + ticker_n = int(conn.execute(text("SELECT COUNT(*) FROM tickers")).scalar_one()) + ohlcv_n = int( + conn.execute(text("SELECT COUNT(*) FROM ohlcv_records")).scalar_one() + ) + d_range = conn.execute( + text("SELECT MIN(date), MAX(date) FROM ohlcv_records") + ).fetchone() + # Bars per calendar year (global). + by_year = conn.execute( + text( + """ + SELECT substr(date, 1, 4) AS y, COUNT(*) AS n, + COUNT(DISTINCT ticker_id) AS tickers + FROM ohlcv_records + GROUP BY substr(date, 1, 4) + ORDER BY y + """ + ) + ).fetchall() + # Per-symbol min/max date + bar count (summary percentiles). + per_sym = conn.execute( + text( + """ + SELECT t.symbol, COUNT(*) AS n, MIN(o.date), MAX(o.date) + FROM ohlcv_records o + JOIN tickers t ON t.id = o.ticker_id + GROUP BY t.symbol + """ + ) + ).fetchall() + finally: + engine.dispose() + + ns = sorted(int(r[1]) for r in per_sym) + def pct(p: float) -> int | None: + if not ns: + return None + i = int(round(p * (len(ns) - 1))) + return ns[i] + + starts = sorted(str(r[2]) for r in per_sym if r[2]) + start_hist: dict[str, int] = defaultdict(int) + for s in starts: + start_hist[s[:4]] += 1 + + return { + "snapshot": str(snapshot.resolve()), + "ticker_count": ticker_n, + "ohlcv_row_count": ohlcv_n, + "date_range": {"min": d_range[0], "max": d_range[1]}, + "bars_per_year": [ + {"year": y, "bars": n, "tickers_with_bars": t} for y, n, t in by_year + ], + "bars_per_symbol": { + "min": ns[0] if ns else None, + "p10": pct(0.10), + "p50": pct(0.50), + "p90": pct(0.90), + "max": ns[-1] if ns else None, + }, + "symbols_by_start_year": dict(sorted(start_hist.items())), + "note": ( + "Where ticker counts drop in early years, the feed (or listing history) " + "thins — do not treat those years as a full 505-name cross-section." + ), + "survivorship_banner": SURVIVORSHIP_BANNER, + } + + +def _assert_complete(snapshot: Path) -> dict[str, Any]: + from scripts.research_snapshot_manifest import ( # type: ignore + assert_research_snapshot_complete, + load_manifest, + ) + + m = load_manifest(snapshot) + if m is None: + # Prod snapshot may lack manifest; still require healthy bar depth. + eng = create_engine( + f"sqlite:///{snapshot.resolve().as_posix()}", + future=True, + ) + try: + with eng.connect() as conn: + avg = conn.execute( + text( + """ + SELECT AVG(c) FROM ( + SELECT COUNT(*) AS c FROM ohlcv_records GROUP BY ticker_id + ) + """ + ) + ).scalar_one() + finally: + eng.dispose() + if avg is None or float(avg) < 400: + raise SystemExit( + f"No completion manifest and avg bars={avg} look short. " + "Rebuild research.sqlite via extend_snapshot_universe.py" + ) + return {"manifest": None, "avg_bars": float(avg), "ok": True} + return {"manifest": assert_research_snapshot_complete(snapshot), "ok": True} + + +async def _harness(snapshot: Path, *, workers: int, quiet: bool) -> dict[str, Any]: + from app.config import settings + from app.services.backtest_service import run_backtest + + os.environ["BACKTEST_SNAPSHOT_OFFLINE"] = "1" + os.environ["BACKTEST_SIGNAL_EVAL_ONLY"] = "1" + if Path("data/research/ticker_sector_map.json").exists(): + os.environ["BACKTEST_SECTOR_MAP_PATH"] = str( + Path("data/research/ticker_sector_map.json").resolve() + ) + settings.backtest_workers = workers + + engine = create_async_engine(_sqlite_url(snapshot), pool_pre_ping=True) + Session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) + + def progress(done: int, total: int, symbol: str) -> None: + if quiet: + return + print(f" progress {done}/{total} {symbol}", end="\r", flush=True) + + try: + async with Session() as db: + report = await run_backtest(db, progress_cb=progress, cadence="weekly") + finally: + await engine.dispose() + if not quiet: + print() + + signal_eval = report.get("signal_eval") or [] + + # Era-split IC: recompute from collected is not available post-run. + # Approximate via second pass is expensive; instead document that era split + # requires collecting weekly ICs. We re-run evaluation if the report embeds + # nothing — for v1, call internal collection is too heavy to duplicate. + # Lightweight approach: mark era_split as requiring BACKTEST with custom + # filter — implemented below by re-scoring from a dedicated collection pass. + era = await _era_split_ics(snapshot, workers=workers, quiet=quiet) + + return { + "survivorship_banner": SURVIVORSHIP_BANNER, + "signal_eval": signal_eval, + "era_split": era, + "params": report.get("params"), + "tickers": report.get("tickers"), + "generated_at_run": report.get("generated_at"), + } + + +async def _era_split_ics( + snapshot: Path, *, workers: int, quiet: bool +) -> dict[str, Any]: + """Collect weekly signal series and evaluate pre/post ERA_SPLIT separately.""" + from app.config import settings + from app.services import backtest_service as bt + from app.services.benchmark_service import load_benchmark_closes + from app.models.ticker import Ticker + from sqlalchemy import select + from collections import defaultdict as dd + + os.environ["BACKTEST_SNAPSHOT_OFFLINE"] = "1" + settings.backtest_workers = max(1, workers) + + engine = create_async_engine(_sqlite_url(snapshot), pool_pre_ping=True) + Session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) + + collected: dict = dd(lambda: dd(list)) + try: + async with Session() as db: + tickers = list( + (await db.execute(select(Ticker).order_by(Ticker.symbol))).scalars() + ) + spy = await load_benchmark_closes(db, "SPY") + symbol_to_sector = {} + sector_etf: dict = {} + try: + from app.services.sector_map import ( + SECTOR_ETFS, + load_ticker_sector_map, + ) + + symbol_to_sector = load_ticker_sector_map() + for etf in SECTOR_ETFS: + series = await load_benchmark_closes(db, etf) + if series: + sector_etf[etf] = series + except Exception: + pass + + for idx, t in enumerate(tickers): + if not quiet and idx % 100 == 0: + print(f" era-collect {idx}/{len(tickers)}", end="\r", flush=True) + cols = await bt._fetch_columns(db, t.symbol) + if cols is None: + continue + records = [ + type( + "R", + (), + { + "date": date.fromordinal(int(cols[0][i])), + "close": cols[4][i], + "high": cols[2][i], + "volume": cols[5][i] if len(cols) > 5 else 0, + }, + )() + for i in range(len(cols[0])) + ] + series = bt._signal_series( + records, + spy, + symbol=t.symbol, + sector_etf_closes=bt._sector_etf_closes_for_symbol( + t.symbol, symbol_to_sector, sector_etf + ), + ) + for name, weeks in series.items(): + for wk, pairs in weeks.items(): + collected[name][wk].extend(pairs) + if symbol_to_sector: + bt._inject_sector_demeaned_momentum(collected, symbol_to_sector) + finally: + await engine.dispose() + if not quiet: + print() + + def _filter_era(coll: dict, *, pre: bool) -> dict: + out: dict = dd(lambda: dd(list)) + for name, weeks in coll.items(): + for wk, recs in weeks.items(): + # ISO week key (year, week) — approximate era by ISO year. + year = int(wk[0]) if isinstance(wk, tuple) else int(str(wk)[:4]) + if pre and year >= ERA_SPLIT.year: + continue + if not pre and year < ERA_SPLIT.year: + continue + out[name][wk].extend(recs) + return out + + pre_eval = bt._signal_evaluation(_filter_era(collected, pre=True)) + post_eval = bt._signal_evaluation(_filter_era(collected, pre=False)) + full_eval = bt._signal_evaluation(collected) + + def _index(rows: list[dict]) -> dict[str, dict]: + return {r["signal"]: r for r in rows} + + return { + "era_split_date": ERA_SPLIT.isoformat(), + "note": "Diagnostic only — not a tuning input. Nested lookbacks are not OOS.", + "full": _index(full_eval), + "pre_2021": _index(pre_eval), + "post_2021": _index(post_eval), + } + + +def _write_md(path: Path, payload: dict) -> None: + pre = path.read_text(encoding="utf-8") if path.exists() else "" + marker = "## Results" + idx = pre.find(marker) + header = pre[:idx] if idx >= 0 else pre.split("## Verdict")[0] + + lines = [ + header.rstrip(), + "", + "## Results", + "", + f"Generated: `{payload.get('generated_at')}`", + "", + f"> **{SURVIVORSHIP_BANNER}**", + "", + "### Coverage", + "", + f"```json\n{json.dumps(payload.get('coverage') or {}, indent=2, default=str)}\n```", + "", + "### Race guard", + "", + f"```json\n{json.dumps(payload.get('race_guard') or {}, indent=2, default=str)}\n```", + "", + "### Signal IC (full extended window)", + "", + ] + harness = payload.get("harness") or {} + rows = harness.get("signal_eval") or [] + if rows: + lines.extend([ + "| signal | mean_ic | ic_t_stat | weeks | avg_N | reliable |", + "|---|---:|---:|---:|---:|---|", + ]) + for r in rows: + lines.append( + f"| {r.get('signal')} | {r.get('mean_ic')} | {r.get('ic_t_stat')} | " + f"{r.get('weeks')} | {r.get('avg_cross_section')} | {r.get('reliable')} |" + ) + else: + lines.append("_Harness not run this pass._") + + era = (harness.get("era_split") or {}) + lines.extend(["", "### Era split (diagnostic only)", ""]) + if era: + for label in ("full", "pre_2021", "post_2021"): + block = era.get(label) or {} + lines.append(f"#### {label}") + lines.append("") + lines.append("| signal | mean_ic | t | weeks | N |") + lines.append("|---|---:|---:|---:|---:|") + for name in sorted(block): + r = block[name] + lines.append( + f"| {name} | {r.get('mean_ic')} | {r.get('ic_t_stat')} | " + f"{r.get('weeks')} | {r.get('avg_cross_section')} |" + ) + lines.append("") + else: + lines.append("_No era split._") + + lines.extend([ + "", + "## Verdict", + "", + f"**{payload.get('verdict')}**", + "", + payload.get("verdict_detail") or "", + "", + "## What a human must decide next", + "", + payload.get("human_next") + or "- Do not retune production knobs from this report without review.", + "", + f"Artifacts: `{payload.get('report_path')}`", + "", + ]) + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + + +async def _main() -> None: + args = _parse_args() + snapshot = Path(args.snapshot) + if not snapshot.exists(): + raise SystemExit(f"Missing snapshot: {snapshot}") + if args.allow_spawn: + os.environ["BACKTEST_ALLOW_SPAWN"] = "1" + + coverage = None + race = None + harness = None + + if args.phase in ("coverage", "all"): + print("Coverage probe…") + coverage = _coverage_report(snapshot) + print( + f" tickers={coverage['ticker_count']} ohlcv={coverage['ohlcv_row_count']} " + f"range={coverage['date_range']}" + ) + for row in coverage["bars_per_year"]: + print( + f" year {row['year']}: bars={row['bars']} " + f"tickers={row['tickers_with_bars']}" + ) + + if args.phase in ("harness", "all"): + print("Race guard…") + race = _assert_complete(snapshot) + print(f" ok={race.get('ok')}") + print("Full harness + era split (LONG)…") + print(f" {SURVIVORSHIP_BANNER}") + harness = await _harness( + snapshot, workers=args.workers, quiet=args.quiet + ) + + stamp = datetime.now().strftime("%Y%m%d-%H%M%S") + out = ( + Path(args.out) + if args.out + else Path("reports") / f"history-depth-{stamp}.json" + ) + payload = { + "generated_at": datetime.now().isoformat(), + "survivorship_banner": SURVIVORSHIP_BANNER, + "coverage": coverage, + "race_guard": race, + "harness": harness, + "verdict": "PENDING_HUMAN" if harness else "COVERAGE_ONLY", + "verdict_detail": ( + "Harness complete — human interprets relative IC / era stability. " + "No production retune from this artifact." + if harness + else "Coverage probe only; run --phase harness after deep rebuild." + ), + "human_next": ( + "- Compare sector residual vs market residual across eras.\n" + "- If pre-2021 IC collapses, park Task 1 wire-in.\n" + "- Do not retune production knobs on deep history levels." + ), + "report_path": str(out.as_posix()), + } + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(payload, indent=2, default=str) + "\n", encoding="utf-8") + md = Path("docs/research/history-depth-extension.md") + _write_md(md, payload) + out.with_suffix(".md").write_text(md.read_text(encoding="utf-8"), encoding="utf-8") + print(f"Wrote {out}") + print(f"Wrote {md}") + + +if __name__ == "__main__": + asyncio.run(_main()) diff --git a/scripts/run_sector_residual_research.py b/scripts/run_sector_residual_research.py new file mode 100644 index 0000000..290987d --- /dev/null +++ b/scripts/run_sector_residual_research.py @@ -0,0 +1,1014 @@ +"""Sector-residual momentum research runner (local only). + +Protocol +-------- +1. Race-guard the research/prod snapshot (completion manifest when present). +2. Require sector map + sector ETFs in ``benchmark_prices``. +3. Run signal IC harness on the production ~505-name snapshot + (``BACKTEST_SIGNAL_EVAL_ONLY=1``) with sector context loaded. +4. Grade candidates vs pre-registered iron rule + t-stat vs ``mom_12_1_resid``. +5. If a candidate promotes: portfolio A/B with candidate as momentum leg + + gate percentile (``fill_mode=close``). Optional sector-cap arm. + +Does not modify production DB, gate, scanner, or schedule. + +Example +------- + python scripts/run_sector_residual_research.py \\ + --snapshot backtest_snapshots/prod.sqlite \\ + --workers 6 --allow-spawn +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import math +import os +import sys +from collections import defaultdict +from copy import deepcopy +from datetime import date, datetime +from pathlib import Path +from typing import Any + +from sqlalchemy import create_engine, text +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from app.services.sector_map import ( # noqa: E402 + DEFAULT_SECTOR_MAP_PATH, + SECTOR_ETFS, + coverage_stats, + load_ticker_sector_map, + normalise_symbol, + sector_to_etf, +) + +VALIDATION_SPLIT = date(2024, 7, 1) +IRON_IC_BAR = 0.03 +MIN_RELIABLE = 12 + + +def _sqlite_url(path: Path) -> str: + return f"sqlite+aiosqlite:///{path.resolve().as_posix()}" + + +def _parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--snapshot", default="backtest_snapshots/prod.sqlite") + p.add_argument( + "--sector-map", + default=str(DEFAULT_SECTOR_MAP_PATH), + ) + p.add_argument("--workers", type=int, default=6) + p.add_argument("--allow-spawn", action="store_true") + p.add_argument( + "--skip-ab", + action="store_true", + help="IC only — never run portfolio A/B even if promotion fires.", + ) + p.add_argument( + "--force-ab", + action="store_true", + help="Run A/B for diagnostic even if IC bar fails (still reported as non-promote).", + ) + p.add_argument( + "--sector-cap", + type=int, + default=None, + help="Optional max positions per sector (e.g. 3). Only used in A/B.", + ) + p.add_argument("--quiet", action="store_true") + p.add_argument( + "--out", + default=None, + help="JSON report path (default reports/sector-residual-YYYYMMDD-HHMMSS.json)", + ) + return p.parse_args() + + +def _assert_snapshot_ready(snapshot: Path) -> dict[str, Any]: + """Race guard: prefer completion manifest; always check live bar sanity.""" + from scripts.research_snapshot_manifest import ( # type: ignore + assert_research_snapshot_complete, + load_manifest, + ) + + guard: dict[str, Any] = {"snapshot": str(snapshot.resolve())} + manifest = load_manifest(snapshot) + if manifest is not None: + # Full assert when a manifest exists (research.sqlite path). + try: + m = assert_research_snapshot_complete(snapshot) + guard["manifest"] = m + guard["manifest_ok"] = True + except SystemExit as exc: + raise SystemExit(str(exc)) from exc + else: + guard["manifest"] = None + guard["manifest_ok"] = None + guard["note"] = ( + "No completion manifest (prod.sqlite is expected without one). " + "Bar-count sanity still applied." + ) + + engine = create_engine( + f"sqlite:///{snapshot.resolve().as_posix()}", + future=True, + ) + try: + with engine.connect() as conn: + ticker_n = int(conn.execute(text("SELECT COUNT(*) FROM tickers")).scalar_one()) + ohlcv_n = int( + conn.execute(text("SELECT COUNT(*) FROM ohlcv_records")).scalar_one() + ) + bar_stats = conn.execute( + text( + """ + SELECT MIN(c), AVG(c), MAX(c) FROM ( + SELECT COUNT(*) AS c FROM ohlcv_records GROUP BY ticker_id + ) + """ + ) + ).fetchone() + bench = conn.execute( + text( + "SELECT symbol, COUNT(*), MIN(date), MAX(date) " + "FROM benchmark_prices GROUP BY symbol ORDER BY symbol" + ) + ).fetchall() + d_range = conn.execute( + text("SELECT MIN(date), MAX(date) FROM ohlcv_records") + ).fetchone() + finally: + engine.dispose() + + guard["ticker_count"] = ticker_n + guard["ohlcv_row_count"] = ohlcv_n + guard["bars_min_avg_max"] = { + "min": bar_stats[0], + "avg": round(float(bar_stats[1]), 1) if bar_stats[1] is not None else None, + "max": bar_stats[2], + } + guard["ohlcv_date_range"] = {"min": d_range[0], "max": d_range[1]} + guard["benchmark_prices"] = [ + {"symbol": s, "n": n, "min": d0, "max": d1} for s, n, d0, d1 in bench + ] + + # Sanity: a half-built snapshot would show many tickers with tiny bar counts. + min_bars = int(bar_stats[0] or 0) + avg_bars = float(bar_stats[1] or 0) + if ticker_n < 400: + raise SystemExit( + f"Snapshot looks short: only {ticker_n} tickers (expected ~505 prod)." + ) + if avg_bars < 200: + raise SystemExit( + f"Snapshot bar counts look short (avg={avg_bars:.0f}). Rebuild before research." + ) + # Allow a few thin names; refuse if median path is collapsed. + if min_bars < 10 and avg_bars < 500: + raise SystemExit( + f"Snapshot min bars={min_bars}, avg={avg_bars:.0f} — possible partial build." + ) + + present_etfs = {row[0] for row in bench} + missing_etfs = [e for e in SECTOR_ETFS if e not in present_etfs] + guard["missing_sector_etfs"] = missing_etfs + if missing_etfs: + raise SystemExit( + "Sector ETFs missing from benchmark_prices: " + f"{missing_etfs}. Run scripts/fetch_sector_etfs_to_snapshot.py first." + ) + if "SPY" not in present_etfs: + raise SystemExit("SPY missing from benchmark_prices") + + return guard + + +def _find_signal(rows: list[dict], name: str) -> dict | None: + for row in rows or []: + if row.get("signal") == name: + return row + return None + + +def _grade_ic( + candidate: dict | None, + resid: dict | None, + *, + expected_sign: float = 1.0, +) -> dict[str, Any]: + """Iron rule + t-stat ≥ mom_12_1_resid.""" + if candidate is None: + return { + "promote_to_ab": False, + "reason": "signal missing from signal_eval", + } + mean_ic = candidate.get("mean_ic") + t_stat = candidate.get("ic_t_stat") + reliable = bool(candidate.get("reliable")) + weeks = int(candidate.get("weeks") or 0) + if mean_ic is None or t_stat is None: + return {"promote_to_ab": False, "reason": "missing mean_ic or t", "row": candidate} + + sign_ok = (float(mean_ic) * expected_sign) > 0 + mag_ok = abs(float(mean_ic)) >= IRON_IC_BAR + reliable_ok = reliable and weeks >= MIN_RELIABLE + resid_t = resid.get("ic_t_stat") if resid else None + t_ok = resid_t is not None and float(t_stat) >= float(resid_t) + + promote = sign_ok and mag_ok and reliable_ok and t_ok + return { + "promote_to_ab": promote, + "checks": { + "sign_ok": sign_ok, + "abs_mean_ic_ge_0_03": mag_ok, + "reliable": reliable_ok, + "t_ge_resid": t_ok, + "mean_ic": mean_ic, + "ic_t_stat": t_stat, + "resid_ic_t_stat": resid_t, + "weeks": weeks, + }, + "reason": ( + "clears iron rule and t ≥ mom_12_1_resid — authorized for A/B only" + if promote + else "does not clear pre-registered IC promotion bar" + ), + "row": candidate, + } + + +async def _run_signal_eval( + snapshot: Path, + *, + workers: int, + quiet: bool, + sector_map_path: Path, +) -> dict: + from app.config import settings + from app.services.backtest_service import run_backtest + + os.environ["BACKTEST_SNAPSHOT_OFFLINE"] = "1" + os.environ["BACKTEST_SIGNAL_EVAL_ONLY"] = "1" + os.environ["BACKTEST_SECTOR_MAP_PATH"] = str(sector_map_path.resolve()) + # Clear liquid-breadth — this is the 505-name prod IC, not breadth. + os.environ.pop("BACKTEST_LIQUID_BREADTH", None) + settings.backtest_workers = workers + + engine = create_async_engine(_sqlite_url(snapshot), pool_pre_ping=True) + Session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) + + def progress(done: int, total: int, symbol: str) -> None: + if quiet: + return + print(f" progress {done}/{total} {symbol}", end="\r", flush=True) + + try: + async with Session() as db: + report = await run_backtest(db, progress_cb=progress, cadence="weekly") + finally: + await engine.dispose() + if not quiet: + print() + return report + + +def _period_percentiles(rows: list[dict], value_key: str) -> dict[tuple, dict[str, float]]: + by_period: dict[tuple, list[dict]] = defaultdict(list) + for row in rows: + if row.get(value_key) is None: + continue + period = row.get("ranking_period") or row.get("iso_week") + by_period[period].append(row) + out: dict[tuple, dict[str, float]] = {} + for period, group in by_period.items(): + ordered = sorted(group, key=lambda r: float(r[value_key])) + n = len(ordered) + for rank, row in enumerate(ordered): + key = (str(row["symbol"]), str(row["date"])) + pct = (rank / (n - 1) * 100.0) if n > 1 else 100.0 + out.setdefault(key, {})[value_key] = float(row[value_key]) + out[key][f"{value_key}_percentile"] = pct + return out + + +async def _load_prices_and_benchmarks(snapshot: Path) -> tuple[dict, dict, dict]: + """Return (price_columns, spy_closes, sector_etf_closes).""" + from app.services import backtest_service as bt + from app.services.benchmark_service import load_benchmark_closes + from app.models.ticker import Ticker + from sqlalchemy import select + + engine = create_async_engine(_sqlite_url(snapshot), pool_pre_ping=True) + Session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) + prices: dict[str, tuple] = {} + try: + async with Session() as db: + tickers = list( + (await db.execute(select(Ticker).order_by(Ticker.symbol))).scalars() + ) + for t in tickers: + cols = await bt._fetch_columns(db, t.symbol) + if cols is not None: + prices[t.symbol] = cols + spy = await load_benchmark_closes(db, "SPY") + sector: dict[str, dict] = {} + for etf in SECTOR_ETFS: + series = await load_benchmark_closes(db, etf) + if series: + sector[etf] = series + finally: + await engine.dispose() + return prices, spy, sector + + +def _recompute_sector_residual_on_candidates( + candidates: list[dict], + prices: dict[str, tuple], + spy_closes: dict, + sector_etf_closes: dict[str, dict], + symbol_to_sector: dict[str, str], + *, + momentum_field: str, +) -> list[dict]: + """Attach alternative residual momentum on each candidate as-of date.""" + from app.services import backtest_service as bt + from app.services.sector_map import etf_for_symbol + + # Index price series once. + series_cache: dict[str, tuple[list, list, list]] = {} + for sym, cols in prices.items(): + ords, _o, _h, _l, closes, _v = cols + dates = [date.fromordinal(int(o)) for o in ords] + series_cache[sym] = (dates, list(closes), list(ords)) + + out: list[dict] = [] + for cand in candidates: + c = dict(cand) + sym = str(c["symbol"]) + if sym not in series_cache: + out.append(c) + continue + dates, closes, ords = series_cache[sym] + asof = date.fromisoformat(str(c["date"])) + # Find as-of index. + try: + i = next(idx for idx, d in enumerate(dates) if d == asof) + except StopIteration: + # nearest on/before + i = max((idx for idx, d in enumerate(dates) if d <= asof), default=-1) + if i < 0: + out.append(c) + continue + + if momentum_field == "mom_12_1_sector_resid": + etf = etf_for_symbol(sym, symbol_to_sector) + etf_series = sector_etf_closes.get(etf or "") + val = None + if spy_closes and etf_series: + val = bt._multi_factor_residual_momentum_12_1( + dates, closes, i, [spy_closes, etf_series] + ) + c["residual_momentum"] = val + c["_alt_momentum_signal"] = momentum_field + c["_alt_momentum_value"] = val + elif momentum_field == "mom_12_1_sector_demeaned": + # Placeholder: demean requires cross-section; filled in a second pass. + raw = None + if i >= 252 and closes[i - 252] > 0: + raw = closes[i - 21] / closes[i - 252] - 1.0 + c["_raw_mom_12_1"] = raw + c["_alt_momentum_signal"] = momentum_field + else: + raise ValueError(momentum_field) + out.append(c) + + if momentum_field == "mom_12_1_sector_demeaned": + # Cross-sectional demean within ranking period × sector. + by_period: dict[Any, list[dict]] = defaultdict(list) + for c in out: + if c.get("_raw_mom_12_1") is None: + continue + period = c.get("ranking_period") or c.get("iso_week") + by_period[period].append(c) + for period, group in by_period.items(): + by_sec: dict[str, list[float]] = defaultdict(list) + for c in group: + sec = symbol_to_sector.get(normalise_symbol(str(c["symbol"]))) + if sec: + by_sec[sec].append(float(c["_raw_mom_12_1"])) + means = { + s: sum(vs) / len(vs) for s, vs in by_sec.items() if len(vs) >= 2 + } + for c in group: + sec = symbol_to_sector.get(normalise_symbol(str(c["symbol"]))) + raw = float(c["_raw_mom_12_1"]) + if sec in means: + val = raw - means[sec] + c["residual_momentum"] = val + c["_alt_momentum_value"] = val + else: + c["residual_momentum"] = None + c["_alt_momentum_value"] = None + + return out + + +def _assign_prod_ranks(candidates: list[dict]) -> None: + from app.services import backtest_service as bt + + bt._assign_momentum_percentiles(candidates) + bt._assign_residual_momentum_percentiles(candidates) + bt._assign_low_volatility_percentiles(candidates) + bt._assign_activation_momentum_percentiles(candidates) + bt._assign_residual_high_vol_blend(candidates) + for c in candidates: + c["qualified"] = bt._momentum_qualifies(c, 80.0) + + +async def _run_ab( + snapshot: Path, + *, + sector_map: dict[str, str], + signal_name: str, + sector_cap: int | None, + quiet: bool, + workers: int, +) -> dict[str, Any]: + """Control vs treatment book with candidate as momentum residual.""" + from app.services import backtest_service as bt + from app.config import settings + from app.models.ticker import Ticker + from app.services.admin_service import get_activation_config + from app.services.recommendation_service import get_recommendation_config + from app.services.paper_trade_service import get_exit_policy + from app.services.benchmark_service import load_benchmark_closes + from sqlalchemy import select + + os.environ["BACKTEST_SNAPSHOT_OFFLINE"] = "1" + os.environ.pop("BACKTEST_SIGNAL_EVAL_ONLY", None) + settings.backtest_workers = max(1, int(workers)) + + engine = create_async_engine(_sqlite_url(snapshot), pool_pre_ping=True) + Session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) + try: + async with Session() as db: + config = await get_recommendation_config(db) + activation = await get_activation_config(db) + exit_config = await get_exit_policy(db) + tickers = list( + (await db.execute(select(Ticker).order_by(Ticker.symbol))).scalars() + ) + spy = await load_benchmark_closes(db, "SPY") + sector_etf: dict[str, dict] = {} + for etf in SECTOR_ETFS: + series = await load_benchmark_closes(db, etf) + if series: + sector_etf[etf] = series + + prices: dict[str, tuple] = {} + candidates: list[dict] = [] + for idx, t in enumerate(tickers): + if not quiet and idx % 25 == 0: + print(f" fetch {idx}/{len(tickers)}", end="\r", flush=True) + cols = await bt._fetch_columns(db, t.symbol) + if cols is None: + continue + prices[t.symbol] = cols + cands, _series = bt._replay_and_signals( + t.symbol, + cols, + config, + activation, + spy, + bt.PRODUCTION_GTL_TARGET_MODEL, + "weekly", + False, + sector_etf, + sector_map, + ) + candidates.extend(cands) + finally: + await engine.dispose() + if not quiet: + print() + + # Control ranks (production residual). + control = [dict(c) for c in candidates] + _assign_prod_ranks(control) + control_longs = [ + c for c in control if c.get("qualified") and c.get("direction") == "long" + ] + + # Treatment: replace residual with sector signal, re-rank. + treatment = _recompute_sector_residual_on_candidates( + candidates, + prices, + spy, + sector_etf, + sector_map, + momentum_field=signal_name, + ) + _assign_prod_ranks(treatment) + treatment_longs = [ + c for c in treatment if c.get("qualified") and c.get("direction") == "long" + ] + + strategy = next(s for s in bt.PORTFOLIO_MONITOR_STRATEGIES if s.get("is_production")) + entry_config = bt._entry_variant_config(str(strategy["entry_variant"])) + assert entry_config is not None + ranking_key = str( + entry_config.get("ranking_key") or entry_config["percentile_key"] + ) + # Production ranking key is residual_high_vol_blend_80_20. + if ranking_key not in (bt.RESIDUAL_HIGH_VOL_BLEND_80_20_KEY, bt.PRODUCTION_PERCENTILE_KEY): + ranking_key = bt.RESIDUAL_HIGH_VOL_BLEND_80_20_KEY + + exit_policy = bt.LIVE_EXIT_MODE_TO_SIM.get( + str(exit_config.get("mode", "atr_trailing")), "atr_trail3" + ) + hold_days = int(exit_config.get("hold_days", 30)) + trail = float(exit_config.get("atr_multiplier", bt.ATR_TRAIL_MULTIPLIER)) + risk = float(entry_config["risk_per_trade"]) + max_pos = int(entry_config["max_positions"]) + + def _sim_book(longs: list[dict], *, label: str, cap: int | None = None) -> dict: + reentry = bt._make_gate_reset_reentry_fn( + longs, prices, cadence="weekly", ranking_key=ranking_key + ) + windows = {} + for wname, start, end in ( + ("train", None, VALIDATION_SPLIT), + ("validation", VALIDATION_SPLIT, None), + ("full", None, None), + ): + sim = bt._simulate_portfolio( + longs, + prices, + spy, + exit_policy, + hold_days, + ranking_key=ranking_key, + max_positions=max_pos, + risk_per_trade=risk, + atr_trail_multiplier=trail, + post_stop_reentry_fn=reentry, + start_date=start, + end_date=end, + fill_mode=bt.FILL_MODE_CLOSE, + include_trades=True, + ) + if sim is None: + windows[wname] = {"error": "no_trades"} + continue + # Optional sector cap: filter trade_details is post-hoc; real cap needs + # simulator support. For research, re-sim with a wrapper ranking that + # drops overflow sector names is approximate — we implement a simple + # pre-filter on daily entry sets via max_positions only when cap is None. + # When cap is set, apply a post-sim diagnostic on entries. + payload = { + k: sim.get(k) + for k in ( + "sharpe", + "sharpe_se", + "cagr_pct", + "max_drawdown_pct", + "total_return_pct", + "trades", + "win_rate_pct", + "avg_r", + "n_returns", + "return_skew", + "return_kurtosis", + "psr", + ) + } + details = sim.get("trade_details") or [] + rs = [ + float(t["realized_r"]) + for t in details + if t.get("realized_r") is not None + ] + if rs: + rs_sorted = sorted(rs) + payload["r_p05"] = rs_sorted[max(0, int(0.05 * (len(rs_sorted) - 1)))] + payload["r_p50"] = rs_sorted[len(rs_sorted) // 2] + payload["r_p95"] = rs_sorted[min(len(rs_sorted) - 1, int(0.95 * (len(rs_sorted) - 1)))] + payload["entry_count"] = len(rs) + if cap is not None and details: + # Diagnostic: count how often a calendar day would exceed cap. + from collections import Counter + + # Use entry dates; sector from map. + day_sector: dict[str, Counter] = defaultdict(Counter) + for t in details: + sec = sector_map.get(normalise_symbol(str(t.get("symbol", "")))) or "?" + day_sector[str(t.get("entry_date") or t.get("date") or "")][sec] += 1 + breaches = sum( + 1 + for day, ctr in day_sector.items() + if any(v > cap for v in ctr.values()) + ) + payload["sector_cap"] = cap + payload["entry_days_with_sector_over_cap"] = breaches + windows[wname] = payload + return {"label": label, "n_qualified_longs": len(longs), "windows": windows} + + control_result = _sim_book(control_longs, label="control_mom_12_1_resid") + treatment_result = _sim_book( + treatment_longs, label=f"treatment_{signal_name}", cap=None + ) + out: dict[str, Any] = { + "signal": signal_name, + "ranking_key": ranking_key, + "fill_mode": "close", + "validation_split": VALIDATION_SPLIT.isoformat(), + "control": control_result, + "treatment": treatment_result, + "promotion": _grade_ab(control_result, treatment_result), + } + if sector_cap is not None: + # Approximate sector-cap book: when selecting, prefer higher rank but + # refuse a 4th name in the same sector among concurrent opens. + # Implemented by tagging candidates and using a custom sim is heavy; + # instead report diagnostic on unconstrained treatment + a filtered + # re-rank that zeros residual for overflow names within each period. + capped = _apply_sector_cap_to_ranks( + treatment, sector_map, cap=sector_cap, ranking_key=ranking_key + ) + capped_longs = [ + c for c in capped if c.get("qualified") and c.get("direction") == "long" + ] + out["sector_cap_arm"] = _sim_book( + capped_longs, label=f"treatment_{signal_name}_cap{sector_cap}", cap=sector_cap + ) + out["sector_cap_promotion"] = _grade_ab( + control_result, out["sector_cap_arm"] + ) + return out + + +def _apply_sector_cap_to_ranks( + candidates: list[dict], + sector_map: dict[str, str], + *, + cap: int, + ranking_key: str, +) -> list[dict]: + """Within each ranking period, keep top `cap` per sector by ranking_key.""" + by_period: dict[Any, list[dict]] = defaultdict(list) + for c in candidates: + period = c.get("ranking_period") or c.get("iso_week") + by_period[period].append(dict(c)) + out: list[dict] = [] + for period, group in by_period.items(): + ordered = sorted( + group, + key=lambda r: float(r.get(ranking_key) or r.get("residual_momentum") or -1e9), + reverse=True, + ) + sector_counts: dict[str, int] = defaultdict(int) + for c in ordered: + sec = sector_map.get(normalise_symbol(str(c["symbol"]))) or "_unknown" + if sector_counts[sec] >= cap: + # Push below gate by nulling activation percentile. + c["qualified"] = False + c["_sector_cap_blocked"] = True + else: + if c.get("qualified"): + sector_counts[sec] += 1 + out.append(c) + return out + + +def _grade_ab(control: dict, treatment: dict) -> dict[str, Any]: + """Pre-registered: val Sharpe ≥ control − 0.5·SE; full Sharpe & maxDD not worse.""" + def win(arm: dict, name: str) -> dict: + return (arm.get("windows") or {}).get(name) or {} + + c_val = win(control, "validation") + t_val = win(treatment, "validation") + c_full = win(control, "full") + t_full = win(treatment, "full") + + def _f(d: dict, k: str) -> float | None: + v = d.get(k) + return None if v is None else float(v) + + c_sh = _f(c_val, "sharpe") + t_sh = _f(t_val, "sharpe") + # Use treatment SE if present else control SE. + se = _f(t_val, "sharpe_se") + if se is None: + se = _f(c_val, "sharpe_se") + if se is None: + se = 0.0 + + val_ok = ( + c_sh is not None + and t_sh is not None + and t_sh >= (c_sh - 0.5 * se) + ) + c_full_sh = _f(c_full, "sharpe") + t_full_sh = _f(t_full, "sharpe") + full_sh_ok = ( + c_full_sh is not None + and t_full_sh is not None + and t_full_sh >= c_full_sh + ) + # max DD: higher absolute drawdown is worse; stored as positive pct typically. + c_dd = _f(c_full, "max_drawdown_pct") + t_dd = _f(t_full, "max_drawdown_pct") + full_dd_ok = ( + c_dd is not None and t_dd is not None and abs(t_dd) <= abs(c_dd) + 1e-9 + ) + promote = bool(val_ok and full_sh_ok and full_dd_ok) + return { + "promote": promote, + "checks": { + "validation_sharpe_ge_control_minus_half_se": val_ok, + "full_sharpe_not_worse": full_sh_ok, + "full_maxdd_not_worse": full_dd_ok, + "control_validation_sharpe": c_sh, + "treatment_validation_sharpe": t_sh, + "se_used": se, + "control_full_sharpe": c_full_sh, + "treatment_full_sharpe": t_full_sh, + "control_full_maxdd": c_dd, + "treatment_full_maxdd": t_dd, + }, + "reason": ( + "clears pre-registered A/B bar — human decides wire-in" + if promote + else "fails pre-registered A/B bar" + ), + } + + +def _write_md(path: Path, payload: dict) -> None: + """Refresh the results sections of the research doc (preserve pre-reg header).""" + # Always write a standalone results companion + update the main doc's + # results block by rewriting the full file with pre-reg + results. + pre = Path("docs/research/sector-residual-momentum.md") + # Keep pre-registration by reading until '## Results' if present. + header = "" + if pre.exists(): + text = pre.read_text(encoding="utf-8") + marker = "## Results" + idx = text.find(marker) + header = text[:idx] if idx >= 0 else text.split("## Verdict")[0] + + guard = payload.get("snapshot_guard") or {} + cov = payload.get("sector_coverage") or {} + ic_rows = payload.get("signal_eval") or [] + grades = payload.get("ic_grades") or {} + lines = [ + header.rstrip(), + "", + "## Results", + "", + f"Generated: `{payload.get('generated_at')}`", + "", + "### Snapshot race guard", + "", + f"- Snapshot: `{guard.get('snapshot')}`", + f"- Tickers: **{guard.get('ticker_count')}** OHLCV rows: **{guard.get('ohlcv_row_count')}**", + f"- Bars min/avg/max: `{guard.get('bars_min_avg_max')}`", + f"- OHLCV range: `{guard.get('ohlcv_date_range')}`", + f"- Manifest ok: `{guard.get('manifest_ok')}`", + f"- Missing sector ETFs at start: `{guard.get('missing_sector_etfs')}`", + "", + "### Sector label coverage", + "", + f"```json\n{json.dumps(cov, indent=2, default=str)}\n```", + "", + "### IC harness (identical cross-sections)", + "", + "| signal | mean_ic | ic_t_stat | weeks | avg_N | reliable | ic+_pct |", + "|---|---:|---:|---:|---:|---|---:|", + ] + want = [ + "mom_12_1", + "mom_12_1_resid", + "mom_12_1_sector_resid", + "mom_12_1_sector_demeaned", + ] + by_name = {r.get("signal"): r for r in ic_rows} + for name in want: + r = by_name.get(name) or {} + lines.append( + f"| {name} | {r.get('mean_ic', '')} | {r.get('ic_t_stat', '')} | " + f"{r.get('weeks', '')} | {r.get('avg_cross_section', '')} | " + f"{r.get('reliable', '')} | {r.get('ic_positive_pct', '')} |" + ) + lines.extend(["", "### IC promotion grades", ""]) + for name, g in grades.items(): + lines.append(f"- **{name}**: promote_to_ab=`{g.get('promote_to_ab')}` — {g.get('reason')}") + lines.append(f" - checks: `{json.dumps(g.get('checks') or {}, default=str)}`") + + ab = payload.get("portfolio_ab") + lines.extend(["", "### Portfolio A/B", ""]) + if not ab: + lines.append("_Not run (IC bar not cleared, or --skip-ab)._") + else: + lines.append(f"```json\n{json.dumps(ab, indent=2, default=str)}\n```") + + lines.extend([ + "", + "## Verdict", + "", + f"**{payload.get('verdict')}**", + "", + payload.get("verdict_detail") or "", + "", + "## What a human must decide next", + "", + payload.get("human_next") or "- Review numbers; do not merge into strategy docs without approval.", + "", + "## Artifacts", + "", + f"- JSON: `{payload.get('report_path')}`", + "", + ]) + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + + +async def _main() -> None: + args = _parse_args() + snapshot = Path(args.snapshot) + sector_map_path = Path(args.sector_map) + if not snapshot.exists(): + raise SystemExit(f"Snapshot missing: {snapshot}") + if not sector_map_path.exists(): + raise SystemExit( + f"Sector map missing: {sector_map_path}. " + "Run scripts/build_ticker_sector_map.py first." + ) + + if args.allow_spawn: + os.environ["BACKTEST_ALLOW_SPAWN"] = "1" + + print("Race-guarding snapshot…") + guard = _assert_snapshot_ready(snapshot) + print( + f" tickers={guard['ticker_count']} ohlcv={guard['ohlcv_row_count']} " + f"bars={guard['bars_min_avg_max']}" + ) + + mapping = load_ticker_sector_map(sector_map_path) + engine = create_engine( + f"sqlite:///{snapshot.resolve().as_posix()}", + future=True, + ) + try: + with engine.connect() as conn: + symbols = [ + normalise_symbol(r[0]) + for r in conn.execute(text("SELECT symbol FROM tickers")).fetchall() + ] + finally: + engine.dispose() + cov = coverage_stats(symbols, mapping) + print( + f"Sector map: {cov['mapped']}/{cov['universe']} " + f"({cov['mapped_pct']}%) with_etf={cov['with_etf']}" + ) + if cov["mapped_pct"] < 90: + print( + f"WARNING: sector coverage {cov['mapped_pct']}% < 90%; " + f"missing e.g. {cov['missing'][:20]}" + ) + + print("Running IC harness (signal-eval only)…") + report = await _run_signal_eval( + snapshot, + workers=args.workers, + quiet=args.quiet, + sector_map_path=sector_map_path, + ) + signal_eval = report.get("signal_eval") or report.get("signals") or [] + # Locate key in report — backtest uses "signal_edge" historically. + if not signal_eval: + for key in ("signal_edge", "signal_evaluation", "factor_ic"): + if key in report and isinstance(report[key], list): + signal_eval = report[key] + break + + resid = _find_signal(signal_eval, "mom_12_1_resid") + grades = { + name: _grade_ic(_find_signal(signal_eval, name), resid) + for name in ("mom_12_1_sector_resid", "mom_12_1_sector_demeaned") + } + for name, g in grades.items(): + print( + f" {name}: promote_to_ab={g['promote_to_ab']} " + f"ic={((g.get('row') or {}).get('mean_ic'))} " + f"t={((g.get('row') or {}).get('ic_t_stat'))}" + ) + + ab_results: dict[str, Any] | None = None + promote_names = [n for n, g in grades.items() if g.get("promote_to_ab")] + run_ab = (bool(promote_names) or args.force_ab) and not args.skip_ab + if run_ab: + # Prefer sector_resid if both; else the one that promoted / force resid. + if "mom_12_1_sector_resid" in promote_names or ( + args.force_ab and not promote_names + ): + ab_signal = "mom_12_1_sector_resid" + else: + ab_signal = promote_names[0] + print(f"Running portfolio A/B for {ab_signal}…") + ab_results = await _run_ab( + snapshot, + sector_map=mapping, + signal_name=ab_signal, + sector_cap=args.sector_cap, + quiet=args.quiet, + workers=args.workers, + ) + print( + f" A/B promote={ab_results.get('promotion', {}).get('promote')} " + f"— {ab_results.get('promotion', {}).get('reason')}" + ) + else: + print("Skipping portfolio A/B (no IC promotion; use --force-ab to override).") + + # Verdict + if ab_results and ab_results.get("promotion", {}).get("promote"): + verdict = "PROMOTE" + detail = ( + f"{ab_results['signal']} cleared IC + A/B bars. " + "Human must design wire-in; do not ship from this branch." + ) + human = ( + "- Approve or reject production residual swap vs dual-signal design.\n" + "- If sector-cap arm ran, review tail-trim diagnostics before any cap." + ) + elif any(g.get("promote_to_ab") for g in grades.values()): + verdict = "PARK" + detail = ( + "IC promotion bar cleared but A/B did not promote " + "(or A/B skipped). Park for human review." + ) + human = "- Inspect A/B windows; decide whether to re-run or park." + elif any( + (g.get("row") or {}).get("mean_ic") is not None + and abs(float((g.get("row") or {}).get("mean_ic") or 0)) >= IRON_IC_BAR * 0.5 + for g in grades.values() + ): + verdict = "PARK" + detail = "Weak / partial IC — not dead, not green. Machinery kept." + human = "- No book change. Revisit after history-depth extension (Task 3)." + else: + verdict = "DEAD" + detail = ( + "Neither sector residual nor sector demean cleared the iron-rule bar " + "with t ≥ mom_12_1_resid on this window." + ) + human = "- Do not wire sector residual. Optional: re-check after Task 3 depth." + + stamp = datetime.now().strftime("%Y%m%d-%H%M%S") + out_path = Path(args.out) if args.out else Path("reports") / f"sector-residual-{stamp}.json" + payload = { + "generated_at": datetime.now().isoformat(), + "snapshot_guard": guard, + "sector_coverage": cov, + "sector_map_path": str(sector_map_path.resolve()), + "signal_eval": signal_eval, + "ic_grades": grades, + "portfolio_ab": ab_results, + "verdict": verdict, + "verdict_detail": detail, + "human_next": human, + "report_path": str(out_path.as_posix()), + "pre_registration": { + "iron_ic_bar": IRON_IC_BAR, + "validation_split": VALIDATION_SPLIT.isoformat(), + "fill_mode": "close", + "cost_per_side": 0.001, + "ab_rule": "val Sharpe >= control - 0.5*SE; full Sharpe & maxDD not worse", + }, + } + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text(json.dumps(payload, indent=2, default=str) + "\n", encoding="utf-8") + + md_path = Path("docs/research/sector-residual-momentum.md") + _write_md(md_path, payload) + # Companion md under reports/ + md_report = out_path.with_suffix(".md") + md_report.write_text(md_path.read_text(encoding="utf-8"), encoding="utf-8") + + print(f"Verdict: {verdict}") + print(f"Wrote {out_path}") + print(f"Wrote {md_path}") + + +if __name__ == "__main__": + asyncio.run(_main()) diff --git a/tests/unit/test_backtest_service.py b/tests/unit/test_backtest_service.py index aff0eb6..b811a27 100644 --- a/tests/unit/test_backtest_service.py +++ b/tests/unit/test_backtest_service.py @@ -100,6 +100,75 @@ def test_residual_momentum_removes_market_beta_but_keeps_specific_drift(): assert drift["mom_12_1_resid"] > pure["mom_12_1_resid"] + 0.12 +def test_sector_residual_momentum_two_factor(): + """Pure market+sector beta stock → sector resid ~0; idiosyncratic drift kept.""" + dates, pure_beta, highs, benchmark = _signal_test_series(extra_return=0.0) + # Sector ETF = leveraged market (collinear-ish but not identical). + sector = {d: benchmark[d] * 1.02 + 0.5 for d in dates} + # Stock with pure exposure to market + sector, no alpha. + closes = [100.0] + for i in range(1, len(dates)): + m_prev = benchmark[dates[i - 1]] + m_cur = benchmark[dates[i]] + s_prev = sector[dates[i - 1]] + s_cur = sector[dates[i]] + m_ret = m_cur / m_prev - 1.0 + s_ret = s_cur / s_prev - 1.0 + closes.append(closes[-1] * (1.0 + 0.7 * m_ret + 0.5 * s_ret)) + highs_p = [c * 1.01 for c in closes] + + pure = bt._signal_values( + dates, closes, highs_p, 260, benchmark, sector_etf_closes=sector + ) + assert "mom_12_1_sector_resid" in pure + assert pure["mom_12_1_sector_resid"] == pytest.approx(0.0, abs=0.05) + + # Add idiosyncratic drift — sector residual should keep it. + drift_closes = [100.0] + for i in range(1, len(dates)): + m_prev = benchmark[dates[i - 1]] + m_cur = benchmark[dates[i]] + s_prev = sector[dates[i - 1]] + s_cur = sector[dates[i]] + m_ret = m_cur / m_prev - 1.0 + s_ret = s_cur / s_prev - 1.0 + drift_closes.append( + drift_closes[-1] * (1.0 + 0.7 * m_ret + 0.5 * s_ret + 0.0008) + ) + drift_highs = [c * 1.01 for c in drift_closes] + drift = bt._signal_values( + dates, drift_closes, drift_highs, 260, benchmark, sector_etf_closes=sector + ) + assert drift["mom_12_1_sector_resid"] > pure["mom_12_1_sector_resid"] + 0.10 + + +def test_inject_sector_demeaned_momentum(): + collected = { + "mom_12_1": { + (2024, 1): [ + {"val": 0.20, "fwd": 0.01, "symbol": "AAA"}, + {"val": 0.10, "fwd": 0.02, "symbol": "BBB"}, + {"val": 0.40, "fwd": -0.01, "symbol": "CCC"}, + {"val": 0.00, "fwd": 0.03, "symbol": "DDD"}, + ] + } + } + symbol_to_sector = { + "AAA": "Information Technology", + "BBB": "Information Technology", + "CCC": "Energy", + "DDD": "Energy", + } + bt._inject_sector_demeaned_momentum(collected, symbol_to_sector) + dem = collected["mom_12_1_sector_demeaned"][(2024, 1)] + by_sym = {r["symbol"]: r["val"] for r in dem} + # IT mean = 0.15 → AAA +0.05, BBB -0.05; Energy mean = 0.20 → CCC +0.20, DDD -0.20 + assert by_sym["AAA"] == pytest.approx(0.05) + assert by_sym["BBB"] == pytest.approx(-0.05) + assert by_sym["CCC"] == pytest.approx(0.20) + assert by_sym["DDD"] == pytest.approx(-0.20) + + def test_assigns_raw_and_residual_percentiles_independently(): cands = [ {"iso_week": (2026, 1), "momentum": 0.10, "residual_momentum": 0.30}, From 32bf9c9297a8019cdce4ccc9f139f4d412c954af Mon Sep 17 00:00:00 2001 From: Dennis Thiessen Date: Sun, 19 Jul 2026 09:34:49 +0200 Subject: [PATCH 02/14] research: bundle MacBook tier-1 pipeline into one bash script scripts/run_tier1_macbook.sh wraps earnings resume, coverage probe, deep snapshot rebuild, sector ETF refresh, and history-depth harness with phase flags. --- docs/research/history-depth-extension.md | 50 +++---- scripts/run_tier1_macbook.sh | 169 +++++++++++++++++++++++ 2 files changed, 187 insertions(+), 32 deletions(-) create mode 100644 scripts/run_tier1_macbook.sh diff --git a/docs/research/history-depth-extension.md b/docs/research/history-depth-extension.md index 6ffcc16..e25fb92 100644 --- a/docs/research/history-depth-extension.md +++ b/docs/research/history-depth-extension.md @@ -45,45 +45,31 @@ the 2018 vol shock and full 2020 crash (where the feed allows). ## MacBook runbook +Prefer the bundled script (one entry point): + ```bash -# 0. Repo + env -git fetch origin -git checkout research/earnings-gap-and-sue # or history-depth branch once pushed -# ensure .env has ALPACA_* (and FMP if resuming earnings) +git fetch origin && git checkout research/earnings-gap-and-sue +# .env: ALPACA_* required; FMP_* if resuming earnings +# copy backtest_snapshots/prod.sqlite if not already local -# 1. (Optional) finish earnings backfill first — multi-day free tier -python scripts/backfill_earnings_events.py \ - --snapshot backtest_snapshots/prod.sqlite \ - --provider fmp --force-symbol --limit 250 --sleep 0.35 +chmod +x scripts/run_tier1_macbook.sh -# 2. Coverage probe (before long rebuild) -python scripts/run_history_depth_research.py --phase coverage \ - --snapshot backtest_snapshots/prod.sqlite +# Default: coverage → deep rebuild → harness (+ era split) +./scripts/run_tier1_macbook.sh -# 3. Full deep rebuild of research.sqlite (LONG — Alpaca per symbol) -# Clears prior completion manifest; writes complete=true only at end. -python scripts/extend_snapshot_universe.py \ - --source backtest_snapshots/prod.sqlite \ - --output backtest_snapshots/research.sqlite \ - --force-copy \ - --history-days 5000 \ - --min-bars 260 \ - --sleep 0.15 +# Optional variants +./scripts/run_tier1_macbook.sh --all # + earnings resume first +./scripts/run_tier1_macbook.sh --earnings-only # multi-day FMP + 2a/2b only +./scripts/run_tier1_macbook.sh --harness-only # skip rebuild +./scripts/run_tier1_macbook.sh --coverage-only -# 4. Also refresh SPY + sector ETFs to the same depth on BOTH snapshots -python scripts/fetch_sector_etfs_to_snapshot.py \ - --snapshot backtest_snapshots/research.sqlite --history-days 5000 -python scripts/fetch_sector_etfs_to_snapshot.py \ - --snapshot backtest_snapshots/prod.sqlite --history-days 5000 - -# 5. Harness + era split (after race guard passes) -python scripts/run_history_depth_research.py --phase harness \ - --snapshot backtest_snapshots/research.sqlite \ - --workers 8 --allow-spawn - -# 6. Copy reports/ + docs/research/history-depth-extension.md results back +# Tunables +WORKERS=12 HISTORY_DAYS=5000 ./scripts/run_tier1_macbook.sh +./scripts/run_tier1_macbook.sh --workers 12 --fmp-limit 250 ``` +Then commit `reports/` + updated research docs, or copy them back to Windows. + --- ## Data provenance diff --git a/scripts/run_tier1_macbook.sh b/scripts/run_tier1_macbook.sh new file mode 100644 index 0000000..5afbe20 --- /dev/null +++ b/scripts/run_tier1_macbook.sh @@ -0,0 +1,169 @@ +#!/usr/bin/env bash +# Tier-1 alpha research runner for a high-CPU MacBook (local only). +# +# Prerequisites +# - git checkout research/earnings-gap-and-sue (or later research branch) +# - .env with ALPACA_* (required for OHLCV/ETFs); FMP_* for earnings resume; +# optional ALPHA_VANTAGE_* as earnings fallback +# - Python venv with project deps installed +# - backtest_snapshots/prod.sqlite present (gitignored — copy or rebuild) +# +# Usage +# chmod +x scripts/run_tier1_macbook.sh +# ./scripts/run_tier1_macbook.sh # coverage + deep rebuild + harness +# ./scripts/run_tier1_macbook.sh --all # earnings resume + full depth pipeline +# ./scripts/run_tier1_macbook.sh --earnings-only # multi-day FMP backfill + re-run 2a/2b +# ./scripts/run_tier1_macbook.sh --harness-only # skip rebuild; race-guard + IC only +# ./scripts/run_tier1_macbook.sh --coverage-only # bars-per-year probe only +# +# Does NOT touch production Postgres, scheduler, gates, or prod config. + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT" + +# --- defaults (override via flags or env) --- +PROD_SNAP="${PROD_SNAP:-backtest_snapshots/prod.sqlite}" +RESEARCH_SNAP="${RESEARCH_SNAP:-backtest_snapshots/research.sqlite}" +HISTORY_DAYS="${HISTORY_DAYS:-5000}" +MIN_BARS="${MIN_BARS:-260}" +WORKERS="${WORKERS:-8}" +ALPACA_SLEEP="${ALPACA_SLEEP:-0.15}" +FMP_LIMIT="${FMP_LIMIT:-250}" +FMP_SLEEP="${FMP_SLEEP:-0.35}" +PYTHON="${PYTHON:-python3}" + +PHASE="depth" # depth | all | earnings | harness | coverage + +usage() { + sed -n '2,22p' "$0" | sed 's/^# \?//' + exit "${1:-0}" +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --all) PHASE=all; shift ;; + --earnings-only) PHASE=earnings; shift ;; + --harness-only) PHASE=harness; shift ;; + --coverage-only) PHASE=coverage; shift ;; + --depth) PHASE=depth; shift ;; + --prod-snap) PROD_SNAP="$2"; shift 2 ;; + --research-snap) RESEARCH_SNAP="$2"; shift 2 ;; + --history-days) HISTORY_DAYS="$2"; shift 2 ;; + --workers) WORKERS="$2"; shift 2 ;; + --fmp-limit) FMP_LIMIT="$2"; shift 2 ;; + --python) PYTHON="$2"; shift 2 ;; + -h|--help) usage 0 ;; + *) echo "Unknown flag: $1" >&2; usage 1 ;; + esac +done + +if [[ -x .venv/bin/python ]]; then + PYTHON=".venv/bin/python" +elif command -v "$PYTHON" >/dev/null 2>&1; then + : +else + echo "ERROR: no Python found (tried .venv/bin/python and $PYTHON)" >&2 + exit 1 +fi + +log() { printf '\n==> %s\n' "$*"; } +die() { echo "ERROR: $*" >&2; exit 1; } + +need_file() { + [[ -f "$1" ]] || die "missing $1" +} + +require_prod() { + need_file "$PROD_SNAP" +} + +run_earnings() { + require_prod + log "Earnings backfill (FMP free tier ~${FMP_LIMIT}/day; resume-safe)" + "$PYTHON" scripts/backfill_earnings_events.py \ + --snapshot "$PROD_SNAP" \ + --provider fmp \ + --force-symbol \ + --limit "$FMP_LIMIT" \ + --sleep "$FMP_SLEEP" + + log "Earnings research 2a+2b (report only; no filters shipped)" + "$PYTHON" scripts/run_earnings_research.py \ + --snapshot "$PROD_SNAP" \ + --workers "$WORKERS" \ + --allow-spawn +} + +run_coverage() { + require_prod + log "Coverage probe (bars per year) on $PROD_SNAP" + "$PYTHON" scripts/run_history_depth_research.py \ + --phase coverage \ + --snapshot "$PROD_SNAP" +} + +run_rebuild() { + require_prod + log "Deep rebuild $PROD_SNAP → $RESEARCH_SNAP (history-days=$HISTORY_DAYS)" + log "SURVIVORSHIP: today's constituents backfilled — relative IC only, not levels" + "$PYTHON" scripts/extend_snapshot_universe.py \ + --source "$PROD_SNAP" \ + --output "$RESEARCH_SNAP" \ + --force-copy \ + --history-days "$HISTORY_DAYS" \ + --min-bars "$MIN_BARS" \ + --sleep "$ALPACA_SLEEP" + + log "Refresh SPY + 11 sector ETFs on research snapshot" + "$PYTHON" scripts/fetch_sector_etfs_to_snapshot.py \ + --snapshot "$RESEARCH_SNAP" \ + --history-days "$HISTORY_DAYS" + + log "Also deepen sector ETFs on prod snapshot (for local A/B parity)" + "$PYTHON" scripts/fetch_sector_etfs_to_snapshot.py \ + --snapshot "$PROD_SNAP" \ + --history-days "$HISTORY_DAYS" +} + +run_harness() { + need_file "$RESEARCH_SNAP" + log "Race-guard + full signal harness + era split on $RESEARCH_SNAP" + "$PYTHON" scripts/run_history_depth_research.py \ + --phase harness \ + --snapshot "$RESEARCH_SNAP" \ + --workers "$WORKERS" \ + --allow-spawn +} + +log "cwd=$ROOT python=$PYTHON phase=$PHASE workers=$WORKERS" + +case "$PHASE" in + coverage) + run_coverage + ;; + earnings) + run_earnings + ;; + harness) + run_harness + ;; + depth) + run_coverage + run_rebuild + run_harness + ;; + all) + run_earnings + run_coverage + run_rebuild + run_harness + ;; + *) + die "unknown phase $PHASE" + ;; +esac + +log "Done. Check reports/ and docs/research/history-depth-extension.md" +log "Commit reports on this machine if they look good, or copy them back to Windows." From 06cf054f60ba8f91b452a711a018f07e55eb1148 Mon Sep 17 00:00:00 2001 From: Dennis Thiessen Date: Sun, 19 Jul 2026 09:46:07 +0200 Subject: [PATCH 03/14] fix: bootstrap SSL/CA for research CLI on corporate MacBooks Extract app/ssl_bootstrap.py (shared with FastAPI main), wire it into research scripts, and teach run_tier1_macbook.sh to locate combined-ca-bundle.pem, certifi, optional USE_CORP_PROXY, plus --ssl-check diagnostics. --- app/main.py | 51 +-------- app/ssl_bootstrap.py | 136 +++++++++++++++++++++++ scripts/backfill_earnings_events.py | 4 + scripts/build_ticker_sector_map.py | 4 + scripts/extend_snapshot_universe.py | 4 + scripts/fetch_sector_etfs_to_snapshot.py | 4 + scripts/run_earnings_research.py | 4 + scripts/run_history_depth_research.py | 4 + scripts/run_sector_residual_research.py | 4 + scripts/run_tier1_macbook.sh | 94 +++++++++++++++- 10 files changed, 258 insertions(+), 51 deletions(-) create mode 100644 app/ssl_bootstrap.py diff --git a/app/main.py b/app/main.py index 9bc5f8f..c8064ad 100644 --- a/app/main.py +++ b/app/main.py @@ -3,56 +3,9 @@ # --------------------------------------------------------------------------- # SSL + proxy injection — MUST happen before any HTTP client imports # --------------------------------------------------------------------------- -import os as _os -import ssl as _ssl -from pathlib import Path as _Path +from app.ssl_bootstrap import bootstrap_ssl -_COMBINED_CERT = _Path(__file__).resolve().parent.parent / "combined-ca-bundle.pem" - -if _COMBINED_CERT.exists(): - _cert_path = str(_COMBINED_CERT) - # Env vars for libraries that respect them (requests, urllib3) - _os.environ["SSL_CERT_FILE"] = _cert_path - _os.environ["REQUESTS_CA_BUNDLE"] = _cert_path - _os.environ["CURL_CA_BUNDLE"] = _cert_path - - # Monkey-patch ssl.create_default_context so that ALL libraries - # (aiohttp, httpx, google-genai, alpaca-py, etc.) automatically - # use our combined CA bundle that includes the corporate root cert. - _original_create_default_context = _ssl.create_default_context - - def _patched_create_default_context( - purpose=_ssl.Purpose.SERVER_AUTH, *, cafile=None, capath=None, cadata=None - ): - ctx = _original_create_default_context( - purpose, cafile=cafile, capath=capath, cadata=cadata - ) - # Always load our combined bundle on top of whatever was loaded - ctx.load_verify_locations(cafile=_cert_path) - return ctx - - _ssl.create_default_context = _patched_create_default_context - - # Also patch aiohttp's cached SSL context objects directly, since - # aiohttp creates them at import time and may have already cached - # a context without our corporate CA bundle. - try: - import aiohttp.connector as _aio_conn - if hasattr(_aio_conn, '_SSL_CONTEXT_VERIFIED') and _aio_conn._SSL_CONTEXT_VERIFIED is not None: - _aio_conn._SSL_CONTEXT_VERIFIED.load_verify_locations(cafile=_cert_path) - if hasattr(_aio_conn, '_SSL_CONTEXT_UNVERIFIED') and _aio_conn._SSL_CONTEXT_UNVERIFIED is not None: - _aio_conn._SSL_CONTEXT_UNVERIFIED.load_verify_locations(cafile=_cert_path) - except ImportError: - pass - -# Corporate proxy — needed when Kiro spawns the process (no .zshrc sourced) -# Only enable this if explicitly requested via environment variable. -if _os.environ.get("USE_CORP_PROXY", "0") == "1": - _PROXY = "http://aproxy.corproot.net:8080" - _NO_PROXY = "corproot.net,sharedtcs.net,127.0.0.1,localhost,bix.swisscom.com,swisscom.com" - _os.environ.setdefault("HTTP_PROXY", _PROXY) - _os.environ.setdefault("HTTPS_PROXY", _PROXY) - _os.environ.setdefault("NO_PROXY", _NO_PROXY) +bootstrap_ssl() import logging import sys diff --git a/app/ssl_bootstrap.py b/app/ssl_bootstrap.py new file mode 100644 index 0000000..e3f2e4e --- /dev/null +++ b/app/ssl_bootstrap.py @@ -0,0 +1,136 @@ +"""TLS / corporate-proxy bootstrap for CLI scripts and the API. + +Must run **before** httpx / alpaca / aiohttp open connections. + +Resolution order for the CA bundle: +1. ``combined-ca-bundle.pem`` in the repo root (gitignored corporate bundle) +2. ``$HOME/combined-ca-bundle.pem`` (MacBook path used by existing tooling) +3. ``SSL_CERT_FILE`` / ``REQUESTS_CA_BUNDLE`` if already set and present +4. ``certifi.where()`` when the package is installed +5. System defaults (no patch) + +Optional corporate proxy (Swisscom-style) when ``USE_CORP_PROXY=1``. +""" + +from __future__ import annotations + +import os +import ssl +from pathlib import Path + +_BOOTSTRAPPED = False + + +def _candidate_ca_paths() -> list[Path]: + root = Path(__file__).resolve().parent.parent + home = Path.home() + env_paths = [ + os.environ.get("SSL_CERT_FILE", ""), + os.environ.get("REQUESTS_CA_BUNDLE", ""), + os.environ.get("CURL_CA_BUNDLE", ""), + ] + paths = [ + root / "combined-ca-bundle.pem", + home / "combined-ca-bundle.pem", + *[Path(p) for p in env_paths if p], + ] + try: + import certifi + + paths.append(Path(certifi.where())) + except Exception: + pass + return paths + + +def resolve_ca_bundle() -> str | None: + for path in _candidate_ca_paths(): + try: + if path.is_file() and path.stat().st_size > 0: + return str(path.resolve()) + except OSError: + continue + return None + + +def apply_corp_proxy_if_requested() -> None: + if os.environ.get("USE_CORP_PROXY", "0") != "1": + return + proxy = os.environ.get("CORP_HTTP_PROXY", "http://aproxy.corproot.net:8080") + no_proxy = os.environ.get( + "CORP_NO_PROXY", + "corproot.net,sharedtcs.net,127.0.0.1,localhost,bix.swisscom.com,swisscom.com", + ) + os.environ.setdefault("HTTP_PROXY", proxy) + os.environ.setdefault("HTTPS_PROXY", proxy) + os.environ.setdefault("NO_PROXY", no_proxy) + os.environ.setdefault("http_proxy", proxy) + os.environ.setdefault("https_proxy", proxy) + os.environ.setdefault("no_proxy", no_proxy) + + +def bootstrap_ssl(*, force: bool = False) -> str | None: + """Install CA env vars + patch ``ssl.create_default_context``. + + Returns the CA path used, or None if nothing was applied. + Safe to call multiple times. + """ + global _BOOTSTRAPPED + if _BOOTSTRAPPED and not force: + return os.environ.get("SSL_CERT_FILE") or None + + apply_corp_proxy_if_requested() + + cert_path = resolve_ca_bundle() + if not cert_path: + _BOOTSTRAPPED = True + return None + + os.environ["SSL_CERT_FILE"] = cert_path + os.environ["REQUESTS_CA_BUNDLE"] = cert_path + os.environ["CURL_CA_BUNDLE"] = cert_path + + original = ssl.create_default_context + + def _patched( + purpose=ssl.Purpose.SERVER_AUTH, *, cafile=None, capath=None, cadata=None + ): + ctx = original(purpose, cafile=cafile, capath=capath, cadata=cadata) + try: + ctx.load_verify_locations(cafile=cert_path) + except Exception: + pass + return ctx + + ssl.create_default_context = _patched # type: ignore[assignment] + + # aiohttp may cache SSL contexts at import time. + try: + import aiohttp.connector as aio_conn + + for attr in ("_SSL_CONTEXT_VERIFIED", "_SSL_CONTEXT_UNVERIFIED"): + ctx = getattr(aio_conn, attr, None) + if ctx is not None: + try: + ctx.load_verify_locations(cafile=cert_path) + except Exception: + pass + except ImportError: + pass + + _BOOTSTRAPPED = True + return cert_path + + +def ssl_status() -> dict: + """Diagnostic blob for research scripts / MacBook troubleshooting.""" + ca = resolve_ca_bundle() + return { + "ca_bundle": ca, + "ssl_cert_file_env": os.environ.get("SSL_CERT_FILE"), + "use_corp_proxy": os.environ.get("USE_CORP_PROXY", "0"), + "http_proxy": os.environ.get("HTTPS_PROXY") or os.environ.get("HTTP_PROXY"), + "candidates_exist": { + str(p): p.is_file() for p in _candidate_ca_paths()[:4] + }, + } diff --git a/scripts/backfill_earnings_events.py b/scripts/backfill_earnings_events.py index 971d555..665eabc 100644 --- a/scripts/backfill_earnings_events.py +++ b/scripts/backfill_earnings_events.py @@ -29,6 +29,10 @@ ROOT = Path(__file__).resolve().parents[1] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) +from app.ssl_bootstrap import bootstrap_ssl # noqa: E402 + +bootstrap_ssl() + FMP_STABLE = "https://financialmodelingprep.com/stable" DDL = """ CREATE TABLE IF NOT EXISTS earnings_events ( diff --git a/scripts/build_ticker_sector_map.py b/scripts/build_ticker_sector_map.py index f4a9ae9..e200b76 100644 --- a/scripts/build_ticker_sector_map.py +++ b/scripts/build_ticker_sector_map.py @@ -34,6 +34,10 @@ ROOT = Path(__file__).resolve().parents[1] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) +from app.ssl_bootstrap import bootstrap_ssl # noqa: E402 + +bootstrap_ssl() + from app.services.sector_map import ( # noqa: E402 DEFAULT_SECTOR_MAP_PATH, coverage_stats, diff --git a/scripts/extend_snapshot_universe.py b/scripts/extend_snapshot_universe.py index 886d9e2..8f10b94 100644 --- a/scripts/extend_snapshot_universe.py +++ b/scripts/extend_snapshot_universe.py @@ -45,6 +45,10 @@ ROOT = Path(__file__).resolve().parents[1] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) +from app.ssl_bootstrap import bootstrap_ssl # noqa: E402 + +bootstrap_ssl() + def _parse_args() -> argparse.Namespace: p = argparse.ArgumentParser(description=__doc__) diff --git a/scripts/fetch_sector_etfs_to_snapshot.py b/scripts/fetch_sector_etfs_to_snapshot.py index db558e0..5968a03 100644 --- a/scripts/fetch_sector_etfs_to_snapshot.py +++ b/scripts/fetch_sector_etfs_to_snapshot.py @@ -27,6 +27,10 @@ ROOT = Path(__file__).resolve().parents[1] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) +from app.ssl_bootstrap import bootstrap_ssl # noqa: E402 + +bootstrap_ssl() + from app.services.sector_map import SECTOR_ETFS # noqa: E402 diff --git a/scripts/run_earnings_research.py b/scripts/run_earnings_research.py index 20c695f..1e12264 100644 --- a/scripts/run_earnings_research.py +++ b/scripts/run_earnings_research.py @@ -28,6 +28,10 @@ ROOT = Path(__file__).resolve().parents[1] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) +from app.ssl_bootstrap import bootstrap_ssl # noqa: E402 + +bootstrap_ssl() + IRON_IC_BAR = 0.03 MIN_RELIABLE = 12 SUE_CARRY_DAYS = 63 diff --git a/scripts/run_history_depth_research.py b/scripts/run_history_depth_research.py index 02bb229..cfe2285 100644 --- a/scripts/run_history_depth_research.py +++ b/scripts/run_history_depth_research.py @@ -35,6 +35,10 @@ ROOT = Path(__file__).resolve().parents[1] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) +from app.ssl_bootstrap import bootstrap_ssl # noqa: E402 + +bootstrap_ssl() + ERA_SPLIT = date(2021, 1, 1) SURVIVORSHIP_BANNER = ( "SURVIVORSHIP BIAS: today's constituents backfilled historically. " diff --git a/scripts/run_sector_residual_research.py b/scripts/run_sector_residual_research.py index 290987d..1f60590 100644 --- a/scripts/run_sector_residual_research.py +++ b/scripts/run_sector_residual_research.py @@ -40,6 +40,10 @@ ROOT = Path(__file__).resolve().parents[1] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) +from app.ssl_bootstrap import bootstrap_ssl # noqa: E402 + +bootstrap_ssl() + from app.services.sector_map import ( # noqa: E402 DEFAULT_SECTOR_MAP_PATH, SECTOR_ETFS, diff --git a/scripts/run_tier1_macbook.sh b/scripts/run_tier1_macbook.sh index 5afbe20..1472cf4 100644 --- a/scripts/run_tier1_macbook.sh +++ b/scripts/run_tier1_macbook.sh @@ -33,11 +33,21 @@ ALPACA_SLEEP="${ALPACA_SLEEP:-0.15}" FMP_LIMIT="${FMP_LIMIT:-250}" FMP_SLEEP="${FMP_SLEEP:-0.35}" PYTHON="${PYTHON:-python3}" +USE_CORP_PROXY="${USE_CORP_PROXY:-0}" -PHASE="depth" # depth | all | earnings | harness | coverage +PHASE="depth" # depth | all | earnings | harness | coverage | ssl-check usage() { - sed -n '2,22p' "$0" | sed 's/^# \?//' + sed -n '2,25p' "$0" | sed 's/^# \?//' + cat <<'EOF' + +SSL / network (corporate MacBook) + SSL errors usually mean the corp root CA is missing from Python. + 1) Put combined-ca-bundle.pem in the repo root OR $HOME + 2) Or: export SSL_CERT_FILE=/path/to/combined-ca-bundle.pem + 3) Behind corp proxy: USE_CORP_PROXY=1 ./scripts/run_tier1_macbook.sh + 4) Diagnose: ./scripts/run_tier1_macbook.sh --ssl-check +EOF exit "${1:-0}" } @@ -48,6 +58,8 @@ while [[ $# -gt 0 ]]; do --harness-only) PHASE=harness; shift ;; --coverage-only) PHASE=coverage; shift ;; --depth) PHASE=depth; shift ;; + --ssl-check) PHASE=ssl; shift ;; + --corp-proxy) USE_CORP_PROXY=1; shift ;; --prod-snap) PROD_SNAP="$2"; shift 2 ;; --research-snap) RESEARCH_SNAP="$2"; shift 2 ;; --history-days) HISTORY_DAYS="$2"; shift 2 ;; @@ -71,6 +83,80 @@ fi log() { printf '\n==> %s\n' "$*"; } die() { echo "ERROR: $*" >&2; exit 1; } +# --------------------------------------------------------------------------- +# TLS bootstrap — same corp CA path the FastAPI app uses +# --------------------------------------------------------------------------- +setup_ssl() { + export USE_CORP_PROXY + + # Prefer explicit env, then repo / home corporate bundle, then certifi. + if [[ -z "${SSL_CERT_FILE:-}" ]]; then + if [[ -f "$ROOT/combined-ca-bundle.pem" ]]; then + export SSL_CERT_FILE="$ROOT/combined-ca-bundle.pem" + elif [[ -f "$HOME/combined-ca-bundle.pem" ]]; then + export SSL_CERT_FILE="$HOME/combined-ca-bundle.pem" + fi + fi + + if [[ -n "${SSL_CERT_FILE:-}" && -f "$SSL_CERT_FILE" ]]; then + export REQUESTS_CA_BUNDLE="$SSL_CERT_FILE" + export CURL_CA_BUNDLE="$SSL_CERT_FILE" + log "SSL CA bundle: $SSL_CERT_FILE" + else + # Fall back to certifi if installed + local certifi_path + certifi_path="$("$PYTHON" -c 'import certifi; print(certifi.where())' 2>/dev/null || true)" + if [[ -n "$certifi_path" && -f "$certifi_path" ]]; then + export SSL_CERT_FILE="$certifi_path" + export REQUESTS_CA_BUNDLE="$certifi_path" + export CURL_CA_BUNDLE="$certifi_path" + log "SSL CA bundle (certifi): $SSL_CERT_FILE" + else + log "WARNING: no CA bundle found — SSL may fail on corp networks" + log " Copy combined-ca-bundle.pem to $ROOT/ or \$HOME/" + log " Or: export SSL_CERT_FILE=/path/to/combined-ca-bundle.pem" + fi + fi + + if [[ "$USE_CORP_PROXY" == "1" ]]; then + export HTTP_PROXY="${HTTP_PROXY:-http://aproxy.corproot.net:8080}" + export HTTPS_PROXY="${HTTPS_PROXY:-http://aproxy.corproot.net:8080}" + export NO_PROXY="${NO_PROXY:-corproot.net,sharedtcs.net,127.0.0.1,localhost,bix.swisscom.com,swisscom.com}" + export http_proxy="$HTTP_PROXY" https_proxy="$HTTPS_PROXY" no_proxy="$NO_PROXY" + log "Corp proxy enabled: $HTTPS_PROXY" + fi + + # Ensure Python process sees the same bootstrap (patches ssl for alpaca-py). + export PYTHONPATH="${ROOT}${PYTHONPATH:+:$PYTHONPATH}" +} + +ssl_check() { + setup_ssl + log "SSL diagnostic" + "$PYTHON" - <<'PY' +from app.ssl_bootstrap import bootstrap_ssl, ssl_status +import json +import urllib.request + +ca = bootstrap_ssl() +print(json.dumps(ssl_status(), indent=2)) +print("bootstrap_ssl ->", ca) +urls = [ + "https://data.alpaca.markets/v2/stocks/SPY/bars?timeframe=1Day&limit=1", + "https://financialmodelingprep.com/stable/profile?symbol=AAPL", + "https://www.alphavantage.co/query?function=TIME_SERIES_DAILY&symbol=IBM", +] +for url in urls: + try: + req = urllib.request.Request(url, headers={"User-Agent": "signal-platform-ssl-check"}) + with urllib.request.urlopen(req, timeout=20) as resp: + print(f"OK {resp.status} {url[:60]}...") + except Exception as exc: + print(f"FAIL {type(exc).__name__}: {exc}") + print(f" {url[:80]}") +PY +} + need_file() { [[ -f "$1" ]] || die "missing $1" } @@ -138,8 +224,12 @@ run_harness() { } log "cwd=$ROOT python=$PYTHON phase=$PHASE workers=$WORKERS" +setup_ssl case "$PHASE" in + ssl) + ssl_check + ;; coverage) run_coverage ;; From f6e0ca734feeba3ce36da5bc27cde85226fe8680 Mon Sep 17 00:00:00 2001 From: Dennis Thiessen Date: Sun, 19 Jul 2026 10:40:19 +0200 Subject: [PATCH 04/14] tests done --- docs/research/history-depth-extension.md | 124 +++++- reports/history-depth-20260719-093853.json | 68 +++ reports/history-depth-20260719-093853.md | 177 ++++++++ reports/history-depth-20260719-094134.json | 68 +++ reports/history-depth-20260719-094134.md | 177 ++++++++ reports/history-depth-20260719-094344.json | 68 +++ reports/history-depth-20260719-094344.md | 177 ++++++++ reports/history-depth-20260719-095156.json | 68 +++ reports/history-depth-20260719-095156.md | 177 ++++++++ reports/history-depth-20260719-103315.json | 494 +++++++++++++++++++++ reports/history-depth-20260719-103315.md | 208 +++++++++ scripts/run_tier1_macbook.sh | 0 12 files changed, 1801 insertions(+), 5 deletions(-) create mode 100644 reports/history-depth-20260719-093853.json create mode 100644 reports/history-depth-20260719-093853.md create mode 100644 reports/history-depth-20260719-094134.json create mode 100644 reports/history-depth-20260719-094134.md create mode 100644 reports/history-depth-20260719-094344.json create mode 100644 reports/history-depth-20260719-094344.md create mode 100644 reports/history-depth-20260719-095156.json create mode 100644 reports/history-depth-20260719-095156.md create mode 100644 reports/history-depth-20260719-103315.json create mode 100644 reports/history-depth-20260719-103315.md mode change 100644 => 100755 scripts/run_tier1_macbook.sh diff --git a/docs/research/history-depth-extension.md b/docs/research/history-depth-extension.md index e25fb92..79da918 100644 --- a/docs/research/history-depth-extension.md +++ b/docs/research/history-depth-extension.md @@ -80,15 +80,129 @@ Then commit `reports/` + updated research docs, or copy them back to Windows. ## Results -*(filled at run time)* +Generated: `2026-07-19T10:33:15.322673` + +> **SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only — not levels.** + +### Coverage + +```json +{} +``` + +### Race guard + +```json +{ + "manifest": { + "schema_version": 1, + "snapshot": "research.sqlite", + "snapshot_resolved": "/Users/taathde3/git/lab/signal_platform/backtest_snapshots/research.sqlite", + "complete": true, + "finished_at": "2026-07-19T08:19:02.992206+00:00", + "ticker_count": 4655, + "ohlcv_row_count": 6609926, + "rank_only_count": 4149, + "sources": { + "nasdaq_all": "nasdaq_trader", + "sp500": "wikipedia_sp500" + }, + "history_days": 5000, + "min_bars": 260, + "fetch_ok": 4152, + "fetch_fail": 0, + "limit": null, + "extra": { + "prod_symbols_at_start": 506, + "pool_size": 4648, + "to_fetch": 4152 + }, + "live_counts": { + "ticker_count": 4655, + "ohlcv_row_count": 6609926, + "rank_only_count": 4149 + } + }, + "ok": true +} +``` + +### Signal IC (full extended window) + +| signal | mean_ic | ic_t_stat | weeks | avg_N | reliable | +|---|---:|---:|---:|---:|---| +| high_52w | 0.1111 | 6.41 | 84 | 2280.3 | True | +| mom_12_1 | 0.0663 | 5.11 | 83 | 2277.5 | True | +| mom_12_1_sector_resid | 0.0578 | 2.34 | 35 | 497.7 | True | +| trend_200 | 0.0546 | 4.3 | 85 | 2308.1 | True | +| mom_6_1 | 0.0493 | 4.91 | 88 | 2375.7 | True | +| mom_3_1 | 0.0363 | 3.58 | 90 | 2425.6 | True | +| mom_12_1_sector_demeaned | 0.034 | 1.32 | 35 | 496.7 | True | +| fip_id | 0.0267 | 3.25 | 83 | 2277.5 | True | +| mom_12_1_resid | 0.0256 | 2.21 | 83 | 2277.5 | True | +| reversal_1m | 0.003 | 0.31 | 91 | 2443.2 | True | +| vol_6m | -0.1226 | -6.34 | 88 | 2375.7 | True | + +### Era split (diagnostic only) + +#### full + +| signal | mean_ic | t | weeks | N | +|---|---:|---:|---:|---:| +| fip_id | 0.0267 | 3.25 | 83 | 2277.5 | +| high_52w | 0.1111 | 6.41 | 84 | 2280.3 | +| mom_12_1 | 0.0663 | 5.11 | 83 | 2277.5 | +| mom_12_1_resid | 0.0256 | 2.21 | 83 | 2277.5 | +| mom_12_1_sector_demeaned | 0.034 | 1.32 | 35 | 496.7 | +| mom_12_1_sector_resid | 0.0578 | 2.34 | 35 | 497.7 | +| mom_3_1 | 0.0363 | 3.58 | 90 | 2425.6 | +| mom_6_1 | 0.0493 | 4.91 | 88 | 2375.7 | +| reversal_1m | 0.003 | 0.31 | 91 | 2443.2 | +| trend_200 | 0.0546 | 4.3 | 85 | 2308.1 | +| vol_6m | -0.1226 | -6.34 | 88 | 2375.7 | + +#### pre_2021 + +| signal | mean_ic | t | weeks | N | +|---|---:|---:|---:|---:| +| fip_id | 0.0116 | 1.35 | 36 | 1424.8 | +| high_52w | 0.0494 | 2.0 | 36 | 1422.2 | +| mom_12_1 | 0.0413 | 2.94 | 36 | 1424.8 | +| mom_12_1_resid | 0.0226 | 1.68 | 36 | 1424.8 | +| mom_3_1 | 0.0217 | 1.84 | 42 | 1480.0 | +| mom_6_1 | 0.0205 | 1.63 | 40 | 1461.2 | +| reversal_1m | 0.0019 | 0.15 | 44 | 1501.4 | +| trend_200 | 0.0322 | 2.23 | 38 | 1440.9 | +| vol_6m | -0.056 | -2.16 | 40 | 1461.2 | + +#### post_2021 + +| signal | mean_ic | t | weeks | N | +|---|---:|---:|---:|---:| +| fip_id | 0.0366 | 2.91 | 48 | 2913.0 | +| high_52w | 0.1375 | 4.34 | 48 | 2914.5 | +| mom_12_1 | 0.0791 | 3.64 | 48 | 2913.0 | +| mom_12_1_resid | 0.0265 | 1.37 | 48 | 2913.0 | +| mom_12_1_sector_demeaned | 0.034 | 1.32 | 35 | 496.7 | +| mom_12_1_sector_resid | 0.0578 | 2.34 | 35 | 497.7 | +| mom_3_1 | 0.0291 | 1.54 | 48 | 3234.2 | +| mom_6_1 | 0.0779 | 4.36 | 48 | 3122.6 | +| reversal_1m | -0.0126 | -0.72 | 48 | 3315.8 | +| trend_200 | 0.0585 | 2.68 | 48 | 3001.5 | +| vol_6m | -0.1623 | -4.94 | 48 | 3122.6 | ---- ## Verdict -**Pending MacBook run.** +**PENDING_HUMAN** + +Harness complete — human interprets relative IC / era stability. No production retune from this artifact. ## What a human must decide next -- Do not retune production from deep history without explicit review. -- Use relative IC stability to accept/reject Task 1 sector residual wire-in. +- Compare sector residual vs market residual across eras. +- If pre-2021 IC collapses, park Task 1 wire-in. +- Do not retune production knobs on deep history levels. + +Artifacts: `reports/history-depth-20260719-103315.json` + diff --git a/reports/history-depth-20260719-093853.json b/reports/history-depth-20260719-093853.json new file mode 100644 index 0000000..6ee84fb --- /dev/null +++ b/reports/history-depth-20260719-093853.json @@ -0,0 +1,68 @@ +{ + "generated_at": "2026-07-19T09:38:53.936226", + "survivorship_banner": "SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only \u2014 not levels.", + "coverage": { + "snapshot": "/Users/taathde3/git/lab/signal_platform/backtest_snapshots/prod.sqlite", + "ticker_count": 506, + "ohlcv_row_count": 629263, + "date_range": { + "min": "2021-06-24", + "max": "2026-07-02" + }, + "bars_per_year": [ + { + "year": "2021", + "bars": 65678, + "tickers_with_bars": 494 + }, + { + "year": "2022", + "bars": 124423, + "tickers_with_bars": 497 + }, + { + "year": "2023", + "bars": 124479, + "tickers_with_bars": 499 + }, + { + "year": "2024", + "bars": 126129, + "tickers_with_bars": 501 + }, + { + "year": "2025", + "bars": 125615, + "tickers_with_bars": 504 + }, + { + "year": "2026", + "bars": 62939, + "tickers_with_bars": 505 + } + ], + "bars_per_symbol": { + "min": 14, + "p10": 1261, + "p50": 1261, + "p90": 1261, + "max": 1261 + }, + "symbols_by_start_year": { + "2021": 494, + "2022": 3, + "2023": 2, + "2024": 2, + "2025": 3, + "2026": 1 + }, + "note": "Where ticker counts drop in early years, the feed (or listing history) thins \u2014 do not treat those years as a full 505-name cross-section.", + "survivorship_banner": "SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only \u2014 not levels." + }, + "race_guard": null, + "harness": null, + "verdict": "COVERAGE_ONLY", + "verdict_detail": "Coverage probe only; run --phase harness after deep rebuild.", + "human_next": "- Compare sector residual vs market residual across eras.\n- If pre-2021 IC collapses, park Task 1 wire-in.\n- Do not retune production knobs on deep history levels.", + "report_path": "reports/history-depth-20260719-093853.json" +} diff --git a/reports/history-depth-20260719-093853.md b/reports/history-depth-20260719-093853.md new file mode 100644 index 0000000..658f7d3 --- /dev/null +++ b/reports/history-depth-20260719-093853.md @@ -0,0 +1,177 @@ +# History-depth extension (Tier-1 alpha research) + +**Status:** PRE-REGISTERED — run on MacBook (heavy I/O + full harness). +**Branch:** `research/history-depth-extension` (create from latest research stack). +**Production impact:** none. **Do not retune any production knob on deep history.** + +--- + +## Pre-registration (locked before rebuild) + +### Motivation + +All current conclusions rest on ~35 non-overlapping weekly windows in essentially +one post-2021 regime. Extending history toward max Alpaca daily-bar depth adds +the 2018 vol shock and full 2020 crash (where the feed allows). + +### Protocol + +1. **Empirical coverage first** — bars per calendar year per symbol; document + where the feed thins out. Do **not** assume a uniform start date. +2. **Rebuild the research snapshot completely** from prod source + max history + per symbol (`Adjustment.SPLIT`, ~200 req/min pacing via existing extender). +3. **Race guard (rule 6)** — refuse analysis until completion manifest is + `complete=true` and live counts match. +4. **Re-run full signal harness** (all existing signals incl. sector residual / + SUE if present) on the extended window. +5. **Report per signal:** mean IC, t, window count, and **era split** + (pre-/post-2021) — diagnostic only, **not a tuning input**. +6. **Log prominently:** survivorship bias grows with depth (today’s constituents + backfilled). Absolute Sharpe/CAGR on deep history is optimistic; payload is + **relative** signal comparisons and IC stability, not levels. +7. **Do not retune** production knobs. If a knob’s confirmation looks + overturned on deep history → report only; human decides. + +### Success / interpretation (not promotion of a new signal) + +| outcome | meaning | +|---|---| +| Sector residual still ≥ market residual on deep IC + stable sign | strengthens Task 1 PROMOTE case | +| Sector residual collapses pre-2021 | **PARK** Task 1 wire-in | +| SUE remains weak after full earnings + depth | **DEAD** SUE for this stack | +| Any production knob looks worse deep | report; no auto-retune | + +--- + +## MacBook runbook + +Prefer the bundled script (one entry point): + +```bash +git fetch origin && git checkout research/earnings-gap-and-sue +# .env: ALPACA_* required; FMP_* if resuming earnings +# copy backtest_snapshots/prod.sqlite if not already local + +chmod +x scripts/run_tier1_macbook.sh + +# Default: coverage → deep rebuild → harness (+ era split) +./scripts/run_tier1_macbook.sh + +# Optional variants +./scripts/run_tier1_macbook.sh --all # + earnings resume first +./scripts/run_tier1_macbook.sh --earnings-only # multi-day FMP + 2a/2b only +./scripts/run_tier1_macbook.sh --harness-only # skip rebuild +./scripts/run_tier1_macbook.sh --coverage-only + +# Tunables +WORKERS=12 HISTORY_DAYS=5000 ./scripts/run_tier1_macbook.sh +./scripts/run_tier1_macbook.sh --workers 12 --fmp-limit 250 +``` + +Then commit `reports/` + updated research docs, or copy them back to Windows. + +--- + +## Data provenance + +*(filled at run time)* + +--- + +## Results + +Generated: `2026-07-19T09:38:53.936226` + +> **SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only — not levels.** + +### Coverage + +```json +{ + "snapshot": "/Users/taathde3/git/lab/signal_platform/backtest_snapshots/prod.sqlite", + "ticker_count": 506, + "ohlcv_row_count": 629263, + "date_range": { + "min": "2021-06-24", + "max": "2026-07-02" + }, + "bars_per_year": [ + { + "year": "2021", + "bars": 65678, + "tickers_with_bars": 494 + }, + { + "year": "2022", + "bars": 124423, + "tickers_with_bars": 497 + }, + { + "year": "2023", + "bars": 124479, + "tickers_with_bars": 499 + }, + { + "year": "2024", + "bars": 126129, + "tickers_with_bars": 501 + }, + { + "year": "2025", + "bars": 125615, + "tickers_with_bars": 504 + }, + { + "year": "2026", + "bars": 62939, + "tickers_with_bars": 505 + } + ], + "bars_per_symbol": { + "min": 14, + "p10": 1261, + "p50": 1261, + "p90": 1261, + "max": 1261 + }, + "symbols_by_start_year": { + "2021": 494, + "2022": 3, + "2023": 2, + "2024": 2, + "2025": 3, + "2026": 1 + }, + "note": "Where ticker counts drop in early years, the feed (or listing history) thins \u2014 do not treat those years as a full 505-name cross-section.", + "survivorship_banner": "SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only \u2014 not levels." +} +``` + +### Race guard + +```json +{} +``` + +### Signal IC (full extended window) + +_Harness not run this pass._ + +### Era split (diagnostic only) + +_No era split._ + +## Verdict + +**COVERAGE_ONLY** + +Coverage probe only; run --phase harness after deep rebuild. + +## What a human must decide next + +- Compare sector residual vs market residual across eras. +- If pre-2021 IC collapses, park Task 1 wire-in. +- Do not retune production knobs on deep history levels. + +Artifacts: `reports/history-depth-20260719-093853.json` + diff --git a/reports/history-depth-20260719-094134.json b/reports/history-depth-20260719-094134.json new file mode 100644 index 0000000..826ced0 --- /dev/null +++ b/reports/history-depth-20260719-094134.json @@ -0,0 +1,68 @@ +{ + "generated_at": "2026-07-19T09:41:34.109378", + "survivorship_banner": "SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only \u2014 not levels.", + "coverage": { + "snapshot": "/Users/taathde3/git/lab/signal_platform/backtest_snapshots/prod.sqlite", + "ticker_count": 506, + "ohlcv_row_count": 629263, + "date_range": { + "min": "2021-06-24", + "max": "2026-07-02" + }, + "bars_per_year": [ + { + "year": "2021", + "bars": 65678, + "tickers_with_bars": 494 + }, + { + "year": "2022", + "bars": 124423, + "tickers_with_bars": 497 + }, + { + "year": "2023", + "bars": 124479, + "tickers_with_bars": 499 + }, + { + "year": "2024", + "bars": 126129, + "tickers_with_bars": 501 + }, + { + "year": "2025", + "bars": 125615, + "tickers_with_bars": 504 + }, + { + "year": "2026", + "bars": 62939, + "tickers_with_bars": 505 + } + ], + "bars_per_symbol": { + "min": 14, + "p10": 1261, + "p50": 1261, + "p90": 1261, + "max": 1261 + }, + "symbols_by_start_year": { + "2021": 494, + "2022": 3, + "2023": 2, + "2024": 2, + "2025": 3, + "2026": 1 + }, + "note": "Where ticker counts drop in early years, the feed (or listing history) thins \u2014 do not treat those years as a full 505-name cross-section.", + "survivorship_banner": "SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only \u2014 not levels." + }, + "race_guard": null, + "harness": null, + "verdict": "COVERAGE_ONLY", + "verdict_detail": "Coverage probe only; run --phase harness after deep rebuild.", + "human_next": "- Compare sector residual vs market residual across eras.\n- If pre-2021 IC collapses, park Task 1 wire-in.\n- Do not retune production knobs on deep history levels.", + "report_path": "reports/history-depth-20260719-094134.json" +} diff --git a/reports/history-depth-20260719-094134.md b/reports/history-depth-20260719-094134.md new file mode 100644 index 0000000..7b23512 --- /dev/null +++ b/reports/history-depth-20260719-094134.md @@ -0,0 +1,177 @@ +# History-depth extension (Tier-1 alpha research) + +**Status:** PRE-REGISTERED — run on MacBook (heavy I/O + full harness). +**Branch:** `research/history-depth-extension` (create from latest research stack). +**Production impact:** none. **Do not retune any production knob on deep history.** + +--- + +## Pre-registration (locked before rebuild) + +### Motivation + +All current conclusions rest on ~35 non-overlapping weekly windows in essentially +one post-2021 regime. Extending history toward max Alpaca daily-bar depth adds +the 2018 vol shock and full 2020 crash (where the feed allows). + +### Protocol + +1. **Empirical coverage first** — bars per calendar year per symbol; document + where the feed thins out. Do **not** assume a uniform start date. +2. **Rebuild the research snapshot completely** from prod source + max history + per symbol (`Adjustment.SPLIT`, ~200 req/min pacing via existing extender). +3. **Race guard (rule 6)** — refuse analysis until completion manifest is + `complete=true` and live counts match. +4. **Re-run full signal harness** (all existing signals incl. sector residual / + SUE if present) on the extended window. +5. **Report per signal:** mean IC, t, window count, and **era split** + (pre-/post-2021) — diagnostic only, **not a tuning input**. +6. **Log prominently:** survivorship bias grows with depth (today’s constituents + backfilled). Absolute Sharpe/CAGR on deep history is optimistic; payload is + **relative** signal comparisons and IC stability, not levels. +7. **Do not retune** production knobs. If a knob’s confirmation looks + overturned on deep history → report only; human decides. + +### Success / interpretation (not promotion of a new signal) + +| outcome | meaning | +|---|---| +| Sector residual still ≥ market residual on deep IC + stable sign | strengthens Task 1 PROMOTE case | +| Sector residual collapses pre-2021 | **PARK** Task 1 wire-in | +| SUE remains weak after full earnings + depth | **DEAD** SUE for this stack | +| Any production knob looks worse deep | report; no auto-retune | + +--- + +## MacBook runbook + +Prefer the bundled script (one entry point): + +```bash +git fetch origin && git checkout research/earnings-gap-and-sue +# .env: ALPACA_* required; FMP_* if resuming earnings +# copy backtest_snapshots/prod.sqlite if not already local + +chmod +x scripts/run_tier1_macbook.sh + +# Default: coverage → deep rebuild → harness (+ era split) +./scripts/run_tier1_macbook.sh + +# Optional variants +./scripts/run_tier1_macbook.sh --all # + earnings resume first +./scripts/run_tier1_macbook.sh --earnings-only # multi-day FMP + 2a/2b only +./scripts/run_tier1_macbook.sh --harness-only # skip rebuild +./scripts/run_tier1_macbook.sh --coverage-only + +# Tunables +WORKERS=12 HISTORY_DAYS=5000 ./scripts/run_tier1_macbook.sh +./scripts/run_tier1_macbook.sh --workers 12 --fmp-limit 250 +``` + +Then commit `reports/` + updated research docs, or copy them back to Windows. + +--- + +## Data provenance + +*(filled at run time)* + +--- + +## Results + +Generated: `2026-07-19T09:41:34.109378` + +> **SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only — not levels.** + +### Coverage + +```json +{ + "snapshot": "/Users/taathde3/git/lab/signal_platform/backtest_snapshots/prod.sqlite", + "ticker_count": 506, + "ohlcv_row_count": 629263, + "date_range": { + "min": "2021-06-24", + "max": "2026-07-02" + }, + "bars_per_year": [ + { + "year": "2021", + "bars": 65678, + "tickers_with_bars": 494 + }, + { + "year": "2022", + "bars": 124423, + "tickers_with_bars": 497 + }, + { + "year": "2023", + "bars": 124479, + "tickers_with_bars": 499 + }, + { + "year": "2024", + "bars": 126129, + "tickers_with_bars": 501 + }, + { + "year": "2025", + "bars": 125615, + "tickers_with_bars": 504 + }, + { + "year": "2026", + "bars": 62939, + "tickers_with_bars": 505 + } + ], + "bars_per_symbol": { + "min": 14, + "p10": 1261, + "p50": 1261, + "p90": 1261, + "max": 1261 + }, + "symbols_by_start_year": { + "2021": 494, + "2022": 3, + "2023": 2, + "2024": 2, + "2025": 3, + "2026": 1 + }, + "note": "Where ticker counts drop in early years, the feed (or listing history) thins \u2014 do not treat those years as a full 505-name cross-section.", + "survivorship_banner": "SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only \u2014 not levels." +} +``` + +### Race guard + +```json +{} +``` + +### Signal IC (full extended window) + +_Harness not run this pass._ + +### Era split (diagnostic only) + +_No era split._ + +## Verdict + +**COVERAGE_ONLY** + +Coverage probe only; run --phase harness after deep rebuild. + +## What a human must decide next + +- Compare sector residual vs market residual across eras. +- If pre-2021 IC collapses, park Task 1 wire-in. +- Do not retune production knobs on deep history levels. + +Artifacts: `reports/history-depth-20260719-094134.json` + diff --git a/reports/history-depth-20260719-094344.json b/reports/history-depth-20260719-094344.json new file mode 100644 index 0000000..5c35f97 --- /dev/null +++ b/reports/history-depth-20260719-094344.json @@ -0,0 +1,68 @@ +{ + "generated_at": "2026-07-19T09:43:44.520126", + "survivorship_banner": "SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only \u2014 not levels.", + "coverage": { + "snapshot": "/Users/taathde3/git/lab/signal_platform/backtest_snapshots/prod.sqlite", + "ticker_count": 506, + "ohlcv_row_count": 629263, + "date_range": { + "min": "2021-06-24", + "max": "2026-07-02" + }, + "bars_per_year": [ + { + "year": "2021", + "bars": 65678, + "tickers_with_bars": 494 + }, + { + "year": "2022", + "bars": 124423, + "tickers_with_bars": 497 + }, + { + "year": "2023", + "bars": 124479, + "tickers_with_bars": 499 + }, + { + "year": "2024", + "bars": 126129, + "tickers_with_bars": 501 + }, + { + "year": "2025", + "bars": 125615, + "tickers_with_bars": 504 + }, + { + "year": "2026", + "bars": 62939, + "tickers_with_bars": 505 + } + ], + "bars_per_symbol": { + "min": 14, + "p10": 1261, + "p50": 1261, + "p90": 1261, + "max": 1261 + }, + "symbols_by_start_year": { + "2021": 494, + "2022": 3, + "2023": 2, + "2024": 2, + "2025": 3, + "2026": 1 + }, + "note": "Where ticker counts drop in early years, the feed (or listing history) thins \u2014 do not treat those years as a full 505-name cross-section.", + "survivorship_banner": "SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only \u2014 not levels." + }, + "race_guard": null, + "harness": null, + "verdict": "COVERAGE_ONLY", + "verdict_detail": "Coverage probe only; run --phase harness after deep rebuild.", + "human_next": "- Compare sector residual vs market residual across eras.\n- If pre-2021 IC collapses, park Task 1 wire-in.\n- Do not retune production knobs on deep history levels.", + "report_path": "reports/history-depth-20260719-094344.json" +} diff --git a/reports/history-depth-20260719-094344.md b/reports/history-depth-20260719-094344.md new file mode 100644 index 0000000..d52b1bb --- /dev/null +++ b/reports/history-depth-20260719-094344.md @@ -0,0 +1,177 @@ +# History-depth extension (Tier-1 alpha research) + +**Status:** PRE-REGISTERED — run on MacBook (heavy I/O + full harness). +**Branch:** `research/history-depth-extension` (create from latest research stack). +**Production impact:** none. **Do not retune any production knob on deep history.** + +--- + +## Pre-registration (locked before rebuild) + +### Motivation + +All current conclusions rest on ~35 non-overlapping weekly windows in essentially +one post-2021 regime. Extending history toward max Alpaca daily-bar depth adds +the 2018 vol shock and full 2020 crash (where the feed allows). + +### Protocol + +1. **Empirical coverage first** — bars per calendar year per symbol; document + where the feed thins out. Do **not** assume a uniform start date. +2. **Rebuild the research snapshot completely** from prod source + max history + per symbol (`Adjustment.SPLIT`, ~200 req/min pacing via existing extender). +3. **Race guard (rule 6)** — refuse analysis until completion manifest is + `complete=true` and live counts match. +4. **Re-run full signal harness** (all existing signals incl. sector residual / + SUE if present) on the extended window. +5. **Report per signal:** mean IC, t, window count, and **era split** + (pre-/post-2021) — diagnostic only, **not a tuning input**. +6. **Log prominently:** survivorship bias grows with depth (today’s constituents + backfilled). Absolute Sharpe/CAGR on deep history is optimistic; payload is + **relative** signal comparisons and IC stability, not levels. +7. **Do not retune** production knobs. If a knob’s confirmation looks + overturned on deep history → report only; human decides. + +### Success / interpretation (not promotion of a new signal) + +| outcome | meaning | +|---|---| +| Sector residual still ≥ market residual on deep IC + stable sign | strengthens Task 1 PROMOTE case | +| Sector residual collapses pre-2021 | **PARK** Task 1 wire-in | +| SUE remains weak after full earnings + depth | **DEAD** SUE for this stack | +| Any production knob looks worse deep | report; no auto-retune | + +--- + +## MacBook runbook + +Prefer the bundled script (one entry point): + +```bash +git fetch origin && git checkout research/earnings-gap-and-sue +# .env: ALPACA_* required; FMP_* if resuming earnings +# copy backtest_snapshots/prod.sqlite if not already local + +chmod +x scripts/run_tier1_macbook.sh + +# Default: coverage → deep rebuild → harness (+ era split) +./scripts/run_tier1_macbook.sh + +# Optional variants +./scripts/run_tier1_macbook.sh --all # + earnings resume first +./scripts/run_tier1_macbook.sh --earnings-only # multi-day FMP + 2a/2b only +./scripts/run_tier1_macbook.sh --harness-only # skip rebuild +./scripts/run_tier1_macbook.sh --coverage-only + +# Tunables +WORKERS=12 HISTORY_DAYS=5000 ./scripts/run_tier1_macbook.sh +./scripts/run_tier1_macbook.sh --workers 12 --fmp-limit 250 +``` + +Then commit `reports/` + updated research docs, or copy them back to Windows. + +--- + +## Data provenance + +*(filled at run time)* + +--- + +## Results + +Generated: `2026-07-19T09:43:44.520126` + +> **SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only — not levels.** + +### Coverage + +```json +{ + "snapshot": "/Users/taathde3/git/lab/signal_platform/backtest_snapshots/prod.sqlite", + "ticker_count": 506, + "ohlcv_row_count": 629263, + "date_range": { + "min": "2021-06-24", + "max": "2026-07-02" + }, + "bars_per_year": [ + { + "year": "2021", + "bars": 65678, + "tickers_with_bars": 494 + }, + { + "year": "2022", + "bars": 124423, + "tickers_with_bars": 497 + }, + { + "year": "2023", + "bars": 124479, + "tickers_with_bars": 499 + }, + { + "year": "2024", + "bars": 126129, + "tickers_with_bars": 501 + }, + { + "year": "2025", + "bars": 125615, + "tickers_with_bars": 504 + }, + { + "year": "2026", + "bars": 62939, + "tickers_with_bars": 505 + } + ], + "bars_per_symbol": { + "min": 14, + "p10": 1261, + "p50": 1261, + "p90": 1261, + "max": 1261 + }, + "symbols_by_start_year": { + "2021": 494, + "2022": 3, + "2023": 2, + "2024": 2, + "2025": 3, + "2026": 1 + }, + "note": "Where ticker counts drop in early years, the feed (or listing history) thins \u2014 do not treat those years as a full 505-name cross-section.", + "survivorship_banner": "SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only \u2014 not levels." +} +``` + +### Race guard + +```json +{} +``` + +### Signal IC (full extended window) + +_Harness not run this pass._ + +### Era split (diagnostic only) + +_No era split._ + +## Verdict + +**COVERAGE_ONLY** + +Coverage probe only; run --phase harness after deep rebuild. + +## What a human must decide next + +- Compare sector residual vs market residual across eras. +- If pre-2021 IC collapses, park Task 1 wire-in. +- Do not retune production knobs on deep history levels. + +Artifacts: `reports/history-depth-20260719-094344.json` + diff --git a/reports/history-depth-20260719-095156.json b/reports/history-depth-20260719-095156.json new file mode 100644 index 0000000..416c34e --- /dev/null +++ b/reports/history-depth-20260719-095156.json @@ -0,0 +1,68 @@ +{ + "generated_at": "2026-07-19T09:51:56.525634", + "survivorship_banner": "SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only \u2014 not levels.", + "coverage": { + "snapshot": "/Users/taathde3/git/lab/signal_platform/backtest_snapshots/prod.sqlite", + "ticker_count": 506, + "ohlcv_row_count": 629263, + "date_range": { + "min": "2021-06-24", + "max": "2026-07-02" + }, + "bars_per_year": [ + { + "year": "2021", + "bars": 65678, + "tickers_with_bars": 494 + }, + { + "year": "2022", + "bars": 124423, + "tickers_with_bars": 497 + }, + { + "year": "2023", + "bars": 124479, + "tickers_with_bars": 499 + }, + { + "year": "2024", + "bars": 126129, + "tickers_with_bars": 501 + }, + { + "year": "2025", + "bars": 125615, + "tickers_with_bars": 504 + }, + { + "year": "2026", + "bars": 62939, + "tickers_with_bars": 505 + } + ], + "bars_per_symbol": { + "min": 14, + "p10": 1261, + "p50": 1261, + "p90": 1261, + "max": 1261 + }, + "symbols_by_start_year": { + "2021": 494, + "2022": 3, + "2023": 2, + "2024": 2, + "2025": 3, + "2026": 1 + }, + "note": "Where ticker counts drop in early years, the feed (or listing history) thins \u2014 do not treat those years as a full 505-name cross-section.", + "survivorship_banner": "SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only \u2014 not levels." + }, + "race_guard": null, + "harness": null, + "verdict": "COVERAGE_ONLY", + "verdict_detail": "Coverage probe only; run --phase harness after deep rebuild.", + "human_next": "- Compare sector residual vs market residual across eras.\n- If pre-2021 IC collapses, park Task 1 wire-in.\n- Do not retune production knobs on deep history levels.", + "report_path": "reports/history-depth-20260719-095156.json" +} diff --git a/reports/history-depth-20260719-095156.md b/reports/history-depth-20260719-095156.md new file mode 100644 index 0000000..a6edf52 --- /dev/null +++ b/reports/history-depth-20260719-095156.md @@ -0,0 +1,177 @@ +# History-depth extension (Tier-1 alpha research) + +**Status:** PRE-REGISTERED — run on MacBook (heavy I/O + full harness). +**Branch:** `research/history-depth-extension` (create from latest research stack). +**Production impact:** none. **Do not retune any production knob on deep history.** + +--- + +## Pre-registration (locked before rebuild) + +### Motivation + +All current conclusions rest on ~35 non-overlapping weekly windows in essentially +one post-2021 regime. Extending history toward max Alpaca daily-bar depth adds +the 2018 vol shock and full 2020 crash (where the feed allows). + +### Protocol + +1. **Empirical coverage first** — bars per calendar year per symbol; document + where the feed thins out. Do **not** assume a uniform start date. +2. **Rebuild the research snapshot completely** from prod source + max history + per symbol (`Adjustment.SPLIT`, ~200 req/min pacing via existing extender). +3. **Race guard (rule 6)** — refuse analysis until completion manifest is + `complete=true` and live counts match. +4. **Re-run full signal harness** (all existing signals incl. sector residual / + SUE if present) on the extended window. +5. **Report per signal:** mean IC, t, window count, and **era split** + (pre-/post-2021) — diagnostic only, **not a tuning input**. +6. **Log prominently:** survivorship bias grows with depth (today’s constituents + backfilled). Absolute Sharpe/CAGR on deep history is optimistic; payload is + **relative** signal comparisons and IC stability, not levels. +7. **Do not retune** production knobs. If a knob’s confirmation looks + overturned on deep history → report only; human decides. + +### Success / interpretation (not promotion of a new signal) + +| outcome | meaning | +|---|---| +| Sector residual still ≥ market residual on deep IC + stable sign | strengthens Task 1 PROMOTE case | +| Sector residual collapses pre-2021 | **PARK** Task 1 wire-in | +| SUE remains weak after full earnings + depth | **DEAD** SUE for this stack | +| Any production knob looks worse deep | report; no auto-retune | + +--- + +## MacBook runbook + +Prefer the bundled script (one entry point): + +```bash +git fetch origin && git checkout research/earnings-gap-and-sue +# .env: ALPACA_* required; FMP_* if resuming earnings +# copy backtest_snapshots/prod.sqlite if not already local + +chmod +x scripts/run_tier1_macbook.sh + +# Default: coverage → deep rebuild → harness (+ era split) +./scripts/run_tier1_macbook.sh + +# Optional variants +./scripts/run_tier1_macbook.sh --all # + earnings resume first +./scripts/run_tier1_macbook.sh --earnings-only # multi-day FMP + 2a/2b only +./scripts/run_tier1_macbook.sh --harness-only # skip rebuild +./scripts/run_tier1_macbook.sh --coverage-only + +# Tunables +WORKERS=12 HISTORY_DAYS=5000 ./scripts/run_tier1_macbook.sh +./scripts/run_tier1_macbook.sh --workers 12 --fmp-limit 250 +``` + +Then commit `reports/` + updated research docs, or copy them back to Windows. + +--- + +## Data provenance + +*(filled at run time)* + +--- + +## Results + +Generated: `2026-07-19T09:51:56.525634` + +> **SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only — not levels.** + +### Coverage + +```json +{ + "snapshot": "/Users/taathde3/git/lab/signal_platform/backtest_snapshots/prod.sqlite", + "ticker_count": 506, + "ohlcv_row_count": 629263, + "date_range": { + "min": "2021-06-24", + "max": "2026-07-02" + }, + "bars_per_year": [ + { + "year": "2021", + "bars": 65678, + "tickers_with_bars": 494 + }, + { + "year": "2022", + "bars": 124423, + "tickers_with_bars": 497 + }, + { + "year": "2023", + "bars": 124479, + "tickers_with_bars": 499 + }, + { + "year": "2024", + "bars": 126129, + "tickers_with_bars": 501 + }, + { + "year": "2025", + "bars": 125615, + "tickers_with_bars": 504 + }, + { + "year": "2026", + "bars": 62939, + "tickers_with_bars": 505 + } + ], + "bars_per_symbol": { + "min": 14, + "p10": 1261, + "p50": 1261, + "p90": 1261, + "max": 1261 + }, + "symbols_by_start_year": { + "2021": 494, + "2022": 3, + "2023": 2, + "2024": 2, + "2025": 3, + "2026": 1 + }, + "note": "Where ticker counts drop in early years, the feed (or listing history) thins \u2014 do not treat those years as a full 505-name cross-section.", + "survivorship_banner": "SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only \u2014 not levels." +} +``` + +### Race guard + +```json +{} +``` + +### Signal IC (full extended window) + +_Harness not run this pass._ + +### Era split (diagnostic only) + +_No era split._ + +## Verdict + +**COVERAGE_ONLY** + +Coverage probe only; run --phase harness after deep rebuild. + +## What a human must decide next + +- Compare sector residual vs market residual across eras. +- If pre-2021 IC collapses, park Task 1 wire-in. +- Do not retune production knobs on deep history levels. + +Artifacts: `reports/history-depth-20260719-095156.json` + diff --git a/reports/history-depth-20260719-103315.json b/reports/history-depth-20260719-103315.json new file mode 100644 index 0000000..5f4f66a --- /dev/null +++ b/reports/history-depth-20260719-103315.json @@ -0,0 +1,494 @@ +{ + "generated_at": "2026-07-19T10:33:15.322673", + "survivorship_banner": "SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only \u2014 not levels.", + "coverage": null, + "race_guard": { + "manifest": { + "schema_version": 1, + "snapshot": "research.sqlite", + "snapshot_resolved": "/Users/taathde3/git/lab/signal_platform/backtest_snapshots/research.sqlite", + "complete": true, + "finished_at": "2026-07-19T08:19:02.992206+00:00", + "ticker_count": 4655, + "ohlcv_row_count": 6609926, + "rank_only_count": 4149, + "sources": { + "nasdaq_all": "nasdaq_trader", + "sp500": "wikipedia_sp500" + }, + "history_days": 5000, + "min_bars": 260, + "fetch_ok": 4152, + "fetch_fail": 0, + "limit": null, + "extra": { + "prod_symbols_at_start": 506, + "pool_size": 4648, + "to_fetch": 4152 + }, + "live_counts": { + "ticker_count": 4655, + "ohlcv_row_count": 6609926, + "rank_only_count": 4149 + } + }, + "ok": true + }, + "harness": { + "survivorship_banner": "SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only \u2014 not levels.", + "signal_eval": [ + { + "signal": "high_52w", + "weeks": 84, + "avg_cross_section": 2280.3, + "mean_ic": 0.1111, + "ic_t_stat": 6.41, + "ic_positive_pct": 76.2, + "mean_quintile_spread": -13.5104, + "reliable": true + }, + { + "signal": "mom_12_1", + "weeks": 83, + "avg_cross_section": 2277.5, + "mean_ic": 0.0663, + "ic_t_stat": 5.11, + "ic_positive_pct": 74.7, + "mean_quintile_spread": -10.1818, + "reliable": true + }, + { + "signal": "mom_12_1_sector_resid", + "weeks": 35, + "avg_cross_section": 497.7, + "mean_ic": 0.0578, + "ic_t_stat": 2.34, + "ic_positive_pct": 65.7, + "mean_quintile_spread": 0.0245, + "reliable": true + }, + { + "signal": "trend_200", + "weeks": 85, + "avg_cross_section": 2308.1, + "mean_ic": 0.0546, + "ic_t_stat": 4.3, + "ic_positive_pct": 70.6, + "mean_quintile_spread": -12.4094, + "reliable": true + }, + { + "signal": "mom_6_1", + "weeks": 88, + "avg_cross_section": 2375.7, + "mean_ic": 0.0493, + "ic_t_stat": 4.91, + "ic_positive_pct": 70.5, + "mean_quintile_spread": 3.4224, + "reliable": true + }, + { + "signal": "mom_3_1", + "weeks": 90, + "avg_cross_section": 2425.6, + "mean_ic": 0.0363, + "ic_t_stat": 3.58, + "ic_positive_pct": 72.2, + "mean_quintile_spread": 0.9289, + "reliable": true + }, + { + "signal": "mom_12_1_sector_demeaned", + "weeks": 35, + "avg_cross_section": 496.7, + "mean_ic": 0.034, + "ic_t_stat": 1.32, + "ic_positive_pct": 62.9, + "mean_quintile_spread": 0.0154, + "reliable": true + }, + { + "signal": "fip_id", + "weeks": 83, + "avg_cross_section": 2277.5, + "mean_ic": 0.0267, + "ic_t_stat": 3.25, + "ic_positive_pct": 67.5, + "mean_quintile_spread": -0.0017, + "reliable": true + }, + { + "signal": "mom_12_1_resid", + "weeks": 83, + "avg_cross_section": 2277.5, + "mean_ic": 0.0256, + "ic_t_stat": 2.21, + "ic_positive_pct": 65.1, + "mean_quintile_spread": 10.1373, + "reliable": true + }, + { + "signal": "reversal_1m", + "weeks": 91, + "avg_cross_section": 2443.2, + "mean_ic": 0.003, + "ic_t_stat": 0.31, + "ic_positive_pct": 48.4, + "mean_quintile_spread": -7.0153, + "reliable": true + }, + { + "signal": "vol_6m", + "weeks": 88, + "avg_cross_section": 2375.7, + "mean_ic": -0.1226, + "ic_t_stat": -6.34, + "ic_positive_pct": 21.6, + "mean_quintile_spread": 1.4646, + "reliable": true + } + ], + "era_split": { + "era_split_date": "2021-01-01", + "note": "Diagnostic only \u2014 not a tuning input. Nested lookbacks are not OOS.", + "full": { + "high_52w": { + "signal": "high_52w", + "weeks": 84, + "avg_cross_section": 2280.3, + "mean_ic": 0.1111, + "ic_t_stat": 6.41, + "ic_positive_pct": 76.2, + "mean_quintile_spread": -13.5104, + "reliable": true + }, + "mom_12_1": { + "signal": "mom_12_1", + "weeks": 83, + "avg_cross_section": 2277.5, + "mean_ic": 0.0663, + "ic_t_stat": 5.11, + "ic_positive_pct": 74.7, + "mean_quintile_spread": -10.1818, + "reliable": true + }, + "mom_12_1_sector_resid": { + "signal": "mom_12_1_sector_resid", + "weeks": 35, + "avg_cross_section": 497.7, + "mean_ic": 0.0578, + "ic_t_stat": 2.34, + "ic_positive_pct": 65.7, + "mean_quintile_spread": 0.0245, + "reliable": true + }, + "trend_200": { + "signal": "trend_200", + "weeks": 85, + "avg_cross_section": 2308.1, + "mean_ic": 0.0546, + "ic_t_stat": 4.3, + "ic_positive_pct": 70.6, + "mean_quintile_spread": -12.4094, + "reliable": true + }, + "mom_6_1": { + "signal": "mom_6_1", + "weeks": 88, + "avg_cross_section": 2375.7, + "mean_ic": 0.0493, + "ic_t_stat": 4.91, + "ic_positive_pct": 70.5, + "mean_quintile_spread": 3.4224, + "reliable": true + }, + "mom_3_1": { + "signal": "mom_3_1", + "weeks": 90, + "avg_cross_section": 2425.6, + "mean_ic": 0.0363, + "ic_t_stat": 3.58, + "ic_positive_pct": 72.2, + "mean_quintile_spread": 0.9289, + "reliable": true + }, + "mom_12_1_sector_demeaned": { + "signal": "mom_12_1_sector_demeaned", + "weeks": 35, + "avg_cross_section": 496.7, + "mean_ic": 0.034, + "ic_t_stat": 1.32, + "ic_positive_pct": 62.9, + "mean_quintile_spread": 0.0154, + "reliable": true + }, + "fip_id": { + "signal": "fip_id", + "weeks": 83, + "avg_cross_section": 2277.5, + "mean_ic": 0.0267, + "ic_t_stat": 3.25, + "ic_positive_pct": 67.5, + "mean_quintile_spread": -0.0017, + "reliable": true + }, + "mom_12_1_resid": { + "signal": "mom_12_1_resid", + "weeks": 83, + "avg_cross_section": 2277.5, + "mean_ic": 0.0256, + "ic_t_stat": 2.21, + "ic_positive_pct": 65.1, + "mean_quintile_spread": 10.1373, + "reliable": true + }, + "reversal_1m": { + "signal": "reversal_1m", + "weeks": 91, + "avg_cross_section": 2443.2, + "mean_ic": 0.003, + "ic_t_stat": 0.31, + "ic_positive_pct": 48.4, + "mean_quintile_spread": -7.0153, + "reliable": true + }, + "vol_6m": { + "signal": "vol_6m", + "weeks": 88, + "avg_cross_section": 2375.7, + "mean_ic": -0.1226, + "ic_t_stat": -6.34, + "ic_positive_pct": 21.6, + "mean_quintile_spread": 1.4646, + "reliable": true + } + }, + "pre_2021": { + "high_52w": { + "signal": "high_52w", + "weeks": 36, + "avg_cross_section": 1422.2, + "mean_ic": 0.0494, + "ic_t_stat": 2.0, + "ic_positive_pct": 63.9, + "mean_quintile_spread": -31.5146, + "reliable": true + }, + "mom_12_1": { + "signal": "mom_12_1", + "weeks": 36, + "avg_cross_section": 1424.8, + "mean_ic": 0.0413, + "ic_t_stat": 2.94, + "ic_positive_pct": 66.7, + "mean_quintile_spread": -23.3935, + "reliable": true + }, + "trend_200": { + "signal": "trend_200", + "weeks": 38, + "avg_cross_section": 1440.9, + "mean_ic": 0.0322, + "ic_t_stat": 2.23, + "ic_positive_pct": 68.4, + "mean_quintile_spread": -27.5982, + "reliable": true + }, + "mom_12_1_resid": { + "signal": "mom_12_1_resid", + "weeks": 36, + "avg_cross_section": 1424.8, + "mean_ic": 0.0226, + "ic_t_stat": 1.68, + "ic_positive_pct": 69.4, + "mean_quintile_spread": 23.4097, + "reliable": true + }, + "mom_3_1": { + "signal": "mom_3_1", + "weeks": 42, + "avg_cross_section": 1480.0, + "mean_ic": 0.0217, + "ic_t_stat": 1.84, + "ic_positive_pct": 73.8, + "mean_quintile_spread": 2.0077, + "reliable": true + }, + "mom_6_1": { + "signal": "mom_6_1", + "weeks": 40, + "avg_cross_section": 1461.2, + "mean_ic": 0.0205, + "ic_t_stat": 1.63, + "ic_positive_pct": 65.0, + "mean_quintile_spread": 7.4512, + "reliable": true + }, + "fip_id": { + "signal": "fip_id", + "weeks": 36, + "avg_cross_section": 1424.8, + "mean_ic": 0.0116, + "ic_t_stat": 1.35, + "ic_positive_pct": 58.3, + "mean_quintile_spread": -0.0118, + "reliable": true + }, + "reversal_1m": { + "signal": "reversal_1m", + "weeks": 44, + "avg_cross_section": 1501.4, + "mean_ic": 0.0019, + "ic_t_stat": 0.15, + "ic_positive_pct": 50.0, + "mean_quintile_spread": -14.4299, + "reliable": true + }, + "vol_6m": { + "signal": "vol_6m", + "weeks": 40, + "avg_cross_section": 1461.2, + "mean_ic": -0.056, + "ic_t_stat": -2.16, + "ic_positive_pct": 35.0, + "mean_quintile_spread": 3.1335, + "reliable": true + } + }, + "post_2021": { + "high_52w": { + "signal": "high_52w", + "weeks": 48, + "avg_cross_section": 2914.5, + "mean_ic": 0.1375, + "ic_t_stat": 4.34, + "ic_positive_pct": 79.2, + "mean_quintile_spread": -0.0856, + "reliable": true + }, + "mom_12_1": { + "signal": "mom_12_1", + "weeks": 48, + "avg_cross_section": 2913.0, + "mean_ic": 0.0791, + "ic_t_stat": 3.64, + "ic_positive_pct": 77.1, + "mean_quintile_spread": -0.0858, + "reliable": true + }, + "mom_6_1": { + "signal": "mom_6_1", + "weeks": 48, + "avg_cross_section": 3122.6, + "mean_ic": 0.0779, + "ic_t_stat": 4.36, + "ic_positive_pct": 75.0, + "mean_quintile_spread": -0.0781, + "reliable": true + }, + "trend_200": { + "signal": "trend_200", + "weeks": 48, + "avg_cross_section": 3001.5, + "mean_ic": 0.0585, + "ic_t_stat": 2.68, + "ic_positive_pct": 72.9, + "mean_quintile_spread": -0.1129, + "reliable": true + }, + "mom_12_1_sector_resid": { + "signal": "mom_12_1_sector_resid", + "weeks": 35, + "avg_cross_section": 497.7, + "mean_ic": 0.0578, + "ic_t_stat": 2.34, + "ic_positive_pct": 65.7, + "mean_quintile_spread": 0.0245, + "reliable": true + }, + "fip_id": { + "signal": "fip_id", + "weeks": 48, + "avg_cross_section": 2913.0, + "mean_ic": 0.0366, + "ic_t_stat": 2.91, + "ic_positive_pct": 70.8, + "mean_quintile_spread": -0.0151, + "reliable": true + }, + "mom_12_1_sector_demeaned": { + "signal": "mom_12_1_sector_demeaned", + "weeks": 35, + "avg_cross_section": 496.7, + "mean_ic": 0.034, + "ic_t_stat": 1.32, + "ic_positive_pct": 62.9, + "mean_quintile_spread": 0.0154, + "reliable": true + }, + "mom_3_1": { + "signal": "mom_3_1", + "weeks": 48, + "avg_cross_section": 3234.2, + "mean_ic": 0.0291, + "ic_t_stat": 1.54, + "ic_positive_pct": 66.7, + "mean_quintile_spread": -0.0356, + "reliable": true + }, + "mom_12_1_resid": { + "signal": "mom_12_1_resid", + "weeks": 48, + "avg_cross_section": 2913.0, + "mean_ic": 0.0265, + "ic_t_stat": 1.37, + "ic_positive_pct": 66.7, + "mean_quintile_spread": -0.0405, + "reliable": true + }, + "reversal_1m": { + "signal": "reversal_1m", + "weeks": 48, + "avg_cross_section": 3315.8, + "mean_ic": -0.0126, + "ic_t_stat": -0.72, + "ic_positive_pct": 45.8, + "mean_quintile_spread": -0.0387, + "reliable": true + }, + "vol_6m": { + "signal": "vol_6m", + "weeks": 48, + "avg_cross_section": 3122.6, + "mean_ic": -0.1623, + "ic_t_stat": -4.94, + "ic_positive_pct": 22.9, + "mean_quintile_spread": 0.0059, + "reliable": true + } + } + }, + "params": { + "step_days": 5, + "step_sessions": 5, + "entry_cadence": "weekly", + "signal_eval_cadence": "weekly", + "horizon_days": 30, + "min_lookback": 60, + "cost_per_side_pct": 0.1, + "target_model": "production_gtl", + "target_model_label": "Live GTL (production)", + "is_production_target_model": true, + "production_reentry_policy": "gate_reset", + "liquid_breadth_top_n": null, + "liquid_min_price": null, + "signal_eval_only": true + }, + "tickers": 4655, + "generated_at_run": "2026-07-19T08:27:27.206022+00:00" + }, + "verdict": "PENDING_HUMAN", + "verdict_detail": "Harness complete \u2014 human interprets relative IC / era stability. No production retune from this artifact.", + "human_next": "- Compare sector residual vs market residual across eras.\n- If pre-2021 IC collapses, park Task 1 wire-in.\n- Do not retune production knobs on deep history levels.", + "report_path": "reports/history-depth-20260719-103315.json" +} diff --git a/reports/history-depth-20260719-103315.md b/reports/history-depth-20260719-103315.md new file mode 100644 index 0000000..79da918 --- /dev/null +++ b/reports/history-depth-20260719-103315.md @@ -0,0 +1,208 @@ +# History-depth extension (Tier-1 alpha research) + +**Status:** PRE-REGISTERED — run on MacBook (heavy I/O + full harness). +**Branch:** `research/history-depth-extension` (create from latest research stack). +**Production impact:** none. **Do not retune any production knob on deep history.** + +--- + +## Pre-registration (locked before rebuild) + +### Motivation + +All current conclusions rest on ~35 non-overlapping weekly windows in essentially +one post-2021 regime. Extending history toward max Alpaca daily-bar depth adds +the 2018 vol shock and full 2020 crash (where the feed allows). + +### Protocol + +1. **Empirical coverage first** — bars per calendar year per symbol; document + where the feed thins out. Do **not** assume a uniform start date. +2. **Rebuild the research snapshot completely** from prod source + max history + per symbol (`Adjustment.SPLIT`, ~200 req/min pacing via existing extender). +3. **Race guard (rule 6)** — refuse analysis until completion manifest is + `complete=true` and live counts match. +4. **Re-run full signal harness** (all existing signals incl. sector residual / + SUE if present) on the extended window. +5. **Report per signal:** mean IC, t, window count, and **era split** + (pre-/post-2021) — diagnostic only, **not a tuning input**. +6. **Log prominently:** survivorship bias grows with depth (today’s constituents + backfilled). Absolute Sharpe/CAGR on deep history is optimistic; payload is + **relative** signal comparisons and IC stability, not levels. +7. **Do not retune** production knobs. If a knob’s confirmation looks + overturned on deep history → report only; human decides. + +### Success / interpretation (not promotion of a new signal) + +| outcome | meaning | +|---|---| +| Sector residual still ≥ market residual on deep IC + stable sign | strengthens Task 1 PROMOTE case | +| Sector residual collapses pre-2021 | **PARK** Task 1 wire-in | +| SUE remains weak after full earnings + depth | **DEAD** SUE for this stack | +| Any production knob looks worse deep | report; no auto-retune | + +--- + +## MacBook runbook + +Prefer the bundled script (one entry point): + +```bash +git fetch origin && git checkout research/earnings-gap-and-sue +# .env: ALPACA_* required; FMP_* if resuming earnings +# copy backtest_snapshots/prod.sqlite if not already local + +chmod +x scripts/run_tier1_macbook.sh + +# Default: coverage → deep rebuild → harness (+ era split) +./scripts/run_tier1_macbook.sh + +# Optional variants +./scripts/run_tier1_macbook.sh --all # + earnings resume first +./scripts/run_tier1_macbook.sh --earnings-only # multi-day FMP + 2a/2b only +./scripts/run_tier1_macbook.sh --harness-only # skip rebuild +./scripts/run_tier1_macbook.sh --coverage-only + +# Tunables +WORKERS=12 HISTORY_DAYS=5000 ./scripts/run_tier1_macbook.sh +./scripts/run_tier1_macbook.sh --workers 12 --fmp-limit 250 +``` + +Then commit `reports/` + updated research docs, or copy them back to Windows. + +--- + +## Data provenance + +*(filled at run time)* + +--- + +## Results + +Generated: `2026-07-19T10:33:15.322673` + +> **SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only — not levels.** + +### Coverage + +```json +{} +``` + +### Race guard + +```json +{ + "manifest": { + "schema_version": 1, + "snapshot": "research.sqlite", + "snapshot_resolved": "/Users/taathde3/git/lab/signal_platform/backtest_snapshots/research.sqlite", + "complete": true, + "finished_at": "2026-07-19T08:19:02.992206+00:00", + "ticker_count": 4655, + "ohlcv_row_count": 6609926, + "rank_only_count": 4149, + "sources": { + "nasdaq_all": "nasdaq_trader", + "sp500": "wikipedia_sp500" + }, + "history_days": 5000, + "min_bars": 260, + "fetch_ok": 4152, + "fetch_fail": 0, + "limit": null, + "extra": { + "prod_symbols_at_start": 506, + "pool_size": 4648, + "to_fetch": 4152 + }, + "live_counts": { + "ticker_count": 4655, + "ohlcv_row_count": 6609926, + "rank_only_count": 4149 + } + }, + "ok": true +} +``` + +### Signal IC (full extended window) + +| signal | mean_ic | ic_t_stat | weeks | avg_N | reliable | +|---|---:|---:|---:|---:|---| +| high_52w | 0.1111 | 6.41 | 84 | 2280.3 | True | +| mom_12_1 | 0.0663 | 5.11 | 83 | 2277.5 | True | +| mom_12_1_sector_resid | 0.0578 | 2.34 | 35 | 497.7 | True | +| trend_200 | 0.0546 | 4.3 | 85 | 2308.1 | True | +| mom_6_1 | 0.0493 | 4.91 | 88 | 2375.7 | True | +| mom_3_1 | 0.0363 | 3.58 | 90 | 2425.6 | True | +| mom_12_1_sector_demeaned | 0.034 | 1.32 | 35 | 496.7 | True | +| fip_id | 0.0267 | 3.25 | 83 | 2277.5 | True | +| mom_12_1_resid | 0.0256 | 2.21 | 83 | 2277.5 | True | +| reversal_1m | 0.003 | 0.31 | 91 | 2443.2 | True | +| vol_6m | -0.1226 | -6.34 | 88 | 2375.7 | True | + +### Era split (diagnostic only) + +#### full + +| signal | mean_ic | t | weeks | N | +|---|---:|---:|---:|---:| +| fip_id | 0.0267 | 3.25 | 83 | 2277.5 | +| high_52w | 0.1111 | 6.41 | 84 | 2280.3 | +| mom_12_1 | 0.0663 | 5.11 | 83 | 2277.5 | +| mom_12_1_resid | 0.0256 | 2.21 | 83 | 2277.5 | +| mom_12_1_sector_demeaned | 0.034 | 1.32 | 35 | 496.7 | +| mom_12_1_sector_resid | 0.0578 | 2.34 | 35 | 497.7 | +| mom_3_1 | 0.0363 | 3.58 | 90 | 2425.6 | +| mom_6_1 | 0.0493 | 4.91 | 88 | 2375.7 | +| reversal_1m | 0.003 | 0.31 | 91 | 2443.2 | +| trend_200 | 0.0546 | 4.3 | 85 | 2308.1 | +| vol_6m | -0.1226 | -6.34 | 88 | 2375.7 | + +#### pre_2021 + +| signal | mean_ic | t | weeks | N | +|---|---:|---:|---:|---:| +| fip_id | 0.0116 | 1.35 | 36 | 1424.8 | +| high_52w | 0.0494 | 2.0 | 36 | 1422.2 | +| mom_12_1 | 0.0413 | 2.94 | 36 | 1424.8 | +| mom_12_1_resid | 0.0226 | 1.68 | 36 | 1424.8 | +| mom_3_1 | 0.0217 | 1.84 | 42 | 1480.0 | +| mom_6_1 | 0.0205 | 1.63 | 40 | 1461.2 | +| reversal_1m | 0.0019 | 0.15 | 44 | 1501.4 | +| trend_200 | 0.0322 | 2.23 | 38 | 1440.9 | +| vol_6m | -0.056 | -2.16 | 40 | 1461.2 | + +#### post_2021 + +| signal | mean_ic | t | weeks | N | +|---|---:|---:|---:|---:| +| fip_id | 0.0366 | 2.91 | 48 | 2913.0 | +| high_52w | 0.1375 | 4.34 | 48 | 2914.5 | +| mom_12_1 | 0.0791 | 3.64 | 48 | 2913.0 | +| mom_12_1_resid | 0.0265 | 1.37 | 48 | 2913.0 | +| mom_12_1_sector_demeaned | 0.034 | 1.32 | 35 | 496.7 | +| mom_12_1_sector_resid | 0.0578 | 2.34 | 35 | 497.7 | +| mom_3_1 | 0.0291 | 1.54 | 48 | 3234.2 | +| mom_6_1 | 0.0779 | 4.36 | 48 | 3122.6 | +| reversal_1m | -0.0126 | -0.72 | 48 | 3315.8 | +| trend_200 | 0.0585 | 2.68 | 48 | 3001.5 | +| vol_6m | -0.1623 | -4.94 | 48 | 3122.6 | + + +## Verdict + +**PENDING_HUMAN** + +Harness complete — human interprets relative IC / era stability. No production retune from this artifact. + +## What a human must decide next + +- Compare sector residual vs market residual across eras. +- If pre-2021 IC collapses, park Task 1 wire-in. +- Do not retune production knobs on deep history levels. + +Artifacts: `reports/history-depth-20260719-103315.json` + diff --git a/scripts/run_tier1_macbook.sh b/scripts/run_tier1_macbook.sh old mode 100644 new mode 100755 From 64761f38bac3c313f1dec6eac7b353f9e01c13e7 Mon Sep 17 00:00:00 2001 From: Dennis Thiessen Date: Sun, 19 Jul 2026 10:42:01 +0200 Subject: [PATCH 05/14] research: interpret history-depth MacBook harness (PARK sector residual wire-in) Authoritative report history-depth-20260719-103315: race guard pass on deep research.sqlite. Sector residual still short-window only (no pre-2021); fip sign flips on broad deep sample; no production retune. --- docs/research/history-depth-extension.md | 256 ++++++++++------------- reports/history-depth-20260719-103315.md | 256 ++++++++++------------- 2 files changed, 230 insertions(+), 282 deletions(-) diff --git a/docs/research/history-depth-extension.md b/docs/research/history-depth-extension.md index 79da918..522c57e 100644 --- a/docs/research/history-depth-extension.md +++ b/docs/research/history-depth-extension.md @@ -1,7 +1,8 @@ # History-depth extension (Tier-1 alpha research) -**Status:** PRE-REGISTERED — run on MacBook (heavy I/O + full harness). -**Branch:** `research/history-depth-extension` (create from latest research stack). +**Status:** **RUN COMPLETE — human interpretation below.** +**Branch:** `research/earnings-gap-and-sue` (MacBook commit `f6e0ca7`) +**Authoritative artifact:** `reports/history-depth-20260719-103315.json` **Production impact:** none. **Do not retune any production knob on deep history.** --- @@ -43,166 +44,139 @@ the 2018 vol shock and full 2020 crash (where the feed allows). --- -## MacBook runbook - -Prefer the bundled script (one entry point): - -```bash -git fetch origin && git checkout research/earnings-gap-and-sue -# .env: ALPACA_* required; FMP_* if resuming earnings -# copy backtest_snapshots/prod.sqlite if not already local - -chmod +x scripts/run_tier1_macbook.sh - -# Default: coverage → deep rebuild → harness (+ era split) -./scripts/run_tier1_macbook.sh - -# Optional variants -./scripts/run_tier1_macbook.sh --all # + earnings resume first -./scripts/run_tier1_macbook.sh --earnings-only # multi-day FMP + 2a/2b only -./scripts/run_tier1_macbook.sh --harness-only # skip rebuild -./scripts/run_tier1_macbook.sh --coverage-only - -# Tunables -WORKERS=12 HISTORY_DAYS=5000 ./scripts/run_tier1_macbook.sh -./scripts/run_tier1_macbook.sh --workers 12 --fmp-limit 250 -``` - -Then commit `reports/` + updated research docs, or copy them back to Windows. - ---- - ## Data provenance -*(filled at run time)* +| check | result | +|---|---| +| Snapshot | MacBook `research.sqlite` | +| Manifest `complete` | **true** (finished 2026-07-19T08:19Z) | +| Live counts match | yes — 4655 tickers / 6,609,926 OHLCV / 4149 rank_only | +| `history_days` | 5000 | +| fetch_ok / fail | 4152 / 0 | +| Race guard | **pass** | + +> **SURVIVORSHIP BIAS:** today’s constituents backfilled historically. Absolute +> Sharpe/CAGR levels on deep history are optimistic. Use **relative** signal IC +> comparisons and era stability only — not levels. + +**Coverage JSON was empty in the auto-written doc** (harness-only phase after +rebuild). Manifest is the race-guard source of truth for this run. + +Earlier MacBook files `history-depth-20260719-093853` … `095156` are intermediate +/ incomplete passes — **do not cite**. Only **103315** is authoritative. --- -## Results +## Results (authoritative: 103315) -Generated: `2026-07-19T10:33:15.322673` +### Full-window signal IC (broad research universe, deep bars) -> **SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only — not levels.** - -### Coverage - -```json -{} -``` - -### Race guard - -```json -{ - "manifest": { - "schema_version": 1, - "snapshot": "research.sqlite", - "snapshot_resolved": "/Users/taathde3/git/lab/signal_platform/backtest_snapshots/research.sqlite", - "complete": true, - "finished_at": "2026-07-19T08:19:02.992206+00:00", - "ticker_count": 4655, - "ohlcv_row_count": 6609926, - "rank_only_count": 4149, - "sources": { - "nasdaq_all": "nasdaq_trader", - "sp500": "wikipedia_sp500" - }, - "history_days": 5000, - "min_bars": 260, - "fetch_ok": 4152, - "fetch_fail": 0, - "limit": null, - "extra": { - "prod_symbols_at_start": 506, - "pool_size": 4648, - "to_fetch": 4152 - }, - "live_counts": { - "ticker_count": 4655, - "ohlcv_row_count": 6609926, - "rank_only_count": 4149 - } - }, - "ok": true -} -``` - -### Signal IC (full extended window) - -| signal | mean_ic | ic_t_stat | weeks | avg_N | reliable | +| signal | mean_ic | t | weeks | avg_N | notes | |---|---:|---:|---:|---:|---| -| high_52w | 0.1111 | 6.41 | 84 | 2280.3 | True | -| mom_12_1 | 0.0663 | 5.11 | 83 | 2277.5 | True | -| mom_12_1_sector_resid | 0.0578 | 2.34 | 35 | 497.7 | True | -| trend_200 | 0.0546 | 4.3 | 85 | 2308.1 | True | -| mom_6_1 | 0.0493 | 4.91 | 88 | 2375.7 | True | -| mom_3_1 | 0.0363 | 3.58 | 90 | 2425.6 | True | -| mom_12_1_sector_demeaned | 0.034 | 1.32 | 35 | 496.7 | True | -| fip_id | 0.0267 | 3.25 | 83 | 2277.5 | True | -| mom_12_1_resid | 0.0256 | 2.21 | 83 | 2277.5 | True | -| reversal_1m | 0.003 | 0.31 | 91 | 2443.2 | True | -| vol_6m | -0.1226 | -6.34 | 88 | 2375.7 | True | +| high_52w | **0.111** | **6.41** | 84 | 2280 | strong on deep breadth | +| mom_12_1 | **0.066** | **5.11** | 83 | 2278 | raw momentum strong | +| trend_200 | 0.055 | 4.30 | 85 | 2308 | | +| mom_6_1 | 0.049 | 4.91 | 88 | 2376 | | +| mom_3_1 | 0.036 | 3.58 | 90 | 2426 | | +| fip_id | **+0.027** | **3.25** | 83 | 2278 | **sign flip vs prod fingerprint** | +| mom_12_1_resid | 0.026 | 2.21 | 83 | 2278 | market residual still + but weaker than raw | +| reversal_1m | ~0 | 0.31 | 91 | 2443 | dead | +| vol_6m | **−0.123** | **−6.34** | 88 | 2376 | low-vol anomaly strong | +| mom_12_1_sector_resid | 0.058 | 2.34 | **35** | **498** | **not deep-sample — see caveats** | +| mom_12_1_sector_demeaned | 0.034 | 1.32 | **35** | **497** | same short fingerprint | -### Era split (diagnostic only) +### Era split (diagnostic only — not a tuning input) -#### full +| signal | pre-2021 IC / t / w / N | post-2021 IC / t / w / N | +|---|---|---| +| mom_12_1 | +0.041 / 2.94 / 36 / 1425 | +0.079 / 3.64 / 48 / 2913 | +| mom_12_1_resid | +0.023 / 1.68 / 36 / 1425 | +0.027 / 1.37 / 48 / 2913 | +| fip_id | +0.012 / 1.35 / 36 / 1425 | +0.037 / 2.91 / 48 / 2913 | +| vol_6m | −0.056 / −2.16 / 40 / 1461 | −0.162 / −4.94 / 48 / 3123 | +| high_52w | +0.049 / 2.0 / 36 / 1422 | +0.138 / 4.34 / 48 / 2915 | +| **sector_resid** | **absent** | 0.058 / 2.34 / 35 / 498 (short only) | +| **sector_demeaned** | **absent** | 0.034 / 1.32 / 35 / 497 (short only) | -| signal | mean_ic | t | weeks | N | -|---|---:|---:|---:|---:| -| fip_id | 0.0267 | 3.25 | 83 | 2277.5 | -| high_52w | 0.1111 | 6.41 | 84 | 2280.3 | -| mom_12_1 | 0.0663 | 5.11 | 83 | 2277.5 | -| mom_12_1_resid | 0.0256 | 2.21 | 83 | 2277.5 | -| mom_12_1_sector_demeaned | 0.034 | 1.32 | 35 | 496.7 | -| mom_12_1_sector_resid | 0.0578 | 2.34 | 35 | 497.7 | -| mom_3_1 | 0.0363 | 3.58 | 90 | 2425.6 | -| mom_6_1 | 0.0493 | 4.91 | 88 | 2375.7 | -| reversal_1m | 0.003 | 0.31 | 91 | 2443.2 | -| trend_200 | 0.0546 | 4.3 | 85 | 2308.1 | -| vol_6m | -0.1226 | -6.34 | 88 | 2375.7 | +--- -#### pre_2021 +## Critical caveats (must read) -| signal | mean_ic | t | weeks | N | -|---|---:|---:|---:|---:| -| fip_id | 0.0116 | 1.35 | 36 | 1424.8 | -| high_52w | 0.0494 | 2.0 | 36 | 1422.2 | -| mom_12_1 | 0.0413 | 2.94 | 36 | 1424.8 | -| mom_12_1_resid | 0.0226 | 1.68 | 36 | 1424.8 | -| mom_3_1 | 0.0217 | 1.84 | 42 | 1480.0 | -| mom_6_1 | 0.0205 | 1.63 | 40 | 1461.2 | -| reversal_1m | 0.0019 | 0.15 | 44 | 1501.4 | -| trend_200 | 0.0322 | 2.23 | 38 | 1440.9 | -| vol_6m | -0.056 | -2.16 | 40 | 1461.2 | +### 1. Sector residual did **not** get a deep-history stress test -#### post_2021 +`mom_12_1_sector_resid` / `_demeaned` still show **exactly** the Task‑1 short-window +fingerprint: **35 weeks, N≈498, IC 0.0578, t 2.34**. -| signal | mean_ic | t | weeks | N | -|---|---:|---:|---:|---:| -| fip_id | 0.0366 | 2.91 | 48 | 2913.0 | -| high_52w | 0.1375 | 4.34 | 48 | 2914.5 | -| mom_12_1 | 0.0791 | 3.64 | 48 | 2913.0 | -| mom_12_1_resid | 0.0265 | 1.37 | 48 | 2913.0 | -| mom_12_1_sector_demeaned | 0.034 | 1.32 | 35 | 496.7 | -| mom_12_1_sector_resid | 0.0578 | 2.34 | 35 | 497.7 | -| mom_3_1 | 0.0291 | 1.54 | 48 | 3234.2 | -| mom_6_1 | 0.0779 | 4.36 | 48 | 3122.6 | -| reversal_1m | -0.0126 | -0.72 | 48 | 3315.8 | -| trend_200 | 0.0585 | 2.68 | 48 | 3001.5 | -| vol_6m | -0.1623 | -4.94 | 48 | 3122.6 | +On the same run, raw `mom_12_1` has **83 weeks, N≈2278**. So depth worked for +price-only signals, but sector residual is still limited to the **~505 labeled +prod names × short factor calendar** (sector map only covers prod; and/or sector +ETF / two-factor path did not extend usable residual weeks). +**Pre-registered rule:** “Sector residual collapses pre-2021 → PARK Task 1 +wire-in.” Pre-2021 sector residual is **absent** from the era table. That is a +**PARK**, not a confirmation of the short-window PROMOTE. -## Verdict +Do **not** claim “sector residual beats market residual on deep history” from +this table — the two rows are **not the same cross-section or window count**. -**PENDING_HUMAN** +### 2. `fip_id` sign flips vs production fingerprint -Harness complete — human interprets relative IC / era stability. No production retune from this artifact. +| sample | fip mean IC | t | +|---|---:|---:| +| Prod 505, ~5y (fingerprint) | **−0.045** | −2.91 | +| Research breadth, deep (this run) | **+0.027** | +3.25 | + +This does **not** authorize resurrecting unconditional FIP as a book filter. It +confirms earlier Phase‑B caution: FIP edge is **universe- and sample-dependent**. +Production display card can stay context-only. Nested lookbacks still not OOS. + +### 3. Market residual vs raw momentum on deep breadth + +On deep broad IC, **raw 12‑1 (0.066 / t 5.1) ≫ market residual (0.026 / t 2.2)**. +That does **not** by itself overturn production residual ranking (book A/B was +on 505 + GTL gate, not pure factor IC), but it is a yellow flag for “residual is +always the better rank key” stories on broad history. **No auto-retune.** + +### 4. Low-vol anomaly is the cleanest deep-history result + +`vol_6m` IC −0.12 / t −6.3 full; stronger post-2021. Consistent sign across eras. +Production already blends **high**-vol (not low-vol) into the 80/20 rank — this +report does not change that without a separate A/B. Flag for human awareness only. + +--- + +## Verdicts (vs pre-registration) + +| question | verdict | +|---|---| +| Task 1 sector residual wire-in | **PARK** — no pre-2021 sector residual; deep-sample IC not established; short-window PROMOTE stays “human design only,” **not strengthened** by this run | +| Sector demean | still **DEAD** for promotion (t 1.32, short only) | +| SUE | **not re-scored here** (no `sue_latest` in harness table) — leave Task 2 **PARK** until full earnings backfill | +| fip unconditional book filter | remains **rejected / parked** despite sign flip on broad deep sample | +| Production residual / 80/20 / trail knobs | **no retune** from this report | +| Overall Task 3 | **COMPLETE as diagnostic** — payload is relative IC + caveats above | + +--- ## What a human must decide next -- Compare sector residual vs market residual across eras. -- If pre-2021 IC collapses, park Task 1 wire-in. -- Do not retune production knobs on deep history levels. +1. **Sector residual:** keep research-only until either + (a) sector ETF + sector map cover the full deep window **and** IC is re-run + with weeks ≫ 35 on a documented universe, or + (b) explicitly accept short-window-only evidence (weaker case). +2. **Do not** merge sector residual into production from this depth run. +3. **Do not** retune residual vs raw, FIP, or vol blend from these IC tables + without a pre-registered book A/B on the intended universe. +4. Optional follow-up: extend sector ETF history + sector labels to nasdaq_all, + re-run **only** sector residual IC on deep research.sqlite with race guard. +5. Optional: finish earnings backfill (48→506) and re-run SUE; depth alone did + not include SUE. -Artifacts: `reports/history-depth-20260719-103315.json` +--- +## Artifacts + +| file | role | +|---|---| +| `reports/history-depth-20260719-103315.json` | **authoritative** | +| `reports/history-depth-20260719-103315.md` | companion dump | +| `reports/history-depth-20260719-093853` … `095156` | **ignore** (partial) | diff --git a/reports/history-depth-20260719-103315.md b/reports/history-depth-20260719-103315.md index 79da918..522c57e 100644 --- a/reports/history-depth-20260719-103315.md +++ b/reports/history-depth-20260719-103315.md @@ -1,7 +1,8 @@ # History-depth extension (Tier-1 alpha research) -**Status:** PRE-REGISTERED — run on MacBook (heavy I/O + full harness). -**Branch:** `research/history-depth-extension` (create from latest research stack). +**Status:** **RUN COMPLETE — human interpretation below.** +**Branch:** `research/earnings-gap-and-sue` (MacBook commit `f6e0ca7`) +**Authoritative artifact:** `reports/history-depth-20260719-103315.json` **Production impact:** none. **Do not retune any production knob on deep history.** --- @@ -43,166 +44,139 @@ the 2018 vol shock and full 2020 crash (where the feed allows). --- -## MacBook runbook - -Prefer the bundled script (one entry point): - -```bash -git fetch origin && git checkout research/earnings-gap-and-sue -# .env: ALPACA_* required; FMP_* if resuming earnings -# copy backtest_snapshots/prod.sqlite if not already local - -chmod +x scripts/run_tier1_macbook.sh - -# Default: coverage → deep rebuild → harness (+ era split) -./scripts/run_tier1_macbook.sh - -# Optional variants -./scripts/run_tier1_macbook.sh --all # + earnings resume first -./scripts/run_tier1_macbook.sh --earnings-only # multi-day FMP + 2a/2b only -./scripts/run_tier1_macbook.sh --harness-only # skip rebuild -./scripts/run_tier1_macbook.sh --coverage-only - -# Tunables -WORKERS=12 HISTORY_DAYS=5000 ./scripts/run_tier1_macbook.sh -./scripts/run_tier1_macbook.sh --workers 12 --fmp-limit 250 -``` - -Then commit `reports/` + updated research docs, or copy them back to Windows. - ---- - ## Data provenance -*(filled at run time)* +| check | result | +|---|---| +| Snapshot | MacBook `research.sqlite` | +| Manifest `complete` | **true** (finished 2026-07-19T08:19Z) | +| Live counts match | yes — 4655 tickers / 6,609,926 OHLCV / 4149 rank_only | +| `history_days` | 5000 | +| fetch_ok / fail | 4152 / 0 | +| Race guard | **pass** | + +> **SURVIVORSHIP BIAS:** today’s constituents backfilled historically. Absolute +> Sharpe/CAGR levels on deep history are optimistic. Use **relative** signal IC +> comparisons and era stability only — not levels. + +**Coverage JSON was empty in the auto-written doc** (harness-only phase after +rebuild). Manifest is the race-guard source of truth for this run. + +Earlier MacBook files `history-depth-20260719-093853` … `095156` are intermediate +/ incomplete passes — **do not cite**. Only **103315** is authoritative. --- -## Results +## Results (authoritative: 103315) -Generated: `2026-07-19T10:33:15.322673` +### Full-window signal IC (broad research universe, deep bars) -> **SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only — not levels.** - -### Coverage - -```json -{} -``` - -### Race guard - -```json -{ - "manifest": { - "schema_version": 1, - "snapshot": "research.sqlite", - "snapshot_resolved": "/Users/taathde3/git/lab/signal_platform/backtest_snapshots/research.sqlite", - "complete": true, - "finished_at": "2026-07-19T08:19:02.992206+00:00", - "ticker_count": 4655, - "ohlcv_row_count": 6609926, - "rank_only_count": 4149, - "sources": { - "nasdaq_all": "nasdaq_trader", - "sp500": "wikipedia_sp500" - }, - "history_days": 5000, - "min_bars": 260, - "fetch_ok": 4152, - "fetch_fail": 0, - "limit": null, - "extra": { - "prod_symbols_at_start": 506, - "pool_size": 4648, - "to_fetch": 4152 - }, - "live_counts": { - "ticker_count": 4655, - "ohlcv_row_count": 6609926, - "rank_only_count": 4149 - } - }, - "ok": true -} -``` - -### Signal IC (full extended window) - -| signal | mean_ic | ic_t_stat | weeks | avg_N | reliable | +| signal | mean_ic | t | weeks | avg_N | notes | |---|---:|---:|---:|---:|---| -| high_52w | 0.1111 | 6.41 | 84 | 2280.3 | True | -| mom_12_1 | 0.0663 | 5.11 | 83 | 2277.5 | True | -| mom_12_1_sector_resid | 0.0578 | 2.34 | 35 | 497.7 | True | -| trend_200 | 0.0546 | 4.3 | 85 | 2308.1 | True | -| mom_6_1 | 0.0493 | 4.91 | 88 | 2375.7 | True | -| mom_3_1 | 0.0363 | 3.58 | 90 | 2425.6 | True | -| mom_12_1_sector_demeaned | 0.034 | 1.32 | 35 | 496.7 | True | -| fip_id | 0.0267 | 3.25 | 83 | 2277.5 | True | -| mom_12_1_resid | 0.0256 | 2.21 | 83 | 2277.5 | True | -| reversal_1m | 0.003 | 0.31 | 91 | 2443.2 | True | -| vol_6m | -0.1226 | -6.34 | 88 | 2375.7 | True | +| high_52w | **0.111** | **6.41** | 84 | 2280 | strong on deep breadth | +| mom_12_1 | **0.066** | **5.11** | 83 | 2278 | raw momentum strong | +| trend_200 | 0.055 | 4.30 | 85 | 2308 | | +| mom_6_1 | 0.049 | 4.91 | 88 | 2376 | | +| mom_3_1 | 0.036 | 3.58 | 90 | 2426 | | +| fip_id | **+0.027** | **3.25** | 83 | 2278 | **sign flip vs prod fingerprint** | +| mom_12_1_resid | 0.026 | 2.21 | 83 | 2278 | market residual still + but weaker than raw | +| reversal_1m | ~0 | 0.31 | 91 | 2443 | dead | +| vol_6m | **−0.123** | **−6.34** | 88 | 2376 | low-vol anomaly strong | +| mom_12_1_sector_resid | 0.058 | 2.34 | **35** | **498** | **not deep-sample — see caveats** | +| mom_12_1_sector_demeaned | 0.034 | 1.32 | **35** | **497** | same short fingerprint | -### Era split (diagnostic only) +### Era split (diagnostic only — not a tuning input) -#### full +| signal | pre-2021 IC / t / w / N | post-2021 IC / t / w / N | +|---|---|---| +| mom_12_1 | +0.041 / 2.94 / 36 / 1425 | +0.079 / 3.64 / 48 / 2913 | +| mom_12_1_resid | +0.023 / 1.68 / 36 / 1425 | +0.027 / 1.37 / 48 / 2913 | +| fip_id | +0.012 / 1.35 / 36 / 1425 | +0.037 / 2.91 / 48 / 2913 | +| vol_6m | −0.056 / −2.16 / 40 / 1461 | −0.162 / −4.94 / 48 / 3123 | +| high_52w | +0.049 / 2.0 / 36 / 1422 | +0.138 / 4.34 / 48 / 2915 | +| **sector_resid** | **absent** | 0.058 / 2.34 / 35 / 498 (short only) | +| **sector_demeaned** | **absent** | 0.034 / 1.32 / 35 / 497 (short only) | -| signal | mean_ic | t | weeks | N | -|---|---:|---:|---:|---:| -| fip_id | 0.0267 | 3.25 | 83 | 2277.5 | -| high_52w | 0.1111 | 6.41 | 84 | 2280.3 | -| mom_12_1 | 0.0663 | 5.11 | 83 | 2277.5 | -| mom_12_1_resid | 0.0256 | 2.21 | 83 | 2277.5 | -| mom_12_1_sector_demeaned | 0.034 | 1.32 | 35 | 496.7 | -| mom_12_1_sector_resid | 0.0578 | 2.34 | 35 | 497.7 | -| mom_3_1 | 0.0363 | 3.58 | 90 | 2425.6 | -| mom_6_1 | 0.0493 | 4.91 | 88 | 2375.7 | -| reversal_1m | 0.003 | 0.31 | 91 | 2443.2 | -| trend_200 | 0.0546 | 4.3 | 85 | 2308.1 | -| vol_6m | -0.1226 | -6.34 | 88 | 2375.7 | +--- -#### pre_2021 +## Critical caveats (must read) -| signal | mean_ic | t | weeks | N | -|---|---:|---:|---:|---:| -| fip_id | 0.0116 | 1.35 | 36 | 1424.8 | -| high_52w | 0.0494 | 2.0 | 36 | 1422.2 | -| mom_12_1 | 0.0413 | 2.94 | 36 | 1424.8 | -| mom_12_1_resid | 0.0226 | 1.68 | 36 | 1424.8 | -| mom_3_1 | 0.0217 | 1.84 | 42 | 1480.0 | -| mom_6_1 | 0.0205 | 1.63 | 40 | 1461.2 | -| reversal_1m | 0.0019 | 0.15 | 44 | 1501.4 | -| trend_200 | 0.0322 | 2.23 | 38 | 1440.9 | -| vol_6m | -0.056 | -2.16 | 40 | 1461.2 | +### 1. Sector residual did **not** get a deep-history stress test -#### post_2021 +`mom_12_1_sector_resid` / `_demeaned` still show **exactly** the Task‑1 short-window +fingerprint: **35 weeks, N≈498, IC 0.0578, t 2.34**. -| signal | mean_ic | t | weeks | N | -|---|---:|---:|---:|---:| -| fip_id | 0.0366 | 2.91 | 48 | 2913.0 | -| high_52w | 0.1375 | 4.34 | 48 | 2914.5 | -| mom_12_1 | 0.0791 | 3.64 | 48 | 2913.0 | -| mom_12_1_resid | 0.0265 | 1.37 | 48 | 2913.0 | -| mom_12_1_sector_demeaned | 0.034 | 1.32 | 35 | 496.7 | -| mom_12_1_sector_resid | 0.0578 | 2.34 | 35 | 497.7 | -| mom_3_1 | 0.0291 | 1.54 | 48 | 3234.2 | -| mom_6_1 | 0.0779 | 4.36 | 48 | 3122.6 | -| reversal_1m | -0.0126 | -0.72 | 48 | 3315.8 | -| trend_200 | 0.0585 | 2.68 | 48 | 3001.5 | -| vol_6m | -0.1623 | -4.94 | 48 | 3122.6 | +On the same run, raw `mom_12_1` has **83 weeks, N≈2278**. So depth worked for +price-only signals, but sector residual is still limited to the **~505 labeled +prod names × short factor calendar** (sector map only covers prod; and/or sector +ETF / two-factor path did not extend usable residual weeks). +**Pre-registered rule:** “Sector residual collapses pre-2021 → PARK Task 1 +wire-in.” Pre-2021 sector residual is **absent** from the era table. That is a +**PARK**, not a confirmation of the short-window PROMOTE. -## Verdict +Do **not** claim “sector residual beats market residual on deep history” from +this table — the two rows are **not the same cross-section or window count**. -**PENDING_HUMAN** +### 2. `fip_id` sign flips vs production fingerprint -Harness complete — human interprets relative IC / era stability. No production retune from this artifact. +| sample | fip mean IC | t | +|---|---:|---:| +| Prod 505, ~5y (fingerprint) | **−0.045** | −2.91 | +| Research breadth, deep (this run) | **+0.027** | +3.25 | + +This does **not** authorize resurrecting unconditional FIP as a book filter. It +confirms earlier Phase‑B caution: FIP edge is **universe- and sample-dependent**. +Production display card can stay context-only. Nested lookbacks still not OOS. + +### 3. Market residual vs raw momentum on deep breadth + +On deep broad IC, **raw 12‑1 (0.066 / t 5.1) ≫ market residual (0.026 / t 2.2)**. +That does **not** by itself overturn production residual ranking (book A/B was +on 505 + GTL gate, not pure factor IC), but it is a yellow flag for “residual is +always the better rank key” stories on broad history. **No auto-retune.** + +### 4. Low-vol anomaly is the cleanest deep-history result + +`vol_6m` IC −0.12 / t −6.3 full; stronger post-2021. Consistent sign across eras. +Production already blends **high**-vol (not low-vol) into the 80/20 rank — this +report does not change that without a separate A/B. Flag for human awareness only. + +--- + +## Verdicts (vs pre-registration) + +| question | verdict | +|---|---| +| Task 1 sector residual wire-in | **PARK** — no pre-2021 sector residual; deep-sample IC not established; short-window PROMOTE stays “human design only,” **not strengthened** by this run | +| Sector demean | still **DEAD** for promotion (t 1.32, short only) | +| SUE | **not re-scored here** (no `sue_latest` in harness table) — leave Task 2 **PARK** until full earnings backfill | +| fip unconditional book filter | remains **rejected / parked** despite sign flip on broad deep sample | +| Production residual / 80/20 / trail knobs | **no retune** from this report | +| Overall Task 3 | **COMPLETE as diagnostic** — payload is relative IC + caveats above | + +--- ## What a human must decide next -- Compare sector residual vs market residual across eras. -- If pre-2021 IC collapses, park Task 1 wire-in. -- Do not retune production knobs on deep history levels. +1. **Sector residual:** keep research-only until either + (a) sector ETF + sector map cover the full deep window **and** IC is re-run + with weeks ≫ 35 on a documented universe, or + (b) explicitly accept short-window-only evidence (weaker case). +2. **Do not** merge sector residual into production from this depth run. +3. **Do not** retune residual vs raw, FIP, or vol blend from these IC tables + without a pre-registered book A/B on the intended universe. +4. Optional follow-up: extend sector ETF history + sector labels to nasdaq_all, + re-run **only** sector residual IC on deep research.sqlite with race guard. +5. Optional: finish earnings backfill (48→506) and re-run SUE; depth alone did + not include SUE. -Artifacts: `reports/history-depth-20260719-103315.json` +--- +## Artifacts + +| file | role | +|---|---| +| `reports/history-depth-20260719-103315.json` | **authoritative** | +| `reports/history-depth-20260719-103315.md` | companion dump | +| `reports/history-depth-20260719-093853` … `095156` | **ignore** (partial) | From a9841d92b7d9aa1f506bfd18b462999e0dfb0234 Mon Sep 17 00:00:00 2001 From: Dennis Thiessen Date: Sun, 19 Jul 2026 10:58:03 +0200 Subject: [PATCH 06/14] research: sector-resid deep test (deepen shallow + one masked PASS/FAIL) Terminal follow-up for Task 1: detect/refetch shallow two-tier symbols and sector ETFs at 5000d, regenerate manifest, run ONE liquid-1500 harness with era split, grade mom_12_1_sector_resid mechanically. Bundled as run_tier1_macbook.sh --sector-resid-deep. --- scripts/run_sector_resid_deep_test.py | 998 ++++++++++++++++++++++++++ scripts/run_tier1_macbook.sh | 19 +- 2 files changed, 1016 insertions(+), 1 deletion(-) create mode 100644 scripts/run_sector_resid_deep_test.py diff --git a/scripts/run_sector_resid_deep_test.py b/scripts/run_sector_resid_deep_test.py new file mode 100644 index 0000000..54de8bf --- /dev/null +++ b/scripts/run_sector_resid_deep_test.py @@ -0,0 +1,998 @@ +#!/usr/bin/env python3 +"""Terminal sector-residual deep test: deepen shallow symbols → ONE masked run → PASS/FAIL. + +Repairs the two-tier history-depth defect (prod/ETF names left at ~5y while breadth +got 5000d), then runs a single liquid-breadth signal harness and grades +``mom_12_1_sector_resid`` against the pre-registered rule. + +Local research only. No production changes. + +MacBook +------- + # On deep research.sqlite from the prior history-depth rebuild: + python scripts/run_sector_resid_deep_test.py \\ + --snapshot backtest_snapshots/research.sqlite \\ + --workers 8 --allow-spawn + + # Skip re-fetch if Step-1 already done and sanity-check passes: + python scripts/run_sector_resid_deep_test.py --skip-deepen --workers 8 --allow-spawn +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import math +import os +import sys +import time +from collections import defaultdict +from datetime import date, datetime, timedelta, timezone +from pathlib import Path +from typing import Any + +from sqlalchemy import create_engine, text +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from app.ssl_bootstrap import bootstrap_ssl # noqa: E402 + +bootstrap_ssl() + +from app.services.sector_map import ( # noqa: E402 + DEFAULT_SECTOR_MAP_PATH, + SECTOR_ETFS, + load_ticker_sector_map, +) +from scripts.research_snapshot_manifest import ( # noqa: E402 + assert_research_snapshot_complete, + clear_manifest, + write_completion_manifest, +) + +ERA_SPLIT = date(2021, 1, 1) +IRON_IC = 0.03 +# "weeks ≫ 35 (expect ~80)" — mechanical floor for "data fix worked" +MIN_WEEKS_DEEP = 50 +SANITY_MEGACAPS = ("AAPL", "MSFT", "JPM", "XOM", "JNJ") +SURVIVORSHIP = ( + "SURVIVORSHIP BIAS: today's constituents backfilled. Relative IC only — not levels." +) + + +def _parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--snapshot", default="backtest_snapshots/research.sqlite") + p.add_argument("--history-days", type=int, default=5000) + p.add_argument("--sleep", type=float, default=0.15) + p.add_argument("--workers", type=int, default=8) + p.add_argument("--allow-spawn", action="store_true") + p.add_argument( + "--skip-deepen", + action="store_true", + help="Skip Step-1 re-fetch; only sanity-check + harness.", + ) + p.add_argument( + "--sector-map", + default=str(DEFAULT_SECTOR_MAP_PATH), + ) + p.add_argument("--liquid-breadth", type=int, default=1500) + p.add_argument("--min-price", type=float, default=5.0) + p.add_argument("--quiet", action="store_true") + p.add_argument("--out", default=None) + return p.parse_args() + + +def _sqlite_url(path: Path) -> str: + return f"sqlite+aiosqlite:///{path.resolve().as_posix()}" + + +def _symbol_depth(snapshot: Path) -> list[dict[str, Any]]: + engine = create_engine( + f"sqlite:///{snapshot.resolve().as_posix()}", + future=True, + ) + try: + with engine.connect() as conn: + rows = conn.execute( + text( + """ + SELECT t.symbol, COUNT(*) AS n, MIN(o.date), MAX(o.date) + FROM ohlcv_records o + JOIN tickers t ON t.id = o.ticker_id + GROUP BY t.symbol + """ + ) + ).fetchall() + finally: + engine.dispose() + out = [] + for sym, n, d0, d1 in rows: + out.append({ + "symbol": str(sym).upper(), + "bars": int(n), + "min_date": str(d0)[:10] if d0 else None, + "max_date": str(d1)[:10] if d1 else None, + }) + return out + + +def _derive_shallow( + depths: list[dict[str, Any]], + *, + lag_days: int = 400, +) -> tuple[list[str], dict[str, Any]]: + """Symbols whose earliest bar starts materially later than the deep cohort.""" + starts: list[tuple[str, date]] = [] + for row in depths: + if not row.get("min_date"): + continue + starts.append((row["symbol"], date.fromisoformat(row["min_date"]))) + if not starts: + return [], {"error": "no symbols with min_date"} + + # Deep cohort start ≈ 10th percentile of earliest dates (early = deep). + ordered = sorted(d for _, d in starts) + p10 = ordered[max(0, int(0.10 * (len(ordered) - 1)))] + cutoff = p10 + timedelta(days=lag_days) + shallow = sorted({sym for sym, d in starts if d > cutoff}) + meta = { + "n_symbols": len(starts), + "deep_cohort_p10_start": p10.isoformat(), + "shallow_cutoff": cutoff.isoformat(), + "lag_days": lag_days, + "n_shallow": len(shallow), + "shallow_start_histogram": _year_hist( + [d for sym, d in starts if sym in set(shallow)] + ), + "deep_start_histogram": _year_hist( + [d for sym, d in starts if sym not in set(shallow)] + ), + "shallow_sample": shallow[:30], + } + return shallow, meta + + +def _year_hist(dates: list[date]) -> dict[str, int]: + h: dict[str, int] = defaultdict(int) + for d in dates: + h[str(d.year)] += 1 + return dict(sorted(h.items())) + + +async def _fetch_and_replace_ohlcv( + engine, + provider, + symbol: str, + start: date, + end: date, + *, + sleep_s: float, + max_retries: int = 5, +) -> int: + from app.exceptions import ProviderError, RateLimitError + + bars = [] + for attempt in range(max_retries): + try: + bars = await provider.fetch_ohlcv(symbol, start, end) + break + except RateLimitError: + wait = min(60.0, 2.0 ** attempt) + print(f" rate limited {symbol}; sleep {wait:.0f}s") + await asyncio.sleep(wait) + except ProviderError as exc: + if attempt + 1 >= max_retries: + raise + await asyncio.sleep(1.0) + _ = exc + if sleep_s > 0: + await asyncio.sleep(sleep_s) + if not bars: + return 0 + + with engine.begin() as write: + tid = write.execute( + text("SELECT id FROM tickers WHERE symbol = :s"), + {"s": symbol}, + ).scalar_one_or_none() + if tid is None: + write.execute( + text( + "INSERT INTO tickers (symbol, name, created_at) " + "VALUES (:s, NULL, :c)" + ), + {"s": symbol, "c": datetime.now(timezone.utc).isoformat()}, + ) + tid = write.execute( + text("SELECT id FROM tickers WHERE symbol = :s"), + {"s": symbol}, + ).scalar_one() + # Full replace for this symbol so shallow tails cannot linger. + write.execute( + text("DELETE FROM ohlcv_records WHERE ticker_id = :tid"), + {"tid": int(tid)}, + ) + now = datetime.now(timezone.utc).replace(tzinfo=None).isoformat() + write.execute( + text( + """ + INSERT INTO ohlcv_records + (ticker_id, date, open, high, low, close, volume, created_at) + VALUES + (:ticker_id, :date, :open, :high, :low, :close, :volume, :created_at) + """ + ), + [ + { + "ticker_id": int(tid), + "date": b.date.isoformat() + if hasattr(b.date, "isoformat") + else str(b.date), + "open": float(b.open), + "high": float(b.high), + "low": float(b.low), + "close": float(b.close), + "volume": int(b.volume), + "created_at": now, + } + for b in bars + ], + ) + return len(bars) + + +async def _deepen_sector_etfs( + snapshot: Path, *, history_days: int, sleep_s: float +) -> dict[str, Any]: + """Refresh SPY + 11 sector ETFs in benchmark_prices to full depth.""" + # Reuse the existing CLI helper for consistency. + from scripts.fetch_sector_etfs_to_snapshot import _fetch_and_upsert + from app.config import settings + from app.providers.alpaca import AlpacaOHLCVProvider + + if not settings.alpaca_api_key or not settings.alpaca_api_secret: + raise SystemExit("ALPACA keys required to deepen sector ETFs") + + provider = AlpacaOHLCVProvider(settings.alpaca_api_key, settings.alpaca_api_secret) + end = date.today() + start = end - timedelta(days=history_days) + engine = create_engine( + f"sqlite:///{snapshot.resolve().as_posix()}", + future=True, + ) + symbols = ["SPY", *SECTOR_ETFS] + written: dict[str, int] = {} + try: + for sym in symbols: + n = await _fetch_and_upsert( + engine, provider, sym, start, end, sleep_s=sleep_s + ) + written[sym] = n + finally: + engine.dispose() + + engine = create_engine( + f"sqlite:///{snapshot.resolve().as_posix()}", + future=True, + ) + try: + with engine.connect() as conn: + rows = conn.execute( + text( + "SELECT symbol, COUNT(*), MIN(date), MAX(date) " + "FROM benchmark_prices GROUP BY symbol ORDER BY symbol" + ) + ).fetchall() + finally: + engine.dispose() + return { + "written": written, + "benchmark_summary": [ + {"symbol": s, "n": n, "min": d0, "max": d1} for s, n, d0, d1 in rows + ], + } + + +def _sanity_check(snapshot: Path, *, history_days: int) -> dict[str, Any]: + depths = {r["symbol"]: r for r in _symbol_depth(snapshot)} + end = date.today() + target_start = end - timedelta(days=history_days) + # Allow ~1 year slack for IPO/listing limits (not a hard fail for all names). + megacap_deadline = target_start + timedelta(days=400) + + megacap = {} + ok_mega = True + for sym in SANITY_MEGACAPS: + row = depths.get(sym) + megacap[sym] = row + if row is None or not row.get("min_date"): + ok_mega = False + continue + if date.fromisoformat(row["min_date"]) > megacap_deadline: + ok_mega = False + + engine = create_engine( + f"sqlite:///{snapshot.resolve().as_posix()}", + future=True, + ) + try: + with engine.connect() as conn: + etf_rows = conn.execute( + text( + "SELECT symbol, COUNT(*), MIN(date), MAX(date) " + "FROM benchmark_prices WHERE symbol IN " + f"({','.join(repr(s) for s in SECTOR_ETFS)}) " + "GROUP BY symbol" + ) + ).fetchall() + finally: + engine.dispose() + + etf_info = { + s: {"n": n, "min": d0, "max": d1} for s, n, d0, d1 in etf_rows + } + deep_etfs = 0 + for sym in SECTOR_ETFS: + info = etf_info.get(sym) + if not info or not info["min"]: + continue + # XLC lists mid-2018 — accept that floor. + floor = date(2018, 6, 1) if sym == "XLC" else megacap_deadline + if date.fromisoformat(str(info["min"])[:10]) <= floor + timedelta(days=60): + deep_etfs += 1 + elif date.fromisoformat(str(info["min"])[:10]) <= date(2019, 1, 1): + # moderately deep still counts for non-XLC if near 2018-19 + if sym != "XLC": + deep_etfs += 1 + + # Require 10 of 11 sector ETFs deep (XLC may be the exception with mid-2018 start). + ok_etf = deep_etfs >= 10 + + # Shallow residual after deepen: few names should still start after 2020. + still_shallow, _ = _derive_shallow(list(depths.values()), lag_days=400) + # After fix, shallow set should shrink dramatically vs ~500. + note_xlc = ( + "XLC lists mid-2018 → Communication Services residual coverage from ~mid-2019." + ) + + passed = bool(ok_mega and ok_etf) + return { + "passed": passed, + "megacap": megacap, + "megacap_ok": ok_mega, + "megacap_deadline": megacap_deadline.isoformat(), + "sector_etfs": etf_info, + "sector_etfs_deep_count": deep_etfs, + "sector_etfs_ok": ok_etf, + "still_shallow_count": len(still_shallow), + "still_shallow_sample": still_shallow[:20], + "xlc_note": note_xlc, + "target_history_days": history_days, + } + + +async def _step1_deepen( + snapshot: Path, + *, + history_days: int, + sleep_s: float, + quiet: bool, +) -> dict[str, Any]: + from app.config import settings + from app.providers.alpaca import AlpacaOHLCVProvider + + if not settings.alpaca_api_key or not settings.alpaca_api_secret: + raise SystemExit("ALPACA_API_KEY / ALPACA_API_SECRET required") + + clear_manifest(snapshot) + depths = _symbol_depth(snapshot) + shallow, shallow_meta = _derive_shallow(depths) + print( + f"Shallow symbols to deepen: {len(shallow)} " + f"(p10 deep start={shallow_meta.get('deep_cohort_p10_start')}, " + f"cutoff={shallow_meta.get('shallow_cutoff')})" + ) + if not shallow: + print("WARNING: no shallow symbols detected — snapshot may already be uniform") + + provider = AlpacaOHLCVProvider(settings.alpaca_api_key, settings.alpaca_api_secret) + end = date.today() + start = end - timedelta(days=history_days) + engine = create_engine( + f"sqlite:///{snapshot.resolve().as_posix()}", + future=True, + ) + + ok = fail = 0 + t0 = time.monotonic() + try: + for i, sym in enumerate(shallow, 1): + try: + n = await _fetch_and_replace_ohlcv( + engine, provider, sym, start, end, sleep_s=sleep_s + ) + if n <= 0: + fail += 1 + if not quiet: + print(f" [{i}/{len(shallow)}] {sym} empty") + continue + ok += 1 + if not quiet and (i % 25 == 0 or i == len(shallow)): + print( + f" progress {i}/{len(shallow)} ok={ok} fail={fail} " + f"last={sym} bars={n} elapsed={(time.monotonic()-t0)/60:.1f}m" + ) + except Exception as exc: + fail += 1 + print(f" [{i}/{len(shallow)}] {sym} FAIL {exc}") + finally: + engine.dispose() + + print("Deepening SPY + sector ETFs in benchmark_prices…") + etf_result = await _deepen_sector_etfs( + snapshot, history_days=history_days, sleep_s=sleep_s + ) + + # Manifest: full completion after deepen (no --limit). + from scripts.research_snapshot_manifest import _count_snapshot + + counts = _count_snapshot(snapshot) + manifest_path = write_completion_manifest( + snapshot, + complete=True, + sources={"deepen": "sector_resid_deep_test step1"}, + history_days=history_days, + min_bars=None, + fetch_ok=ok, + fetch_fail=fail, + limit=None, + extra={ + "shallow_meta": shallow_meta, + "shallow_fetched_ok": ok, + "shallow_fetched_fail": fail, + "etf_refresh": etf_result.get("written"), + "counts_after": counts, + }, + ) + print(f"Manifest written: {manifest_path}") + + sanity = _sanity_check(snapshot, history_days=history_days) + return { + "shallow_meta": shallow_meta, + "shallow_list_n": len(shallow), + "fetch_ok": ok, + "fetch_fail": fail, + "etf_refresh": etf_result, + "sanity": sanity, + "manifest_path": str(manifest_path), + } + + +async def _one_masked_run( + snapshot: Path, + *, + sector_map_path: Path, + liquid_breadth: int, + min_price: float, + workers: int, + quiet: bool, +) -> dict[str, Any]: + """Single collection under liquid mask; full + era IC from the same series.""" + from app.config import settings + from app.services import backtest_service as bt + from app.services.benchmark_service import load_benchmark_closes + from app.models.ticker import Ticker + from sqlalchemy import select + + os.environ["BACKTEST_SNAPSHOT_OFFLINE"] = "1" + os.environ["BACKTEST_SIGNAL_EVAL_ONLY"] = "1" + os.environ["BACKTEST_LIQUID_BREADTH"] = str(int(liquid_breadth)) + os.environ["BACKTEST_LIQUID_MIN_PRICE"] = str(float(min_price)) + os.environ["BACKTEST_SECTOR_MAP_PATH"] = str(sector_map_path.resolve()) + if workers: + settings.backtest_workers = workers + + # One collection pass (not run_backtest twice). + engine = create_async_engine(_sqlite_url(snapshot), pool_pre_ping=True) + Session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) + collected: dict = defaultdict(lambda: defaultdict(list)) + symbol_to_sector = load_ticker_sector_map(sector_map_path) + + try: + async with Session() as db: + tickers = list( + (await db.execute(select(Ticker).order_by(Ticker.symbol))).scalars() + ) + spy = await load_benchmark_closes(db, "SPY") + sector_etf: dict[str, dict] = {} + for etf in SECTOR_ETFS: + series = await load_benchmark_closes(db, etf) + if series: + sector_etf[etf] = series + + total = len(tickers) + for idx, t in enumerate(tickers): + if not quiet and idx % 100 == 0: + print(f" collect {idx}/{total}", end="\r", flush=True) + cols = await bt._fetch_columns(db, t.symbol) + if cols is None: + continue + records = [ + type( + "R", + (), + { + "date": date.fromordinal(int(cols[0][i])), + "close": cols[4][i], + "high": cols[2][i], + "volume": cols[5][i] if len(cols) > 5 else 0, + }, + )() + for i in range(len(cols[0])) + ] + etf_closes = bt._sector_etf_closes_for_symbol( + t.symbol, symbol_to_sector, sector_etf + ) + series = bt._signal_series( + records, + spy, + symbol=t.symbol, + sector_etf_closes=etf_closes, + ) + for name, weeks in series.items(): + for wk, pairs in weeks.items(): + collected[name][wk].extend(pairs) + finally: + await engine.dispose() + if not quiet: + print() + + if symbol_to_sector: + bt._inject_sector_demeaned_momentum(collected, symbol_to_sector) + + full_eval = bt._signal_evaluation(dict(collected)) + + def _filter_era(coll: dict, *, pre: bool) -> dict: + out: dict = defaultdict(lambda: defaultdict(list)) + for name, weeks in coll.items(): + for wk, recs in weeks.items(): + year = int(wk[0]) if isinstance(wk, tuple) else int(str(wk)[:4]) + if pre and year >= ERA_SPLIT.year: + continue + if not pre and year < ERA_SPLIT.year: + continue + out[name][wk].extend(recs) + return out + + pre_eval = bt._signal_evaluation(_filter_era(collected, pre=True)) + post_eval = bt._signal_evaluation(_filter_era(collected, pre=False)) + + # Identical-subset: resid IC only where sector_resid exists (same CS for t rule). + identical = _identical_subset_eval(collected, bt) + + def _idx(rows: list[dict]) -> dict[str, dict]: + return {r["signal"]: r for r in rows} + + # Mask bind diagnostics from liquid-aware rows if present. + mask_diag = _mask_diagnostics(full_eval) + + return { + "liquid_breadth_top_n": liquid_breadth, + "liquid_min_price": min_price, + "survivorship_banner": SURVIVORSHIP, + "signal_eval": full_eval, + "signal_eval_by_name": _idx(full_eval), + "era_split": { + "era_split_date": ERA_SPLIT.isoformat(), + "note": "Diagnostic only — not a tuning input.", + "pre_2021": _idx(pre_eval), + "post_2021": _idx(post_eval), + }, + "identical_subset_sector_cs": identical, + "mask_diagnostics": mask_diag, + "sector_map_size": len(symbol_to_sector), + "sector_etfs_loaded": sorted(sector_etf), + "spy_bars": len(spy), + } + + +def _identical_subset_eval(collected: dict, bt) -> dict[str, Any]: + """Re-score mom_12_1_resid on the same (week, symbol) cells as sector_resid.""" + sector_weeks = collected.get("mom_12_1_sector_resid") or {} + resid_weeks = collected.get("mom_12_1_resid") or {} + demean_weeks = collected.get("mom_12_1_sector_demeaned") or {} + mom_weeks = collected.get("mom_12_1") or {} + + restricted: dict = defaultdict(lambda: defaultdict(list)) + for wk, recs in sector_weeks.items(): + syms = set() + for rec in recs: + if isinstance(rec, dict) and rec.get("symbol"): + syms.add(str(rec["symbol"]).upper()) + restricted["mom_12_1_sector_resid"][wk].append(rec) + for name, source in ( + ("mom_12_1_resid", resid_weeks), + ("mom_12_1", mom_weeks), + ("mom_12_1_sector_demeaned", demean_weeks), + ): + for rec in source.get(wk) or []: + if not isinstance(rec, dict): + continue + sym = rec.get("symbol") + if sym and str(sym).upper() in syms: + restricted[name][wk].append(rec) + + rows = bt._signal_evaluation(dict(restricted)) + return {r["signal"]: r for r in rows} + + +def _mask_diagnostics(signal_eval: list[dict]) -> dict[str, Any]: + # Prefer a dense signal for mask stats. + for name in ("vol_6m", "mom_12_1", "fip_id"): + for row in signal_eval: + if row.get("signal") == name and row.get("mask_binds_pct") is not None: + return { + "reference_signal": name, + "avg_cross_section": row.get("avg_cross_section"), + "avg_raw_pool": row.get("avg_raw_pool"), + "avg_eligible_pre_mask": row.get("avg_eligible_pre_mask"), + "mask_binds_pct": row.get("mask_binds_pct"), + "weeks": row.get("weeks"), + } + # Fallback: any row with liquid fields + for row in signal_eval: + if row.get("liquid_breadth_top_n"): + return { + "reference_signal": row.get("signal"), + "avg_cross_section": row.get("avg_cross_section"), + "mask_binds_pct": row.get("mask_binds_pct"), + "weeks": row.get("weeks"), + } + return {"note": "no liquid mask diagnostics on rows (mask may be off)"} + + +def _grade(harness: dict[str, Any]) -> dict[str, Any]: + """Pre-registered PASS/FAIL for mom_12_1_sector_resid — mechanical.""" + by = harness.get("signal_eval_by_name") or {} + era = harness.get("era_split") or {} + identical = harness.get("identical_subset_sector_cs") or {} + + sector = by.get("mom_12_1_sector_resid") + # Prefer identical-subset resid for t comparison; fall back to full-table resid. + resid = identical.get("mom_12_1_resid") or by.get("mom_12_1_resid") + pre = (era.get("pre_2021") or {}).get("mom_12_1_sector_resid") + post = (era.get("post_2021") or {}).get("mom_12_1_sector_resid") + + checks: dict[str, Any] = { + "sector_row": sector, + "resid_row_for_t": resid, + "resid_t_source": ( + "identical_subset" if identical.get("mom_12_1_resid") else "full_table" + ), + "pre_2021": pre, + "post_2021": post, + } + + if sector is None: + return { + "verdict": "FAIL", + "reason": "mom_12_1_sector_resid missing from signal_eval", + "checks": checks, + "headline": "Task 1 CLOSED — sector residual dead on deep evidence.", + } + + mean_ic = sector.get("mean_ic") + t_stat = sector.get("ic_t_stat") + weeks = int(sector.get("weeks") or 0) + reliable = bool(sector.get("reliable")) + resid_t = resid.get("ic_t_stat") if resid else None + + mag_ok = mean_ic is not None and abs(float(mean_ic)) >= IRON_IC + sign_ok = mean_ic is not None and float(mean_ic) > 0 + reliable_ok = reliable and weeks >= 12 + weeks_ok = weeks >= MIN_WEEKS_DEEP + t_ok = ( + t_stat is not None + and resid_t is not None + and float(t_stat) >= float(resid_t) + ) + + pre_ic = pre.get("mean_ic") if pre else None + post_ic = post.get("mean_ic") if post else None + era_sign_ok = ( + pre_ic is not None + and post_ic is not None + and float(pre_ic) > 0 + and float(post_ic) > 0 + ) + # If pre era has no row, data fix failed for depth / era coverage. + era_present = pre is not None and post is not None + + checks.update({ + "abs_mean_ic_ge_0_03": mag_ok, + "sign_positive": sign_ok, + "reliable": reliable_ok, + "weeks_ge_50": weeks_ok, + "weeks": weeks, + "t_ge_resid_same_cs": t_ok, + "sector_t": t_stat, + "resid_t": resid_t, + "era_both_present": era_present, + "era_sign_consistent_positive": era_sign_ok, + "pre_ic": pre_ic, + "post_ic": post_ic, + "avg_cross_section": sector.get("avg_cross_section"), + }) + + if not weeks_ok: + return { + "verdict": "FAIL", + "reason": ( + f"weeks={weeks} did not extend (need ≥{MIN_WEEKS_DEEP}) — " + "data fix did not work or sector residual still shallow" + ), + "checks": checks, + "headline": "Task 1 CLOSED — sector residual dead on deep evidence.", + } + + passed = ( + mag_ok + and sign_ok + and reliable_ok + and weeks_ok + and t_ok + and era_present + and era_sign_ok + ) + if passed: + return { + "verdict": "PASS", + "reason": ( + "iron bar + weeks extended + t≥resid on same CS + era sign consistent" + ), + "checks": checks, + "headline": ( + "PROMOTE case strengthened — portfolio A/B is the next human decision." + ), + } + return { + "verdict": "FAIL", + "reason": "failed one or more pre-registered checks (see checks)", + "checks": checks, + "headline": "Task 1 CLOSED — sector residual dead on deep evidence.", + } + + +def _write_reports(payload: dict, out_json: Path, doc_path: Path) -> None: + out_json.parent.mkdir(parents=True, exist_ok=True) + out_json.write_text( + json.dumps(payload, indent=2, default=str) + "\n", encoding="utf-8" + ) + + grade = payload.get("grade") or {} + harness = payload.get("harness") or {} + by = harness.get("signal_eval_by_name") or {} + era = harness.get("era_split") or {} + lines = [ + "# Sector-residual deep test (masked, repaired snapshot)", + "", + f"Generated: `{payload.get('generated_at')}`", + "", + f"> **{SURVIVORSHIP}**", + "", + "## Pre-registered grade (mechanical)", + "", + f"**Verdict: {grade.get('verdict')}**", + "", + f"{grade.get('headline')}", + "", + f"Reason: {grade.get('reason')}", + "", + f"```json\n{json.dumps(grade.get('checks') or {}, indent=2, default=str)}\n```", + "", + "## Step-1 sanity", + "", + f"```json\n{json.dumps(payload.get('step1') or {}, indent=2, default=str)}\n```", + "", + "## Mask diagnostics", + "", + f"```json\n{json.dumps(harness.get('mask_diagnostics') or {}, indent=2, default=str)}\n```", + "", + "## Signal table (rows only — no narrative for non-sector signals)", + "", + "| signal | mean_ic | t | weeks | avg_N | reliable |", + "|---|---:|---:|---:|---:|---|", + ] + for name in sorted(by): + r = by[name] + lines.append( + f"| {name} | {r.get('mean_ic')} | {r.get('ic_t_stat')} | " + f"{r.get('weeks')} | {r.get('avg_cross_section')} | {r.get('reliable')} |" + ) + lines.extend([ + "", + "### Era split — mom_12_1_sector_resid only (for grade)", + "", + f"| era | IC | t | weeks | N |", + f"|---|---:|---:|---:|---:|", + ]) + for label in ("pre_2021", "post_2021"): + r = (era.get(label) or {}).get("mom_12_1_sector_resid") or {} + lines.append( + f"| {label} | {r.get('mean_ic', '')} | {r.get('ic_t_stat', '')} | " + f"{r.get('weeks', '')} | {r.get('avg_cross_section', '')} |" + ) + lines.extend([ + "", + "### Identical-subset baselines (sector CS)", + "", + f"```json\n{json.dumps(harness.get('identical_subset_sector_cs') or {}, indent=2, default=str)}\n```", + "", + "## Status", + "", + "PENDING_HUMAN beyond the mechanical PASS/FAIL above. " + "Nothing merged into production docs or prod code.", + "", + f"JSON: `{out_json.as_posix()}`", + "", + ]) + md_path = out_json.with_suffix(".md") + md_path.write_text("\n".join(lines) + "\n", encoding="utf-8") + + # Update history-depth-extension.md + _update_history_doc(doc_path, payload, out_json) + + +def _update_history_doc(doc_path: Path, payload: dict, out_json: Path) -> None: + grade = payload.get("grade") or {} + banner = ( + "\n\n---\n\n" + "## Supersession notice (2026-07-19 sector-resid deep test)\n\n" + "The table and interpretation from **`history-depth-20260719-103315`** are " + "**UNMASKED, TWO-TIER SNAPSHOT — superseded, directional only, do not cite**. " + "Prod-universe names (and sector residual coverage) were left shallow while " + "breadth names were deepened; sector residual weeks=35 was a data gap.\n\n" + f"### Sector-residual deep test outcome: **{grade.get('verdict')}**\n\n" + f"{grade.get('headline')}\n\n" + f"- Reason: {grade.get('reason')}\n" + f"- Artifact: `{out_json.as_posix()}`\n" + f"- Mechanical checks: see that report.\n\n" + "**Future snapshot rebuilds must verify per-symbol depth** (earliest-bar " + "uniformity across the intended universe) — guard is a to-do, not part of " + "this order.\n" + ) + if doc_path.exists(): + text = doc_path.read_text(encoding="utf-8") + # Insert supersession after status line / near top results if not already there. + marker = "## Supersession notice (2026-07-19 sector-resid deep test)" + if marker in text: + # Replace from marker to end of that section or append fresh block at end. + pre = text.split(marker)[0].rstrip() + text = pre + banner + else: + # Mark 103315 in place if mentioned. + text = text.replace( + "Authoritative artifact:** `reports/history-depth-20260719-103315.json`", + "Superseded artifact (do not cite):** `reports/history-depth-20260719-103315.json` " + "— **UNMASKED, TWO-TIER SNAPSHOT**", + ) + text = text.rstrip() + banner + # Soften old PARK-only language if present — leave body but status at top. + if text.startswith("#"): + lines = text.splitlines() + for i, line in enumerate(lines[:15]): + if line.startswith("**Status:**"): + lines[i] = ( + f"**Status:** sector-resid deep test **{grade.get('verdict')}** " + f"— see supersession section. PENDING_HUMAN beyond PASS/FAIL." + ) + break + text = "\n".join(lines) + doc_path.write_text(text if text.endswith("\n") else text + "\n", encoding="utf-8") + else: + doc_path.write_text( + "# History-depth extension\n" + banner, encoding="utf-8" + ) + + +async def _main() -> None: + args = _parse_args() + snapshot = Path(args.snapshot) + if not snapshot.exists(): + raise SystemExit(f"Snapshot missing: {snapshot}") + if args.allow_spawn: + os.environ["BACKTEST_ALLOW_SPAWN"] = "1" + + sector_map = Path(args.sector_map) + if not sector_map.exists(): + raise SystemExit(f"Sector map missing: {sector_map}") + + step1: dict[str, Any] + if args.skip_deepen: + print("Skip deepen — race guard + sanity only…") + assert_research_snapshot_complete(snapshot) + sanity = _sanity_check(snapshot, history_days=args.history_days) + step1 = {"skipped": True, "sanity": sanity} + if not sanity["passed"]: + raise SystemExit( + "Sanity check FAILED with --skip-deepen. " + f"Details: {json.dumps(sanity, default=str)}" + ) + else: + print("Step 1 — deepen shallow symbols…") + step1 = await _step1_deepen( + snapshot, + history_days=args.history_days, + sleep_s=args.sleep, + quiet=args.quiet, + ) + if not step1["sanity"]["passed"]: + print("SANITY CHECK FAILED — refusing harness.") + print(json.dumps(step1["sanity"], indent=2, default=str)) + stamp = datetime.now().strftime("%Y%m%d-%H%M%S") + fail_path = Path("reports") / f"sector-resid-deep-{stamp}-SANITY-FAIL.json" + fail_path.parent.mkdir(parents=True, exist_ok=True) + fail_path.write_text( + json.dumps({"step1": step1, "harness": None}, indent=2, default=str) + + "\n", + encoding="utf-8", + ) + raise SystemExit( + f"Stop: sanity failed. Wrote {fail_path}. Do not run harness on two-tier data." + ) + print("Sanity check PASSED.") + assert_research_snapshot_complete(snapshot) + + print( + f"Step 2 — ONE masked harness " + f"(top {args.liquid_breadth}, min_price={args.min_price})…" + ) + harness = await _one_masked_run( + snapshot, + sector_map_path=sector_map, + liquid_breadth=args.liquid_breadth, + min_price=args.min_price, + workers=args.workers, + quiet=args.quiet, + ) + grade = _grade(harness) + print(f"GRADE: {grade['verdict']} — {grade['headline']}") + + stamp = datetime.now().strftime("%Y%m%d-%H%M%S") + out = Path(args.out) if args.out else Path("reports") / f"sector-resid-deep-{stamp}.json" + payload = { + "generated_at": datetime.now().isoformat(), + "snapshot": str(snapshot.resolve()), + "pre_registration": { + "iron_ic": IRON_IC, + "min_weeks_deep": MIN_WEEKS_DEEP, + "liquid_breadth": args.liquid_breadth, + "min_price": args.min_price, + "rule": ( + "PASS = |IC|>=0.03, +sign, reliable, weeks>=50, " + "t>=resid on same CS, era signs both +" + ), + }, + "step1": step1, + "harness": harness, + "grade": grade, + "pending_human": True, + "note": "Nothing merged into production. Thread ends at PASS/FAIL.", + } + _write_reports( + payload, + out, + Path("docs/research/history-depth-extension.md"), + ) + print(f"Wrote {out}") + print(f"Wrote {out.with_suffix('.md')}") + print("Updated docs/research/history-depth-extension.md") + + +if __name__ == "__main__": + asyncio.run(_main()) diff --git a/scripts/run_tier1_macbook.sh b/scripts/run_tier1_macbook.sh index 1472cf4..72f8aef 100755 --- a/scripts/run_tier1_macbook.sh +++ b/scripts/run_tier1_macbook.sh @@ -15,6 +15,7 @@ # ./scripts/run_tier1_macbook.sh --earnings-only # multi-day FMP backfill + re-run 2a/2b # ./scripts/run_tier1_macbook.sh --harness-only # skip rebuild; race-guard + IC only # ./scripts/run_tier1_macbook.sh --coverage-only # bars-per-year probe only +# ./scripts/run_tier1_macbook.sh --sector-resid-deep # deepen shallow + ONE masked grade # # Does NOT touch production Postgres, scheduler, gates, or prod config. @@ -35,7 +36,7 @@ FMP_SLEEP="${FMP_SLEEP:-0.35}" PYTHON="${PYTHON:-python3}" USE_CORP_PROXY="${USE_CORP_PROXY:-0}" -PHASE="depth" # depth | all | earnings | harness | coverage | ssl-check +PHASE="depth" # depth | all | earnings | harness | coverage | ssl | sector-resid-deep usage() { sed -n '2,25p' "$0" | sed 's/^# \?//' @@ -59,6 +60,7 @@ while [[ $# -gt 0 ]]; do --coverage-only) PHASE=coverage; shift ;; --depth) PHASE=depth; shift ;; --ssl-check) PHASE=ssl; shift ;; + --sector-resid-deep) PHASE=sector_resid_deep; shift ;; --corp-proxy) USE_CORP_PROXY=1; shift ;; --prod-snap) PROD_SNAP="$2"; shift 2 ;; --research-snap) RESEARCH_SNAP="$2"; shift 2 ;; @@ -223,6 +225,18 @@ run_harness() { --allow-spawn } +run_sector_resid_deep() { + need_file "$RESEARCH_SNAP" + log "Sector-resid deep test: deepen shallow symbols + ONE liquid-1500 masked grade" + log "Pre-registered PASS/FAIL only — thread ends after this run" + "$PYTHON" scripts/run_sector_resid_deep_test.py \ + --snapshot "$RESEARCH_SNAP" \ + --history-days "$HISTORY_DAYS" \ + --sleep "$ALPACA_SLEEP" \ + --workers "$WORKERS" \ + --allow-spawn +} + log "cwd=$ROOT python=$PYTHON phase=$PHASE workers=$WORKERS" setup_ssl @@ -230,6 +244,9 @@ case "$PHASE" in ssl) ssl_check ;; + sector_resid_deep) + run_sector_resid_deep + ;; coverage) run_coverage ;; From f3d1312a69d5c0b65b0738e1132530e181da5ffa Mon Sep 17 00:00:00 2001 From: Dennis Thiessen Date: Sun, 19 Jul 2026 11:21:05 +0200 Subject: [PATCH 07/14] tests done --- ...esid-deep-20260719-111923-SANITY-FAIL.json | 275 ++++++++++++++++++ 1 file changed, 275 insertions(+) create mode 100644 reports/sector-resid-deep-20260719-111923-SANITY-FAIL.json diff --git a/reports/sector-resid-deep-20260719-111923-SANITY-FAIL.json b/reports/sector-resid-deep-20260719-111923-SANITY-FAIL.json new file mode 100644 index 0000000..6b03e61 --- /dev/null +++ b/reports/sector-resid-deep-20260719-111923-SANITY-FAIL.json @@ -0,0 +1,275 @@ +{ + "step1": { + "shallow_meta": { + "n_symbols": 4654, + "deep_cohort_p10_start": "2016-01-04", + "shallow_cutoff": "2017-02-07", + "lag_days": 400, + "n_shallow": 3349, + "shallow_start_histogram": { + "2017": 117, + "2018": 162, + "2019": 163, + "2020": 269, + "2021": 1060, + "2022": 211, + "2023": 176, + "2024": 281, + "2025": 543, + "2026": 367 + }, + "deep_start_histogram": { + "2016": 1296, + "2017": 9 + }, + "shallow_sample": [ + "A", + "AACB", + "AACBR", + "AACBU", + "AACI", + "AACIU", + "AACIW", + "AACO", + "AACOU", + "AACOW", + "AACP", + "AACPR", + "AACPU", + "AACPW", + "AAPG", + "AAPL", + "AARD", + "ABAT", + "ABBV", + "ABCL", + "ABLV", + "ABLVW", + "ABNB", + "ABOS", + "ABSI", + "ABT", + "ABTC", + "ABVC", + "ABVX", + "ACAA" + ] + }, + "shallow_list_n": 3349, + "fetch_ok": 3349, + "fetch_fail": 0, + "etf_refresh": { + "written": { + "SPY": 0, + "XLB": 0, + "XLC": 0, + "XLE": 0, + "XLF": 0, + "XLI": 0, + "XLK": 0, + "XLP": 0, + "XLRE": 0, + "XLU": 0, + "XLV": 0, + "XLY": 0 + }, + "benchmark_summary": [ + { + "symbol": "SPY", + "n": 2649, + "min": "2016-01-04", + "max": "2026-07-17" + }, + { + "symbol": "XLB", + "n": 2649, + "min": "2016-01-04", + "max": "2026-07-17" + }, + { + "symbol": "XLC", + "n": 2030, + "min": "2018-06-19", + "max": "2026-07-17" + }, + { + "symbol": "XLE", + "n": 2649, + "min": "2016-01-04", + "max": "2026-07-17" + }, + { + "symbol": "XLF", + "n": 2649, + "min": "2016-01-04", + "max": "2026-07-17" + }, + { + "symbol": "XLI", + "n": 2649, + "min": "2016-01-04", + "max": "2026-07-17" + }, + { + "symbol": "XLK", + "n": 2649, + "min": "2016-01-04", + "max": "2026-07-17" + }, + { + "symbol": "XLP", + "n": 2649, + "min": "2016-01-04", + "max": "2026-07-17" + }, + { + "symbol": "XLRE", + "n": 2649, + "min": "2016-01-04", + "max": "2026-07-17" + }, + { + "symbol": "XLU", + "n": 2649, + "min": "2016-01-04", + "max": "2026-07-17" + }, + { + "symbol": "XLV", + "n": 2649, + "min": "2016-01-04", + "max": "2026-07-17" + }, + { + "symbol": "XLY", + "n": 2649, + "min": "2016-01-04", + "max": "2026-07-17" + } + ] + }, + "sanity": { + "passed": false, + "megacap": { + "AAPL": { + "symbol": "AAPL", + "bars": 2649, + "min_date": "2016-01-04", + "max_date": "2026-07-17" + }, + "MSFT": { + "symbol": "MSFT", + "bars": 2649, + "min_date": "2016-01-04", + "max_date": "2026-07-17" + }, + "JPM": { + "symbol": "JPM", + "bars": 2649, + "min_date": "2016-01-04", + "max_date": "2026-07-17" + }, + "XOM": { + "symbol": "XOM", + "bars": 2649, + "min_date": "2016-01-04", + "max_date": "2026-07-17" + }, + "JNJ": { + "symbol": "JNJ", + "bars": 2649, + "min_date": "2016-01-04", + "max_date": "2026-07-17" + } + }, + "megacap_ok": false, + "megacap_deadline": "2013-12-14", + "sector_etfs": { + "XLB": { + "n": 2649, + "min": "2016-01-04", + "max": "2026-07-17" + }, + "XLC": { + "n": 2030, + "min": "2018-06-19", + "max": "2026-07-17" + }, + "XLE": { + "n": 2649, + "min": "2016-01-04", + "max": "2026-07-17" + }, + "XLF": { + "n": 2649, + "min": "2016-01-04", + "max": "2026-07-17" + }, + "XLI": { + "n": 2649, + "min": "2016-01-04", + "max": "2026-07-17" + }, + "XLK": { + "n": 2649, + "min": "2016-01-04", + "max": "2026-07-17" + }, + "XLP": { + "n": 2649, + "min": "2016-01-04", + "max": "2026-07-17" + }, + "XLRE": { + "n": 2649, + "min": "2016-01-04", + "max": "2026-07-17" + }, + "XLU": { + "n": 2649, + "min": "2016-01-04", + "max": "2026-07-17" + }, + "XLV": { + "n": 2649, + "min": "2016-01-04", + "max": "2026-07-17" + }, + "XLY": { + "n": 2649, + "min": "2016-01-04", + "max": "2026-07-17" + } + }, + "sector_etfs_deep_count": 11, + "sector_etfs_ok": true, + "still_shallow_count": 2882, + "still_shallow_sample": [ + "AACB", + "AACBR", + "AACBU", + "AACI", + "AACIU", + "AACIW", + "AACO", + "AACOU", + "AACOW", + "AACP", + "AACPR", + "AACPU", + "AACPW", + "AAPG", + "AARD", + "ABAT", + "ABCL", + "ABLV", + "ABLVW", + "ABNB" + ], + "xlc_note": "XLC lists mid-2018 \u2192 Communication Services residual coverage from ~mid-2019.", + "target_history_days": 5000 + }, + "manifest_path": "backtest_snapshots/research.sqlite.manifest.json" + }, + "harness": null +} From 003f20de199f519441778e90809a20768b0e37b3 Mon Sep 17 00:00:00 2001 From: Dennis Thiessen Date: Sun, 19 Jul 2026 11:22:06 +0200 Subject: [PATCH 08/14] fix: sector-resid sanity grades against Alpaca feed floor, not calendar 5000d SANITY-FAIL report showed megacaps/ETFs already at empirical 2016-01-04 floor (2649 bars) after deepen; check wrongly required ~2013. Pass when megacaps leave the old 2021 two-tier floor and match SPY; XLC listing exception retained. --- scripts/run_sector_resid_deep_test.py | 103 +++++++++++++++++++------- 1 file changed, 76 insertions(+), 27 deletions(-) diff --git a/scripts/run_sector_resid_deep_test.py b/scripts/run_sector_resid_deep_test.py index 54de8bf..a5cbfff 100644 --- a/scripts/run_sector_resid_deep_test.py +++ b/scripts/run_sector_resid_deep_test.py @@ -299,22 +299,14 @@ async def _deepen_sector_etfs( def _sanity_check(snapshot: Path, *, history_days: int) -> dict[str, Any]: - depths = {r["symbol"]: r for r in _symbol_depth(snapshot)} - end = date.today() - target_start = end - timedelta(days=history_days) - # Allow ~1 year slack for IPO/listing limits (not a hard fail for all names). - megacap_deadline = target_start + timedelta(days=400) + """Pass if megacaps + sector ETFs sit at the *empirical feed floor*, not calendar 5000d. - megacap = {} - ok_mega = True - for sym in SANITY_MEGACAPS: - row = depths.get(sym) - megacap[sym] = row - if row is None or not row.get("min_date"): - ok_mega = False - continue - if date.fromisoformat(row["min_date"]) > megacap_deadline: - ok_mega = False + Alpaca daily history for this stack bottoms out around 2016-01-04 (~2649 bars) + even when history_days=5000 is requested. That is feed coverage, not a two-tier + snapshot bug. Fail only if megacaps are still stuck near the old ~2021 prod floor + or if sector ETFs are missing / shorter than the SPY series (except XLC listing). + """ + depths = {r["symbol"]: r for r in _symbol_depth(snapshot)} engine = create_engine( f"sqlite:///{snapshot.resolve().as_posix()}", @@ -322,6 +314,12 @@ def _sanity_check(snapshot: Path, *, history_days: int) -> dict[str, Any]: ) try: with engine.connect() as conn: + spy_row = conn.execute( + text( + "SELECT COUNT(*), MIN(date), MAX(date) FROM benchmark_prices " + "WHERE symbol = 'SPY'" + ) + ).fetchone() etf_rows = conn.execute( text( "SELECT symbol, COUNT(*), MIN(date), MAX(date) " @@ -333,45 +331,96 @@ def _sanity_check(snapshot: Path, *, history_days: int) -> dict[str, Any]: finally: engine.dispose() + spy_n, spy_min, spy_max = spy_row if spy_row else (0, None, None) + feed_floor = ( + date.fromisoformat(str(spy_min)[:10]) + if spy_min + else date(2016, 1, 4) + ) + # Megacaps must match the feed floor within a few sessions (not calendar-5000). + megacap_slack_days = 10 + # Old two-tier defect left prod names at ~2021-06; anything still after this fails. + old_shallow_floor = date(2020, 1, 1) + + megacap = {} + ok_mega = True + mega_reasons: list[str] = [] + for sym in SANITY_MEGACAPS: + row = depths.get(sym) + megacap[sym] = row + if row is None or not row.get("min_date"): + ok_mega = False + mega_reasons.append(f"{sym}: missing") + continue + d0 = date.fromisoformat(row["min_date"]) + if d0 > old_shallow_floor: + ok_mega = False + mega_reasons.append( + f"{sym}: min_date={d0} still after {old_shallow_floor} (two-tier unrepaired)" + ) + elif d0 > feed_floor + timedelta(days=megacap_slack_days): + ok_mega = False + mega_reasons.append( + f"{sym}: min_date={d0} later than SPY feed floor {feed_floor}" + ) + etf_info = { s: {"n": n, "min": d0, "max": d1} for s, n, d0, d1 in etf_rows } deep_etfs = 0 + etf_reasons: list[str] = [] for sym in SECTOR_ETFS: info = etf_info.get(sym) if not info or not info["min"]: + etf_reasons.append(f"{sym}: missing") continue - # XLC lists mid-2018 — accept that floor. - floor = date(2018, 6, 1) if sym == "XLC" else megacap_deadline - if date.fromisoformat(str(info["min"])[:10]) <= floor + timedelta(days=60): - deep_etfs += 1 - elif date.fromisoformat(str(info["min"])[:10]) <= date(2019, 1, 1): - # moderately deep still counts for non-XLC if near 2018-19 - if sym != "XLC": + d0 = date.fromisoformat(str(info["min"])[:10]) + if sym == "XLC": + # Listed 2018-06-18/19. + if d0 <= date(2018, 7, 15): deep_etfs += 1 + else: + etf_reasons.append(f"XLC: min_date={d0} later than listing floor") + else: + if d0 <= feed_floor + timedelta(days=megacap_slack_days): + deep_etfs += 1 + else: + etf_reasons.append( + f"{sym}: min_date={d0} later than SPY feed floor {feed_floor}" + ) - # Require 10 of 11 sector ETFs deep (XLC may be the exception with mid-2018 start). ok_etf = deep_etfs >= 10 - - # Shallow residual after deepen: few names should still start after 2020. still_shallow, _ = _derive_shallow(list(depths.values()), lag_days=400) - # After fix, shallow set should shrink dramatically vs ~500. note_xlc = ( "XLC lists mid-2018 → Communication Services residual coverage from ~mid-2019." ) + note_feed = ( + f"Empirical Alpaca floor observed via SPY: {feed_floor.isoformat()} " + f"(n={spy_n}). Calendar history_days={history_days} is a request cap, not a " + "guarantee — sanity grades against the feed floor, not 5000 calendar days." + ) passed = bool(ok_mega and ok_etf) return { "passed": passed, "megacap": megacap, "megacap_ok": ok_mega, - "megacap_deadline": megacap_deadline.isoformat(), + "megacap_reasons": mega_reasons, + "feed_floor": feed_floor.isoformat(), + "spy_benchmark": {"n": spy_n, "min": spy_min, "max": spy_max}, + "old_shallow_floor": old_shallow_floor.isoformat(), "sector_etfs": etf_info, "sector_etfs_deep_count": deep_etfs, "sector_etfs_ok": ok_etf, + "sector_etf_reasons": etf_reasons, "still_shallow_count": len(still_shallow), "still_shallow_sample": still_shallow[:20], + "still_shallow_note": ( + "Remaining 'shallow' names are mostly post-2017 IPOs/listings — expected, " + "not a two-tier defect." + ), "xlc_note": note_xlc, + "feed_note": note_feed, "target_history_days": history_days, } From 01007fb6dd1eee67c3257555ebb07983293fd54f Mon Sep 17 00:00:00 2001 From: Dennis Thiessen Date: Sun, 19 Jul 2026 11:39:43 +0200 Subject: [PATCH 09/14] tests done --- docs/research/history-depth-extension.md | 20 +- .../sector-resid-deep-20260719-113319.json | 990 ++++++++++++++++++ reports/sector-resid-deep-20260719-113319.md | 341 ++++++ 3 files changed, 1349 insertions(+), 2 deletions(-) create mode 100644 reports/sector-resid-deep-20260719-113319.json create mode 100644 reports/sector-resid-deep-20260719-113319.md diff --git a/docs/research/history-depth-extension.md b/docs/research/history-depth-extension.md index 522c57e..3f9d97a 100644 --- a/docs/research/history-depth-extension.md +++ b/docs/research/history-depth-extension.md @@ -1,8 +1,8 @@ # History-depth extension (Tier-1 alpha research) -**Status:** **RUN COMPLETE — human interpretation below.** +**Status:** sector-resid deep test **FAIL** — see supersession section. PENDING_HUMAN beyond PASS/FAIL. **Branch:** `research/earnings-gap-and-sue` (MacBook commit `f6e0ca7`) -**Authoritative artifact:** `reports/history-depth-20260719-103315.json` +**Superseded artifact (do not cite):** `reports/history-depth-20260719-103315.json` — **UNMASKED, TWO-TIER SNAPSHOT** **Production impact:** none. **Do not retune any production knob on deep history.** --- @@ -180,3 +180,19 @@ report does not change that without a separate A/B. Flag for human awareness onl | `reports/history-depth-20260719-103315.json` | **authoritative** | | `reports/history-depth-20260719-103315.md` | companion dump | | `reports/history-depth-20260719-093853` … `095156` | **ignore** (partial) | + +--- + +## Supersession notice (2026-07-19 sector-resid deep test) + +The table and interpretation from **`history-depth-20260719-103315`** are **UNMASKED, TWO-TIER SNAPSHOT — superseded, directional only, do not cite**. Prod-universe names (and sector residual coverage) were left shallow while breadth names were deepened; sector residual weeks=35 was a data gap. + +### Sector-residual deep test outcome: **FAIL** + +Task 1 CLOSED — sector residual dead on deep evidence. + +- Reason: failed one or more pre-registered checks (see checks) +- Artifact: `reports/sector-resid-deep-20260719-113319.json` +- Mechanical checks: see that report. + +**Future snapshot rebuilds must verify per-symbol depth** (earliest-bar uniformity across the intended universe) — guard is a to-do, not part of this order. diff --git a/reports/sector-resid-deep-20260719-113319.json b/reports/sector-resid-deep-20260719-113319.json new file mode 100644 index 0000000..2e6d9f9 --- /dev/null +++ b/reports/sector-resid-deep-20260719-113319.json @@ -0,0 +1,990 @@ +{ + "generated_at": "2026-07-19T11:33:19.102779", + "snapshot": "/Users/taathde3/git/lab/signal_platform/backtest_snapshots/research.sqlite", + "pre_registration": { + "iron_ic": 0.03, + "min_weeks_deep": 50, + "liquid_breadth": 1500, + "min_price": 5.0, + "rule": "PASS = |IC|>=0.03, +sign, reliable, weeks>=50, t>=resid on same CS, era signs both +" + }, + "step1": { + "skipped": true, + "sanity": { + "passed": true, + "megacap": { + "AAPL": { + "symbol": "AAPL", + "bars": 2649, + "min_date": "2016-01-04", + "max_date": "2026-07-17" + }, + "MSFT": { + "symbol": "MSFT", + "bars": 2649, + "min_date": "2016-01-04", + "max_date": "2026-07-17" + }, + "JPM": { + "symbol": "JPM", + "bars": 2649, + "min_date": "2016-01-04", + "max_date": "2026-07-17" + }, + "XOM": { + "symbol": "XOM", + "bars": 2649, + "min_date": "2016-01-04", + "max_date": "2026-07-17" + }, + "JNJ": { + "symbol": "JNJ", + "bars": 2649, + "min_date": "2016-01-04", + "max_date": "2026-07-17" + } + }, + "megacap_ok": true, + "megacap_reasons": [], + "feed_floor": "2016-01-04", + "spy_benchmark": { + "n": 2649, + "min": "2016-01-04", + "max": "2026-07-17" + }, + "old_shallow_floor": "2020-01-01", + "sector_etfs": { + "XLB": { + "n": 2649, + "min": "2016-01-04", + "max": "2026-07-17" + }, + "XLC": { + "n": 2030, + "min": "2018-06-19", + "max": "2026-07-17" + }, + "XLE": { + "n": 2649, + "min": "2016-01-04", + "max": "2026-07-17" + }, + "XLF": { + "n": 2649, + "min": "2016-01-04", + "max": "2026-07-17" + }, + "XLI": { + "n": 2649, + "min": "2016-01-04", + "max": "2026-07-17" + }, + "XLK": { + "n": 2649, + "min": "2016-01-04", + "max": "2026-07-17" + }, + "XLP": { + "n": 2649, + "min": "2016-01-04", + "max": "2026-07-17" + }, + "XLRE": { + "n": 2649, + "min": "2016-01-04", + "max": "2026-07-17" + }, + "XLU": { + "n": 2649, + "min": "2016-01-04", + "max": "2026-07-17" + }, + "XLV": { + "n": 2649, + "min": "2016-01-04", + "max": "2026-07-17" + }, + "XLY": { + "n": 2649, + "min": "2016-01-04", + "max": "2026-07-17" + } + }, + "sector_etfs_deep_count": 11, + "sector_etfs_ok": true, + "sector_etf_reasons": [], + "still_shallow_count": 2882, + "still_shallow_sample": [ + "AACB", + "AACBR", + "AACBU", + "AACI", + "AACIU", + "AACIW", + "AACO", + "AACOU", + "AACOW", + "AACP", + "AACPR", + "AACPU", + "AACPW", + "AAPG", + "AARD", + "ABAT", + "ABCL", + "ABLV", + "ABLVW", + "ABNB" + ], + "still_shallow_note": "Remaining 'shallow' names are mostly post-2017 IPOs/listings \u2014 expected, not a two-tier defect.", + "xlc_note": "XLC lists mid-2018 \u2192 Communication Services residual coverage from ~mid-2019.", + "feed_note": "Empirical Alpaca floor observed via SPY: 2016-01-04 (n=2649). Calendar history_days=5000 is a request cap, not a guarantee \u2014 sanity grades against the feed floor, not 5000 calendar days.", + "target_history_days": 5000 + } + }, + "harness": { + "liquid_breadth_top_n": 1500, + "liquid_min_price": 5.0, + "survivorship_banner": "SURVIVORSHIP BIAS: today's constituents backfilled. Relative IC only \u2014 not levels.", + "signal_eval": [ + { + "signal": "high_52w", + "weeks": 84, + "avg_cross_section": 1499.2, + "mean_ic": 0.0761, + "ic_t_stat": 4.46, + "ic_positive_pct": 71.4, + "mean_quintile_spread": 0.0118, + "reliable": true, + "liquid_breadth_top_n": 1500, + "liquid_min_price": 5.0, + "avg_raw_pool": 2564.4, + "avg_eligible_pre_mask": 2047.0, + "mask_binds_pct": 94.0 + }, + { + "signal": "trend_200", + "weeks": 85, + "avg_cross_section": 1499.6, + "mean_ic": 0.0371, + "ic_t_stat": 2.63, + "ic_positive_pct": 62.4, + "mean_quintile_spread": -0.7839, + "reliable": true, + "liquid_breadth_top_n": 1500, + "liquid_min_price": 5.0, + "avg_raw_pool": 2583.0, + "avg_eligible_pre_mask": 2065.8, + "mask_binds_pct": 96.5 + }, + { + "signal": "mom_12_1", + "weeks": 83, + "avg_cross_section": 1499.4, + "mean_ic": 0.0355, + "ic_t_stat": 2.41, + "ic_positive_pct": 62.7, + "mean_quintile_spread": 0.0141, + "reliable": true, + "liquid_breadth_top_n": 1500, + "liquid_min_price": 5.0, + "avg_raw_pool": 2553.0, + "avg_eligible_pre_mask": 2045.4, + "mask_binds_pct": 95.2 + }, + { + "signal": "mom_12_1_sector_resid", + "weeks": 83, + "avg_cross_section": 480.0, + "mean_ic": 0.0268, + "ic_t_stat": 1.69, + "ic_positive_pct": 60.2, + "mean_quintile_spread": 0.0126, + "reliable": true, + "liquid_breadth_top_n": 1500, + "liquid_min_price": 5.0, + "avg_raw_pool": 481.2, + "avg_eligible_pre_mask": 480.0, + "mask_binds_pct": 0.0 + }, + { + "signal": "mom_3_1", + "weeks": 90, + "avg_cross_section": 1498.6, + "mean_ic": 0.0256, + "ic_t_stat": 2.24, + "ic_positive_pct": 61.1, + "mean_quintile_spread": -0.0048, + "reliable": true, + "liquid_breadth_top_n": 1500, + "liquid_min_price": 5.0, + "avg_raw_pool": 2679.7, + "avg_eligible_pre_mask": 2128.6, + "mask_binds_pct": 95.6 + }, + { + "signal": "reversal_1m", + "weeks": 89, + "avg_cross_section": 1499.0, + "mean_ic": 0.0156, + "ic_t_stat": 1.37, + "ic_positive_pct": 59.6, + "mean_quintile_spread": -0.7882, + "reliable": true, + "liquid_breadth_top_n": 1500, + "liquid_min_price": 5.0, + "avg_raw_pool": 2700.0, + "avg_eligible_pre_mask": 2085.8, + "mask_binds_pct": 94.5 + }, + { + "signal": "mom_12_1_resid", + "weeks": 83, + "avg_cross_section": 1499.4, + "mean_ic": 0.0148, + "ic_t_stat": 1.02, + "ic_positive_pct": 57.8, + "mean_quintile_spread": 0.0181, + "reliable": true, + "liquid_breadth_top_n": 1500, + "liquid_min_price": 5.0, + "avg_raw_pool": 2553.0, + "avg_eligible_pre_mask": 2045.4, + "mask_binds_pct": 95.2 + }, + { + "signal": "mom_6_1", + "weeks": 88, + "avg_cross_section": 1499.3, + "mean_ic": 0.0101, + "ic_t_stat": 0.91, + "ic_positive_pct": 58.0, + "mean_quintile_spread": 0.0108, + "reliable": true, + "liquid_breadth_top_n": 1500, + "liquid_min_price": 5.0, + "avg_raw_pool": 2641.3, + "avg_eligible_pre_mask": 2098.8, + "mask_binds_pct": 95.5 + }, + { + "signal": "mom_12_1_sector_demeaned", + "weeks": 83, + "avg_cross_section": 483.2, + "mean_ic": 0.0076, + "ic_t_stat": 0.46, + "ic_positive_pct": 55.4, + "mean_quintile_spread": 0.0054, + "reliable": true, + "liquid_breadth_top_n": 1500, + "liquid_min_price": 5.0, + "avg_raw_pool": 484.4, + "avg_eligible_pre_mask": 483.2, + "mask_binds_pct": 0.0 + }, + { + "signal": "fip_id", + "weeks": 83, + "avg_cross_section": 1499.4, + "mean_ic": -0.0184, + "ic_t_stat": -2.11, + "ic_positive_pct": 45.8, + "mean_quintile_spread": -0.0059, + "reliable": true, + "liquid_breadth_top_n": 1500, + "liquid_min_price": 5.0, + "avg_raw_pool": 2553.0, + "avg_eligible_pre_mask": 2045.4, + "mask_binds_pct": 95.2 + }, + { + "signal": "vol_6m", + "weeks": 88, + "avg_cross_section": 1499.3, + "mean_ic": -0.0704, + "ic_t_stat": -3.14, + "ic_positive_pct": 35.2, + "mean_quintile_spread": 0.0051, + "reliable": true, + "liquid_breadth_top_n": 1500, + "liquid_min_price": 5.0, + "avg_raw_pool": 2641.3, + "avg_eligible_pre_mask": 2098.8, + "mask_binds_pct": 95.5 + } + ], + "signal_eval_by_name": { + "high_52w": { + "signal": "high_52w", + "weeks": 84, + "avg_cross_section": 1499.2, + "mean_ic": 0.0761, + "ic_t_stat": 4.46, + "ic_positive_pct": 71.4, + "mean_quintile_spread": 0.0118, + "reliable": true, + "liquid_breadth_top_n": 1500, + "liquid_min_price": 5.0, + "avg_raw_pool": 2564.4, + "avg_eligible_pre_mask": 2047.0, + "mask_binds_pct": 94.0 + }, + "trend_200": { + "signal": "trend_200", + "weeks": 85, + "avg_cross_section": 1499.6, + "mean_ic": 0.0371, + "ic_t_stat": 2.63, + "ic_positive_pct": 62.4, + "mean_quintile_spread": -0.7839, + "reliable": true, + "liquid_breadth_top_n": 1500, + "liquid_min_price": 5.0, + "avg_raw_pool": 2583.0, + "avg_eligible_pre_mask": 2065.8, + "mask_binds_pct": 96.5 + }, + "mom_12_1": { + "signal": "mom_12_1", + "weeks": 83, + "avg_cross_section": 1499.4, + "mean_ic": 0.0355, + "ic_t_stat": 2.41, + "ic_positive_pct": 62.7, + "mean_quintile_spread": 0.0141, + "reliable": true, + "liquid_breadth_top_n": 1500, + "liquid_min_price": 5.0, + "avg_raw_pool": 2553.0, + "avg_eligible_pre_mask": 2045.4, + "mask_binds_pct": 95.2 + }, + "mom_12_1_sector_resid": { + "signal": "mom_12_1_sector_resid", + "weeks": 83, + "avg_cross_section": 480.0, + "mean_ic": 0.0268, + "ic_t_stat": 1.69, + "ic_positive_pct": 60.2, + "mean_quintile_spread": 0.0126, + "reliable": true, + "liquid_breadth_top_n": 1500, + "liquid_min_price": 5.0, + "avg_raw_pool": 481.2, + "avg_eligible_pre_mask": 480.0, + "mask_binds_pct": 0.0 + }, + "mom_3_1": { + "signal": "mom_3_1", + "weeks": 90, + "avg_cross_section": 1498.6, + "mean_ic": 0.0256, + "ic_t_stat": 2.24, + "ic_positive_pct": 61.1, + "mean_quintile_spread": -0.0048, + "reliable": true, + "liquid_breadth_top_n": 1500, + "liquid_min_price": 5.0, + "avg_raw_pool": 2679.7, + "avg_eligible_pre_mask": 2128.6, + "mask_binds_pct": 95.6 + }, + "reversal_1m": { + "signal": "reversal_1m", + "weeks": 89, + "avg_cross_section": 1499.0, + "mean_ic": 0.0156, + "ic_t_stat": 1.37, + "ic_positive_pct": 59.6, + "mean_quintile_spread": -0.7882, + "reliable": true, + "liquid_breadth_top_n": 1500, + "liquid_min_price": 5.0, + "avg_raw_pool": 2700.0, + "avg_eligible_pre_mask": 2085.8, + "mask_binds_pct": 94.5 + }, + "mom_12_1_resid": { + "signal": "mom_12_1_resid", + "weeks": 83, + "avg_cross_section": 1499.4, + "mean_ic": 0.0148, + "ic_t_stat": 1.02, + "ic_positive_pct": 57.8, + "mean_quintile_spread": 0.0181, + "reliable": true, + "liquid_breadth_top_n": 1500, + "liquid_min_price": 5.0, + "avg_raw_pool": 2553.0, + "avg_eligible_pre_mask": 2045.4, + "mask_binds_pct": 95.2 + }, + "mom_6_1": { + "signal": "mom_6_1", + "weeks": 88, + "avg_cross_section": 1499.3, + "mean_ic": 0.0101, + "ic_t_stat": 0.91, + "ic_positive_pct": 58.0, + "mean_quintile_spread": 0.0108, + "reliable": true, + "liquid_breadth_top_n": 1500, + "liquid_min_price": 5.0, + "avg_raw_pool": 2641.3, + "avg_eligible_pre_mask": 2098.8, + "mask_binds_pct": 95.5 + }, + "mom_12_1_sector_demeaned": { + "signal": "mom_12_1_sector_demeaned", + "weeks": 83, + "avg_cross_section": 483.2, + "mean_ic": 0.0076, + "ic_t_stat": 0.46, + "ic_positive_pct": 55.4, + "mean_quintile_spread": 0.0054, + "reliable": true, + "liquid_breadth_top_n": 1500, + "liquid_min_price": 5.0, + "avg_raw_pool": 484.4, + "avg_eligible_pre_mask": 483.2, + "mask_binds_pct": 0.0 + }, + "fip_id": { + "signal": "fip_id", + "weeks": 83, + "avg_cross_section": 1499.4, + "mean_ic": -0.0184, + "ic_t_stat": -2.11, + "ic_positive_pct": 45.8, + "mean_quintile_spread": -0.0059, + "reliable": true, + "liquid_breadth_top_n": 1500, + "liquid_min_price": 5.0, + "avg_raw_pool": 2553.0, + "avg_eligible_pre_mask": 2045.4, + "mask_binds_pct": 95.2 + }, + "vol_6m": { + "signal": "vol_6m", + "weeks": 88, + "avg_cross_section": 1499.3, + "mean_ic": -0.0704, + "ic_t_stat": -3.14, + "ic_positive_pct": 35.2, + "mean_quintile_spread": 0.0051, + "reliable": true, + "liquid_breadth_top_n": 1500, + "liquid_min_price": 5.0, + "avg_raw_pool": 2641.3, + "avg_eligible_pre_mask": 2098.8, + "mask_binds_pct": 95.5 + } + }, + "era_split": { + "era_split_date": "2021-01-01", + "note": "Diagnostic only \u2014 not a tuning input.", + "pre_2021": { + "high_52w": { + "signal": "high_52w", + "weeks": 36, + "avg_cross_section": 1498.1, + "mean_ic": 0.0535, + "ic_t_stat": 2.26, + "ic_positive_pct": 69.4, + "mean_quintile_spread": -0.0027, + "reliable": true, + "liquid_breadth_top_n": 1500, + "liquid_min_price": 5.0, + "avg_raw_pool": 1894.7, + "avg_eligible_pre_mask": 1670.6, + "mask_binds_pct": 86.1 + }, + "trend_200": { + "signal": "trend_200", + "weeks": 38, + "avg_cross_section": 1499.1, + "mean_ic": 0.0373, + "ic_t_stat": 1.99, + "ic_positive_pct": 65.8, + "mean_quintile_spread": -1.7748, + "reliable": true, + "liquid_breadth_top_n": 1500, + "liquid_min_price": 5.0, + "avg_raw_pool": 1913.9, + "avg_eligible_pre_mask": 1687.4, + "mask_binds_pct": 92.1 + }, + "mom_12_1": { + "signal": "mom_12_1", + "weeks": 36, + "avg_cross_section": 1498.6, + "mean_ic": 0.037, + "ic_t_stat": 1.92, + "ic_positive_pct": 63.9, + "mean_quintile_spread": 0.0096, + "reliable": true, + "liquid_breadth_top_n": 1500, + "liquid_min_price": 5.0, + "avg_raw_pool": 1897.3, + "avg_eligible_pre_mask": 1673.1, + "mask_binds_pct": 88.9 + }, + "mom_3_1": { + "signal": "mom_3_1", + "weeks": 42, + "avg_cross_section": 1497.0, + "mean_ic": 0.0273, + "ic_t_stat": 2.03, + "ic_positive_pct": 64.3, + "mean_quintile_spread": -0.027, + "reliable": true, + "liquid_breadth_top_n": 1500, + "liquid_min_price": 5.0, + "avg_raw_pool": 1954.5, + "avg_eligible_pre_mask": 1717.3, + "mask_binds_pct": 90.5 + }, + "mom_12_1_resid": { + "signal": "mom_12_1_resid", + "weeks": 36, + "avg_cross_section": 1498.6, + "mean_ic": 0.0216, + "ic_t_stat": 1.09, + "ic_positive_pct": 63.9, + "mean_quintile_spread": 0.0284, + "reliable": true, + "liquid_breadth_top_n": 1500, + "liquid_min_price": 5.0, + "avg_raw_pool": 1897.3, + "avg_eligible_pre_mask": 1673.1, + "mask_binds_pct": 88.9 + }, + "reversal_1m": { + "signal": "reversal_1m", + "weeks": 42, + "avg_cross_section": 1497.8, + "mean_ic": 0.0208, + "ic_t_stat": 1.45, + "ic_positive_pct": 64.3, + "mean_quintile_spread": -1.6753, + "reliable": true, + "liquid_breadth_top_n": 1500, + "liquid_min_price": 5.0, + "avg_raw_pool": 1976.6, + "avg_eligible_pre_mask": 1650.9, + "mask_binds_pct": 88.6 + }, + "mom_12_1_sector_resid": { + "signal": "mom_12_1_sector_resid", + "weeks": 36, + "avg_cross_section": 460.7, + "mean_ic": 0.0149, + "ic_t_stat": 0.64, + "ic_positive_pct": 58.3, + "mean_quintile_spread": 0.0063, + "reliable": true, + "liquid_breadth_top_n": 1500, + "liquid_min_price": 5.0, + "avg_raw_pool": 462.8, + "avg_eligible_pre_mask": 460.7, + "mask_binds_pct": 0.0 + }, + "mom_6_1": { + "signal": "mom_6_1", + "weeks": 40, + "avg_cross_section": 1498.5, + "mean_ic": 0.0082, + "ic_t_stat": 0.58, + "ic_positive_pct": 57.5, + "mean_quintile_spread": 0.0163, + "reliable": true, + "liquid_breadth_top_n": 1500, + "liquid_min_price": 5.0, + "avg_raw_pool": 1935.1, + "avg_eligible_pre_mask": 1702.2, + "mask_binds_pct": 90.0 + }, + "mom_12_1_sector_demeaned": { + "signal": "mom_12_1_sector_demeaned", + "weeks": 36, + "avg_cross_section": 469.4, + "mean_ic": 0.0031, + "ic_t_stat": 0.13, + "ic_positive_pct": 55.6, + "mean_quintile_spread": 0.0009, + "reliable": true, + "liquid_breadth_top_n": 1500, + "liquid_min_price": 5.0, + "avg_raw_pool": 471.5, + "avg_eligible_pre_mask": 469.4, + "mask_binds_pct": 0.0 + }, + "fip_id": { + "signal": "fip_id", + "weeks": 36, + "avg_cross_section": 1498.6, + "mean_ic": -0.0116, + "ic_t_stat": -0.93, + "ic_positive_pct": 52.8, + "mean_quintile_spread": -0.0037, + "reliable": true, + "liquid_breadth_top_n": 1500, + "liquid_min_price": 5.0, + "avg_raw_pool": 1897.3, + "avg_eligible_pre_mask": 1673.1, + "mask_binds_pct": 88.9 + }, + "vol_6m": { + "signal": "vol_6m", + "weeks": 40, + "avg_cross_section": 1498.5, + "mean_ic": -0.0219, + "ic_t_stat": -0.77, + "ic_positive_pct": 40.0, + "mean_quintile_spread": 0.0381, + "reliable": true, + "liquid_breadth_top_n": 1500, + "liquid_min_price": 5.0, + "avg_raw_pool": 1935.1, + "avg_eligible_pre_mask": 1702.2, + "mask_binds_pct": 90.0 + } + }, + "post_2021": { + "high_52w": { + "signal": "high_52w", + "weeks": 48, + "avg_cross_section": 1500.0, + "mean_ic": 0.0831, + "ic_t_stat": 2.36, + "ic_positive_pct": 68.8, + "mean_quintile_spread": 0.0102, + "reliable": true, + "liquid_breadth_top_n": 1500, + "liquid_min_price": 5.0, + "avg_raw_pool": 3046.7, + "avg_eligible_pre_mask": 2325.3, + "mask_binds_pct": 100.0 + }, + "mom_12_1_sector_resid": { + "signal": "mom_12_1_sector_resid", + "weeks": 48, + "avg_cross_section": 494.5, + "mean_ic": 0.033, + "ic_t_stat": 1.39, + "ic_positive_pct": 60.4, + "mean_quintile_spread": 0.0158, + "reliable": true, + "liquid_breadth_top_n": 1500, + "liquid_min_price": 5.0, + "avg_raw_pool": 495.1, + "avg_eligible_pre_mask": 494.5, + "mask_binds_pct": 0.0 + }, + "mom_6_1": { + "signal": "mom_6_1", + "weeks": 48, + "avg_cross_section": 1500.0, + "mean_ic": 0.0314, + "ic_t_stat": 1.51, + "ic_positive_pct": 58.3, + "mean_quintile_spread": 0.0109, + "reliable": true, + "liquid_breadth_top_n": 1500, + "liquid_min_price": 5.0, + "avg_raw_pool": 3214.4, + "avg_eligible_pre_mask": 2425.9, + "mask_binds_pct": 100.0 + }, + "mom_12_1": { + "signal": "mom_12_1", + "weeks": 48, + "avg_cross_section": 1500.0, + "mean_ic": 0.0279, + "ic_t_stat": 1.14, + "ic_positive_pct": 62.5, + "mean_quintile_spread": 0.0206, + "reliable": true, + "liquid_breadth_top_n": 1500, + "liquid_min_price": 5.0, + "avg_raw_pool": 3045.2, + "avg_eligible_pre_mask": 2324.2, + "mask_binds_pct": 100.0 + }, + "trend_200": { + "signal": "trend_200", + "weeks": 48, + "avg_cross_section": 1500.0, + "mean_ic": 0.0254, + "ic_t_stat": 1.03, + "ic_positive_pct": 60.4, + "mean_quintile_spread": 0.0064, + "reliable": true, + "liquid_breadth_top_n": 1500, + "liquid_min_price": 5.0, + "avg_raw_pool": 3113.5, + "avg_eligible_pre_mask": 2365.0, + "mask_binds_pct": 100.0 + }, + "mom_12_1_sector_demeaned": { + "signal": "mom_12_1_sector_demeaned", + "weeks": 48, + "avg_cross_section": 493.5, + "mean_ic": 0.0071, + "ic_t_stat": 0.3, + "ic_positive_pct": 52.1, + "mean_quintile_spread": 0.0078, + "reliable": true, + "liquid_breadth_top_n": 1500, + "liquid_min_price": 5.0, + "avg_raw_pool": 494.1, + "avg_eligible_pre_mask": 493.5, + "mask_binds_pct": 0.0 + }, + "mom_12_1_resid": { + "signal": "mom_12_1_resid", + "weeks": 48, + "avg_cross_section": 1500.0, + "mean_ic": 0.0067, + "ic_t_stat": 0.29, + "ic_positive_pct": 54.2, + "mean_quintile_spread": 0.0149, + "reliable": true, + "liquid_breadth_top_n": 1500, + "liquid_min_price": 5.0, + "avg_raw_pool": 3045.2, + "avg_eligible_pre_mask": 2324.2, + "mask_binds_pct": 100.0 + }, + "mom_3_1": { + "signal": "mom_3_1", + "weeks": 48, + "avg_cross_section": 1500.0, + "mean_ic": -0.0014, + "ic_t_stat": -0.06, + "ic_positive_pct": 52.1, + "mean_quintile_spread": -0.0028, + "reliable": true, + "liquid_breadth_top_n": 1500, + "liquid_min_price": 5.0, + "avg_raw_pool": 3305.8, + "avg_eligible_pre_mask": 2486.0, + "mask_binds_pct": 100.0 + }, + "reversal_1m": { + "signal": "reversal_1m", + "weeks": 48, + "avg_cross_section": 1500.0, + "mean_ic": -0.013, + "ic_t_stat": -0.69, + "ic_positive_pct": 41.7, + "mean_quintile_spread": -0.0099, + "reliable": true, + "liquid_breadth_top_n": 1500, + "liquid_min_price": 5.0, + "avg_raw_pool": 3367.0, + "avg_eligible_pre_mask": 2486.9, + "mask_binds_pct": 100.0 + }, + "fip_id": { + "signal": "fip_id", + "weeks": 48, + "avg_cross_section": 1500.0, + "mean_ic": -0.019, + "ic_t_stat": -1.56, + "ic_positive_pct": 41.7, + "mean_quintile_spread": -0.015, + "reliable": true, + "liquid_breadth_top_n": 1500, + "liquid_min_price": 5.0, + "avg_raw_pool": 3045.2, + "avg_eligible_pre_mask": 2324.2, + "mask_binds_pct": 100.0 + }, + "vol_6m": { + "signal": "vol_6m", + "weeks": 48, + "avg_cross_section": 1500.0, + "mean_ic": -0.0943, + "ic_t_stat": -2.41, + "ic_positive_pct": 31.2, + "mean_quintile_spread": -0.0031, + "reliable": true, + "liquid_breadth_top_n": 1500, + "liquid_min_price": 5.0, + "avg_raw_pool": 3214.4, + "avg_eligible_pre_mask": 2425.9, + "mask_binds_pct": 100.0 + } + } + }, + "identical_subset_sector_cs": { + "mom_12_1_sector_resid": { + "signal": "mom_12_1_sector_resid", + "weeks": 83, + "avg_cross_section": 480.0, + "mean_ic": 0.0268, + "ic_t_stat": 1.69, + "ic_positive_pct": 60.2, + "mean_quintile_spread": 0.0126, + "reliable": true, + "liquid_breadth_top_n": 1500, + "liquid_min_price": 5.0, + "avg_raw_pool": 481.2, + "avg_eligible_pre_mask": 480.0, + "mask_binds_pct": 0.0 + }, + "mom_12_1_resid": { + "signal": "mom_12_1_resid", + "weeks": 83, + "avg_cross_section": 480.0, + "mean_ic": 0.0251, + "ic_t_stat": 1.3, + "ic_positive_pct": 57.8, + "mean_quintile_spread": 0.011, + "reliable": true, + "liquid_breadth_top_n": 1500, + "liquid_min_price": 5.0, + "avg_raw_pool": 481.2, + "avg_eligible_pre_mask": 480.0, + "mask_binds_pct": 0.0 + }, + "mom_12_1": { + "signal": "mom_12_1", + "weeks": 83, + "avg_cross_section": 480.0, + "mean_ic": 0.0192, + "ic_t_stat": 0.92, + "ic_positive_pct": 56.6, + "mean_quintile_spread": 0.0099, + "reliable": true, + "liquid_breadth_top_n": 1500, + "liquid_min_price": 5.0, + "avg_raw_pool": 481.2, + "avg_eligible_pre_mask": 480.0, + "mask_binds_pct": 0.0 + }, + "mom_12_1_sector_demeaned": { + "signal": "mom_12_1_sector_demeaned", + "weeks": 83, + "avg_cross_section": 479.0, + "mean_ic": 0.0063, + "ic_t_stat": 0.39, + "ic_positive_pct": 55.4, + "mean_quintile_spread": 0.0048, + "reliable": true, + "liquid_breadth_top_n": 1500, + "liquid_min_price": 5.0, + "avg_raw_pool": 480.2, + "avg_eligible_pre_mask": 479.0, + "mask_binds_pct": 0.0 + } + }, + "mask_diagnostics": { + "reference_signal": "vol_6m", + "avg_cross_section": 1499.3, + "avg_raw_pool": 2641.3, + "avg_eligible_pre_mask": 2098.8, + "mask_binds_pct": 95.5, + "weeks": 88 + }, + "sector_map_size": 505, + "sector_etfs_loaded": [ + "XLB", + "XLC", + "XLE", + "XLF", + "XLI", + "XLK", + "XLP", + "XLRE", + "XLU", + "XLV", + "XLY" + ], + "spy_bars": 2649 + }, + "grade": { + "verdict": "FAIL", + "reason": "failed one or more pre-registered checks (see checks)", + "checks": { + "sector_row": { + "signal": "mom_12_1_sector_resid", + "weeks": 83, + "avg_cross_section": 480.0, + "mean_ic": 0.0268, + "ic_t_stat": 1.69, + "ic_positive_pct": 60.2, + "mean_quintile_spread": 0.0126, + "reliable": true, + "liquid_breadth_top_n": 1500, + "liquid_min_price": 5.0, + "avg_raw_pool": 481.2, + "avg_eligible_pre_mask": 480.0, + "mask_binds_pct": 0.0 + }, + "resid_row_for_t": { + "signal": "mom_12_1_resid", + "weeks": 83, + "avg_cross_section": 480.0, + "mean_ic": 0.0251, + "ic_t_stat": 1.3, + "ic_positive_pct": 57.8, + "mean_quintile_spread": 0.011, + "reliable": true, + "liquid_breadth_top_n": 1500, + "liquid_min_price": 5.0, + "avg_raw_pool": 481.2, + "avg_eligible_pre_mask": 480.0, + "mask_binds_pct": 0.0 + }, + "resid_t_source": "identical_subset", + "pre_2021": { + "signal": "mom_12_1_sector_resid", + "weeks": 36, + "avg_cross_section": 460.7, + "mean_ic": 0.0149, + "ic_t_stat": 0.64, + "ic_positive_pct": 58.3, + "mean_quintile_spread": 0.0063, + "reliable": true, + "liquid_breadth_top_n": 1500, + "liquid_min_price": 5.0, + "avg_raw_pool": 462.8, + "avg_eligible_pre_mask": 460.7, + "mask_binds_pct": 0.0 + }, + "post_2021": { + "signal": "mom_12_1_sector_resid", + "weeks": 48, + "avg_cross_section": 494.5, + "mean_ic": 0.033, + "ic_t_stat": 1.39, + "ic_positive_pct": 60.4, + "mean_quintile_spread": 0.0158, + "reliable": true, + "liquid_breadth_top_n": 1500, + "liquid_min_price": 5.0, + "avg_raw_pool": 495.1, + "avg_eligible_pre_mask": 494.5, + "mask_binds_pct": 0.0 + }, + "abs_mean_ic_ge_0_03": false, + "sign_positive": true, + "reliable": true, + "weeks_ge_50": true, + "weeks": 83, + "t_ge_resid_same_cs": true, + "sector_t": 1.69, + "resid_t": 1.3, + "era_both_present": true, + "era_sign_consistent_positive": true, + "pre_ic": 0.0149, + "post_ic": 0.033, + "avg_cross_section": 480.0 + }, + "headline": "Task 1 CLOSED \u2014 sector residual dead on deep evidence." + }, + "pending_human": true, + "note": "Nothing merged into production. Thread ends at PASS/FAIL." +} diff --git a/reports/sector-resid-deep-20260719-113319.md b/reports/sector-resid-deep-20260719-113319.md new file mode 100644 index 0000000..ceae7fe --- /dev/null +++ b/reports/sector-resid-deep-20260719-113319.md @@ -0,0 +1,341 @@ +# Sector-residual deep test (masked, repaired snapshot) + +Generated: `2026-07-19T11:33:19.102779` + +> **SURVIVORSHIP BIAS: today's constituents backfilled. Relative IC only — not levels.** + +## Pre-registered grade (mechanical) + +**Verdict: FAIL** + +Task 1 CLOSED — sector residual dead on deep evidence. + +Reason: failed one or more pre-registered checks (see checks) + +```json +{ + "sector_row": { + "signal": "mom_12_1_sector_resid", + "weeks": 83, + "avg_cross_section": 480.0, + "mean_ic": 0.0268, + "ic_t_stat": 1.69, + "ic_positive_pct": 60.2, + "mean_quintile_spread": 0.0126, + "reliable": true, + "liquid_breadth_top_n": 1500, + "liquid_min_price": 5.0, + "avg_raw_pool": 481.2, + "avg_eligible_pre_mask": 480.0, + "mask_binds_pct": 0.0 + }, + "resid_row_for_t": { + "signal": "mom_12_1_resid", + "weeks": 83, + "avg_cross_section": 480.0, + "mean_ic": 0.0251, + "ic_t_stat": 1.3, + "ic_positive_pct": 57.8, + "mean_quintile_spread": 0.011, + "reliable": true, + "liquid_breadth_top_n": 1500, + "liquid_min_price": 5.0, + "avg_raw_pool": 481.2, + "avg_eligible_pre_mask": 480.0, + "mask_binds_pct": 0.0 + }, + "resid_t_source": "identical_subset", + "pre_2021": { + "signal": "mom_12_1_sector_resid", + "weeks": 36, + "avg_cross_section": 460.7, + "mean_ic": 0.0149, + "ic_t_stat": 0.64, + "ic_positive_pct": 58.3, + "mean_quintile_spread": 0.0063, + "reliable": true, + "liquid_breadth_top_n": 1500, + "liquid_min_price": 5.0, + "avg_raw_pool": 462.8, + "avg_eligible_pre_mask": 460.7, + "mask_binds_pct": 0.0 + }, + "post_2021": { + "signal": "mom_12_1_sector_resid", + "weeks": 48, + "avg_cross_section": 494.5, + "mean_ic": 0.033, + "ic_t_stat": 1.39, + "ic_positive_pct": 60.4, + "mean_quintile_spread": 0.0158, + "reliable": true, + "liquid_breadth_top_n": 1500, + "liquid_min_price": 5.0, + "avg_raw_pool": 495.1, + "avg_eligible_pre_mask": 494.5, + "mask_binds_pct": 0.0 + }, + "abs_mean_ic_ge_0_03": false, + "sign_positive": true, + "reliable": true, + "weeks_ge_50": true, + "weeks": 83, + "t_ge_resid_same_cs": true, + "sector_t": 1.69, + "resid_t": 1.3, + "era_both_present": true, + "era_sign_consistent_positive": true, + "pre_ic": 0.0149, + "post_ic": 0.033, + "avg_cross_section": 480.0 +} +``` + +## Step-1 sanity + +```json +{ + "skipped": true, + "sanity": { + "passed": true, + "megacap": { + "AAPL": { + "symbol": "AAPL", + "bars": 2649, + "min_date": "2016-01-04", + "max_date": "2026-07-17" + }, + "MSFT": { + "symbol": "MSFT", + "bars": 2649, + "min_date": "2016-01-04", + "max_date": "2026-07-17" + }, + "JPM": { + "symbol": "JPM", + "bars": 2649, + "min_date": "2016-01-04", + "max_date": "2026-07-17" + }, + "XOM": { + "symbol": "XOM", + "bars": 2649, + "min_date": "2016-01-04", + "max_date": "2026-07-17" + }, + "JNJ": { + "symbol": "JNJ", + "bars": 2649, + "min_date": "2016-01-04", + "max_date": "2026-07-17" + } + }, + "megacap_ok": true, + "megacap_reasons": [], + "feed_floor": "2016-01-04", + "spy_benchmark": { + "n": 2649, + "min": "2016-01-04", + "max": "2026-07-17" + }, + "old_shallow_floor": "2020-01-01", + "sector_etfs": { + "XLB": { + "n": 2649, + "min": "2016-01-04", + "max": "2026-07-17" + }, + "XLC": { + "n": 2030, + "min": "2018-06-19", + "max": "2026-07-17" + }, + "XLE": { + "n": 2649, + "min": "2016-01-04", + "max": "2026-07-17" + }, + "XLF": { + "n": 2649, + "min": "2016-01-04", + "max": "2026-07-17" + }, + "XLI": { + "n": 2649, + "min": "2016-01-04", + "max": "2026-07-17" + }, + "XLK": { + "n": 2649, + "min": "2016-01-04", + "max": "2026-07-17" + }, + "XLP": { + "n": 2649, + "min": "2016-01-04", + "max": "2026-07-17" + }, + "XLRE": { + "n": 2649, + "min": "2016-01-04", + "max": "2026-07-17" + }, + "XLU": { + "n": 2649, + "min": "2016-01-04", + "max": "2026-07-17" + }, + "XLV": { + "n": 2649, + "min": "2016-01-04", + "max": "2026-07-17" + }, + "XLY": { + "n": 2649, + "min": "2016-01-04", + "max": "2026-07-17" + } + }, + "sector_etfs_deep_count": 11, + "sector_etfs_ok": true, + "sector_etf_reasons": [], + "still_shallow_count": 2882, + "still_shallow_sample": [ + "AACB", + "AACBR", + "AACBU", + "AACI", + "AACIU", + "AACIW", + "AACO", + "AACOU", + "AACOW", + "AACP", + "AACPR", + "AACPU", + "AACPW", + "AAPG", + "AARD", + "ABAT", + "ABCL", + "ABLV", + "ABLVW", + "ABNB" + ], + "still_shallow_note": "Remaining 'shallow' names are mostly post-2017 IPOs/listings \u2014 expected, not a two-tier defect.", + "xlc_note": "XLC lists mid-2018 \u2192 Communication Services residual coverage from ~mid-2019.", + "feed_note": "Empirical Alpaca floor observed via SPY: 2016-01-04 (n=2649). Calendar history_days=5000 is a request cap, not a guarantee \u2014 sanity grades against the feed floor, not 5000 calendar days.", + "target_history_days": 5000 + } +} +``` + +## Mask diagnostics + +```json +{ + "reference_signal": "vol_6m", + "avg_cross_section": 1499.3, + "avg_raw_pool": 2641.3, + "avg_eligible_pre_mask": 2098.8, + "mask_binds_pct": 95.5, + "weeks": 88 +} +``` + +## Signal table (rows only — no narrative for non-sector signals) + +| signal | mean_ic | t | weeks | avg_N | reliable | +|---|---:|---:|---:|---:|---| +| fip_id | -0.0184 | -2.11 | 83 | 1499.4 | True | +| high_52w | 0.0761 | 4.46 | 84 | 1499.2 | True | +| mom_12_1 | 0.0355 | 2.41 | 83 | 1499.4 | True | +| mom_12_1_resid | 0.0148 | 1.02 | 83 | 1499.4 | True | +| mom_12_1_sector_demeaned | 0.0076 | 0.46 | 83 | 483.2 | True | +| mom_12_1_sector_resid | 0.0268 | 1.69 | 83 | 480.0 | True | +| mom_3_1 | 0.0256 | 2.24 | 90 | 1498.6 | True | +| mom_6_1 | 0.0101 | 0.91 | 88 | 1499.3 | True | +| reversal_1m | 0.0156 | 1.37 | 89 | 1499.0 | True | +| trend_200 | 0.0371 | 2.63 | 85 | 1499.6 | True | +| vol_6m | -0.0704 | -3.14 | 88 | 1499.3 | True | + +### Era split — mom_12_1_sector_resid only (for grade) + +| era | IC | t | weeks | N | +|---|---:|---:|---:|---:| +| pre_2021 | 0.0149 | 0.64 | 36 | 460.7 | +| post_2021 | 0.033 | 1.39 | 48 | 494.5 | + +### Identical-subset baselines (sector CS) + +```json +{ + "mom_12_1_sector_resid": { + "signal": "mom_12_1_sector_resid", + "weeks": 83, + "avg_cross_section": 480.0, + "mean_ic": 0.0268, + "ic_t_stat": 1.69, + "ic_positive_pct": 60.2, + "mean_quintile_spread": 0.0126, + "reliable": true, + "liquid_breadth_top_n": 1500, + "liquid_min_price": 5.0, + "avg_raw_pool": 481.2, + "avg_eligible_pre_mask": 480.0, + "mask_binds_pct": 0.0 + }, + "mom_12_1_resid": { + "signal": "mom_12_1_resid", + "weeks": 83, + "avg_cross_section": 480.0, + "mean_ic": 0.0251, + "ic_t_stat": 1.3, + "ic_positive_pct": 57.8, + "mean_quintile_spread": 0.011, + "reliable": true, + "liquid_breadth_top_n": 1500, + "liquid_min_price": 5.0, + "avg_raw_pool": 481.2, + "avg_eligible_pre_mask": 480.0, + "mask_binds_pct": 0.0 + }, + "mom_12_1": { + "signal": "mom_12_1", + "weeks": 83, + "avg_cross_section": 480.0, + "mean_ic": 0.0192, + "ic_t_stat": 0.92, + "ic_positive_pct": 56.6, + "mean_quintile_spread": 0.0099, + "reliable": true, + "liquid_breadth_top_n": 1500, + "liquid_min_price": 5.0, + "avg_raw_pool": 481.2, + "avg_eligible_pre_mask": 480.0, + "mask_binds_pct": 0.0 + }, + "mom_12_1_sector_demeaned": { + "signal": "mom_12_1_sector_demeaned", + "weeks": 83, + "avg_cross_section": 479.0, + "mean_ic": 0.0063, + "ic_t_stat": 0.39, + "ic_positive_pct": 55.4, + "mean_quintile_spread": 0.0048, + "reliable": true, + "liquid_breadth_top_n": 1500, + "liquid_min_price": 5.0, + "avg_raw_pool": 480.2, + "avg_eligible_pre_mask": 479.0, + "mask_binds_pct": 0.0 + } +} +``` + +## Status + +PENDING_HUMAN beyond the mechanical PASS/FAIL above. Nothing merged into production docs or prod code. + +JSON: `reports/sector-resid-deep-20260719-113319.json` + From 9717d8176bdb68cf072c71644679ee6dd8732266 Mon Sep 17 00:00:00 2001 From: Dennis Thiessen Date: Sun, 19 Jul 2026 11:45:15 +0200 Subject: [PATCH 10/14] research: archive Task 1 sector residual as CLOSED/REJECTED Deep masked retest failed the iron IC bar (0.027 < 0.03). Log as rejected #13 in research README; close sector-residual and history-depth docs. Production market residual unchanged. --- docs/research/README.md | 1 + docs/research/history-depth-extension.md | 39 +++++++------ docs/research/sector-residual-momentum.md | 67 ++++++++++++----------- 3 files changed, 57 insertions(+), 50 deletions(-) diff --git a/docs/research/README.md b/docs/research/README.md index 47e5add..0bd30b6 100644 --- a/docs/research/README.md +++ b/docs/research/README.md @@ -47,6 +47,7 @@ score, Structural S/R, the Gate Target Ladder, sentiment, fundamentals) is | 10 | **Inverse-vol position sizing** | The apparent "win" was **mis-attributed**: the 20% notional cap bound on 95% of entries, so it measured concentration, not vol-sizing. Genuine inverse-vol cuts DD to −18.2% but costs ~58pp return at flat Sharpe | **Rejected** as edge; it's a risk-preference trade | `backtest-20260709-position-sizing*.json` | | 11 | **FIP path-smoothness** as tie-breaker/filter | Non-monotonic within the qualified set; thinning the entry stream costs more compounding than the tilt returns | **Rejected as a filter** — but see §4, it's the strongest raw signal we've measured | — | | 12 | **Fixed take-profit sweep** (R-multiples) | No interior optimum ever found — the best TP is "no TP" | **Rejected.** Momentum's edge lives in the right tail | `backtest_service.py:450` | +| 13 | **Sector-residual 12-1** (`mom_12_1_sector_resid` / sector demean) as replacement for market residual | Short-window IC/A/B looked knife-edge green; deep repaired + **liquid-1500** retest: weeks 83, mild +IC **0.027** / t 1.69, **below iron bar 0.03** (FAIL). Demean already weaker | **Rejected / closed.** Keep production market residual. Do not resurrect without a new pre-registered protocol | [sector-residual-momentum.md](sector-residual-momentum.md) · `sector-resid-deep-20260719-113319.json` · history-depth supersession note | --- diff --git a/docs/research/history-depth-extension.md b/docs/research/history-depth-extension.md index 3f9d97a..182b315 100644 --- a/docs/research/history-depth-extension.md +++ b/docs/research/history-depth-extension.md @@ -1,8 +1,9 @@ # History-depth extension (Tier-1 alpha research) -**Status:** sector-resid deep test **FAIL** — see supersession section. PENDING_HUMAN beyond PASS/FAIL. -**Branch:** `research/earnings-gap-and-sue` (MacBook commit `f6e0ca7`) +**Status:** **CLOSED.** Sector-residual deep test **FAIL** — Task 1 archived as rejected (see supersession). +**Branch:** `research/earnings-gap-and-sue` **Superseded artifact (do not cite):** `reports/history-depth-20260719-103315.json` — **UNMASKED, TWO-TIER SNAPSHOT** +**Authoritative sector grade:** `reports/sector-resid-deep-20260719-113319.json` **Production impact:** none. **Do not retune any production knob on deep history.** --- @@ -159,17 +160,11 @@ report does not change that without a separate A/B. Flag for human awareness onl ## What a human must decide next -1. **Sector residual:** keep research-only until either - (a) sector ETF + sector map cover the full deep window **and** IC is re-run - with weeks ≫ 35 on a documented universe, or - (b) explicitly accept short-window-only evidence (weaker case). -2. **Do not** merge sector residual into production from this depth run. -3. **Do not** retune residual vs raw, FIP, or vol blend from these IC tables +1. **Sector residual — decided:** CLOSED / REJECTED (archive complete). No wire-in. +2. **Do not** retune residual vs raw, FIP, or vol blend from deep IC tables without a pre-registered book A/B on the intended universe. -4. Optional follow-up: extend sector ETF history + sector labels to nasdaq_all, - re-run **only** sector residual IC on deep research.sqlite with race guard. -5. Optional: finish earnings backfill (48→506) and re-run SUE; depth alone did - not include SUE. +3. Optional (separate threads only): finish earnings backfill and re-run SUE; + snapshot per-symbol depth guard as tooling. --- @@ -187,12 +182,22 @@ report does not change that without a separate A/B. Flag for human awareness onl The table and interpretation from **`history-depth-20260719-103315`** are **UNMASKED, TWO-TIER SNAPSHOT — superseded, directional only, do not cite**. Prod-universe names (and sector residual coverage) were left shallow while breadth names were deepened; sector residual weeks=35 was a data gap. -### Sector-residual deep test outcome: **FAIL** +### Sector-residual deep test outcome: **FAIL** (archived) -Task 1 CLOSED — sector residual dead on deep evidence. +**Task 1 CLOSED / REJECTED** — sector residual dead on deep evidence. Archived in +the research log rejected table (#13). Do not resurrect without a new +pre-registered protocol. + +| check | result | +|---|---| +| weeks | **83** (data fix worked) | +| mean IC | **0.0268** (below 0.03 bar) → FAIL | +| t vs resid same CS | 1.69 ≥ 1.30 pass | +| era signs | both + pass | -- Reason: failed one or more pre-registered checks (see checks) - Artifact: `reports/sector-resid-deep-20260719-113319.json` -- Mechanical checks: see that report. +- Summary write-up: [sector-residual-momentum.md](sector-residual-momentum.md) -**Future snapshot rebuilds must verify per-symbol depth** (earliest-bar uniformity across the intended universe) — guard is a to-do, not part of this order. +**Future snapshot rebuilds must verify per-symbol depth** (earliest-bar +uniformity across the intended universe) — guard is a to-do, not part of this +order. diff --git a/docs/research/sector-residual-momentum.md b/docs/research/sector-residual-momentum.md index 90a15ee..8400bf0 100644 --- a/docs/research/sector-residual-momentum.md +++ b/docs/research/sector-residual-momentum.md @@ -1,9 +1,24 @@ # Sector-residual momentum (Tier-1 alpha research) -**Status:** **PROMOTE (to human design decision only)** — IC + A/B bars cleared; **do not ship**. -**Branch:** `research/sector-residual-momentum` -**Production impact:** none. Local research only. No scheduler / gate / prod-config changes. -**Artifacts:** `reports/sector-residual-20260719-083356.json` (+ companion `.md`) +**Status:** **CLOSED / REJECTED** — do not resurrect without a new pre-registered protocol. +**Branch:** `research/earnings-gap-and-sue` (final grade) · earlier short-window work on `research/sector-residual-momentum` +**Production impact:** none. Market residual 12-1 remains the production momentum leg. +**Authoritative deep grade:** `reports/sector-resid-deep-20260719-113319.json` (**FAIL**) +**Short-window A/B (superseded for promotion):** `reports/sector-residual-20260719-083356.json` — knife-edge only; not decisive after deep masked retest. + +### Closure (2026-07-19) + +Pre-registered deep test on repaired snapshot + liquid-1500 mask: + +| check | result | +|---|---| +| weeks extended (≫ 35) | pass (83) | +| sign +, reliable, eras both + | pass | +| t ≥ `mom_12_1_resid` same CS | pass (1.69 ≥ 1.30) | +| \|mean IC\| ≥ 0.03 | **fail (0.0268)** | + +**Verdict:** Task 1 CLOSED — sector residual dead on deep evidence. +`mom_12_1_sector_demeaned` remains DEAD for promotion. No further sector-residual variants from this thread. --- @@ -176,45 +191,31 @@ gates a slightly larger set). --- -## Verdict +## Verdict (final — archived) | signal | verdict | note | |---|---|---| -| **`mom_12_1_sector_resid`** | **PROMOTE → human wire-in decision** | IC modestly beats market residual; A/B clears pre-reg bar narrowly. **Do not ship from this branch.** | -| **`mom_12_1_sector_demeaned`** | **DEAD** (for promotion) | Iron-rule IC magnitude ok, but t-stat loses to `mom_12_1_resid`. Cheap variant not competitive. | +| **`mom_12_1_sector_resid`** | **CLOSED / REJECTED** | Deep masked IC 0.0268 < 0.03 bar (`sector-resid-deep-20260719-113319`). Short-window PROMOTE superseded. | +| **`mom_12_1_sector_demeaned`** | **DEAD** | Never cleared t vs market residual; stays dead. | -### Read carefully (for the human) +Short-window evidence below is **historical only** (pre-deep retest). Do not use it +to reopen promotion. -1. **IC edge is real but small.** Sector residual IC 0.0578 / t 2.34 vs market - residual 0.0552 / t 1.98 on the **same** 35 windows — better consistency - (ic+ 65.7% vs 60%) and slightly higher mean, not a different factor class. -2. **A/B is not a clear Sharpe win.** Full-period Sharpe is flat (2.09). - Validation Sharpe is **lower** than control (2.57 vs 2.92) and only clears - the pre-registered “within 0.5 SE” cushion by ~0.001. Train improves; - validation worsens — classic regime-split noise on ~2 years. -3. **Risk side is friendly.** Full max DD improves (19.8% vs 21.4%); train DD - also better. Matches the “lower factor vol” half of the hypothesis more than - the “higher Sharpe” half on this window. -4. **Survivorship / short history.** Same caveats as all current research: - today’s constituents, ~35 independent weekly windows, one post-2021 regime - dominant. Task 3 (history depth) should re-check IC stability before any - wire-in. -5. **Not shipped.** Machinery lives on the research branch; production residual - path is untouched. +### Read carefully (archived context) + +1. Short-window IC (0.058 / t 2.34 vs resid 0.055 / t 1.98 on 35 weeks) and knife-edge + A/B looked openable — that was the data gap era (shallow prod bars). +2. Deep repaired + liquid-1500 retest closed the case: weeks 83, mild +IC, **below bar**. +3. Production keeps **market** residual 12-1. Research harness may still *emit* + sector residual for diagnostics; it is not a promotion candidate. +4. **Do not resurrect** without a new pre-registered protocol and new data. --- ## What a human must decide next -1. **Accept or reject** replacing `mom_12_1_resid` with `mom_12_1_sector_resid` - as the production residual (gate + 80/20 mom leg), **or** keep market residual - and treat sector residual as research-only. -2. If leaning accept: require **Task 3 history-depth** confirmation (IC era split - pre/post-2021) before any production PR. -3. Optional: run **sector-cap ≤3** A/B with full tail diagnostics (not run here). -4. **Do not** merge this verdict into main strategy docs without review. -5. Wire-in design (live sector map refresh, ETF series ops, fallback when sector - missing) is a **separate** approved engineering step. +**Nothing on Task 1** — archived. Optional: leave research machinery in tree +(harmless) or delete later as cleanup; not a strategy decision. --- From 9171e366ee58dc01bcdb1af6efb82d87cb16c601 Mon Sep 17 00:00:00 2001 From: Dennis Thiessen Date: Sun, 19 Jul 2026 11:58:52 +0200 Subject: [PATCH 11/14] research: prepare prod book universe x horizon 4-arm matrix Pre-register A-D (4y/2016 x 505/505+liquid) with unchanged production knobs. Runner caches full GTL candidates then re-ranks per arm; MacBook entry via run_tier1_macbook.sh --prod-book-matrix. --- docs/research/prod-book-universe-horizon.md | 87 +++ scripts/run_prod_book_universe_matrix.py | 706 ++++++++++++++++++++ scripts/run_tier1_macbook.sh | 17 +- 3 files changed, 809 insertions(+), 1 deletion(-) create mode 100644 docs/research/prod-book-universe-horizon.md create mode 100644 scripts/run_prod_book_universe_matrix.py diff --git a/docs/research/prod-book-universe-horizon.md b/docs/research/prod-book-universe-horizon.md new file mode 100644 index 0000000..070ad8f --- /dev/null +++ b/docs/research/prod-book-universe-horizon.md @@ -0,0 +1,87 @@ +# Production book × universe × horizon matrix + +**Status:** PRE-REGISTERED — prepare / MacBook run; no production changes. +**Branch:** `research/earnings-gap-and-sue` +**Runner:** `scripts/run_prod_book_universe_matrix.py` + +--- + +## Question + +How does the **live production book** (unchanged knobs) behave when we only vary: + +1. **History length** used for entries (≈4y vs since 2016-07) +2. **Tradable universe** (prod ~505 vs 505 + PIT liquid Nasdaq/breadth) + +No strategy modifications: same residual gate, 80/20 high-vol rank, GTL entry +machinery, 3× ATR trail, 30d max hold, gate-reset re-entry, `fill_mode=close`, +cost 10 bps/side, max 10, 1% risk. + +--- + +## Pre-registered arms (locked) + +| id | label | Entry start | Tradable universe | +|---|---|---|---| +| **A** | prod_4y_505 | **2022-07-01** | Prod ~505 only | +| **B** | prod_4y_505_liquid | **2022-07-01** | Prod ∪ liquid top-1500 | +| **C** | prod_2016_505 | **2016-07-01** | Prod ~505 only | +| **D** | prod_2016_505_liquid | **2016-07-01** | Prod ∪ liquid top-1500 | + +- **End:** last available bar in snapshot (no artificial end). +- **4y start** chosen to align with recent Phase‑A / book baselines (~mid‑2022 → mid‑2026). +- **2016-07-01** = first full month after typical Alpaca floor (~2016-01); residual 12‑1 needs ~1y bars so first residual ranks appear mid‑2017 where feed allows. + +### Universe definitions + +| set | definition | +|---|---| +| **Prod ~505** | Symbols **not** in `research_rank_only` on the research snapshot (the original prod-universe copy). | +| **Liquid top-1500** | Point-in-time: among names with as-of close ≥ **$5** and valid 63d median $vol, keep top **1500** by that $vol. Same definition as breadth IC research. | +| **Prod ∪ liquid** | A name may enter the book on date *t* if it is prod **or** in the liquid top-1500 at *t*. | + +Cross-sectional residual / vol / 80/20 ranks are **recomputed inside each arm’s +eligible candidate set** that period (so breadth arms are not ranked against +non-eligible thin names). + +### Explicit non-goals + +- No sector residual, SUE, FIP filter, gap-cap, take-profit, vol-target, corr-cap +- No retune of trail / cutoff / min_rr +- Survivorship: report levels with the standard caveat; **compare arms relatively** + +### Reporting (required table) + +Per arm: Sharpe, Sharpe SE (Mertens), CAGR %, max DD %, total return %, trades, +win rate if available, start/end, n qualified longs. One markdown table + JSON. + +**No promotion rule** — descriptive matrix only. Human decides whether breadth +or depth changes the risk story. + +--- + +## Snapshot requirements + +- Prefer MacBook **deep** `research.sqlite` after sector-resid deepen (prod names + from ~2016, breadth deep, completion manifest `complete=true`). +- Race-guard before run. +- Sector map / sector ETFs optional (not used for ranking). + +--- + +## Results + +*(filled after run)* + +| arm | universe | entry start | Sharpe | SE | CAGR % | max DD % | trades | notes | +|---|---|---|---:|---:|---:|---:|---:|---| +| A | 505 | 2022-07-01 | | | | | | | +| B | 505+liquid | 2022-07-01 | | | | | | | +| C | 505 | 2016-07-01 | | | | | | | +| D | 505+liquid | 2016-07-01 | | | | | | | + +--- + +## Verdict + +**PENDING_HUMAN** after numbers land. diff --git a/scripts/run_prod_book_universe_matrix.py b/scripts/run_prod_book_universe_matrix.py new file mode 100644 index 0000000..6d5b9cc --- /dev/null +++ b/scripts/run_prod_book_universe_matrix.py @@ -0,0 +1,706 @@ +#!/usr/bin/env python3 +"""Production book × universe × horizon matrix (research only). + +Four pre-registered arms — same live strategy knobs; only entry start date and +tradable universe change. See docs/research/prod-book-universe-horizon.md. + + A 2022-07-01 prod ~505 + B 2022-07-01 prod ∪ liquid top-1500 + C 2016-07-01 prod ~505 + D 2016-07-01 prod ∪ liquid top-1500 + +Example (MacBook, deep research.sqlite) +--------------------------------------- + python scripts/run_prod_book_universe_matrix.py \\ + --snapshot backtest_snapshots/research.sqlite \\ + --workers 8 --allow-spawn \\ + --candidate-cache reports/.cache/prod-book-univ-cands.pkl +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import os +import pickle +import sys +import time +from collections import defaultdict +from concurrent.futures import ProcessPoolExecutor +from datetime import date, datetime +from pathlib import Path +from typing import Any + +from sqlalchemy import create_engine, text +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from app.ssl_bootstrap import bootstrap_ssl # noqa: E402 + +bootstrap_ssl() + +SHORT_START = date(2022, 7, 1) +LONG_START = date(2016, 7, 1) +LIQUID_TOP_N = 1500 +LIQUID_MIN_PRICE = 5.0 +CACHE_VERSION = "prod-book-universe-horizon-v1" + +ARMS: tuple[dict[str, Any], ...] = ( + { + "id": "A_prod_4y_505", + "label": "Prod book · ~4y · 505 only", + "start": SHORT_START, + "universe": "prod_505", + }, + { + "id": "B_prod_4y_505_liquid", + "label": "Prod book · ~4y · 505 + liquid top-1500", + "start": SHORT_START, + "universe": "prod_plus_liquid", + }, + { + "id": "C_prod_2016_505", + "label": "Prod book · since 2016-07 · 505 only", + "start": LONG_START, + "universe": "prod_505", + }, + { + "id": "D_prod_2016_505_liquid", + "label": "Prod book · since 2016-07 · 505 + liquid top-1500", + "start": LONG_START, + "universe": "prod_plus_liquid", + }, +) + + +def _parse_args() -> argparse.Namespace: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--snapshot", default="backtest_snapshots/research.sqlite") + p.add_argument("--workers", type=int, default=8) + p.add_argument("--allow-spawn", action="store_true") + p.add_argument("--quiet", action="store_true") + p.add_argument( + "--candidate-cache", + default="reports/.cache/prod-book-universe-cands.pkl", + help="Pickle cache for full GTL candidate pass (expensive).", + ) + p.add_argument( + "--rebuild-cache", + action="store_true", + help="Ignore existing candidate cache.", + ) + p.add_argument("--out", default=None) + p.add_argument( + "--skip-race-guard", + action="store_true", + help="Allow run without completion manifest (not recommended).", + ) + return p.parse_args() + + +def _sqlite_url(path: Path) -> str: + return f"sqlite+aiosqlite:///{path.resolve().as_posix()}" + + +def _load_prod_and_all_symbols(snapshot: Path) -> tuple[set[str], list[str]]: + engine = create_engine( + f"sqlite:///{snapshot.resolve().as_posix()}", + future=True, + ) + try: + with engine.connect() as conn: + all_syms = [ + str(r[0]).upper() + for r in conn.execute(text("SELECT symbol FROM tickers ORDER BY 1")) + ] + try: + rank_only = { + str(r[0]).upper() + for r in conn.execute(text("SELECT symbol FROM research_rank_only")) + } + except Exception: + rank_only = set() + finally: + engine.dispose() + prod = {s for s in all_syms if s not in rank_only} + return prod, all_syms + + +def _median(xs: list[float]) -> float | None: + if len(xs) < 20: + return None + s = sorted(xs) + mid = len(s) // 2 + if len(s) % 2: + return s[mid] + return 0.5 * (s[mid - 1] + s[mid]) + + +def _build_liquid_membership( + prices: dict[str, tuple], + *, + top_n: int, + min_price: float, +) -> dict[date, set[str]]: + """For each calendar date present in any series, top-N by 63d median $vol.""" + # Collect per-symbol (date -> (close, dvol63)) + per_sym: dict[str, dict[date, tuple[float, float | None]]] = {} + all_dates: set[date] = set() + for sym, cols in prices.items(): + ords, _o, _h, _l, closes, vols = cols + dates = [date.fromordinal(int(o)) for o in ords] + n = len(dates) + series: dict[date, tuple[float, float | None]] = {} + for i in range(n): + d = dates[i] + c = float(closes[i]) + dvol = None + if i + 1 >= 63: + dvs = [] + for k in range(i - 62, i + 1): + ck = float(closes[k]) + vk = float(vols[k] or 0) + if ck > 0 and vk >= 0: + dvs.append(ck * vk) + dvol = _median(dvs) + series[d] = (c, dvol) + all_dates.add(d) + per_sym[sym] = series + + membership: dict[date, set[str]] = {} + for d in sorted(all_dates): + eligible: list[tuple[float, str]] = [] + for sym, series in per_sym.items(): + row = series.get(d) + if row is None: + continue + c, dvol = row + if c < min_price or dvol is None or dvol <= 0: + continue + eligible.append((-dvol, sym)) # highest dvol first + eligible.sort() + membership[d] = {sym for _, sym in eligible[:top_n]} + return membership + + +def _worker_replay( + symbol: str, + columns: tuple, + config: dict, + activation: dict, + spy: dict, + cadence: str, +) -> list[dict]: + """Picklable full GTL+signals candidate replay (no signal-only).""" + from app.services import backtest_service as bt + + cands, _series = bt._replay_and_signals( + symbol, + columns, + config, + activation, + spy, + bt.PRODUCTION_GTL_TARGET_MODEL, + cadence, + False, # always full replay for book matrix + None, + None, + ) + return cands + + +async def _load_or_build_candidates( + snapshot: Path, + *, + cache_path: Path | None, + rebuild: bool, + workers: int, + quiet: bool, +) -> tuple[list[dict], dict[str, tuple], dict, set[str], dict]: + from app.config import settings + from app.services import backtest_service as bt + from app.services.admin_service import get_activation_config + from app.services.recommendation_service import get_recommendation_config + from app.services.paper_trade_service import get_exit_policy + from app.services.benchmark_service import load_benchmark_closes + from app.models.ticker import Ticker + from sqlalchemy import select + + os.environ["BACKTEST_SNAPSHOT_OFFLINE"] = "1" + settings.backtest_workers = max(1, workers) + + prod_set, all_syms = _load_prod_and_all_symbols(snapshot) + print(f"Symbols: all={len(all_syms)} prod_505={len(prod_set)}") + + cache_key = { + "version": CACHE_VERSION, + "snapshot": str(snapshot.resolve()), + "prod_n": len(prod_set), + "all_n": len(all_syms), + } + if cache_path and cache_path.exists() and not rebuild: + with cache_path.open("rb") as fh: + blob = pickle.load(fh) + if blob.get("key") == cache_key and blob.get("candidates"): + print(f"Loaded candidate cache: {cache_path} ({len(blob['candidates'])} rows)") + return ( + blob["candidates"], + blob["prices"], + blob["spy"], + set(blob["prod_set"]), + blob["exit_config"], + ) + print("Cache key mismatch — rebuilding candidates") + + engine = create_async_engine(_sqlite_url(snapshot), pool_pre_ping=True) + Session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) + + candidates: list[dict] = [] + prices: dict[str, tuple] = {} + try: + async with Session() as db: + config = await get_recommendation_config(db) + activation = await get_activation_config(db) + exit_config = await get_exit_policy(db) + spy = await load_benchmark_closes(db, "SPY") + tickers = list( + (await db.execute(select(Ticker).order_by(Ticker.symbol))).scalars() + ) + + # Fetch all price columns first (I/O). + for idx, t in enumerate(tickers): + if not quiet and idx % 100 == 0: + print(f" fetch prices {idx}/{len(tickers)}", end="\r", flush=True) + cols = await bt._fetch_columns(db, t.symbol) + if cols is not None: + prices[t.symbol.upper()] = cols + if not quiet: + print() + + # Parallel GTL replay for every symbol with prices. + syms = sorted(prices) + print(f"GTL replay on {len(syms)} symbols (workers={workers})…") + t0 = time.monotonic() + if workers <= 1: + for i, sym in enumerate(syms): + if not quiet and i % 50 == 0: + print(f" replay {i}/{len(syms)}", end="\r", flush=True) + candidates.extend( + _worker_replay( + sym, prices[sym], config, activation, spy, "weekly" + ) + ) + else: + # Process pool: pass column batches. + import multiprocessing as mp + + ctx = mp.get_context("spawn") + chunk = max(1, workers * 2) + with ProcessPoolExecutor(max_workers=workers, mp_context=ctx) as pool: + for start in range(0, len(syms), chunk): + batch = syms[start : start + chunk] + futs = [ + pool.submit( + _worker_replay, + sym, + prices[sym], + config, + activation, + spy, + "weekly", + ) + for sym in batch + ] + for fut in futs: + try: + candidates.extend(fut.result()) + except Exception as exc: + print(f" worker error: {exc}") + if not quiet: + print( + f" replay {min(start+chunk, len(syms))}/{len(syms)} " + f"cands={len(candidates)} " + f"elapsed={(time.monotonic()-t0)/60:.1f}m", + end="\r", + flush=True, + ) + if not quiet: + print() + finally: + await engine.dispose() + + print(f"Total raw candidates: {len(candidates)}") + if cache_path: + cache_path.parent.mkdir(parents=True, exist_ok=True) + with cache_path.open("wb") as fh: + pickle.dump( + { + "key": cache_key, + "candidates": candidates, + "prices": prices, + "spy": spy, + "prod_set": sorted(prod_set), + "exit_config": exit_config, + }, + fh, + protocol=pickle.HIGHEST_PROTOCOL, + ) + print(f"Wrote cache {cache_path}") + + return candidates, prices, spy, prod_set, exit_config + + +def _candidate_eligible( + cand: dict, + *, + prod_set: set[str], + universe: str, + liquid_by_date: dict[date, set[str]], +) -> bool: + if cand.get("direction") != "long": + return False + sym = str(cand.get("symbol") or "").upper() + if not sym: + return False + if universe == "prod_505": + return sym in prod_set + # prod_plus_liquid + if sym in prod_set: + return True + try: + d = date.fromisoformat(str(cand["date"])[:10]) + except Exception: + return False + return sym in (liquid_by_date.get(d) or set()) + + +def _run_arm( + arm: dict[str, Any], + *, + all_candidates: list[dict], + prices: dict[str, tuple], + spy: dict, + prod_set: set[str], + liquid_by_date: dict[date, set[str]], + exit_config: dict, +) -> dict[str, Any]: + from app.services import backtest_service as bt + + start: date = arm["start"] + universe: str = arm["universe"] + + filtered: list[dict] = [] + for c in all_candidates: + try: + d = date.fromisoformat(str(c["date"])[:10]) + except Exception: + continue + if d < start: + continue + if not _candidate_eligible( + c, prod_set=prod_set, universe=universe, liquid_by_date=liquid_by_date + ): + continue + filtered.append(dict(c)) + + # Re-rank inside this arm's universe (production percentile logic). + bt._assign_momentum_percentiles(filtered) + bt._assign_residual_momentum_percentiles(filtered) + bt._assign_low_volatility_percentiles(filtered) + bt._assign_activation_momentum_percentiles(filtered) + bt._assign_residual_high_vol_blend(filtered) + for c in filtered: + c["qualified"] = bt._momentum_qualifies(c, 80.0) + + longs = [ + c for c in filtered if c.get("qualified") and c.get("direction") == "long" + ] + + strategy = next(s for s in bt.PORTFOLIO_MONITOR_STRATEGIES if s.get("is_production")) + entry_cfg = bt._entry_variant_config(str(strategy["entry_variant"])) + assert entry_cfg is not None + ranking_key = str( + entry_cfg.get("ranking_key") or entry_cfg["percentile_key"] + ) + exit_policy = bt.LIVE_EXIT_MODE_TO_SIM.get( + str(exit_config.get("mode", "atr_trailing")), "atr_trail3" + ) + hold_days = int(exit_config.get("hold_days", 30)) + trail = float(exit_config.get("atr_multiplier", bt.ATR_TRAIL_MULTIPLIER)) + risk = float(entry_cfg["risk_per_trade"]) + max_pos = int(entry_cfg["max_positions"]) + + reentry = bt._make_gate_reset_reentry_fn( + longs, prices, cadence="weekly", ranking_key=ranking_key + ) + sim = bt._simulate_portfolio( + longs, + prices, + spy, + exit_policy, + hold_days, + ranking_key=ranking_key, + max_positions=max_pos, + risk_per_trade=risk, + atr_trail_multiplier=trail, + post_stop_reentry_fn=reentry, + start_date=start, + end_date=None, + fill_mode=bt.FILL_MODE_CLOSE, + include_trades=False, + ) + if sim is None: + return { + "id": arm["id"], + "label": arm["label"], + "start": start.isoformat(), + "universe": universe, + "n_candidates": len(filtered), + "n_qualified_longs": 0, + "error": "no_trades", + } + + keep = { + k: sim.get(k) + for k in ( + "sharpe", + "sharpe_se", + "cagr_pct", + "max_drawdown_pct", + "total_return_pct", + "calmar", + "trades", + "win_rate", + "n_returns", + "psr", + "start_date", + "end_date", + "spy_return_pct", + "final_equity", + ) + } + return { + "id": arm["id"], + "label": arm["label"], + "start": start.isoformat(), + "universe": universe, + "n_candidates": len(filtered), + "n_qualified_longs": len(longs), + "fill_mode": "close", + "ranking_key": ranking_key, + "exit_policy": exit_policy, + "hold_days": hold_days, + **keep, + } + + +def _write_outputs(payload: dict, out_json: Path, doc_path: Path) -> None: + out_json.parent.mkdir(parents=True, exist_ok=True) + out_json.write_text( + json.dumps(payload, indent=2, default=str) + "\n", encoding="utf-8" + ) + + lines = [ + "# Production book × universe × horizon — results", + "", + f"Generated: `{payload.get('generated_at')}`", + "", + "> Survivorship: today's constituents backfilled. Compare arms relatively; " + "do not treat deep CAGR/Sharpe levels as deployable forecasts.", + "", + "## Arms", + "", + "| arm | universe | entry start | Sharpe | SE | CAGR % | max DD % | ret % | trades | qual longs | span |", + "|---|---|---|---:|---:|---:|---:|---:|---:|---:|---|", + ] + for row in payload.get("arms") or []: + if row.get("error"): + lines.append( + f"| {row.get('id')} | {row.get('universe')} | {row.get('start')} | " + f"ERR | | | | | | {row.get('n_qualified_longs')} | {row.get('error')} |" + ) + continue + lines.append( + f"| {row.get('id')} | {row.get('universe')} | {row.get('start')} | " + f"{row.get('sharpe')} | {row.get('sharpe_se')} | {row.get('cagr_pct')} | " + f"{row.get('max_drawdown_pct')} | {row.get('total_return_pct')} | " + f"{row.get('trades')} | {row.get('n_qualified_longs')} | " + f"{row.get('start_date')}→{row.get('end_date')} |" + ) + lines.extend([ + "", + "## Config (production, unchanged)", + "", + f"```json\n{json.dumps(payload.get('strategy') or {}, indent=2)}\n```", + "", + "## Snapshot", + "", + f"```json\n{json.dumps(payload.get('snapshot_meta') or {}, indent=2, default=str)}\n```", + "", + "PENDING_HUMAN — descriptive matrix only; no auto promotion.", + "", + f"JSON: `{out_json.as_posix()}`", + "", + ]) + out_json.with_suffix(".md").write_text("\n".join(lines) + "\n", encoding="utf-8") + + # Fill results section of the research doc. + if doc_path.exists(): + text = doc_path.read_text(encoding="utf-8") + marker = "## Results" + idx = text.find(marker) + header = text[:idx] if idx >= 0 else text + # Drop old results/verdict tail + for m in ("## Results", "## Verdict"): + pass + body = [ + header.rstrip(), + "", + "## Results", + "", + f"Generated: `{payload.get('generated_at')}`", + "", + "| arm | universe | entry start | Sharpe | SE | CAGR % | max DD % | trades |", + "|---|---|---|---:|---:|---:|---:|---:|", + ] + for row in payload.get("arms") or []: + body.append( + f"| {row.get('id')} | {row.get('universe')} | {row.get('start')} | " + f"{row.get('sharpe', '')} | {row.get('sharpe_se', '')} | " + f"{row.get('cagr_pct', '')} | {row.get('max_drawdown_pct', '')} | " + f"{row.get('trades', '')} |" + ) + body.extend([ + "", + f"Full report: `{out_json.as_posix()}`", + "", + "## Verdict", + "", + "**PENDING_HUMAN** — descriptive only; production knobs unchanged.", + "", + ]) + doc_path.write_text("\n".join(body) + "\n", encoding="utf-8") + + +async def _main() -> None: + args = _parse_args() + snapshot = Path(args.snapshot) + if not snapshot.exists(): + raise SystemExit(f"Missing snapshot: {snapshot}") + if args.allow_spawn: + os.environ["BACKTEST_ALLOW_SPAWN"] = "1" + + if not args.skip_race_guard: + try: + from scripts.research_snapshot_manifest import ( + assert_research_snapshot_complete, + ) + + manifest = assert_research_snapshot_complete(snapshot) + print( + f"Race guard OK: tickers={manifest.get('ticker_count')} " + f"ohlcv={manifest.get('ohlcv_row_count')}" + ) + except SystemExit as exc: + # Prod-only snapshot without manifest: allow with warning if ~505. + engine = create_engine( + f"sqlite:///{snapshot.resolve().as_posix()}", + future=True, + ) + try: + with engine.connect() as conn: + n = int(conn.execute(text("SELECT COUNT(*) FROM tickers")).scalar_one()) + finally: + engine.dispose() + if n < 400: + raise + print(f"WARNING: no research manifest ({exc}); proceeding n_tickers={n}") + + cache = Path(args.candidate_cache) if args.candidate_cache else None + candidates, prices, spy, prod_set, exit_config = await _load_or_build_candidates( + snapshot, + cache_path=cache, + rebuild=args.rebuild_cache, + workers=args.workers, + quiet=args.quiet, + ) + + print("Building PIT liquid membership (top-1500, price≥5)…") + t0 = time.monotonic() + liquid_by_date = _build_liquid_membership( + prices, top_n=LIQUID_TOP_N, min_price=LIQUID_MIN_PRICE + ) + print( + f" liquid dates={len(liquid_by_date)} " + f"elapsed={(time.monotonic()-t0)/60:.1f}m" + ) + + arms_out = [] + for arm in ARMS: + print(f"Running arm {arm['id']}…") + row = _run_arm( + arm, + all_candidates=candidates, + prices=prices, + spy=spy, + prod_set=prod_set, + liquid_by_date=liquid_by_date, + exit_config=exit_config, + ) + arms_out.append(row) + print( + f" Sharpe={row.get('sharpe')} CAGR={row.get('cagr_pct')} " + f"DD={row.get('max_drawdown_pct')} trades={row.get('trades')} " + f"qual={row.get('n_qualified_longs')}" + ) + + stamp = datetime.now().strftime("%Y%m%d-%H%M%S") + out = ( + Path(args.out) + if args.out + else Path("reports") / f"prod-book-universe-horizon-{stamp}.json" + ) + payload = { + "generated_at": datetime.now().isoformat(), + "snapshot": str(snapshot.resolve()), + "snapshot_meta": { + "prod_universe_n": len(prod_set), + "price_symbols_n": len(prices), + "raw_candidates": len(candidates), + "liquid_top_n": LIQUID_TOP_N, + "liquid_min_price": LIQUID_MIN_PRICE, + "short_start": SHORT_START.isoformat(), + "long_start": LONG_START.isoformat(), + }, + "strategy": { + "note": "Live production knobs — no modifications", + "momentum": "residual_12_1 gate 80", + "rank": "residual_high_vol_blend_80_20", + "fill_mode": "close", + "cost_per_side": 0.001, + "exit": exit_config, + "max_positions": 10, + "risk_per_trade": 0.01, + "reentry": "gate_reset", + }, + "arms": arms_out, + "survivorship_banner": ( + "Today's constituents backfilled. Relative arm comparison only." + ), + "pending_human": True, + } + _write_outputs( + payload, + out, + Path("docs/research/prod-book-universe-horizon.md"), + ) + print(f"Wrote {out}") + print(f"Wrote {out.with_suffix('.md')}") + + +if __name__ == "__main__": + asyncio.run(_main()) diff --git a/scripts/run_tier1_macbook.sh b/scripts/run_tier1_macbook.sh index 72f8aef..584eb4a 100755 --- a/scripts/run_tier1_macbook.sh +++ b/scripts/run_tier1_macbook.sh @@ -16,6 +16,7 @@ # ./scripts/run_tier1_macbook.sh --harness-only # skip rebuild; race-guard + IC only # ./scripts/run_tier1_macbook.sh --coverage-only # bars-per-year probe only # ./scripts/run_tier1_macbook.sh --sector-resid-deep # deepen shallow + ONE masked grade +# ./scripts/run_tier1_macbook.sh --prod-book-matrix # 4-arm universe×horizon book matrix # # Does NOT touch production Postgres, scheduler, gates, or prod config. @@ -36,7 +37,7 @@ FMP_SLEEP="${FMP_SLEEP:-0.35}" PYTHON="${PYTHON:-python3}" USE_CORP_PROXY="${USE_CORP_PROXY:-0}" -PHASE="depth" # depth | all | earnings | harness | coverage | ssl | sector-resid-deep +PHASE="depth" # depth | all | earnings | harness | coverage | ssl | sector-resid-deep | prod-book usage() { sed -n '2,25p' "$0" | sed 's/^# \?//' @@ -61,6 +62,7 @@ while [[ $# -gt 0 ]]; do --depth) PHASE=depth; shift ;; --ssl-check) PHASE=ssl; shift ;; --sector-resid-deep) PHASE=sector_resid_deep; shift ;; + --prod-book-matrix) PHASE=prod_book; shift ;; --corp-proxy) USE_CORP_PROXY=1; shift ;; --prod-snap) PROD_SNAP="$2"; shift 2 ;; --research-snap) RESEARCH_SNAP="$2"; shift 2 ;; @@ -237,6 +239,16 @@ run_sector_resid_deep() { --allow-spawn } +run_prod_book_matrix() { + need_file "$RESEARCH_SNAP" + log "Production book × universe × horizon (4 arms, strategy unchanged)" + "$PYTHON" scripts/run_prod_book_universe_matrix.py \ + --snapshot "$RESEARCH_SNAP" \ + --workers "$WORKERS" \ + --allow-spawn \ + --candidate-cache reports/.cache/prod-book-universe-cands.pkl +} + log "cwd=$ROOT python=$PYTHON phase=$PHASE workers=$WORKERS" setup_ssl @@ -247,6 +259,9 @@ case "$PHASE" in sector_resid_deep) run_sector_resid_deep ;; + prod_book) + run_prod_book_matrix + ;; coverage) run_coverage ;; From a4d5ed7a93acf058d9fe4e3da89b3cbe490caf71 Mon Sep 17 00:00:00 2001 From: Dennis Thiessen Date: Sun, 19 Jul 2026 14:21:05 +0200 Subject: [PATCH 12/14] tests done --- docs/research/prod-book-universe-horizon.md | 19 +-- ...book-universe-horizon-20260719-140737.json | 137 ++++++++++++++++++ ...d-book-universe-horizon-20260719-140737.md | 54 +++++++ 3 files changed, 201 insertions(+), 9 deletions(-) create mode 100644 reports/prod-book-universe-horizon-20260719-140737.json create mode 100644 reports/prod-book-universe-horizon-20260719-140737.md diff --git a/docs/research/prod-book-universe-horizon.md b/docs/research/prod-book-universe-horizon.md index 070ad8f..b7110b0 100644 --- a/docs/research/prod-book-universe-horizon.md +++ b/docs/research/prod-book-universe-horizon.md @@ -71,17 +71,18 @@ or depth changes the risk story. ## Results -*(filled after run)* +Generated: `2026-07-19T14:07:37.458454` -| arm | universe | entry start | Sharpe | SE | CAGR % | max DD % | trades | notes | -|---|---|---|---:|---:|---:|---:|---:|---| -| A | 505 | 2022-07-01 | | | | | | | -| B | 505+liquid | 2022-07-01 | | | | | | | -| C | 505 | 2016-07-01 | | | | | | | -| D | 505+liquid | 2016-07-01 | | | | | | | +| arm | universe | entry start | Sharpe | SE | CAGR % | max DD % | trades | +|---|---|---|---:|---:|---:|---:|---:| +| A_prod_4y_505 | prod_505 | 2022-07-01 | 1.32 | 0.49 | 31.8 | 18.9 | 374 | +| B_prod_4y_505_liquid | prod_plus_liquid | 2022-07-01 | 0.14 | 0.499 | -1.8 | 55.3 | 706 | +| C_prod_2016_505 | prod_505 | 2016-07-01 | 0.88 | 0.314 | 16.7 | 24.4 | 763 | +| D_prod_2016_505_liquid | prod_plus_liquid | 2016-07-01 | -0.06 | 0.316 | -7.0 | 73.9 | 1567 | ---- +Full report: `reports/prod-book-universe-horizon-20260719-140737.json` ## Verdict -**PENDING_HUMAN** after numbers land. +**PENDING_HUMAN** — descriptive only; production knobs unchanged. + diff --git a/reports/prod-book-universe-horizon-20260719-140737.json b/reports/prod-book-universe-horizon-20260719-140737.json new file mode 100644 index 0000000..c76d2cf --- /dev/null +++ b/reports/prod-book-universe-horizon-20260719-140737.json @@ -0,0 +1,137 @@ +{ + "generated_at": "2026-07-19T14:07:37.458454", + "snapshot": "/Users/taathde3/git/lab/signal_platform/backtest_snapshots/research.sqlite", + "snapshot_meta": { + "prod_universe_n": 506, + "price_symbols_n": 4654, + "raw_candidates": 2389258, + "liquid_top_n": 1500, + "liquid_min_price": 5.0, + "short_start": "2022-07-01", + "long_start": "2016-07-01" + }, + "strategy": { + "note": "Live production knobs \u2014 no modifications", + "momentum": "residual_12_1 gate 80", + "rank": "residual_high_vol_blend_80_20", + "fill_mode": "close", + "cost_per_side": 0.001, + "exit": { + "mode": "atr_trailing", + "trailing_pct": 12.0, + "atr_multiplier": 3.0, + "hold_days": 30 + }, + "max_positions": 10, + "risk_per_trade": 0.01, + "reentry": "gate_reset" + }, + "arms": [ + { + "id": "A_prod_4y_505", + "label": "Prod book \u00b7 ~4y \u00b7 505 only", + "start": "2022-07-01", + "universe": "prod_505", + "n_candidates": 81626, + "n_qualified_longs": 1448, + "fill_mode": "close", + "ranking_key": "residual_high_vol_blend_80_20_score", + "exit_policy": "atr_trail3", + "hold_days": 30, + "sharpe": 1.32, + "sharpe_se": 0.49, + "cagr_pct": 31.8, + "max_drawdown_pct": 18.9, + "total_return_pct": 204.7, + "calmar": 1.68, + "trades": 374, + "win_rate": 35.6, + "n_returns": 1009, + "psr": 0.9965, + "start_date": "2022-07-01", + "end_date": "2026-07-13", + "spy_return_pct": 96.5, + "final_equity": 30467.86 + }, + { + "id": "B_prod_4y_505_liquid", + "label": "Prod book \u00b7 ~4y \u00b7 505 + liquid top-1500", + "start": "2022-07-01", + "universe": "prod_plus_liquid", + "n_candidates": 267579, + "n_qualified_longs": 6587, + "fill_mode": "close", + "ranking_key": "residual_high_vol_blend_80_20_score", + "exit_policy": "atr_trail3", + "hold_days": 30, + "sharpe": 0.14, + "sharpe_se": 0.499, + "cagr_pct": -1.8, + "max_drawdown_pct": 55.3, + "total_return_pct": -7.1, + "calmar": -0.03, + "trades": 706, + "win_rate": 29.3, + "n_returns": 1013, + "psr": 0.6107, + "start_date": "2022-07-01", + "end_date": "2026-07-17", + "spy_return_pct": 95.0, + "final_equity": 9287.52 + }, + { + "id": "C_prod_2016_505", + "label": "Prod book \u00b7 since 2016-07 \u00b7 505 only", + "start": "2016-07-01", + "universe": "prod_505", + "n_candidates": 190179, + "n_qualified_longs": 2450, + "fill_mode": "close", + "ranking_key": "residual_high_vol_blend_80_20_score", + "exit_policy": "atr_trail3", + "hold_days": 30, + "sharpe": 0.88, + "sharpe_se": 0.314, + "cagr_pct": 16.7, + "max_drawdown_pct": 24.4, + "total_return_pct": 369.4, + "calmar": 0.68, + "trades": 763, + "win_rate": 36.7, + "n_returns": 2519, + "psr": 0.9975, + "start_date": "2016-07-01", + "end_date": "2026-07-13", + "spy_return_pct": 256.9, + "final_equity": 46938.66 + }, + { + "id": "D_prod_2016_505_liquid", + "label": "Prod book \u00b7 since 2016-07 \u00b7 505 + liquid top-1500", + "start": "2016-07-01", + "universe": "prod_plus_liquid", + "n_candidates": 649305, + "n_qualified_longs": 11551, + "fill_mode": "close", + "ranking_key": "residual_high_vol_blend_80_20_score", + "exit_policy": "atr_trail3", + "hold_days": 30, + "sharpe": -0.06, + "sharpe_se": 0.316, + "cagr_pct": -7.0, + "max_drawdown_pct": 73.9, + "total_return_pct": -52.0, + "calmar": -0.1, + "trades": 1567, + "win_rate": 28.0, + "n_returns": 2523, + "psr": 0.4287, + "start_date": "2016-07-01", + "end_date": "2026-07-17", + "spy_return_pct": 254.1, + "final_equity": 4800.5 + } + ], + "survivorship_banner": "Today's constituents backfilled. Relative arm comparison only.", + "pending_human": true +} diff --git a/reports/prod-book-universe-horizon-20260719-140737.md b/reports/prod-book-universe-horizon-20260719-140737.md new file mode 100644 index 0000000..4628100 --- /dev/null +++ b/reports/prod-book-universe-horizon-20260719-140737.md @@ -0,0 +1,54 @@ +# Production book × universe × horizon — results + +Generated: `2026-07-19T14:07:37.458454` + +> Survivorship: today's constituents backfilled. Compare arms relatively; do not treat deep CAGR/Sharpe levels as deployable forecasts. + +## Arms + +| arm | universe | entry start | Sharpe | SE | CAGR % | max DD % | ret % | trades | qual longs | span | +|---|---|---|---:|---:|---:|---:|---:|---:|---:|---| +| A_prod_4y_505 | prod_505 | 2022-07-01 | 1.32 | 0.49 | 31.8 | 18.9 | 204.7 | 374 | 1448 | 2022-07-01→2026-07-13 | +| B_prod_4y_505_liquid | prod_plus_liquid | 2022-07-01 | 0.14 | 0.499 | -1.8 | 55.3 | -7.1 | 706 | 6587 | 2022-07-01→2026-07-17 | +| C_prod_2016_505 | prod_505 | 2016-07-01 | 0.88 | 0.314 | 16.7 | 24.4 | 369.4 | 763 | 2450 | 2016-07-01→2026-07-13 | +| D_prod_2016_505_liquid | prod_plus_liquid | 2016-07-01 | -0.06 | 0.316 | -7.0 | 73.9 | -52.0 | 1567 | 11551 | 2016-07-01→2026-07-17 | + +## Config (production, unchanged) + +```json +{ + "note": "Live production knobs \u2014 no modifications", + "momentum": "residual_12_1 gate 80", + "rank": "residual_high_vol_blend_80_20", + "fill_mode": "close", + "cost_per_side": 0.001, + "exit": { + "mode": "atr_trailing", + "trailing_pct": 12.0, + "atr_multiplier": 3.0, + "hold_days": 30 + }, + "max_positions": 10, + "risk_per_trade": 0.01, + "reentry": "gate_reset" +} +``` + +## Snapshot + +```json +{ + "prod_universe_n": 506, + "price_symbols_n": 4654, + "raw_candidates": 2389258, + "liquid_top_n": 1500, + "liquid_min_price": 5.0, + "short_start": "2022-07-01", + "long_start": "2016-07-01" +} +``` + +PENDING_HUMAN — descriptive matrix only; no auto promotion. + +JSON: `reports/prod-book-universe-horizon-20260719-140737.json` + From 1c38a94dd0ca3a8de09eb65a2b4f863a00cd06a8 Mon Sep 17 00:00:00 2001 From: Dennis Thiessen Date: Sun, 19 Jul 2026 14:25:06 +0200 Subject: [PATCH 13/14] research: interpret prod book universe x horizon matrix; ignore candidate cache Four-arm results: 505 stays positive (softer on deep history); liquid breadth destroys book under current knobs. Stop tracking 1.3GB pkl cache under reports/.cache. --- .gitignore | 1 + docs/research/prod-book-universe-horizon.md | 57 +++++++++++++++++---- 2 files changed, 49 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index 2d776d9..0d70f55 100644 --- a/.gitignore +++ b/.gitignore @@ -46,3 +46,4 @@ backtest_snapshots/ # Rebuildable pickle caches are local accelerators, not decision evidence. reports/*.pkl reports/*.pk1 +reports/.cache/ diff --git a/docs/research/prod-book-universe-horizon.md b/docs/research/prod-book-universe-horizon.md index b7110b0..8cef4a1 100644 --- a/docs/research/prod-book-universe-horizon.md +++ b/docs/research/prod-book-universe-horizon.md @@ -71,18 +71,57 @@ or depth changes the risk story. ## Results -Generated: `2026-07-19T14:07:37.458454` +Generated: `2026-07-19T14:07:37` · artifact +`reports/prod-book-universe-horizon-20260719-140737.json` +Snapshot: MacBook deep `research.sqlite` (506 prod + breadth prices; 2.39M raw +GTL candidates). Strategy knobs = live production (residual 80, 80/20 high-vol +rank, ATR trail 3×, hold 30, gate-reset, `fill_mode=close`). -| arm | universe | entry start | Sharpe | SE | CAGR % | max DD % | trades | -|---|---|---|---:|---:|---:|---:|---:| -| A_prod_4y_505 | prod_505 | 2022-07-01 | 1.32 | 0.49 | 31.8 | 18.9 | 374 | -| B_prod_4y_505_liquid | prod_plus_liquid | 2022-07-01 | 0.14 | 0.499 | -1.8 | 55.3 | 706 | -| C_prod_2016_505 | prod_505 | 2016-07-01 | 0.88 | 0.314 | 16.7 | 24.4 | 763 | -| D_prod_2016_505_liquid | prod_plus_liquid | 2016-07-01 | -0.06 | 0.316 | -7.0 | 73.9 | 1567 | +> Survivorship: today's constituents backfilled. **Compare arms relatively.** +> Absolute deep CAGR/Sharpe are not deployable forecasts. -Full report: `reports/prod-book-universe-horizon-20260719-140737.json` +| arm | universe | entries from | Sharpe | SE | CAGR % | max DD % | total ret % | trades | win % | vs SPY | +|---|---|---|---:|---:|---:|---:|---:|---:|---:|---:| +| **A** | 505 only | 2022-07-01 | **1.32** | 0.49 | **31.8** | **18.9** | +205 | 374 | 35.6 | +96.5 | +| **B** | 505 + liquid 1500 | 2022-07-01 | 0.14 | 0.50 | −1.8 | 55.3 | −7 | 706 | 29.3 | +95.0 | +| **C** | 505 only | 2016-07-01 | **0.88** | 0.31 | **16.7** | **24.4** | +369 | 763 | 36.7 | +257 | +| **D** | 505 + liquid 1500 | 2016-07-01 | −0.06 | 0.32 | −7.0 | 73.9 | −52 | 1567 | 28.0 | +254 | + +Qualified longs: A 1 448 · B 6 587 · C 2 450 · D 11 551. + +### Read (relative only) + +1. **Same strategy, broader liquid universe kills the book** (A→B and C→D). + Sharpe collapses; DD roughly triples; win rate drops ~6–8pp; trade count + ~doubles. This matches earlier breadth IC work: the production residual + + high-vol package is a **large-cap / prod-universe** edge, not a + “more names = better” edge. + +2. **Longer history on 505 stays positive but softer** (A→C). Sharpe 1.32 → 0.88, + CAGR 32% → 17%, DD 19% → 24%. Still well above the liquid-breadth arms. + Levels are optimistic (survivorship); the useful message is “edge does not + vanish when 2018/2020 are included,” not “expect 17% CAGR forever.” + +3. **Arm A vs older Phase‑A / short-window controls** (~Sharpe 1.7–2.1): this + matrix re-ranked on deep research.sqlite with a fixed entry start; numbers + need not match prior reports row-for-row. Use **this table for A–D + comparisons**, not for rewriting the production baseline number. + +4. **No production change implied.** Keep the live ~505 universe. Do not broaden + the tradable set to liquid Nasdaq under current knobs without a new + pre-registered design (and almost certainly a different rank/tilt package). ## Verdict -**PENDING_HUMAN** — descriptive only; production knobs unchanged. +**Descriptive matrix complete.** + +| question | answer from this matrix | +|---|---| +| Prod book @ ~4y / 505 | Positive (arm A) | +| Same + liquid Nasdaq | **No** — large degradation (arm B) | +| Prod book since 2016 / 505 | Still positive, milder (arm C) | +| Same + liquid Nasdaq deep | **No** — worst arm (arm D) | + +**PENDING_HUMAN** only for whether to log “universe broaden under current knobs” +as rejected in the main research index. Strategy knobs unchanged either way. From bb8aa655a19c694c6358504e4a17a881520ac8df Mon Sep 17 00:00:00 2001 From: Dennis Thiessen Date: Sun, 19 Jul 2026 14:41:52 +0200 Subject: [PATCH 14/14] research: clean up closed Tier-1 scaffolding from branch Drop intermediate history-depth reports, sector-residual runners/map/code hooks (evidence stays in final reports + docs), and slim MacBook helper to ssl/earnings/ prod-book-matrix only. SSL bootstrap and archived research conclusions retained. --- app/services/backtest_service.py | 263 +---- app/services/sector_map.py | 145 --- data/research/ticker_sector_map.json | 543 --------- docs/research/history-depth-extension.md | 9 +- docs/research/sector-residual-momentum.md | 19 +- reports/history-depth-20260719-093853.json | 68 -- reports/history-depth-20260719-093853.md | 177 --- reports/history-depth-20260719-094134.json | 68 -- reports/history-depth-20260719-094134.md | 177 --- reports/history-depth-20260719-094344.json | 68 -- reports/history-depth-20260719-094344.md | 177 --- reports/history-depth-20260719-095156.json | 68 -- reports/history-depth-20260719-095156.md | 177 --- ...esid-deep-20260719-111923-SANITY-FAIL.json | 275 ----- scripts/build_ticker_sector_map.py | 233 ---- scripts/fetch_sector_etfs_to_snapshot.py | 187 --- scripts/run_earnings_research.py | 26 - scripts/run_history_depth_research.py | 480 -------- scripts/run_sector_resid_deep_test.py | 1047 ----------------- scripts/run_sector_residual_research.py | 1018 ---------------- scripts/run_tier1_macbook.sh | 244 +--- tests/unit/test_backtest_service.py | 69 -- 22 files changed, 88 insertions(+), 5450 deletions(-) delete mode 100644 app/services/sector_map.py delete mode 100644 data/research/ticker_sector_map.json delete mode 100644 reports/history-depth-20260719-093853.json delete mode 100644 reports/history-depth-20260719-093853.md delete mode 100644 reports/history-depth-20260719-094134.json delete mode 100644 reports/history-depth-20260719-094134.md delete mode 100644 reports/history-depth-20260719-094344.json delete mode 100644 reports/history-depth-20260719-094344.md delete mode 100644 reports/history-depth-20260719-095156.json delete mode 100644 reports/history-depth-20260719-095156.md delete mode 100644 reports/sector-resid-deep-20260719-111923-SANITY-FAIL.json delete mode 100644 scripts/build_ticker_sector_map.py delete mode 100644 scripts/fetch_sector_etfs_to_snapshot.py delete mode 100644 scripts/run_history_depth_research.py delete mode 100644 scripts/run_sector_resid_deep_test.py delete mode 100644 scripts/run_sector_residual_research.py diff --git a/app/services/backtest_service.py b/app/services/backtest_service.py index 55280db..1c06c87 100644 --- a/app/services/backtest_service.py +++ b/app/services/backtest_service.py @@ -791,86 +791,34 @@ def _residual_momentum_12_1( with an intercept estimated over the same window, the arithmetic residuals sum to ~zero by construction, which would destroy the signal. """ - return _multi_factor_residual_momentum_12_1( - dates, closes, i, [benchmark_closes] if benchmark_closes else None - ) - - -def _multi_factor_residual_momentum_12_1( - dates: list[date], - closes: list[float], - i: int, - factor_closes: list[dict[date, float]] | None, -) -> float | None: - """12-1 residual momentum vs one or more factors (OLS, no intercept). - - Same formation window as raw / single-factor residual momentum: - daily returns from close[i-252] → close[i-21], require ≥100 paired obs. - Factors are stacked as columns; betas are OLS without intercept so the - cumulative residual is not forced to zero. - """ - if not factor_closes or i - 252 < 0: - return None - n_factors = len(factor_closes) - if n_factors < 1: + if not benchmark_closes or i - 252 < 0: return None stock_rets: list[float] = [] - factor_rets: list[list[float]] = [[] for _ in range(n_factors)] + market_rets: list[float] = [] + # Same daily intervals as mom_12_1: close[i-252] -> close[i-21]. for k in range(i - 251, i - 20): prev_close = closes[k - 1] - if prev_close <= 0: - continue - f_day: list[float] = [] - ok = True - for fc in factor_closes: - f_prev = fc.get(dates[k - 1]) - f_cur = fc.get(dates[k]) - if f_prev is None or f_cur is None or f_prev <= 0: - ok = False - break - f_day.append(f_cur / f_prev - 1.0) - if not ok: + bench_prev = benchmark_closes.get(dates[k - 1]) + bench_cur = benchmark_closes.get(dates[k]) + if prev_close <= 0 or bench_prev is None or bench_cur is None or bench_prev <= 0: continue stock_rets.append(closes[k] / prev_close - 1.0) - for j, r in enumerate(f_day): - factor_rets[j].append(r) + market_rets.append(bench_cur / bench_prev - 1.0) - n = len(stock_rets) - if n < 100: + if len(stock_rets) < 100: return None - - if n_factors == 1: - # Fast path: identical algebra to the historical single-factor form. - market_rets = factor_rets[0] - mean_market = sum(market_rets) / n - mean_stock = sum(stock_rets) / n - var_market = sum((x - mean_market) ** 2 for x in market_rets) - if var_market <= 0: - return None - cov = sum( - (stock_rets[k] - mean_stock) * (market_rets[k] - mean_market) - for k in range(n) - ) - beta = cov / var_market - return sum(stock_rets[k] - beta * market_rets[k] for k in range(n)) - - # OLS without intercept: β = (X'X)^{-1} X'y for X columns = factor returns. - # Implemented for exactly two factors (market + sector); refuse larger. - if n_factors != 2: + mean_market = sum(market_rets) / len(market_rets) + mean_stock = sum(stock_rets) / len(stock_rets) + var_market = sum((x - mean_market) ** 2 for x in market_rets) + if var_market <= 0: return None - f1, f2 = factor_rets[0], factor_rets[1] - s11 = sum(a * a for a in f1) - s22 = sum(a * a for a in f2) - s12 = sum(f1[k] * f2[k] for k in range(n)) - sy1 = sum(stock_rets[k] * f1[k] for k in range(n)) - sy2 = sum(stock_rets[k] * f2[k] for k in range(n)) - det = s11 * s22 - s12 * s12 - if abs(det) < 1e-18: - return None - b1 = (s22 * sy1 - s12 * sy2) / det - b2 = (s11 * sy2 - s12 * sy1) / det - return sum(stock_rets[k] - b1 * f1[k] - b2 * f2[k] for k in range(n)) + cov = sum( + (stock_rets[k] - mean_stock) * (market_rets[k] - mean_market) + for k in range(len(stock_rets)) + ) + beta = cov / var_market + return sum(stock_rets[k] - beta * market_rets[k] for k in range(len(stock_rets))) def _realized_vol_6m(closes: list[float], i: int) -> float | None: @@ -895,7 +843,6 @@ def _signal_values( highs: list[float], i: int, benchmark_closes: dict[date, float] | None = None, - sector_etf_closes: dict[date, float] | None = None, ) -> dict[str, float]: """Point-in-time candidate signals at as-of index ``i`` (price-only). @@ -907,11 +854,6 @@ def _signal_values( higher = nearer the high, expect positive IC). ``vol_6m`` is 126-day realized volatility (expect negative IC if the low-volatility anomaly holds). ``fip_id`` is Da/Gurun/Warachka information discreteness (expect negative IC). - - When ``sector_etf_closes`` is supplied (research path), also emit - ``mom_12_1_sector_resid``: two-factor residual vs SPY + sector ETF. - Cross-sectional ``mom_12_1_sector_demeaned`` is injected later from the - full weekly cross-section (cannot be computed per-ticker alone). """ out: dict[str, float] = {} if i - 252 >= 0 and closes[i - 252] > 0: @@ -919,12 +861,6 @@ def _signal_values( residual = _residual_momentum_12_1(dates, closes, i, benchmark_closes) if residual is not None: out["mom_12_1_resid"] = residual - if benchmark_closes and sector_etf_closes: - sector_resid = _multi_factor_residual_momentum_12_1( - dates, closes, i, [benchmark_closes, sector_etf_closes] - ) - if sector_resid is not None: - out["mom_12_1_sector_resid"] = sector_resid fip = _fip_id(closes, i) if fip is not None: out["fip_id"] = fip @@ -1011,16 +947,14 @@ def _accumulate_signal_series( benchmark_closes: dict[date, float] | None = None, *, symbol: str | None = None, - sector_etf_closes: dict[date, float] | None = None, ) -> None: """For each weekly as-of bar, emit (signal, forward-return) pairs keyed by ISO week into ``collected[name][week_key]``. Forward return is close-to-close over HORIZON trading days. Mutates ``collected`` (a dict of dict of list). When ``BACKTEST_LIQUID_BREADTH`` is set, observations are dicts with PIT - liquidity fields for the mask. When ``symbol`` is provided, observations are - also dicts (so sector demeaning can group by name); otherwise plain - ``(val, fwd)`` tuples keep the production path unchanged. + liquidity fields for the mask; otherwise plain ``(val, fwd)`` tuples so the + production signal path stays unchanged. """ n = len(records) if n < HORIZON + 21: @@ -1030,7 +964,6 @@ def _accumulate_signal_series( volumes = [float(getattr(r, "volume", 0) or 0) for r in records] dates = [r.date for r in records] liquid_mode = _liquid_breadth_top_n() > 0 - rich = liquid_mode or symbol is not None for i in _weekly_asof_indices(records): j = i + HORIZON if j >= n or closes[i] <= 0: @@ -1039,79 +972,19 @@ def _accumulate_signal_series( iso = records[i].date.isocalendar() week_key = (iso[0], iso[1]) dvol = _median_dollar_vol_63(closes, volumes, i) if liquid_mode else None - for name, val in _signal_values( - dates, closes, highs, i, benchmark_closes, sector_etf_closes - ).items(): - if rich: - row = { + for name, val in _signal_values(dates, closes, highs, i, benchmark_closes).items(): + if liquid_mode: + collected[name][week_key].append({ "val": val, "fwd": fwd, + "close": closes[i], + "median_dvol_63": dvol, "symbol": symbol, - } - if liquid_mode: - row["close"] = closes[i] - row["median_dvol_63"] = dvol - collected[name][week_key].append(row) + }) else: collected[name][week_key].append((val, fwd)) -def _inject_sector_demeaned_momentum( - collected: dict, - symbol_to_sector: dict[str, str], - *, - min_sector_names: int = 2, -) -> None: - """Cross-sectional demean of ``mom_12_1`` within GICS sector per week. - - ``mom_12_1_sector_demeaned[i] = mom_12_1[i] − mean(mom_12_1 | sector_i)``. - Requires rich observations with a ``symbol`` field (research path). Names - without a sector label, or sectors with fewer than ``min_sector_names`` - members that week, are dropped from the demeaned series. - """ - if not symbol_to_sector or "mom_12_1" not in collected: - return - from app.services.sector_map import normalise_symbol - - demeaned: dict = defaultdict(list) - for week_key, recs in collected["mom_12_1"].items(): - parsed: list[tuple[str, float, float, object]] = [] - by_sector: dict[str, list[float]] = defaultdict(list) - for rec in recs: - pair = _obs_val_fwd(rec) - if pair is None: - continue - val, fwd = pair - if isinstance(rec, dict): - sym = rec.get("symbol") - else: - sym = None - if not sym: - continue - sector = symbol_to_sector.get(normalise_symbol(str(sym))) - if not sector: - continue - parsed.append((sector, val, fwd, rec)) - by_sector[sector].append(val) - means = { - sec: sum(vs) / len(vs) - for sec, vs in by_sector.items() - if len(vs) >= min_sector_names - } - for sector, val, fwd, rec in parsed: - if sector not in means: - continue - dval = val - means[sector] - if isinstance(rec, dict): - row = dict(rec) - row["val"] = dval - demeaned[week_key].append(row) - else: - demeaned[week_key].append((dval, fwd)) - if demeaned: - collected["mom_12_1_sector_demeaned"] = demeaned - - def _rank(xs: list[float]) -> list[float]: """Average (tie-corrected) ranks, 1-based.""" order = sorted(range(len(xs)), key=lambda k: xs[k]) @@ -1388,38 +1261,14 @@ def _signal_series( benchmark_closes: dict[date, float] | None = None, *, symbol: str | None = None, - sector_etf_closes: dict[date, float] | None = None, ) -> dict: """Per-ticker signal/forward-return series as a PLAIN (picklable) nested dict — no defaultdict/lambda — so it can cross a process boundary.""" tmp: dict = defaultdict(lambda: defaultdict(list)) - _accumulate_signal_series( - records, - tmp, - benchmark_closes, - symbol=symbol, - sector_etf_closes=sector_etf_closes, - ) + _accumulate_signal_series(records, tmp, benchmark_closes, symbol=symbol) return {name: dict(weeks) for name, weeks in tmp.items()} -def _sector_etf_closes_for_symbol( - symbol: str, - symbol_to_sector: dict[str, str] | None, - sector_etf_closes: dict[str, dict[date, float]] | None, -) -> dict[date, float] | None: - """Resolve the sector-ETF close series for one ticker, or None.""" - if not symbol_to_sector or not sector_etf_closes: - return None - from app.services.sector_map import etf_for_symbol - - etf = etf_for_symbol(symbol, symbol_to_sector) - if not etf: - return None - series = sector_etf_closes.get(etf) - return series or None - - def _replay_and_signals( symbol: str, columns: tuple, @@ -1429,8 +1278,6 @@ def _replay_and_signals( target_model: str = PRODUCTION_GTL_TARGET_MODEL, cadence: str = DEFAULT_BACKTEST_CADENCE, signal_only: bool = False, - sector_etf_closes: dict[str, dict[date, float]] | None = None, - symbol_to_sector: dict[str, str] | None = None, ) -> tuple[list[dict], dict]: """The CPU-bound per-ticker work, as a top-level (picklable) function so it can run in a worker process. Takes primitive column arrays (cheap to pickle), @@ -1457,17 +1304,9 @@ def _replay_and_signals( target_model, cadence, ) - etf_closes = _sector_etf_closes_for_symbol( - symbol, symbol_to_sector, sector_etf_closes - ) return ( candidates, - _signal_series( - bars, - benchmark_closes, - symbol=symbol, - sector_etf_closes=etf_closes, - ), + _signal_series(bars, benchmark_closes, symbol=symbol), ) @@ -4218,41 +4057,6 @@ async def run_backtest( except Exception: logger.exception("Benchmark load for residual momentum failed") - # Optional sector residualisation (research): local ticker→sector map + sector - # ETF closes stored in benchmark_prices. Absent map/series → no sector signals. - symbol_to_sector: dict[str, str] = {} - sector_etf_closes: dict[str, dict[date, float]] = {} - try: - from app.services.sector_map import ( - SECTOR_ETFS, - load_ticker_sector_map, - normalise_symbol, - ) - from app.services.benchmark_service import load_benchmark_closes - - map_path = os.getenv("BACKTEST_SECTOR_MAP_PATH", "").strip() or None - symbol_to_sector = { - normalise_symbol(k): v - for k, v in load_ticker_sector_map(map_path).items() - } - if symbol_to_sector: - for etf in SECTOR_ETFS: - try: - series = await load_benchmark_closes(db, etf) - except Exception: - series = {} - if series: - sector_etf_closes[etf] = series - logger.info(json.dumps({ - "event": "backtest_sector_context_loaded", - "sector_map_size": len(symbol_to_sector), - "sector_etfs_loaded": sorted(sector_etf_closes), - })) - except Exception: - logger.exception("Sector residual context load failed; continuing without") - symbol_to_sector = {} - sector_etf_closes = {} - def _merge(result: tuple[list[dict], dict]) -> None: cands, series = result candidates.extend(cands) @@ -4303,8 +4107,6 @@ async def run_backtest( target_model, cadence, ticker.symbol in rank_only_symbols, - sector_etf_closes or None, - symbol_to_sector or None, )) for result in await asyncio.gather(*futures, return_exceptions=True): if isinstance(result, Exception): @@ -4333,8 +4135,6 @@ async def run_backtest( target_model, cadence, ticker.symbol in rank_only_symbols, - sector_etf_closes or None, - symbol_to_sector or None, )) except Exception: logger.exception("Backtest replay failed for %s", ticker.symbol) @@ -4342,13 +4142,6 @@ async def run_backtest( if progress_cb is not None and total: progress_cb(total, total, "") - # Cross-sectional sector demean needs the full weekly universe. - if symbol_to_sector: - try: - _inject_sector_demeaned_momentum(collected, symbol_to_sector) - except Exception: - logger.exception("Sector demeaned momentum injection failed") - # Cross-sectional momentum: rank every week's universe, then "qualified" means # floors + top ``min_momentum_percentile`` by promoted residual 12-1 momentum # (raw 12-1 fallback only when benchmark data is unavailable). diff --git a/app/services/sector_map.py b/app/services/sector_map.py deleted file mode 100644 index f72230c..0000000 --- a/app/services/sector_map.py +++ /dev/null @@ -1,145 +0,0 @@ -"""Ticker → GICS sector → SPDR sector ETF mapping (research only). - -Sector residual momentum residualizes 12-1 momentum against SPY and the name's -sector ETF. Labels are persisted under ``data/research/ticker_sector_map.json`` -so research runs do not depend on live FMP calls. -""" - -from __future__ import annotations - -import json -from pathlib import Path -from typing import Any - -# Eleven SPDR sector ETFs. Auxiliary series only — never tradable book members. -SECTOR_ETFS: tuple[str, ...] = ( - "XLB", - "XLC", - "XLE", - "XLF", - "XLI", - "XLK", - "XLP", - "XLRE", - "XLU", - "XLV", - "XLY", -) - -# GICS sector name (and common aliases) → SPDR ETF. -# Keys are lower-case for matching. -GICS_SECTOR_TO_ETF: dict[str, str] = { - "materials": "XLB", - "basic materials": "XLB", - "communication services": "XLC", - "communications": "XLC", - "telecommunication services": "XLC", - "energy": "XLE", - "financials": "XLF", - "financial services": "XLF", - "financial": "XLF", - "industrials": "XLI", - "industrial goods": "XLI", - "information technology": "XLK", - "technology": "XLK", - "consumer staples": "XLP", - "consumer defensive": "XLP", - "real estate": "XLRE", - "utilities": "XLU", - "health care": "XLV", - "healthcare": "XLV", - "consumer discretionary": "XLY", - "consumer cyclical": "XLY", -} - -DEFAULT_SECTOR_MAP_PATH = Path("data/research/ticker_sector_map.json") - - -def normalise_symbol(symbol: str) -> str: - """Alpaca-style symbols: BRK.B / BRK/B → BRK-B.""" - s = str(symbol or "").strip().upper() - s = s.replace(".", "-").replace("/", "-") - return s - - -def sector_to_etf(sector: str | None) -> str | None: - if not sector: - return None - return GICS_SECTOR_TO_ETF.get(str(sector).strip().lower()) - - -def etf_for_symbol(symbol: str, symbol_to_sector: dict[str, str]) -> str | None: - sector = symbol_to_sector.get(normalise_symbol(symbol)) - return sector_to_etf(sector) - - -def load_ticker_sector_map(path: Path | str | None = None) -> dict[str, str]: - """Load ``{symbol: gics_sector}`` from JSON. Empty dict if missing.""" - p = Path(path) if path is not None else DEFAULT_SECTOR_MAP_PATH - if not p.exists(): - return {} - raw = json.loads(p.read_text(encoding="utf-8")) - if not isinstance(raw, dict): - return {} - out: dict[str, str] = {} - # Accept either flat map or {"map": {...}, "meta": ...} - payload = raw.get("map") if "map" in raw and isinstance(raw.get("map"), dict) else raw - if not isinstance(payload, dict): - return {} - for sym, sector in payload.items(): - if sym in ("meta", "schema_version", "map"): - continue - if sector is None: - continue - ns = normalise_symbol(str(sym)) - if ns: - out[ns] = str(sector).strip() - return out - - -def save_ticker_sector_map( - mapping: dict[str, str], - path: Path | str | None = None, - *, - meta: dict[str, Any] | None = None, -) -> Path: - p = Path(path) if path is not None else DEFAULT_SECTOR_MAP_PATH - p.parent.mkdir(parents=True, exist_ok=True) - # Normalise keys on write. - clean = { - normalise_symbol(k): str(v).strip() - for k, v in mapping.items() - if k and v and normalise_symbol(k) - } - payload: dict[str, Any] = { - "schema_version": 1, - "map": clean, - "meta": meta or {}, - } - p.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") - return p - - -def coverage_stats( - symbols: list[str], mapping: dict[str, str] -) -> dict[str, Any]: - total = len(symbols) - mapped = [s for s in symbols if normalise_symbol(s) in mapping] - with_etf = [ - s - for s in mapped - if sector_to_etf(mapping[normalise_symbol(s)]) is not None - ] - missing = [s for s in symbols if normalise_symbol(s) not in mapping] - by_sector: dict[str, int] = {} - for s in mapped: - sec = mapping[normalise_symbol(s)] - by_sector[sec] = by_sector.get(sec, 0) + 1 - return { - "universe": total, - "mapped": len(mapped), - "mapped_pct": round(100.0 * len(mapped) / total, 1) if total else 0.0, - "with_etf": len(with_etf), - "missing": missing, - "by_sector": dict(sorted(by_sector.items(), key=lambda kv: (-kv[1], kv[0]))), - } diff --git a/data/research/ticker_sector_map.json b/data/research/ticker_sector_map.json deleted file mode 100644 index 31cc0db..0000000 --- a/data/research/ticker_sector_map.json +++ /dev/null @@ -1,543 +0,0 @@ -{ - "map": { - "A": "Health Care", - "AAPL": "Information Technology", - "ABBV": "Health Care", - "ABNB": "Consumer Discretionary", - "ABT": "Health Care", - "ACGL": "Financials", - "ACN": "Information Technology", - "ADBE": "Information Technology", - "ADI": "Information Technology", - "ADM": "Consumer Staples", - "ADP": "Industrials", - "ADSK": "Information Technology", - "AEE": "Utilities", - "AEP": "Utilities", - "AES": "Utilities", - "AFL": "Financials", - "AIG": "Financials", - "AIZ": "Financials", - "AJG": "Financials", - "AKAM": "Information Technology", - "ALB": "Materials", - "ALGN": "Health Care", - "ALL": "Financials", - "ALLE": "Industrials", - "AMAT": "Information Technology", - "AMCR": "Materials", - "AMD": "Information Technology", - "AME": "Industrials", - "AMGN": "Health Care", - "AMP": "Financials", - "AMT": "Real Estate", - "AMZN": "Consumer Discretionary", - "ANET": "Information Technology", - "AON": "Financials", - "AOS": "Industrials", - "APA": "Energy", - "APD": "Materials", - "APH": "Information Technology", - "APO": "Financials", - "APP": "Information Technology", - "APTV": "Consumer Discretionary", - "ARE": "Real Estate", - "ARES": "Financials", - "ATO": "Utilities", - "AVB": "Real Estate", - "AVGO": "Information Technology", - "AVY": "Materials", - "AWK": "Utilities", - "AXON": "Industrials", - "AXP": "Financials", - "AZO": "Consumer Discretionary", - "BA": "Industrials", - "BAC": "Financials", - "BALL": "Materials", - "BAX": "Health Care", - "BBY": "Consumer Discretionary", - "BDX": "Health Care", - "BEN": "Financials", - "BF-B": "Consumer Staples", - "BG": "Consumer Staples", - "BIIB": "Health Care", - "BK": "Financial Services", - "BKNG": "Consumer Discretionary", - "BKR": "Energy", - "BLDR": "Industrials", - "BLK": "Financials", - "BMY": "Health Care", - "BR": "Industrials", - "BRK-B": "Financials", - "BRO": "Financials", - "BSX": "Health Care", - "BX": "Financials", - "BXP": "Real Estate", - "C": "Financials", - "CAG": "Consumer Defensive", - "CAH": "Health Care", - "CARR": "Industrials", - "CASY": "Consumer Staples", - "CAT": "Industrials", - "CB": "Financials", - "CBOE": "Financials", - "CBRE": "Real Estate", - "CCI": "Real Estate", - "CCL": "Consumer Discretionary", - "CDNS": "Information Technology", - "CDW": "Information Technology", - "CEG": "Utilities", - "CF": "Materials", - "CFG": "Financials", - "CHD": "Consumer Staples", - "CHRW": "Industrials", - "CHTR": "Communication Services", - "CI": "Health Care", - "CIEN": "Information Technology", - "CINF": "Financials", - "CL": "Consumer Staples", - "CLX": "Consumer Staples", - "CMCSA": "Communication Services", - "CME": "Financials", - "CMG": "Consumer Discretionary", - "CMI": "Industrials", - "CMS": "Utilities", - "CNC": "Health Care", - "CNP": "Utilities", - "COF": "Financials", - "COHR": "Information Technology", - "COIN": "Financials", - "COO": "Health Care", - "COP": "Energy", - "COR": "Health Care", - "COST": "Consumer Staples", - "CPAY": "Financials", - "CPB": "Consumer Defensive", - "CPRT": "Industrials", - "CPT": "Real Estate", - "CRH": "Materials", - "CRL": "Health Care", - "CRM": "Information Technology", - "CRWD": "Information Technology", - "CSCO": "Information Technology", - "CSGP": "Real Estate", - "CSX": "Industrials", - "CTAS": "Industrials", - "CTRA": "Energy", - "CTSH": "Information Technology", - "CTVA": "Materials", - "CVNA": "Consumer Discretionary", - "CVS": "Health Care", - "CVX": "Energy", - "D": "Utilities", - "DAL": "Industrials", - "DASH": "Consumer Discretionary", - "DD": "Materials", - "DDOG": "Information Technology", - "DE": "Industrials", - "DECK": "Consumer Discretionary", - "DELL": "Information Technology", - "DG": "Consumer Staples", - "DGX": "Health Care", - "DHI": "Consumer Discretionary", - "DHR": "Health Care", - "DIS": "Communication Services", - "DLR": "Real Estate", - "DLTR": "Consumer Staples", - "DOC": "Real Estate", - "DOV": "Industrials", - "DOW": "Materials", - "DPZ": "Consumer Discretionary", - "DRI": "Consumer Discretionary", - "DTE": "Utilities", - "DUK": "Utilities", - "DVA": "Health Care", - "DVN": "Energy", - "DXCM": "Health Care", - "EA": "Communication Services", - "EBAY": "Consumer Discretionary", - "ECL": "Materials", - "ED": "Utilities", - "EFX": "Industrials", - "EG": "Financials", - "EIX": "Utilities", - "EL": "Consumer Staples", - "ELV": "Health Care", - "EME": "Industrials", - "EMR": "Industrials", - "EOG": "Energy", - "EPAM": "Technology", - "EQIX": "Real Estate", - "EQR": "Real Estate", - "EQT": "Energy", - "ERIE": "Financials", - "ES": "Utilities", - "ESS": "Real Estate", - "ETN": "Industrials", - "ETR": "Utilities", - "EVRG": "Utilities", - "EW": "Health Care", - "EXC": "Utilities", - "EXE": "Energy", - "EXPD": "Industrials", - "EXPE": "Consumer Discretionary", - "EXR": "Real Estate", - "F": "Consumer Discretionary", - "FANG": "Energy", - "FAST": "Industrials", - "FCX": "Materials", - "FDS": "Financials", - "FDX": "Industrials", - "FE": "Utilities", - "FFIV": "Information Technology", - "FICO": "Information Technology", - "FIS": "Financials", - "FISV": "Financials", - "FITB": "Financials", - "FIX": "Industrials", - "FOX": "Communication Services", - "FOXA": "Communication Services", - "FRT": "Real Estate", - "FSLR": "Information Technology", - "FTNT": "Information Technology", - "FTV": "Industrials", - "GD": "Industrials", - "GDDY": "Information Technology", - "GE": "Industrials", - "GEHC": "Health Care", - "GEN": "Information Technology", - "GEV": "Industrials", - "GILD": "Health Care", - "GIS": "Consumer Staples", - "GL": "Financials", - "GLW": "Information Technology", - "GM": "Consumer Discretionary", - "GNRC": "Industrials", - "GOOG": "Communication Services", - "GOOGL": "Communication Services", - "GPC": "Consumer Discretionary", - "GPN": "Financials", - "GRMN": "Consumer Discretionary", - "GS": "Financials", - "GWW": "Industrials", - "HAL": "Energy", - "HAS": "Consumer Discretionary", - "HBAN": "Financials", - "HCA": "Health Care", - "HD": "Consumer Discretionary", - "HIG": "Financials", - "HII": "Industrials", - "HLT": "Consumer Discretionary", - "HON": "Industrials", - "HOOD": "Financials", - "HPE": "Information Technology", - "HPQ": "Information Technology", - "HRL": "Consumer Staples", - "HSIC": "Health Care", - "HST": "Real Estate", - "HSY": "Consumer Staples", - "HUBB": "Industrials", - "HUM": "Health Care", - "HWM": "Industrials", - "IBKR": "Financials", - "IBM": "Information Technology", - "ICE": "Financials", - "IDXX": "Health Care", - "IEX": "Industrials", - "IFF": "Materials", - "INCY": "Health Care", - "INTC": "Information Technology", - "INTU": "Information Technology", - "INVH": "Real Estate", - "IP": "Materials", - "IQV": "Health Care", - "IR": "Industrials", - "IRM": "Real Estate", - "ISRG": "Health Care", - "IT": "Information Technology", - "ITW": "Industrials", - "IVZ": "Financials", - "J": "Industrials", - "JBHT": "Industrials", - "JBL": "Information Technology", - "JCI": "Industrials", - "JKHY": "Financials", - "JNJ": "Health Care", - "JPM": "Financials", - "KDP": "Consumer Staples", - "KEY": "Financials", - "KEYS": "Information Technology", - "KHC": "Consumer Staples", - "KIM": "Real Estate", - "KKR": "Financials", - "KLAC": "Information Technology", - "KMB": "Consumer Staples", - "KMI": "Energy", - "KO": "Consumer Staples", - "KR": "Consumer Staples", - "KVUE": "Consumer Staples", - "L": "Financials", - "LDOS": "Industrials", - "LEN": "Consumer Discretionary", - "LH": "Health Care", - "LHX": "Industrials", - "LII": "Industrials", - "LIN": "Materials", - "LITE": "Information Technology", - "LLY": "Health Care", - "LMT": "Industrials", - "LNT": "Utilities", - "LOW": "Consumer Discretionary", - "LRCX": "Information Technology", - "LULU": "Consumer Discretionary", - "LUV": "Industrials", - "LVS": "Consumer Discretionary", - "LYB": "Materials", - "LYV": "Communication Services", - "MA": "Financials", - "MAA": "Real Estate", - "MAR": "Consumer Discretionary", - "MAS": "Industrials", - "MCD": "Consumer Discretionary", - "MCHP": "Information Technology", - "MCK": "Health Care", - "MCO": "Financials", - "MDLZ": "Consumer Staples", - "MDT": "Health Care", - "MET": "Financials", - "META": "Communication Services", - "MGM": "Consumer Discretionary", - "MKC": "Consumer Staples", - "MLM": "Materials", - "MMM": "Industrials", - "MNST": "Consumer Staples", - "MO": "Consumer Staples", - "MOS": "Materials", - "MPC": "Energy", - "MPWR": "Information Technology", - "MRK": "Health Care", - "MRNA": "Health Care", - "MRSH": "Financials", - "MS": "Financials", - "MSCI": "Financials", - "MSFT": "Information Technology", - "MSI": "Information Technology", - "MSTR": "Technology", - "MTB": "Financials", - "MTD": "Health Care", - "MU": "Information Technology", - "NCLH": "Consumer Discretionary", - "NDAQ": "Financials", - "NDSN": "Industrials", - "NEE": "Utilities", - "NEM": "Materials", - "NFLX": "Communication Services", - "NI": "Utilities", - "NKE": "Consumer Discretionary", - "NOC": "Industrials", - "NOW": "Information Technology", - "NRG": "Utilities", - "NSC": "Industrials", - "NTAP": "Information Technology", - "NTRS": "Financials", - "NUE": "Materials", - "NVDA": "Information Technology", - "NVR": "Consumer Discretionary", - "NWS": "Communication Services", - "NWSA": "Communication Services", - "NXPI": "Information Technology", - "O": "Real Estate", - "ODFL": "Industrials", - "OKE": "Energy", - "OMC": "Communication Services", - "ON": "Information Technology", - "ORCL": "Information Technology", - "ORLY": "Consumer Discretionary", - "OTIS": "Industrials", - "OXY": "Energy", - "PANW": "Information Technology", - "PAYX": "Industrials", - "PCAR": "Industrials", - "PCG": "Utilities", - "PEG": "Utilities", - "PEP": "Consumer Staples", - "PFE": "Health Care", - "PFG": "Financials", - "PG": "Consumer Staples", - "PGR": "Financials", - "PH": "Industrials", - "PHM": "Consumer Discretionary", - "PKG": "Materials", - "PLD": "Real Estate", - "PLTR": "Information Technology", - "PM": "Consumer Staples", - "PNC": "Financials", - "PNR": "Industrials", - "PNW": "Utilities", - "PODD": "Health Care", - "POOL": "Industrials", - "PPG": "Materials", - "PPL": "Utilities", - "PRU": "Financials", - "PSA": "Real Estate", - "PSKY": "Communication Services", - "PSX": "Energy", - "PTC": "Information Technology", - "PWR": "Industrials", - "PYPL": "Financials", - "Q": "Information Technology", - "QCOM": "Information Technology", - "RCL": "Consumer Discretionary", - "REG": "Real Estate", - "REGN": "Health Care", - "RF": "Financials", - "RJF": "Financials", - "RL": "Consumer Discretionary", - "RMD": "Health Care", - "ROK": "Industrials", - "ROL": "Industrials", - "ROP": "Information Technology", - "ROST": "Consumer Discretionary", - "RSG": "Industrials", - "RTX": "Industrials", - "RVTY": "Health Care", - "SATS": "Communication Services", - "SBAC": "Real Estate", - "SBUX": "Consumer Discretionary", - "SCHW": "Financials", - "SHW": "Materials", - "SJM": "Consumer Staples", - "SLB": "Energy", - "SMCI": "Information Technology", - "SNA": "Industrials", - "SNDK": "Information Technology", - "SNPS": "Information Technology", - "SO": "Utilities", - "SOLV": "Health Care", - "SPCX": "Industrials", - "SPG": "Real Estate", - "SPGI": "Financials", - "SRE": "Utilities", - "STE": "Health Care", - "STLD": "Materials", - "STT": "Financials", - "STX": "Information Technology", - "STZ": "Consumer Staples", - "SW": "Materials", - "SWK": "Industrials", - "SWKS": "Information Technology", - "SYF": "Financials", - "SYK": "Health Care", - "SYY": "Consumer Staples", - "T": "Communication Services", - "TAP": "Consumer Staples", - "TDG": "Industrials", - "TDY": "Information Technology", - "TECH": "Health Care", - "TEL": "Information Technology", - "TER": "Information Technology", - "TFC": "Financials", - "TGT": "Consumer Staples", - "TJX": "Consumer Discretionary", - "TKO": "Communication Services", - "TMO": "Health Care", - "TMUS": "Communication Services", - "TPL": "Energy", - "TPR": "Consumer Discretionary", - "TRGP": "Energy", - "TRMB": "Information Technology", - "TROW": "Financials", - "TRV": "Financials", - "TSCO": "Consumer Discretionary", - "TSLA": "Consumer Discretionary", - "TSN": "Consumer Staples", - "TT": "Industrials", - "TTD": "Communication Services", - "TTWO": "Communication Services", - "TXN": "Information Technology", - "TXT": "Industrials", - "TYL": "Information Technology", - "UAL": "Industrials", - "UBER": "Industrials", - "UDR": "Real Estate", - "UHS": "Health Care", - "ULTA": "Consumer Discretionary", - "UNH": "Health Care", - "UNP": "Industrials", - "UPS": "Industrials", - "URI": "Industrials", - "USB": "Financials", - "V": "Financials", - "VICI": "Real Estate", - "VLO": "Energy", - "VLTO": "Industrials", - "VMC": "Materials", - "VRSK": "Industrials", - "VRSN": "Information Technology", - "VRT": "Industrials", - "VRTX": "Health Care", - "VST": "Utilities", - "VTR": "Real Estate", - "VTRS": "Health Care", - "VZ": "Communication Services", - "WAB": "Industrials", - "WAT": "Health Care", - "WBD": "Communication Services", - "WDAY": "Information Technology", - "WDC": "Information Technology", - "WEC": "Utilities", - "WELL": "Real Estate", - "WFC": "Financials", - "WM": "Industrials", - "WMB": "Energy", - "WMT": "Consumer Staples", - "WRB": "Financials", - "WSM": "Consumer Discretionary", - "WST": "Health Care", - "WTW": "Financials", - "WY": "Real Estate", - "WYNN": "Consumer Discretionary", - "XEL": "Utilities", - "XOM": "Energy", - "XYL": "Industrials", - "XYZ": "Financials", - "YUM": "Consumer Discretionary", - "ZBH": "Health Care", - "ZBRA": "Information Technology", - "ZTS": "Health Care" - }, - "meta": { - "built_at": "2026-07-19T05:35:41.184460+00:00", - "coverage": { - "by_sector": { - "Communication Services": 23, - "Consumer Defensive": 2, - "Consumer Discretionary": 47, - "Consumer Staples": 34, - "Energy": 22, - "Financial Services": 1, - "Financials": 75, - "Health Care": 58, - "Industrials": 81, - "Information Technology": 72, - "Materials": 26, - "Real Estate": 31, - "Technology": 2, - "Utilities": 31 - }, - "mapped": 505, - "mapped_pct": 99.8, - "universe": 506, - "with_etf": 505 - }, - "fmp_requests": 10, - "from_existing": 0, - "from_fmp": 9, - "from_sp500_csv": 496, - "snapshot": "C:\\Workspace\\signal-platform\\backtest_snapshots\\prod.sqlite", - "still_missing": [ - "RHM" - ] - }, - "schema_version": 1 -} diff --git a/docs/research/history-depth-extension.md b/docs/research/history-depth-extension.md index 182b315..59857d0 100644 --- a/docs/research/history-depth-extension.md +++ b/docs/research/history-depth-extension.md @@ -172,9 +172,12 @@ report does not change that without a separate A/B. Flag for human awareness onl | file | role | |---|---| -| `reports/history-depth-20260719-103315.json` | **authoritative** | -| `reports/history-depth-20260719-103315.md` | companion dump | -| `reports/history-depth-20260719-093853` … `095156` | **ignore** (partial) | +| `reports/history-depth-20260719-103315.json` | Superseded unmasked/two-tier IC dump (do not cite for sector residual) | +| `reports/sector-resid-deep-20260719-113319.json` | Authoritative sector-resid deep grade | +| `reports/prod-book-universe-horizon-20260719-140737.json` | 505 vs liquid × horizon book matrix | + +Intermediate history-depth partials (093853–095156) and SANITY-FAIL noise were +removed in branch cleanup. --- diff --git a/docs/research/sector-residual-momentum.md b/docs/research/sector-residual-momentum.md index 8400bf0..aaf9656 100644 --- a/docs/research/sector-residual-momentum.md +++ b/docs/research/sector-residual-momentum.md @@ -219,20 +219,17 @@ to reopen promotion. --- -## Implementation notes (research machinery) +## Implementation notes -| piece | role | -|---|---| -| `app/services/sector_map.py` | GICS→ETF map, symbol normalise, JSON load/save | -| `app/services/backtest_service.py` | multi-factor residual; `mom_12_1_sector_resid` in `_signal_values`; demean inject | -| `scripts/build_ticker_sector_map.py` | SP500 CSV + FMP gap fill | -| `scripts/fetch_sector_etfs_to_snapshot.py` | Alpaca → snapshot `benchmark_prices` | -| `scripts/run_sector_residual_research.py` | race guard, IC, optional A/B, reports | -| `data/research/ticker_sector_map.json` | persisted labels (research only) | +Research runners and sector-residual harness hooks were **removed after close** +(2026-07-19 cleanup). Evidence remains in the report artifacts below. Do not +re-add without a new pre-registered protocol. --- ## Artifacts -- JSON: `reports/sector-residual-20260719-083356.json` -- MD copy: `reports/sector-residual-20260719-083356.md` +| file | role | +|---|---| +| `reports/sector-resid-deep-20260719-113319.json` | **Authoritative deep FAIL** | +| `reports/sector-residual-20260719-083356.json` | Short-window IC/A/B (superseded for promotion) | diff --git a/reports/history-depth-20260719-093853.json b/reports/history-depth-20260719-093853.json deleted file mode 100644 index 6ee84fb..0000000 --- a/reports/history-depth-20260719-093853.json +++ /dev/null @@ -1,68 +0,0 @@ -{ - "generated_at": "2026-07-19T09:38:53.936226", - "survivorship_banner": "SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only \u2014 not levels.", - "coverage": { - "snapshot": "/Users/taathde3/git/lab/signal_platform/backtest_snapshots/prod.sqlite", - "ticker_count": 506, - "ohlcv_row_count": 629263, - "date_range": { - "min": "2021-06-24", - "max": "2026-07-02" - }, - "bars_per_year": [ - { - "year": "2021", - "bars": 65678, - "tickers_with_bars": 494 - }, - { - "year": "2022", - "bars": 124423, - "tickers_with_bars": 497 - }, - { - "year": "2023", - "bars": 124479, - "tickers_with_bars": 499 - }, - { - "year": "2024", - "bars": 126129, - "tickers_with_bars": 501 - }, - { - "year": "2025", - "bars": 125615, - "tickers_with_bars": 504 - }, - { - "year": "2026", - "bars": 62939, - "tickers_with_bars": 505 - } - ], - "bars_per_symbol": { - "min": 14, - "p10": 1261, - "p50": 1261, - "p90": 1261, - "max": 1261 - }, - "symbols_by_start_year": { - "2021": 494, - "2022": 3, - "2023": 2, - "2024": 2, - "2025": 3, - "2026": 1 - }, - "note": "Where ticker counts drop in early years, the feed (or listing history) thins \u2014 do not treat those years as a full 505-name cross-section.", - "survivorship_banner": "SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only \u2014 not levels." - }, - "race_guard": null, - "harness": null, - "verdict": "COVERAGE_ONLY", - "verdict_detail": "Coverage probe only; run --phase harness after deep rebuild.", - "human_next": "- Compare sector residual vs market residual across eras.\n- If pre-2021 IC collapses, park Task 1 wire-in.\n- Do not retune production knobs on deep history levels.", - "report_path": "reports/history-depth-20260719-093853.json" -} diff --git a/reports/history-depth-20260719-093853.md b/reports/history-depth-20260719-093853.md deleted file mode 100644 index 658f7d3..0000000 --- a/reports/history-depth-20260719-093853.md +++ /dev/null @@ -1,177 +0,0 @@ -# History-depth extension (Tier-1 alpha research) - -**Status:** PRE-REGISTERED — run on MacBook (heavy I/O + full harness). -**Branch:** `research/history-depth-extension` (create from latest research stack). -**Production impact:** none. **Do not retune any production knob on deep history.** - ---- - -## Pre-registration (locked before rebuild) - -### Motivation - -All current conclusions rest on ~35 non-overlapping weekly windows in essentially -one post-2021 regime. Extending history toward max Alpaca daily-bar depth adds -the 2018 vol shock and full 2020 crash (where the feed allows). - -### Protocol - -1. **Empirical coverage first** — bars per calendar year per symbol; document - where the feed thins out. Do **not** assume a uniform start date. -2. **Rebuild the research snapshot completely** from prod source + max history - per symbol (`Adjustment.SPLIT`, ~200 req/min pacing via existing extender). -3. **Race guard (rule 6)** — refuse analysis until completion manifest is - `complete=true` and live counts match. -4. **Re-run full signal harness** (all existing signals incl. sector residual / - SUE if present) on the extended window. -5. **Report per signal:** mean IC, t, window count, and **era split** - (pre-/post-2021) — diagnostic only, **not a tuning input**. -6. **Log prominently:** survivorship bias grows with depth (today’s constituents - backfilled). Absolute Sharpe/CAGR on deep history is optimistic; payload is - **relative** signal comparisons and IC stability, not levels. -7. **Do not retune** production knobs. If a knob’s confirmation looks - overturned on deep history → report only; human decides. - -### Success / interpretation (not promotion of a new signal) - -| outcome | meaning | -|---|---| -| Sector residual still ≥ market residual on deep IC + stable sign | strengthens Task 1 PROMOTE case | -| Sector residual collapses pre-2021 | **PARK** Task 1 wire-in | -| SUE remains weak after full earnings + depth | **DEAD** SUE for this stack | -| Any production knob looks worse deep | report; no auto-retune | - ---- - -## MacBook runbook - -Prefer the bundled script (one entry point): - -```bash -git fetch origin && git checkout research/earnings-gap-and-sue -# .env: ALPACA_* required; FMP_* if resuming earnings -# copy backtest_snapshots/prod.sqlite if not already local - -chmod +x scripts/run_tier1_macbook.sh - -# Default: coverage → deep rebuild → harness (+ era split) -./scripts/run_tier1_macbook.sh - -# Optional variants -./scripts/run_tier1_macbook.sh --all # + earnings resume first -./scripts/run_tier1_macbook.sh --earnings-only # multi-day FMP + 2a/2b only -./scripts/run_tier1_macbook.sh --harness-only # skip rebuild -./scripts/run_tier1_macbook.sh --coverage-only - -# Tunables -WORKERS=12 HISTORY_DAYS=5000 ./scripts/run_tier1_macbook.sh -./scripts/run_tier1_macbook.sh --workers 12 --fmp-limit 250 -``` - -Then commit `reports/` + updated research docs, or copy them back to Windows. - ---- - -## Data provenance - -*(filled at run time)* - ---- - -## Results - -Generated: `2026-07-19T09:38:53.936226` - -> **SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only — not levels.** - -### Coverage - -```json -{ - "snapshot": "/Users/taathde3/git/lab/signal_platform/backtest_snapshots/prod.sqlite", - "ticker_count": 506, - "ohlcv_row_count": 629263, - "date_range": { - "min": "2021-06-24", - "max": "2026-07-02" - }, - "bars_per_year": [ - { - "year": "2021", - "bars": 65678, - "tickers_with_bars": 494 - }, - { - "year": "2022", - "bars": 124423, - "tickers_with_bars": 497 - }, - { - "year": "2023", - "bars": 124479, - "tickers_with_bars": 499 - }, - { - "year": "2024", - "bars": 126129, - "tickers_with_bars": 501 - }, - { - "year": "2025", - "bars": 125615, - "tickers_with_bars": 504 - }, - { - "year": "2026", - "bars": 62939, - "tickers_with_bars": 505 - } - ], - "bars_per_symbol": { - "min": 14, - "p10": 1261, - "p50": 1261, - "p90": 1261, - "max": 1261 - }, - "symbols_by_start_year": { - "2021": 494, - "2022": 3, - "2023": 2, - "2024": 2, - "2025": 3, - "2026": 1 - }, - "note": "Where ticker counts drop in early years, the feed (or listing history) thins \u2014 do not treat those years as a full 505-name cross-section.", - "survivorship_banner": "SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only \u2014 not levels." -} -``` - -### Race guard - -```json -{} -``` - -### Signal IC (full extended window) - -_Harness not run this pass._ - -### Era split (diagnostic only) - -_No era split._ - -## Verdict - -**COVERAGE_ONLY** - -Coverage probe only; run --phase harness after deep rebuild. - -## What a human must decide next - -- Compare sector residual vs market residual across eras. -- If pre-2021 IC collapses, park Task 1 wire-in. -- Do not retune production knobs on deep history levels. - -Artifacts: `reports/history-depth-20260719-093853.json` - diff --git a/reports/history-depth-20260719-094134.json b/reports/history-depth-20260719-094134.json deleted file mode 100644 index 826ced0..0000000 --- a/reports/history-depth-20260719-094134.json +++ /dev/null @@ -1,68 +0,0 @@ -{ - "generated_at": "2026-07-19T09:41:34.109378", - "survivorship_banner": "SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only \u2014 not levels.", - "coverage": { - "snapshot": "/Users/taathde3/git/lab/signal_platform/backtest_snapshots/prod.sqlite", - "ticker_count": 506, - "ohlcv_row_count": 629263, - "date_range": { - "min": "2021-06-24", - "max": "2026-07-02" - }, - "bars_per_year": [ - { - "year": "2021", - "bars": 65678, - "tickers_with_bars": 494 - }, - { - "year": "2022", - "bars": 124423, - "tickers_with_bars": 497 - }, - { - "year": "2023", - "bars": 124479, - "tickers_with_bars": 499 - }, - { - "year": "2024", - "bars": 126129, - "tickers_with_bars": 501 - }, - { - "year": "2025", - "bars": 125615, - "tickers_with_bars": 504 - }, - { - "year": "2026", - "bars": 62939, - "tickers_with_bars": 505 - } - ], - "bars_per_symbol": { - "min": 14, - "p10": 1261, - "p50": 1261, - "p90": 1261, - "max": 1261 - }, - "symbols_by_start_year": { - "2021": 494, - "2022": 3, - "2023": 2, - "2024": 2, - "2025": 3, - "2026": 1 - }, - "note": "Where ticker counts drop in early years, the feed (or listing history) thins \u2014 do not treat those years as a full 505-name cross-section.", - "survivorship_banner": "SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only \u2014 not levels." - }, - "race_guard": null, - "harness": null, - "verdict": "COVERAGE_ONLY", - "verdict_detail": "Coverage probe only; run --phase harness after deep rebuild.", - "human_next": "- Compare sector residual vs market residual across eras.\n- If pre-2021 IC collapses, park Task 1 wire-in.\n- Do not retune production knobs on deep history levels.", - "report_path": "reports/history-depth-20260719-094134.json" -} diff --git a/reports/history-depth-20260719-094134.md b/reports/history-depth-20260719-094134.md deleted file mode 100644 index 7b23512..0000000 --- a/reports/history-depth-20260719-094134.md +++ /dev/null @@ -1,177 +0,0 @@ -# History-depth extension (Tier-1 alpha research) - -**Status:** PRE-REGISTERED — run on MacBook (heavy I/O + full harness). -**Branch:** `research/history-depth-extension` (create from latest research stack). -**Production impact:** none. **Do not retune any production knob on deep history.** - ---- - -## Pre-registration (locked before rebuild) - -### Motivation - -All current conclusions rest on ~35 non-overlapping weekly windows in essentially -one post-2021 regime. Extending history toward max Alpaca daily-bar depth adds -the 2018 vol shock and full 2020 crash (where the feed allows). - -### Protocol - -1. **Empirical coverage first** — bars per calendar year per symbol; document - where the feed thins out. Do **not** assume a uniform start date. -2. **Rebuild the research snapshot completely** from prod source + max history - per symbol (`Adjustment.SPLIT`, ~200 req/min pacing via existing extender). -3. **Race guard (rule 6)** — refuse analysis until completion manifest is - `complete=true` and live counts match. -4. **Re-run full signal harness** (all existing signals incl. sector residual / - SUE if present) on the extended window. -5. **Report per signal:** mean IC, t, window count, and **era split** - (pre-/post-2021) — diagnostic only, **not a tuning input**. -6. **Log prominently:** survivorship bias grows with depth (today’s constituents - backfilled). Absolute Sharpe/CAGR on deep history is optimistic; payload is - **relative** signal comparisons and IC stability, not levels. -7. **Do not retune** production knobs. If a knob’s confirmation looks - overturned on deep history → report only; human decides. - -### Success / interpretation (not promotion of a new signal) - -| outcome | meaning | -|---|---| -| Sector residual still ≥ market residual on deep IC + stable sign | strengthens Task 1 PROMOTE case | -| Sector residual collapses pre-2021 | **PARK** Task 1 wire-in | -| SUE remains weak after full earnings + depth | **DEAD** SUE for this stack | -| Any production knob looks worse deep | report; no auto-retune | - ---- - -## MacBook runbook - -Prefer the bundled script (one entry point): - -```bash -git fetch origin && git checkout research/earnings-gap-and-sue -# .env: ALPACA_* required; FMP_* if resuming earnings -# copy backtest_snapshots/prod.sqlite if not already local - -chmod +x scripts/run_tier1_macbook.sh - -# Default: coverage → deep rebuild → harness (+ era split) -./scripts/run_tier1_macbook.sh - -# Optional variants -./scripts/run_tier1_macbook.sh --all # + earnings resume first -./scripts/run_tier1_macbook.sh --earnings-only # multi-day FMP + 2a/2b only -./scripts/run_tier1_macbook.sh --harness-only # skip rebuild -./scripts/run_tier1_macbook.sh --coverage-only - -# Tunables -WORKERS=12 HISTORY_DAYS=5000 ./scripts/run_tier1_macbook.sh -./scripts/run_tier1_macbook.sh --workers 12 --fmp-limit 250 -``` - -Then commit `reports/` + updated research docs, or copy them back to Windows. - ---- - -## Data provenance - -*(filled at run time)* - ---- - -## Results - -Generated: `2026-07-19T09:41:34.109378` - -> **SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only — not levels.** - -### Coverage - -```json -{ - "snapshot": "/Users/taathde3/git/lab/signal_platform/backtest_snapshots/prod.sqlite", - "ticker_count": 506, - "ohlcv_row_count": 629263, - "date_range": { - "min": "2021-06-24", - "max": "2026-07-02" - }, - "bars_per_year": [ - { - "year": "2021", - "bars": 65678, - "tickers_with_bars": 494 - }, - { - "year": "2022", - "bars": 124423, - "tickers_with_bars": 497 - }, - { - "year": "2023", - "bars": 124479, - "tickers_with_bars": 499 - }, - { - "year": "2024", - "bars": 126129, - "tickers_with_bars": 501 - }, - { - "year": "2025", - "bars": 125615, - "tickers_with_bars": 504 - }, - { - "year": "2026", - "bars": 62939, - "tickers_with_bars": 505 - } - ], - "bars_per_symbol": { - "min": 14, - "p10": 1261, - "p50": 1261, - "p90": 1261, - "max": 1261 - }, - "symbols_by_start_year": { - "2021": 494, - "2022": 3, - "2023": 2, - "2024": 2, - "2025": 3, - "2026": 1 - }, - "note": "Where ticker counts drop in early years, the feed (or listing history) thins \u2014 do not treat those years as a full 505-name cross-section.", - "survivorship_banner": "SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only \u2014 not levels." -} -``` - -### Race guard - -```json -{} -``` - -### Signal IC (full extended window) - -_Harness not run this pass._ - -### Era split (diagnostic only) - -_No era split._ - -## Verdict - -**COVERAGE_ONLY** - -Coverage probe only; run --phase harness after deep rebuild. - -## What a human must decide next - -- Compare sector residual vs market residual across eras. -- If pre-2021 IC collapses, park Task 1 wire-in. -- Do not retune production knobs on deep history levels. - -Artifacts: `reports/history-depth-20260719-094134.json` - diff --git a/reports/history-depth-20260719-094344.json b/reports/history-depth-20260719-094344.json deleted file mode 100644 index 5c35f97..0000000 --- a/reports/history-depth-20260719-094344.json +++ /dev/null @@ -1,68 +0,0 @@ -{ - "generated_at": "2026-07-19T09:43:44.520126", - "survivorship_banner": "SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only \u2014 not levels.", - "coverage": { - "snapshot": "/Users/taathde3/git/lab/signal_platform/backtest_snapshots/prod.sqlite", - "ticker_count": 506, - "ohlcv_row_count": 629263, - "date_range": { - "min": "2021-06-24", - "max": "2026-07-02" - }, - "bars_per_year": [ - { - "year": "2021", - "bars": 65678, - "tickers_with_bars": 494 - }, - { - "year": "2022", - "bars": 124423, - "tickers_with_bars": 497 - }, - { - "year": "2023", - "bars": 124479, - "tickers_with_bars": 499 - }, - { - "year": "2024", - "bars": 126129, - "tickers_with_bars": 501 - }, - { - "year": "2025", - "bars": 125615, - "tickers_with_bars": 504 - }, - { - "year": "2026", - "bars": 62939, - "tickers_with_bars": 505 - } - ], - "bars_per_symbol": { - "min": 14, - "p10": 1261, - "p50": 1261, - "p90": 1261, - "max": 1261 - }, - "symbols_by_start_year": { - "2021": 494, - "2022": 3, - "2023": 2, - "2024": 2, - "2025": 3, - "2026": 1 - }, - "note": "Where ticker counts drop in early years, the feed (or listing history) thins \u2014 do not treat those years as a full 505-name cross-section.", - "survivorship_banner": "SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only \u2014 not levels." - }, - "race_guard": null, - "harness": null, - "verdict": "COVERAGE_ONLY", - "verdict_detail": "Coverage probe only; run --phase harness after deep rebuild.", - "human_next": "- Compare sector residual vs market residual across eras.\n- If pre-2021 IC collapses, park Task 1 wire-in.\n- Do not retune production knobs on deep history levels.", - "report_path": "reports/history-depth-20260719-094344.json" -} diff --git a/reports/history-depth-20260719-094344.md b/reports/history-depth-20260719-094344.md deleted file mode 100644 index d52b1bb..0000000 --- a/reports/history-depth-20260719-094344.md +++ /dev/null @@ -1,177 +0,0 @@ -# History-depth extension (Tier-1 alpha research) - -**Status:** PRE-REGISTERED — run on MacBook (heavy I/O + full harness). -**Branch:** `research/history-depth-extension` (create from latest research stack). -**Production impact:** none. **Do not retune any production knob on deep history.** - ---- - -## Pre-registration (locked before rebuild) - -### Motivation - -All current conclusions rest on ~35 non-overlapping weekly windows in essentially -one post-2021 regime. Extending history toward max Alpaca daily-bar depth adds -the 2018 vol shock and full 2020 crash (where the feed allows). - -### Protocol - -1. **Empirical coverage first** — bars per calendar year per symbol; document - where the feed thins out. Do **not** assume a uniform start date. -2. **Rebuild the research snapshot completely** from prod source + max history - per symbol (`Adjustment.SPLIT`, ~200 req/min pacing via existing extender). -3. **Race guard (rule 6)** — refuse analysis until completion manifest is - `complete=true` and live counts match. -4. **Re-run full signal harness** (all existing signals incl. sector residual / - SUE if present) on the extended window. -5. **Report per signal:** mean IC, t, window count, and **era split** - (pre-/post-2021) — diagnostic only, **not a tuning input**. -6. **Log prominently:** survivorship bias grows with depth (today’s constituents - backfilled). Absolute Sharpe/CAGR on deep history is optimistic; payload is - **relative** signal comparisons and IC stability, not levels. -7. **Do not retune** production knobs. If a knob’s confirmation looks - overturned on deep history → report only; human decides. - -### Success / interpretation (not promotion of a new signal) - -| outcome | meaning | -|---|---| -| Sector residual still ≥ market residual on deep IC + stable sign | strengthens Task 1 PROMOTE case | -| Sector residual collapses pre-2021 | **PARK** Task 1 wire-in | -| SUE remains weak after full earnings + depth | **DEAD** SUE for this stack | -| Any production knob looks worse deep | report; no auto-retune | - ---- - -## MacBook runbook - -Prefer the bundled script (one entry point): - -```bash -git fetch origin && git checkout research/earnings-gap-and-sue -# .env: ALPACA_* required; FMP_* if resuming earnings -# copy backtest_snapshots/prod.sqlite if not already local - -chmod +x scripts/run_tier1_macbook.sh - -# Default: coverage → deep rebuild → harness (+ era split) -./scripts/run_tier1_macbook.sh - -# Optional variants -./scripts/run_tier1_macbook.sh --all # + earnings resume first -./scripts/run_tier1_macbook.sh --earnings-only # multi-day FMP + 2a/2b only -./scripts/run_tier1_macbook.sh --harness-only # skip rebuild -./scripts/run_tier1_macbook.sh --coverage-only - -# Tunables -WORKERS=12 HISTORY_DAYS=5000 ./scripts/run_tier1_macbook.sh -./scripts/run_tier1_macbook.sh --workers 12 --fmp-limit 250 -``` - -Then commit `reports/` + updated research docs, or copy them back to Windows. - ---- - -## Data provenance - -*(filled at run time)* - ---- - -## Results - -Generated: `2026-07-19T09:43:44.520126` - -> **SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only — not levels.** - -### Coverage - -```json -{ - "snapshot": "/Users/taathde3/git/lab/signal_platform/backtest_snapshots/prod.sqlite", - "ticker_count": 506, - "ohlcv_row_count": 629263, - "date_range": { - "min": "2021-06-24", - "max": "2026-07-02" - }, - "bars_per_year": [ - { - "year": "2021", - "bars": 65678, - "tickers_with_bars": 494 - }, - { - "year": "2022", - "bars": 124423, - "tickers_with_bars": 497 - }, - { - "year": "2023", - "bars": 124479, - "tickers_with_bars": 499 - }, - { - "year": "2024", - "bars": 126129, - "tickers_with_bars": 501 - }, - { - "year": "2025", - "bars": 125615, - "tickers_with_bars": 504 - }, - { - "year": "2026", - "bars": 62939, - "tickers_with_bars": 505 - } - ], - "bars_per_symbol": { - "min": 14, - "p10": 1261, - "p50": 1261, - "p90": 1261, - "max": 1261 - }, - "symbols_by_start_year": { - "2021": 494, - "2022": 3, - "2023": 2, - "2024": 2, - "2025": 3, - "2026": 1 - }, - "note": "Where ticker counts drop in early years, the feed (or listing history) thins \u2014 do not treat those years as a full 505-name cross-section.", - "survivorship_banner": "SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only \u2014 not levels." -} -``` - -### Race guard - -```json -{} -``` - -### Signal IC (full extended window) - -_Harness not run this pass._ - -### Era split (diagnostic only) - -_No era split._ - -## Verdict - -**COVERAGE_ONLY** - -Coverage probe only; run --phase harness after deep rebuild. - -## What a human must decide next - -- Compare sector residual vs market residual across eras. -- If pre-2021 IC collapses, park Task 1 wire-in. -- Do not retune production knobs on deep history levels. - -Artifacts: `reports/history-depth-20260719-094344.json` - diff --git a/reports/history-depth-20260719-095156.json b/reports/history-depth-20260719-095156.json deleted file mode 100644 index 416c34e..0000000 --- a/reports/history-depth-20260719-095156.json +++ /dev/null @@ -1,68 +0,0 @@ -{ - "generated_at": "2026-07-19T09:51:56.525634", - "survivorship_banner": "SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only \u2014 not levels.", - "coverage": { - "snapshot": "/Users/taathde3/git/lab/signal_platform/backtest_snapshots/prod.sqlite", - "ticker_count": 506, - "ohlcv_row_count": 629263, - "date_range": { - "min": "2021-06-24", - "max": "2026-07-02" - }, - "bars_per_year": [ - { - "year": "2021", - "bars": 65678, - "tickers_with_bars": 494 - }, - { - "year": "2022", - "bars": 124423, - "tickers_with_bars": 497 - }, - { - "year": "2023", - "bars": 124479, - "tickers_with_bars": 499 - }, - { - "year": "2024", - "bars": 126129, - "tickers_with_bars": 501 - }, - { - "year": "2025", - "bars": 125615, - "tickers_with_bars": 504 - }, - { - "year": "2026", - "bars": 62939, - "tickers_with_bars": 505 - } - ], - "bars_per_symbol": { - "min": 14, - "p10": 1261, - "p50": 1261, - "p90": 1261, - "max": 1261 - }, - "symbols_by_start_year": { - "2021": 494, - "2022": 3, - "2023": 2, - "2024": 2, - "2025": 3, - "2026": 1 - }, - "note": "Where ticker counts drop in early years, the feed (or listing history) thins \u2014 do not treat those years as a full 505-name cross-section.", - "survivorship_banner": "SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only \u2014 not levels." - }, - "race_guard": null, - "harness": null, - "verdict": "COVERAGE_ONLY", - "verdict_detail": "Coverage probe only; run --phase harness after deep rebuild.", - "human_next": "- Compare sector residual vs market residual across eras.\n- If pre-2021 IC collapses, park Task 1 wire-in.\n- Do not retune production knobs on deep history levels.", - "report_path": "reports/history-depth-20260719-095156.json" -} diff --git a/reports/history-depth-20260719-095156.md b/reports/history-depth-20260719-095156.md deleted file mode 100644 index a6edf52..0000000 --- a/reports/history-depth-20260719-095156.md +++ /dev/null @@ -1,177 +0,0 @@ -# History-depth extension (Tier-1 alpha research) - -**Status:** PRE-REGISTERED — run on MacBook (heavy I/O + full harness). -**Branch:** `research/history-depth-extension` (create from latest research stack). -**Production impact:** none. **Do not retune any production knob on deep history.** - ---- - -## Pre-registration (locked before rebuild) - -### Motivation - -All current conclusions rest on ~35 non-overlapping weekly windows in essentially -one post-2021 regime. Extending history toward max Alpaca daily-bar depth adds -the 2018 vol shock and full 2020 crash (where the feed allows). - -### Protocol - -1. **Empirical coverage first** — bars per calendar year per symbol; document - where the feed thins out. Do **not** assume a uniform start date. -2. **Rebuild the research snapshot completely** from prod source + max history - per symbol (`Adjustment.SPLIT`, ~200 req/min pacing via existing extender). -3. **Race guard (rule 6)** — refuse analysis until completion manifest is - `complete=true` and live counts match. -4. **Re-run full signal harness** (all existing signals incl. sector residual / - SUE if present) on the extended window. -5. **Report per signal:** mean IC, t, window count, and **era split** - (pre-/post-2021) — diagnostic only, **not a tuning input**. -6. **Log prominently:** survivorship bias grows with depth (today’s constituents - backfilled). Absolute Sharpe/CAGR on deep history is optimistic; payload is - **relative** signal comparisons and IC stability, not levels. -7. **Do not retune** production knobs. If a knob’s confirmation looks - overturned on deep history → report only; human decides. - -### Success / interpretation (not promotion of a new signal) - -| outcome | meaning | -|---|---| -| Sector residual still ≥ market residual on deep IC + stable sign | strengthens Task 1 PROMOTE case | -| Sector residual collapses pre-2021 | **PARK** Task 1 wire-in | -| SUE remains weak after full earnings + depth | **DEAD** SUE for this stack | -| Any production knob looks worse deep | report; no auto-retune | - ---- - -## MacBook runbook - -Prefer the bundled script (one entry point): - -```bash -git fetch origin && git checkout research/earnings-gap-and-sue -# .env: ALPACA_* required; FMP_* if resuming earnings -# copy backtest_snapshots/prod.sqlite if not already local - -chmod +x scripts/run_tier1_macbook.sh - -# Default: coverage → deep rebuild → harness (+ era split) -./scripts/run_tier1_macbook.sh - -# Optional variants -./scripts/run_tier1_macbook.sh --all # + earnings resume first -./scripts/run_tier1_macbook.sh --earnings-only # multi-day FMP + 2a/2b only -./scripts/run_tier1_macbook.sh --harness-only # skip rebuild -./scripts/run_tier1_macbook.sh --coverage-only - -# Tunables -WORKERS=12 HISTORY_DAYS=5000 ./scripts/run_tier1_macbook.sh -./scripts/run_tier1_macbook.sh --workers 12 --fmp-limit 250 -``` - -Then commit `reports/` + updated research docs, or copy them back to Windows. - ---- - -## Data provenance - -*(filled at run time)* - ---- - -## Results - -Generated: `2026-07-19T09:51:56.525634` - -> **SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only — not levels.** - -### Coverage - -```json -{ - "snapshot": "/Users/taathde3/git/lab/signal_platform/backtest_snapshots/prod.sqlite", - "ticker_count": 506, - "ohlcv_row_count": 629263, - "date_range": { - "min": "2021-06-24", - "max": "2026-07-02" - }, - "bars_per_year": [ - { - "year": "2021", - "bars": 65678, - "tickers_with_bars": 494 - }, - { - "year": "2022", - "bars": 124423, - "tickers_with_bars": 497 - }, - { - "year": "2023", - "bars": 124479, - "tickers_with_bars": 499 - }, - { - "year": "2024", - "bars": 126129, - "tickers_with_bars": 501 - }, - { - "year": "2025", - "bars": 125615, - "tickers_with_bars": 504 - }, - { - "year": "2026", - "bars": 62939, - "tickers_with_bars": 505 - } - ], - "bars_per_symbol": { - "min": 14, - "p10": 1261, - "p50": 1261, - "p90": 1261, - "max": 1261 - }, - "symbols_by_start_year": { - "2021": 494, - "2022": 3, - "2023": 2, - "2024": 2, - "2025": 3, - "2026": 1 - }, - "note": "Where ticker counts drop in early years, the feed (or listing history) thins \u2014 do not treat those years as a full 505-name cross-section.", - "survivorship_banner": "SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only \u2014 not levels." -} -``` - -### Race guard - -```json -{} -``` - -### Signal IC (full extended window) - -_Harness not run this pass._ - -### Era split (diagnostic only) - -_No era split._ - -## Verdict - -**COVERAGE_ONLY** - -Coverage probe only; run --phase harness after deep rebuild. - -## What a human must decide next - -- Compare sector residual vs market residual across eras. -- If pre-2021 IC collapses, park Task 1 wire-in. -- Do not retune production knobs on deep history levels. - -Artifacts: `reports/history-depth-20260719-095156.json` - diff --git a/reports/sector-resid-deep-20260719-111923-SANITY-FAIL.json b/reports/sector-resid-deep-20260719-111923-SANITY-FAIL.json deleted file mode 100644 index 6b03e61..0000000 --- a/reports/sector-resid-deep-20260719-111923-SANITY-FAIL.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "step1": { - "shallow_meta": { - "n_symbols": 4654, - "deep_cohort_p10_start": "2016-01-04", - "shallow_cutoff": "2017-02-07", - "lag_days": 400, - "n_shallow": 3349, - "shallow_start_histogram": { - "2017": 117, - "2018": 162, - "2019": 163, - "2020": 269, - "2021": 1060, - "2022": 211, - "2023": 176, - "2024": 281, - "2025": 543, - "2026": 367 - }, - "deep_start_histogram": { - "2016": 1296, - "2017": 9 - }, - "shallow_sample": [ - "A", - "AACB", - "AACBR", - "AACBU", - "AACI", - "AACIU", - "AACIW", - "AACO", - "AACOU", - "AACOW", - "AACP", - "AACPR", - "AACPU", - "AACPW", - "AAPG", - "AAPL", - "AARD", - "ABAT", - "ABBV", - "ABCL", - "ABLV", - "ABLVW", - "ABNB", - "ABOS", - "ABSI", - "ABT", - "ABTC", - "ABVC", - "ABVX", - "ACAA" - ] - }, - "shallow_list_n": 3349, - "fetch_ok": 3349, - "fetch_fail": 0, - "etf_refresh": { - "written": { - "SPY": 0, - "XLB": 0, - "XLC": 0, - "XLE": 0, - "XLF": 0, - "XLI": 0, - "XLK": 0, - "XLP": 0, - "XLRE": 0, - "XLU": 0, - "XLV": 0, - "XLY": 0 - }, - "benchmark_summary": [ - { - "symbol": "SPY", - "n": 2649, - "min": "2016-01-04", - "max": "2026-07-17" - }, - { - "symbol": "XLB", - "n": 2649, - "min": "2016-01-04", - "max": "2026-07-17" - }, - { - "symbol": "XLC", - "n": 2030, - "min": "2018-06-19", - "max": "2026-07-17" - }, - { - "symbol": "XLE", - "n": 2649, - "min": "2016-01-04", - "max": "2026-07-17" - }, - { - "symbol": "XLF", - "n": 2649, - "min": "2016-01-04", - "max": "2026-07-17" - }, - { - "symbol": "XLI", - "n": 2649, - "min": "2016-01-04", - "max": "2026-07-17" - }, - { - "symbol": "XLK", - "n": 2649, - "min": "2016-01-04", - "max": "2026-07-17" - }, - { - "symbol": "XLP", - "n": 2649, - "min": "2016-01-04", - "max": "2026-07-17" - }, - { - "symbol": "XLRE", - "n": 2649, - "min": "2016-01-04", - "max": "2026-07-17" - }, - { - "symbol": "XLU", - "n": 2649, - "min": "2016-01-04", - "max": "2026-07-17" - }, - { - "symbol": "XLV", - "n": 2649, - "min": "2016-01-04", - "max": "2026-07-17" - }, - { - "symbol": "XLY", - "n": 2649, - "min": "2016-01-04", - "max": "2026-07-17" - } - ] - }, - "sanity": { - "passed": false, - "megacap": { - "AAPL": { - "symbol": "AAPL", - "bars": 2649, - "min_date": "2016-01-04", - "max_date": "2026-07-17" - }, - "MSFT": { - "symbol": "MSFT", - "bars": 2649, - "min_date": "2016-01-04", - "max_date": "2026-07-17" - }, - "JPM": { - "symbol": "JPM", - "bars": 2649, - "min_date": "2016-01-04", - "max_date": "2026-07-17" - }, - "XOM": { - "symbol": "XOM", - "bars": 2649, - "min_date": "2016-01-04", - "max_date": "2026-07-17" - }, - "JNJ": { - "symbol": "JNJ", - "bars": 2649, - "min_date": "2016-01-04", - "max_date": "2026-07-17" - } - }, - "megacap_ok": false, - "megacap_deadline": "2013-12-14", - "sector_etfs": { - "XLB": { - "n": 2649, - "min": "2016-01-04", - "max": "2026-07-17" - }, - "XLC": { - "n": 2030, - "min": "2018-06-19", - "max": "2026-07-17" - }, - "XLE": { - "n": 2649, - "min": "2016-01-04", - "max": "2026-07-17" - }, - "XLF": { - "n": 2649, - "min": "2016-01-04", - "max": "2026-07-17" - }, - "XLI": { - "n": 2649, - "min": "2016-01-04", - "max": "2026-07-17" - }, - "XLK": { - "n": 2649, - "min": "2016-01-04", - "max": "2026-07-17" - }, - "XLP": { - "n": 2649, - "min": "2016-01-04", - "max": "2026-07-17" - }, - "XLRE": { - "n": 2649, - "min": "2016-01-04", - "max": "2026-07-17" - }, - "XLU": { - "n": 2649, - "min": "2016-01-04", - "max": "2026-07-17" - }, - "XLV": { - "n": 2649, - "min": "2016-01-04", - "max": "2026-07-17" - }, - "XLY": { - "n": 2649, - "min": "2016-01-04", - "max": "2026-07-17" - } - }, - "sector_etfs_deep_count": 11, - "sector_etfs_ok": true, - "still_shallow_count": 2882, - "still_shallow_sample": [ - "AACB", - "AACBR", - "AACBU", - "AACI", - "AACIU", - "AACIW", - "AACO", - "AACOU", - "AACOW", - "AACP", - "AACPR", - "AACPU", - "AACPW", - "AAPG", - "AARD", - "ABAT", - "ABCL", - "ABLV", - "ABLVW", - "ABNB" - ], - "xlc_note": "XLC lists mid-2018 \u2192 Communication Services residual coverage from ~mid-2019.", - "target_history_days": 5000 - }, - "manifest_path": "backtest_snapshots/research.sqlite.manifest.json" - }, - "harness": null -} diff --git a/scripts/build_ticker_sector_map.py b/scripts/build_ticker_sector_map.py deleted file mode 100644 index e200b76..0000000 --- a/scripts/build_ticker_sector_map.py +++ /dev/null @@ -1,233 +0,0 @@ -"""Build a local ticker → GICS sector map for research residualization. - -Sources (in order): -1. Public S&P 500 constituents CSV (datasets/s-and-p-500-companies) — bulk, free. -2. Existing map file (resume). -3. FMP stable ``profile`` for still-missing symbols (budget ~250 req/day). - -Writes ``data/research/ticker_sector_map.json``. Never touches production Postgres. - -Example -------- - python scripts/build_ticker_sector_map.py \\ - --snapshot backtest_snapshots/prod.sqlite - - python scripts/build_ticker_sector_map.py --fmp-limit 50 -""" - -from __future__ import annotations - -import argparse -import asyncio -import csv -import io -import json -import sys -import time -from datetime import datetime, timezone -from pathlib import Path - -import httpx -from sqlalchemy import create_engine, text - -ROOT = Path(__file__).resolve().parents[1] -if str(ROOT) not in sys.path: - sys.path.insert(0, str(ROOT)) - -from app.ssl_bootstrap import bootstrap_ssl # noqa: E402 - -bootstrap_ssl() - -from app.services.sector_map import ( # noqa: E402 - DEFAULT_SECTOR_MAP_PATH, - coverage_stats, - load_ticker_sector_map, - normalise_symbol, - save_ticker_sector_map, - sector_to_etf, -) - -SP500_CSV_URL = ( - "https://raw.githubusercontent.com/datasets/s-and-p-500-companies/" - "master/data/constituents.csv" -) -FMP_STABLE = "https://financialmodelingprep.com/stable" - - -def _parse_args() -> argparse.Namespace: - p = argparse.ArgumentParser(description=__doc__) - p.add_argument( - "--snapshot", - default="backtest_snapshots/prod.sqlite", - help="Snapshot whose tickers define the universe.", - ) - p.add_argument( - "--out", - default=str(DEFAULT_SECTOR_MAP_PATH), - help="Output JSON path.", - ) - p.add_argument( - "--fmp-limit", - type=int, - default=200, - help="Max FMP profile requests this run (free-tier cushion).", - ) - p.add_argument( - "--skip-fmp", - action="store_true", - help="Only use public SP500 CSV + existing map.", - ) - p.add_argument("--sleep", type=float, default=0.35, help="Pause between FMP calls.") - return p.parse_args() - - -def _snapshot_symbols(snapshot: Path) -> list[str]: - engine = create_engine(f"sqlite:///{snapshot.resolve().as_posix()}", future=True) - try: - with engine.connect() as conn: - rows = conn.execute(text("SELECT symbol FROM tickers ORDER BY symbol")).fetchall() - finally: - engine.dispose() - return [normalise_symbol(r[0]) for r in rows if r[0]] - - -def _fetch_sp500_map() -> dict[str, str]: - with httpx.Client(timeout=60.0, follow_redirects=True) as client: - resp = client.get(SP500_CSV_URL) - resp.raise_for_status() - reader = csv.DictReader(io.StringIO(resp.text)) - out: dict[str, str] = {} - for row in reader: - sym = normalise_symbol(row.get("Symbol") or "") - sector = (row.get("GICS Sector") or "").strip() - if sym and sector: - out[sym] = sector - return out - - -async def _fmp_profile_sector(client: httpx.AsyncClient, api_key: str, symbol: str) -> str | None: - resp = await client.get( - f"{FMP_STABLE}/profile", - params={"symbol": symbol, "apikey": api_key}, - ) - if resp.status_code == 429: - raise RuntimeError(f"FMP rate limited on {symbol}") - if resp.status_code == 402: - return None - resp.raise_for_status() - data = resp.json() - if isinstance(data, list): - data = data[0] if data else {} - if not isinstance(data, dict): - return None - sector = (data.get("sector") or data.get("industry") or "").strip() - # industry alone is not a GICS sector — only accept if we can map to an ETF - if sector and sector_to_etf(sector): - return sector - # FMP sometimes returns industry under sector when sector missing; try sector field only - sec = (data.get("sector") or "").strip() - return sec or None - - -async def _fill_from_fmp( - missing: list[str], - *, - api_key: str, - limit: int, - sleep_s: float, -) -> tuple[dict[str, str], int]: - filled: dict[str, str] = {} - used = 0 - async with httpx.AsyncClient(timeout=30.0) as client: - for sym in missing: - if used >= limit: - break - try: - sector = await _fmp_profile_sector(client, api_key, sym) - except Exception as exc: - print(f" FMP fail {sym}: {exc}") - used += 1 - await asyncio.sleep(sleep_s) - continue - used += 1 - if sector: - filled[sym] = sector - print(f" FMP {sym} → {sector}") - else: - print(f" FMP {sym} → (no sector)") - if sleep_s > 0: - await asyncio.sleep(sleep_s) - return filled, used - - -async def _main() -> None: - args = _parse_args() - snapshot = Path(args.snapshot) - if not snapshot.exists(): - raise SystemExit(f"Snapshot not found: {snapshot}") - - symbols = _snapshot_symbols(snapshot) - print(f"Universe: {len(symbols)} symbols from {snapshot}") - - existing = load_ticker_sector_map(args.out) - print(f"Existing map entries: {len(existing)}") - - print("Fetching public S&P 500 sector CSV…") - sp500 = _fetch_sp500_map() - print(f" SP500 CSV rows: {len(sp500)}") - - mapping = dict(existing) - from_sp500 = 0 - for sym in symbols: - if sym in mapping: - continue - if sym in sp500: - mapping[sym] = sp500[sym] - from_sp500 += 1 - print(f" Newly filled from SP500 CSV: {from_sp500}") - - missing = [s for s in symbols if s not in mapping] - fmp_used = 0 - from_fmp = 0 - if missing and not args.skip_fmp: - from app.config import settings - - if not settings.fmp_api_key: - print("WARNING: FMP key missing; leaving gaps unfilled") - else: - print(f"FMP fill for {len(missing)} missing (limit={args.fmp_limit})…") - filled, fmp_used = await _fill_from_fmp( - missing, - api_key=settings.fmp_api_key, - limit=int(args.fmp_limit), - sleep_s=float(args.sleep), - ) - mapping.update(filled) - from_fmp = len(filled) - - still_missing = [s for s in symbols if s not in mapping] - stats = coverage_stats(symbols, mapping) - meta = { - "built_at": datetime.now(timezone.utc).isoformat(), - "snapshot": str(snapshot.resolve()), - "from_existing": len(existing), - "from_sp500_csv": from_sp500, - "from_fmp": from_fmp, - "fmp_requests": fmp_used, - "still_missing": still_missing, - "coverage": { - k: stats[k] - for k in ("universe", "mapped", "mapped_pct", "with_etf", "by_sector") - }, - } - out_path = save_ticker_sector_map(mapping, args.out, meta=meta) - print(f"Wrote {out_path}") - print(json.dumps(meta["coverage"], indent=2)) - if still_missing: - print(f"Still missing ({len(still_missing)}): {still_missing[:40]}") - if len(still_missing) > 40: - print(f" … +{len(still_missing) - 40} more") - - -if __name__ == "__main__": - asyncio.run(_main()) diff --git a/scripts/fetch_sector_etfs_to_snapshot.py b/scripts/fetch_sector_etfs_to_snapshot.py deleted file mode 100644 index 5968a03..0000000 --- a/scripts/fetch_sector_etfs_to_snapshot.py +++ /dev/null @@ -1,187 +0,0 @@ -"""Fetch the 11 SPDR sector ETFs into a snapshot's ``benchmark_prices``. - -Research-only. Sector ETFs are auxiliary series (like SPY) — they must not -enter the tradable ticker universe or candidate replay. Storing them in -``benchmark_prices`` keeps that invariant. - -Also refreshes SPY on the same window so residual factors share a calendar. - -Example -------- - python scripts/fetch_sector_etfs_to_snapshot.py \\ - --snapshot backtest_snapshots/prod.sqlite --history-days 2200 -""" - -from __future__ import annotations - -import argparse -import asyncio -import sys -import time -from datetime import date, timedelta -from pathlib import Path - -from sqlalchemy import create_engine, text - -ROOT = Path(__file__).resolve().parents[1] -if str(ROOT) not in sys.path: - sys.path.insert(0, str(ROOT)) - -from app.ssl_bootstrap import bootstrap_ssl # noqa: E402 - -bootstrap_ssl() - -from app.services.sector_map import SECTOR_ETFS # noqa: E402 - - -def _parse_args() -> argparse.Namespace: - p = argparse.ArgumentParser(description=__doc__) - p.add_argument("--snapshot", default="backtest_snapshots/prod.sqlite") - p.add_argument( - "--history-days", - type=int, - default=2200, - help="Lookback calendar days (default ~6y; covers 5y snapshot + cushion).", - ) - p.add_argument("--sleep", type=float, default=0.25) - p.add_argument( - "--symbols", - default=None, - help="Comma-separated override (default: SPY + 11 sector ETFs).", - ) - return p.parse_args() - - -async def _fetch_and_upsert( - engine, - provider, - symbol: str, - start: date, - end: date, - *, - sleep_s: float, -) -> int: - from app.exceptions import ProviderError, RateLimitError - - for attempt in range(5): - try: - bars = await provider.fetch_ohlcv(symbol, start, end) - break - except RateLimitError: - wait = min(60.0, 2.0 ** attempt) - print(f" rate limited {symbol}; sleep {wait:.0f}s") - await asyncio.sleep(wait) - bars = [] - except ProviderError as exc: - if attempt + 1 >= 5: - raise - await asyncio.sleep(1.0) - print(f" retry {symbol}: {exc}") - bars = [] - else: - bars = [] - - if sleep_s > 0: - await asyncio.sleep(sleep_s) - - if not bars: - print(f" {symbol}: empty") - return 0 - - written = 0 - with engine.begin() as conn: - for bar in bars: - d = bar.date.isoformat() if hasattr(bar.date, "isoformat") else str(bar.date) - close = float(bar.close) - existing = conn.execute( - text( - "SELECT id, close FROM benchmark_prices " - "WHERE symbol = :sym AND date = :d" - ), - {"sym": symbol, "d": d}, - ).fetchone() - if existing is None: - # id is INTEGER PK — let sqlite autoincrement if possible - conn.execute( - text( - "INSERT INTO benchmark_prices (symbol, date, close) " - "VALUES (:sym, :d, :c)" - ), - {"sym": symbol, "d": d, "c": close}, - ) - written += 1 - elif abs(float(existing[1]) - close) > 1e-9: - conn.execute( - text( - "UPDATE benchmark_prices SET close = :c WHERE id = :id" - ), - {"c": close, "id": int(existing[0])}, - ) - written += 1 - print(f" {symbol}: {len(bars)} bars, {written} rows written/updated") - return written - - -async def _main() -> None: - args = _parse_args() - snapshot = Path(args.snapshot) - if not snapshot.exists(): - raise SystemExit(f"Snapshot not found: {snapshot}") - - from app.config import settings - from app.providers.alpaca import AlpacaOHLCVProvider - - if not settings.alpaca_api_key or not settings.alpaca_api_secret: - raise SystemExit("ALPACA_API_KEY / ALPACA_API_SECRET required") - - if args.symbols: - symbols = [s.strip().upper() for s in args.symbols.split(",") if s.strip()] - else: - symbols = ["SPY", *SECTOR_ETFS] - - end = date.today() - start = end - timedelta(days=int(args.history_days)) - provider = AlpacaOHLCVProvider(settings.alpaca_api_key, settings.alpaca_api_secret) - engine = create_engine( - f"sqlite:///{snapshot.resolve().as_posix()}", - future=True, - ) - - print(f"Snapshot: {snapshot}") - print(f"Window: {start} → {end}") - print(f"Symbols: {symbols}") - - t0 = time.monotonic() - total = 0 - try: - for sym in symbols: - n = await _fetch_and_upsert( - engine, provider, sym, start, end, sleep_s=float(args.sleep) - ) - total += n - finally: - engine.dispose() - - # Summary counts - engine = create_engine( - f"sqlite:///{snapshot.resolve().as_posix()}", - future=True, - ) - try: - with engine.connect() as conn: - rows = conn.execute( - text( - "SELECT symbol, COUNT(*), MIN(date), MAX(date) " - "FROM benchmark_prices GROUP BY symbol ORDER BY symbol" - ) - ).fetchall() - finally: - engine.dispose() - - print(f"Done in {(time.monotonic() - t0) / 60:.1f}m; rows touched={total}") - for sym, n, d0, d1 in rows: - print(f" {sym}: n={n} {d0}→{d1}") - - -if __name__ == "__main__": - asyncio.run(_main()) diff --git a/scripts/run_earnings_research.py b/scripts/run_earnings_research.py index 1e12264..bfc353e 100644 --- a/scripts/run_earnings_research.py +++ b/scripts/run_earnings_research.py @@ -466,11 +466,6 @@ async def _run_2b_ic( os.environ["BACKTEST_SNAPSHOT_OFFLINE"] = "1" os.environ["BACKTEST_SIGNAL_EVAL_ONLY"] = "1" - # Load sector map if present so sector signals also appear (side-by-side optional). - if Path("data/research/ticker_sector_map.json").exists(): - os.environ["BACKTEST_SECTOR_MAP_PATH"] = str( - Path("data/research/ticker_sector_map.json").resolve() - ) settings.backtest_workers = workers engine = create_async_engine(_sqlite_url(snapshot), pool_pre_ping=True) @@ -484,18 +479,6 @@ async def _run_2b_ic( (await db.execute(select(Ticker).order_by(Ticker.symbol))).scalars() ) spy = await load_benchmark_closes(db, "SPY") - sector_etf: dict[str, dict] = {} - try: - from app.services.sector_map import SECTOR_ETFS, load_ticker_sector_map - - symbol_to_sector = load_ticker_sector_map() - for etf in SECTOR_ETFS: - series = await load_benchmark_closes(db, etf) - if series: - sector_etf[etf] = series - except Exception: - symbol_to_sector = {} - sector_etf = {} prices: dict[str, tuple] = {} for idx, t in enumerate(tickers): @@ -521,9 +504,6 @@ async def _run_2b_ic( ], spy, symbol=t.symbol, - sector_etf_closes=bt._sector_etf_closes_for_symbol( - t.symbol, symbol_to_sector, sector_etf - ), ) for name, weeks in series.items(): for wk, pairs in weeks.items(): @@ -533,9 +513,6 @@ async def _run_2b_ic( if not quiet: print() - if symbol_to_sector: - bt._inject_sector_demeaned_momentum(collected, symbol_to_sector) - # SUE series. events_by_sym: dict[str, list[dict]] = defaultdict(list) for ev in events: @@ -709,8 +686,6 @@ async def _run_2b_ic( for name in ( "mom_12_1", "mom_12_1_resid", - "mom_12_1_sector_resid", - "mom_12_1_sector_demeaned", "sue_latest", "fip_id", ) @@ -784,7 +759,6 @@ def _write_md(path: Path, payload: dict) -> None: "mom_12_1", "mom_12_1_resid", "sue_latest", - "mom_12_1_sector_resid", "fip_id", ): r = side.get(name) or {} diff --git a/scripts/run_history_depth_research.py b/scripts/run_history_depth_research.py deleted file mode 100644 index cfe2285..0000000 --- a/scripts/run_history_depth_research.py +++ /dev/null @@ -1,480 +0,0 @@ -"""History-depth extension research (local / MacBook). - -Phases ------- - coverage — bars per calendar year; no rebuild - harness — race-guard snapshot, full signal_eval, era split pre/post-2021 - -Does not retune production knobs. Does not modify scheduler/gates. - -Example -------- - python scripts/run_history_depth_research.py --phase coverage \\ - --snapshot backtest_snapshots/prod.sqlite - - python scripts/run_history_depth_research.py --phase harness \\ - --snapshot backtest_snapshots/research.sqlite --workers 8 --allow-spawn -""" - -from __future__ import annotations - -import argparse -import asyncio -import json -import os -import sys -from collections import defaultdict -from datetime import date, datetime -from pathlib import Path -from typing import Any - -from sqlalchemy import create_engine, text -from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine - -ROOT = Path(__file__).resolve().parents[1] -if str(ROOT) not in sys.path: - sys.path.insert(0, str(ROOT)) - -from app.ssl_bootstrap import bootstrap_ssl # noqa: E402 - -bootstrap_ssl() - -ERA_SPLIT = date(2021, 1, 1) -SURVIVORSHIP_BANNER = ( - "SURVIVORSHIP BIAS: today's constituents backfilled historically. " - "Absolute Sharpe/CAGR levels on deep history are optimistic. " - "Use RELATIVE signal IC comparisons and era stability only — not levels." -) - - -def _sqlite_url(path: Path) -> str: - return f"sqlite+aiosqlite:///{path.resolve().as_posix()}" - - -def _parse_args() -> argparse.Namespace: - p = argparse.ArgumentParser(description=__doc__) - p.add_argument("--phase", choices=("coverage", "harness", "all"), default="all") - p.add_argument("--snapshot", default="backtest_snapshots/research.sqlite") - p.add_argument("--workers", type=int, default=8) - p.add_argument("--allow-spawn", action="store_true") - p.add_argument("--quiet", action="store_true") - p.add_argument("--out", default=None) - return p.parse_args() - - -def _coverage_report(snapshot: Path) -> dict[str, Any]: - engine = create_engine( - f"sqlite:///{snapshot.resolve().as_posix()}", - future=True, - ) - try: - with engine.connect() as conn: - ticker_n = int(conn.execute(text("SELECT COUNT(*) FROM tickers")).scalar_one()) - ohlcv_n = int( - conn.execute(text("SELECT COUNT(*) FROM ohlcv_records")).scalar_one() - ) - d_range = conn.execute( - text("SELECT MIN(date), MAX(date) FROM ohlcv_records") - ).fetchone() - # Bars per calendar year (global). - by_year = conn.execute( - text( - """ - SELECT substr(date, 1, 4) AS y, COUNT(*) AS n, - COUNT(DISTINCT ticker_id) AS tickers - FROM ohlcv_records - GROUP BY substr(date, 1, 4) - ORDER BY y - """ - ) - ).fetchall() - # Per-symbol min/max date + bar count (summary percentiles). - per_sym = conn.execute( - text( - """ - SELECT t.symbol, COUNT(*) AS n, MIN(o.date), MAX(o.date) - FROM ohlcv_records o - JOIN tickers t ON t.id = o.ticker_id - GROUP BY t.symbol - """ - ) - ).fetchall() - finally: - engine.dispose() - - ns = sorted(int(r[1]) for r in per_sym) - def pct(p: float) -> int | None: - if not ns: - return None - i = int(round(p * (len(ns) - 1))) - return ns[i] - - starts = sorted(str(r[2]) for r in per_sym if r[2]) - start_hist: dict[str, int] = defaultdict(int) - for s in starts: - start_hist[s[:4]] += 1 - - return { - "snapshot": str(snapshot.resolve()), - "ticker_count": ticker_n, - "ohlcv_row_count": ohlcv_n, - "date_range": {"min": d_range[0], "max": d_range[1]}, - "bars_per_year": [ - {"year": y, "bars": n, "tickers_with_bars": t} for y, n, t in by_year - ], - "bars_per_symbol": { - "min": ns[0] if ns else None, - "p10": pct(0.10), - "p50": pct(0.50), - "p90": pct(0.90), - "max": ns[-1] if ns else None, - }, - "symbols_by_start_year": dict(sorted(start_hist.items())), - "note": ( - "Where ticker counts drop in early years, the feed (or listing history) " - "thins — do not treat those years as a full 505-name cross-section." - ), - "survivorship_banner": SURVIVORSHIP_BANNER, - } - - -def _assert_complete(snapshot: Path) -> dict[str, Any]: - from scripts.research_snapshot_manifest import ( # type: ignore - assert_research_snapshot_complete, - load_manifest, - ) - - m = load_manifest(snapshot) - if m is None: - # Prod snapshot may lack manifest; still require healthy bar depth. - eng = create_engine( - f"sqlite:///{snapshot.resolve().as_posix()}", - future=True, - ) - try: - with eng.connect() as conn: - avg = conn.execute( - text( - """ - SELECT AVG(c) FROM ( - SELECT COUNT(*) AS c FROM ohlcv_records GROUP BY ticker_id - ) - """ - ) - ).scalar_one() - finally: - eng.dispose() - if avg is None or float(avg) < 400: - raise SystemExit( - f"No completion manifest and avg bars={avg} look short. " - "Rebuild research.sqlite via extend_snapshot_universe.py" - ) - return {"manifest": None, "avg_bars": float(avg), "ok": True} - return {"manifest": assert_research_snapshot_complete(snapshot), "ok": True} - - -async def _harness(snapshot: Path, *, workers: int, quiet: bool) -> dict[str, Any]: - from app.config import settings - from app.services.backtest_service import run_backtest - - os.environ["BACKTEST_SNAPSHOT_OFFLINE"] = "1" - os.environ["BACKTEST_SIGNAL_EVAL_ONLY"] = "1" - if Path("data/research/ticker_sector_map.json").exists(): - os.environ["BACKTEST_SECTOR_MAP_PATH"] = str( - Path("data/research/ticker_sector_map.json").resolve() - ) - settings.backtest_workers = workers - - engine = create_async_engine(_sqlite_url(snapshot), pool_pre_ping=True) - Session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) - - def progress(done: int, total: int, symbol: str) -> None: - if quiet: - return - print(f" progress {done}/{total} {symbol}", end="\r", flush=True) - - try: - async with Session() as db: - report = await run_backtest(db, progress_cb=progress, cadence="weekly") - finally: - await engine.dispose() - if not quiet: - print() - - signal_eval = report.get("signal_eval") or [] - - # Era-split IC: recompute from collected is not available post-run. - # Approximate via second pass is expensive; instead document that era split - # requires collecting weekly ICs. We re-run evaluation if the report embeds - # nothing — for v1, call internal collection is too heavy to duplicate. - # Lightweight approach: mark era_split as requiring BACKTEST with custom - # filter — implemented below by re-scoring from a dedicated collection pass. - era = await _era_split_ics(snapshot, workers=workers, quiet=quiet) - - return { - "survivorship_banner": SURVIVORSHIP_BANNER, - "signal_eval": signal_eval, - "era_split": era, - "params": report.get("params"), - "tickers": report.get("tickers"), - "generated_at_run": report.get("generated_at"), - } - - -async def _era_split_ics( - snapshot: Path, *, workers: int, quiet: bool -) -> dict[str, Any]: - """Collect weekly signal series and evaluate pre/post ERA_SPLIT separately.""" - from app.config import settings - from app.services import backtest_service as bt - from app.services.benchmark_service import load_benchmark_closes - from app.models.ticker import Ticker - from sqlalchemy import select - from collections import defaultdict as dd - - os.environ["BACKTEST_SNAPSHOT_OFFLINE"] = "1" - settings.backtest_workers = max(1, workers) - - engine = create_async_engine(_sqlite_url(snapshot), pool_pre_ping=True) - Session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) - - collected: dict = dd(lambda: dd(list)) - try: - async with Session() as db: - tickers = list( - (await db.execute(select(Ticker).order_by(Ticker.symbol))).scalars() - ) - spy = await load_benchmark_closes(db, "SPY") - symbol_to_sector = {} - sector_etf: dict = {} - try: - from app.services.sector_map import ( - SECTOR_ETFS, - load_ticker_sector_map, - ) - - symbol_to_sector = load_ticker_sector_map() - for etf in SECTOR_ETFS: - series = await load_benchmark_closes(db, etf) - if series: - sector_etf[etf] = series - except Exception: - pass - - for idx, t in enumerate(tickers): - if not quiet and idx % 100 == 0: - print(f" era-collect {idx}/{len(tickers)}", end="\r", flush=True) - cols = await bt._fetch_columns(db, t.symbol) - if cols is None: - continue - records = [ - type( - "R", - (), - { - "date": date.fromordinal(int(cols[0][i])), - "close": cols[4][i], - "high": cols[2][i], - "volume": cols[5][i] if len(cols) > 5 else 0, - }, - )() - for i in range(len(cols[0])) - ] - series = bt._signal_series( - records, - spy, - symbol=t.symbol, - sector_etf_closes=bt._sector_etf_closes_for_symbol( - t.symbol, symbol_to_sector, sector_etf - ), - ) - for name, weeks in series.items(): - for wk, pairs in weeks.items(): - collected[name][wk].extend(pairs) - if symbol_to_sector: - bt._inject_sector_demeaned_momentum(collected, symbol_to_sector) - finally: - await engine.dispose() - if not quiet: - print() - - def _filter_era(coll: dict, *, pre: bool) -> dict: - out: dict = dd(lambda: dd(list)) - for name, weeks in coll.items(): - for wk, recs in weeks.items(): - # ISO week key (year, week) — approximate era by ISO year. - year = int(wk[0]) if isinstance(wk, tuple) else int(str(wk)[:4]) - if pre and year >= ERA_SPLIT.year: - continue - if not pre and year < ERA_SPLIT.year: - continue - out[name][wk].extend(recs) - return out - - pre_eval = bt._signal_evaluation(_filter_era(collected, pre=True)) - post_eval = bt._signal_evaluation(_filter_era(collected, pre=False)) - full_eval = bt._signal_evaluation(collected) - - def _index(rows: list[dict]) -> dict[str, dict]: - return {r["signal"]: r for r in rows} - - return { - "era_split_date": ERA_SPLIT.isoformat(), - "note": "Diagnostic only — not a tuning input. Nested lookbacks are not OOS.", - "full": _index(full_eval), - "pre_2021": _index(pre_eval), - "post_2021": _index(post_eval), - } - - -def _write_md(path: Path, payload: dict) -> None: - pre = path.read_text(encoding="utf-8") if path.exists() else "" - marker = "## Results" - idx = pre.find(marker) - header = pre[:idx] if idx >= 0 else pre.split("## Verdict")[0] - - lines = [ - header.rstrip(), - "", - "## Results", - "", - f"Generated: `{payload.get('generated_at')}`", - "", - f"> **{SURVIVORSHIP_BANNER}**", - "", - "### Coverage", - "", - f"```json\n{json.dumps(payload.get('coverage') or {}, indent=2, default=str)}\n```", - "", - "### Race guard", - "", - f"```json\n{json.dumps(payload.get('race_guard') or {}, indent=2, default=str)}\n```", - "", - "### Signal IC (full extended window)", - "", - ] - harness = payload.get("harness") or {} - rows = harness.get("signal_eval") or [] - if rows: - lines.extend([ - "| signal | mean_ic | ic_t_stat | weeks | avg_N | reliable |", - "|---|---:|---:|---:|---:|---|", - ]) - for r in rows: - lines.append( - f"| {r.get('signal')} | {r.get('mean_ic')} | {r.get('ic_t_stat')} | " - f"{r.get('weeks')} | {r.get('avg_cross_section')} | {r.get('reliable')} |" - ) - else: - lines.append("_Harness not run this pass._") - - era = (harness.get("era_split") or {}) - lines.extend(["", "### Era split (diagnostic only)", ""]) - if era: - for label in ("full", "pre_2021", "post_2021"): - block = era.get(label) or {} - lines.append(f"#### {label}") - lines.append("") - lines.append("| signal | mean_ic | t | weeks | N |") - lines.append("|---|---:|---:|---:|---:|") - for name in sorted(block): - r = block[name] - lines.append( - f"| {name} | {r.get('mean_ic')} | {r.get('ic_t_stat')} | " - f"{r.get('weeks')} | {r.get('avg_cross_section')} |" - ) - lines.append("") - else: - lines.append("_No era split._") - - lines.extend([ - "", - "## Verdict", - "", - f"**{payload.get('verdict')}**", - "", - payload.get("verdict_detail") or "", - "", - "## What a human must decide next", - "", - payload.get("human_next") - or "- Do not retune production knobs from this report without review.", - "", - f"Artifacts: `{payload.get('report_path')}`", - "", - ]) - path.write_text("\n".join(lines) + "\n", encoding="utf-8") - - -async def _main() -> None: - args = _parse_args() - snapshot = Path(args.snapshot) - if not snapshot.exists(): - raise SystemExit(f"Missing snapshot: {snapshot}") - if args.allow_spawn: - os.environ["BACKTEST_ALLOW_SPAWN"] = "1" - - coverage = None - race = None - harness = None - - if args.phase in ("coverage", "all"): - print("Coverage probe…") - coverage = _coverage_report(snapshot) - print( - f" tickers={coverage['ticker_count']} ohlcv={coverage['ohlcv_row_count']} " - f"range={coverage['date_range']}" - ) - for row in coverage["bars_per_year"]: - print( - f" year {row['year']}: bars={row['bars']} " - f"tickers={row['tickers_with_bars']}" - ) - - if args.phase in ("harness", "all"): - print("Race guard…") - race = _assert_complete(snapshot) - print(f" ok={race.get('ok')}") - print("Full harness + era split (LONG)…") - print(f" {SURVIVORSHIP_BANNER}") - harness = await _harness( - snapshot, workers=args.workers, quiet=args.quiet - ) - - stamp = datetime.now().strftime("%Y%m%d-%H%M%S") - out = ( - Path(args.out) - if args.out - else Path("reports") / f"history-depth-{stamp}.json" - ) - payload = { - "generated_at": datetime.now().isoformat(), - "survivorship_banner": SURVIVORSHIP_BANNER, - "coverage": coverage, - "race_guard": race, - "harness": harness, - "verdict": "PENDING_HUMAN" if harness else "COVERAGE_ONLY", - "verdict_detail": ( - "Harness complete — human interprets relative IC / era stability. " - "No production retune from this artifact." - if harness - else "Coverage probe only; run --phase harness after deep rebuild." - ), - "human_next": ( - "- Compare sector residual vs market residual across eras.\n" - "- If pre-2021 IC collapses, park Task 1 wire-in.\n" - "- Do not retune production knobs on deep history levels." - ), - "report_path": str(out.as_posix()), - } - out.parent.mkdir(parents=True, exist_ok=True) - out.write_text(json.dumps(payload, indent=2, default=str) + "\n", encoding="utf-8") - md = Path("docs/research/history-depth-extension.md") - _write_md(md, payload) - out.with_suffix(".md").write_text(md.read_text(encoding="utf-8"), encoding="utf-8") - print(f"Wrote {out}") - print(f"Wrote {md}") - - -if __name__ == "__main__": - asyncio.run(_main()) diff --git a/scripts/run_sector_resid_deep_test.py b/scripts/run_sector_resid_deep_test.py deleted file mode 100644 index a5cbfff..0000000 --- a/scripts/run_sector_resid_deep_test.py +++ /dev/null @@ -1,1047 +0,0 @@ -#!/usr/bin/env python3 -"""Terminal sector-residual deep test: deepen shallow symbols → ONE masked run → PASS/FAIL. - -Repairs the two-tier history-depth defect (prod/ETF names left at ~5y while breadth -got 5000d), then runs a single liquid-breadth signal harness and grades -``mom_12_1_sector_resid`` against the pre-registered rule. - -Local research only. No production changes. - -MacBook -------- - # On deep research.sqlite from the prior history-depth rebuild: - python scripts/run_sector_resid_deep_test.py \\ - --snapshot backtest_snapshots/research.sqlite \\ - --workers 8 --allow-spawn - - # Skip re-fetch if Step-1 already done and sanity-check passes: - python scripts/run_sector_resid_deep_test.py --skip-deepen --workers 8 --allow-spawn -""" - -from __future__ import annotations - -import argparse -import asyncio -import json -import math -import os -import sys -import time -from collections import defaultdict -from datetime import date, datetime, timedelta, timezone -from pathlib import Path -from typing import Any - -from sqlalchemy import create_engine, text -from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine - -ROOT = Path(__file__).resolve().parents[1] -if str(ROOT) not in sys.path: - sys.path.insert(0, str(ROOT)) - -from app.ssl_bootstrap import bootstrap_ssl # noqa: E402 - -bootstrap_ssl() - -from app.services.sector_map import ( # noqa: E402 - DEFAULT_SECTOR_MAP_PATH, - SECTOR_ETFS, - load_ticker_sector_map, -) -from scripts.research_snapshot_manifest import ( # noqa: E402 - assert_research_snapshot_complete, - clear_manifest, - write_completion_manifest, -) - -ERA_SPLIT = date(2021, 1, 1) -IRON_IC = 0.03 -# "weeks ≫ 35 (expect ~80)" — mechanical floor for "data fix worked" -MIN_WEEKS_DEEP = 50 -SANITY_MEGACAPS = ("AAPL", "MSFT", "JPM", "XOM", "JNJ") -SURVIVORSHIP = ( - "SURVIVORSHIP BIAS: today's constituents backfilled. Relative IC only — not levels." -) - - -def _parse_args() -> argparse.Namespace: - p = argparse.ArgumentParser(description=__doc__) - p.add_argument("--snapshot", default="backtest_snapshots/research.sqlite") - p.add_argument("--history-days", type=int, default=5000) - p.add_argument("--sleep", type=float, default=0.15) - p.add_argument("--workers", type=int, default=8) - p.add_argument("--allow-spawn", action="store_true") - p.add_argument( - "--skip-deepen", - action="store_true", - help="Skip Step-1 re-fetch; only sanity-check + harness.", - ) - p.add_argument( - "--sector-map", - default=str(DEFAULT_SECTOR_MAP_PATH), - ) - p.add_argument("--liquid-breadth", type=int, default=1500) - p.add_argument("--min-price", type=float, default=5.0) - p.add_argument("--quiet", action="store_true") - p.add_argument("--out", default=None) - return p.parse_args() - - -def _sqlite_url(path: Path) -> str: - return f"sqlite+aiosqlite:///{path.resolve().as_posix()}" - - -def _symbol_depth(snapshot: Path) -> list[dict[str, Any]]: - engine = create_engine( - f"sqlite:///{snapshot.resolve().as_posix()}", - future=True, - ) - try: - with engine.connect() as conn: - rows = conn.execute( - text( - """ - SELECT t.symbol, COUNT(*) AS n, MIN(o.date), MAX(o.date) - FROM ohlcv_records o - JOIN tickers t ON t.id = o.ticker_id - GROUP BY t.symbol - """ - ) - ).fetchall() - finally: - engine.dispose() - out = [] - for sym, n, d0, d1 in rows: - out.append({ - "symbol": str(sym).upper(), - "bars": int(n), - "min_date": str(d0)[:10] if d0 else None, - "max_date": str(d1)[:10] if d1 else None, - }) - return out - - -def _derive_shallow( - depths: list[dict[str, Any]], - *, - lag_days: int = 400, -) -> tuple[list[str], dict[str, Any]]: - """Symbols whose earliest bar starts materially later than the deep cohort.""" - starts: list[tuple[str, date]] = [] - for row in depths: - if not row.get("min_date"): - continue - starts.append((row["symbol"], date.fromisoformat(row["min_date"]))) - if not starts: - return [], {"error": "no symbols with min_date"} - - # Deep cohort start ≈ 10th percentile of earliest dates (early = deep). - ordered = sorted(d for _, d in starts) - p10 = ordered[max(0, int(0.10 * (len(ordered) - 1)))] - cutoff = p10 + timedelta(days=lag_days) - shallow = sorted({sym for sym, d in starts if d > cutoff}) - meta = { - "n_symbols": len(starts), - "deep_cohort_p10_start": p10.isoformat(), - "shallow_cutoff": cutoff.isoformat(), - "lag_days": lag_days, - "n_shallow": len(shallow), - "shallow_start_histogram": _year_hist( - [d for sym, d in starts if sym in set(shallow)] - ), - "deep_start_histogram": _year_hist( - [d for sym, d in starts if sym not in set(shallow)] - ), - "shallow_sample": shallow[:30], - } - return shallow, meta - - -def _year_hist(dates: list[date]) -> dict[str, int]: - h: dict[str, int] = defaultdict(int) - for d in dates: - h[str(d.year)] += 1 - return dict(sorted(h.items())) - - -async def _fetch_and_replace_ohlcv( - engine, - provider, - symbol: str, - start: date, - end: date, - *, - sleep_s: float, - max_retries: int = 5, -) -> int: - from app.exceptions import ProviderError, RateLimitError - - bars = [] - for attempt in range(max_retries): - try: - bars = await provider.fetch_ohlcv(symbol, start, end) - break - except RateLimitError: - wait = min(60.0, 2.0 ** attempt) - print(f" rate limited {symbol}; sleep {wait:.0f}s") - await asyncio.sleep(wait) - except ProviderError as exc: - if attempt + 1 >= max_retries: - raise - await asyncio.sleep(1.0) - _ = exc - if sleep_s > 0: - await asyncio.sleep(sleep_s) - if not bars: - return 0 - - with engine.begin() as write: - tid = write.execute( - text("SELECT id FROM tickers WHERE symbol = :s"), - {"s": symbol}, - ).scalar_one_or_none() - if tid is None: - write.execute( - text( - "INSERT INTO tickers (symbol, name, created_at) " - "VALUES (:s, NULL, :c)" - ), - {"s": symbol, "c": datetime.now(timezone.utc).isoformat()}, - ) - tid = write.execute( - text("SELECT id FROM tickers WHERE symbol = :s"), - {"s": symbol}, - ).scalar_one() - # Full replace for this symbol so shallow tails cannot linger. - write.execute( - text("DELETE FROM ohlcv_records WHERE ticker_id = :tid"), - {"tid": int(tid)}, - ) - now = datetime.now(timezone.utc).replace(tzinfo=None).isoformat() - write.execute( - text( - """ - INSERT INTO ohlcv_records - (ticker_id, date, open, high, low, close, volume, created_at) - VALUES - (:ticker_id, :date, :open, :high, :low, :close, :volume, :created_at) - """ - ), - [ - { - "ticker_id": int(tid), - "date": b.date.isoformat() - if hasattr(b.date, "isoformat") - else str(b.date), - "open": float(b.open), - "high": float(b.high), - "low": float(b.low), - "close": float(b.close), - "volume": int(b.volume), - "created_at": now, - } - for b in bars - ], - ) - return len(bars) - - -async def _deepen_sector_etfs( - snapshot: Path, *, history_days: int, sleep_s: float -) -> dict[str, Any]: - """Refresh SPY + 11 sector ETFs in benchmark_prices to full depth.""" - # Reuse the existing CLI helper for consistency. - from scripts.fetch_sector_etfs_to_snapshot import _fetch_and_upsert - from app.config import settings - from app.providers.alpaca import AlpacaOHLCVProvider - - if not settings.alpaca_api_key or not settings.alpaca_api_secret: - raise SystemExit("ALPACA keys required to deepen sector ETFs") - - provider = AlpacaOHLCVProvider(settings.alpaca_api_key, settings.alpaca_api_secret) - end = date.today() - start = end - timedelta(days=history_days) - engine = create_engine( - f"sqlite:///{snapshot.resolve().as_posix()}", - future=True, - ) - symbols = ["SPY", *SECTOR_ETFS] - written: dict[str, int] = {} - try: - for sym in symbols: - n = await _fetch_and_upsert( - engine, provider, sym, start, end, sleep_s=sleep_s - ) - written[sym] = n - finally: - engine.dispose() - - engine = create_engine( - f"sqlite:///{snapshot.resolve().as_posix()}", - future=True, - ) - try: - with engine.connect() as conn: - rows = conn.execute( - text( - "SELECT symbol, COUNT(*), MIN(date), MAX(date) " - "FROM benchmark_prices GROUP BY symbol ORDER BY symbol" - ) - ).fetchall() - finally: - engine.dispose() - return { - "written": written, - "benchmark_summary": [ - {"symbol": s, "n": n, "min": d0, "max": d1} for s, n, d0, d1 in rows - ], - } - - -def _sanity_check(snapshot: Path, *, history_days: int) -> dict[str, Any]: - """Pass if megacaps + sector ETFs sit at the *empirical feed floor*, not calendar 5000d. - - Alpaca daily history for this stack bottoms out around 2016-01-04 (~2649 bars) - even when history_days=5000 is requested. That is feed coverage, not a two-tier - snapshot bug. Fail only if megacaps are still stuck near the old ~2021 prod floor - or if sector ETFs are missing / shorter than the SPY series (except XLC listing). - """ - depths = {r["symbol"]: r for r in _symbol_depth(snapshot)} - - engine = create_engine( - f"sqlite:///{snapshot.resolve().as_posix()}", - future=True, - ) - try: - with engine.connect() as conn: - spy_row = conn.execute( - text( - "SELECT COUNT(*), MIN(date), MAX(date) FROM benchmark_prices " - "WHERE symbol = 'SPY'" - ) - ).fetchone() - etf_rows = conn.execute( - text( - "SELECT symbol, COUNT(*), MIN(date), MAX(date) " - "FROM benchmark_prices WHERE symbol IN " - f"({','.join(repr(s) for s in SECTOR_ETFS)}) " - "GROUP BY symbol" - ) - ).fetchall() - finally: - engine.dispose() - - spy_n, spy_min, spy_max = spy_row if spy_row else (0, None, None) - feed_floor = ( - date.fromisoformat(str(spy_min)[:10]) - if spy_min - else date(2016, 1, 4) - ) - # Megacaps must match the feed floor within a few sessions (not calendar-5000). - megacap_slack_days = 10 - # Old two-tier defect left prod names at ~2021-06; anything still after this fails. - old_shallow_floor = date(2020, 1, 1) - - megacap = {} - ok_mega = True - mega_reasons: list[str] = [] - for sym in SANITY_MEGACAPS: - row = depths.get(sym) - megacap[sym] = row - if row is None or not row.get("min_date"): - ok_mega = False - mega_reasons.append(f"{sym}: missing") - continue - d0 = date.fromisoformat(row["min_date"]) - if d0 > old_shallow_floor: - ok_mega = False - mega_reasons.append( - f"{sym}: min_date={d0} still after {old_shallow_floor} (two-tier unrepaired)" - ) - elif d0 > feed_floor + timedelta(days=megacap_slack_days): - ok_mega = False - mega_reasons.append( - f"{sym}: min_date={d0} later than SPY feed floor {feed_floor}" - ) - - etf_info = { - s: {"n": n, "min": d0, "max": d1} for s, n, d0, d1 in etf_rows - } - deep_etfs = 0 - etf_reasons: list[str] = [] - for sym in SECTOR_ETFS: - info = etf_info.get(sym) - if not info or not info["min"]: - etf_reasons.append(f"{sym}: missing") - continue - d0 = date.fromisoformat(str(info["min"])[:10]) - if sym == "XLC": - # Listed 2018-06-18/19. - if d0 <= date(2018, 7, 15): - deep_etfs += 1 - else: - etf_reasons.append(f"XLC: min_date={d0} later than listing floor") - else: - if d0 <= feed_floor + timedelta(days=megacap_slack_days): - deep_etfs += 1 - else: - etf_reasons.append( - f"{sym}: min_date={d0} later than SPY feed floor {feed_floor}" - ) - - ok_etf = deep_etfs >= 10 - still_shallow, _ = _derive_shallow(list(depths.values()), lag_days=400) - note_xlc = ( - "XLC lists mid-2018 → Communication Services residual coverage from ~mid-2019." - ) - note_feed = ( - f"Empirical Alpaca floor observed via SPY: {feed_floor.isoformat()} " - f"(n={spy_n}). Calendar history_days={history_days} is a request cap, not a " - "guarantee — sanity grades against the feed floor, not 5000 calendar days." - ) - - passed = bool(ok_mega and ok_etf) - return { - "passed": passed, - "megacap": megacap, - "megacap_ok": ok_mega, - "megacap_reasons": mega_reasons, - "feed_floor": feed_floor.isoformat(), - "spy_benchmark": {"n": spy_n, "min": spy_min, "max": spy_max}, - "old_shallow_floor": old_shallow_floor.isoformat(), - "sector_etfs": etf_info, - "sector_etfs_deep_count": deep_etfs, - "sector_etfs_ok": ok_etf, - "sector_etf_reasons": etf_reasons, - "still_shallow_count": len(still_shallow), - "still_shallow_sample": still_shallow[:20], - "still_shallow_note": ( - "Remaining 'shallow' names are mostly post-2017 IPOs/listings — expected, " - "not a two-tier defect." - ), - "xlc_note": note_xlc, - "feed_note": note_feed, - "target_history_days": history_days, - } - - -async def _step1_deepen( - snapshot: Path, - *, - history_days: int, - sleep_s: float, - quiet: bool, -) -> dict[str, Any]: - from app.config import settings - from app.providers.alpaca import AlpacaOHLCVProvider - - if not settings.alpaca_api_key or not settings.alpaca_api_secret: - raise SystemExit("ALPACA_API_KEY / ALPACA_API_SECRET required") - - clear_manifest(snapshot) - depths = _symbol_depth(snapshot) - shallow, shallow_meta = _derive_shallow(depths) - print( - f"Shallow symbols to deepen: {len(shallow)} " - f"(p10 deep start={shallow_meta.get('deep_cohort_p10_start')}, " - f"cutoff={shallow_meta.get('shallow_cutoff')})" - ) - if not shallow: - print("WARNING: no shallow symbols detected — snapshot may already be uniform") - - provider = AlpacaOHLCVProvider(settings.alpaca_api_key, settings.alpaca_api_secret) - end = date.today() - start = end - timedelta(days=history_days) - engine = create_engine( - f"sqlite:///{snapshot.resolve().as_posix()}", - future=True, - ) - - ok = fail = 0 - t0 = time.monotonic() - try: - for i, sym in enumerate(shallow, 1): - try: - n = await _fetch_and_replace_ohlcv( - engine, provider, sym, start, end, sleep_s=sleep_s - ) - if n <= 0: - fail += 1 - if not quiet: - print(f" [{i}/{len(shallow)}] {sym} empty") - continue - ok += 1 - if not quiet and (i % 25 == 0 or i == len(shallow)): - print( - f" progress {i}/{len(shallow)} ok={ok} fail={fail} " - f"last={sym} bars={n} elapsed={(time.monotonic()-t0)/60:.1f}m" - ) - except Exception as exc: - fail += 1 - print(f" [{i}/{len(shallow)}] {sym} FAIL {exc}") - finally: - engine.dispose() - - print("Deepening SPY + sector ETFs in benchmark_prices…") - etf_result = await _deepen_sector_etfs( - snapshot, history_days=history_days, sleep_s=sleep_s - ) - - # Manifest: full completion after deepen (no --limit). - from scripts.research_snapshot_manifest import _count_snapshot - - counts = _count_snapshot(snapshot) - manifest_path = write_completion_manifest( - snapshot, - complete=True, - sources={"deepen": "sector_resid_deep_test step1"}, - history_days=history_days, - min_bars=None, - fetch_ok=ok, - fetch_fail=fail, - limit=None, - extra={ - "shallow_meta": shallow_meta, - "shallow_fetched_ok": ok, - "shallow_fetched_fail": fail, - "etf_refresh": etf_result.get("written"), - "counts_after": counts, - }, - ) - print(f"Manifest written: {manifest_path}") - - sanity = _sanity_check(snapshot, history_days=history_days) - return { - "shallow_meta": shallow_meta, - "shallow_list_n": len(shallow), - "fetch_ok": ok, - "fetch_fail": fail, - "etf_refresh": etf_result, - "sanity": sanity, - "manifest_path": str(manifest_path), - } - - -async def _one_masked_run( - snapshot: Path, - *, - sector_map_path: Path, - liquid_breadth: int, - min_price: float, - workers: int, - quiet: bool, -) -> dict[str, Any]: - """Single collection under liquid mask; full + era IC from the same series.""" - from app.config import settings - from app.services import backtest_service as bt - from app.services.benchmark_service import load_benchmark_closes - from app.models.ticker import Ticker - from sqlalchemy import select - - os.environ["BACKTEST_SNAPSHOT_OFFLINE"] = "1" - os.environ["BACKTEST_SIGNAL_EVAL_ONLY"] = "1" - os.environ["BACKTEST_LIQUID_BREADTH"] = str(int(liquid_breadth)) - os.environ["BACKTEST_LIQUID_MIN_PRICE"] = str(float(min_price)) - os.environ["BACKTEST_SECTOR_MAP_PATH"] = str(sector_map_path.resolve()) - if workers: - settings.backtest_workers = workers - - # One collection pass (not run_backtest twice). - engine = create_async_engine(_sqlite_url(snapshot), pool_pre_ping=True) - Session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) - collected: dict = defaultdict(lambda: defaultdict(list)) - symbol_to_sector = load_ticker_sector_map(sector_map_path) - - try: - async with Session() as db: - tickers = list( - (await db.execute(select(Ticker).order_by(Ticker.symbol))).scalars() - ) - spy = await load_benchmark_closes(db, "SPY") - sector_etf: dict[str, dict] = {} - for etf in SECTOR_ETFS: - series = await load_benchmark_closes(db, etf) - if series: - sector_etf[etf] = series - - total = len(tickers) - for idx, t in enumerate(tickers): - if not quiet and idx % 100 == 0: - print(f" collect {idx}/{total}", end="\r", flush=True) - cols = await bt._fetch_columns(db, t.symbol) - if cols is None: - continue - records = [ - type( - "R", - (), - { - "date": date.fromordinal(int(cols[0][i])), - "close": cols[4][i], - "high": cols[2][i], - "volume": cols[5][i] if len(cols) > 5 else 0, - }, - )() - for i in range(len(cols[0])) - ] - etf_closes = bt._sector_etf_closes_for_symbol( - t.symbol, symbol_to_sector, sector_etf - ) - series = bt._signal_series( - records, - spy, - symbol=t.symbol, - sector_etf_closes=etf_closes, - ) - for name, weeks in series.items(): - for wk, pairs in weeks.items(): - collected[name][wk].extend(pairs) - finally: - await engine.dispose() - if not quiet: - print() - - if symbol_to_sector: - bt._inject_sector_demeaned_momentum(collected, symbol_to_sector) - - full_eval = bt._signal_evaluation(dict(collected)) - - def _filter_era(coll: dict, *, pre: bool) -> dict: - out: dict = defaultdict(lambda: defaultdict(list)) - for name, weeks in coll.items(): - for wk, recs in weeks.items(): - year = int(wk[0]) if isinstance(wk, tuple) else int(str(wk)[:4]) - if pre and year >= ERA_SPLIT.year: - continue - if not pre and year < ERA_SPLIT.year: - continue - out[name][wk].extend(recs) - return out - - pre_eval = bt._signal_evaluation(_filter_era(collected, pre=True)) - post_eval = bt._signal_evaluation(_filter_era(collected, pre=False)) - - # Identical-subset: resid IC only where sector_resid exists (same CS for t rule). - identical = _identical_subset_eval(collected, bt) - - def _idx(rows: list[dict]) -> dict[str, dict]: - return {r["signal"]: r for r in rows} - - # Mask bind diagnostics from liquid-aware rows if present. - mask_diag = _mask_diagnostics(full_eval) - - return { - "liquid_breadth_top_n": liquid_breadth, - "liquid_min_price": min_price, - "survivorship_banner": SURVIVORSHIP, - "signal_eval": full_eval, - "signal_eval_by_name": _idx(full_eval), - "era_split": { - "era_split_date": ERA_SPLIT.isoformat(), - "note": "Diagnostic only — not a tuning input.", - "pre_2021": _idx(pre_eval), - "post_2021": _idx(post_eval), - }, - "identical_subset_sector_cs": identical, - "mask_diagnostics": mask_diag, - "sector_map_size": len(symbol_to_sector), - "sector_etfs_loaded": sorted(sector_etf), - "spy_bars": len(spy), - } - - -def _identical_subset_eval(collected: dict, bt) -> dict[str, Any]: - """Re-score mom_12_1_resid on the same (week, symbol) cells as sector_resid.""" - sector_weeks = collected.get("mom_12_1_sector_resid") or {} - resid_weeks = collected.get("mom_12_1_resid") or {} - demean_weeks = collected.get("mom_12_1_sector_demeaned") or {} - mom_weeks = collected.get("mom_12_1") or {} - - restricted: dict = defaultdict(lambda: defaultdict(list)) - for wk, recs in sector_weeks.items(): - syms = set() - for rec in recs: - if isinstance(rec, dict) and rec.get("symbol"): - syms.add(str(rec["symbol"]).upper()) - restricted["mom_12_1_sector_resid"][wk].append(rec) - for name, source in ( - ("mom_12_1_resid", resid_weeks), - ("mom_12_1", mom_weeks), - ("mom_12_1_sector_demeaned", demean_weeks), - ): - for rec in source.get(wk) or []: - if not isinstance(rec, dict): - continue - sym = rec.get("symbol") - if sym and str(sym).upper() in syms: - restricted[name][wk].append(rec) - - rows = bt._signal_evaluation(dict(restricted)) - return {r["signal"]: r for r in rows} - - -def _mask_diagnostics(signal_eval: list[dict]) -> dict[str, Any]: - # Prefer a dense signal for mask stats. - for name in ("vol_6m", "mom_12_1", "fip_id"): - for row in signal_eval: - if row.get("signal") == name and row.get("mask_binds_pct") is not None: - return { - "reference_signal": name, - "avg_cross_section": row.get("avg_cross_section"), - "avg_raw_pool": row.get("avg_raw_pool"), - "avg_eligible_pre_mask": row.get("avg_eligible_pre_mask"), - "mask_binds_pct": row.get("mask_binds_pct"), - "weeks": row.get("weeks"), - } - # Fallback: any row with liquid fields - for row in signal_eval: - if row.get("liquid_breadth_top_n"): - return { - "reference_signal": row.get("signal"), - "avg_cross_section": row.get("avg_cross_section"), - "mask_binds_pct": row.get("mask_binds_pct"), - "weeks": row.get("weeks"), - } - return {"note": "no liquid mask diagnostics on rows (mask may be off)"} - - -def _grade(harness: dict[str, Any]) -> dict[str, Any]: - """Pre-registered PASS/FAIL for mom_12_1_sector_resid — mechanical.""" - by = harness.get("signal_eval_by_name") or {} - era = harness.get("era_split") or {} - identical = harness.get("identical_subset_sector_cs") or {} - - sector = by.get("mom_12_1_sector_resid") - # Prefer identical-subset resid for t comparison; fall back to full-table resid. - resid = identical.get("mom_12_1_resid") or by.get("mom_12_1_resid") - pre = (era.get("pre_2021") or {}).get("mom_12_1_sector_resid") - post = (era.get("post_2021") or {}).get("mom_12_1_sector_resid") - - checks: dict[str, Any] = { - "sector_row": sector, - "resid_row_for_t": resid, - "resid_t_source": ( - "identical_subset" if identical.get("mom_12_1_resid") else "full_table" - ), - "pre_2021": pre, - "post_2021": post, - } - - if sector is None: - return { - "verdict": "FAIL", - "reason": "mom_12_1_sector_resid missing from signal_eval", - "checks": checks, - "headline": "Task 1 CLOSED — sector residual dead on deep evidence.", - } - - mean_ic = sector.get("mean_ic") - t_stat = sector.get("ic_t_stat") - weeks = int(sector.get("weeks") or 0) - reliable = bool(sector.get("reliable")) - resid_t = resid.get("ic_t_stat") if resid else None - - mag_ok = mean_ic is not None and abs(float(mean_ic)) >= IRON_IC - sign_ok = mean_ic is not None and float(mean_ic) > 0 - reliable_ok = reliable and weeks >= 12 - weeks_ok = weeks >= MIN_WEEKS_DEEP - t_ok = ( - t_stat is not None - and resid_t is not None - and float(t_stat) >= float(resid_t) - ) - - pre_ic = pre.get("mean_ic") if pre else None - post_ic = post.get("mean_ic") if post else None - era_sign_ok = ( - pre_ic is not None - and post_ic is not None - and float(pre_ic) > 0 - and float(post_ic) > 0 - ) - # If pre era has no row, data fix failed for depth / era coverage. - era_present = pre is not None and post is not None - - checks.update({ - "abs_mean_ic_ge_0_03": mag_ok, - "sign_positive": sign_ok, - "reliable": reliable_ok, - "weeks_ge_50": weeks_ok, - "weeks": weeks, - "t_ge_resid_same_cs": t_ok, - "sector_t": t_stat, - "resid_t": resid_t, - "era_both_present": era_present, - "era_sign_consistent_positive": era_sign_ok, - "pre_ic": pre_ic, - "post_ic": post_ic, - "avg_cross_section": sector.get("avg_cross_section"), - }) - - if not weeks_ok: - return { - "verdict": "FAIL", - "reason": ( - f"weeks={weeks} did not extend (need ≥{MIN_WEEKS_DEEP}) — " - "data fix did not work or sector residual still shallow" - ), - "checks": checks, - "headline": "Task 1 CLOSED — sector residual dead on deep evidence.", - } - - passed = ( - mag_ok - and sign_ok - and reliable_ok - and weeks_ok - and t_ok - and era_present - and era_sign_ok - ) - if passed: - return { - "verdict": "PASS", - "reason": ( - "iron bar + weeks extended + t≥resid on same CS + era sign consistent" - ), - "checks": checks, - "headline": ( - "PROMOTE case strengthened — portfolio A/B is the next human decision." - ), - } - return { - "verdict": "FAIL", - "reason": "failed one or more pre-registered checks (see checks)", - "checks": checks, - "headline": "Task 1 CLOSED — sector residual dead on deep evidence.", - } - - -def _write_reports(payload: dict, out_json: Path, doc_path: Path) -> None: - out_json.parent.mkdir(parents=True, exist_ok=True) - out_json.write_text( - json.dumps(payload, indent=2, default=str) + "\n", encoding="utf-8" - ) - - grade = payload.get("grade") or {} - harness = payload.get("harness") or {} - by = harness.get("signal_eval_by_name") or {} - era = harness.get("era_split") or {} - lines = [ - "# Sector-residual deep test (masked, repaired snapshot)", - "", - f"Generated: `{payload.get('generated_at')}`", - "", - f"> **{SURVIVORSHIP}**", - "", - "## Pre-registered grade (mechanical)", - "", - f"**Verdict: {grade.get('verdict')}**", - "", - f"{grade.get('headline')}", - "", - f"Reason: {grade.get('reason')}", - "", - f"```json\n{json.dumps(grade.get('checks') or {}, indent=2, default=str)}\n```", - "", - "## Step-1 sanity", - "", - f"```json\n{json.dumps(payload.get('step1') or {}, indent=2, default=str)}\n```", - "", - "## Mask diagnostics", - "", - f"```json\n{json.dumps(harness.get('mask_diagnostics') or {}, indent=2, default=str)}\n```", - "", - "## Signal table (rows only — no narrative for non-sector signals)", - "", - "| signal | mean_ic | t | weeks | avg_N | reliable |", - "|---|---:|---:|---:|---:|---|", - ] - for name in sorted(by): - r = by[name] - lines.append( - f"| {name} | {r.get('mean_ic')} | {r.get('ic_t_stat')} | " - f"{r.get('weeks')} | {r.get('avg_cross_section')} | {r.get('reliable')} |" - ) - lines.extend([ - "", - "### Era split — mom_12_1_sector_resid only (for grade)", - "", - f"| era | IC | t | weeks | N |", - f"|---|---:|---:|---:|---:|", - ]) - for label in ("pre_2021", "post_2021"): - r = (era.get(label) or {}).get("mom_12_1_sector_resid") or {} - lines.append( - f"| {label} | {r.get('mean_ic', '')} | {r.get('ic_t_stat', '')} | " - f"{r.get('weeks', '')} | {r.get('avg_cross_section', '')} |" - ) - lines.extend([ - "", - "### Identical-subset baselines (sector CS)", - "", - f"```json\n{json.dumps(harness.get('identical_subset_sector_cs') or {}, indent=2, default=str)}\n```", - "", - "## Status", - "", - "PENDING_HUMAN beyond the mechanical PASS/FAIL above. " - "Nothing merged into production docs or prod code.", - "", - f"JSON: `{out_json.as_posix()}`", - "", - ]) - md_path = out_json.with_suffix(".md") - md_path.write_text("\n".join(lines) + "\n", encoding="utf-8") - - # Update history-depth-extension.md - _update_history_doc(doc_path, payload, out_json) - - -def _update_history_doc(doc_path: Path, payload: dict, out_json: Path) -> None: - grade = payload.get("grade") or {} - banner = ( - "\n\n---\n\n" - "## Supersession notice (2026-07-19 sector-resid deep test)\n\n" - "The table and interpretation from **`history-depth-20260719-103315`** are " - "**UNMASKED, TWO-TIER SNAPSHOT — superseded, directional only, do not cite**. " - "Prod-universe names (and sector residual coverage) were left shallow while " - "breadth names were deepened; sector residual weeks=35 was a data gap.\n\n" - f"### Sector-residual deep test outcome: **{grade.get('verdict')}**\n\n" - f"{grade.get('headline')}\n\n" - f"- Reason: {grade.get('reason')}\n" - f"- Artifact: `{out_json.as_posix()}`\n" - f"- Mechanical checks: see that report.\n\n" - "**Future snapshot rebuilds must verify per-symbol depth** (earliest-bar " - "uniformity across the intended universe) — guard is a to-do, not part of " - "this order.\n" - ) - if doc_path.exists(): - text = doc_path.read_text(encoding="utf-8") - # Insert supersession after status line / near top results if not already there. - marker = "## Supersession notice (2026-07-19 sector-resid deep test)" - if marker in text: - # Replace from marker to end of that section or append fresh block at end. - pre = text.split(marker)[0].rstrip() - text = pre + banner - else: - # Mark 103315 in place if mentioned. - text = text.replace( - "Authoritative artifact:** `reports/history-depth-20260719-103315.json`", - "Superseded artifact (do not cite):** `reports/history-depth-20260719-103315.json` " - "— **UNMASKED, TWO-TIER SNAPSHOT**", - ) - text = text.rstrip() + banner - # Soften old PARK-only language if present — leave body but status at top. - if text.startswith("#"): - lines = text.splitlines() - for i, line in enumerate(lines[:15]): - if line.startswith("**Status:**"): - lines[i] = ( - f"**Status:** sector-resid deep test **{grade.get('verdict')}** " - f"— see supersession section. PENDING_HUMAN beyond PASS/FAIL." - ) - break - text = "\n".join(lines) - doc_path.write_text(text if text.endswith("\n") else text + "\n", encoding="utf-8") - else: - doc_path.write_text( - "# History-depth extension\n" + banner, encoding="utf-8" - ) - - -async def _main() -> None: - args = _parse_args() - snapshot = Path(args.snapshot) - if not snapshot.exists(): - raise SystemExit(f"Snapshot missing: {snapshot}") - if args.allow_spawn: - os.environ["BACKTEST_ALLOW_SPAWN"] = "1" - - sector_map = Path(args.sector_map) - if not sector_map.exists(): - raise SystemExit(f"Sector map missing: {sector_map}") - - step1: dict[str, Any] - if args.skip_deepen: - print("Skip deepen — race guard + sanity only…") - assert_research_snapshot_complete(snapshot) - sanity = _sanity_check(snapshot, history_days=args.history_days) - step1 = {"skipped": True, "sanity": sanity} - if not sanity["passed"]: - raise SystemExit( - "Sanity check FAILED with --skip-deepen. " - f"Details: {json.dumps(sanity, default=str)}" - ) - else: - print("Step 1 — deepen shallow symbols…") - step1 = await _step1_deepen( - snapshot, - history_days=args.history_days, - sleep_s=args.sleep, - quiet=args.quiet, - ) - if not step1["sanity"]["passed"]: - print("SANITY CHECK FAILED — refusing harness.") - print(json.dumps(step1["sanity"], indent=2, default=str)) - stamp = datetime.now().strftime("%Y%m%d-%H%M%S") - fail_path = Path("reports") / f"sector-resid-deep-{stamp}-SANITY-FAIL.json" - fail_path.parent.mkdir(parents=True, exist_ok=True) - fail_path.write_text( - json.dumps({"step1": step1, "harness": None}, indent=2, default=str) - + "\n", - encoding="utf-8", - ) - raise SystemExit( - f"Stop: sanity failed. Wrote {fail_path}. Do not run harness on two-tier data." - ) - print("Sanity check PASSED.") - assert_research_snapshot_complete(snapshot) - - print( - f"Step 2 — ONE masked harness " - f"(top {args.liquid_breadth}, min_price={args.min_price})…" - ) - harness = await _one_masked_run( - snapshot, - sector_map_path=sector_map, - liquid_breadth=args.liquid_breadth, - min_price=args.min_price, - workers=args.workers, - quiet=args.quiet, - ) - grade = _grade(harness) - print(f"GRADE: {grade['verdict']} — {grade['headline']}") - - stamp = datetime.now().strftime("%Y%m%d-%H%M%S") - out = Path(args.out) if args.out else Path("reports") / f"sector-resid-deep-{stamp}.json" - payload = { - "generated_at": datetime.now().isoformat(), - "snapshot": str(snapshot.resolve()), - "pre_registration": { - "iron_ic": IRON_IC, - "min_weeks_deep": MIN_WEEKS_DEEP, - "liquid_breadth": args.liquid_breadth, - "min_price": args.min_price, - "rule": ( - "PASS = |IC|>=0.03, +sign, reliable, weeks>=50, " - "t>=resid on same CS, era signs both +" - ), - }, - "step1": step1, - "harness": harness, - "grade": grade, - "pending_human": True, - "note": "Nothing merged into production. Thread ends at PASS/FAIL.", - } - _write_reports( - payload, - out, - Path("docs/research/history-depth-extension.md"), - ) - print(f"Wrote {out}") - print(f"Wrote {out.with_suffix('.md')}") - print("Updated docs/research/history-depth-extension.md") - - -if __name__ == "__main__": - asyncio.run(_main()) diff --git a/scripts/run_sector_residual_research.py b/scripts/run_sector_residual_research.py deleted file mode 100644 index 1f60590..0000000 --- a/scripts/run_sector_residual_research.py +++ /dev/null @@ -1,1018 +0,0 @@ -"""Sector-residual momentum research runner (local only). - -Protocol --------- -1. Race-guard the research/prod snapshot (completion manifest when present). -2. Require sector map + sector ETFs in ``benchmark_prices``. -3. Run signal IC harness on the production ~505-name snapshot - (``BACKTEST_SIGNAL_EVAL_ONLY=1``) with sector context loaded. -4. Grade candidates vs pre-registered iron rule + t-stat vs ``mom_12_1_resid``. -5. If a candidate promotes: portfolio A/B with candidate as momentum leg + - gate percentile (``fill_mode=close``). Optional sector-cap arm. - -Does not modify production DB, gate, scanner, or schedule. - -Example -------- - python scripts/run_sector_residual_research.py \\ - --snapshot backtest_snapshots/prod.sqlite \\ - --workers 6 --allow-spawn -""" - -from __future__ import annotations - -import argparse -import asyncio -import json -import math -import os -import sys -from collections import defaultdict -from copy import deepcopy -from datetime import date, datetime -from pathlib import Path -from typing import Any - -from sqlalchemy import create_engine, text -from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine - -ROOT = Path(__file__).resolve().parents[1] -if str(ROOT) not in sys.path: - sys.path.insert(0, str(ROOT)) - -from app.ssl_bootstrap import bootstrap_ssl # noqa: E402 - -bootstrap_ssl() - -from app.services.sector_map import ( # noqa: E402 - DEFAULT_SECTOR_MAP_PATH, - SECTOR_ETFS, - coverage_stats, - load_ticker_sector_map, - normalise_symbol, - sector_to_etf, -) - -VALIDATION_SPLIT = date(2024, 7, 1) -IRON_IC_BAR = 0.03 -MIN_RELIABLE = 12 - - -def _sqlite_url(path: Path) -> str: - return f"sqlite+aiosqlite:///{path.resolve().as_posix()}" - - -def _parse_args() -> argparse.Namespace: - p = argparse.ArgumentParser(description=__doc__) - p.add_argument("--snapshot", default="backtest_snapshots/prod.sqlite") - p.add_argument( - "--sector-map", - default=str(DEFAULT_SECTOR_MAP_PATH), - ) - p.add_argument("--workers", type=int, default=6) - p.add_argument("--allow-spawn", action="store_true") - p.add_argument( - "--skip-ab", - action="store_true", - help="IC only — never run portfolio A/B even if promotion fires.", - ) - p.add_argument( - "--force-ab", - action="store_true", - help="Run A/B for diagnostic even if IC bar fails (still reported as non-promote).", - ) - p.add_argument( - "--sector-cap", - type=int, - default=None, - help="Optional max positions per sector (e.g. 3). Only used in A/B.", - ) - p.add_argument("--quiet", action="store_true") - p.add_argument( - "--out", - default=None, - help="JSON report path (default reports/sector-residual-YYYYMMDD-HHMMSS.json)", - ) - return p.parse_args() - - -def _assert_snapshot_ready(snapshot: Path) -> dict[str, Any]: - """Race guard: prefer completion manifest; always check live bar sanity.""" - from scripts.research_snapshot_manifest import ( # type: ignore - assert_research_snapshot_complete, - load_manifest, - ) - - guard: dict[str, Any] = {"snapshot": str(snapshot.resolve())} - manifest = load_manifest(snapshot) - if manifest is not None: - # Full assert when a manifest exists (research.sqlite path). - try: - m = assert_research_snapshot_complete(snapshot) - guard["manifest"] = m - guard["manifest_ok"] = True - except SystemExit as exc: - raise SystemExit(str(exc)) from exc - else: - guard["manifest"] = None - guard["manifest_ok"] = None - guard["note"] = ( - "No completion manifest (prod.sqlite is expected without one). " - "Bar-count sanity still applied." - ) - - engine = create_engine( - f"sqlite:///{snapshot.resolve().as_posix()}", - future=True, - ) - try: - with engine.connect() as conn: - ticker_n = int(conn.execute(text("SELECT COUNT(*) FROM tickers")).scalar_one()) - ohlcv_n = int( - conn.execute(text("SELECT COUNT(*) FROM ohlcv_records")).scalar_one() - ) - bar_stats = conn.execute( - text( - """ - SELECT MIN(c), AVG(c), MAX(c) FROM ( - SELECT COUNT(*) AS c FROM ohlcv_records GROUP BY ticker_id - ) - """ - ) - ).fetchone() - bench = conn.execute( - text( - "SELECT symbol, COUNT(*), MIN(date), MAX(date) " - "FROM benchmark_prices GROUP BY symbol ORDER BY symbol" - ) - ).fetchall() - d_range = conn.execute( - text("SELECT MIN(date), MAX(date) FROM ohlcv_records") - ).fetchone() - finally: - engine.dispose() - - guard["ticker_count"] = ticker_n - guard["ohlcv_row_count"] = ohlcv_n - guard["bars_min_avg_max"] = { - "min": bar_stats[0], - "avg": round(float(bar_stats[1]), 1) if bar_stats[1] is not None else None, - "max": bar_stats[2], - } - guard["ohlcv_date_range"] = {"min": d_range[0], "max": d_range[1]} - guard["benchmark_prices"] = [ - {"symbol": s, "n": n, "min": d0, "max": d1} for s, n, d0, d1 in bench - ] - - # Sanity: a half-built snapshot would show many tickers with tiny bar counts. - min_bars = int(bar_stats[0] or 0) - avg_bars = float(bar_stats[1] or 0) - if ticker_n < 400: - raise SystemExit( - f"Snapshot looks short: only {ticker_n} tickers (expected ~505 prod)." - ) - if avg_bars < 200: - raise SystemExit( - f"Snapshot bar counts look short (avg={avg_bars:.0f}). Rebuild before research." - ) - # Allow a few thin names; refuse if median path is collapsed. - if min_bars < 10 and avg_bars < 500: - raise SystemExit( - f"Snapshot min bars={min_bars}, avg={avg_bars:.0f} — possible partial build." - ) - - present_etfs = {row[0] for row in bench} - missing_etfs = [e for e in SECTOR_ETFS if e not in present_etfs] - guard["missing_sector_etfs"] = missing_etfs - if missing_etfs: - raise SystemExit( - "Sector ETFs missing from benchmark_prices: " - f"{missing_etfs}. Run scripts/fetch_sector_etfs_to_snapshot.py first." - ) - if "SPY" not in present_etfs: - raise SystemExit("SPY missing from benchmark_prices") - - return guard - - -def _find_signal(rows: list[dict], name: str) -> dict | None: - for row in rows or []: - if row.get("signal") == name: - return row - return None - - -def _grade_ic( - candidate: dict | None, - resid: dict | None, - *, - expected_sign: float = 1.0, -) -> dict[str, Any]: - """Iron rule + t-stat ≥ mom_12_1_resid.""" - if candidate is None: - return { - "promote_to_ab": False, - "reason": "signal missing from signal_eval", - } - mean_ic = candidate.get("mean_ic") - t_stat = candidate.get("ic_t_stat") - reliable = bool(candidate.get("reliable")) - weeks = int(candidate.get("weeks") or 0) - if mean_ic is None or t_stat is None: - return {"promote_to_ab": False, "reason": "missing mean_ic or t", "row": candidate} - - sign_ok = (float(mean_ic) * expected_sign) > 0 - mag_ok = abs(float(mean_ic)) >= IRON_IC_BAR - reliable_ok = reliable and weeks >= MIN_RELIABLE - resid_t = resid.get("ic_t_stat") if resid else None - t_ok = resid_t is not None and float(t_stat) >= float(resid_t) - - promote = sign_ok and mag_ok and reliable_ok and t_ok - return { - "promote_to_ab": promote, - "checks": { - "sign_ok": sign_ok, - "abs_mean_ic_ge_0_03": mag_ok, - "reliable": reliable_ok, - "t_ge_resid": t_ok, - "mean_ic": mean_ic, - "ic_t_stat": t_stat, - "resid_ic_t_stat": resid_t, - "weeks": weeks, - }, - "reason": ( - "clears iron rule and t ≥ mom_12_1_resid — authorized for A/B only" - if promote - else "does not clear pre-registered IC promotion bar" - ), - "row": candidate, - } - - -async def _run_signal_eval( - snapshot: Path, - *, - workers: int, - quiet: bool, - sector_map_path: Path, -) -> dict: - from app.config import settings - from app.services.backtest_service import run_backtest - - os.environ["BACKTEST_SNAPSHOT_OFFLINE"] = "1" - os.environ["BACKTEST_SIGNAL_EVAL_ONLY"] = "1" - os.environ["BACKTEST_SECTOR_MAP_PATH"] = str(sector_map_path.resolve()) - # Clear liquid-breadth — this is the 505-name prod IC, not breadth. - os.environ.pop("BACKTEST_LIQUID_BREADTH", None) - settings.backtest_workers = workers - - engine = create_async_engine(_sqlite_url(snapshot), pool_pre_ping=True) - Session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) - - def progress(done: int, total: int, symbol: str) -> None: - if quiet: - return - print(f" progress {done}/{total} {symbol}", end="\r", flush=True) - - try: - async with Session() as db: - report = await run_backtest(db, progress_cb=progress, cadence="weekly") - finally: - await engine.dispose() - if not quiet: - print() - return report - - -def _period_percentiles(rows: list[dict], value_key: str) -> dict[tuple, dict[str, float]]: - by_period: dict[tuple, list[dict]] = defaultdict(list) - for row in rows: - if row.get(value_key) is None: - continue - period = row.get("ranking_period") or row.get("iso_week") - by_period[period].append(row) - out: dict[tuple, dict[str, float]] = {} - for period, group in by_period.items(): - ordered = sorted(group, key=lambda r: float(r[value_key])) - n = len(ordered) - for rank, row in enumerate(ordered): - key = (str(row["symbol"]), str(row["date"])) - pct = (rank / (n - 1) * 100.0) if n > 1 else 100.0 - out.setdefault(key, {})[value_key] = float(row[value_key]) - out[key][f"{value_key}_percentile"] = pct - return out - - -async def _load_prices_and_benchmarks(snapshot: Path) -> tuple[dict, dict, dict]: - """Return (price_columns, spy_closes, sector_etf_closes).""" - from app.services import backtest_service as bt - from app.services.benchmark_service import load_benchmark_closes - from app.models.ticker import Ticker - from sqlalchemy import select - - engine = create_async_engine(_sqlite_url(snapshot), pool_pre_ping=True) - Session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) - prices: dict[str, tuple] = {} - try: - async with Session() as db: - tickers = list( - (await db.execute(select(Ticker).order_by(Ticker.symbol))).scalars() - ) - for t in tickers: - cols = await bt._fetch_columns(db, t.symbol) - if cols is not None: - prices[t.symbol] = cols - spy = await load_benchmark_closes(db, "SPY") - sector: dict[str, dict] = {} - for etf in SECTOR_ETFS: - series = await load_benchmark_closes(db, etf) - if series: - sector[etf] = series - finally: - await engine.dispose() - return prices, spy, sector - - -def _recompute_sector_residual_on_candidates( - candidates: list[dict], - prices: dict[str, tuple], - spy_closes: dict, - sector_etf_closes: dict[str, dict], - symbol_to_sector: dict[str, str], - *, - momentum_field: str, -) -> list[dict]: - """Attach alternative residual momentum on each candidate as-of date.""" - from app.services import backtest_service as bt - from app.services.sector_map import etf_for_symbol - - # Index price series once. - series_cache: dict[str, tuple[list, list, list]] = {} - for sym, cols in prices.items(): - ords, _o, _h, _l, closes, _v = cols - dates = [date.fromordinal(int(o)) for o in ords] - series_cache[sym] = (dates, list(closes), list(ords)) - - out: list[dict] = [] - for cand in candidates: - c = dict(cand) - sym = str(c["symbol"]) - if sym not in series_cache: - out.append(c) - continue - dates, closes, ords = series_cache[sym] - asof = date.fromisoformat(str(c["date"])) - # Find as-of index. - try: - i = next(idx for idx, d in enumerate(dates) if d == asof) - except StopIteration: - # nearest on/before - i = max((idx for idx, d in enumerate(dates) if d <= asof), default=-1) - if i < 0: - out.append(c) - continue - - if momentum_field == "mom_12_1_sector_resid": - etf = etf_for_symbol(sym, symbol_to_sector) - etf_series = sector_etf_closes.get(etf or "") - val = None - if spy_closes and etf_series: - val = bt._multi_factor_residual_momentum_12_1( - dates, closes, i, [spy_closes, etf_series] - ) - c["residual_momentum"] = val - c["_alt_momentum_signal"] = momentum_field - c["_alt_momentum_value"] = val - elif momentum_field == "mom_12_1_sector_demeaned": - # Placeholder: demean requires cross-section; filled in a second pass. - raw = None - if i >= 252 and closes[i - 252] > 0: - raw = closes[i - 21] / closes[i - 252] - 1.0 - c["_raw_mom_12_1"] = raw - c["_alt_momentum_signal"] = momentum_field - else: - raise ValueError(momentum_field) - out.append(c) - - if momentum_field == "mom_12_1_sector_demeaned": - # Cross-sectional demean within ranking period × sector. - by_period: dict[Any, list[dict]] = defaultdict(list) - for c in out: - if c.get("_raw_mom_12_1") is None: - continue - period = c.get("ranking_period") or c.get("iso_week") - by_period[period].append(c) - for period, group in by_period.items(): - by_sec: dict[str, list[float]] = defaultdict(list) - for c in group: - sec = symbol_to_sector.get(normalise_symbol(str(c["symbol"]))) - if sec: - by_sec[sec].append(float(c["_raw_mom_12_1"])) - means = { - s: sum(vs) / len(vs) for s, vs in by_sec.items() if len(vs) >= 2 - } - for c in group: - sec = symbol_to_sector.get(normalise_symbol(str(c["symbol"]))) - raw = float(c["_raw_mom_12_1"]) - if sec in means: - val = raw - means[sec] - c["residual_momentum"] = val - c["_alt_momentum_value"] = val - else: - c["residual_momentum"] = None - c["_alt_momentum_value"] = None - - return out - - -def _assign_prod_ranks(candidates: list[dict]) -> None: - from app.services import backtest_service as bt - - bt._assign_momentum_percentiles(candidates) - bt._assign_residual_momentum_percentiles(candidates) - bt._assign_low_volatility_percentiles(candidates) - bt._assign_activation_momentum_percentiles(candidates) - bt._assign_residual_high_vol_blend(candidates) - for c in candidates: - c["qualified"] = bt._momentum_qualifies(c, 80.0) - - -async def _run_ab( - snapshot: Path, - *, - sector_map: dict[str, str], - signal_name: str, - sector_cap: int | None, - quiet: bool, - workers: int, -) -> dict[str, Any]: - """Control vs treatment book with candidate as momentum residual.""" - from app.services import backtest_service as bt - from app.config import settings - from app.models.ticker import Ticker - from app.services.admin_service import get_activation_config - from app.services.recommendation_service import get_recommendation_config - from app.services.paper_trade_service import get_exit_policy - from app.services.benchmark_service import load_benchmark_closes - from sqlalchemy import select - - os.environ["BACKTEST_SNAPSHOT_OFFLINE"] = "1" - os.environ.pop("BACKTEST_SIGNAL_EVAL_ONLY", None) - settings.backtest_workers = max(1, int(workers)) - - engine = create_async_engine(_sqlite_url(snapshot), pool_pre_ping=True) - Session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) - try: - async with Session() as db: - config = await get_recommendation_config(db) - activation = await get_activation_config(db) - exit_config = await get_exit_policy(db) - tickers = list( - (await db.execute(select(Ticker).order_by(Ticker.symbol))).scalars() - ) - spy = await load_benchmark_closes(db, "SPY") - sector_etf: dict[str, dict] = {} - for etf in SECTOR_ETFS: - series = await load_benchmark_closes(db, etf) - if series: - sector_etf[etf] = series - - prices: dict[str, tuple] = {} - candidates: list[dict] = [] - for idx, t in enumerate(tickers): - if not quiet and idx % 25 == 0: - print(f" fetch {idx}/{len(tickers)}", end="\r", flush=True) - cols = await bt._fetch_columns(db, t.symbol) - if cols is None: - continue - prices[t.symbol] = cols - cands, _series = bt._replay_and_signals( - t.symbol, - cols, - config, - activation, - spy, - bt.PRODUCTION_GTL_TARGET_MODEL, - "weekly", - False, - sector_etf, - sector_map, - ) - candidates.extend(cands) - finally: - await engine.dispose() - if not quiet: - print() - - # Control ranks (production residual). - control = [dict(c) for c in candidates] - _assign_prod_ranks(control) - control_longs = [ - c for c in control if c.get("qualified") and c.get("direction") == "long" - ] - - # Treatment: replace residual with sector signal, re-rank. - treatment = _recompute_sector_residual_on_candidates( - candidates, - prices, - spy, - sector_etf, - sector_map, - momentum_field=signal_name, - ) - _assign_prod_ranks(treatment) - treatment_longs = [ - c for c in treatment if c.get("qualified") and c.get("direction") == "long" - ] - - strategy = next(s for s in bt.PORTFOLIO_MONITOR_STRATEGIES if s.get("is_production")) - entry_config = bt._entry_variant_config(str(strategy["entry_variant"])) - assert entry_config is not None - ranking_key = str( - entry_config.get("ranking_key") or entry_config["percentile_key"] - ) - # Production ranking key is residual_high_vol_blend_80_20. - if ranking_key not in (bt.RESIDUAL_HIGH_VOL_BLEND_80_20_KEY, bt.PRODUCTION_PERCENTILE_KEY): - ranking_key = bt.RESIDUAL_HIGH_VOL_BLEND_80_20_KEY - - exit_policy = bt.LIVE_EXIT_MODE_TO_SIM.get( - str(exit_config.get("mode", "atr_trailing")), "atr_trail3" - ) - hold_days = int(exit_config.get("hold_days", 30)) - trail = float(exit_config.get("atr_multiplier", bt.ATR_TRAIL_MULTIPLIER)) - risk = float(entry_config["risk_per_trade"]) - max_pos = int(entry_config["max_positions"]) - - def _sim_book(longs: list[dict], *, label: str, cap: int | None = None) -> dict: - reentry = bt._make_gate_reset_reentry_fn( - longs, prices, cadence="weekly", ranking_key=ranking_key - ) - windows = {} - for wname, start, end in ( - ("train", None, VALIDATION_SPLIT), - ("validation", VALIDATION_SPLIT, None), - ("full", None, None), - ): - sim = bt._simulate_portfolio( - longs, - prices, - spy, - exit_policy, - hold_days, - ranking_key=ranking_key, - max_positions=max_pos, - risk_per_trade=risk, - atr_trail_multiplier=trail, - post_stop_reentry_fn=reentry, - start_date=start, - end_date=end, - fill_mode=bt.FILL_MODE_CLOSE, - include_trades=True, - ) - if sim is None: - windows[wname] = {"error": "no_trades"} - continue - # Optional sector cap: filter trade_details is post-hoc; real cap needs - # simulator support. For research, re-sim with a wrapper ranking that - # drops overflow sector names is approximate — we implement a simple - # pre-filter on daily entry sets via max_positions only when cap is None. - # When cap is set, apply a post-sim diagnostic on entries. - payload = { - k: sim.get(k) - for k in ( - "sharpe", - "sharpe_se", - "cagr_pct", - "max_drawdown_pct", - "total_return_pct", - "trades", - "win_rate_pct", - "avg_r", - "n_returns", - "return_skew", - "return_kurtosis", - "psr", - ) - } - details = sim.get("trade_details") or [] - rs = [ - float(t["realized_r"]) - for t in details - if t.get("realized_r") is not None - ] - if rs: - rs_sorted = sorted(rs) - payload["r_p05"] = rs_sorted[max(0, int(0.05 * (len(rs_sorted) - 1)))] - payload["r_p50"] = rs_sorted[len(rs_sorted) // 2] - payload["r_p95"] = rs_sorted[min(len(rs_sorted) - 1, int(0.95 * (len(rs_sorted) - 1)))] - payload["entry_count"] = len(rs) - if cap is not None and details: - # Diagnostic: count how often a calendar day would exceed cap. - from collections import Counter - - # Use entry dates; sector from map. - day_sector: dict[str, Counter] = defaultdict(Counter) - for t in details: - sec = sector_map.get(normalise_symbol(str(t.get("symbol", "")))) or "?" - day_sector[str(t.get("entry_date") or t.get("date") or "")][sec] += 1 - breaches = sum( - 1 - for day, ctr in day_sector.items() - if any(v > cap for v in ctr.values()) - ) - payload["sector_cap"] = cap - payload["entry_days_with_sector_over_cap"] = breaches - windows[wname] = payload - return {"label": label, "n_qualified_longs": len(longs), "windows": windows} - - control_result = _sim_book(control_longs, label="control_mom_12_1_resid") - treatment_result = _sim_book( - treatment_longs, label=f"treatment_{signal_name}", cap=None - ) - out: dict[str, Any] = { - "signal": signal_name, - "ranking_key": ranking_key, - "fill_mode": "close", - "validation_split": VALIDATION_SPLIT.isoformat(), - "control": control_result, - "treatment": treatment_result, - "promotion": _grade_ab(control_result, treatment_result), - } - if sector_cap is not None: - # Approximate sector-cap book: when selecting, prefer higher rank but - # refuse a 4th name in the same sector among concurrent opens. - # Implemented by tagging candidates and using a custom sim is heavy; - # instead report diagnostic on unconstrained treatment + a filtered - # re-rank that zeros residual for overflow names within each period. - capped = _apply_sector_cap_to_ranks( - treatment, sector_map, cap=sector_cap, ranking_key=ranking_key - ) - capped_longs = [ - c for c in capped if c.get("qualified") and c.get("direction") == "long" - ] - out["sector_cap_arm"] = _sim_book( - capped_longs, label=f"treatment_{signal_name}_cap{sector_cap}", cap=sector_cap - ) - out["sector_cap_promotion"] = _grade_ab( - control_result, out["sector_cap_arm"] - ) - return out - - -def _apply_sector_cap_to_ranks( - candidates: list[dict], - sector_map: dict[str, str], - *, - cap: int, - ranking_key: str, -) -> list[dict]: - """Within each ranking period, keep top `cap` per sector by ranking_key.""" - by_period: dict[Any, list[dict]] = defaultdict(list) - for c in candidates: - period = c.get("ranking_period") or c.get("iso_week") - by_period[period].append(dict(c)) - out: list[dict] = [] - for period, group in by_period.items(): - ordered = sorted( - group, - key=lambda r: float(r.get(ranking_key) or r.get("residual_momentum") or -1e9), - reverse=True, - ) - sector_counts: dict[str, int] = defaultdict(int) - for c in ordered: - sec = sector_map.get(normalise_symbol(str(c["symbol"]))) or "_unknown" - if sector_counts[sec] >= cap: - # Push below gate by nulling activation percentile. - c["qualified"] = False - c["_sector_cap_blocked"] = True - else: - if c.get("qualified"): - sector_counts[sec] += 1 - out.append(c) - return out - - -def _grade_ab(control: dict, treatment: dict) -> dict[str, Any]: - """Pre-registered: val Sharpe ≥ control − 0.5·SE; full Sharpe & maxDD not worse.""" - def win(arm: dict, name: str) -> dict: - return (arm.get("windows") or {}).get(name) or {} - - c_val = win(control, "validation") - t_val = win(treatment, "validation") - c_full = win(control, "full") - t_full = win(treatment, "full") - - def _f(d: dict, k: str) -> float | None: - v = d.get(k) - return None if v is None else float(v) - - c_sh = _f(c_val, "sharpe") - t_sh = _f(t_val, "sharpe") - # Use treatment SE if present else control SE. - se = _f(t_val, "sharpe_se") - if se is None: - se = _f(c_val, "sharpe_se") - if se is None: - se = 0.0 - - val_ok = ( - c_sh is not None - and t_sh is not None - and t_sh >= (c_sh - 0.5 * se) - ) - c_full_sh = _f(c_full, "sharpe") - t_full_sh = _f(t_full, "sharpe") - full_sh_ok = ( - c_full_sh is not None - and t_full_sh is not None - and t_full_sh >= c_full_sh - ) - # max DD: higher absolute drawdown is worse; stored as positive pct typically. - c_dd = _f(c_full, "max_drawdown_pct") - t_dd = _f(t_full, "max_drawdown_pct") - full_dd_ok = ( - c_dd is not None and t_dd is not None and abs(t_dd) <= abs(c_dd) + 1e-9 - ) - promote = bool(val_ok and full_sh_ok and full_dd_ok) - return { - "promote": promote, - "checks": { - "validation_sharpe_ge_control_minus_half_se": val_ok, - "full_sharpe_not_worse": full_sh_ok, - "full_maxdd_not_worse": full_dd_ok, - "control_validation_sharpe": c_sh, - "treatment_validation_sharpe": t_sh, - "se_used": se, - "control_full_sharpe": c_full_sh, - "treatment_full_sharpe": t_full_sh, - "control_full_maxdd": c_dd, - "treatment_full_maxdd": t_dd, - }, - "reason": ( - "clears pre-registered A/B bar — human decides wire-in" - if promote - else "fails pre-registered A/B bar" - ), - } - - -def _write_md(path: Path, payload: dict) -> None: - """Refresh the results sections of the research doc (preserve pre-reg header).""" - # Always write a standalone results companion + update the main doc's - # results block by rewriting the full file with pre-reg + results. - pre = Path("docs/research/sector-residual-momentum.md") - # Keep pre-registration by reading until '## Results' if present. - header = "" - if pre.exists(): - text = pre.read_text(encoding="utf-8") - marker = "## Results" - idx = text.find(marker) - header = text[:idx] if idx >= 0 else text.split("## Verdict")[0] - - guard = payload.get("snapshot_guard") or {} - cov = payload.get("sector_coverage") or {} - ic_rows = payload.get("signal_eval") or [] - grades = payload.get("ic_grades") or {} - lines = [ - header.rstrip(), - "", - "## Results", - "", - f"Generated: `{payload.get('generated_at')}`", - "", - "### Snapshot race guard", - "", - f"- Snapshot: `{guard.get('snapshot')}`", - f"- Tickers: **{guard.get('ticker_count')}** OHLCV rows: **{guard.get('ohlcv_row_count')}**", - f"- Bars min/avg/max: `{guard.get('bars_min_avg_max')}`", - f"- OHLCV range: `{guard.get('ohlcv_date_range')}`", - f"- Manifest ok: `{guard.get('manifest_ok')}`", - f"- Missing sector ETFs at start: `{guard.get('missing_sector_etfs')}`", - "", - "### Sector label coverage", - "", - f"```json\n{json.dumps(cov, indent=2, default=str)}\n```", - "", - "### IC harness (identical cross-sections)", - "", - "| signal | mean_ic | ic_t_stat | weeks | avg_N | reliable | ic+_pct |", - "|---|---:|---:|---:|---:|---|---:|", - ] - want = [ - "mom_12_1", - "mom_12_1_resid", - "mom_12_1_sector_resid", - "mom_12_1_sector_demeaned", - ] - by_name = {r.get("signal"): r for r in ic_rows} - for name in want: - r = by_name.get(name) or {} - lines.append( - f"| {name} | {r.get('mean_ic', '')} | {r.get('ic_t_stat', '')} | " - f"{r.get('weeks', '')} | {r.get('avg_cross_section', '')} | " - f"{r.get('reliable', '')} | {r.get('ic_positive_pct', '')} |" - ) - lines.extend(["", "### IC promotion grades", ""]) - for name, g in grades.items(): - lines.append(f"- **{name}**: promote_to_ab=`{g.get('promote_to_ab')}` — {g.get('reason')}") - lines.append(f" - checks: `{json.dumps(g.get('checks') or {}, default=str)}`") - - ab = payload.get("portfolio_ab") - lines.extend(["", "### Portfolio A/B", ""]) - if not ab: - lines.append("_Not run (IC bar not cleared, or --skip-ab)._") - else: - lines.append(f"```json\n{json.dumps(ab, indent=2, default=str)}\n```") - - lines.extend([ - "", - "## Verdict", - "", - f"**{payload.get('verdict')}**", - "", - payload.get("verdict_detail") or "", - "", - "## What a human must decide next", - "", - payload.get("human_next") or "- Review numbers; do not merge into strategy docs without approval.", - "", - "## Artifacts", - "", - f"- JSON: `{payload.get('report_path')}`", - "", - ]) - path.write_text("\n".join(lines) + "\n", encoding="utf-8") - - -async def _main() -> None: - args = _parse_args() - snapshot = Path(args.snapshot) - sector_map_path = Path(args.sector_map) - if not snapshot.exists(): - raise SystemExit(f"Snapshot missing: {snapshot}") - if not sector_map_path.exists(): - raise SystemExit( - f"Sector map missing: {sector_map_path}. " - "Run scripts/build_ticker_sector_map.py first." - ) - - if args.allow_spawn: - os.environ["BACKTEST_ALLOW_SPAWN"] = "1" - - print("Race-guarding snapshot…") - guard = _assert_snapshot_ready(snapshot) - print( - f" tickers={guard['ticker_count']} ohlcv={guard['ohlcv_row_count']} " - f"bars={guard['bars_min_avg_max']}" - ) - - mapping = load_ticker_sector_map(sector_map_path) - engine = create_engine( - f"sqlite:///{snapshot.resolve().as_posix()}", - future=True, - ) - try: - with engine.connect() as conn: - symbols = [ - normalise_symbol(r[0]) - for r in conn.execute(text("SELECT symbol FROM tickers")).fetchall() - ] - finally: - engine.dispose() - cov = coverage_stats(symbols, mapping) - print( - f"Sector map: {cov['mapped']}/{cov['universe']} " - f"({cov['mapped_pct']}%) with_etf={cov['with_etf']}" - ) - if cov["mapped_pct"] < 90: - print( - f"WARNING: sector coverage {cov['mapped_pct']}% < 90%; " - f"missing e.g. {cov['missing'][:20]}" - ) - - print("Running IC harness (signal-eval only)…") - report = await _run_signal_eval( - snapshot, - workers=args.workers, - quiet=args.quiet, - sector_map_path=sector_map_path, - ) - signal_eval = report.get("signal_eval") or report.get("signals") or [] - # Locate key in report — backtest uses "signal_edge" historically. - if not signal_eval: - for key in ("signal_edge", "signal_evaluation", "factor_ic"): - if key in report and isinstance(report[key], list): - signal_eval = report[key] - break - - resid = _find_signal(signal_eval, "mom_12_1_resid") - grades = { - name: _grade_ic(_find_signal(signal_eval, name), resid) - for name in ("mom_12_1_sector_resid", "mom_12_1_sector_demeaned") - } - for name, g in grades.items(): - print( - f" {name}: promote_to_ab={g['promote_to_ab']} " - f"ic={((g.get('row') or {}).get('mean_ic'))} " - f"t={((g.get('row') or {}).get('ic_t_stat'))}" - ) - - ab_results: dict[str, Any] | None = None - promote_names = [n for n, g in grades.items() if g.get("promote_to_ab")] - run_ab = (bool(promote_names) or args.force_ab) and not args.skip_ab - if run_ab: - # Prefer sector_resid if both; else the one that promoted / force resid. - if "mom_12_1_sector_resid" in promote_names or ( - args.force_ab and not promote_names - ): - ab_signal = "mom_12_1_sector_resid" - else: - ab_signal = promote_names[0] - print(f"Running portfolio A/B for {ab_signal}…") - ab_results = await _run_ab( - snapshot, - sector_map=mapping, - signal_name=ab_signal, - sector_cap=args.sector_cap, - quiet=args.quiet, - workers=args.workers, - ) - print( - f" A/B promote={ab_results.get('promotion', {}).get('promote')} " - f"— {ab_results.get('promotion', {}).get('reason')}" - ) - else: - print("Skipping portfolio A/B (no IC promotion; use --force-ab to override).") - - # Verdict - if ab_results and ab_results.get("promotion", {}).get("promote"): - verdict = "PROMOTE" - detail = ( - f"{ab_results['signal']} cleared IC + A/B bars. " - "Human must design wire-in; do not ship from this branch." - ) - human = ( - "- Approve or reject production residual swap vs dual-signal design.\n" - "- If sector-cap arm ran, review tail-trim diagnostics before any cap." - ) - elif any(g.get("promote_to_ab") for g in grades.values()): - verdict = "PARK" - detail = ( - "IC promotion bar cleared but A/B did not promote " - "(or A/B skipped). Park for human review." - ) - human = "- Inspect A/B windows; decide whether to re-run or park." - elif any( - (g.get("row") or {}).get("mean_ic") is not None - and abs(float((g.get("row") or {}).get("mean_ic") or 0)) >= IRON_IC_BAR * 0.5 - for g in grades.values() - ): - verdict = "PARK" - detail = "Weak / partial IC — not dead, not green. Machinery kept." - human = "- No book change. Revisit after history-depth extension (Task 3)." - else: - verdict = "DEAD" - detail = ( - "Neither sector residual nor sector demean cleared the iron-rule bar " - "with t ≥ mom_12_1_resid on this window." - ) - human = "- Do not wire sector residual. Optional: re-check after Task 3 depth." - - stamp = datetime.now().strftime("%Y%m%d-%H%M%S") - out_path = Path(args.out) if args.out else Path("reports") / f"sector-residual-{stamp}.json" - payload = { - "generated_at": datetime.now().isoformat(), - "snapshot_guard": guard, - "sector_coverage": cov, - "sector_map_path": str(sector_map_path.resolve()), - "signal_eval": signal_eval, - "ic_grades": grades, - "portfolio_ab": ab_results, - "verdict": verdict, - "verdict_detail": detail, - "human_next": human, - "report_path": str(out_path.as_posix()), - "pre_registration": { - "iron_ic_bar": IRON_IC_BAR, - "validation_split": VALIDATION_SPLIT.isoformat(), - "fill_mode": "close", - "cost_per_side": 0.001, - "ab_rule": "val Sharpe >= control - 0.5*SE; full Sharpe & maxDD not worse", - }, - } - out_path.parent.mkdir(parents=True, exist_ok=True) - out_path.write_text(json.dumps(payload, indent=2, default=str) + "\n", encoding="utf-8") - - md_path = Path("docs/research/sector-residual-momentum.md") - _write_md(md_path, payload) - # Companion md under reports/ - md_report = out_path.with_suffix(".md") - md_report.write_text(md_path.read_text(encoding="utf-8"), encoding="utf-8") - - print(f"Verdict: {verdict}") - print(f"Wrote {out_path}") - print(f"Wrote {md_path}") - - -if __name__ == "__main__": - asyncio.run(_main()) diff --git a/scripts/run_tier1_macbook.sh b/scripts/run_tier1_macbook.sh index 584eb4a..3254e4e 100755 --- a/scripts/run_tier1_macbook.sh +++ b/scripts/run_tier1_macbook.sh @@ -1,99 +1,70 @@ #!/usr/bin/env bash -# Tier-1 alpha research runner for a high-CPU MacBook (local only). +# Research helpers for a high-CPU MacBook (local only). # -# Prerequisites -# - git checkout research/earnings-gap-and-sue (or later research branch) -# - .env with ALPACA_* (required for OHLCV/ETFs); FMP_* for earnings resume; -# optional ALPHA_VANTAGE_* as earnings fallback -# - Python venv with project deps installed -# - backtest_snapshots/prod.sqlite present (gitignored — copy or rebuild) +# Kept after Tier-1 cleanup: +# --ssl-check diagnose corporate CA / proxy +# --earnings-only resume FMP earnings backfill + 2a/2b (parked) +# --prod-book-matrix re-run 505 vs liquid universe × horizon book matrix +# +# Prerequisites: git checkout research branch, .env, deep research.sqlite for +# book matrix, combined-ca-bundle.pem or certifi when on corp network. # -# Usage # chmod +x scripts/run_tier1_macbook.sh -# ./scripts/run_tier1_macbook.sh # coverage + deep rebuild + harness -# ./scripts/run_tier1_macbook.sh --all # earnings resume + full depth pipeline -# ./scripts/run_tier1_macbook.sh --earnings-only # multi-day FMP backfill + re-run 2a/2b -# ./scripts/run_tier1_macbook.sh --harness-only # skip rebuild; race-guard + IC only -# ./scripts/run_tier1_macbook.sh --coverage-only # bars-per-year probe only -# ./scripts/run_tier1_macbook.sh --sector-resid-deep # deepen shallow + ONE masked grade -# ./scripts/run_tier1_macbook.sh --prod-book-matrix # 4-arm universe×horizon book matrix -# -# Does NOT touch production Postgres, scheduler, gates, or prod config. +# ./scripts/run_tier1_macbook.sh --ssl-check +# ./scripts/run_tier1_macbook.sh --prod-book-matrix set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$ROOT" -# --- defaults (override via flags or env) --- -PROD_SNAP="${PROD_SNAP:-backtest_snapshots/prod.sqlite}" RESEARCH_SNAP="${RESEARCH_SNAP:-backtest_snapshots/research.sqlite}" -HISTORY_DAYS="${HISTORY_DAYS:-5000}" -MIN_BARS="${MIN_BARS:-260}" +PROD_SNAP="${PROD_SNAP:-backtest_snapshots/prod.sqlite}" WORKERS="${WORKERS:-8}" -ALPACA_SLEEP="${ALPACA_SLEEP:-0.15}" FMP_LIMIT="${FMP_LIMIT:-250}" FMP_SLEEP="${FMP_SLEEP:-0.35}" PYTHON="${PYTHON:-python3}" USE_CORP_PROXY="${USE_CORP_PROXY:-0}" - -PHASE="depth" # depth | all | earnings | harness | coverage | ssl | sector-resid-deep | prod-book +PHASE="" usage() { - sed -n '2,25p' "$0" | sed 's/^# \?//' - cat <<'EOF' - -SSL / network (corporate MacBook) - SSL errors usually mean the corp root CA is missing from Python. - 1) Put combined-ca-bundle.pem in the repo root OR $HOME - 2) Or: export SSL_CERT_FILE=/path/to/combined-ca-bundle.pem - 3) Behind corp proxy: USE_CORP_PROXY=1 ./scripts/run_tier1_macbook.sh - 4) Diagnose: ./scripts/run_tier1_macbook.sh --ssl-check -EOF + sed -n '2,16p' "$0" | sed 's/^# \?//' exit "${1:-0}" } while [[ $# -gt 0 ]]; do case "$1" in - --all) PHASE=all; shift ;; - --earnings-only) PHASE=earnings; shift ;; - --harness-only) PHASE=harness; shift ;; - --coverage-only) PHASE=coverage; shift ;; - --depth) PHASE=depth; shift ;; --ssl-check) PHASE=ssl; shift ;; - --sector-resid-deep) PHASE=sector_resid_deep; shift ;; + --earnings-only) PHASE=earnings; shift ;; --prod-book-matrix) PHASE=prod_book; shift ;; --corp-proxy) USE_CORP_PROXY=1; shift ;; - --prod-snap) PROD_SNAP="$2"; shift 2 ;; - --research-snap) RESEARCH_SNAP="$2"; shift 2 ;; - --history-days) HISTORY_DAYS="$2"; shift 2 ;; --workers) WORKERS="$2"; shift 2 ;; - --fmp-limit) FMP_LIMIT="$2"; shift 2 ;; --python) PYTHON="$2"; shift 2 ;; -h|--help) usage 0 ;; *) echo "Unknown flag: $1" >&2; usage 1 ;; esac done +if [[ -z "$PHASE" ]]; then + echo "Pick a phase: --ssl-check | --earnings-only | --prod-book-matrix" >&2 + usage 1 +fi + if [[ -x .venv/bin/python ]]; then PYTHON=".venv/bin/python" elif command -v "$PYTHON" >/dev/null 2>&1; then : else - echo "ERROR: no Python found (tried .venv/bin/python and $PYTHON)" >&2 + echo "ERROR: no Python found" >&2 exit 1 fi log() { printf '\n==> %s\n' "$*"; } die() { echo "ERROR: $*" >&2; exit 1; } +need_file() { [[ -f "$1" ]] || die "missing $1"; } -# --------------------------------------------------------------------------- -# TLS bootstrap — same corp CA path the FastAPI app uses -# --------------------------------------------------------------------------- setup_ssl() { export USE_CORP_PROXY - - # Prefer explicit env, then repo / home corporate bundle, then certifi. if [[ -z "${SSL_CERT_FILE:-}" ]]; then if [[ -f "$ROOT/combined-ca-bundle.pem" ]]; then export SSL_CERT_FILE="$ROOT/combined-ca-bundle.pem" @@ -101,191 +72,68 @@ setup_ssl() { export SSL_CERT_FILE="$HOME/combined-ca-bundle.pem" fi fi - if [[ -n "${SSL_CERT_FILE:-}" && -f "$SSL_CERT_FILE" ]]; then - export REQUESTS_CA_BUNDLE="$SSL_CERT_FILE" - export CURL_CA_BUNDLE="$SSL_CERT_FILE" + export REQUESTS_CA_BUNDLE="$SSL_CERT_FILE" CURL_CA_BUNDLE="$SSL_CERT_FILE" log "SSL CA bundle: $SSL_CERT_FILE" else - # Fall back to certifi if installed local certifi_path certifi_path="$("$PYTHON" -c 'import certifi; print(certifi.where())' 2>/dev/null || true)" if [[ -n "$certifi_path" && -f "$certifi_path" ]]; then - export SSL_CERT_FILE="$certifi_path" - export REQUESTS_CA_BUNDLE="$certifi_path" - export CURL_CA_BUNDLE="$certifi_path" + export SSL_CERT_FILE="$certifi_path" REQUESTS_CA_BUNDLE="$certifi_path" CURL_CA_BUNDLE="$certifi_path" log "SSL CA bundle (certifi): $SSL_CERT_FILE" else log "WARNING: no CA bundle found — SSL may fail on corp networks" - log " Copy combined-ca-bundle.pem to $ROOT/ or \$HOME/" - log " Or: export SSL_CERT_FILE=/path/to/combined-ca-bundle.pem" fi fi - if [[ "$USE_CORP_PROXY" == "1" ]]; then export HTTP_PROXY="${HTTP_PROXY:-http://aproxy.corproot.net:8080}" export HTTPS_PROXY="${HTTPS_PROXY:-http://aproxy.corproot.net:8080}" - export NO_PROXY="${NO_PROXY:-corproot.net,sharedtcs.net,127.0.0.1,localhost,bix.swisscom.com,swisscom.com}" + export NO_PROXY="${NO_PROXY:-corproot.net,sharedtcs.net,127.0.0.1,localhost}" export http_proxy="$HTTP_PROXY" https_proxy="$HTTPS_PROXY" no_proxy="$NO_PROXY" log "Corp proxy enabled: $HTTPS_PROXY" fi - - # Ensure Python process sees the same bootstrap (patches ssl for alpaca-py). export PYTHONPATH="${ROOT}${PYTHONPATH:+:$PYTHONPATH}" } ssl_check() { setup_ssl - log "SSL diagnostic" "$PYTHON" - <<'PY' from app.ssl_bootstrap import bootstrap_ssl, ssl_status -import json -import urllib.request - -ca = bootstrap_ssl() +import json, urllib.request print(json.dumps(ssl_status(), indent=2)) -print("bootstrap_ssl ->", ca) -urls = [ +print("bootstrap ->", bootstrap_ssl()) +for url in ( "https://data.alpaca.markets/v2/stocks/SPY/bars?timeframe=1Day&limit=1", "https://financialmodelingprep.com/stable/profile?symbol=AAPL", - "https://www.alphavantage.co/query?function=TIME_SERIES_DAILY&symbol=IBM", -] -for url in urls: +): try: - req = urllib.request.Request(url, headers={"User-Agent": "signal-platform-ssl-check"}) + req = urllib.request.Request(url, headers={"User-Agent": "ssl-check"}) with urllib.request.urlopen(req, timeout=20) as resp: - print(f"OK {resp.status} {url[:60]}...") + print(f"OK {resp.status} {url[:60]}") except Exception as exc: print(f"FAIL {type(exc).__name__}: {exc}") - print(f" {url[:80]}") PY } -need_file() { - [[ -f "$1" ]] || die "missing $1" -} - -require_prod() { - need_file "$PROD_SNAP" -} - -run_earnings() { - require_prod - log "Earnings backfill (FMP free tier ~${FMP_LIMIT}/day; resume-safe)" - "$PYTHON" scripts/backfill_earnings_events.py \ - --snapshot "$PROD_SNAP" \ - --provider fmp \ - --force-symbol \ - --limit "$FMP_LIMIT" \ - --sleep "$FMP_SLEEP" - - log "Earnings research 2a+2b (report only; no filters shipped)" - "$PYTHON" scripts/run_earnings_research.py \ - --snapshot "$PROD_SNAP" \ - --workers "$WORKERS" \ - --allow-spawn -} - -run_coverage() { - require_prod - log "Coverage probe (bars per year) on $PROD_SNAP" - "$PYTHON" scripts/run_history_depth_research.py \ - --phase coverage \ - --snapshot "$PROD_SNAP" -} - -run_rebuild() { - require_prod - log "Deep rebuild $PROD_SNAP → $RESEARCH_SNAP (history-days=$HISTORY_DAYS)" - log "SURVIVORSHIP: today's constituents backfilled — relative IC only, not levels" - "$PYTHON" scripts/extend_snapshot_universe.py \ - --source "$PROD_SNAP" \ - --output "$RESEARCH_SNAP" \ - --force-copy \ - --history-days "$HISTORY_DAYS" \ - --min-bars "$MIN_BARS" \ - --sleep "$ALPACA_SLEEP" - - log "Refresh SPY + 11 sector ETFs on research snapshot" - "$PYTHON" scripts/fetch_sector_etfs_to_snapshot.py \ - --snapshot "$RESEARCH_SNAP" \ - --history-days "$HISTORY_DAYS" - - log "Also deepen sector ETFs on prod snapshot (for local A/B parity)" - "$PYTHON" scripts/fetch_sector_etfs_to_snapshot.py \ - --snapshot "$PROD_SNAP" \ - --history-days "$HISTORY_DAYS" -} - -run_harness() { - need_file "$RESEARCH_SNAP" - log "Race-guard + full signal harness + era split on $RESEARCH_SNAP" - "$PYTHON" scripts/run_history_depth_research.py \ - --phase harness \ - --snapshot "$RESEARCH_SNAP" \ - --workers "$WORKERS" \ - --allow-spawn -} - -run_sector_resid_deep() { - need_file "$RESEARCH_SNAP" - log "Sector-resid deep test: deepen shallow symbols + ONE liquid-1500 masked grade" - log "Pre-registered PASS/FAIL only — thread ends after this run" - "$PYTHON" scripts/run_sector_resid_deep_test.py \ - --snapshot "$RESEARCH_SNAP" \ - --history-days "$HISTORY_DAYS" \ - --sleep "$ALPACA_SLEEP" \ - --workers "$WORKERS" \ - --allow-spawn -} - -run_prod_book_matrix() { - need_file "$RESEARCH_SNAP" - log "Production book × universe × horizon (4 arms, strategy unchanged)" - "$PYTHON" scripts/run_prod_book_universe_matrix.py \ - --snapshot "$RESEARCH_SNAP" \ - --workers "$WORKERS" \ - --allow-spawn \ - --candidate-cache reports/.cache/prod-book-universe-cands.pkl -} - -log "cwd=$ROOT python=$PYTHON phase=$PHASE workers=$WORKERS" setup_ssl - case "$PHASE" in - ssl) - ssl_check - ;; - sector_resid_deep) - run_sector_resid_deep + ssl) ssl_check ;; + earnings) + need_file "$PROD_SNAP" + log "Earnings backfill + research (parked experiment)" + "$PYTHON" scripts/backfill_earnings_events.py \ + --snapshot "$PROD_SNAP" --provider fmp --force-symbol \ + --limit "$FMP_LIMIT" --sleep "$FMP_SLEEP" + "$PYTHON" scripts/run_earnings_research.py \ + --snapshot "$PROD_SNAP" --workers "$WORKERS" --allow-spawn ;; prod_book) - run_prod_book_matrix - ;; - coverage) - run_coverage - ;; - earnings) - run_earnings - ;; - harness) - run_harness - ;; - depth) - run_coverage - run_rebuild - run_harness - ;; - all) - run_earnings - run_coverage - run_rebuild - run_harness - ;; - *) - die "unknown phase $PHASE" + need_file "$RESEARCH_SNAP" + log "Production book universe × horizon matrix" + "$PYTHON" scripts/run_prod_book_universe_matrix.py \ + --snapshot "$RESEARCH_SNAP" --workers "$WORKERS" --allow-spawn \ + --candidate-cache reports/.cache/prod-book-universe-cands.pkl ;; + *) die "unknown phase $PHASE" ;; esac - -log "Done. Check reports/ and docs/research/history-depth-extension.md" -log "Commit reports on this machine if they look good, or copy them back to Windows." +log "Done." diff --git a/tests/unit/test_backtest_service.py b/tests/unit/test_backtest_service.py index b811a27..aff0eb6 100644 --- a/tests/unit/test_backtest_service.py +++ b/tests/unit/test_backtest_service.py @@ -100,75 +100,6 @@ def test_residual_momentum_removes_market_beta_but_keeps_specific_drift(): assert drift["mom_12_1_resid"] > pure["mom_12_1_resid"] + 0.12 -def test_sector_residual_momentum_two_factor(): - """Pure market+sector beta stock → sector resid ~0; idiosyncratic drift kept.""" - dates, pure_beta, highs, benchmark = _signal_test_series(extra_return=0.0) - # Sector ETF = leveraged market (collinear-ish but not identical). - sector = {d: benchmark[d] * 1.02 + 0.5 for d in dates} - # Stock with pure exposure to market + sector, no alpha. - closes = [100.0] - for i in range(1, len(dates)): - m_prev = benchmark[dates[i - 1]] - m_cur = benchmark[dates[i]] - s_prev = sector[dates[i - 1]] - s_cur = sector[dates[i]] - m_ret = m_cur / m_prev - 1.0 - s_ret = s_cur / s_prev - 1.0 - closes.append(closes[-1] * (1.0 + 0.7 * m_ret + 0.5 * s_ret)) - highs_p = [c * 1.01 for c in closes] - - pure = bt._signal_values( - dates, closes, highs_p, 260, benchmark, sector_etf_closes=sector - ) - assert "mom_12_1_sector_resid" in pure - assert pure["mom_12_1_sector_resid"] == pytest.approx(0.0, abs=0.05) - - # Add idiosyncratic drift — sector residual should keep it. - drift_closes = [100.0] - for i in range(1, len(dates)): - m_prev = benchmark[dates[i - 1]] - m_cur = benchmark[dates[i]] - s_prev = sector[dates[i - 1]] - s_cur = sector[dates[i]] - m_ret = m_cur / m_prev - 1.0 - s_ret = s_cur / s_prev - 1.0 - drift_closes.append( - drift_closes[-1] * (1.0 + 0.7 * m_ret + 0.5 * s_ret + 0.0008) - ) - drift_highs = [c * 1.01 for c in drift_closes] - drift = bt._signal_values( - dates, drift_closes, drift_highs, 260, benchmark, sector_etf_closes=sector - ) - assert drift["mom_12_1_sector_resid"] > pure["mom_12_1_sector_resid"] + 0.10 - - -def test_inject_sector_demeaned_momentum(): - collected = { - "mom_12_1": { - (2024, 1): [ - {"val": 0.20, "fwd": 0.01, "symbol": "AAA"}, - {"val": 0.10, "fwd": 0.02, "symbol": "BBB"}, - {"val": 0.40, "fwd": -0.01, "symbol": "CCC"}, - {"val": 0.00, "fwd": 0.03, "symbol": "DDD"}, - ] - } - } - symbol_to_sector = { - "AAA": "Information Technology", - "BBB": "Information Technology", - "CCC": "Energy", - "DDD": "Energy", - } - bt._inject_sector_demeaned_momentum(collected, symbol_to_sector) - dem = collected["mom_12_1_sector_demeaned"][(2024, 1)] - by_sym = {r["symbol"]: r["val"] for r in dem} - # IT mean = 0.15 → AAA +0.05, BBB -0.05; Energy mean = 0.20 → CCC +0.20, DDD -0.20 - assert by_sym["AAA"] == pytest.approx(0.05) - assert by_sym["BBB"] == pytest.approx(-0.05) - assert by_sym["CCC"] == pytest.approx(0.20) - assert by_sym["DDD"] == pytest.approx(-0.20) - - def test_assigns_raw_and_residual_percentiles_independently(): cands = [ {"iso_week": (2026, 1), "momentum": 0.10, "residual_momentum": 0.30},