diff --git a/app/services/backtest_service.py b/app/services/backtest_service.py index 55280db..1c06c87 100644 --- a/app/services/backtest_service.py +++ b/app/services/backtest_service.py @@ -791,86 +791,34 @@ def _residual_momentum_12_1( with an intercept estimated over the same window, the arithmetic residuals sum to ~zero by construction, which would destroy the signal. """ - return _multi_factor_residual_momentum_12_1( - dates, closes, i, [benchmark_closes] if benchmark_closes else None - ) - - -def _multi_factor_residual_momentum_12_1( - dates: list[date], - closes: list[float], - i: int, - factor_closes: list[dict[date, float]] | None, -) -> float | None: - """12-1 residual momentum vs one or more factors (OLS, no intercept). - - Same formation window as raw / single-factor residual momentum: - daily returns from close[i-252] → close[i-21], require ≥100 paired obs. - Factors are stacked as columns; betas are OLS without intercept so the - cumulative residual is not forced to zero. - """ - if not factor_closes or i - 252 < 0: - return None - n_factors = len(factor_closes) - if n_factors < 1: + if not benchmark_closes or i - 252 < 0: return None stock_rets: list[float] = [] - factor_rets: list[list[float]] = [[] for _ in range(n_factors)] + market_rets: list[float] = [] + # Same daily intervals as mom_12_1: close[i-252] -> close[i-21]. for k in range(i - 251, i - 20): prev_close = closes[k - 1] - if prev_close <= 0: - continue - f_day: list[float] = [] - ok = True - for fc in factor_closes: - f_prev = fc.get(dates[k - 1]) - f_cur = fc.get(dates[k]) - if f_prev is None or f_cur is None or f_prev <= 0: - ok = False - break - f_day.append(f_cur / f_prev - 1.0) - if not ok: + bench_prev = benchmark_closes.get(dates[k - 1]) + bench_cur = benchmark_closes.get(dates[k]) + if prev_close <= 0 or bench_prev is None or bench_cur is None or bench_prev <= 0: continue stock_rets.append(closes[k] / prev_close - 1.0) - for j, r in enumerate(f_day): - factor_rets[j].append(r) + market_rets.append(bench_cur / bench_prev - 1.0) - n = len(stock_rets) - if n < 100: + if len(stock_rets) < 100: return None - - if n_factors == 1: - # Fast path: identical algebra to the historical single-factor form. - market_rets = factor_rets[0] - mean_market = sum(market_rets) / n - mean_stock = sum(stock_rets) / n - var_market = sum((x - mean_market) ** 2 for x in market_rets) - if var_market <= 0: - return None - cov = sum( - (stock_rets[k] - mean_stock) * (market_rets[k] - mean_market) - for k in range(n) - ) - beta = cov / var_market - return sum(stock_rets[k] - beta * market_rets[k] for k in range(n)) - - # OLS without intercept: β = (X'X)^{-1} X'y for X columns = factor returns. - # Implemented for exactly two factors (market + sector); refuse larger. - if n_factors != 2: + mean_market = sum(market_rets) / len(market_rets) + mean_stock = sum(stock_rets) / len(stock_rets) + var_market = sum((x - mean_market) ** 2 for x in market_rets) + if var_market <= 0: return None - f1, f2 = factor_rets[0], factor_rets[1] - s11 = sum(a * a for a in f1) - s22 = sum(a * a for a in f2) - s12 = sum(f1[k] * f2[k] for k in range(n)) - sy1 = sum(stock_rets[k] * f1[k] for k in range(n)) - sy2 = sum(stock_rets[k] * f2[k] for k in range(n)) - det = s11 * s22 - s12 * s12 - if abs(det) < 1e-18: - return None - b1 = (s22 * sy1 - s12 * sy2) / det - b2 = (s11 * sy2 - s12 * sy1) / det - return sum(stock_rets[k] - b1 * f1[k] - b2 * f2[k] for k in range(n)) + cov = sum( + (stock_rets[k] - mean_stock) * (market_rets[k] - mean_market) + for k in range(len(stock_rets)) + ) + beta = cov / var_market + return sum(stock_rets[k] - beta * market_rets[k] for k in range(len(stock_rets))) def _realized_vol_6m(closes: list[float], i: int) -> float | None: @@ -895,7 +843,6 @@ def _signal_values( highs: list[float], i: int, benchmark_closes: dict[date, float] | None = None, - sector_etf_closes: dict[date, float] | None = None, ) -> dict[str, float]: """Point-in-time candidate signals at as-of index ``i`` (price-only). @@ -907,11 +854,6 @@ def _signal_values( higher = nearer the high, expect positive IC). ``vol_6m`` is 126-day realized volatility (expect negative IC if the low-volatility anomaly holds). ``fip_id`` is Da/Gurun/Warachka information discreteness (expect negative IC). - - When ``sector_etf_closes`` is supplied (research path), also emit - ``mom_12_1_sector_resid``: two-factor residual vs SPY + sector ETF. - Cross-sectional ``mom_12_1_sector_demeaned`` is injected later from the - full weekly cross-section (cannot be computed per-ticker alone). """ out: dict[str, float] = {} if i - 252 >= 0 and closes[i - 252] > 0: @@ -919,12 +861,6 @@ def _signal_values( residual = _residual_momentum_12_1(dates, closes, i, benchmark_closes) if residual is not None: out["mom_12_1_resid"] = residual - if benchmark_closes and sector_etf_closes: - sector_resid = _multi_factor_residual_momentum_12_1( - dates, closes, i, [benchmark_closes, sector_etf_closes] - ) - if sector_resid is not None: - out["mom_12_1_sector_resid"] = sector_resid fip = _fip_id(closes, i) if fip is not None: out["fip_id"] = fip @@ -1011,16 +947,14 @@ def _accumulate_signal_series( benchmark_closes: dict[date, float] | None = None, *, symbol: str | None = None, - sector_etf_closes: dict[date, float] | None = None, ) -> None: """For each weekly as-of bar, emit (signal, forward-return) pairs keyed by ISO week into ``collected[name][week_key]``. Forward return is close-to-close over HORIZON trading days. Mutates ``collected`` (a dict of dict of list). When ``BACKTEST_LIQUID_BREADTH`` is set, observations are dicts with PIT - liquidity fields for the mask. When ``symbol`` is provided, observations are - also dicts (so sector demeaning can group by name); otherwise plain - ``(val, fwd)`` tuples keep the production path unchanged. + liquidity fields for the mask; otherwise plain ``(val, fwd)`` tuples so the + production signal path stays unchanged. """ n = len(records) if n < HORIZON + 21: @@ -1030,7 +964,6 @@ def _accumulate_signal_series( volumes = [float(getattr(r, "volume", 0) or 0) for r in records] dates = [r.date for r in records] liquid_mode = _liquid_breadth_top_n() > 0 - rich = liquid_mode or symbol is not None for i in _weekly_asof_indices(records): j = i + HORIZON if j >= n or closes[i] <= 0: @@ -1039,79 +972,19 @@ def _accumulate_signal_series( iso = records[i].date.isocalendar() week_key = (iso[0], iso[1]) dvol = _median_dollar_vol_63(closes, volumes, i) if liquid_mode else None - for name, val in _signal_values( - dates, closes, highs, i, benchmark_closes, sector_etf_closes - ).items(): - if rich: - row = { + for name, val in _signal_values(dates, closes, highs, i, benchmark_closes).items(): + if liquid_mode: + collected[name][week_key].append({ "val": val, "fwd": fwd, + "close": closes[i], + "median_dvol_63": dvol, "symbol": symbol, - } - if liquid_mode: - row["close"] = closes[i] - row["median_dvol_63"] = dvol - collected[name][week_key].append(row) + }) else: collected[name][week_key].append((val, fwd)) -def _inject_sector_demeaned_momentum( - collected: dict, - symbol_to_sector: dict[str, str], - *, - min_sector_names: int = 2, -) -> None: - """Cross-sectional demean of ``mom_12_1`` within GICS sector per week. - - ``mom_12_1_sector_demeaned[i] = mom_12_1[i] − mean(mom_12_1 | sector_i)``. - Requires rich observations with a ``symbol`` field (research path). Names - without a sector label, or sectors with fewer than ``min_sector_names`` - members that week, are dropped from the demeaned series. - """ - if not symbol_to_sector or "mom_12_1" not in collected: - return - from app.services.sector_map import normalise_symbol - - demeaned: dict = defaultdict(list) - for week_key, recs in collected["mom_12_1"].items(): - parsed: list[tuple[str, float, float, object]] = [] - by_sector: dict[str, list[float]] = defaultdict(list) - for rec in recs: - pair = _obs_val_fwd(rec) - if pair is None: - continue - val, fwd = pair - if isinstance(rec, dict): - sym = rec.get("symbol") - else: - sym = None - if not sym: - continue - sector = symbol_to_sector.get(normalise_symbol(str(sym))) - if not sector: - continue - parsed.append((sector, val, fwd, rec)) - by_sector[sector].append(val) - means = { - sec: sum(vs) / len(vs) - for sec, vs in by_sector.items() - if len(vs) >= min_sector_names - } - for sector, val, fwd, rec in parsed: - if sector not in means: - continue - dval = val - means[sector] - if isinstance(rec, dict): - row = dict(rec) - row["val"] = dval - demeaned[week_key].append(row) - else: - demeaned[week_key].append((dval, fwd)) - if demeaned: - collected["mom_12_1_sector_demeaned"] = demeaned - - def _rank(xs: list[float]) -> list[float]: """Average (tie-corrected) ranks, 1-based.""" order = sorted(range(len(xs)), key=lambda k: xs[k]) @@ -1388,38 +1261,14 @@ def _signal_series( benchmark_closes: dict[date, float] | None = None, *, symbol: str | None = None, - sector_etf_closes: dict[date, float] | None = None, ) -> dict: """Per-ticker signal/forward-return series as a PLAIN (picklable) nested dict — no defaultdict/lambda — so it can cross a process boundary.""" tmp: dict = defaultdict(lambda: defaultdict(list)) - _accumulate_signal_series( - records, - tmp, - benchmark_closes, - symbol=symbol, - sector_etf_closes=sector_etf_closes, - ) + _accumulate_signal_series(records, tmp, benchmark_closes, symbol=symbol) return {name: dict(weeks) for name, weeks in tmp.items()} -def _sector_etf_closes_for_symbol( - symbol: str, - symbol_to_sector: dict[str, str] | None, - sector_etf_closes: dict[str, dict[date, float]] | None, -) -> dict[date, float] | None: - """Resolve the sector-ETF close series for one ticker, or None.""" - if not symbol_to_sector or not sector_etf_closes: - return None - from app.services.sector_map import etf_for_symbol - - etf = etf_for_symbol(symbol, symbol_to_sector) - if not etf: - return None - series = sector_etf_closes.get(etf) - return series or None - - def _replay_and_signals( symbol: str, columns: tuple, @@ -1429,8 +1278,6 @@ def _replay_and_signals( target_model: str = PRODUCTION_GTL_TARGET_MODEL, cadence: str = DEFAULT_BACKTEST_CADENCE, signal_only: bool = False, - sector_etf_closes: dict[str, dict[date, float]] | None = None, - symbol_to_sector: dict[str, str] | None = None, ) -> tuple[list[dict], dict]: """The CPU-bound per-ticker work, as a top-level (picklable) function so it can run in a worker process. Takes primitive column arrays (cheap to pickle), @@ -1457,17 +1304,9 @@ def _replay_and_signals( target_model, cadence, ) - etf_closes = _sector_etf_closes_for_symbol( - symbol, symbol_to_sector, sector_etf_closes - ) return ( candidates, - _signal_series( - bars, - benchmark_closes, - symbol=symbol, - sector_etf_closes=etf_closes, - ), + _signal_series(bars, benchmark_closes, symbol=symbol), ) @@ -4218,41 +4057,6 @@ async def run_backtest( except Exception: logger.exception("Benchmark load for residual momentum failed") - # Optional sector residualisation (research): local ticker→sector map + sector - # ETF closes stored in benchmark_prices. Absent map/series → no sector signals. - symbol_to_sector: dict[str, str] = {} - sector_etf_closes: dict[str, dict[date, float]] = {} - try: - from app.services.sector_map import ( - SECTOR_ETFS, - load_ticker_sector_map, - normalise_symbol, - ) - from app.services.benchmark_service import load_benchmark_closes - - map_path = os.getenv("BACKTEST_SECTOR_MAP_PATH", "").strip() or None - symbol_to_sector = { - normalise_symbol(k): v - for k, v in load_ticker_sector_map(map_path).items() - } - if symbol_to_sector: - for etf in SECTOR_ETFS: - try: - series = await load_benchmark_closes(db, etf) - except Exception: - series = {} - if series: - sector_etf_closes[etf] = series - logger.info(json.dumps({ - "event": "backtest_sector_context_loaded", - "sector_map_size": len(symbol_to_sector), - "sector_etfs_loaded": sorted(sector_etf_closes), - })) - except Exception: - logger.exception("Sector residual context load failed; continuing without") - symbol_to_sector = {} - sector_etf_closes = {} - def _merge(result: tuple[list[dict], dict]) -> None: cands, series = result candidates.extend(cands) @@ -4303,8 +4107,6 @@ async def run_backtest( target_model, cadence, ticker.symbol in rank_only_symbols, - sector_etf_closes or None, - symbol_to_sector or None, )) for result in await asyncio.gather(*futures, return_exceptions=True): if isinstance(result, Exception): @@ -4333,8 +4135,6 @@ async def run_backtest( target_model, cadence, ticker.symbol in rank_only_symbols, - sector_etf_closes or None, - symbol_to_sector or None, )) except Exception: logger.exception("Backtest replay failed for %s", ticker.symbol) @@ -4342,13 +4142,6 @@ async def run_backtest( if progress_cb is not None and total: progress_cb(total, total, "") - # Cross-sectional sector demean needs the full weekly universe. - if symbol_to_sector: - try: - _inject_sector_demeaned_momentum(collected, symbol_to_sector) - except Exception: - logger.exception("Sector demeaned momentum injection failed") - # Cross-sectional momentum: rank every week's universe, then "qualified" means # floors + top ``min_momentum_percentile`` by promoted residual 12-1 momentum # (raw 12-1 fallback only when benchmark data is unavailable). diff --git a/app/services/sector_map.py b/app/services/sector_map.py deleted file mode 100644 index f72230c..0000000 --- a/app/services/sector_map.py +++ /dev/null @@ -1,145 +0,0 @@ -"""Ticker → GICS sector → SPDR sector ETF mapping (research only). - -Sector residual momentum residualizes 12-1 momentum against SPY and the name's -sector ETF. Labels are persisted under ``data/research/ticker_sector_map.json`` -so research runs do not depend on live FMP calls. -""" - -from __future__ import annotations - -import json -from pathlib import Path -from typing import Any - -# Eleven SPDR sector ETFs. Auxiliary series only — never tradable book members. -SECTOR_ETFS: tuple[str, ...] = ( - "XLB", - "XLC", - "XLE", - "XLF", - "XLI", - "XLK", - "XLP", - "XLRE", - "XLU", - "XLV", - "XLY", -) - -# GICS sector name (and common aliases) → SPDR ETF. -# Keys are lower-case for matching. -GICS_SECTOR_TO_ETF: dict[str, str] = { - "materials": "XLB", - "basic materials": "XLB", - "communication services": "XLC", - "communications": "XLC", - "telecommunication services": "XLC", - "energy": "XLE", - "financials": "XLF", - "financial services": "XLF", - "financial": "XLF", - "industrials": "XLI", - "industrial goods": "XLI", - "information technology": "XLK", - "technology": "XLK", - "consumer staples": "XLP", - "consumer defensive": "XLP", - "real estate": "XLRE", - "utilities": "XLU", - "health care": "XLV", - "healthcare": "XLV", - "consumer discretionary": "XLY", - "consumer cyclical": "XLY", -} - -DEFAULT_SECTOR_MAP_PATH = Path("data/research/ticker_sector_map.json") - - -def normalise_symbol(symbol: str) -> str: - """Alpaca-style symbols: BRK.B / BRK/B → BRK-B.""" - s = str(symbol or "").strip().upper() - s = s.replace(".", "-").replace("/", "-") - return s - - -def sector_to_etf(sector: str | None) -> str | None: - if not sector: - return None - return GICS_SECTOR_TO_ETF.get(str(sector).strip().lower()) - - -def etf_for_symbol(symbol: str, symbol_to_sector: dict[str, str]) -> str | None: - sector = symbol_to_sector.get(normalise_symbol(symbol)) - return sector_to_etf(sector) - - -def load_ticker_sector_map(path: Path | str | None = None) -> dict[str, str]: - """Load ``{symbol: gics_sector}`` from JSON. Empty dict if missing.""" - p = Path(path) if path is not None else DEFAULT_SECTOR_MAP_PATH - if not p.exists(): - return {} - raw = json.loads(p.read_text(encoding="utf-8")) - if not isinstance(raw, dict): - return {} - out: dict[str, str] = {} - # Accept either flat map or {"map": {...}, "meta": ...} - payload = raw.get("map") if "map" in raw and isinstance(raw.get("map"), dict) else raw - if not isinstance(payload, dict): - return {} - for sym, sector in payload.items(): - if sym in ("meta", "schema_version", "map"): - continue - if sector is None: - continue - ns = normalise_symbol(str(sym)) - if ns: - out[ns] = str(sector).strip() - return out - - -def save_ticker_sector_map( - mapping: dict[str, str], - path: Path | str | None = None, - *, - meta: dict[str, Any] | None = None, -) -> Path: - p = Path(path) if path is not None else DEFAULT_SECTOR_MAP_PATH - p.parent.mkdir(parents=True, exist_ok=True) - # Normalise keys on write. - clean = { - normalise_symbol(k): str(v).strip() - for k, v in mapping.items() - if k and v and normalise_symbol(k) - } - payload: dict[str, Any] = { - "schema_version": 1, - "map": clean, - "meta": meta or {}, - } - p.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") - return p - - -def coverage_stats( - symbols: list[str], mapping: dict[str, str] -) -> dict[str, Any]: - total = len(symbols) - mapped = [s for s in symbols if normalise_symbol(s) in mapping] - with_etf = [ - s - for s in mapped - if sector_to_etf(mapping[normalise_symbol(s)]) is not None - ] - missing = [s for s in symbols if normalise_symbol(s) not in mapping] - by_sector: dict[str, int] = {} - for s in mapped: - sec = mapping[normalise_symbol(s)] - by_sector[sec] = by_sector.get(sec, 0) + 1 - return { - "universe": total, - "mapped": len(mapped), - "mapped_pct": round(100.0 * len(mapped) / total, 1) if total else 0.0, - "with_etf": len(with_etf), - "missing": missing, - "by_sector": dict(sorted(by_sector.items(), key=lambda kv: (-kv[1], kv[0]))), - } diff --git a/data/research/ticker_sector_map.json b/data/research/ticker_sector_map.json deleted file mode 100644 index 31cc0db..0000000 --- a/data/research/ticker_sector_map.json +++ /dev/null @@ -1,543 +0,0 @@ -{ - "map": { - "A": "Health Care", - "AAPL": "Information Technology", - "ABBV": "Health Care", - "ABNB": "Consumer Discretionary", - "ABT": "Health Care", - "ACGL": "Financials", - "ACN": "Information Technology", - "ADBE": "Information Technology", - "ADI": "Information Technology", - "ADM": "Consumer Staples", - "ADP": "Industrials", - "ADSK": "Information Technology", - "AEE": "Utilities", - "AEP": "Utilities", - "AES": "Utilities", - "AFL": "Financials", - "AIG": "Financials", - "AIZ": "Financials", - "AJG": "Financials", - "AKAM": "Information Technology", - "ALB": "Materials", - "ALGN": "Health Care", - "ALL": "Financials", - "ALLE": "Industrials", - "AMAT": "Information Technology", - "AMCR": "Materials", - "AMD": "Information Technology", - "AME": "Industrials", - "AMGN": "Health Care", - "AMP": "Financials", - "AMT": "Real Estate", - "AMZN": "Consumer Discretionary", - "ANET": "Information Technology", - "AON": "Financials", - "AOS": "Industrials", - "APA": "Energy", - "APD": "Materials", - "APH": "Information Technology", - "APO": "Financials", - "APP": "Information Technology", - "APTV": "Consumer Discretionary", - "ARE": "Real Estate", - "ARES": "Financials", - "ATO": "Utilities", - "AVB": "Real Estate", - "AVGO": "Information Technology", - "AVY": "Materials", - "AWK": "Utilities", - "AXON": "Industrials", - "AXP": "Financials", - "AZO": "Consumer Discretionary", - "BA": "Industrials", - "BAC": "Financials", - "BALL": "Materials", - "BAX": "Health Care", - "BBY": "Consumer Discretionary", - "BDX": "Health Care", - "BEN": "Financials", - "BF-B": "Consumer Staples", - "BG": "Consumer Staples", - "BIIB": "Health Care", - "BK": "Financial Services", - "BKNG": "Consumer Discretionary", - "BKR": "Energy", - "BLDR": "Industrials", - "BLK": "Financials", - "BMY": "Health Care", - "BR": "Industrials", - "BRK-B": "Financials", - "BRO": "Financials", - "BSX": "Health Care", - "BX": "Financials", - "BXP": "Real Estate", - "C": "Financials", - "CAG": "Consumer Defensive", - "CAH": "Health Care", - "CARR": "Industrials", - "CASY": "Consumer Staples", - "CAT": "Industrials", - "CB": "Financials", - "CBOE": "Financials", - "CBRE": "Real Estate", - "CCI": "Real Estate", - "CCL": "Consumer Discretionary", - "CDNS": "Information Technology", - "CDW": "Information Technology", - "CEG": "Utilities", - "CF": "Materials", - "CFG": "Financials", - "CHD": "Consumer Staples", - "CHRW": "Industrials", - "CHTR": "Communication Services", - "CI": "Health Care", - "CIEN": "Information Technology", - "CINF": "Financials", - "CL": "Consumer Staples", - "CLX": "Consumer Staples", - "CMCSA": "Communication Services", - "CME": "Financials", - "CMG": "Consumer Discretionary", - "CMI": "Industrials", - "CMS": "Utilities", - "CNC": "Health Care", - "CNP": "Utilities", - "COF": "Financials", - "COHR": "Information Technology", - "COIN": "Financials", - "COO": "Health Care", - "COP": "Energy", - "COR": "Health Care", - "COST": "Consumer Staples", - "CPAY": "Financials", - "CPB": "Consumer Defensive", - "CPRT": "Industrials", - "CPT": "Real Estate", - "CRH": "Materials", - "CRL": "Health Care", - "CRM": "Information Technology", - "CRWD": "Information Technology", - "CSCO": "Information Technology", - "CSGP": "Real Estate", - "CSX": "Industrials", - "CTAS": "Industrials", - "CTRA": "Energy", - "CTSH": "Information Technology", - "CTVA": "Materials", - "CVNA": "Consumer Discretionary", - "CVS": "Health Care", - "CVX": "Energy", - "D": "Utilities", - "DAL": "Industrials", - "DASH": "Consumer Discretionary", - "DD": "Materials", - "DDOG": "Information Technology", - "DE": "Industrials", - "DECK": "Consumer Discretionary", - "DELL": "Information Technology", - "DG": "Consumer Staples", - "DGX": "Health Care", - "DHI": "Consumer Discretionary", - "DHR": "Health Care", - "DIS": "Communication Services", - "DLR": "Real Estate", - "DLTR": "Consumer Staples", - "DOC": "Real Estate", - "DOV": "Industrials", - "DOW": "Materials", - "DPZ": "Consumer Discretionary", - "DRI": "Consumer Discretionary", - "DTE": "Utilities", - "DUK": "Utilities", - "DVA": "Health Care", - "DVN": "Energy", - "DXCM": "Health Care", - "EA": "Communication Services", - "EBAY": "Consumer Discretionary", - "ECL": "Materials", - "ED": "Utilities", - "EFX": "Industrials", - "EG": "Financials", - "EIX": "Utilities", - "EL": "Consumer Staples", - "ELV": "Health Care", - "EME": "Industrials", - "EMR": "Industrials", - "EOG": "Energy", - "EPAM": "Technology", - "EQIX": "Real Estate", - "EQR": "Real Estate", - "EQT": "Energy", - "ERIE": "Financials", - "ES": "Utilities", - "ESS": "Real Estate", - "ETN": "Industrials", - "ETR": "Utilities", - "EVRG": "Utilities", - "EW": "Health Care", - "EXC": "Utilities", - "EXE": "Energy", - "EXPD": "Industrials", - "EXPE": "Consumer Discretionary", - "EXR": "Real Estate", - "F": "Consumer Discretionary", - "FANG": "Energy", - "FAST": "Industrials", - "FCX": "Materials", - "FDS": "Financials", - "FDX": "Industrials", - "FE": "Utilities", - "FFIV": "Information Technology", - "FICO": "Information Technology", - "FIS": "Financials", - "FISV": "Financials", - "FITB": "Financials", - "FIX": "Industrials", - "FOX": "Communication Services", - "FOXA": "Communication Services", - "FRT": "Real Estate", - "FSLR": "Information Technology", - "FTNT": "Information Technology", - "FTV": "Industrials", - "GD": "Industrials", - "GDDY": "Information Technology", - "GE": "Industrials", - "GEHC": "Health Care", - "GEN": "Information Technology", - "GEV": "Industrials", - "GILD": "Health Care", - "GIS": "Consumer Staples", - "GL": "Financials", - "GLW": "Information Technology", - "GM": "Consumer Discretionary", - "GNRC": "Industrials", - "GOOG": "Communication Services", - "GOOGL": "Communication Services", - "GPC": "Consumer Discretionary", - "GPN": "Financials", - "GRMN": "Consumer Discretionary", - "GS": "Financials", - "GWW": "Industrials", - "HAL": "Energy", - "HAS": "Consumer Discretionary", - "HBAN": "Financials", - "HCA": "Health Care", - "HD": "Consumer Discretionary", - "HIG": "Financials", - "HII": "Industrials", - "HLT": "Consumer Discretionary", - "HON": "Industrials", - "HOOD": "Financials", - "HPE": "Information Technology", - "HPQ": "Information Technology", - "HRL": "Consumer Staples", - "HSIC": "Health Care", - "HST": "Real Estate", - "HSY": "Consumer Staples", - "HUBB": "Industrials", - "HUM": "Health Care", - "HWM": "Industrials", - "IBKR": "Financials", - "IBM": "Information Technology", - "ICE": "Financials", - "IDXX": "Health Care", - "IEX": "Industrials", - "IFF": "Materials", - "INCY": "Health Care", - "INTC": "Information Technology", - "INTU": "Information Technology", - "INVH": "Real Estate", - "IP": "Materials", - "IQV": "Health Care", - "IR": "Industrials", - "IRM": "Real Estate", - "ISRG": "Health Care", - "IT": "Information Technology", - "ITW": "Industrials", - "IVZ": "Financials", - "J": "Industrials", - "JBHT": "Industrials", - "JBL": "Information Technology", - "JCI": "Industrials", - "JKHY": "Financials", - "JNJ": "Health Care", - "JPM": "Financials", - "KDP": "Consumer Staples", - "KEY": "Financials", - "KEYS": "Information Technology", - "KHC": "Consumer Staples", - "KIM": "Real Estate", - "KKR": "Financials", - "KLAC": "Information Technology", - "KMB": "Consumer Staples", - "KMI": "Energy", - "KO": "Consumer Staples", - "KR": "Consumer Staples", - "KVUE": "Consumer Staples", - "L": "Financials", - "LDOS": "Industrials", - "LEN": "Consumer Discretionary", - "LH": "Health Care", - "LHX": "Industrials", - "LII": "Industrials", - "LIN": "Materials", - "LITE": "Information Technology", - "LLY": "Health Care", - "LMT": "Industrials", - "LNT": "Utilities", - "LOW": "Consumer Discretionary", - "LRCX": "Information Technology", - "LULU": "Consumer Discretionary", - "LUV": "Industrials", - "LVS": "Consumer Discretionary", - "LYB": "Materials", - "LYV": "Communication Services", - "MA": "Financials", - "MAA": "Real Estate", - "MAR": "Consumer Discretionary", - "MAS": "Industrials", - "MCD": "Consumer Discretionary", - "MCHP": "Information Technology", - "MCK": "Health Care", - "MCO": "Financials", - "MDLZ": "Consumer Staples", - "MDT": "Health Care", - "MET": "Financials", - "META": "Communication Services", - "MGM": "Consumer Discretionary", - "MKC": "Consumer Staples", - "MLM": "Materials", - "MMM": "Industrials", - "MNST": "Consumer Staples", - "MO": "Consumer Staples", - "MOS": "Materials", - "MPC": "Energy", - "MPWR": "Information Technology", - "MRK": "Health Care", - "MRNA": "Health Care", - "MRSH": "Financials", - "MS": "Financials", - "MSCI": "Financials", - "MSFT": "Information Technology", - "MSI": "Information Technology", - "MSTR": "Technology", - "MTB": "Financials", - "MTD": "Health Care", - "MU": "Information Technology", - "NCLH": "Consumer Discretionary", - "NDAQ": "Financials", - "NDSN": "Industrials", - "NEE": "Utilities", - "NEM": "Materials", - "NFLX": "Communication Services", - "NI": "Utilities", - "NKE": "Consumer Discretionary", - "NOC": "Industrials", - "NOW": "Information Technology", - "NRG": "Utilities", - "NSC": "Industrials", - "NTAP": "Information Technology", - "NTRS": "Financials", - "NUE": "Materials", - "NVDA": "Information Technology", - "NVR": "Consumer Discretionary", - "NWS": "Communication Services", - "NWSA": "Communication Services", - "NXPI": "Information Technology", - "O": "Real Estate", - "ODFL": "Industrials", - "OKE": "Energy", - "OMC": "Communication Services", - "ON": "Information Technology", - "ORCL": "Information Technology", - "ORLY": "Consumer Discretionary", - "OTIS": "Industrials", - "OXY": "Energy", - "PANW": "Information Technology", - "PAYX": "Industrials", - "PCAR": "Industrials", - "PCG": "Utilities", - "PEG": "Utilities", - "PEP": "Consumer Staples", - "PFE": "Health Care", - "PFG": "Financials", - "PG": "Consumer Staples", - "PGR": "Financials", - "PH": "Industrials", - "PHM": "Consumer Discretionary", - "PKG": "Materials", - "PLD": "Real Estate", - "PLTR": "Information Technology", - "PM": "Consumer Staples", - "PNC": "Financials", - "PNR": "Industrials", - "PNW": "Utilities", - "PODD": "Health Care", - "POOL": "Industrials", - "PPG": "Materials", - "PPL": "Utilities", - "PRU": "Financials", - "PSA": "Real Estate", - "PSKY": "Communication Services", - "PSX": "Energy", - "PTC": "Information Technology", - "PWR": "Industrials", - "PYPL": "Financials", - "Q": "Information Technology", - "QCOM": "Information Technology", - "RCL": "Consumer Discretionary", - "REG": "Real Estate", - "REGN": "Health Care", - "RF": "Financials", - "RJF": "Financials", - "RL": "Consumer Discretionary", - "RMD": "Health Care", - "ROK": "Industrials", - "ROL": "Industrials", - "ROP": "Information Technology", - "ROST": "Consumer Discretionary", - "RSG": "Industrials", - "RTX": "Industrials", - "RVTY": "Health Care", - "SATS": "Communication Services", - "SBAC": "Real Estate", - "SBUX": "Consumer Discretionary", - "SCHW": "Financials", - "SHW": "Materials", - "SJM": "Consumer Staples", - "SLB": "Energy", - "SMCI": "Information Technology", - "SNA": "Industrials", - "SNDK": "Information Technology", - "SNPS": "Information Technology", - "SO": "Utilities", - "SOLV": "Health Care", - "SPCX": "Industrials", - "SPG": "Real Estate", - "SPGI": "Financials", - "SRE": "Utilities", - "STE": "Health Care", - "STLD": "Materials", - "STT": "Financials", - "STX": "Information Technology", - "STZ": "Consumer Staples", - "SW": "Materials", - "SWK": "Industrials", - "SWKS": "Information Technology", - "SYF": "Financials", - "SYK": "Health Care", - "SYY": "Consumer Staples", - "T": "Communication Services", - "TAP": "Consumer Staples", - "TDG": "Industrials", - "TDY": "Information Technology", - "TECH": "Health Care", - "TEL": "Information Technology", - "TER": "Information Technology", - "TFC": "Financials", - "TGT": "Consumer Staples", - "TJX": "Consumer Discretionary", - "TKO": "Communication Services", - "TMO": "Health Care", - "TMUS": "Communication Services", - "TPL": "Energy", - "TPR": "Consumer Discretionary", - "TRGP": "Energy", - "TRMB": "Information Technology", - "TROW": "Financials", - "TRV": "Financials", - "TSCO": "Consumer Discretionary", - "TSLA": "Consumer Discretionary", - "TSN": "Consumer Staples", - "TT": "Industrials", - "TTD": "Communication Services", - "TTWO": "Communication Services", - "TXN": "Information Technology", - "TXT": "Industrials", - "TYL": "Information Technology", - "UAL": "Industrials", - "UBER": "Industrials", - "UDR": "Real Estate", - "UHS": "Health Care", - "ULTA": "Consumer Discretionary", - "UNH": "Health Care", - "UNP": "Industrials", - "UPS": "Industrials", - "URI": "Industrials", - "USB": "Financials", - "V": "Financials", - "VICI": "Real Estate", - "VLO": "Energy", - "VLTO": "Industrials", - "VMC": "Materials", - "VRSK": "Industrials", - "VRSN": "Information Technology", - "VRT": "Industrials", - "VRTX": "Health Care", - "VST": "Utilities", - "VTR": "Real Estate", - "VTRS": "Health Care", - "VZ": "Communication Services", - "WAB": "Industrials", - "WAT": "Health Care", - "WBD": "Communication Services", - "WDAY": "Information Technology", - "WDC": "Information Technology", - "WEC": "Utilities", - "WELL": "Real Estate", - "WFC": "Financials", - "WM": "Industrials", - "WMB": "Energy", - "WMT": "Consumer Staples", - "WRB": "Financials", - "WSM": "Consumer Discretionary", - "WST": "Health Care", - "WTW": "Financials", - "WY": "Real Estate", - "WYNN": "Consumer Discretionary", - "XEL": "Utilities", - "XOM": "Energy", - "XYL": "Industrials", - "XYZ": "Financials", - "YUM": "Consumer Discretionary", - "ZBH": "Health Care", - "ZBRA": "Information Technology", - "ZTS": "Health Care" - }, - "meta": { - "built_at": "2026-07-19T05:35:41.184460+00:00", - "coverage": { - "by_sector": { - "Communication Services": 23, - "Consumer Defensive": 2, - "Consumer Discretionary": 47, - "Consumer Staples": 34, - "Energy": 22, - "Financial Services": 1, - "Financials": 75, - "Health Care": 58, - "Industrials": 81, - "Information Technology": 72, - "Materials": 26, - "Real Estate": 31, - "Technology": 2, - "Utilities": 31 - }, - "mapped": 505, - "mapped_pct": 99.8, - "universe": 506, - "with_etf": 505 - }, - "fmp_requests": 10, - "from_existing": 0, - "from_fmp": 9, - "from_sp500_csv": 496, - "snapshot": "C:\\Workspace\\signal-platform\\backtest_snapshots\\prod.sqlite", - "still_missing": [ - "RHM" - ] - }, - "schema_version": 1 -} diff --git a/docs/research/history-depth-extension.md b/docs/research/history-depth-extension.md index 182b315..59857d0 100644 --- a/docs/research/history-depth-extension.md +++ b/docs/research/history-depth-extension.md @@ -172,9 +172,12 @@ report does not change that without a separate A/B. Flag for human awareness onl | file | role | |---|---| -| `reports/history-depth-20260719-103315.json` | **authoritative** | -| `reports/history-depth-20260719-103315.md` | companion dump | -| `reports/history-depth-20260719-093853` … `095156` | **ignore** (partial) | +| `reports/history-depth-20260719-103315.json` | Superseded unmasked/two-tier IC dump (do not cite for sector residual) | +| `reports/sector-resid-deep-20260719-113319.json` | Authoritative sector-resid deep grade | +| `reports/prod-book-universe-horizon-20260719-140737.json` | 505 vs liquid × horizon book matrix | + +Intermediate history-depth partials (093853–095156) and SANITY-FAIL noise were +removed in branch cleanup. --- diff --git a/docs/research/sector-residual-momentum.md b/docs/research/sector-residual-momentum.md index 8400bf0..aaf9656 100644 --- a/docs/research/sector-residual-momentum.md +++ b/docs/research/sector-residual-momentum.md @@ -219,20 +219,17 @@ to reopen promotion. --- -## Implementation notes (research machinery) +## Implementation notes -| piece | role | -|---|---| -| `app/services/sector_map.py` | GICS→ETF map, symbol normalise, JSON load/save | -| `app/services/backtest_service.py` | multi-factor residual; `mom_12_1_sector_resid` in `_signal_values`; demean inject | -| `scripts/build_ticker_sector_map.py` | SP500 CSV + FMP gap fill | -| `scripts/fetch_sector_etfs_to_snapshot.py` | Alpaca → snapshot `benchmark_prices` | -| `scripts/run_sector_residual_research.py` | race guard, IC, optional A/B, reports | -| `data/research/ticker_sector_map.json` | persisted labels (research only) | +Research runners and sector-residual harness hooks were **removed after close** +(2026-07-19 cleanup). Evidence remains in the report artifacts below. Do not +re-add without a new pre-registered protocol. --- ## Artifacts -- JSON: `reports/sector-residual-20260719-083356.json` -- MD copy: `reports/sector-residual-20260719-083356.md` +| file | role | +|---|---| +| `reports/sector-resid-deep-20260719-113319.json` | **Authoritative deep FAIL** | +| `reports/sector-residual-20260719-083356.json` | Short-window IC/A/B (superseded for promotion) | diff --git a/reports/history-depth-20260719-093853.json b/reports/history-depth-20260719-093853.json deleted file mode 100644 index 6ee84fb..0000000 --- a/reports/history-depth-20260719-093853.json +++ /dev/null @@ -1,68 +0,0 @@ -{ - "generated_at": "2026-07-19T09:38:53.936226", - "survivorship_banner": "SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only \u2014 not levels.", - "coverage": { - "snapshot": "/Users/taathde3/git/lab/signal_platform/backtest_snapshots/prod.sqlite", - "ticker_count": 506, - "ohlcv_row_count": 629263, - "date_range": { - "min": "2021-06-24", - "max": "2026-07-02" - }, - "bars_per_year": [ - { - "year": "2021", - "bars": 65678, - "tickers_with_bars": 494 - }, - { - "year": "2022", - "bars": 124423, - "tickers_with_bars": 497 - }, - { - "year": "2023", - "bars": 124479, - "tickers_with_bars": 499 - }, - { - "year": "2024", - "bars": 126129, - "tickers_with_bars": 501 - }, - { - "year": "2025", - "bars": 125615, - "tickers_with_bars": 504 - }, - { - "year": "2026", - "bars": 62939, - "tickers_with_bars": 505 - } - ], - "bars_per_symbol": { - "min": 14, - "p10": 1261, - "p50": 1261, - "p90": 1261, - "max": 1261 - }, - "symbols_by_start_year": { - "2021": 494, - "2022": 3, - "2023": 2, - "2024": 2, - "2025": 3, - "2026": 1 - }, - "note": "Where ticker counts drop in early years, the feed (or listing history) thins \u2014 do not treat those years as a full 505-name cross-section.", - "survivorship_banner": "SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only \u2014 not levels." - }, - "race_guard": null, - "harness": null, - "verdict": "COVERAGE_ONLY", - "verdict_detail": "Coverage probe only; run --phase harness after deep rebuild.", - "human_next": "- Compare sector residual vs market residual across eras.\n- If pre-2021 IC collapses, park Task 1 wire-in.\n- Do not retune production knobs on deep history levels.", - "report_path": "reports/history-depth-20260719-093853.json" -} diff --git a/reports/history-depth-20260719-093853.md b/reports/history-depth-20260719-093853.md deleted file mode 100644 index 658f7d3..0000000 --- a/reports/history-depth-20260719-093853.md +++ /dev/null @@ -1,177 +0,0 @@ -# History-depth extension (Tier-1 alpha research) - -**Status:** PRE-REGISTERED — run on MacBook (heavy I/O + full harness). -**Branch:** `research/history-depth-extension` (create from latest research stack). -**Production impact:** none. **Do not retune any production knob on deep history.** - ---- - -## Pre-registration (locked before rebuild) - -### Motivation - -All current conclusions rest on ~35 non-overlapping weekly windows in essentially -one post-2021 regime. Extending history toward max Alpaca daily-bar depth adds -the 2018 vol shock and full 2020 crash (where the feed allows). - -### Protocol - -1. **Empirical coverage first** — bars per calendar year per symbol; document - where the feed thins out. Do **not** assume a uniform start date. -2. **Rebuild the research snapshot completely** from prod source + max history - per symbol (`Adjustment.SPLIT`, ~200 req/min pacing via existing extender). -3. **Race guard (rule 6)** — refuse analysis until completion manifest is - `complete=true` and live counts match. -4. **Re-run full signal harness** (all existing signals incl. sector residual / - SUE if present) on the extended window. -5. **Report per signal:** mean IC, t, window count, and **era split** - (pre-/post-2021) — diagnostic only, **not a tuning input**. -6. **Log prominently:** survivorship bias grows with depth (today’s constituents - backfilled). Absolute Sharpe/CAGR on deep history is optimistic; payload is - **relative** signal comparisons and IC stability, not levels. -7. **Do not retune** production knobs. If a knob’s confirmation looks - overturned on deep history → report only; human decides. - -### Success / interpretation (not promotion of a new signal) - -| outcome | meaning | -|---|---| -| Sector residual still ≥ market residual on deep IC + stable sign | strengthens Task 1 PROMOTE case | -| Sector residual collapses pre-2021 | **PARK** Task 1 wire-in | -| SUE remains weak after full earnings + depth | **DEAD** SUE for this stack | -| Any production knob looks worse deep | report; no auto-retune | - ---- - -## MacBook runbook - -Prefer the bundled script (one entry point): - -```bash -git fetch origin && git checkout research/earnings-gap-and-sue -# .env: ALPACA_* required; FMP_* if resuming earnings -# copy backtest_snapshots/prod.sqlite if not already local - -chmod +x scripts/run_tier1_macbook.sh - -# Default: coverage → deep rebuild → harness (+ era split) -./scripts/run_tier1_macbook.sh - -# Optional variants -./scripts/run_tier1_macbook.sh --all # + earnings resume first -./scripts/run_tier1_macbook.sh --earnings-only # multi-day FMP + 2a/2b only -./scripts/run_tier1_macbook.sh --harness-only # skip rebuild -./scripts/run_tier1_macbook.sh --coverage-only - -# Tunables -WORKERS=12 HISTORY_DAYS=5000 ./scripts/run_tier1_macbook.sh -./scripts/run_tier1_macbook.sh --workers 12 --fmp-limit 250 -``` - -Then commit `reports/` + updated research docs, or copy them back to Windows. - ---- - -## Data provenance - -*(filled at run time)* - ---- - -## Results - -Generated: `2026-07-19T09:38:53.936226` - -> **SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only — not levels.** - -### Coverage - -```json -{ - "snapshot": "/Users/taathde3/git/lab/signal_platform/backtest_snapshots/prod.sqlite", - "ticker_count": 506, - "ohlcv_row_count": 629263, - "date_range": { - "min": "2021-06-24", - "max": "2026-07-02" - }, - "bars_per_year": [ - { - "year": "2021", - "bars": 65678, - "tickers_with_bars": 494 - }, - { - "year": "2022", - "bars": 124423, - "tickers_with_bars": 497 - }, - { - "year": "2023", - "bars": 124479, - "tickers_with_bars": 499 - }, - { - "year": "2024", - "bars": 126129, - "tickers_with_bars": 501 - }, - { - "year": "2025", - "bars": 125615, - "tickers_with_bars": 504 - }, - { - "year": "2026", - "bars": 62939, - "tickers_with_bars": 505 - } - ], - "bars_per_symbol": { - "min": 14, - "p10": 1261, - "p50": 1261, - "p90": 1261, - "max": 1261 - }, - "symbols_by_start_year": { - "2021": 494, - "2022": 3, - "2023": 2, - "2024": 2, - "2025": 3, - "2026": 1 - }, - "note": "Where ticker counts drop in early years, the feed (or listing history) thins \u2014 do not treat those years as a full 505-name cross-section.", - "survivorship_banner": "SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only \u2014 not levels." -} -``` - -### Race guard - -```json -{} -``` - -### Signal IC (full extended window) - -_Harness not run this pass._ - -### Era split (diagnostic only) - -_No era split._ - -## Verdict - -**COVERAGE_ONLY** - -Coverage probe only; run --phase harness after deep rebuild. - -## What a human must decide next - -- Compare sector residual vs market residual across eras. -- If pre-2021 IC collapses, park Task 1 wire-in. -- Do not retune production knobs on deep history levels. - -Artifacts: `reports/history-depth-20260719-093853.json` - diff --git a/reports/history-depth-20260719-094134.json b/reports/history-depth-20260719-094134.json deleted file mode 100644 index 826ced0..0000000 --- a/reports/history-depth-20260719-094134.json +++ /dev/null @@ -1,68 +0,0 @@ -{ - "generated_at": "2026-07-19T09:41:34.109378", - "survivorship_banner": "SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only \u2014 not levels.", - "coverage": { - "snapshot": "/Users/taathde3/git/lab/signal_platform/backtest_snapshots/prod.sqlite", - "ticker_count": 506, - "ohlcv_row_count": 629263, - "date_range": { - "min": "2021-06-24", - "max": "2026-07-02" - }, - "bars_per_year": [ - { - "year": "2021", - "bars": 65678, - "tickers_with_bars": 494 - }, - { - "year": "2022", - "bars": 124423, - "tickers_with_bars": 497 - }, - { - "year": "2023", - "bars": 124479, - "tickers_with_bars": 499 - }, - { - "year": "2024", - "bars": 126129, - "tickers_with_bars": 501 - }, - { - "year": "2025", - "bars": 125615, - "tickers_with_bars": 504 - }, - { - "year": "2026", - "bars": 62939, - "tickers_with_bars": 505 - } - ], - "bars_per_symbol": { - "min": 14, - "p10": 1261, - "p50": 1261, - "p90": 1261, - "max": 1261 - }, - "symbols_by_start_year": { - "2021": 494, - "2022": 3, - "2023": 2, - "2024": 2, - "2025": 3, - "2026": 1 - }, - "note": "Where ticker counts drop in early years, the feed (or listing history) thins \u2014 do not treat those years as a full 505-name cross-section.", - "survivorship_banner": "SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only \u2014 not levels." - }, - "race_guard": null, - "harness": null, - "verdict": "COVERAGE_ONLY", - "verdict_detail": "Coverage probe only; run --phase harness after deep rebuild.", - "human_next": "- Compare sector residual vs market residual across eras.\n- If pre-2021 IC collapses, park Task 1 wire-in.\n- Do not retune production knobs on deep history levels.", - "report_path": "reports/history-depth-20260719-094134.json" -} diff --git a/reports/history-depth-20260719-094134.md b/reports/history-depth-20260719-094134.md deleted file mode 100644 index 7b23512..0000000 --- a/reports/history-depth-20260719-094134.md +++ /dev/null @@ -1,177 +0,0 @@ -# History-depth extension (Tier-1 alpha research) - -**Status:** PRE-REGISTERED — run on MacBook (heavy I/O + full harness). -**Branch:** `research/history-depth-extension` (create from latest research stack). -**Production impact:** none. **Do not retune any production knob on deep history.** - ---- - -## Pre-registration (locked before rebuild) - -### Motivation - -All current conclusions rest on ~35 non-overlapping weekly windows in essentially -one post-2021 regime. Extending history toward max Alpaca daily-bar depth adds -the 2018 vol shock and full 2020 crash (where the feed allows). - -### Protocol - -1. **Empirical coverage first** — bars per calendar year per symbol; document - where the feed thins out. Do **not** assume a uniform start date. -2. **Rebuild the research snapshot completely** from prod source + max history - per symbol (`Adjustment.SPLIT`, ~200 req/min pacing via existing extender). -3. **Race guard (rule 6)** — refuse analysis until completion manifest is - `complete=true` and live counts match. -4. **Re-run full signal harness** (all existing signals incl. sector residual / - SUE if present) on the extended window. -5. **Report per signal:** mean IC, t, window count, and **era split** - (pre-/post-2021) — diagnostic only, **not a tuning input**. -6. **Log prominently:** survivorship bias grows with depth (today’s constituents - backfilled). Absolute Sharpe/CAGR on deep history is optimistic; payload is - **relative** signal comparisons and IC stability, not levels. -7. **Do not retune** production knobs. If a knob’s confirmation looks - overturned on deep history → report only; human decides. - -### Success / interpretation (not promotion of a new signal) - -| outcome | meaning | -|---|---| -| Sector residual still ≥ market residual on deep IC + stable sign | strengthens Task 1 PROMOTE case | -| Sector residual collapses pre-2021 | **PARK** Task 1 wire-in | -| SUE remains weak after full earnings + depth | **DEAD** SUE for this stack | -| Any production knob looks worse deep | report; no auto-retune | - ---- - -## MacBook runbook - -Prefer the bundled script (one entry point): - -```bash -git fetch origin && git checkout research/earnings-gap-and-sue -# .env: ALPACA_* required; FMP_* if resuming earnings -# copy backtest_snapshots/prod.sqlite if not already local - -chmod +x scripts/run_tier1_macbook.sh - -# Default: coverage → deep rebuild → harness (+ era split) -./scripts/run_tier1_macbook.sh - -# Optional variants -./scripts/run_tier1_macbook.sh --all # + earnings resume first -./scripts/run_tier1_macbook.sh --earnings-only # multi-day FMP + 2a/2b only -./scripts/run_tier1_macbook.sh --harness-only # skip rebuild -./scripts/run_tier1_macbook.sh --coverage-only - -# Tunables -WORKERS=12 HISTORY_DAYS=5000 ./scripts/run_tier1_macbook.sh -./scripts/run_tier1_macbook.sh --workers 12 --fmp-limit 250 -``` - -Then commit `reports/` + updated research docs, or copy them back to Windows. - ---- - -## Data provenance - -*(filled at run time)* - ---- - -## Results - -Generated: `2026-07-19T09:41:34.109378` - -> **SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only — not levels.** - -### Coverage - -```json -{ - "snapshot": "/Users/taathde3/git/lab/signal_platform/backtest_snapshots/prod.sqlite", - "ticker_count": 506, - "ohlcv_row_count": 629263, - "date_range": { - "min": "2021-06-24", - "max": "2026-07-02" - }, - "bars_per_year": [ - { - "year": "2021", - "bars": 65678, - "tickers_with_bars": 494 - }, - { - "year": "2022", - "bars": 124423, - "tickers_with_bars": 497 - }, - { - "year": "2023", - "bars": 124479, - "tickers_with_bars": 499 - }, - { - "year": "2024", - "bars": 126129, - "tickers_with_bars": 501 - }, - { - "year": "2025", - "bars": 125615, - "tickers_with_bars": 504 - }, - { - "year": "2026", - "bars": 62939, - "tickers_with_bars": 505 - } - ], - "bars_per_symbol": { - "min": 14, - "p10": 1261, - "p50": 1261, - "p90": 1261, - "max": 1261 - }, - "symbols_by_start_year": { - "2021": 494, - "2022": 3, - "2023": 2, - "2024": 2, - "2025": 3, - "2026": 1 - }, - "note": "Where ticker counts drop in early years, the feed (or listing history) thins \u2014 do not treat those years as a full 505-name cross-section.", - "survivorship_banner": "SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only \u2014 not levels." -} -``` - -### Race guard - -```json -{} -``` - -### Signal IC (full extended window) - -_Harness not run this pass._ - -### Era split (diagnostic only) - -_No era split._ - -## Verdict - -**COVERAGE_ONLY** - -Coverage probe only; run --phase harness after deep rebuild. - -## What a human must decide next - -- Compare sector residual vs market residual across eras. -- If pre-2021 IC collapses, park Task 1 wire-in. -- Do not retune production knobs on deep history levels. - -Artifacts: `reports/history-depth-20260719-094134.json` - diff --git a/reports/history-depth-20260719-094344.json b/reports/history-depth-20260719-094344.json deleted file mode 100644 index 5c35f97..0000000 --- a/reports/history-depth-20260719-094344.json +++ /dev/null @@ -1,68 +0,0 @@ -{ - "generated_at": "2026-07-19T09:43:44.520126", - "survivorship_banner": "SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only \u2014 not levels.", - "coverage": { - "snapshot": "/Users/taathde3/git/lab/signal_platform/backtest_snapshots/prod.sqlite", - "ticker_count": 506, - "ohlcv_row_count": 629263, - "date_range": { - "min": "2021-06-24", - "max": "2026-07-02" - }, - "bars_per_year": [ - { - "year": "2021", - "bars": 65678, - "tickers_with_bars": 494 - }, - { - "year": "2022", - "bars": 124423, - "tickers_with_bars": 497 - }, - { - "year": "2023", - "bars": 124479, - "tickers_with_bars": 499 - }, - { - "year": "2024", - "bars": 126129, - "tickers_with_bars": 501 - }, - { - "year": "2025", - "bars": 125615, - "tickers_with_bars": 504 - }, - { - "year": "2026", - "bars": 62939, - "tickers_with_bars": 505 - } - ], - "bars_per_symbol": { - "min": 14, - "p10": 1261, - "p50": 1261, - "p90": 1261, - "max": 1261 - }, - "symbols_by_start_year": { - "2021": 494, - "2022": 3, - "2023": 2, - "2024": 2, - "2025": 3, - "2026": 1 - }, - "note": "Where ticker counts drop in early years, the feed (or listing history) thins \u2014 do not treat those years as a full 505-name cross-section.", - "survivorship_banner": "SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only \u2014 not levels." - }, - "race_guard": null, - "harness": null, - "verdict": "COVERAGE_ONLY", - "verdict_detail": "Coverage probe only; run --phase harness after deep rebuild.", - "human_next": "- Compare sector residual vs market residual across eras.\n- If pre-2021 IC collapses, park Task 1 wire-in.\n- Do not retune production knobs on deep history levels.", - "report_path": "reports/history-depth-20260719-094344.json" -} diff --git a/reports/history-depth-20260719-094344.md b/reports/history-depth-20260719-094344.md deleted file mode 100644 index d52b1bb..0000000 --- a/reports/history-depth-20260719-094344.md +++ /dev/null @@ -1,177 +0,0 @@ -# History-depth extension (Tier-1 alpha research) - -**Status:** PRE-REGISTERED — run on MacBook (heavy I/O + full harness). -**Branch:** `research/history-depth-extension` (create from latest research stack). -**Production impact:** none. **Do not retune any production knob on deep history.** - ---- - -## Pre-registration (locked before rebuild) - -### Motivation - -All current conclusions rest on ~35 non-overlapping weekly windows in essentially -one post-2021 regime. Extending history toward max Alpaca daily-bar depth adds -the 2018 vol shock and full 2020 crash (where the feed allows). - -### Protocol - -1. **Empirical coverage first** — bars per calendar year per symbol; document - where the feed thins out. Do **not** assume a uniform start date. -2. **Rebuild the research snapshot completely** from prod source + max history - per symbol (`Adjustment.SPLIT`, ~200 req/min pacing via existing extender). -3. **Race guard (rule 6)** — refuse analysis until completion manifest is - `complete=true` and live counts match. -4. **Re-run full signal harness** (all existing signals incl. sector residual / - SUE if present) on the extended window. -5. **Report per signal:** mean IC, t, window count, and **era split** - (pre-/post-2021) — diagnostic only, **not a tuning input**. -6. **Log prominently:** survivorship bias grows with depth (today’s constituents - backfilled). Absolute Sharpe/CAGR on deep history is optimistic; payload is - **relative** signal comparisons and IC stability, not levels. -7. **Do not retune** production knobs. If a knob’s confirmation looks - overturned on deep history → report only; human decides. - -### Success / interpretation (not promotion of a new signal) - -| outcome | meaning | -|---|---| -| Sector residual still ≥ market residual on deep IC + stable sign | strengthens Task 1 PROMOTE case | -| Sector residual collapses pre-2021 | **PARK** Task 1 wire-in | -| SUE remains weak after full earnings + depth | **DEAD** SUE for this stack | -| Any production knob looks worse deep | report; no auto-retune | - ---- - -## MacBook runbook - -Prefer the bundled script (one entry point): - -```bash -git fetch origin && git checkout research/earnings-gap-and-sue -# .env: ALPACA_* required; FMP_* if resuming earnings -# copy backtest_snapshots/prod.sqlite if not already local - -chmod +x scripts/run_tier1_macbook.sh - -# Default: coverage → deep rebuild → harness (+ era split) -./scripts/run_tier1_macbook.sh - -# Optional variants -./scripts/run_tier1_macbook.sh --all # + earnings resume first -./scripts/run_tier1_macbook.sh --earnings-only # multi-day FMP + 2a/2b only -./scripts/run_tier1_macbook.sh --harness-only # skip rebuild -./scripts/run_tier1_macbook.sh --coverage-only - -# Tunables -WORKERS=12 HISTORY_DAYS=5000 ./scripts/run_tier1_macbook.sh -./scripts/run_tier1_macbook.sh --workers 12 --fmp-limit 250 -``` - -Then commit `reports/` + updated research docs, or copy them back to Windows. - ---- - -## Data provenance - -*(filled at run time)* - ---- - -## Results - -Generated: `2026-07-19T09:43:44.520126` - -> **SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only — not levels.** - -### Coverage - -```json -{ - "snapshot": "/Users/taathde3/git/lab/signal_platform/backtest_snapshots/prod.sqlite", - "ticker_count": 506, - "ohlcv_row_count": 629263, - "date_range": { - "min": "2021-06-24", - "max": "2026-07-02" - }, - "bars_per_year": [ - { - "year": "2021", - "bars": 65678, - "tickers_with_bars": 494 - }, - { - "year": "2022", - "bars": 124423, - "tickers_with_bars": 497 - }, - { - "year": "2023", - "bars": 124479, - "tickers_with_bars": 499 - }, - { - "year": "2024", - "bars": 126129, - "tickers_with_bars": 501 - }, - { - "year": "2025", - "bars": 125615, - "tickers_with_bars": 504 - }, - { - "year": "2026", - "bars": 62939, - "tickers_with_bars": 505 - } - ], - "bars_per_symbol": { - "min": 14, - "p10": 1261, - "p50": 1261, - "p90": 1261, - "max": 1261 - }, - "symbols_by_start_year": { - "2021": 494, - "2022": 3, - "2023": 2, - "2024": 2, - "2025": 3, - "2026": 1 - }, - "note": "Where ticker counts drop in early years, the feed (or listing history) thins \u2014 do not treat those years as a full 505-name cross-section.", - "survivorship_banner": "SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only \u2014 not levels." -} -``` - -### Race guard - -```json -{} -``` - -### Signal IC (full extended window) - -_Harness not run this pass._ - -### Era split (diagnostic only) - -_No era split._ - -## Verdict - -**COVERAGE_ONLY** - -Coverage probe only; run --phase harness after deep rebuild. - -## What a human must decide next - -- Compare sector residual vs market residual across eras. -- If pre-2021 IC collapses, park Task 1 wire-in. -- Do not retune production knobs on deep history levels. - -Artifacts: `reports/history-depth-20260719-094344.json` - diff --git a/reports/history-depth-20260719-095156.json b/reports/history-depth-20260719-095156.json deleted file mode 100644 index 416c34e..0000000 --- a/reports/history-depth-20260719-095156.json +++ /dev/null @@ -1,68 +0,0 @@ -{ - "generated_at": "2026-07-19T09:51:56.525634", - "survivorship_banner": "SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only \u2014 not levels.", - "coverage": { - "snapshot": "/Users/taathde3/git/lab/signal_platform/backtest_snapshots/prod.sqlite", - "ticker_count": 506, - "ohlcv_row_count": 629263, - "date_range": { - "min": "2021-06-24", - "max": "2026-07-02" - }, - "bars_per_year": [ - { - "year": "2021", - "bars": 65678, - "tickers_with_bars": 494 - }, - { - "year": "2022", - "bars": 124423, - "tickers_with_bars": 497 - }, - { - "year": "2023", - "bars": 124479, - "tickers_with_bars": 499 - }, - { - "year": "2024", - "bars": 126129, - "tickers_with_bars": 501 - }, - { - "year": "2025", - "bars": 125615, - "tickers_with_bars": 504 - }, - { - "year": "2026", - "bars": 62939, - "tickers_with_bars": 505 - } - ], - "bars_per_symbol": { - "min": 14, - "p10": 1261, - "p50": 1261, - "p90": 1261, - "max": 1261 - }, - "symbols_by_start_year": { - "2021": 494, - "2022": 3, - "2023": 2, - "2024": 2, - "2025": 3, - "2026": 1 - }, - "note": "Where ticker counts drop in early years, the feed (or listing history) thins \u2014 do not treat those years as a full 505-name cross-section.", - "survivorship_banner": "SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only \u2014 not levels." - }, - "race_guard": null, - "harness": null, - "verdict": "COVERAGE_ONLY", - "verdict_detail": "Coverage probe only; run --phase harness after deep rebuild.", - "human_next": "- Compare sector residual vs market residual across eras.\n- If pre-2021 IC collapses, park Task 1 wire-in.\n- Do not retune production knobs on deep history levels.", - "report_path": "reports/history-depth-20260719-095156.json" -} diff --git a/reports/history-depth-20260719-095156.md b/reports/history-depth-20260719-095156.md deleted file mode 100644 index a6edf52..0000000 --- a/reports/history-depth-20260719-095156.md +++ /dev/null @@ -1,177 +0,0 @@ -# History-depth extension (Tier-1 alpha research) - -**Status:** PRE-REGISTERED — run on MacBook (heavy I/O + full harness). -**Branch:** `research/history-depth-extension` (create from latest research stack). -**Production impact:** none. **Do not retune any production knob on deep history.** - ---- - -## Pre-registration (locked before rebuild) - -### Motivation - -All current conclusions rest on ~35 non-overlapping weekly windows in essentially -one post-2021 regime. Extending history toward max Alpaca daily-bar depth adds -the 2018 vol shock and full 2020 crash (where the feed allows). - -### Protocol - -1. **Empirical coverage first** — bars per calendar year per symbol; document - where the feed thins out. Do **not** assume a uniform start date. -2. **Rebuild the research snapshot completely** from prod source + max history - per symbol (`Adjustment.SPLIT`, ~200 req/min pacing via existing extender). -3. **Race guard (rule 6)** — refuse analysis until completion manifest is - `complete=true` and live counts match. -4. **Re-run full signal harness** (all existing signals incl. sector residual / - SUE if present) on the extended window. -5. **Report per signal:** mean IC, t, window count, and **era split** - (pre-/post-2021) — diagnostic only, **not a tuning input**. -6. **Log prominently:** survivorship bias grows with depth (today’s constituents - backfilled). Absolute Sharpe/CAGR on deep history is optimistic; payload is - **relative** signal comparisons and IC stability, not levels. -7. **Do not retune** production knobs. If a knob’s confirmation looks - overturned on deep history → report only; human decides. - -### Success / interpretation (not promotion of a new signal) - -| outcome | meaning | -|---|---| -| Sector residual still ≥ market residual on deep IC + stable sign | strengthens Task 1 PROMOTE case | -| Sector residual collapses pre-2021 | **PARK** Task 1 wire-in | -| SUE remains weak after full earnings + depth | **DEAD** SUE for this stack | -| Any production knob looks worse deep | report; no auto-retune | - ---- - -## MacBook runbook - -Prefer the bundled script (one entry point): - -```bash -git fetch origin && git checkout research/earnings-gap-and-sue -# .env: ALPACA_* required; FMP_* if resuming earnings -# copy backtest_snapshots/prod.sqlite if not already local - -chmod +x scripts/run_tier1_macbook.sh - -# Default: coverage → deep rebuild → harness (+ era split) -./scripts/run_tier1_macbook.sh - -# Optional variants -./scripts/run_tier1_macbook.sh --all # + earnings resume first -./scripts/run_tier1_macbook.sh --earnings-only # multi-day FMP + 2a/2b only -./scripts/run_tier1_macbook.sh --harness-only # skip rebuild -./scripts/run_tier1_macbook.sh --coverage-only - -# Tunables -WORKERS=12 HISTORY_DAYS=5000 ./scripts/run_tier1_macbook.sh -./scripts/run_tier1_macbook.sh --workers 12 --fmp-limit 250 -``` - -Then commit `reports/` + updated research docs, or copy them back to Windows. - ---- - -## Data provenance - -*(filled at run time)* - ---- - -## Results - -Generated: `2026-07-19T09:51:56.525634` - -> **SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only — not levels.** - -### Coverage - -```json -{ - "snapshot": "/Users/taathde3/git/lab/signal_platform/backtest_snapshots/prod.sqlite", - "ticker_count": 506, - "ohlcv_row_count": 629263, - "date_range": { - "min": "2021-06-24", - "max": "2026-07-02" - }, - "bars_per_year": [ - { - "year": "2021", - "bars": 65678, - "tickers_with_bars": 494 - }, - { - "year": "2022", - "bars": 124423, - "tickers_with_bars": 497 - }, - { - "year": "2023", - "bars": 124479, - "tickers_with_bars": 499 - }, - { - "year": "2024", - "bars": 126129, - "tickers_with_bars": 501 - }, - { - "year": "2025", - "bars": 125615, - "tickers_with_bars": 504 - }, - { - "year": "2026", - "bars": 62939, - "tickers_with_bars": 505 - } - ], - "bars_per_symbol": { - "min": 14, - "p10": 1261, - "p50": 1261, - "p90": 1261, - "max": 1261 - }, - "symbols_by_start_year": { - "2021": 494, - "2022": 3, - "2023": 2, - "2024": 2, - "2025": 3, - "2026": 1 - }, - "note": "Where ticker counts drop in early years, the feed (or listing history) thins \u2014 do not treat those years as a full 505-name cross-section.", - "survivorship_banner": "SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only \u2014 not levels." -} -``` - -### Race guard - -```json -{} -``` - -### Signal IC (full extended window) - -_Harness not run this pass._ - -### Era split (diagnostic only) - -_No era split._ - -## Verdict - -**COVERAGE_ONLY** - -Coverage probe only; run --phase harness after deep rebuild. - -## What a human must decide next - -- Compare sector residual vs market residual across eras. -- If pre-2021 IC collapses, park Task 1 wire-in. -- Do not retune production knobs on deep history levels. - -Artifacts: `reports/history-depth-20260719-095156.json` - diff --git a/reports/sector-resid-deep-20260719-111923-SANITY-FAIL.json b/reports/sector-resid-deep-20260719-111923-SANITY-FAIL.json deleted file mode 100644 index 6b03e61..0000000 --- a/reports/sector-resid-deep-20260719-111923-SANITY-FAIL.json +++ /dev/null @@ -1,275 +0,0 @@ -{ - "step1": { - "shallow_meta": { - "n_symbols": 4654, - "deep_cohort_p10_start": "2016-01-04", - "shallow_cutoff": "2017-02-07", - "lag_days": 400, - "n_shallow": 3349, - "shallow_start_histogram": { - "2017": 117, - "2018": 162, - "2019": 163, - "2020": 269, - "2021": 1060, - "2022": 211, - "2023": 176, - "2024": 281, - "2025": 543, - "2026": 367 - }, - "deep_start_histogram": { - "2016": 1296, - "2017": 9 - }, - "shallow_sample": [ - "A", - "AACB", - "AACBR", - "AACBU", - "AACI", - "AACIU", - "AACIW", - "AACO", - "AACOU", - "AACOW", - "AACP", - "AACPR", - "AACPU", - "AACPW", - "AAPG", - "AAPL", - "AARD", - "ABAT", - "ABBV", - "ABCL", - "ABLV", - "ABLVW", - "ABNB", - "ABOS", - "ABSI", - "ABT", - "ABTC", - "ABVC", - "ABVX", - "ACAA" - ] - }, - "shallow_list_n": 3349, - "fetch_ok": 3349, - "fetch_fail": 0, - "etf_refresh": { - "written": { - "SPY": 0, - "XLB": 0, - "XLC": 0, - "XLE": 0, - "XLF": 0, - "XLI": 0, - "XLK": 0, - "XLP": 0, - "XLRE": 0, - "XLU": 0, - "XLV": 0, - "XLY": 0 - }, - "benchmark_summary": [ - { - "symbol": "SPY", - "n": 2649, - "min": "2016-01-04", - "max": "2026-07-17" - }, - { - "symbol": "XLB", - "n": 2649, - "min": "2016-01-04", - "max": "2026-07-17" - }, - { - "symbol": "XLC", - "n": 2030, - "min": "2018-06-19", - "max": "2026-07-17" - }, - { - "symbol": "XLE", - "n": 2649, - "min": "2016-01-04", - "max": "2026-07-17" - }, - { - "symbol": "XLF", - "n": 2649, - "min": "2016-01-04", - "max": "2026-07-17" - }, - { - "symbol": "XLI", - "n": 2649, - "min": "2016-01-04", - "max": "2026-07-17" - }, - { - "symbol": "XLK", - "n": 2649, - "min": "2016-01-04", - "max": "2026-07-17" - }, - { - "symbol": "XLP", - "n": 2649, - "min": "2016-01-04", - "max": "2026-07-17" - }, - { - "symbol": "XLRE", - "n": 2649, - "min": "2016-01-04", - "max": "2026-07-17" - }, - { - "symbol": "XLU", - "n": 2649, - "min": "2016-01-04", - "max": "2026-07-17" - }, - { - "symbol": "XLV", - "n": 2649, - "min": "2016-01-04", - "max": "2026-07-17" - }, - { - "symbol": "XLY", - "n": 2649, - "min": "2016-01-04", - "max": "2026-07-17" - } - ] - }, - "sanity": { - "passed": false, - "megacap": { - "AAPL": { - "symbol": "AAPL", - "bars": 2649, - "min_date": "2016-01-04", - "max_date": "2026-07-17" - }, - "MSFT": { - "symbol": "MSFT", - "bars": 2649, - "min_date": "2016-01-04", - "max_date": "2026-07-17" - }, - "JPM": { - "symbol": "JPM", - "bars": 2649, - "min_date": "2016-01-04", - "max_date": "2026-07-17" - }, - "XOM": { - "symbol": "XOM", - "bars": 2649, - "min_date": "2016-01-04", - "max_date": "2026-07-17" - }, - "JNJ": { - "symbol": "JNJ", - "bars": 2649, - "min_date": "2016-01-04", - "max_date": "2026-07-17" - } - }, - "megacap_ok": false, - "megacap_deadline": "2013-12-14", - "sector_etfs": { - "XLB": { - "n": 2649, - "min": "2016-01-04", - "max": "2026-07-17" - }, - "XLC": { - "n": 2030, - "min": "2018-06-19", - "max": "2026-07-17" - }, - "XLE": { - "n": 2649, - "min": "2016-01-04", - "max": "2026-07-17" - }, - "XLF": { - "n": 2649, - "min": "2016-01-04", - "max": "2026-07-17" - }, - "XLI": { - "n": 2649, - "min": "2016-01-04", - "max": "2026-07-17" - }, - "XLK": { - "n": 2649, - "min": "2016-01-04", - "max": "2026-07-17" - }, - "XLP": { - "n": 2649, - "min": "2016-01-04", - "max": "2026-07-17" - }, - "XLRE": { - "n": 2649, - "min": "2016-01-04", - "max": "2026-07-17" - }, - "XLU": { - "n": 2649, - "min": "2016-01-04", - "max": "2026-07-17" - }, - "XLV": { - "n": 2649, - "min": "2016-01-04", - "max": "2026-07-17" - }, - "XLY": { - "n": 2649, - "min": "2016-01-04", - "max": "2026-07-17" - } - }, - "sector_etfs_deep_count": 11, - "sector_etfs_ok": true, - "still_shallow_count": 2882, - "still_shallow_sample": [ - "AACB", - "AACBR", - "AACBU", - "AACI", - "AACIU", - "AACIW", - "AACO", - "AACOU", - "AACOW", - "AACP", - "AACPR", - "AACPU", - "AACPW", - "AAPG", - "AARD", - "ABAT", - "ABCL", - "ABLV", - "ABLVW", - "ABNB" - ], - "xlc_note": "XLC lists mid-2018 \u2192 Communication Services residual coverage from ~mid-2019.", - "target_history_days": 5000 - }, - "manifest_path": "backtest_snapshots/research.sqlite.manifest.json" - }, - "harness": null -} diff --git a/scripts/build_ticker_sector_map.py b/scripts/build_ticker_sector_map.py deleted file mode 100644 index e200b76..0000000 --- a/scripts/build_ticker_sector_map.py +++ /dev/null @@ -1,233 +0,0 @@ -"""Build a local ticker → GICS sector map for research residualization. - -Sources (in order): -1. Public S&P 500 constituents CSV (datasets/s-and-p-500-companies) — bulk, free. -2. Existing map file (resume). -3. FMP stable ``profile`` for still-missing symbols (budget ~250 req/day). - -Writes ``data/research/ticker_sector_map.json``. Never touches production Postgres. - -Example -------- - python scripts/build_ticker_sector_map.py \\ - --snapshot backtest_snapshots/prod.sqlite - - python scripts/build_ticker_sector_map.py --fmp-limit 50 -""" - -from __future__ import annotations - -import argparse -import asyncio -import csv -import io -import json -import sys -import time -from datetime import datetime, timezone -from pathlib import Path - -import httpx -from sqlalchemy import create_engine, text - -ROOT = Path(__file__).resolve().parents[1] -if str(ROOT) not in sys.path: - sys.path.insert(0, str(ROOT)) - -from app.ssl_bootstrap import bootstrap_ssl # noqa: E402 - -bootstrap_ssl() - -from app.services.sector_map import ( # noqa: E402 - DEFAULT_SECTOR_MAP_PATH, - coverage_stats, - load_ticker_sector_map, - normalise_symbol, - save_ticker_sector_map, - sector_to_etf, -) - -SP500_CSV_URL = ( - "https://raw.githubusercontent.com/datasets/s-and-p-500-companies/" - "master/data/constituents.csv" -) -FMP_STABLE = "https://financialmodelingprep.com/stable" - - -def _parse_args() -> argparse.Namespace: - p = argparse.ArgumentParser(description=__doc__) - p.add_argument( - "--snapshot", - default="backtest_snapshots/prod.sqlite", - help="Snapshot whose tickers define the universe.", - ) - p.add_argument( - "--out", - default=str(DEFAULT_SECTOR_MAP_PATH), - help="Output JSON path.", - ) - p.add_argument( - "--fmp-limit", - type=int, - default=200, - help="Max FMP profile requests this run (free-tier cushion).", - ) - p.add_argument( - "--skip-fmp", - action="store_true", - help="Only use public SP500 CSV + existing map.", - ) - p.add_argument("--sleep", type=float, default=0.35, help="Pause between FMP calls.") - return p.parse_args() - - -def _snapshot_symbols(snapshot: Path) -> list[str]: - engine = create_engine(f"sqlite:///{snapshot.resolve().as_posix()}", future=True) - try: - with engine.connect() as conn: - rows = conn.execute(text("SELECT symbol FROM tickers ORDER BY symbol")).fetchall() - finally: - engine.dispose() - return [normalise_symbol(r[0]) for r in rows if r[0]] - - -def _fetch_sp500_map() -> dict[str, str]: - with httpx.Client(timeout=60.0, follow_redirects=True) as client: - resp = client.get(SP500_CSV_URL) - resp.raise_for_status() - reader = csv.DictReader(io.StringIO(resp.text)) - out: dict[str, str] = {} - for row in reader: - sym = normalise_symbol(row.get("Symbol") or "") - sector = (row.get("GICS Sector") or "").strip() - if sym and sector: - out[sym] = sector - return out - - -async def _fmp_profile_sector(client: httpx.AsyncClient, api_key: str, symbol: str) -> str | None: - resp = await client.get( - f"{FMP_STABLE}/profile", - params={"symbol": symbol, "apikey": api_key}, - ) - if resp.status_code == 429: - raise RuntimeError(f"FMP rate limited on {symbol}") - if resp.status_code == 402: - return None - resp.raise_for_status() - data = resp.json() - if isinstance(data, list): - data = data[0] if data else {} - if not isinstance(data, dict): - return None - sector = (data.get("sector") or data.get("industry") or "").strip() - # industry alone is not a GICS sector — only accept if we can map to an ETF - if sector and sector_to_etf(sector): - return sector - # FMP sometimes returns industry under sector when sector missing; try sector field only - sec = (data.get("sector") or "").strip() - return sec or None - - -async def _fill_from_fmp( - missing: list[str], - *, - api_key: str, - limit: int, - sleep_s: float, -) -> tuple[dict[str, str], int]: - filled: dict[str, str] = {} - used = 0 - async with httpx.AsyncClient(timeout=30.0) as client: - for sym in missing: - if used >= limit: - break - try: - sector = await _fmp_profile_sector(client, api_key, sym) - except Exception as exc: - print(f" FMP fail {sym}: {exc}") - used += 1 - await asyncio.sleep(sleep_s) - continue - used += 1 - if sector: - filled[sym] = sector - print(f" FMP {sym} → {sector}") - else: - print(f" FMP {sym} → (no sector)") - if sleep_s > 0: - await asyncio.sleep(sleep_s) - return filled, used - - -async def _main() -> None: - args = _parse_args() - snapshot = Path(args.snapshot) - if not snapshot.exists(): - raise SystemExit(f"Snapshot not found: {snapshot}") - - symbols = _snapshot_symbols(snapshot) - print(f"Universe: {len(symbols)} symbols from {snapshot}") - - existing = load_ticker_sector_map(args.out) - print(f"Existing map entries: {len(existing)}") - - print("Fetching public S&P 500 sector CSV…") - sp500 = _fetch_sp500_map() - print(f" SP500 CSV rows: {len(sp500)}") - - mapping = dict(existing) - from_sp500 = 0 - for sym in symbols: - if sym in mapping: - continue - if sym in sp500: - mapping[sym] = sp500[sym] - from_sp500 += 1 - print(f" Newly filled from SP500 CSV: {from_sp500}") - - missing = [s for s in symbols if s not in mapping] - fmp_used = 0 - from_fmp = 0 - if missing and not args.skip_fmp: - from app.config import settings - - if not settings.fmp_api_key: - print("WARNING: FMP key missing; leaving gaps unfilled") - else: - print(f"FMP fill for {len(missing)} missing (limit={args.fmp_limit})…") - filled, fmp_used = await _fill_from_fmp( - missing, - api_key=settings.fmp_api_key, - limit=int(args.fmp_limit), - sleep_s=float(args.sleep), - ) - mapping.update(filled) - from_fmp = len(filled) - - still_missing = [s for s in symbols if s not in mapping] - stats = coverage_stats(symbols, mapping) - meta = { - "built_at": datetime.now(timezone.utc).isoformat(), - "snapshot": str(snapshot.resolve()), - "from_existing": len(existing), - "from_sp500_csv": from_sp500, - "from_fmp": from_fmp, - "fmp_requests": fmp_used, - "still_missing": still_missing, - "coverage": { - k: stats[k] - for k in ("universe", "mapped", "mapped_pct", "with_etf", "by_sector") - }, - } - out_path = save_ticker_sector_map(mapping, args.out, meta=meta) - print(f"Wrote {out_path}") - print(json.dumps(meta["coverage"], indent=2)) - if still_missing: - print(f"Still missing ({len(still_missing)}): {still_missing[:40]}") - if len(still_missing) > 40: - print(f" … +{len(still_missing) - 40} more") - - -if __name__ == "__main__": - asyncio.run(_main()) diff --git a/scripts/fetch_sector_etfs_to_snapshot.py b/scripts/fetch_sector_etfs_to_snapshot.py deleted file mode 100644 index 5968a03..0000000 --- a/scripts/fetch_sector_etfs_to_snapshot.py +++ /dev/null @@ -1,187 +0,0 @@ -"""Fetch the 11 SPDR sector ETFs into a snapshot's ``benchmark_prices``. - -Research-only. Sector ETFs are auxiliary series (like SPY) — they must not -enter the tradable ticker universe or candidate replay. Storing them in -``benchmark_prices`` keeps that invariant. - -Also refreshes SPY on the same window so residual factors share a calendar. - -Example -------- - python scripts/fetch_sector_etfs_to_snapshot.py \\ - --snapshot backtest_snapshots/prod.sqlite --history-days 2200 -""" - -from __future__ import annotations - -import argparse -import asyncio -import sys -import time -from datetime import date, timedelta -from pathlib import Path - -from sqlalchemy import create_engine, text - -ROOT = Path(__file__).resolve().parents[1] -if str(ROOT) not in sys.path: - sys.path.insert(0, str(ROOT)) - -from app.ssl_bootstrap import bootstrap_ssl # noqa: E402 - -bootstrap_ssl() - -from app.services.sector_map import SECTOR_ETFS # noqa: E402 - - -def _parse_args() -> argparse.Namespace: - p = argparse.ArgumentParser(description=__doc__) - p.add_argument("--snapshot", default="backtest_snapshots/prod.sqlite") - p.add_argument( - "--history-days", - type=int, - default=2200, - help="Lookback calendar days (default ~6y; covers 5y snapshot + cushion).", - ) - p.add_argument("--sleep", type=float, default=0.25) - p.add_argument( - "--symbols", - default=None, - help="Comma-separated override (default: SPY + 11 sector ETFs).", - ) - return p.parse_args() - - -async def _fetch_and_upsert( - engine, - provider, - symbol: str, - start: date, - end: date, - *, - sleep_s: float, -) -> int: - from app.exceptions import ProviderError, RateLimitError - - for attempt in range(5): - try: - bars = await provider.fetch_ohlcv(symbol, start, end) - break - except RateLimitError: - wait = min(60.0, 2.0 ** attempt) - print(f" rate limited {symbol}; sleep {wait:.0f}s") - await asyncio.sleep(wait) - bars = [] - except ProviderError as exc: - if attempt + 1 >= 5: - raise - await asyncio.sleep(1.0) - print(f" retry {symbol}: {exc}") - bars = [] - else: - bars = [] - - if sleep_s > 0: - await asyncio.sleep(sleep_s) - - if not bars: - print(f" {symbol}: empty") - return 0 - - written = 0 - with engine.begin() as conn: - for bar in bars: - d = bar.date.isoformat() if hasattr(bar.date, "isoformat") else str(bar.date) - close = float(bar.close) - existing = conn.execute( - text( - "SELECT id, close FROM benchmark_prices " - "WHERE symbol = :sym AND date = :d" - ), - {"sym": symbol, "d": d}, - ).fetchone() - if existing is None: - # id is INTEGER PK — let sqlite autoincrement if possible - conn.execute( - text( - "INSERT INTO benchmark_prices (symbol, date, close) " - "VALUES (:sym, :d, :c)" - ), - {"sym": symbol, "d": d, "c": close}, - ) - written += 1 - elif abs(float(existing[1]) - close) > 1e-9: - conn.execute( - text( - "UPDATE benchmark_prices SET close = :c WHERE id = :id" - ), - {"c": close, "id": int(existing[0])}, - ) - written += 1 - print(f" {symbol}: {len(bars)} bars, {written} rows written/updated") - return written - - -async def _main() -> None: - args = _parse_args() - snapshot = Path(args.snapshot) - if not snapshot.exists(): - raise SystemExit(f"Snapshot not found: {snapshot}") - - from app.config import settings - from app.providers.alpaca import AlpacaOHLCVProvider - - if not settings.alpaca_api_key or not settings.alpaca_api_secret: - raise SystemExit("ALPACA_API_KEY / ALPACA_API_SECRET required") - - if args.symbols: - symbols = [s.strip().upper() for s in args.symbols.split(",") if s.strip()] - else: - symbols = ["SPY", *SECTOR_ETFS] - - end = date.today() - start = end - timedelta(days=int(args.history_days)) - provider = AlpacaOHLCVProvider(settings.alpaca_api_key, settings.alpaca_api_secret) - engine = create_engine( - f"sqlite:///{snapshot.resolve().as_posix()}", - future=True, - ) - - print(f"Snapshot: {snapshot}") - print(f"Window: {start} → {end}") - print(f"Symbols: {symbols}") - - t0 = time.monotonic() - total = 0 - try: - for sym in symbols: - n = await _fetch_and_upsert( - engine, provider, sym, start, end, sleep_s=float(args.sleep) - ) - total += n - finally: - engine.dispose() - - # Summary counts - engine = create_engine( - f"sqlite:///{snapshot.resolve().as_posix()}", - future=True, - ) - try: - with engine.connect() as conn: - rows = conn.execute( - text( - "SELECT symbol, COUNT(*), MIN(date), MAX(date) " - "FROM benchmark_prices GROUP BY symbol ORDER BY symbol" - ) - ).fetchall() - finally: - engine.dispose() - - print(f"Done in {(time.monotonic() - t0) / 60:.1f}m; rows touched={total}") - for sym, n, d0, d1 in rows: - print(f" {sym}: n={n} {d0}→{d1}") - - -if __name__ == "__main__": - asyncio.run(_main()) diff --git a/scripts/run_earnings_research.py b/scripts/run_earnings_research.py index 1e12264..bfc353e 100644 --- a/scripts/run_earnings_research.py +++ b/scripts/run_earnings_research.py @@ -466,11 +466,6 @@ async def _run_2b_ic( os.environ["BACKTEST_SNAPSHOT_OFFLINE"] = "1" os.environ["BACKTEST_SIGNAL_EVAL_ONLY"] = "1" - # Load sector map if present so sector signals also appear (side-by-side optional). - if Path("data/research/ticker_sector_map.json").exists(): - os.environ["BACKTEST_SECTOR_MAP_PATH"] = str( - Path("data/research/ticker_sector_map.json").resolve() - ) settings.backtest_workers = workers engine = create_async_engine(_sqlite_url(snapshot), pool_pre_ping=True) @@ -484,18 +479,6 @@ async def _run_2b_ic( (await db.execute(select(Ticker).order_by(Ticker.symbol))).scalars() ) spy = await load_benchmark_closes(db, "SPY") - sector_etf: dict[str, dict] = {} - try: - from app.services.sector_map import SECTOR_ETFS, load_ticker_sector_map - - symbol_to_sector = load_ticker_sector_map() - for etf in SECTOR_ETFS: - series = await load_benchmark_closes(db, etf) - if series: - sector_etf[etf] = series - except Exception: - symbol_to_sector = {} - sector_etf = {} prices: dict[str, tuple] = {} for idx, t in enumerate(tickers): @@ -521,9 +504,6 @@ async def _run_2b_ic( ], spy, symbol=t.symbol, - sector_etf_closes=bt._sector_etf_closes_for_symbol( - t.symbol, symbol_to_sector, sector_etf - ), ) for name, weeks in series.items(): for wk, pairs in weeks.items(): @@ -533,9 +513,6 @@ async def _run_2b_ic( if not quiet: print() - if symbol_to_sector: - bt._inject_sector_demeaned_momentum(collected, symbol_to_sector) - # SUE series. events_by_sym: dict[str, list[dict]] = defaultdict(list) for ev in events: @@ -709,8 +686,6 @@ async def _run_2b_ic( for name in ( "mom_12_1", "mom_12_1_resid", - "mom_12_1_sector_resid", - "mom_12_1_sector_demeaned", "sue_latest", "fip_id", ) @@ -784,7 +759,6 @@ def _write_md(path: Path, payload: dict) -> None: "mom_12_1", "mom_12_1_resid", "sue_latest", - "mom_12_1_sector_resid", "fip_id", ): r = side.get(name) or {} diff --git a/scripts/run_history_depth_research.py b/scripts/run_history_depth_research.py deleted file mode 100644 index cfe2285..0000000 --- a/scripts/run_history_depth_research.py +++ /dev/null @@ -1,480 +0,0 @@ -"""History-depth extension research (local / MacBook). - -Phases ------- - coverage — bars per calendar year; no rebuild - harness — race-guard snapshot, full signal_eval, era split pre/post-2021 - -Does not retune production knobs. Does not modify scheduler/gates. - -Example -------- - python scripts/run_history_depth_research.py --phase coverage \\ - --snapshot backtest_snapshots/prod.sqlite - - python scripts/run_history_depth_research.py --phase harness \\ - --snapshot backtest_snapshots/research.sqlite --workers 8 --allow-spawn -""" - -from __future__ import annotations - -import argparse -import asyncio -import json -import os -import sys -from collections import defaultdict -from datetime import date, datetime -from pathlib import Path -from typing import Any - -from sqlalchemy import create_engine, text -from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine - -ROOT = Path(__file__).resolve().parents[1] -if str(ROOT) not in sys.path: - sys.path.insert(0, str(ROOT)) - -from app.ssl_bootstrap import bootstrap_ssl # noqa: E402 - -bootstrap_ssl() - -ERA_SPLIT = date(2021, 1, 1) -SURVIVORSHIP_BANNER = ( - "SURVIVORSHIP BIAS: today's constituents backfilled historically. " - "Absolute Sharpe/CAGR levels on deep history are optimistic. " - "Use RELATIVE signal IC comparisons and era stability only — not levels." -) - - -def _sqlite_url(path: Path) -> str: - return f"sqlite+aiosqlite:///{path.resolve().as_posix()}" - - -def _parse_args() -> argparse.Namespace: - p = argparse.ArgumentParser(description=__doc__) - p.add_argument("--phase", choices=("coverage", "harness", "all"), default="all") - p.add_argument("--snapshot", default="backtest_snapshots/research.sqlite") - p.add_argument("--workers", type=int, default=8) - p.add_argument("--allow-spawn", action="store_true") - p.add_argument("--quiet", action="store_true") - p.add_argument("--out", default=None) - return p.parse_args() - - -def _coverage_report(snapshot: Path) -> dict[str, Any]: - engine = create_engine( - f"sqlite:///{snapshot.resolve().as_posix()}", - future=True, - ) - try: - with engine.connect() as conn: - ticker_n = int(conn.execute(text("SELECT COUNT(*) FROM tickers")).scalar_one()) - ohlcv_n = int( - conn.execute(text("SELECT COUNT(*) FROM ohlcv_records")).scalar_one() - ) - d_range = conn.execute( - text("SELECT MIN(date), MAX(date) FROM ohlcv_records") - ).fetchone() - # Bars per calendar year (global). - by_year = conn.execute( - text( - """ - SELECT substr(date, 1, 4) AS y, COUNT(*) AS n, - COUNT(DISTINCT ticker_id) AS tickers - FROM ohlcv_records - GROUP BY substr(date, 1, 4) - ORDER BY y - """ - ) - ).fetchall() - # Per-symbol min/max date + bar count (summary percentiles). - per_sym = conn.execute( - text( - """ - SELECT t.symbol, COUNT(*) AS n, MIN(o.date), MAX(o.date) - FROM ohlcv_records o - JOIN tickers t ON t.id = o.ticker_id - GROUP BY t.symbol - """ - ) - ).fetchall() - finally: - engine.dispose() - - ns = sorted(int(r[1]) for r in per_sym) - def pct(p: float) -> int | None: - if not ns: - return None - i = int(round(p * (len(ns) - 1))) - return ns[i] - - starts = sorted(str(r[2]) for r in per_sym if r[2]) - start_hist: dict[str, int] = defaultdict(int) - for s in starts: - start_hist[s[:4]] += 1 - - return { - "snapshot": str(snapshot.resolve()), - "ticker_count": ticker_n, - "ohlcv_row_count": ohlcv_n, - "date_range": {"min": d_range[0], "max": d_range[1]}, - "bars_per_year": [ - {"year": y, "bars": n, "tickers_with_bars": t} for y, n, t in by_year - ], - "bars_per_symbol": { - "min": ns[0] if ns else None, - "p10": pct(0.10), - "p50": pct(0.50), - "p90": pct(0.90), - "max": ns[-1] if ns else None, - }, - "symbols_by_start_year": dict(sorted(start_hist.items())), - "note": ( - "Where ticker counts drop in early years, the feed (or listing history) " - "thins — do not treat those years as a full 505-name cross-section." - ), - "survivorship_banner": SURVIVORSHIP_BANNER, - } - - -def _assert_complete(snapshot: Path) -> dict[str, Any]: - from scripts.research_snapshot_manifest import ( # type: ignore - assert_research_snapshot_complete, - load_manifest, - ) - - m = load_manifest(snapshot) - if m is None: - # Prod snapshot may lack manifest; still require healthy bar depth. - eng = create_engine( - f"sqlite:///{snapshot.resolve().as_posix()}", - future=True, - ) - try: - with eng.connect() as conn: - avg = conn.execute( - text( - """ - SELECT AVG(c) FROM ( - SELECT COUNT(*) AS c FROM ohlcv_records GROUP BY ticker_id - ) - """ - ) - ).scalar_one() - finally: - eng.dispose() - if avg is None or float(avg) < 400: - raise SystemExit( - f"No completion manifest and avg bars={avg} look short. " - "Rebuild research.sqlite via extend_snapshot_universe.py" - ) - return {"manifest": None, "avg_bars": float(avg), "ok": True} - return {"manifest": assert_research_snapshot_complete(snapshot), "ok": True} - - -async def _harness(snapshot: Path, *, workers: int, quiet: bool) -> dict[str, Any]: - from app.config import settings - from app.services.backtest_service import run_backtest - - os.environ["BACKTEST_SNAPSHOT_OFFLINE"] = "1" - os.environ["BACKTEST_SIGNAL_EVAL_ONLY"] = "1" - if Path("data/research/ticker_sector_map.json").exists(): - os.environ["BACKTEST_SECTOR_MAP_PATH"] = str( - Path("data/research/ticker_sector_map.json").resolve() - ) - settings.backtest_workers = workers - - engine = create_async_engine(_sqlite_url(snapshot), pool_pre_ping=True) - Session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) - - def progress(done: int, total: int, symbol: str) -> None: - if quiet: - return - print(f" progress {done}/{total} {symbol}", end="\r", flush=True) - - try: - async with Session() as db: - report = await run_backtest(db, progress_cb=progress, cadence="weekly") - finally: - await engine.dispose() - if not quiet: - print() - - signal_eval = report.get("signal_eval") or [] - - # Era-split IC: recompute from collected is not available post-run. - # Approximate via second pass is expensive; instead document that era split - # requires collecting weekly ICs. We re-run evaluation if the report embeds - # nothing — for v1, call internal collection is too heavy to duplicate. - # Lightweight approach: mark era_split as requiring BACKTEST with custom - # filter — implemented below by re-scoring from a dedicated collection pass. - era = await _era_split_ics(snapshot, workers=workers, quiet=quiet) - - return { - "survivorship_banner": SURVIVORSHIP_BANNER, - "signal_eval": signal_eval, - "era_split": era, - "params": report.get("params"), - "tickers": report.get("tickers"), - "generated_at_run": report.get("generated_at"), - } - - -async def _era_split_ics( - snapshot: Path, *, workers: int, quiet: bool -) -> dict[str, Any]: - """Collect weekly signal series and evaluate pre/post ERA_SPLIT separately.""" - from app.config import settings - from app.services import backtest_service as bt - from app.services.benchmark_service import load_benchmark_closes - from app.models.ticker import Ticker - from sqlalchemy import select - from collections import defaultdict as dd - - os.environ["BACKTEST_SNAPSHOT_OFFLINE"] = "1" - settings.backtest_workers = max(1, workers) - - engine = create_async_engine(_sqlite_url(snapshot), pool_pre_ping=True) - Session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) - - collected: dict = dd(lambda: dd(list)) - try: - async with Session() as db: - tickers = list( - (await db.execute(select(Ticker).order_by(Ticker.symbol))).scalars() - ) - spy = await load_benchmark_closes(db, "SPY") - symbol_to_sector = {} - sector_etf: dict = {} - try: - from app.services.sector_map import ( - SECTOR_ETFS, - load_ticker_sector_map, - ) - - symbol_to_sector = load_ticker_sector_map() - for etf in SECTOR_ETFS: - series = await load_benchmark_closes(db, etf) - if series: - sector_etf[etf] = series - except Exception: - pass - - for idx, t in enumerate(tickers): - if not quiet and idx % 100 == 0: - print(f" era-collect {idx}/{len(tickers)}", end="\r", flush=True) - cols = await bt._fetch_columns(db, t.symbol) - if cols is None: - continue - records = [ - type( - "R", - (), - { - "date": date.fromordinal(int(cols[0][i])), - "close": cols[4][i], - "high": cols[2][i], - "volume": cols[5][i] if len(cols) > 5 else 0, - }, - )() - for i in range(len(cols[0])) - ] - series = bt._signal_series( - records, - spy, - symbol=t.symbol, - sector_etf_closes=bt._sector_etf_closes_for_symbol( - t.symbol, symbol_to_sector, sector_etf - ), - ) - for name, weeks in series.items(): - for wk, pairs in weeks.items(): - collected[name][wk].extend(pairs) - if symbol_to_sector: - bt._inject_sector_demeaned_momentum(collected, symbol_to_sector) - finally: - await engine.dispose() - if not quiet: - print() - - def _filter_era(coll: dict, *, pre: bool) -> dict: - out: dict = dd(lambda: dd(list)) - for name, weeks in coll.items(): - for wk, recs in weeks.items(): - # ISO week key (year, week) — approximate era by ISO year. - year = int(wk[0]) if isinstance(wk, tuple) else int(str(wk)[:4]) - if pre and year >= ERA_SPLIT.year: - continue - if not pre and year < ERA_SPLIT.year: - continue - out[name][wk].extend(recs) - return out - - pre_eval = bt._signal_evaluation(_filter_era(collected, pre=True)) - post_eval = bt._signal_evaluation(_filter_era(collected, pre=False)) - full_eval = bt._signal_evaluation(collected) - - def _index(rows: list[dict]) -> dict[str, dict]: - return {r["signal"]: r for r in rows} - - return { - "era_split_date": ERA_SPLIT.isoformat(), - "note": "Diagnostic only — not a tuning input. Nested lookbacks are not OOS.", - "full": _index(full_eval), - "pre_2021": _index(pre_eval), - "post_2021": _index(post_eval), - } - - -def _write_md(path: Path, payload: dict) -> None: - pre = path.read_text(encoding="utf-8") if path.exists() else "" - marker = "## Results" - idx = pre.find(marker) - header = pre[:idx] if idx >= 0 else pre.split("## Verdict")[0] - - lines = [ - header.rstrip(), - "", - "## Results", - "", - f"Generated: `{payload.get('generated_at')}`", - "", - f"> **{SURVIVORSHIP_BANNER}**", - "", - "### Coverage", - "", - f"```json\n{json.dumps(payload.get('coverage') or {}, indent=2, default=str)}\n```", - "", - "### Race guard", - "", - f"```json\n{json.dumps(payload.get('race_guard') or {}, indent=2, default=str)}\n```", - "", - "### Signal IC (full extended window)", - "", - ] - harness = payload.get("harness") or {} - rows = harness.get("signal_eval") or [] - if rows: - lines.extend([ - "| signal | mean_ic | ic_t_stat | weeks | avg_N | reliable |", - "|---|---:|---:|---:|---:|---|", - ]) - for r in rows: - lines.append( - f"| {r.get('signal')} | {r.get('mean_ic')} | {r.get('ic_t_stat')} | " - f"{r.get('weeks')} | {r.get('avg_cross_section')} | {r.get('reliable')} |" - ) - else: - lines.append("_Harness not run this pass._") - - era = (harness.get("era_split") or {}) - lines.extend(["", "### Era split (diagnostic only)", ""]) - if era: - for label in ("full", "pre_2021", "post_2021"): - block = era.get(label) or {} - lines.append(f"#### {label}") - lines.append("") - lines.append("| signal | mean_ic | t | weeks | N |") - lines.append("|---|---:|---:|---:|---:|") - for name in sorted(block): - r = block[name] - lines.append( - f"| {name} | {r.get('mean_ic')} | {r.get('ic_t_stat')} | " - f"{r.get('weeks')} | {r.get('avg_cross_section')} |" - ) - lines.append("") - else: - lines.append("_No era split._") - - lines.extend([ - "", - "## Verdict", - "", - f"**{payload.get('verdict')}**", - "", - payload.get("verdict_detail") or "", - "", - "## What a human must decide next", - "", - payload.get("human_next") - or "- Do not retune production knobs from this report without review.", - "", - f"Artifacts: `{payload.get('report_path')}`", - "", - ]) - path.write_text("\n".join(lines) + "\n", encoding="utf-8") - - -async def _main() -> None: - args = _parse_args() - snapshot = Path(args.snapshot) - if not snapshot.exists(): - raise SystemExit(f"Missing snapshot: {snapshot}") - if args.allow_spawn: - os.environ["BACKTEST_ALLOW_SPAWN"] = "1" - - coverage = None - race = None - harness = None - - if args.phase in ("coverage", "all"): - print("Coverage probe…") - coverage = _coverage_report(snapshot) - print( - f" tickers={coverage['ticker_count']} ohlcv={coverage['ohlcv_row_count']} " - f"range={coverage['date_range']}" - ) - for row in coverage["bars_per_year"]: - print( - f" year {row['year']}: bars={row['bars']} " - f"tickers={row['tickers_with_bars']}" - ) - - if args.phase in ("harness", "all"): - print("Race guard…") - race = _assert_complete(snapshot) - print(f" ok={race.get('ok')}") - print("Full harness + era split (LONG)…") - print(f" {SURVIVORSHIP_BANNER}") - harness = await _harness( - snapshot, workers=args.workers, quiet=args.quiet - ) - - stamp = datetime.now().strftime("%Y%m%d-%H%M%S") - out = ( - Path(args.out) - if args.out - else Path("reports") / f"history-depth-{stamp}.json" - ) - payload = { - "generated_at": datetime.now().isoformat(), - "survivorship_banner": SURVIVORSHIP_BANNER, - "coverage": coverage, - "race_guard": race, - "harness": harness, - "verdict": "PENDING_HUMAN" if harness else "COVERAGE_ONLY", - "verdict_detail": ( - "Harness complete — human interprets relative IC / era stability. " - "No production retune from this artifact." - if harness - else "Coverage probe only; run --phase harness after deep rebuild." - ), - "human_next": ( - "- Compare sector residual vs market residual across eras.\n" - "- If pre-2021 IC collapses, park Task 1 wire-in.\n" - "- Do not retune production knobs on deep history levels." - ), - "report_path": str(out.as_posix()), - } - out.parent.mkdir(parents=True, exist_ok=True) - out.write_text(json.dumps(payload, indent=2, default=str) + "\n", encoding="utf-8") - md = Path("docs/research/history-depth-extension.md") - _write_md(md, payload) - out.with_suffix(".md").write_text(md.read_text(encoding="utf-8"), encoding="utf-8") - print(f"Wrote {out}") - print(f"Wrote {md}") - - -if __name__ == "__main__": - asyncio.run(_main()) diff --git a/scripts/run_sector_resid_deep_test.py b/scripts/run_sector_resid_deep_test.py deleted file mode 100644 index a5cbfff..0000000 --- a/scripts/run_sector_resid_deep_test.py +++ /dev/null @@ -1,1047 +0,0 @@ -#!/usr/bin/env python3 -"""Terminal sector-residual deep test: deepen shallow symbols → ONE masked run → PASS/FAIL. - -Repairs the two-tier history-depth defect (prod/ETF names left at ~5y while breadth -got 5000d), then runs a single liquid-breadth signal harness and grades -``mom_12_1_sector_resid`` against the pre-registered rule. - -Local research only. No production changes. - -MacBook -------- - # On deep research.sqlite from the prior history-depth rebuild: - python scripts/run_sector_resid_deep_test.py \\ - --snapshot backtest_snapshots/research.sqlite \\ - --workers 8 --allow-spawn - - # Skip re-fetch if Step-1 already done and sanity-check passes: - python scripts/run_sector_resid_deep_test.py --skip-deepen --workers 8 --allow-spawn -""" - -from __future__ import annotations - -import argparse -import asyncio -import json -import math -import os -import sys -import time -from collections import defaultdict -from datetime import date, datetime, timedelta, timezone -from pathlib import Path -from typing import Any - -from sqlalchemy import create_engine, text -from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine - -ROOT = Path(__file__).resolve().parents[1] -if str(ROOT) not in sys.path: - sys.path.insert(0, str(ROOT)) - -from app.ssl_bootstrap import bootstrap_ssl # noqa: E402 - -bootstrap_ssl() - -from app.services.sector_map import ( # noqa: E402 - DEFAULT_SECTOR_MAP_PATH, - SECTOR_ETFS, - load_ticker_sector_map, -) -from scripts.research_snapshot_manifest import ( # noqa: E402 - assert_research_snapshot_complete, - clear_manifest, - write_completion_manifest, -) - -ERA_SPLIT = date(2021, 1, 1) -IRON_IC = 0.03 -# "weeks ≫ 35 (expect ~80)" — mechanical floor for "data fix worked" -MIN_WEEKS_DEEP = 50 -SANITY_MEGACAPS = ("AAPL", "MSFT", "JPM", "XOM", "JNJ") -SURVIVORSHIP = ( - "SURVIVORSHIP BIAS: today's constituents backfilled. Relative IC only — not levels." -) - - -def _parse_args() -> argparse.Namespace: - p = argparse.ArgumentParser(description=__doc__) - p.add_argument("--snapshot", default="backtest_snapshots/research.sqlite") - p.add_argument("--history-days", type=int, default=5000) - p.add_argument("--sleep", type=float, default=0.15) - p.add_argument("--workers", type=int, default=8) - p.add_argument("--allow-spawn", action="store_true") - p.add_argument( - "--skip-deepen", - action="store_true", - help="Skip Step-1 re-fetch; only sanity-check + harness.", - ) - p.add_argument( - "--sector-map", - default=str(DEFAULT_SECTOR_MAP_PATH), - ) - p.add_argument("--liquid-breadth", type=int, default=1500) - p.add_argument("--min-price", type=float, default=5.0) - p.add_argument("--quiet", action="store_true") - p.add_argument("--out", default=None) - return p.parse_args() - - -def _sqlite_url(path: Path) -> str: - return f"sqlite+aiosqlite:///{path.resolve().as_posix()}" - - -def _symbol_depth(snapshot: Path) -> list[dict[str, Any]]: - engine = create_engine( - f"sqlite:///{snapshot.resolve().as_posix()}", - future=True, - ) - try: - with engine.connect() as conn: - rows = conn.execute( - text( - """ - SELECT t.symbol, COUNT(*) AS n, MIN(o.date), MAX(o.date) - FROM ohlcv_records o - JOIN tickers t ON t.id = o.ticker_id - GROUP BY t.symbol - """ - ) - ).fetchall() - finally: - engine.dispose() - out = [] - for sym, n, d0, d1 in rows: - out.append({ - "symbol": str(sym).upper(), - "bars": int(n), - "min_date": str(d0)[:10] if d0 else None, - "max_date": str(d1)[:10] if d1 else None, - }) - return out - - -def _derive_shallow( - depths: list[dict[str, Any]], - *, - lag_days: int = 400, -) -> tuple[list[str], dict[str, Any]]: - """Symbols whose earliest bar starts materially later than the deep cohort.""" - starts: list[tuple[str, date]] = [] - for row in depths: - if not row.get("min_date"): - continue - starts.append((row["symbol"], date.fromisoformat(row["min_date"]))) - if not starts: - return [], {"error": "no symbols with min_date"} - - # Deep cohort start ≈ 10th percentile of earliest dates (early = deep). - ordered = sorted(d for _, d in starts) - p10 = ordered[max(0, int(0.10 * (len(ordered) - 1)))] - cutoff = p10 + timedelta(days=lag_days) - shallow = sorted({sym for sym, d in starts if d > cutoff}) - meta = { - "n_symbols": len(starts), - "deep_cohort_p10_start": p10.isoformat(), - "shallow_cutoff": cutoff.isoformat(), - "lag_days": lag_days, - "n_shallow": len(shallow), - "shallow_start_histogram": _year_hist( - [d for sym, d in starts if sym in set(shallow)] - ), - "deep_start_histogram": _year_hist( - [d for sym, d in starts if sym not in set(shallow)] - ), - "shallow_sample": shallow[:30], - } - return shallow, meta - - -def _year_hist(dates: list[date]) -> dict[str, int]: - h: dict[str, int] = defaultdict(int) - for d in dates: - h[str(d.year)] += 1 - return dict(sorted(h.items())) - - -async def _fetch_and_replace_ohlcv( - engine, - provider, - symbol: str, - start: date, - end: date, - *, - sleep_s: float, - max_retries: int = 5, -) -> int: - from app.exceptions import ProviderError, RateLimitError - - bars = [] - for attempt in range(max_retries): - try: - bars = await provider.fetch_ohlcv(symbol, start, end) - break - except RateLimitError: - wait = min(60.0, 2.0 ** attempt) - print(f" rate limited {symbol}; sleep {wait:.0f}s") - await asyncio.sleep(wait) - except ProviderError as exc: - if attempt + 1 >= max_retries: - raise - await asyncio.sleep(1.0) - _ = exc - if sleep_s > 0: - await asyncio.sleep(sleep_s) - if not bars: - return 0 - - with engine.begin() as write: - tid = write.execute( - text("SELECT id FROM tickers WHERE symbol = :s"), - {"s": symbol}, - ).scalar_one_or_none() - if tid is None: - write.execute( - text( - "INSERT INTO tickers (symbol, name, created_at) " - "VALUES (:s, NULL, :c)" - ), - {"s": symbol, "c": datetime.now(timezone.utc).isoformat()}, - ) - tid = write.execute( - text("SELECT id FROM tickers WHERE symbol = :s"), - {"s": symbol}, - ).scalar_one() - # Full replace for this symbol so shallow tails cannot linger. - write.execute( - text("DELETE FROM ohlcv_records WHERE ticker_id = :tid"), - {"tid": int(tid)}, - ) - now = datetime.now(timezone.utc).replace(tzinfo=None).isoformat() - write.execute( - text( - """ - INSERT INTO ohlcv_records - (ticker_id, date, open, high, low, close, volume, created_at) - VALUES - (:ticker_id, :date, :open, :high, :low, :close, :volume, :created_at) - """ - ), - [ - { - "ticker_id": int(tid), - "date": b.date.isoformat() - if hasattr(b.date, "isoformat") - else str(b.date), - "open": float(b.open), - "high": float(b.high), - "low": float(b.low), - "close": float(b.close), - "volume": int(b.volume), - "created_at": now, - } - for b in bars - ], - ) - return len(bars) - - -async def _deepen_sector_etfs( - snapshot: Path, *, history_days: int, sleep_s: float -) -> dict[str, Any]: - """Refresh SPY + 11 sector ETFs in benchmark_prices to full depth.""" - # Reuse the existing CLI helper for consistency. - from scripts.fetch_sector_etfs_to_snapshot import _fetch_and_upsert - from app.config import settings - from app.providers.alpaca import AlpacaOHLCVProvider - - if not settings.alpaca_api_key or not settings.alpaca_api_secret: - raise SystemExit("ALPACA keys required to deepen sector ETFs") - - provider = AlpacaOHLCVProvider(settings.alpaca_api_key, settings.alpaca_api_secret) - end = date.today() - start = end - timedelta(days=history_days) - engine = create_engine( - f"sqlite:///{snapshot.resolve().as_posix()}", - future=True, - ) - symbols = ["SPY", *SECTOR_ETFS] - written: dict[str, int] = {} - try: - for sym in symbols: - n = await _fetch_and_upsert( - engine, provider, sym, start, end, sleep_s=sleep_s - ) - written[sym] = n - finally: - engine.dispose() - - engine = create_engine( - f"sqlite:///{snapshot.resolve().as_posix()}", - future=True, - ) - try: - with engine.connect() as conn: - rows = conn.execute( - text( - "SELECT symbol, COUNT(*), MIN(date), MAX(date) " - "FROM benchmark_prices GROUP BY symbol ORDER BY symbol" - ) - ).fetchall() - finally: - engine.dispose() - return { - "written": written, - "benchmark_summary": [ - {"symbol": s, "n": n, "min": d0, "max": d1} for s, n, d0, d1 in rows - ], - } - - -def _sanity_check(snapshot: Path, *, history_days: int) -> dict[str, Any]: - """Pass if megacaps + sector ETFs sit at the *empirical feed floor*, not calendar 5000d. - - Alpaca daily history for this stack bottoms out around 2016-01-04 (~2649 bars) - even when history_days=5000 is requested. That is feed coverage, not a two-tier - snapshot bug. Fail only if megacaps are still stuck near the old ~2021 prod floor - or if sector ETFs are missing / shorter than the SPY series (except XLC listing). - """ - depths = {r["symbol"]: r for r in _symbol_depth(snapshot)} - - engine = create_engine( - f"sqlite:///{snapshot.resolve().as_posix()}", - future=True, - ) - try: - with engine.connect() as conn: - spy_row = conn.execute( - text( - "SELECT COUNT(*), MIN(date), MAX(date) FROM benchmark_prices " - "WHERE symbol = 'SPY'" - ) - ).fetchone() - etf_rows = conn.execute( - text( - "SELECT symbol, COUNT(*), MIN(date), MAX(date) " - "FROM benchmark_prices WHERE symbol IN " - f"({','.join(repr(s) for s in SECTOR_ETFS)}) " - "GROUP BY symbol" - ) - ).fetchall() - finally: - engine.dispose() - - spy_n, spy_min, spy_max = spy_row if spy_row else (0, None, None) - feed_floor = ( - date.fromisoformat(str(spy_min)[:10]) - if spy_min - else date(2016, 1, 4) - ) - # Megacaps must match the feed floor within a few sessions (not calendar-5000). - megacap_slack_days = 10 - # Old two-tier defect left prod names at ~2021-06; anything still after this fails. - old_shallow_floor = date(2020, 1, 1) - - megacap = {} - ok_mega = True - mega_reasons: list[str] = [] - for sym in SANITY_MEGACAPS: - row = depths.get(sym) - megacap[sym] = row - if row is None or not row.get("min_date"): - ok_mega = False - mega_reasons.append(f"{sym}: missing") - continue - d0 = date.fromisoformat(row["min_date"]) - if d0 > old_shallow_floor: - ok_mega = False - mega_reasons.append( - f"{sym}: min_date={d0} still after {old_shallow_floor} (two-tier unrepaired)" - ) - elif d0 > feed_floor + timedelta(days=megacap_slack_days): - ok_mega = False - mega_reasons.append( - f"{sym}: min_date={d0} later than SPY feed floor {feed_floor}" - ) - - etf_info = { - s: {"n": n, "min": d0, "max": d1} for s, n, d0, d1 in etf_rows - } - deep_etfs = 0 - etf_reasons: list[str] = [] - for sym in SECTOR_ETFS: - info = etf_info.get(sym) - if not info or not info["min"]: - etf_reasons.append(f"{sym}: missing") - continue - d0 = date.fromisoformat(str(info["min"])[:10]) - if sym == "XLC": - # Listed 2018-06-18/19. - if d0 <= date(2018, 7, 15): - deep_etfs += 1 - else: - etf_reasons.append(f"XLC: min_date={d0} later than listing floor") - else: - if d0 <= feed_floor + timedelta(days=megacap_slack_days): - deep_etfs += 1 - else: - etf_reasons.append( - f"{sym}: min_date={d0} later than SPY feed floor {feed_floor}" - ) - - ok_etf = deep_etfs >= 10 - still_shallow, _ = _derive_shallow(list(depths.values()), lag_days=400) - note_xlc = ( - "XLC lists mid-2018 → Communication Services residual coverage from ~mid-2019." - ) - note_feed = ( - f"Empirical Alpaca floor observed via SPY: {feed_floor.isoformat()} " - f"(n={spy_n}). Calendar history_days={history_days} is a request cap, not a " - "guarantee — sanity grades against the feed floor, not 5000 calendar days." - ) - - passed = bool(ok_mega and ok_etf) - return { - "passed": passed, - "megacap": megacap, - "megacap_ok": ok_mega, - "megacap_reasons": mega_reasons, - "feed_floor": feed_floor.isoformat(), - "spy_benchmark": {"n": spy_n, "min": spy_min, "max": spy_max}, - "old_shallow_floor": old_shallow_floor.isoformat(), - "sector_etfs": etf_info, - "sector_etfs_deep_count": deep_etfs, - "sector_etfs_ok": ok_etf, - "sector_etf_reasons": etf_reasons, - "still_shallow_count": len(still_shallow), - "still_shallow_sample": still_shallow[:20], - "still_shallow_note": ( - "Remaining 'shallow' names are mostly post-2017 IPOs/listings — expected, " - "not a two-tier defect." - ), - "xlc_note": note_xlc, - "feed_note": note_feed, - "target_history_days": history_days, - } - - -async def _step1_deepen( - snapshot: Path, - *, - history_days: int, - sleep_s: float, - quiet: bool, -) -> dict[str, Any]: - from app.config import settings - from app.providers.alpaca import AlpacaOHLCVProvider - - if not settings.alpaca_api_key or not settings.alpaca_api_secret: - raise SystemExit("ALPACA_API_KEY / ALPACA_API_SECRET required") - - clear_manifest(snapshot) - depths = _symbol_depth(snapshot) - shallow, shallow_meta = _derive_shallow(depths) - print( - f"Shallow symbols to deepen: {len(shallow)} " - f"(p10 deep start={shallow_meta.get('deep_cohort_p10_start')}, " - f"cutoff={shallow_meta.get('shallow_cutoff')})" - ) - if not shallow: - print("WARNING: no shallow symbols detected — snapshot may already be uniform") - - provider = AlpacaOHLCVProvider(settings.alpaca_api_key, settings.alpaca_api_secret) - end = date.today() - start = end - timedelta(days=history_days) - engine = create_engine( - f"sqlite:///{snapshot.resolve().as_posix()}", - future=True, - ) - - ok = fail = 0 - t0 = time.monotonic() - try: - for i, sym in enumerate(shallow, 1): - try: - n = await _fetch_and_replace_ohlcv( - engine, provider, sym, start, end, sleep_s=sleep_s - ) - if n <= 0: - fail += 1 - if not quiet: - print(f" [{i}/{len(shallow)}] {sym} empty") - continue - ok += 1 - if not quiet and (i % 25 == 0 or i == len(shallow)): - print( - f" progress {i}/{len(shallow)} ok={ok} fail={fail} " - f"last={sym} bars={n} elapsed={(time.monotonic()-t0)/60:.1f}m" - ) - except Exception as exc: - fail += 1 - print(f" [{i}/{len(shallow)}] {sym} FAIL {exc}") - finally: - engine.dispose() - - print("Deepening SPY + sector ETFs in benchmark_prices…") - etf_result = await _deepen_sector_etfs( - snapshot, history_days=history_days, sleep_s=sleep_s - ) - - # Manifest: full completion after deepen (no --limit). - from scripts.research_snapshot_manifest import _count_snapshot - - counts = _count_snapshot(snapshot) - manifest_path = write_completion_manifest( - snapshot, - complete=True, - sources={"deepen": "sector_resid_deep_test step1"}, - history_days=history_days, - min_bars=None, - fetch_ok=ok, - fetch_fail=fail, - limit=None, - extra={ - "shallow_meta": shallow_meta, - "shallow_fetched_ok": ok, - "shallow_fetched_fail": fail, - "etf_refresh": etf_result.get("written"), - "counts_after": counts, - }, - ) - print(f"Manifest written: {manifest_path}") - - sanity = _sanity_check(snapshot, history_days=history_days) - return { - "shallow_meta": shallow_meta, - "shallow_list_n": len(shallow), - "fetch_ok": ok, - "fetch_fail": fail, - "etf_refresh": etf_result, - "sanity": sanity, - "manifest_path": str(manifest_path), - } - - -async def _one_masked_run( - snapshot: Path, - *, - sector_map_path: Path, - liquid_breadth: int, - min_price: float, - workers: int, - quiet: bool, -) -> dict[str, Any]: - """Single collection under liquid mask; full + era IC from the same series.""" - from app.config import settings - from app.services import backtest_service as bt - from app.services.benchmark_service import load_benchmark_closes - from app.models.ticker import Ticker - from sqlalchemy import select - - os.environ["BACKTEST_SNAPSHOT_OFFLINE"] = "1" - os.environ["BACKTEST_SIGNAL_EVAL_ONLY"] = "1" - os.environ["BACKTEST_LIQUID_BREADTH"] = str(int(liquid_breadth)) - os.environ["BACKTEST_LIQUID_MIN_PRICE"] = str(float(min_price)) - os.environ["BACKTEST_SECTOR_MAP_PATH"] = str(sector_map_path.resolve()) - if workers: - settings.backtest_workers = workers - - # One collection pass (not run_backtest twice). - engine = create_async_engine(_sqlite_url(snapshot), pool_pre_ping=True) - Session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) - collected: dict = defaultdict(lambda: defaultdict(list)) - symbol_to_sector = load_ticker_sector_map(sector_map_path) - - try: - async with Session() as db: - tickers = list( - (await db.execute(select(Ticker).order_by(Ticker.symbol))).scalars() - ) - spy = await load_benchmark_closes(db, "SPY") - sector_etf: dict[str, dict] = {} - for etf in SECTOR_ETFS: - series = await load_benchmark_closes(db, etf) - if series: - sector_etf[etf] = series - - total = len(tickers) - for idx, t in enumerate(tickers): - if not quiet and idx % 100 == 0: - print(f" collect {idx}/{total}", end="\r", flush=True) - cols = await bt._fetch_columns(db, t.symbol) - if cols is None: - continue - records = [ - type( - "R", - (), - { - "date": date.fromordinal(int(cols[0][i])), - "close": cols[4][i], - "high": cols[2][i], - "volume": cols[5][i] if len(cols) > 5 else 0, - }, - )() - for i in range(len(cols[0])) - ] - etf_closes = bt._sector_etf_closes_for_symbol( - t.symbol, symbol_to_sector, sector_etf - ) - series = bt._signal_series( - records, - spy, - symbol=t.symbol, - sector_etf_closes=etf_closes, - ) - for name, weeks in series.items(): - for wk, pairs in weeks.items(): - collected[name][wk].extend(pairs) - finally: - await engine.dispose() - if not quiet: - print() - - if symbol_to_sector: - bt._inject_sector_demeaned_momentum(collected, symbol_to_sector) - - full_eval = bt._signal_evaluation(dict(collected)) - - def _filter_era(coll: dict, *, pre: bool) -> dict: - out: dict = defaultdict(lambda: defaultdict(list)) - for name, weeks in coll.items(): - for wk, recs in weeks.items(): - year = int(wk[0]) if isinstance(wk, tuple) else int(str(wk)[:4]) - if pre and year >= ERA_SPLIT.year: - continue - if not pre and year < ERA_SPLIT.year: - continue - out[name][wk].extend(recs) - return out - - pre_eval = bt._signal_evaluation(_filter_era(collected, pre=True)) - post_eval = bt._signal_evaluation(_filter_era(collected, pre=False)) - - # Identical-subset: resid IC only where sector_resid exists (same CS for t rule). - identical = _identical_subset_eval(collected, bt) - - def _idx(rows: list[dict]) -> dict[str, dict]: - return {r["signal"]: r for r in rows} - - # Mask bind diagnostics from liquid-aware rows if present. - mask_diag = _mask_diagnostics(full_eval) - - return { - "liquid_breadth_top_n": liquid_breadth, - "liquid_min_price": min_price, - "survivorship_banner": SURVIVORSHIP, - "signal_eval": full_eval, - "signal_eval_by_name": _idx(full_eval), - "era_split": { - "era_split_date": ERA_SPLIT.isoformat(), - "note": "Diagnostic only — not a tuning input.", - "pre_2021": _idx(pre_eval), - "post_2021": _idx(post_eval), - }, - "identical_subset_sector_cs": identical, - "mask_diagnostics": mask_diag, - "sector_map_size": len(symbol_to_sector), - "sector_etfs_loaded": sorted(sector_etf), - "spy_bars": len(spy), - } - - -def _identical_subset_eval(collected: dict, bt) -> dict[str, Any]: - """Re-score mom_12_1_resid on the same (week, symbol) cells as sector_resid.""" - sector_weeks = collected.get("mom_12_1_sector_resid") or {} - resid_weeks = collected.get("mom_12_1_resid") or {} - demean_weeks = collected.get("mom_12_1_sector_demeaned") or {} - mom_weeks = collected.get("mom_12_1") or {} - - restricted: dict = defaultdict(lambda: defaultdict(list)) - for wk, recs in sector_weeks.items(): - syms = set() - for rec in recs: - if isinstance(rec, dict) and rec.get("symbol"): - syms.add(str(rec["symbol"]).upper()) - restricted["mom_12_1_sector_resid"][wk].append(rec) - for name, source in ( - ("mom_12_1_resid", resid_weeks), - ("mom_12_1", mom_weeks), - ("mom_12_1_sector_demeaned", demean_weeks), - ): - for rec in source.get(wk) or []: - if not isinstance(rec, dict): - continue - sym = rec.get("symbol") - if sym and str(sym).upper() in syms: - restricted[name][wk].append(rec) - - rows = bt._signal_evaluation(dict(restricted)) - return {r["signal"]: r for r in rows} - - -def _mask_diagnostics(signal_eval: list[dict]) -> dict[str, Any]: - # Prefer a dense signal for mask stats. - for name in ("vol_6m", "mom_12_1", "fip_id"): - for row in signal_eval: - if row.get("signal") == name and row.get("mask_binds_pct") is not None: - return { - "reference_signal": name, - "avg_cross_section": row.get("avg_cross_section"), - "avg_raw_pool": row.get("avg_raw_pool"), - "avg_eligible_pre_mask": row.get("avg_eligible_pre_mask"), - "mask_binds_pct": row.get("mask_binds_pct"), - "weeks": row.get("weeks"), - } - # Fallback: any row with liquid fields - for row in signal_eval: - if row.get("liquid_breadth_top_n"): - return { - "reference_signal": row.get("signal"), - "avg_cross_section": row.get("avg_cross_section"), - "mask_binds_pct": row.get("mask_binds_pct"), - "weeks": row.get("weeks"), - } - return {"note": "no liquid mask diagnostics on rows (mask may be off)"} - - -def _grade(harness: dict[str, Any]) -> dict[str, Any]: - """Pre-registered PASS/FAIL for mom_12_1_sector_resid — mechanical.""" - by = harness.get("signal_eval_by_name") or {} - era = harness.get("era_split") or {} - identical = harness.get("identical_subset_sector_cs") or {} - - sector = by.get("mom_12_1_sector_resid") - # Prefer identical-subset resid for t comparison; fall back to full-table resid. - resid = identical.get("mom_12_1_resid") or by.get("mom_12_1_resid") - pre = (era.get("pre_2021") or {}).get("mom_12_1_sector_resid") - post = (era.get("post_2021") or {}).get("mom_12_1_sector_resid") - - checks: dict[str, Any] = { - "sector_row": sector, - "resid_row_for_t": resid, - "resid_t_source": ( - "identical_subset" if identical.get("mom_12_1_resid") else "full_table" - ), - "pre_2021": pre, - "post_2021": post, - } - - if sector is None: - return { - "verdict": "FAIL", - "reason": "mom_12_1_sector_resid missing from signal_eval", - "checks": checks, - "headline": "Task 1 CLOSED — sector residual dead on deep evidence.", - } - - mean_ic = sector.get("mean_ic") - t_stat = sector.get("ic_t_stat") - weeks = int(sector.get("weeks") or 0) - reliable = bool(sector.get("reliable")) - resid_t = resid.get("ic_t_stat") if resid else None - - mag_ok = mean_ic is not None and abs(float(mean_ic)) >= IRON_IC - sign_ok = mean_ic is not None and float(mean_ic) > 0 - reliable_ok = reliable and weeks >= 12 - weeks_ok = weeks >= MIN_WEEKS_DEEP - t_ok = ( - t_stat is not None - and resid_t is not None - and float(t_stat) >= float(resid_t) - ) - - pre_ic = pre.get("mean_ic") if pre else None - post_ic = post.get("mean_ic") if post else None - era_sign_ok = ( - pre_ic is not None - and post_ic is not None - and float(pre_ic) > 0 - and float(post_ic) > 0 - ) - # If pre era has no row, data fix failed for depth / era coverage. - era_present = pre is not None and post is not None - - checks.update({ - "abs_mean_ic_ge_0_03": mag_ok, - "sign_positive": sign_ok, - "reliable": reliable_ok, - "weeks_ge_50": weeks_ok, - "weeks": weeks, - "t_ge_resid_same_cs": t_ok, - "sector_t": t_stat, - "resid_t": resid_t, - "era_both_present": era_present, - "era_sign_consistent_positive": era_sign_ok, - "pre_ic": pre_ic, - "post_ic": post_ic, - "avg_cross_section": sector.get("avg_cross_section"), - }) - - if not weeks_ok: - return { - "verdict": "FAIL", - "reason": ( - f"weeks={weeks} did not extend (need ≥{MIN_WEEKS_DEEP}) — " - "data fix did not work or sector residual still shallow" - ), - "checks": checks, - "headline": "Task 1 CLOSED — sector residual dead on deep evidence.", - } - - passed = ( - mag_ok - and sign_ok - and reliable_ok - and weeks_ok - and t_ok - and era_present - and era_sign_ok - ) - if passed: - return { - "verdict": "PASS", - "reason": ( - "iron bar + weeks extended + t≥resid on same CS + era sign consistent" - ), - "checks": checks, - "headline": ( - "PROMOTE case strengthened — portfolio A/B is the next human decision." - ), - } - return { - "verdict": "FAIL", - "reason": "failed one or more pre-registered checks (see checks)", - "checks": checks, - "headline": "Task 1 CLOSED — sector residual dead on deep evidence.", - } - - -def _write_reports(payload: dict, out_json: Path, doc_path: Path) -> None: - out_json.parent.mkdir(parents=True, exist_ok=True) - out_json.write_text( - json.dumps(payload, indent=2, default=str) + "\n", encoding="utf-8" - ) - - grade = payload.get("grade") or {} - harness = payload.get("harness") or {} - by = harness.get("signal_eval_by_name") or {} - era = harness.get("era_split") or {} - lines = [ - "# Sector-residual deep test (masked, repaired snapshot)", - "", - f"Generated: `{payload.get('generated_at')}`", - "", - f"> **{SURVIVORSHIP}**", - "", - "## Pre-registered grade (mechanical)", - "", - f"**Verdict: {grade.get('verdict')}**", - "", - f"{grade.get('headline')}", - "", - f"Reason: {grade.get('reason')}", - "", - f"```json\n{json.dumps(grade.get('checks') or {}, indent=2, default=str)}\n```", - "", - "## Step-1 sanity", - "", - f"```json\n{json.dumps(payload.get('step1') or {}, indent=2, default=str)}\n```", - "", - "## Mask diagnostics", - "", - f"```json\n{json.dumps(harness.get('mask_diagnostics') or {}, indent=2, default=str)}\n```", - "", - "## Signal table (rows only — no narrative for non-sector signals)", - "", - "| signal | mean_ic | t | weeks | avg_N | reliable |", - "|---|---:|---:|---:|---:|---|", - ] - for name in sorted(by): - r = by[name] - lines.append( - f"| {name} | {r.get('mean_ic')} | {r.get('ic_t_stat')} | " - f"{r.get('weeks')} | {r.get('avg_cross_section')} | {r.get('reliable')} |" - ) - lines.extend([ - "", - "### Era split — mom_12_1_sector_resid only (for grade)", - "", - f"| era | IC | t | weeks | N |", - f"|---|---:|---:|---:|---:|", - ]) - for label in ("pre_2021", "post_2021"): - r = (era.get(label) or {}).get("mom_12_1_sector_resid") or {} - lines.append( - f"| {label} | {r.get('mean_ic', '')} | {r.get('ic_t_stat', '')} | " - f"{r.get('weeks', '')} | {r.get('avg_cross_section', '')} |" - ) - lines.extend([ - "", - "### Identical-subset baselines (sector CS)", - "", - f"```json\n{json.dumps(harness.get('identical_subset_sector_cs') or {}, indent=2, default=str)}\n```", - "", - "## Status", - "", - "PENDING_HUMAN beyond the mechanical PASS/FAIL above. " - "Nothing merged into production docs or prod code.", - "", - f"JSON: `{out_json.as_posix()}`", - "", - ]) - md_path = out_json.with_suffix(".md") - md_path.write_text("\n".join(lines) + "\n", encoding="utf-8") - - # Update history-depth-extension.md - _update_history_doc(doc_path, payload, out_json) - - -def _update_history_doc(doc_path: Path, payload: dict, out_json: Path) -> None: - grade = payload.get("grade") or {} - banner = ( - "\n\n---\n\n" - "## Supersession notice (2026-07-19 sector-resid deep test)\n\n" - "The table and interpretation from **`history-depth-20260719-103315`** are " - "**UNMASKED, TWO-TIER SNAPSHOT — superseded, directional only, do not cite**. " - "Prod-universe names (and sector residual coverage) were left shallow while " - "breadth names were deepened; sector residual weeks=35 was a data gap.\n\n" - f"### Sector-residual deep test outcome: **{grade.get('verdict')}**\n\n" - f"{grade.get('headline')}\n\n" - f"- Reason: {grade.get('reason')}\n" - f"- Artifact: `{out_json.as_posix()}`\n" - f"- Mechanical checks: see that report.\n\n" - "**Future snapshot rebuilds must verify per-symbol depth** (earliest-bar " - "uniformity across the intended universe) — guard is a to-do, not part of " - "this order.\n" - ) - if doc_path.exists(): - text = doc_path.read_text(encoding="utf-8") - # Insert supersession after status line / near top results if not already there. - marker = "## Supersession notice (2026-07-19 sector-resid deep test)" - if marker in text: - # Replace from marker to end of that section or append fresh block at end. - pre = text.split(marker)[0].rstrip() - text = pre + banner - else: - # Mark 103315 in place if mentioned. - text = text.replace( - "Authoritative artifact:** `reports/history-depth-20260719-103315.json`", - "Superseded artifact (do not cite):** `reports/history-depth-20260719-103315.json` " - "— **UNMASKED, TWO-TIER SNAPSHOT**", - ) - text = text.rstrip() + banner - # Soften old PARK-only language if present — leave body but status at top. - if text.startswith("#"): - lines = text.splitlines() - for i, line in enumerate(lines[:15]): - if line.startswith("**Status:**"): - lines[i] = ( - f"**Status:** sector-resid deep test **{grade.get('verdict')}** " - f"— see supersession section. PENDING_HUMAN beyond PASS/FAIL." - ) - break - text = "\n".join(lines) - doc_path.write_text(text if text.endswith("\n") else text + "\n", encoding="utf-8") - else: - doc_path.write_text( - "# History-depth extension\n" + banner, encoding="utf-8" - ) - - -async def _main() -> None: - args = _parse_args() - snapshot = Path(args.snapshot) - if not snapshot.exists(): - raise SystemExit(f"Snapshot missing: {snapshot}") - if args.allow_spawn: - os.environ["BACKTEST_ALLOW_SPAWN"] = "1" - - sector_map = Path(args.sector_map) - if not sector_map.exists(): - raise SystemExit(f"Sector map missing: {sector_map}") - - step1: dict[str, Any] - if args.skip_deepen: - print("Skip deepen — race guard + sanity only…") - assert_research_snapshot_complete(snapshot) - sanity = _sanity_check(snapshot, history_days=args.history_days) - step1 = {"skipped": True, "sanity": sanity} - if not sanity["passed"]: - raise SystemExit( - "Sanity check FAILED with --skip-deepen. " - f"Details: {json.dumps(sanity, default=str)}" - ) - else: - print("Step 1 — deepen shallow symbols…") - step1 = await _step1_deepen( - snapshot, - history_days=args.history_days, - sleep_s=args.sleep, - quiet=args.quiet, - ) - if not step1["sanity"]["passed"]: - print("SANITY CHECK FAILED — refusing harness.") - print(json.dumps(step1["sanity"], indent=2, default=str)) - stamp = datetime.now().strftime("%Y%m%d-%H%M%S") - fail_path = Path("reports") / f"sector-resid-deep-{stamp}-SANITY-FAIL.json" - fail_path.parent.mkdir(parents=True, exist_ok=True) - fail_path.write_text( - json.dumps({"step1": step1, "harness": None}, indent=2, default=str) - + "\n", - encoding="utf-8", - ) - raise SystemExit( - f"Stop: sanity failed. Wrote {fail_path}. Do not run harness on two-tier data." - ) - print("Sanity check PASSED.") - assert_research_snapshot_complete(snapshot) - - print( - f"Step 2 — ONE masked harness " - f"(top {args.liquid_breadth}, min_price={args.min_price})…" - ) - harness = await _one_masked_run( - snapshot, - sector_map_path=sector_map, - liquid_breadth=args.liquid_breadth, - min_price=args.min_price, - workers=args.workers, - quiet=args.quiet, - ) - grade = _grade(harness) - print(f"GRADE: {grade['verdict']} — {grade['headline']}") - - stamp = datetime.now().strftime("%Y%m%d-%H%M%S") - out = Path(args.out) if args.out else Path("reports") / f"sector-resid-deep-{stamp}.json" - payload = { - "generated_at": datetime.now().isoformat(), - "snapshot": str(snapshot.resolve()), - "pre_registration": { - "iron_ic": IRON_IC, - "min_weeks_deep": MIN_WEEKS_DEEP, - "liquid_breadth": args.liquid_breadth, - "min_price": args.min_price, - "rule": ( - "PASS = |IC|>=0.03, +sign, reliable, weeks>=50, " - "t>=resid on same CS, era signs both +" - ), - }, - "step1": step1, - "harness": harness, - "grade": grade, - "pending_human": True, - "note": "Nothing merged into production. Thread ends at PASS/FAIL.", - } - _write_reports( - payload, - out, - Path("docs/research/history-depth-extension.md"), - ) - print(f"Wrote {out}") - print(f"Wrote {out.with_suffix('.md')}") - print("Updated docs/research/history-depth-extension.md") - - -if __name__ == "__main__": - asyncio.run(_main()) diff --git a/scripts/run_sector_residual_research.py b/scripts/run_sector_residual_research.py deleted file mode 100644 index 1f60590..0000000 --- a/scripts/run_sector_residual_research.py +++ /dev/null @@ -1,1018 +0,0 @@ -"""Sector-residual momentum research runner (local only). - -Protocol --------- -1. Race-guard the research/prod snapshot (completion manifest when present). -2. Require sector map + sector ETFs in ``benchmark_prices``. -3. Run signal IC harness on the production ~505-name snapshot - (``BACKTEST_SIGNAL_EVAL_ONLY=1``) with sector context loaded. -4. Grade candidates vs pre-registered iron rule + t-stat vs ``mom_12_1_resid``. -5. If a candidate promotes: portfolio A/B with candidate as momentum leg + - gate percentile (``fill_mode=close``). Optional sector-cap arm. - -Does not modify production DB, gate, scanner, or schedule. - -Example -------- - python scripts/run_sector_residual_research.py \\ - --snapshot backtest_snapshots/prod.sqlite \\ - --workers 6 --allow-spawn -""" - -from __future__ import annotations - -import argparse -import asyncio -import json -import math -import os -import sys -from collections import defaultdict -from copy import deepcopy -from datetime import date, datetime -from pathlib import Path -from typing import Any - -from sqlalchemy import create_engine, text -from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine - -ROOT = Path(__file__).resolve().parents[1] -if str(ROOT) not in sys.path: - sys.path.insert(0, str(ROOT)) - -from app.ssl_bootstrap import bootstrap_ssl # noqa: E402 - -bootstrap_ssl() - -from app.services.sector_map import ( # noqa: E402 - DEFAULT_SECTOR_MAP_PATH, - SECTOR_ETFS, - coverage_stats, - load_ticker_sector_map, - normalise_symbol, - sector_to_etf, -) - -VALIDATION_SPLIT = date(2024, 7, 1) -IRON_IC_BAR = 0.03 -MIN_RELIABLE = 12 - - -def _sqlite_url(path: Path) -> str: - return f"sqlite+aiosqlite:///{path.resolve().as_posix()}" - - -def _parse_args() -> argparse.Namespace: - p = argparse.ArgumentParser(description=__doc__) - p.add_argument("--snapshot", default="backtest_snapshots/prod.sqlite") - p.add_argument( - "--sector-map", - default=str(DEFAULT_SECTOR_MAP_PATH), - ) - p.add_argument("--workers", type=int, default=6) - p.add_argument("--allow-spawn", action="store_true") - p.add_argument( - "--skip-ab", - action="store_true", - help="IC only — never run portfolio A/B even if promotion fires.", - ) - p.add_argument( - "--force-ab", - action="store_true", - help="Run A/B for diagnostic even if IC bar fails (still reported as non-promote).", - ) - p.add_argument( - "--sector-cap", - type=int, - default=None, - help="Optional max positions per sector (e.g. 3). Only used in A/B.", - ) - p.add_argument("--quiet", action="store_true") - p.add_argument( - "--out", - default=None, - help="JSON report path (default reports/sector-residual-YYYYMMDD-HHMMSS.json)", - ) - return p.parse_args() - - -def _assert_snapshot_ready(snapshot: Path) -> dict[str, Any]: - """Race guard: prefer completion manifest; always check live bar sanity.""" - from scripts.research_snapshot_manifest import ( # type: ignore - assert_research_snapshot_complete, - load_manifest, - ) - - guard: dict[str, Any] = {"snapshot": str(snapshot.resolve())} - manifest = load_manifest(snapshot) - if manifest is not None: - # Full assert when a manifest exists (research.sqlite path). - try: - m = assert_research_snapshot_complete(snapshot) - guard["manifest"] = m - guard["manifest_ok"] = True - except SystemExit as exc: - raise SystemExit(str(exc)) from exc - else: - guard["manifest"] = None - guard["manifest_ok"] = None - guard["note"] = ( - "No completion manifest (prod.sqlite is expected without one). " - "Bar-count sanity still applied." - ) - - engine = create_engine( - f"sqlite:///{snapshot.resolve().as_posix()}", - future=True, - ) - try: - with engine.connect() as conn: - ticker_n = int(conn.execute(text("SELECT COUNT(*) FROM tickers")).scalar_one()) - ohlcv_n = int( - conn.execute(text("SELECT COUNT(*) FROM ohlcv_records")).scalar_one() - ) - bar_stats = conn.execute( - text( - """ - SELECT MIN(c), AVG(c), MAX(c) FROM ( - SELECT COUNT(*) AS c FROM ohlcv_records GROUP BY ticker_id - ) - """ - ) - ).fetchone() - bench = conn.execute( - text( - "SELECT symbol, COUNT(*), MIN(date), MAX(date) " - "FROM benchmark_prices GROUP BY symbol ORDER BY symbol" - ) - ).fetchall() - d_range = conn.execute( - text("SELECT MIN(date), MAX(date) FROM ohlcv_records") - ).fetchone() - finally: - engine.dispose() - - guard["ticker_count"] = ticker_n - guard["ohlcv_row_count"] = ohlcv_n - guard["bars_min_avg_max"] = { - "min": bar_stats[0], - "avg": round(float(bar_stats[1]), 1) if bar_stats[1] is not None else None, - "max": bar_stats[2], - } - guard["ohlcv_date_range"] = {"min": d_range[0], "max": d_range[1]} - guard["benchmark_prices"] = [ - {"symbol": s, "n": n, "min": d0, "max": d1} for s, n, d0, d1 in bench - ] - - # Sanity: a half-built snapshot would show many tickers with tiny bar counts. - min_bars = int(bar_stats[0] or 0) - avg_bars = float(bar_stats[1] or 0) - if ticker_n < 400: - raise SystemExit( - f"Snapshot looks short: only {ticker_n} tickers (expected ~505 prod)." - ) - if avg_bars < 200: - raise SystemExit( - f"Snapshot bar counts look short (avg={avg_bars:.0f}). Rebuild before research." - ) - # Allow a few thin names; refuse if median path is collapsed. - if min_bars < 10 and avg_bars < 500: - raise SystemExit( - f"Snapshot min bars={min_bars}, avg={avg_bars:.0f} — possible partial build." - ) - - present_etfs = {row[0] for row in bench} - missing_etfs = [e for e in SECTOR_ETFS if e not in present_etfs] - guard["missing_sector_etfs"] = missing_etfs - if missing_etfs: - raise SystemExit( - "Sector ETFs missing from benchmark_prices: " - f"{missing_etfs}. Run scripts/fetch_sector_etfs_to_snapshot.py first." - ) - if "SPY" not in present_etfs: - raise SystemExit("SPY missing from benchmark_prices") - - return guard - - -def _find_signal(rows: list[dict], name: str) -> dict | None: - for row in rows or []: - if row.get("signal") == name: - return row - return None - - -def _grade_ic( - candidate: dict | None, - resid: dict | None, - *, - expected_sign: float = 1.0, -) -> dict[str, Any]: - """Iron rule + t-stat ≥ mom_12_1_resid.""" - if candidate is None: - return { - "promote_to_ab": False, - "reason": "signal missing from signal_eval", - } - mean_ic = candidate.get("mean_ic") - t_stat = candidate.get("ic_t_stat") - reliable = bool(candidate.get("reliable")) - weeks = int(candidate.get("weeks") or 0) - if mean_ic is None or t_stat is None: - return {"promote_to_ab": False, "reason": "missing mean_ic or t", "row": candidate} - - sign_ok = (float(mean_ic) * expected_sign) > 0 - mag_ok = abs(float(mean_ic)) >= IRON_IC_BAR - reliable_ok = reliable and weeks >= MIN_RELIABLE - resid_t = resid.get("ic_t_stat") if resid else None - t_ok = resid_t is not None and float(t_stat) >= float(resid_t) - - promote = sign_ok and mag_ok and reliable_ok and t_ok - return { - "promote_to_ab": promote, - "checks": { - "sign_ok": sign_ok, - "abs_mean_ic_ge_0_03": mag_ok, - "reliable": reliable_ok, - "t_ge_resid": t_ok, - "mean_ic": mean_ic, - "ic_t_stat": t_stat, - "resid_ic_t_stat": resid_t, - "weeks": weeks, - }, - "reason": ( - "clears iron rule and t ≥ mom_12_1_resid — authorized for A/B only" - if promote - else "does not clear pre-registered IC promotion bar" - ), - "row": candidate, - } - - -async def _run_signal_eval( - snapshot: Path, - *, - workers: int, - quiet: bool, - sector_map_path: Path, -) -> dict: - from app.config import settings - from app.services.backtest_service import run_backtest - - os.environ["BACKTEST_SNAPSHOT_OFFLINE"] = "1" - os.environ["BACKTEST_SIGNAL_EVAL_ONLY"] = "1" - os.environ["BACKTEST_SECTOR_MAP_PATH"] = str(sector_map_path.resolve()) - # Clear liquid-breadth — this is the 505-name prod IC, not breadth. - os.environ.pop("BACKTEST_LIQUID_BREADTH", None) - settings.backtest_workers = workers - - engine = create_async_engine(_sqlite_url(snapshot), pool_pre_ping=True) - Session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) - - def progress(done: int, total: int, symbol: str) -> None: - if quiet: - return - print(f" progress {done}/{total} {symbol}", end="\r", flush=True) - - try: - async with Session() as db: - report = await run_backtest(db, progress_cb=progress, cadence="weekly") - finally: - await engine.dispose() - if not quiet: - print() - return report - - -def _period_percentiles(rows: list[dict], value_key: str) -> dict[tuple, dict[str, float]]: - by_period: dict[tuple, list[dict]] = defaultdict(list) - for row in rows: - if row.get(value_key) is None: - continue - period = row.get("ranking_period") or row.get("iso_week") - by_period[period].append(row) - out: dict[tuple, dict[str, float]] = {} - for period, group in by_period.items(): - ordered = sorted(group, key=lambda r: float(r[value_key])) - n = len(ordered) - for rank, row in enumerate(ordered): - key = (str(row["symbol"]), str(row["date"])) - pct = (rank / (n - 1) * 100.0) if n > 1 else 100.0 - out.setdefault(key, {})[value_key] = float(row[value_key]) - out[key][f"{value_key}_percentile"] = pct - return out - - -async def _load_prices_and_benchmarks(snapshot: Path) -> tuple[dict, dict, dict]: - """Return (price_columns, spy_closes, sector_etf_closes).""" - from app.services import backtest_service as bt - from app.services.benchmark_service import load_benchmark_closes - from app.models.ticker import Ticker - from sqlalchemy import select - - engine = create_async_engine(_sqlite_url(snapshot), pool_pre_ping=True) - Session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) - prices: dict[str, tuple] = {} - try: - async with Session() as db: - tickers = list( - (await db.execute(select(Ticker).order_by(Ticker.symbol))).scalars() - ) - for t in tickers: - cols = await bt._fetch_columns(db, t.symbol) - if cols is not None: - prices[t.symbol] = cols - spy = await load_benchmark_closes(db, "SPY") - sector: dict[str, dict] = {} - for etf in SECTOR_ETFS: - series = await load_benchmark_closes(db, etf) - if series: - sector[etf] = series - finally: - await engine.dispose() - return prices, spy, sector - - -def _recompute_sector_residual_on_candidates( - candidates: list[dict], - prices: dict[str, tuple], - spy_closes: dict, - sector_etf_closes: dict[str, dict], - symbol_to_sector: dict[str, str], - *, - momentum_field: str, -) -> list[dict]: - """Attach alternative residual momentum on each candidate as-of date.""" - from app.services import backtest_service as bt - from app.services.sector_map import etf_for_symbol - - # Index price series once. - series_cache: dict[str, tuple[list, list, list]] = {} - for sym, cols in prices.items(): - ords, _o, _h, _l, closes, _v = cols - dates = [date.fromordinal(int(o)) for o in ords] - series_cache[sym] = (dates, list(closes), list(ords)) - - out: list[dict] = [] - for cand in candidates: - c = dict(cand) - sym = str(c["symbol"]) - if sym not in series_cache: - out.append(c) - continue - dates, closes, ords = series_cache[sym] - asof = date.fromisoformat(str(c["date"])) - # Find as-of index. - try: - i = next(idx for idx, d in enumerate(dates) if d == asof) - except StopIteration: - # nearest on/before - i = max((idx for idx, d in enumerate(dates) if d <= asof), default=-1) - if i < 0: - out.append(c) - continue - - if momentum_field == "mom_12_1_sector_resid": - etf = etf_for_symbol(sym, symbol_to_sector) - etf_series = sector_etf_closes.get(etf or "") - val = None - if spy_closes and etf_series: - val = bt._multi_factor_residual_momentum_12_1( - dates, closes, i, [spy_closes, etf_series] - ) - c["residual_momentum"] = val - c["_alt_momentum_signal"] = momentum_field - c["_alt_momentum_value"] = val - elif momentum_field == "mom_12_1_sector_demeaned": - # Placeholder: demean requires cross-section; filled in a second pass. - raw = None - if i >= 252 and closes[i - 252] > 0: - raw = closes[i - 21] / closes[i - 252] - 1.0 - c["_raw_mom_12_1"] = raw - c["_alt_momentum_signal"] = momentum_field - else: - raise ValueError(momentum_field) - out.append(c) - - if momentum_field == "mom_12_1_sector_demeaned": - # Cross-sectional demean within ranking period × sector. - by_period: dict[Any, list[dict]] = defaultdict(list) - for c in out: - if c.get("_raw_mom_12_1") is None: - continue - period = c.get("ranking_period") or c.get("iso_week") - by_period[period].append(c) - for period, group in by_period.items(): - by_sec: dict[str, list[float]] = defaultdict(list) - for c in group: - sec = symbol_to_sector.get(normalise_symbol(str(c["symbol"]))) - if sec: - by_sec[sec].append(float(c["_raw_mom_12_1"])) - means = { - s: sum(vs) / len(vs) for s, vs in by_sec.items() if len(vs) >= 2 - } - for c in group: - sec = symbol_to_sector.get(normalise_symbol(str(c["symbol"]))) - raw = float(c["_raw_mom_12_1"]) - if sec in means: - val = raw - means[sec] - c["residual_momentum"] = val - c["_alt_momentum_value"] = val - else: - c["residual_momentum"] = None - c["_alt_momentum_value"] = None - - return out - - -def _assign_prod_ranks(candidates: list[dict]) -> None: - from app.services import backtest_service as bt - - bt._assign_momentum_percentiles(candidates) - bt._assign_residual_momentum_percentiles(candidates) - bt._assign_low_volatility_percentiles(candidates) - bt._assign_activation_momentum_percentiles(candidates) - bt._assign_residual_high_vol_blend(candidates) - for c in candidates: - c["qualified"] = bt._momentum_qualifies(c, 80.0) - - -async def _run_ab( - snapshot: Path, - *, - sector_map: dict[str, str], - signal_name: str, - sector_cap: int | None, - quiet: bool, - workers: int, -) -> dict[str, Any]: - """Control vs treatment book with candidate as momentum residual.""" - from app.services import backtest_service as bt - from app.config import settings - from app.models.ticker import Ticker - from app.services.admin_service import get_activation_config - from app.services.recommendation_service import get_recommendation_config - from app.services.paper_trade_service import get_exit_policy - from app.services.benchmark_service import load_benchmark_closes - from sqlalchemy import select - - os.environ["BACKTEST_SNAPSHOT_OFFLINE"] = "1" - os.environ.pop("BACKTEST_SIGNAL_EVAL_ONLY", None) - settings.backtest_workers = max(1, int(workers)) - - engine = create_async_engine(_sqlite_url(snapshot), pool_pre_ping=True) - Session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) - try: - async with Session() as db: - config = await get_recommendation_config(db) - activation = await get_activation_config(db) - exit_config = await get_exit_policy(db) - tickers = list( - (await db.execute(select(Ticker).order_by(Ticker.symbol))).scalars() - ) - spy = await load_benchmark_closes(db, "SPY") - sector_etf: dict[str, dict] = {} - for etf in SECTOR_ETFS: - series = await load_benchmark_closes(db, etf) - if series: - sector_etf[etf] = series - - prices: dict[str, tuple] = {} - candidates: list[dict] = [] - for idx, t in enumerate(tickers): - if not quiet and idx % 25 == 0: - print(f" fetch {idx}/{len(tickers)}", end="\r", flush=True) - cols = await bt._fetch_columns(db, t.symbol) - if cols is None: - continue - prices[t.symbol] = cols - cands, _series = bt._replay_and_signals( - t.symbol, - cols, - config, - activation, - spy, - bt.PRODUCTION_GTL_TARGET_MODEL, - "weekly", - False, - sector_etf, - sector_map, - ) - candidates.extend(cands) - finally: - await engine.dispose() - if not quiet: - print() - - # Control ranks (production residual). - control = [dict(c) for c in candidates] - _assign_prod_ranks(control) - control_longs = [ - c for c in control if c.get("qualified") and c.get("direction") == "long" - ] - - # Treatment: replace residual with sector signal, re-rank. - treatment = _recompute_sector_residual_on_candidates( - candidates, - prices, - spy, - sector_etf, - sector_map, - momentum_field=signal_name, - ) - _assign_prod_ranks(treatment) - treatment_longs = [ - c for c in treatment if c.get("qualified") and c.get("direction") == "long" - ] - - strategy = next(s for s in bt.PORTFOLIO_MONITOR_STRATEGIES if s.get("is_production")) - entry_config = bt._entry_variant_config(str(strategy["entry_variant"])) - assert entry_config is not None - ranking_key = str( - entry_config.get("ranking_key") or entry_config["percentile_key"] - ) - # Production ranking key is residual_high_vol_blend_80_20. - if ranking_key not in (bt.RESIDUAL_HIGH_VOL_BLEND_80_20_KEY, bt.PRODUCTION_PERCENTILE_KEY): - ranking_key = bt.RESIDUAL_HIGH_VOL_BLEND_80_20_KEY - - exit_policy = bt.LIVE_EXIT_MODE_TO_SIM.get( - str(exit_config.get("mode", "atr_trailing")), "atr_trail3" - ) - hold_days = int(exit_config.get("hold_days", 30)) - trail = float(exit_config.get("atr_multiplier", bt.ATR_TRAIL_MULTIPLIER)) - risk = float(entry_config["risk_per_trade"]) - max_pos = int(entry_config["max_positions"]) - - def _sim_book(longs: list[dict], *, label: str, cap: int | None = None) -> dict: - reentry = bt._make_gate_reset_reentry_fn( - longs, prices, cadence="weekly", ranking_key=ranking_key - ) - windows = {} - for wname, start, end in ( - ("train", None, VALIDATION_SPLIT), - ("validation", VALIDATION_SPLIT, None), - ("full", None, None), - ): - sim = bt._simulate_portfolio( - longs, - prices, - spy, - exit_policy, - hold_days, - ranking_key=ranking_key, - max_positions=max_pos, - risk_per_trade=risk, - atr_trail_multiplier=trail, - post_stop_reentry_fn=reentry, - start_date=start, - end_date=end, - fill_mode=bt.FILL_MODE_CLOSE, - include_trades=True, - ) - if sim is None: - windows[wname] = {"error": "no_trades"} - continue - # Optional sector cap: filter trade_details is post-hoc; real cap needs - # simulator support. For research, re-sim with a wrapper ranking that - # drops overflow sector names is approximate — we implement a simple - # pre-filter on daily entry sets via max_positions only when cap is None. - # When cap is set, apply a post-sim diagnostic on entries. - payload = { - k: sim.get(k) - for k in ( - "sharpe", - "sharpe_se", - "cagr_pct", - "max_drawdown_pct", - "total_return_pct", - "trades", - "win_rate_pct", - "avg_r", - "n_returns", - "return_skew", - "return_kurtosis", - "psr", - ) - } - details = sim.get("trade_details") or [] - rs = [ - float(t["realized_r"]) - for t in details - if t.get("realized_r") is not None - ] - if rs: - rs_sorted = sorted(rs) - payload["r_p05"] = rs_sorted[max(0, int(0.05 * (len(rs_sorted) - 1)))] - payload["r_p50"] = rs_sorted[len(rs_sorted) // 2] - payload["r_p95"] = rs_sorted[min(len(rs_sorted) - 1, int(0.95 * (len(rs_sorted) - 1)))] - payload["entry_count"] = len(rs) - if cap is not None and details: - # Diagnostic: count how often a calendar day would exceed cap. - from collections import Counter - - # Use entry dates; sector from map. - day_sector: dict[str, Counter] = defaultdict(Counter) - for t in details: - sec = sector_map.get(normalise_symbol(str(t.get("symbol", "")))) or "?" - day_sector[str(t.get("entry_date") or t.get("date") or "")][sec] += 1 - breaches = sum( - 1 - for day, ctr in day_sector.items() - if any(v > cap for v in ctr.values()) - ) - payload["sector_cap"] = cap - payload["entry_days_with_sector_over_cap"] = breaches - windows[wname] = payload - return {"label": label, "n_qualified_longs": len(longs), "windows": windows} - - control_result = _sim_book(control_longs, label="control_mom_12_1_resid") - treatment_result = _sim_book( - treatment_longs, label=f"treatment_{signal_name}", cap=None - ) - out: dict[str, Any] = { - "signal": signal_name, - "ranking_key": ranking_key, - "fill_mode": "close", - "validation_split": VALIDATION_SPLIT.isoformat(), - "control": control_result, - "treatment": treatment_result, - "promotion": _grade_ab(control_result, treatment_result), - } - if sector_cap is not None: - # Approximate sector-cap book: when selecting, prefer higher rank but - # refuse a 4th name in the same sector among concurrent opens. - # Implemented by tagging candidates and using a custom sim is heavy; - # instead report diagnostic on unconstrained treatment + a filtered - # re-rank that zeros residual for overflow names within each period. - capped = _apply_sector_cap_to_ranks( - treatment, sector_map, cap=sector_cap, ranking_key=ranking_key - ) - capped_longs = [ - c for c in capped if c.get("qualified") and c.get("direction") == "long" - ] - out["sector_cap_arm"] = _sim_book( - capped_longs, label=f"treatment_{signal_name}_cap{sector_cap}", cap=sector_cap - ) - out["sector_cap_promotion"] = _grade_ab( - control_result, out["sector_cap_arm"] - ) - return out - - -def _apply_sector_cap_to_ranks( - candidates: list[dict], - sector_map: dict[str, str], - *, - cap: int, - ranking_key: str, -) -> list[dict]: - """Within each ranking period, keep top `cap` per sector by ranking_key.""" - by_period: dict[Any, list[dict]] = defaultdict(list) - for c in candidates: - period = c.get("ranking_period") or c.get("iso_week") - by_period[period].append(dict(c)) - out: list[dict] = [] - for period, group in by_period.items(): - ordered = sorted( - group, - key=lambda r: float(r.get(ranking_key) or r.get("residual_momentum") or -1e9), - reverse=True, - ) - sector_counts: dict[str, int] = defaultdict(int) - for c in ordered: - sec = sector_map.get(normalise_symbol(str(c["symbol"]))) or "_unknown" - if sector_counts[sec] >= cap: - # Push below gate by nulling activation percentile. - c["qualified"] = False - c["_sector_cap_blocked"] = True - else: - if c.get("qualified"): - sector_counts[sec] += 1 - out.append(c) - return out - - -def _grade_ab(control: dict, treatment: dict) -> dict[str, Any]: - """Pre-registered: val Sharpe ≥ control − 0.5·SE; full Sharpe & maxDD not worse.""" - def win(arm: dict, name: str) -> dict: - return (arm.get("windows") or {}).get(name) or {} - - c_val = win(control, "validation") - t_val = win(treatment, "validation") - c_full = win(control, "full") - t_full = win(treatment, "full") - - def _f(d: dict, k: str) -> float | None: - v = d.get(k) - return None if v is None else float(v) - - c_sh = _f(c_val, "sharpe") - t_sh = _f(t_val, "sharpe") - # Use treatment SE if present else control SE. - se = _f(t_val, "sharpe_se") - if se is None: - se = _f(c_val, "sharpe_se") - if se is None: - se = 0.0 - - val_ok = ( - c_sh is not None - and t_sh is not None - and t_sh >= (c_sh - 0.5 * se) - ) - c_full_sh = _f(c_full, "sharpe") - t_full_sh = _f(t_full, "sharpe") - full_sh_ok = ( - c_full_sh is not None - and t_full_sh is not None - and t_full_sh >= c_full_sh - ) - # max DD: higher absolute drawdown is worse; stored as positive pct typically. - c_dd = _f(c_full, "max_drawdown_pct") - t_dd = _f(t_full, "max_drawdown_pct") - full_dd_ok = ( - c_dd is not None and t_dd is not None and abs(t_dd) <= abs(c_dd) + 1e-9 - ) - promote = bool(val_ok and full_sh_ok and full_dd_ok) - return { - "promote": promote, - "checks": { - "validation_sharpe_ge_control_minus_half_se": val_ok, - "full_sharpe_not_worse": full_sh_ok, - "full_maxdd_not_worse": full_dd_ok, - "control_validation_sharpe": c_sh, - "treatment_validation_sharpe": t_sh, - "se_used": se, - "control_full_sharpe": c_full_sh, - "treatment_full_sharpe": t_full_sh, - "control_full_maxdd": c_dd, - "treatment_full_maxdd": t_dd, - }, - "reason": ( - "clears pre-registered A/B bar — human decides wire-in" - if promote - else "fails pre-registered A/B bar" - ), - } - - -def _write_md(path: Path, payload: dict) -> None: - """Refresh the results sections of the research doc (preserve pre-reg header).""" - # Always write a standalone results companion + update the main doc's - # results block by rewriting the full file with pre-reg + results. - pre = Path("docs/research/sector-residual-momentum.md") - # Keep pre-registration by reading until '## Results' if present. - header = "" - if pre.exists(): - text = pre.read_text(encoding="utf-8") - marker = "## Results" - idx = text.find(marker) - header = text[:idx] if idx >= 0 else text.split("## Verdict")[0] - - guard = payload.get("snapshot_guard") or {} - cov = payload.get("sector_coverage") or {} - ic_rows = payload.get("signal_eval") or [] - grades = payload.get("ic_grades") or {} - lines = [ - header.rstrip(), - "", - "## Results", - "", - f"Generated: `{payload.get('generated_at')}`", - "", - "### Snapshot race guard", - "", - f"- Snapshot: `{guard.get('snapshot')}`", - f"- Tickers: **{guard.get('ticker_count')}** OHLCV rows: **{guard.get('ohlcv_row_count')}**", - f"- Bars min/avg/max: `{guard.get('bars_min_avg_max')}`", - f"- OHLCV range: `{guard.get('ohlcv_date_range')}`", - f"- Manifest ok: `{guard.get('manifest_ok')}`", - f"- Missing sector ETFs at start: `{guard.get('missing_sector_etfs')}`", - "", - "### Sector label coverage", - "", - f"```json\n{json.dumps(cov, indent=2, default=str)}\n```", - "", - "### IC harness (identical cross-sections)", - "", - "| signal | mean_ic | ic_t_stat | weeks | avg_N | reliable | ic+_pct |", - "|---|---:|---:|---:|---:|---|---:|", - ] - want = [ - "mom_12_1", - "mom_12_1_resid", - "mom_12_1_sector_resid", - "mom_12_1_sector_demeaned", - ] - by_name = {r.get("signal"): r for r in ic_rows} - for name in want: - r = by_name.get(name) or {} - lines.append( - f"| {name} | {r.get('mean_ic', '')} | {r.get('ic_t_stat', '')} | " - f"{r.get('weeks', '')} | {r.get('avg_cross_section', '')} | " - f"{r.get('reliable', '')} | {r.get('ic_positive_pct', '')} |" - ) - lines.extend(["", "### IC promotion grades", ""]) - for name, g in grades.items(): - lines.append(f"- **{name}**: promote_to_ab=`{g.get('promote_to_ab')}` — {g.get('reason')}") - lines.append(f" - checks: `{json.dumps(g.get('checks') or {}, default=str)}`") - - ab = payload.get("portfolio_ab") - lines.extend(["", "### Portfolio A/B", ""]) - if not ab: - lines.append("_Not run (IC bar not cleared, or --skip-ab)._") - else: - lines.append(f"```json\n{json.dumps(ab, indent=2, default=str)}\n```") - - lines.extend([ - "", - "## Verdict", - "", - f"**{payload.get('verdict')}**", - "", - payload.get("verdict_detail") or "", - "", - "## What a human must decide next", - "", - payload.get("human_next") or "- Review numbers; do not merge into strategy docs without approval.", - "", - "## Artifacts", - "", - f"- JSON: `{payload.get('report_path')}`", - "", - ]) - path.write_text("\n".join(lines) + "\n", encoding="utf-8") - - -async def _main() -> None: - args = _parse_args() - snapshot = Path(args.snapshot) - sector_map_path = Path(args.sector_map) - if not snapshot.exists(): - raise SystemExit(f"Snapshot missing: {snapshot}") - if not sector_map_path.exists(): - raise SystemExit( - f"Sector map missing: {sector_map_path}. " - "Run scripts/build_ticker_sector_map.py first." - ) - - if args.allow_spawn: - os.environ["BACKTEST_ALLOW_SPAWN"] = "1" - - print("Race-guarding snapshot…") - guard = _assert_snapshot_ready(snapshot) - print( - f" tickers={guard['ticker_count']} ohlcv={guard['ohlcv_row_count']} " - f"bars={guard['bars_min_avg_max']}" - ) - - mapping = load_ticker_sector_map(sector_map_path) - engine = create_engine( - f"sqlite:///{snapshot.resolve().as_posix()}", - future=True, - ) - try: - with engine.connect() as conn: - symbols = [ - normalise_symbol(r[0]) - for r in conn.execute(text("SELECT symbol FROM tickers")).fetchall() - ] - finally: - engine.dispose() - cov = coverage_stats(symbols, mapping) - print( - f"Sector map: {cov['mapped']}/{cov['universe']} " - f"({cov['mapped_pct']}%) with_etf={cov['with_etf']}" - ) - if cov["mapped_pct"] < 90: - print( - f"WARNING: sector coverage {cov['mapped_pct']}% < 90%; " - f"missing e.g. {cov['missing'][:20]}" - ) - - print("Running IC harness (signal-eval only)…") - report = await _run_signal_eval( - snapshot, - workers=args.workers, - quiet=args.quiet, - sector_map_path=sector_map_path, - ) - signal_eval = report.get("signal_eval") or report.get("signals") or [] - # Locate key in report — backtest uses "signal_edge" historically. - if not signal_eval: - for key in ("signal_edge", "signal_evaluation", "factor_ic"): - if key in report and isinstance(report[key], list): - signal_eval = report[key] - break - - resid = _find_signal(signal_eval, "mom_12_1_resid") - grades = { - name: _grade_ic(_find_signal(signal_eval, name), resid) - for name in ("mom_12_1_sector_resid", "mom_12_1_sector_demeaned") - } - for name, g in grades.items(): - print( - f" {name}: promote_to_ab={g['promote_to_ab']} " - f"ic={((g.get('row') or {}).get('mean_ic'))} " - f"t={((g.get('row') or {}).get('ic_t_stat'))}" - ) - - ab_results: dict[str, Any] | None = None - promote_names = [n for n, g in grades.items() if g.get("promote_to_ab")] - run_ab = (bool(promote_names) or args.force_ab) and not args.skip_ab - if run_ab: - # Prefer sector_resid if both; else the one that promoted / force resid. - if "mom_12_1_sector_resid" in promote_names or ( - args.force_ab and not promote_names - ): - ab_signal = "mom_12_1_sector_resid" - else: - ab_signal = promote_names[0] - print(f"Running portfolio A/B for {ab_signal}…") - ab_results = await _run_ab( - snapshot, - sector_map=mapping, - signal_name=ab_signal, - sector_cap=args.sector_cap, - quiet=args.quiet, - workers=args.workers, - ) - print( - f" A/B promote={ab_results.get('promotion', {}).get('promote')} " - f"— {ab_results.get('promotion', {}).get('reason')}" - ) - else: - print("Skipping portfolio A/B (no IC promotion; use --force-ab to override).") - - # Verdict - if ab_results and ab_results.get("promotion", {}).get("promote"): - verdict = "PROMOTE" - detail = ( - f"{ab_results['signal']} cleared IC + A/B bars. " - "Human must design wire-in; do not ship from this branch." - ) - human = ( - "- Approve or reject production residual swap vs dual-signal design.\n" - "- If sector-cap arm ran, review tail-trim diagnostics before any cap." - ) - elif any(g.get("promote_to_ab") for g in grades.values()): - verdict = "PARK" - detail = ( - "IC promotion bar cleared but A/B did not promote " - "(or A/B skipped). Park for human review." - ) - human = "- Inspect A/B windows; decide whether to re-run or park." - elif any( - (g.get("row") or {}).get("mean_ic") is not None - and abs(float((g.get("row") or {}).get("mean_ic") or 0)) >= IRON_IC_BAR * 0.5 - for g in grades.values() - ): - verdict = "PARK" - detail = "Weak / partial IC — not dead, not green. Machinery kept." - human = "- No book change. Revisit after history-depth extension (Task 3)." - else: - verdict = "DEAD" - detail = ( - "Neither sector residual nor sector demean cleared the iron-rule bar " - "with t ≥ mom_12_1_resid on this window." - ) - human = "- Do not wire sector residual. Optional: re-check after Task 3 depth." - - stamp = datetime.now().strftime("%Y%m%d-%H%M%S") - out_path = Path(args.out) if args.out else Path("reports") / f"sector-residual-{stamp}.json" - payload = { - "generated_at": datetime.now().isoformat(), - "snapshot_guard": guard, - "sector_coverage": cov, - "sector_map_path": str(sector_map_path.resolve()), - "signal_eval": signal_eval, - "ic_grades": grades, - "portfolio_ab": ab_results, - "verdict": verdict, - "verdict_detail": detail, - "human_next": human, - "report_path": str(out_path.as_posix()), - "pre_registration": { - "iron_ic_bar": IRON_IC_BAR, - "validation_split": VALIDATION_SPLIT.isoformat(), - "fill_mode": "close", - "cost_per_side": 0.001, - "ab_rule": "val Sharpe >= control - 0.5*SE; full Sharpe & maxDD not worse", - }, - } - out_path.parent.mkdir(parents=True, exist_ok=True) - out_path.write_text(json.dumps(payload, indent=2, default=str) + "\n", encoding="utf-8") - - md_path = Path("docs/research/sector-residual-momentum.md") - _write_md(md_path, payload) - # Companion md under reports/ - md_report = out_path.with_suffix(".md") - md_report.write_text(md_path.read_text(encoding="utf-8"), encoding="utf-8") - - print(f"Verdict: {verdict}") - print(f"Wrote {out_path}") - print(f"Wrote {md_path}") - - -if __name__ == "__main__": - asyncio.run(_main()) diff --git a/scripts/run_tier1_macbook.sh b/scripts/run_tier1_macbook.sh index 584eb4a..3254e4e 100755 --- a/scripts/run_tier1_macbook.sh +++ b/scripts/run_tier1_macbook.sh @@ -1,99 +1,70 @@ #!/usr/bin/env bash -# Tier-1 alpha research runner for a high-CPU MacBook (local only). +# Research helpers for a high-CPU MacBook (local only). # -# Prerequisites -# - git checkout research/earnings-gap-and-sue (or later research branch) -# - .env with ALPACA_* (required for OHLCV/ETFs); FMP_* for earnings resume; -# optional ALPHA_VANTAGE_* as earnings fallback -# - Python venv with project deps installed -# - backtest_snapshots/prod.sqlite present (gitignored — copy or rebuild) +# Kept after Tier-1 cleanup: +# --ssl-check diagnose corporate CA / proxy +# --earnings-only resume FMP earnings backfill + 2a/2b (parked) +# --prod-book-matrix re-run 505 vs liquid universe × horizon book matrix +# +# Prerequisites: git checkout research branch, .env, deep research.sqlite for +# book matrix, combined-ca-bundle.pem or certifi when on corp network. # -# Usage # chmod +x scripts/run_tier1_macbook.sh -# ./scripts/run_tier1_macbook.sh # coverage + deep rebuild + harness -# ./scripts/run_tier1_macbook.sh --all # earnings resume + full depth pipeline -# ./scripts/run_tier1_macbook.sh --earnings-only # multi-day FMP backfill + re-run 2a/2b -# ./scripts/run_tier1_macbook.sh --harness-only # skip rebuild; race-guard + IC only -# ./scripts/run_tier1_macbook.sh --coverage-only # bars-per-year probe only -# ./scripts/run_tier1_macbook.sh --sector-resid-deep # deepen shallow + ONE masked grade -# ./scripts/run_tier1_macbook.sh --prod-book-matrix # 4-arm universe×horizon book matrix -# -# Does NOT touch production Postgres, scheduler, gates, or prod config. +# ./scripts/run_tier1_macbook.sh --ssl-check +# ./scripts/run_tier1_macbook.sh --prod-book-matrix set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$ROOT" -# --- defaults (override via flags or env) --- -PROD_SNAP="${PROD_SNAP:-backtest_snapshots/prod.sqlite}" RESEARCH_SNAP="${RESEARCH_SNAP:-backtest_snapshots/research.sqlite}" -HISTORY_DAYS="${HISTORY_DAYS:-5000}" -MIN_BARS="${MIN_BARS:-260}" +PROD_SNAP="${PROD_SNAP:-backtest_snapshots/prod.sqlite}" WORKERS="${WORKERS:-8}" -ALPACA_SLEEP="${ALPACA_SLEEP:-0.15}" FMP_LIMIT="${FMP_LIMIT:-250}" FMP_SLEEP="${FMP_SLEEP:-0.35}" PYTHON="${PYTHON:-python3}" USE_CORP_PROXY="${USE_CORP_PROXY:-0}" - -PHASE="depth" # depth | all | earnings | harness | coverage | ssl | sector-resid-deep | prod-book +PHASE="" usage() { - sed -n '2,25p' "$0" | sed 's/^# \?//' - cat <<'EOF' - -SSL / network (corporate MacBook) - SSL errors usually mean the corp root CA is missing from Python. - 1) Put combined-ca-bundle.pem in the repo root OR $HOME - 2) Or: export SSL_CERT_FILE=/path/to/combined-ca-bundle.pem - 3) Behind corp proxy: USE_CORP_PROXY=1 ./scripts/run_tier1_macbook.sh - 4) Diagnose: ./scripts/run_tier1_macbook.sh --ssl-check -EOF + sed -n '2,16p' "$0" | sed 's/^# \?//' exit "${1:-0}" } while [[ $# -gt 0 ]]; do case "$1" in - --all) PHASE=all; shift ;; - --earnings-only) PHASE=earnings; shift ;; - --harness-only) PHASE=harness; shift ;; - --coverage-only) PHASE=coverage; shift ;; - --depth) PHASE=depth; shift ;; --ssl-check) PHASE=ssl; shift ;; - --sector-resid-deep) PHASE=sector_resid_deep; shift ;; + --earnings-only) PHASE=earnings; shift ;; --prod-book-matrix) PHASE=prod_book; shift ;; --corp-proxy) USE_CORP_PROXY=1; shift ;; - --prod-snap) PROD_SNAP="$2"; shift 2 ;; - --research-snap) RESEARCH_SNAP="$2"; shift 2 ;; - --history-days) HISTORY_DAYS="$2"; shift 2 ;; --workers) WORKERS="$2"; shift 2 ;; - --fmp-limit) FMP_LIMIT="$2"; shift 2 ;; --python) PYTHON="$2"; shift 2 ;; -h|--help) usage 0 ;; *) echo "Unknown flag: $1" >&2; usage 1 ;; esac done +if [[ -z "$PHASE" ]]; then + echo "Pick a phase: --ssl-check | --earnings-only | --prod-book-matrix" >&2 + usage 1 +fi + if [[ -x .venv/bin/python ]]; then PYTHON=".venv/bin/python" elif command -v "$PYTHON" >/dev/null 2>&1; then : else - echo "ERROR: no Python found (tried .venv/bin/python and $PYTHON)" >&2 + echo "ERROR: no Python found" >&2 exit 1 fi log() { printf '\n==> %s\n' "$*"; } die() { echo "ERROR: $*" >&2; exit 1; } +need_file() { [[ -f "$1" ]] || die "missing $1"; } -# --------------------------------------------------------------------------- -# TLS bootstrap — same corp CA path the FastAPI app uses -# --------------------------------------------------------------------------- setup_ssl() { export USE_CORP_PROXY - - # Prefer explicit env, then repo / home corporate bundle, then certifi. if [[ -z "${SSL_CERT_FILE:-}" ]]; then if [[ -f "$ROOT/combined-ca-bundle.pem" ]]; then export SSL_CERT_FILE="$ROOT/combined-ca-bundle.pem" @@ -101,191 +72,68 @@ setup_ssl() { export SSL_CERT_FILE="$HOME/combined-ca-bundle.pem" fi fi - if [[ -n "${SSL_CERT_FILE:-}" && -f "$SSL_CERT_FILE" ]]; then - export REQUESTS_CA_BUNDLE="$SSL_CERT_FILE" - export CURL_CA_BUNDLE="$SSL_CERT_FILE" + export REQUESTS_CA_BUNDLE="$SSL_CERT_FILE" CURL_CA_BUNDLE="$SSL_CERT_FILE" log "SSL CA bundle: $SSL_CERT_FILE" else - # Fall back to certifi if installed local certifi_path certifi_path="$("$PYTHON" -c 'import certifi; print(certifi.where())' 2>/dev/null || true)" if [[ -n "$certifi_path" && -f "$certifi_path" ]]; then - export SSL_CERT_FILE="$certifi_path" - export REQUESTS_CA_BUNDLE="$certifi_path" - export CURL_CA_BUNDLE="$certifi_path" + export SSL_CERT_FILE="$certifi_path" REQUESTS_CA_BUNDLE="$certifi_path" CURL_CA_BUNDLE="$certifi_path" log "SSL CA bundle (certifi): $SSL_CERT_FILE" else log "WARNING: no CA bundle found — SSL may fail on corp networks" - log " Copy combined-ca-bundle.pem to $ROOT/ or \$HOME/" - log " Or: export SSL_CERT_FILE=/path/to/combined-ca-bundle.pem" fi fi - if [[ "$USE_CORP_PROXY" == "1" ]]; then export HTTP_PROXY="${HTTP_PROXY:-http://aproxy.corproot.net:8080}" export HTTPS_PROXY="${HTTPS_PROXY:-http://aproxy.corproot.net:8080}" - export NO_PROXY="${NO_PROXY:-corproot.net,sharedtcs.net,127.0.0.1,localhost,bix.swisscom.com,swisscom.com}" + export NO_PROXY="${NO_PROXY:-corproot.net,sharedtcs.net,127.0.0.1,localhost}" export http_proxy="$HTTP_PROXY" https_proxy="$HTTPS_PROXY" no_proxy="$NO_PROXY" log "Corp proxy enabled: $HTTPS_PROXY" fi - - # Ensure Python process sees the same bootstrap (patches ssl for alpaca-py). export PYTHONPATH="${ROOT}${PYTHONPATH:+:$PYTHONPATH}" } ssl_check() { setup_ssl - log "SSL diagnostic" "$PYTHON" - <<'PY' from app.ssl_bootstrap import bootstrap_ssl, ssl_status -import json -import urllib.request - -ca = bootstrap_ssl() +import json, urllib.request print(json.dumps(ssl_status(), indent=2)) -print("bootstrap_ssl ->", ca) -urls = [ +print("bootstrap ->", bootstrap_ssl()) +for url in ( "https://data.alpaca.markets/v2/stocks/SPY/bars?timeframe=1Day&limit=1", "https://financialmodelingprep.com/stable/profile?symbol=AAPL", - "https://www.alphavantage.co/query?function=TIME_SERIES_DAILY&symbol=IBM", -] -for url in urls: +): try: - req = urllib.request.Request(url, headers={"User-Agent": "signal-platform-ssl-check"}) + req = urllib.request.Request(url, headers={"User-Agent": "ssl-check"}) with urllib.request.urlopen(req, timeout=20) as resp: - print(f"OK {resp.status} {url[:60]}...") + print(f"OK {resp.status} {url[:60]}") except Exception as exc: print(f"FAIL {type(exc).__name__}: {exc}") - print(f" {url[:80]}") PY } -need_file() { - [[ -f "$1" ]] || die "missing $1" -} - -require_prod() { - need_file "$PROD_SNAP" -} - -run_earnings() { - require_prod - log "Earnings backfill (FMP free tier ~${FMP_LIMIT}/day; resume-safe)" - "$PYTHON" scripts/backfill_earnings_events.py \ - --snapshot "$PROD_SNAP" \ - --provider fmp \ - --force-symbol \ - --limit "$FMP_LIMIT" \ - --sleep "$FMP_SLEEP" - - log "Earnings research 2a+2b (report only; no filters shipped)" - "$PYTHON" scripts/run_earnings_research.py \ - --snapshot "$PROD_SNAP" \ - --workers "$WORKERS" \ - --allow-spawn -} - -run_coverage() { - require_prod - log "Coverage probe (bars per year) on $PROD_SNAP" - "$PYTHON" scripts/run_history_depth_research.py \ - --phase coverage \ - --snapshot "$PROD_SNAP" -} - -run_rebuild() { - require_prod - log "Deep rebuild $PROD_SNAP → $RESEARCH_SNAP (history-days=$HISTORY_DAYS)" - log "SURVIVORSHIP: today's constituents backfilled — relative IC only, not levels" - "$PYTHON" scripts/extend_snapshot_universe.py \ - --source "$PROD_SNAP" \ - --output "$RESEARCH_SNAP" \ - --force-copy \ - --history-days "$HISTORY_DAYS" \ - --min-bars "$MIN_BARS" \ - --sleep "$ALPACA_SLEEP" - - log "Refresh SPY + 11 sector ETFs on research snapshot" - "$PYTHON" scripts/fetch_sector_etfs_to_snapshot.py \ - --snapshot "$RESEARCH_SNAP" \ - --history-days "$HISTORY_DAYS" - - log "Also deepen sector ETFs on prod snapshot (for local A/B parity)" - "$PYTHON" scripts/fetch_sector_etfs_to_snapshot.py \ - --snapshot "$PROD_SNAP" \ - --history-days "$HISTORY_DAYS" -} - -run_harness() { - need_file "$RESEARCH_SNAP" - log "Race-guard + full signal harness + era split on $RESEARCH_SNAP" - "$PYTHON" scripts/run_history_depth_research.py \ - --phase harness \ - --snapshot "$RESEARCH_SNAP" \ - --workers "$WORKERS" \ - --allow-spawn -} - -run_sector_resid_deep() { - need_file "$RESEARCH_SNAP" - log "Sector-resid deep test: deepen shallow symbols + ONE liquid-1500 masked grade" - log "Pre-registered PASS/FAIL only — thread ends after this run" - "$PYTHON" scripts/run_sector_resid_deep_test.py \ - --snapshot "$RESEARCH_SNAP" \ - --history-days "$HISTORY_DAYS" \ - --sleep "$ALPACA_SLEEP" \ - --workers "$WORKERS" \ - --allow-spawn -} - -run_prod_book_matrix() { - need_file "$RESEARCH_SNAP" - log "Production book × universe × horizon (4 arms, strategy unchanged)" - "$PYTHON" scripts/run_prod_book_universe_matrix.py \ - --snapshot "$RESEARCH_SNAP" \ - --workers "$WORKERS" \ - --allow-spawn \ - --candidate-cache reports/.cache/prod-book-universe-cands.pkl -} - -log "cwd=$ROOT python=$PYTHON phase=$PHASE workers=$WORKERS" setup_ssl - case "$PHASE" in - ssl) - ssl_check - ;; - sector_resid_deep) - run_sector_resid_deep + ssl) ssl_check ;; + earnings) + need_file "$PROD_SNAP" + log "Earnings backfill + research (parked experiment)" + "$PYTHON" scripts/backfill_earnings_events.py \ + --snapshot "$PROD_SNAP" --provider fmp --force-symbol \ + --limit "$FMP_LIMIT" --sleep "$FMP_SLEEP" + "$PYTHON" scripts/run_earnings_research.py \ + --snapshot "$PROD_SNAP" --workers "$WORKERS" --allow-spawn ;; prod_book) - run_prod_book_matrix - ;; - coverage) - run_coverage - ;; - earnings) - run_earnings - ;; - harness) - run_harness - ;; - depth) - run_coverage - run_rebuild - run_harness - ;; - all) - run_earnings - run_coverage - run_rebuild - run_harness - ;; - *) - die "unknown phase $PHASE" + need_file "$RESEARCH_SNAP" + log "Production book universe × horizon matrix" + "$PYTHON" scripts/run_prod_book_universe_matrix.py \ + --snapshot "$RESEARCH_SNAP" --workers "$WORKERS" --allow-spawn \ + --candidate-cache reports/.cache/prod-book-universe-cands.pkl ;; + *) die "unknown phase $PHASE" ;; esac - -log "Done. Check reports/ and docs/research/history-depth-extension.md" -log "Commit reports on this machine if they look good, or copy them back to Windows." +log "Done." diff --git a/tests/unit/test_backtest_service.py b/tests/unit/test_backtest_service.py index b811a27..aff0eb6 100644 --- a/tests/unit/test_backtest_service.py +++ b/tests/unit/test_backtest_service.py @@ -100,75 +100,6 @@ def test_residual_momentum_removes_market_beta_but_keeps_specific_drift(): assert drift["mom_12_1_resid"] > pure["mom_12_1_resid"] + 0.12 -def test_sector_residual_momentum_two_factor(): - """Pure market+sector beta stock → sector resid ~0; idiosyncratic drift kept.""" - dates, pure_beta, highs, benchmark = _signal_test_series(extra_return=0.0) - # Sector ETF = leveraged market (collinear-ish but not identical). - sector = {d: benchmark[d] * 1.02 + 0.5 for d in dates} - # Stock with pure exposure to market + sector, no alpha. - closes = [100.0] - for i in range(1, len(dates)): - m_prev = benchmark[dates[i - 1]] - m_cur = benchmark[dates[i]] - s_prev = sector[dates[i - 1]] - s_cur = sector[dates[i]] - m_ret = m_cur / m_prev - 1.0 - s_ret = s_cur / s_prev - 1.0 - closes.append(closes[-1] * (1.0 + 0.7 * m_ret + 0.5 * s_ret)) - highs_p = [c * 1.01 for c in closes] - - pure = bt._signal_values( - dates, closes, highs_p, 260, benchmark, sector_etf_closes=sector - ) - assert "mom_12_1_sector_resid" in pure - assert pure["mom_12_1_sector_resid"] == pytest.approx(0.0, abs=0.05) - - # Add idiosyncratic drift — sector residual should keep it. - drift_closes = [100.0] - for i in range(1, len(dates)): - m_prev = benchmark[dates[i - 1]] - m_cur = benchmark[dates[i]] - s_prev = sector[dates[i - 1]] - s_cur = sector[dates[i]] - m_ret = m_cur / m_prev - 1.0 - s_ret = s_cur / s_prev - 1.0 - drift_closes.append( - drift_closes[-1] * (1.0 + 0.7 * m_ret + 0.5 * s_ret + 0.0008) - ) - drift_highs = [c * 1.01 for c in drift_closes] - drift = bt._signal_values( - dates, drift_closes, drift_highs, 260, benchmark, sector_etf_closes=sector - ) - assert drift["mom_12_1_sector_resid"] > pure["mom_12_1_sector_resid"] + 0.10 - - -def test_inject_sector_demeaned_momentum(): - collected = { - "mom_12_1": { - (2024, 1): [ - {"val": 0.20, "fwd": 0.01, "symbol": "AAA"}, - {"val": 0.10, "fwd": 0.02, "symbol": "BBB"}, - {"val": 0.40, "fwd": -0.01, "symbol": "CCC"}, - {"val": 0.00, "fwd": 0.03, "symbol": "DDD"}, - ] - } - } - symbol_to_sector = { - "AAA": "Information Technology", - "BBB": "Information Technology", - "CCC": "Energy", - "DDD": "Energy", - } - bt._inject_sector_demeaned_momentum(collected, symbol_to_sector) - dem = collected["mom_12_1_sector_demeaned"][(2024, 1)] - by_sym = {r["symbol"]: r["val"] for r in dem} - # IT mean = 0.15 → AAA +0.05, BBB -0.05; Energy mean = 0.20 → CCC +0.20, DDD -0.20 - assert by_sym["AAA"] == pytest.approx(0.05) - assert by_sym["BBB"] == pytest.approx(-0.05) - assert by_sym["CCC"] == pytest.approx(0.20) - assert by_sym["DDD"] == pytest.approx(-0.20) - - def test_assigns_raw_and_residual_percentiles_independently(): cands = [ {"iso_week": (2026, 1), "momentum": 0.10, "residual_momentum": 0.30},