research: clean up closed Tier-1 scaffolding from branch
Drop intermediate history-depth reports, sector-residual runners/map/code hooks (evidence stays in final reports + docs), and slim MacBook helper to ssl/earnings/ prod-book-matrix only. SSL bootstrap and archived research conclusions retained.
This commit is contained in:
@@ -791,86 +791,34 @@ def _residual_momentum_12_1(
|
||||
with an intercept estimated over the same window, the arithmetic residuals
|
||||
sum to ~zero by construction, which would destroy the signal.
|
||||
"""
|
||||
return _multi_factor_residual_momentum_12_1(
|
||||
dates, closes, i, [benchmark_closes] if benchmark_closes else None
|
||||
)
|
||||
|
||||
|
||||
def _multi_factor_residual_momentum_12_1(
|
||||
dates: list[date],
|
||||
closes: list[float],
|
||||
i: int,
|
||||
factor_closes: list[dict[date, float]] | None,
|
||||
) -> float | None:
|
||||
"""12-1 residual momentum vs one or more factors (OLS, no intercept).
|
||||
|
||||
Same formation window as raw / single-factor residual momentum:
|
||||
daily returns from close[i-252] → close[i-21], require ≥100 paired obs.
|
||||
Factors are stacked as columns; betas are OLS without intercept so the
|
||||
cumulative residual is not forced to zero.
|
||||
"""
|
||||
if not factor_closes or i - 252 < 0:
|
||||
return None
|
||||
n_factors = len(factor_closes)
|
||||
if n_factors < 1:
|
||||
if not benchmark_closes or i - 252 < 0:
|
||||
return None
|
||||
|
||||
stock_rets: list[float] = []
|
||||
factor_rets: list[list[float]] = [[] for _ in range(n_factors)]
|
||||
market_rets: list[float] = []
|
||||
# Same daily intervals as mom_12_1: close[i-252] -> close[i-21].
|
||||
for k in range(i - 251, i - 20):
|
||||
prev_close = closes[k - 1]
|
||||
if prev_close <= 0:
|
||||
continue
|
||||
f_day: list[float] = []
|
||||
ok = True
|
||||
for fc in factor_closes:
|
||||
f_prev = fc.get(dates[k - 1])
|
||||
f_cur = fc.get(dates[k])
|
||||
if f_prev is None or f_cur is None or f_prev <= 0:
|
||||
ok = False
|
||||
break
|
||||
f_day.append(f_cur / f_prev - 1.0)
|
||||
if not ok:
|
||||
bench_prev = benchmark_closes.get(dates[k - 1])
|
||||
bench_cur = benchmark_closes.get(dates[k])
|
||||
if prev_close <= 0 or bench_prev is None or bench_cur is None or bench_prev <= 0:
|
||||
continue
|
||||
stock_rets.append(closes[k] / prev_close - 1.0)
|
||||
for j, r in enumerate(f_day):
|
||||
factor_rets[j].append(r)
|
||||
market_rets.append(bench_cur / bench_prev - 1.0)
|
||||
|
||||
n = len(stock_rets)
|
||||
if n < 100:
|
||||
if len(stock_rets) < 100:
|
||||
return None
|
||||
|
||||
if n_factors == 1:
|
||||
# Fast path: identical algebra to the historical single-factor form.
|
||||
market_rets = factor_rets[0]
|
||||
mean_market = sum(market_rets) / n
|
||||
mean_stock = sum(stock_rets) / n
|
||||
var_market = sum((x - mean_market) ** 2 for x in market_rets)
|
||||
if var_market <= 0:
|
||||
return None
|
||||
cov = sum(
|
||||
(stock_rets[k] - mean_stock) * (market_rets[k] - mean_market)
|
||||
for k in range(n)
|
||||
)
|
||||
beta = cov / var_market
|
||||
return sum(stock_rets[k] - beta * market_rets[k] for k in range(n))
|
||||
|
||||
# OLS without intercept: β = (X'X)^{-1} X'y for X columns = factor returns.
|
||||
# Implemented for exactly two factors (market + sector); refuse larger.
|
||||
if n_factors != 2:
|
||||
mean_market = sum(market_rets) / len(market_rets)
|
||||
mean_stock = sum(stock_rets) / len(stock_rets)
|
||||
var_market = sum((x - mean_market) ** 2 for x in market_rets)
|
||||
if var_market <= 0:
|
||||
return None
|
||||
f1, f2 = factor_rets[0], factor_rets[1]
|
||||
s11 = sum(a * a for a in f1)
|
||||
s22 = sum(a * a for a in f2)
|
||||
s12 = sum(f1[k] * f2[k] for k in range(n))
|
||||
sy1 = sum(stock_rets[k] * f1[k] for k in range(n))
|
||||
sy2 = sum(stock_rets[k] * f2[k] for k in range(n))
|
||||
det = s11 * s22 - s12 * s12
|
||||
if abs(det) < 1e-18:
|
||||
return None
|
||||
b1 = (s22 * sy1 - s12 * sy2) / det
|
||||
b2 = (s11 * sy2 - s12 * sy1) / det
|
||||
return sum(stock_rets[k] - b1 * f1[k] - b2 * f2[k] for k in range(n))
|
||||
cov = sum(
|
||||
(stock_rets[k] - mean_stock) * (market_rets[k] - mean_market)
|
||||
for k in range(len(stock_rets))
|
||||
)
|
||||
beta = cov / var_market
|
||||
return sum(stock_rets[k] - beta * market_rets[k] for k in range(len(stock_rets)))
|
||||
|
||||
|
||||
def _realized_vol_6m(closes: list[float], i: int) -> float | None:
|
||||
@@ -895,7 +843,6 @@ def _signal_values(
|
||||
highs: list[float],
|
||||
i: int,
|
||||
benchmark_closes: dict[date, float] | None = None,
|
||||
sector_etf_closes: dict[date, float] | None = None,
|
||||
) -> dict[str, float]:
|
||||
"""Point-in-time candidate signals at as-of index ``i`` (price-only).
|
||||
|
||||
@@ -907,11 +854,6 @@ def _signal_values(
|
||||
higher = nearer the high, expect positive IC). ``vol_6m`` is 126-day realized
|
||||
volatility (expect negative IC if the low-volatility anomaly holds).
|
||||
``fip_id`` is Da/Gurun/Warachka information discreteness (expect negative IC).
|
||||
|
||||
When ``sector_etf_closes`` is supplied (research path), also emit
|
||||
``mom_12_1_sector_resid``: two-factor residual vs SPY + sector ETF.
|
||||
Cross-sectional ``mom_12_1_sector_demeaned`` is injected later from the
|
||||
full weekly cross-section (cannot be computed per-ticker alone).
|
||||
"""
|
||||
out: dict[str, float] = {}
|
||||
if i - 252 >= 0 and closes[i - 252] > 0:
|
||||
@@ -919,12 +861,6 @@ def _signal_values(
|
||||
residual = _residual_momentum_12_1(dates, closes, i, benchmark_closes)
|
||||
if residual is not None:
|
||||
out["mom_12_1_resid"] = residual
|
||||
if benchmark_closes and sector_etf_closes:
|
||||
sector_resid = _multi_factor_residual_momentum_12_1(
|
||||
dates, closes, i, [benchmark_closes, sector_etf_closes]
|
||||
)
|
||||
if sector_resid is not None:
|
||||
out["mom_12_1_sector_resid"] = sector_resid
|
||||
fip = _fip_id(closes, i)
|
||||
if fip is not None:
|
||||
out["fip_id"] = fip
|
||||
@@ -1011,16 +947,14 @@ def _accumulate_signal_series(
|
||||
benchmark_closes: dict[date, float] | None = None,
|
||||
*,
|
||||
symbol: str | None = None,
|
||||
sector_etf_closes: dict[date, float] | None = None,
|
||||
) -> None:
|
||||
"""For each weekly as-of bar, emit (signal, forward-return) pairs keyed by ISO
|
||||
week into ``collected[name][week_key]``. Forward return is close-to-close over
|
||||
HORIZON trading days. Mutates ``collected`` (a dict of dict of list).
|
||||
|
||||
When ``BACKTEST_LIQUID_BREADTH`` is set, observations are dicts with PIT
|
||||
liquidity fields for the mask. When ``symbol`` is provided, observations are
|
||||
also dicts (so sector demeaning can group by name); otherwise plain
|
||||
``(val, fwd)`` tuples keep the production path unchanged.
|
||||
liquidity fields for the mask; otherwise plain ``(val, fwd)`` tuples so the
|
||||
production signal path stays unchanged.
|
||||
"""
|
||||
n = len(records)
|
||||
if n < HORIZON + 21:
|
||||
@@ -1030,7 +964,6 @@ def _accumulate_signal_series(
|
||||
volumes = [float(getattr(r, "volume", 0) or 0) for r in records]
|
||||
dates = [r.date for r in records]
|
||||
liquid_mode = _liquid_breadth_top_n() > 0
|
||||
rich = liquid_mode or symbol is not None
|
||||
for i in _weekly_asof_indices(records):
|
||||
j = i + HORIZON
|
||||
if j >= n or closes[i] <= 0:
|
||||
@@ -1039,79 +972,19 @@ def _accumulate_signal_series(
|
||||
iso = records[i].date.isocalendar()
|
||||
week_key = (iso[0], iso[1])
|
||||
dvol = _median_dollar_vol_63(closes, volumes, i) if liquid_mode else None
|
||||
for name, val in _signal_values(
|
||||
dates, closes, highs, i, benchmark_closes, sector_etf_closes
|
||||
).items():
|
||||
if rich:
|
||||
row = {
|
||||
for name, val in _signal_values(dates, closes, highs, i, benchmark_closes).items():
|
||||
if liquid_mode:
|
||||
collected[name][week_key].append({
|
||||
"val": val,
|
||||
"fwd": fwd,
|
||||
"close": closes[i],
|
||||
"median_dvol_63": dvol,
|
||||
"symbol": symbol,
|
||||
}
|
||||
if liquid_mode:
|
||||
row["close"] = closes[i]
|
||||
row["median_dvol_63"] = dvol
|
||||
collected[name][week_key].append(row)
|
||||
})
|
||||
else:
|
||||
collected[name][week_key].append((val, fwd))
|
||||
|
||||
|
||||
def _inject_sector_demeaned_momentum(
|
||||
collected: dict,
|
||||
symbol_to_sector: dict[str, str],
|
||||
*,
|
||||
min_sector_names: int = 2,
|
||||
) -> None:
|
||||
"""Cross-sectional demean of ``mom_12_1`` within GICS sector per week.
|
||||
|
||||
``mom_12_1_sector_demeaned[i] = mom_12_1[i] − mean(mom_12_1 | sector_i)``.
|
||||
Requires rich observations with a ``symbol`` field (research path). Names
|
||||
without a sector label, or sectors with fewer than ``min_sector_names``
|
||||
members that week, are dropped from the demeaned series.
|
||||
"""
|
||||
if not symbol_to_sector or "mom_12_1" not in collected:
|
||||
return
|
||||
from app.services.sector_map import normalise_symbol
|
||||
|
||||
demeaned: dict = defaultdict(list)
|
||||
for week_key, recs in collected["mom_12_1"].items():
|
||||
parsed: list[tuple[str, float, float, object]] = []
|
||||
by_sector: dict[str, list[float]] = defaultdict(list)
|
||||
for rec in recs:
|
||||
pair = _obs_val_fwd(rec)
|
||||
if pair is None:
|
||||
continue
|
||||
val, fwd = pair
|
||||
if isinstance(rec, dict):
|
||||
sym = rec.get("symbol")
|
||||
else:
|
||||
sym = None
|
||||
if not sym:
|
||||
continue
|
||||
sector = symbol_to_sector.get(normalise_symbol(str(sym)))
|
||||
if not sector:
|
||||
continue
|
||||
parsed.append((sector, val, fwd, rec))
|
||||
by_sector[sector].append(val)
|
||||
means = {
|
||||
sec: sum(vs) / len(vs)
|
||||
for sec, vs in by_sector.items()
|
||||
if len(vs) >= min_sector_names
|
||||
}
|
||||
for sector, val, fwd, rec in parsed:
|
||||
if sector not in means:
|
||||
continue
|
||||
dval = val - means[sector]
|
||||
if isinstance(rec, dict):
|
||||
row = dict(rec)
|
||||
row["val"] = dval
|
||||
demeaned[week_key].append(row)
|
||||
else:
|
||||
demeaned[week_key].append((dval, fwd))
|
||||
if demeaned:
|
||||
collected["mom_12_1_sector_demeaned"] = demeaned
|
||||
|
||||
|
||||
def _rank(xs: list[float]) -> list[float]:
|
||||
"""Average (tie-corrected) ranks, 1-based."""
|
||||
order = sorted(range(len(xs)), key=lambda k: xs[k])
|
||||
@@ -1388,38 +1261,14 @@ def _signal_series(
|
||||
benchmark_closes: dict[date, float] | None = None,
|
||||
*,
|
||||
symbol: str | None = None,
|
||||
sector_etf_closes: dict[date, float] | None = None,
|
||||
) -> dict:
|
||||
"""Per-ticker signal/forward-return series as a PLAIN (picklable) nested dict
|
||||
— no defaultdict/lambda — so it can cross a process boundary."""
|
||||
tmp: dict = defaultdict(lambda: defaultdict(list))
|
||||
_accumulate_signal_series(
|
||||
records,
|
||||
tmp,
|
||||
benchmark_closes,
|
||||
symbol=symbol,
|
||||
sector_etf_closes=sector_etf_closes,
|
||||
)
|
||||
_accumulate_signal_series(records, tmp, benchmark_closes, symbol=symbol)
|
||||
return {name: dict(weeks) for name, weeks in tmp.items()}
|
||||
|
||||
|
||||
def _sector_etf_closes_for_symbol(
|
||||
symbol: str,
|
||||
symbol_to_sector: dict[str, str] | None,
|
||||
sector_etf_closes: dict[str, dict[date, float]] | None,
|
||||
) -> dict[date, float] | None:
|
||||
"""Resolve the sector-ETF close series for one ticker, or None."""
|
||||
if not symbol_to_sector or not sector_etf_closes:
|
||||
return None
|
||||
from app.services.sector_map import etf_for_symbol
|
||||
|
||||
etf = etf_for_symbol(symbol, symbol_to_sector)
|
||||
if not etf:
|
||||
return None
|
||||
series = sector_etf_closes.get(etf)
|
||||
return series or None
|
||||
|
||||
|
||||
def _replay_and_signals(
|
||||
symbol: str,
|
||||
columns: tuple,
|
||||
@@ -1429,8 +1278,6 @@ def _replay_and_signals(
|
||||
target_model: str = PRODUCTION_GTL_TARGET_MODEL,
|
||||
cadence: str = DEFAULT_BACKTEST_CADENCE,
|
||||
signal_only: bool = False,
|
||||
sector_etf_closes: dict[str, dict[date, float]] | None = None,
|
||||
symbol_to_sector: dict[str, str] | None = None,
|
||||
) -> tuple[list[dict], dict]:
|
||||
"""The CPU-bound per-ticker work, as a top-level (picklable) function so it can
|
||||
run in a worker process. Takes primitive column arrays (cheap to pickle),
|
||||
@@ -1457,17 +1304,9 @@ def _replay_and_signals(
|
||||
target_model,
|
||||
cadence,
|
||||
)
|
||||
etf_closes = _sector_etf_closes_for_symbol(
|
||||
symbol, symbol_to_sector, sector_etf_closes
|
||||
)
|
||||
return (
|
||||
candidates,
|
||||
_signal_series(
|
||||
bars,
|
||||
benchmark_closes,
|
||||
symbol=symbol,
|
||||
sector_etf_closes=etf_closes,
|
||||
),
|
||||
_signal_series(bars, benchmark_closes, symbol=symbol),
|
||||
)
|
||||
|
||||
|
||||
@@ -4218,41 +4057,6 @@ async def run_backtest(
|
||||
except Exception:
|
||||
logger.exception("Benchmark load for residual momentum failed")
|
||||
|
||||
# Optional sector residualisation (research): local ticker→sector map + sector
|
||||
# ETF closes stored in benchmark_prices. Absent map/series → no sector signals.
|
||||
symbol_to_sector: dict[str, str] = {}
|
||||
sector_etf_closes: dict[str, dict[date, float]] = {}
|
||||
try:
|
||||
from app.services.sector_map import (
|
||||
SECTOR_ETFS,
|
||||
load_ticker_sector_map,
|
||||
normalise_symbol,
|
||||
)
|
||||
from app.services.benchmark_service import load_benchmark_closes
|
||||
|
||||
map_path = os.getenv("BACKTEST_SECTOR_MAP_PATH", "").strip() or None
|
||||
symbol_to_sector = {
|
||||
normalise_symbol(k): v
|
||||
for k, v in load_ticker_sector_map(map_path).items()
|
||||
}
|
||||
if symbol_to_sector:
|
||||
for etf in SECTOR_ETFS:
|
||||
try:
|
||||
series = await load_benchmark_closes(db, etf)
|
||||
except Exception:
|
||||
series = {}
|
||||
if series:
|
||||
sector_etf_closes[etf] = series
|
||||
logger.info(json.dumps({
|
||||
"event": "backtest_sector_context_loaded",
|
||||
"sector_map_size": len(symbol_to_sector),
|
||||
"sector_etfs_loaded": sorted(sector_etf_closes),
|
||||
}))
|
||||
except Exception:
|
||||
logger.exception("Sector residual context load failed; continuing without")
|
||||
symbol_to_sector = {}
|
||||
sector_etf_closes = {}
|
||||
|
||||
def _merge(result: tuple[list[dict], dict]) -> None:
|
||||
cands, series = result
|
||||
candidates.extend(cands)
|
||||
@@ -4303,8 +4107,6 @@ async def run_backtest(
|
||||
target_model,
|
||||
cadence,
|
||||
ticker.symbol in rank_only_symbols,
|
||||
sector_etf_closes or None,
|
||||
symbol_to_sector or None,
|
||||
))
|
||||
for result in await asyncio.gather(*futures, return_exceptions=True):
|
||||
if isinstance(result, Exception):
|
||||
@@ -4333,8 +4135,6 @@ async def run_backtest(
|
||||
target_model,
|
||||
cadence,
|
||||
ticker.symbol in rank_only_symbols,
|
||||
sector_etf_closes or None,
|
||||
symbol_to_sector or None,
|
||||
))
|
||||
except Exception:
|
||||
logger.exception("Backtest replay failed for %s", ticker.symbol)
|
||||
@@ -4342,13 +4142,6 @@ async def run_backtest(
|
||||
if progress_cb is not None and total:
|
||||
progress_cb(total, total, "")
|
||||
|
||||
# Cross-sectional sector demean needs the full weekly universe.
|
||||
if symbol_to_sector:
|
||||
try:
|
||||
_inject_sector_demeaned_momentum(collected, symbol_to_sector)
|
||||
except Exception:
|
||||
logger.exception("Sector demeaned momentum injection failed")
|
||||
|
||||
# Cross-sectional momentum: rank every week's universe, then "qualified" means
|
||||
# floors + top ``min_momentum_percentile`` by promoted residual 12-1 momentum
|
||||
# (raw 12-1 fallback only when benchmark data is unavailable).
|
||||
|
||||
@@ -1,145 +0,0 @@
|
||||
"""Ticker → GICS sector → SPDR sector ETF mapping (research only).
|
||||
|
||||
Sector residual momentum residualizes 12-1 momentum against SPY and the name's
|
||||
sector ETF. Labels are persisted under ``data/research/ticker_sector_map.json``
|
||||
so research runs do not depend on live FMP calls.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
# Eleven SPDR sector ETFs. Auxiliary series only — never tradable book members.
|
||||
SECTOR_ETFS: tuple[str, ...] = (
|
||||
"XLB",
|
||||
"XLC",
|
||||
"XLE",
|
||||
"XLF",
|
||||
"XLI",
|
||||
"XLK",
|
||||
"XLP",
|
||||
"XLRE",
|
||||
"XLU",
|
||||
"XLV",
|
||||
"XLY",
|
||||
)
|
||||
|
||||
# GICS sector name (and common aliases) → SPDR ETF.
|
||||
# Keys are lower-case for matching.
|
||||
GICS_SECTOR_TO_ETF: dict[str, str] = {
|
||||
"materials": "XLB",
|
||||
"basic materials": "XLB",
|
||||
"communication services": "XLC",
|
||||
"communications": "XLC",
|
||||
"telecommunication services": "XLC",
|
||||
"energy": "XLE",
|
||||
"financials": "XLF",
|
||||
"financial services": "XLF",
|
||||
"financial": "XLF",
|
||||
"industrials": "XLI",
|
||||
"industrial goods": "XLI",
|
||||
"information technology": "XLK",
|
||||
"technology": "XLK",
|
||||
"consumer staples": "XLP",
|
||||
"consumer defensive": "XLP",
|
||||
"real estate": "XLRE",
|
||||
"utilities": "XLU",
|
||||
"health care": "XLV",
|
||||
"healthcare": "XLV",
|
||||
"consumer discretionary": "XLY",
|
||||
"consumer cyclical": "XLY",
|
||||
}
|
||||
|
||||
DEFAULT_SECTOR_MAP_PATH = Path("data/research/ticker_sector_map.json")
|
||||
|
||||
|
||||
def normalise_symbol(symbol: str) -> str:
|
||||
"""Alpaca-style symbols: BRK.B / BRK/B → BRK-B."""
|
||||
s = str(symbol or "").strip().upper()
|
||||
s = s.replace(".", "-").replace("/", "-")
|
||||
return s
|
||||
|
||||
|
||||
def sector_to_etf(sector: str | None) -> str | None:
|
||||
if not sector:
|
||||
return None
|
||||
return GICS_SECTOR_TO_ETF.get(str(sector).strip().lower())
|
||||
|
||||
|
||||
def etf_for_symbol(symbol: str, symbol_to_sector: dict[str, str]) -> str | None:
|
||||
sector = symbol_to_sector.get(normalise_symbol(symbol))
|
||||
return sector_to_etf(sector)
|
||||
|
||||
|
||||
def load_ticker_sector_map(path: Path | str | None = None) -> dict[str, str]:
|
||||
"""Load ``{symbol: gics_sector}`` from JSON. Empty dict if missing."""
|
||||
p = Path(path) if path is not None else DEFAULT_SECTOR_MAP_PATH
|
||||
if not p.exists():
|
||||
return {}
|
||||
raw = json.loads(p.read_text(encoding="utf-8"))
|
||||
if not isinstance(raw, dict):
|
||||
return {}
|
||||
out: dict[str, str] = {}
|
||||
# Accept either flat map or {"map": {...}, "meta": ...}
|
||||
payload = raw.get("map") if "map" in raw and isinstance(raw.get("map"), dict) else raw
|
||||
if not isinstance(payload, dict):
|
||||
return {}
|
||||
for sym, sector in payload.items():
|
||||
if sym in ("meta", "schema_version", "map"):
|
||||
continue
|
||||
if sector is None:
|
||||
continue
|
||||
ns = normalise_symbol(str(sym))
|
||||
if ns:
|
||||
out[ns] = str(sector).strip()
|
||||
return out
|
||||
|
||||
|
||||
def save_ticker_sector_map(
|
||||
mapping: dict[str, str],
|
||||
path: Path | str | None = None,
|
||||
*,
|
||||
meta: dict[str, Any] | None = None,
|
||||
) -> Path:
|
||||
p = Path(path) if path is not None else DEFAULT_SECTOR_MAP_PATH
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
# Normalise keys on write.
|
||||
clean = {
|
||||
normalise_symbol(k): str(v).strip()
|
||||
for k, v in mapping.items()
|
||||
if k and v and normalise_symbol(k)
|
||||
}
|
||||
payload: dict[str, Any] = {
|
||||
"schema_version": 1,
|
||||
"map": clean,
|
||||
"meta": meta or {},
|
||||
}
|
||||
p.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
return p
|
||||
|
||||
|
||||
def coverage_stats(
|
||||
symbols: list[str], mapping: dict[str, str]
|
||||
) -> dict[str, Any]:
|
||||
total = len(symbols)
|
||||
mapped = [s for s in symbols if normalise_symbol(s) in mapping]
|
||||
with_etf = [
|
||||
s
|
||||
for s in mapped
|
||||
if sector_to_etf(mapping[normalise_symbol(s)]) is not None
|
||||
]
|
||||
missing = [s for s in symbols if normalise_symbol(s) not in mapping]
|
||||
by_sector: dict[str, int] = {}
|
||||
for s in mapped:
|
||||
sec = mapping[normalise_symbol(s)]
|
||||
by_sector[sec] = by_sector.get(sec, 0) + 1
|
||||
return {
|
||||
"universe": total,
|
||||
"mapped": len(mapped),
|
||||
"mapped_pct": round(100.0 * len(mapped) / total, 1) if total else 0.0,
|
||||
"with_etf": len(with_etf),
|
||||
"missing": missing,
|
||||
"by_sector": dict(sorted(by_sector.items(), key=lambda kv: (-kv[1], kv[0]))),
|
||||
}
|
||||
@@ -1,543 +0,0 @@
|
||||
{
|
||||
"map": {
|
||||
"A": "Health Care",
|
||||
"AAPL": "Information Technology",
|
||||
"ABBV": "Health Care",
|
||||
"ABNB": "Consumer Discretionary",
|
||||
"ABT": "Health Care",
|
||||
"ACGL": "Financials",
|
||||
"ACN": "Information Technology",
|
||||
"ADBE": "Information Technology",
|
||||
"ADI": "Information Technology",
|
||||
"ADM": "Consumer Staples",
|
||||
"ADP": "Industrials",
|
||||
"ADSK": "Information Technology",
|
||||
"AEE": "Utilities",
|
||||
"AEP": "Utilities",
|
||||
"AES": "Utilities",
|
||||
"AFL": "Financials",
|
||||
"AIG": "Financials",
|
||||
"AIZ": "Financials",
|
||||
"AJG": "Financials",
|
||||
"AKAM": "Information Technology",
|
||||
"ALB": "Materials",
|
||||
"ALGN": "Health Care",
|
||||
"ALL": "Financials",
|
||||
"ALLE": "Industrials",
|
||||
"AMAT": "Information Technology",
|
||||
"AMCR": "Materials",
|
||||
"AMD": "Information Technology",
|
||||
"AME": "Industrials",
|
||||
"AMGN": "Health Care",
|
||||
"AMP": "Financials",
|
||||
"AMT": "Real Estate",
|
||||
"AMZN": "Consumer Discretionary",
|
||||
"ANET": "Information Technology",
|
||||
"AON": "Financials",
|
||||
"AOS": "Industrials",
|
||||
"APA": "Energy",
|
||||
"APD": "Materials",
|
||||
"APH": "Information Technology",
|
||||
"APO": "Financials",
|
||||
"APP": "Information Technology",
|
||||
"APTV": "Consumer Discretionary",
|
||||
"ARE": "Real Estate",
|
||||
"ARES": "Financials",
|
||||
"ATO": "Utilities",
|
||||
"AVB": "Real Estate",
|
||||
"AVGO": "Information Technology",
|
||||
"AVY": "Materials",
|
||||
"AWK": "Utilities",
|
||||
"AXON": "Industrials",
|
||||
"AXP": "Financials",
|
||||
"AZO": "Consumer Discretionary",
|
||||
"BA": "Industrials",
|
||||
"BAC": "Financials",
|
||||
"BALL": "Materials",
|
||||
"BAX": "Health Care",
|
||||
"BBY": "Consumer Discretionary",
|
||||
"BDX": "Health Care",
|
||||
"BEN": "Financials",
|
||||
"BF-B": "Consumer Staples",
|
||||
"BG": "Consumer Staples",
|
||||
"BIIB": "Health Care",
|
||||
"BK": "Financial Services",
|
||||
"BKNG": "Consumer Discretionary",
|
||||
"BKR": "Energy",
|
||||
"BLDR": "Industrials",
|
||||
"BLK": "Financials",
|
||||
"BMY": "Health Care",
|
||||
"BR": "Industrials",
|
||||
"BRK-B": "Financials",
|
||||
"BRO": "Financials",
|
||||
"BSX": "Health Care",
|
||||
"BX": "Financials",
|
||||
"BXP": "Real Estate",
|
||||
"C": "Financials",
|
||||
"CAG": "Consumer Defensive",
|
||||
"CAH": "Health Care",
|
||||
"CARR": "Industrials",
|
||||
"CASY": "Consumer Staples",
|
||||
"CAT": "Industrials",
|
||||
"CB": "Financials",
|
||||
"CBOE": "Financials",
|
||||
"CBRE": "Real Estate",
|
||||
"CCI": "Real Estate",
|
||||
"CCL": "Consumer Discretionary",
|
||||
"CDNS": "Information Technology",
|
||||
"CDW": "Information Technology",
|
||||
"CEG": "Utilities",
|
||||
"CF": "Materials",
|
||||
"CFG": "Financials",
|
||||
"CHD": "Consumer Staples",
|
||||
"CHRW": "Industrials",
|
||||
"CHTR": "Communication Services",
|
||||
"CI": "Health Care",
|
||||
"CIEN": "Information Technology",
|
||||
"CINF": "Financials",
|
||||
"CL": "Consumer Staples",
|
||||
"CLX": "Consumer Staples",
|
||||
"CMCSA": "Communication Services",
|
||||
"CME": "Financials",
|
||||
"CMG": "Consumer Discretionary",
|
||||
"CMI": "Industrials",
|
||||
"CMS": "Utilities",
|
||||
"CNC": "Health Care",
|
||||
"CNP": "Utilities",
|
||||
"COF": "Financials",
|
||||
"COHR": "Information Technology",
|
||||
"COIN": "Financials",
|
||||
"COO": "Health Care",
|
||||
"COP": "Energy",
|
||||
"COR": "Health Care",
|
||||
"COST": "Consumer Staples",
|
||||
"CPAY": "Financials",
|
||||
"CPB": "Consumer Defensive",
|
||||
"CPRT": "Industrials",
|
||||
"CPT": "Real Estate",
|
||||
"CRH": "Materials",
|
||||
"CRL": "Health Care",
|
||||
"CRM": "Information Technology",
|
||||
"CRWD": "Information Technology",
|
||||
"CSCO": "Information Technology",
|
||||
"CSGP": "Real Estate",
|
||||
"CSX": "Industrials",
|
||||
"CTAS": "Industrials",
|
||||
"CTRA": "Energy",
|
||||
"CTSH": "Information Technology",
|
||||
"CTVA": "Materials",
|
||||
"CVNA": "Consumer Discretionary",
|
||||
"CVS": "Health Care",
|
||||
"CVX": "Energy",
|
||||
"D": "Utilities",
|
||||
"DAL": "Industrials",
|
||||
"DASH": "Consumer Discretionary",
|
||||
"DD": "Materials",
|
||||
"DDOG": "Information Technology",
|
||||
"DE": "Industrials",
|
||||
"DECK": "Consumer Discretionary",
|
||||
"DELL": "Information Technology",
|
||||
"DG": "Consumer Staples",
|
||||
"DGX": "Health Care",
|
||||
"DHI": "Consumer Discretionary",
|
||||
"DHR": "Health Care",
|
||||
"DIS": "Communication Services",
|
||||
"DLR": "Real Estate",
|
||||
"DLTR": "Consumer Staples",
|
||||
"DOC": "Real Estate",
|
||||
"DOV": "Industrials",
|
||||
"DOW": "Materials",
|
||||
"DPZ": "Consumer Discretionary",
|
||||
"DRI": "Consumer Discretionary",
|
||||
"DTE": "Utilities",
|
||||
"DUK": "Utilities",
|
||||
"DVA": "Health Care",
|
||||
"DVN": "Energy",
|
||||
"DXCM": "Health Care",
|
||||
"EA": "Communication Services",
|
||||
"EBAY": "Consumer Discretionary",
|
||||
"ECL": "Materials",
|
||||
"ED": "Utilities",
|
||||
"EFX": "Industrials",
|
||||
"EG": "Financials",
|
||||
"EIX": "Utilities",
|
||||
"EL": "Consumer Staples",
|
||||
"ELV": "Health Care",
|
||||
"EME": "Industrials",
|
||||
"EMR": "Industrials",
|
||||
"EOG": "Energy",
|
||||
"EPAM": "Technology",
|
||||
"EQIX": "Real Estate",
|
||||
"EQR": "Real Estate",
|
||||
"EQT": "Energy",
|
||||
"ERIE": "Financials",
|
||||
"ES": "Utilities",
|
||||
"ESS": "Real Estate",
|
||||
"ETN": "Industrials",
|
||||
"ETR": "Utilities",
|
||||
"EVRG": "Utilities",
|
||||
"EW": "Health Care",
|
||||
"EXC": "Utilities",
|
||||
"EXE": "Energy",
|
||||
"EXPD": "Industrials",
|
||||
"EXPE": "Consumer Discretionary",
|
||||
"EXR": "Real Estate",
|
||||
"F": "Consumer Discretionary",
|
||||
"FANG": "Energy",
|
||||
"FAST": "Industrials",
|
||||
"FCX": "Materials",
|
||||
"FDS": "Financials",
|
||||
"FDX": "Industrials",
|
||||
"FE": "Utilities",
|
||||
"FFIV": "Information Technology",
|
||||
"FICO": "Information Technology",
|
||||
"FIS": "Financials",
|
||||
"FISV": "Financials",
|
||||
"FITB": "Financials",
|
||||
"FIX": "Industrials",
|
||||
"FOX": "Communication Services",
|
||||
"FOXA": "Communication Services",
|
||||
"FRT": "Real Estate",
|
||||
"FSLR": "Information Technology",
|
||||
"FTNT": "Information Technology",
|
||||
"FTV": "Industrials",
|
||||
"GD": "Industrials",
|
||||
"GDDY": "Information Technology",
|
||||
"GE": "Industrials",
|
||||
"GEHC": "Health Care",
|
||||
"GEN": "Information Technology",
|
||||
"GEV": "Industrials",
|
||||
"GILD": "Health Care",
|
||||
"GIS": "Consumer Staples",
|
||||
"GL": "Financials",
|
||||
"GLW": "Information Technology",
|
||||
"GM": "Consumer Discretionary",
|
||||
"GNRC": "Industrials",
|
||||
"GOOG": "Communication Services",
|
||||
"GOOGL": "Communication Services",
|
||||
"GPC": "Consumer Discretionary",
|
||||
"GPN": "Financials",
|
||||
"GRMN": "Consumer Discretionary",
|
||||
"GS": "Financials",
|
||||
"GWW": "Industrials",
|
||||
"HAL": "Energy",
|
||||
"HAS": "Consumer Discretionary",
|
||||
"HBAN": "Financials",
|
||||
"HCA": "Health Care",
|
||||
"HD": "Consumer Discretionary",
|
||||
"HIG": "Financials",
|
||||
"HII": "Industrials",
|
||||
"HLT": "Consumer Discretionary",
|
||||
"HON": "Industrials",
|
||||
"HOOD": "Financials",
|
||||
"HPE": "Information Technology",
|
||||
"HPQ": "Information Technology",
|
||||
"HRL": "Consumer Staples",
|
||||
"HSIC": "Health Care",
|
||||
"HST": "Real Estate",
|
||||
"HSY": "Consumer Staples",
|
||||
"HUBB": "Industrials",
|
||||
"HUM": "Health Care",
|
||||
"HWM": "Industrials",
|
||||
"IBKR": "Financials",
|
||||
"IBM": "Information Technology",
|
||||
"ICE": "Financials",
|
||||
"IDXX": "Health Care",
|
||||
"IEX": "Industrials",
|
||||
"IFF": "Materials",
|
||||
"INCY": "Health Care",
|
||||
"INTC": "Information Technology",
|
||||
"INTU": "Information Technology",
|
||||
"INVH": "Real Estate",
|
||||
"IP": "Materials",
|
||||
"IQV": "Health Care",
|
||||
"IR": "Industrials",
|
||||
"IRM": "Real Estate",
|
||||
"ISRG": "Health Care",
|
||||
"IT": "Information Technology",
|
||||
"ITW": "Industrials",
|
||||
"IVZ": "Financials",
|
||||
"J": "Industrials",
|
||||
"JBHT": "Industrials",
|
||||
"JBL": "Information Technology",
|
||||
"JCI": "Industrials",
|
||||
"JKHY": "Financials",
|
||||
"JNJ": "Health Care",
|
||||
"JPM": "Financials",
|
||||
"KDP": "Consumer Staples",
|
||||
"KEY": "Financials",
|
||||
"KEYS": "Information Technology",
|
||||
"KHC": "Consumer Staples",
|
||||
"KIM": "Real Estate",
|
||||
"KKR": "Financials",
|
||||
"KLAC": "Information Technology",
|
||||
"KMB": "Consumer Staples",
|
||||
"KMI": "Energy",
|
||||
"KO": "Consumer Staples",
|
||||
"KR": "Consumer Staples",
|
||||
"KVUE": "Consumer Staples",
|
||||
"L": "Financials",
|
||||
"LDOS": "Industrials",
|
||||
"LEN": "Consumer Discretionary",
|
||||
"LH": "Health Care",
|
||||
"LHX": "Industrials",
|
||||
"LII": "Industrials",
|
||||
"LIN": "Materials",
|
||||
"LITE": "Information Technology",
|
||||
"LLY": "Health Care",
|
||||
"LMT": "Industrials",
|
||||
"LNT": "Utilities",
|
||||
"LOW": "Consumer Discretionary",
|
||||
"LRCX": "Information Technology",
|
||||
"LULU": "Consumer Discretionary",
|
||||
"LUV": "Industrials",
|
||||
"LVS": "Consumer Discretionary",
|
||||
"LYB": "Materials",
|
||||
"LYV": "Communication Services",
|
||||
"MA": "Financials",
|
||||
"MAA": "Real Estate",
|
||||
"MAR": "Consumer Discretionary",
|
||||
"MAS": "Industrials",
|
||||
"MCD": "Consumer Discretionary",
|
||||
"MCHP": "Information Technology",
|
||||
"MCK": "Health Care",
|
||||
"MCO": "Financials",
|
||||
"MDLZ": "Consumer Staples",
|
||||
"MDT": "Health Care",
|
||||
"MET": "Financials",
|
||||
"META": "Communication Services",
|
||||
"MGM": "Consumer Discretionary",
|
||||
"MKC": "Consumer Staples",
|
||||
"MLM": "Materials",
|
||||
"MMM": "Industrials",
|
||||
"MNST": "Consumer Staples",
|
||||
"MO": "Consumer Staples",
|
||||
"MOS": "Materials",
|
||||
"MPC": "Energy",
|
||||
"MPWR": "Information Technology",
|
||||
"MRK": "Health Care",
|
||||
"MRNA": "Health Care",
|
||||
"MRSH": "Financials",
|
||||
"MS": "Financials",
|
||||
"MSCI": "Financials",
|
||||
"MSFT": "Information Technology",
|
||||
"MSI": "Information Technology",
|
||||
"MSTR": "Technology",
|
||||
"MTB": "Financials",
|
||||
"MTD": "Health Care",
|
||||
"MU": "Information Technology",
|
||||
"NCLH": "Consumer Discretionary",
|
||||
"NDAQ": "Financials",
|
||||
"NDSN": "Industrials",
|
||||
"NEE": "Utilities",
|
||||
"NEM": "Materials",
|
||||
"NFLX": "Communication Services",
|
||||
"NI": "Utilities",
|
||||
"NKE": "Consumer Discretionary",
|
||||
"NOC": "Industrials",
|
||||
"NOW": "Information Technology",
|
||||
"NRG": "Utilities",
|
||||
"NSC": "Industrials",
|
||||
"NTAP": "Information Technology",
|
||||
"NTRS": "Financials",
|
||||
"NUE": "Materials",
|
||||
"NVDA": "Information Technology",
|
||||
"NVR": "Consumer Discretionary",
|
||||
"NWS": "Communication Services",
|
||||
"NWSA": "Communication Services",
|
||||
"NXPI": "Information Technology",
|
||||
"O": "Real Estate",
|
||||
"ODFL": "Industrials",
|
||||
"OKE": "Energy",
|
||||
"OMC": "Communication Services",
|
||||
"ON": "Information Technology",
|
||||
"ORCL": "Information Technology",
|
||||
"ORLY": "Consumer Discretionary",
|
||||
"OTIS": "Industrials",
|
||||
"OXY": "Energy",
|
||||
"PANW": "Information Technology",
|
||||
"PAYX": "Industrials",
|
||||
"PCAR": "Industrials",
|
||||
"PCG": "Utilities",
|
||||
"PEG": "Utilities",
|
||||
"PEP": "Consumer Staples",
|
||||
"PFE": "Health Care",
|
||||
"PFG": "Financials",
|
||||
"PG": "Consumer Staples",
|
||||
"PGR": "Financials",
|
||||
"PH": "Industrials",
|
||||
"PHM": "Consumer Discretionary",
|
||||
"PKG": "Materials",
|
||||
"PLD": "Real Estate",
|
||||
"PLTR": "Information Technology",
|
||||
"PM": "Consumer Staples",
|
||||
"PNC": "Financials",
|
||||
"PNR": "Industrials",
|
||||
"PNW": "Utilities",
|
||||
"PODD": "Health Care",
|
||||
"POOL": "Industrials",
|
||||
"PPG": "Materials",
|
||||
"PPL": "Utilities",
|
||||
"PRU": "Financials",
|
||||
"PSA": "Real Estate",
|
||||
"PSKY": "Communication Services",
|
||||
"PSX": "Energy",
|
||||
"PTC": "Information Technology",
|
||||
"PWR": "Industrials",
|
||||
"PYPL": "Financials",
|
||||
"Q": "Information Technology",
|
||||
"QCOM": "Information Technology",
|
||||
"RCL": "Consumer Discretionary",
|
||||
"REG": "Real Estate",
|
||||
"REGN": "Health Care",
|
||||
"RF": "Financials",
|
||||
"RJF": "Financials",
|
||||
"RL": "Consumer Discretionary",
|
||||
"RMD": "Health Care",
|
||||
"ROK": "Industrials",
|
||||
"ROL": "Industrials",
|
||||
"ROP": "Information Technology",
|
||||
"ROST": "Consumer Discretionary",
|
||||
"RSG": "Industrials",
|
||||
"RTX": "Industrials",
|
||||
"RVTY": "Health Care",
|
||||
"SATS": "Communication Services",
|
||||
"SBAC": "Real Estate",
|
||||
"SBUX": "Consumer Discretionary",
|
||||
"SCHW": "Financials",
|
||||
"SHW": "Materials",
|
||||
"SJM": "Consumer Staples",
|
||||
"SLB": "Energy",
|
||||
"SMCI": "Information Technology",
|
||||
"SNA": "Industrials",
|
||||
"SNDK": "Information Technology",
|
||||
"SNPS": "Information Technology",
|
||||
"SO": "Utilities",
|
||||
"SOLV": "Health Care",
|
||||
"SPCX": "Industrials",
|
||||
"SPG": "Real Estate",
|
||||
"SPGI": "Financials",
|
||||
"SRE": "Utilities",
|
||||
"STE": "Health Care",
|
||||
"STLD": "Materials",
|
||||
"STT": "Financials",
|
||||
"STX": "Information Technology",
|
||||
"STZ": "Consumer Staples",
|
||||
"SW": "Materials",
|
||||
"SWK": "Industrials",
|
||||
"SWKS": "Information Technology",
|
||||
"SYF": "Financials",
|
||||
"SYK": "Health Care",
|
||||
"SYY": "Consumer Staples",
|
||||
"T": "Communication Services",
|
||||
"TAP": "Consumer Staples",
|
||||
"TDG": "Industrials",
|
||||
"TDY": "Information Technology",
|
||||
"TECH": "Health Care",
|
||||
"TEL": "Information Technology",
|
||||
"TER": "Information Technology",
|
||||
"TFC": "Financials",
|
||||
"TGT": "Consumer Staples",
|
||||
"TJX": "Consumer Discretionary",
|
||||
"TKO": "Communication Services",
|
||||
"TMO": "Health Care",
|
||||
"TMUS": "Communication Services",
|
||||
"TPL": "Energy",
|
||||
"TPR": "Consumer Discretionary",
|
||||
"TRGP": "Energy",
|
||||
"TRMB": "Information Technology",
|
||||
"TROW": "Financials",
|
||||
"TRV": "Financials",
|
||||
"TSCO": "Consumer Discretionary",
|
||||
"TSLA": "Consumer Discretionary",
|
||||
"TSN": "Consumer Staples",
|
||||
"TT": "Industrials",
|
||||
"TTD": "Communication Services",
|
||||
"TTWO": "Communication Services",
|
||||
"TXN": "Information Technology",
|
||||
"TXT": "Industrials",
|
||||
"TYL": "Information Technology",
|
||||
"UAL": "Industrials",
|
||||
"UBER": "Industrials",
|
||||
"UDR": "Real Estate",
|
||||
"UHS": "Health Care",
|
||||
"ULTA": "Consumer Discretionary",
|
||||
"UNH": "Health Care",
|
||||
"UNP": "Industrials",
|
||||
"UPS": "Industrials",
|
||||
"URI": "Industrials",
|
||||
"USB": "Financials",
|
||||
"V": "Financials",
|
||||
"VICI": "Real Estate",
|
||||
"VLO": "Energy",
|
||||
"VLTO": "Industrials",
|
||||
"VMC": "Materials",
|
||||
"VRSK": "Industrials",
|
||||
"VRSN": "Information Technology",
|
||||
"VRT": "Industrials",
|
||||
"VRTX": "Health Care",
|
||||
"VST": "Utilities",
|
||||
"VTR": "Real Estate",
|
||||
"VTRS": "Health Care",
|
||||
"VZ": "Communication Services",
|
||||
"WAB": "Industrials",
|
||||
"WAT": "Health Care",
|
||||
"WBD": "Communication Services",
|
||||
"WDAY": "Information Technology",
|
||||
"WDC": "Information Technology",
|
||||
"WEC": "Utilities",
|
||||
"WELL": "Real Estate",
|
||||
"WFC": "Financials",
|
||||
"WM": "Industrials",
|
||||
"WMB": "Energy",
|
||||
"WMT": "Consumer Staples",
|
||||
"WRB": "Financials",
|
||||
"WSM": "Consumer Discretionary",
|
||||
"WST": "Health Care",
|
||||
"WTW": "Financials",
|
||||
"WY": "Real Estate",
|
||||
"WYNN": "Consumer Discretionary",
|
||||
"XEL": "Utilities",
|
||||
"XOM": "Energy",
|
||||
"XYL": "Industrials",
|
||||
"XYZ": "Financials",
|
||||
"YUM": "Consumer Discretionary",
|
||||
"ZBH": "Health Care",
|
||||
"ZBRA": "Information Technology",
|
||||
"ZTS": "Health Care"
|
||||
},
|
||||
"meta": {
|
||||
"built_at": "2026-07-19T05:35:41.184460+00:00",
|
||||
"coverage": {
|
||||
"by_sector": {
|
||||
"Communication Services": 23,
|
||||
"Consumer Defensive": 2,
|
||||
"Consumer Discretionary": 47,
|
||||
"Consumer Staples": 34,
|
||||
"Energy": 22,
|
||||
"Financial Services": 1,
|
||||
"Financials": 75,
|
||||
"Health Care": 58,
|
||||
"Industrials": 81,
|
||||
"Information Technology": 72,
|
||||
"Materials": 26,
|
||||
"Real Estate": 31,
|
||||
"Technology": 2,
|
||||
"Utilities": 31
|
||||
},
|
||||
"mapped": 505,
|
||||
"mapped_pct": 99.8,
|
||||
"universe": 506,
|
||||
"with_etf": 505
|
||||
},
|
||||
"fmp_requests": 10,
|
||||
"from_existing": 0,
|
||||
"from_fmp": 9,
|
||||
"from_sp500_csv": 496,
|
||||
"snapshot": "C:\\Workspace\\signal-platform\\backtest_snapshots\\prod.sqlite",
|
||||
"still_missing": [
|
||||
"RHM"
|
||||
]
|
||||
},
|
||||
"schema_version": 1
|
||||
}
|
||||
@@ -172,9 +172,12 @@ report does not change that without a separate A/B. Flag for human awareness onl
|
||||
|
||||
| file | role |
|
||||
|---|---|
|
||||
| `reports/history-depth-20260719-103315.json` | **authoritative** |
|
||||
| `reports/history-depth-20260719-103315.md` | companion dump |
|
||||
| `reports/history-depth-20260719-093853` … `095156` | **ignore** (partial) |
|
||||
| `reports/history-depth-20260719-103315.json` | Superseded unmasked/two-tier IC dump (do not cite for sector residual) |
|
||||
| `reports/sector-resid-deep-20260719-113319.json` | Authoritative sector-resid deep grade |
|
||||
| `reports/prod-book-universe-horizon-20260719-140737.json` | 505 vs liquid × horizon book matrix |
|
||||
|
||||
Intermediate history-depth partials (093853–095156) and SANITY-FAIL noise were
|
||||
removed in branch cleanup.
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -219,20 +219,17 @@ to reopen promotion.
|
||||
|
||||
---
|
||||
|
||||
## Implementation notes (research machinery)
|
||||
## Implementation notes
|
||||
|
||||
| piece | role |
|
||||
|---|---|
|
||||
| `app/services/sector_map.py` | GICS→ETF map, symbol normalise, JSON load/save |
|
||||
| `app/services/backtest_service.py` | multi-factor residual; `mom_12_1_sector_resid` in `_signal_values`; demean inject |
|
||||
| `scripts/build_ticker_sector_map.py` | SP500 CSV + FMP gap fill |
|
||||
| `scripts/fetch_sector_etfs_to_snapshot.py` | Alpaca → snapshot `benchmark_prices` |
|
||||
| `scripts/run_sector_residual_research.py` | race guard, IC, optional A/B, reports |
|
||||
| `data/research/ticker_sector_map.json` | persisted labels (research only) |
|
||||
Research runners and sector-residual harness hooks were **removed after close**
|
||||
(2026-07-19 cleanup). Evidence remains in the report artifacts below. Do not
|
||||
re-add without a new pre-registered protocol.
|
||||
|
||||
---
|
||||
|
||||
## Artifacts
|
||||
|
||||
- JSON: `reports/sector-residual-20260719-083356.json`
|
||||
- MD copy: `reports/sector-residual-20260719-083356.md`
|
||||
| file | role |
|
||||
|---|---|
|
||||
| `reports/sector-resid-deep-20260719-113319.json` | **Authoritative deep FAIL** |
|
||||
| `reports/sector-residual-20260719-083356.json` | Short-window IC/A/B (superseded for promotion) |
|
||||
|
||||
@@ -1,68 +0,0 @@
|
||||
{
|
||||
"generated_at": "2026-07-19T09:38:53.936226",
|
||||
"survivorship_banner": "SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only \u2014 not levels.",
|
||||
"coverage": {
|
||||
"snapshot": "/Users/taathde3/git/lab/signal_platform/backtest_snapshots/prod.sqlite",
|
||||
"ticker_count": 506,
|
||||
"ohlcv_row_count": 629263,
|
||||
"date_range": {
|
||||
"min": "2021-06-24",
|
||||
"max": "2026-07-02"
|
||||
},
|
||||
"bars_per_year": [
|
||||
{
|
||||
"year": "2021",
|
||||
"bars": 65678,
|
||||
"tickers_with_bars": 494
|
||||
},
|
||||
{
|
||||
"year": "2022",
|
||||
"bars": 124423,
|
||||
"tickers_with_bars": 497
|
||||
},
|
||||
{
|
||||
"year": "2023",
|
||||
"bars": 124479,
|
||||
"tickers_with_bars": 499
|
||||
},
|
||||
{
|
||||
"year": "2024",
|
||||
"bars": 126129,
|
||||
"tickers_with_bars": 501
|
||||
},
|
||||
{
|
||||
"year": "2025",
|
||||
"bars": 125615,
|
||||
"tickers_with_bars": 504
|
||||
},
|
||||
{
|
||||
"year": "2026",
|
||||
"bars": 62939,
|
||||
"tickers_with_bars": 505
|
||||
}
|
||||
],
|
||||
"bars_per_symbol": {
|
||||
"min": 14,
|
||||
"p10": 1261,
|
||||
"p50": 1261,
|
||||
"p90": 1261,
|
||||
"max": 1261
|
||||
},
|
||||
"symbols_by_start_year": {
|
||||
"2021": 494,
|
||||
"2022": 3,
|
||||
"2023": 2,
|
||||
"2024": 2,
|
||||
"2025": 3,
|
||||
"2026": 1
|
||||
},
|
||||
"note": "Where ticker counts drop in early years, the feed (or listing history) thins \u2014 do not treat those years as a full 505-name cross-section.",
|
||||
"survivorship_banner": "SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only \u2014 not levels."
|
||||
},
|
||||
"race_guard": null,
|
||||
"harness": null,
|
||||
"verdict": "COVERAGE_ONLY",
|
||||
"verdict_detail": "Coverage probe only; run --phase harness after deep rebuild.",
|
||||
"human_next": "- Compare sector residual vs market residual across eras.\n- If pre-2021 IC collapses, park Task 1 wire-in.\n- Do not retune production knobs on deep history levels.",
|
||||
"report_path": "reports/history-depth-20260719-093853.json"
|
||||
}
|
||||
@@ -1,177 +0,0 @@
|
||||
# History-depth extension (Tier-1 alpha research)
|
||||
|
||||
**Status:** PRE-REGISTERED — run on MacBook (heavy I/O + full harness).
|
||||
**Branch:** `research/history-depth-extension` (create from latest research stack).
|
||||
**Production impact:** none. **Do not retune any production knob on deep history.**
|
||||
|
||||
---
|
||||
|
||||
## Pre-registration (locked before rebuild)
|
||||
|
||||
### Motivation
|
||||
|
||||
All current conclusions rest on ~35 non-overlapping weekly windows in essentially
|
||||
one post-2021 regime. Extending history toward max Alpaca daily-bar depth adds
|
||||
the 2018 vol shock and full 2020 crash (where the feed allows).
|
||||
|
||||
### Protocol
|
||||
|
||||
1. **Empirical coverage first** — bars per calendar year per symbol; document
|
||||
where the feed thins out. Do **not** assume a uniform start date.
|
||||
2. **Rebuild the research snapshot completely** from prod source + max history
|
||||
per symbol (`Adjustment.SPLIT`, ~200 req/min pacing via existing extender).
|
||||
3. **Race guard (rule 6)** — refuse analysis until completion manifest is
|
||||
`complete=true` and live counts match.
|
||||
4. **Re-run full signal harness** (all existing signals incl. sector residual /
|
||||
SUE if present) on the extended window.
|
||||
5. **Report per signal:** mean IC, t, window count, and **era split**
|
||||
(pre-/post-2021) — diagnostic only, **not a tuning input**.
|
||||
6. **Log prominently:** survivorship bias grows with depth (today’s constituents
|
||||
backfilled). Absolute Sharpe/CAGR on deep history is optimistic; payload is
|
||||
**relative** signal comparisons and IC stability, not levels.
|
||||
7. **Do not retune** production knobs. If a knob’s confirmation looks
|
||||
overturned on deep history → report only; human decides.
|
||||
|
||||
### Success / interpretation (not promotion of a new signal)
|
||||
|
||||
| outcome | meaning |
|
||||
|---|---|
|
||||
| Sector residual still ≥ market residual on deep IC + stable sign | strengthens Task 1 PROMOTE case |
|
||||
| Sector residual collapses pre-2021 | **PARK** Task 1 wire-in |
|
||||
| SUE remains weak after full earnings + depth | **DEAD** SUE for this stack |
|
||||
| Any production knob looks worse deep | report; no auto-retune |
|
||||
|
||||
---
|
||||
|
||||
## MacBook runbook
|
||||
|
||||
Prefer the bundled script (one entry point):
|
||||
|
||||
```bash
|
||||
git fetch origin && git checkout research/earnings-gap-and-sue
|
||||
# .env: ALPACA_* required; FMP_* if resuming earnings
|
||||
# copy backtest_snapshots/prod.sqlite if not already local
|
||||
|
||||
chmod +x scripts/run_tier1_macbook.sh
|
||||
|
||||
# Default: coverage → deep rebuild → harness (+ era split)
|
||||
./scripts/run_tier1_macbook.sh
|
||||
|
||||
# Optional variants
|
||||
./scripts/run_tier1_macbook.sh --all # + earnings resume first
|
||||
./scripts/run_tier1_macbook.sh --earnings-only # multi-day FMP + 2a/2b only
|
||||
./scripts/run_tier1_macbook.sh --harness-only # skip rebuild
|
||||
./scripts/run_tier1_macbook.sh --coverage-only
|
||||
|
||||
# Tunables
|
||||
WORKERS=12 HISTORY_DAYS=5000 ./scripts/run_tier1_macbook.sh
|
||||
./scripts/run_tier1_macbook.sh --workers 12 --fmp-limit 250
|
||||
```
|
||||
|
||||
Then commit `reports/` + updated research docs, or copy them back to Windows.
|
||||
|
||||
---
|
||||
|
||||
## Data provenance
|
||||
|
||||
*(filled at run time)*
|
||||
|
||||
---
|
||||
|
||||
## Results
|
||||
|
||||
Generated: `2026-07-19T09:38:53.936226`
|
||||
|
||||
> **SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only — not levels.**
|
||||
|
||||
### Coverage
|
||||
|
||||
```json
|
||||
{
|
||||
"snapshot": "/Users/taathde3/git/lab/signal_platform/backtest_snapshots/prod.sqlite",
|
||||
"ticker_count": 506,
|
||||
"ohlcv_row_count": 629263,
|
||||
"date_range": {
|
||||
"min": "2021-06-24",
|
||||
"max": "2026-07-02"
|
||||
},
|
||||
"bars_per_year": [
|
||||
{
|
||||
"year": "2021",
|
||||
"bars": 65678,
|
||||
"tickers_with_bars": 494
|
||||
},
|
||||
{
|
||||
"year": "2022",
|
||||
"bars": 124423,
|
||||
"tickers_with_bars": 497
|
||||
},
|
||||
{
|
||||
"year": "2023",
|
||||
"bars": 124479,
|
||||
"tickers_with_bars": 499
|
||||
},
|
||||
{
|
||||
"year": "2024",
|
||||
"bars": 126129,
|
||||
"tickers_with_bars": 501
|
||||
},
|
||||
{
|
||||
"year": "2025",
|
||||
"bars": 125615,
|
||||
"tickers_with_bars": 504
|
||||
},
|
||||
{
|
||||
"year": "2026",
|
||||
"bars": 62939,
|
||||
"tickers_with_bars": 505
|
||||
}
|
||||
],
|
||||
"bars_per_symbol": {
|
||||
"min": 14,
|
||||
"p10": 1261,
|
||||
"p50": 1261,
|
||||
"p90": 1261,
|
||||
"max": 1261
|
||||
},
|
||||
"symbols_by_start_year": {
|
||||
"2021": 494,
|
||||
"2022": 3,
|
||||
"2023": 2,
|
||||
"2024": 2,
|
||||
"2025": 3,
|
||||
"2026": 1
|
||||
},
|
||||
"note": "Where ticker counts drop in early years, the feed (or listing history) thins \u2014 do not treat those years as a full 505-name cross-section.",
|
||||
"survivorship_banner": "SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only \u2014 not levels."
|
||||
}
|
||||
```
|
||||
|
||||
### Race guard
|
||||
|
||||
```json
|
||||
{}
|
||||
```
|
||||
|
||||
### Signal IC (full extended window)
|
||||
|
||||
_Harness not run this pass._
|
||||
|
||||
### Era split (diagnostic only)
|
||||
|
||||
_No era split._
|
||||
|
||||
## Verdict
|
||||
|
||||
**COVERAGE_ONLY**
|
||||
|
||||
Coverage probe only; run --phase harness after deep rebuild.
|
||||
|
||||
## What a human must decide next
|
||||
|
||||
- Compare sector residual vs market residual across eras.
|
||||
- If pre-2021 IC collapses, park Task 1 wire-in.
|
||||
- Do not retune production knobs on deep history levels.
|
||||
|
||||
Artifacts: `reports/history-depth-20260719-093853.json`
|
||||
|
||||
@@ -1,68 +0,0 @@
|
||||
{
|
||||
"generated_at": "2026-07-19T09:41:34.109378",
|
||||
"survivorship_banner": "SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only \u2014 not levels.",
|
||||
"coverage": {
|
||||
"snapshot": "/Users/taathde3/git/lab/signal_platform/backtest_snapshots/prod.sqlite",
|
||||
"ticker_count": 506,
|
||||
"ohlcv_row_count": 629263,
|
||||
"date_range": {
|
||||
"min": "2021-06-24",
|
||||
"max": "2026-07-02"
|
||||
},
|
||||
"bars_per_year": [
|
||||
{
|
||||
"year": "2021",
|
||||
"bars": 65678,
|
||||
"tickers_with_bars": 494
|
||||
},
|
||||
{
|
||||
"year": "2022",
|
||||
"bars": 124423,
|
||||
"tickers_with_bars": 497
|
||||
},
|
||||
{
|
||||
"year": "2023",
|
||||
"bars": 124479,
|
||||
"tickers_with_bars": 499
|
||||
},
|
||||
{
|
||||
"year": "2024",
|
||||
"bars": 126129,
|
||||
"tickers_with_bars": 501
|
||||
},
|
||||
{
|
||||
"year": "2025",
|
||||
"bars": 125615,
|
||||
"tickers_with_bars": 504
|
||||
},
|
||||
{
|
||||
"year": "2026",
|
||||
"bars": 62939,
|
||||
"tickers_with_bars": 505
|
||||
}
|
||||
],
|
||||
"bars_per_symbol": {
|
||||
"min": 14,
|
||||
"p10": 1261,
|
||||
"p50": 1261,
|
||||
"p90": 1261,
|
||||
"max": 1261
|
||||
},
|
||||
"symbols_by_start_year": {
|
||||
"2021": 494,
|
||||
"2022": 3,
|
||||
"2023": 2,
|
||||
"2024": 2,
|
||||
"2025": 3,
|
||||
"2026": 1
|
||||
},
|
||||
"note": "Where ticker counts drop in early years, the feed (or listing history) thins \u2014 do not treat those years as a full 505-name cross-section.",
|
||||
"survivorship_banner": "SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only \u2014 not levels."
|
||||
},
|
||||
"race_guard": null,
|
||||
"harness": null,
|
||||
"verdict": "COVERAGE_ONLY",
|
||||
"verdict_detail": "Coverage probe only; run --phase harness after deep rebuild.",
|
||||
"human_next": "- Compare sector residual vs market residual across eras.\n- If pre-2021 IC collapses, park Task 1 wire-in.\n- Do not retune production knobs on deep history levels.",
|
||||
"report_path": "reports/history-depth-20260719-094134.json"
|
||||
}
|
||||
@@ -1,177 +0,0 @@
|
||||
# History-depth extension (Tier-1 alpha research)
|
||||
|
||||
**Status:** PRE-REGISTERED — run on MacBook (heavy I/O + full harness).
|
||||
**Branch:** `research/history-depth-extension` (create from latest research stack).
|
||||
**Production impact:** none. **Do not retune any production knob on deep history.**
|
||||
|
||||
---
|
||||
|
||||
## Pre-registration (locked before rebuild)
|
||||
|
||||
### Motivation
|
||||
|
||||
All current conclusions rest on ~35 non-overlapping weekly windows in essentially
|
||||
one post-2021 regime. Extending history toward max Alpaca daily-bar depth adds
|
||||
the 2018 vol shock and full 2020 crash (where the feed allows).
|
||||
|
||||
### Protocol
|
||||
|
||||
1. **Empirical coverage first** — bars per calendar year per symbol; document
|
||||
where the feed thins out. Do **not** assume a uniform start date.
|
||||
2. **Rebuild the research snapshot completely** from prod source + max history
|
||||
per symbol (`Adjustment.SPLIT`, ~200 req/min pacing via existing extender).
|
||||
3. **Race guard (rule 6)** — refuse analysis until completion manifest is
|
||||
`complete=true` and live counts match.
|
||||
4. **Re-run full signal harness** (all existing signals incl. sector residual /
|
||||
SUE if present) on the extended window.
|
||||
5. **Report per signal:** mean IC, t, window count, and **era split**
|
||||
(pre-/post-2021) — diagnostic only, **not a tuning input**.
|
||||
6. **Log prominently:** survivorship bias grows with depth (today’s constituents
|
||||
backfilled). Absolute Sharpe/CAGR on deep history is optimistic; payload is
|
||||
**relative** signal comparisons and IC stability, not levels.
|
||||
7. **Do not retune** production knobs. If a knob’s confirmation looks
|
||||
overturned on deep history → report only; human decides.
|
||||
|
||||
### Success / interpretation (not promotion of a new signal)
|
||||
|
||||
| outcome | meaning |
|
||||
|---|---|
|
||||
| Sector residual still ≥ market residual on deep IC + stable sign | strengthens Task 1 PROMOTE case |
|
||||
| Sector residual collapses pre-2021 | **PARK** Task 1 wire-in |
|
||||
| SUE remains weak after full earnings + depth | **DEAD** SUE for this stack |
|
||||
| Any production knob looks worse deep | report; no auto-retune |
|
||||
|
||||
---
|
||||
|
||||
## MacBook runbook
|
||||
|
||||
Prefer the bundled script (one entry point):
|
||||
|
||||
```bash
|
||||
git fetch origin && git checkout research/earnings-gap-and-sue
|
||||
# .env: ALPACA_* required; FMP_* if resuming earnings
|
||||
# copy backtest_snapshots/prod.sqlite if not already local
|
||||
|
||||
chmod +x scripts/run_tier1_macbook.sh
|
||||
|
||||
# Default: coverage → deep rebuild → harness (+ era split)
|
||||
./scripts/run_tier1_macbook.sh
|
||||
|
||||
# Optional variants
|
||||
./scripts/run_tier1_macbook.sh --all # + earnings resume first
|
||||
./scripts/run_tier1_macbook.sh --earnings-only # multi-day FMP + 2a/2b only
|
||||
./scripts/run_tier1_macbook.sh --harness-only # skip rebuild
|
||||
./scripts/run_tier1_macbook.sh --coverage-only
|
||||
|
||||
# Tunables
|
||||
WORKERS=12 HISTORY_DAYS=5000 ./scripts/run_tier1_macbook.sh
|
||||
./scripts/run_tier1_macbook.sh --workers 12 --fmp-limit 250
|
||||
```
|
||||
|
||||
Then commit `reports/` + updated research docs, or copy them back to Windows.
|
||||
|
||||
---
|
||||
|
||||
## Data provenance
|
||||
|
||||
*(filled at run time)*
|
||||
|
||||
---
|
||||
|
||||
## Results
|
||||
|
||||
Generated: `2026-07-19T09:41:34.109378`
|
||||
|
||||
> **SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only — not levels.**
|
||||
|
||||
### Coverage
|
||||
|
||||
```json
|
||||
{
|
||||
"snapshot": "/Users/taathde3/git/lab/signal_platform/backtest_snapshots/prod.sqlite",
|
||||
"ticker_count": 506,
|
||||
"ohlcv_row_count": 629263,
|
||||
"date_range": {
|
||||
"min": "2021-06-24",
|
||||
"max": "2026-07-02"
|
||||
},
|
||||
"bars_per_year": [
|
||||
{
|
||||
"year": "2021",
|
||||
"bars": 65678,
|
||||
"tickers_with_bars": 494
|
||||
},
|
||||
{
|
||||
"year": "2022",
|
||||
"bars": 124423,
|
||||
"tickers_with_bars": 497
|
||||
},
|
||||
{
|
||||
"year": "2023",
|
||||
"bars": 124479,
|
||||
"tickers_with_bars": 499
|
||||
},
|
||||
{
|
||||
"year": "2024",
|
||||
"bars": 126129,
|
||||
"tickers_with_bars": 501
|
||||
},
|
||||
{
|
||||
"year": "2025",
|
||||
"bars": 125615,
|
||||
"tickers_with_bars": 504
|
||||
},
|
||||
{
|
||||
"year": "2026",
|
||||
"bars": 62939,
|
||||
"tickers_with_bars": 505
|
||||
}
|
||||
],
|
||||
"bars_per_symbol": {
|
||||
"min": 14,
|
||||
"p10": 1261,
|
||||
"p50": 1261,
|
||||
"p90": 1261,
|
||||
"max": 1261
|
||||
},
|
||||
"symbols_by_start_year": {
|
||||
"2021": 494,
|
||||
"2022": 3,
|
||||
"2023": 2,
|
||||
"2024": 2,
|
||||
"2025": 3,
|
||||
"2026": 1
|
||||
},
|
||||
"note": "Where ticker counts drop in early years, the feed (or listing history) thins \u2014 do not treat those years as a full 505-name cross-section.",
|
||||
"survivorship_banner": "SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only \u2014 not levels."
|
||||
}
|
||||
```
|
||||
|
||||
### Race guard
|
||||
|
||||
```json
|
||||
{}
|
||||
```
|
||||
|
||||
### Signal IC (full extended window)
|
||||
|
||||
_Harness not run this pass._
|
||||
|
||||
### Era split (diagnostic only)
|
||||
|
||||
_No era split._
|
||||
|
||||
## Verdict
|
||||
|
||||
**COVERAGE_ONLY**
|
||||
|
||||
Coverage probe only; run --phase harness after deep rebuild.
|
||||
|
||||
## What a human must decide next
|
||||
|
||||
- Compare sector residual vs market residual across eras.
|
||||
- If pre-2021 IC collapses, park Task 1 wire-in.
|
||||
- Do not retune production knobs on deep history levels.
|
||||
|
||||
Artifacts: `reports/history-depth-20260719-094134.json`
|
||||
|
||||
@@ -1,68 +0,0 @@
|
||||
{
|
||||
"generated_at": "2026-07-19T09:43:44.520126",
|
||||
"survivorship_banner": "SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only \u2014 not levels.",
|
||||
"coverage": {
|
||||
"snapshot": "/Users/taathde3/git/lab/signal_platform/backtest_snapshots/prod.sqlite",
|
||||
"ticker_count": 506,
|
||||
"ohlcv_row_count": 629263,
|
||||
"date_range": {
|
||||
"min": "2021-06-24",
|
||||
"max": "2026-07-02"
|
||||
},
|
||||
"bars_per_year": [
|
||||
{
|
||||
"year": "2021",
|
||||
"bars": 65678,
|
||||
"tickers_with_bars": 494
|
||||
},
|
||||
{
|
||||
"year": "2022",
|
||||
"bars": 124423,
|
||||
"tickers_with_bars": 497
|
||||
},
|
||||
{
|
||||
"year": "2023",
|
||||
"bars": 124479,
|
||||
"tickers_with_bars": 499
|
||||
},
|
||||
{
|
||||
"year": "2024",
|
||||
"bars": 126129,
|
||||
"tickers_with_bars": 501
|
||||
},
|
||||
{
|
||||
"year": "2025",
|
||||
"bars": 125615,
|
||||
"tickers_with_bars": 504
|
||||
},
|
||||
{
|
||||
"year": "2026",
|
||||
"bars": 62939,
|
||||
"tickers_with_bars": 505
|
||||
}
|
||||
],
|
||||
"bars_per_symbol": {
|
||||
"min": 14,
|
||||
"p10": 1261,
|
||||
"p50": 1261,
|
||||
"p90": 1261,
|
||||
"max": 1261
|
||||
},
|
||||
"symbols_by_start_year": {
|
||||
"2021": 494,
|
||||
"2022": 3,
|
||||
"2023": 2,
|
||||
"2024": 2,
|
||||
"2025": 3,
|
||||
"2026": 1
|
||||
},
|
||||
"note": "Where ticker counts drop in early years, the feed (or listing history) thins \u2014 do not treat those years as a full 505-name cross-section.",
|
||||
"survivorship_banner": "SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only \u2014 not levels."
|
||||
},
|
||||
"race_guard": null,
|
||||
"harness": null,
|
||||
"verdict": "COVERAGE_ONLY",
|
||||
"verdict_detail": "Coverage probe only; run --phase harness after deep rebuild.",
|
||||
"human_next": "- Compare sector residual vs market residual across eras.\n- If pre-2021 IC collapses, park Task 1 wire-in.\n- Do not retune production knobs on deep history levels.",
|
||||
"report_path": "reports/history-depth-20260719-094344.json"
|
||||
}
|
||||
@@ -1,177 +0,0 @@
|
||||
# History-depth extension (Tier-1 alpha research)
|
||||
|
||||
**Status:** PRE-REGISTERED — run on MacBook (heavy I/O + full harness).
|
||||
**Branch:** `research/history-depth-extension` (create from latest research stack).
|
||||
**Production impact:** none. **Do not retune any production knob on deep history.**
|
||||
|
||||
---
|
||||
|
||||
## Pre-registration (locked before rebuild)
|
||||
|
||||
### Motivation
|
||||
|
||||
All current conclusions rest on ~35 non-overlapping weekly windows in essentially
|
||||
one post-2021 regime. Extending history toward max Alpaca daily-bar depth adds
|
||||
the 2018 vol shock and full 2020 crash (where the feed allows).
|
||||
|
||||
### Protocol
|
||||
|
||||
1. **Empirical coverage first** — bars per calendar year per symbol; document
|
||||
where the feed thins out. Do **not** assume a uniform start date.
|
||||
2. **Rebuild the research snapshot completely** from prod source + max history
|
||||
per symbol (`Adjustment.SPLIT`, ~200 req/min pacing via existing extender).
|
||||
3. **Race guard (rule 6)** — refuse analysis until completion manifest is
|
||||
`complete=true` and live counts match.
|
||||
4. **Re-run full signal harness** (all existing signals incl. sector residual /
|
||||
SUE if present) on the extended window.
|
||||
5. **Report per signal:** mean IC, t, window count, and **era split**
|
||||
(pre-/post-2021) — diagnostic only, **not a tuning input**.
|
||||
6. **Log prominently:** survivorship bias grows with depth (today’s constituents
|
||||
backfilled). Absolute Sharpe/CAGR on deep history is optimistic; payload is
|
||||
**relative** signal comparisons and IC stability, not levels.
|
||||
7. **Do not retune** production knobs. If a knob’s confirmation looks
|
||||
overturned on deep history → report only; human decides.
|
||||
|
||||
### Success / interpretation (not promotion of a new signal)
|
||||
|
||||
| outcome | meaning |
|
||||
|---|---|
|
||||
| Sector residual still ≥ market residual on deep IC + stable sign | strengthens Task 1 PROMOTE case |
|
||||
| Sector residual collapses pre-2021 | **PARK** Task 1 wire-in |
|
||||
| SUE remains weak after full earnings + depth | **DEAD** SUE for this stack |
|
||||
| Any production knob looks worse deep | report; no auto-retune |
|
||||
|
||||
---
|
||||
|
||||
## MacBook runbook
|
||||
|
||||
Prefer the bundled script (one entry point):
|
||||
|
||||
```bash
|
||||
git fetch origin && git checkout research/earnings-gap-and-sue
|
||||
# .env: ALPACA_* required; FMP_* if resuming earnings
|
||||
# copy backtest_snapshots/prod.sqlite if not already local
|
||||
|
||||
chmod +x scripts/run_tier1_macbook.sh
|
||||
|
||||
# Default: coverage → deep rebuild → harness (+ era split)
|
||||
./scripts/run_tier1_macbook.sh
|
||||
|
||||
# Optional variants
|
||||
./scripts/run_tier1_macbook.sh --all # + earnings resume first
|
||||
./scripts/run_tier1_macbook.sh --earnings-only # multi-day FMP + 2a/2b only
|
||||
./scripts/run_tier1_macbook.sh --harness-only # skip rebuild
|
||||
./scripts/run_tier1_macbook.sh --coverage-only
|
||||
|
||||
# Tunables
|
||||
WORKERS=12 HISTORY_DAYS=5000 ./scripts/run_tier1_macbook.sh
|
||||
./scripts/run_tier1_macbook.sh --workers 12 --fmp-limit 250
|
||||
```
|
||||
|
||||
Then commit `reports/` + updated research docs, or copy them back to Windows.
|
||||
|
||||
---
|
||||
|
||||
## Data provenance
|
||||
|
||||
*(filled at run time)*
|
||||
|
||||
---
|
||||
|
||||
## Results
|
||||
|
||||
Generated: `2026-07-19T09:43:44.520126`
|
||||
|
||||
> **SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only — not levels.**
|
||||
|
||||
### Coverage
|
||||
|
||||
```json
|
||||
{
|
||||
"snapshot": "/Users/taathde3/git/lab/signal_platform/backtest_snapshots/prod.sqlite",
|
||||
"ticker_count": 506,
|
||||
"ohlcv_row_count": 629263,
|
||||
"date_range": {
|
||||
"min": "2021-06-24",
|
||||
"max": "2026-07-02"
|
||||
},
|
||||
"bars_per_year": [
|
||||
{
|
||||
"year": "2021",
|
||||
"bars": 65678,
|
||||
"tickers_with_bars": 494
|
||||
},
|
||||
{
|
||||
"year": "2022",
|
||||
"bars": 124423,
|
||||
"tickers_with_bars": 497
|
||||
},
|
||||
{
|
||||
"year": "2023",
|
||||
"bars": 124479,
|
||||
"tickers_with_bars": 499
|
||||
},
|
||||
{
|
||||
"year": "2024",
|
||||
"bars": 126129,
|
||||
"tickers_with_bars": 501
|
||||
},
|
||||
{
|
||||
"year": "2025",
|
||||
"bars": 125615,
|
||||
"tickers_with_bars": 504
|
||||
},
|
||||
{
|
||||
"year": "2026",
|
||||
"bars": 62939,
|
||||
"tickers_with_bars": 505
|
||||
}
|
||||
],
|
||||
"bars_per_symbol": {
|
||||
"min": 14,
|
||||
"p10": 1261,
|
||||
"p50": 1261,
|
||||
"p90": 1261,
|
||||
"max": 1261
|
||||
},
|
||||
"symbols_by_start_year": {
|
||||
"2021": 494,
|
||||
"2022": 3,
|
||||
"2023": 2,
|
||||
"2024": 2,
|
||||
"2025": 3,
|
||||
"2026": 1
|
||||
},
|
||||
"note": "Where ticker counts drop in early years, the feed (or listing history) thins \u2014 do not treat those years as a full 505-name cross-section.",
|
||||
"survivorship_banner": "SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only \u2014 not levels."
|
||||
}
|
||||
```
|
||||
|
||||
### Race guard
|
||||
|
||||
```json
|
||||
{}
|
||||
```
|
||||
|
||||
### Signal IC (full extended window)
|
||||
|
||||
_Harness not run this pass._
|
||||
|
||||
### Era split (diagnostic only)
|
||||
|
||||
_No era split._
|
||||
|
||||
## Verdict
|
||||
|
||||
**COVERAGE_ONLY**
|
||||
|
||||
Coverage probe only; run --phase harness after deep rebuild.
|
||||
|
||||
## What a human must decide next
|
||||
|
||||
- Compare sector residual vs market residual across eras.
|
||||
- If pre-2021 IC collapses, park Task 1 wire-in.
|
||||
- Do not retune production knobs on deep history levels.
|
||||
|
||||
Artifacts: `reports/history-depth-20260719-094344.json`
|
||||
|
||||
@@ -1,68 +0,0 @@
|
||||
{
|
||||
"generated_at": "2026-07-19T09:51:56.525634",
|
||||
"survivorship_banner": "SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only \u2014 not levels.",
|
||||
"coverage": {
|
||||
"snapshot": "/Users/taathde3/git/lab/signal_platform/backtest_snapshots/prod.sqlite",
|
||||
"ticker_count": 506,
|
||||
"ohlcv_row_count": 629263,
|
||||
"date_range": {
|
||||
"min": "2021-06-24",
|
||||
"max": "2026-07-02"
|
||||
},
|
||||
"bars_per_year": [
|
||||
{
|
||||
"year": "2021",
|
||||
"bars": 65678,
|
||||
"tickers_with_bars": 494
|
||||
},
|
||||
{
|
||||
"year": "2022",
|
||||
"bars": 124423,
|
||||
"tickers_with_bars": 497
|
||||
},
|
||||
{
|
||||
"year": "2023",
|
||||
"bars": 124479,
|
||||
"tickers_with_bars": 499
|
||||
},
|
||||
{
|
||||
"year": "2024",
|
||||
"bars": 126129,
|
||||
"tickers_with_bars": 501
|
||||
},
|
||||
{
|
||||
"year": "2025",
|
||||
"bars": 125615,
|
||||
"tickers_with_bars": 504
|
||||
},
|
||||
{
|
||||
"year": "2026",
|
||||
"bars": 62939,
|
||||
"tickers_with_bars": 505
|
||||
}
|
||||
],
|
||||
"bars_per_symbol": {
|
||||
"min": 14,
|
||||
"p10": 1261,
|
||||
"p50": 1261,
|
||||
"p90": 1261,
|
||||
"max": 1261
|
||||
},
|
||||
"symbols_by_start_year": {
|
||||
"2021": 494,
|
||||
"2022": 3,
|
||||
"2023": 2,
|
||||
"2024": 2,
|
||||
"2025": 3,
|
||||
"2026": 1
|
||||
},
|
||||
"note": "Where ticker counts drop in early years, the feed (or listing history) thins \u2014 do not treat those years as a full 505-name cross-section.",
|
||||
"survivorship_banner": "SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only \u2014 not levels."
|
||||
},
|
||||
"race_guard": null,
|
||||
"harness": null,
|
||||
"verdict": "COVERAGE_ONLY",
|
||||
"verdict_detail": "Coverage probe only; run --phase harness after deep rebuild.",
|
||||
"human_next": "- Compare sector residual vs market residual across eras.\n- If pre-2021 IC collapses, park Task 1 wire-in.\n- Do not retune production knobs on deep history levels.",
|
||||
"report_path": "reports/history-depth-20260719-095156.json"
|
||||
}
|
||||
@@ -1,177 +0,0 @@
|
||||
# History-depth extension (Tier-1 alpha research)
|
||||
|
||||
**Status:** PRE-REGISTERED — run on MacBook (heavy I/O + full harness).
|
||||
**Branch:** `research/history-depth-extension` (create from latest research stack).
|
||||
**Production impact:** none. **Do not retune any production knob on deep history.**
|
||||
|
||||
---
|
||||
|
||||
## Pre-registration (locked before rebuild)
|
||||
|
||||
### Motivation
|
||||
|
||||
All current conclusions rest on ~35 non-overlapping weekly windows in essentially
|
||||
one post-2021 regime. Extending history toward max Alpaca daily-bar depth adds
|
||||
the 2018 vol shock and full 2020 crash (where the feed allows).
|
||||
|
||||
### Protocol
|
||||
|
||||
1. **Empirical coverage first** — bars per calendar year per symbol; document
|
||||
where the feed thins out. Do **not** assume a uniform start date.
|
||||
2. **Rebuild the research snapshot completely** from prod source + max history
|
||||
per symbol (`Adjustment.SPLIT`, ~200 req/min pacing via existing extender).
|
||||
3. **Race guard (rule 6)** — refuse analysis until completion manifest is
|
||||
`complete=true` and live counts match.
|
||||
4. **Re-run full signal harness** (all existing signals incl. sector residual /
|
||||
SUE if present) on the extended window.
|
||||
5. **Report per signal:** mean IC, t, window count, and **era split**
|
||||
(pre-/post-2021) — diagnostic only, **not a tuning input**.
|
||||
6. **Log prominently:** survivorship bias grows with depth (today’s constituents
|
||||
backfilled). Absolute Sharpe/CAGR on deep history is optimistic; payload is
|
||||
**relative** signal comparisons and IC stability, not levels.
|
||||
7. **Do not retune** production knobs. If a knob’s confirmation looks
|
||||
overturned on deep history → report only; human decides.
|
||||
|
||||
### Success / interpretation (not promotion of a new signal)
|
||||
|
||||
| outcome | meaning |
|
||||
|---|---|
|
||||
| Sector residual still ≥ market residual on deep IC + stable sign | strengthens Task 1 PROMOTE case |
|
||||
| Sector residual collapses pre-2021 | **PARK** Task 1 wire-in |
|
||||
| SUE remains weak after full earnings + depth | **DEAD** SUE for this stack |
|
||||
| Any production knob looks worse deep | report; no auto-retune |
|
||||
|
||||
---
|
||||
|
||||
## MacBook runbook
|
||||
|
||||
Prefer the bundled script (one entry point):
|
||||
|
||||
```bash
|
||||
git fetch origin && git checkout research/earnings-gap-and-sue
|
||||
# .env: ALPACA_* required; FMP_* if resuming earnings
|
||||
# copy backtest_snapshots/prod.sqlite if not already local
|
||||
|
||||
chmod +x scripts/run_tier1_macbook.sh
|
||||
|
||||
# Default: coverage → deep rebuild → harness (+ era split)
|
||||
./scripts/run_tier1_macbook.sh
|
||||
|
||||
# Optional variants
|
||||
./scripts/run_tier1_macbook.sh --all # + earnings resume first
|
||||
./scripts/run_tier1_macbook.sh --earnings-only # multi-day FMP + 2a/2b only
|
||||
./scripts/run_tier1_macbook.sh --harness-only # skip rebuild
|
||||
./scripts/run_tier1_macbook.sh --coverage-only
|
||||
|
||||
# Tunables
|
||||
WORKERS=12 HISTORY_DAYS=5000 ./scripts/run_tier1_macbook.sh
|
||||
./scripts/run_tier1_macbook.sh --workers 12 --fmp-limit 250
|
||||
```
|
||||
|
||||
Then commit `reports/` + updated research docs, or copy them back to Windows.
|
||||
|
||||
---
|
||||
|
||||
## Data provenance
|
||||
|
||||
*(filled at run time)*
|
||||
|
||||
---
|
||||
|
||||
## Results
|
||||
|
||||
Generated: `2026-07-19T09:51:56.525634`
|
||||
|
||||
> **SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only — not levels.**
|
||||
|
||||
### Coverage
|
||||
|
||||
```json
|
||||
{
|
||||
"snapshot": "/Users/taathde3/git/lab/signal_platform/backtest_snapshots/prod.sqlite",
|
||||
"ticker_count": 506,
|
||||
"ohlcv_row_count": 629263,
|
||||
"date_range": {
|
||||
"min": "2021-06-24",
|
||||
"max": "2026-07-02"
|
||||
},
|
||||
"bars_per_year": [
|
||||
{
|
||||
"year": "2021",
|
||||
"bars": 65678,
|
||||
"tickers_with_bars": 494
|
||||
},
|
||||
{
|
||||
"year": "2022",
|
||||
"bars": 124423,
|
||||
"tickers_with_bars": 497
|
||||
},
|
||||
{
|
||||
"year": "2023",
|
||||
"bars": 124479,
|
||||
"tickers_with_bars": 499
|
||||
},
|
||||
{
|
||||
"year": "2024",
|
||||
"bars": 126129,
|
||||
"tickers_with_bars": 501
|
||||
},
|
||||
{
|
||||
"year": "2025",
|
||||
"bars": 125615,
|
||||
"tickers_with_bars": 504
|
||||
},
|
||||
{
|
||||
"year": "2026",
|
||||
"bars": 62939,
|
||||
"tickers_with_bars": 505
|
||||
}
|
||||
],
|
||||
"bars_per_symbol": {
|
||||
"min": 14,
|
||||
"p10": 1261,
|
||||
"p50": 1261,
|
||||
"p90": 1261,
|
||||
"max": 1261
|
||||
},
|
||||
"symbols_by_start_year": {
|
||||
"2021": 494,
|
||||
"2022": 3,
|
||||
"2023": 2,
|
||||
"2024": 2,
|
||||
"2025": 3,
|
||||
"2026": 1
|
||||
},
|
||||
"note": "Where ticker counts drop in early years, the feed (or listing history) thins \u2014 do not treat those years as a full 505-name cross-section.",
|
||||
"survivorship_banner": "SURVIVORSHIP BIAS: today's constituents backfilled historically. Absolute Sharpe/CAGR levels on deep history are optimistic. Use RELATIVE signal IC comparisons and era stability only \u2014 not levels."
|
||||
}
|
||||
```
|
||||
|
||||
### Race guard
|
||||
|
||||
```json
|
||||
{}
|
||||
```
|
||||
|
||||
### Signal IC (full extended window)
|
||||
|
||||
_Harness not run this pass._
|
||||
|
||||
### Era split (diagnostic only)
|
||||
|
||||
_No era split._
|
||||
|
||||
## Verdict
|
||||
|
||||
**COVERAGE_ONLY**
|
||||
|
||||
Coverage probe only; run --phase harness after deep rebuild.
|
||||
|
||||
## What a human must decide next
|
||||
|
||||
- Compare sector residual vs market residual across eras.
|
||||
- If pre-2021 IC collapses, park Task 1 wire-in.
|
||||
- Do not retune production knobs on deep history levels.
|
||||
|
||||
Artifacts: `reports/history-depth-20260719-095156.json`
|
||||
|
||||
@@ -1,275 +0,0 @@
|
||||
{
|
||||
"step1": {
|
||||
"shallow_meta": {
|
||||
"n_symbols": 4654,
|
||||
"deep_cohort_p10_start": "2016-01-04",
|
||||
"shallow_cutoff": "2017-02-07",
|
||||
"lag_days": 400,
|
||||
"n_shallow": 3349,
|
||||
"shallow_start_histogram": {
|
||||
"2017": 117,
|
||||
"2018": 162,
|
||||
"2019": 163,
|
||||
"2020": 269,
|
||||
"2021": 1060,
|
||||
"2022": 211,
|
||||
"2023": 176,
|
||||
"2024": 281,
|
||||
"2025": 543,
|
||||
"2026": 367
|
||||
},
|
||||
"deep_start_histogram": {
|
||||
"2016": 1296,
|
||||
"2017": 9
|
||||
},
|
||||
"shallow_sample": [
|
||||
"A",
|
||||
"AACB",
|
||||
"AACBR",
|
||||
"AACBU",
|
||||
"AACI",
|
||||
"AACIU",
|
||||
"AACIW",
|
||||
"AACO",
|
||||
"AACOU",
|
||||
"AACOW",
|
||||
"AACP",
|
||||
"AACPR",
|
||||
"AACPU",
|
||||
"AACPW",
|
||||
"AAPG",
|
||||
"AAPL",
|
||||
"AARD",
|
||||
"ABAT",
|
||||
"ABBV",
|
||||
"ABCL",
|
||||
"ABLV",
|
||||
"ABLVW",
|
||||
"ABNB",
|
||||
"ABOS",
|
||||
"ABSI",
|
||||
"ABT",
|
||||
"ABTC",
|
||||
"ABVC",
|
||||
"ABVX",
|
||||
"ACAA"
|
||||
]
|
||||
},
|
||||
"shallow_list_n": 3349,
|
||||
"fetch_ok": 3349,
|
||||
"fetch_fail": 0,
|
||||
"etf_refresh": {
|
||||
"written": {
|
||||
"SPY": 0,
|
||||
"XLB": 0,
|
||||
"XLC": 0,
|
||||
"XLE": 0,
|
||||
"XLF": 0,
|
||||
"XLI": 0,
|
||||
"XLK": 0,
|
||||
"XLP": 0,
|
||||
"XLRE": 0,
|
||||
"XLU": 0,
|
||||
"XLV": 0,
|
||||
"XLY": 0
|
||||
},
|
||||
"benchmark_summary": [
|
||||
{
|
||||
"symbol": "SPY",
|
||||
"n": 2649,
|
||||
"min": "2016-01-04",
|
||||
"max": "2026-07-17"
|
||||
},
|
||||
{
|
||||
"symbol": "XLB",
|
||||
"n": 2649,
|
||||
"min": "2016-01-04",
|
||||
"max": "2026-07-17"
|
||||
},
|
||||
{
|
||||
"symbol": "XLC",
|
||||
"n": 2030,
|
||||
"min": "2018-06-19",
|
||||
"max": "2026-07-17"
|
||||
},
|
||||
{
|
||||
"symbol": "XLE",
|
||||
"n": 2649,
|
||||
"min": "2016-01-04",
|
||||
"max": "2026-07-17"
|
||||
},
|
||||
{
|
||||
"symbol": "XLF",
|
||||
"n": 2649,
|
||||
"min": "2016-01-04",
|
||||
"max": "2026-07-17"
|
||||
},
|
||||
{
|
||||
"symbol": "XLI",
|
||||
"n": 2649,
|
||||
"min": "2016-01-04",
|
||||
"max": "2026-07-17"
|
||||
},
|
||||
{
|
||||
"symbol": "XLK",
|
||||
"n": 2649,
|
||||
"min": "2016-01-04",
|
||||
"max": "2026-07-17"
|
||||
},
|
||||
{
|
||||
"symbol": "XLP",
|
||||
"n": 2649,
|
||||
"min": "2016-01-04",
|
||||
"max": "2026-07-17"
|
||||
},
|
||||
{
|
||||
"symbol": "XLRE",
|
||||
"n": 2649,
|
||||
"min": "2016-01-04",
|
||||
"max": "2026-07-17"
|
||||
},
|
||||
{
|
||||
"symbol": "XLU",
|
||||
"n": 2649,
|
||||
"min": "2016-01-04",
|
||||
"max": "2026-07-17"
|
||||
},
|
||||
{
|
||||
"symbol": "XLV",
|
||||
"n": 2649,
|
||||
"min": "2016-01-04",
|
||||
"max": "2026-07-17"
|
||||
},
|
||||
{
|
||||
"symbol": "XLY",
|
||||
"n": 2649,
|
||||
"min": "2016-01-04",
|
||||
"max": "2026-07-17"
|
||||
}
|
||||
]
|
||||
},
|
||||
"sanity": {
|
||||
"passed": false,
|
||||
"megacap": {
|
||||
"AAPL": {
|
||||
"symbol": "AAPL",
|
||||
"bars": 2649,
|
||||
"min_date": "2016-01-04",
|
||||
"max_date": "2026-07-17"
|
||||
},
|
||||
"MSFT": {
|
||||
"symbol": "MSFT",
|
||||
"bars": 2649,
|
||||
"min_date": "2016-01-04",
|
||||
"max_date": "2026-07-17"
|
||||
},
|
||||
"JPM": {
|
||||
"symbol": "JPM",
|
||||
"bars": 2649,
|
||||
"min_date": "2016-01-04",
|
||||
"max_date": "2026-07-17"
|
||||
},
|
||||
"XOM": {
|
||||
"symbol": "XOM",
|
||||
"bars": 2649,
|
||||
"min_date": "2016-01-04",
|
||||
"max_date": "2026-07-17"
|
||||
},
|
||||
"JNJ": {
|
||||
"symbol": "JNJ",
|
||||
"bars": 2649,
|
||||
"min_date": "2016-01-04",
|
||||
"max_date": "2026-07-17"
|
||||
}
|
||||
},
|
||||
"megacap_ok": false,
|
||||
"megacap_deadline": "2013-12-14",
|
||||
"sector_etfs": {
|
||||
"XLB": {
|
||||
"n": 2649,
|
||||
"min": "2016-01-04",
|
||||
"max": "2026-07-17"
|
||||
},
|
||||
"XLC": {
|
||||
"n": 2030,
|
||||
"min": "2018-06-19",
|
||||
"max": "2026-07-17"
|
||||
},
|
||||
"XLE": {
|
||||
"n": 2649,
|
||||
"min": "2016-01-04",
|
||||
"max": "2026-07-17"
|
||||
},
|
||||
"XLF": {
|
||||
"n": 2649,
|
||||
"min": "2016-01-04",
|
||||
"max": "2026-07-17"
|
||||
},
|
||||
"XLI": {
|
||||
"n": 2649,
|
||||
"min": "2016-01-04",
|
||||
"max": "2026-07-17"
|
||||
},
|
||||
"XLK": {
|
||||
"n": 2649,
|
||||
"min": "2016-01-04",
|
||||
"max": "2026-07-17"
|
||||
},
|
||||
"XLP": {
|
||||
"n": 2649,
|
||||
"min": "2016-01-04",
|
||||
"max": "2026-07-17"
|
||||
},
|
||||
"XLRE": {
|
||||
"n": 2649,
|
||||
"min": "2016-01-04",
|
||||
"max": "2026-07-17"
|
||||
},
|
||||
"XLU": {
|
||||
"n": 2649,
|
||||
"min": "2016-01-04",
|
||||
"max": "2026-07-17"
|
||||
},
|
||||
"XLV": {
|
||||
"n": 2649,
|
||||
"min": "2016-01-04",
|
||||
"max": "2026-07-17"
|
||||
},
|
||||
"XLY": {
|
||||
"n": 2649,
|
||||
"min": "2016-01-04",
|
||||
"max": "2026-07-17"
|
||||
}
|
||||
},
|
||||
"sector_etfs_deep_count": 11,
|
||||
"sector_etfs_ok": true,
|
||||
"still_shallow_count": 2882,
|
||||
"still_shallow_sample": [
|
||||
"AACB",
|
||||
"AACBR",
|
||||
"AACBU",
|
||||
"AACI",
|
||||
"AACIU",
|
||||
"AACIW",
|
||||
"AACO",
|
||||
"AACOU",
|
||||
"AACOW",
|
||||
"AACP",
|
||||
"AACPR",
|
||||
"AACPU",
|
||||
"AACPW",
|
||||
"AAPG",
|
||||
"AARD",
|
||||
"ABAT",
|
||||
"ABCL",
|
||||
"ABLV",
|
||||
"ABLVW",
|
||||
"ABNB"
|
||||
],
|
||||
"xlc_note": "XLC lists mid-2018 \u2192 Communication Services residual coverage from ~mid-2019.",
|
||||
"target_history_days": 5000
|
||||
},
|
||||
"manifest_path": "backtest_snapshots/research.sqlite.manifest.json"
|
||||
},
|
||||
"harness": null
|
||||
}
|
||||
@@ -1,233 +0,0 @@
|
||||
"""Build a local ticker → GICS sector map for research residualization.
|
||||
|
||||
Sources (in order):
|
||||
1. Public S&P 500 constituents CSV (datasets/s-and-p-500-companies) — bulk, free.
|
||||
2. Existing map file (resume).
|
||||
3. FMP stable ``profile`` for still-missing symbols (budget ~250 req/day).
|
||||
|
||||
Writes ``data/research/ticker_sector_map.json``. Never touches production Postgres.
|
||||
|
||||
Example
|
||||
-------
|
||||
python scripts/build_ticker_sector_map.py \\
|
||||
--snapshot backtest_snapshots/prod.sqlite
|
||||
|
||||
python scripts/build_ticker_sector_map.py --fmp-limit 50
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
from sqlalchemy import create_engine, text
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from app.ssl_bootstrap import bootstrap_ssl # noqa: E402
|
||||
|
||||
bootstrap_ssl()
|
||||
|
||||
from app.services.sector_map import ( # noqa: E402
|
||||
DEFAULT_SECTOR_MAP_PATH,
|
||||
coverage_stats,
|
||||
load_ticker_sector_map,
|
||||
normalise_symbol,
|
||||
save_ticker_sector_map,
|
||||
sector_to_etf,
|
||||
)
|
||||
|
||||
SP500_CSV_URL = (
|
||||
"https://raw.githubusercontent.com/datasets/s-and-p-500-companies/"
|
||||
"master/data/constituents.csv"
|
||||
)
|
||||
FMP_STABLE = "https://financialmodelingprep.com/stable"
|
||||
|
||||
|
||||
def _parse_args() -> argparse.Namespace:
|
||||
p = argparse.ArgumentParser(description=__doc__)
|
||||
p.add_argument(
|
||||
"--snapshot",
|
||||
default="backtest_snapshots/prod.sqlite",
|
||||
help="Snapshot whose tickers define the universe.",
|
||||
)
|
||||
p.add_argument(
|
||||
"--out",
|
||||
default=str(DEFAULT_SECTOR_MAP_PATH),
|
||||
help="Output JSON path.",
|
||||
)
|
||||
p.add_argument(
|
||||
"--fmp-limit",
|
||||
type=int,
|
||||
default=200,
|
||||
help="Max FMP profile requests this run (free-tier cushion).",
|
||||
)
|
||||
p.add_argument(
|
||||
"--skip-fmp",
|
||||
action="store_true",
|
||||
help="Only use public SP500 CSV + existing map.",
|
||||
)
|
||||
p.add_argument("--sleep", type=float, default=0.35, help="Pause between FMP calls.")
|
||||
return p.parse_args()
|
||||
|
||||
|
||||
def _snapshot_symbols(snapshot: Path) -> list[str]:
|
||||
engine = create_engine(f"sqlite:///{snapshot.resolve().as_posix()}", future=True)
|
||||
try:
|
||||
with engine.connect() as conn:
|
||||
rows = conn.execute(text("SELECT symbol FROM tickers ORDER BY symbol")).fetchall()
|
||||
finally:
|
||||
engine.dispose()
|
||||
return [normalise_symbol(r[0]) for r in rows if r[0]]
|
||||
|
||||
|
||||
def _fetch_sp500_map() -> dict[str, str]:
|
||||
with httpx.Client(timeout=60.0, follow_redirects=True) as client:
|
||||
resp = client.get(SP500_CSV_URL)
|
||||
resp.raise_for_status()
|
||||
reader = csv.DictReader(io.StringIO(resp.text))
|
||||
out: dict[str, str] = {}
|
||||
for row in reader:
|
||||
sym = normalise_symbol(row.get("Symbol") or "")
|
||||
sector = (row.get("GICS Sector") or "").strip()
|
||||
if sym and sector:
|
||||
out[sym] = sector
|
||||
return out
|
||||
|
||||
|
||||
async def _fmp_profile_sector(client: httpx.AsyncClient, api_key: str, symbol: str) -> str | None:
|
||||
resp = await client.get(
|
||||
f"{FMP_STABLE}/profile",
|
||||
params={"symbol": symbol, "apikey": api_key},
|
||||
)
|
||||
if resp.status_code == 429:
|
||||
raise RuntimeError(f"FMP rate limited on {symbol}")
|
||||
if resp.status_code == 402:
|
||||
return None
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
if isinstance(data, list):
|
||||
data = data[0] if data else {}
|
||||
if not isinstance(data, dict):
|
||||
return None
|
||||
sector = (data.get("sector") or data.get("industry") or "").strip()
|
||||
# industry alone is not a GICS sector — only accept if we can map to an ETF
|
||||
if sector and sector_to_etf(sector):
|
||||
return sector
|
||||
# FMP sometimes returns industry under sector when sector missing; try sector field only
|
||||
sec = (data.get("sector") or "").strip()
|
||||
return sec or None
|
||||
|
||||
|
||||
async def _fill_from_fmp(
|
||||
missing: list[str],
|
||||
*,
|
||||
api_key: str,
|
||||
limit: int,
|
||||
sleep_s: float,
|
||||
) -> tuple[dict[str, str], int]:
|
||||
filled: dict[str, str] = {}
|
||||
used = 0
|
||||
async with httpx.AsyncClient(timeout=30.0) as client:
|
||||
for sym in missing:
|
||||
if used >= limit:
|
||||
break
|
||||
try:
|
||||
sector = await _fmp_profile_sector(client, api_key, sym)
|
||||
except Exception as exc:
|
||||
print(f" FMP fail {sym}: {exc}")
|
||||
used += 1
|
||||
await asyncio.sleep(sleep_s)
|
||||
continue
|
||||
used += 1
|
||||
if sector:
|
||||
filled[sym] = sector
|
||||
print(f" FMP {sym} → {sector}")
|
||||
else:
|
||||
print(f" FMP {sym} → (no sector)")
|
||||
if sleep_s > 0:
|
||||
await asyncio.sleep(sleep_s)
|
||||
return filled, used
|
||||
|
||||
|
||||
async def _main() -> None:
|
||||
args = _parse_args()
|
||||
snapshot = Path(args.snapshot)
|
||||
if not snapshot.exists():
|
||||
raise SystemExit(f"Snapshot not found: {snapshot}")
|
||||
|
||||
symbols = _snapshot_symbols(snapshot)
|
||||
print(f"Universe: {len(symbols)} symbols from {snapshot}")
|
||||
|
||||
existing = load_ticker_sector_map(args.out)
|
||||
print(f"Existing map entries: {len(existing)}")
|
||||
|
||||
print("Fetching public S&P 500 sector CSV…")
|
||||
sp500 = _fetch_sp500_map()
|
||||
print(f" SP500 CSV rows: {len(sp500)}")
|
||||
|
||||
mapping = dict(existing)
|
||||
from_sp500 = 0
|
||||
for sym in symbols:
|
||||
if sym in mapping:
|
||||
continue
|
||||
if sym in sp500:
|
||||
mapping[sym] = sp500[sym]
|
||||
from_sp500 += 1
|
||||
print(f" Newly filled from SP500 CSV: {from_sp500}")
|
||||
|
||||
missing = [s for s in symbols if s not in mapping]
|
||||
fmp_used = 0
|
||||
from_fmp = 0
|
||||
if missing and not args.skip_fmp:
|
||||
from app.config import settings
|
||||
|
||||
if not settings.fmp_api_key:
|
||||
print("WARNING: FMP key missing; leaving gaps unfilled")
|
||||
else:
|
||||
print(f"FMP fill for {len(missing)} missing (limit={args.fmp_limit})…")
|
||||
filled, fmp_used = await _fill_from_fmp(
|
||||
missing,
|
||||
api_key=settings.fmp_api_key,
|
||||
limit=int(args.fmp_limit),
|
||||
sleep_s=float(args.sleep),
|
||||
)
|
||||
mapping.update(filled)
|
||||
from_fmp = len(filled)
|
||||
|
||||
still_missing = [s for s in symbols if s not in mapping]
|
||||
stats = coverage_stats(symbols, mapping)
|
||||
meta = {
|
||||
"built_at": datetime.now(timezone.utc).isoformat(),
|
||||
"snapshot": str(snapshot.resolve()),
|
||||
"from_existing": len(existing),
|
||||
"from_sp500_csv": from_sp500,
|
||||
"from_fmp": from_fmp,
|
||||
"fmp_requests": fmp_used,
|
||||
"still_missing": still_missing,
|
||||
"coverage": {
|
||||
k: stats[k]
|
||||
for k in ("universe", "mapped", "mapped_pct", "with_etf", "by_sector")
|
||||
},
|
||||
}
|
||||
out_path = save_ticker_sector_map(mapping, args.out, meta=meta)
|
||||
print(f"Wrote {out_path}")
|
||||
print(json.dumps(meta["coverage"], indent=2))
|
||||
if still_missing:
|
||||
print(f"Still missing ({len(still_missing)}): {still_missing[:40]}")
|
||||
if len(still_missing) > 40:
|
||||
print(f" … +{len(still_missing) - 40} more")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(_main())
|
||||
@@ -1,187 +0,0 @@
|
||||
"""Fetch the 11 SPDR sector ETFs into a snapshot's ``benchmark_prices``.
|
||||
|
||||
Research-only. Sector ETFs are auxiliary series (like SPY) — they must not
|
||||
enter the tradable ticker universe or candidate replay. Storing them in
|
||||
``benchmark_prices`` keeps that invariant.
|
||||
|
||||
Also refreshes SPY on the same window so residual factors share a calendar.
|
||||
|
||||
Example
|
||||
-------
|
||||
python scripts/fetch_sector_etfs_to_snapshot.py \\
|
||||
--snapshot backtest_snapshots/prod.sqlite --history-days 2200
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import sys
|
||||
import time
|
||||
from datetime import date, timedelta
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import create_engine, text
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from app.ssl_bootstrap import bootstrap_ssl # noqa: E402
|
||||
|
||||
bootstrap_ssl()
|
||||
|
||||
from app.services.sector_map import SECTOR_ETFS # noqa: E402
|
||||
|
||||
|
||||
def _parse_args() -> argparse.Namespace:
|
||||
p = argparse.ArgumentParser(description=__doc__)
|
||||
p.add_argument("--snapshot", default="backtest_snapshots/prod.sqlite")
|
||||
p.add_argument(
|
||||
"--history-days",
|
||||
type=int,
|
||||
default=2200,
|
||||
help="Lookback calendar days (default ~6y; covers 5y snapshot + cushion).",
|
||||
)
|
||||
p.add_argument("--sleep", type=float, default=0.25)
|
||||
p.add_argument(
|
||||
"--symbols",
|
||||
default=None,
|
||||
help="Comma-separated override (default: SPY + 11 sector ETFs).",
|
||||
)
|
||||
return p.parse_args()
|
||||
|
||||
|
||||
async def _fetch_and_upsert(
|
||||
engine,
|
||||
provider,
|
||||
symbol: str,
|
||||
start: date,
|
||||
end: date,
|
||||
*,
|
||||
sleep_s: float,
|
||||
) -> int:
|
||||
from app.exceptions import ProviderError, RateLimitError
|
||||
|
||||
for attempt in range(5):
|
||||
try:
|
||||
bars = await provider.fetch_ohlcv(symbol, start, end)
|
||||
break
|
||||
except RateLimitError:
|
||||
wait = min(60.0, 2.0 ** attempt)
|
||||
print(f" rate limited {symbol}; sleep {wait:.0f}s")
|
||||
await asyncio.sleep(wait)
|
||||
bars = []
|
||||
except ProviderError as exc:
|
||||
if attempt + 1 >= 5:
|
||||
raise
|
||||
await asyncio.sleep(1.0)
|
||||
print(f" retry {symbol}: {exc}")
|
||||
bars = []
|
||||
else:
|
||||
bars = []
|
||||
|
||||
if sleep_s > 0:
|
||||
await asyncio.sleep(sleep_s)
|
||||
|
||||
if not bars:
|
||||
print(f" {symbol}: empty")
|
||||
return 0
|
||||
|
||||
written = 0
|
||||
with engine.begin() as conn:
|
||||
for bar in bars:
|
||||
d = bar.date.isoformat() if hasattr(bar.date, "isoformat") else str(bar.date)
|
||||
close = float(bar.close)
|
||||
existing = conn.execute(
|
||||
text(
|
||||
"SELECT id, close FROM benchmark_prices "
|
||||
"WHERE symbol = :sym AND date = :d"
|
||||
),
|
||||
{"sym": symbol, "d": d},
|
||||
).fetchone()
|
||||
if existing is None:
|
||||
# id is INTEGER PK — let sqlite autoincrement if possible
|
||||
conn.execute(
|
||||
text(
|
||||
"INSERT INTO benchmark_prices (symbol, date, close) "
|
||||
"VALUES (:sym, :d, :c)"
|
||||
),
|
||||
{"sym": symbol, "d": d, "c": close},
|
||||
)
|
||||
written += 1
|
||||
elif abs(float(existing[1]) - close) > 1e-9:
|
||||
conn.execute(
|
||||
text(
|
||||
"UPDATE benchmark_prices SET close = :c WHERE id = :id"
|
||||
),
|
||||
{"c": close, "id": int(existing[0])},
|
||||
)
|
||||
written += 1
|
||||
print(f" {symbol}: {len(bars)} bars, {written} rows written/updated")
|
||||
return written
|
||||
|
||||
|
||||
async def _main() -> None:
|
||||
args = _parse_args()
|
||||
snapshot = Path(args.snapshot)
|
||||
if not snapshot.exists():
|
||||
raise SystemExit(f"Snapshot not found: {snapshot}")
|
||||
|
||||
from app.config import settings
|
||||
from app.providers.alpaca import AlpacaOHLCVProvider
|
||||
|
||||
if not settings.alpaca_api_key or not settings.alpaca_api_secret:
|
||||
raise SystemExit("ALPACA_API_KEY / ALPACA_API_SECRET required")
|
||||
|
||||
if args.symbols:
|
||||
symbols = [s.strip().upper() for s in args.symbols.split(",") if s.strip()]
|
||||
else:
|
||||
symbols = ["SPY", *SECTOR_ETFS]
|
||||
|
||||
end = date.today()
|
||||
start = end - timedelta(days=int(args.history_days))
|
||||
provider = AlpacaOHLCVProvider(settings.alpaca_api_key, settings.alpaca_api_secret)
|
||||
engine = create_engine(
|
||||
f"sqlite:///{snapshot.resolve().as_posix()}",
|
||||
future=True,
|
||||
)
|
||||
|
||||
print(f"Snapshot: {snapshot}")
|
||||
print(f"Window: {start} → {end}")
|
||||
print(f"Symbols: {symbols}")
|
||||
|
||||
t0 = time.monotonic()
|
||||
total = 0
|
||||
try:
|
||||
for sym in symbols:
|
||||
n = await _fetch_and_upsert(
|
||||
engine, provider, sym, start, end, sleep_s=float(args.sleep)
|
||||
)
|
||||
total += n
|
||||
finally:
|
||||
engine.dispose()
|
||||
|
||||
# Summary counts
|
||||
engine = create_engine(
|
||||
f"sqlite:///{snapshot.resolve().as_posix()}",
|
||||
future=True,
|
||||
)
|
||||
try:
|
||||
with engine.connect() as conn:
|
||||
rows = conn.execute(
|
||||
text(
|
||||
"SELECT symbol, COUNT(*), MIN(date), MAX(date) "
|
||||
"FROM benchmark_prices GROUP BY symbol ORDER BY symbol"
|
||||
)
|
||||
).fetchall()
|
||||
finally:
|
||||
engine.dispose()
|
||||
|
||||
print(f"Done in {(time.monotonic() - t0) / 60:.1f}m; rows touched={total}")
|
||||
for sym, n, d0, d1 in rows:
|
||||
print(f" {sym}: n={n} {d0}→{d1}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(_main())
|
||||
@@ -466,11 +466,6 @@ async def _run_2b_ic(
|
||||
|
||||
os.environ["BACKTEST_SNAPSHOT_OFFLINE"] = "1"
|
||||
os.environ["BACKTEST_SIGNAL_EVAL_ONLY"] = "1"
|
||||
# Load sector map if present so sector signals also appear (side-by-side optional).
|
||||
if Path("data/research/ticker_sector_map.json").exists():
|
||||
os.environ["BACKTEST_SECTOR_MAP_PATH"] = str(
|
||||
Path("data/research/ticker_sector_map.json").resolve()
|
||||
)
|
||||
settings.backtest_workers = workers
|
||||
|
||||
engine = create_async_engine(_sqlite_url(snapshot), pool_pre_ping=True)
|
||||
@@ -484,18 +479,6 @@ async def _run_2b_ic(
|
||||
(await db.execute(select(Ticker).order_by(Ticker.symbol))).scalars()
|
||||
)
|
||||
spy = await load_benchmark_closes(db, "SPY")
|
||||
sector_etf: dict[str, dict] = {}
|
||||
try:
|
||||
from app.services.sector_map import SECTOR_ETFS, load_ticker_sector_map
|
||||
|
||||
symbol_to_sector = load_ticker_sector_map()
|
||||
for etf in SECTOR_ETFS:
|
||||
series = await load_benchmark_closes(db, etf)
|
||||
if series:
|
||||
sector_etf[etf] = series
|
||||
except Exception:
|
||||
symbol_to_sector = {}
|
||||
sector_etf = {}
|
||||
|
||||
prices: dict[str, tuple] = {}
|
||||
for idx, t in enumerate(tickers):
|
||||
@@ -521,9 +504,6 @@ async def _run_2b_ic(
|
||||
],
|
||||
spy,
|
||||
symbol=t.symbol,
|
||||
sector_etf_closes=bt._sector_etf_closes_for_symbol(
|
||||
t.symbol, symbol_to_sector, sector_etf
|
||||
),
|
||||
)
|
||||
for name, weeks in series.items():
|
||||
for wk, pairs in weeks.items():
|
||||
@@ -533,9 +513,6 @@ async def _run_2b_ic(
|
||||
if not quiet:
|
||||
print()
|
||||
|
||||
if symbol_to_sector:
|
||||
bt._inject_sector_demeaned_momentum(collected, symbol_to_sector)
|
||||
|
||||
# SUE series.
|
||||
events_by_sym: dict[str, list[dict]] = defaultdict(list)
|
||||
for ev in events:
|
||||
@@ -709,8 +686,6 @@ async def _run_2b_ic(
|
||||
for name in (
|
||||
"mom_12_1",
|
||||
"mom_12_1_resid",
|
||||
"mom_12_1_sector_resid",
|
||||
"mom_12_1_sector_demeaned",
|
||||
"sue_latest",
|
||||
"fip_id",
|
||||
)
|
||||
@@ -784,7 +759,6 @@ def _write_md(path: Path, payload: dict) -> None:
|
||||
"mom_12_1",
|
||||
"mom_12_1_resid",
|
||||
"sue_latest",
|
||||
"mom_12_1_sector_resid",
|
||||
"fip_id",
|
||||
):
|
||||
r = side.get(name) or {}
|
||||
|
||||
@@ -1,480 +0,0 @@
|
||||
"""History-depth extension research (local / MacBook).
|
||||
|
||||
Phases
|
||||
------
|
||||
coverage — bars per calendar year; no rebuild
|
||||
harness — race-guard snapshot, full signal_eval, era split pre/post-2021
|
||||
|
||||
Does not retune production knobs. Does not modify scheduler/gates.
|
||||
|
||||
Example
|
||||
-------
|
||||
python scripts/run_history_depth_research.py --phase coverage \\
|
||||
--snapshot backtest_snapshots/prod.sqlite
|
||||
|
||||
python scripts/run_history_depth_research.py --phase harness \\
|
||||
--snapshot backtest_snapshots/research.sqlite --workers 8 --allow-spawn
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from datetime import date, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import create_engine, text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from app.ssl_bootstrap import bootstrap_ssl # noqa: E402
|
||||
|
||||
bootstrap_ssl()
|
||||
|
||||
ERA_SPLIT = date(2021, 1, 1)
|
||||
SURVIVORSHIP_BANNER = (
|
||||
"SURVIVORSHIP BIAS: today's constituents backfilled historically. "
|
||||
"Absolute Sharpe/CAGR levels on deep history are optimistic. "
|
||||
"Use RELATIVE signal IC comparisons and era stability only — not levels."
|
||||
)
|
||||
|
||||
|
||||
def _sqlite_url(path: Path) -> str:
|
||||
return f"sqlite+aiosqlite:///{path.resolve().as_posix()}"
|
||||
|
||||
|
||||
def _parse_args() -> argparse.Namespace:
|
||||
p = argparse.ArgumentParser(description=__doc__)
|
||||
p.add_argument("--phase", choices=("coverage", "harness", "all"), default="all")
|
||||
p.add_argument("--snapshot", default="backtest_snapshots/research.sqlite")
|
||||
p.add_argument("--workers", type=int, default=8)
|
||||
p.add_argument("--allow-spawn", action="store_true")
|
||||
p.add_argument("--quiet", action="store_true")
|
||||
p.add_argument("--out", default=None)
|
||||
return p.parse_args()
|
||||
|
||||
|
||||
def _coverage_report(snapshot: Path) -> dict[str, Any]:
|
||||
engine = create_engine(
|
||||
f"sqlite:///{snapshot.resolve().as_posix()}",
|
||||
future=True,
|
||||
)
|
||||
try:
|
||||
with engine.connect() as conn:
|
||||
ticker_n = int(conn.execute(text("SELECT COUNT(*) FROM tickers")).scalar_one())
|
||||
ohlcv_n = int(
|
||||
conn.execute(text("SELECT COUNT(*) FROM ohlcv_records")).scalar_one()
|
||||
)
|
||||
d_range = conn.execute(
|
||||
text("SELECT MIN(date), MAX(date) FROM ohlcv_records")
|
||||
).fetchone()
|
||||
# Bars per calendar year (global).
|
||||
by_year = conn.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT substr(date, 1, 4) AS y, COUNT(*) AS n,
|
||||
COUNT(DISTINCT ticker_id) AS tickers
|
||||
FROM ohlcv_records
|
||||
GROUP BY substr(date, 1, 4)
|
||||
ORDER BY y
|
||||
"""
|
||||
)
|
||||
).fetchall()
|
||||
# Per-symbol min/max date + bar count (summary percentiles).
|
||||
per_sym = conn.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT t.symbol, COUNT(*) AS n, MIN(o.date), MAX(o.date)
|
||||
FROM ohlcv_records o
|
||||
JOIN tickers t ON t.id = o.ticker_id
|
||||
GROUP BY t.symbol
|
||||
"""
|
||||
)
|
||||
).fetchall()
|
||||
finally:
|
||||
engine.dispose()
|
||||
|
||||
ns = sorted(int(r[1]) for r in per_sym)
|
||||
def pct(p: float) -> int | None:
|
||||
if not ns:
|
||||
return None
|
||||
i = int(round(p * (len(ns) - 1)))
|
||||
return ns[i]
|
||||
|
||||
starts = sorted(str(r[2]) for r in per_sym if r[2])
|
||||
start_hist: dict[str, int] = defaultdict(int)
|
||||
for s in starts:
|
||||
start_hist[s[:4]] += 1
|
||||
|
||||
return {
|
||||
"snapshot": str(snapshot.resolve()),
|
||||
"ticker_count": ticker_n,
|
||||
"ohlcv_row_count": ohlcv_n,
|
||||
"date_range": {"min": d_range[0], "max": d_range[1]},
|
||||
"bars_per_year": [
|
||||
{"year": y, "bars": n, "tickers_with_bars": t} for y, n, t in by_year
|
||||
],
|
||||
"bars_per_symbol": {
|
||||
"min": ns[0] if ns else None,
|
||||
"p10": pct(0.10),
|
||||
"p50": pct(0.50),
|
||||
"p90": pct(0.90),
|
||||
"max": ns[-1] if ns else None,
|
||||
},
|
||||
"symbols_by_start_year": dict(sorted(start_hist.items())),
|
||||
"note": (
|
||||
"Where ticker counts drop in early years, the feed (or listing history) "
|
||||
"thins — do not treat those years as a full 505-name cross-section."
|
||||
),
|
||||
"survivorship_banner": SURVIVORSHIP_BANNER,
|
||||
}
|
||||
|
||||
|
||||
def _assert_complete(snapshot: Path) -> dict[str, Any]:
|
||||
from scripts.research_snapshot_manifest import ( # type: ignore
|
||||
assert_research_snapshot_complete,
|
||||
load_manifest,
|
||||
)
|
||||
|
||||
m = load_manifest(snapshot)
|
||||
if m is None:
|
||||
# Prod snapshot may lack manifest; still require healthy bar depth.
|
||||
eng = create_engine(
|
||||
f"sqlite:///{snapshot.resolve().as_posix()}",
|
||||
future=True,
|
||||
)
|
||||
try:
|
||||
with eng.connect() as conn:
|
||||
avg = conn.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT AVG(c) FROM (
|
||||
SELECT COUNT(*) AS c FROM ohlcv_records GROUP BY ticker_id
|
||||
)
|
||||
"""
|
||||
)
|
||||
).scalar_one()
|
||||
finally:
|
||||
eng.dispose()
|
||||
if avg is None or float(avg) < 400:
|
||||
raise SystemExit(
|
||||
f"No completion manifest and avg bars={avg} look short. "
|
||||
"Rebuild research.sqlite via extend_snapshot_universe.py"
|
||||
)
|
||||
return {"manifest": None, "avg_bars": float(avg), "ok": True}
|
||||
return {"manifest": assert_research_snapshot_complete(snapshot), "ok": True}
|
||||
|
||||
|
||||
async def _harness(snapshot: Path, *, workers: int, quiet: bool) -> dict[str, Any]:
|
||||
from app.config import settings
|
||||
from app.services.backtest_service import run_backtest
|
||||
|
||||
os.environ["BACKTEST_SNAPSHOT_OFFLINE"] = "1"
|
||||
os.environ["BACKTEST_SIGNAL_EVAL_ONLY"] = "1"
|
||||
if Path("data/research/ticker_sector_map.json").exists():
|
||||
os.environ["BACKTEST_SECTOR_MAP_PATH"] = str(
|
||||
Path("data/research/ticker_sector_map.json").resolve()
|
||||
)
|
||||
settings.backtest_workers = workers
|
||||
|
||||
engine = create_async_engine(_sqlite_url(snapshot), pool_pre_ping=True)
|
||||
Session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
|
||||
|
||||
def progress(done: int, total: int, symbol: str) -> None:
|
||||
if quiet:
|
||||
return
|
||||
print(f" progress {done}/{total} {symbol}", end="\r", flush=True)
|
||||
|
||||
try:
|
||||
async with Session() as db:
|
||||
report = await run_backtest(db, progress_cb=progress, cadence="weekly")
|
||||
finally:
|
||||
await engine.dispose()
|
||||
if not quiet:
|
||||
print()
|
||||
|
||||
signal_eval = report.get("signal_eval") or []
|
||||
|
||||
# Era-split IC: recompute from collected is not available post-run.
|
||||
# Approximate via second pass is expensive; instead document that era split
|
||||
# requires collecting weekly ICs. We re-run evaluation if the report embeds
|
||||
# nothing — for v1, call internal collection is too heavy to duplicate.
|
||||
# Lightweight approach: mark era_split as requiring BACKTEST with custom
|
||||
# filter — implemented below by re-scoring from a dedicated collection pass.
|
||||
era = await _era_split_ics(snapshot, workers=workers, quiet=quiet)
|
||||
|
||||
return {
|
||||
"survivorship_banner": SURVIVORSHIP_BANNER,
|
||||
"signal_eval": signal_eval,
|
||||
"era_split": era,
|
||||
"params": report.get("params"),
|
||||
"tickers": report.get("tickers"),
|
||||
"generated_at_run": report.get("generated_at"),
|
||||
}
|
||||
|
||||
|
||||
async def _era_split_ics(
|
||||
snapshot: Path, *, workers: int, quiet: bool
|
||||
) -> dict[str, Any]:
|
||||
"""Collect weekly signal series and evaluate pre/post ERA_SPLIT separately."""
|
||||
from app.config import settings
|
||||
from app.services import backtest_service as bt
|
||||
from app.services.benchmark_service import load_benchmark_closes
|
||||
from app.models.ticker import Ticker
|
||||
from sqlalchemy import select
|
||||
from collections import defaultdict as dd
|
||||
|
||||
os.environ["BACKTEST_SNAPSHOT_OFFLINE"] = "1"
|
||||
settings.backtest_workers = max(1, workers)
|
||||
|
||||
engine = create_async_engine(_sqlite_url(snapshot), pool_pre_ping=True)
|
||||
Session = async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
|
||||
|
||||
collected: dict = dd(lambda: dd(list))
|
||||
try:
|
||||
async with Session() as db:
|
||||
tickers = list(
|
||||
(await db.execute(select(Ticker).order_by(Ticker.symbol))).scalars()
|
||||
)
|
||||
spy = await load_benchmark_closes(db, "SPY")
|
||||
symbol_to_sector = {}
|
||||
sector_etf: dict = {}
|
||||
try:
|
||||
from app.services.sector_map import (
|
||||
SECTOR_ETFS,
|
||||
load_ticker_sector_map,
|
||||
)
|
||||
|
||||
symbol_to_sector = load_ticker_sector_map()
|
||||
for etf in SECTOR_ETFS:
|
||||
series = await load_benchmark_closes(db, etf)
|
||||
if series:
|
||||
sector_etf[etf] = series
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
for idx, t in enumerate(tickers):
|
||||
if not quiet and idx % 100 == 0:
|
||||
print(f" era-collect {idx}/{len(tickers)}", end="\r", flush=True)
|
||||
cols = await bt._fetch_columns(db, t.symbol)
|
||||
if cols is None:
|
||||
continue
|
||||
records = [
|
||||
type(
|
||||
"R",
|
||||
(),
|
||||
{
|
||||
"date": date.fromordinal(int(cols[0][i])),
|
||||
"close": cols[4][i],
|
||||
"high": cols[2][i],
|
||||
"volume": cols[5][i] if len(cols) > 5 else 0,
|
||||
},
|
||||
)()
|
||||
for i in range(len(cols[0]))
|
||||
]
|
||||
series = bt._signal_series(
|
||||
records,
|
||||
spy,
|
||||
symbol=t.symbol,
|
||||
sector_etf_closes=bt._sector_etf_closes_for_symbol(
|
||||
t.symbol, symbol_to_sector, sector_etf
|
||||
),
|
||||
)
|
||||
for name, weeks in series.items():
|
||||
for wk, pairs in weeks.items():
|
||||
collected[name][wk].extend(pairs)
|
||||
if symbol_to_sector:
|
||||
bt._inject_sector_demeaned_momentum(collected, symbol_to_sector)
|
||||
finally:
|
||||
await engine.dispose()
|
||||
if not quiet:
|
||||
print()
|
||||
|
||||
def _filter_era(coll: dict, *, pre: bool) -> dict:
|
||||
out: dict = dd(lambda: dd(list))
|
||||
for name, weeks in coll.items():
|
||||
for wk, recs in weeks.items():
|
||||
# ISO week key (year, week) — approximate era by ISO year.
|
||||
year = int(wk[0]) if isinstance(wk, tuple) else int(str(wk)[:4])
|
||||
if pre and year >= ERA_SPLIT.year:
|
||||
continue
|
||||
if not pre and year < ERA_SPLIT.year:
|
||||
continue
|
||||
out[name][wk].extend(recs)
|
||||
return out
|
||||
|
||||
pre_eval = bt._signal_evaluation(_filter_era(collected, pre=True))
|
||||
post_eval = bt._signal_evaluation(_filter_era(collected, pre=False))
|
||||
full_eval = bt._signal_evaluation(collected)
|
||||
|
||||
def _index(rows: list[dict]) -> dict[str, dict]:
|
||||
return {r["signal"]: r for r in rows}
|
||||
|
||||
return {
|
||||
"era_split_date": ERA_SPLIT.isoformat(),
|
||||
"note": "Diagnostic only — not a tuning input. Nested lookbacks are not OOS.",
|
||||
"full": _index(full_eval),
|
||||
"pre_2021": _index(pre_eval),
|
||||
"post_2021": _index(post_eval),
|
||||
}
|
||||
|
||||
|
||||
def _write_md(path: Path, payload: dict) -> None:
|
||||
pre = path.read_text(encoding="utf-8") if path.exists() else ""
|
||||
marker = "## Results"
|
||||
idx = pre.find(marker)
|
||||
header = pre[:idx] if idx >= 0 else pre.split("## Verdict")[0]
|
||||
|
||||
lines = [
|
||||
header.rstrip(),
|
||||
"",
|
||||
"## Results",
|
||||
"",
|
||||
f"Generated: `{payload.get('generated_at')}`",
|
||||
"",
|
||||
f"> **{SURVIVORSHIP_BANNER}**",
|
||||
"",
|
||||
"### Coverage",
|
||||
"",
|
||||
f"```json\n{json.dumps(payload.get('coverage') or {}, indent=2, default=str)}\n```",
|
||||
"",
|
||||
"### Race guard",
|
||||
"",
|
||||
f"```json\n{json.dumps(payload.get('race_guard') or {}, indent=2, default=str)}\n```",
|
||||
"",
|
||||
"### Signal IC (full extended window)",
|
||||
"",
|
||||
]
|
||||
harness = payload.get("harness") or {}
|
||||
rows = harness.get("signal_eval") or []
|
||||
if rows:
|
||||
lines.extend([
|
||||
"| signal | mean_ic | ic_t_stat | weeks | avg_N | reliable |",
|
||||
"|---|---:|---:|---:|---:|---|",
|
||||
])
|
||||
for r in rows:
|
||||
lines.append(
|
||||
f"| {r.get('signal')} | {r.get('mean_ic')} | {r.get('ic_t_stat')} | "
|
||||
f"{r.get('weeks')} | {r.get('avg_cross_section')} | {r.get('reliable')} |"
|
||||
)
|
||||
else:
|
||||
lines.append("_Harness not run this pass._")
|
||||
|
||||
era = (harness.get("era_split") or {})
|
||||
lines.extend(["", "### Era split (diagnostic only)", ""])
|
||||
if era:
|
||||
for label in ("full", "pre_2021", "post_2021"):
|
||||
block = era.get(label) or {}
|
||||
lines.append(f"#### {label}")
|
||||
lines.append("")
|
||||
lines.append("| signal | mean_ic | t | weeks | N |")
|
||||
lines.append("|---|---:|---:|---:|---:|")
|
||||
for name in sorted(block):
|
||||
r = block[name]
|
||||
lines.append(
|
||||
f"| {name} | {r.get('mean_ic')} | {r.get('ic_t_stat')} | "
|
||||
f"{r.get('weeks')} | {r.get('avg_cross_section')} |"
|
||||
)
|
||||
lines.append("")
|
||||
else:
|
||||
lines.append("_No era split._")
|
||||
|
||||
lines.extend([
|
||||
"",
|
||||
"## Verdict",
|
||||
"",
|
||||
f"**{payload.get('verdict')}**",
|
||||
"",
|
||||
payload.get("verdict_detail") or "",
|
||||
"",
|
||||
"## What a human must decide next",
|
||||
"",
|
||||
payload.get("human_next")
|
||||
or "- Do not retune production knobs from this report without review.",
|
||||
"",
|
||||
f"Artifacts: `{payload.get('report_path')}`",
|
||||
"",
|
||||
])
|
||||
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
async def _main() -> None:
|
||||
args = _parse_args()
|
||||
snapshot = Path(args.snapshot)
|
||||
if not snapshot.exists():
|
||||
raise SystemExit(f"Missing snapshot: {snapshot}")
|
||||
if args.allow_spawn:
|
||||
os.environ["BACKTEST_ALLOW_SPAWN"] = "1"
|
||||
|
||||
coverage = None
|
||||
race = None
|
||||
harness = None
|
||||
|
||||
if args.phase in ("coverage", "all"):
|
||||
print("Coverage probe…")
|
||||
coverage = _coverage_report(snapshot)
|
||||
print(
|
||||
f" tickers={coverage['ticker_count']} ohlcv={coverage['ohlcv_row_count']} "
|
||||
f"range={coverage['date_range']}"
|
||||
)
|
||||
for row in coverage["bars_per_year"]:
|
||||
print(
|
||||
f" year {row['year']}: bars={row['bars']} "
|
||||
f"tickers={row['tickers_with_bars']}"
|
||||
)
|
||||
|
||||
if args.phase in ("harness", "all"):
|
||||
print("Race guard…")
|
||||
race = _assert_complete(snapshot)
|
||||
print(f" ok={race.get('ok')}")
|
||||
print("Full harness + era split (LONG)…")
|
||||
print(f" {SURVIVORSHIP_BANNER}")
|
||||
harness = await _harness(
|
||||
snapshot, workers=args.workers, quiet=args.quiet
|
||||
)
|
||||
|
||||
stamp = datetime.now().strftime("%Y%m%d-%H%M%S")
|
||||
out = (
|
||||
Path(args.out)
|
||||
if args.out
|
||||
else Path("reports") / f"history-depth-{stamp}.json"
|
||||
)
|
||||
payload = {
|
||||
"generated_at": datetime.now().isoformat(),
|
||||
"survivorship_banner": SURVIVORSHIP_BANNER,
|
||||
"coverage": coverage,
|
||||
"race_guard": race,
|
||||
"harness": harness,
|
||||
"verdict": "PENDING_HUMAN" if harness else "COVERAGE_ONLY",
|
||||
"verdict_detail": (
|
||||
"Harness complete — human interprets relative IC / era stability. "
|
||||
"No production retune from this artifact."
|
||||
if harness
|
||||
else "Coverage probe only; run --phase harness after deep rebuild."
|
||||
),
|
||||
"human_next": (
|
||||
"- Compare sector residual vs market residual across eras.\n"
|
||||
"- If pre-2021 IC collapses, park Task 1 wire-in.\n"
|
||||
"- Do not retune production knobs on deep history levels."
|
||||
),
|
||||
"report_path": str(out.as_posix()),
|
||||
}
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
out.write_text(json.dumps(payload, indent=2, default=str) + "\n", encoding="utf-8")
|
||||
md = Path("docs/research/history-depth-extension.md")
|
||||
_write_md(md, payload)
|
||||
out.with_suffix(".md").write_text(md.read_text(encoding="utf-8"), encoding="utf-8")
|
||||
print(f"Wrote {out}")
|
||||
print(f"Wrote {md}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(_main())
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
+46
-198
@@ -1,99 +1,70 @@
|
||||
#!/usr/bin/env bash
|
||||
# Tier-1 alpha research runner for a high-CPU MacBook (local only).
|
||||
# Research helpers for a high-CPU MacBook (local only).
|
||||
#
|
||||
# Prerequisites
|
||||
# - git checkout research/earnings-gap-and-sue (or later research branch)
|
||||
# - .env with ALPACA_* (required for OHLCV/ETFs); FMP_* for earnings resume;
|
||||
# optional ALPHA_VANTAGE_* as earnings fallback
|
||||
# - Python venv with project deps installed
|
||||
# - backtest_snapshots/prod.sqlite present (gitignored — copy or rebuild)
|
||||
# Kept after Tier-1 cleanup:
|
||||
# --ssl-check diagnose corporate CA / proxy
|
||||
# --earnings-only resume FMP earnings backfill + 2a/2b (parked)
|
||||
# --prod-book-matrix re-run 505 vs liquid universe × horizon book matrix
|
||||
#
|
||||
# Prerequisites: git checkout research branch, .env, deep research.sqlite for
|
||||
# book matrix, combined-ca-bundle.pem or certifi when on corp network.
|
||||
#
|
||||
# Usage
|
||||
# chmod +x scripts/run_tier1_macbook.sh
|
||||
# ./scripts/run_tier1_macbook.sh # coverage + deep rebuild + harness
|
||||
# ./scripts/run_tier1_macbook.sh --all # earnings resume + full depth pipeline
|
||||
# ./scripts/run_tier1_macbook.sh --earnings-only # multi-day FMP backfill + re-run 2a/2b
|
||||
# ./scripts/run_tier1_macbook.sh --harness-only # skip rebuild; race-guard + IC only
|
||||
# ./scripts/run_tier1_macbook.sh --coverage-only # bars-per-year probe only
|
||||
# ./scripts/run_tier1_macbook.sh --sector-resid-deep # deepen shallow + ONE masked grade
|
||||
# ./scripts/run_tier1_macbook.sh --prod-book-matrix # 4-arm universe×horizon book matrix
|
||||
#
|
||||
# Does NOT touch production Postgres, scheduler, gates, or prod config.
|
||||
# ./scripts/run_tier1_macbook.sh --ssl-check
|
||||
# ./scripts/run_tier1_macbook.sh --prod-book-matrix
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
cd "$ROOT"
|
||||
|
||||
# --- defaults (override via flags or env) ---
|
||||
PROD_SNAP="${PROD_SNAP:-backtest_snapshots/prod.sqlite}"
|
||||
RESEARCH_SNAP="${RESEARCH_SNAP:-backtest_snapshots/research.sqlite}"
|
||||
HISTORY_DAYS="${HISTORY_DAYS:-5000}"
|
||||
MIN_BARS="${MIN_BARS:-260}"
|
||||
PROD_SNAP="${PROD_SNAP:-backtest_snapshots/prod.sqlite}"
|
||||
WORKERS="${WORKERS:-8}"
|
||||
ALPACA_SLEEP="${ALPACA_SLEEP:-0.15}"
|
||||
FMP_LIMIT="${FMP_LIMIT:-250}"
|
||||
FMP_SLEEP="${FMP_SLEEP:-0.35}"
|
||||
PYTHON="${PYTHON:-python3}"
|
||||
USE_CORP_PROXY="${USE_CORP_PROXY:-0}"
|
||||
|
||||
PHASE="depth" # depth | all | earnings | harness | coverage | ssl | sector-resid-deep | prod-book
|
||||
PHASE=""
|
||||
|
||||
usage() {
|
||||
sed -n '2,25p' "$0" | sed 's/^# \?//'
|
||||
cat <<'EOF'
|
||||
|
||||
SSL / network (corporate MacBook)
|
||||
SSL errors usually mean the corp root CA is missing from Python.
|
||||
1) Put combined-ca-bundle.pem in the repo root OR $HOME
|
||||
2) Or: export SSL_CERT_FILE=/path/to/combined-ca-bundle.pem
|
||||
3) Behind corp proxy: USE_CORP_PROXY=1 ./scripts/run_tier1_macbook.sh
|
||||
4) Diagnose: ./scripts/run_tier1_macbook.sh --ssl-check
|
||||
EOF
|
||||
sed -n '2,16p' "$0" | sed 's/^# \?//'
|
||||
exit "${1:-0}"
|
||||
}
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--all) PHASE=all; shift ;;
|
||||
--earnings-only) PHASE=earnings; shift ;;
|
||||
--harness-only) PHASE=harness; shift ;;
|
||||
--coverage-only) PHASE=coverage; shift ;;
|
||||
--depth) PHASE=depth; shift ;;
|
||||
--ssl-check) PHASE=ssl; shift ;;
|
||||
--sector-resid-deep) PHASE=sector_resid_deep; shift ;;
|
||||
--earnings-only) PHASE=earnings; shift ;;
|
||||
--prod-book-matrix) PHASE=prod_book; shift ;;
|
||||
--corp-proxy) USE_CORP_PROXY=1; shift ;;
|
||||
--prod-snap) PROD_SNAP="$2"; shift 2 ;;
|
||||
--research-snap) RESEARCH_SNAP="$2"; shift 2 ;;
|
||||
--history-days) HISTORY_DAYS="$2"; shift 2 ;;
|
||||
--workers) WORKERS="$2"; shift 2 ;;
|
||||
--fmp-limit) FMP_LIMIT="$2"; shift 2 ;;
|
||||
--python) PYTHON="$2"; shift 2 ;;
|
||||
-h|--help) usage 0 ;;
|
||||
*) echo "Unknown flag: $1" >&2; usage 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ -z "$PHASE" ]]; then
|
||||
echo "Pick a phase: --ssl-check | --earnings-only | --prod-book-matrix" >&2
|
||||
usage 1
|
||||
fi
|
||||
|
||||
if [[ -x .venv/bin/python ]]; then
|
||||
PYTHON=".venv/bin/python"
|
||||
elif command -v "$PYTHON" >/dev/null 2>&1; then
|
||||
:
|
||||
else
|
||||
echo "ERROR: no Python found (tried .venv/bin/python and $PYTHON)" >&2
|
||||
echo "ERROR: no Python found" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log() { printf '\n==> %s\n' "$*"; }
|
||||
die() { echo "ERROR: $*" >&2; exit 1; }
|
||||
need_file() { [[ -f "$1" ]] || die "missing $1"; }
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# TLS bootstrap — same corp CA path the FastAPI app uses
|
||||
# ---------------------------------------------------------------------------
|
||||
setup_ssl() {
|
||||
export USE_CORP_PROXY
|
||||
|
||||
# Prefer explicit env, then repo / home corporate bundle, then certifi.
|
||||
if [[ -z "${SSL_CERT_FILE:-}" ]]; then
|
||||
if [[ -f "$ROOT/combined-ca-bundle.pem" ]]; then
|
||||
export SSL_CERT_FILE="$ROOT/combined-ca-bundle.pem"
|
||||
@@ -101,191 +72,68 @@ setup_ssl() {
|
||||
export SSL_CERT_FILE="$HOME/combined-ca-bundle.pem"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ -n "${SSL_CERT_FILE:-}" && -f "$SSL_CERT_FILE" ]]; then
|
||||
export REQUESTS_CA_BUNDLE="$SSL_CERT_FILE"
|
||||
export CURL_CA_BUNDLE="$SSL_CERT_FILE"
|
||||
export REQUESTS_CA_BUNDLE="$SSL_CERT_FILE" CURL_CA_BUNDLE="$SSL_CERT_FILE"
|
||||
log "SSL CA bundle: $SSL_CERT_FILE"
|
||||
else
|
||||
# Fall back to certifi if installed
|
||||
local certifi_path
|
||||
certifi_path="$("$PYTHON" -c 'import certifi; print(certifi.where())' 2>/dev/null || true)"
|
||||
if [[ -n "$certifi_path" && -f "$certifi_path" ]]; then
|
||||
export SSL_CERT_FILE="$certifi_path"
|
||||
export REQUESTS_CA_BUNDLE="$certifi_path"
|
||||
export CURL_CA_BUNDLE="$certifi_path"
|
||||
export SSL_CERT_FILE="$certifi_path" REQUESTS_CA_BUNDLE="$certifi_path" CURL_CA_BUNDLE="$certifi_path"
|
||||
log "SSL CA bundle (certifi): $SSL_CERT_FILE"
|
||||
else
|
||||
log "WARNING: no CA bundle found — SSL may fail on corp networks"
|
||||
log " Copy combined-ca-bundle.pem to $ROOT/ or \$HOME/"
|
||||
log " Or: export SSL_CERT_FILE=/path/to/combined-ca-bundle.pem"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ "$USE_CORP_PROXY" == "1" ]]; then
|
||||
export HTTP_PROXY="${HTTP_PROXY:-http://aproxy.corproot.net:8080}"
|
||||
export HTTPS_PROXY="${HTTPS_PROXY:-http://aproxy.corproot.net:8080}"
|
||||
export NO_PROXY="${NO_PROXY:-corproot.net,sharedtcs.net,127.0.0.1,localhost,bix.swisscom.com,swisscom.com}"
|
||||
export NO_PROXY="${NO_PROXY:-corproot.net,sharedtcs.net,127.0.0.1,localhost}"
|
||||
export http_proxy="$HTTP_PROXY" https_proxy="$HTTPS_PROXY" no_proxy="$NO_PROXY"
|
||||
log "Corp proxy enabled: $HTTPS_PROXY"
|
||||
fi
|
||||
|
||||
# Ensure Python process sees the same bootstrap (patches ssl for alpaca-py).
|
||||
export PYTHONPATH="${ROOT}${PYTHONPATH:+:$PYTHONPATH}"
|
||||
}
|
||||
|
||||
ssl_check() {
|
||||
setup_ssl
|
||||
log "SSL diagnostic"
|
||||
"$PYTHON" - <<'PY'
|
||||
from app.ssl_bootstrap import bootstrap_ssl, ssl_status
|
||||
import json
|
||||
import urllib.request
|
||||
|
||||
ca = bootstrap_ssl()
|
||||
import json, urllib.request
|
||||
print(json.dumps(ssl_status(), indent=2))
|
||||
print("bootstrap_ssl ->", ca)
|
||||
urls = [
|
||||
print("bootstrap ->", bootstrap_ssl())
|
||||
for url in (
|
||||
"https://data.alpaca.markets/v2/stocks/SPY/bars?timeframe=1Day&limit=1",
|
||||
"https://financialmodelingprep.com/stable/profile?symbol=AAPL",
|
||||
"https://www.alphavantage.co/query?function=TIME_SERIES_DAILY&symbol=IBM",
|
||||
]
|
||||
for url in urls:
|
||||
):
|
||||
try:
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "signal-platform-ssl-check"})
|
||||
req = urllib.request.Request(url, headers={"User-Agent": "ssl-check"})
|
||||
with urllib.request.urlopen(req, timeout=20) as resp:
|
||||
print(f"OK {resp.status} {url[:60]}...")
|
||||
print(f"OK {resp.status} {url[:60]}")
|
||||
except Exception as exc:
|
||||
print(f"FAIL {type(exc).__name__}: {exc}")
|
||||
print(f" {url[:80]}")
|
||||
PY
|
||||
}
|
||||
|
||||
need_file() {
|
||||
[[ -f "$1" ]] || die "missing $1"
|
||||
}
|
||||
|
||||
require_prod() {
|
||||
need_file "$PROD_SNAP"
|
||||
}
|
||||
|
||||
run_earnings() {
|
||||
require_prod
|
||||
log "Earnings backfill (FMP free tier ~${FMP_LIMIT}/day; resume-safe)"
|
||||
"$PYTHON" scripts/backfill_earnings_events.py \
|
||||
--snapshot "$PROD_SNAP" \
|
||||
--provider fmp \
|
||||
--force-symbol \
|
||||
--limit "$FMP_LIMIT" \
|
||||
--sleep "$FMP_SLEEP"
|
||||
|
||||
log "Earnings research 2a+2b (report only; no filters shipped)"
|
||||
"$PYTHON" scripts/run_earnings_research.py \
|
||||
--snapshot "$PROD_SNAP" \
|
||||
--workers "$WORKERS" \
|
||||
--allow-spawn
|
||||
}
|
||||
|
||||
run_coverage() {
|
||||
require_prod
|
||||
log "Coverage probe (bars per year) on $PROD_SNAP"
|
||||
"$PYTHON" scripts/run_history_depth_research.py \
|
||||
--phase coverage \
|
||||
--snapshot "$PROD_SNAP"
|
||||
}
|
||||
|
||||
run_rebuild() {
|
||||
require_prod
|
||||
log "Deep rebuild $PROD_SNAP → $RESEARCH_SNAP (history-days=$HISTORY_DAYS)"
|
||||
log "SURVIVORSHIP: today's constituents backfilled — relative IC only, not levels"
|
||||
"$PYTHON" scripts/extend_snapshot_universe.py \
|
||||
--source "$PROD_SNAP" \
|
||||
--output "$RESEARCH_SNAP" \
|
||||
--force-copy \
|
||||
--history-days "$HISTORY_DAYS" \
|
||||
--min-bars "$MIN_BARS" \
|
||||
--sleep "$ALPACA_SLEEP"
|
||||
|
||||
log "Refresh SPY + 11 sector ETFs on research snapshot"
|
||||
"$PYTHON" scripts/fetch_sector_etfs_to_snapshot.py \
|
||||
--snapshot "$RESEARCH_SNAP" \
|
||||
--history-days "$HISTORY_DAYS"
|
||||
|
||||
log "Also deepen sector ETFs on prod snapshot (for local A/B parity)"
|
||||
"$PYTHON" scripts/fetch_sector_etfs_to_snapshot.py \
|
||||
--snapshot "$PROD_SNAP" \
|
||||
--history-days "$HISTORY_DAYS"
|
||||
}
|
||||
|
||||
run_harness() {
|
||||
need_file "$RESEARCH_SNAP"
|
||||
log "Race-guard + full signal harness + era split on $RESEARCH_SNAP"
|
||||
"$PYTHON" scripts/run_history_depth_research.py \
|
||||
--phase harness \
|
||||
--snapshot "$RESEARCH_SNAP" \
|
||||
--workers "$WORKERS" \
|
||||
--allow-spawn
|
||||
}
|
||||
|
||||
run_sector_resid_deep() {
|
||||
need_file "$RESEARCH_SNAP"
|
||||
log "Sector-resid deep test: deepen shallow symbols + ONE liquid-1500 masked grade"
|
||||
log "Pre-registered PASS/FAIL only — thread ends after this run"
|
||||
"$PYTHON" scripts/run_sector_resid_deep_test.py \
|
||||
--snapshot "$RESEARCH_SNAP" \
|
||||
--history-days "$HISTORY_DAYS" \
|
||||
--sleep "$ALPACA_SLEEP" \
|
||||
--workers "$WORKERS" \
|
||||
--allow-spawn
|
||||
}
|
||||
|
||||
run_prod_book_matrix() {
|
||||
need_file "$RESEARCH_SNAP"
|
||||
log "Production book × universe × horizon (4 arms, strategy unchanged)"
|
||||
"$PYTHON" scripts/run_prod_book_universe_matrix.py \
|
||||
--snapshot "$RESEARCH_SNAP" \
|
||||
--workers "$WORKERS" \
|
||||
--allow-spawn \
|
||||
--candidate-cache reports/.cache/prod-book-universe-cands.pkl
|
||||
}
|
||||
|
||||
log "cwd=$ROOT python=$PYTHON phase=$PHASE workers=$WORKERS"
|
||||
setup_ssl
|
||||
|
||||
case "$PHASE" in
|
||||
ssl)
|
||||
ssl_check
|
||||
;;
|
||||
sector_resid_deep)
|
||||
run_sector_resid_deep
|
||||
ssl) ssl_check ;;
|
||||
earnings)
|
||||
need_file "$PROD_SNAP"
|
||||
log "Earnings backfill + research (parked experiment)"
|
||||
"$PYTHON" scripts/backfill_earnings_events.py \
|
||||
--snapshot "$PROD_SNAP" --provider fmp --force-symbol \
|
||||
--limit "$FMP_LIMIT" --sleep "$FMP_SLEEP"
|
||||
"$PYTHON" scripts/run_earnings_research.py \
|
||||
--snapshot "$PROD_SNAP" --workers "$WORKERS" --allow-spawn
|
||||
;;
|
||||
prod_book)
|
||||
run_prod_book_matrix
|
||||
;;
|
||||
coverage)
|
||||
run_coverage
|
||||
;;
|
||||
earnings)
|
||||
run_earnings
|
||||
;;
|
||||
harness)
|
||||
run_harness
|
||||
;;
|
||||
depth)
|
||||
run_coverage
|
||||
run_rebuild
|
||||
run_harness
|
||||
;;
|
||||
all)
|
||||
run_earnings
|
||||
run_coverage
|
||||
run_rebuild
|
||||
run_harness
|
||||
;;
|
||||
*)
|
||||
die "unknown phase $PHASE"
|
||||
need_file "$RESEARCH_SNAP"
|
||||
log "Production book universe × horizon matrix"
|
||||
"$PYTHON" scripts/run_prod_book_universe_matrix.py \
|
||||
--snapshot "$RESEARCH_SNAP" --workers "$WORKERS" --allow-spawn \
|
||||
--candidate-cache reports/.cache/prod-book-universe-cands.pkl
|
||||
;;
|
||||
*) die "unknown phase $PHASE" ;;
|
||||
esac
|
||||
|
||||
log "Done. Check reports/ and docs/research/history-depth-extension.md"
|
||||
log "Commit reports on this machine if they look good, or copy them back to Windows."
|
||||
log "Done."
|
||||
|
||||
@@ -100,75 +100,6 @@ def test_residual_momentum_removes_market_beta_but_keeps_specific_drift():
|
||||
assert drift["mom_12_1_resid"] > pure["mom_12_1_resid"] + 0.12
|
||||
|
||||
|
||||
def test_sector_residual_momentum_two_factor():
|
||||
"""Pure market+sector beta stock → sector resid ~0; idiosyncratic drift kept."""
|
||||
dates, pure_beta, highs, benchmark = _signal_test_series(extra_return=0.0)
|
||||
# Sector ETF = leveraged market (collinear-ish but not identical).
|
||||
sector = {d: benchmark[d] * 1.02 + 0.5 for d in dates}
|
||||
# Stock with pure exposure to market + sector, no alpha.
|
||||
closes = [100.0]
|
||||
for i in range(1, len(dates)):
|
||||
m_prev = benchmark[dates[i - 1]]
|
||||
m_cur = benchmark[dates[i]]
|
||||
s_prev = sector[dates[i - 1]]
|
||||
s_cur = sector[dates[i]]
|
||||
m_ret = m_cur / m_prev - 1.0
|
||||
s_ret = s_cur / s_prev - 1.0
|
||||
closes.append(closes[-1] * (1.0 + 0.7 * m_ret + 0.5 * s_ret))
|
||||
highs_p = [c * 1.01 for c in closes]
|
||||
|
||||
pure = bt._signal_values(
|
||||
dates, closes, highs_p, 260, benchmark, sector_etf_closes=sector
|
||||
)
|
||||
assert "mom_12_1_sector_resid" in pure
|
||||
assert pure["mom_12_1_sector_resid"] == pytest.approx(0.0, abs=0.05)
|
||||
|
||||
# Add idiosyncratic drift — sector residual should keep it.
|
||||
drift_closes = [100.0]
|
||||
for i in range(1, len(dates)):
|
||||
m_prev = benchmark[dates[i - 1]]
|
||||
m_cur = benchmark[dates[i]]
|
||||
s_prev = sector[dates[i - 1]]
|
||||
s_cur = sector[dates[i]]
|
||||
m_ret = m_cur / m_prev - 1.0
|
||||
s_ret = s_cur / s_prev - 1.0
|
||||
drift_closes.append(
|
||||
drift_closes[-1] * (1.0 + 0.7 * m_ret + 0.5 * s_ret + 0.0008)
|
||||
)
|
||||
drift_highs = [c * 1.01 for c in drift_closes]
|
||||
drift = bt._signal_values(
|
||||
dates, drift_closes, drift_highs, 260, benchmark, sector_etf_closes=sector
|
||||
)
|
||||
assert drift["mom_12_1_sector_resid"] > pure["mom_12_1_sector_resid"] + 0.10
|
||||
|
||||
|
||||
def test_inject_sector_demeaned_momentum():
|
||||
collected = {
|
||||
"mom_12_1": {
|
||||
(2024, 1): [
|
||||
{"val": 0.20, "fwd": 0.01, "symbol": "AAA"},
|
||||
{"val": 0.10, "fwd": 0.02, "symbol": "BBB"},
|
||||
{"val": 0.40, "fwd": -0.01, "symbol": "CCC"},
|
||||
{"val": 0.00, "fwd": 0.03, "symbol": "DDD"},
|
||||
]
|
||||
}
|
||||
}
|
||||
symbol_to_sector = {
|
||||
"AAA": "Information Technology",
|
||||
"BBB": "Information Technology",
|
||||
"CCC": "Energy",
|
||||
"DDD": "Energy",
|
||||
}
|
||||
bt._inject_sector_demeaned_momentum(collected, symbol_to_sector)
|
||||
dem = collected["mom_12_1_sector_demeaned"][(2024, 1)]
|
||||
by_sym = {r["symbol"]: r["val"] for r in dem}
|
||||
# IT mean = 0.15 → AAA +0.05, BBB -0.05; Energy mean = 0.20 → CCC +0.20, DDD -0.20
|
||||
assert by_sym["AAA"] == pytest.approx(0.05)
|
||||
assert by_sym["BBB"] == pytest.approx(-0.05)
|
||||
assert by_sym["CCC"] == pytest.approx(0.20)
|
||||
assert by_sym["DDD"] == pytest.approx(-0.20)
|
||||
|
||||
|
||||
def test_assigns_raw_and_residual_percentiles_independently():
|
||||
cands = [
|
||||
{"iso_week": (2026, 1), "momentum": 0.10, "residual_momentum": 0.30},
|
||||
|
||||
Reference in New Issue
Block a user