dennisthiessen/signal-platform
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: sector-resid sanity grades against Alpaca feed floor, not calendar 5000d

Browse Source 
SANITY-FAIL report showed megacaps/ETFs already at empirical 2016-01-04 floor
(2649 bars) after deepen; check wrongly required ~2013. Pass when megacaps leave
the old 2021 two-tier floor and match SPY; XLC listing exception retained.
This commit is contained in:
dennisthiessen
2026-07-19 11:22:06 +02:00
parent f3d1312a69
commit 003f20de19
1 changed files with 76 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/run_sector_resid_deep_test.py
+75 -26
View File
@@ -299,22 +299,14 @@ async def _deepen_sector_etfs(
def _sanity_check(snapshot: Path, *, history_days: int) -> dict[str, Any]:
depths = {r["symbol"]: r for r in _symbol_depth(snapshot)}
end = date.today()
target_start = end - timedelta(days=history_days)
# Allow ~1 year slack for IPO/listing limits (not a hard fail for all names).
megacap_deadline = target_start + timedelta(days=400)
"""Pass if megacaps + sector ETFs sit at the *empirical feed floor*, not calendar 5000d.
megacap = {}
ok_mega = True
for sym in SANITY_MEGACAPS:
row = depths.get(sym)
megacap[sym] = row
if row is None or not row.get("min_date"):
ok_mega = False
continue
if date.fromisoformat(row["min_date"]) > megacap_deadline:
ok_mega = False
Alpaca daily history for this stack bottoms out around 2016-01-04 (~2649 bars)
even when history_days=5000 is requested. That is feed coverage, not a two-tier
snapshot bug. Fail only if megacaps are still stuck near the old ~2021 prod floor
or if sector ETFs are missing / shorter than the SPY series (except XLC listing).
"""
depths = {r["symbol"]: r for r in _symbol_depth(snapshot)}
engine = create_engine(
f"sqlite:///{snapshot.resolve().as_posix()}",
@@ -322,6 +314,12 @@ def _sanity_check(snapshot: Path, *, history_days: int) -> dict[str, Any]:
)
try:
with engine.connect() as conn:
spy_row = conn.execute(
text(
"SELECT COUNT(*), MIN(date), MAX(date) FROM benchmark_prices "
"WHERE symbol = 'SPY'"
)
).fetchone()
etf_rows = conn.execute(
text(
"SELECT symbol, COUNT(*), MIN(date), MAX(date) "
@@ -333,45 +331,96 @@ def _sanity_check(snapshot: Path, *, history_days: int) -> dict[str, Any]:
finally:
engine.dispose()
spy_n, spy_min, spy_max = spy_row if spy_row else (0, None, None)
feed_floor = (
date.fromisoformat(str(spy_min)[:10])
if spy_min
else date(2016, 1, 4)
)
# Megacaps must match the feed floor within a few sessions (not calendar-5000).
megacap_slack_days = 10
# Old two-tier defect left prod names at ~2021-06; anything still after this fails.
old_shallow_floor = date(2020, 1, 1)
megacap = {}
ok_mega = True
mega_reasons: list[str] = []
for sym in SANITY_MEGACAPS:
row = depths.get(sym)
megacap[sym] = row
if row is None or not row.get("min_date"):
ok_mega = False
mega_reasons.append(f"{sym}: missing")
continue
d0 = date.fromisoformat(row["min_date"])
if d0 > old_shallow_floor:
ok_mega = False
mega_reasons.append(
f"{sym}: min_date={d0} still after {old_shallow_floor} (two-tier unrepaired)"
)
elif d0 > feed_floor + timedelta(days=megacap_slack_days):
ok_mega = False
mega_reasons.append(
f"{sym}: min_date={d0} later than SPY feed floor {feed_floor}"
)
etf_info = {
s: {"n": n, "min": d0, "max": d1} for s, n, d0, d1 in etf_rows
}
deep_etfs = 0
etf_reasons: list[str] = []
for sym in SECTOR_ETFS:
info = etf_info.get(sym)
if not info or not info["min"]:
etf_reasons.append(f"{sym}: missing")
continue
# XLC lists mid-2018 — accept that floor.
floor = date(2018, 6, 1) if sym == "XLC" else megacap_deadline
if date.fromisoformat(str(info["min"])[:10]) <= floor + timedelta(days=60):
d0 = date.fromisoformat(str(info["min"])[:10])
if sym == "XLC":
# Listed 2018-06-18/19.
if d0 <= date(2018, 7, 15):
deep_etfs += 1
elif date.fromisoformat(str(info["min"])[:10]) <= date(2019, 1, 1):
# moderately deep still counts for non-XLC if near 2018-19
if sym != "XLC":
else:
etf_reasons.append(f"XLC: min_date={d0} later than listing floor")
else:
if d0 <= feed_floor + timedelta(days=megacap_slack_days):
deep_etfs += 1
else:
etf_reasons.append(
f"{sym}: min_date={d0} later than SPY feed floor {feed_floor}"
)
# Require 10 of 11 sector ETFs deep (XLC may be the exception with mid-2018 start).
ok_etf = deep_etfs >= 10
# Shallow residual after deepen: few names should still start after 2020.
still_shallow, _ = _derive_shallow(list(depths.values()), lag_days=400)
# After fix, shallow set should shrink dramatically vs ~500.
note_xlc = (
"XLC lists mid-2018 → Communication Services residual coverage from ~mid-2019."
)
note_feed = (
f"Empirical Alpaca floor observed via SPY: {feed_floor.isoformat()} "
f"(n={spy_n}). Calendar history_days={history_days} is a request cap, not a "
"guarantee — sanity grades against the feed floor, not 5000 calendar days."
)
passed = bool(ok_mega and ok_etf)
return {
"passed": passed,
"megacap": megacap,
"megacap_ok": ok_mega,
"megacap_deadline": megacap_deadline.isoformat(),
"megacap_reasons": mega_reasons,
"feed_floor": feed_floor.isoformat(),
"spy_benchmark": {"n": spy_n, "min": spy_min, "max": spy_max},
"old_shallow_floor": old_shallow_floor.isoformat(),
"sector_etfs": etf_info,
"sector_etfs_deep_count": deep_etfs,
"sector_etfs_ok": ok_etf,
"sector_etf_reasons": etf_reasons,
"still_shallow_count": len(still_shallow),
"still_shallow_sample": still_shallow[:20],
"still_shallow_note": (
"Remaining 'shallow' names are mostly post-2017 IPOs/listings — expected, "
"not a two-tier defect."
),
"xlc_note": note_xlc,
"feed_note": note_feed,
"target_history_days": history_days,
}