From 003f20de199f519441778e90809a20768b0e37b3 Mon Sep 17 00:00:00 2001 From: Dennis Thiessen Date: Sun, 19 Jul 2026 11:22:06 +0200 Subject: [PATCH] fix: sector-resid sanity grades against Alpaca feed floor, not calendar 5000d SANITY-FAIL report showed megacaps/ETFs already at empirical 2016-01-04 floor (2649 bars) after deepen; check wrongly required ~2013. Pass when megacaps leave the old 2021 two-tier floor and match SPY; XLC listing exception retained. --- scripts/run_sector_resid_deep_test.py | 103 +++++++++++++++++++------- 1 file changed, 76 insertions(+), 27 deletions(-) diff --git a/scripts/run_sector_resid_deep_test.py b/scripts/run_sector_resid_deep_test.py index 54de8bf..a5cbfff 100644 --- a/scripts/run_sector_resid_deep_test.py +++ b/scripts/run_sector_resid_deep_test.py @@ -299,22 +299,14 @@ async def _deepen_sector_etfs( def _sanity_check(snapshot: Path, *, history_days: int) -> dict[str, Any]: - depths = {r["symbol"]: r for r in _symbol_depth(snapshot)} - end = date.today() - target_start = end - timedelta(days=history_days) - # Allow ~1 year slack for IPO/listing limits (not a hard fail for all names). - megacap_deadline = target_start + timedelta(days=400) + """Pass if megacaps + sector ETFs sit at the *empirical feed floor*, not calendar 5000d. - megacap = {} - ok_mega = True - for sym in SANITY_MEGACAPS: - row = depths.get(sym) - megacap[sym] = row - if row is None or not row.get("min_date"): - ok_mega = False - continue - if date.fromisoformat(row["min_date"]) > megacap_deadline: - ok_mega = False + Alpaca daily history for this stack bottoms out around 2016-01-04 (~2649 bars) + even when history_days=5000 is requested. That is feed coverage, not a two-tier + snapshot bug. Fail only if megacaps are still stuck near the old ~2021 prod floor + or if sector ETFs are missing / shorter than the SPY series (except XLC listing). + """ + depths = {r["symbol"]: r for r in _symbol_depth(snapshot)} engine = create_engine( f"sqlite:///{snapshot.resolve().as_posix()}", @@ -322,6 +314,12 @@ def _sanity_check(snapshot: Path, *, history_days: int) -> dict[str, Any]: ) try: with engine.connect() as conn: + spy_row = conn.execute( + text( + "SELECT COUNT(*), MIN(date), MAX(date) FROM benchmark_prices " + "WHERE symbol = 'SPY'" + ) + ).fetchone() etf_rows = conn.execute( text( "SELECT symbol, COUNT(*), MIN(date), MAX(date) " @@ -333,45 +331,96 @@ def _sanity_check(snapshot: Path, *, history_days: int) -> dict[str, Any]: finally: engine.dispose() + spy_n, spy_min, spy_max = spy_row if spy_row else (0, None, None) + feed_floor = ( + date.fromisoformat(str(spy_min)[:10]) + if spy_min + else date(2016, 1, 4) + ) + # Megacaps must match the feed floor within a few sessions (not calendar-5000). + megacap_slack_days = 10 + # Old two-tier defect left prod names at ~2021-06; anything still after this fails. + old_shallow_floor = date(2020, 1, 1) + + megacap = {} + ok_mega = True + mega_reasons: list[str] = [] + for sym in SANITY_MEGACAPS: + row = depths.get(sym) + megacap[sym] = row + if row is None or not row.get("min_date"): + ok_mega = False + mega_reasons.append(f"{sym}: missing") + continue + d0 = date.fromisoformat(row["min_date"]) + if d0 > old_shallow_floor: + ok_mega = False + mega_reasons.append( + f"{sym}: min_date={d0} still after {old_shallow_floor} (two-tier unrepaired)" + ) + elif d0 > feed_floor + timedelta(days=megacap_slack_days): + ok_mega = False + mega_reasons.append( + f"{sym}: min_date={d0} later than SPY feed floor {feed_floor}" + ) + etf_info = { s: {"n": n, "min": d0, "max": d1} for s, n, d0, d1 in etf_rows } deep_etfs = 0 + etf_reasons: list[str] = [] for sym in SECTOR_ETFS: info = etf_info.get(sym) if not info or not info["min"]: + etf_reasons.append(f"{sym}: missing") continue - # XLC lists mid-2018 — accept that floor. - floor = date(2018, 6, 1) if sym == "XLC" else megacap_deadline - if date.fromisoformat(str(info["min"])[:10]) <= floor + timedelta(days=60): - deep_etfs += 1 - elif date.fromisoformat(str(info["min"])[:10]) <= date(2019, 1, 1): - # moderately deep still counts for non-XLC if near 2018-19 - if sym != "XLC": + d0 = date.fromisoformat(str(info["min"])[:10]) + if sym == "XLC": + # Listed 2018-06-18/19. + if d0 <= date(2018, 7, 15): deep_etfs += 1 + else: + etf_reasons.append(f"XLC: min_date={d0} later than listing floor") + else: + if d0 <= feed_floor + timedelta(days=megacap_slack_days): + deep_etfs += 1 + else: + etf_reasons.append( + f"{sym}: min_date={d0} later than SPY feed floor {feed_floor}" + ) - # Require 10 of 11 sector ETFs deep (XLC may be the exception with mid-2018 start). ok_etf = deep_etfs >= 10 - - # Shallow residual after deepen: few names should still start after 2020. still_shallow, _ = _derive_shallow(list(depths.values()), lag_days=400) - # After fix, shallow set should shrink dramatically vs ~500. note_xlc = ( "XLC lists mid-2018 → Communication Services residual coverage from ~mid-2019." ) + note_feed = ( + f"Empirical Alpaca floor observed via SPY: {feed_floor.isoformat()} " + f"(n={spy_n}). Calendar history_days={history_days} is a request cap, not a " + "guarantee — sanity grades against the feed floor, not 5000 calendar days." + ) passed = bool(ok_mega and ok_etf) return { "passed": passed, "megacap": megacap, "megacap_ok": ok_mega, - "megacap_deadline": megacap_deadline.isoformat(), + "megacap_reasons": mega_reasons, + "feed_floor": feed_floor.isoformat(), + "spy_benchmark": {"n": spy_n, "min": spy_min, "max": spy_max}, + "old_shallow_floor": old_shallow_floor.isoformat(), "sector_etfs": etf_info, "sector_etfs_deep_count": deep_etfs, "sector_etfs_ok": ok_etf, + "sector_etf_reasons": etf_reasons, "still_shallow_count": len(still_shallow), "still_shallow_sample": still_shallow[:20], + "still_shallow_note": ( + "Remaining 'shallow' names are mostly post-2017 IPOs/listings — expected, " + "not a two-tier defect." + ), "xlc_note": note_xlc, + "feed_note": note_feed, "target_history_days": history_days, }