fix: sector-resid sanity grades against Alpaca feed floor, not calendar 5000d

SANITY-FAIL report showed megacaps/ETFs already at empirical 2016-01-04 floor
(2649 bars) after deepen; check wrongly required ~2013. Pass when megacaps leave
the old 2021 two-tier floor and match SPY; XLC listing exception retained.

scripts/run_sector_resid_deep_test.py

@@ -299,22 +299,14 @@ async def _deepen_sector_etfs(
 
 
 def _sanity_check(snapshot: Path, *, history_days: int) -> dict[str, Any]:
-    depths = {r["symbol"]: r for r in _symbol_depth(snapshot)}
+    """Pass if megacaps + sector ETFs sit at the *empirical feed floor*, not calendar 5000d.
-    end = date.today()
-    target_start = end - timedelta(days=history_days)
-    # Allow ~1 year slack for IPO/listing limits (not a hard fail for all names).
-    megacap_deadline = target_start + timedelta(days=400)
 
-    megacap = {}
+    Alpaca daily history for this stack bottoms out around 2016-01-04 (~2649 bars)
-    ok_mega = True
+    even when history_days=5000 is requested. That is feed coverage, not a two-tier
-    for sym in SANITY_MEGACAPS:
+    snapshot bug. Fail only if megacaps are still stuck near the old ~2021 prod floor
-        row = depths.get(sym)
+    or if sector ETFs are missing / shorter than the SPY series (except XLC listing).
-        megacap[sym] = row
+    """
-        if row is None or not row.get("min_date"):
+    depths = {r["symbol"]: r for r in _symbol_depth(snapshot)}
-            ok_mega = False
-            continue
-        if date.fromisoformat(row["min_date"]) > megacap_deadline:
-            ok_mega = False
 
     engine = create_engine(
         f"sqlite:///{snapshot.resolve().as_posix()}",
@@ -322,6 +314,12 @@ def _sanity_check(snapshot: Path, *, history_days: int) -> dict[str, Any]:
     )
     try:
         with engine.connect() as conn:
+            spy_row = conn.execute(
+                text(
+                    "SELECT COUNT(*), MIN(date), MAX(date) FROM benchmark_prices "
+                    "WHERE symbol = 'SPY'"
+                )
+            ).fetchone()
             etf_rows = conn.execute(
                 text(
                     "SELECT symbol, COUNT(*), MIN(date), MAX(date) "
@@ -333,45 +331,96 @@ def _sanity_check(snapshot: Path, *, history_days: int) -> dict[str, Any]:
     finally:
         engine.dispose()
 
+    spy_n, spy_min, spy_max = spy_row if spy_row else (0, None, None)
+    feed_floor = (
+        date.fromisoformat(str(spy_min)[:10])
+        if spy_min
+        else date(2016, 1, 4)
+    )
+    # Megacaps must match the feed floor within a few sessions (not calendar-5000).
+    megacap_slack_days = 10
+    # Old two-tier defect left prod names at ~2021-06; anything still after this fails.
+    old_shallow_floor = date(2020, 1, 1)
+
+    megacap = {}
+    ok_mega = True
+    mega_reasons: list[str] = []
+    for sym in SANITY_MEGACAPS:
+        row = depths.get(sym)
+        megacap[sym] = row
+        if row is None or not row.get("min_date"):
+            ok_mega = False
+            mega_reasons.append(f"{sym}: missing")
+            continue
+        d0 = date.fromisoformat(row["min_date"])
+        if d0 > old_shallow_floor:
+            ok_mega = False
+            mega_reasons.append(
+                f"{sym}: min_date={d0} still after {old_shallow_floor} (two-tier unrepaired)"
+            )
+        elif d0 > feed_floor + timedelta(days=megacap_slack_days):
+            ok_mega = False
+            mega_reasons.append(
+                f"{sym}: min_date={d0} later than SPY feed floor {feed_floor}"
+            )
+
     etf_info = {
         s: {"n": n, "min": d0, "max": d1} for s, n, d0, d1 in etf_rows
     }
     deep_etfs = 0
+    etf_reasons: list[str] = []
     for sym in SECTOR_ETFS:
         info = etf_info.get(sym)
         if not info or not info["min"]:
+            etf_reasons.append(f"{sym}: missing")
             continue
-        # XLC lists mid-2018 — accept that floor.
+        d0 = date.fromisoformat(str(info["min"])[:10])
-        floor = date(2018, 6, 1) if sym == "XLC" else megacap_deadline
+        if sym == "XLC":
-        if date.fromisoformat(str(info["min"])[:10]) <= floor + timedelta(days=60):
+            # Listed 2018-06-18/19.
-            deep_etfs += 1
+            if d0 <= date(2018, 7, 15):
-        elif date.fromisoformat(str(info["min"])[:10]) <= date(2019, 1, 1):
-            # moderately deep still counts for non-XLC if near 2018-19
-            if sym != "XLC":
                 deep_etfs += 1
+            else:
+                etf_reasons.append(f"XLC: min_date={d0} later than listing floor")
+        else:
+            if d0 <= feed_floor + timedelta(days=megacap_slack_days):
+                deep_etfs += 1
+            else:
+                etf_reasons.append(
+                    f"{sym}: min_date={d0} later than SPY feed floor {feed_floor}"
+                )
 
-    # Require 10 of 11 sector ETFs deep (XLC may be the exception with mid-2018 start).
     ok_etf = deep_etfs >= 10
-
-    # Shallow residual after deepen: few names should still start after 2020.
     still_shallow, _ = _derive_shallow(list(depths.values()), lag_days=400)
-    # After fix, shallow set should shrink dramatically vs ~500.
     note_xlc = (
         "XLC lists mid-2018 → Communication Services residual coverage from ~mid-2019."
     )
+    note_feed = (
+        f"Empirical Alpaca floor observed via SPY: {feed_floor.isoformat()} "
+        f"(n={spy_n}). Calendar history_days={history_days} is a request cap, not a "
+        "guarantee — sanity grades against the feed floor, not 5000 calendar days."
+    )
 
     passed = bool(ok_mega and ok_etf)
     return {
         "passed": passed,
         "megacap": megacap,
         "megacap_ok": ok_mega,
-        "megacap_deadline": megacap_deadline.isoformat(),
+        "megacap_reasons": mega_reasons,
+        "feed_floor": feed_floor.isoformat(),
+        "spy_benchmark": {"n": spy_n, "min": spy_min, "max": spy_max},
+        "old_shallow_floor": old_shallow_floor.isoformat(),
         "sector_etfs": etf_info,
         "sector_etfs_deep_count": deep_etfs,
         "sector_etfs_ok": ok_etf,
+        "sector_etf_reasons": etf_reasons,
         "still_shallow_count": len(still_shallow),
         "still_shallow_sample": still_shallow[:20],
+        "still_shallow_note": (
+            "Remaining 'shallow' names are mostly post-2017 IPOs/listings — expected, "
+            "not a two-tier defect."
+        ),
         "xlc_note": note_xlc,
+        "feed_note": note_feed,
         "target_history_days": history_days,
     }