Event study: report its own statistical limits

The v3 cutover run scored 2/4 corrections warned against v2's 3/4, which reads
like a regression and is not one. Only 4 of the 11 detected corrections fall in
the holdout, so recall is one event from a different headline -- and the event
that flips is decided by threshold placement, not by what the score saw. "v3
without the credit sensor" catches 2025-02-21 at a *higher* threshold (35.5)
than shipped v3 misses it at (32.3), because the alarm rule needs a rising edge
and a lower threshold can fire outside the horizon then never reset below.

Two caveats are now computed and surfaced rather than left for the reader to
infer:

- Holdout event count against MIN_EVENTS_FOR_CONFIDENCE. The summary sentence
  states how many of the detected corrections actually fall in the test period.
- Warning-sensor coverage across the split. The score renormalises over what is
  available, so a training window predating a sensor's history freezes the
  threshold on a different construct than the holdout is measured against. At
  the cutover that is 39% of training sessions with all three sensors versus
  100% of the test period, credit history beginning 2023-07-25.

Restricting the threshold to sensor-matched training sessions was tested and
rejected: those sessions are a calm recent stretch, so the threshold falls from
32.3 to 22.5 and false alarms rise from 3.3 to 8.6/yr. It swaps a coverage bias
for a regime-selection bias. The report states its limits instead.

_warning_series now returns per-session sensor counts alongside the scores.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

app/services/event_study_service.py

@@ -28,6 +28,10 @@ DRAWDOWN_LOOKBACK = 252
 HORIZON_DAYS = 20
 WARN_PERCENTILE = 80.0
 TRAIN_FRACTION = 0.70
+# Below this many holdout corrections, recall is one event away from a very
+# different headline and should not be read as a property of the score.
+MIN_EVENTS_FOR_CONFIDENCE = 8
+SENSOR_MISMATCH_TOLERANCE = 0.10
 
 
 def _median(values: list[float]) -> float | None:
@@ -150,18 +154,23 @@ def _warning_series(
     dates: list[date],
     config: dict,
     oas_series: rms.Series | None = None,
-) -> dict[date, float]:
-    """Warning score per session, from the monitor's own sensor definitions.
+) -> tuple[dict[date, float], dict[date, int]]:
+    """Warning score per session plus how many sensors backed it.
 
     v2 re-derived this by hand from ``WARNING_WEIGHTS`` and so would have kept
     measuring the old construct after a scoring change.  Since v3 dropped
     fundamentals from the score, this is now exactly the live Warning score
     rather than a technical-only approximation of it.
+
+    The sensor count matters because the score renormalises over whatever is
+    available: a session backed by two sensors is not drawn from the same
+    distribution as one backed by three, and the frozen threshold assumes it is.
     """
     tickers = config["tickers"]
     smh_full = prices.get(tickers["leaders"][0], [])
     spy_full = prices.get(tickers["market"], [])
     out: dict[date, float] = {}
+    backing: dict[date, int] = {}
     for session in dates:
         sensors = rms.warning_sensor_scores(
             breadth_divergence.get(session),
@@ -172,7 +181,44 @@ def _warning_series(
         score = rms.score_warning_sensors(sensors)
         if score is not None:
             out[session] = round(score, 2)
-    return out
+            backing[session] = sum(1 for value in sensors.values() if value is not None)
+    return out, backing
+
+
+def _reliability(
+    dates: list[date],
+    split: int,
+    backing: dict[date, int],
+    events_detected: int,
+    events_in_holdout: int,
+) -> dict:
+    """How far the headline metrics can actually be trusted.
+
+    Two things repeatedly invite over-reading this report:
+
+    * The holdout carries only the corrections that fall in the last 30% of the
+      sample. A "2/4" is one event away from "3/4", and in practice the events
+      that flip are decided by where the frozen threshold happens to land rather
+      than by whether the score saw anything.
+    * The score renormalises over available sensors, so a training window that
+      predates a sensor's history freezes a threshold on a different construct
+      than the holdout is measured against.
+    """
+    expected = len(rms.WARNING_WEIGHTS)
+    train = [backing[d] for d in dates[:split] if d in backing]
+    holdout = [backing[d] for d in dates[split:] if d in backing]
+    train_full = sum(1 for n in train if n == expected) / len(train) if train else 0.0
+    holdout_full = sum(1 for n in holdout if n == expected) / len(holdout) if holdout else 0.0
+    return {
+        "events_detected": events_detected,
+        "events_in_holdout": events_in_holdout,
+        "minimum_events": MIN_EVENTS_FOR_CONFIDENCE,
+        "underpowered": events_in_holdout < MIN_EVENTS_FOR_CONFIDENCE,
+        "sensors_expected": expected,
+        "train_full_sensor_share": round(train_full * 100, 1),
+        "holdout_full_sensor_share": round(holdout_full * 100, 1),
+        "sensor_coverage_mismatch": abs(train_full - holdout_full) > SENSOR_MISMATCH_TOLERANCE,
+    }
 
 
 async def run_event_study(
@@ -196,7 +242,7 @@ async def run_event_study(
     )
     divergence = breadth_service.compute_divergence_series(breadth, benchmark)
     oas_series = await rms._fetch_fred_series("BAMLH0A0HYM2", start, end)
-    warning = _warning_series(prices, divergence, dates, config, oas_series)
+    warning, backing = _warning_series(prices, divergence, dates, config, oas_series)
     # The credit sensor cannot reach back as far as the price history does (the
     # upstream series is capped at ~3 years), so the earlier part of the sample
     # scores on W1+W2 alone via renormalisation. Report where W3 starts rather
@@ -218,6 +264,8 @@ async def run_event_study(
         metrics["false_alarms"] / (holdout_sessions / 252.0), 2
     )
 
+    reliability = _reliability(dates, split, backing, len(all_events), len(holdout_events))
+
     basket_asof = date.fromisoformat(config["basket_asof"])
     retrospective = dates[split] < basket_asof
     evaluation = "exploratory" if retrospective else "holdout"
@@ -230,7 +278,14 @@ async def run_event_study(
         f"{evaluation.capitalize()} chronological test: warning episodes preceded "
         f"{metrics['events_warned']}/{metrics['events']} 10% corrections; "
         f"{metrics['events_missed']} missed, {metrics['false_alarms_per_year']:.1f} "
-        f"false alarms/year, {lead_text}."
+        f"false alarms/year, {lead_text}. "
+        f"{metrics['events']} of {reliability['events_detected']} detected corrections "
+        f"fall in the test period"
+        + (
+            f"; too few to read recall as a property of the score."
+            if reliability["underpowered"]
+            else "."
+        )
     )
     per_event = metrics.pop("per_event")
 
@@ -262,6 +317,7 @@ async def run_event_study(
             "holdout_sessions": holdout_sessions,
         },
         "metrics": metrics,
+        "reliability": reliability,
         "events": per_event,
         "recent_breadth": [
             {"date": d.isoformat(), "breadth": breadth[d], "warning": warning.get(d)}
@@ -273,8 +329,11 @@ async def run_event_study(
         "event": "regime_event_study_complete",
         "evaluation": evaluation,
         "events": metrics["events"],
+        "events_detected": reliability["events_detected"],
         "warned": metrics["events_warned"],
         "false_alarms_per_year": metrics["false_alarms_per_year"],
+        "underpowered": reliability["underpowered"],
+        "sensor_coverage_mismatch": reliability["sensor_coverage_mismatch"],
     }))
     return report