Event study: report its own statistical limits

The v3 cutover run scored 2/4 corrections warned against v2's 3/4, which reads
like a regression and is not one. Only 4 of the 11 detected corrections fall in
the holdout, so recall is one event from a different headline -- and the event
that flips is decided by threshold placement, not by what the score saw. "v3
without the credit sensor" catches 2025-02-21 at a *higher* threshold (35.5)
than shipped v3 misses it at (32.3), because the alarm rule needs a rising edge
and a lower threshold can fire outside the horizon then never reset below.

Two caveats are now computed and surfaced rather than left for the reader to
infer:

- Holdout event count against MIN_EVENTS_FOR_CONFIDENCE. The summary sentence
  states how many of the detected corrections actually fall in the test period.
- Warning-sensor coverage across the split. The score renormalises over what is
  available, so a training window predating a sensor's history freezes the
  threshold on a different construct than the holdout is measured against. At
  the cutover that is 39% of training sessions with all three sensors versus
  100% of the test period, credit history beginning 2023-07-25.

Restricting the threshold to sensor-matched training sessions was tested and
rejected: those sessions are a calm recent stretch, so the threshold falls from
32.3 to 22.5 and false alarms rise from 3.3 to 8.6/yr. It swaps a coverage bias
for a regime-selection bias. The report states its limits instead.

_warning_series now returns per-session sensor counts alongside the scores.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

app/services/event_study_service.py

@@ -28,6 +28,10 @@ DRAWDOWN_LOOKBACK = 252
 HORIZON_DAYS = 20
 WARN_PERCENTILE = 80.0
 TRAIN_FRACTION = 0.70
+# Below this many holdout corrections, recall is one event away from a very
+# different headline and should not be read as a property of the score.
+MIN_EVENTS_FOR_CONFIDENCE = 8
+SENSOR_MISMATCH_TOLERANCE = 0.10
 
 
 def _median(values: list[float]) -> float | None:
@@ -150,18 +154,23 @@ def _warning_series(
     dates: list[date],
     config: dict,
     oas_series: rms.Series | None = None,
-) -> dict[date, float]:
-    """Warning score per session, from the monitor's own sensor definitions.
+) -> tuple[dict[date, float], dict[date, int]]:
+    """Warning score per session plus how many sensors backed it.
 
     v2 re-derived this by hand from ``WARNING_WEIGHTS`` and so would have kept
     measuring the old construct after a scoring change.  Since v3 dropped
     fundamentals from the score, this is now exactly the live Warning score
     rather than a technical-only approximation of it.
+
+    The sensor count matters because the score renormalises over whatever is
+    available: a session backed by two sensors is not drawn from the same
+    distribution as one backed by three, and the frozen threshold assumes it is.
     """
     tickers = config["tickers"]
     smh_full = prices.get(tickers["leaders"][0], [])
     spy_full = prices.get(tickers["market"], [])
     out: dict[date, float] = {}
+    backing: dict[date, int] = {}
     for session in dates:
         sensors = rms.warning_sensor_scores(
             breadth_divergence.get(session),
@@ -172,7 +181,44 @@ def _warning_series(
         score = rms.score_warning_sensors(sensors)
         if score is not None:
             out[session] = round(score, 2)
-    return out
+            backing[session] = sum(1 for value in sensors.values() if value is not None)
+    return out, backing
+
+
+def _reliability(
+    dates: list[date],
+    split: int,
+    backing: dict[date, int],
+    events_detected: int,
+    events_in_holdout: int,
+) -> dict:
+    """How far the headline metrics can actually be trusted.
+
+    Two things repeatedly invite over-reading this report:
+
+    * The holdout carries only the corrections that fall in the last 30% of the
+      sample. A "2/4" is one event away from "3/4", and in practice the events
+      that flip are decided by where the frozen threshold happens to land rather
+      than by whether the score saw anything.
+    * The score renormalises over available sensors, so a training window that
+      predates a sensor's history freezes a threshold on a different construct
+      than the holdout is measured against.
+    """
+    expected = len(rms.WARNING_WEIGHTS)
+    train = [backing[d] for d in dates[:split] if d in backing]
+    holdout = [backing[d] for d in dates[split:] if d in backing]
+    train_full = sum(1 for n in train if n == expected) / len(train) if train else 0.0
+    holdout_full = sum(1 for n in holdout if n == expected) / len(holdout) if holdout else 0.0
+    return {
+        "events_detected": events_detected,
+        "events_in_holdout": events_in_holdout,
+        "minimum_events": MIN_EVENTS_FOR_CONFIDENCE,
+        "underpowered": events_in_holdout < MIN_EVENTS_FOR_CONFIDENCE,
+        "sensors_expected": expected,
+        "train_full_sensor_share": round(train_full * 100, 1),
+        "holdout_full_sensor_share": round(holdout_full * 100, 1),
+        "sensor_coverage_mismatch": abs(train_full - holdout_full) > SENSOR_MISMATCH_TOLERANCE,
+    }
 
 
 async def run_event_study(
@@ -196,7 +242,7 @@ async def run_event_study(
     )
     divergence = breadth_service.compute_divergence_series(breadth, benchmark)
     oas_series = await rms._fetch_fred_series("BAMLH0A0HYM2", start, end)
-    warning = _warning_series(prices, divergence, dates, config, oas_series)
+    warning, backing = _warning_series(prices, divergence, dates, config, oas_series)
     # The credit sensor cannot reach back as far as the price history does (the
     # upstream series is capped at ~3 years), so the earlier part of the sample
     # scores on W1+W2 alone via renormalisation. Report where W3 starts rather
@@ -218,6 +264,8 @@ async def run_event_study(
         metrics["false_alarms"] / (holdout_sessions / 252.0), 2
     )
 
+    reliability = _reliability(dates, split, backing, len(all_events), len(holdout_events))
+
     basket_asof = date.fromisoformat(config["basket_asof"])
     retrospective = dates[split] < basket_asof
     evaluation = "exploratory" if retrospective else "holdout"
@@ -230,7 +278,14 @@ async def run_event_study(
         f"{evaluation.capitalize()} chronological test: warning episodes preceded "
         f"{metrics['events_warned']}/{metrics['events']} 10% corrections; "
         f"{metrics['events_missed']} missed, {metrics['false_alarms_per_year']:.1f} "
-        f"false alarms/year, {lead_text}."
+        f"false alarms/year, {lead_text}. "
+        f"{metrics['events']} of {reliability['events_detected']} detected corrections "
+        f"fall in the test period"
+        + (
+            f"; too few to read recall as a property of the score."
+            if reliability["underpowered"]
+            else "."
+        )
     )
     per_event = metrics.pop("per_event")
 
@@ -262,6 +317,7 @@ async def run_event_study(
             "holdout_sessions": holdout_sessions,
         },
         "metrics": metrics,
+        "reliability": reliability,
         "events": per_event,
         "recent_breadth": [
             {"date": d.isoformat(), "breadth": breadth[d], "warning": warning.get(d)}
@@ -273,8 +329,11 @@ async def run_event_study(
         "event": "regime_event_study_complete",
         "evaluation": evaluation,
         "events": metrics["events"],
+        "events_detected": reliability["events_detected"],
         "warned": metrics["events_warned"],
         "false_alarms_per_year": metrics["false_alarms_per_year"],
+        "underpowered": reliability["underpowered"],
+        "sensor_coverage_mismatch": reliability["sensor_coverage_mismatch"],
     }))
     return report
 

docs/research/regime-monitor-v3.md

@@ -160,6 +160,40 @@ A cached report is discarded when its methodology no longer matches, so the pane
 reverts to "not run yet" after a bump rather than showing stale numbers. **Re-run
 the Event Study job after cutting over to v3.**
 
+### Reading the result
+
+The report carries a `reliability` block and the UI renders its warnings, because
+the headline numbers invite over-reading in two specific ways.
+
+**The holdout is thin.** The study detects 11 corrections across 5 years but the
+70/30 split leaves only 4 in the test period. Recall is therefore one event away
+from a materially different headline, and in practice the event that flips is
+decided by where the frozen threshold happens to land rather than by whether the
+score saw anything. The v3 cutover run illustrates it: v3 scored 2/4 against v2's
+3/4, but "v3 without the credit sensor" scores 3/4 at a *higher* threshold
+(35.5) than shipped v3 misses it at (32.3) — because the alarm rule needs a
+rising edge, and a lower threshold can mean the alarm already fired outside the
+20-session horizon and never reset below. Below `MIN_EVENTS_FOR_CONFIDENCE`
+holdout events the report says so explicitly.
+
+Some events carry no information at all for comparison: in that run every
+variant caught 2026-03-06, every variant missed 2026-06-05, and every variant
+"caught" 2025-11-20 with a 1-session lead, which is coincident rather than a
+warning.
+
+**Sensor coverage can straddle the split.** The score renormalises over available
+sensors, so a training window predating a sensor's history freezes the threshold
+on a different construct than the holdout is measured against. At the v3 cutover
+only 39% of training sessions had all three Warning sensors versus 100% of the
+test period, because credit history begins 2023-07-25.
+
+Restricting the threshold to sensor-matched training sessions was tried and is
+*not* the fix: those sessions are a calm recent stretch, so the threshold drops
+from 32.3 to 22.5 and false alarms rise from 3.3 to 8.6 per year. It trades a
+coverage bias for a regime-selection bias. The honest position is that the
+threshold is hypersensitive to window choice at this sample size; the report
+states its limits rather than pretending to a precision it does not have.
+
 ## Operator rule
 
 Quadrant alerts default off for new/reset configurations. When enabled they

frontend/src/lib/types.ts

@@ -604,6 +604,18 @@ export interface EventStudyReport {
     warn_threshold: number;
     basket_hash: string;
     basket_asof: string;
+    credit_sensor_from?: string | null;
+  };
+  /** How far the headline metrics can be trusted. See _reliability(). */
+  reliability?: {
+    events_detected: number;
+    events_in_holdout: number;
+    minimum_events: number;
+    underpowered: boolean;
+    sensors_expected: number;
+    train_full_sensor_share: number;
+    holdout_full_sensor_share: number;
+    sensor_coverage_mismatch: boolean;
   };
   sample?: {
     start: string;

frontend/src/pages/RegimePage.tsx

@@ -271,6 +271,32 @@ function EventStudyBody({ report }: { report: EventStudyReport }) {
           </table>
         </div>
       )}
+      {report.reliability && (report.reliability.underpowered || report.reliability.sensor_coverage_mismatch) && (
+        <Callout variant="warning">
+          <div className="space-y-1.5">
+            {report.reliability.underpowered && (
+              <p>
+                <strong>Underpowered.</strong> Only {report.reliability.events_in_holdout} of{' '}
+                {report.reliability.events_detected} detected corrections fall in the test period (
+                {report.reliability.minimum_events}+ needed). Recall is one event away from a materially
+                different headline, and which events flip is usually decided by where the frozen threshold
+                lands rather than by what the score saw. Read the direction, not the ratio.
+              </p>
+            )}
+            {report.reliability.sensor_coverage_mismatch && (
+              <p>
+                <strong>Sensor coverage differs across the split.</strong>{' '}
+                {report.reliability.train_full_sensor_share}% of training sessions had all{' '}
+                {report.reliability.sensors_expected} Warning sensors versus{' '}
+                {report.reliability.holdout_full_sensor_share}% of test sessions
+                {report.params?.credit_sensor_from && ` — credit history begins ${report.params.credit_sensor_from}`}
+                . The score renormalises over what is available, so the threshold was frozen on a partly
+                different construct than it is measured against.
+              </p>
+            )}
+          </div>
+        </Callout>
+      )}
       <p className="text-[11px] leading-relaxed text-gray-600">
         The threshold is frozen on the training period and measured on the chronological test period. Reconstructed
         pre-freeze basket history remains exploratory.

tests/unit/test_event_study.py

@@ -1,4 +1,4 @@
-"""Tests for v2 correction events and warning alarm episodes."""
+"""Tests for v3 correction events, warning alarm episodes, and report caveats."""
 
 from __future__ import annotations
 
@@ -6,7 +6,9 @@ from datetime import date, timedelta
 
 from app.services.breadth_service import _breadth_from_closes, compute_divergence_series
 from app.services.event_study_service import (
+    MIN_EVENTS_FOR_CONFIDENCE,
     _percentile,
+    _reliability,
     alarm_episodes,
     detect_events,
     evaluate_alarms,
@@ -23,6 +25,40 @@ def test_detect_events_uses_rising_edge_and_cooldown():
     assert [event["index"] for event in events] == [300, 355]
 
 
+def test_reliability_flags_a_thin_holdout():
+    """2/4 must not read like a property of the score."""
+    dates = _days(100)
+    backing = dict.fromkeys(dates, 3)
+
+    thin = _reliability(dates, 70, backing, events_detected=11, events_in_holdout=4)
+    assert thin["underpowered"] is True
+    assert thin["events_detected"] == 11
+    assert thin["events_in_holdout"] == 4
+    assert thin["minimum_events"] == MIN_EVENTS_FOR_CONFIDENCE
+
+    ample = _reliability(dates, 70, backing, events_detected=20, events_in_holdout=12)
+    assert ample["underpowered"] is False
+
+
+def test_reliability_flags_a_sensor_coverage_split():
+    """The threshold must not be frozen on a different construct than it is tested on.
+
+    Credit history starts partway through the training window, so the score
+    renormalises over two sensors early and three later.
+    """
+    dates = _days(100)
+    matched = dict.fromkeys(dates, 3)
+    assert _reliability(dates, 70, matched, 20, 12)["sensor_coverage_mismatch"] is False
+
+    # Training is 40% three-sensor; the holdout is entirely three-sensor.
+    split_backing = {d: (3 if index >= 42 else 2) for index, d in enumerate(dates)}
+    mismatched = _reliability(dates, 70, split_backing, 20, 12)
+    assert mismatched["sensor_coverage_mismatch"] is True
+    assert mismatched["train_full_sensor_share"] == 40.0
+    assert mismatched["holdout_full_sensor_share"] == 100.0
+    assert mismatched["sensors_expected"] == 3
+
+
 def test_percentile_is_fixed_from_supplied_values():
     values = [float(value) for value in range(0, 101, 10)]
     assert _percentile(values, 50) == 50.0